Entirely remove libonion (WIP)

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "third-party/onion"]
-	path = third-party/onion
-	url = https://github.com/davidmoreno/onion
 [submodule "third-party/libscan"]
 	path = third-party/libscan
 	url = https://github.com/simon987/libscan

CMakeLists.txt

@@ -6,6 +6,7 @@ project(sist2 C)
 option(SIST_DEBUG "Build a debug executable" on)
 
 add_subdirectory(third-party/libscan)
+set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
 add_executable(
@@ -20,7 +21,6 @@ add_executable(
         src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
         src/index/web.c src/index/web.h
         src/web/serve.c src/web/serve.h
-        src/web/auth_basic.h src/web/auth_basic.c
         src/index/elastic.c src/index/elastic.h
         src/util.c src/util.h
         src/ctx.h src/types.h
@@ -104,9 +104,6 @@ target_link_libraries(
         unofficial::mongoose::mongoose
 #        OpenSSL::SSL OpenSSL::Crypto
 
-        # onion
-        ${PROJECT_SOURCE_DIR}/third-party/onion/src/onion/libonion_static.a
-
         uuid
         pthread
         magic

README.md

@@ -123,6 +123,7 @@ binaries (GCC 7+ required).
 
 2. Build
     ```bash
+    git clone --recursive https://github.com/simon987/sist2/
     cmake -D <VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

src/cli.c

@@ -1,6 +1,5 @@
 #include "cli.h"
 #include "ctx.h"
-#include <onion/codecs.h>
 #include <tesseract/capi.h>
 
 #define DEFAULT_OUTPUT "index.sist2/"
@@ -285,11 +284,12 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         args->port = DEFAULT_PORT;
     }
 
-    if (args->credentials != NULL) {
+    //TODO
-        args->b64credentials = onion_base64_encode(args->credentials, (int) strlen(args->credentials));
+//    if (args->credentials != NULL) {
-        //Remove trailing newline
+//        args->b64credentials = onion_base64_encode(args->credentials, (int) strlen(args->credentials));
-        *(args->b64credentials + strlen(args->b64credentials) - 1) = '\0';
+//        //Remove trailing newline
-    }
+//        *(args->b64credentials + strlen(args->b64credentials) - 1) = '\0';
+//    }
 
     args->index_count = argc - 1;
     args->indices = argv + 1;

src/io/serialize.c

@@ -202,6 +202,7 @@ void write_document(document_t *doc) {
 void thread_cleanup() {
     close(index_fd);
     cleanup_parse();
+    cleanup_font();
 }
 
 

src/io/walk.c

@@ -18,7 +18,7 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info,
         job->ext = len;
     }
 
-    job->info = *info;
+    job->vfile.info = *info;
 
     memset(job->parent, 0, 16);
 

src/main.c

@@ -86,6 +86,7 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.media_ctx.log = sist_log;
     ScanCtx.media_ctx.logf = sist_logf;
     ScanCtx.media_ctx.store = _store;
+    init_media();
 
     // OOXML
     ScanCtx.ooxml_ctx.content_size = args->content_size;

src/parsing/parse.c

@@ -3,11 +3,9 @@
 #include "src/sist.h"
 #include "src/ctx.h"
 #include "mime.h"
-#include "libscan/scan.h"
 #include "src/io/serialize.h"
 
 #include <magic.h>
-#include <src/ctx.h>
 
 __thread magic_t Magic = NULL;
 
@@ -40,9 +38,9 @@ void parse(void *arg) {
     parse_job_t *job = arg;
     document_t doc;
 
-    int inc_ts = incremental_get(ScanCtx.original_table, job->info.st_ino);
+    int inc_ts = incremental_get(ScanCtx.original_table, job->vfile.info.st_ino);
-    if (inc_ts != 0 && inc_ts == job->info.st_mtim.tv_sec) {
+    if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) {
-        incremental_mark_file_for_copy(ScanCtx.copy_table, job->info.st_ino);
+        incremental_mark_file_for_copy(ScanCtx.copy_table, job->vfile.info.st_ino);
         return;
     }
 
@@ -57,9 +55,9 @@ void parse(void *arg) {
     doc.meta_head = NULL;
     doc.meta_tail = NULL;
     doc.mime = 0;
-    doc.size = job->info.st_size;
+    doc.size = job->vfile.info.st_size;
-    doc.ino = job->info.st_ino;
+    doc.ino = job->vfile.info.st_ino;
-    doc.mtime = job->info.st_mtim.tv_sec;
+    doc.mtime = job->vfile.info.st_mtim.tv_sec;
 
     uuid_generate(doc.uuid);
     char *buf[PARSE_BUF_SIZE];
@@ -70,7 +68,7 @@ void parse(void *arg) {
         LOG_DEBUGF(job->filepath, "Starting parse job {%s}", uuid_str)
     }
 
-    if (job->info.st_size == 0) {
+    if (job->vfile.info.st_size == 0) {
         doc.mime = MIME_EMPTY;
     } else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) {
         doc.mime = mime_get_mime_by_ext(ScanCtx.ext_table, job->filepath + job->ext);

src/web/auth_basic.c

@@ -1,59 +0,0 @@
-#include "auth_basic.h"
-
-#define UNAUTHORIZED_TEXT "Unauthorized"
-
-typedef struct auth_basic_data {
-    onion_handler *inside;
-    const char *b64credentials;
-} auth_basic_data_t;
-
-
-int authenticate(const char *expected, const char *credentials) {
-
-    if (expected == NULL) {
-        return TRUE;
-    }
-
-    if (credentials && strncmp(credentials, "Basic ", 6) == 0) {
-        if (strcmp((credentials + 6), expected) == 0) {
-            return TRUE;
-        }
-    }
-
-    return FALSE;
-}
-
-int auth_basic_handler(auth_basic_data_t *d,
-                       onion_request *req,
-                       onion_response *res) {
-
-    const char *credentials = onion_request_get_header(req, "Authorization");
-
-    if (authenticate(d->b64credentials, credentials)) {
-        return onion_handler_handle(d->inside, req, res);
-    }
-
-    onion_response_set_header(res, "WWW-Authenticate", "Basic realm=\"sist2\"");
-    onion_response_set_code(res, HTTP_UNAUTHORIZED);
-    onion_response_write(res, UNAUTHORIZED_TEXT, sizeof(UNAUTHORIZED_TEXT));
-    onion_response_set_length(res, sizeof(UNAUTHORIZED_TEXT));
-
-    return OCS_PROCESSED;
-}
-
-void auth_basic_free(auth_basic_data_t *data) {
-    onion_handler_free(data->inside);
-    free(data);
-}
-
-onion_handler *auth_basic(const char *b64credentials, onion_handler *inside_level) {
-
-    auth_basic_data_t *privdata = malloc(sizeof(auth_basic_data_t));
-
-    privdata->b64credentials = b64credentials;
-    privdata->inside = inside_level;
-
-    return onion_handler_new((onion_handler_handler) auth_basic_handler, privdata,
-                             (onion_handler_private_data_free) auth_basic_free);
-}
-

src/web/auth_basic.h

@@ -1,6 +0,0 @@
-#include "src/sist.h"
-
-#include <onion/handler.h>
-#include <onion/onion.h>
-
-onion_handler *auth_basic(const char *b64credentials, onion_handler *inside_level);

src/web/serve.c

@@ -5,15 +5,9 @@
 #include "static_generated.c"
 #include "src/index/elastic.h"
 #include "src/index/web.h"
-#include "src/web/auth_basic.h"
 
 #include <src/ctx.h>
 
-#include <onion/onion.h>
-#include <onion/handler.h>
-#include <onion/block.h>
-#include <onion/shortcuts.h>
-
 #include <mongoose.h>
 
 #define CHUNK_SIZE 1024 * 1024 * 10
@@ -119,7 +113,7 @@ int img_sprite_skin_flat(struct mg_connection *nc, struct http_message *hm) {
 void thumbnail(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
 
     if (path->len != UUID_STR_LEN * 2 + 2) {
-        LOG_DEBUGF("serve.c", "Invalid thumnail path: %.*s", (int) path->len, path->p);
+        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) path->len, path->p);
         nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
@@ -176,22 +170,23 @@ void search(struct mg_connection *nc, struct http_message *hm) {
     free(body);
 }
 
-int serve_file_from_url(cJSON *json, index_t *idx, onion_request *req, onion_response *res) {
+//TODO
-
+//int serve_file_from_url(cJSON *json, index_t *idx, onion_request *req, onion_response *res) {
-    const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
+//
-    const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
+//    const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
-    const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
+//    const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
-
+//    const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
-    char url[8196];
+//
-    snprintf(url, sizeof(url),
+//    char url[8196];
-             "%s%s/%s%s%s",
+//    snprintf(url, sizeof(url),
-             idx->desc.rewrite_url, path, name, strlen(ext) == 0 ? "" : ".", ext);
+//             "%s%s/%s%s%s",
-
+//             idx->desc.rewrite_url, path, name, strlen(ext) == 0 ? "" : ".", ext);
-    dyn_buffer_t encoded = url_escape(url);
+//
-    int ret = onion_shortcut_redirect(encoded.buf, req, res);
+//    dyn_buffer_t encoded = url_escape(url);
-    dyn_buffer_destroy(&encoded);
+//    int ret = onion_shortcut_redirect(encoded.buf, req, res);
-    return ret;
+//    dyn_buffer_destroy(&encoded);
-}
+//    return ret;
+//}
 
 void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, struct http_message *hm) {
 
@@ -240,36 +235,41 @@ int index_info(struct mg_connection *nc) {
 }
 
 
-int document_info(UNUSED(void *p), onion_request *req, onion_response *res) {
+void document_info(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
 
-    const char *arg_uuid = onion_request_get_query(req, "1");
+    if (path->len != UUID_STR_LEN + 2) {
-    if (arg_uuid == NULL) {
+        LOG_DEBUGF("serve.c", "Invalid document_info path: %.*s", (int) path->len, path->p);
-        return OCS_PROCESSED;
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return;
     }
 
+    char arg_uuid[UUID_STR_LEN];
+    memcpy(arg_uuid, hm->uri.p + 3, UUID_STR_LEN);
+    *(arg_uuid + UUID_STR_LEN - 1) = '\0';
+
     cJSON *doc = elastic_get_document(arg_uuid);
     cJSON *source = cJSON_GetObjectItem(doc, "_source");
 
     cJSON *index_id = cJSON_GetObjectItem(source, "index");
     if (index_id == NULL) {
         cJSON_Delete(doc);
-        return OCS_NOT_PROCESSED;
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return;
     }
 
     index_t *idx = get_index_by_id(index_id->valuestring);
     if (idx == NULL) {
         cJSON_Delete(doc);
-        return OCS_NOT_PROCESSED;
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return;
     }
 
-    onion_response_set_header(res, "Content-Type", "application/json");
-
     char *json_str = cJSON_PrintUnformatted(source);
-    onion_response_write0(res, json_str);
+    send_response_line(nc, 200, (int)strlen(json_str), "Content-Type: application/json");
     free(json_str);
     cJSON_Delete(doc);
 
-    return OCS_PROCESSED;
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
@@ -322,19 +322,17 @@ void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path
     cJSON_Delete(doc);
 }
 
-int status(UNUSED(void *p), UNUSED(onion_request *req), onion_response *res) {
+int status(struct mg_connection *nc) {
-    onion_response_set_header(res, "Content-Type", "application/x-empty");
-
     char *status = elastic_get_status();
     if (strcmp(status, "open") == 0) {
-        onion_response_set_code(res, 204);
+        send_response_line(nc, 204, 0, "Content-Type: application/json");
     } else {
-        onion_response_set_code(res, 500);
+        send_response_line(nc, 500, 0, "Content-Type: application/json");
     }
 
     free(status);
 
-    return OCS_PROCESSED;
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 typedef struct {
@@ -370,16 +368,18 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
             search(nc, hm);
         } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/i")))) {
             index_info(nc);
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/status")))) {
+            status(nc);
         } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/f/")))) {
             file(nc, hm, &path);
         } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/t/")))) {
             thumbnail(nc, hm, &path);
+        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/d/")))) {
+            document_info(nc, hm, &path);
         } else {
-
             nc->flags |= MG_F_SEND_AND_CLOSE;
         }
 
-
     } else if (ev == MG_EV_POLL) {
         if (nc->user_data != NULL) {
             //Waiting for ES reply
@@ -419,6 +419,8 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
 
 void serve(const char *hostname, const char *port) {
 
+    printf("Starting web server @ http://%s:%s\n", hostname, port);
+
     struct mg_mgr mgr;
     mg_mgr_init(&mgr, NULL);
 
@@ -433,38 +435,5 @@ void serve(const char *hostname, const char *port) {
         mg_mgr_poll(&mgr, 10);
     }
 
-    onion *o = onion_new(O_POOL);
+//    onion_set_root_handler(o, auth_basic(WebCtx.b64credentials, onion_url_to_handler(urls)));
-    onion_set_timeout(o, 3500);
-
-    onion_set_hostname(o, hostname);
-    onion_set_port(o, port);
-
-    onion_url *urls = onion_url_new();
-
-    // Static paths
-    onion_set_root_handler(o, auth_basic(WebCtx.b64credentials, onion_url_to_handler(urls)));
-
-    onion_url_add(urls, "", search_index);
-    onion_url_add(urls, "css", style);
-    onion_url_add(urls, "js", javascript);
-    onion_url_add(urls, "img/sprite-skin-flat.png", img_sprite_skin_flat);
-
-    onion_url_add(urls, "es", search);
-    onion_url_add(urls, "status", status);
-    onion_url_add(
-            urls,
-            "^t/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})/"
-            "([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})$",
-            thumbnail
-    );
-    onion_url_add(urls, "^f/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})$", file);
-    onion_url_add(urls, "^d/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})$",
-                  document_info);
-    onion_url_add(urls, "i", index_info);
-
-
-    printf("Starting web server @ http://%s:%s\n", hostname, port);
-
-    onion_listen(o);
-    onion_free(o);
 }