From a0a18c79e3fd28125f5e60660a1c3c1d645584dd Mon Sep 17 00:00:00 2001 From: simon987 Date: Sun, 5 Apr 2020 20:25:27 -0400 Subject: [PATCH] Entirely remove libonion (WIP) --- .gitmodules | 3 -- CMakeLists.txt | 5 +- README.md | 1 + src/cli.c | 12 ++--- src/io/serialize.c | 1 + src/io/walk.c | 2 +- src/main.c | 1 + src/parsing/parse.c | 16 +++--- src/web/auth_basic.c | 59 --------------------- src/web/auth_basic.h | 6 --- src/web/serve.c | 119 ++++++++++++++++--------------------------- 11 files changed, 62 insertions(+), 163 deletions(-) delete mode 100644 src/web/auth_basic.c delete mode 100644 src/web/auth_basic.h diff --git a/.gitmodules b/.gitmodules index a3dfdb4..cc56582 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "third-party/onion"] - path = third-party/onion - url = https://github.com/davidmoreno/onion [submodule "third-party/libscan"] path = third-party/libscan url = https://github.com/simon987/libscan diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a1b79e..469b5c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ project(sist2 C) option(SIST_DEBUG "Build a debug executable" on) add_subdirectory(third-party/libscan) +set(ARGPARSE_SHARED off) add_subdirectory(third-party/argparse) add_executable( @@ -20,7 +21,6 @@ add_executable( src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c src/index/web.c src/index/web.h src/web/serve.c src/web/serve.h - src/web/auth_basic.h src/web/auth_basic.c src/index/elastic.c src/index/elastic.h src/util.c src/util.h src/ctx.h src/types.h @@ -104,9 +104,6 @@ target_link_libraries( unofficial::mongoose::mongoose # OpenSSL::SSL OpenSSL::Crypto - # onion - ${PROJECT_SOURCE_DIR}/third-party/onion/src/onion/libonion_static.a - uuid pthread magic diff --git a/README.md b/README.md index f22a7fe..22e9849 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ binaries (GCC 7+ required). 2. Build ```bash + git clone --recursive https://github.com/simon987/sist2/ cmake -D /scripts/buildsystems/vcpkg.cmake . make ``` diff --git a/src/cli.c b/src/cli.c index 2165631..0e52754 100644 --- a/src/cli.c +++ b/src/cli.c @@ -1,6 +1,5 @@ #include "cli.h" #include "ctx.h" -#include #include #define DEFAULT_OUTPUT "index.sist2/" @@ -285,11 +284,12 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) { args->port = DEFAULT_PORT; } - if (args->credentials != NULL) { - args->b64credentials = onion_base64_encode(args->credentials, (int) strlen(args->credentials)); - //Remove trailing newline - *(args->b64credentials + strlen(args->b64credentials) - 1) = '\0'; - } + //TODO +// if (args->credentials != NULL) { +// args->b64credentials = onion_base64_encode(args->credentials, (int) strlen(args->credentials)); +// //Remove trailing newline +// *(args->b64credentials + strlen(args->b64credentials) - 1) = '\0'; +// } args->index_count = argc - 1; args->indices = argv + 1; diff --git a/src/io/serialize.c b/src/io/serialize.c index 946a36b..d7c18cd 100644 --- a/src/io/serialize.c +++ b/src/io/serialize.c @@ -202,6 +202,7 @@ void write_document(document_t *doc) { void thread_cleanup() { close(index_fd); cleanup_parse(); + cleanup_font(); } diff --git a/src/io/walk.c b/src/io/walk.c index 460c088..28341bc 100644 --- a/src/io/walk.c +++ b/src/io/walk.c @@ -18,7 +18,7 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info, job->ext = len; } - job->info = *info; + job->vfile.info = *info; memset(job->parent, 0, 16); diff --git a/src/main.c b/src/main.c index e7951f2..f0bc0e3 100644 --- a/src/main.c +++ b/src/main.c @@ -86,6 +86,7 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.media_ctx.log = sist_log; ScanCtx.media_ctx.logf = sist_logf; ScanCtx.media_ctx.store = _store; + init_media(); // OOXML ScanCtx.ooxml_ctx.content_size = args->content_size; diff --git a/src/parsing/parse.c b/src/parsing/parse.c index fb9f94a..cfb6d73 100644 --- a/src/parsing/parse.c +++ b/src/parsing/parse.c @@ -3,11 +3,9 @@ #include "src/sist.h" #include "src/ctx.h" #include "mime.h" -#include "libscan/scan.h" #include "src/io/serialize.h" #include -#include __thread magic_t Magic = NULL; @@ -40,9 +38,9 @@ void parse(void *arg) { parse_job_t *job = arg; document_t doc; - int inc_ts = incremental_get(ScanCtx.original_table, job->info.st_ino); - if (inc_ts != 0 && inc_ts == job->info.st_mtim.tv_sec) { - incremental_mark_file_for_copy(ScanCtx.copy_table, job->info.st_ino); + int inc_ts = incremental_get(ScanCtx.original_table, job->vfile.info.st_ino); + if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) { + incremental_mark_file_for_copy(ScanCtx.copy_table, job->vfile.info.st_ino); return; } @@ -57,9 +55,9 @@ void parse(void *arg) { doc.meta_head = NULL; doc.meta_tail = NULL; doc.mime = 0; - doc.size = job->info.st_size; - doc.ino = job->info.st_ino; - doc.mtime = job->info.st_mtim.tv_sec; + doc.size = job->vfile.info.st_size; + doc.ino = job->vfile.info.st_ino; + doc.mtime = job->vfile.info.st_mtim.tv_sec; uuid_generate(doc.uuid); char *buf[PARSE_BUF_SIZE]; @@ -70,7 +68,7 @@ void parse(void *arg) { LOG_DEBUGF(job->filepath, "Starting parse job {%s}", uuid_str) } - if (job->info.st_size == 0) { + if (job->vfile.info.st_size == 0) { doc.mime = MIME_EMPTY; } else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) { doc.mime = mime_get_mime_by_ext(ScanCtx.ext_table, job->filepath + job->ext); diff --git a/src/web/auth_basic.c b/src/web/auth_basic.c deleted file mode 100644 index 80ea9c9..0000000 --- a/src/web/auth_basic.c +++ /dev/null @@ -1,59 +0,0 @@ -#include "auth_basic.h" - -#define UNAUTHORIZED_TEXT "Unauthorized" - -typedef struct auth_basic_data { - onion_handler *inside; - const char *b64credentials; -} auth_basic_data_t; - - -int authenticate(const char *expected, const char *credentials) { - - if (expected == NULL) { - return TRUE; - } - - if (credentials && strncmp(credentials, "Basic ", 6) == 0) { - if (strcmp((credentials + 6), expected) == 0) { - return TRUE; - } - } - - return FALSE; -} - -int auth_basic_handler(auth_basic_data_t *d, - onion_request *req, - onion_response *res) { - - const char *credentials = onion_request_get_header(req, "Authorization"); - - if (authenticate(d->b64credentials, credentials)) { - return onion_handler_handle(d->inside, req, res); - } - - onion_response_set_header(res, "WWW-Authenticate", "Basic realm=\"sist2\""); - onion_response_set_code(res, HTTP_UNAUTHORIZED); - onion_response_write(res, UNAUTHORIZED_TEXT, sizeof(UNAUTHORIZED_TEXT)); - onion_response_set_length(res, sizeof(UNAUTHORIZED_TEXT)); - - return OCS_PROCESSED; -} - -void auth_basic_free(auth_basic_data_t *data) { - onion_handler_free(data->inside); - free(data); -} - -onion_handler *auth_basic(const char *b64credentials, onion_handler *inside_level) { - - auth_basic_data_t *privdata = malloc(sizeof(auth_basic_data_t)); - - privdata->b64credentials = b64credentials; - privdata->inside = inside_level; - - return onion_handler_new((onion_handler_handler) auth_basic_handler, privdata, - (onion_handler_private_data_free) auth_basic_free); -} - diff --git a/src/web/auth_basic.h b/src/web/auth_basic.h deleted file mode 100644 index 15a6301..0000000 --- a/src/web/auth_basic.h +++ /dev/null @@ -1,6 +0,0 @@ -#include "src/sist.h" - -#include -#include - -onion_handler *auth_basic(const char *b64credentials, onion_handler *inside_level); diff --git a/src/web/serve.c b/src/web/serve.c index 5aa1160..56460f5 100644 --- a/src/web/serve.c +++ b/src/web/serve.c @@ -5,15 +5,9 @@ #include "static_generated.c" #include "src/index/elastic.h" #include "src/index/web.h" -#include "src/web/auth_basic.h" #include -#include -#include -#include -#include - #include #define CHUNK_SIZE 1024 * 1024 * 10 @@ -119,7 +113,7 @@ int img_sprite_skin_flat(struct mg_connection *nc, struct http_message *hm) { void thumbnail(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) { if (path->len != UUID_STR_LEN * 2 + 2) { - LOG_DEBUGF("serve.c", "Invalid thumnail path: %.*s", (int) path->len, path->p); + LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) path->len, path->p); nc->flags |= MG_F_SEND_AND_CLOSE; return; } @@ -176,22 +170,23 @@ void search(struct mg_connection *nc, struct http_message *hm) { free(body); } -int serve_file_from_url(cJSON *json, index_t *idx, onion_request *req, onion_response *res) { - - const char *path = cJSON_GetObjectItem(json, "path")->valuestring; - const char *name = cJSON_GetObjectItem(json, "name")->valuestring; - const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring; - - char url[8196]; - snprintf(url, sizeof(url), - "%s%s/%s%s%s", - idx->desc.rewrite_url, path, name, strlen(ext) == 0 ? "" : ".", ext); - - dyn_buffer_t encoded = url_escape(url); - int ret = onion_shortcut_redirect(encoded.buf, req, res); - dyn_buffer_destroy(&encoded); - return ret; -} +//TODO +//int serve_file_from_url(cJSON *json, index_t *idx, onion_request *req, onion_response *res) { +// +// const char *path = cJSON_GetObjectItem(json, "path")->valuestring; +// const char *name = cJSON_GetObjectItem(json, "name")->valuestring; +// const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring; +// +// char url[8196]; +// snprintf(url, sizeof(url), +// "%s%s/%s%s%s", +// idx->desc.rewrite_url, path, name, strlen(ext) == 0 ? "" : ".", ext); +// +// dyn_buffer_t encoded = url_escape(url); +// int ret = onion_shortcut_redirect(encoded.buf, req, res); +// dyn_buffer_destroy(&encoded); +// return ret; +//} void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, struct http_message *hm) { @@ -240,36 +235,41 @@ int index_info(struct mg_connection *nc) { } -int document_info(UNUSED(void *p), onion_request *req, onion_response *res) { +void document_info(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) { - const char *arg_uuid = onion_request_get_query(req, "1"); - if (arg_uuid == NULL) { - return OCS_PROCESSED; + if (path->len != UUID_STR_LEN + 2) { + LOG_DEBUGF("serve.c", "Invalid document_info path: %.*s", (int) path->len, path->p); + nc->flags |= MG_F_SEND_AND_CLOSE; + return; } + char arg_uuid[UUID_STR_LEN]; + memcpy(arg_uuid, hm->uri.p + 3, UUID_STR_LEN); + *(arg_uuid + UUID_STR_LEN - 1) = '\0'; + cJSON *doc = elastic_get_document(arg_uuid); cJSON *source = cJSON_GetObjectItem(doc, "_source"); cJSON *index_id = cJSON_GetObjectItem(source, "index"); if (index_id == NULL) { cJSON_Delete(doc); - return OCS_NOT_PROCESSED; + nc->flags |= MG_F_SEND_AND_CLOSE; + return; } index_t *idx = get_index_by_id(index_id->valuestring); if (idx == NULL) { cJSON_Delete(doc); - return OCS_NOT_PROCESSED; + nc->flags |= MG_F_SEND_AND_CLOSE; + return; } - onion_response_set_header(res, "Content-Type", "application/json"); - char *json_str = cJSON_PrintUnformatted(source); - onion_response_write0(res, json_str); + send_response_line(nc, 200, (int)strlen(json_str), "Content-Type: application/json"); free(json_str); cJSON_Delete(doc); - return OCS_PROCESSED; + nc->flags |= MG_F_SEND_AND_CLOSE; } void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) { @@ -322,19 +322,17 @@ void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path cJSON_Delete(doc); } -int status(UNUSED(void *p), UNUSED(onion_request *req), onion_response *res) { - onion_response_set_header(res, "Content-Type", "application/x-empty"); - +int status(struct mg_connection *nc) { char *status = elastic_get_status(); if (strcmp(status, "open") == 0) { - onion_response_set_code(res, 204); + send_response_line(nc, 204, 0, "Content-Type: application/json"); } else { - onion_response_set_code(res, 500); + send_response_line(nc, 500, 0, "Content-Type: application/json"); } free(status); - return OCS_PROCESSED; + nc->flags |= MG_F_SEND_AND_CLOSE; } typedef struct { @@ -370,16 +368,18 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) { search(nc, hm); } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/i")))) { index_info(nc); + } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/status")))) { + status(nc); } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/f/")))) { file(nc, hm, &path); } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/t/")))) { thumbnail(nc, hm, &path); + } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/d/")))) { + document_info(nc, hm, &path); } else { - nc->flags |= MG_F_SEND_AND_CLOSE; } - } else if (ev == MG_EV_POLL) { if (nc->user_data != NULL) { //Waiting for ES reply @@ -419,6 +419,8 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) { void serve(const char *hostname, const char *port) { + printf("Starting web server @ http://%s:%s\n", hostname, port); + struct mg_mgr mgr; mg_mgr_init(&mgr, NULL); @@ -433,38 +435,5 @@ void serve(const char *hostname, const char *port) { mg_mgr_poll(&mgr, 10); } - onion *o = onion_new(O_POOL); - onion_set_timeout(o, 3500); - - onion_set_hostname(o, hostname); - onion_set_port(o, port); - - onion_url *urls = onion_url_new(); - - // Static paths - onion_set_root_handler(o, auth_basic(WebCtx.b64credentials, onion_url_to_handler(urls))); - - onion_url_add(urls, "", search_index); - onion_url_add(urls, "css", style); - onion_url_add(urls, "js", javascript); - onion_url_add(urls, "img/sprite-skin-flat.png", img_sprite_skin_flat); - - onion_url_add(urls, "es", search); - onion_url_add(urls, "status", status); - onion_url_add( - urls, - "^t/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})/" - "([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})$", - thumbnail - ); - onion_url_add(urls, "^f/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})$", file); - onion_url_add(urls, "^d/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})$", - document_info); - onion_url_add(urls, "i", index_info); - - - printf("Starting web server @ http://%s:%s\n", hostname, port); - - onion_listen(o); - onion_free(o); +// onion_set_root_handler(o, auth_basic(WebCtx.b64credentials, onion_url_to_handler(urls))); }