simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-24 12:45:56 +00:00
Code Issues Projects Releases Wiki Activity

Entirely remove libonion (WIP)

Browse Source
This commit is contained in:
simon987 2020-04-05 20:25:27 -04:00
parent e6e35811c6
commit a0a18c79e3
11 changed files with 62 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -1,6 +1,3 @@
[submodule "third-party/onion"]
path = third-party/onion
url = https://github.com/davidmoreno/onion
[submodule "third-party/libscan"]
path = third-party/libscan
url = https://github.com/simon987/libscan

5
CMakeLists.txt
View File

@ -6,6 +6,7 @@ project(sist2 C)
option(SIST_DEBUG "Build a debug executable" on)
add_subdirectory(third-party/libscan)
set(ARGPARSE_SHARED off)
add_subdirectory(third-party/argparse)
add_executable(
@ -20,7 +21,6 @@ add_executable(
src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
src/index/web.c src/index/web.h
src/web/serve.c src/web/serve.h
src/web/auth_basic.h src/web/auth_basic.c
src/index/elastic.c src/index/elastic.h
src/util.c src/util.h
src/ctx.h src/types.h
@ -104,9 +104,6 @@ target_link_libraries(
unofficial::mongoose::mongoose
# OpenSSL::SSL OpenSSL::Crypto
# onion
${PROJECT_SOURCE_DIR}/third-party/onion/src/onion/libonion_static.a
uuid
pthread
magic

1
README.md
View File

@ -123,6 +123,7 @@ binaries (GCC 7+ required).
2. Build
```bash
git clone --recursive https://github.com/simon987/sist2/
cmake -D <VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
make
```

12
src/cli.c
View File

@ -1,6 +1,5 @@
#include "cli.h"
#include "ctx.h"
#include <onion/codecs.h>
#include <tesseract/capi.h>
#define DEFAULT_OUTPUT "index.sist2/"
@ -285,11 +284,12 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
args->port = DEFAULT_PORT;
}
if (args->credentials != NULL) {
args->b64credentials = onion_base64_encode(args->credentials, (int) strlen(args->credentials));
//Remove trailing newline
*(args->b64credentials + strlen(args->b64credentials) - 1) = '\0';
}
//TODO
// if (args->credentials != NULL) {
// args->b64credentials = onion_base64_encode(args->credentials, (int) strlen(args->credentials));
// //Remove trailing newline
// *(args->b64credentials + strlen(args->b64credentials) - 1) = '\0';
// }
args->index_count = argc - 1;
args->indices = argv + 1;

1
src/io/serialize.c
View File

@ -202,6 +202,7 @@ void write_document(document_t *doc) {
void thread_cleanup() {
close(index_fd);
cleanup_parse();
cleanup_font();
}

2
src/io/walk.c
View File

@ -18,7 +18,7 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info,
job->ext = len;
}
job->info = *info;
job->vfile.info = *info;
memset(job->parent, 0, 16);

1
src/main.c
View File

@ -86,6 +86,7 @@ void initialize_scan_context(scan_args_t *args) {
ScanCtx.media_ctx.log = sist_log;
ScanCtx.media_ctx.logf = sist_logf;
ScanCtx.media_ctx.store = _store;
init_media();
// OOXML
ScanCtx.ooxml_ctx.content_size = args->content_size;

16
src/parsing/parse.c
View File

@ -3,11 +3,9 @@
#include "src/sist.h"
#include "src/ctx.h"
#include "mime.h"
#include "libscan/scan.h"
#include "src/io/serialize.h"
#include <magic.h>
#include <src/ctx.h>
__thread magic_t Magic = NULL;
@ -40,9 +38,9 @@ void parse(void *arg) {
parse_job_t *job = arg;
document_t doc;
int inc_ts = incremental_get(ScanCtx.original_table, job->info.st_ino);
if (inc_ts != 0 && inc_ts == job->info.st_mtim.tv_sec) {
incremental_mark_file_for_copy(ScanCtx.copy_table, job->info.st_ino);
int inc_ts = incremental_get(ScanCtx.original_table, job->vfile.info.st_ino);
if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) {
incremental_mark_file_for_copy(ScanCtx.copy_table, job->vfile.info.st_ino);
return;
}
@ -57,9 +55,9 @@ void parse(void *arg) {
doc.meta_head = NULL;
doc.meta_tail = NULL;
doc.mime = 0;
doc.size = job->info.st_size;
doc.ino = job->info.st_ino;
doc.mtime = job->info.st_mtim.tv_sec;
doc.size = job->vfile.info.st_size;
doc.ino = job->vfile.info.st_ino;
doc.mtime = job->vfile.info.st_mtim.tv_sec;
uuid_generate(doc.uuid);
char *buf[PARSE_BUF_SIZE];
@ -70,7 +68,7 @@ void parse(void *arg) {
LOG_DEBUGF(job->filepath, "Starting parse job {%s}", uuid_str)
}
if (job->info.st_size == 0) {
if (job->vfile.info.st_size == 0) {
doc.mime = MIME_EMPTY;
} else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) {
doc.mime = mime_get_mime_by_ext(ScanCtx.ext_table, job->filepath + job->ext);

59
src/web/auth_basic.c
View File

@ -1,59 +0,0 @@
#include "auth_basic.h"
#define UNAUTHORIZED_TEXT "Unauthorized"
typedef struct auth_basic_data {
onion_handler *inside;
const char *b64credentials;
} auth_basic_data_t;
int authenticate(const char *expected, const char *credentials) {
if (expected == NULL) {
return TRUE;
}
if (credentials && strncmp(credentials, "Basic ", 6) == 0) {
if (strcmp((credentials + 6), expected) == 0) {
return TRUE;
}
}
return FALSE;
}
int auth_basic_handler(auth_basic_data_t *d,
onion_request *req,
onion_response *res) {
const char *credentials = onion_request_get_header(req, "Authorization");
if (authenticate(d->b64credentials, credentials)) {
return onion_handler_handle(d->inside, req, res);
}
onion_response_set_header(res, "WWW-Authenticate", "Basic realm=\"sist2\"");
onion_response_set_code(res, HTTP_UNAUTHORIZED);
onion_response_write(res, UNAUTHORIZED_TEXT, sizeof(UNAUTHORIZED_TEXT));
onion_response_set_length(res, sizeof(UNAUTHORIZED_TEXT));
return OCS_PROCESSED;
}
void auth_basic_free(auth_basic_data_t *data) {
onion_handler_free(data->inside);
free(data);
}
onion_handler *auth_basic(const char *b64credentials, onion_handler *inside_level) {
auth_basic_data_t *privdata = malloc(sizeof(auth_basic_data_t));
privdata->b64credentials = b64credentials;
privdata->inside = inside_level;
return onion_handler_new((onion_handler_handler) auth_basic_handler, privdata,
(onion_handler_private_data_free) auth_basic_free);
}

6
src/web/auth_basic.h
View File

@ -1,6 +0,0 @@
#include "src/sist.h"
#include <onion/handler.h>
#include <onion/onion.h>
onion_handler *auth_basic(const char *b64credentials, onion_handler *inside_level);

119
src/web/serve.c
View File

@ -5,15 +5,9 @@
#include "static_generated.c"
#include "src/index/elastic.h"
#include "src/index/web.h"
#include "src/web/auth_basic.h"
#include <src/ctx.h>
#include <onion/onion.h>
#include <onion/handler.h>
#include <onion/block.h>
#include <onion/shortcuts.h>
#include <mongoose.h>
#define CHUNK_SIZE 1024 * 1024 * 10
@ -119,7 +113,7 @@ int img_sprite_skin_flat(struct mg_connection *nc, struct http_message *hm) {
void thumbnail(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
if (path->len != UUID_STR_LEN * 2 + 2) {
LOG_DEBUGF("serve.c", "Invalid thumnail path: %.*s", (int) path->len, path->p);
LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) path->len, path->p);
nc->flags |= MG_F_SEND_AND_CLOSE;
return;
}
@ -176,22 +170,23 @@ void search(struct mg_connection *nc, struct http_message *hm) {
free(body);
}
int serve_file_from_url(cJSON *json, index_t *idx, onion_request *req, onion_response *res) {
const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
char url[8196];
snprintf(url, sizeof(url),
"%s%s/%s%s%s",
idx->desc.rewrite_url, path, name, strlen(ext) == 0 ? "" : ".", ext);
dyn_buffer_t encoded = url_escape(url);
int ret = onion_shortcut_redirect(encoded.buf, req, res);
dyn_buffer_destroy(&encoded);
return ret;
}
//TODO
//int serve_file_from_url(cJSON *json, index_t *idx, onion_request *req, onion_response *res) {
//
// const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
// const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
// const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
//
// char url[8196];
// snprintf(url, sizeof(url),
// "%s%s/%s%s%s",
// idx->desc.rewrite_url, path, name, strlen(ext) == 0 ? "" : ".", ext);
//
// dyn_buffer_t encoded = url_escape(url);
// int ret = onion_shortcut_redirect(encoded.buf, req, res);
// dyn_buffer_destroy(&encoded);
// return ret;
//}
void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, struct http_message *hm) {
@ -240,36 +235,41 @@ int index_info(struct mg_connection *nc) {
}
int document_info(UNUSED(void *p), onion_request *req, onion_response *res) {
void document_info(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
const char *arg_uuid = onion_request_get_query(req, "1");
if (arg_uuid == NULL) {
return OCS_PROCESSED;
if (path->len != UUID_STR_LEN + 2) {
LOG_DEBUGF("serve.c", "Invalid document_info path: %.*s", (int) path->len, path->p);
nc->flags |= MG_F_SEND_AND_CLOSE;
return;
}
char arg_uuid[UUID_STR_LEN];
memcpy(arg_uuid, hm->uri.p + 3, UUID_STR_LEN);
*(arg_uuid + UUID_STR_LEN - 1) = '\0';
cJSON *doc = elastic_get_document(arg_uuid);
cJSON *source = cJSON_GetObjectItem(doc, "_source");
cJSON *index_id = cJSON_GetObjectItem(source, "index");
if (index_id == NULL) {
cJSON_Delete(doc);
return OCS_NOT_PROCESSED;
nc->flags |= MG_F_SEND_AND_CLOSE;
return;
}
index_t *idx = get_index_by_id(index_id->valuestring);
if (idx == NULL) {
cJSON_Delete(doc);
return OCS_NOT_PROCESSED;
nc->flags |= MG_F_SEND_AND_CLOSE;
return;
}
onion_response_set_header(res, "Content-Type", "application/json");
char *json_str = cJSON_PrintUnformatted(source);
onion_response_write0(res, json_str);
send_response_line(nc, 200, (int)strlen(json_str), "Content-Type: application/json");
free(json_str);
cJSON_Delete(doc);
return OCS_PROCESSED;
nc->flags |= MG_F_SEND_AND_CLOSE;
}
void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
@ -322,19 +322,17 @@ void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path
cJSON_Delete(doc);
}
int status(UNUSED(void *p), UNUSED(onion_request *req), onion_response *res) {
onion_response_set_header(res, "Content-Type", "application/x-empty");
int status(struct mg_connection *nc) {
char *status = elastic_get_status();
if (strcmp(status, "open") == 0) {
onion_response_set_code(res, 204);
send_response_line(nc, 204, 0, "Content-Type: application/json");
} else {
onion_response_set_code(res, 500);
send_response_line(nc, 500, 0, "Content-Type: application/json");
}
free(status);
return OCS_PROCESSED;
nc->flags |= MG_F_SEND_AND_CLOSE;
}
typedef struct {
@ -370,16 +368,18 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
search(nc, hm);
} else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/i")))) {
index_info(nc);
} else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/status")))) {
status(nc);
} else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/f/")))) {
file(nc, hm, &path);
} else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/t/")))) {
thumbnail(nc, hm, &path);
} else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/d/")))) {
document_info(nc, hm, &path);
} else {
nc->flags |= MG_F_SEND_AND_CLOSE;
}
} else if (ev == MG_EV_POLL) {
if (nc->user_data != NULL) {
//Waiting for ES reply
@ -419,6 +419,8 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
void serve(const char *hostname, const char *port) {
printf("Starting web server @ http://%s:%s\n", hostname, port);
struct mg_mgr mgr;
mg_mgr_init(&mgr, NULL);
@ -433,38 +435,5 @@ void serve(const char *hostname, const char *port) {
mg_mgr_poll(&mgr, 10);
}
onion *o = onion_new(O_POOL);
onion_set_timeout(o, 3500);
onion_set_hostname(o, hostname);
onion_set_port(o, port);
onion_url *urls = onion_url_new();
// Static paths
onion_set_root_handler(o, auth_basic(WebCtx.b64credentials, onion_url_to_handler(urls)));
onion_url_add(urls, "", search_index);
onion_url_add(urls, "css", style);
onion_url_add(urls, "js", javascript);
onion_url_add(urls, "img/sprite-skin-flat.png", img_sprite_skin_flat);
onion_url_add(urls, "es", search);
onion_url_add(urls, "status", status);
onion_url_add(
urls,
"^t/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})/"
"([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})$",
thumbnail
);
onion_url_add(urls, "^f/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})$", file);
onion_url_add(urls, "^d/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})$",
document_info);
onion_url_add(urls, "i", index_info);
printf("Starting web server @ http://%s:%s\n", hostname, port);
onion_listen(o);
onion_free(o);
// onion_set_root_handler(o, auth_basic(WebCtx.b64credentials, onion_url_to_handler(urls)));
}