From 8c662bb8f844aa6dd1a78831e7f13aef3ef5747c Mon Sep 17 00:00:00 2001 From: simon987 Date: Mon, 27 Feb 2023 20:44:25 -0500 Subject: [PATCH 1/7] Adjust some structs --- src/io/serialize.c | 1 - src/io/walk.c | 2 +- src/parsing/parse.c | 5 ----- third-party/libscan/libscan/scan.h | 5 ++--- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/io/serialize.c b/src/io/serialize.c index 0d7d709..f00d737 100644 --- a/src/io/serialize.c +++ b/src/io/serialize.c @@ -248,7 +248,6 @@ void write_document_func(void *arg) { zstd_write_string(json_str, json_str_len + 1); free(json_str); - free(doc->filepath); } void zstd_close() { diff --git a/src/io/walk.c b/src/io/walk.c index f87f5c6..cdaeda5 100644 --- a/src/io/walk.c +++ b/src/io/walk.c @@ -9,7 +9,7 @@ __always_inline parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info, int base) { int len = (int) strlen(filepath); - parse_job_t *job = malloc(sizeof(parse_job_t) + len); + parse_job_t *job = malloc(sizeof(parse_job_t)); strcpy(job->filepath, filepath); job->base = base; diff --git a/src/parsing/parse.c b/src/parsing/parse.c index 10435d7..b6882bc 100644 --- a/src/parsing/parse.c +++ b/src/parsing/parse.c @@ -61,7 +61,6 @@ void parse(void *arg) { parse_job_t *job = arg; document_t *doc = malloc(sizeof(document_t)); - doc->filepath = malloc(strlen(job->filepath) + 1); set_dbg_current_file(job); @@ -230,10 +229,6 @@ void parse(void *arg) { meta_parent->key = MetaParent; strcpy(meta_parent->str_val, job->parent); APPEND_META((doc), meta_parent) - - doc->has_parent = TRUE; - } else { - doc->has_parent = FALSE; } CLOSE_FILE(job->vfile) diff --git a/third-party/libscan/libscan/scan.h b/third-party/libscan/libscan/scan.h index f474a47..187161e 100644 --- a/third-party/libscan/libscan/scan.h +++ b/third-party/libscan/libscan/scan.h @@ -112,10 +112,9 @@ typedef struct document { int mtime; short base; short ext; - char has_parent; meta_line_t *meta_head; meta_line_t *meta_tail; - char *filepath; + char filepath[PATH_MAX]; } document_t; typedef struct vfile vfile_t; @@ -163,7 +162,7 @@ typedef struct parse_job_t { int ext; struct vfile vfile; char parent[SIST_DOC_ID_LEN]; - char filepath[1]; + char filepath[PATH_MAX]; } parse_job_t; From f8abffba81236e699a9075923ae30a22ad06c281 Mon Sep 17 00:00:00 2001 From: simon987 Date: Thu, 9 Mar 2023 22:11:21 -0500 Subject: [PATCH 2/7] process pool mostly works, still WIP --- CMakeLists.txt | 1 + docs/USAGE.md | 2 + src/cli.c | 4 +- src/index/elastic.c | 27 +- src/io/serialize.c | 32 +- src/io/store.c | 150 +++-- src/io/store.h | 18 +- src/io/walk.c | 20 +- src/main.c | 17 +- src/mempool/mempool.c | 757 ++++++++++++++++++++++ src/mempool/mempool.h | 62 ++ src/parsing/parse.c | 31 +- src/parsing/parse.h | 4 +- src/tpool.c | 255 ++++---- src/tpool.h | 19 +- src/util.c | 2 + third-party/libscan/libscan/arc/arc.c | 22 +- third-party/libscan/libscan/ebook/ebook.c | 14 +- third-party/libscan/libscan/ebook/ebook.h | 1 - third-party/libscan/libscan/json/json.c | 2 +- third-party/libscan/libscan/media/media.c | 8 +- third-party/libscan/libscan/scan.h | 14 +- third-party/libscan/libscan/text/text.c | 4 +- third-party/libscan/libscan/util.h | 6 +- third-party/libscan/test/test_util.cpp | 14 +- 25 files changed, 1219 insertions(+), 267 deletions(-) create mode 100644 src/mempool/mempool.c create mode 100644 src/mempool/mempool.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d7517ad..6857720 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ add_executable(sist2 src/cli.c src/cli.h src/stats.c src/stats.h src/ctx.c src/parsing/sidecar.c src/parsing/sidecar.h + src/mempool/mempool.c src/mempool/mempool.h src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp diff --git a/docs/USAGE.md b/docs/USAGE.md index 3dfc31d..dffe11a 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -164,6 +164,8 @@ that is about `8000000 * 36kB = 288GB`. ![thumbnail_size](thumbnail_size.png) +// TODO: add note about LMDB page size 4096 + ### Scan examples Simple scan diff --git a/src/cli.c b/src/cli.c index 7349aa3..1bafba6 100644 --- a/src/cli.c +++ b/src/cli.c @@ -140,8 +140,8 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { if (args->threads == 0) { args->threads = 1; - } else if (args->threads < 0) { - fprintf(stderr, "Invalid value for --threads: %d. Must be a positive number\n", args->threads); + } else if (args->threads < 0 || args->threads > 256) { + fprintf(stderr, "Invalid value for --threads: %d. Must be a positive number <= 256\n", args->threads); return 1; } diff --git a/src/index/elastic.c b/src/index/elastic.c index 0012767..0b41b07 100644 --- a/src/index/elastic.c +++ b/src/index/elastic.c @@ -64,18 +64,26 @@ void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) { cJSON_Delete(line); } -void index_json_func(void *arg) { - es_bulk_line_t *line = arg; +void index_json_func(tpool_work_arg_shm_t *arg) { + // Copy arg to heap because it's going to be free immediately after this function returns + es_bulk_line_t *line = malloc(arg->arg_size); + memcpy(line, arg->arg, arg->arg_size); + elastic_index_line(line); } -void delete_document(const char* document_id_str, void* UNUSED(_data)) { +void delete_document(const char *document_id_str, void *UNUSED(_data)) { es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t)); + bulk_line->type = ES_BULK_LINE_DELETE; bulk_line->next = NULL; - strcpy(bulk_line->doc_id, document_id_str); - tpool_add_work(IndexCtx.pool, index_json_func, bulk_line); + + tpool_work_arg_t arg = { + .arg_size = sizeof(es_bulk_line_t), + .arg = bulk_line + }; + tpool_add_work(IndexCtx.pool, index_json_func, &arg); } @@ -92,7 +100,11 @@ void index_json(cJSON *document, const char doc_id[SIST_DOC_ID_LEN]) { bulk_line->next = NULL; cJSON_free(json); - tpool_add_work(IndexCtx.pool, index_json_func, bulk_line); + tpool_work_arg_t arg = { + .arg_size = sizeof(es_bulk_line_t) + json_len + 2, + .arg = bulk_line + }; + tpool_add_work(IndexCtx.pool, index_json_func, &arg); } void execute_update_script(const char *script, int async, const char index_id[SIST_INDEX_ID_LEN]) { @@ -538,7 +550,8 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s free_response(r); if (IS_LEGACY_VERSION(es_version)) { - snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index); + snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, + IndexCtx.es_index); } else { snprintf(url, sizeof(url), "%s/%s/_mappings", IndexCtx.es_url, IndexCtx.es_index); } diff --git a/src/io/serialize.c b/src/io/serialize.c index f00d737..c438025 100644 --- a/src/io/serialize.c +++ b/src/io/serialize.c @@ -197,7 +197,7 @@ static struct { ZSTD_CCtx *cctx; } WriterCtx = { - .out_file = NULL + .out_file = NULL }; #define ZSTD_COMPRESSION_LEVEL 10 @@ -229,7 +229,9 @@ void zstd_write_string(const char *string, const size_t len) { } while (input.pos != input.size); } -void write_document_func(void *arg) { +void write_document_func(tpool_work_arg_shm_t *arg) { + + const char *json_str = arg->arg; if (WriterCtx.out_file == NULL) { char dstfile[PATH_MAX]; @@ -237,17 +239,7 @@ void write_document_func(void *arg) { initialize_writer_ctx(dstfile); } - document_t *doc = arg; - - char *json_str = build_json_string(doc); - const size_t json_str_len = strlen(json_str); - - json_str = realloc(json_str, json_str_len + 1); - *(json_str + json_str_len) = '\n'; - - zstd_write_string(json_str, json_str_len + 1); - - free(json_str); + zstd_write_string(json_str, arg->arg_size); } void zstd_close() { @@ -345,7 +337,19 @@ index_descriptor_t read_index_descriptor(char *path) { void write_document(document_t *doc) { - tpool_add_work(ScanCtx.writer_pool, write_document_func, doc); + char *json_str = build_json_string(doc); + free(doc); + const size_t json_str_len = strlen(json_str); + + json_str = realloc(json_str, json_str_len + 1); + *(json_str + json_str_len) = '\n'; + + tpool_work_arg_t arg = { + .arg_size = json_str_len + 1, + .arg = json_str + }; + + tpool_add_work(ScanCtx.writer_pool, write_document_func, &arg); } void thread_cleanup() { diff --git a/src/io/store.c b/src/io/store.c index 1cdc754..2e03e8d 100644 --- a/src/io/store.c +++ b/src/io/store.c @@ -1,18 +1,13 @@ +#include #include "store.h" #include "src/ctx.h" -store_t *store_create(const char *path, size_t chunk_size) { - store_t *store = malloc(sizeof(struct store_t)); - mkdir(path, S_IWUSR | S_IRUSR | S_IXUSR); - strcpy(store->path, path); +//#define SIST_FAKE_STORE 1 -#if (SIST_FAKE_STORE != 1) - store->chunk_size = chunk_size; - pthread_rwlock_init(&store->lock, NULL); +void open_env(const char *path, MDB_env **env, MDB_dbi *dbi) { + mdb_env_create(env); - mdb_env_create(&store->env); - - int open_ret = mdb_env_open(store->env, + int open_ret = mdb_env_open(*env, path, MDB_WRITEMAP | MDB_MAPASYNC, S_IRUSR | S_IWUSR @@ -22,14 +17,33 @@ store_t *store_create(const char *path, size_t chunk_size) { LOG_FATALF("store.c", "Error while opening store: %s (%s)\n", mdb_strerror(open_ret), path) } - store->size = (size_t) store->chunk_size; - mdb_env_set_mapsize(store->env, store->size); - - // Open dbi MDB_txn *txn; - mdb_txn_begin(store->env, NULL, 0, &txn); - mdb_dbi_open(txn, NULL, 0, &store->dbi); + mdb_txn_begin(*env, NULL, 0, &txn); + mdb_dbi_open(txn, NULL, 0, dbi); mdb_txn_commit(txn); +} + +store_t *store_create(const char *path, size_t chunk_size) { + store_t *store = calloc(1, sizeof(struct store_t)); + mkdir(path, S_IWUSR | S_IRUSR | S_IXUSR); + strcpy(store->path, path); + + MDB_env *env; + MDB_dbi dbi; + +#if (SIST_FAKE_STORE != 1) + store->chunk_size = chunk_size; + + store->shared_memory = mmap(NULL, sizeof(*store->shm), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + store->shm = store->shared_memory; + + open_env(path, &env, &dbi); + + store->shm->size = (size_t) store->chunk_size; + mdb_env_set_mapsize(env, store->shm->size); + + // Close, child processes will open the environment again + mdb_env_close(env); #endif return store; @@ -37,26 +51,35 @@ store_t *store_create(const char *path, size_t chunk_size) { void store_destroy(store_t *store) { + LOG_DEBUG("store.c", "store_destroy()") #if (SIST_FAKE_STORE != 1) - pthread_rwlock_destroy(&store->lock); - mdb_dbi_close(store->env, store->dbi); - mdb_env_close(store->env); + munmap(store->shared_memory, sizeof(*store->shm)); + + mdb_dbi_close(store->proc.env, store->proc.dbi); + mdb_env_close(store->proc.env); #endif free(store); } void store_flush(store_t *store) { - mdb_env_sync(store->env, TRUE); + mdb_env_sync(store->proc.env, TRUE); } void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len) { + ScanCtx.stat_tn_size += buf_len; + if (LogCtx.very_verbose) { LOG_DEBUGF("store.c", "Store write %s@{%s} %lu bytes", store->path, key, buf_len) } #if (SIST_FAKE_STORE != 1) + if (store->proc.env == NULL) { + open_env(store->path, &store->proc.env, &store->proc.dbi); + LOG_DEBUGF("store.c", "Opening mdb environment %s", store->path) + } + MDB_val mdb_key; mdb_key.mv_data = key; mdb_key.mv_size = key_len; @@ -66,70 +89,80 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu mdb_value.mv_size = buf_len; MDB_txn *txn; - pthread_rwlock_rdlock(&store->lock); - mdb_txn_begin(store->env, NULL, 0, &txn); - - int put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0); - ScanCtx.stat_tn_size += buf_len; int db_full = FALSE; + int put_ret = 0; int should_abort_transaction = FALSE; + int should_increase_size = TRUE; - if (put_ret == MDB_MAP_FULL) { + int begin_ret = mdb_txn_begin(store->proc.env, NULL, 0, &txn); + + if (begin_ret == MDB_MAP_RESIZED) { + // mapsize was increased by another process. We don't need to increase the size again, but we need + // to update the size of the environment for the current process. db_full = TRUE; - should_abort_transaction = TRUE; + should_increase_size = FALSE; } else { - int commit_ret = mdb_txn_commit(txn); + put_ret = mdb_put(txn, store->proc.dbi, &mdb_key, &mdb_value, 0); - if (commit_ret == MDB_MAP_FULL) { + if (put_ret == MDB_MAP_FULL) { + // Database is full, we need to increase the environment size db_full = TRUE; + should_abort_transaction = TRUE; + } else { + int commit_ret = mdb_txn_commit(txn); + + if (commit_ret == MDB_MAP_FULL) { + db_full = TRUE; + } } } if (db_full) { - LOG_DEBUGF("store.c", "Updating mdb mapsize to %lu bytes", store->size) + LOG_DEBUGF("store.c", "Updating mdb mapsize to %lu bytes", store->shm->size) if (should_abort_transaction) { mdb_txn_abort(txn); } - pthread_rwlock_unlock(&store->lock); - - // Cannot resize when there is a opened transaction. + // Cannot resize when there is an opened transaction in this process. // Resize take effect on the next commit. - pthread_rwlock_wrlock(&store->lock); - store->size += store->chunk_size; - int resize_ret = mdb_env_set_mapsize(store->env, store->size); - if (resize_ret != 0) { - LOG_ERROR("store.c", mdb_strerror(put_ret)) + if (should_increase_size) { + store->shm->size += store->chunk_size; } - mdb_txn_begin(store->env, NULL, 0, &txn); - int put_ret_retry = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0); + int resize_ret = mdb_env_set_mapsize(store->proc.env, store->shm->size); + if (resize_ret != 0) { + LOG_ERRORF("store.c", "mdb_env_set_mapsize() failed: %s", mdb_strerror(resize_ret)) + } + mdb_txn_begin(store->proc.env, NULL, 0, &txn); + int put_ret_retry = mdb_put(txn, store->proc.dbi, &mdb_key, &mdb_value, 0); if (put_ret_retry != 0) { - LOG_ERROR("store.c", mdb_strerror(put_ret)) + LOG_ERRORF("store.c", "mdb_put() (retry) failed: %s", mdb_strerror(put_ret_retry)) } int ret = mdb_txn_commit(txn); if (ret != 0) { LOG_FATALF("store.c", "FIXME: Could not commit to store %s: %s (%d), %d, %d %d", store->path, mdb_strerror(ret), ret, - put_ret, put_ret_retry); + ret, put_ret_retry) } - LOG_DEBUGF("store.c", "Updated mdb mapsize to %lu bytes", store->size) + LOG_DEBUGF("store.c", "Updated mdb mapsize to %lu bytes", store->shm->size) } else if (put_ret != 0) { - LOG_ERROR("store.c", mdb_strerror(put_ret)) + LOG_ERRORF("store.c", "mdb_put() failed: %s", mdb_strerror(put_ret)) } - pthread_rwlock_unlock(&store->lock); - #endif } -char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen) { +char *store_read(store_t *store, char *key, size_t key_len, size_t *return_value_len) { char *buf = NULL; #if (SIST_FAKE_STORE != 1) + if (store->proc.env == NULL) { + open_env(store->path, &store->proc.env, &store->proc.dbi); + } + MDB_val mdb_key; mdb_key.mv_data = key; mdb_key.mv_size = key_len; @@ -137,14 +170,14 @@ char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen) MDB_val mdb_value; MDB_txn *txn; - mdb_txn_begin(store->env, NULL, MDB_RDONLY, &txn); + mdb_txn_begin(store->proc.env, NULL, MDB_RDONLY, &txn); - int get_ret = mdb_get(txn, store->dbi, &mdb_key, &mdb_value); + int get_ret = mdb_get(txn, store->proc.dbi, &mdb_key, &mdb_value); if (get_ret == MDB_NOTFOUND) { - *ret_vallen = 0; + *return_value_len = 0; } else { - *ret_vallen = mdb_value.mv_size; + *return_value_len = mdb_value.mv_size; buf = malloc(mdb_value.mv_size); memcpy(buf, mdb_value.mv_data, mdb_value.mv_size); } @@ -156,15 +189,20 @@ char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen) GHashTable *store_read_all(store_t *store) { + if (store->proc.env == NULL) { + open_env(store->path, &store->proc.env, &store->proc.dbi); + LOG_DEBUGF("store.c", "Opening mdb environment %s", store->path) + } + int count = 0; GHashTable *table = g_hash_table_new_full(g_str_hash, g_str_equal, free, free); MDB_txn *txn = NULL; - mdb_txn_begin(store->env, NULL, MDB_RDONLY, &txn); + mdb_txn_begin(store->proc.env, NULL, MDB_RDONLY, &txn); MDB_cursor *cur = NULL; - mdb_cursor_open(txn, store->dbi, &cur); + mdb_cursor_open(txn, store->proc.dbi, &cur); MDB_val key; MDB_val value; @@ -180,8 +218,8 @@ GHashTable *store_read_all(store_t *store) { } const char *path; - mdb_env_get_path(store->env, &path); - LOG_DEBUGF("store.c", "Read %d entries from %s", count, path); + mdb_env_get_path(store->proc.env, &path); + LOG_DEBUGF("store.c", "Read %d entries from %s", count, path) mdb_cursor_close(cur); mdb_txn_abort(txn); @@ -191,5 +229,5 @@ GHashTable *store_read_all(store_t *store) { void store_copy(store_t *store, const char *destination) { mkdir(destination, S_IWUSR | S_IRUSR | S_IXUSR); - mdb_env_copy(store->env, destination); + mdb_env_copy(store->proc.env, destination); } diff --git a/src/io/store.h b/src/io/store.h index a1f7217..b6c8472 100644 --- a/src/io/store.h +++ b/src/io/store.h @@ -10,14 +10,20 @@ #define STORE_SIZE_TAG (1024 * 1024) #define STORE_SIZE_META STORE_SIZE_TAG + typedef struct store_t { char path[PATH_MAX]; - char *tmp_path; - MDB_dbi dbi; - MDB_env *env; - size_t size; size_t chunk_size; - pthread_rwlock_t lock; + void *shared_memory; + + struct { + MDB_dbi dbi; + MDB_env *env; + } proc; + + struct { + size_t size; + } *shm; } store_t; store_t *store_create(const char *path, size_t chunk_size); @@ -28,7 +34,7 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu void store_flush(store_t *store); -char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen); +char *store_read(store_t *store, char *key, size_t key_len, size_t *return_value_len); GHashTable *store_read_all(store_t *store); diff --git a/src/io/walk.c b/src/io/walk.c index cdaeda5..b6019cf 100644 --- a/src/io/walk.c +++ b/src/io/walk.c @@ -20,11 +20,13 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info, job->ext = len; } - job->vfile.info = *info; + job->vfile.st_size = info->st_size; + job->vfile.st_mode = info->st_mode; + job->vfile.mtime = (int) info->st_mtim.tv_sec; job->parent[0] = '\0'; - job->vfile.filepath = job->filepath; + memcpy(job->vfile.filepath, job->filepath, sizeof(job->vfile.filepath)); job->vfile.read = fs_read; // Filesystem reads are always rewindable job->vfile.read_rewindable = fs_read; @@ -68,7 +70,12 @@ int handle_entry(const char *filepath, const struct stat *info, int typeflag, st if (typeflag == FTW_F && S_ISREG(info->st_mode)) { parse_job_t *job = create_fs_parse_job(filepath, info, ftw->base); - tpool_add_work(ScanCtx.pool, parse, job); + + tpool_work_arg_t arg = { + .arg_size = sizeof(parse_job_t), + .arg = job + }; + tpool_add_work(ScanCtx.pool, parse, &arg); } return FTW_CONTINUE; @@ -128,7 +135,12 @@ int iterate_file_list(void *input_file) { parse_job_t *job = create_fs_parse_job(absolute_path, &info, base); free(absolute_path); - tpool_add_work(ScanCtx.pool, parse, job); + + tpool_work_arg_t arg = { + .arg = job, + .arg_size = sizeof(parse_job_t) + }; + tpool_add_work(ScanCtx.pool, parse, &arg); } return 0; diff --git a/src/main.c b/src/main.c index d5ad271..03e40a5 100644 --- a/src/main.c +++ b/src/main.c @@ -188,7 +188,7 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.arc_ctx.mode = args->archive_mode; ScanCtx.arc_ctx.log = _log; ScanCtx.arc_ctx.logf = _logf; - ScanCtx.arc_ctx.parse = (parse_callback_t) parse; + ScanCtx.arc_ctx.parse = (parse_callback_t) parse_job; if (args->archive_passphrase != NULL) { strcpy(ScanCtx.arc_ctx.passphrase, args->archive_passphrase); } else { @@ -206,7 +206,6 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.comic_ctx.cbz_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbz"); // Ebook - pthread_mutex_init(&ScanCtx.ebook_ctx.mupdf_mutex, NULL); ScanCtx.ebook_ctx.content_size = args->content_size; ScanCtx.ebook_ctx.enable_tn = args->tn_count > 0; ScanCtx.ebook_ctx.tn_size = args->tn_size; @@ -407,12 +406,12 @@ void sist2_scan(scan_args_t *args) { load_incremental_index(args); } - ScanCtx.pool = tpool_create(ScanCtx.threads, thread_cleanup, TRUE, TRUE, ScanCtx.mem_limit); - tpool_start(ScanCtx.pool); - - ScanCtx.writer_pool = tpool_create(1, writer_cleanup, TRUE, FALSE, 0); + ScanCtx.writer_pool = tpool_create(1, writer_cleanup, FALSE, 0); tpool_start(ScanCtx.writer_pool); + ScanCtx.pool = tpool_create(ScanCtx.threads, thread_cleanup, TRUE, ScanCtx.mem_limit); + tpool_start(ScanCtx.pool); + if (args->list_path) { // Scan using file list int list_ret = iterate_file_list(args->list_file); @@ -494,7 +493,7 @@ void sist2_index(index_args_t *args) { f = index_json; } - IndexCtx.pool = tpool_create(args->threads, elastic_cleanup, FALSE, args->print == 0, 0); + IndexCtx.pool = tpool_create(args->threads, elastic_cleanup, args->print == 0, 0); tpool_start(IndexCtx.pool); READ_INDICES(file_path, args->index_path, { @@ -616,8 +615,8 @@ int set_to_negative_if_value_is_zero(struct argparse *self, const struct argpars int main(int argc, const char *argv[]) { - sigsegv_handler = signal(SIGSEGV, sig_handler); - sigabrt_handler = signal(SIGABRT, sig_handler); +// sigsegv_handler = signal(SIGSEGV, sig_handler); +// sigabrt_handler = signal(SIGABRT, sig_handler); setlocale(LC_ALL, ""); diff --git a/src/mempool/mempool.c b/src/mempool/mempool.c new file mode 100644 index 0000000..d8dd0a8 --- /dev/null +++ b/src/mempool/mempool.c @@ -0,0 +1,757 @@ +#include "mempool.h" +#include + +#define NCX_SLAB_PAGE_MASK 3 +#define NCX_SLAB_PAGE 0 +#define NCX_SLAB_BIG 1 +#define NCX_SLAB_EXACT 2 +#define NCX_SLAB_SMALL 3 + +#define NCX_SLAB_PAGE_FREE 0 +#define NCX_SLAB_PAGE_BUSY 0xffffffffffffffff +#define NCX_SLAB_PAGE_START 0x8000000000000000 + +#define NCX_SLAB_SHIFT_MASK 0x000000000000000f +#define NCX_SLAB_MAP_MASK 0xffffffff00000000 +#define NCX_SLAB_MAP_SHIFT 32 + +#define NCX_SLAB_BUSY 0xffffffffffffffff + + +static ncx_slab_page_t *ncx_slab_alloc_pages(ncx_slab_pool_t *pool, ncx_uint_t pages); + +static void ncx_slab_free_pages(ncx_slab_pool_t *pool, ncx_slab_page_t *page, ncx_uint_t pages); + +static bool ncx_slab_empty(ncx_slab_pool_t *pool, ncx_slab_page_t *page); + +static ncx_uint_t ncx_slab_max_size; +static ncx_uint_t ncx_slab_exact_size; +static ncx_uint_t ncx_slab_exact_shift; +static ncx_uint_t ncx_pagesize; +static ncx_uint_t ncx_pagesize_shift; +static ncx_uint_t ncx_real_pages; + +void ncx_slab_init(ncx_slab_pool_t *pool) { + u_char *p; + size_t size; + ncx_uint_t i, n, pages; + ncx_slab_page_t *slots; + + /*pagesize*/ + ncx_pagesize = getpagesize(); + for (n = ncx_pagesize, ncx_pagesize_shift = 0; + n >>= 1; ncx_pagesize_shift++) { /* void */ } + + /* STUB */ + if (ncx_slab_max_size == 0) { + ncx_slab_max_size = ncx_pagesize / 2; + ncx_slab_exact_size = ncx_pagesize / (8 * sizeof(uintptr_t)); + for (n = ncx_slab_exact_size; n >>= 1; ncx_slab_exact_shift++) { + /* void */ + } + } + + pool->min_size = 1 << pool->min_shift; + + p = (u_char *) pool + sizeof(ncx_slab_pool_t); + slots = (ncx_slab_page_t *) p; + + n = ncx_pagesize_shift - pool->min_shift; + for (i = 0; i < n; i++) { + slots[i].slab = 0; + slots[i].next = &slots[i]; + slots[i].prev = 0; + } + + p += n * sizeof(ncx_slab_page_t); + + size = pool->end - p; + + pages = (ncx_uint_t) (size / (ncx_pagesize + sizeof(ncx_slab_page_t))); + + ncx_memzero(p, pages * sizeof(ncx_slab_page_t)); + + pool->pages = (ncx_slab_page_t *) p; + + pool->free.prev = 0; + pool->free.next = (ncx_slab_page_t *) p; + + pool->pages->slab = pages; + pool->pages->next = &pool->free; + pool->pages->prev = (uintptr_t) &pool->free; + + pool->start = (u_char *) + ncx_align_ptr((uintptr_t) p + pages * sizeof(ncx_slab_page_t), + ncx_pagesize); + + ncx_real_pages = (pool->end - pool->start) / ncx_pagesize; + pool->pages->slab = ncx_real_pages; +} + + +void *ncx_slab_alloc(ncx_slab_pool_t *pool, size_t size) { + size_t s; + uintptr_t p, n, m, mask, *bitmap; + ncx_uint_t i, slot, shift, map; + ncx_slab_page_t *page, *prev, *slots; + + if (size >= ncx_slab_max_size) { + + page = ncx_slab_alloc_pages(pool, (size >> ncx_pagesize_shift) + + ((size % ncx_pagesize) ? 1 : 0)); + if (page) { + p = (page - pool->pages) << ncx_pagesize_shift; + p += (uintptr_t) pool->start; + + } else { + p = 0; + } + + goto done; + } + + if (size > pool->min_size) { + shift = 1; + for (s = size - 1; s >>= 1; shift++) { /* void */ } + slot = shift - pool->min_shift; + + } else { + shift = pool->min_shift; + slot = 0; + } + + slots = (ncx_slab_page_t *) ((u_char *) pool + sizeof(ncx_slab_pool_t)); + page = slots[slot].next; + + if (page->next != page) { + + if (shift < ncx_slab_exact_shift) { + + do { + p = (page - pool->pages) << ncx_pagesize_shift; + bitmap = (uintptr_t *) (pool->start + p); + + map = (1 << (ncx_pagesize_shift - shift)) + / (sizeof(uintptr_t) * 8); + + for (n = 0; n < map; n++) { + + if (bitmap[n] != NCX_SLAB_BUSY) { + + for (m = 1, i = 0; m; m <<= 1, i++) { + if ((bitmap[n] & m)) { + continue; + } + + bitmap[n] |= m; + + i = ((n * sizeof(uintptr_t) * 8) << shift) + + (i << shift); + + if (bitmap[n] == NCX_SLAB_BUSY) { + for (n = n + 1; n < map; n++) { + if (bitmap[n] != NCX_SLAB_BUSY) { + p = (uintptr_t) bitmap + i; + + goto done; + } + } + + prev = (ncx_slab_page_t *) + (page->prev & ~NCX_SLAB_PAGE_MASK); + prev->next = page->next; + page->next->prev = page->prev; + + page->next = NULL; + page->prev = NCX_SLAB_SMALL; + } + + p = (uintptr_t) bitmap + i; + + goto done; + } + } + } + + page = page->next; + + } while (page); + + } else if (shift == ncx_slab_exact_shift) { + + do { + if (page->slab != NCX_SLAB_BUSY) { + + for (m = 1, i = 0; m; m <<= 1, i++) { + if ((page->slab & m)) { + continue; + } + + page->slab |= m; + + if (page->slab == NCX_SLAB_BUSY) { + prev = (ncx_slab_page_t *) + (page->prev & ~NCX_SLAB_PAGE_MASK); + prev->next = page->next; + page->next->prev = page->prev; + + page->next = NULL; + page->prev = NCX_SLAB_EXACT; + } + + p = (page - pool->pages) << ncx_pagesize_shift; + p += i << shift; + p += (uintptr_t) pool->start; + + goto done; + } + } + + page = page->next; + + } while (page); + + } else { /* shift > ncx_slab_exact_shift */ + + n = ncx_pagesize_shift - (page->slab & NCX_SLAB_SHIFT_MASK); + n = 1 << n; + n = ((uintptr_t) 1 << n) - 1; + mask = n << NCX_SLAB_MAP_SHIFT; + + do { + if ((page->slab & NCX_SLAB_MAP_MASK) != mask) { + + for (m = (uintptr_t) 1 << NCX_SLAB_MAP_SHIFT, i = 0; + m & mask; + m <<= 1, i++) { + if ((page->slab & m)) { + continue; + } + + page->slab |= m; + + if ((page->slab & NCX_SLAB_MAP_MASK) == mask) { + prev = (ncx_slab_page_t *) + (page->prev & ~NCX_SLAB_PAGE_MASK); + prev->next = page->next; + page->next->prev = page->prev; + + page->next = NULL; + page->prev = NCX_SLAB_BIG; + } + + p = (page - pool->pages) << ncx_pagesize_shift; + p += i << shift; + p += (uintptr_t) pool->start; + + goto done; + } + } + + page = page->next; + + } while (page); + } + } + + page = ncx_slab_alloc_pages(pool, 1); + + if (page) { + if (shift < ncx_slab_exact_shift) { + p = (page - pool->pages) << ncx_pagesize_shift; + bitmap = (uintptr_t *) (pool->start + p); + + s = 1 << shift; + n = (1 << (ncx_pagesize_shift - shift)) / 8 / s; + + if (n == 0) { + n = 1; + } + + bitmap[0] = (2 << n) - 1; + + map = (1 << (ncx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8); + + for (i = 1; i < map; i++) { + bitmap[i] = 0; + } + + page->slab = shift; + page->next = &slots[slot]; + page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_SMALL; + + slots[slot].next = page; + + p = ((page - pool->pages) << ncx_pagesize_shift) + s * n; + p += (uintptr_t) pool->start; + + goto done; + + } else if (shift == ncx_slab_exact_shift) { + + page->slab = 1; + page->next = &slots[slot]; + page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_EXACT; + + slots[slot].next = page; + + p = (page - pool->pages) << ncx_pagesize_shift; + p += (uintptr_t) pool->start; + + goto done; + + } else { /* shift > ncx_slab_exact_shift */ + + page->slab = ((uintptr_t) 1 << NCX_SLAB_MAP_SHIFT) | shift; + page->next = &slots[slot]; + page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_BIG; + + slots[slot].next = page; + + p = (page - pool->pages) << ncx_pagesize_shift; + p += (uintptr_t) pool->start; + + goto done; + } + } + + p = 0; + + done: + + return (void *) p; +} + + +void ncx_slab_free(ncx_slab_pool_t *pool, void *p) { + size_t size; + uintptr_t slab, m, *bitmap; + ncx_uint_t n, type, slot, shift, map; + ncx_slab_page_t *slots, *page; + + if ((u_char *) p < pool->start || (u_char *) p > pool->end) { +// error("ncx_slab_free(): outside of pool"); + goto fail; + } + + n = ((u_char *) p - pool->start) >> ncx_pagesize_shift; + page = &pool->pages[n]; + slab = page->slab; + type = page->prev & NCX_SLAB_PAGE_MASK; + + switch (type) { + + case NCX_SLAB_SMALL: + + shift = slab & NCX_SLAB_SHIFT_MASK; + size = 1 << shift; + + if ((uintptr_t) p & (size - 1)) { + goto wrong_chunk; + } + + n = ((uintptr_t) p & (ncx_pagesize - 1)) >> shift; + m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1)); + n /= (sizeof(uintptr_t) * 8); + bitmap = (uintptr_t *) ((uintptr_t) p & ~(ncx_pagesize - 1)); + + if (bitmap[n] & m) { + + if (page->next == NULL) { + slots = (ncx_slab_page_t *) + ((u_char *) pool + sizeof(ncx_slab_pool_t)); + slot = shift - pool->min_shift; + + page->next = slots[slot].next; + slots[slot].next = page; + + page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_SMALL; + page->next->prev = (uintptr_t) page | NCX_SLAB_SMALL; + } + + bitmap[n] &= ~m; + + n = (1 << (ncx_pagesize_shift - shift)) / 8 / (1 << shift); + + if (n == 0) { + n = 1; + } + + if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) { + goto done; + } + + map = (1 << (ncx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8); + + for (n = 1; n < map; n++) { + if (bitmap[n]) { + goto done; + } + } + + ncx_slab_free_pages(pool, page, 1); + + goto done; + } + + goto chunk_already_free; + + case NCX_SLAB_EXACT: + + m = (uintptr_t) 1 << + (((uintptr_t) p & (ncx_pagesize - 1)) >> ncx_slab_exact_shift); + size = ncx_slab_exact_size; + + if ((uintptr_t) p & (size - 1)) { + goto wrong_chunk; + } + + if (slab & m) { + if (slab == NCX_SLAB_BUSY) { + slots = (ncx_slab_page_t *) + ((u_char *) pool + sizeof(ncx_slab_pool_t)); + slot = ncx_slab_exact_shift - pool->min_shift; + + page->next = slots[slot].next; + slots[slot].next = page; + + page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_EXACT; + page->next->prev = (uintptr_t) page | NCX_SLAB_EXACT; + } + + page->slab &= ~m; + + if (page->slab) { + goto done; + } + + ncx_slab_free_pages(pool, page, 1); + + goto done; + } + + goto chunk_already_free; + + case NCX_SLAB_BIG: + + shift = slab & NCX_SLAB_SHIFT_MASK; + size = 1 << shift; + + if ((uintptr_t) p & (size - 1)) { + goto wrong_chunk; + } + + m = (uintptr_t) 1 << ((((uintptr_t) p & (ncx_pagesize - 1)) >> shift) + + NCX_SLAB_MAP_SHIFT); + + if (slab & m) { + + if (page->next == NULL) { + slots = (ncx_slab_page_t *) + ((u_char *) pool + sizeof(ncx_slab_pool_t)); + slot = shift - pool->min_shift; + + page->next = slots[slot].next; + slots[slot].next = page; + + page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_BIG; + page->next->prev = (uintptr_t) page | NCX_SLAB_BIG; + } + + page->slab &= ~m; + + if (page->slab & NCX_SLAB_MAP_MASK) { + goto done; + } + + ncx_slab_free_pages(pool, page, 1); + + goto done; + } + + goto chunk_already_free; + + case NCX_SLAB_PAGE: + + if ((uintptr_t) p & (ncx_pagesize - 1)) { + goto wrong_chunk; + } + + if (slab == NCX_SLAB_PAGE_FREE) { +// alert("ncx_slab_free(): page is already free"); + goto fail; + } + + if (slab == NCX_SLAB_PAGE_BUSY) { +// alert("ncx_slab_free(): pointer to wrong page"); + goto fail; + } + + n = ((u_char *) p - pool->start) >> ncx_pagesize_shift; + size = slab & ~NCX_SLAB_PAGE_START; + + ncx_slab_free_pages(pool, &pool->pages[n], size); + + return; + } + + /* not reached */ + + return; + + done: + + return; + + wrong_chunk: + +// error("ncx_slab_free(): pointer to wrong chunk"); + + goto fail; + + chunk_already_free: + +// error("ncx_slab_free(): chunk is already free"); + + fail: + + return; +} + + +static ncx_slab_page_t *ncx_slab_alloc_pages(ncx_slab_pool_t *pool, ncx_uint_t pages) { + ncx_slab_page_t *page, *p; + + for (page = pool->free.next; page != &pool->free; page = page->next) { + + if (page->slab >= pages) { + + if (page->slab > pages) { + page[pages].slab = page->slab - pages; + page[pages].next = page->next; + page[pages].prev = page->prev; + + p = (ncx_slab_page_t *) page->prev; + p->next = &page[pages]; + page->next->prev = (uintptr_t) &page[pages]; + + } else { + p = (ncx_slab_page_t *) page->prev; + p->next = page->next; + page->next->prev = page->prev; + } + + page->slab = pages | NCX_SLAB_PAGE_START; + page->next = NULL; + page->prev = NCX_SLAB_PAGE; + + if (--pages == 0) { + return page; + } + + for (p = page + 1; pages; pages--) { + p->slab = NCX_SLAB_PAGE_BUSY; + p->next = NULL; + p->prev = NCX_SLAB_PAGE; + p++; + } + + return page; + } + } + +// error("ncx_slab_alloc() failed: no memory"); + + return NULL; +} + +static void ncx_slab_free_pages(ncx_slab_pool_t *pool, ncx_slab_page_t *page, ncx_uint_t pages) { + ncx_slab_page_t *prev; + + if (pages > 1) { + ncx_memzero(&page[1], (pages - 1) * sizeof(ncx_slab_page_t)); + } + + if (page->next) { + prev = (ncx_slab_page_t *) (page->prev & ~NCX_SLAB_PAGE_MASK); + prev->next = page->next; + page->next->prev = page->prev; + } + + page->slab = pages; + page->prev = (uintptr_t) &pool->free; + page->next = pool->free.next; + page->next->prev = (uintptr_t) page; + + pool->free.next = page; + +#ifdef PAGE_MERGE + if (pool->pages != page) { + prev = page - 1; + if (ncx_slab_empty(pool, prev)) { + for (; prev >= pool->pages; prev--) { + if (prev->slab != 0) + { + pool->free.next = page->next; + page->next->prev = (uintptr_t) &pool->free; + + prev->slab += pages; + ncx_memzero(page, sizeof(ncx_slab_page_t)); + + page = prev; + + break; + } + } + } + } + + if ((page - pool->pages + page->slab) < ncx_real_pages) { + next = page + page->slab; + if (ncx_slab_empty(pool, next)) + { + prev = (ncx_slab_page_t *) (next->prev); + prev->next = next->next; + next->next->prev = next->prev; + + page->slab += next->slab; + ncx_memzero(next, sizeof(ncx_slab_page_t)); + } + } + +#endif +} + +void ncx_slab_stat(ncx_slab_pool_t *pool, ncx_slab_stat_t *stat) { + uintptr_t m, n, mask, slab; + uintptr_t *bitmap; + ncx_uint_t i, j, map, type, obj_size; + ncx_slab_page_t *page; + + ncx_memzero(stat, sizeof(ncx_slab_stat_t)); + + page = pool->pages; + stat->pages = (pool->end - pool->start) / ncx_pagesize; + + for (i = 0; i < stat->pages; i++) { + slab = page->slab; + type = page->prev & NCX_SLAB_PAGE_MASK; + + switch (type) { + + case NCX_SLAB_SMALL: + + n = (page - pool->pages) << ncx_pagesize_shift; + bitmap = (uintptr_t *) (pool->start + n); + + obj_size = 1 << slab; + map = (1 << (ncx_pagesize_shift - slab)) + / (sizeof(uintptr_t) * 8); + + for (j = 0; j < map; j++) { + for (m = 1; m; m <<= 1) { + if ((bitmap[j] & m)) { + stat->used_size += obj_size; + stat->b_small += obj_size; + } + + } + } + + stat->p_small++; + + break; + + case NCX_SLAB_EXACT: + + if (slab == NCX_SLAB_BUSY) { + stat->used_size += sizeof(uintptr_t) * 8 * ncx_slab_exact_size; + stat->b_exact += sizeof(uintptr_t) * 8 * ncx_slab_exact_size; + } else { + for (m = 1; m; m <<= 1) { + if (slab & m) { + stat->used_size += ncx_slab_exact_size; + stat->b_exact += ncx_slab_exact_size; + } + } + } + + stat->p_exact++; + + break; + + case NCX_SLAB_BIG: + + j = ncx_pagesize_shift - (slab & NCX_SLAB_SHIFT_MASK); + j = 1 << j; + j = ((uintptr_t) 1 << j) - 1; + mask = j << NCX_SLAB_MAP_SHIFT; + obj_size = 1 << (slab & NCX_SLAB_SHIFT_MASK); + + for (m = (uintptr_t) 1 << NCX_SLAB_MAP_SHIFT; m & mask; m <<= 1) { + if ((page->slab & m)) { + stat->used_size += obj_size; + stat->b_big += obj_size; + } + } + + stat->p_big++; + + break; + + case NCX_SLAB_PAGE: + + if (page->prev == NCX_SLAB_PAGE) { + slab = slab & ~NCX_SLAB_PAGE_START; + stat->used_size += slab * ncx_pagesize; + stat->b_page += slab * ncx_pagesize; + stat->p_page += slab; + + i += (slab - 1); + + break; + } + + default: + + if (slab > stat->max_free_pages) { + stat->max_free_pages = page->slab; + } + + stat->free_page += slab; + + i += (slab - 1); + + break; + } + + page = pool->pages + i + 1; + } + + stat->pool_size = pool->end - pool->start; + stat->used_pct = stat->used_size * 100 / stat->pool_size; +} + +static bool ncx_slab_empty(ncx_slab_pool_t *pool, ncx_slab_page_t *page) { + ncx_slab_page_t *prev; + + if (page->slab == 0) { + return true; + } + + //page->prev == PAGE | SMALL | EXACT | BIG + if (page->next == NULL) { + return false; + } + + prev = (ncx_slab_page_t *) (page->prev & ~NCX_SLAB_PAGE_MASK); + while (prev >= pool->pages) { + prev = (ncx_slab_page_t *) (prev->prev & ~NCX_SLAB_PAGE_MASK); + } + + if (prev == &pool->free) { + return true; + } + + return false; +} \ No newline at end of file diff --git a/src/mempool/mempool.h b/src/mempool/mempool.h new file mode 100644 index 0000000..e903158 --- /dev/null +++ b/src/mempool/mempool.h @@ -0,0 +1,62 @@ +#ifndef SIST2_MEMPOOL_H +#define SIST2_MEMPOOL_H + +#include +#include +#include +#include +#include + +typedef unsigned char u_char; +typedef uintptr_t ncx_uint_t; + +#ifndef NCX_ALIGNMENT +#define NCX_ALIGNMENT sizeof(unsigned long) +#endif + +#define ncx_align(d, a) (((d) + (a - 1)) & ~(a - 1)) +#define ncx_align_ptr(p, a) (u_char *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1)) + +#define ncx_memzero(buf, n) (void) memset(buf, 0, n) +#define ncx_memset(buf, c, n) (void) memset(buf, c, n) + +typedef struct ncx_slab_page_s ncx_slab_page_t; + +struct ncx_slab_page_s { + uintptr_t slab; + ncx_slab_page_t *next; + uintptr_t prev; +}; + +typedef struct { + size_t min_size; + size_t min_shift; + + ncx_slab_page_t *pages; + ncx_slab_page_t free; + + u_char *start; + u_char *end; + + //ncx_shmtx_t mutex; + + void *addr; +} ncx_slab_pool_t; + +typedef struct { + size_t pool_size, used_size, used_pct; + size_t pages, free_page; + size_t p_small, p_exact, p_big, p_page; + size_t b_small, b_exact, b_big, b_page; + size_t max_free_pages; +} ncx_slab_stat_t; + +void ncx_slab_init(ncx_slab_pool_t *mempool); + +void *ncx_slab_alloc(ncx_slab_pool_t *mempool, size_t size); + +void ncx_slab_free(ncx_slab_pool_t *mempool, void *p); + +void ncx_slab_stat(ncx_slab_pool_t *mempool, ncx_slab_stat_t *stat); + +#endif //SIST2_MEMPOOL_H diff --git a/src/parsing/parse.c b/src/parsing/parse.c index b6882bc..a792ad6 100644 --- a/src/parsing/parse.c +++ b/src/parsing/parse.c @@ -56,9 +56,20 @@ void set_dbg_current_file(parse_job_t *job) { pthread_mutex_unlock(&ScanCtx.dbg_current_files_mu); } -void parse(void *arg) { +void parse_job(parse_job_t *job) { + tpool_work_arg_shm_t *arg = malloc(sizeof(tpool_work_arg_shm_t) + sizeof(*job)); - parse_job_t *job = arg; + memcpy(arg->arg, job, sizeof(*job)); + arg->arg_size = -1; + + parse(arg); + + free(arg); +} + +void parse(tpool_work_arg_shm_t *arg) { + + parse_job_t *job = (void*)arg->arg; document_t *doc = malloc(sizeof(document_t)); @@ -74,11 +85,11 @@ void parse(void *arg) { doc->meta_head = NULL; doc->meta_tail = NULL; doc->mime = 0; - doc->size = job->vfile.info.st_size; - doc->mtime = (int) job->vfile.info.st_mtim.tv_sec; + doc->size = job->vfile.st_size; + doc->mtime = (int) job->vfile.mtime; int inc_ts = incremental_get(ScanCtx.original_table, doc->doc_id); - if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) { + if (inc_ts != 0 && inc_ts == job->vfile.mtime) { pthread_mutex_lock(&ScanCtx.copy_table_mu); incremental_mark_file(ScanCtx.copy_table, doc->doc_id); pthread_mutex_unlock(&ScanCtx.copy_table_mu); @@ -88,7 +99,6 @@ void parse(void *arg) { pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu); CLOSE_FILE(job->vfile) - free(doc->filepath); free(doc); return; @@ -106,13 +116,16 @@ void parse(void *arg) { LOG_DEBUGF(job->filepath, "Starting parse job {%s}", doc->doc_id) } - if (job->vfile.info.st_size == 0) { + if (job->ext > 4096) { + fprintf(stderr, "Ext is %d, filename is %s\n", job->ext, job->filepath); + } + + if (job->vfile.st_size == 0) { doc->mime = MIME_EMPTY; } else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) { doc->mime = mime_get_mime_by_ext(ScanCtx.ext_table, job->filepath + job->ext); } - if (doc->mime == 0 && !ScanCtx.fast) { // Get mime type with libmagic @@ -136,7 +149,6 @@ void parse(void *arg) { pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu); CLOSE_FILE(job->vfile) - free(doc->filepath); free(doc); return; @@ -210,7 +222,6 @@ void parse(void *arg) { } else if (doc->mime == MIME_SIST2_SIDECAR) { parse_sidecar(&job->vfile, doc); CLOSE_FILE(job->vfile) - free(doc->filepath); free(doc); return; } else if (is_msdoc(&ScanCtx.msdoc_ctx, doc->mime)) { diff --git a/src/parsing/parse.h b/src/parsing/parse.h index a62dcc4..55ecf18 100644 --- a/src/parsing/parse.h +++ b/src/parsing/parse.h @@ -2,6 +2,7 @@ #define SIST2_PARSE_H #include "../sist.h" +#include "src/tpool.h" #define MAGIC_BUF_SIZE (4096 * 6) @@ -9,7 +10,8 @@ int fs_read(struct vfile *f, void *buf, size_t size); void fs_close(struct vfile *f); void fs_reset(struct vfile *f); -void parse(void *arg); +void parse_job(parse_job_t *job); +void parse(tpool_work_arg_shm_t *arg); void cleanup_parse(); diff --git a/src/tpool.c b/src/tpool.c index adf4423..7abea86 100644 --- a/src/tpool.c +++ b/src/tpool.c @@ -2,13 +2,14 @@ #include "ctx.h" #include "sist.h" #include +#include +#include +#include "mempool/mempool.h" -#define MAX_QUEUE_SIZE 1000000 - -typedef void (*thread_func_t)(void *arg); +#define MAX_QUEUE_SIZE 5000 typedef struct tpool_work { - void *arg; + tpool_work_arg_shm_t *arg; thread_func_t func; struct tpool_work *next; } tpool_work_t; @@ -18,11 +19,12 @@ typedef struct tpool { tpool_work_t *work_tail; pthread_mutex_t work_mutex; + pthread_mutex_t mem_mutex; pthread_cond_t has_work_cond; pthread_cond_t working_cond; - pthread_t *threads; + pthread_t threads[256]; int thread_cnt; int work_cnt; @@ -32,28 +34,46 @@ typedef struct tpool { size_t mem_limit; size_t page_size; - int free_arg; int stop; int waiting; int print_progress; void (*cleanup_func)(); + + // ========= + + void *shared_memory; + size_t shared_memory_size; + ncx_slab_pool_t *mempool; } tpool_t; /** * Create a work object */ -static tpool_work_t *tpool_work_create(thread_func_t func, void *arg) { +static tpool_work_t *tpool_work_create(tpool_t *pool, thread_func_t func, tpool_work_arg_t *arg) { if (func == NULL) { return NULL; } - tpool_work_t *work = malloc(sizeof(tpool_work_t)); + // Copy heap arg to shm arg + pthread_mutex_lock(&pool->mem_mutex); + + tpool_work_arg_shm_t *shm_arg = ncx_slab_alloc(pool->mempool, sizeof(tpool_work_arg_shm_t) + arg->arg_size); + + shm_arg->arg_size = arg->arg_size; + memcpy(shm_arg->arg, arg->arg, arg->arg_size); + + free(arg->arg); + + tpool_work_t *work = ncx_slab_alloc(pool->mempool, sizeof(tpool_work_t)); + + pthread_mutex_unlock(&pool->mem_mutex); + work->func = func; - work->arg = arg; + work->arg = shm_arg; work->next = NULL; return work; @@ -90,16 +110,15 @@ static tpool_work_t *tpool_work_get(tpool_t *pool) { /** * Push work object to thread pool */ -int tpool_add_work(tpool_t *pool, thread_func_t func, void *arg) { - - tpool_work_t *work = tpool_work_create(func, arg); - if (work == NULL) { - return 0; - } +int tpool_add_work(tpool_t *pool, thread_func_t func, tpool_work_arg_t *arg) { while ((pool->work_cnt - pool->done_cnt) >= MAX_QUEUE_SIZE) { usleep(10000); } + tpool_work_t *work = tpool_work_create(pool, func, arg); + if (work == NULL) { + return 0; + } pthread_mutex_lock(&(pool->work_mutex)); if (pool->work_head == NULL) { @@ -118,127 +137,92 @@ int tpool_add_work(tpool_t *pool, thread_func_t func, void *arg) { return 1; } -/** - * see: https://github.com/htop-dev/htop/blob/f782f821f7f8081cb43bbad1c37f32830a260a81/linux/LinuxProcessList.c - */ -__always_inline -static size_t _get_total_mem(tpool_t *pool) { - FILE *statmfile = fopen("/proc/self/statm", "r"); - if (!statmfile) - return 0; - - long int dummy, dummy2, dummy3, dummy4, dummy5, dummy6; - long int m_resident; - - int r = fscanf(statmfile, "%ld %ld %ld %ld %ld %ld %ld", - &dummy, /* m_virt */ - &m_resident, - &dummy2, /* m_share */ - &dummy3, /* m_trs */ - &dummy4, /* unused since Linux 2.6; always 0 */ - &dummy5, /* m_drs */ - &dummy6); /* unused since Linux 2.6; always 0 */ - fclose(statmfile); - - if (r == 7) { - return m_resident * pool->page_size; - } else { - return 0; - } -} - /** * Thread worker function */ static void *tpool_worker(void *arg) { tpool_t *pool = arg; - int stuck_notified = 0; - int throttle_ms = 0; - while (TRUE) { - pthread_mutex_lock(&pool->work_mutex); - if (pool->stop) { - break; - } - - if (pool->work_head == NULL) { - pthread_cond_wait(&(pool->has_work_cond), &(pool->work_mutex)); - } - - tpool_work_t *work = tpool_work_get(pool); - if (work != NULL) { - pool->busy_cnt += 1; - } - - pthread_mutex_unlock(&(pool->work_mutex)); - - if (work != NULL) { - stuck_notified = 0; - throttle_ms = 0; - while (!pool->stop && pool->mem_limit > 0 && _get_total_mem(pool) >= pool->mem_limit) { - if (!stuck_notified && throttle_ms >= 90000) { - // notify the pool that this thread is stuck. - pthread_mutex_lock(&(pool->work_mutex)); - pool->throttle_stuck_cnt += 1; - if (pool->throttle_stuck_cnt == pool->thread_cnt) { - LOG_ERROR("tpool.c", "Throttle memory limit too low, cannot proceed!"); - pool->stop = TRUE; - } - pthread_mutex_unlock(&(pool->work_mutex)); - stuck_notified = 1; - } - usleep(10000); - throttle_ms += 10; - } + int pid = fork(); + if (pid == 0) { + while (TRUE) { + pthread_mutex_lock(&pool->work_mutex); if (pool->stop) { break; } - // we are not stuck anymore. cancel our notification. - if (stuck_notified) { - pthread_mutex_lock(&(pool->work_mutex)); - pool->throttle_stuck_cnt -= 1; - pthread_mutex_unlock(&(pool->work_mutex)); + if (pool->work_head == NULL) { + pthread_cond_wait(&(pool->has_work_cond), &(pool->work_mutex)); } - work->func(work->arg); - if (pool->free_arg) { - free(work->arg); + tpool_work_t *work = tpool_work_get(pool); + + if (work != NULL) { + pool->busy_cnt += 1; } - free(work); + + pthread_mutex_unlock(&(pool->work_mutex)); + + if (work != NULL) { + if (pool->stop) { + break; + } + + work->func(work->arg); + + pthread_mutex_lock(&pool->mem_mutex); + ncx_slab_free(pool->mempool, work->arg); + ncx_slab_free(pool->mempool, work); + pthread_mutex_unlock(&pool->mem_mutex); + } + + pthread_mutex_lock(&(pool->work_mutex)); + if (work != NULL) { + pool->busy_cnt -= 1; + pool->done_cnt++; + } + + if (pool->print_progress) { + if (LogCtx.json_logs) { + progress_bar_print_json(pool->done_cnt, pool->work_cnt, ScanCtx.stat_tn_size, + ScanCtx.stat_index_size, pool->waiting); + } else { + progress_bar_print((double) pool->done_cnt / pool->work_cnt, ScanCtx.stat_tn_size, + ScanCtx.stat_index_size); + } + } + + if (pool->work_head == NULL) { + pthread_cond_signal(&(pool->working_cond)); + } + pthread_mutex_unlock(&(pool->work_mutex)); } - pthread_mutex_lock(&(pool->work_mutex)); - if (work != NULL) { + if (pool->cleanup_func != NULL) { + LOG_INFO("tpool.c", "Executing cleanup function") + pool->cleanup_func(); + LOG_DEBUG("tpool.c", "Done executing cleanup function") + } + + pthread_cond_signal(&(pool->working_cond)); + pthread_mutex_unlock(&(pool->work_mutex)); + exit(0); + + } else { + int status; + waitpid(pid, &status, 0); + + LOG_ERRORF("tpool.c", "child processed terminated with status code %d, signal=%d", WEXITSTATUS(status), WIFSTOPPED(status) ? WSTOPSIG(status) : -1) + + if (WIFSTOPPED(status)) { + pthread_mutex_lock(&(pool->work_mutex)); pool->busy_cnt -= 1; pool->done_cnt++; + pthread_mutex_unlock(&(pool->work_mutex)); } - - if (pool->print_progress) { - if (LogCtx.json_logs) { - progress_bar_print_json(pool->done_cnt, pool->work_cnt, ScanCtx.stat_tn_size, - ScanCtx.stat_index_size, pool->waiting); - } else { - progress_bar_print((double) pool->done_cnt / pool->work_cnt, ScanCtx.stat_tn_size, - ScanCtx.stat_index_size); - } - } - - if (pool->work_head == NULL) { - pthread_cond_signal(&(pool->working_cond)); - } - pthread_mutex_unlock(&(pool->work_mutex)); } - if (pool->cleanup_func != NULL) { - LOG_INFO("tpool.c", "Executing cleanup function") - pool->cleanup_func(); - LOG_DEBUG("tpool.c", "Done executing cleanup function") - } - - pthread_cond_signal(&(pool->working_cond)); - pthread_mutex_unlock(&(pool->work_mutex)); return NULL; } @@ -304,17 +288,32 @@ void tpool_destroy(tpool_t *pool) { pthread_cond_destroy(&(pool->has_work_cond)); pthread_cond_destroy(&(pool->working_cond)); - free(pool->threads); - free(pool); + munmap(pool->shared_memory, pool->shared_memory_size); } /** * Create a thread pool * @param thread_cnt Worker threads count */ -tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int free_arg, int print_progress, size_t mem_limit) { +tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int print_progress, size_t mem_limit) { + + // ============= + size_t shm_size = 1024 * 1024 * 2000; + + void *shared_memory = mmap(NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + tpool_t *pool = (tpool_t *) shared_memory; + pool->shared_memory = shared_memory; + pool->shared_memory_size = shm_size; + pool->mempool = (ncx_slab_pool_t *) (pool->shared_memory + sizeof(tpool_t)); + pool->mempool->addr = pool->mempool; + pool->mempool->min_shift = 4; + pool->mempool->end = pool->shared_memory + shm_size; + + ncx_slab_init(pool->mempool); + + // ============= - tpool_t *pool = malloc(sizeof(tpool_t)); pool->thread_cnt = thread_cnt; pool->work_cnt = 0; pool->done_cnt = 0; @@ -323,16 +322,24 @@ tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int free_arg, int pri pool->mem_limit = mem_limit; pool->stop = FALSE; pool->waiting = FALSE; - pool->free_arg = free_arg; pool->cleanup_func = cleanup_func; - pool->threads = calloc(sizeof(pthread_t), thread_cnt); + memset(pool->threads, 0, sizeof(pool->threads)); pool->print_progress = print_progress; pool->page_size = getpagesize(); - pthread_mutex_init(&(pool->work_mutex), NULL); + pthread_mutexattr_t mutexattr; + pthread_mutexattr_init(&mutexattr); + pthread_mutexattr_setpshared(&mutexattr, TRUE); - pthread_cond_init(&(pool->has_work_cond), NULL); - pthread_cond_init(&(pool->working_cond), NULL); + pthread_mutex_init(&(pool->work_mutex), &mutexattr); + pthread_mutex_init(&(pool->mem_mutex), &mutexattr); + + pthread_condattr_t condattr; + pthread_condattr_init(&condattr); + pthread_condattr_setpshared(&condattr, TRUE); + + pthread_cond_init(&(pool->has_work_cond), &condattr); + pthread_cond_init(&(pool->working_cond),&condattr); pool->work_head = NULL; pool->work_tail = NULL; diff --git a/src/tpool.h b/src/tpool.h index 71d202f..c1898f3 100644 --- a/src/tpool.h +++ b/src/tpool.h @@ -6,13 +6,26 @@ struct tpool; typedef struct tpool tpool_t; -typedef void (*thread_func_t)(void *arg); +typedef struct { + size_t arg_size; + void *arg; +} tpool_work_arg_t; + +typedef struct { + size_t arg_size; + char arg[0]; +} tpool_work_arg_shm_t; + +typedef void (*thread_func_t)(tpool_work_arg_shm_t *arg); + +tpool_t *tpool_create(int num, void (*cleanup_func)(), int print_progress, size_t mem_limit); -tpool_t *tpool_create(int num, void (*cleanup_func)(), int free_arg, int print_progress, size_t mem_limit); void tpool_start(tpool_t *pool); + void tpool_destroy(tpool_t *pool); -int tpool_add_work(tpool_t *pool, thread_func_t func, void *arg); +int tpool_add_work(tpool_t *pool, thread_func_t func, tpool_work_arg_t *arg); + void tpool_wait(tpool_t *pool); void tpool_dump_debug_info(tpool_t *pool); diff --git a/src/util.c b/src/util.c index f58cb6d..79d5bd2 100644 --- a/src/util.c +++ b/src/util.c @@ -103,7 +103,9 @@ void progress_bar_print_json(size_t done, size_t count, size_t tn_size, size_t i void progress_bar_print(double percentage, size_t tn_size, size_t index_size) { + // TODO: Fix this with shm/ctx static int last_val = -1; + int val = (int) (percentage * 100); if (last_val == val || val > 100) { return; diff --git a/third-party/libscan/libscan/arc/arc.c b/third-party/libscan/libscan/arc/arc.c index e547231..092bc8c 100644 --- a/third-party/libscan/libscan/arc/arc.c +++ b/third-party/libscan/libscan/arc/arc.c @@ -188,14 +188,13 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre } else { - parse_job_t *sub_job = malloc(sizeof(parse_job_t) + PATH_MAX * 2); + parse_job_t *sub_job = malloc(sizeof(parse_job_t)); sub_job->vfile.close = arc_close; sub_job->vfile.read = arc_read; sub_job->vfile.read_rewindable = arc_read_rewindable; sub_job->vfile.reset = NULL; sub_job->vfile.arc = a; - sub_job->vfile.filepath = sub_job->filepath; sub_job->vfile.is_fs_file = FALSE; sub_job->vfile.rewind_buffer_size = 0; sub_job->vfile.rewind_buffer = NULL; @@ -206,22 +205,29 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre strcpy(sub_job->parent, doc->doc_id); while (archive_read_next_header(a, &entry) == ARCHIVE_OK) { - sub_job->vfile.info = *archive_entry_stat(entry); + struct stat entry_stat = *archive_entry_stat(entry); + sub_job->vfile.st_mode = entry_stat.st_mode; + sub_job->vfile.st_size = entry_stat.st_size; + sub_job->vfile.mtime = (int) entry_stat.st_mtim.tv_sec; - double decompressed_size_ratio = (double) sub_job->vfile.info.st_size / (double) f->info.st_size; + double decompressed_size_ratio = (double) sub_job->vfile.st_size / (double) f->st_size; if (decompressed_size_ratio > MAX_DECOMPRESSED_SIZE_RATIO) { - CTX_LOG_DEBUGF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)", sub_job->filepath, decompressed_size_ratio) + CTX_LOG_DEBUGF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)", sub_job->filepath, + decompressed_size_ratio) continue; } - if (S_ISREG(sub_job->vfile.info.st_mode)) { + if (S_ISREG(sub_job->vfile.st_mode)) { const char *utf8_name = archive_entry_pathname_utf8(entry); if (utf8_name == NULL) { - sprintf(sub_job->filepath, "%s#/%s", f->filepath, archive_entry_pathname(entry)); + snprintf(sub_job->filepath, sizeof(sub_job->filepath), "%s#/%s", f->filepath, + archive_entry_pathname(entry)); + strcpy(sub_job->vfile.filepath, sub_job->filepath); } else { - sprintf(sub_job->filepath, "%s#/%s", f->filepath, utf8_name); + snprintf(sub_job->filepath, sizeof(sub_job->filepath), "%s#/%s", f->filepath, utf8_name); + strcpy(sub_job->vfile.filepath, sub_job->filepath); } sub_job->base = (int) (strrchr(sub_job->filepath, '/') - sub_job->filepath) + 1; diff --git a/third-party/libscan/libscan/ebook/ebook.c b/third-party/libscan/libscan/ebook/ebook.c index b0d6e78..acf70fd 100644 --- a/third-party/libscan/libscan/ebook/ebook.c +++ b/third-party/libscan/libscan/ebook/ebook.c @@ -1,28 +1,34 @@ #include "ebook.h" #include -#include #include #include "../media/media.h" #include "../arc/arc.h" #include "../ocr/ocr.h" +#if EBOOK_LOCKS +#include +pthread_mutex_t Mutex; +#endif + /* fill_image callback doesn't let us pass opaque pointers unless I create my own device */ __thread text_buffer_t thread_buffer; __thread scan_ebook_ctx_t thread_ctx; -pthread_mutex_t Mutex; - static void my_fz_lock(UNUSED(void *user), int lock) { +#if EBOOK_LOCKS if (lock == FZ_LOCK_FREETYPE) { pthread_mutex_lock(&Mutex); } +#endif } static void my_fz_unlock(UNUSED(void *user), int lock) { +#if EBOOK_LOCKS if (lock == FZ_LOCK_FREETYPE) { pthread_mutex_unlock(&Mutex); } +#endif } @@ -187,11 +193,13 @@ void fz_warn_callback(void *user, const char *message) { static void init_fzctx(fz_context *fzctx, document_t *doc) { fz_register_document_handlers(fzctx); +#if EBOOK_LOCKS static int mu_is_initialized = FALSE; if (!mu_is_initialized) { pthread_mutex_init(&Mutex, NULL); mu_is_initialized = TRUE; } +#endif fzctx->warn.print_user = doc; fzctx->warn.print = fz_warn_callback; diff --git a/third-party/libscan/libscan/ebook/ebook.h b/third-party/libscan/libscan/ebook/ebook.h index cae9d9a..7ee37c1 100644 --- a/third-party/libscan/libscan/ebook/ebook.h +++ b/third-party/libscan/libscan/ebook/ebook.h @@ -9,7 +9,6 @@ typedef struct { int enable_tn; const char *tesseract_lang; const char *tesseract_path; - pthread_mutex_t mupdf_mutex; log_callback_t log; logf_callback_t logf; diff --git a/third-party/libscan/libscan/json/json.c b/third-party/libscan/libscan/json/json.c index 3ba69f0..ef93405 100644 --- a/third-party/libscan/libscan/json/json.c +++ b/third-party/libscan/libscan/json/json.c @@ -32,7 +32,7 @@ int json_extract_text(cJSON *json, text_buffer_t *tex) { scan_code_t parse_json(scan_json_ctx_t *ctx, vfile_t *f, document_t *doc) { - if (f->info.st_size > JSON_MAX_FILE_SIZE) { + if (f->st_size > JSON_MAX_FILE_SIZE) { CTX_LOG_WARNINGF("json.c", "File larger than maximum allowed [%s]", f->filepath) return SCAN_ERR_SKIP; } diff --git a/third-party/libscan/libscan/media/media.c b/third-party/libscan/libscan/media/media.c index 956d3a4..fe9360e 100644 --- a/third-party/libscan/libscan/media/media.c +++ b/third-party/libscan/libscan/media/media.c @@ -687,7 +687,7 @@ long memfile_seek(void *ptr, long offset, int whence) { } int memfile_open(vfile_t *f, memfile_t *mem) { - mem->size = f->info.st_size; + mem->size = f->st_size; mem->buf = malloc(mem->size); if (mem->buf == NULL) { @@ -737,16 +737,16 @@ void parse_media_vfile(scan_media_ctx_t *ctx, struct vfile *f, document_t *doc, const char *filepath = get_filepath_with_ext(doc, f->filepath, mime_str); - if (f->info.st_size <= ctx->max_media_buffer) { + if (f->st_size <= ctx->max_media_buffer) { int ret = memfile_open(f, &memfile); if (ret == 0) { - CTX_LOG_DEBUGF(f->filepath, "Loading media file in memory (%ldB)", f->info.st_size) + CTX_LOG_DEBUGF(f->filepath, "Loading media file in memory (%ldB)", f->st_size) io_ctx = avio_alloc_context(buffer, AVIO_BUF_SIZE, 0, &memfile, memfile_read, NULL, memfile_seek); } } if (io_ctx == NULL) { - CTX_LOG_DEBUGF(f->filepath, "Reading media file without seek support", f->info.st_size) + CTX_LOG_DEBUGF(f->filepath, "Reading media file without seek support", f->st_size) io_ctx = avio_alloc_context(buffer, AVIO_BUF_SIZE, 0, f, vfile_read, NULL, NULL); } diff --git a/third-party/libscan/libscan/scan.h b/third-party/libscan/libscan/scan.h index 187161e..fd3fd1f 100644 --- a/third-party/libscan/libscan/scan.h +++ b/third-party/libscan/libscan/scan.h @@ -51,6 +51,8 @@ typedef int scan_code_t; #define SIST_DOC_ID_LEN MD5_STR_LENGTH #define SIST_INDEX_ID_LEN MD5_STR_LENGTH +#define EBOOK_LOCKS 0 + enum metakey { // String MetaContent = 1, @@ -100,7 +102,6 @@ typedef struct meta_line { union { char str_val[0]; unsigned long long_val; - double double_val; }; } meta_line_t; @@ -114,7 +115,7 @@ typedef struct document { short ext; meta_line_t *meta_head; meta_line_t *meta_tail; - char filepath[PATH_MAX]; + char filepath[PATH_MAX * 2 + 1]; } document_t; typedef struct vfile vfile_t; @@ -139,8 +140,11 @@ typedef struct vfile { int is_fs_file; int has_checksum; int calculate_checksum; - const char *filepath; - struct stat info; + char filepath[PATH_MAX * 2 + 1]; + + int mtime; + size_t st_size; + unsigned int st_mode; SHA_CTX sha1_ctx; unsigned char sha1_digest[SHA1_DIGEST_LENGTH]; @@ -162,7 +166,7 @@ typedef struct parse_job_t { int ext; struct vfile vfile; char parent[SIST_DOC_ID_LEN]; - char filepath[PATH_MAX]; + char filepath[PATH_MAX * 2 + 1]; } parse_job_t; diff --git a/third-party/libscan/libscan/text/text.c b/third-party/libscan/libscan/text/text.c index ba9403e..b4ffe33 100644 --- a/third-party/libscan/libscan/text/text.c +++ b/third-party/libscan/libscan/text/text.c @@ -2,7 +2,7 @@ scan_code_t parse_text(scan_text_ctx_t *ctx, vfile_t *f, document_t *doc) { - int to_read = MIN(ctx->content_size, f->info.st_size); + int to_read = MIN(ctx->content_size, f->st_size); if (to_read <= 2) { return SCAN_OK; @@ -39,7 +39,7 @@ scan_code_t parse_text(scan_text_ctx_t *ctx, vfile_t *f, document_t *doc) { scan_code_t parse_markup(scan_text_ctx_t *ctx, vfile_t *f, document_t *doc) { - int to_read = MIN(MAX_MARKUP_SIZE, f->info.st_size); + int to_read = MIN(MAX_MARKUP_SIZE, f->st_size); char *buf = malloc(to_read + 1); int ret = f->read(f, buf, to_read); diff --git a/third-party/libscan/libscan/util.h b/third-party/libscan/libscan/util.h index 33923fa..09f7ad5 100644 --- a/third-party/libscan/libscan/util.h +++ b/third-party/libscan/libscan/util.h @@ -325,10 +325,10 @@ static int text_buffer_append_markup(text_buffer_t *buf, const char *markup) { } static void *read_all(vfile_t *f, size_t *size) { - void *buf = malloc(f->info.st_size); - *size = f->read(f, buf, f->info.st_size); + void *buf = malloc(f->st_size); + *size = f->read(f, buf, f->st_size); - if (*size != f->info.st_size) { + if (*size != f->st_size) { free(buf); return NULL; } diff --git a/third-party/libscan/test/test_util.cpp b/third-party/libscan/test/test_util.cpp index ee78f10..fb9aa5c 100644 --- a/third-party/libscan/test/test_util.cpp +++ b/third-party/libscan/test/test_util.cpp @@ -50,14 +50,20 @@ void cleanup(document_t *doc, vfile_t *f) { } void load_file(const char *filepath, vfile_t *f) { - stat(filepath, &f->info); + struct stat info = {}; + stat(filepath, &info); + + f->mtime = (int)info.st_mtim.tv_sec; + f->st_size = info.st_size; + f->st_mode = info.st_mode; + f->fd = open(filepath, O_RDONLY); if (f->fd == -1) { FAIL() << FILE_NOT_FOUND_ERR; } - f->filepath = filepath; + memcpy(f->filepath, filepath, sizeof(f->filepath)); f->read = fs_read; f->close = fs_close; f->is_fs_file = TRUE; @@ -66,9 +72,9 @@ void load_file(const char *filepath, vfile_t *f) { } void load_mem(void *mem, size_t size, vfile_t *f) { - f->filepath = "_mem_"; + memcpy(f->filepath, "_mem_", strlen("_mem_")); f->_test_data = mem; - f->info.st_size = (int) size; + f->st_size = size; f->read = mem_read; f->close = nullptr; f->is_fs_file = TRUE; From ca973d63a47f073f058adf6b6eae1aa35bbc8e95 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sun, 12 Mar 2023 11:38:31 -0400 Subject: [PATCH 3/7] Still WIP.. --- README.md | 4 +- docker-compose.yml | 4 +- scripts/start_dev_es.sh | 2 +- src/cli.c | 4 -- src/cli.h | 1 - src/ctx.h | 1 - src/index/elastic.c | 2 +- src/io/store.c | 5 +- src/io/store.h | 1 - src/log.h | 5 ++ src/main.c | 11 ++- src/sist.h | 2 +- src/tpool.c | 151 +++++++++++++++++++++------------------- src/tpool.h | 2 +- 14 files changed, 99 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 2703260..298dbe2 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,12 @@ sist2 (Simple incremental search tool) 1. Download [from official website](https://www.elastic.co/downloads/elasticsearch) 1. *(or)* Run using docker: ```bash - docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.14.0 + docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9 ``` 1. *(or)* Run using docker-compose: ```yaml elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9 environment: - discovery.type=single-node - "ES_JAVA_OPTS=-Xms1G -Xmx2G" diff --git a/docker-compose.yml b/docker-compose.yml index d6d891c..3b9aef6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: elasticsearch: - image: elasticsearch:7.14.0 + image: elasticsearch:7.17.9 container_name: sist2-es environment: - "discovery.type=single-node" @@ -15,9 +15,9 @@ services: - /mnt/array/sist2-admin-data/:/sist2-admin/ - /:/host ports: + - 4090:4090 # NOTE: Don't export this port publicly! - 8080:8080 - - 4090:4090 working_dir: /root/sist2-admin/ entrypoint: python3 command: diff --git a/scripts/start_dev_es.sh b/scripts/start_dev_es.sh index 6bd0bd5..ce091b5 100755 --- a/scripts/start_dev_es.sh +++ b/scripts/start_dev_es.sh @@ -1,3 +1,3 @@ docker run --rm -it --name "sist2-dev-es"\ -p 9200:9200 -e "discovery.type=single-node" \ - -e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.14.0 + -e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.17.9 diff --git a/src/cli.c b/src/cli.c index 1bafba6..8ff8875 100644 --- a/src/cli.c +++ b/src/cli.c @@ -273,10 +273,6 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { args->max_memory_buffer_mib = DEFAULT_MAX_MEM_BUFFER; } - if (args->scan_mem_limit_mib == OPTION_VALUE_UNSPECIFIED || args->scan_mem_limit_mib == OPTION_VALUE_DISABLE) { - args->scan_mem_limit_mib = DEFAULT_THROTTLE_MEMORY_THRESHOLD; - } - if (args->list_path != OPTION_VALUE_UNSPECIFIED) { if (strcmp(args->list_path, "-") == 0) { args->list_file = stdin; diff --git a/src/cli.h b/src/cli.h index 556d1d6..d953621 100644 --- a/src/cli.h +++ b/src/cli.h @@ -13,7 +13,6 @@ typedef struct scan_args { int tn_size; int content_size; int threads; - int scan_mem_limit_mib; char *incremental; char *output; char *rewrite_url; diff --git a/src/ctx.h b/src/ctx.h index a859eee..49fdbb7 100644 --- a/src/ctx.h +++ b/src/ctx.h @@ -35,7 +35,6 @@ typedef struct { int threads; int depth; int calculate_checksums; - size_t mem_limit; size_t stat_tn_size; size_t stat_index_size; diff --git a/src/index/elastic.c b/src/index/elastic.c index 0b41b07..9d582fc 100644 --- a/src/index/elastic.c +++ b/src/index/elastic.c @@ -65,7 +65,7 @@ void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) { } void index_json_func(tpool_work_arg_shm_t *arg) { - // Copy arg to heap because it's going to be free immediately after this function returns + // Copy arg to heap because it's going to be freed immediately after this function returns es_bulk_line_t *line = malloc(arg->arg_size); memcpy(line, arg->arg, arg->arg_size); diff --git a/src/io/store.c b/src/io/store.c index 2e03e8d..dad686b 100644 --- a/src/io/store.c +++ b/src/io/store.c @@ -34,8 +34,7 @@ store_t *store_create(const char *path, size_t chunk_size) { #if (SIST_FAKE_STORE != 1) store->chunk_size = chunk_size; - store->shared_memory = mmap(NULL, sizeof(*store->shm), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - store->shm = store->shared_memory; + store->shm = mmap(NULL, sizeof(*store->shm), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); open_env(path, &env, &dbi); @@ -53,7 +52,7 @@ void store_destroy(store_t *store) { LOG_DEBUG("store.c", "store_destroy()") #if (SIST_FAKE_STORE != 1) - munmap(store->shared_memory, sizeof(*store->shm)); + munmap(store->shm, sizeof(*store->shm)); mdb_dbi_close(store->proc.env, store->proc.dbi); mdb_env_close(store->proc.env); diff --git a/src/io/store.h b/src/io/store.h index b6c8472..ce27ded 100644 --- a/src/io/store.h +++ b/src/io/store.h @@ -14,7 +14,6 @@ typedef struct store_t { char path[PATH_MAX]; size_t chunk_size; - void *shared_memory; struct { MDB_dbi dbi; diff --git a/src/log.h b/src/log.h index e2e7cd0..113a577 100644 --- a/src/log.h +++ b/src/log.h @@ -37,6 +37,11 @@ sist_log(filepath, LOG_SIST_FATAL, str);\ exit(-1); +#define LOG_FATALF_NO_EXIT(filepath, fmt, ...) \ + sist_logf(filepath, LOG_SIST_FATAL, fmt, __VA_ARGS__); +#define LOG_FATAL_NO_EXIT(filepath, str) \ + sist_log(filepath, LOG_SIST_FATAL, str); + #include "sist.h" void sist_logf(const char *filepath, int level, char *format, ...); diff --git a/src/main.c b/src/main.c index 03e40a5..38a5dd5 100644 --- a/src/main.c +++ b/src/main.c @@ -17,6 +17,7 @@ #include #include +#include #include "stats.h" @@ -268,7 +269,6 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.threads = args->threads; ScanCtx.depth = args->depth; - ScanCtx.mem_limit = (size_t) args->scan_mem_limit_mib * 1024 * 1024; strncpy(ScanCtx.index.path, args->output, sizeof(ScanCtx.index.path)); strncpy(ScanCtx.index.desc.name, args->name, sizeof(ScanCtx.index.desc.name)); @@ -406,10 +406,10 @@ void sist2_scan(scan_args_t *args) { load_incremental_index(args); } - ScanCtx.writer_pool = tpool_create(1, writer_cleanup, FALSE, 0); + ScanCtx.writer_pool = tpool_create(1, writer_cleanup, FALSE); tpool_start(ScanCtx.writer_pool); - ScanCtx.pool = tpool_create(ScanCtx.threads, thread_cleanup, TRUE, ScanCtx.mem_limit); + ScanCtx.pool = tpool_create(ScanCtx.threads, thread_cleanup, TRUE); tpool_start(ScanCtx.pool); if (args->list_path) { @@ -493,7 +493,7 @@ void sist2_index(index_args_t *args) { f = index_json; } - IndexCtx.pool = tpool_create(args->threads, elastic_cleanup, args->print == 0, 0); + IndexCtx.pool = tpool_create(args->threads, elastic_cleanup, args->print == 0); tpool_start(IndexCtx.pool); READ_INDICES(file_path, args->index_path, { @@ -644,9 +644,6 @@ int main(int argc, const char *argv[]) { OPT_GROUP("Scan options"), OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"), - OPT_INTEGER(0, "mem-throttle", &scan_args->scan_mem_limit_mib, - "Total memory threshold in MiB for scan throttling. DEFAULT=0", - set_to_negative_if_value_is_zero, (intptr_t) &scan_args->scan_mem_limit_mib), OPT_INTEGER('q', "thumbnail-quality", &scan_args->tn_quality, "Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT=2", set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_quality), diff --git a/src/sist.h b/src/sist.h index 6dff0d3..30ded55 100644 --- a/src/sist.h +++ b/src/sist.h @@ -49,7 +49,7 @@ #include #include "git_hash.h" -#define VERSION "2.14.2" +#define VERSION "2.14.3" static const char *const Version = VERSION; #ifndef SIST_PLATFORM diff --git a/src/tpool.c b/src/tpool.c index 7abea86..8678e22 100644 --- a/src/tpool.c +++ b/src/tpool.c @@ -6,7 +6,9 @@ #include #include "mempool/mempool.h" -#define MAX_QUEUE_SIZE 5000 +#define BLANK_STR " " +// TODO: Use slab OOM to control queue size +#define MAX_QUEUE_SIZE 100000 typedef struct tpool_work { tpool_work_arg_shm_t *arg; @@ -21,6 +23,7 @@ typedef struct tpool { pthread_mutex_t work_mutex; pthread_mutex_t mem_mutex; + // TODO: Initialize with SHARED attr pthread_cond_t has_work_cond; pthread_cond_t working_cond; @@ -30,9 +33,6 @@ typedef struct tpool { int work_cnt; int done_cnt; int busy_cnt; - int throttle_stuck_cnt; - size_t mem_limit; - size_t page_size; int stop; int waiting; @@ -41,8 +41,6 @@ typedef struct tpool { void (*cleanup_func)(); - // ========= - void *shared_memory; size_t shared_memory_size; ncx_slab_pool_t *mempool; @@ -137,6 +135,61 @@ int tpool_add_work(tpool_t *pool, thread_func_t func, tpool_work_arg_t *arg) { return 1; } +static void worker_thread_loop(tpool_t *pool) { + while (TRUE) { + pthread_mutex_lock(&pool->work_mutex); + if (pool->stop) { + break; + } + + if (pool->work_head == NULL) { + pthread_cond_wait(&(pool->has_work_cond), &(pool->work_mutex)); + } + + tpool_work_t *work = tpool_work_get(pool); + + if (work != NULL) { + pool->busy_cnt += 1; + } + + pthread_mutex_unlock(&(pool->work_mutex)); + + if (work != NULL) { + if (pool->stop) { + break; + } + + work->func(work->arg); + + pthread_mutex_lock(&pool->mem_mutex); + ncx_slab_free(pool->mempool, work->arg); + ncx_slab_free(pool->mempool, work); + pthread_mutex_unlock(&pool->mem_mutex); + } + + pthread_mutex_lock(&(pool->work_mutex)); + if (work != NULL) { + pool->busy_cnt -= 1; + pool->done_cnt++; + } + + if (pool->print_progress) { + if (LogCtx.json_logs) { + progress_bar_print_json(pool->done_cnt, pool->work_cnt, ScanCtx.stat_tn_size, + ScanCtx.stat_index_size, pool->waiting); + } else { + progress_bar_print((double) pool->done_cnt / pool->work_cnt, ScanCtx.stat_tn_size, + ScanCtx.stat_index_size); + } + } + + if (pool->work_head == NULL) { + pthread_cond_signal(&(pool->working_cond)); + } + pthread_mutex_unlock(&(pool->work_mutex)); + } +} + /** * Thread worker function */ @@ -146,58 +199,8 @@ static void *tpool_worker(void *arg) { int pid = fork(); if (pid == 0) { - while (TRUE) { - pthread_mutex_lock(&pool->work_mutex); - if (pool->stop) { - break; - } - if (pool->work_head == NULL) { - pthread_cond_wait(&(pool->has_work_cond), &(pool->work_mutex)); - } - - tpool_work_t *work = tpool_work_get(pool); - - if (work != NULL) { - pool->busy_cnt += 1; - } - - pthread_mutex_unlock(&(pool->work_mutex)); - - if (work != NULL) { - if (pool->stop) { - break; - } - - work->func(work->arg); - - pthread_mutex_lock(&pool->mem_mutex); - ncx_slab_free(pool->mempool, work->arg); - ncx_slab_free(pool->mempool, work); - pthread_mutex_unlock(&pool->mem_mutex); - } - - pthread_mutex_lock(&(pool->work_mutex)); - if (work != NULL) { - pool->busy_cnt -= 1; - pool->done_cnt++; - } - - if (pool->print_progress) { - if (LogCtx.json_logs) { - progress_bar_print_json(pool->done_cnt, pool->work_cnt, ScanCtx.stat_tn_size, - ScanCtx.stat_index_size, pool->waiting); - } else { - progress_bar_print((double) pool->done_cnt / pool->work_cnt, ScanCtx.stat_tn_size, - ScanCtx.stat_index_size); - } - } - - if (pool->work_head == NULL) { - pthread_cond_signal(&(pool->working_cond)); - } - pthread_mutex_unlock(&(pool->work_mutex)); - } + worker_thread_loop(pool); if (pool->cleanup_func != NULL) { LOG_INFO("tpool.c", "Executing cleanup function") @@ -211,15 +214,27 @@ static void *tpool_worker(void *arg) { } else { int status; + // TODO: On crash, print debug info and resume thread waitpid(pid, &status, 0); - LOG_ERRORF("tpool.c", "child processed terminated with status code %d, signal=%d", WEXITSTATUS(status), WIFSTOPPED(status) ? WSTOPSIG(status) : -1) + LOG_DEBUGF("tpool.c", "Child process terminated with status code %d", WEXITSTATUS(status)) - if (WIFSTOPPED(status)) { - pthread_mutex_lock(&(pool->work_mutex)); - pool->busy_cnt -= 1; - pool->done_cnt++; - pthread_mutex_unlock(&(pool->work_mutex)); + pthread_mutex_lock(&(pool->work_mutex)); + pool->busy_cnt -= 1; + pool->done_cnt++; + pthread_mutex_unlock(&(pool->work_mutex)); + + if (WIFSIGNALED(status)) { +// parse_job_t *job = g_hash_table_lookup(ScanCtx.dbg_current_files, GINT_TO_POINTER(pthread_self())); + const char *job_filepath = "TODO"; + + LOG_FATALF_NO_EXIT( + "tpool.c", + "Child process was terminated by signal (%s).\n" + BLANK_STR "The process was working on %s", + strsignal(WTERMSIG(status)), + job_filepath + ) } } @@ -295,9 +310,8 @@ void tpool_destroy(tpool_t *pool) { * Create a thread pool * @param thread_cnt Worker threads count */ -tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int print_progress, size_t mem_limit) { +tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int print_progress) { - // ============= size_t shm_size = 1024 * 1024 * 2000; void *shared_memory = mmap(NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); @@ -312,20 +326,15 @@ tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int print_progress, s ncx_slab_init(pool->mempool); - // ============= - pool->thread_cnt = thread_cnt; pool->work_cnt = 0; pool->done_cnt = 0; pool->busy_cnt = 0; - pool->throttle_stuck_cnt = 0; - pool->mem_limit = mem_limit; pool->stop = FALSE; pool->waiting = FALSE; pool->cleanup_func = cleanup_func; memset(pool->threads, 0, sizeof(pool->threads)); pool->print_progress = print_progress; - pool->page_size = getpagesize(); pthread_mutexattr_t mutexattr; pthread_mutexattr_init(&mutexattr); @@ -339,7 +348,7 @@ tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int print_progress, s pthread_condattr_setpshared(&condattr, TRUE); pthread_cond_init(&(pool->has_work_cond), &condattr); - pthread_cond_init(&(pool->working_cond),&condattr); + pthread_cond_init(&(pool->working_cond), &condattr); pool->work_head = NULL; pool->work_tail = NULL; diff --git a/src/tpool.h b/src/tpool.h index c1898f3..8f73449 100644 --- a/src/tpool.h +++ b/src/tpool.h @@ -18,7 +18,7 @@ typedef struct { typedef void (*thread_func_t)(tpool_work_arg_shm_t *arg); -tpool_t *tpool_create(int num, void (*cleanup_func)(), int print_progress, size_t mem_limit); +tpool_t *tpool_create(int num, void (*cleanup_func)(), int print_progress); void tpool_start(tpool_t *pool); From fc36f33d528ae89ab36a382c9c3542f7700e2d76 Mon Sep 17 00:00:00 2001 From: simon987 Date: Mon, 3 Apr 2023 21:39:50 -0400 Subject: [PATCH 4/7] use sqlite to save index, major thread pool refactor --- .gitignore | 2 + CMakeLists.txt | 27 +- README.md | 2 +- scripts/before_build.sh | 15 +- scripts/mime.csv | 6 +- scripts/mime.py | 50 +- sist2-admin/frontend/public/index.html | 2 +- src/auth0/auth0_c_api.h | 3 +- src/cli.c | 163 +- src/cli.h | 2 +- src/ctx.c | 5 +- src/ctx.h | 27 +- src/database/database.c | 586 +++++ src/database/database.h | 147 ++ src/database/database_schema.c | 78 + src/database/database_stats.c | 159 ++ src/database/database_stats.h | 5 + src/index/elastic.c | 52 +- src/index/elastic.h | 2 +- src/index/web.c | 8 +- src/io/serialize.c | 398 +-- src/io/serialize.h | 48 - src/io/store.c | 232 -- src/io/store.h | 42 - src/io/walk.c | 68 +- src/log.c | 28 +- src/log.h | 43 +- src/main.c | 437 ++-- src/mempool/mempool.c | 757 ------ src/mempool/mempool.h | 62 - src/parsing/fs_util.h | 42 + src/parsing/magic_util.c | 32 + src/parsing/magic_util.h | 8 + src/parsing/mime.c | 24 +- src/parsing/mime.h | 10 +- src/parsing/mime_generated.c | 2730 ++++++++++----------- src/parsing/parse.c | 381 +-- src/parsing/parse.h | 10 +- src/parsing/sidecar.c | 9 +- src/sist.h | 5 +- src/stats.c | 343 --- src/stats.h | 6 - src/tpool.c | 456 ++-- src/tpool.h | 21 +- src/types.h | 14 +- src/util.c | 53 +- src/util.h | 45 +- src/web/serve.c | 451 ++-- src/web/web_util.c | 63 + src/web/web_util.h | 32 + third-party/libscan/CMakeLists.txt | 3 +- third-party/libscan/libscan/arc/arc.c | 37 +- third-party/libscan/libscan/arc/arc.h | 2 +- third-party/libscan/libscan/ebook/ebook.c | 2 +- third-party/libscan/libscan/font/font.c | 2 +- third-party/libscan/libscan/media/media.c | 16 +- third-party/libscan/libscan/ooxml/ooxml.c | 2 +- third-party/libscan/libscan/scan.h | 10 +- third-party/libscan/libscan/util.h | 33 + third-party/libscan/test/test_util.cpp | 1 - third-party/libscan/test/test_util.h | 2 +- third-party/libscan/third-party/antiword | 2 +- 62 files changed, 3630 insertions(+), 4673 deletions(-) create mode 100644 src/database/database.c create mode 100644 src/database/database.h create mode 100644 src/database/database_schema.c create mode 100644 src/database/database_stats.c create mode 100644 src/database/database_stats.h delete mode 100644 src/io/store.c delete mode 100644 src/io/store.h delete mode 100644 src/mempool/mempool.c delete mode 100644 src/mempool/mempool.h create mode 100644 src/parsing/fs_util.h create mode 100644 src/parsing/magic_util.c create mode 100644 src/parsing/magic_util.h delete mode 100644 src/stats.c delete mode 100644 src/stats.h create mode 100644 src/web/web_util.c create mode 100644 src/web/web_util.h diff --git a/.gitignore b/.gitignore index 3a42293..8f9f8a3 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,5 @@ build.ninja src/web/static_generated.c src/magic_generated.c src/index/static_generated.c +*.sist2 +*-shm \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 6857720..d3869f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,30 +22,33 @@ set(ARGPARSE_SHARED off) add_subdirectory(third-party/argparse) add_executable(sist2 + # argparse + third-party/argparse/argparse.h third-party/argparse/argparse.c + src/main.c src/sist.h src/io/walk.h src/io/walk.c - src/io/store.h src/io/store.c src/tpool.h src/tpool.c src/parsing/parse.h src/parsing/parse.c + src/parsing/magic_util.c src/parsing/magic_util.h src/io/serialize.h src/io/serialize.c src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c src/index/web.c src/index/web.h src/web/serve.c src/web/serve.h + src/web/web_util.c src/web/web_util.h src/index/elastic.c src/index/elastic.h src/util.c src/util.h - src/ctx.h src/types.h + src/ctx.c src/ctx.h + src/types.h src/log.c src/log.h src/cli.c src/cli.h - src/stats.c src/stats.h src/ctx.c src/parsing/sidecar.c src/parsing/sidecar.h - src/mempool/mempool.c src/mempool/mempool.h + src/database/database.c src/database/database.h + src/parsing/fs_util.h src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp - # argparse - third-party/argparse/argparse.h third-party/argparse/argparse.c - ) + src/database/database_stats.c src/database/database_stats.h src/database/database_schema.c) set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C) target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/) @@ -53,8 +56,6 @@ set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib) find_package(PkgConfig REQUIRED) -pkg_search_module(GLIB REQUIRED glib-2.0) - find_package(lmdb CONFIG REQUIRED) find_package(cJSON CONFIG REQUIRED) find_package(unofficial-mongoose CONFIG REQUIRED) @@ -63,6 +64,7 @@ find_library(MAGIC_LIB NAMES libmagic.so.1 magic PATHS /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/ ) +find_package(unofficial-sqlite3 CONFIG REQUIRED) target_include_directories( @@ -71,7 +73,6 @@ target_include_directories( ${CMAKE_SOURCE_DIR}/third-party/utf8.h/ ${CMAKE_SOURCE_DIR}/third-party/libscan/ ${CMAKE_SOURCE_DIR}/ - ${GLIB_INCLUDE_DIRS} ) target_compile_options( @@ -90,6 +91,7 @@ if (SIST_DEBUG) -fsanitize=address -fno-inline # -O2 + -w ) target_link_options( sist2 @@ -121,6 +123,7 @@ else () -Ofast -fno-stack-protector -fomit-frame-pointer + -w ) endif () @@ -137,17 +140,15 @@ target_link_libraries( lmdb cjson argparse - ${GLIB_LDFLAGS} unofficial::mongoose::mongoose CURL::libcurl pthread - c - scan ${MAGIC_LIB} + unofficial::sqlite3::sqlite3 ) add_custom_target( diff --git a/README.md b/README.md index 298dbe2..39d0783 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux ```bash vcpkg install curl[core,openssl] - vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw jasper lcms gumbo + vcpkg install lmdb sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw jasper lcms gumbo ``` 1. Build diff --git a/scripts/before_build.sh b/scripts/before_build.sh index 5e393e5..8b32302 100755 --- a/scripts/before_build.sh +++ b/scripts/before_build.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash -rm -rf index.sist2/ +( + cd .. + rm -rf index.sist2 -python3 scripts/mime.py > src/parsing/mime_generated.c -python3 scripts/serve_static.py > src/web/static_generated.c -python3 scripts/index_static.py > src/index/static_generated.c -python3 scripts/magic_static.py > src/magic_generated.c + python3 scripts/mime.py > src/parsing/mime_generated.c + python3 scripts/serve_static.py > src/web/static_generated.c + python3 scripts/index_static.py > src/index/static_generated.c + python3 scripts/magic_static.py > src/magic_generated.c -printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h \ No newline at end of file + printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h +) \ No newline at end of file diff --git a/scripts/mime.csv b/scripts/mime.csv index 5b30d31..4b32f32 100644 --- a/scripts/mime.csv +++ b/scripts/mime.csv @@ -29,7 +29,7 @@ application/mime, aps application/mspowerpoint, ppz application/msword, doc|dot|w6w|wiz|word application/netmc, mcp -application/octet-stream, bin|dump|gpg +application/octet-stream, bin|dump|gpg|pack|idx application/oda, oda application/ogg, ogv application/pdf, pdf @@ -243,7 +243,7 @@ audio/make, funk|my|pfunk audio/midi, kar audio/mid, rmi audio/mp4, m4b -audio/mpeg, m2a|mpa +audio/mpeg, m2a|mpa|mpga audio/ogg, ogg audio/s3m, s3m audio/tsp-audio, tsi @@ -382,7 +382,7 @@ text/x-pascal, p text/x-perl, pl text/x-php, php text/x-po, po -text/x-python, py +text/x-python, py|pyi text/x-ruby, rb text/x-sass, sass text/x-scss, scss diff --git a/scripts/mime.py b/scripts/mime.py index d0a539a..9eae391 100644 --- a/scripts/mime.py +++ b/scripts/mime.py @@ -1,3 +1,5 @@ +import zlib + mimes = {} noparse = set() ext_in_hash = set() @@ -135,24 +137,40 @@ def clean(t): return t.replace("/", "_").replace(".", "_").replace("+", "_").replace("-", "_") +def crc(s): + return zlib.crc32(s.encode()) & 0xffffffff + + with open("scripts/mime.csv") as f: for l in f: mime, ext_list = l.split(",") if l.startswith("!"): mime = mime[1:] noparse.add(mime) - ext = [x.strip() for x in ext_list.split("|")] + ext = [x.strip() for x in ext_list.split("|") if x.strip() != ""] mimes[mime] = ext + seen_crc = set() + for ext in mimes.values(): + for e in ext: + if crc(e) in seen_crc: + raise Exception("CRC32 collision") + seen_crc.add(crc(e)) + + seen_crc = set() + for mime in mimes.keys(): + if crc(mime) in seen_crc: + raise Exception("CRC32 collision") + seen_crc.add(crc(mime)) + print("// **Generated by mime.py**") print("#ifndef MIME_GENERATED_C") print("#define MIME_GENERATED_C") - print("#include \n") print("#include \n") # Enum print("enum mime {") for mime, ext in sorted(mimes.items()): - print(" " + clean(mime) + "=" + mime_id(mime) + ",") + print(f"{clean(mime)}={mime_id(mime)},") print("};") # Enum -> string @@ -163,20 +181,20 @@ with open("scripts/mime.csv") as f: print("default: return NULL;}}") # Ext -> Enum - print("GHashTable *mime_get_ext_table() {" - "GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);") + print("unsigned int mime_extension_lookup(unsigned long extension_crc32) {" + "switch (extension_crc32) {") for mime, ext in mimes.items(): - for e in [e for e in ext if e]: - print("g_hash_table_insert(ext_table, \"" + e + "\", (gpointer)" + clean(mime) + ");") - if e in ext_in_hash: - raise Exception("extension already in hash: " + e) - ext_in_hash.add(e) - print("return ext_table;}") + if len(ext) > 0: + for e in ext: + print(f"case {crc(e)}:", end="") + print(f"return {clean(mime)};") + print("default: return 0;}}") # string -> Enum - print("GHashTable *mime_get_mime_table() {" - "GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);") - for mime, ext in mimes.items(): - print("g_hash_table_insert(mime_table, \"" + mime + "\", (gpointer)" + clean(mime) + ");") - print("return mime_table;}") + print("unsigned int mime_name_lookup(unsigned long mime_crc32) {" + "switch (mime_crc32) {") + for mime in mimes.keys(): + print(f"case {crc(mime)}: return {clean(mime)};") + + print("default: return 0;}}") print("#endif") diff --git a/sist2-admin/frontend/public/index.html b/sist2-admin/frontend/public/index.html index 78f9a73..9af01df 100644 --- a/sist2-admin/frontend/public/index.html +++ b/sist2-admin/frontend/public/index.html @@ -4,7 +4,7 @@ - + sist2-admin diff --git a/src/auth0/auth0_c_api.h b/src/auth0/auth0_c_api.h index c90e322..a46ebfa 100644 --- a/src/auth0/auth0_c_api.h +++ b/src/auth0/auth0_c_api.h @@ -1,12 +1,13 @@ #ifndef SIST2_AUTH0_C_API_H #define SIST2_AUTH0_C_API_H -#include "stdlib.h" #ifdef __cplusplus #define EXTERNC extern "C" +#include "cstdlib" #else #define EXTERNC +#include "stdlib.h" #endif #define AUTH0_OK (0) diff --git a/src/cli.c b/src/cli.c index 8ff8875..e2540ba 100644 --- a/src/cli.c +++ b/src/cli.c @@ -2,16 +2,17 @@ #include "ctx.h" #include -#define DEFAULT_OUTPUT "index.sist2/" +#define DEFAULT_OUTPUT "index.sist2" +#define DEFAULT_NAME "index" #define DEFAULT_CONTENT_SIZE 32768 #define DEFAULT_QUALITY 2 -#define DEFAULT_THUMBNAIL_SIZE 500 +#define DEFAULT_THUMBNAIL_SIZE 552 #define DEFAULT_THUMBNAIL_COUNT 1 #define DEFAULT_REWRITE_URL "" #define DEFAULT_ES_URL "http://localhost:9200" #define DEFAULT_ES_INDEX "sist2" -#define DEFAULT_BATCH_SIZE 100 +#define DEFAULT_BATCH_SIZE 70 #define DEFAULT_TAGLINE "Lightning-fast file system indexer and search tool" #define DEFAULT_LANG "en" @@ -20,8 +21,6 @@ #define DEFAULT_MAX_MEM_BUFFER 2000 -#define DEFAULT_THROTTLE_MEMORY_THRESHOLD 0 - const char *TESS_DATAPATHS[] = { "/usr/share/tessdata/", "/usr/share/tesseract-ocr/tessdata/", @@ -48,9 +47,6 @@ void scan_args_destroy(scan_args_t *args) { if (args->name != NULL) { free(args->name); } - if (args->incremental != NULL) { - free(args->incremental); - } if (args->path != NULL) { free(args->path); } @@ -61,7 +57,6 @@ void scan_args_destroy(scan_args_t *args) { } void index_args_destroy(index_args_t *args) { - //todo if (args->es_mappings_path) { free(args->es_mappings); } @@ -76,7 +71,6 @@ void index_args_destroy(index_args_t *args) { } void web_args_destroy(web_args_t *args) { - //todo free(args); } @@ -97,19 +91,13 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { char *abs_path = abspath(argv[1]); if (abs_path == NULL) { - LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]) + LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]); } else { + abs_path = realloc(abs_path, strlen(abs_path) + 2); + strcat(abs_path, "/"); args->path = abs_path; } - if (args->incremental != OPTION_VALUE_UNSPECIFIED) { - args->incremental = abspath(args->incremental); - if (abs_path == NULL) { - sist_log("main.c", LOG_SIST_WARNING, "Could not open original index! Disabled incremental scan feature."); - args->incremental = NULL; - } - } - if (args->tn_quality == OPTION_VALUE_UNSPECIFIED) { args->tn_quality = DEFAULT_QUALITY; } else if (args->tn_quality < 2 || args->tn_quality > 31) { @@ -152,20 +140,24 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { args->output = expandpath(args->output); } - int ret = mkdir(args->output, S_IRUSR | S_IWUSR | S_IXUSR); - if (ret != 0) { - fprintf(stderr, "Invalid output: '%s' (%s).\n", args->output, strerror(errno)); - return 1; + char *abs_output = abspath(args->output); + if (args->incremental && abs_output == NULL) { + LOG_WARNINGF("main.c", "Could not open original index for incremental scan: %s. Will not perform incremental scan.", abs_output); + args->incremental = FALSE; + } else if (!args->incremental && abs_output != NULL) { + LOG_FATALF("main.c", "Index already exists: %s. If you wish to perform incremental scan, you must specify --incremental", abs_output); } + free(abs_output); if (args->depth <= 0) { - args->depth = G_MAXINT32; + args->depth = 2147483647; } else { args->depth += 1; } if (args->name == OPTION_VALUE_UNSPECIFIED) { - args->name = g_path_get_basename(args->output); + args->name = malloc(strlen(DEFAULT_NAME) + 1); + strcpy(args->name, DEFAULT_NAME); } else { char *tmp = malloc(strlen(args->name) + 1); strcpy(tmp, args->name); @@ -224,7 +216,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { } if (trained_data_path != NULL && path != trained_data_path) { LOG_FATAL("cli.c", "When specifying more than one tesseract language, all the traineddata " - "files must be in the same folder") + "files must be in the same folder"); } trained_data_path = path; @@ -232,7 +224,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { } free(lang); - ret = TessBaseAPIInit3(api, trained_data_path, args->tesseract_lang); + int ret = TessBaseAPIInit3(api, trained_data_path, args->tesseract_lang); if (ret != 0) { fprintf(stderr, "Could not initialize tesseract with lang '%s'\n", args->tesseract_lang); return 1; @@ -249,12 +241,12 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { pcre *re = pcre_compile(args->exclude_regex, 0, &error, &error_offset, 0); if (error != NULL) { - LOG_FATALF("cli.c", "pcre_compile returned error: %s (offset:%d)", error, error_offset) + LOG_FATALF("cli.c", "pcre_compile returned error: %s (offset:%d)", error, error_offset); } pcre_extra *re_extra = pcre_study(re, 0, &error); if (error != NULL) { - LOG_FATALF("cli.c", "pcre_study returned error: %s", error) + LOG_FATALF("cli.c", "pcre_study returned error: %s", error); } ScanCtx.exclude = re; @@ -276,7 +268,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { if (args->list_path != OPTION_VALUE_UNSPECIFIED) { if (strcmp(args->list_path, "-") == 0) { args->list_file = stdin; - LOG_DEBUG("cli.c", "Using stdin as list file") + LOG_DEBUG("cli.c", "Using stdin as list file"); } else { args->list_file = fopen(args->list_path, "r"); @@ -286,27 +278,27 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { } } - LOG_DEBUGF("cli.c", "arg tn_quality=%f", args->tn_quality) - LOG_DEBUGF("cli.c", "arg tn_size=%d", args->tn_size) - LOG_DEBUGF("cli.c", "arg tn_count=%d", args->tn_count) - LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size) - LOG_DEBUGF("cli.c", "arg threads=%d", args->threads) - LOG_DEBUGF("cli.c", "arg incremental=%s", args->incremental) - LOG_DEBUGF("cli.c", "arg output=%s", args->output) - LOG_DEBUGF("cli.c", "arg rewrite_url=%s", args->rewrite_url) - LOG_DEBUGF("cli.c", "arg name=%s", args->name) - LOG_DEBUGF("cli.c", "arg depth=%d", args->depth) - LOG_DEBUGF("cli.c", "arg path=%s", args->path) - LOG_DEBUGF("cli.c", "arg archive=%s", args->archive) - LOG_DEBUGF("cli.c", "arg archive_passphrase=%s", args->archive_passphrase) - LOG_DEBUGF("cli.c", "arg tesseract_lang=%s", args->tesseract_lang) - LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path) - LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex) - LOG_DEBUGF("cli.c", "arg fast=%d", args->fast) - LOG_DEBUGF("cli.c", "arg fast_epub=%d", args->fast_epub) - LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold) - LOG_DEBUGF("cli.c", "arg max_memory_buffer_mib=%d", args->max_memory_buffer_mib) - LOG_DEBUGF("cli.c", "arg list_path=%s", args->list_path) + LOG_DEBUGF("cli.c", "arg tn_quality=%f", args->tn_quality); + LOG_DEBUGF("cli.c", "arg tn_size=%d", args->tn_size); + LOG_DEBUGF("cli.c", "arg tn_count=%d", args->tn_count); + LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size); + LOG_DEBUGF("cli.c", "arg threads=%d", args->threads); + LOG_DEBUGF("cli.c", "arg incremental=%d", args->incremental); + LOG_DEBUGF("cli.c", "arg output=%s", args->output); + LOG_DEBUGF("cli.c", "arg rewrite_url=%s", args->rewrite_url); + LOG_DEBUGF("cli.c", "arg name=%s", args->name); + LOG_DEBUGF("cli.c", "arg depth=%d", args->depth); + LOG_DEBUGF("cli.c", "arg path=%s", args->path); + LOG_DEBUGF("cli.c", "arg archive=%s", args->archive); + LOG_DEBUGF("cli.c", "arg archive_passphrase=%s", args->archive_passphrase); + LOG_DEBUGF("cli.c", "arg tesseract_lang=%s", args->tesseract_lang); + LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path); + LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex); + LOG_DEBUGF("cli.c", "arg fast=%d", args->fast); + LOG_DEBUGF("cli.c", "arg fast_epub=%d", args->fast_epub); + LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold); + LOG_DEBUGF("cli.c", "arg max_memory_buffer_mib=%d", args->max_memory_buffer_mib); + LOG_DEBUGF("cli.c", "arg list_path=%s", args->list_path); return 0; } @@ -316,20 +308,20 @@ int load_external_file(const char *file_path, char **dst) { int res = stat(file_path, &info); if (res == -1) { - LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno)) + LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno)); return 1; } int fd = open(file_path, O_RDONLY); if (fd == -1) { - LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno)) + LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno)); return 1; } *dst = malloc(info.st_size + 1); res = read(fd, *dst, info.st_size); if (res < 0) { - LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno)) + LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno)); return 1; } @@ -357,7 +349,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) { char *index_path = abspath(argv[1]); if (index_path == NULL) { - LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]) + LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]); } else { args->index_path = index_path; } @@ -392,28 +384,28 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) { args->batch_size = DEFAULT_BATCH_SIZE; } - LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url) - LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index) - LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl) - LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path) - LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path) - LOG_DEBUGF("cli.c", "arg async_script=%d", args->async_script) + LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url); + LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index); + LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl); + LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path); + LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path); + LOG_DEBUGF("cli.c", "arg async_script=%d", args->async_script); if (args->script) { char log_buf[5000]; strncpy(log_buf, args->script, sizeof(log_buf)); *(log_buf + sizeof(log_buf) - 1) = '\0'; - LOG_DEBUGF("cli.c", "arg script=%s", log_buf) + LOG_DEBUGF("cli.c", "arg script=%s", log_buf); } - LOG_DEBUGF("cli.c", "arg print=%d", args->print) - LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path) - LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings) - LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path) - LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings) - LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size) - LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset) + LOG_DEBUGF("cli.c", "arg print=%d", args->print); + LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path); + LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings); + LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path); + LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings); + LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size); + LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset); return 0; } @@ -534,23 +526,24 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) { for (int i = 0; i < args->index_count; i++) { char *abs_path = abspath(args->indices[i]); if (abs_path == NULL) { - LOG_FATALF("cli.c", "Index not found: %s", args->indices[i]) + LOG_FATALF("cli.c", "Index not found: %s", args->indices[i]); } + free(abs_path); } - LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url) - LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index) - LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl) - LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline) - LOG_DEBUGF("cli.c", "arg dev=%d", args->dev) - LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address) - LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials) - LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials) - LOG_DEBUGF("cli.c", "arg auth_user=%s", args->auth_user) - LOG_DEBUGF("cli.c", "arg auth_pass=%s", args->auth_pass) - LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count) + LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url); + LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index); + LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl); + LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline); + LOG_DEBUGF("cli.c", "arg dev=%d", args->dev); + LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address); + LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials); + LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials); + LOG_DEBUGF("cli.c", "arg auth_user=%s", args->auth_user); + LOG_DEBUGF("cli.c", "arg auth_pass=%s", args->auth_pass); + LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count); for (int i = 0; i < args->index_count; i++) { - LOG_DEBUGF("cli.c", "arg indices[%d]=%s", i, args->indices[i]) + LOG_DEBUGF("cli.c", "arg indices[%d]=%s", i, args->indices[i]); } return 0; @@ -575,7 +568,7 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) { char *index_path = abspath(argv[1]); if (index_path == NULL) { - LOG_FATALF("cli.c", "Invalid index PATH argument. File not found: %s", argv[1]) + LOG_FATALF("cli.c", "Invalid index PATH argument. File not found: %s", argv[1]); } else { args->index_path = index_path; } @@ -596,12 +589,12 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) { return 1; } - LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path) + LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path); char log_buf[5000]; strncpy(log_buf, args->script, sizeof(log_buf)); *(log_buf + sizeof(log_buf) - 1) = '\0'; - LOG_DEBUGF("cli.c", "arg script=%s", log_buf) + LOG_DEBUGF("cli.c", "arg script=%s", log_buf); return 0; } diff --git a/src/cli.h b/src/cli.h index d953621..e1e039c 100644 --- a/src/cli.h +++ b/src/cli.h @@ -13,7 +13,7 @@ typedef struct scan_args { int tn_size; int content_size; int threads; - char *incremental; + int incremental; char *output; char *rewrite_url; char *name; diff --git a/src/ctx.c b/src/ctx.c index 3c24a9f..fe6e8a7 100644 --- a/src/ctx.c +++ b/src/ctx.c @@ -3,9 +3,10 @@ ScanCtx_t ScanCtx = { .stat_index_size = 0, .stat_tn_size = 0, - .dbg_current_files = NULL, - .pool = NULL + .pool = NULL, + .index.path = {0,}, }; WebCtx_t WebCtx; IndexCtx_t IndexCtx; LogCtx_t LogCtx; +__thread ProcData_t ProcData; diff --git a/src/ctx.h b/src/ctx.h index 49fdbb7..f56afd8 100644 --- a/src/ctx.h +++ b/src/ctx.h @@ -16,22 +16,17 @@ #include "libscan/msdoc/msdoc.h" #include "libscan/wpd/wpd.h" #include "libscan/json/json.h" -#include "src/io/store.h" +#include "src/database/database.h" #include "src/index/elastic.h" +#include "sqlite3.h" -#include #include typedef struct { struct index_t index; - GHashTable *mime_table; - GHashTable *ext_table; - tpool_t *pool; - tpool_t *writer_pool; - int threads; int depth; int calculate_checksums; @@ -39,16 +34,10 @@ typedef struct { size_t stat_tn_size; size_t stat_index_size; - GHashTable *original_table; - GHashTable *copy_table; - GHashTable *new_table; - pthread_mutex_t copy_table_mu; - pcre *exclude; pcre_extra *exclude_extra; int fast; - GHashTable *dbg_current_files; pthread_mutex_t dbg_current_files_mu; int dbg_failed_files_count; @@ -84,10 +73,6 @@ typedef struct { char *es_index; int batch_size; tpool_t *pool; - store_t *tag_store; - GHashTable *tags; - store_t *meta_store; - GHashTable *meta; /** * Set to false when using --print */ @@ -117,10 +102,18 @@ typedef struct { int dev; } WebCtx_t; + +typedef struct { + int thread_id; + database_t *ipc_db; + database_t *index_db; +} ProcData_t; + extern ScanCtx_t ScanCtx; extern WebCtx_t WebCtx; extern IndexCtx_t IndexCtx; extern LogCtx_t LogCtx; +extern __thread ProcData_t ProcData; #endif diff --git a/src/database/database.c b/src/database/database.c new file mode 100644 index 0000000..741187c --- /dev/null +++ b/src/database/database.c @@ -0,0 +1,586 @@ +#include "database.h" +#include "malloc.h" +#include "src/ctx.h" +#include +#include +#include "src/util.h" + +#include + + + +database_t *database_create(const char *filename, database_type_t type) { + database_t *db = malloc(sizeof(database_t)); + + strcpy(db->filename, filename); + db->type = type; + db->select_thumbnail_stmt = NULL; + + db->ipc_ctx = NULL; + + return db; +} + +__always_inline +static int sep_rfind(const char *str) { + for (int i = (int) strlen(str); i >= 0; i--) { + if (str[i] == '/') { + return i; + } + } + return -1; +} + +void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) { + if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(ctx, "Invalid parameters", -1); + } + + const char *value = (const char *) sqlite3_value_text(argv[0]); + + int stop = sep_rfind(value); + if (stop == -1) { + sqlite3_result_null(ctx); + return; + } + char parent[PATH_MAX * 3]; + strncpy(parent, value, stop); + + sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT); +} + + +void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv) { + if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) { + sqlite3_result_error(ctx, "Invalid parameters", -1); + } + + database_ipc_ctx_t *ipc_ctx = sqlite3_user_data(ctx); + + const char *current_job = (const char *) sqlite3_value_text(argv[0]); + + char buf[PATH_MAX]; + strcpy(buf, current_job); + + strcpy(ipc_ctx->current_job[ProcData.thread_id], current_job); + + sqlite3_result_text(ctx, "ok", -1, SQLITE_STATIC); +} + +void database_initialize(database_t *db) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_open(db->filename, &db->db)); + + LOG_DEBUGF("database.c", "Initializing database %s", db->filename); + if (db->type == INDEX_DATABASE) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IndexDatabaseSchema, NULL, NULL, NULL)); + } else if (db->type == IPC_CONSUMER_DATABASE || db->type == IPC_PRODUCER_DATABASE) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IpcDatabaseSchema, NULL, NULL, NULL)); + } + + sqlite3_close(db->db); +} + +void database_open(database_t *db) { + LOG_DEBUGF("tpool.c", "Opening database %s (%d)", db->filename, db->type); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_open(db->filename, &db->db)); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA cache_size = -200000;", NULL, NULL, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL)); + + if (db->type == INDEX_DATABASE) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA temp_store = memory;", NULL, NULL, NULL)); + } + + if (db->type == INDEX_DATABASE) { + // Prepare statements; + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, + "SELECT data FROM thumbnail WHERE id=? AND num=? LIMIT 1;", -1, + &db->select_thumbnail_stmt, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, + "UPDATE document SET marked=1 WHERE id=? AND mtime=? RETURNING id", + -1, + &db->mark_document_stmt, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, + "REPLACE INTO document_sidecar (id, json_data) VALUES (?,?)", -1, + &db->write_document_sidecar_stmt, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, + "REPLACE INTO document (id, mtime, size, json_data) VALUES (?, ?, ?, ?);", -1, + &db->write_document_stmt, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, + "INSERT INTO thumbnail (id, num, data) VALUES (?,?,?) ON CONFLICT DO UPDATE SET data=excluded.data;", -1, + &db->write_thumbnail_stmt, NULL)); + + // Create functions + sqlite3_create_function( + db->db, + "path_parent", + 1, + SQLITE_UTF8, + NULL, + path_parent_func, + NULL, + NULL + ); + } else if (db->type == IPC_CONSUMER_DATABASE) { + + sqlite3_create_function( + db->db, + "save_current_job_info", + 1, + SQLITE_UTF8, + db->ipc_ctx, + save_current_job_info, + NULL, + NULL + ); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, + "DELETE FROM parse_job WHERE id = (SELECT MIN(id) FROM parse_job)" + " RETURNING filepath,mtime,st_size,save_current_job_info(filepath);", + -1, &db->pop_parse_job_stmt, NULL + )); + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, + "DELETE FROM index_job WHERE id = (SELECT MIN(id) FROM index_job)" + " RETURNING doc_id,type,line;", + -1, &db->pop_index_job_stmt, NULL + )); + + } else if (db->type == IPC_PRODUCER_DATABASE) { + char sql[40]; + int max_size_mb = 10; // TODO: read from args. + + snprintf(sql, sizeof(sql), "PRAGMA max_page_count=%d", (max_size_mb * 1024 * 1024) / 4096); + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, sql, NULL, NULL, NULL)); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, "INSERT INTO parse_job (filepath,mtime,st_size) VALUES (?,?,?);", -1, + &db->insert_parse_job_stmt, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, "INSERT INTO index_job (doc_id,type,line) VALUES (?,?,?);", -1, + &db->insert_index_job_stmt, NULL)); + + sqlite3_create_function( + db->db, + "path_parent", + 1, + SQLITE_UTF8, + NULL, + path_parent_func, + NULL, + NULL + ); + } + +} + +void database_close(database_t *db, int optimize) { + LOG_DEBUGF("database.c", "Closing database %s", db->filename); + + if (optimize) { + LOG_DEBUG("database.c", "Optimizing database"); + // TODO: This should be an optional argument +// CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "VACUUM;", NULL, NULL, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA optimize;", NULL, NULL, NULL)); + } + + sqlite3_close(db->db); + free(db); + db = NULL; +} + +void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *return_value_len) { + sqlite3_bind_text(db->select_thumbnail_stmt, 1, id, -1, SQLITE_STATIC); + sqlite3_bind_int(db->select_thumbnail_stmt, 2, num); + + int ret = sqlite3_step(db->select_thumbnail_stmt); + + // TODO: if row not found, return null + if (ret != SQLITE_ROW) { + LOG_FATALF("database.c", "FIXME: tn step returned %d", ret); + } + + const void *blob = sqlite3_column_blob(db->select_thumbnail_stmt, 0); + const int blob_size = sqlite3_column_bytes(db->select_thumbnail_stmt, 0); + + *return_value_len = blob_size; + void *return_data = malloc(blob_size); + memcpy(return_data, blob, blob_size); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->select_thumbnail_stmt)); + + return return_data; +} + +void database_write_index_descriptor(database_t *db, index_descriptor_t *desc) { + + sqlite3_exec(db->db, "DELETE FROM descriptor;", NULL, NULL, NULL); + + sqlite3_stmt *stmt; + + sqlite3_prepare_v2(db->db, "INSERT INTO descriptor (id, version_major, version_minor, version_patch," + " root, name, rewrite_url, timestamp) VALUES (?,?,?,?,?,?,?,?);", -1, &stmt, NULL); + sqlite3_bind_text(stmt, 1, desc->id, -1, SQLITE_STATIC); + sqlite3_bind_int(stmt, 2, desc->version_major); + sqlite3_bind_int(stmt, 3, desc->version_minor); + sqlite3_bind_int(stmt, 4, desc->version_patch); + sqlite3_bind_text(stmt, 5, desc->root, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 6, desc->name, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 7, desc->rewrite_url, -1, SQLITE_STATIC); + sqlite3_bind_int64(stmt, 8, desc->timestamp); + + CRASH_IF_STMT_FAIL(sqlite3_step(stmt)); + + sqlite3_finalize(stmt); +} + +index_descriptor_t *database_read_index_descriptor(database_t *db) { + + sqlite3_stmt *stmt; + + sqlite3_prepare_v2(db->db, "SELECT id, version_major, version_minor, version_patch," + " root, name, rewrite_url, timestamp FROM descriptor;", -1, &stmt, NULL); + + CRASH_IF_STMT_FAIL(sqlite3_step(stmt)); + + const char *id = (char *) sqlite3_column_text(stmt, 0); + int v_major = sqlite3_column_int(stmt, 1); + int v_minor = sqlite3_column_int(stmt, 2); + int v_patch = sqlite3_column_int(stmt, 3); + const char *root = (char *) sqlite3_column_text(stmt, 4); + const char *name = (char *) sqlite3_column_text(stmt, 5); + const char *rewrite_url = (char *) sqlite3_column_text(stmt, 6); + int timestamp = sqlite3_column_int(stmt, 7); + + index_descriptor_t *desc = malloc(sizeof(index_descriptor_t)); + strcpy(desc->id, id); + snprintf(desc->version, sizeof(desc->version), "%d.%d.%d", v_major, v_minor, v_patch); + desc->version_major = v_major; + desc->version_minor = v_minor; + desc->version_patch = v_patch; + strcpy(desc->root, root); + strcpy(desc->name, name); + strcpy(desc->rewrite_url, rewrite_url); + desc->timestamp = timestamp; + + CRASH_IF_NOT_SQLITE_OK(sqlite3_finalize(stmt)); + + return desc; +} + +database_iterator_t *database_create_document_iterator(database_t *db) { + + sqlite3_stmt *stmt; + + // TODO: remove mtime, size, _id from json_data + + sqlite3_prepare_v2(db->db, "WITH doc (j) AS (SELECT CASE" + " WHEN sc.json_data IS NULL THEN" + " CASE" + " WHEN t.tag IS NULL THEN" + " document.json_data" + " ELSE" + " json_set(document.json_data, '$.tag', json_group_array(t.tag))" + " END" + " ELSE" + " CASE" + " WHEN t.tag IS NULL THEN" + " json_patch(document.json_data, sc.json_data)" + " ELSE" + // This will overwrite any tags specified in the sidecar file! + // TODO: concatenate the two arrays? + " json_set(json_patch(document.json_data, sc.json_data), '$.tag', json_group_array(t.tag))" + " END" + " END" + " FROM document" + " LEFT JOIN document_sidecar sc ON document.id = sc.id" + " LEFT JOIN tag t ON document.id = t.id" + " GROUP BY document.id)" + " SELECT json_set(j, '$.index', (SELECT id FROM descriptor)) FROM doc", -1, &stmt, NULL); + + database_iterator_t *iter = malloc(sizeof(database_iterator_t)); + + iter->stmt = stmt; + iter->db = db; + + return iter; +} + +cJSON *database_document_iter(database_iterator_t *iter) { + + if (iter->stmt == NULL) { + LOG_ERROR("database.c", "FIXME: database_document_iter() called after iteration stopped"); + return NULL; + } + + int ret = sqlite3_step(iter->stmt); + + if (ret == SQLITE_ROW) { + const char *json_string = (const char *) sqlite3_column_text(iter->stmt, 0); + return cJSON_Parse(json_string); + } + + if (ret != SQLITE_DONE) { + LOG_FATALF("database.c", "FIXME: doc iter returned %s", sqlite3_errmsg(iter->db->db)); + } + + if (sqlite3_finalize(iter->stmt) != SQLITE_OK) { + LOG_FATALF("database.c", "FIXME: doc iter returned %s", sqlite3_errmsg(iter->db->db)); + } + + iter->stmt = NULL; + + return NULL; +} + +cJSON *database_incremental_scan_begin(database_t *db) { + LOG_DEBUG("database.c", "Preparing database for incremental scan"); + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "UPDATE document SET marked=0;", NULL, NULL, NULL)); +} + +cJSON *database_incremental_scan_end(database_t *db) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec( + db->db, + "DELETE FROM delete_list WHERE id IN (SELECT id FROM document WHERE marked=1);", + NULL, NULL, NULL + )); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec( + db->db, + "DELETE FROM thumbnail WHERE id IN (SELECT id FROM document WHERE marked=0);", + NULL, NULL, NULL + )); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec( + db->db, + "INSERT INTO delete_list (id) SELECT id FROM document WHERE marked=0;", + NULL, NULL, NULL + )); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec( + db->db, + "DELETE FROM document_sidecar WHERE id IN (SELECT id FROM document WHERE marked=0);", + NULL, NULL, NULL + )); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec( + db->db, + "DELETE FROM document WHERE marked=0;", + NULL, NULL, NULL + )); +} + +int database_mark_document(database_t *db, const char *id, int mtime) { + sqlite3_bind_text(db->mark_document_stmt, 1, id, -1, SQLITE_STATIC); + sqlite3_bind_int(db->mark_document_stmt, 2, mtime); + + pthread_mutex_lock(&db->ipc_ctx->index_db_mutex); + int ret = sqlite3_step(db->mark_document_stmt); + + if (ret == SQLITE_ROW) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->mark_document_stmt)); + pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex); + return TRUE; + } + + if (ret == SQLITE_DONE) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->mark_document_stmt)); + pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex); + return FALSE; + } + pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex); + + CRASH_IF_STMT_FAIL(ret); +} + +void database_write_document(database_t *db, document_t *doc, const char *json_data) { + sqlite3_bind_text(db->write_document_stmt, 1, doc->doc_id, -1, SQLITE_STATIC); + sqlite3_bind_int(db->write_document_stmt, 2, doc->mtime); + sqlite3_bind_int64(db->write_document_stmt, 3, (long) doc->size); + sqlite3_bind_text(db->write_document_stmt, 4, json_data, -1, SQLITE_STATIC); + + pthread_mutex_lock(&db->ipc_ctx->index_db_mutex); + CRASH_IF_STMT_FAIL(sqlite3_step(db->write_document_stmt)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_document_stmt)); + pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex); +} + + +void database_write_document_sidecar(database_t *db, const char *id, const char *json_data) { + sqlite3_bind_text(db->write_document_sidecar_stmt, 1, id, -1, SQLITE_STATIC); + sqlite3_bind_text(db->write_document_sidecar_stmt, 2, json_data, -1, SQLITE_STATIC); + + pthread_mutex_lock(&db->ipc_ctx->index_db_mutex); + CRASH_IF_STMT_FAIL(sqlite3_step(db->write_document_sidecar_stmt)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_document_sidecar_stmt)); + pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex); +} + +void database_write_thumbnail(database_t *db, const char *id, int num, void *data, size_t data_size) { + sqlite3_bind_text(db->write_thumbnail_stmt, 1, id, -1, SQLITE_STATIC); + sqlite3_bind_int(db->write_thumbnail_stmt, 2, num); + sqlite3_bind_blob(db->write_thumbnail_stmt, 3, data, (int) data_size, SQLITE_STATIC); + + pthread_mutex_lock(&db->ipc_ctx->index_db_mutex); + CRASH_IF_STMT_FAIL(sqlite3_step(db->write_thumbnail_stmt)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_thumbnail_stmt)); + pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex); +} + + +//void database_create_fts_index(database_t *db, database_t *fts_db) { +// // In a separate file, +// +// // use database_initialize() to create FTS schema +// // if --force-reset, then truncate the tables first +// +// /* +// * create/append fts table +// * +// * create/append scalar index table with +// * id,index,size,mtime,mime +// * +// * create/append path index table with +// * index,path,depth +// * +// * content table is a view with SELECT UNION for all attached tables +// * random_seed column +// */ +// +// // INSERT INTO ft(ft) VALUES('optimize'); +//} + +job_t *database_get_work(database_t *db, job_type_t job_type) { + job_t *job; + + pthread_mutex_lock(&db->ipc_ctx->mutex); + while (db->ipc_ctx->job_count == 0 && !db->ipc_ctx->no_more_jobs) { + pthread_cond_timedwait_ms(&db->ipc_ctx->has_work_cond, &db->ipc_ctx->mutex, 10); + } + pthread_mutex_unlock(&db->ipc_ctx->mutex); + + pthread_mutex_lock(&db->ipc_ctx->db_mutex); + + if (job_type == JOB_PARSE_JOB) { + int ret = sqlite3_step(db->pop_parse_job_stmt); + if (ret == SQLITE_DONE) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_parse_job_stmt)); + pthread_mutex_unlock(&db->ipc_ctx->db_mutex); + return NULL; + } else { + CRASH_IF_STMT_FAIL(ret); + } + + job = malloc(sizeof(*job)); + + job->parse_job = create_parse_job( + (const char *) sqlite3_column_text(db->pop_parse_job_stmt, 0), + sqlite3_column_int(db->pop_parse_job_stmt, 1), + sqlite3_column_int64(db->pop_parse_job_stmt, 2)); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_parse_job_stmt)); + } else { + + int ret = sqlite3_step(db->pop_index_job_stmt); + + if (ret == SQLITE_DONE) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_index_job_stmt)); + pthread_mutex_unlock(&db->ipc_ctx->db_mutex); + return NULL; + } else { + CRASH_IF_STMT_FAIL(ret); + } + + job = malloc(sizeof(*job)); + + const char *line = (const char *) sqlite3_column_text(db->pop_index_job_stmt, 2); + if (line != NULL) { + job->bulk_line = malloc(sizeof(es_bulk_line_t) + strlen(line) + 1); + strcpy(job->bulk_line->line, line); + } else { + job->bulk_line = malloc(sizeof(es_bulk_line_t)); + } + strcpy(job->bulk_line->doc_id, (const char *) sqlite3_column_text(db->pop_index_job_stmt, 0)); + job->bulk_line->type = sqlite3_column_int(db->pop_index_job_stmt, 1); + job->bulk_line->next = NULL; + + // TODO CRASH IF NOT OK + sqlite3_step(db->pop_parse_job_stmt); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_index_job_stmt)); + } + + pthread_mutex_unlock(&db->ipc_ctx->db_mutex); + + pthread_mutex_lock(&db->ipc_ctx->mutex); + db->ipc_ctx->job_count -= 1; + pthread_mutex_unlock(&db->ipc_ctx->mutex); + + job->type = job_type; + return job; +} + +void database_add_work(database_t *db, job_t *job) { + int ret; + + pthread_mutex_lock(&db->ipc_ctx->db_mutex); + + if (job->type == JOB_PARSE_JOB) { + do { + sqlite3_bind_text(db->insert_parse_job_stmt, 1, job->parse_job->filepath, -1, SQLITE_STATIC); + sqlite3_bind_int(db->insert_parse_job_stmt, 2, job->parse_job->vfile.mtime); + sqlite3_bind_int64(db->insert_parse_job_stmt, 3, (long) job->parse_job->vfile.st_size); + + ret = sqlite3_step(db->insert_parse_job_stmt); + + if (ret == SQLITE_FULL) { + usleep(1000000); + } else { + CRASH_IF_STMT_FAIL(ret); + } + + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->insert_parse_job_stmt)); + } while (ret != SQLITE_DONE); + } else if (job->type == JOB_BULK_LINE) { + do { + sqlite3_bind_text(db->insert_index_job_stmt, 1, job->bulk_line->doc_id, -1, SQLITE_STATIC); + sqlite3_bind_int(db->insert_index_job_stmt, 2, job->bulk_line->type); + sqlite3_bind_text(db->insert_index_job_stmt, 3, job->bulk_line->line, -1, SQLITE_STATIC); + + ret = sqlite3_step(db->insert_index_job_stmt); + + if (ret == SQLITE_FULL) { + sqlite3_reset(db->insert_index_job_stmt); + pthread_mutex_unlock(&db->ipc_ctx->db_mutex); + usleep(100000); + pthread_mutex_lock(&db->ipc_ctx->db_mutex); + continue; + } else { + CRASH_IF_STMT_FAIL(ret); + } + + ret = sqlite3_reset(db->insert_index_job_stmt); + if (ret == SQLITE_FULL) { + pthread_mutex_unlock(&db->ipc_ctx->db_mutex); + usleep(100000); + pthread_mutex_lock(&db->ipc_ctx->db_mutex); + } + + } while (ret != SQLITE_DONE && ret != SQLITE_OK); + } else { + LOG_FATAL("database.c", "FIXME: invalid job type"); + } + pthread_mutex_unlock(&db->ipc_ctx->db_mutex); + + pthread_mutex_lock(&db->ipc_ctx->mutex); + db->ipc_ctx->job_count += 1; + pthread_cond_signal(&db->ipc_ctx->has_work_cond); + pthread_mutex_unlock(&db->ipc_ctx->mutex); +} diff --git a/src/database/database.h b/src/database/database.h new file mode 100644 index 0000000..d36f802 --- /dev/null +++ b/src/database/database.h @@ -0,0 +1,147 @@ +#ifndef SIST2_DATABASE_H +#define SIST2_DATABASE_H + +#include +#include +#include "src/sist.h" +#include "src/index/elastic.h" + +typedef struct index_descriptor index_descriptor_t; + +extern const char *IpcDatabaseSchema; +extern const char *IndexDatabaseSchema; + +typedef enum { + INDEX_DATABASE, + IPC_CONSUMER_DATABASE, + IPC_PRODUCER_DATABASE, + FTS_DATABASE +} database_type_t; + +typedef enum { + JOB_UNDEFINED, + JOB_BULK_LINE, + JOB_PARSE_JOB +} job_type_t; + +typedef struct { + job_type_t type; + union { + parse_job_t *parse_job; + es_bulk_line_t *bulk_line; + }; +} job_t; + +typedef struct { + int job_count; + int no_more_jobs; + int completed_job_count; + + pthread_mutex_t mutex; + pthread_mutex_t db_mutex; + pthread_mutex_t index_db_mutex; + pthread_cond_t has_work_cond; + char current_job[256][PATH_MAX * 2]; +} database_ipc_ctx_t; + +typedef struct database { + char filename[PATH_MAX]; + database_type_t type; + sqlite3 *db; + + // Prepared statements + sqlite3_stmt *select_thumbnail_stmt; + sqlite3_stmt *treemap_merge_up_update_stmt; + sqlite3_stmt *treemap_merge_up_delete_stmt; + + sqlite3_stmt *mark_document_stmt; + sqlite3_stmt *write_document_stmt; + sqlite3_stmt *write_document_sidecar_stmt; + sqlite3_stmt *write_thumbnail_stmt; + + sqlite3_stmt *insert_parse_job_stmt; + sqlite3_stmt *insert_index_job_stmt; + sqlite3_stmt *pop_parse_job_stmt; + sqlite3_stmt *pop_index_job_stmt; + + database_ipc_ctx_t *ipc_ctx; +} database_t; + +typedef struct { + database_t *db; + sqlite3_stmt *stmt; +} database_iterator_t; + +typedef struct { + const char *path; + const char *parent; + long size; +} treemap_row_t; + +static treemap_row_t null_treemap_row = {0, 0, 0}; + + +database_t *database_create(const char *filename, database_type_t type); + +void database_initialize(database_t *db); + +void database_open(database_t *db); + +void database_close(database_t *, int optimize); + +void database_write_thumbnail(database_t *db, const char *id, int num, void *data, size_t data_size); + +void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *return_value_len); + +void database_write_index_descriptor(database_t *db, index_descriptor_t *desc); + +index_descriptor_t *database_read_index_descriptor(database_t *db); + +void database_write_document(database_t *db, document_t *doc, const char *json_data); + +database_iterator_t *database_create_document_iterator(database_t *db); + +cJSON *database_document_iter(database_iterator_t *); + +#define database_document_iter_foreach(element, iter) \ + for (cJSON *element = database_document_iter(iter); element != NULL; element = database_document_iter(iter)) + +cJSON *database_incremental_scan_begin(database_t *db); + +cJSON *database_incremental_scan_end(database_t *db); + +int database_mark_document(database_t *db, const char *id, int mtime); + +void database_write_document_sidecar(database_t *db, const char *id, const char *json_data); + +database_iterator_t *database_create_treemap_iterator(database_t *db, long threshold); + +treemap_row_t database_treemap_iter(database_iterator_t *iter); + +#define database_treemap_iter_foreach(element, iter) \ + for (treemap_row_t element = database_treemap_iter(iter); element.path != NULL; element = database_treemap_iter(iter)) + + +void database_generate_stats(database_t *db, double treemap_threshold); + +job_t *database_get_work(database_t *db, job_type_t job_type); + +void database_add_work(database_t *db, job_t *job); + +//void database_index(database_t *db); + +#define CRASH_IF_STMT_FAIL(x) do { \ + int return_value = x; \ + if (return_value != SQLITE_DONE && return_value != SQLITE_ROW) { \ + LOG_FATALF("database.c", "Sqlite error @ database.c:%d : (%d) %s", __LINE__, return_value, sqlite3_errmsg(db->db)); \ + } \ + } while (0) + +#define CRASH_IF_NOT_SQLITE_OK(x) do { \ + int return_value = x; \ + if (return_value != SQLITE_OK) { \ + LOG_FATALF("database.c", "Sqlite error @ database.c:%d : (%d) %s", __LINE__, return_value, sqlite3_errmsg(db->db)); \ + } \ + } while (0) + +#endif //SIST2_DATABASE_H \ No newline at end of file diff --git a/src/database/database_schema.c b/src/database/database_schema.c new file mode 100644 index 0000000..23cb05f --- /dev/null +++ b/src/database/database_schema.c @@ -0,0 +1,78 @@ + +const char *IpcDatabaseSchema = + "CREATE TABLE parse_job (" + " id INTEGER PRIMARY KEY," + " filepath TEXT NOT NULL," + " mtime INTEGER NOT NULL," + " st_size INTEGER NOT NULL" + ");" + "" + "CREATE TABLE index_job (" + " id INTEGER PRIMARY KEY," + " doc_id TEXT NOT NULL CHECK ( length(doc_id) = 32 )," + " type INTEGER NOT NULL," + " line TEXT" + ");"; + +const char *IndexDatabaseSchema = + "CREATE TABLE thumbnail (" + " id TEXT NOT NULL CHECK ( length(id) = 32 )," + " num INTEGER NOT NULL," + " data BLOB NOT NULL," + " PRIMARY KEY(id, num)" + ") WITHOUT ROWID;" + "" + "CREATE TABLE document (" + " id TEXT PRIMARY KEY CHECK ( length(id) = 32 )," + " marked INTEGER NOT NULL DEFAULT (1)," + " mtime INTEGER NOT NULL," + " size INTEGER NOT NULL," + " json_data TEXT NOT NULL CHECK ( json_valid(json_data) )" + ") WITHOUT ROWID;" + "" + "CREATE TABLE delete_list (" + " id TEXT PRIMARY KEY CHECK ( length(id) = 32 )" + ") WITHOUT ROWID;" + "" + "CREATE TABLE tag (" + " id TEXT NOT NULL," + " tag TEXT NOT NULL" + ");" + "" + "CREATE TABLE document_sidecar (" + " id TEXT PRIMARY KEY NOT NULL," + " json_data TEXT NOT NULL" + ") WITHOUT ROWID;" + "" + "CREATE TABLE descriptor (" + " id TEXT NOT NULL," + " version_major INTEGER NOT NULL," + " version_minor INTEGER NOT NULL," + " version_patch INTEGER NOT NULL," + " root TEXT NOT NULL," + " name TEXT NOT NULL," + " rewrite_url TEXT," + " timestamp INTEGER NOT NULL" + ");" + "" + "CREATE TABLE stats_treemap (" + " path TEXT NOT NULL," + " size INTEGER NOT NULL" + ");" + "" + "CREATE TABLE stats_size_agg (" + " bucket INTEGER NOT NULL," + " count INTEGER NOT NULL" + ");" + "" + "CREATE TABLE stats_date_agg (" + " bucket INTEGER NOT NULL," + " count INTEGER NOT NULL" + ");" + "" + "CREATE TABLE stats_mime_agg (" + " mime TEXT NOT NULL," + " size INTEGER NOT NULL," + " count INTEGER NOT NULL" + ");"; + diff --git a/src/database/database_stats.c b/src/database/database_stats.c new file mode 100644 index 0000000..2f2c73f --- /dev/null +++ b/src/database/database_stats.c @@ -0,0 +1,159 @@ +#include "database.h" +#include "src/sist.h" +#include "src/ctx.h" + +#define TREEMAP_MINIMUM_MERGES_TO_CONTINUE (100) +#define SIZE_BUCKET (long)(5 * 1000 * 1000) +#define DATE_BUCKET (long)(2629800) // ~30 days + +database_iterator_t *database_create_treemap_iterator(database_t *db, long threshold) { + + sqlite3_stmt *stmt; + + sqlite3_prepare_v2(db->db, + "SELECT path, path_parent(path), size FROM tm" + " WHERE path_parent(path) IN (SELECT path FROM tm)" + " AND sizestmt = stmt; + iter->db = db; + + return iter; +} + +treemap_row_t database_treemap_iter(database_iterator_t *iter) { + + if (iter->stmt == NULL) { + LOG_FATAL("database.c", "FIXME: database_treemap_iter() called after iteration stopped"); + } + + int ret = sqlite3_step(iter->stmt); + + if (ret == SQLITE_ROW) { + treemap_row_t row = { + .path = (const char *) sqlite3_column_text(iter->stmt, 0), + .parent = (const char *) sqlite3_column_text(iter->stmt, 1), + .size = sqlite3_column_int64(iter->stmt, 2) + }; + + return row; + } + + if (ret != SQLITE_DONE) { + LOG_FATALF("database.c", "FIXME: doc iter returned %s", sqlite3_errmsg(iter->db->db)); + } + + sqlite3_finalize(iter->stmt); + iter->stmt = NULL; + + return (treemap_row_t) {NULL, NULL, 0}; +} + +void database_generate_stats(database_t *db, double treemap_threshold) { + + LOG_INFO("database.c", "Generating stats"); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "DELETE FROM stats_size_agg;", NULL, NULL, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "DELETE FROM stats_date_agg;", NULL, NULL, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "DELETE FROM stats_mime_agg;", NULL, NULL, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "DELETE FROM stats_treemap;", NULL, NULL, NULL)); + + CRASH_IF_NOT_SQLITE_OK( + sqlite3_exec(db->db, "CREATE TEMP TABLE tm(path TEXT PRIMARY KEY, size INT);", NULL, NULL, NULL)); + + sqlite3_prepare_v2(db->db, "UPDATE tm SET size=size+? WHERE path=?;", -1, &db->treemap_merge_up_update_stmt, NULL); + sqlite3_prepare_v2(db->db, "DELETE FROM tm WHERE path = ?;", -1, &db->treemap_merge_up_delete_stmt, NULL); + + // size aggregation + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db->db, "INSERT INTO stats_size_agg" + " SELECT" + " cast(size / ?1 as int) * ?1 as bucket," + " count(*) as count" + " FROM document" + " GROUP BY bucket", -1, &stmt, NULL); + sqlite3_bind_int(stmt, 1, SIZE_BUCKET); + CRASH_IF_STMT_FAIL(sqlite3_step(stmt)); + + sqlite3_finalize(stmt); + + // date aggregation + sqlite3_prepare_v2(db->db, "INSERT INTO stats_date_agg" + " SELECT" + " cast(mtime / ?1 as int) * ?1 as bucket," + " count(*) as count" + " FROM document" + " GROUP BY bucket", -1, &stmt, NULL); + sqlite3_bind_int(stmt, 1, DATE_BUCKET); + CRASH_IF_STMT_FAIL(sqlite3_step(stmt)); + + sqlite3_finalize(stmt); + + // mime aggregation + sqlite3_prepare_v2(db->db, "INSERT INTO stats_mime_agg" + " SELECT" + " (json_data->>'mime') as bucket," + " sum(size)," + " count(*)" + " FROM document" + " WHERE bucket IS NOT NULL" + " GROUP BY bucket", -1, &stmt, NULL); + CRASH_IF_STMT_FAIL(sqlite3_step(stmt)); + + sqlite3_finalize(stmt); + + // Treemap + sqlite3_prepare_v2(db->db, "SELECT SUM(size) FROM document;", -1, &stmt, NULL); + CRASH_IF_STMT_FAIL(sqlite3_step(stmt)); + long total_size = sqlite3_column_int64(stmt, 0); + long threshold = (long) ((double) total_size * treemap_threshold); + sqlite3_finalize(stmt); + + // flat map + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, + "INSERT INTO tm (path, size) SELECT json_data->>'path' as path, sum(size)" + " FROM document WHERE json_data->>'parent' IS NULL GROUP BY path;", + NULL, NULL, NULL)); + + // Merge up + int merged_rows = 0; + do { + if (merged_rows) { + LOG_INFOF("database.c", "Treemap merge iteration (%d rows changed)", merged_rows); + } + merged_rows = 0; + + sqlite3_prepare_v2(db->db, + "INSERT INTO tm (path, size) SELECT path_parent(path) as parent, 0 " + " FROM tm WHERE parent not IN (SELECT path FROM tm) AND sizetreemap_merge_up_update_stmt, 1, row.size); + sqlite3_bind_text(db->treemap_merge_up_update_stmt, 2, row.parent, -1, SQLITE_STATIC); + CRASH_IF_STMT_FAIL(sqlite3_step(db->treemap_merge_up_update_stmt)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->treemap_merge_up_update_stmt)); + + sqlite3_bind_text(db->treemap_merge_up_delete_stmt, 1, row.path, -1, SQLITE_STATIC); + CRASH_IF_STMT_FAIL(sqlite3_step(db->treemap_merge_up_delete_stmt)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->treemap_merge_up_delete_stmt)); + + merged_rows += 1; + } + } while (merged_rows > TREEMAP_MINIMUM_MERGES_TO_CONTINUE); + + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, + "INSERT INTO stats_treemap (path, size) SELECT path,size FROM tm;", + NULL, NULL, NULL)); + + LOG_INFO("database.c", "Done!"); +} + diff --git a/src/database/database_stats.h b/src/database/database_stats.h new file mode 100644 index 0000000..fe4d507 --- /dev/null +++ b/src/database/database_stats.h @@ -0,0 +1,5 @@ +#ifndef SIST2_DATABASE_STATS_H +#define SIST2_DATABASE_STATS_H + + +#endif //SIST2_DATABASE_STATS_H diff --git a/src/index/elastic.c b/src/index/elastic.c index 9d582fc..337da0e 100644 --- a/src/index/elastic.c +++ b/src/index/elastic.c @@ -29,7 +29,7 @@ void destroy_indexer(es_indexer_t *indexer) { return; } - LOG_DEBUG("elastic.c", "Destroying indexer") + LOG_DEBUG("elastic.c", "Destroying indexer"); if (indexer->es_url != NULL) { free(indexer->es_url); @@ -64,26 +64,21 @@ void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) { cJSON_Delete(line); } -void index_json_func(tpool_work_arg_shm_t *arg) { - // Copy arg to heap because it's going to be freed immediately after this function returns - es_bulk_line_t *line = malloc(arg->arg_size); - memcpy(line, arg->arg, arg->arg_size); - - elastic_index_line(line); +void index_json_func(job_t *job) { + elastic_index_line(job->bulk_line); } -void delete_document(const char *document_id_str, void *UNUSED(_data)) { +void delete_document(const char *document_id) { es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t)); bulk_line->type = ES_BULK_LINE_DELETE; bulk_line->next = NULL; - strcpy(bulk_line->doc_id, document_id_str); + strcpy(bulk_line->doc_id, document_id); - tpool_work_arg_t arg = { - .arg_size = sizeof(es_bulk_line_t), - .arg = bulk_line - }; - tpool_add_work(IndexCtx.pool, index_json_func, &arg); + tpool_add_work(IndexCtx.pool, &(job_t) { + .type = JOB_BULK_LINE, + .bulk_line = bulk_line, + }); } @@ -100,11 +95,10 @@ void index_json(cJSON *document, const char doc_id[SIST_DOC_ID_LEN]) { bulk_line->next = NULL; cJSON_free(json); - tpool_work_arg_t arg = { - .arg_size = sizeof(es_bulk_line_t) + json_len + 2, - .arg = bulk_line - }; - tpool_add_work(IndexCtx.pool, index_json_func, &arg); + tpool_add_work(IndexCtx.pool, &(job_t) { + .type = JOB_BULK_LINE, + .bulk_line = bulk_line, + }); } void execute_update_script(const char *script, int async, const char index_id[SIST_INDEX_ID_LEN]) { @@ -278,7 +272,7 @@ void print_error(response_t *r) { void _elastic_flush(int max) { if (max == 0) { - LOG_WARNING("elastic.c", "calling _elastic_flush with 0 in queue") + LOG_WARNING("elastic.c", "calling _elastic_flush with 0 in queue"); return; } @@ -291,13 +285,13 @@ void _elastic_flush(int max) { response_t *r = web_post(bulk_url, buf, IndexCtx.es_insecure_ssl); if (r->status_code == 0) { - LOG_FATALF("elastic.c", "Could not connect to %s, make sure that elasticsearch is running!\n", IndexCtx.es_url) + LOG_FATALF("elastic.c", "Could not connect to %s, make sure that elasticsearch is running!\n", IndexCtx.es_url); } if (r->status_code == 413) { if (max <= 1) { - LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->doc_id) + LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->doc_id); free_response(r); free(buf); free_queue(1); @@ -318,7 +312,7 @@ void _elastic_flush(int max) { free_response(r); free(buf); - LOG_WARNING("elastic.c", "Got 429 status, will retry after delay") + LOG_WARNING("elastic.c", "Got 429 status, will retry after delay"); usleep(1000000 * 20); _elastic_flush(max); return; @@ -453,7 +447,7 @@ es_version_t *elastic_get_version(const char *es_url, int insecure) { } if (cJSON_GetObjectItem(response, "error") != NULL) { - LOG_WARNING("elastic.c", "Could not get Elasticsearch version") + LOG_WARNING("elastic.c", "Could not get Elasticsearch version"); print_error(r); free_response(r); return NULL; @@ -489,7 +483,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s IndexCtx.es_version = es_version; if (es_version == NULL) { - LOG_FATAL("elastic.c", "Could not get ES version") + LOG_FATAL("elastic.c", "Could not get ES version"); } LOG_INFOF("elastic.c", @@ -497,7 +491,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s format_es_version(es_version), IS_SUPPORTED_ES_VERSION(es_version), IS_LEGACY_VERSION(es_version)); if (!IS_SUPPORTED_ES_VERSION(es_version)) { - LOG_FATAL("elastic.c", "This elasticsearch version is not supported!") + LOG_FATAL("elastic.c", "This elasticsearch version is not supported!"); } char *settings = NULL; @@ -524,7 +518,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s if (r->status_code != 200) { print_error(r); - LOG_FATAL("elastic.c", "Could not create index") + LOG_FATAL("elastic.c", "Could not create index"); } LOG_INFOF("elastic.c", "Create index <%d>", r->status_code); @@ -545,7 +539,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s LOG_INFOF("elastic.c", "Update ES settings <%d>", r->status_code); if (r->status_code != 200) { print_error(r); - LOG_FATAL("elastic.c", "Could not update user settings") + LOG_FATAL("elastic.c", "Could not update user settings"); } free_response(r); @@ -560,7 +554,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s LOG_INFOF("elastic.c", "Update ES mappings <%d>", r->status_code); if (r->status_code != 200) { print_error(r); - LOG_FATAL("elastic.c", "Could not update user mappings") + LOG_FATAL("elastic.c", "Could not update user mappings"); } free_response(r); diff --git a/src/index/elastic.h b/src/index/elastic.h index 319fe71..94c847d 100644 --- a/src/index/elastic.h +++ b/src/index/elastic.h @@ -46,7 +46,7 @@ void print_json(cJSON *document, const char index_id_str[SIST_INDEX_ID_LEN]); void index_json(cJSON *document, const char doc_id[SIST_INDEX_ID_LEN]); -void delete_document(const char *document_id_str, void* data); +void delete_document(const char *document_id); es_indexer_t *create_indexer(const char *url, const char *index); diff --git a/src/index/web.c b/src/index/web.c index f608da1..4dcd9bc 100644 --- a/src/index/web.c +++ b/src/index/web.c @@ -65,7 +65,7 @@ void web_post_async_poll(subreq_ctx_t *req) { curl_easy_getinfo(req->handle, CURLINFO_RESPONSE_CODE, &req->response->status_code); if (req->response->status_code == 0) { - LOG_ERRORF("web.c", "CURL Error: %s", req->curl_err_buffer) + LOG_ERRORF("web.c", "CURL Error: %s", req->curl_err_buffer); } curl_multi_cleanup(req->multi); @@ -104,7 +104,7 @@ subreq_ctx_t *web_post_async(const char *url, char *data, int insecure) { curl_multi_add_handle(req->multi, curl); curl_multi_perform(req->multi, &req->running_handles); - LOG_DEBUGF("web.c", "async request POST %s", url) + LOG_DEBUGF("web.c", "async request POST %s", url); return req; } @@ -136,7 +136,7 @@ response_t *web_get(const char *url, int timeout, int insecure) { curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code); if (resp->status_code == 0) { - LOG_ERRORF("web.c", "CURL Error: %s", err_buffer) + LOG_ERRORF("web.c", "CURL Error: %s", err_buffer); } curl_easy_cleanup(curl); @@ -180,7 +180,7 @@ response_t *web_post(const char *url, const char *data, int insecure) { resp->size = buffer.cur; if (resp->status_code == 0) { - LOG_ERRORF("web.c", "CURL Error: %s", err_buffer) + LOG_ERRORF("web.c", "CURL Error: %s", err_buffer); } curl_easy_cleanup(curl); diff --git a/src/io/serialize.c b/src/io/serialize.c index c438025..a1432d7 100644 --- a/src/io/serialize.c +++ b/src/io/serialize.c @@ -1,9 +1,7 @@ #include "src/ctx.h" #include "serialize.h" -#include "src/parsing/parse.h" #include "src/parsing/mime.h" -#include char *get_meta_key_text(enum metakey meta_key) { @@ -79,7 +77,7 @@ char *get_meta_key_text(enum metakey meta_key) { case MetaChecksum: return "checksum"; default: - LOG_FATALF("serialize.c", "FIXME: Unknown meta key: %d", meta_key) + LOG_FATALF("serialize.c", "FIXME: Unknown meta key: %d", meta_key); } } @@ -175,7 +173,7 @@ char *build_json_string(document_t *doc) { break; } default: - LOG_FATALF("serialize.c", "Invalid meta key: %x %s", meta->key, get_meta_key_text(meta->key)) + LOG_FATALF("serialize.c", "Invalid meta key: %x %s", meta->key, get_meta_key_text(meta->key)); } meta_line_t *tmp = meta; @@ -189,394 +187,10 @@ char *build_json_string(document_t *doc) { return json_str; } -static struct { - FILE *out_file; - size_t buf_out_size; - - void *buf_out; - - ZSTD_CCtx *cctx; -} WriterCtx = { - .out_file = NULL -}; - -#define ZSTD_COMPRESSION_LEVEL 10 - -void initialize_writer_ctx(const char *file_path) { - WriterCtx.out_file = fopen(file_path, "wb"); - - WriterCtx.buf_out_size = ZSTD_CStreamOutSize(); - WriterCtx.buf_out = malloc(WriterCtx.buf_out_size); - - WriterCtx.cctx = ZSTD_createCCtx(); - - ZSTD_CCtx_setParameter(WriterCtx.cctx, ZSTD_c_compressionLevel, ZSTD_COMPRESSION_LEVEL); - ZSTD_CCtx_setParameter(WriterCtx.cctx, ZSTD_c_checksumFlag, FALSE); - - LOG_DEBUGF("serialize.c", "Open index file for writing %s", file_path) -} - -void zstd_write_string(const char *string, const size_t len) { - ZSTD_inBuffer input = {string, len, 0}; - - do { - ZSTD_outBuffer output = {WriterCtx.buf_out, WriterCtx.buf_out_size, 0}; - ZSTD_compressStream2(WriterCtx.cctx, &output, &input, ZSTD_e_continue); - - if (output.pos > 0) { - ScanCtx.stat_index_size += fwrite(WriterCtx.buf_out, 1, output.pos, WriterCtx.out_file); - } - } while (input.pos != input.size); -} - -void write_document_func(tpool_work_arg_shm_t *arg) { - - const char *json_str = arg->arg; - - if (WriterCtx.out_file == NULL) { - char dstfile[PATH_MAX]; - snprintf(dstfile, PATH_MAX, "%s_index_main.ndjson.zst", ScanCtx.index.path); - initialize_writer_ctx(dstfile); - } - - zstd_write_string(json_str, arg->arg_size); -} - -void zstd_close() { - if (WriterCtx.out_file == NULL) { - LOG_DEBUG("serialize.c", "No zstd stream to close, skipping cleanup") - return; - } - - size_t remaining; - do { - ZSTD_outBuffer output = {WriterCtx.buf_out, WriterCtx.buf_out_size, 0}; - remaining = ZSTD_endStream(WriterCtx.cctx, &output); - - if (output.pos > 0) { - ScanCtx.stat_index_size += fwrite(WriterCtx.buf_out, 1, output.pos, WriterCtx.out_file); - } - } while (remaining != 0); - - ZSTD_freeCCtx(WriterCtx.cctx); - free(WriterCtx.buf_out); - fclose(WriterCtx.out_file); - - LOG_DEBUG("serialize.c", "End zstd stream & close index file") -} - -void writer_cleanup() { - zstd_close(); - WriterCtx.out_file = NULL; -} - -void write_index_descriptor(char *path, index_descriptor_t *desc) { - cJSON *json = cJSON_CreateObject(); - cJSON_AddStringToObject(json, "id", desc->id); - cJSON_AddStringToObject(json, "version", desc->version); - cJSON_AddStringToObject(json, "root", desc->root); - cJSON_AddStringToObject(json, "name", desc->name); - cJSON_AddStringToObject(json, "type", desc->type); - cJSON_AddStringToObject(json, "rewrite_url", desc->rewrite_url); - cJSON_AddNumberToObject(json, "timestamp", (double) desc->timestamp); - - int fd = open(path, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); - if (fd < 0) { - LOG_FATALF("serialize.c", "Could not open index descriptor: %s", strerror(errno)); - } - char *str = cJSON_Print(json); - size_t ret = write(fd, str, strlen(str)); - if (ret == -1) { - LOG_FATALF("serialize.c", "Could not write index descriptor: %s", strerror(errno)); - } - free(str); - close(fd); - - cJSON_Delete(json); -} - -index_descriptor_t read_index_descriptor(char *path) { - - struct stat info; - stat(path, &info); - int fd = open(path, O_RDONLY); - - if (fd == -1) { - LOG_FATALF("serialize.c", "Invalid/corrupt index (Could not find descriptor): %s: %s\n", path, strerror(errno)) - } - - char *buf = malloc(info.st_size + 1); - size_t ret = read(fd, buf, info.st_size); - if (ret == -1) { - LOG_FATALF("serialize.c", "Could not read index descriptor: %s", strerror(errno)); - } - *(buf + info.st_size) = '\0'; - close(fd); - - cJSON *json = cJSON_Parse(buf); - - index_descriptor_t descriptor; - descriptor.timestamp = (long) cJSON_GetObjectItem(json, "timestamp")->valuedouble; - strcpy(descriptor.root, cJSON_GetObjectItem(json, "root")->valuestring); - strcpy(descriptor.name, cJSON_GetObjectItem(json, "name")->valuestring); - strcpy(descriptor.rewrite_url, cJSON_GetObjectItem(json, "rewrite_url")->valuestring); - descriptor.root_len = (short) strlen(descriptor.root); - strcpy(descriptor.version, cJSON_GetObjectItem(json, "version")->valuestring); - strcpy(descriptor.id, cJSON_GetObjectItem(json, "id")->valuestring); - if (cJSON_GetObjectItem(json, "type") == NULL) { - strcpy(descriptor.type, INDEX_TYPE_NDJSON); - } else { - strcpy(descriptor.type, cJSON_GetObjectItem(json, "type")->valuestring); - } - - cJSON_Delete(json); - free(buf); - - return descriptor; -} - - void write_document(document_t *doc) { char *json_str = build_json_string(doc); + + database_write_document(ProcData.index_db, doc, json_str); free(doc); - const size_t json_str_len = strlen(json_str); - - json_str = realloc(json_str, json_str_len + 1); - *(json_str + json_str_len) = '\n'; - - tpool_work_arg_t arg = { - .arg_size = json_str_len + 1, - .arg = json_str - }; - - tpool_add_work(ScanCtx.writer_pool, write_document_func, &arg); -} - -void thread_cleanup() { - cleanup_parse(); - cleanup_font(); -} - -void read_index_bin_handle_line(const char *line, const char *index_id, index_func func) { - - cJSON *document = cJSON_Parse(line); - const char *path_md5_str = cJSON_GetObjectItem(document, "_id")->valuestring; - - cJSON_AddStringToObject(document, "index", index_id); - - // Load meta from sidecar files - cJSON *meta_obj = NULL; - if (IndexCtx.meta != NULL) { - const char *meta_string = g_hash_table_lookup(IndexCtx.meta, path_md5_str); - if (meta_string != NULL) { - meta_obj = cJSON_Parse(meta_string); - - cJSON *child; - for (child = meta_obj->child; child != NULL; child = child->next) { - char meta_key[4096]; - strcpy(meta_key, child->string); - cJSON_DeleteItemFromObject(document, meta_key); - cJSON_AddItemReferenceToObject(document, meta_key, child); - } - } - } - - // Load tags from tags DB - if (IndexCtx.tags != NULL) { - const char *tags_string = g_hash_table_lookup(IndexCtx.tags, path_md5_str); - if (tags_string != NULL) { - cJSON *tags_arr = cJSON_Parse(tags_string); - cJSON_DeleteItemFromObject(document, "tag"); - cJSON_AddItemToObject(document, "tag", tags_arr); - } - } - - func(document, path_md5_str); - cJSON_DeleteItemFromObject(document, "_id"); - cJSON_Delete(document); - if (meta_obj) { - cJSON_Delete(meta_obj); - } -} - -void read_lines(const char *path, const line_processor_t processor) { - dyn_buffer_t buf = dyn_buffer_create(); - - // Initialize zstd things - FILE *file = fopen(path, "rb"); - - size_t const buf_in_size = ZSTD_DStreamInSize(); - void *const buf_in = malloc(buf_in_size); - - size_t const buf_out_size = ZSTD_DStreamOutSize(); - void *const buf_out = malloc(buf_out_size); - - ZSTD_DCtx *const dctx = ZSTD_createDCtx(); - - size_t read; - size_t last_ret = 0; - while ((read = fread(buf_in, 1, buf_in_size, file))) { - ZSTD_inBuffer input = {buf_in, read, 0}; - - while (input.pos < input.size) { - ZSTD_outBuffer output = {buf_out, buf_out_size, 0}; - - size_t const ret = ZSTD_decompressStream(dctx, &output, &input); - - for (int i = 0; i < output.pos; i++) { - char c = ((char *) output.dst)[i]; - - if (c == '\n') { - dyn_buffer_write_char(&buf, '\0'); - processor.func(buf.buf, processor.data); - buf.cur = 0; - } else { - dyn_buffer_write_char(&buf, c); - } - } - - last_ret = ret; - } - } - - if (last_ret != 0) { - /* The last return value from ZSTD_decompressStream did not end on a - * frame, but we reached the end of the file! We assume this is an - * error, and the input was truncated. - */ - LOG_FATALF("serialize.c", "EOF before end of stream: %zu", last_ret) - } - - ZSTD_freeDCtx(dctx); - free(buf_in); - free(buf_out); - - dyn_buffer_destroy(&buf); - fclose(file); -} - -void read_index_ndjson(const char *line, void *_data) { - void **data = _data; - const char *index_id = data[0]; - index_func func = data[1]; - read_index_bin_handle_line(line, index_id, func); -} - -void read_index(const char *path, const char index_id[SIST_INDEX_ID_LEN], const char *type, index_func func) { - if (strcmp(type, INDEX_TYPE_NDJSON) == 0) { - read_lines(path, (line_processor_t) { - .data = (void *[2]) {(void *) index_id, func}, - .func = read_index_ndjson, - }); - } -} - -static __thread GHashTable *IncrementalReadTable = NULL; - -void json_put_incremental(cJSON *document, UNUSED(const char doc_id[SIST_DOC_ID_LEN])) { - const char *path_md5_str = cJSON_GetObjectItem(document, "_id")->valuestring; - const int mtime = cJSON_GetObjectItem(document, "mtime")->valueint; - - incremental_put(IncrementalReadTable, path_md5_str, mtime); -} - -void incremental_read(GHashTable *table, const char *filepath, index_descriptor_t *desc) { - IncrementalReadTable = table; - read_index(filepath, desc->id, desc->type, json_put_incremental); -} - -static __thread GHashTable *IncrementalCopyTable = NULL; -static __thread GHashTable *IncrementalNewTable = NULL; -static __thread store_t *IncrementalCopySourceStore = NULL; -static __thread store_t *IncrementalCopyDestinationStore = NULL; - -void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[SIST_DOC_ID_LEN])) { - - const char *doc_id = cJSON_GetObjectItem(document, "_id")->valuestring; - - if (cJSON_GetObjectItem(document, "parent") != NULL || incremental_get(IncrementalCopyTable, doc_id)) { - // Copy index line - cJSON_DeleteItemFromObject(document, "index"); - char *json_str = cJSON_PrintUnformatted(document); - const size_t json_str_len = strlen(json_str); - - json_str = realloc(json_str, json_str_len + 1); - *(json_str + json_str_len) = '\n'; - - // Copy tn store contents - size_t buf_len; - char *buf = store_read(IncrementalCopySourceStore, (char *) doc_id, SIST_DOC_ID_LEN, &buf_len); - if (buf_len != 0) { - store_write(IncrementalCopyDestinationStore, (char *) doc_id, SIST_DOC_ID_LEN, buf, buf_len); - free(buf); - } - - // Also copy additional thumbnails - if (cJSON_GetObjectItem(document, "thumbnail") != NULL) { - const int thumbnail_count = cJSON_GetObjectItem(document, "thumbnail")->valueint; - - for (int i = 1; i < thumbnail_count; i++) { - char tn_key[SIST_DOC_ID_LEN + sizeof(char) * 4]; - - snprintf(tn_key, sizeof(tn_key), "%s%04d", doc_id, i); - - buf = store_read(IncrementalCopySourceStore, tn_key, sizeof(tn_key), &buf_len); - if (buf_len != 0) { - store_write(IncrementalCopyDestinationStore, tn_key, sizeof(tn_key), buf, buf_len); - free(buf); - } - } - } - - zstd_write_string(json_str, json_str_len + 1); - free(json_str); - } -} - -/** - * Copy items from an index that are in the copy_table. Also copies from - * the store. - */ -void incremental_copy(store_t *store, store_t *dst_store, const char *filepath, - const char *dst_filepath, GHashTable *copy_table) { - - if (WriterCtx.out_file == NULL) { - initialize_writer_ctx(dst_filepath); - } - - IncrementalCopyTable = copy_table; - IncrementalCopySourceStore = store; - IncrementalCopyDestinationStore = dst_store; - - read_index(filepath, "", INDEX_TYPE_NDJSON, incremental_copy_handle_doc); -} - -void incremental_delete_handle_doc(cJSON *document, UNUSED(const char id_str[SIST_DOC_ID_LEN])) { - - char doc_id_n[SIST_DOC_ID_LEN + 1]; - doc_id_n[SIST_DOC_ID_LEN] = '\0'; - doc_id_n[SIST_DOC_ID_LEN - 1] = '\n'; - const char *doc_id = cJSON_GetObjectItem(document, "_id")->valuestring; - - // do not delete archive virtual entries - if (cJSON_GetObjectItem(document, "parent") == NULL - && !incremental_get(IncrementalCopyTable, doc_id) - && !incremental_get(IncrementalNewTable, doc_id) - ) { - memcpy(doc_id_n, doc_id, SIST_DOC_ID_LEN - 1); - zstd_write_string(doc_id, sizeof(doc_id_n)); - } -} - -void incremental_delete(const char *del_filepath, const char *index_filepath, - GHashTable *copy_table, GHashTable *new_table) { - - if (WriterCtx.out_file == NULL) { - initialize_writer_ctx(del_filepath); - } - - IncrementalCopyTable = copy_table; - IncrementalNewTable = new_table; - - read_index(index_filepath, "", INDEX_TYPE_NDJSON, incremental_delete_handle_doc); -} + free(json_str); +} \ No newline at end of file diff --git a/src/io/serialize.h b/src/io/serialize.h index 2da3cb3..83614bf 100644 --- a/src/io/serialize.h +++ b/src/io/serialize.h @@ -2,55 +2,7 @@ #define SIST2_SERIALIZE_H #include "src/sist.h" -#include "store.h" - -#include -#include - -typedef struct line_processor { - void* data; - void (*func)(const char*, void*); -} line_processor_t; - -typedef void(*index_func)(cJSON *, const char[SIST_DOC_ID_LEN]); - -void incremental_copy(store_t *store, store_t *dst_store, const char *filepath, - const char *dst_filepath, GHashTable *copy_table); - -void incremental_delete(const char *del_filepath, const char* index_filepath, - GHashTable *copy_table, GHashTable *new_table); void write_document(document_t *doc); -void read_lines(const char *path, const line_processor_t processor); - -void read_index(const char *path, const char index_id[SIST_INDEX_ID_LEN], const char *type, index_func); - -void incremental_read(GHashTable *table, const char *filepath, index_descriptor_t *desc); - -/** - * Must be called after write_document - */ -void thread_cleanup(); - -void writer_cleanup(); - -void write_index_descriptor(char *path, index_descriptor_t *desc); - -index_descriptor_t read_index_descriptor(char *path); - -// caller ensures char file_path[PATH_MAX] -#define READ_INDICES(file_path, index_path, action_ok, action_main_fail, cond_original) \ - snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", index_path); \ - if (access(file_path, R_OK) == 0) { \ - action_ok; \ - } else { \ - action_main_fail; \ - } \ - snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", index_path); \ - if ((cond_original) && access(file_path, R_OK) == 0) { \ - action_ok; \ - } \ - - #endif diff --git a/src/io/store.c b/src/io/store.c deleted file mode 100644 index dad686b..0000000 --- a/src/io/store.c +++ /dev/null @@ -1,232 +0,0 @@ -#include -#include "store.h" -#include "src/ctx.h" - -//#define SIST_FAKE_STORE 1 - -void open_env(const char *path, MDB_env **env, MDB_dbi *dbi) { - mdb_env_create(env); - - int open_ret = mdb_env_open(*env, - path, - MDB_WRITEMAP | MDB_MAPASYNC, - S_IRUSR | S_IWUSR - ); - - if (open_ret != 0) { - LOG_FATALF("store.c", "Error while opening store: %s (%s)\n", mdb_strerror(open_ret), path) - } - - MDB_txn *txn; - mdb_txn_begin(*env, NULL, 0, &txn); - mdb_dbi_open(txn, NULL, 0, dbi); - mdb_txn_commit(txn); -} - -store_t *store_create(const char *path, size_t chunk_size) { - store_t *store = calloc(1, sizeof(struct store_t)); - mkdir(path, S_IWUSR | S_IRUSR | S_IXUSR); - strcpy(store->path, path); - - MDB_env *env; - MDB_dbi dbi; - -#if (SIST_FAKE_STORE != 1) - store->chunk_size = chunk_size; - - store->shm = mmap(NULL, sizeof(*store->shm), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - - open_env(path, &env, &dbi); - - store->shm->size = (size_t) store->chunk_size; - mdb_env_set_mapsize(env, store->shm->size); - - // Close, child processes will open the environment again - mdb_env_close(env); -#endif - - return store; -} - -void store_destroy(store_t *store) { - - LOG_DEBUG("store.c", "store_destroy()") -#if (SIST_FAKE_STORE != 1) - munmap(store->shm, sizeof(*store->shm)); - - mdb_dbi_close(store->proc.env, store->proc.dbi); - mdb_env_close(store->proc.env); -#endif - free(store); -} - -void store_flush(store_t *store) { - mdb_env_sync(store->proc.env, TRUE); -} - -void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len) { - - ScanCtx.stat_tn_size += buf_len; - - if (LogCtx.very_verbose) { - LOG_DEBUGF("store.c", "Store write %s@{%s} %lu bytes", store->path, key, buf_len) - } - -#if (SIST_FAKE_STORE != 1) - - if (store->proc.env == NULL) { - open_env(store->path, &store->proc.env, &store->proc.dbi); - LOG_DEBUGF("store.c", "Opening mdb environment %s", store->path) - } - - MDB_val mdb_key; - mdb_key.mv_data = key; - mdb_key.mv_size = key_len; - - MDB_val mdb_value; - mdb_value.mv_data = buf; - mdb_value.mv_size = buf_len; - - MDB_txn *txn; - - int db_full = FALSE; - int put_ret = 0; - int should_abort_transaction = FALSE; - int should_increase_size = TRUE; - - int begin_ret = mdb_txn_begin(store->proc.env, NULL, 0, &txn); - - if (begin_ret == MDB_MAP_RESIZED) { - // mapsize was increased by another process. We don't need to increase the size again, but we need - // to update the size of the environment for the current process. - db_full = TRUE; - should_increase_size = FALSE; - } else { - put_ret = mdb_put(txn, store->proc.dbi, &mdb_key, &mdb_value, 0); - - if (put_ret == MDB_MAP_FULL) { - // Database is full, we need to increase the environment size - db_full = TRUE; - should_abort_transaction = TRUE; - } else { - int commit_ret = mdb_txn_commit(txn); - - if (commit_ret == MDB_MAP_FULL) { - db_full = TRUE; - } - } - } - - if (db_full) { - LOG_DEBUGF("store.c", "Updating mdb mapsize to %lu bytes", store->shm->size) - - if (should_abort_transaction) { - mdb_txn_abort(txn); - } - - // Cannot resize when there is an opened transaction in this process. - // Resize take effect on the next commit. - if (should_increase_size) { - store->shm->size += store->chunk_size; - } - int resize_ret = mdb_env_set_mapsize(store->proc.env, store->shm->size); - if (resize_ret != 0) { - LOG_ERRORF("store.c", "mdb_env_set_mapsize() failed: %s", mdb_strerror(resize_ret)) - } - mdb_txn_begin(store->proc.env, NULL, 0, &txn); - int put_ret_retry = mdb_put(txn, store->proc.dbi, &mdb_key, &mdb_value, 0); - - if (put_ret_retry != 0) { - LOG_ERRORF("store.c", "mdb_put() (retry) failed: %s", mdb_strerror(put_ret_retry)) - } - - int ret = mdb_txn_commit(txn); - if (ret != 0) { - LOG_FATALF("store.c", "FIXME: Could not commit to store %s: %s (%d), %d, %d %d", - store->path, mdb_strerror(ret), ret, - ret, put_ret_retry) - } - LOG_DEBUGF("store.c", "Updated mdb mapsize to %lu bytes", store->shm->size) - } else if (put_ret != 0) { - LOG_ERRORF("store.c", "mdb_put() failed: %s", mdb_strerror(put_ret)) - } - -#endif -} - -char *store_read(store_t *store, char *key, size_t key_len, size_t *return_value_len) { - char *buf = NULL; - -#if (SIST_FAKE_STORE != 1) - if (store->proc.env == NULL) { - open_env(store->path, &store->proc.env, &store->proc.dbi); - } - - MDB_val mdb_key; - mdb_key.mv_data = key; - mdb_key.mv_size = key_len; - - MDB_val mdb_value; - - MDB_txn *txn; - mdb_txn_begin(store->proc.env, NULL, MDB_RDONLY, &txn); - - int get_ret = mdb_get(txn, store->proc.dbi, &mdb_key, &mdb_value); - - if (get_ret == MDB_NOTFOUND) { - *return_value_len = 0; - } else { - *return_value_len = mdb_value.mv_size; - buf = malloc(mdb_value.mv_size); - memcpy(buf, mdb_value.mv_data, mdb_value.mv_size); - } - - mdb_txn_abort(txn); -#endif - return buf; -} - -GHashTable *store_read_all(store_t *store) { - - if (store->proc.env == NULL) { - open_env(store->path, &store->proc.env, &store->proc.dbi); - LOG_DEBUGF("store.c", "Opening mdb environment %s", store->path) - } - - int count = 0; - - GHashTable *table = g_hash_table_new_full(g_str_hash, g_str_equal, free, free); - - MDB_txn *txn = NULL; - mdb_txn_begin(store->proc.env, NULL, MDB_RDONLY, &txn); - - MDB_cursor *cur = NULL; - mdb_cursor_open(txn, store->proc.dbi, &cur); - - MDB_val key; - MDB_val value; - - while (mdb_cursor_get(cur, &key, &value, MDB_NEXT) == 0) { - char *key_str = malloc(key.mv_size); - memcpy(key_str, key.mv_data, key.mv_size); - char *val_str = malloc(value.mv_size); - memcpy(val_str, value.mv_data, value.mv_size); - - g_hash_table_insert(table, key_str, val_str); - count += 1; - } - - const char *path; - mdb_env_get_path(store->proc.env, &path); - LOG_DEBUGF("store.c", "Read %d entries from %s", count, path) - - mdb_cursor_close(cur); - mdb_txn_abort(txn); - return table; -} - - -void store_copy(store_t *store, const char *destination) { - mkdir(destination, S_IWUSR | S_IRUSR | S_IXUSR); - mdb_env_copy(store->proc.env, destination); -} diff --git a/src/io/store.h b/src/io/store.h deleted file mode 100644 index ce27ded..0000000 --- a/src/io/store.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef SIST2_STORE_H -#define SIST2_STORE_H - -#include -#include - -#include - -#define STORE_SIZE_TN (1024 * 1024 * 5) -#define STORE_SIZE_TAG (1024 * 1024) -#define STORE_SIZE_META STORE_SIZE_TAG - - -typedef struct store_t { - char path[PATH_MAX]; - size_t chunk_size; - - struct { - MDB_dbi dbi; - MDB_env *env; - } proc; - - struct { - size_t size; - } *shm; -} store_t; - -store_t *store_create(const char *path, size_t chunk_size); - -void store_destroy(store_t *store); - -void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len); - -void store_flush(store_t *store); - -char *store_read(store_t *store, char *key, size_t key_len, size_t *return_value_len); - -GHashTable *store_read_all(store_t *store); - -void store_copy(store_t *store, const char *destination); - -#endif diff --git a/src/io/walk.c b/src/io/walk.c index b6019cf..c9fa8b0 100644 --- a/src/io/walk.c +++ b/src/io/walk.c @@ -1,46 +1,12 @@ #include "walk.h" #include "src/ctx.h" -#include "src/parsing/parse.h" +#include "src/parsing/fs_util.h" #include +#include #define STR_STARTS_WITH(x, y) (strncmp(y, x, strlen(y) - 1) == 0) -__always_inline -parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info, int base) { - int len = (int) strlen(filepath); - parse_job_t *job = malloc(sizeof(parse_job_t)); - - strcpy(job->filepath, filepath); - job->base = base; - char *p = strrchr(filepath + base, '.'); - if (p != NULL) { - job->ext = (int) (p - filepath + 1); - } else { - job->ext = len; - } - - job->vfile.st_size = info->st_size; - job->vfile.st_mode = info->st_mode; - job->vfile.mtime = (int) info->st_mtim.tv_sec; - - job->parent[0] = '\0'; - - memcpy(job->vfile.filepath, job->filepath, sizeof(job->vfile.filepath)); - job->vfile.read = fs_read; - // Filesystem reads are always rewindable - job->vfile.read_rewindable = fs_read; - job->vfile.reset = fs_reset; - job->vfile.close = fs_close; - job->vfile.fd = -1; - job->vfile.is_fs_file = TRUE; - job->vfile.has_checksum = FALSE; - job->vfile.rewind_buffer_size = 0; - job->vfile.rewind_buffer = NULL; - job->vfile.calculate_checksum = ScanCtx.calculate_checksums; - - return job; -} int sub_strings[30]; #define EXCLUDED(str) (pcre_exec(ScanCtx.exclude, ScanCtx.exclude_extra, str, strlen(str), 0, 0, sub_strings, sizeof(sub_strings)) >= 0) @@ -55,7 +21,7 @@ int handle_entry(const char *filepath, const struct stat *info, int typeflag, st } if (ScanCtx.exclude != NULL && EXCLUDED(filepath)) { - LOG_DEBUGF("walk.c", "Excluded: %s", filepath) + LOG_DEBUGF("walk.c", "Excluded: %s", filepath); if (typeflag == FTW_F && S_ISREG(info->st_mode)) { pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu); @@ -69,13 +35,13 @@ int handle_entry(const char *filepath, const struct stat *info, int typeflag, st } if (typeflag == FTW_F && S_ISREG(info->st_mode)) { - parse_job_t *job = create_fs_parse_job(filepath, info, ftw->base); + parse_job_t *job = create_parse_job(filepath, (int) info->st_mtim.tv_sec, info->st_size); - tpool_work_arg_t arg = { - .arg_size = sizeof(parse_job_t), - .arg = job - }; - tpool_add_work(ScanCtx.pool, parse, &arg); + tpool_add_work(ScanCtx.pool, &(job_t) { + .type = JOB_PARSE_JOB, + .parse_job = job + }); + free(job); } return FTW_CONTINUE; @@ -116,7 +82,7 @@ int iterate_file_list(void *input_file) { } if (ScanCtx.exclude != NULL && EXCLUDED(absolute_path)) { - LOG_DEBUGF("walk.c", "Excluded: %s", absolute_path) + LOG_DEBUGF("walk.c", "Excluded: %s", absolute_path); if (S_ISREG(info.st_mode)) { pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu); @@ -131,16 +97,14 @@ int iterate_file_list(void *input_file) { LOG_FATALF("walk.c", "File is not a children of root folder (%s): %s", ScanCtx.index.desc.root, buf); } - int base = (int) (strrchr(buf, '/') - buf) + 1; - - parse_job_t *job = create_fs_parse_job(absolute_path, &info, base); + parse_job_t *job = create_parse_job(absolute_path, (int) info.st_mtim.tv_sec, info.st_size); free(absolute_path); - tpool_work_arg_t arg = { - .arg = job, - .arg_size = sizeof(parse_job_t) - }; - tpool_add_work(ScanCtx.pool, parse, &arg); + tpool_add_work(ScanCtx.pool, &(job_t) { + .type = JOB_PARSE_JOB, + .parse_job = job + }); + free(job); } return 0; diff --git a/src/log.c b/src/log.c index b5e1ece..4ca6869 100644 --- a/src/log.c +++ b/src/log.c @@ -21,8 +21,6 @@ void vsist_logf(const char *filepath, int level, char *format, va_list ap) { char log_str[LOG_MAX_LENGTH]; - unsigned long long pid = (unsigned long long) pthread_self(); - char datetime[32]; time_t t; struct tm result; @@ -42,8 +40,8 @@ void vsist_logf(const char *filepath, int level, char *format, va_list ap) { log_len = snprintf( log_str, sizeof(log_str), - "{\"thread\":\"%04llX\",\"datetime\":\"%s\",\"level\":\"%s\",\"filepath\":%s,\"message\":%s}\n", - pid, datetime, log_levels[level], filepath_json_str, log_str_json_str + "{\"thread\":\"T%d\",\"datetime\":\"%s\",\"level\":\"%s\",\"filepath\":%s,\"message\":%s}\n", + ProcData.thread_id, datetime, log_levels[level], filepath_json_str, log_str_json_str ); cJSON_Delete(filepath_json); @@ -58,15 +56,15 @@ void vsist_logf(const char *filepath, int level, char *format, va_list ap) { if (is_tty) { log_len = snprintf( log_str, sizeof(log_str), - "\033[%dm[%04llX]%s [%s] [%s %s] ", - 31 + ((unsigned int) (pid)) % 7, pid, log_colors[level], + "\033[%dmT%d%s [%s] [%s %s] ", + 31 + ProcData.thread_id % 7, ProcData.thread_id, log_colors[level], datetime, log_levels[level], filepath ); } else { log_len = snprintf( log_str, sizeof(log_str), - "[%04llX] [%s] [%s %s] ", - pid, datetime, log_levels[level], filepath + "T%d [%s] [%s %s] ", + ProcData.thread_id, datetime, log_levels[level], filepath ); } @@ -112,8 +110,6 @@ void sist_log(const char *filepath, int level, char *str) { char log_str[LOG_MAX_LENGTH]; - unsigned long long pid = (unsigned long long) pthread_self(); - char datetime[32]; time_t t; struct tm result; @@ -132,8 +128,8 @@ void sist_log(const char *filepath, int level, char *str) { log_len = snprintf( log_str, sizeof(log_str), - "{\"thread\":\"%04llX\",\"datetime\":\"%s\",\"level\":\"%s\",\"filepath\":%s,\"message\":%s}\n", - pid, datetime, log_levels[level], filepath_json_str, log_str_json_str + "{\"thread\":\"T%d\",\"datetime\":\"%s\",\"level\":\"%s\",\"filepath\":%s,\"message\":%s}\n", + ProcData.thread_id, datetime, log_levels[level], filepath_json_str, log_str_json_str ); cJSON_Delete(log_str_json); @@ -147,16 +143,16 @@ void sist_log(const char *filepath, int level, char *str) { if (is_tty) { log_len = snprintf( log_str, sizeof(log_str), - "\033[%dm[%04llX]%s [%s] [%s %s] %s \033[0m\n", - 31 + ((unsigned int) (pid)) % 7, pid, log_colors[level], + "\033[%dmT%d%s [%s] [%s %s] %s \033[0m\n", + 31 + ProcData.thread_id % 7, ProcData.thread_id, log_colors[level], datetime, log_levels[level], filepath, str ); } else { log_len = snprintf( log_str, sizeof(log_str), - "[%04llX] [%s] [%s %s] %s \n", - pid, datetime, log_levels[level], filepath, + "T%d [%s] [%s %s] %s \n", + ProcData.thread_id, datetime, log_levels[level], filepath, str ); } diff --git a/src/log.h b/src/log.h index 113a577..2426be3 100644 --- a/src/log.h +++ b/src/log.h @@ -2,6 +2,7 @@ #define SIST2_LOG_H +#include #define LOG_MAX_LENGTH 8192 #define LOG_SIST_DEBUG 0 @@ -10,37 +11,37 @@ #define LOG_SIST_ERROR 3 #define LOG_SIST_FATAL 4 -#define LOG_DEBUGF(filepath, fmt, ...) \ - if (LogCtx.very_verbose) {sist_logf(filepath, LOG_SIST_DEBUG, fmt, __VA_ARGS__);} -#define LOG_DEBUG(filepath, str) \ - if (LogCtx.very_verbose) {sist_log(filepath, LOG_SIST_DEBUG, str);} +#define LOG_DEBUGF(filepath, fmt, ...) do{\ + if (LogCtx.very_verbose) {sist_logf(filepath, LOG_SIST_DEBUG, fmt, __VA_ARGS__);}}while(0) +#define LOG_DEBUG(filepath, str) do{\ + if (LogCtx.very_verbose) {sist_log(filepath, LOG_SIST_DEBUG, str);}}while(0) -#define LOG_INFOF(filepath, fmt, ...) \ - if (LogCtx.verbose) {sist_logf(filepath, LOG_SIST_INFO, fmt, __VA_ARGS__);} -#define LOG_INFO(filepath, str) \ - if (LogCtx.verbose) {sist_log(filepath, LOG_SIST_INFO, str);} +#define LOG_INFOF(filepath, fmt, ...) do {\ + if (LogCtx.verbose) {sist_logf(filepath, LOG_SIST_INFO, fmt, __VA_ARGS__);}} while(0) +#define LOG_INFO(filepath, str) do {\ + if (LogCtx.verbose) {sist_log(filepath, LOG_SIST_INFO, str);}} while(0) -#define LOG_WARNINGF(filepath, fmt, ...) \ - if (LogCtx.verbose) {sist_logf(filepath, LOG_SIST_WARNING, fmt, __VA_ARGS__);} -#define LOG_WARNING(filepath, str) \ - if (LogCtx.verbose) {sist_log(filepath, LOG_SIST_WARNING, str);} +#define LOG_WARNINGF(filepath, fmt, ...) do {\ + if (LogCtx.verbose) {sist_logf(filepath, LOG_SIST_WARNING, fmt, __VA_ARGS__);}}while(0) +#define LOG_WARNING(filepath, str) do{\ + if (LogCtx.verbose) {sist_log(filepath, LOG_SIST_WARNING, str);}}while(0) -#define LOG_ERRORF(filepath, fmt, ...) \ - if (LogCtx.verbose) {sist_logf(filepath, LOG_SIST_ERROR, fmt, __VA_ARGS__);} -#define LOG_ERROR(filepath, str) \ - if (LogCtx.verbose) {sist_log(filepath, LOG_SIST_ERROR, str);} +#define LOG_ERRORF(filepath, fmt, ...) do {\ + if (LogCtx.verbose) {sist_logf(filepath, LOG_SIST_ERROR, fmt, __VA_ARGS__);}}while(0) +#define LOG_ERROR(filepath, str) do{\ + if (LogCtx.verbose) {sist_log(filepath, LOG_SIST_ERROR, str);}}while(0) -#define LOG_FATALF(filepath, fmt, ...) \ +#define LOG_FATALF(filepath, fmt, ...)\ sist_logf(filepath, LOG_SIST_FATAL, fmt, __VA_ARGS__);\ - exit(-1); + raise(SIGUSR1) #define LOG_FATAL(filepath, str) \ sist_log(filepath, LOG_SIST_FATAL, str);\ - exit(-1); + exit(SIGUSR1) #define LOG_FATALF_NO_EXIT(filepath, fmt, ...) \ - sist_logf(filepath, LOG_SIST_FATAL, fmt, __VA_ARGS__); + sist_logf(filepath, LOG_SIST_FATAL, fmt, __VA_ARGS__) #define LOG_FATAL_NO_EXIT(filepath, str) \ - sist_log(filepath, LOG_SIST_FATAL, str); + sist_log(filepath, LOG_SIST_FATAL, str) #include "sist.h" diff --git a/src/main.c b/src/main.c index 38a5dd5..72bdc89 100644 --- a/src/main.c +++ b/src/main.c @@ -5,8 +5,6 @@ #include #include "cli.h" -#include "io/serialize.h" -#include "io/store.h" #include "tpool.h" #include "io/walk.h" #include "index/elastic.h" @@ -16,10 +14,9 @@ #include "auth0/auth0_c_api.h" #include -#include -#include +#include -#include "stats.h" +#include "src/database/database.h" #define DESCRIPTION "Lightning-fast file system indexer and search tool." @@ -46,30 +43,31 @@ void sig_handler(int signum) { LOG_ERROR("*SIGNAL HANDLER*", "=============================================\n\n"); LOG_ERRORF("*SIGNAL HANDLER*", "Uh oh! Caught fatal signal: %s", strsignal(signum)); - if (ScanCtx.dbg_current_files != NULL) { - GHashTableIter iter; - g_hash_table_iter_init(&iter, ScanCtx.dbg_current_files); - - void *key; - void *value; - while (g_hash_table_iter_next(&iter, &key, &value)) { - parse_job_t *job = value; - - if (isatty(STDERR_FILENO)) { - LOG_DEBUGF( - "*SIGNAL HANDLER*", - "Thread \033[%dm[%04llX]\033[0m was working on job '%s'", - 31 + ((unsigned int) key) % 7, key, job->filepath - ); - } else { - LOG_DEBUGF( - "*SIGNAL HANDLER*", - "THREAD [%04llX] was working on job %s", - key, job->filepath - ); - } - } - } + // TODO: Print debug info +// if (ScanCtx.dbg_current_files != NULL) { +// GHashTableIter iter; +// g_hash_table_iter_init(&iter, ScanCtx.dbg_current_files); +// +// void *key; +// void *value; +// while (g_hash_table_iter_next(&iter, &key, &value)) { +// parse_job_t *job = value; +// +// if (isatty(STDERR_FILENO)) { +// LOG_DEBUGF( +// "*SIGNAL HANDLER*", +// "Thread \033[%dm[%04llX]\033[0m was working on job '%s'", +// 31 + ((unsigned int) key) % 7, key, job->filepath +// ); +// } else { +// LOG_DEBUGF( +// "*SIGNAL HANDLER*", +// "THREAD [%04llX] was working on job %s", +// key, job->filepath +// ); +// } +// } +// } if (ScanCtx.pool != NULL) { tpool_dump_debug_info(ScanCtx.pool); @@ -82,18 +80,18 @@ void sig_handler(int signum) { LOG_INFO( "*SIGNAL HANDLER*", "Please consider creating a bug report at https://github.com/simon987/sist2/issues !" - ) + ); LOG_INFO( "*SIGNAL HANDLER*", "sist2 is an open source project and relies on the collaboration of its users to diagnose and fix bugs" - ) + ); #ifndef SIST_DEBUG LOG_WARNING( "*SIGNAL HANDLER*", "You are running sist2 in release mode! Please consider downloading the debug binary from the Github " "releases page to provide additionnal information when submitting a bug report." - ) + ); #endif if (signum == SIGSEGV && sigsegv_handler != NULL) { @@ -105,36 +103,59 @@ void sig_handler(int signum) { exit(-1); } -void init_dir(const char *dirpath, scan_args_t *args) { - char path[PATH_MAX]; - snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath); +void database_scan_begin(scan_args_t *args) { + index_descriptor_t *desc = &ScanCtx.index.desc; - time(&ScanCtx.index.desc.timestamp); - strcpy(ScanCtx.index.desc.version, Version); - strcpy(ScanCtx.index.desc.type, INDEX_TYPE_NDJSON); + database_t *db = database_create(args->output, INDEX_DATABASE); + + if (args->incremental) { + // Update existing descriptor + database_open(db); + index_descriptor_t *original_desc = database_read_index_descriptor(db); + + // copy original index id + strcpy(desc->id, original_desc->id); + + if (original_desc->version_major != VersionMajor) { + LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc->version, Version); + } + + strcpy(original_desc->root, desc->root); + original_desc->root_len = desc->root_len; + strcpy(original_desc->rewrite_url, desc->rewrite_url); + strcpy(original_desc->name, desc->name); + + time(&original_desc->timestamp); + + database_write_index_descriptor(db, original_desc); + free(original_desc); + + database_incremental_scan_begin(db); - if (args->incremental != NULL) { - // copy old index id - char descriptor_path[PATH_MAX]; - snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental); - index_descriptor_t original_desc = read_index_descriptor(descriptor_path); - memcpy(ScanCtx.index.desc.id, original_desc.id, sizeof(original_desc.id)); } else { + // Create new descriptor + + time(&desc->timestamp); + strcpy(desc->version, Version); + desc->version_major = VersionMajor; + desc->version_minor = VersionMinor; + desc->version_patch = VersionPatch; + // generate new index id based on timestamp unsigned char index_md5[MD5_DIGEST_LENGTH]; MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5); buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id); + + database_initialize(db); + database_open(db); + database_write_index_descriptor(db, desc); } - write_index_descriptor(path, &ScanCtx.index.desc); + database_close(db, FALSE); } -void scan_print_header() { - LOG_INFOF("main.c", "sist2 v%s", Version) -} - -void _store(char *key, size_t key_len, char *buf, size_t buf_len) { - store_write(ScanCtx.index.store, key, key_len, buf, buf_len); +void write_thumbnail_callback(char *key, int num, void *buf, size_t buf_len) { + database_write_thumbnail(ProcData.index_db, key, num, buf, buf_len); } void _log(const char *filepath, int level, char *str) { @@ -177,11 +198,8 @@ void _logf(const char *filepath, int level, char *format, ...) { } void initialize_scan_context(scan_args_t *args) { - - ScanCtx.dbg_current_files = g_hash_table_new_full(g_int64_hash, g_int64_equal, NULL, NULL); - pthread_mutex_init(&ScanCtx.dbg_current_files_mu, NULL); + // TODO: shared pthread_mutex_init(&ScanCtx.dbg_file_counts_mu, NULL); - pthread_mutex_init(&ScanCtx.copy_table_mu, NULL); ScanCtx.calculate_checksums = args->calculate_checksums; @@ -189,7 +207,7 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.arc_ctx.mode = args->archive_mode; ScanCtx.arc_ctx.log = _log; ScanCtx.arc_ctx.logf = _logf; - ScanCtx.arc_ctx.parse = (parse_callback_t) parse_job; + ScanCtx.arc_ctx.parse = (parse_callback_t) parse; if (args->archive_passphrase != NULL) { strcpy(ScanCtx.arc_ctx.passphrase, args->archive_passphrase); } else { @@ -199,12 +217,12 @@ void initialize_scan_context(scan_args_t *args) { // Comic ScanCtx.comic_ctx.log = _log; ScanCtx.comic_ctx.logf = _logf; - ScanCtx.comic_ctx.store = _store; + ScanCtx.comic_ctx.store = write_thumbnail_callback; ScanCtx.comic_ctx.enable_tn = args->tn_count > 0; ScanCtx.comic_ctx.tn_size = args->tn_size; ScanCtx.comic_ctx.tn_qscale = args->tn_quality; - ScanCtx.comic_ctx.cbr_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbr"); - ScanCtx.comic_ctx.cbz_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbz"); + ScanCtx.comic_ctx.cbr_mime = mime_get_mime_by_string("application/x-cbr"); + ScanCtx.comic_ctx.cbz_mime = mime_get_mime_by_string("application/x-cbz"); // Ebook ScanCtx.ebook_ctx.content_size = args->content_size; @@ -216,7 +234,7 @@ void initialize_scan_context(scan_args_t *args) { } ScanCtx.ebook_ctx.log = _log; ScanCtx.ebook_ctx.logf = _logf; - ScanCtx.ebook_ctx.store = _store; + ScanCtx.ebook_ctx.store = write_thumbnail_callback; ScanCtx.ebook_ctx.fast_epub_parse = args->fast_epub; ScanCtx.ebook_ctx.tn_qscale = args->tn_quality; @@ -224,7 +242,7 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.font_ctx.enable_tn = args->tn_count > 0; ScanCtx.font_ctx.log = _log; ScanCtx.font_ctx.logf = _logf; - ScanCtx.font_ctx.store = _store; + ScanCtx.font_ctx.store = write_thumbnail_callback; // Media ScanCtx.media_ctx.tn_qscale = args->tn_quality; @@ -232,7 +250,7 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.media_ctx.tn_count = args->tn_count; ScanCtx.media_ctx.log = _log; ScanCtx.media_ctx.logf = _logf; - ScanCtx.media_ctx.store = _store; + ScanCtx.media_ctx.store = write_thumbnail_callback; ScanCtx.media_ctx.max_media_buffer = (long) args->max_memory_buffer_mib * 1024 * 1024; ScanCtx.media_ctx.read_subtitles = args->read_subtitles; ScanCtx.media_ctx.read_subtitles = args->tn_count; @@ -248,7 +266,7 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.ooxml_ctx.content_size = args->content_size; ScanCtx.ooxml_ctx.log = _log; ScanCtx.ooxml_ctx.logf = _logf; - ScanCtx.ooxml_ctx.store = _store; + ScanCtx.ooxml_ctx.store = write_thumbnail_callback; // MOBI ScanCtx.mobi_ctx.content_size = args->content_size; @@ -264,8 +282,8 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.msdoc_ctx.content_size = args->content_size; ScanCtx.msdoc_ctx.log = _log; ScanCtx.msdoc_ctx.logf = _logf; - ScanCtx.msdoc_ctx.store = _store; - ScanCtx.msdoc_ctx.msdoc_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/msword"); + ScanCtx.msdoc_ctx.store = write_thumbnail_callback; + ScanCtx.msdoc_ctx.msdoc_mime = mime_get_mime_by_string("application/msword"); ScanCtx.threads = args->threads; ScanCtx.depth = args->depth; @@ -283,174 +301,67 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.raw_ctx.tn_size = args->tn_size; ScanCtx.raw_ctx.log = _log; ScanCtx.raw_ctx.logf = _logf; - ScanCtx.raw_ctx.store = _store; + ScanCtx.raw_ctx.store = write_thumbnail_callback; // Wpd ScanCtx.wpd_ctx.content_size = args->content_size; ScanCtx.wpd_ctx.log = _log; ScanCtx.wpd_ctx.logf = _logf; - ScanCtx.wpd_ctx.wpd_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/wordperfect"); + ScanCtx.wpd_ctx.wpd_mime = mime_get_mime_by_string("application/wordperfect"); // Json ScanCtx.json_ctx.content_size = args->content_size; ScanCtx.json_ctx.log = _log; ScanCtx.json_ctx.logf = _logf; - ScanCtx.json_ctx.json_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/json"); - ScanCtx.json_ctx.ndjson_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/ndjson"); + ScanCtx.json_ctx.json_mime = mime_get_mime_by_string("application/json"); + ScanCtx.json_ctx.ndjson_mime = mime_get_mime_by_string("application/ndjson"); } -/** - * Loads an existing index as the baseline for incremental scanning. - * 1. load old index files (original+main) => original_table - * 2. allocate empty table => copy_table - * 3. allocate empty table => new_table - * the original_table/copy_table/new_table will be populated in parsing/parse.c:parse - * and consumed in main.c:save_incremental_index - * - * Note: the existing index may or may not be of incremental index form. - */ -void load_incremental_index(const scan_args_t *args) { - char file_path[PATH_MAX]; - - ScanCtx.original_table = incremental_get_table(); - ScanCtx.copy_table = incremental_get_table(); - ScanCtx.new_table = incremental_get_table(); - - char descriptor_path[PATH_MAX]; - snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental); - index_descriptor_t original_desc = read_index_descriptor(descriptor_path); - - if (strcmp(original_desc.version, Version) != 0) { - LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc.version, Version) - } - - READ_INDICES( - file_path, - args->incremental, - incremental_read(ScanCtx.original_table, file_path, &original_desc), - LOG_DEBUG("main.c", "The base index for incremental scan does not have a main index"), - TRUE - ); - - LOG_INFOF("main.c", "Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table)) -} - -/** - * Saves an incremental index. - * Before calling this function, the scanner should have finished writing the main index. - * 1. Build original_table - new_table => delete_table - * 2. Incrementally copy from old index files [(original+main) /\ copy_table] => index_original.ndjson.zst & store - */ -void save_incremental_index(scan_args_t *args) { - char dst_path[PATH_MAX]; - char store_path[PATH_MAX]; - char file_path[PATH_MAX]; - char del_path[PATH_MAX]; - snprintf(store_path, PATH_MAX, "%sthumbs", args->incremental); - snprintf(dst_path, PATH_MAX, "%s_index_original.ndjson.zst", ScanCtx.index.path); - store_t *source = store_create(store_path, STORE_SIZE_TN); - - LOG_INFOF("main.c", "incremental_delete: original size = %u, copy size = %u, new size = %u", - g_hash_table_size(ScanCtx.original_table), - g_hash_table_size(ScanCtx.copy_table), - g_hash_table_size(ScanCtx.new_table)); - snprintf(del_path, PATH_MAX, "%s_index_delete.list.zst", ScanCtx.index.path); - READ_INDICES(file_path, args->incremental, - incremental_delete(del_path, file_path, ScanCtx.copy_table, ScanCtx.new_table), - perror("incremental_delete"), 1); - writer_cleanup(); - - READ_INDICES(file_path, args->incremental, - incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table), - perror("incremental_copy"), 1); - writer_cleanup(); - - store_destroy(source); - - snprintf(store_path, PATH_MAX, "%stags", args->incremental); - snprintf(dst_path, PATH_MAX, "%stags", ScanCtx.index.path); - store_t *source_tags = store_create(store_path, STORE_SIZE_TAG); - store_copy(source_tags, dst_path); - store_destroy(source_tags); -} - -/** - * An index can be either incremental or non-incremental (initial index). - * For an initial index, there is only the "main" index. - * For an incremental index, there are, additionally: - * - An "original" index, referencing all files unchanged since the previous index. - * - A "delete" index, referencing all files that exist in the previous index, but deleted since then. - * Therefore, for an incremental index, "main"+"original" covers all the current files in the live filesystem, - * and is orthognal with the "delete" index. When building an incremental index upon an old incremental index, - * the old "delete" index can be safely ignored. - */ void sist2_scan(scan_args_t *args) { - - ScanCtx.mime_table = mime_get_mime_table(); - ScanCtx.ext_table = mime_get_ext_table(); - initialize_scan_context(args); - init_dir(ScanCtx.index.path, args); + database_scan_begin(args); - char store_path[PATH_MAX]; - snprintf(store_path, PATH_MAX, "%sthumbs", ScanCtx.index.path); - ScanCtx.index.store = store_create(store_path, STORE_SIZE_TN); + LOG_INFOF("main.c", "sist2 v%s", Version); - snprintf(store_path, PATH_MAX, "%smeta", ScanCtx.index.path); - ScanCtx.index.meta_store = store_create(store_path, STORE_SIZE_META); - - scan_print_header(); - - if (args->incremental != NULL) { - load_incremental_index(args); - } - - ScanCtx.writer_pool = tpool_create(1, writer_cleanup, FALSE); - tpool_start(ScanCtx.writer_pool); - - ScanCtx.pool = tpool_create(ScanCtx.threads, thread_cleanup, TRUE); + ScanCtx.pool = tpool_create(ScanCtx.threads, TRUE); tpool_start(ScanCtx.pool); if (args->list_path) { // Scan using file list int list_ret = iterate_file_list(args->list_file); if (list_ret != 0) { - LOG_FATALF("main.c", "iterate_file_list() failed! (%d)", list_ret) + LOG_FATALF("main.c", "iterate_file_list() failed! (%d)", list_ret); } } else { // Scan directory recursively int walk_ret = walk_directory_tree(ScanCtx.index.desc.root); if (walk_ret == -1) { - LOG_FATALF("main.c", "walk_directory_tree() failed! %s (%d)", strerror(errno), errno) + LOG_FATALF("main.c", "walk_directory_tree() failed! %s (%d)", strerror(errno), errno); } } tpool_wait(ScanCtx.pool); tpool_destroy(ScanCtx.pool); - tpool_wait(ScanCtx.writer_pool); - tpool_destroy(ScanCtx.writer_pool); + LOG_DEBUGF("main.c", "Skipped files: %d", ScanCtx.dbg_skipped_files_count); + LOG_DEBUGF("main.c", "Excluded files: %d", ScanCtx.dbg_excluded_files_count); + LOG_DEBUGF("main.c", "Failed files: %d", ScanCtx.dbg_failed_files_count); + LOG_DEBUGF("main.c", "Thumbnail store size: %lu", ScanCtx.stat_tn_size); + LOG_DEBUGF("main.c", "Index size: %lu", ScanCtx.stat_index_size); - LOG_DEBUGF("main.c", "Skipped files: %d", ScanCtx.dbg_skipped_files_count) - LOG_DEBUGF("main.c", "Excluded files: %d", ScanCtx.dbg_excluded_files_count) - LOG_DEBUGF("main.c", "Failed files: %d", ScanCtx.dbg_failed_files_count) - LOG_DEBUGF("main.c", "Thumbnail store size: %lu", ScanCtx.stat_tn_size) - LOG_DEBUGF("main.c", "Index size: %lu", ScanCtx.stat_index_size) + database_t *db = database_create(args->output, INDEX_DATABASE); + database_open(db); - if (args->incremental != NULL) { - save_incremental_index(args); + if (args->incremental != FALSE) { + database_incremental_scan_end(db); } - generate_stats(&ScanCtx.index, args->treemap_threshold, ScanCtx.index.path); - - store_destroy(ScanCtx.index.store); - store_destroy(ScanCtx.index.meta_store); + database_generate_stats(db, args->treemap_threshold); + database_close(db, TRUE); } void sist2_index(index_args_t *args) { - char file_path[PATH_MAX]; - IndexCtx.es_url = args->es_url; IndexCtx.es_index = args->es_index; IndexCtx.es_insecure_ssl = args->es_insecure_ssl; @@ -461,91 +372,69 @@ void sist2_index(index_args_t *args) { elastic_init(args->force_reset, args->es_mappings, args->es_settings); } - char descriptor_path[PATH_MAX]; - snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->index_path); + database_t *db = database_create(args->index_path, INDEX_DATABASE); + database_open(db); + index_descriptor_t *desc = database_read_index_descriptor(db); + database_close(db, FALSE); - index_descriptor_t desc = read_index_descriptor(descriptor_path); + LOG_DEBUGF("main.c", "Index version %s", desc->version); - LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type) - - if (strcmp(desc.version, Version) != 0) { - LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", desc.version, Version) + if (desc->version_major != VersionMajor) { + LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", desc->version, Version); } - DIR *dir = opendir(args->index_path); - if (dir == NULL) { - LOG_FATALF("main.c", "Could not open index %s: %s", args->index_path, strerror(errno)) - } - - char path_tmp[PATH_MAX]; - snprintf(path_tmp, sizeof(path_tmp), "%stags", args->index_path); - IndexCtx.tag_store = store_create(path_tmp, STORE_SIZE_TAG); - IndexCtx.tags = store_read_all(IndexCtx.tag_store); - - snprintf(path_tmp, sizeof(path_tmp), "%smeta", args->index_path); - IndexCtx.meta_store = store_create(path_tmp, STORE_SIZE_META); - IndexCtx.meta = store_read_all(IndexCtx.meta_store); - - index_func f; - if (args->print) { - f = print_json; - } else { - f = index_json; - } - - IndexCtx.pool = tpool_create(args->threads, elastic_cleanup, args->print == 0); + IndexCtx.pool = tpool_create(args->threads, args->print == FALSE); tpool_start(IndexCtx.pool); - READ_INDICES(file_path, args->index_path, { - read_index(file_path, desc.id, desc.type, f); - LOG_DEBUGF("main.c", "Read index file %s (%s)", file_path, desc.type); - }, {}, !args->incremental); + int cnt = 0; - // Only read the _delete index if we're sending data to ES - if (!args->print) { - snprintf(file_path, PATH_MAX, "%s_index_delete.list.zst", args->index_path); - if (0 == access(file_path, R_OK)) { - read_lines(file_path, (line_processor_t) { - .data = NULL, - .func = delete_document - }); - LOG_DEBUGF("main.c", "Read index file %s (%s)", file_path, desc.type) + db = database_create(args->index_path, INDEX_DATABASE); + database_open(db); + database_iterator_t *iterator = database_create_document_iterator(db); + database_document_iter_foreach(json, iterator) { + const char *doc_id = cJSON_GetObjectItem(json, "_id")->valuestring; + if (args->print) { + print_json(json, doc_id); + } else { + index_json(json, doc_id); + cnt +=1; } } - closedir(dir); + free(iterator); + database_close(db, FALSE); + + // Only read the _delete index if we're sending data to ES + if (!args->print) { + // TODO: (delete_list iterator) + } tpool_wait(IndexCtx.pool); - tpool_destroy(IndexCtx.pool); if (IndexCtx.needs_es_connection) { - finish_indexer(args->script, args->async_script, desc.id); + finish_indexer(args->script, args->async_script, desc->id); } - - store_destroy(IndexCtx.tag_store); - store_destroy(IndexCtx.meta_store); - g_hash_table_remove_all(IndexCtx.tags); - g_hash_table_destroy(IndexCtx.tags); + free(desc); } void sist2_exec_script(exec_args_t *args) { - LogCtx.verbose = TRUE; - char descriptor_path[PATH_MAX]; - snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->index_path); - index_descriptor_t desc = read_index_descriptor(descriptor_path); - IndexCtx.es_url = args->es_url; IndexCtx.es_index = args->es_index; IndexCtx.es_insecure_ssl = args->es_insecure_ssl; IndexCtx.needs_es_connection = TRUE; - LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type) + database_t *db = database_create(args->index_path, INDEX_DATABASE); + database_open(db); - execute_update_script(args->script, args->async_script, desc.id); + index_descriptor_t *desc = database_read_index_descriptor(db); + LOG_DEBUGF("main.c", "Index version %s", desc->version); + + execute_update_script(args->script, args->async_script, desc->id); free(args->script); + database_close(db, FALSE); } void sist2_web(web_args_t *args) { @@ -569,23 +458,17 @@ void sist2_web(web_args_t *args) { for (int i = 0; i < args->index_count; i++) { char *abs_path = abspath(args->indices[i]); - if (abs_path == NULL) { - return; - } - char path_tmp[PATH_MAX]; - - snprintf(path_tmp, PATH_MAX, "%sthumbs", abs_path); - WebCtx.indices[i].store = store_create(path_tmp, STORE_SIZE_TN); - - snprintf(path_tmp, PATH_MAX, "%stags", abs_path); - mkdir(path_tmp, S_IWUSR | S_IRUSR | S_IXUSR); - WebCtx.indices[i].tag_store = store_create(path_tmp, STORE_SIZE_TAG); - - snprintf(path_tmp, PATH_MAX, "%sdescriptor.json", abs_path); - WebCtx.indices[i].desc = read_index_descriptor(path_tmp); strcpy(WebCtx.indices[i].path, abs_path); - LOG_INFOF("main.c", "Loaded index: [%s]", WebCtx.indices[i].desc.name) + + WebCtx.indices[i].db = database_create(abs_path, INDEX_DATABASE); + database_open(WebCtx.indices[i].db); + + index_descriptor_t *desc = database_read_index_descriptor(WebCtx.indices[i].db); + WebCtx.indices[i].desc = *desc; + free(desc); + + LOG_INFOF("main.c", "Loaded index: [%s]", WebCtx.indices[i].desc.name); free(abs_path); } @@ -600,7 +483,7 @@ void sist2_web(web_args_t *args) { * Negative number -> Raise error * Specified a valid number -> Continue as normal */ -int set_to_negative_if_value_is_zero(struct argparse *self, const struct argparse_option *option) { +int set_to_negative_if_value_is_zero(UNUSED(struct argparse *self), const struct argparse_option *option) { int specified_value = *(int *) option->value; if (specified_value == 0) { @@ -613,6 +496,7 @@ int set_to_negative_if_value_is_zero(struct argparse *self, const struct argpars } } +#include int main(int argc, const char *argv[]) { // sigsegv_handler = signal(SIGSEGV, sig_handler); @@ -645,8 +529,8 @@ int main(int argc, const char *argv[]) { OPT_GROUP("Scan options"), OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"), OPT_INTEGER('q', "thumbnail-quality", &scan_args->tn_quality, - "Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT=2", - set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_quality), + "Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT=2", + set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_quality), OPT_INTEGER(0, "thumbnail-size", &scan_args->tn_size, "Thumbnail size, in pixels. DEFAULT=500", set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_size), @@ -656,7 +540,8 @@ int main(int argc, const char *argv[]) { OPT_INTEGER(0, "content-size", &scan_args->content_size, "Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT=32768", set_to_negative_if_value_is_zero, (intptr_t) &scan_args->content_size), - OPT_STRING(0, "incremental", &scan_args->incremental, + OPT_BOOLEAN(0, "incremental", &scan_args->incremental, + // TODO: Update help string "Reuse an existing index and only scan modified files."), OPT_STRING('o', "output", &scan_args->output, "Output directory. DEFAULT=index.sist2/"), OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."), @@ -692,7 +577,8 @@ int main(int argc, const char *argv[]) { OPT_GROUP("Index options"), OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"), OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"), - OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."), + OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, + "Do not verify SSL connections to Elasticsearch."), OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."), OPT_BOOLEAN(0, "incremental-index", &index_args->incremental, @@ -701,20 +587,22 @@ int main(int argc, const char *argv[]) { OPT_STRING(0, "mappings-file", &index_args->es_mappings_path, "Path to Elasticsearch mappings."), OPT_STRING(0, "settings-file", &index_args->es_settings_path, "Path to Elasticsearch settings."), OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."), - OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 100"), + OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 70"), OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. " "(You must use this option the first time you use the index command)"), OPT_GROUP("Web options"), OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"), - OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."), + OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, + "Do not verify SSL connections to Elasticsearch."), OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), OPT_STRING(0, "bind", &web_args->listen_address, "Listen on this address. DEFAULT=localhost:4090"), OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"), OPT_STRING(0, "auth0-audience", &web_args->auth0_audience, "API audience/identifier"), OPT_STRING(0, "auth0-domain", &web_args->auth0_domain, "Application domain"), OPT_STRING(0, "auth0-client-id", &web_args->auth0_client_id, "Application client ID"), - OPT_STRING(0, "auth0-public-key-file", &web_args->auth0_public_key_path, "Path to Auth0 public key file extracted from /pem"), + OPT_STRING(0, "auth0-public-key-file", &web_args->auth0_public_key_path, + "Path to Auth0 public key file extracted from /pem"), OPT_STRING(0, "tag-auth", &web_args->tag_credentials, "Basic auth in user:password format for tagging"), OPT_STRING(0, "tagline", &web_args->tagline, "Tagline in navbar"), OPT_BOOLEAN(0, "dev", &web_args->dev, "Serve html & js files from disk (for development)"), @@ -722,7 +610,8 @@ int main(int argc, const char *argv[]) { OPT_GROUP("Exec-script options"), OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"), - OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."), + OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, + "Do not verify SSL connections to Elasticsearch."), OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), OPT_STRING(0, "script-file", &common_script_path, "Path to user script."), OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."), @@ -800,7 +689,7 @@ int main(int argc, const char *argv[]) { } else { argparse_usage(&argparse); - LOG_FATALF("main.c", "Invalid command: '%s'\n", argv[0]) + LOG_FATALF("main.c", "Invalid command: '%s'\n", argv[0]); } printf("\n"); diff --git a/src/mempool/mempool.c b/src/mempool/mempool.c deleted file mode 100644 index d8dd0a8..0000000 --- a/src/mempool/mempool.c +++ /dev/null @@ -1,757 +0,0 @@ -#include "mempool.h" -#include - -#define NCX_SLAB_PAGE_MASK 3 -#define NCX_SLAB_PAGE 0 -#define NCX_SLAB_BIG 1 -#define NCX_SLAB_EXACT 2 -#define NCX_SLAB_SMALL 3 - -#define NCX_SLAB_PAGE_FREE 0 -#define NCX_SLAB_PAGE_BUSY 0xffffffffffffffff -#define NCX_SLAB_PAGE_START 0x8000000000000000 - -#define NCX_SLAB_SHIFT_MASK 0x000000000000000f -#define NCX_SLAB_MAP_MASK 0xffffffff00000000 -#define NCX_SLAB_MAP_SHIFT 32 - -#define NCX_SLAB_BUSY 0xffffffffffffffff - - -static ncx_slab_page_t *ncx_slab_alloc_pages(ncx_slab_pool_t *pool, ncx_uint_t pages); - -static void ncx_slab_free_pages(ncx_slab_pool_t *pool, ncx_slab_page_t *page, ncx_uint_t pages); - -static bool ncx_slab_empty(ncx_slab_pool_t *pool, ncx_slab_page_t *page); - -static ncx_uint_t ncx_slab_max_size; -static ncx_uint_t ncx_slab_exact_size; -static ncx_uint_t ncx_slab_exact_shift; -static ncx_uint_t ncx_pagesize; -static ncx_uint_t ncx_pagesize_shift; -static ncx_uint_t ncx_real_pages; - -void ncx_slab_init(ncx_slab_pool_t *pool) { - u_char *p; - size_t size; - ncx_uint_t i, n, pages; - ncx_slab_page_t *slots; - - /*pagesize*/ - ncx_pagesize = getpagesize(); - for (n = ncx_pagesize, ncx_pagesize_shift = 0; - n >>= 1; ncx_pagesize_shift++) { /* void */ } - - /* STUB */ - if (ncx_slab_max_size == 0) { - ncx_slab_max_size = ncx_pagesize / 2; - ncx_slab_exact_size = ncx_pagesize / (8 * sizeof(uintptr_t)); - for (n = ncx_slab_exact_size; n >>= 1; ncx_slab_exact_shift++) { - /* void */ - } - } - - pool->min_size = 1 << pool->min_shift; - - p = (u_char *) pool + sizeof(ncx_slab_pool_t); - slots = (ncx_slab_page_t *) p; - - n = ncx_pagesize_shift - pool->min_shift; - for (i = 0; i < n; i++) { - slots[i].slab = 0; - slots[i].next = &slots[i]; - slots[i].prev = 0; - } - - p += n * sizeof(ncx_slab_page_t); - - size = pool->end - p; - - pages = (ncx_uint_t) (size / (ncx_pagesize + sizeof(ncx_slab_page_t))); - - ncx_memzero(p, pages * sizeof(ncx_slab_page_t)); - - pool->pages = (ncx_slab_page_t *) p; - - pool->free.prev = 0; - pool->free.next = (ncx_slab_page_t *) p; - - pool->pages->slab = pages; - pool->pages->next = &pool->free; - pool->pages->prev = (uintptr_t) &pool->free; - - pool->start = (u_char *) - ncx_align_ptr((uintptr_t) p + pages * sizeof(ncx_slab_page_t), - ncx_pagesize); - - ncx_real_pages = (pool->end - pool->start) / ncx_pagesize; - pool->pages->slab = ncx_real_pages; -} - - -void *ncx_slab_alloc(ncx_slab_pool_t *pool, size_t size) { - size_t s; - uintptr_t p, n, m, mask, *bitmap; - ncx_uint_t i, slot, shift, map; - ncx_slab_page_t *page, *prev, *slots; - - if (size >= ncx_slab_max_size) { - - page = ncx_slab_alloc_pages(pool, (size >> ncx_pagesize_shift) - + ((size % ncx_pagesize) ? 1 : 0)); - if (page) { - p = (page - pool->pages) << ncx_pagesize_shift; - p += (uintptr_t) pool->start; - - } else { - p = 0; - } - - goto done; - } - - if (size > pool->min_size) { - shift = 1; - for (s = size - 1; s >>= 1; shift++) { /* void */ } - slot = shift - pool->min_shift; - - } else { - shift = pool->min_shift; - slot = 0; - } - - slots = (ncx_slab_page_t *) ((u_char *) pool + sizeof(ncx_slab_pool_t)); - page = slots[slot].next; - - if (page->next != page) { - - if (shift < ncx_slab_exact_shift) { - - do { - p = (page - pool->pages) << ncx_pagesize_shift; - bitmap = (uintptr_t *) (pool->start + p); - - map = (1 << (ncx_pagesize_shift - shift)) - / (sizeof(uintptr_t) * 8); - - for (n = 0; n < map; n++) { - - if (bitmap[n] != NCX_SLAB_BUSY) { - - for (m = 1, i = 0; m; m <<= 1, i++) { - if ((bitmap[n] & m)) { - continue; - } - - bitmap[n] |= m; - - i = ((n * sizeof(uintptr_t) * 8) << shift) - + (i << shift); - - if (bitmap[n] == NCX_SLAB_BUSY) { - for (n = n + 1; n < map; n++) { - if (bitmap[n] != NCX_SLAB_BUSY) { - p = (uintptr_t) bitmap + i; - - goto done; - } - } - - prev = (ncx_slab_page_t *) - (page->prev & ~NCX_SLAB_PAGE_MASK); - prev->next = page->next; - page->next->prev = page->prev; - - page->next = NULL; - page->prev = NCX_SLAB_SMALL; - } - - p = (uintptr_t) bitmap + i; - - goto done; - } - } - } - - page = page->next; - - } while (page); - - } else if (shift == ncx_slab_exact_shift) { - - do { - if (page->slab != NCX_SLAB_BUSY) { - - for (m = 1, i = 0; m; m <<= 1, i++) { - if ((page->slab & m)) { - continue; - } - - page->slab |= m; - - if (page->slab == NCX_SLAB_BUSY) { - prev = (ncx_slab_page_t *) - (page->prev & ~NCX_SLAB_PAGE_MASK); - prev->next = page->next; - page->next->prev = page->prev; - - page->next = NULL; - page->prev = NCX_SLAB_EXACT; - } - - p = (page - pool->pages) << ncx_pagesize_shift; - p += i << shift; - p += (uintptr_t) pool->start; - - goto done; - } - } - - page = page->next; - - } while (page); - - } else { /* shift > ncx_slab_exact_shift */ - - n = ncx_pagesize_shift - (page->slab & NCX_SLAB_SHIFT_MASK); - n = 1 << n; - n = ((uintptr_t) 1 << n) - 1; - mask = n << NCX_SLAB_MAP_SHIFT; - - do { - if ((page->slab & NCX_SLAB_MAP_MASK) != mask) { - - for (m = (uintptr_t) 1 << NCX_SLAB_MAP_SHIFT, i = 0; - m & mask; - m <<= 1, i++) { - if ((page->slab & m)) { - continue; - } - - page->slab |= m; - - if ((page->slab & NCX_SLAB_MAP_MASK) == mask) { - prev = (ncx_slab_page_t *) - (page->prev & ~NCX_SLAB_PAGE_MASK); - prev->next = page->next; - page->next->prev = page->prev; - - page->next = NULL; - page->prev = NCX_SLAB_BIG; - } - - p = (page - pool->pages) << ncx_pagesize_shift; - p += i << shift; - p += (uintptr_t) pool->start; - - goto done; - } - } - - page = page->next; - - } while (page); - } - } - - page = ncx_slab_alloc_pages(pool, 1); - - if (page) { - if (shift < ncx_slab_exact_shift) { - p = (page - pool->pages) << ncx_pagesize_shift; - bitmap = (uintptr_t *) (pool->start + p); - - s = 1 << shift; - n = (1 << (ncx_pagesize_shift - shift)) / 8 / s; - - if (n == 0) { - n = 1; - } - - bitmap[0] = (2 << n) - 1; - - map = (1 << (ncx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8); - - for (i = 1; i < map; i++) { - bitmap[i] = 0; - } - - page->slab = shift; - page->next = &slots[slot]; - page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_SMALL; - - slots[slot].next = page; - - p = ((page - pool->pages) << ncx_pagesize_shift) + s * n; - p += (uintptr_t) pool->start; - - goto done; - - } else if (shift == ncx_slab_exact_shift) { - - page->slab = 1; - page->next = &slots[slot]; - page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_EXACT; - - slots[slot].next = page; - - p = (page - pool->pages) << ncx_pagesize_shift; - p += (uintptr_t) pool->start; - - goto done; - - } else { /* shift > ncx_slab_exact_shift */ - - page->slab = ((uintptr_t) 1 << NCX_SLAB_MAP_SHIFT) | shift; - page->next = &slots[slot]; - page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_BIG; - - slots[slot].next = page; - - p = (page - pool->pages) << ncx_pagesize_shift; - p += (uintptr_t) pool->start; - - goto done; - } - } - - p = 0; - - done: - - return (void *) p; -} - - -void ncx_slab_free(ncx_slab_pool_t *pool, void *p) { - size_t size; - uintptr_t slab, m, *bitmap; - ncx_uint_t n, type, slot, shift, map; - ncx_slab_page_t *slots, *page; - - if ((u_char *) p < pool->start || (u_char *) p > pool->end) { -// error("ncx_slab_free(): outside of pool"); - goto fail; - } - - n = ((u_char *) p - pool->start) >> ncx_pagesize_shift; - page = &pool->pages[n]; - slab = page->slab; - type = page->prev & NCX_SLAB_PAGE_MASK; - - switch (type) { - - case NCX_SLAB_SMALL: - - shift = slab & NCX_SLAB_SHIFT_MASK; - size = 1 << shift; - - if ((uintptr_t) p & (size - 1)) { - goto wrong_chunk; - } - - n = ((uintptr_t) p & (ncx_pagesize - 1)) >> shift; - m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1)); - n /= (sizeof(uintptr_t) * 8); - bitmap = (uintptr_t *) ((uintptr_t) p & ~(ncx_pagesize - 1)); - - if (bitmap[n] & m) { - - if (page->next == NULL) { - slots = (ncx_slab_page_t *) - ((u_char *) pool + sizeof(ncx_slab_pool_t)); - slot = shift - pool->min_shift; - - page->next = slots[slot].next; - slots[slot].next = page; - - page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_SMALL; - page->next->prev = (uintptr_t) page | NCX_SLAB_SMALL; - } - - bitmap[n] &= ~m; - - n = (1 << (ncx_pagesize_shift - shift)) / 8 / (1 << shift); - - if (n == 0) { - n = 1; - } - - if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) { - goto done; - } - - map = (1 << (ncx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8); - - for (n = 1; n < map; n++) { - if (bitmap[n]) { - goto done; - } - } - - ncx_slab_free_pages(pool, page, 1); - - goto done; - } - - goto chunk_already_free; - - case NCX_SLAB_EXACT: - - m = (uintptr_t) 1 << - (((uintptr_t) p & (ncx_pagesize - 1)) >> ncx_slab_exact_shift); - size = ncx_slab_exact_size; - - if ((uintptr_t) p & (size - 1)) { - goto wrong_chunk; - } - - if (slab & m) { - if (slab == NCX_SLAB_BUSY) { - slots = (ncx_slab_page_t *) - ((u_char *) pool + sizeof(ncx_slab_pool_t)); - slot = ncx_slab_exact_shift - pool->min_shift; - - page->next = slots[slot].next; - slots[slot].next = page; - - page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_EXACT; - page->next->prev = (uintptr_t) page | NCX_SLAB_EXACT; - } - - page->slab &= ~m; - - if (page->slab) { - goto done; - } - - ncx_slab_free_pages(pool, page, 1); - - goto done; - } - - goto chunk_already_free; - - case NCX_SLAB_BIG: - - shift = slab & NCX_SLAB_SHIFT_MASK; - size = 1 << shift; - - if ((uintptr_t) p & (size - 1)) { - goto wrong_chunk; - } - - m = (uintptr_t) 1 << ((((uintptr_t) p & (ncx_pagesize - 1)) >> shift) - + NCX_SLAB_MAP_SHIFT); - - if (slab & m) { - - if (page->next == NULL) { - slots = (ncx_slab_page_t *) - ((u_char *) pool + sizeof(ncx_slab_pool_t)); - slot = shift - pool->min_shift; - - page->next = slots[slot].next; - slots[slot].next = page; - - page->prev = (uintptr_t) &slots[slot] | NCX_SLAB_BIG; - page->next->prev = (uintptr_t) page | NCX_SLAB_BIG; - } - - page->slab &= ~m; - - if (page->slab & NCX_SLAB_MAP_MASK) { - goto done; - } - - ncx_slab_free_pages(pool, page, 1); - - goto done; - } - - goto chunk_already_free; - - case NCX_SLAB_PAGE: - - if ((uintptr_t) p & (ncx_pagesize - 1)) { - goto wrong_chunk; - } - - if (slab == NCX_SLAB_PAGE_FREE) { -// alert("ncx_slab_free(): page is already free"); - goto fail; - } - - if (slab == NCX_SLAB_PAGE_BUSY) { -// alert("ncx_slab_free(): pointer to wrong page"); - goto fail; - } - - n = ((u_char *) p - pool->start) >> ncx_pagesize_shift; - size = slab & ~NCX_SLAB_PAGE_START; - - ncx_slab_free_pages(pool, &pool->pages[n], size); - - return; - } - - /* not reached */ - - return; - - done: - - return; - - wrong_chunk: - -// error("ncx_slab_free(): pointer to wrong chunk"); - - goto fail; - - chunk_already_free: - -// error("ncx_slab_free(): chunk is already free"); - - fail: - - return; -} - - -static ncx_slab_page_t *ncx_slab_alloc_pages(ncx_slab_pool_t *pool, ncx_uint_t pages) { - ncx_slab_page_t *page, *p; - - for (page = pool->free.next; page != &pool->free; page = page->next) { - - if (page->slab >= pages) { - - if (page->slab > pages) { - page[pages].slab = page->slab - pages; - page[pages].next = page->next; - page[pages].prev = page->prev; - - p = (ncx_slab_page_t *) page->prev; - p->next = &page[pages]; - page->next->prev = (uintptr_t) &page[pages]; - - } else { - p = (ncx_slab_page_t *) page->prev; - p->next = page->next; - page->next->prev = page->prev; - } - - page->slab = pages | NCX_SLAB_PAGE_START; - page->next = NULL; - page->prev = NCX_SLAB_PAGE; - - if (--pages == 0) { - return page; - } - - for (p = page + 1; pages; pages--) { - p->slab = NCX_SLAB_PAGE_BUSY; - p->next = NULL; - p->prev = NCX_SLAB_PAGE; - p++; - } - - return page; - } - } - -// error("ncx_slab_alloc() failed: no memory"); - - return NULL; -} - -static void ncx_slab_free_pages(ncx_slab_pool_t *pool, ncx_slab_page_t *page, ncx_uint_t pages) { - ncx_slab_page_t *prev; - - if (pages > 1) { - ncx_memzero(&page[1], (pages - 1) * sizeof(ncx_slab_page_t)); - } - - if (page->next) { - prev = (ncx_slab_page_t *) (page->prev & ~NCX_SLAB_PAGE_MASK); - prev->next = page->next; - page->next->prev = page->prev; - } - - page->slab = pages; - page->prev = (uintptr_t) &pool->free; - page->next = pool->free.next; - page->next->prev = (uintptr_t) page; - - pool->free.next = page; - -#ifdef PAGE_MERGE - if (pool->pages != page) { - prev = page - 1; - if (ncx_slab_empty(pool, prev)) { - for (; prev >= pool->pages; prev--) { - if (prev->slab != 0) - { - pool->free.next = page->next; - page->next->prev = (uintptr_t) &pool->free; - - prev->slab += pages; - ncx_memzero(page, sizeof(ncx_slab_page_t)); - - page = prev; - - break; - } - } - } - } - - if ((page - pool->pages + page->slab) < ncx_real_pages) { - next = page + page->slab; - if (ncx_slab_empty(pool, next)) - { - prev = (ncx_slab_page_t *) (next->prev); - prev->next = next->next; - next->next->prev = next->prev; - - page->slab += next->slab; - ncx_memzero(next, sizeof(ncx_slab_page_t)); - } - } - -#endif -} - -void ncx_slab_stat(ncx_slab_pool_t *pool, ncx_slab_stat_t *stat) { - uintptr_t m, n, mask, slab; - uintptr_t *bitmap; - ncx_uint_t i, j, map, type, obj_size; - ncx_slab_page_t *page; - - ncx_memzero(stat, sizeof(ncx_slab_stat_t)); - - page = pool->pages; - stat->pages = (pool->end - pool->start) / ncx_pagesize; - - for (i = 0; i < stat->pages; i++) { - slab = page->slab; - type = page->prev & NCX_SLAB_PAGE_MASK; - - switch (type) { - - case NCX_SLAB_SMALL: - - n = (page - pool->pages) << ncx_pagesize_shift; - bitmap = (uintptr_t *) (pool->start + n); - - obj_size = 1 << slab; - map = (1 << (ncx_pagesize_shift - slab)) - / (sizeof(uintptr_t) * 8); - - for (j = 0; j < map; j++) { - for (m = 1; m; m <<= 1) { - if ((bitmap[j] & m)) { - stat->used_size += obj_size; - stat->b_small += obj_size; - } - - } - } - - stat->p_small++; - - break; - - case NCX_SLAB_EXACT: - - if (slab == NCX_SLAB_BUSY) { - stat->used_size += sizeof(uintptr_t) * 8 * ncx_slab_exact_size; - stat->b_exact += sizeof(uintptr_t) * 8 * ncx_slab_exact_size; - } else { - for (m = 1; m; m <<= 1) { - if (slab & m) { - stat->used_size += ncx_slab_exact_size; - stat->b_exact += ncx_slab_exact_size; - } - } - } - - stat->p_exact++; - - break; - - case NCX_SLAB_BIG: - - j = ncx_pagesize_shift - (slab & NCX_SLAB_SHIFT_MASK); - j = 1 << j; - j = ((uintptr_t) 1 << j) - 1; - mask = j << NCX_SLAB_MAP_SHIFT; - obj_size = 1 << (slab & NCX_SLAB_SHIFT_MASK); - - for (m = (uintptr_t) 1 << NCX_SLAB_MAP_SHIFT; m & mask; m <<= 1) { - if ((page->slab & m)) { - stat->used_size += obj_size; - stat->b_big += obj_size; - } - } - - stat->p_big++; - - break; - - case NCX_SLAB_PAGE: - - if (page->prev == NCX_SLAB_PAGE) { - slab = slab & ~NCX_SLAB_PAGE_START; - stat->used_size += slab * ncx_pagesize; - stat->b_page += slab * ncx_pagesize; - stat->p_page += slab; - - i += (slab - 1); - - break; - } - - default: - - if (slab > stat->max_free_pages) { - stat->max_free_pages = page->slab; - } - - stat->free_page += slab; - - i += (slab - 1); - - break; - } - - page = pool->pages + i + 1; - } - - stat->pool_size = pool->end - pool->start; - stat->used_pct = stat->used_size * 100 / stat->pool_size; -} - -static bool ncx_slab_empty(ncx_slab_pool_t *pool, ncx_slab_page_t *page) { - ncx_slab_page_t *prev; - - if (page->slab == 0) { - return true; - } - - //page->prev == PAGE | SMALL | EXACT | BIG - if (page->next == NULL) { - return false; - } - - prev = (ncx_slab_page_t *) (page->prev & ~NCX_SLAB_PAGE_MASK); - while (prev >= pool->pages) { - prev = (ncx_slab_page_t *) (prev->prev & ~NCX_SLAB_PAGE_MASK); - } - - if (prev == &pool->free) { - return true; - } - - return false; -} \ No newline at end of file diff --git a/src/mempool/mempool.h b/src/mempool/mempool.h deleted file mode 100644 index e903158..0000000 --- a/src/mempool/mempool.h +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef SIST2_MEMPOOL_H -#define SIST2_MEMPOOL_H - -#include -#include -#include -#include -#include - -typedef unsigned char u_char; -typedef uintptr_t ncx_uint_t; - -#ifndef NCX_ALIGNMENT -#define NCX_ALIGNMENT sizeof(unsigned long) -#endif - -#define ncx_align(d, a) (((d) + (a - 1)) & ~(a - 1)) -#define ncx_align_ptr(p, a) (u_char *) (((uintptr_t) (p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1)) - -#define ncx_memzero(buf, n) (void) memset(buf, 0, n) -#define ncx_memset(buf, c, n) (void) memset(buf, c, n) - -typedef struct ncx_slab_page_s ncx_slab_page_t; - -struct ncx_slab_page_s { - uintptr_t slab; - ncx_slab_page_t *next; - uintptr_t prev; -}; - -typedef struct { - size_t min_size; - size_t min_shift; - - ncx_slab_page_t *pages; - ncx_slab_page_t free; - - u_char *start; - u_char *end; - - //ncx_shmtx_t mutex; - - void *addr; -} ncx_slab_pool_t; - -typedef struct { - size_t pool_size, used_size, used_pct; - size_t pages, free_page; - size_t p_small, p_exact, p_big, p_page; - size_t b_small, b_exact, b_big, b_page; - size_t max_free_pages; -} ncx_slab_stat_t; - -void ncx_slab_init(ncx_slab_pool_t *mempool); - -void *ncx_slab_alloc(ncx_slab_pool_t *mempool, size_t size); - -void ncx_slab_free(ncx_slab_pool_t *mempool, void *p); - -void ncx_slab_stat(ncx_slab_pool_t *mempool, ncx_slab_stat_t *stat); - -#endif //SIST2_MEMPOOL_H diff --git a/src/parsing/fs_util.h b/src/parsing/fs_util.h new file mode 100644 index 0000000..a3b257e --- /dev/null +++ b/src/parsing/fs_util.h @@ -0,0 +1,42 @@ +#ifndef SIST2_FS_UTIL_H +#define SIST2_FS_UTIL_H + +#include "src/sist.h" + +#define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));}; + +static int fs_read(struct vfile *f, void *buf, size_t size) { + + if (f->fd == -1) { + SHA1_Init(&f->sha1_ctx); + + f->fd = open(f->filepath, O_RDONLY); + if (f->fd == -1) { + return -1; + } + } + + int ret = (int) read(f->fd, buf, size); + + if (ret != 0 && f->calculate_checksum) { + f->has_checksum = TRUE; + safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret); + } + + return ret; +} + +static void fs_close(struct vfile *f) { + if (f->fd != -1) { + SHA1_Final(f->sha1_digest, &f->sha1_ctx); + close(f->fd); + } +} + +static void fs_reset(struct vfile *f) { + if (f->fd != -1) { + lseek(f->fd, 0, SEEK_SET); + } +} + +#endif diff --git a/src/parsing/magic_util.c b/src/parsing/magic_util.c new file mode 100644 index 0000000..e5443a8 --- /dev/null +++ b/src/parsing/magic_util.c @@ -0,0 +1,32 @@ +#include "magic_util.h" +#include "src/log.h" +#include "mime.h" +#include +#include "src/magic_generated.c" + + +char *magic_buffer_embedded(void *buffer, size_t buffer_size) { + + magic_t magic = magic_open(MAGIC_MIME_TYPE); + + const char *magic_buffers[1] = {magic_database_buffer,}; + size_t sizes[1] = {sizeof(magic_database_buffer),}; + + // TODO: check if we can reuse the magic instance + int load_ret = magic_load_buffers(magic, (void **) &magic_buffers, sizes, 1); + + if (load_ret != 0) { + LOG_FATALF("parse.c", "Could not load libmagic database: (%d)", load_ret); + } + + const char *magic_mime_str = magic_buffer(magic, buffer, buffer_size); + char *return_value = NULL; + + if (magic_mime_str != NULL) { + return_value = malloc(strlen(magic_mime_str) + 1); + strcpy(return_value, magic_mime_str); + } + + magic_close(magic); + return return_value; +} \ No newline at end of file diff --git a/src/parsing/magic_util.h b/src/parsing/magic_util.h new file mode 100644 index 0000000..8c40cb4 --- /dev/null +++ b/src/parsing/magic_util.h @@ -0,0 +1,8 @@ +#ifndef SIST2_MAGIC_UTIL_H +#define SIST2_MAGIC_UTIL_H + +#include + +char *magic_buffer_embedded(void *buffer, size_t buffer_size); + +#endif //SIST2_MAGIC_UTIL_H diff --git a/src/parsing/mime.c b/src/parsing/mime.c index 4a218df..49a26c0 100644 --- a/src/parsing/mime.c +++ b/src/parsing/mime.c @@ -1,22 +1,30 @@ #include "mime.h" +#include -unsigned int mime_get_mime_by_ext(GHashTable *ext_table, const char * ext) { - char lower[8]; - char *p = lower; +unsigned int mime_get_mime_by_ext(const char *ext) { + unsigned char lower[16]; + unsigned char *p = lower; int cnt = 0; while ((*ext) != '\0' && cnt + 1 < sizeof(lower)) { - *p++ = (char)tolower(*ext++); + *p++ = tolower(*ext++); cnt++; } *p = '\0'; - return (size_t) g_hash_table_lookup(ext_table, lower); + + unsigned long crc = crc32(0, lower, cnt); + + unsigned int mime = mime_extension_lookup(crc); + return mime; } -unsigned int mime_get_mime_by_string(GHashTable *mime_table, const char * str) { +unsigned int mime_get_mime_by_string(const char *str) { - const char * ptr = str; + const char *ptr = str; while (*ptr == ' ' || *ptr == '[') { ptr++; } - return (size_t) g_hash_table_lookup(mime_table, ptr); + + unsigned long crc = crc32(0, (unsigned char *) ptr, strlen(ptr)); + + return mime_name_lookup(crc); } diff --git a/src/parsing/mime.h b/src/parsing/mime.h index de7b1be..50b380a 100644 --- a/src/parsing/mime.h +++ b/src/parsing/mime.h @@ -51,14 +51,14 @@ enum major_mime { enum mime; -GHashTable *mime_get_mime_table(); +unsigned int mime_name_lookup(unsigned long mime_crc32); -GHashTable *mime_get_ext_table(); +unsigned int mime_extension_lookup(unsigned long extension_crc32); -char *mime_get_mime_text(unsigned int); +const char *mime_get_mime_text(unsigned int); -unsigned int mime_get_mime_by_ext(GHashTable *ext_table, const char * ext); +unsigned int mime_get_mime_by_ext(const char *ext); -unsigned int mime_get_mime_by_string(GHashTable *mime_table, const char * str); +unsigned int mime_get_mime_by_string(const char *str); #endif diff --git a/src/parsing/mime_generated.c b/src/parsing/mime_generated.c index 3b8fed8..6eedeae 100644 --- a/src/parsing/mime_generated.c +++ b/src/parsing/mime_generated.c @@ -1,460 +1,458 @@ // **Generated by mime.py** #ifndef MIME_GENERATED_C #define MIME_GENERATED_C -#include - #include enum mime { - application_CDFV2=655361, - application_CDFV2_corrupt=655362, - application_arj=655363, - application_base64=655364, - application_binhex=655365, - application_book=655366, - application_clariscad=655367, - application_commonground=655368, - application_csv=655369, - application_dicom=655370, - application_drafting=655371, - application_epub_zip=655372 | 0x40000000, - application_freeloader=655373, - application_futuresplash=655374, - application_groupwise=655375, - application_gzip=655376 | 0x08000000, - application_hta=655377, - application_i_deas=655378, - application_iges=655379, - application_inf=655380, - application_java=655381, - application_java_archive=655382, - application_javascript=655383, - application_json=655384, - application_marc=655385, - application_mbedlet=655386, - application_mime=655387, - application_mspowerpoint=655388, - application_msword=655389, - application_ndjson=655390, - application_netmc=655391, - application_octet_stream=655392, - application_oda=655393, - application_ogg=655394, - application_pdf=655395 | 0x40000000, - application_pgp_keys=655396, - application_pgp_signature=655397, - application_pkcs7_signature=655398, - application_pkix_cert=655399, - application_postscript=655400, - application_pro_eng=655401, - application_ringing_tones=655402, - application_smil=655403, - application_solids=655404, - application_sounder=655405, - application_step=655406, - application_streamingmedia=655407, - application_vda=655408, - application_vnd_amazon_mobi8_ebook=655409 | 0x02000000, - application_vnd_coffeescript=655410, - application_vnd_fdf=655411, - application_vnd_font_fontforge_sfd=655412, - application_vnd_hp_hpgl=655413, - application_vnd_iccprofile=655414, - application_vnd_lotus_1_2_3=655415, - application_vnd_ms_cab_compressed=655416, - application_vnd_ms_excel=655417, - application_vnd_ms_fontobject=655418, - application_vnd_ms_opentype=655419 | 0x20000000, - application_vnd_ms_outlook=655420, - application_vnd_ms_pki_certstore=655421, - application_vnd_ms_pki_pko=655422, - application_vnd_ms_pki_seccat=655423, - application_vnd_ms_powerpoint=655424, - application_vnd_ms_project=655425, - application_vnd_oasis_opendocument_base=655426, - application_vnd_oasis_opendocument_formula=655427, - application_vnd_oasis_opendocument_graphics=655428, - application_vnd_oasis_opendocument_presentation=655429, - application_vnd_oasis_opendocument_spreadsheet=655430, - application_vnd_oasis_opendocument_text=655431, - application_vnd_openxmlformats_officedocument_presentationml_presentation=655432 | 0x04000000, - application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655433 | 0x04000000, - application_vnd_openxmlformats_officedocument_wordprocessingml_document=655434 | 0x04000000, - application_vnd_symbian_install=655435, - application_vnd_tcpdump_pcap=655436, - application_vnd_wap_wmlc=655437, - application_vnd_wap_wmlscriptc=655438, - application_vnd_xara=655439, - application_vocaltec_media_desc=655440, - application_vocaltec_media_file=655441, - application_warc=655442, - application_winhelp=655443, - application_wordperfect=655444, - application_x_123=655445, - application_x_7z_compressed=655446 | 0x10000000, - application_x_aim=655447, - application_x_apple_diskimage=655448, - application_x_arc=655449 | 0x10000000, - application_x_archive=655450, - application_x_atari_7800_rom=655451, - application_x_authorware_bin=655452, - application_x_authorware_map=655453, - application_x_authorware_seg=655454, - application_x_avira_qua=655455, - application_x_bcpio=655456, - application_x_bittorrent=655457, - application_x_bsh=655458, - application_x_bytecode_python=655459, - application_x_bzip=655460, - application_x_bzip2=655461 | 0x08000000, - application_x_cbr=655462, - application_x_cbz=655463, - application_x_cdlink=655464, - application_x_chat=655465, - application_x_chrome_extension=655466, - application_x_cocoa=655467, - application_x_conference=655468, - application_x_coredump=655469, - application_x_cpio=655470, - application_x_dbf=655471, - application_x_dbt=655472, - application_x_debian_package=655473, - application_x_deepv=655474, - application_x_director=655475, - application_x_dmp=655476, - application_x_dosdriver=655477, - application_x_dosexec=655478, - application_x_dvi=655479, - application_x_elc=655480, - application_x_empty=1, - application_x_envoy=655481, - application_x_esrehber=655482, - application_x_excel=655483, - application_x_executable=655484, - application_x_font_gdos=655485, - application_x_font_pf2=655486, - application_x_font_pfm=655487, - application_x_font_sfn=655488, - application_x_font_ttf=655489 | 0x20000000, - application_x_fptapplication_x_dbt=655490, - application_x_freelance=655491, - application_x_gamecube_rom=655492, - application_x_gdbm=655493, - application_x_gettext_translation=655494, - application_x_git=655495, - application_x_gsp=655496, - application_x_gss=655497, - application_x_gtar=655498, - application_x_gzip=655499, - application_x_hdf=655500, - application_x_helpfile=655501, - application_x_httpd_imap=655502, - application_x_ima=655503, - application_x_innosetup=655504, - application_x_internett_signup=655505, - application_x_inventor=655506, - application_x_ip2=655507, - application_x_java_applet=655508, - application_x_java_commerce=655509, - application_x_java_image=655510, - application_x_java_jmod=655511, - application_x_java_keystore=655512, - application_x_kdelnk=655513, - application_x_koan=655514, - application_x_latex=655515, - application_x_livescreen=655516, - application_x_lotus=655517, - application_x_lz4=655518 | 0x08000000, - application_x_lz4_json=655519, - application_x_lzh=655520, - application_x_lzh_compressed=655521, - application_x_lzip=655522 | 0x08000000, - application_x_lzma=655523 | 0x08000000, - application_x_lzop=655524 | 0x08000000, - application_x_lzx=655525, - application_x_mach_binary=655526, - application_x_mach_executable=655527, - application_x_magic_cap_package_1_0=655528, - application_x_mathcad=655529, - application_x_maxis_dbpf=655530, - application_x_meme=655531, - application_x_midi=655532, - application_x_mif=655533, - application_x_mix_transfer=655534, - application_x_mobipocket_ebook=655535 | 0x02000000, - application_x_ms_compress_szdd=655536, - application_x_ms_pdb=655537, - application_x_ms_reader=655538, - application_x_msaccess=655539, - application_x_n64_rom=655540, - application_x_navi_animation=655541, - application_x_navidoc=655542, - application_x_navimap=655543, - application_x_navistyle=655544, - application_x_nes_rom=655545, - application_x_netcdf=655546, - application_x_newton_compatible_pkg=655547, - application_x_nintendo_ds_rom=655548, - application_x_object=655549, - application_x_omc=655550, - application_x_omcdatamaker=655551, - application_x_omcregerator=655552, - application_x_pagemaker=655553, - application_x_pcl=655554, - application_x_pgp_keyring=655555, - application_x_pixclscript=655556, - application_x_pkcs7_certreqresp=655557, - application_x_pkcs7_signature=655558, - application_x_project=655559, - application_x_qpro=655560, - application_x_rar=655561 | 0x10000000, - application_x_rpm=655562, - application_x_sdp=655563, - application_x_sea=655564, - application_x_seelogo=655565, - application_x_setupscript=655566, - application_x_shar=655567, - application_x_sharedlib=655568, - application_x_shockwave_flash=655569, - application_x_snappy_framed=655570, - application_x_sprite=655571, - application_x_sqlite3=655572, - application_x_stargallery_thm=655573, - application_x_stuffit=655574, - application_x_sv4cpio=655575, - application_x_sv4crc=655576, - application_x_tar=655577 | 0x10000000, - application_x_tbook=655578, - application_x_terminfo=655579, - application_x_terminfo2=655580, - application_x_tex_tfm=655581, - application_x_texinfo=655582, - application_x_ustar=655583, - application_x_visio=655584, - application_x_vnd_audioexplosion_mzz=655585, - application_x_vnd_ls_xpix=655586, - application_x_vrml=655587, - application_x_wais_source=655588, - application_x_wine_extension_ini=655589, - application_x_wintalk=655590, - application_x_world=655591, - application_x_wri=655592, - application_x_x509_ca_cert=655593, - application_x_xz=655594 | 0x08000000, - application_x_zip=655595, - application_x_zstd=655596 | 0x08000000, - application_x_zstd_dictionary=655597, - application_xml=655598, - application_zip=655599 | 0x10000000, - application_zlib=655600, - audio_basic=458993 | 0x80000000, - audio_it=458994, - audio_make=458995, - audio_mid=458996, - audio_midi=458997, - audio_mp4=458998, - audio_mpeg=458999, - audio_ogg=459000, - audio_s3m=459001, - audio_tsp_audio=459002, - audio_tsplayer=459003, - audio_vnd_qcelp=459004, - audio_voxware=459005, - audio_x_aiff=459006, - audio_x_flac=459007, - audio_x_gsm=459008, - audio_x_hx_aac_adts=459009, - audio_x_jam=459010, - audio_x_liveaudio=459011, - audio_x_m4a=459012, - audio_x_midi=459013, - audio_x_mod=459014, - audio_x_mp4a_latm=459015, - audio_x_mpeg_3=459016, - audio_x_mpequrl=459017, - audio_x_nspaudio=459018, - audio_x_pn_realaudio=459019, - audio_x_psid=459020, - audio_x_realaudio=459021, - audio_x_s3m=459022, - audio_x_twinvq=459023, - audio_x_twinvq_plugin=459024, - audio_x_voc=459025, - audio_x_wav=459026, - audio_x_xbox_executable=459027 | 0x80000000, - audio_x_xbox360_executable=459028 | 0x80000000, - audio_xm=459029, - font_otf=327958 | 0x20000000, - font_sfnt=327959 | 0x20000000, - font_woff=327960 | 0x20000000, - font_woff2=327961 | 0x20000000, - image_bmp=524570, - image_cmu_raster=524571, - image_fif=524572, - image_florian=524573, - image_g3fax=524574, - image_gif=524575, - image_heic=524576, - image_ief=524577, - image_jpeg=524578, - image_jutvision=524579, - image_naplps=524580, - image_pict=524581, - image_png=524582, - image_svg=524583 | 0x80000000, - image_svg_xml=524584 | 0x80000000, - image_tiff=524585, - image_vnd_adobe_photoshop=524586 | 0x80000000, - image_vnd_djvu=524587 | 0x80000000, - image_vnd_fpx=524588, - image_vnd_microsoft_icon=524589, - image_vnd_rn_realflash=524590, - image_vnd_rn_realpix=524591, - image_vnd_wap_wbmp=524592, - image_vnd_xiff=524593, - image_webp=524594, - image_wmf=524595, - image_x_3ds=524596, - image_x_adobe_dng=524597 | 0x00800000, - image_x_award_bioslogo=524598, - image_x_canon_cr2=524599 | 0x00800000, - image_x_canon_crw=524600 | 0x00800000, - image_x_cmu_raster=524601, - image_x_cur=524602, - image_x_dcraw=524603 | 0x00800000, - image_x_dwg=524604, - image_x_eps=524605, - image_x_epson_erf=524606 | 0x00800000, - image_x_exr=524607, - image_x_fuji_raf=524608 | 0x00800000, - image_x_gem=524609, - image_x_icns=524610, - image_x_icon=524611 | 0x80000000, - image_x_jg=524612, - image_x_jps=524613, - image_x_kodak_dcr=524614 | 0x00800000, - image_x_kodak_k25=524615 | 0x00800000, - image_x_kodak_kdc=524616 | 0x00800000, - image_x_minolta_mrw=524617 | 0x00800000, - image_x_ms_bmp=524618, - image_x_niff=524619, - image_x_nikon_nef=524620 | 0x00800000, - image_x_olympus_orf=524621 | 0x00800000, - image_x_panasonic_raw=524622 | 0x00800000, - image_x_pcx=524623, - image_x_pentax_pef=524624 | 0x00800000, - image_x_pict=524625, - image_x_portable_bitmap=524626, - image_x_portable_graymap=524627, - image_x_portable_pixmap=524628, - image_x_quicktime=524629, - image_x_rgb=524630, - image_x_sigma_x3f=524631 | 0x00800000, - image_x_sony_arw=524632 | 0x00800000, - image_x_sony_sr2=524633 | 0x00800000, - image_x_sony_srf=524634 | 0x00800000, - image_x_tga=524635, - image_x_tiff=524636, - image_x_win_bitmap=524637, - image_x_xcf=524638 | 0x80000000, - image_x_xpixmap=524639 | 0x80000000, - image_x_xwindowdump=524640, - message_news=196961, - message_rfc822=196962, - model_vnd_dwf=65891, - model_vnd_gdl=65892, - model_vnd_gs_gdl=65893, - model_vrml=65894, - model_x_pov=65895, - sist2_sidecar=2, - text_PGP=590184, - text_asp=590185, - text_css=590186, - text_html=590187 | 0x01000000, - text_javascript=590188, - text_mcf=590189, - text_pascal=590190, - text_plain=590191, - text_richtext=590192, - text_rtf=590193, - text_scriplet=590194, - text_tab_separated_values=590195, - text_troff=590196, - text_uri_list=590197, - text_vnd_abc=590198, - text_vnd_fmi_flexstor=590199, - text_vnd_wap_wml=590200, - text_vnd_wap_wmlscript=590201, - text_webviewhtml=590202, - text_x_Algol68=590203, - text_x_asm=590204, - text_x_audiosoft_intra=590205, - text_x_awk=590206, - text_x_bcpl=590207, - text_x_c=590208, - text_x_c__=590209, - text_x_component=590210, - text_x_diff=590211, - text_x_fortran=590212, - text_x_java=590213, - text_x_la_asf=590214, - text_x_lisp=590215, - text_x_m=590216, - text_x_m4=590217, - text_x_makefile=590218, - text_x_ms_regedit=590219, - text_x_msdos_batch=590220, - text_x_objective_c=590221, - text_x_pascal=590222, - text_x_perl=590223, - text_x_php=590224, - text_x_po=590225, - text_x_python=590226, - text_x_ruby=590227, - text_x_sass=590228, - text_x_scss=590229, - text_x_server_parsed_html=590230, - text_x_setext=590231, - text_x_sgml=590232 | 0x01000000, - text_x_shellscript=590233, - text_x_speech=590234, - text_x_tcl=590235, - text_x_tex=590236, - text_x_uil=590237, - text_x_uuencode=590238, - text_x_vcalendar=590239, - text_x_vcard=590240, - text_xml=590241 | 0x01000000, - video_MP2T=393634, - video_animaflex=393635, - video_avi=393636, - video_avs_video=393637, - video_mp4=393638, - video_mpeg=393639, - video_quicktime=393640, - video_vdo=393641, - video_vivo=393642, - video_vnd_rn_realvideo=393643, - video_vosaic=393644, - video_webm=393645, - video_x_amt_demorun=393646, - video_x_amt_showrun=393647, - video_x_atomic3d_feature=393648, - video_x_dl=393649, - video_x_dv=393650, - video_x_fli=393651, - video_x_flv=393652, - video_x_isvideo=393653, - video_x_jng=393654 | 0x80000000, - video_x_m4v=393655, - video_x_matroska=393656, - video_x_mng=393657, - video_x_motion_jpeg=393658, - video_x_ms_asf=393659, - video_x_msvideo=393660, - video_x_qtc=393661, - video_x_sgi_movie=393662, - x_epoc_x_sisx_app=721343, +application_CDFV2=655361, +application_CDFV2_corrupt=655362, +application_arj=655363, +application_base64=655364, +application_binhex=655365, +application_book=655366, +application_clariscad=655367, +application_commonground=655368, +application_csv=655369, +application_dicom=655370, +application_drafting=655371, +application_epub_zip=655372 | 0x40000000, +application_freeloader=655373, +application_futuresplash=655374, +application_groupwise=655375, +application_gzip=655376 | 0x08000000, +application_hta=655377, +application_i_deas=655378, +application_iges=655379, +application_inf=655380, +application_java=655381, +application_java_archive=655382, +application_javascript=655383, +application_json=655384, +application_marc=655385, +application_mbedlet=655386, +application_mime=655387, +application_mspowerpoint=655388, +application_msword=655389, +application_ndjson=655390, +application_netmc=655391, +application_octet_stream=655392, +application_oda=655393, +application_ogg=655394, +application_pdf=655395 | 0x40000000, +application_pgp_keys=655396, +application_pgp_signature=655397, +application_pkcs7_signature=655398, +application_pkix_cert=655399, +application_postscript=655400, +application_pro_eng=655401, +application_ringing_tones=655402, +application_smil=655403, +application_solids=655404, +application_sounder=655405, +application_step=655406, +application_streamingmedia=655407, +application_vda=655408, +application_vnd_amazon_mobi8_ebook=655409 | 0x02000000, +application_vnd_coffeescript=655410, +application_vnd_fdf=655411, +application_vnd_font_fontforge_sfd=655412, +application_vnd_hp_hpgl=655413, +application_vnd_iccprofile=655414, +application_vnd_lotus_1_2_3=655415, +application_vnd_ms_cab_compressed=655416, +application_vnd_ms_excel=655417, +application_vnd_ms_fontobject=655418, +application_vnd_ms_opentype=655419 | 0x20000000, +application_vnd_ms_outlook=655420, +application_vnd_ms_pki_certstore=655421, +application_vnd_ms_pki_pko=655422, +application_vnd_ms_pki_seccat=655423, +application_vnd_ms_powerpoint=655424, +application_vnd_ms_project=655425, +application_vnd_oasis_opendocument_base=655426, +application_vnd_oasis_opendocument_formula=655427, +application_vnd_oasis_opendocument_graphics=655428, +application_vnd_oasis_opendocument_presentation=655429, +application_vnd_oasis_opendocument_spreadsheet=655430, +application_vnd_oasis_opendocument_text=655431, +application_vnd_openxmlformats_officedocument_presentationml_presentation=655432 | 0x04000000, +application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655433 | 0x04000000, +application_vnd_openxmlformats_officedocument_wordprocessingml_document=655434 | 0x04000000, +application_vnd_symbian_install=655435, +application_vnd_tcpdump_pcap=655436, +application_vnd_wap_wmlc=655437, +application_vnd_wap_wmlscriptc=655438, +application_vnd_xara=655439, +application_vocaltec_media_desc=655440, +application_vocaltec_media_file=655441, +application_warc=655442, +application_winhelp=655443, +application_wordperfect=655444, +application_x_123=655445, +application_x_7z_compressed=655446 | 0x10000000, +application_x_aim=655447, +application_x_apple_diskimage=655448, +application_x_arc=655449 | 0x10000000, +application_x_archive=655450, +application_x_atari_7800_rom=655451, +application_x_authorware_bin=655452, +application_x_authorware_map=655453, +application_x_authorware_seg=655454, +application_x_avira_qua=655455, +application_x_bcpio=655456, +application_x_bittorrent=655457, +application_x_bsh=655458, +application_x_bytecode_python=655459, +application_x_bzip=655460, +application_x_bzip2=655461 | 0x08000000, +application_x_cbr=655462, +application_x_cbz=655463, +application_x_cdlink=655464, +application_x_chat=655465, +application_x_chrome_extension=655466, +application_x_cocoa=655467, +application_x_conference=655468, +application_x_coredump=655469, +application_x_cpio=655470, +application_x_dbf=655471, +application_x_dbt=655472, +application_x_debian_package=655473, +application_x_deepv=655474, +application_x_director=655475, +application_x_dmp=655476, +application_x_dosdriver=655477, +application_x_dosexec=655478, +application_x_dvi=655479, +application_x_elc=655480, +application_x_empty=1, +application_x_envoy=655481, +application_x_esrehber=655482, +application_x_excel=655483, +application_x_executable=655484, +application_x_font_gdos=655485, +application_x_font_pf2=655486, +application_x_font_pfm=655487, +application_x_font_sfn=655488, +application_x_font_ttf=655489 | 0x20000000, +application_x_fptapplication_x_dbt=655490, +application_x_freelance=655491, +application_x_gamecube_rom=655492, +application_x_gdbm=655493, +application_x_gettext_translation=655494, +application_x_git=655495, +application_x_gsp=655496, +application_x_gss=655497, +application_x_gtar=655498, +application_x_gzip=655499, +application_x_hdf=655500, +application_x_helpfile=655501, +application_x_httpd_imap=655502, +application_x_ima=655503, +application_x_innosetup=655504, +application_x_internett_signup=655505, +application_x_inventor=655506, +application_x_ip2=655507, +application_x_java_applet=655508, +application_x_java_commerce=655509, +application_x_java_image=655510, +application_x_java_jmod=655511, +application_x_java_keystore=655512, +application_x_kdelnk=655513, +application_x_koan=655514, +application_x_latex=655515, +application_x_livescreen=655516, +application_x_lotus=655517, +application_x_lz4=655518 | 0x08000000, +application_x_lz4_json=655519, +application_x_lzh=655520, +application_x_lzh_compressed=655521, +application_x_lzip=655522 | 0x08000000, +application_x_lzma=655523 | 0x08000000, +application_x_lzop=655524 | 0x08000000, +application_x_lzx=655525, +application_x_mach_binary=655526, +application_x_mach_executable=655527, +application_x_magic_cap_package_1_0=655528, +application_x_mathcad=655529, +application_x_maxis_dbpf=655530, +application_x_meme=655531, +application_x_midi=655532, +application_x_mif=655533, +application_x_mix_transfer=655534, +application_x_mobipocket_ebook=655535 | 0x02000000, +application_x_ms_compress_szdd=655536, +application_x_ms_pdb=655537, +application_x_ms_reader=655538, +application_x_msaccess=655539, +application_x_n64_rom=655540, +application_x_navi_animation=655541, +application_x_navidoc=655542, +application_x_navimap=655543, +application_x_navistyle=655544, +application_x_nes_rom=655545, +application_x_netcdf=655546, +application_x_newton_compatible_pkg=655547, +application_x_nintendo_ds_rom=655548, +application_x_object=655549, +application_x_omc=655550, +application_x_omcdatamaker=655551, +application_x_omcregerator=655552, +application_x_pagemaker=655553, +application_x_pcl=655554, +application_x_pgp_keyring=655555, +application_x_pixclscript=655556, +application_x_pkcs7_certreqresp=655557, +application_x_pkcs7_signature=655558, +application_x_project=655559, +application_x_qpro=655560, +application_x_rar=655561 | 0x10000000, +application_x_rpm=655562, +application_x_sdp=655563, +application_x_sea=655564, +application_x_seelogo=655565, +application_x_setupscript=655566, +application_x_shar=655567, +application_x_sharedlib=655568, +application_x_shockwave_flash=655569, +application_x_snappy_framed=655570, +application_x_sprite=655571, +application_x_sqlite3=655572, +application_x_stargallery_thm=655573, +application_x_stuffit=655574, +application_x_sv4cpio=655575, +application_x_sv4crc=655576, +application_x_tar=655577 | 0x10000000, +application_x_tbook=655578, +application_x_terminfo=655579, +application_x_terminfo2=655580, +application_x_tex_tfm=655581, +application_x_texinfo=655582, +application_x_ustar=655583, +application_x_visio=655584, +application_x_vnd_audioexplosion_mzz=655585, +application_x_vnd_ls_xpix=655586, +application_x_vrml=655587, +application_x_wais_source=655588, +application_x_wine_extension_ini=655589, +application_x_wintalk=655590, +application_x_world=655591, +application_x_wri=655592, +application_x_x509_ca_cert=655593, +application_x_xz=655594 | 0x08000000, +application_x_zip=655595, +application_x_zstd=655596 | 0x08000000, +application_x_zstd_dictionary=655597, +application_xml=655598, +application_zip=655599 | 0x10000000, +application_zlib=655600, +audio_basic=458993 | 0x80000000, +audio_it=458994, +audio_make=458995, +audio_mid=458996, +audio_midi=458997, +audio_mp4=458998, +audio_mpeg=458999, +audio_ogg=459000, +audio_s3m=459001, +audio_tsp_audio=459002, +audio_tsplayer=459003, +audio_vnd_qcelp=459004, +audio_voxware=459005, +audio_x_aiff=459006, +audio_x_flac=459007, +audio_x_gsm=459008, +audio_x_hx_aac_adts=459009, +audio_x_jam=459010, +audio_x_liveaudio=459011, +audio_x_m4a=459012, +audio_x_midi=459013, +audio_x_mod=459014, +audio_x_mp4a_latm=459015, +audio_x_mpeg_3=459016, +audio_x_mpequrl=459017, +audio_x_nspaudio=459018, +audio_x_pn_realaudio=459019, +audio_x_psid=459020, +audio_x_realaudio=459021, +audio_x_s3m=459022, +audio_x_twinvq=459023, +audio_x_twinvq_plugin=459024, +audio_x_voc=459025, +audio_x_wav=459026, +audio_x_xbox_executable=459027 | 0x80000000, +audio_x_xbox360_executable=459028 | 0x80000000, +audio_xm=459029, +font_otf=327958 | 0x20000000, +font_sfnt=327959 | 0x20000000, +font_woff=327960 | 0x20000000, +font_woff2=327961 | 0x20000000, +image_bmp=524570, +image_cmu_raster=524571, +image_fif=524572, +image_florian=524573, +image_g3fax=524574, +image_gif=524575, +image_heic=524576, +image_ief=524577, +image_jpeg=524578, +image_jutvision=524579, +image_naplps=524580, +image_pict=524581, +image_png=524582, +image_svg=524583 | 0x80000000, +image_svg_xml=524584 | 0x80000000, +image_tiff=524585, +image_vnd_adobe_photoshop=524586 | 0x80000000, +image_vnd_djvu=524587 | 0x80000000, +image_vnd_fpx=524588, +image_vnd_microsoft_icon=524589, +image_vnd_rn_realflash=524590, +image_vnd_rn_realpix=524591, +image_vnd_wap_wbmp=524592, +image_vnd_xiff=524593, +image_webp=524594, +image_wmf=524595, +image_x_3ds=524596, +image_x_adobe_dng=524597 | 0x00800000, +image_x_award_bioslogo=524598, +image_x_canon_cr2=524599 | 0x00800000, +image_x_canon_crw=524600 | 0x00800000, +image_x_cmu_raster=524601, +image_x_cur=524602, +image_x_dcraw=524603 | 0x00800000, +image_x_dwg=524604, +image_x_eps=524605, +image_x_epson_erf=524606 | 0x00800000, +image_x_exr=524607, +image_x_fuji_raf=524608 | 0x00800000, +image_x_gem=524609, +image_x_icns=524610, +image_x_icon=524611 | 0x80000000, +image_x_jg=524612, +image_x_jps=524613, +image_x_kodak_dcr=524614 | 0x00800000, +image_x_kodak_k25=524615 | 0x00800000, +image_x_kodak_kdc=524616 | 0x00800000, +image_x_minolta_mrw=524617 | 0x00800000, +image_x_ms_bmp=524618, +image_x_niff=524619, +image_x_nikon_nef=524620 | 0x00800000, +image_x_olympus_orf=524621 | 0x00800000, +image_x_panasonic_raw=524622 | 0x00800000, +image_x_pcx=524623, +image_x_pentax_pef=524624 | 0x00800000, +image_x_pict=524625, +image_x_portable_bitmap=524626, +image_x_portable_graymap=524627, +image_x_portable_pixmap=524628, +image_x_quicktime=524629, +image_x_rgb=524630, +image_x_sigma_x3f=524631 | 0x00800000, +image_x_sony_arw=524632 | 0x00800000, +image_x_sony_sr2=524633 | 0x00800000, +image_x_sony_srf=524634 | 0x00800000, +image_x_tga=524635, +image_x_tiff=524636, +image_x_win_bitmap=524637, +image_x_xcf=524638 | 0x80000000, +image_x_xpixmap=524639 | 0x80000000, +image_x_xwindowdump=524640, +message_news=196961, +message_rfc822=196962, +model_vnd_dwf=65891, +model_vnd_gdl=65892, +model_vnd_gs_gdl=65893, +model_vrml=65894, +model_x_pov=65895, +sist2_sidecar=2, +text_PGP=590184, +text_asp=590185, +text_css=590186, +text_html=590187 | 0x01000000, +text_javascript=590188, +text_mcf=590189, +text_pascal=590190, +text_plain=590191, +text_richtext=590192, +text_rtf=590193, +text_scriplet=590194, +text_tab_separated_values=590195, +text_troff=590196, +text_uri_list=590197, +text_vnd_abc=590198, +text_vnd_fmi_flexstor=590199, +text_vnd_wap_wml=590200, +text_vnd_wap_wmlscript=590201, +text_webviewhtml=590202, +text_x_Algol68=590203, +text_x_asm=590204, +text_x_audiosoft_intra=590205, +text_x_awk=590206, +text_x_bcpl=590207, +text_x_c=590208, +text_x_c__=590209, +text_x_component=590210, +text_x_diff=590211, +text_x_fortran=590212, +text_x_java=590213, +text_x_la_asf=590214, +text_x_lisp=590215, +text_x_m=590216, +text_x_m4=590217, +text_x_makefile=590218, +text_x_ms_regedit=590219, +text_x_msdos_batch=590220, +text_x_objective_c=590221, +text_x_pascal=590222, +text_x_perl=590223, +text_x_php=590224, +text_x_po=590225, +text_x_python=590226, +text_x_ruby=590227, +text_x_sass=590228, +text_x_scss=590229, +text_x_server_parsed_html=590230, +text_x_setext=590231, +text_x_sgml=590232 | 0x01000000, +text_x_shellscript=590233, +text_x_speech=590234, +text_x_tcl=590235, +text_x_tex=590236, +text_x_uil=590237, +text_x_uuencode=590238, +text_x_vcalendar=590239, +text_x_vcard=590240, +text_xml=590241 | 0x01000000, +video_MP2T=393634, +video_animaflex=393635, +video_avi=393636, +video_avs_video=393637, +video_mp4=393638, +video_mpeg=393639, +video_quicktime=393640, +video_vdo=393641, +video_vivo=393642, +video_vnd_rn_realvideo=393643, +video_vosaic=393644, +video_webm=393645, +video_x_amt_demorun=393646, +video_x_amt_showrun=393647, +video_x_atomic3d_feature=393648, +video_x_dl=393649, +video_x_dv=393650, +video_x_fli=393651, +video_x_flv=393652, +video_x_isvideo=393653, +video_x_jng=393654 | 0x80000000, +video_x_m4v=393655, +video_x_matroska=393656, +video_x_mng=393657, +video_x_motion_jpeg=393658, +video_x_ms_asf=393659, +video_x_msvideo=393660, +video_x_qtc=393661, +video_x_sgi_movie=393662, +x_epoc_x_sisx_app=721343, }; char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) { case application_arj: return "application/arj"; @@ -907,1001 +905,837 @@ case image_x_sony_srf: return "image/x-sony-srf"; case image_x_epson_erf: return "image/x-epson-erf"; case sist2_sidecar: return "sist2/sidecar"; default: return NULL;}} -GHashTable *mime_get_ext_table() {GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal); -g_hash_table_insert(ext_table, "arj", (gpointer)application_arj); -g_hash_table_insert(ext_table, "mme", (gpointer)application_base64); -g_hash_table_insert(ext_table, "hqx", (gpointer)application_binhex); -g_hash_table_insert(ext_table, "boo", (gpointer)application_book); -g_hash_table_insert(ext_table, "book", (gpointer)application_book); -g_hash_table_insert(ext_table, "sdv", (gpointer)application_CDFV2); -g_hash_table_insert(ext_table, "ccad", (gpointer)application_clariscad); -g_hash_table_insert(ext_table, "dp", (gpointer)application_commonground); -g_hash_table_insert(ext_table, "dcm", (gpointer)application_dicom); -g_hash_table_insert(ext_table, "drw", (gpointer)application_drafting); -g_hash_table_insert(ext_table, "epub", (gpointer)application_epub_zip); -g_hash_table_insert(ext_table, "frl", (gpointer)application_freeloader); -g_hash_table_insert(ext_table, "spl", (gpointer)application_futuresplash); -g_hash_table_insert(ext_table, "vew", (gpointer)application_groupwise); -g_hash_table_insert(ext_table, "gz", (gpointer)application_gzip); -g_hash_table_insert(ext_table, "tgz", (gpointer)application_gzip); -g_hash_table_insert(ext_table, "hta", (gpointer)application_hta); -g_hash_table_insert(ext_table, "unv", (gpointer)application_i_deas); -g_hash_table_insert(ext_table, "iges", (gpointer)application_iges); -g_hash_table_insert(ext_table, "igs", (gpointer)application_iges); -g_hash_table_insert(ext_table, "inf", (gpointer)application_inf); -g_hash_table_insert(ext_table, "jar", (gpointer)application_java_archive); -g_hash_table_insert(ext_table, "class", (gpointer)application_java); -g_hash_table_insert(ext_table, "json", (gpointer)application_json); -g_hash_table_insert(ext_table, "jsonl", (gpointer)application_ndjson); -g_hash_table_insert(ext_table, "ndjson", (gpointer)application_ndjson); -g_hash_table_insert(ext_table, "mrc", (gpointer)application_marc); -g_hash_table_insert(ext_table, "mbd", (gpointer)application_mbedlet); -g_hash_table_insert(ext_table, "aps", (gpointer)application_mime); -g_hash_table_insert(ext_table, "ppz", (gpointer)application_mspowerpoint); -g_hash_table_insert(ext_table, "doc", (gpointer)application_msword); -g_hash_table_insert(ext_table, "dot", (gpointer)application_msword); -g_hash_table_insert(ext_table, "w6w", (gpointer)application_msword); -g_hash_table_insert(ext_table, "wiz", (gpointer)application_msword); -g_hash_table_insert(ext_table, "word", (gpointer)application_msword); -g_hash_table_insert(ext_table, "mcp", (gpointer)application_netmc); -g_hash_table_insert(ext_table, "bin", (gpointer)application_octet_stream); -g_hash_table_insert(ext_table, "dump", (gpointer)application_octet_stream); -g_hash_table_insert(ext_table, "gpg", (gpointer)application_octet_stream); -g_hash_table_insert(ext_table, "oda", (gpointer)application_oda); -g_hash_table_insert(ext_table, "ogv", (gpointer)application_ogg); -g_hash_table_insert(ext_table, "pdf", (gpointer)application_pdf); -g_hash_table_insert(ext_table, "pgp", (gpointer)application_pgp_signature); -g_hash_table_insert(ext_table, "p7s", (gpointer)application_pkcs7_signature); -g_hash_table_insert(ext_table, "cer", (gpointer)application_pkix_cert); -g_hash_table_insert(ext_table, "crt", (gpointer)application_pkix_cert); -g_hash_table_insert(ext_table, "ai", (gpointer)application_postscript); -g_hash_table_insert(ext_table, "ps", (gpointer)application_postscript); -g_hash_table_insert(ext_table, "part", (gpointer)application_pro_eng); -g_hash_table_insert(ext_table, "prt", (gpointer)application_pro_eng); -g_hash_table_insert(ext_table, "rng", (gpointer)application_ringing_tones); -g_hash_table_insert(ext_table, "smi", (gpointer)application_smil); -g_hash_table_insert(ext_table, "smil", (gpointer)application_smil); -g_hash_table_insert(ext_table, "sol", (gpointer)application_solids); -g_hash_table_insert(ext_table, "sdr", (gpointer)application_sounder); -g_hash_table_insert(ext_table, "step", (gpointer)application_step); -g_hash_table_insert(ext_table, "stp", (gpointer)application_step); -g_hash_table_insert(ext_table, "ssm", (gpointer)application_streamingmedia); -g_hash_table_insert(ext_table, "vda", (gpointer)application_vda); -g_hash_table_insert(ext_table, "fdf", (gpointer)application_vnd_fdf); -g_hash_table_insert(ext_table, "sfd", (gpointer)application_vnd_font_fontforge_sfd); -g_hash_table_insert(ext_table, "hgl", (gpointer)application_vnd_hp_hpgl); -g_hash_table_insert(ext_table, "hpg", (gpointer)application_vnd_hp_hpgl); -g_hash_table_insert(ext_table, "hpgl", (gpointer)application_vnd_hp_hpgl); -g_hash_table_insert(ext_table, "icm", (gpointer)application_vnd_iccprofile); -g_hash_table_insert(ext_table, "cab", (gpointer)application_vnd_ms_cab_compressed); -g_hash_table_insert(ext_table, "xlb", (gpointer)application_vnd_ms_excel); -g_hash_table_insert(ext_table, "xlc", (gpointer)application_vnd_ms_excel); -g_hash_table_insert(ext_table, "xll", (gpointer)application_vnd_ms_excel); -g_hash_table_insert(ext_table, "xlm", (gpointer)application_vnd_ms_excel); -g_hash_table_insert(ext_table, "xls", (gpointer)application_vnd_ms_excel); -g_hash_table_insert(ext_table, "xlw", (gpointer)application_vnd_ms_excel); -g_hash_table_insert(ext_table, "eot", (gpointer)application_vnd_ms_fontobject); -g_hash_table_insert(ext_table, "otf", (gpointer)application_vnd_ms_opentype); -g_hash_table_insert(ext_table, "sst", (gpointer)application_vnd_ms_pki_certstore); -g_hash_table_insert(ext_table, "pko", (gpointer)application_vnd_ms_pki_pko); -g_hash_table_insert(ext_table, "cat", (gpointer)application_vnd_ms_pki_seccat); -g_hash_table_insert(ext_table, "pot", (gpointer)application_vnd_ms_powerpoint); -g_hash_table_insert(ext_table, "ppa", (gpointer)application_vnd_ms_powerpoint); -g_hash_table_insert(ext_table, "pps", (gpointer)application_vnd_ms_powerpoint); -g_hash_table_insert(ext_table, "ppt", (gpointer)application_vnd_ms_powerpoint); -g_hash_table_insert(ext_table, "pwz", (gpointer)application_vnd_ms_powerpoint); -g_hash_table_insert(ext_table, "mpp", (gpointer)application_vnd_ms_project); -g_hash_table_insert(ext_table, "odb", (gpointer)application_vnd_oasis_opendocument_base); -g_hash_table_insert(ext_table, "odf", (gpointer)application_vnd_oasis_opendocument_formula); -g_hash_table_insert(ext_table, "odg", (gpointer)application_vnd_oasis_opendocument_graphics); -g_hash_table_insert(ext_table, "odp", (gpointer)application_vnd_oasis_opendocument_presentation); -g_hash_table_insert(ext_table, "ods", (gpointer)application_vnd_oasis_opendocument_spreadsheet); -g_hash_table_insert(ext_table, "odt", (gpointer)application_vnd_oasis_opendocument_text); -g_hash_table_insert(ext_table, "pptx", (gpointer)application_vnd_openxmlformats_officedocument_presentationml_presentation); -g_hash_table_insert(ext_table, "xlsx", (gpointer)application_vnd_openxmlformats_officedocument_spreadsheetml_sheet); -g_hash_table_insert(ext_table, "docx", (gpointer)application_vnd_openxmlformats_officedocument_wordprocessingml_document); -g_hash_table_insert(ext_table, "pcap", (gpointer)application_vnd_tcpdump_pcap); -g_hash_table_insert(ext_table, "wmlc", (gpointer)application_vnd_wap_wmlc); -g_hash_table_insert(ext_table, "wmlsc", (gpointer)application_vnd_wap_wmlscriptc); -g_hash_table_insert(ext_table, "web", (gpointer)application_vnd_xara); -g_hash_table_insert(ext_table, "vmd", (gpointer)application_vocaltec_media_desc); -g_hash_table_insert(ext_table, "vmf", (gpointer)application_vocaltec_media_file); -g_hash_table_insert(ext_table, "warc", (gpointer)application_warc); -g_hash_table_insert(ext_table, "hlp", (gpointer)application_winhelp); -g_hash_table_insert(ext_table, "wp", (gpointer)application_wordperfect); -g_hash_table_insert(ext_table, "wp5", (gpointer)application_wordperfect); -g_hash_table_insert(ext_table, "wp6", (gpointer)application_wordperfect); -g_hash_table_insert(ext_table, "wpd", (gpointer)application_wordperfect); -g_hash_table_insert(ext_table, "w60", (gpointer)application_wordperfect); -g_hash_table_insert(ext_table, "w61", (gpointer)application_wordperfect); -g_hash_table_insert(ext_table, "wk1", (gpointer)application_x_123); -g_hash_table_insert(ext_table, "7z", (gpointer)application_x_7z_compressed); -g_hash_table_insert(ext_table, "aim", (gpointer)application_x_aim); -g_hash_table_insert(ext_table, "a", (gpointer)application_x_archive); -g_hash_table_insert(ext_table, "a78", (gpointer)application_x_atari_7800_rom); -g_hash_table_insert(ext_table, "aab", (gpointer)application_x_authorware_bin); -g_hash_table_insert(ext_table, "aam", (gpointer)application_x_authorware_map); -g_hash_table_insert(ext_table, "aas", (gpointer)application_x_authorware_seg); -g_hash_table_insert(ext_table, "bcpio", (gpointer)application_x_bcpio); -g_hash_table_insert(ext_table, "torrent", (gpointer)application_x_bittorrent); -g_hash_table_insert(ext_table, "bsh", (gpointer)application_x_bsh); -g_hash_table_insert(ext_table, "pyc", (gpointer)application_x_bytecode_python); -g_hash_table_insert(ext_table, "boz", (gpointer)application_x_bzip2); -g_hash_table_insert(ext_table, "bz2", (gpointer)application_x_bzip2); -g_hash_table_insert(ext_table, "bz", (gpointer)application_x_bzip); -g_hash_table_insert(ext_table, "cbr", (gpointer)application_x_cbr); -g_hash_table_insert(ext_table, "cbz", (gpointer)application_x_cbz); -g_hash_table_insert(ext_table, "vcd", (gpointer)application_x_cdlink); -g_hash_table_insert(ext_table, "cha", (gpointer)application_x_chat); -g_hash_table_insert(ext_table, "chat", (gpointer)application_x_chat); -g_hash_table_insert(ext_table, "cco", (gpointer)application_x_cocoa); -g_hash_table_insert(ext_table, "nsc", (gpointer)application_x_conference); -g_hash_table_insert(ext_table, "cpio", (gpointer)application_x_cpio); -g_hash_table_insert(ext_table, "dbf", (gpointer)application_x_dbf); -g_hash_table_insert(ext_table, "deb", (gpointer)application_x_debian_package); -g_hash_table_insert(ext_table, "deepv", (gpointer)application_x_deepv); -g_hash_table_insert(ext_table, "dir", (gpointer)application_x_director); -g_hash_table_insert(ext_table, "dxr", (gpointer)application_x_director); -g_hash_table_insert(ext_table, "dmp", (gpointer)application_x_dmp); -g_hash_table_insert(ext_table, "dll", (gpointer)application_x_dosexec); -g_hash_table_insert(ext_table, "dvi", (gpointer)application_x_dvi); -g_hash_table_insert(ext_table, "elc", (gpointer)application_x_elc); -g_hash_table_insert(ext_table, "env", (gpointer)application_x_envoy); -g_hash_table_insert(ext_table, "evy", (gpointer)application_x_envoy); -g_hash_table_insert(ext_table, "es", (gpointer)application_x_esrehber); -g_hash_table_insert(ext_table, "xla", (gpointer)application_x_excel); -g_hash_table_insert(ext_table, "xld", (gpointer)application_x_excel); -g_hash_table_insert(ext_table, "xlk", (gpointer)application_x_excel); -g_hash_table_insert(ext_table, "xlt", (gpointer)application_x_excel); -g_hash_table_insert(ext_table, "xlv", (gpointer)application_x_excel); -g_hash_table_insert(ext_table, "exe", (gpointer)application_x_executable); -g_hash_table_insert(ext_table, "pf2", (gpointer)application_x_font_pf2); -g_hash_table_insert(ext_table, "pfm", (gpointer)application_x_font_pfm); -g_hash_table_insert(ext_table, "ttf", (gpointer)application_x_font_ttf); -g_hash_table_insert(ext_table, "ttc", (gpointer)application_x_font_ttf); -g_hash_table_insert(ext_table, "pre", (gpointer)application_x_freelance); -g_hash_table_insert(ext_table, "gsp", (gpointer)application_x_gsp); -g_hash_table_insert(ext_table, "gss", (gpointer)application_x_gss); -g_hash_table_insert(ext_table, "gtar", (gpointer)application_x_gtar); -g_hash_table_insert(ext_table, "gzip", (gpointer)application_x_gzip); -g_hash_table_insert(ext_table, "hdf", (gpointer)application_x_hdf); -g_hash_table_insert(ext_table, "help", (gpointer)application_x_helpfile); -g_hash_table_insert(ext_table, "imap", (gpointer)application_x_httpd_imap); -g_hash_table_insert(ext_table, "ima", (gpointer)application_x_ima); -g_hash_table_insert(ext_table, "ins", (gpointer)application_x_internett_signup); -g_hash_table_insert(ext_table, "iv", (gpointer)application_x_inventor); -g_hash_table_insert(ext_table, "ip", (gpointer)application_x_ip2); -g_hash_table_insert(ext_table, "jcm", (gpointer)application_x_java_commerce); -g_hash_table_insert(ext_table, "jmod", (gpointer)application_x_java_jmod); -g_hash_table_insert(ext_table, "skd", (gpointer)application_x_koan); -g_hash_table_insert(ext_table, "skm", (gpointer)application_x_koan); -g_hash_table_insert(ext_table, "skp", (gpointer)application_x_koan); -g_hash_table_insert(ext_table, "skt", (gpointer)application_x_koan); -g_hash_table_insert(ext_table, "latex", (gpointer)application_x_latex); -g_hash_table_insert(ext_table, "ltx", (gpointer)application_x_latex); -g_hash_table_insert(ext_table, "ivy", (gpointer)application_x_livescreen); -g_hash_table_insert(ext_table, "wq1", (gpointer)application_x_lotus); -g_hash_table_insert(ext_table, "jsonlz4", (gpointer)application_x_lz4_json); -g_hash_table_insert(ext_table, "lz4", (gpointer)application_x_lz4); -g_hash_table_insert(ext_table, "lzh", (gpointer)application_x_lzh); -g_hash_table_insert(ext_table, "lz", (gpointer)application_x_lzip); -g_hash_table_insert(ext_table, "lzma", (gpointer)application_x_lzma); -g_hash_table_insert(ext_table, "lzo", (gpointer)application_x_lzop); -g_hash_table_insert(ext_table, "lzx", (gpointer)application_x_lzx); -g_hash_table_insert(ext_table, "jnilib", (gpointer)application_x_mach_binary); -g_hash_table_insert(ext_table, "dylib", (gpointer)application_x_mach_binary); -g_hash_table_insert(ext_table, "mc$", (gpointer)application_x_magic_cap_package_1_0); -g_hash_table_insert(ext_table, "mcd", (gpointer)application_x_mathcad); -g_hash_table_insert(ext_table, "mm", (gpointer)application_x_meme); -g_hash_table_insert(ext_table, "midi", (gpointer)application_x_midi); -g_hash_table_insert(ext_table, "mif", (gpointer)application_x_mif); -g_hash_table_insert(ext_table, "nix", (gpointer)application_x_mix_transfer); -g_hash_table_insert(ext_table, "opf", (gpointer)application_xml); -g_hash_table_insert(ext_table, "mobi", (gpointer)application_x_mobipocket_ebook); -g_hash_table_insert(ext_table, "azw", (gpointer)application_vnd_amazon_mobi8_ebook); -g_hash_table_insert(ext_table, "azw3", (gpointer)application_vnd_amazon_mobi8_ebook); -g_hash_table_insert(ext_table, "accdb", (gpointer)application_x_msaccess); -g_hash_table_insert(ext_table, "fon", (gpointer)application_x_ms_compress_szdd); -g_hash_table_insert(ext_table, "pdb", (gpointer)application_x_ms_pdb); -g_hash_table_insert(ext_table, "lit", (gpointer)application_x_ms_reader); -g_hash_table_insert(ext_table, "z64", (gpointer)application_x_n64_rom); -g_hash_table_insert(ext_table, "ani", (gpointer)application_x_navi_animation); -g_hash_table_insert(ext_table, "nvd", (gpointer)application_x_navidoc); -g_hash_table_insert(ext_table, "map", (gpointer)application_x_navimap); -g_hash_table_insert(ext_table, "stl", (gpointer)application_x_navistyle); -g_hash_table_insert(ext_table, "nes", (gpointer)application_x_nes_rom); -g_hash_table_insert(ext_table, "cdf", (gpointer)application_x_netcdf); -g_hash_table_insert(ext_table, "nc", (gpointer)application_x_netcdf); -g_hash_table_insert(ext_table, "pkg", (gpointer)application_x_newton_compatible_pkg); -g_hash_table_insert(ext_table, "o", (gpointer)application_x_object); -g_hash_table_insert(ext_table, "omcd", (gpointer)application_x_omcdatamaker); -g_hash_table_insert(ext_table, "omc", (gpointer)application_x_omc); -g_hash_table_insert(ext_table, "omcr", (gpointer)application_x_omcregerator); -g_hash_table_insert(ext_table, "pm4", (gpointer)application_x_pagemaker); -g_hash_table_insert(ext_table, "pm5", (gpointer)application_x_pagemaker); -g_hash_table_insert(ext_table, "pcl", (gpointer)application_x_pcl); -g_hash_table_insert(ext_table, "plx", (gpointer)application_x_pixclscript); -g_hash_table_insert(ext_table, "p7r", (gpointer)application_x_pkcs7_certreqresp); -g_hash_table_insert(ext_table, "p7a", (gpointer)application_x_pkcs7_signature); -g_hash_table_insert(ext_table, "mpc", (gpointer)application_x_project); -g_hash_table_insert(ext_table, "mpt", (gpointer)application_x_project); -g_hash_table_insert(ext_table, "mpv", (gpointer)application_x_project); -g_hash_table_insert(ext_table, "mpx", (gpointer)application_x_project); -g_hash_table_insert(ext_table, "wb1", (gpointer)application_x_qpro); -g_hash_table_insert(ext_table, "rar", (gpointer)application_x_rar); -g_hash_table_insert(ext_table, "rpm", (gpointer)application_x_rpm); -g_hash_table_insert(ext_table, "sdp", (gpointer)application_x_sdp); -g_hash_table_insert(ext_table, "sea", (gpointer)application_x_sea); -g_hash_table_insert(ext_table, "sl", (gpointer)application_x_seelogo); -g_hash_table_insert(ext_table, "so", (gpointer)application_x_sharedlib); -g_hash_table_insert(ext_table, "shar", (gpointer)application_x_shar); -g_hash_table_insert(ext_table, "swf", (gpointer)application_x_shockwave_flash); -g_hash_table_insert(ext_table, "spr", (gpointer)application_x_sprite); -g_hash_table_insert(ext_table, "sprite", (gpointer)application_x_sprite); -g_hash_table_insert(ext_table, "sit", (gpointer)application_x_stuffit); -g_hash_table_insert(ext_table, "sv4cpio", (gpointer)application_x_sv4cpio); -g_hash_table_insert(ext_table, "sv4crc", (gpointer)application_x_sv4crc); -g_hash_table_insert(ext_table, "tar", (gpointer)application_x_tar); -g_hash_table_insert(ext_table, "sbk", (gpointer)application_x_tbook); -g_hash_table_insert(ext_table, "tbk", (gpointer)application_x_tbook); -g_hash_table_insert(ext_table, "texi", (gpointer)application_x_texinfo); -g_hash_table_insert(ext_table, "texinfo", (gpointer)application_x_texinfo); -g_hash_table_insert(ext_table, "tfm", (gpointer)application_x_tex_tfm); -g_hash_table_insert(ext_table, "ustar", (gpointer)application_x_ustar); -g_hash_table_insert(ext_table, "vsd", (gpointer)application_x_visio); -g_hash_table_insert(ext_table, "vst", (gpointer)application_x_visio); -g_hash_table_insert(ext_table, "vsw", (gpointer)application_x_visio); -g_hash_table_insert(ext_table, "mzz", (gpointer)application_x_vnd_audioexplosion_mzz); -g_hash_table_insert(ext_table, "xpix", (gpointer)application_x_vnd_ls_xpix); -g_hash_table_insert(ext_table, "vrml", (gpointer)application_x_vrml); -g_hash_table_insert(ext_table, "src", (gpointer)application_x_wais_source); -g_hash_table_insert(ext_table, "wsrc", (gpointer)application_x_wais_source); -g_hash_table_insert(ext_table, "wtk", (gpointer)application_x_wintalk); -g_hash_table_insert(ext_table, "svr", (gpointer)application_x_world); -g_hash_table_insert(ext_table, "wri", (gpointer)application_x_wri); -g_hash_table_insert(ext_table, "der", (gpointer)application_x_x509_ca_cert); -g_hash_table_insert(ext_table, "xz", (gpointer)application_x_xz); -g_hash_table_insert(ext_table, "zst", (gpointer)application_x_zstd); -g_hash_table_insert(ext_table, "zip", (gpointer)application_zip); -g_hash_table_insert(ext_table, "z", (gpointer)application_zlib); -g_hash_table_insert(ext_table, "au", (gpointer)audio_basic); -g_hash_table_insert(ext_table, "it", (gpointer)audio_it); -g_hash_table_insert(ext_table, "funk", (gpointer)audio_make); -g_hash_table_insert(ext_table, "my", (gpointer)audio_make); -g_hash_table_insert(ext_table, "pfunk", (gpointer)audio_make); -g_hash_table_insert(ext_table, "kar", (gpointer)audio_midi); -g_hash_table_insert(ext_table, "rmi", (gpointer)audio_mid); -g_hash_table_insert(ext_table, "m4b", (gpointer)audio_mp4); -g_hash_table_insert(ext_table, "m2a", (gpointer)audio_mpeg); -g_hash_table_insert(ext_table, "mpa", (gpointer)audio_mpeg); -g_hash_table_insert(ext_table, "ogg", (gpointer)audio_ogg); -g_hash_table_insert(ext_table, "s3m", (gpointer)audio_s3m); -g_hash_table_insert(ext_table, "tsi", (gpointer)audio_tsp_audio); -g_hash_table_insert(ext_table, "tsp", (gpointer)audio_tsplayer); -g_hash_table_insert(ext_table, "qcp", (gpointer)audio_vnd_qcelp); -g_hash_table_insert(ext_table, "vox", (gpointer)audio_voxware); -g_hash_table_insert(ext_table, "aiff", (gpointer)audio_x_aiff); -g_hash_table_insert(ext_table, "aif", (gpointer)audio_x_aiff); -g_hash_table_insert(ext_table, "flac", (gpointer)audio_x_flac); -g_hash_table_insert(ext_table, "gsd", (gpointer)audio_x_gsm); -g_hash_table_insert(ext_table, "gsm", (gpointer)audio_x_gsm); -g_hash_table_insert(ext_table, "jam", (gpointer)audio_x_jam); -g_hash_table_insert(ext_table, "lam", (gpointer)audio_x_liveaudio); -g_hash_table_insert(ext_table, "m4a", (gpointer)audio_x_m4a); -g_hash_table_insert(ext_table, "mid", (gpointer)audio_x_midi); -g_hash_table_insert(ext_table, "mp3", (gpointer)audio_x_mpeg_3); -g_hash_table_insert(ext_table, "xm", (gpointer)audio_xm); -g_hash_table_insert(ext_table, "lma", (gpointer)audio_x_nspaudio); -g_hash_table_insert(ext_table, "ram", (gpointer)audio_x_pn_realaudio); -g_hash_table_insert(ext_table, "rm", (gpointer)audio_x_pn_realaudio); -g_hash_table_insert(ext_table, "rmm", (gpointer)audio_x_pn_realaudio); -g_hash_table_insert(ext_table, "rmp", (gpointer)audio_x_pn_realaudio); -g_hash_table_insert(ext_table, "sid", (gpointer)audio_x_psid); -g_hash_table_insert(ext_table, "ra", (gpointer)audio_x_realaudio); -g_hash_table_insert(ext_table, "vqe", (gpointer)audio_x_twinvq_plugin); -g_hash_table_insert(ext_table, "vql", (gpointer)audio_x_twinvq_plugin); -g_hash_table_insert(ext_table, "vqf", (gpointer)audio_x_twinvq); -g_hash_table_insert(ext_table, "voc", (gpointer)audio_x_voc); -g_hash_table_insert(ext_table, "wav", (gpointer)audio_x_wav); -g_hash_table_insert(ext_table, "xex", (gpointer)audio_x_xbox360_executable); -g_hash_table_insert(ext_table, "xbe", (gpointer)audio_x_xbox_executable); -g_hash_table_insert(ext_table, "woff2", (gpointer)font_woff2); -g_hash_table_insert(ext_table, "woff", (gpointer)font_woff); -g_hash_table_insert(ext_table, "rast", (gpointer)image_cmu_raster); -g_hash_table_insert(ext_table, "fif", (gpointer)image_fif); -g_hash_table_insert(ext_table, "flo", (gpointer)image_florian); -g_hash_table_insert(ext_table, "turbot", (gpointer)image_florian); -g_hash_table_insert(ext_table, "g3", (gpointer)image_g3fax); -g_hash_table_insert(ext_table, "gif", (gpointer)image_gif); -g_hash_table_insert(ext_table, "heic", (gpointer)image_heic); -g_hash_table_insert(ext_table, "ief", (gpointer)image_ief); -g_hash_table_insert(ext_table, "iefs", (gpointer)image_ief); -g_hash_table_insert(ext_table, "jfif", (gpointer)image_jpeg); -g_hash_table_insert(ext_table, "jfif-tbnl", (gpointer)image_jpeg); -g_hash_table_insert(ext_table, "jpe", (gpointer)image_jpeg); -g_hash_table_insert(ext_table, "jpeg", (gpointer)image_jpeg); -g_hash_table_insert(ext_table, "jpg", (gpointer)image_jpeg); -g_hash_table_insert(ext_table, "jut", (gpointer)image_jutvision); -g_hash_table_insert(ext_table, "nap", (gpointer)image_naplps); -g_hash_table_insert(ext_table, "naplps", (gpointer)image_naplps); -g_hash_table_insert(ext_table, "pic", (gpointer)image_pict); -g_hash_table_insert(ext_table, "pict", (gpointer)image_pict); -g_hash_table_insert(ext_table, "png", (gpointer)image_png); -g_hash_table_insert(ext_table, "x-png", (gpointer)image_png); -g_hash_table_insert(ext_table, "svg", (gpointer)image_svg); -g_hash_table_insert(ext_table, "psd", (gpointer)image_vnd_adobe_photoshop); -g_hash_table_insert(ext_table, "djvu", (gpointer)image_vnd_djvu); -g_hash_table_insert(ext_table, "fpx", (gpointer)image_vnd_fpx); -g_hash_table_insert(ext_table, "rf", (gpointer)image_vnd_rn_realflash); -g_hash_table_insert(ext_table, "rp", (gpointer)image_vnd_rn_realpix); -g_hash_table_insert(ext_table, "wbmp", (gpointer)image_vnd_wap_wbmp); -g_hash_table_insert(ext_table, "xif", (gpointer)image_vnd_xiff); -g_hash_table_insert(ext_table, "webp", (gpointer)image_webp); -g_hash_table_insert(ext_table, "3ds", (gpointer)image_x_3ds); -g_hash_table_insert(ext_table, "ras", (gpointer)image_x_cmu_raster); -g_hash_table_insert(ext_table, "tga", (gpointer)image_x_cur); -g_hash_table_insert(ext_table, "dwg", (gpointer)image_x_dwg); -g_hash_table_insert(ext_table, "dxf", (gpointer)image_x_dwg); -g_hash_table_insert(ext_table, "svf", (gpointer)image_x_dwg); -g_hash_table_insert(ext_table, "exr", (gpointer)image_x_exr); -g_hash_table_insert(ext_table, "ico", (gpointer)image_x_icon); -g_hash_table_insert(ext_table, "art", (gpointer)image_x_jg); -g_hash_table_insert(ext_table, "jps", (gpointer)image_x_jps); -g_hash_table_insert(ext_table, "bm", (gpointer)image_x_ms_bmp); -g_hash_table_insert(ext_table, "bmp", (gpointer)image_x_ms_bmp); -g_hash_table_insert(ext_table, "nif", (gpointer)image_x_niff); -g_hash_table_insert(ext_table, "niff", (gpointer)image_x_niff); -g_hash_table_insert(ext_table, "pcx", (gpointer)image_x_pcx); -g_hash_table_insert(ext_table, "pct", (gpointer)image_x_pict); -g_hash_table_insert(ext_table, "pbm", (gpointer)image_x_portable_bitmap); -g_hash_table_insert(ext_table, "pgm", (gpointer)image_x_portable_graymap); -g_hash_table_insert(ext_table, "ppm", (gpointer)image_x_portable_pixmap); -g_hash_table_insert(ext_table, "qif", (gpointer)image_x_quicktime); -g_hash_table_insert(ext_table, "qti", (gpointer)image_x_quicktime); -g_hash_table_insert(ext_table, "qtif", (gpointer)image_x_quicktime); -g_hash_table_insert(ext_table, "rgb", (gpointer)image_x_rgb); -g_hash_table_insert(ext_table, "tif", (gpointer)image_x_tiff); -g_hash_table_insert(ext_table, "tiff", (gpointer)image_x_tiff); -g_hash_table_insert(ext_table, "xcf", (gpointer)image_x_xcf); -g_hash_table_insert(ext_table, "xpm", (gpointer)image_x_xpixmap); -g_hash_table_insert(ext_table, "xwd", (gpointer)image_x_xwindowdump); -g_hash_table_insert(ext_table, "mht", (gpointer)message_rfc822); -g_hash_table_insert(ext_table, "mhtml", (gpointer)message_rfc822); -g_hash_table_insert(ext_table, "mime", (gpointer)message_rfc822); -g_hash_table_insert(ext_table, "dwf", (gpointer)model_vnd_dwf); -g_hash_table_insert(ext_table, "gdl", (gpointer)model_vnd_gdl); -g_hash_table_insert(ext_table, "gdsl", (gpointer)model_vnd_gs_gdl); -g_hash_table_insert(ext_table, "wrz", (gpointer)model_vrml); -g_hash_table_insert(ext_table, "pov", (gpointer)model_x_pov); -g_hash_table_insert(ext_table, "asp", (gpointer)text_asp); -g_hash_table_insert(ext_table, "css", (gpointer)text_css); -g_hash_table_insert(ext_table, "acgi", (gpointer)text_html); -g_hash_table_insert(ext_table, "htm", (gpointer)text_html); -g_hash_table_insert(ext_table, "html", (gpointer)text_html); -g_hash_table_insert(ext_table, "htmls", (gpointer)text_html); -g_hash_table_insert(ext_table, "htx", (gpointer)text_html); -g_hash_table_insert(ext_table, "shtml", (gpointer)text_html); -g_hash_table_insert(ext_table, "js", (gpointer)text_javascript); -g_hash_table_insert(ext_table, "mcf", (gpointer)text_mcf); -g_hash_table_insert(ext_table, "pas", (gpointer)text_pascal); -g_hash_table_insert(ext_table, "com", (gpointer)text_plain); -g_hash_table_insert(ext_table, "cmd", (gpointer)text_plain); -g_hash_table_insert(ext_table, "conf", (gpointer)text_plain); -g_hash_table_insert(ext_table, "def", (gpointer)text_plain); -g_hash_table_insert(ext_table, "g", (gpointer)text_plain); -g_hash_table_insert(ext_table, "idc", (gpointer)text_plain); -g_hash_table_insert(ext_table, "list", (gpointer)text_plain); -g_hash_table_insert(ext_table, "lst", (gpointer)text_plain); -g_hash_table_insert(ext_table, "mar", (gpointer)text_plain); -g_hash_table_insert(ext_table, "sdml", (gpointer)text_plain); -g_hash_table_insert(ext_table, "text", (gpointer)text_plain); -g_hash_table_insert(ext_table, "txt", (gpointer)text_plain); -g_hash_table_insert(ext_table, "md", (gpointer)text_plain); -g_hash_table_insert(ext_table, "groovy", (gpointer)text_plain); -g_hash_table_insert(ext_table, "license", (gpointer)text_plain); -g_hash_table_insert(ext_table, "properties", (gpointer)text_plain); -g_hash_table_insert(ext_table, "desktop", (gpointer)text_plain); -g_hash_table_insert(ext_table, "ini", (gpointer)text_plain); -g_hash_table_insert(ext_table, "rst", (gpointer)text_plain); -g_hash_table_insert(ext_table, "cmake", (gpointer)text_plain); -g_hash_table_insert(ext_table, "ipynb", (gpointer)text_plain); -g_hash_table_insert(ext_table, "readme", (gpointer)text_plain); -g_hash_table_insert(ext_table, "less", (gpointer)text_plain); -g_hash_table_insert(ext_table, "lo", (gpointer)text_plain); -g_hash_table_insert(ext_table, "go", (gpointer)text_plain); -g_hash_table_insert(ext_table, "yml", (gpointer)text_plain); -g_hash_table_insert(ext_table, "d", (gpointer)text_plain); -g_hash_table_insert(ext_table, "cs", (gpointer)text_plain); -g_hash_table_insert(ext_table, "hpp", (gpointer)text_plain); -g_hash_table_insert(ext_table, "srt", (gpointer)text_plain); -g_hash_table_insert(ext_table, "nfo", (gpointer)text_plain); -g_hash_table_insert(ext_table, "sfv", (gpointer)text_plain); -g_hash_table_insert(ext_table, "m3u", (gpointer)text_plain); -g_hash_table_insert(ext_table, "csv", (gpointer)text_plain); -g_hash_table_insert(ext_table, "eml", (gpointer)text_plain); -g_hash_table_insert(ext_table, "make", (gpointer)text_plain); -g_hash_table_insert(ext_table, "log", (gpointer)text_plain); -g_hash_table_insert(ext_table, "markdown", (gpointer)text_plain); -g_hash_table_insert(ext_table, "yaml", (gpointer)text_plain); -g_hash_table_insert(ext_table, "coffee", (gpointer)application_vnd_coffeescript); -g_hash_table_insert(ext_table, "rt", (gpointer)text_richtext); -g_hash_table_insert(ext_table, "rtf", (gpointer)text_richtext); -g_hash_table_insert(ext_table, "rtx", (gpointer)text_richtext); -g_hash_table_insert(ext_table, "wsc", (gpointer)text_scriplet); -g_hash_table_insert(ext_table, "tsv", (gpointer)text_tab_separated_values); -g_hash_table_insert(ext_table, "man", (gpointer)text_troff); -g_hash_table_insert(ext_table, "me", (gpointer)text_troff); -g_hash_table_insert(ext_table, "ms", (gpointer)text_troff); -g_hash_table_insert(ext_table, "roff", (gpointer)text_troff); -g_hash_table_insert(ext_table, "t", (gpointer)text_troff); -g_hash_table_insert(ext_table, "tr", (gpointer)text_troff); -g_hash_table_insert(ext_table, "uji", (gpointer)text_uri_list); -g_hash_table_insert(ext_table, "unis", (gpointer)text_uri_list); -g_hash_table_insert(ext_table, "uri", (gpointer)text_uri_list); -g_hash_table_insert(ext_table, "uris", (gpointer)text_uri_list); -g_hash_table_insert(ext_table, "abc", (gpointer)text_vnd_abc); -g_hash_table_insert(ext_table, "flx", (gpointer)text_vnd_fmi_flexstor); -g_hash_table_insert(ext_table, "wmls", (gpointer)text_vnd_wap_wmlscript); -g_hash_table_insert(ext_table, "wml", (gpointer)text_vnd_wap_wml); -g_hash_table_insert(ext_table, "htt", (gpointer)text_webviewhtml); -g_hash_table_insert(ext_table, "asm", (gpointer)text_x_asm); -g_hash_table_insert(ext_table, "s", (gpointer)text_x_asm); -g_hash_table_insert(ext_table, "aip", (gpointer)text_x_audiosoft_intra); -g_hash_table_insert(ext_table, "awk", (gpointer)text_x_awk); -g_hash_table_insert(ext_table, "c", (gpointer)text_x_c); -g_hash_table_insert(ext_table, "cc", (gpointer)text_x_c); -g_hash_table_insert(ext_table, "h", (gpointer)text_x_c); -g_hash_table_insert(ext_table, "cpp", (gpointer)text_x_c__); -g_hash_table_insert(ext_table, "cxx", (gpointer)text_x_c__); -g_hash_table_insert(ext_table, "c++", (gpointer)text_x_c__); -g_hash_table_insert(ext_table, "htc", (gpointer)text_x_component); -g_hash_table_insert(ext_table, "f", (gpointer)text_x_fortran); -g_hash_table_insert(ext_table, "f77", (gpointer)text_x_fortran); -g_hash_table_insert(ext_table, "f90", (gpointer)text_x_fortran); -g_hash_table_insert(ext_table, "for", (gpointer)text_x_fortran); -g_hash_table_insert(ext_table, "jav", (gpointer)text_x_java); -g_hash_table_insert(ext_table, "java", (gpointer)text_x_java); -g_hash_table_insert(ext_table, "lsx", (gpointer)text_x_la_asf); -g_hash_table_insert(ext_table, "el", (gpointer)text_x_lisp); -g_hash_table_insert(ext_table, "m4", (gpointer)text_x_m4); -g_hash_table_insert(ext_table, "ac", (gpointer)text_x_m4); -g_hash_table_insert(ext_table, "am", (gpointer)text_x_makefile); -g_hash_table_insert(ext_table, "mak", (gpointer)text_x_makefile); -g_hash_table_insert(ext_table, "xml", (gpointer)text_xml); -g_hash_table_insert(ext_table, "pom", (gpointer)text_xml); -g_hash_table_insert(ext_table, "iml", (gpointer)text_xml); -g_hash_table_insert(ext_table, "plist", (gpointer)text_xml); -g_hash_table_insert(ext_table, "m", (gpointer)text_x_m); -g_hash_table_insert(ext_table, "bat", (gpointer)text_x_msdos_batch); -g_hash_table_insert(ext_table, "reg", (gpointer)text_x_ms_regedit); -g_hash_table_insert(ext_table, "p", (gpointer)text_x_pascal); -g_hash_table_insert(ext_table, "pl", (gpointer)text_x_perl); -g_hash_table_insert(ext_table, "php", (gpointer)text_x_php); -g_hash_table_insert(ext_table, "po", (gpointer)text_x_po); -g_hash_table_insert(ext_table, "py", (gpointer)text_x_python); -g_hash_table_insert(ext_table, "rb", (gpointer)text_x_ruby); -g_hash_table_insert(ext_table, "sass", (gpointer)text_x_sass); -g_hash_table_insert(ext_table, "scss", (gpointer)text_x_scss); -g_hash_table_insert(ext_table, "ssi", (gpointer)text_x_server_parsed_html); -g_hash_table_insert(ext_table, "etx", (gpointer)text_x_setext); -g_hash_table_insert(ext_table, "sgm", (gpointer)text_x_sgml); -g_hash_table_insert(ext_table, "sgml", (gpointer)text_x_sgml); -g_hash_table_insert(ext_table, "sh", (gpointer)text_x_shellscript); -g_hash_table_insert(ext_table, "talk", (gpointer)text_x_speech); -g_hash_table_insert(ext_table, "tex", (gpointer)text_x_tex); -g_hash_table_insert(ext_table, "uil", (gpointer)text_x_uil); -g_hash_table_insert(ext_table, "uue", (gpointer)text_x_uuencode); -g_hash_table_insert(ext_table, "vcs", (gpointer)text_x_vcalendar); -g_hash_table_insert(ext_table, "vcf", (gpointer)text_x_vcard); -g_hash_table_insert(ext_table, "afl", (gpointer)video_animaflex); -g_hash_table_insert(ext_table, "avi", (gpointer)video_avi); -g_hash_table_insert(ext_table, "avs", (gpointer)video_avs_video); -g_hash_table_insert(ext_table, "mp4", (gpointer)video_mp4); -g_hash_table_insert(ext_table, "m1v", (gpointer)video_mpeg); -g_hash_table_insert(ext_table, "m2v", (gpointer)video_mpeg); -g_hash_table_insert(ext_table, "mpe", (gpointer)video_mpeg); -g_hash_table_insert(ext_table, "mpeg", (gpointer)video_mpeg); -g_hash_table_insert(ext_table, "mpg", (gpointer)video_mpeg); -g_hash_table_insert(ext_table, "moov", (gpointer)video_quicktime); -g_hash_table_insert(ext_table, "mov", (gpointer)video_quicktime); -g_hash_table_insert(ext_table, "qt", (gpointer)video_quicktime); -g_hash_table_insert(ext_table, "vdo", (gpointer)video_vdo); -g_hash_table_insert(ext_table, "viv", (gpointer)video_vivo); -g_hash_table_insert(ext_table, "vivo", (gpointer)video_vivo); -g_hash_table_insert(ext_table, "rv", (gpointer)video_vnd_rn_realvideo); -g_hash_table_insert(ext_table, "vos", (gpointer)video_vosaic); -g_hash_table_insert(ext_table, "webm", (gpointer)video_webm); -g_hash_table_insert(ext_table, "xdr", (gpointer)video_x_amt_demorun); -g_hash_table_insert(ext_table, "xsr", (gpointer)video_x_amt_showrun); -g_hash_table_insert(ext_table, "fmf", (gpointer)video_x_atomic3d_feature); -g_hash_table_insert(ext_table, "dl", (gpointer)video_x_dl); -g_hash_table_insert(ext_table, "dif", (gpointer)video_x_dv); -g_hash_table_insert(ext_table, "dv", (gpointer)video_x_dv); -g_hash_table_insert(ext_table, "fli", (gpointer)video_x_fli); -g_hash_table_insert(ext_table, "flv", (gpointer)video_x_flv); -g_hash_table_insert(ext_table, "isu", (gpointer)video_x_isvideo); -g_hash_table_insert(ext_table, "jng", (gpointer)video_x_jng); -g_hash_table_insert(ext_table, "m4v", (gpointer)video_x_m4v); -g_hash_table_insert(ext_table, "mkv", (gpointer)video_x_matroska); -g_hash_table_insert(ext_table, "mng", (gpointer)video_x_mng); -g_hash_table_insert(ext_table, "mjpg", (gpointer)video_x_motion_jpeg); -g_hash_table_insert(ext_table, "asf", (gpointer)video_x_ms_asf); -g_hash_table_insert(ext_table, "asx", (gpointer)video_x_ms_asf); -g_hash_table_insert(ext_table, "wmv", (gpointer)video_x_ms_asf); -g_hash_table_insert(ext_table, "divx", (gpointer)video_x_msvideo); -g_hash_table_insert(ext_table, "qtc", (gpointer)video_x_qtc); -g_hash_table_insert(ext_table, "movie", (gpointer)video_x_sgi_movie); -g_hash_table_insert(ext_table, "mv", (gpointer)video_x_sgi_movie); -g_hash_table_insert(ext_table, "msg", (gpointer)application_vnd_ms_outlook); -g_hash_table_insert(ext_table, "orf", (gpointer)image_x_olympus_orf); -g_hash_table_insert(ext_table, "nef", (gpointer)image_x_nikon_nef); -g_hash_table_insert(ext_table, "raf", (gpointer)image_x_fuji_raf); -g_hash_table_insert(ext_table, "rw2", (gpointer)image_x_panasonic_raw); -g_hash_table_insert(ext_table, "raw", (gpointer)image_x_panasonic_raw); -g_hash_table_insert(ext_table, "dng", (gpointer)image_x_adobe_dng); -g_hash_table_insert(ext_table, "cr2", (gpointer)image_x_canon_cr2); -g_hash_table_insert(ext_table, "crw", (gpointer)image_x_canon_crw); -g_hash_table_insert(ext_table, "dcr", (gpointer)image_x_kodak_dcr); -g_hash_table_insert(ext_table, "k25", (gpointer)image_x_kodak_k25); -g_hash_table_insert(ext_table, "kdc", (gpointer)image_x_kodak_kdc); -g_hash_table_insert(ext_table, "mrw", (gpointer)image_x_minolta_mrw); -g_hash_table_insert(ext_table, "pef", (gpointer)image_x_pentax_pef); -g_hash_table_insert(ext_table, "xf3", (gpointer)image_x_sigma_x3f); -g_hash_table_insert(ext_table, "arw", (gpointer)image_x_sony_arw); -g_hash_table_insert(ext_table, "sr2", (gpointer)image_x_sony_sr2); -g_hash_table_insert(ext_table, "srf", (gpointer)image_x_sony_srf); -g_hash_table_insert(ext_table, "erf", (gpointer)image_x_epson_erf); -g_hash_table_insert(ext_table, "s2meta", (gpointer)sist2_sidecar); -return ext_table;} -GHashTable *mime_get_mime_table() {GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal); -g_hash_table_insert(mime_table, "application/arj", (gpointer)application_arj); -g_hash_table_insert(mime_table, "application/base64", (gpointer)application_base64); -g_hash_table_insert(mime_table, "application/binhex", (gpointer)application_binhex); -g_hash_table_insert(mime_table, "application/book", (gpointer)application_book); -g_hash_table_insert(mime_table, "application/CDFV2-corrupt", (gpointer)application_CDFV2_corrupt); -g_hash_table_insert(mime_table, "application/CDFV2", (gpointer)application_CDFV2); -g_hash_table_insert(mime_table, "application/clariscad", (gpointer)application_clariscad); -g_hash_table_insert(mime_table, "application/commonground", (gpointer)application_commonground); -g_hash_table_insert(mime_table, "application/csv", (gpointer)application_csv); -g_hash_table_insert(mime_table, "application/dicom", (gpointer)application_dicom); -g_hash_table_insert(mime_table, "application/drafting", (gpointer)application_drafting); -g_hash_table_insert(mime_table, "application/epub+zip", (gpointer)application_epub_zip); -g_hash_table_insert(mime_table, "application/freeloader", (gpointer)application_freeloader); -g_hash_table_insert(mime_table, "application/futuresplash", (gpointer)application_futuresplash); -g_hash_table_insert(mime_table, "application/groupwise", (gpointer)application_groupwise); -g_hash_table_insert(mime_table, "application/gzip", (gpointer)application_gzip); -g_hash_table_insert(mime_table, "application/hta", (gpointer)application_hta); -g_hash_table_insert(mime_table, "application/i-deas", (gpointer)application_i_deas); -g_hash_table_insert(mime_table, "application/iges", (gpointer)application_iges); -g_hash_table_insert(mime_table, "application/inf", (gpointer)application_inf); -g_hash_table_insert(mime_table, "application/java-archive", (gpointer)application_java_archive); -g_hash_table_insert(mime_table, "application/java", (gpointer)application_java); -g_hash_table_insert(mime_table, "application/javascript", (gpointer)application_javascript); -g_hash_table_insert(mime_table, "application/json", (gpointer)application_json); -g_hash_table_insert(mime_table, "application/ndjson", (gpointer)application_ndjson); -g_hash_table_insert(mime_table, "application/marc", (gpointer)application_marc); -g_hash_table_insert(mime_table, "application/mbedlet", (gpointer)application_mbedlet); -g_hash_table_insert(mime_table, "application/mime", (gpointer)application_mime); -g_hash_table_insert(mime_table, "application/mspowerpoint", (gpointer)application_mspowerpoint); -g_hash_table_insert(mime_table, "application/msword", (gpointer)application_msword); -g_hash_table_insert(mime_table, "application/netmc", (gpointer)application_netmc); -g_hash_table_insert(mime_table, "application/octet-stream", (gpointer)application_octet_stream); -g_hash_table_insert(mime_table, "application/oda", (gpointer)application_oda); -g_hash_table_insert(mime_table, "application/ogg", (gpointer)application_ogg); -g_hash_table_insert(mime_table, "application/pdf", (gpointer)application_pdf); -g_hash_table_insert(mime_table, "application/pgp-keys", (gpointer)application_pgp_keys); -g_hash_table_insert(mime_table, "application/pgp-signature", (gpointer)application_pgp_signature); -g_hash_table_insert(mime_table, "application/pkcs7-signature", (gpointer)application_pkcs7_signature); -g_hash_table_insert(mime_table, "application/pkix-cert", (gpointer)application_pkix_cert); -g_hash_table_insert(mime_table, "application/postscript", (gpointer)application_postscript); -g_hash_table_insert(mime_table, "application/pro_eng", (gpointer)application_pro_eng); -g_hash_table_insert(mime_table, "application/ringing-tones", (gpointer)application_ringing_tones); -g_hash_table_insert(mime_table, "application/smil", (gpointer)application_smil); -g_hash_table_insert(mime_table, "application/solids", (gpointer)application_solids); -g_hash_table_insert(mime_table, "application/sounder", (gpointer)application_sounder); -g_hash_table_insert(mime_table, "application/step", (gpointer)application_step); -g_hash_table_insert(mime_table, "application/streamingmedia", (gpointer)application_streamingmedia); -g_hash_table_insert(mime_table, "application/vda", (gpointer)application_vda); -g_hash_table_insert(mime_table, "application/vnd.fdf", (gpointer)application_vnd_fdf); -g_hash_table_insert(mime_table, "application/vnd.font-fontforge-sfd", (gpointer)application_vnd_font_fontforge_sfd); -g_hash_table_insert(mime_table, "application/vnd.hp-hpgl", (gpointer)application_vnd_hp_hpgl); -g_hash_table_insert(mime_table, "application/vnd.iccprofile", (gpointer)application_vnd_iccprofile); -g_hash_table_insert(mime_table, "application/vnd.lotus-1-2-3", (gpointer)application_vnd_lotus_1_2_3); -g_hash_table_insert(mime_table, "application/vnd.ms-cab-compressed", (gpointer)application_vnd_ms_cab_compressed); -g_hash_table_insert(mime_table, "application/vnd.ms-excel", (gpointer)application_vnd_ms_excel); -g_hash_table_insert(mime_table, "application/vnd.ms-fontobject", (gpointer)application_vnd_ms_fontobject); -g_hash_table_insert(mime_table, "application/vnd.ms-opentype", (gpointer)application_vnd_ms_opentype); -g_hash_table_insert(mime_table, "application/vnd.ms-pki.certstore", (gpointer)application_vnd_ms_pki_certstore); -g_hash_table_insert(mime_table, "application/vnd.ms-pki.pko", (gpointer)application_vnd_ms_pki_pko); -g_hash_table_insert(mime_table, "application/vnd.ms-pki.seccat", (gpointer)application_vnd_ms_pki_seccat); -g_hash_table_insert(mime_table, "application/vnd.ms-powerpoint", (gpointer)application_vnd_ms_powerpoint); -g_hash_table_insert(mime_table, "application/vnd.ms-project", (gpointer)application_vnd_ms_project); -g_hash_table_insert(mime_table, "application/vnd.oasis.opendocument.base", (gpointer)application_vnd_oasis_opendocument_base); -g_hash_table_insert(mime_table, "application/vnd.oasis.opendocument.formula", (gpointer)application_vnd_oasis_opendocument_formula); -g_hash_table_insert(mime_table, "application/vnd.oasis.opendocument.graphics", (gpointer)application_vnd_oasis_opendocument_graphics); -g_hash_table_insert(mime_table, "application/vnd.oasis.opendocument.presentation", (gpointer)application_vnd_oasis_opendocument_presentation); -g_hash_table_insert(mime_table, "application/vnd.oasis.opendocument.spreadsheet", (gpointer)application_vnd_oasis_opendocument_spreadsheet); -g_hash_table_insert(mime_table, "application/vnd.oasis.opendocument.text", (gpointer)application_vnd_oasis_opendocument_text); -g_hash_table_insert(mime_table, "application/vnd.openxmlformats-officedocument.presentationml.presentation", (gpointer)application_vnd_openxmlformats_officedocument_presentationml_presentation); -g_hash_table_insert(mime_table, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", (gpointer)application_vnd_openxmlformats_officedocument_spreadsheetml_sheet); -g_hash_table_insert(mime_table, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", (gpointer)application_vnd_openxmlformats_officedocument_wordprocessingml_document); -g_hash_table_insert(mime_table, "application/vnd.symbian.install", (gpointer)application_vnd_symbian_install); -g_hash_table_insert(mime_table, "application/vnd.tcpdump.pcap", (gpointer)application_vnd_tcpdump_pcap); -g_hash_table_insert(mime_table, "application/vnd.wap.wmlc", (gpointer)application_vnd_wap_wmlc); -g_hash_table_insert(mime_table, "application/vnd.wap.wmlscriptc", (gpointer)application_vnd_wap_wmlscriptc); -g_hash_table_insert(mime_table, "application/vnd.xara", (gpointer)application_vnd_xara); -g_hash_table_insert(mime_table, "application/vocaltec-media-desc", (gpointer)application_vocaltec_media_desc); -g_hash_table_insert(mime_table, "application/vocaltec-media-file", (gpointer)application_vocaltec_media_file); -g_hash_table_insert(mime_table, "application/warc", (gpointer)application_warc); -g_hash_table_insert(mime_table, "application/winhelp", (gpointer)application_winhelp); -g_hash_table_insert(mime_table, "application/wordperfect", (gpointer)application_wordperfect); -g_hash_table_insert(mime_table, "application/x-123", (gpointer)application_x_123); -g_hash_table_insert(mime_table, "application/x-7z-compressed", (gpointer)application_x_7z_compressed); -g_hash_table_insert(mime_table, "application/x-aim", (gpointer)application_x_aim); -g_hash_table_insert(mime_table, "application/x-apple-diskimage", (gpointer)application_x_apple_diskimage); -g_hash_table_insert(mime_table, "application/x-arc", (gpointer)application_x_arc); -g_hash_table_insert(mime_table, "application/x-archive", (gpointer)application_x_archive); -g_hash_table_insert(mime_table, "application/x-atari-7800-rom", (gpointer)application_x_atari_7800_rom); -g_hash_table_insert(mime_table, "application/x-authorware-bin", (gpointer)application_x_authorware_bin); -g_hash_table_insert(mime_table, "application/x-authorware-map", (gpointer)application_x_authorware_map); -g_hash_table_insert(mime_table, "application/x-authorware-seg", (gpointer)application_x_authorware_seg); -g_hash_table_insert(mime_table, "application/x-avira-qua", (gpointer)application_x_avira_qua); -g_hash_table_insert(mime_table, "application/x-bcpio", (gpointer)application_x_bcpio); -g_hash_table_insert(mime_table, "application/x-bittorrent", (gpointer)application_x_bittorrent); -g_hash_table_insert(mime_table, "application/x-bsh", (gpointer)application_x_bsh); -g_hash_table_insert(mime_table, "application/x-bytecode.python", (gpointer)application_x_bytecode_python); -g_hash_table_insert(mime_table, "application/x-bzip2", (gpointer)application_x_bzip2); -g_hash_table_insert(mime_table, "application/x-bzip", (gpointer)application_x_bzip); -g_hash_table_insert(mime_table, "application/x-cbr", (gpointer)application_x_cbr); -g_hash_table_insert(mime_table, "application/x-cbz", (gpointer)application_x_cbz); -g_hash_table_insert(mime_table, "application/x-cdlink", (gpointer)application_x_cdlink); -g_hash_table_insert(mime_table, "application/x-chat", (gpointer)application_x_chat); -g_hash_table_insert(mime_table, "application/x-chrome-extension", (gpointer)application_x_chrome_extension); -g_hash_table_insert(mime_table, "application/x-cocoa", (gpointer)application_x_cocoa); -g_hash_table_insert(mime_table, "application/x-conference", (gpointer)application_x_conference); -g_hash_table_insert(mime_table, "application/x-coredump", (gpointer)application_x_coredump); -g_hash_table_insert(mime_table, "application/x-cpio", (gpointer)application_x_cpio); -g_hash_table_insert(mime_table, "application/x-dbf", (gpointer)application_x_dbf); -g_hash_table_insert(mime_table, "application/x-dbt", (gpointer)application_x_dbt); -g_hash_table_insert(mime_table, "application/x-debian-package", (gpointer)application_x_debian_package); -g_hash_table_insert(mime_table, "application/x-deepv", (gpointer)application_x_deepv); -g_hash_table_insert(mime_table, "application/x-director", (gpointer)application_x_director); -g_hash_table_insert(mime_table, "application/x-dmp", (gpointer)application_x_dmp); -g_hash_table_insert(mime_table, "application/x-dosdriver", (gpointer)application_x_dosdriver); -g_hash_table_insert(mime_table, "application/x-dosexec", (gpointer)application_x_dosexec); -g_hash_table_insert(mime_table, "application/x-dvi", (gpointer)application_x_dvi); -g_hash_table_insert(mime_table, "application/x-elc", (gpointer)application_x_elc); -g_hash_table_insert(mime_table, "application/x-empty", (gpointer)application_x_empty); -g_hash_table_insert(mime_table, "application/x-envoy", (gpointer)application_x_envoy); -g_hash_table_insert(mime_table, "application/x-esrehber", (gpointer)application_x_esrehber); -g_hash_table_insert(mime_table, "application/x-excel", (gpointer)application_x_excel); -g_hash_table_insert(mime_table, "application/x-executable", (gpointer)application_x_executable); -g_hash_table_insert(mime_table, "application/x-font-gdos", (gpointer)application_x_font_gdos); -g_hash_table_insert(mime_table, "application/x-font-pf2", (gpointer)application_x_font_pf2); -g_hash_table_insert(mime_table, "application/x-font-pfm", (gpointer)application_x_font_pfm); -g_hash_table_insert(mime_table, "application/x-font-sfn", (gpointer)application_x_font_sfn); -g_hash_table_insert(mime_table, "application/x-font-ttf", (gpointer)application_x_font_ttf); -g_hash_table_insert(mime_table, "application/x-fptapplication/x-dbt", (gpointer)application_x_fptapplication_x_dbt); -g_hash_table_insert(mime_table, "application/x-freelance", (gpointer)application_x_freelance); -g_hash_table_insert(mime_table, "application/x-gamecube-rom", (gpointer)application_x_gamecube_rom); -g_hash_table_insert(mime_table, "application/x-gdbm", (gpointer)application_x_gdbm); -g_hash_table_insert(mime_table, "application/x-gettext-translation", (gpointer)application_x_gettext_translation); -g_hash_table_insert(mime_table, "application/x-git", (gpointer)application_x_git); -g_hash_table_insert(mime_table, "application/x-gsp", (gpointer)application_x_gsp); -g_hash_table_insert(mime_table, "application/x-gss", (gpointer)application_x_gss); -g_hash_table_insert(mime_table, "application/x-gtar", (gpointer)application_x_gtar); -g_hash_table_insert(mime_table, "application/x-gzip", (gpointer)application_x_gzip); -g_hash_table_insert(mime_table, "application/x-hdf", (gpointer)application_x_hdf); -g_hash_table_insert(mime_table, "application/x-helpfile", (gpointer)application_x_helpfile); -g_hash_table_insert(mime_table, "application/x-httpd-imap", (gpointer)application_x_httpd_imap); -g_hash_table_insert(mime_table, "application/x-ima", (gpointer)application_x_ima); -g_hash_table_insert(mime_table, "application/x-innosetup", (gpointer)application_x_innosetup); -g_hash_table_insert(mime_table, "application/x-internett-signup", (gpointer)application_x_internett_signup); -g_hash_table_insert(mime_table, "application/x-inventor", (gpointer)application_x_inventor); -g_hash_table_insert(mime_table, "application/x-ip2", (gpointer)application_x_ip2); -g_hash_table_insert(mime_table, "application/x-java-applet", (gpointer)application_x_java_applet); -g_hash_table_insert(mime_table, "application/x-java-commerce", (gpointer)application_x_java_commerce); -g_hash_table_insert(mime_table, "application/x-java-image", (gpointer)application_x_java_image); -g_hash_table_insert(mime_table, "application/x-java-jmod", (gpointer)application_x_java_jmod); -g_hash_table_insert(mime_table, "application/x-java-keystore", (gpointer)application_x_java_keystore); -g_hash_table_insert(mime_table, "application/x-kdelnk", (gpointer)application_x_kdelnk); -g_hash_table_insert(mime_table, "application/x-koan", (gpointer)application_x_koan); -g_hash_table_insert(mime_table, "application/x-latex", (gpointer)application_x_latex); -g_hash_table_insert(mime_table, "application/x-livescreen", (gpointer)application_x_livescreen); -g_hash_table_insert(mime_table, "application/x-lotus", (gpointer)application_x_lotus); -g_hash_table_insert(mime_table, "application/x-lz4+json", (gpointer)application_x_lz4_json); -g_hash_table_insert(mime_table, "application/x-lz4", (gpointer)application_x_lz4); -g_hash_table_insert(mime_table, "application/x-lzh-compressed", (gpointer)application_x_lzh_compressed); -g_hash_table_insert(mime_table, "application/x-lzh", (gpointer)application_x_lzh); -g_hash_table_insert(mime_table, "application/x-lzip", (gpointer)application_x_lzip); -g_hash_table_insert(mime_table, "application/x-lzma", (gpointer)application_x_lzma); -g_hash_table_insert(mime_table, "application/x-lzop", (gpointer)application_x_lzop); -g_hash_table_insert(mime_table, "application/x-lzx", (gpointer)application_x_lzx); -g_hash_table_insert(mime_table, "application/x-mach-binary", (gpointer)application_x_mach_binary); -g_hash_table_insert(mime_table, "application/x-mach-executable", (gpointer)application_x_mach_executable); -g_hash_table_insert(mime_table, "application/x-magic-cap-package-1.0", (gpointer)application_x_magic_cap_package_1_0); -g_hash_table_insert(mime_table, "application/x-mathcad", (gpointer)application_x_mathcad); -g_hash_table_insert(mime_table, "application/x-maxis-dbpf", (gpointer)application_x_maxis_dbpf); -g_hash_table_insert(mime_table, "application/x-meme", (gpointer)application_x_meme); -g_hash_table_insert(mime_table, "application/x-midi", (gpointer)application_x_midi); -g_hash_table_insert(mime_table, "application/x-mif", (gpointer)application_x_mif); -g_hash_table_insert(mime_table, "application/x-mix-transfer", (gpointer)application_x_mix_transfer); -g_hash_table_insert(mime_table, "application/xml", (gpointer)application_xml); -g_hash_table_insert(mime_table, "application/x-mobipocket-ebook", (gpointer)application_x_mobipocket_ebook); -g_hash_table_insert(mime_table, "application/vnd.amazon.mobi8-ebook", (gpointer)application_vnd_amazon_mobi8_ebook); -g_hash_table_insert(mime_table, "application/x-msaccess", (gpointer)application_x_msaccess); -g_hash_table_insert(mime_table, "application/x-ms-compress-szdd", (gpointer)application_x_ms_compress_szdd); -g_hash_table_insert(mime_table, "application/x-ms-pdb", (gpointer)application_x_ms_pdb); -g_hash_table_insert(mime_table, "application/x-ms-reader", (gpointer)application_x_ms_reader); -g_hash_table_insert(mime_table, "application/x-n64-rom", (gpointer)application_x_n64_rom); -g_hash_table_insert(mime_table, "application/x-navi-animation", (gpointer)application_x_navi_animation); -g_hash_table_insert(mime_table, "application/x-navidoc", (gpointer)application_x_navidoc); -g_hash_table_insert(mime_table, "application/x-navimap", (gpointer)application_x_navimap); -g_hash_table_insert(mime_table, "application/x-navistyle", (gpointer)application_x_navistyle); -g_hash_table_insert(mime_table, "application/x-nes-rom", (gpointer)application_x_nes_rom); -g_hash_table_insert(mime_table, "application/x-netcdf", (gpointer)application_x_netcdf); -g_hash_table_insert(mime_table, "application/x-newton-compatible-pkg", (gpointer)application_x_newton_compatible_pkg); -g_hash_table_insert(mime_table, "application/x-nintendo-ds-rom", (gpointer)application_x_nintendo_ds_rom); -g_hash_table_insert(mime_table, "application/x-object", (gpointer)application_x_object); -g_hash_table_insert(mime_table, "application/x-omcdatamaker", (gpointer)application_x_omcdatamaker); -g_hash_table_insert(mime_table, "application/x-omc", (gpointer)application_x_omc); -g_hash_table_insert(mime_table, "application/x-omcregerator", (gpointer)application_x_omcregerator); -g_hash_table_insert(mime_table, "application/x-pagemaker", (gpointer)application_x_pagemaker); -g_hash_table_insert(mime_table, "application/x-pcl", (gpointer)application_x_pcl); -g_hash_table_insert(mime_table, "application/x-pgp-keyring", (gpointer)application_x_pgp_keyring); -g_hash_table_insert(mime_table, "application/x-pixclscript", (gpointer)application_x_pixclscript); -g_hash_table_insert(mime_table, "application/x-pkcs7-certreqresp", (gpointer)application_x_pkcs7_certreqresp); -g_hash_table_insert(mime_table, "application/x-pkcs7-signature", (gpointer)application_x_pkcs7_signature); -g_hash_table_insert(mime_table, "application/x-project", (gpointer)application_x_project); -g_hash_table_insert(mime_table, "application/x-qpro", (gpointer)application_x_qpro); -g_hash_table_insert(mime_table, "application/x-rar", (gpointer)application_x_rar); -g_hash_table_insert(mime_table, "application/x-rpm", (gpointer)application_x_rpm); -g_hash_table_insert(mime_table, "application/x-sdp", (gpointer)application_x_sdp); -g_hash_table_insert(mime_table, "application/x-sea", (gpointer)application_x_sea); -g_hash_table_insert(mime_table, "application/x-seelogo", (gpointer)application_x_seelogo); -g_hash_table_insert(mime_table, "application/x-setupscript", (gpointer)application_x_setupscript); -g_hash_table_insert(mime_table, "application/x-sharedlib", (gpointer)application_x_sharedlib); -g_hash_table_insert(mime_table, "application/x-shar", (gpointer)application_x_shar); -g_hash_table_insert(mime_table, "application/x-shockwave-flash", (gpointer)application_x_shockwave_flash); -g_hash_table_insert(mime_table, "application/x-snappy-framed", (gpointer)application_x_snappy_framed); -g_hash_table_insert(mime_table, "application/x-sprite", (gpointer)application_x_sprite); -g_hash_table_insert(mime_table, "application/x-sqlite3", (gpointer)application_x_sqlite3); -g_hash_table_insert(mime_table, "application/x-stargallery-thm", (gpointer)application_x_stargallery_thm); -g_hash_table_insert(mime_table, "application/x-stuffit", (gpointer)application_x_stuffit); -g_hash_table_insert(mime_table, "application/x-sv4cpio", (gpointer)application_x_sv4cpio); -g_hash_table_insert(mime_table, "application/x-sv4crc", (gpointer)application_x_sv4crc); -g_hash_table_insert(mime_table, "application/x-tar", (gpointer)application_x_tar); -g_hash_table_insert(mime_table, "application/x-tbook", (gpointer)application_x_tbook); -g_hash_table_insert(mime_table, "application/x-terminfo", (gpointer)application_x_terminfo); -g_hash_table_insert(mime_table, "application/x-terminfo2", (gpointer)application_x_terminfo2); -g_hash_table_insert(mime_table, "application/x-texinfo", (gpointer)application_x_texinfo); -g_hash_table_insert(mime_table, "application/x-tex-tfm", (gpointer)application_x_tex_tfm); -g_hash_table_insert(mime_table, "application/x-ustar", (gpointer)application_x_ustar); -g_hash_table_insert(mime_table, "application/x-visio", (gpointer)application_x_visio); -g_hash_table_insert(mime_table, "application/x-vnd.audioexplosion.mzz", (gpointer)application_x_vnd_audioexplosion_mzz); -g_hash_table_insert(mime_table, "application/x-vnd.ls-xpix", (gpointer)application_x_vnd_ls_xpix); -g_hash_table_insert(mime_table, "application/x-vrml", (gpointer)application_x_vrml); -g_hash_table_insert(mime_table, "application/x-wais-source", (gpointer)application_x_wais_source); -g_hash_table_insert(mime_table, "application/x-wine-extension-ini", (gpointer)application_x_wine_extension_ini); -g_hash_table_insert(mime_table, "application/x-wintalk", (gpointer)application_x_wintalk); -g_hash_table_insert(mime_table, "application/x-world", (gpointer)application_x_world); -g_hash_table_insert(mime_table, "application/x-wri", (gpointer)application_x_wri); -g_hash_table_insert(mime_table, "application/x-x509-ca-cert", (gpointer)application_x_x509_ca_cert); -g_hash_table_insert(mime_table, "application/x-xz", (gpointer)application_x_xz); -g_hash_table_insert(mime_table, "application/x-zip", (gpointer)application_x_zip); -g_hash_table_insert(mime_table, "application/x-zstd", (gpointer)application_x_zstd); -g_hash_table_insert(mime_table, "application/zip", (gpointer)application_zip); -g_hash_table_insert(mime_table, "application/zlib", (gpointer)application_zlib); -g_hash_table_insert(mime_table, "audio/basic", (gpointer)audio_basic); -g_hash_table_insert(mime_table, "audio/it", (gpointer)audio_it); -g_hash_table_insert(mime_table, "audio/make", (gpointer)audio_make); -g_hash_table_insert(mime_table, "audio/midi", (gpointer)audio_midi); -g_hash_table_insert(mime_table, "audio/mid", (gpointer)audio_mid); -g_hash_table_insert(mime_table, "audio/mp4", (gpointer)audio_mp4); -g_hash_table_insert(mime_table, "audio/mpeg", (gpointer)audio_mpeg); -g_hash_table_insert(mime_table, "audio/ogg", (gpointer)audio_ogg); -g_hash_table_insert(mime_table, "audio/s3m", (gpointer)audio_s3m); -g_hash_table_insert(mime_table, "audio/tsp-audio", (gpointer)audio_tsp_audio); -g_hash_table_insert(mime_table, "audio/tsplayer", (gpointer)audio_tsplayer); -g_hash_table_insert(mime_table, "audio/vnd.qcelp", (gpointer)audio_vnd_qcelp); -g_hash_table_insert(mime_table, "audio/voxware", (gpointer)audio_voxware); -g_hash_table_insert(mime_table, "audio/x-aiff", (gpointer)audio_x_aiff); -g_hash_table_insert(mime_table, "audio/x-flac", (gpointer)audio_x_flac); -g_hash_table_insert(mime_table, "audio/x-gsm", (gpointer)audio_x_gsm); -g_hash_table_insert(mime_table, "audio/x-hx-aac-adts", (gpointer)audio_x_hx_aac_adts); -g_hash_table_insert(mime_table, "audio/x-jam", (gpointer)audio_x_jam); -g_hash_table_insert(mime_table, "audio/x-liveaudio", (gpointer)audio_x_liveaudio); -g_hash_table_insert(mime_table, "audio/x-m4a", (gpointer)audio_x_m4a); -g_hash_table_insert(mime_table, "audio/x-midi", (gpointer)audio_x_midi); -g_hash_table_insert(mime_table, "audio/x-mod", (gpointer)audio_x_mod); -g_hash_table_insert(mime_table, "audio/x-mp4a-latm", (gpointer)audio_x_mp4a_latm); -g_hash_table_insert(mime_table, "audio/x-mpeg-3", (gpointer)audio_x_mpeg_3); -g_hash_table_insert(mime_table, "audio/x-mpequrl", (gpointer)audio_x_mpequrl); -g_hash_table_insert(mime_table, "audio/xm", (gpointer)audio_xm); -g_hash_table_insert(mime_table, "audio/x-nspaudio", (gpointer)audio_x_nspaudio); -g_hash_table_insert(mime_table, "audio/x-pn-realaudio", (gpointer)audio_x_pn_realaudio); -g_hash_table_insert(mime_table, "audio/x-psid", (gpointer)audio_x_psid); -g_hash_table_insert(mime_table, "audio/x-realaudio", (gpointer)audio_x_realaudio); -g_hash_table_insert(mime_table, "audio/x-s3m", (gpointer)audio_x_s3m); -g_hash_table_insert(mime_table, "audio/x-twinvq-plugin", (gpointer)audio_x_twinvq_plugin); -g_hash_table_insert(mime_table, "audio/x-twinvq", (gpointer)audio_x_twinvq); -g_hash_table_insert(mime_table, "audio/x-voc", (gpointer)audio_x_voc); -g_hash_table_insert(mime_table, "audio/x-wav", (gpointer)audio_x_wav); -g_hash_table_insert(mime_table, "audio/x-xbox360-executable", (gpointer)audio_x_xbox360_executable); -g_hash_table_insert(mime_table, "audio/x-xbox-executable", (gpointer)audio_x_xbox_executable); -g_hash_table_insert(mime_table, "font/otf", (gpointer)font_otf); -g_hash_table_insert(mime_table, "font/sfnt", (gpointer)font_sfnt); -g_hash_table_insert(mime_table, "font/woff2", (gpointer)font_woff2); -g_hash_table_insert(mime_table, "font/woff", (gpointer)font_woff); -g_hash_table_insert(mime_table, "image/bmp", (gpointer)image_bmp); -g_hash_table_insert(mime_table, "image/cmu-raster", (gpointer)image_cmu_raster); -g_hash_table_insert(mime_table, "image/fif", (gpointer)image_fif); -g_hash_table_insert(mime_table, "image/florian", (gpointer)image_florian); -g_hash_table_insert(mime_table, "image/g3fax", (gpointer)image_g3fax); -g_hash_table_insert(mime_table, "image/gif", (gpointer)image_gif); -g_hash_table_insert(mime_table, "image/heic", (gpointer)image_heic); -g_hash_table_insert(mime_table, "image/ief", (gpointer)image_ief); -g_hash_table_insert(mime_table, "image/jpeg", (gpointer)image_jpeg); -g_hash_table_insert(mime_table, "image/jutvision", (gpointer)image_jutvision); -g_hash_table_insert(mime_table, "image/naplps", (gpointer)image_naplps); -g_hash_table_insert(mime_table, "image/pict", (gpointer)image_pict); -g_hash_table_insert(mime_table, "image/png", (gpointer)image_png); -g_hash_table_insert(mime_table, "image/svg", (gpointer)image_svg); -g_hash_table_insert(mime_table, "image/svg+xml", (gpointer)image_svg_xml); -g_hash_table_insert(mime_table, "image/tiff", (gpointer)image_tiff); -g_hash_table_insert(mime_table, "image/vnd.adobe.photoshop", (gpointer)image_vnd_adobe_photoshop); -g_hash_table_insert(mime_table, "image/vnd.djvu", (gpointer)image_vnd_djvu); -g_hash_table_insert(mime_table, "image/vnd.fpx", (gpointer)image_vnd_fpx); -g_hash_table_insert(mime_table, "image/vnd.microsoft.icon", (gpointer)image_vnd_microsoft_icon); -g_hash_table_insert(mime_table, "image/vnd.rn-realflash", (gpointer)image_vnd_rn_realflash); -g_hash_table_insert(mime_table, "image/vnd.rn-realpix", (gpointer)image_vnd_rn_realpix); -g_hash_table_insert(mime_table, "image/vnd.wap.wbmp", (gpointer)image_vnd_wap_wbmp); -g_hash_table_insert(mime_table, "image/vnd.xiff", (gpointer)image_vnd_xiff); -g_hash_table_insert(mime_table, "image/webp", (gpointer)image_webp); -g_hash_table_insert(mime_table, "image/wmf", (gpointer)image_wmf); -g_hash_table_insert(mime_table, "image/x-3ds", (gpointer)image_x_3ds); -g_hash_table_insert(mime_table, "image/x-award-bioslogo", (gpointer)image_x_award_bioslogo); -g_hash_table_insert(mime_table, "image/x-cmu-raster", (gpointer)image_x_cmu_raster); -g_hash_table_insert(mime_table, "image/x-cur", (gpointer)image_x_cur); -g_hash_table_insert(mime_table, "image/x-dwg", (gpointer)image_x_dwg); -g_hash_table_insert(mime_table, "image/x-eps", (gpointer)image_x_eps); -g_hash_table_insert(mime_table, "image/x-exr", (gpointer)image_x_exr); -g_hash_table_insert(mime_table, "image/x-gem", (gpointer)image_x_gem); -g_hash_table_insert(mime_table, "image/x-icns", (gpointer)image_x_icns); -g_hash_table_insert(mime_table, "image/x-icon", (gpointer)image_x_icon); -g_hash_table_insert(mime_table, "image/x-jg", (gpointer)image_x_jg); -g_hash_table_insert(mime_table, "image/x-jps", (gpointer)image_x_jps); -g_hash_table_insert(mime_table, "image/x-ms-bmp", (gpointer)image_x_ms_bmp); -g_hash_table_insert(mime_table, "image/x-niff", (gpointer)image_x_niff); -g_hash_table_insert(mime_table, "image/x-pcx", (gpointer)image_x_pcx); -g_hash_table_insert(mime_table, "image/x-pict", (gpointer)image_x_pict); -g_hash_table_insert(mime_table, "image/x-portable-bitmap", (gpointer)image_x_portable_bitmap); -g_hash_table_insert(mime_table, "image/x-portable-graymap", (gpointer)image_x_portable_graymap); -g_hash_table_insert(mime_table, "image/x-portable-pixmap", (gpointer)image_x_portable_pixmap); -g_hash_table_insert(mime_table, "image/x-quicktime", (gpointer)image_x_quicktime); -g_hash_table_insert(mime_table, "image/x-rgb", (gpointer)image_x_rgb); -g_hash_table_insert(mime_table, "image/x-tga", (gpointer)image_x_tga); -g_hash_table_insert(mime_table, "image/x-tiff", (gpointer)image_x_tiff); -g_hash_table_insert(mime_table, "image/x-win-bitmap", (gpointer)image_x_win_bitmap); -g_hash_table_insert(mime_table, "image/x-xcf", (gpointer)image_x_xcf); -g_hash_table_insert(mime_table, "image/x-xpixmap", (gpointer)image_x_xpixmap); -g_hash_table_insert(mime_table, "image/x-xwindowdump", (gpointer)image_x_xwindowdump); -g_hash_table_insert(mime_table, "message/news", (gpointer)message_news); -g_hash_table_insert(mime_table, "message/rfc822", (gpointer)message_rfc822); -g_hash_table_insert(mime_table, "model/vnd.dwf", (gpointer)model_vnd_dwf); -g_hash_table_insert(mime_table, "model/vnd.gdl", (gpointer)model_vnd_gdl); -g_hash_table_insert(mime_table, "model/vnd.gs.gdl", (gpointer)model_vnd_gs_gdl); -g_hash_table_insert(mime_table, "model/vrml", (gpointer)model_vrml); -g_hash_table_insert(mime_table, "model/x-pov", (gpointer)model_x_pov); -g_hash_table_insert(mime_table, "text/asp", (gpointer)text_asp); -g_hash_table_insert(mime_table, "text/css", (gpointer)text_css); -g_hash_table_insert(mime_table, "text/html", (gpointer)text_html); -g_hash_table_insert(mime_table, "text/javascript", (gpointer)text_javascript); -g_hash_table_insert(mime_table, "text/mcf", (gpointer)text_mcf); -g_hash_table_insert(mime_table, "text/pascal", (gpointer)text_pascal); -g_hash_table_insert(mime_table, "text/PGP", (gpointer)text_PGP); -g_hash_table_insert(mime_table, "text/plain", (gpointer)text_plain); -g_hash_table_insert(mime_table, "application/vnd.coffeescript", (gpointer)application_vnd_coffeescript); -g_hash_table_insert(mime_table, "text/richtext", (gpointer)text_richtext); -g_hash_table_insert(mime_table, "text/rtf", (gpointer)text_rtf); -g_hash_table_insert(mime_table, "text/scriplet", (gpointer)text_scriplet); -g_hash_table_insert(mime_table, "text/tab-separated-values", (gpointer)text_tab_separated_values); -g_hash_table_insert(mime_table, "text/troff", (gpointer)text_troff); -g_hash_table_insert(mime_table, "text/uri-list", (gpointer)text_uri_list); -g_hash_table_insert(mime_table, "text/vnd.abc", (gpointer)text_vnd_abc); -g_hash_table_insert(mime_table, "text/vnd.fmi.flexstor", (gpointer)text_vnd_fmi_flexstor); -g_hash_table_insert(mime_table, "text/vnd.wap.wmlscript", (gpointer)text_vnd_wap_wmlscript); -g_hash_table_insert(mime_table, "text/vnd.wap.wml", (gpointer)text_vnd_wap_wml); -g_hash_table_insert(mime_table, "text/webviewhtml", (gpointer)text_webviewhtml); -g_hash_table_insert(mime_table, "text/x-Algol68", (gpointer)text_x_Algol68); -g_hash_table_insert(mime_table, "text/x-asm", (gpointer)text_x_asm); -g_hash_table_insert(mime_table, "text/x-audiosoft-intra", (gpointer)text_x_audiosoft_intra); -g_hash_table_insert(mime_table, "text/x-awk", (gpointer)text_x_awk); -g_hash_table_insert(mime_table, "text/x-bcpl", (gpointer)text_x_bcpl); -g_hash_table_insert(mime_table, "text/x-c", (gpointer)text_x_c); -g_hash_table_insert(mime_table, "text/x-c++", (gpointer)text_x_c__); -g_hash_table_insert(mime_table, "text/x-component", (gpointer)text_x_component); -g_hash_table_insert(mime_table, "text/x-diff", (gpointer)text_x_diff); -g_hash_table_insert(mime_table, "text/x-fortran", (gpointer)text_x_fortran); -g_hash_table_insert(mime_table, "text/x-java", (gpointer)text_x_java); -g_hash_table_insert(mime_table, "text/x-la-asf", (gpointer)text_x_la_asf); -g_hash_table_insert(mime_table, "text/x-lisp", (gpointer)text_x_lisp); -g_hash_table_insert(mime_table, "text/x-m4", (gpointer)text_x_m4); -g_hash_table_insert(mime_table, "text/x-makefile", (gpointer)text_x_makefile); -g_hash_table_insert(mime_table, "text/xml", (gpointer)text_xml); -g_hash_table_insert(mime_table, "text/x-m", (gpointer)text_x_m); -g_hash_table_insert(mime_table, "text/x-msdos-batch", (gpointer)text_x_msdos_batch); -g_hash_table_insert(mime_table, "text/x-ms-regedit", (gpointer)text_x_ms_regedit); -g_hash_table_insert(mime_table, "text/x-objective-c", (gpointer)text_x_objective_c); -g_hash_table_insert(mime_table, "text/x-pascal", (gpointer)text_x_pascal); -g_hash_table_insert(mime_table, "text/x-perl", (gpointer)text_x_perl); -g_hash_table_insert(mime_table, "text/x-php", (gpointer)text_x_php); -g_hash_table_insert(mime_table, "text/x-po", (gpointer)text_x_po); -g_hash_table_insert(mime_table, "text/x-python", (gpointer)text_x_python); -g_hash_table_insert(mime_table, "text/x-ruby", (gpointer)text_x_ruby); -g_hash_table_insert(mime_table, "text/x-sass", (gpointer)text_x_sass); -g_hash_table_insert(mime_table, "text/x-scss", (gpointer)text_x_scss); -g_hash_table_insert(mime_table, "text/x-server-parsed-html", (gpointer)text_x_server_parsed_html); -g_hash_table_insert(mime_table, "text/x-setext", (gpointer)text_x_setext); -g_hash_table_insert(mime_table, "text/x-sgml", (gpointer)text_x_sgml); -g_hash_table_insert(mime_table, "text/x-shellscript", (gpointer)text_x_shellscript); -g_hash_table_insert(mime_table, "text/x-speech", (gpointer)text_x_speech); -g_hash_table_insert(mime_table, "text/x-tcl", (gpointer)text_x_tcl); -g_hash_table_insert(mime_table, "text/x-tex", (gpointer)text_x_tex); -g_hash_table_insert(mime_table, "text/x-uil", (gpointer)text_x_uil); -g_hash_table_insert(mime_table, "text/x-uuencode", (gpointer)text_x_uuencode); -g_hash_table_insert(mime_table, "text/x-vcalendar", (gpointer)text_x_vcalendar); -g_hash_table_insert(mime_table, "text/x-vcard", (gpointer)text_x_vcard); -g_hash_table_insert(mime_table, "video/animaflex", (gpointer)video_animaflex); -g_hash_table_insert(mime_table, "video/avi", (gpointer)video_avi); -g_hash_table_insert(mime_table, "video/avs-video", (gpointer)video_avs_video); -g_hash_table_insert(mime_table, "video/MP2T", (gpointer)video_MP2T); -g_hash_table_insert(mime_table, "video/mp4", (gpointer)video_mp4); -g_hash_table_insert(mime_table, "video/mpeg", (gpointer)video_mpeg); -g_hash_table_insert(mime_table, "video/quicktime", (gpointer)video_quicktime); -g_hash_table_insert(mime_table, "video/vdo", (gpointer)video_vdo); -g_hash_table_insert(mime_table, "video/vivo", (gpointer)video_vivo); -g_hash_table_insert(mime_table, "video/vnd.rn-realvideo", (gpointer)video_vnd_rn_realvideo); -g_hash_table_insert(mime_table, "video/vosaic", (gpointer)video_vosaic); -g_hash_table_insert(mime_table, "video/webm", (gpointer)video_webm); -g_hash_table_insert(mime_table, "video/x-amt-demorun", (gpointer)video_x_amt_demorun); -g_hash_table_insert(mime_table, "video/x-amt-showrun", (gpointer)video_x_amt_showrun); -g_hash_table_insert(mime_table, "video/x-atomic3d-feature", (gpointer)video_x_atomic3d_feature); -g_hash_table_insert(mime_table, "video/x-dl", (gpointer)video_x_dl); -g_hash_table_insert(mime_table, "video/x-dv", (gpointer)video_x_dv); -g_hash_table_insert(mime_table, "video/x-fli", (gpointer)video_x_fli); -g_hash_table_insert(mime_table, "video/x-flv", (gpointer)video_x_flv); -g_hash_table_insert(mime_table, "video/x-isvideo", (gpointer)video_x_isvideo); -g_hash_table_insert(mime_table, "video/x-jng", (gpointer)video_x_jng); -g_hash_table_insert(mime_table, "video/x-m4v", (gpointer)video_x_m4v); -g_hash_table_insert(mime_table, "video/x-matroska", (gpointer)video_x_matroska); -g_hash_table_insert(mime_table, "video/x-mng", (gpointer)video_x_mng); -g_hash_table_insert(mime_table, "video/x-motion-jpeg", (gpointer)video_x_motion_jpeg); -g_hash_table_insert(mime_table, "video/x-ms-asf", (gpointer)video_x_ms_asf); -g_hash_table_insert(mime_table, "video/x-msvideo", (gpointer)video_x_msvideo); -g_hash_table_insert(mime_table, "video/x-qtc", (gpointer)video_x_qtc); -g_hash_table_insert(mime_table, "video/x-sgi-movie", (gpointer)video_x_sgi_movie); -g_hash_table_insert(mime_table, "x-epoc/x-sisx-app", (gpointer)x_epoc_x_sisx_app); -g_hash_table_insert(mime_table, "application/x-zstd-dictionary", (gpointer)application_x_zstd_dictionary); -g_hash_table_insert(mime_table, "application/vnd.ms-outlook", (gpointer)application_vnd_ms_outlook); -g_hash_table_insert(mime_table, "image/x-olympus-orf", (gpointer)image_x_olympus_orf); -g_hash_table_insert(mime_table, "image/x-nikon-nef", (gpointer)image_x_nikon_nef); -g_hash_table_insert(mime_table, "image/x-fuji-raf", (gpointer)image_x_fuji_raf); -g_hash_table_insert(mime_table, "image/x-panasonic-raw", (gpointer)image_x_panasonic_raw); -g_hash_table_insert(mime_table, "image/x-adobe-dng", (gpointer)image_x_adobe_dng); -g_hash_table_insert(mime_table, "image/x-canon-cr2", (gpointer)image_x_canon_cr2); -g_hash_table_insert(mime_table, "image/x-canon-crw", (gpointer)image_x_canon_crw); -g_hash_table_insert(mime_table, "image/x-dcraw", (gpointer)image_x_dcraw); -g_hash_table_insert(mime_table, "image/x-kodak-dcr", (gpointer)image_x_kodak_dcr); -g_hash_table_insert(mime_table, "image/x-kodak-k25", (gpointer)image_x_kodak_k25); -g_hash_table_insert(mime_table, "image/x-kodak-kdc", (gpointer)image_x_kodak_kdc); -g_hash_table_insert(mime_table, "image/x-minolta-mrw", (gpointer)image_x_minolta_mrw); -g_hash_table_insert(mime_table, "image/x-pentax-pef", (gpointer)image_x_pentax_pef); -g_hash_table_insert(mime_table, "image/x-sigma-x3f", (gpointer)image_x_sigma_x3f); -g_hash_table_insert(mime_table, "image/x-sony-arw", (gpointer)image_x_sony_arw); -g_hash_table_insert(mime_table, "image/x-sony-sr2", (gpointer)image_x_sony_sr2); -g_hash_table_insert(mime_table, "image/x-sony-srf", (gpointer)image_x_sony_srf); -g_hash_table_insert(mime_table, "image/x-epson-erf", (gpointer)image_x_epson_erf); -g_hash_table_insert(mime_table, "sist2/sidecar", (gpointer)sist2_sidecar); -return mime_table;} +unsigned int mime_extension_lookup(unsigned long extension_crc32) {switch (extension_crc32) { +case 104524599:return application_arj; +case 1388642652:return application_base64; +case 3514823219:return application_binhex; +case 2340081149:case 3420824369:return application_book; +case 285308337:return application_CDFV2; +case 1954483503:return application_clariscad; +case 273534304:return application_commonground; +case 3444350831:return application_dicom; +case 1677149445:return application_drafting; +case 2293841338:return application_epub_zip; +case 3927332999:return application_freeloader; +case 3268238238:return application_futuresplash; +case 2044114573:return application_groupwise; +case 3686102973:case 919960796:return application_gzip; +case 3361975990:return application_hta; +case 4016300425:return application_i_deas; +case 4218684756:case 1542976603:return application_iges; +case 3890785273:return application_inf; +case 2028789010:return application_java_archive; +case 3981121951:return application_java; +case 1795630405:return application_json; +case 2430300356:case 2781733981:return application_ndjson; +case 1996270327:return application_marc; +case 2723818757:return application_mbedlet; +case 1348936053:return application_mime; +case 878661782:return application_mspowerpoint; +case 2252471652:case 93485219:case 3917664813:case 2853236491:case 3287381265:return application_msword; +case 2711341113:return application_netmc; +case 2854705901:case 1198289189:case 1311803834:case 2547932707:case 3876093456:return application_octet_stream; +case 2280565346:return application_oda; +case 789609574:return application_ogg; +case 250665868:return application_pdf; +case 3507043614:return application_pgp_signature; +case 4072354288:return application_pkcs7_signature; +case 324084633:case 4289790522:return application_pkix_cert; +case 156340709:case 2817149839:return application_postscript; +case 1225748678:case 3788578579:return application_pro_eng; +case 2157762558:return application_ringing_tones; +case 1305199373:case 3908728192:return application_smil; +case 261575936:return application_solids; +case 376222120:return application_sounder; +case 1136262716:case 2996880085:return application_step; +case 2665953483:return application_streamingmedia; +case 2484679325:return application_vda; +case 375063630:return application_vnd_fdf; +case 3498993275:return application_vnd_font_fontforge_sfd; +case 3611136921:case 1164758663:case 3852449971:return application_vnd_hp_hpgl; +case 3314814012:return application_vnd_iccprofile; +case 1787492089:return application_vnd_ms_cab_compressed; +case 3478635557:case 3092313267:case 686811426:case 1609095604:case 2783438039:case 2726961358:return application_vnd_ms_excel; +case 72356500:return application_vnd_ms_fontobject; +case 1397239184:return application_vnd_ms_opentype; +case 4203515915:return application_vnd_ms_pki_certstore; +case 4038361063:return application_vnd_ms_pki_pko; +case 2656977832:return application_vnd_ms_pki_seccat; +case 515732239:case 3191511418:case 1300490290:case 3555163537:case 2065614417:return application_vnd_ms_powerpoint; +case 3228862891:return application_vnd_ms_project; +case 518519768:return application_vnd_oasis_opendocument_base; +case 428490689:return application_vnd_oasis_opendocument_formula; +case 1854738263:return application_vnd_oasis_opendocument_graphics; +case 3982396048:return application_vnd_oasis_opendocument_presentation; +case 1951914794:return application_vnd_oasis_opendocument_spreadsheet; +case 3929230985:return application_vnd_oasis_opendocument_text; +case 185029164:return application_vnd_openxmlformats_officedocument_presentationml_presentation; +case 2496574992:return application_vnd_openxmlformats_officedocument_spreadsheetml_sheet; +case 3330667071:return application_vnd_openxmlformats_officedocument_wordprocessingml_document; +case 738795811:return application_vnd_tcpdump_pcap; +case 2320501507:return application_vnd_wap_wmlc; +case 3589852303:return application_vnd_wap_wmlscriptc; +case 365508689:return application_vnd_xara; +case 900824411:return application_vocaltec_media_desc; +case 3686734967:return application_vocaltec_media_file; +case 1460391352:return application_warc; +case 549983773:return application_winhelp; +case 1906478322:case 3614644754:case 1316637608:case 3407838064:case 20579870:case 1983723144:return application_wordperfect; +case 2032670097:return application_x_123; +case 1628318441:return application_x_7z_compressed; +case 828945678:return application_x_aim; +case 3904355907:return application_x_archive; +case 240214191:return application_x_atari_7800_rom; +case 1762534039:return application_x_authorware_bin; +case 4189142790:return application_x_authorware_map; +case 62784101:return application_x_authorware_seg; +case 830492586:return application_x_bcpio; +case 3704076214:return application_x_bittorrent; +case 4083746051:return application_x_bsh; +case 2180399903:return application_x_bytecode_python; +case 3869714710:case 2836727456:return application_x_bzip2; +case 2797765624:return application_x_bzip; +case 1544598878:return application_x_cbr; +case 1389051244:return application_x_cbz; +case 2872201429:return application_x_cdlink; +case 574692362:case 1704850090:return application_x_chat; +case 638445766:return application_x_cocoa; +case 1839277551:return application_x_conference; +case 890520454:return application_x_cpio; +case 1132820390:return application_x_dbf; +case 195634552:return application_x_debian_package; +case 2116912533:return application_x_deepv; +case 3131800080:case 3916585216:return application_x_director; +case 818535992:return application_x_dmp; +case 1037284150:return application_x_dosexec; +case 4254383458:return application_x_dvi; +case 2897134736:return application_x_elc; +case 4081402617:case 3789670193:return application_x_envoy; +case 2422189467:return application_x_esrehber; +case 1449039263:case 640966928:case 3062594689:case 998470004:case 3582791768:return application_x_excel; +case 1801697008:return application_x_executable; +case 1354771683:return application_x_font_pf2; +case 2870239366:return application_x_font_pfm; +case 1127964929:case 860984718:return application_x_font_ttf; +case 2338406369:return application_x_freelance; +case 3872284606:return application_x_gsp; +case 2143751684:return application_x_gss; +case 379624156:return application_x_gtar; +case 988372210:return application_x_gzip; +case 482683204:return application_x_hdf; +case 143088812:return application_x_helpfile; +case 193336380:return application_x_httpd_imap; +case 1386311065:return application_x_ima; +case 2318749458:return application_x_internett_signup; +case 1283462680:return application_x_inventor; +case 2783163181:return application_x_ip2; +case 3352474213:return application_x_java_commerce; +case 2640986657:return application_x_java_jmod; +case 1696602166:case 486318226:case 2147142731:case 2023163986:return application_x_koan; +case 215374679:case 2869014186:return application_x_latex; +case 3908779605:return application_x_livescreen; +case 3372609354:return application_x_lotus; +case 1244562587:return application_x_lz4_json; +case 1256754847:return application_x_lz4; +case 674599744:return application_x_lzh; +case 943828598:return application_x_lzip; +case 3811851781:return application_x_lzma; +case 3058763491:return application_x_lzop; +case 897745700:return application_x_lzx; +case 785325796:case 3941149528:return application_x_mach_binary; +case 3449642452:return application_x_magic_cap_package_1_0; +case 3141606468:return application_x_mathcad; +case 2726894320:return application_x_meme; +case 2599515954:return application_x_midi; +case 2946536930:return application_x_mif; +case 1474893528:return application_x_mix_transfer; +case 925165716:return application_xml; +case 3357977606:return application_x_mobipocket_ebook; +case 2917469670:case 605433051:return application_vnd_amazon_mobi8_ebook; +case 2740923043:return application_x_msaccess; +case 4218705335:return application_x_ms_compress_szdd; +case 161292181:return application_x_ms_pdb; +case 1574669981:return application_x_ms_reader; +case 244273492:return application_x_n64_rom; +case 2034558928:return application_x_navi_animation; +case 2394047497:return application_x_navidoc; +case 2477632187:return application_x_navimap; +case 2795609754:return application_x_navistyle; +case 1821278300:return application_x_nes_rom; +case 277923493:case 1847354420:return application_x_netcdf; +case 4268750805:return application_x_newton_compatible_pkg; +case 252678980:return application_x_object; +case 100793631:return application_x_omcdatamaker; +case 3089264647:return application_x_omc; +case 4074063950:return application_x_omcregerator; +case 1515671581:case 760242315:return application_x_pagemaker; +case 2707727445:return application_x_pcl; +case 1009151207:return application_x_pixclscript; +case 2243690854:return application_x_pkcs7_certreqresp; +case 16924856:return application_x_pkcs7_signature; +case 1154140277:case 3340324274:case 689384606:case 3467575705:return application_x_project; +case 2833952472:return application_x_qpro; +case 1792618458:return application_x_rar; +case 3020423487:return application_x_rpm; +case 4167225476:return application_x_sdp; +case 2345255223:return application_x_sea; +case 30348729:return application_x_seelogo; +case 2563130371:return application_x_sharedlib; +case 3706126336:return application_x_shar; +case 1834558535:return application_x_shockwave_flash; +case 952264445:case 891129758:return application_x_sprite; +case 1252092624:return application_x_stuffit; +case 3714480584:return application_x_sv4cpio; +case 3062692538:return application_x_sv4crc; +case 1851020136:return application_x_tar; +case 610123502:case 554870891:return application_x_tbook; +case 1485687582:case 4263375082:return application_x_texinfo; +case 2887632986:return application_x_tex_tfm; +case 529472938:return application_x_ustar; +case 3790619268:case 4232532704:case 1699611482:return application_x_visio; +case 3662579775:return application_x_vnd_audioexplosion_mzz; +case 836703967:return application_x_vnd_ls_xpix; +case 2889000187:return application_x_vrml; +case 1615078541:case 1219082918:return application_x_wais_source; +case 1070375909:return application_x_wintalk; +case 1855504763:return application_x_world; +case 2274911567:return application_x_wri; +case 371078428:return application_x_x509_ca_cert; +case 384792867:return application_x_xz; +case 4116556676:return application_x_zstd; +case 1109235014:return application_zip; +case 1657960367:return application_zlib; +case 491834794:return audio_basic; +case 2727245620:return audio_it; +case 1278026625:case 3092500109:case 1385818959:return audio_make; +case 2033104677:return audio_midi; +case 1275701562:return audio_mid; +case 2815980258:return audio_mp4; +case 1753973982:case 2865015129:case 2901423164:return audio_mpeg; +case 1168137364:return audio_ogg; +case 1855879118:return audio_s3m; +case 2630216279:return audio_tsp_audio; +case 4172169879:return audio_tsplayer; +case 3030880813:return audio_vnd_qcelp; +case 327566230:return audio_voxware; +case 1924052889:case 2797235334:return audio_x_aiff; +case 3600363395:return audio_x_flac; +case 4229204931:case 2244490087:return audio_x_gsm; +case 4125413607:return audio_x_jam; +case 4050234453:return audio_x_liveaudio; +case 1053893464:return audio_x_m4a; +case 1101984974:return audio_x_midi; +case 799119745:return audio_x_mpeg_3; +case 2503802084:return audio_xm; +case 1416277874:return audio_x_nspaudio; +case 3889242671:case 1876104302:case 1264872739:case 677511674:return audio_x_pn_realaudio; +case 1461090996:return audio_x_psid; +case 1717917765:return audio_x_realaudio; +case 2764116112:case 3709708340:return audio_x_twinvq_plugin; +case 1036534058:return audio_x_twinvq; +case 2581826170:return audio_x_voc; +case 1803495720:return audio_x_wav; +case 3824649750:return audio_x_xbox360_executable; +case 3484449800:return audio_x_xbox_executable; +case 1505691300:return font_woff2; +case 55690088:return font_woff; +case 4196239628:return image_cmu_raster; +case 2750773763:return image_fif; +case 2807969506:case 791360735:return image_florian; +case 3568668297:return image_g3fax; +case 2721517620:return image_gif; +case 4048480567:return image_heic; +case 68973106:case 3554509561:return image_ief; +case 694905706:case 3496410035:case 2833677253:case 3933391666:case 1189616361:return image_jpeg; +case 3206655858:return image_jutvision; +case 2448102626:case 3954137820:return image_naplps; +case 3409203534:case 348455534:return image_pict; +case 2199389072:case 562164082:return image_png; +case 54863248:return image_svg; +case 3850190390:return image_vnd_adobe_photoshop; +case 2303371901:return image_vnd_djvu; +case 3271224952:return image_vnd_fpx; +case 4160867814:return image_vnd_rn_realflash; +case 215316663:return image_vnd_rn_realpix; +case 480853153:return image_vnd_wap_wbmp; +case 3041784953:return image_vnd_xiff; +case 2659071723:return image_webp; +case 301010174:return image_x_3ds; +case 501096268:return image_x_cmu_raster; +case 3165697328:return image_x_cur; +case 53977380:case 4087909757:case 1950487814:return image_x_dwg; +case 3903861559:return image_x_exr; +case 731540752:return image_x_icon; +case 4231386708:return image_x_jg; +case 1546833556:return image_x_jps; +case 621872703:case 876913290:return image_x_ms_bmp; +case 2917542843:case 718120911:return image_x_niff; +case 3149819944:return image_x_pcx; +case 2986884099:return image_x_pict; +case 3480790402:return image_x_portable_bitmap; +case 2987348423:return image_x_portable_graymap; +case 3079460177:return image_x_portable_pixmap; +case 3130839030:case 3578773115:case 2983316714:return image_x_quicktime; +case 557094968:return image_x_rgb; +case 3159821597:case 359206964:return image_x_tiff; +case 1336024307:return image_x_xcf; +case 3114239209:return image_x_xpixmap; +case 2399316618:return image_x_xwindowdump; +case 1157813739:case 2663211800:case 1116045392:return message_rfc822; +case 1949340082:return model_vnd_dwf; +case 4148987751:return model_vnd_gdl; +case 2289923100:return model_vnd_gs_gdl; +case 52831377:return model_vrml; +case 4038267427:return model_x_pov; +case 3796050700:return text_asp; +case 2026809048:return text_css; +case 4137492127:case 3252019869:case 410646757:case 3413549060:case 2886207094:case 1097681659:return text_html; +case 398963028:return text_javascript; +case 1431272808:return text_mcf; +case 509266722:return text_pascal; +case 1689700070:case 794565824:case 351504808:case 214229345:case 30677878:case 1835907068:case 1154021400:case 3992351814:case 2107886487:case 2202503947:case 999008199:case 473390917:case 3679822420:case 1465078094:case 1466496025:case 2277716423:case 157353380:case 2002237032:case 4216257084:case 590894066:case 987584319:case 2268432115:case 3551958239:case 1436306077:case 3060306774:case 808890964:case 2564639436:case 3322219037:case 3334425408:case 3818365258:case 1403162576:case 590812979:case 1800036834:case 144986711:case 621471808:case 449607278:case 2403297477:case 2529069283:case 3929123204:return text_plain; +case 1401235891:return application_vnd_coffeescript; +case 196656302:case 1203117491:case 3183026384:return text_richtext; +case 2119613712:return text_scriplet; +case 298706850:return text_tab_separated_values; +case 1772263384:case 2891092674:case 1485186963:case 881800026:case 2238339752:case 3028401693:return text_troff; +case 101132664:case 1121950192:case 2216472865:case 1474544612:return text_uri_list; +case 891568578:return text_vnd_abc; +case 613266213:return text_vnd_fmi_flexstor; +case 2548505447:return text_vnd_wap_wmlscript; +case 984129374:return text_vnd_wap_wml; +case 2780711517:return text_webviewhtml; +case 2168803285:case 453955339:return text_x_asm; +case 1382996439:return text_x_audiosoft_intra; +case 206188516:return text_x_awk; +case 112844655:case 3685882489:case 2439710439:return text_x_c; +case 3404375201:case 216573595:case 1864149058:return text_x_c__; +case 644730778:return text_x_component; +case 1993550816:case 2611047355:case 2605110166:case 4017424888:return text_x_fortran; +case 2139166987:case 2132469458:return text_x_java; +case 3829413997:return text_x_la_asf; +case 492281966:return text_x_lisp; +case 2956915616:case 3917773051:return text_x_m4; +case 238835196:case 432563031:return text_x_makefile; +case 838129763:case 2060901327:case 739250468:case 1518088406:return text_xml; +case 3775001192:return text_x_m; +case 2677811615:return text_x_msdos_batch; +case 1667763765:return text_x_ms_regedit; +case 2181537457:return text_x_pascal; +case 719472250:return text_x_perl; +case 1452351953:return text_x_php; +case 3018528704:return text_x_po; +case 1195352721:case 1629727233:return text_x_python; +case 4285270527:return text_x_ruby; +case 3703929802:return text_x_sass; +case 3745623972:return text_x_scss; +case 2576022738:return text_x_server_parsed_html; +case 2765133093:return text_x_setext; +case 2957635486:case 2164340050:return text_x_sgml; +case 111333792:return text_x_shellscript; +case 2669991355:return text_x_speech; +case 3941433202:return text_x_tex; +case 1564511796:return text_x_uil; +case 3270208461:return text_x_uuencode; +case 685888786:return text_x_vcalendar; +case 1161572857:return text_x_vcard; +case 3254229335:return video_animaflex; +case 4217339785:return video_avi; +case 104695539:return video_avs_video; +case 2982480930:return video_mp4; +case 3228914394:case 3948413209:case 2913561920:case 2007364491:case 1135021164:return video_mpeg; +case 4034977726:case 3830260224:case 546697069:return video_quicktime; +case 1939939226:return video_vdo; +case 2724514071:case 2357097034:return video_vivo; +case 3853966722:return video_vnd_rn_realvideo; +case 2220136990:return video_vosaic; +case 4252523058:return video_webm; +case 439967305:return video_x_amt_demorun; +case 532339423:return video_x_amt_showrun; +case 3348741895:return video_x_atomic3d_feature; +case 72126767:return video_x_dl; +case 2691804781:case 4180568149:return video_x_dv; +case 1312658391:return video_x_fli; +case 3275071010:return video_x_flv; +case 2621081147:return video_x_isvideo; +case 2460558646:return video_x_jng; +case 3171067551:return video_x_m4v; +case 2149705476:return video_x_matroska; +case 2548446131:return video_x_mng; +case 1369518905:return video_x_motion_jpeg; +case 379033181:case 3969428286:case 3351930404:return video_x_ms_asf; +case 4122831001:return video_x_msvideo; +case 899307365:return video_x_qtc; +case 492761711:case 686609180:return video_x_sgi_movie; +case 1753898927:return application_vnd_ms_outlook; +case 85101078:return image_x_olympus_orf; +case 22242487:return image_x_nikon_nef; +case 1879309223:return image_x_fuji_raf; +case 10345373:case 447994709:return image_x_panasonic_raw; +case 2553743420:return image_x_adobe_dng; +case 1611589279:return image_x_canon_cr2; +case 1723356032:return image_x_canon_crw; +case 1078235802:return image_x_kodak_dcr; +case 28033:return image_x_kodak_k25; +case 1860789138:return image_x_kodak_kdc; +case 1814462090:return image_x_minolta_mrw; +case 401337037:return image_x_pentax_pef; +case 701779405:return image_x_sigma_x3f; +case 1698465774:return image_x_sony_arw; +case 2083014127:return image_x_sony_sr2; +case 271503362:return image_x_sony_srf; +case 142938048:return image_x_epson_erf; +case 287571459:return sist2_sidecar; +default: return 0;}} +unsigned int mime_name_lookup(unsigned long mime_crc32) {switch (mime_crc32) { +case 3812269631: return application_arj; +case 2479484568: return application_base64; +case 3891182180: return application_binhex; +case 3319475062: return application_book; +case 131831009: return application_CDFV2_corrupt; +case 1972415093: return application_CDFV2; +case 2361432233: return application_clariscad; +case 3013534691: return application_commonground; +case 3986958175: return application_csv; +case 2684316502: return application_dicom; +case 1539671880: return application_drafting; +case 749429103: return application_epub_zip; +case 663947845: return application_freeloader; +case 1404675439: return application_futuresplash; +case 366020206: return application_groupwise; +case 886559925: return application_gzip; +case 761519038: return application_hta; +case 3028409237: return application_i_deas; +case 4115303699: return application_iges; +case 48815857: return application_inf; +case 2879859070: return application_java_archive; +case 1898213013: return application_java; +case 223081448: return application_javascript; +case 1698278658: return application_json; +case 4187371699: return application_ndjson; +case 1720840708: return application_marc; +case 2657503299: return application_mbedlet; +case 1287325719: return application_mime; +case 2397876339: return application_mspowerpoint; +case 87314907: return application_msword; +case 1400780351: return application_netmc; +case 3754511218: return application_octet_stream; +case 1659821930: return application_oda; +case 2694850972: return application_ogg; +case 3958419076: return application_pdf; +case 1866620368: return application_pgp_keys; +case 489766691: return application_pgp_signature; +case 4163635168: return application_pkcs7_signature; +case 75695703: return application_pkix_cert; +case 2281931862: return application_postscript; +case 2662888269: return application_pro_eng; +case 2510237148: return application_ringing_tones; +case 3871669703: return application_smil; +case 2233914511: return application_solids; +case 4248059749: return application_sounder; +case 1300723323: return application_step; +case 2336404640: return application_streamingmedia; +case 1897482133: return application_vda; +case 2726368314: return application_vnd_fdf; +case 3075632147: return application_vnd_font_fontforge_sfd; +case 2621280924: return application_vnd_hp_hpgl; +case 2741854226: return application_vnd_iccprofile; +case 1371686262: return application_vnd_lotus_1_2_3; +case 3936927446: return application_vnd_ms_cab_compressed; +case 4257532721: return application_vnd_ms_excel; +case 3556277566: return application_vnd_ms_fontobject; +case 2971868745: return application_vnd_ms_opentype; +case 458899730: return application_vnd_ms_pki_certstore; +case 1930126273: return application_vnd_ms_pki_pko; +case 2755865911: return application_vnd_ms_pki_seccat; +case 1964479319: return application_vnd_ms_powerpoint; +case 2494900263: return application_vnd_ms_project; +case 4144865272: return application_vnd_oasis_opendocument_base; +case 1537772039: return application_vnd_oasis_opendocument_formula; +case 3291349919: return application_vnd_oasis_opendocument_graphics; +case 2829854259: return application_vnd_oasis_opendocument_presentation; +case 2319019141: return application_vnd_oasis_opendocument_spreadsheet; +case 204654174: return application_vnd_oasis_opendocument_text; +case 817338285: return application_vnd_openxmlformats_officedocument_presentationml_presentation; +case 3437874751: return application_vnd_openxmlformats_officedocument_spreadsheetml_sheet; +case 2166600829: return application_vnd_openxmlformats_officedocument_wordprocessingml_document; +case 37472375: return application_vnd_symbian_install; +case 1262135101: return application_vnd_tcpdump_pcap; +case 1817019072: return application_vnd_wap_wmlc; +case 2753880093: return application_vnd_wap_wmlscriptc; +case 3065697271: return application_vnd_xara; +case 838649278: return application_vocaltec_media_desc; +case 3067502586: return application_vocaltec_media_file; +case 1496663551: return application_warc; +case 3547836790: return application_winhelp; +case 3853357533: return application_wordperfect; +case 3261561739: return application_x_123; +case 283325182: return application_x_7z_compressed; +case 2068294999: return application_x_aim; +case 4169136188: return application_x_apple_diskimage; +case 902384586: return application_x_arc; +case 4124671903: return application_x_archive; +case 1188732497: return application_x_atari_7800_rom; +case 496856974: return application_x_authorware_bin; +case 605528024: return application_x_authorware_map; +case 3574616417: return application_x_authorware_seg; +case 2858645924: return application_x_avira_qua; +case 1234533326: return application_x_bcpio; +case 1092178040: return application_x_bittorrent; +case 3108379994: return application_x_bsh; +case 153724964: return application_x_bytecode_python; +case 4188634997: return application_x_bzip2; +case 533359240: return application_x_bzip; +case 373271815: return application_x_cbr; +case 417604917: return application_x_cbz; +case 241171266: return application_x_cdlink; +case 2002793698: return application_x_chat; +case 529743617: return application_x_chrome_extension; +case 2443278958: return application_x_cocoa; +case 2145722326: return application_x_conference; +case 2927741547: return application_x_coredump; +case 669615566: return application_x_cpio; +case 162164735: return application_x_dbf; +case 4195557047: return application_x_dbt; +case 3783917932: return application_x_debian_package; +case 104378865: return application_x_deepv; +case 2277091176: return application_x_director; +case 2061944417: return application_x_dmp; +case 1831097311: return application_x_dosdriver; +case 525018050: return application_x_dosexec; +case 3082528059: return application_x_dvi; +case 3867270345: return application_x_elc; +case 282200480: return application_x_empty; +case 2950435174: return application_x_envoy; +case 3187201087: return application_x_esrehber; +case 1467053279: return application_x_excel; +case 947844639: return application_x_executable; +case 2396718311: return application_x_font_gdos; +case 1409586422: return application_x_font_pf2; +case 2949699731: return application_x_font_pfm; +case 882849648: return application_x_font_sfn; +case 1207949588: return application_x_font_ttf; +case 3696685787: return application_x_fptapplication_x_dbt; +case 3068589175: return application_x_freelance; +case 1117803295: return application_x_gamecube_rom; +case 3193872850: return application_x_gdbm; +case 3979978688: return application_x_gettext_translation; +case 463567141: return application_x_git; +case 2900445159: return application_x_gsp; +case 904402525: return application_x_gss; +case 73227412: return application_x_gtar; +case 672438970: return application_x_gzip; +case 1458188573: return application_x_hdf; +case 229946719: return application_x_helpfile; +case 646033581: return application_x_httpd_imap; +case 411989440: return application_x_ima; +case 3975252160: return application_x_innosetup; +case 1986465192: return application_x_internett_signup; +case 2967475964: return application_x_inventor; +case 360716690: return application_x_ip2; +case 1941969752: return application_x_java_applet; +case 2680192289: return application_x_java_commerce; +case 2138341338: return application_x_java_image; +case 69714645: return application_x_java_jmod; +case 4223869: return application_x_java_keystore; +case 2580542542: return application_x_kdelnk; +case 1257894898: return application_x_koan; +case 1958965043: return application_x_latex; +case 3948850351: return application_x_livescreen; +case 1225577433: return application_x_lotus; +case 1357787287: return application_x_lz4_json; +case 13080262: return application_x_lz4; +case 841824197: return application_x_lzh_compressed; +case 1645906713: return application_x_lzh; +case 4279573947: return application_x_lzip; +case 4056545357: return application_x_lzma; +case 2840574525: return application_x_lzop; +case 2142083965: return application_x_lzx; +case 2343296583: return application_x_mach_binary; +case 3713471041: return application_x_mach_executable; +case 217874104: return application_x_magic_cap_package_1_0; +case 451420058: return application_x_mathcad; +case 3288238855: return application_x_maxis_dbpf; +case 1499643772: return application_x_meme; +case 2282533242: return application_x_midi; +case 3851390395: return application_x_mif; +case 3005113033: return application_x_mix_transfer; +case 3572804971: return application_xml; +case 3049871419: return application_x_mobipocket_ebook; +case 3026870622: return application_vnd_amazon_mobi8_ebook; +case 3721289434: return application_x_msaccess; +case 1771413382: return application_x_ms_compress_szdd; +case 1442391593: return application_x_ms_pdb; +case 1142990575: return application_x_ms_reader; +case 873042275: return application_x_n64_rom; +case 2786166244: return application_x_navi_animation; +case 3973357482: return application_x_navidoc; +case 4181248117: return application_x_navimap; +case 3312986187: return application_x_navistyle; +case 1421897571: return application_x_nes_rom; +case 2479206333: return application_x_netcdf; +case 3639323747: return application_x_newton_compatible_pkg; +case 2023658739: return application_x_nintendo_ds_rom; +case 3792313241: return application_x_object; +case 2690661493: return application_x_omcdatamaker; +case 4060964958: return application_x_omc; +case 963821989: return application_x_omcregerator; +case 3803521399: return application_x_pagemaker; +case 3947609100: return application_x_pcl; +case 1165863721: return application_x_pgp_keyring; +case 3070019447: return application_x_pixclscript; +case 4210405996: return application_x_pkcs7_certreqresp; +case 1106894812: return application_x_pkcs7_signature; +case 261550829: return application_x_project; +case 1959776576: return application_x_qpro; +case 553006979: return application_x_rar; +case 4263975270: return application_x_rpm; +case 2991442141: return application_x_sdp; +case 3253131630: return application_x_sea; +case 2900890640: return application_x_seelogo; +case 3384573178: return application_x_setupscript; +case 177730462: return application_x_sharedlib; +case 3457861192: return application_x_shar; +case 69425328: return application_x_shockwave_flash; +case 497520372: return application_x_snappy_framed; +case 2142912491: return application_x_sprite; +case 1325004050: return application_x_sqlite3; +case 3872402089: return application_x_stargallery_thm; +case 1879600421: return application_x_stuffit; +case 4249050059: return application_x_sv4cpio; +case 4230695631: return application_x_sv4crc; +case 612059953: return application_x_tar; +case 1489303899: return application_x_tbook; +case 3222972068: return application_x_terminfo; +case 3417228122: return application_x_terminfo2; +case 3728265961: return application_x_texinfo; +case 905009673: return application_x_tex_tfm; +case 1738156494: return application_x_ustar; +case 2970490033: return application_x_visio; +case 1138952844: return application_x_vnd_audioexplosion_mzz; +case 1304824724: return application_x_vnd_ls_xpix; +case 3201262259: return application_x_vrml; +case 3932648349: return application_x_wais_source; +case 1235338088: return application_x_wine_extension_ini; +case 589270721: return application_x_wintalk; +case 1113728295: return application_x_world; +case 3451354390: return application_x_wri; +case 57439876: return application_x_x509_ca_cert; +case 3864158535: return application_x_xz; +case 137530655: return application_x_zip; +case 1610651158: return application_x_zstd; +case 2803753038: return application_zip; +case 2109142397: return application_zlib; +case 4190753873: return audio_basic; +case 1937202163: return audio_it; +case 2897185533: return audio_make; +case 747801505: return audio_midi; +case 1149197041: return audio_mid; +case 3021251101: return audio_mp4; +case 3250982680: return audio_mpeg; +case 1081209515: return audio_ogg; +case 1800395249: return audio_s3m; +case 1115987927: return audio_tsp_audio; +case 3332029264: return audio_tsplayer; +case 2061915550: return audio_vnd_qcelp; +case 773784371: return audio_voxware; +case 3530521778: return audio_x_aiff; +case 1985537192: return audio_x_flac; +case 1323393159: return audio_x_gsm; +case 4036043693: return audio_x_hx_aac_adts; +case 1053655815: return audio_x_jam; +case 3004387442: return audio_x_liveaudio; +case 4126699704: return audio_x_m4a; +case 976272409: return audio_x_midi; +case 3705471144: return audio_x_mod; +case 673327857: return audio_x_mp4a_latm; +case 1478949397: return audio_x_mpeg_3; +case 87820768: return audio_x_mpequrl; +case 1153814563: return audio_xm; +case 1178632038: return audio_x_nspaudio; +case 3992820039: return audio_x_pn_realaudio; +case 1122834189: return audio_x_psid; +case 97005398: return audio_x_realaudio; +case 2780242990: return audio_x_s3m; +case 2160563041: return audio_x_twinvq_plugin; +case 1246057794: return audio_x_twinvq; +case 1389025690: return audio_x_voc; +case 2690022088: return audio_x_wav; +case 1048952181: return audio_x_xbox360_executable; +case 544201676: return audio_x_xbox_executable; +case 1689004438: return font_otf; +case 1373033921: return font_sfnt; +case 143151839: return font_woff2; +case 3926229593: return font_woff; +case 126997061: return image_bmp; +case 3256582560: return image_cmu_raster; +case 2418019020: return image_fif; +case 2026029393: return image_florian; +case 2108397085: return image_g3fax; +case 2447532283: return image_gif; +case 4205241190: return image_heic; +case 935920381: return image_ief; +case 3785015651: return image_jpeg; +case 3265402270: return image_jutvision; +case 2832913471: return image_naplps; +case 523044927: return image_pict; +case 2966254431: return image_png; +case 814765407: return image_svg; +case 910409547: return image_svg_xml; +case 511777381: return image_tiff; +case 3977526033: return image_vnd_adobe_photoshop; +case 3999110387: return image_vnd_djvu; +case 143568099: return image_vnd_fpx; +case 517285938: return image_vnd_microsoft_icon; +case 4258027809: return image_vnd_rn_realflash; +case 358996403: return image_vnd_rn_realpix; +case 1399719511: return image_vnd_wap_wbmp; +case 950571266: return image_vnd_xiff; +case 2509651130: return image_webp; +case 3920103055: return image_wmf; +case 2937026420: return image_x_3ds; +case 2981127273: return image_x_award_bioslogo; +case 399334222: return image_x_cmu_raster; +case 3882699330: return image_x_cur; +case 3184122542: return image_x_dwg; +case 3918590499: return image_x_eps; +case 1447971005: return image_x_exr; +case 665779514: return image_x_gem; +case 307945478: return image_x_icns; +case 1749532062: return image_x_icon; +case 4168548187: return image_x_jg; +case 3805066526: return image_x_jps; +case 1491847821: return image_x_ms_bmp; +case 656290698: return image_x_niff; +case 88083362: return image_x_pcx; +case 420910635: return image_x_pict; +case 3189998154: return image_x_portable_bitmap; +case 35768844: return image_x_portable_graymap; +case 3136003682: return image_x_portable_pixmap; +case 1296924133: return image_x_quicktime; +case 2680873906: return image_x_rgb; +case 38719162: return image_x_tga; +case 414876785: return image_x_tiff; +case 1917930393: return image_x_win_bitmap; +case 4049364857: return image_x_xcf; +case 1663705411: return image_x_xpixmap; +case 153652621: return image_x_xwindowdump; +case 1301602886: return message_news; +case 475605908: return message_rfc822; +case 1271393540: return model_vnd_dwf; +case 3367762897: return model_vnd_gdl; +case 857821694: return model_vnd_gs_gdl; +case 2025285843: return model_vrml; +case 1733021175: return model_x_pov; +case 2596712340: return text_asp; +case 4926016: return text_css; +case 3872744991: return text_html; +case 3862088606: return text_javascript; +case 768274928: return text_mcf; +case 3970938585: return text_pascal; +case 1059844876: return text_PGP; +case 1152832851: return text_plain; +case 2809123822: return application_vnd_coffeescript; +case 4000659158: return text_richtext; +case 1060344107: return text_rtf; +case 211439948: return text_scriplet; +case 2273059608: return text_tab_separated_values; +case 2184047304: return text_troff; +case 3483524219: return text_uri_list; +case 1323400122: return text_vnd_abc; +case 1570296745: return text_vnd_fmi_flexstor; +case 1751327861: return text_vnd_wap_wmlscript; +case 4260425760: return text_vnd_wap_wml; +case 2469299651: return text_webviewhtml; +case 4287189747: return text_x_Algol68; +case 2118781611: return text_x_asm; +case 952979666: return text_x_audiosoft_intra; +case 4081412762: return text_x_awk; +case 646156822: return text_x_bcpl; +case 699860591: return text_x_c; +case 2417225020: return text_x_c__; +case 238960228: return text_x_component; +case 4063368405: return text_x_diff; +case 1029004811: return text_x_fortran; +case 3361256876: return text_x_java; +case 2465655719: return text_x_la_asf; +case 4108808063: return text_x_lisp; +case 2953969017: return text_x_m4; +case 4016541460: return text_x_makefile; +case 1232164091: return text_xml; +case 3457100648: return text_x_m; +case 700310341: return text_x_msdos_batch; +case 2079603480: return text_x_ms_regedit; +case 4063403658: return text_x_objective_c; +case 1452612725: return text_x_pascal; +case 3928494898: return text_x_perl; +case 2845729967: return text_x_php; +case 3015973145: return text_x_po; +case 3876215756: return text_x_python; +case 2065206899: return text_x_ruby; +case 1804019892: return text_x_sass; +case 1745091802: return text_x_scss; +case 2730494614: return text_x_server_parsed_html; +case 884456798: return text_x_setext; +case 910386732: return text_x_sgml; +case 709434536: return text_x_shellscript; +case 3374645891: return text_x_speech; +case 1499613687: return text_x_tcl; +case 367147532: return text_x_tex; +case 2723138378: return text_x_uil; +case 1710771134: return text_x_uuencode; +case 704335853: return text_x_vcalendar; +case 820689877: return text_x_vcard; +case 1266973941: return video_animaflex; +case 1794710033: return video_avi; +case 793710652: return video_avs_video; +case 1538592195: return video_MP2T; +case 543386042: return video_mp4; +case 2313178776: return video_mpeg; +case 3777745158: return video_quicktime; +case 3792140802: return video_vdo; +case 1916522329: return video_vivo; +case 82483669: return video_vnd_rn_realvideo; +case 4093612396: return video_vosaic; +case 54388513: return video_webm; +case 2020036618: return video_x_amt_demorun; +case 2314063236: return video_x_amt_showrun; +case 2591549712: return video_x_atomic3d_feature; +case 76200991: return video_x_dl; +case 4192748901: return video_x_dv; +case 1759798614: return video_x_fli; +case 3857475747: return video_x_flv; +case 3528106369: return video_x_isvideo; +case 3027293111: return video_x_jng; +case 2614841374: return video_x_m4v; +case 4159659542: return video_x_matroska; +case 2973746482: return video_x_mng; +case 3962088510: return video_x_motion_jpeg; +case 1565147726: return video_x_ms_asf; +case 637896082: return video_x_msvideo; +case 323199460: return video_x_qtc; +case 4268619377: return video_x_sgi_movie; +case 661782105: return x_epoc_x_sisx_app; +case 2159585521: return application_x_zstd_dictionary; +case 106416856: return application_vnd_ms_outlook; +case 2040256118: return image_x_olympus_orf; +case 3741779740: return image_x_nikon_nef; +case 2950019021: return image_x_fuji_raf; +case 2116379200: return image_x_panasonic_raw; +case 1038006091: return image_x_adobe_dng; +case 3469235825: return image_x_canon_cr2; +case 3363758958: return image_x_canon_crw; +case 1615469233: return image_x_dcraw; +case 373957392: return image_x_kodak_dcr; +case 1443814411: return image_x_kodak_k25; +case 954716696: return image_x_kodak_kdc; +case 1433180503: return image_x_minolta_mrw; +case 698598226: return image_x_pentax_pef; +case 2130369412: return image_x_sigma_x3f; +case 3060720351: return image_x_sony_arw; +case 2944016606: return image_x_sony_sr2; +case 3279729971: return image_x_sony_srf; +case 1665206815: return image_x_epson_erf; +case 521139448: return sist2_sidecar; +default: return 0;}} #endif diff --git a/src/parsing/parse.c b/src/parsing/parse.c index a792ad6..4786407 100644 --- a/src/parsing/parse.c +++ b/src/parsing/parse.c @@ -5,235 +5,242 @@ #include "mime.h" #include "src/io/serialize.h" #include "src/parsing/sidecar.h" -#include "src/magic_generated.c" - -#include +#include "src/parsing/fs_util.h" +#include "src/parsing/magic_util.h" +#include #define MIN_VIDEO_SIZE (1024 * 64) #define MIN_IMAGE_SIZE (512) -int fs_read(struct vfile *f, void *buf, size_t size) { +#define MAGIC_BUF_SIZE (4096 * 6) - if (f->fd == -1) { - SHA1_Init(&f->sha1_ctx); +typedef enum { + FILETYPE_DONT_PARSE, + FILETYPE_RAW, + FILETYPE_MEDIA, + FILETYPE_EBOOK, + FILETYPE_MARKUP, + FILETYPE_TEXT, + FILETYPE_FONT, + FILETYPE_ARCHIVE, + FILETYPE_OOXML, + FILETYPE_COMIC, + FILETYPE_MOBI, + FILETYPE_SIST2_SIDECAR, + FILETYPE_MSDOC, + FILETYPE_JSON, + FILETYPE_NDJSON, +} file_type_t; - f->fd = open(f->filepath, O_RDONLY); - if (f->fd == -1) { - return -1; +file_type_t get_file_type(unsigned int mime, size_t size, const char *filepath) { + + int major_mime = MAJOR_MIME(mime); + + if (!(SHOULD_PARSE(mime))) { + return FILETYPE_DONT_PARSE; + } else if (IS_RAW(mime)) { + return FILETYPE_RAW; + } else if ((major_mime == MimeVideo && size >= MIN_VIDEO_SIZE) || + (major_mime == MimeImage && size >= MIN_IMAGE_SIZE) || major_mime == MimeAudio) { + return FILETYPE_MEDIA; + } else if (IS_PDF(mime)) { + return FILETYPE_EBOOK; + } else if (major_mime == MimeText && ScanCtx.text_ctx.content_size > 0) { + if (IS_MARKUP(mime)) { + return FILETYPE_MARKUP; + } else { + return FILETYPE_TEXT; + } + + } else if (IS_FONT(mime)) { + return FILETYPE_FONT; + } else if ( + ScanCtx.arc_ctx.mode != ARC_MODE_SKIP && ( + IS_ARC(mime) || + (IS_ARC_FILTER(mime) && should_parse_filtered_file(filepath)) + )) { + return FILETYPE_ARCHIVE; + } else if ((ScanCtx.ooxml_ctx.content_size > 0 || ScanCtx.media_ctx.tn_size > 0) && IS_DOC(mime)) { + return FILETYPE_OOXML; + } else if (is_cbr(&ScanCtx.comic_ctx, mime) || is_cbz(&ScanCtx.comic_ctx, mime)) { + return FILETYPE_COMIC; + } else if (IS_MOBI(mime)) { + return FILETYPE_MOBI; + } else if (mime == MIME_SIST2_SIDECAR) { + return FILETYPE_SIST2_SIDECAR; + } else if (is_msdoc(&ScanCtx.msdoc_ctx, mime)) { + return FILETYPE_MSDOC; + } else if (is_json(&ScanCtx.json_ctx, mime)) { + return FILETYPE_JSON; + } else if (is_ndjson(&ScanCtx.json_ctx, mime)) { + return FILETYPE_NDJSON; + } +} + +#define GET_MIME_ERROR_FATAL (-1) + +int get_mime(parse_job_t *job) { + + char *extension = job->filepath + job->ext; + + int mime = 0; + + if (job->vfile.st_size == 0) { + return MIME_EMPTY; + } + + if (*extension != '\0' && (job->ext - job->base != 1)) { + mime = (int) mime_get_mime_by_ext(extension); + + if (mime != 0) { + return mime; } } - int ret = (int) read(f->fd, buf, size); - - if (ret != 0 && f->calculate_checksum) { - f->has_checksum = TRUE; - safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret); + if (strlen(extension) == 0 && strlen(job->filepath + job->base) == 40) { + fprintf(stderr, "GIT? %s", job->filepath); } - return ret; -} - -#define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));}; - -void fs_close(struct vfile *f) { - if (f->fd != -1) { - SHA1_Final(f->sha1_digest, &f->sha1_ctx); - close(f->fd); + if (ScanCtx.fast) { + return 0; } -} -void fs_reset(struct vfile *f) { - if (f->fd != -1) { - lseek(f->fd, 0, SEEK_SET); + // Get mime type with libmagic + if (job->vfile.read_rewindable == NULL) { + LOG_WARNING(job->filepath, + "File does not support rewindable reads, cannot guess Media type"); + return 0; } + + char *buf[MAGIC_BUF_SIZE]; + int bytes_read = job->vfile.read_rewindable(&job->vfile, buf, MAGIC_BUF_SIZE); + if (bytes_read < 0) { + if (job->vfile.is_fs_file) { + LOG_ERRORF(job->filepath, "read(): [%d] %s", errno, strerror(errno)); + } else { + LOG_ERRORF(job->filepath, "(virtual) read(): [%d] %s", bytes_read, archive_error_string(job->vfile.arc)); + } + + + return GET_MIME_ERROR_FATAL; + } + + char *magic_mime_str = magic_buffer_embedded(buf, bytes_read); + + if (magic_mime_str != NULL) { + mime = (int) mime_get_mime_by_string(magic_mime_str); + free(magic_mime_str); + + if (mime == 0) { + LOG_WARNINGF(job->filepath, "Couldn't find mime %s", magic_mime_str); + return 0; + } + } + + if (job->vfile.reset != NULL) { + job->vfile.reset(&job->vfile); + } + + return mime; } -void set_dbg_current_file(parse_job_t *job) { - unsigned long long pid = (unsigned long long) pthread_self(); - pthread_mutex_lock(&ScanCtx.dbg_current_files_mu); - g_hash_table_replace(ScanCtx.dbg_current_files, GINT_TO_POINTER(pid), job); - pthread_mutex_unlock(&ScanCtx.dbg_current_files_mu); -} +void parse(parse_job_t *job) { -void parse_job(parse_job_t *job) { - tpool_work_arg_shm_t *arg = malloc(sizeof(tpool_work_arg_shm_t) + sizeof(*job)); - - memcpy(arg->arg, job, sizeof(*job)); - arg->arg_size = -1; - - parse(arg); - - free(arg); -} - -void parse(tpool_work_arg_shm_t *arg) { - - parse_job_t *job = (void*)arg->arg; + if (job->vfile.is_fs_file) { + job->vfile.read = fs_read; + job->vfile.read_rewindable = fs_read; + job->vfile.reset = fs_reset; + job->vfile.close = fs_close; + job->vfile.calculate_checksum = ScanCtx.calculate_checksums; + } document_t *doc = malloc(sizeof(document_t)); - set_dbg_current_file(job); - strcpy(doc->filepath, job->filepath); - doc->ext = (short) job->ext; - doc->base = (short) job->base; - - char *rel_path = doc->filepath + ScanCtx.index.desc.root_len; - generate_doc_id(rel_path, doc->doc_id); - + doc->ext = job->ext; + doc->base = job->base; doc->meta_head = NULL; doc->meta_tail = NULL; - doc->mime = 0; doc->size = job->vfile.st_size; doc->mtime = (int) job->vfile.mtime; + doc->mime = get_mime(job); + generate_doc_id(doc->filepath + ScanCtx.index.desc.root_len, doc->doc_id); - int inc_ts = incremental_get(ScanCtx.original_table, doc->doc_id); - if (inc_ts != 0 && inc_ts == job->vfile.mtime) { - pthread_mutex_lock(&ScanCtx.copy_table_mu); - incremental_mark_file(ScanCtx.copy_table, doc->doc_id); - pthread_mutex_unlock(&ScanCtx.copy_table_mu); + if (doc->mime == GET_MIME_ERROR_FATAL) { + pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu); + ScanCtx.dbg_failed_files_count += 1; + pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu); + CLOSE_FILE(job->vfile) + free(doc); + return; + } + + if (database_mark_document(ProcData.index_db, doc->doc_id, doc->mtime)) { pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu); ScanCtx.dbg_skipped_files_count += 1; pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu); CLOSE_FILE(job->vfile) free(doc); - return; } - if (ScanCtx.new_table != NULL) { - pthread_mutex_lock(&ScanCtx.copy_table_mu); - incremental_mark_file(ScanCtx.new_table, doc->doc_id); - pthread_mutex_unlock(&ScanCtx.copy_table_mu); - } - - char *buf[MAGIC_BUF_SIZE]; - if (LogCtx.very_verbose) { - LOG_DEBUGF(job->filepath, "Starting parse job {%s}", doc->doc_id) + LOG_DEBUGF(job->filepath, "Starting parse job {%s}", doc->doc_id); } - if (job->ext > 4096) { - fprintf(stderr, "Ext is %d, filename is %s\n", job->ext, job->filepath); - } - - if (job->vfile.st_size == 0) { - doc->mime = MIME_EMPTY; - } else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) { - doc->mime = mime_get_mime_by_ext(ScanCtx.ext_table, job->filepath + job->ext); - } - - if (doc->mime == 0 && !ScanCtx.fast) { - - // Get mime type with libmagic - if (job->vfile.read_rewindable == NULL) { - LOG_WARNING(job->filepath, - "File does not support rewindable reads, cannot guess Media type"); - goto abort; - } - - int bytes_read = job->vfile.read_rewindable(&job->vfile, buf, MAGIC_BUF_SIZE); - if (bytes_read < 0) { - - if (job->vfile.is_fs_file) { - LOG_ERRORF(job->filepath, "read(): [%d] %s", errno, strerror(errno)) - } else { - LOG_ERRORF(job->filepath, "(virtual) read(): [%d] %s", bytes_read, archive_error_string(job->vfile.arc)) - } - - pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu); - ScanCtx.dbg_failed_files_count += 1; - pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu); - + switch (get_file_type(doc->mime, doc->size, doc->filepath)) { + case FILETYPE_RAW: + parse_raw(&ScanCtx.raw_ctx, &job->vfile, doc); + break; + case FILETYPE_MEDIA: + parse_media(&ScanCtx.media_ctx, &job->vfile, doc, mime_get_mime_text(doc->mime)); + break; + case FILETYPE_EBOOK: + parse_ebook(&ScanCtx.ebook_ctx, &job->vfile, mime_get_mime_text(doc->mime), doc); + break; + case FILETYPE_MARKUP: + parse_markup(&ScanCtx.text_ctx, &job->vfile, doc); + break; + case FILETYPE_TEXT: + parse_text(&ScanCtx.text_ctx, &job->vfile, doc); + break; + case FILETYPE_FONT: + parse_font(&ScanCtx.font_ctx, &job->vfile, doc); + break; + case FILETYPE_ARCHIVE: + parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc, ScanCtx.exclude, ScanCtx.exclude_extra); + break; + case FILETYPE_OOXML: + parse_ooxml(&ScanCtx.ooxml_ctx, &job->vfile, doc); + break; + case FILETYPE_COMIC: + parse_comic(&ScanCtx.comic_ctx, &job->vfile, doc); + break; + case FILETYPE_MOBI: + parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, doc); + break; + case FILETYPE_SIST2_SIDECAR: + parse_sidecar(&job->vfile, doc); CLOSE_FILE(job->vfile) free(doc); - return; - } - - magic_t magic = magic_open(MAGIC_MIME_TYPE); - - const char *magic_buffers[1] = {magic_database_buffer,}; - size_t sizes[1] = {sizeof(magic_database_buffer),}; - - int load_ret = magic_load_buffers(magic, (void **) &magic_buffers, sizes, 1); - - if (load_ret != 0) { - LOG_FATALF("parse.c", "Could not load libmagic database: (%d)", load_ret) - } - - const char *magic_mime_str = magic_buffer(magic, buf, bytes_read); - if (magic_mime_str != NULL) { - doc->mime = mime_get_mime_by_string(ScanCtx.mime_table, magic_mime_str); - - LOG_DEBUGF(job->filepath, "libmagic: %s", magic_mime_str); - - if (doc->mime == 0) { - LOG_WARNINGF(job->filepath, "Couldn't find mime %s", magic_mime_str); - } - } - - if (job->vfile.reset != NULL) { - job->vfile.reset(&job->vfile); - } - - magic_close(magic); + case FILETYPE_MSDOC: + parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, doc); + break; + case FILETYPE_JSON: + parse_json(&ScanCtx.json_ctx, &job->vfile, doc); + break; + case FILETYPE_NDJSON: + parse_ndjson(&ScanCtx.json_ctx, &job->vfile, doc); + break; + case FILETYPE_DONT_PARSE: + default: + break; } - int mmime = MAJOR_MIME(doc->mime); - - if (!(SHOULD_PARSE(doc->mime))) { - - } else if (IS_RAW(doc->mime)) { - parse_raw(&ScanCtx.raw_ctx, &job->vfile, doc); - } else if ((mmime == MimeVideo && doc->size >= MIN_VIDEO_SIZE) || - (mmime == MimeImage && doc->size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) { - - parse_media(&ScanCtx.media_ctx, &job->vfile, doc, mime_get_mime_text(doc->mime)); - - } else if (IS_PDF(doc->mime)) { - parse_ebook(&ScanCtx.ebook_ctx, &job->vfile, mime_get_mime_text(doc->mime), doc); - - } else if (mmime == MimeText && ScanCtx.text_ctx.content_size > 0) { - if (IS_MARKUP(doc->mime)) { - parse_markup(&ScanCtx.text_ctx, &job->vfile, doc); - } else { - parse_text(&ScanCtx.text_ctx, &job->vfile, doc); - } - - } else if (IS_FONT(doc->mime)) { - parse_font(&ScanCtx.font_ctx, &job->vfile, doc); - - } else if ( - ScanCtx.arc_ctx.mode != ARC_MODE_SKIP && ( - IS_ARC(doc->mime) || - (IS_ARC_FILTER(doc->mime) && should_parse_filtered_file(doc->filepath, doc->ext)) - )) { - parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc, ScanCtx.exclude, ScanCtx.exclude_extra); - } else if ((ScanCtx.ooxml_ctx.content_size > 0 || ScanCtx.media_ctx.tn_size > 0) && IS_DOC(doc->mime)) { - parse_ooxml(&ScanCtx.ooxml_ctx, &job->vfile, doc); - } else if (is_cbr(&ScanCtx.comic_ctx, doc->mime) || is_cbz(&ScanCtx.comic_ctx, doc->mime)) { - parse_comic(&ScanCtx.comic_ctx, &job->vfile, doc); - } else if (IS_MOBI(doc->mime)) { - parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, doc); - } else if (doc->mime == MIME_SIST2_SIDECAR) { - parse_sidecar(&job->vfile, doc); - CLOSE_FILE(job->vfile) - free(doc); - return; - } else if (is_msdoc(&ScanCtx.msdoc_ctx, doc->mime)) { - parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, doc); - } else if (is_json(&ScanCtx.json_ctx, doc->mime)) { - parse_json(&ScanCtx.json_ctx, &job->vfile, doc); - } else if (is_ndjson(&ScanCtx.json_ctx, doc->mime)) { - parse_ndjson(&ScanCtx.json_ctx, &job->vfile, doc); - } - - abort: - //Parent meta if (job->parent[0] != '\0') { meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + SIST_INDEX_ID_LEN); @@ -247,12 +254,8 @@ void parse(tpool_work_arg_shm_t *arg) { if (job->vfile.has_checksum) { char sha1_digest_str[SHA1_STR_LENGTH]; buf2hex((unsigned char *) job->vfile.sha1_digest, SHA1_DIGEST_LENGTH, (char *) sha1_digest_str); - APPEND_STR_META(doc, MetaChecksum, (const char *) sha1_digest_str); + APPEND_STR_META(doc, MetaChecksum, (const char *) sha1_digest_str) } write_document(doc); } - -void cleanup_parse() { - // noop -} diff --git a/src/parsing/parse.h b/src/parsing/parse.h index 55ecf18..4d090ec 100644 --- a/src/parsing/parse.h +++ b/src/parsing/parse.h @@ -4,15 +4,7 @@ #include "../sist.h" #include "src/tpool.h" -#define MAGIC_BUF_SIZE (4096 * 6) -int fs_read(struct vfile *f, void *buf, size_t size); -void fs_close(struct vfile *f); -void fs_reset(struct vfile *f); - -void parse_job(parse_job_t *job); -void parse(tpool_work_arg_shm_t *arg); - -void cleanup_parse(); +void parse(parse_job_t *arg); #endif diff --git a/src/parsing/sidecar.c b/src/parsing/sidecar.c index 0a8952a..5260649 100644 --- a/src/parsing/sidecar.c +++ b/src/parsing/sidecar.c @@ -4,12 +4,12 @@ void parse_sidecar(vfile_t *vfile, document_t *doc) { - LOG_DEBUGF("sidecar.c", "Parsing sidecar file %s", vfile->filepath) + LOG_DEBUGF("sidecar.c", "Parsing sidecar file %s", vfile->filepath); size_t size; char *buf = read_all(vfile, &size); if (buf == NULL) { - LOG_ERRORF("sidecar.c", "Read error for %s", vfile->filepath) + LOG_ERRORF("sidecar.c", "Read error for %s", vfile->filepath); return; } @@ -18,7 +18,7 @@ void parse_sidecar(vfile_t *vfile, document_t *doc) { cJSON *json = cJSON_Parse(buf); if (json == NULL) { - LOG_ERRORF("sidecar.c", "Could not parse JSON sidecar %s", vfile->filepath) + LOG_ERRORF("sidecar.c", "Could not parse JSON sidecar %s", vfile->filepath); return; } char *json_str = cJSON_PrintUnformatted(json); @@ -32,8 +32,7 @@ void parse_sidecar(vfile_t *vfile, document_t *doc) { generate_doc_id(rel_path, assoc_doc_id); - store_write(ScanCtx.index.meta_store, assoc_doc_id, sizeof(assoc_doc_id), json_str, - strlen(json_str) + 1); + database_write_document_sidecar(ProcData.index_db, assoc_doc_id, json_str); cJSON_Delete(json); free(json_str); diff --git a/src/sist.h b/src/sist.h index 30ded55..8e6eb5a 100644 --- a/src/sist.h +++ b/src/sist.h @@ -49,8 +49,11 @@ #include #include "git_hash.h" -#define VERSION "2.14.3" +#define VERSION "3.0.0" static const char *const Version = VERSION; +static const int VersionMajor = 3; +static const int VersionMinor = 0; +static const int VersionPatch = 0; #ifndef SIST_PLATFORM #define SIST_PLATFORM unknown diff --git a/src/stats.c b/src/stats.c deleted file mode 100644 index f94a1a6..0000000 --- a/src/stats.c +++ /dev/null @@ -1,343 +0,0 @@ -#include "sist.h" -#include "io/serialize.h" -#include "ctx.h" - -static GHashTable *FlatTree; -static GHashTable *BufferTable; - -static GHashTable *AggMime; -static GHashTable *AggSize; -static GHashTable *AggDate; - -#define SIZE_BUCKET (long)(5 * 1024 * 1024) -#define DATE_BUCKET (long)(2629800) - -static long TotalSize = 0; -static long DocumentCount = 0; - -typedef struct { - long size; - long count; -} agg_t; - -void fill_tables(cJSON *document, UNUSED(const char index_id[SIST_INDEX_ID_LEN])) { - - if (cJSON_GetObjectItem(document, "parent") != NULL) { - return; - } - - const char *json_path = cJSON_GetObjectItem(document, "path")->valuestring; - char *path = malloc(strlen(json_path) + 1); - strcpy(path, json_path); - - const char *json_mime = cJSON_GetObjectItem(document, "mime")->valuestring; - char *mime; - if (json_mime == NULL) { - mime = NULL; - } else { - mime = malloc(strlen(json_mime) + 1); - strcpy(mime, json_mime); - } - - long size = (long) cJSON_GetObjectItem(document, "size")->valuedouble; - int mtime = cJSON_GetObjectItem(document, "mtime")->valueint; - - // treemap - void *existing_path = g_hash_table_lookup(FlatTree, path); - if (existing_path == NULL) { - g_hash_table_insert(FlatTree, path, (gpointer) size); - } else { - g_hash_table_replace(FlatTree, path, (gpointer) ((long) existing_path + size)); - } - - // mime agg - if (mime != NULL) { - agg_t *orig_agg = g_hash_table_lookup(AggMime, mime); - if (orig_agg == NULL) { - agg_t *agg = malloc(sizeof(agg_t)); - agg->size = size; - agg->count = 1; - g_hash_table_insert(AggMime, mime, agg); - } else { - orig_agg->size += size; - orig_agg->count += 1; - free(mime); - } - } - - // size agg - long size_bucket = size - (size % SIZE_BUCKET); - agg_t *orig_agg = g_hash_table_lookup(AggSize, (gpointer) size_bucket); - if (orig_agg == NULL) { - agg_t *agg = malloc(sizeof(agg_t)); - agg->size = size; - agg->count = 1; - g_hash_table_insert(AggSize, (gpointer) size_bucket, agg); - } else { - orig_agg->count += 1; - orig_agg->size += size; - } - - // date agg - long date_bucket = mtime - (mtime % DATE_BUCKET); - orig_agg = g_hash_table_lookup(AggDate, (gpointer) date_bucket); - if (orig_agg == NULL) { - agg_t *agg = malloc(sizeof(agg_t)); - agg->size = size; - agg->count = 1; - g_hash_table_insert(AggDate, (gpointer) date_bucket, agg); - } else { - orig_agg->count += 1; - orig_agg->size += size; - } - - TotalSize += size; - DocumentCount += 1; -} - -void read_index_into_tables(index_t *index) { - char file_path[PATH_MAX]; - READ_INDICES(file_path, index->path, read_index(file_path, index->desc.id, index->desc.type, fill_tables), {}, 1); -} - -static size_t rfind(const char *str, int c) { - for (int i = (int)strlen(str); i >= 0; i--) { - if (str[i] == c) { - return i; - } - } - return -1; -} - -int merge_up(double thresh) { - long min_size = (long) (thresh * (double) TotalSize); - - int count = 0; - GHashTableIter iter; - g_hash_table_iter_init(&iter, FlatTree); - - void *key; - void *value; - - while (g_hash_table_iter_next(&iter, &key, &value)) { - long size = (long) value; - - if (size < min_size) { - int stop = rfind(key, '/'); - if (stop == -1) { - stop = 0; - } - char *parent = malloc(stop + 1); - strncpy(parent, key, stop); - *(parent + stop) = '\0'; - - void *existing_parent = g_hash_table_lookup(FlatTree, parent); - if (existing_parent == NULL) { - void *existing_parent2_key; - void *existing_parent2_val; - int found = g_hash_table_lookup_extended(BufferTable, parent, &existing_parent2_key, - &existing_parent2_val); - if (!found) { - g_hash_table_insert(BufferTable, parent, value); - } else { - g_hash_table_replace(BufferTable, parent, (gpointer) ((long) existing_parent2_val + size)); - free(existing_parent2_key); - } - } else { - g_hash_table_replace(FlatTree, parent, (gpointer) ((long) existing_parent + size)); - } - - g_hash_table_iter_remove(&iter); - - count += 1; - } - } - - g_hash_table_iter_init(&iter, BufferTable); - while (g_hash_table_iter_next(&iter, &key, &value)) { - g_hash_table_insert(FlatTree, key, value); - g_hash_table_iter_remove(&iter); - } - - int size = g_hash_table_size(FlatTree); - - LOG_DEBUGF("stats.c", "Merge up iteration (%d merged, %d in tree)", count, size) - return count; -} - -/** - * Assumes out is at at least PATH_MAX *4 - */ -void csv_escape(char *dst, const char *str) { - - const char *ptr = str; - char *out = dst; - - if (rfind(str, ',') == -1 && rfind(str, '"') == -1) { - strcpy(dst, str); - return; - } - - *out++ = '"'; - char c; - while ((c = *ptr++) != 0) { - if (c == '"') { - *out++ = '"'; - *out++ = '"'; - } else { - *out++ = c; - } - } - *out++ = '"'; - *out = '\0'; -} - -int open_or_exit(const char *path) { - int fd = open(path, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); - if (fd < 0) { - LOG_FATALF("stats.c", "Error while creating file: %s [%d]\n", strerror(errno), errno) - } - return fd; -} - -#define TREEMAP_CSV_HEADER "path,size" -#define MIME_AGG_CSV_HEADER "mime,size,count" -#define SIZE_AGG_CSV_HEADER "bucket,size,count" -#define DATE_AGG_CSV_HEADER "bucket,size,count" - -void write_treemap_csv(double thresh, const char *out_path) { - - void *key; - void *value; - - long min_size = (long) (thresh * (double) TotalSize); - - int fd = open_or_exit(out_path); - int ret = write(fd, TREEMAP_CSV_HEADER, sizeof(TREEMAP_CSV_HEADER) - 1); - if (ret == -1) { - LOG_FATALF("stats.c", "Write error: %s", strerror(errno)) - } - - GHashTableIter iter; - g_hash_table_iter_init(&iter, FlatTree); - while (g_hash_table_iter_next(&iter, &key, &value)) { - long size = (long) value; - - if (size >= min_size) { - char path_buf[PATH_MAX * 4]; - char buf[PATH_MAX * 4 + 16]; - - csv_escape(path_buf, key); - size_t written = sprintf(buf, "\n%s,%ld", path_buf, (long) value); - ret = write(fd, buf, written); - if (ret == -1) { - LOG_FATALF("stats.c", "Write error: %s", strerror(errno)) - } - } - } - close(fd); -} - -void write_agg_csv_str(const char *out_path, const char *header, GHashTable *table) { - void *key; - void *value; - char buf[4096]; - - int fd = open_or_exit(out_path); - int ret = write(fd, header, strlen(header)); - if (ret == -1) { - LOG_FATALF("stats.c", "Write error: %s", strerror(errno)) - } - - GHashTableIter iter; - g_hash_table_iter_init(&iter, table); - while (g_hash_table_iter_next(&iter, &key, &value)) { - agg_t *agg = value; - - size_t written = sprintf(buf, "\n%s,%ld,%ld", (const char*)key, agg->size, agg->count); - ret = write(fd, buf, written); - if (ret == -1) { - LOG_FATALF("stats.c", "Write error: %s", strerror(errno)) - } - } - - close(fd); -} - -void write_agg_csv_long(const char *out_path, const char *header, GHashTable *table) { - void *key; - void *value; - char buf[4096]; - - int fd = open_or_exit(out_path); - int ret = write(fd, header, strlen(header)); - if (ret == -1) { - LOG_FATALF("stats.c", "Write error: %s", strerror(errno)) - } - - GHashTableIter iter; - g_hash_table_iter_init(&iter, table); - while (g_hash_table_iter_next(&iter, &key, &value)) { - agg_t *agg = value; - size_t written = sprintf(buf, "\n%ld,%ld,%ld", (long)key, agg->size, agg->count); - ret = write(fd, buf, written); - if (ret == -1) { - LOG_FATALF("stats.c", "Write error: %s", strerror(errno)) - } - } - - close(fd); -} - -int generate_stats(index_t *index, const double threshold, const char *out_prefix) { - - FlatTree = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL); - BufferTable = g_hash_table_new(g_str_hash, g_str_equal); - - AggMime = g_hash_table_new_full(g_str_hash, g_str_equal, free, free); - AggSize = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, free); - AggDate = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, free); - - LOG_INFO("stats.c", "Generating stats...") - - read_index_into_tables(index); - - LOG_DEBUG("stats.c", "Read index into tables") - LOG_DEBUGF("stats.c", "Total size is %ld", TotalSize) - LOG_DEBUGF("stats.c", "Document count is %ld", DocumentCount) - LOG_DEBUGF("stats.c", "Merging small directories upwards with a threshold of %f%%", threshold * 100) - - while (merge_up(threshold) > 100) {} - - char tmp[PATH_MAX]; - - strncpy(tmp, out_prefix, sizeof(tmp)); - strcat(tmp, "treemap.csv"); - write_treemap_csv(threshold, tmp); - - strncpy(tmp, out_prefix, sizeof(tmp)); - strcat(tmp, "mime_agg.csv"); - write_agg_csv_str(tmp, MIME_AGG_CSV_HEADER, AggMime); - - strncpy(tmp, out_prefix, sizeof(tmp)); - strcat(tmp, "size_agg.csv"); - write_agg_csv_long(tmp, SIZE_AGG_CSV_HEADER, AggSize); - - strncpy(tmp, out_prefix, sizeof(tmp)); - strcat(tmp, "date_agg.csv"); - write_agg_csv_long(tmp, DATE_AGG_CSV_HEADER, AggDate); - - g_hash_table_remove_all(FlatTree); - g_hash_table_destroy(FlatTree); - g_hash_table_destroy(BufferTable); - - g_hash_table_remove_all(AggMime); - g_hash_table_destroy(AggMime); - g_hash_table_remove_all(AggSize); - g_hash_table_destroy(AggSize); - g_hash_table_remove_all(AggDate); - g_hash_table_destroy(AggDate); - - return 0; -} - diff --git a/src/stats.h b/src/stats.h deleted file mode 100644 index 250874c..0000000 --- a/src/stats.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef SIST2_STATS_H -#define SIST2_STATS_H - -int generate_stats(index_t *index, double threshold, const char* out_prefix); - -#endif diff --git a/src/tpool.c b/src/tpool.c index 8678e22..8f665f8 100644 --- a/src/tpool.c +++ b/src/tpool.c @@ -4,257 +4,250 @@ #include #include #include -#include "mempool/mempool.h" +#include "parsing/parse.h" #define BLANK_STR " " -// TODO: Use slab OOM to control queue size -#define MAX_QUEUE_SIZE 100000 -typedef struct tpool_work { - tpool_work_arg_shm_t *arg; - thread_func_t func; - struct tpool_work *next; -} tpool_work_t; +typedef struct { + int thread_id; + tpool_t *pool; +} start_thread_arg_t; + typedef struct tpool { - tpool_work_t *work_head; - tpool_work_t *work_tail; - - pthread_mutex_t work_mutex; - pthread_mutex_t mem_mutex; - - // TODO: Initialize with SHARED attr - pthread_cond_t has_work_cond; - pthread_cond_t working_cond; - pthread_t threads[256]; - - int thread_cnt; - int work_cnt; - int done_cnt; - int busy_cnt; - - int stop; - int waiting; + int num_threads; + int fork; int print_progress; - void (*cleanup_func)(); - - void *shared_memory; - size_t shared_memory_size; - ncx_slab_pool_t *mempool; + struct { + job_type_t job_type; + int stop; + int waiting; + database_ipc_ctx_t ipc_ctx; + pthread_mutex_t mutex; + pthread_mutex_t data_mutex; + pthread_cond_t done_working_cond; + pthread_cond_t workers_initialized_cond; + int busy_count; + int initialized_count; + } *shm; } tpool_t; - -/** - * Create a work object - */ -static tpool_work_t *tpool_work_create(tpool_t *pool, thread_func_t func, tpool_work_arg_t *arg) { - - if (func == NULL) { - return NULL; +void job_destroy(job_t *job) { + if (job->type == JOB_PARSE_JOB) { + free(job->parse_job); } - // Copy heap arg to shm arg - pthread_mutex_lock(&pool->mem_mutex); - - tpool_work_arg_shm_t *shm_arg = ncx_slab_alloc(pool->mempool, sizeof(tpool_work_arg_shm_t) + arg->arg_size); - - shm_arg->arg_size = arg->arg_size; - memcpy(shm_arg->arg, arg->arg, arg->arg_size); - - free(arg->arg); - - tpool_work_t *work = ncx_slab_alloc(pool->mempool, sizeof(tpool_work_t)); - - pthread_mutex_unlock(&pool->mem_mutex); - - work->func = func; - work->arg = shm_arg; - work->next = NULL; - - return work; + free(job); } void tpool_dump_debug_info(tpool_t *pool) { - LOG_DEBUGF("tpool.c", "pool->thread_cnt = %d", pool->thread_cnt) - LOG_DEBUGF("tpool.c", "pool->work_cnt = %d", pool->work_cnt) - LOG_DEBUGF("tpool.c", "pool->done_cnt = %d", pool->done_cnt) - LOG_DEBUGF("tpool.c", "pool->busy_cnt = %d", pool->busy_cnt) - LOG_DEBUGF("tpool.c", "pool->stop = %d", pool->stop) -} - -/** - * Pop work object from thread pool - */ -static tpool_work_t *tpool_work_get(tpool_t *pool) { - - tpool_work_t *work = pool->work_head; - if (work == NULL) { - return NULL; - } - - if (work->next == NULL) { - pool->work_head = NULL; - pool->work_tail = NULL; - } else { - pool->work_head = work->next; - } - - return work; + // TODO + LOG_DEBUGF("tpool.c", "pool->num_threads = %d", pool->num_threads); } /** * Push work object to thread pool */ -int tpool_add_work(tpool_t *pool, thread_func_t func, tpool_work_arg_t *arg) { +int tpool_add_work(tpool_t *pool, job_t *job) { - while ((pool->work_cnt - pool->done_cnt) >= MAX_QUEUE_SIZE) { - usleep(10000); - } - tpool_work_t *work = tpool_work_create(pool, func, arg); - if (work == NULL) { - return 0; + if (pool->shm->job_type == JOB_UNDEFINED) { + pool->shm->job_type = job->type; + } else if (pool->shm->job_type != job->type) { + LOG_FATAL("tpool.c", "FIXME: tpool cannot queue jobs with different types!"); } - pthread_mutex_lock(&(pool->work_mutex)); - if (pool->work_head == NULL) { - pool->work_head = work; - pool->work_tail = pool->work_head; - } else { - pool->work_tail->next = work; - pool->work_tail = work; - } + database_add_work(ProcData.ipc_db, job); - pool->work_cnt++; - - pthread_cond_broadcast(&(pool->has_work_cond)); - pthread_mutex_unlock(&(pool->work_mutex)); - - return 1; + return TRUE; } static void worker_thread_loop(tpool_t *pool) { while (TRUE) { - pthread_mutex_lock(&pool->work_mutex); - if (pool->stop) { + if (pool->shm->stop) { break; } - if (pool->work_head == NULL) { - pthread_cond_wait(&(pool->has_work_cond), &(pool->work_mutex)); + if (pool->shm->job_type == JOB_UNDEFINED) { + // Wait before first job is queued + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_timedwait_ms(&pool->shm->ipc_ctx.has_work_cond, &pool->shm->mutex, 1000); + pthread_mutex_unlock(&pool->shm->mutex); } - tpool_work_t *work = tpool_work_get(pool); + job_t *job = database_get_work(ProcData.ipc_db, pool->shm->job_type); - if (work != NULL) { - pool->busy_cnt += 1; - } + if (job != NULL) { + pthread_mutex_lock(&(pool->shm->data_mutex)); + pool->shm->busy_count += 1; + pthread_mutex_unlock(&(pool->shm->data_mutex)); - pthread_mutex_unlock(&(pool->work_mutex)); - - if (work != NULL) { - if (pool->stop) { + if (pool->shm->stop) { break; } - work->func(work->arg); + if (job->type == JOB_PARSE_JOB) { + parse(job->parse_job); + } else if (job->type == JOB_BULK_LINE) { + elastic_index_line(job->bulk_line); + } - pthread_mutex_lock(&pool->mem_mutex); - ncx_slab_free(pool->mempool, work->arg); - ncx_slab_free(pool->mempool, work); - pthread_mutex_unlock(&pool->mem_mutex); - } + job_destroy(job); - pthread_mutex_lock(&(pool->work_mutex)); - if (work != NULL) { - pool->busy_cnt -= 1; - pool->done_cnt++; + pthread_mutex_lock(&(pool->shm->data_mutex)); + pool->shm->busy_count -= 1; + pthread_mutex_unlock(&(pool->shm->data_mutex)); + + pthread_mutex_lock(&(pool->shm->ipc_ctx.mutex)); + pool->shm->ipc_ctx.completed_job_count += 1; + pthread_mutex_unlock(&(pool->shm->ipc_ctx.mutex)); } if (pool->print_progress) { + + int done = pool->shm->ipc_ctx.completed_job_count; + int count = pool->shm->ipc_ctx.completed_job_count + pool->shm->ipc_ctx.job_count; + if (LogCtx.json_logs) { - progress_bar_print_json(pool->done_cnt, pool->work_cnt, ScanCtx.stat_tn_size, - ScanCtx.stat_index_size, pool->waiting); + progress_bar_print_json(done, + count, + ScanCtx.stat_tn_size, + ScanCtx.stat_index_size, pool->shm->waiting); } else { - progress_bar_print((double) pool->done_cnt / pool->work_cnt, ScanCtx.stat_tn_size, - ScanCtx.stat_index_size); + progress_bar_print((double) done / count, + ScanCtx.stat_tn_size, ScanCtx.stat_index_size); } } - if (pool->work_head == NULL) { - pthread_cond_signal(&(pool->working_cond)); + if (job == NULL) { + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_signal(&pool->shm->done_working_cond); + pthread_mutex_unlock(&pool->shm->mutex); } - pthread_mutex_unlock(&(pool->work_mutex)); } } +static void worker_proc_init(tpool_t *pool, int thread_id) { + // TODO create PID -> thread_id mapping for signal handler + + ProcData.thread_id = thread_id; + + if (ScanCtx.index.path[0] != '\0') { + // TODO This should be closed in proc cleanup function + ProcData.index_db = database_create(ScanCtx.index.path, INDEX_DATABASE); + ProcData.index_db->ipc_ctx = &pool->shm->ipc_ctx; + database_open(ProcData.index_db); + } + + // TODO /dev/shm + pthread_mutex_lock(&pool->shm->mutex); + ProcData.ipc_db = database_create("/dev/shm/ipc.sist2", IPC_CONSUMER_DATABASE); + ProcData.ipc_db->ipc_ctx = &pool->shm->ipc_ctx; + database_open(ProcData.ipc_db); + pthread_mutex_unlock(&pool->shm->mutex); +} + +void worker_proc_cleanup(tpool_t* pool) { + if (ProcData.index_db != NULL) { + database_close(ProcData.index_db, FALSE); + } + database_close(ProcData.ipc_db, FALSE); +} + /** * Thread worker function */ static void *tpool_worker(void *arg) { - tpool_t *pool = arg; + tpool_t *pool = ((start_thread_arg_t *) arg)->pool; - int pid = fork(); + if (pool->fork) { + while (TRUE) { + int pid = fork(); - if (pid == 0) { + if (pid == 0) { + worker_proc_init(pool, ((start_thread_arg_t *) arg)->thread_id); + + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_signal(&pool->shm->workers_initialized_cond); + pool->shm->initialized_count += 1; + pthread_mutex_unlock(&pool->shm->mutex); + + worker_thread_loop(pool); + + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_signal(&pool->shm->done_working_cond); + pthread_mutex_unlock(&pool->shm->mutex); + + worker_proc_cleanup(pool); + + exit(0); + + } else { + int status; + // TODO: On crash, print debug info and resume thread + waitpid(pid, &status, 0); + + LOG_DEBUGF("tpool.c", "Child process terminated with status code %d", WEXITSTATUS(status)); + + pthread_mutex_lock(&(pool->shm->ipc_ctx.mutex)); + pool->shm->ipc_ctx.completed_job_count += 1; + pthread_mutex_unlock(&(pool->shm->ipc_ctx.mutex)); + + pthread_mutex_lock(&(pool->shm->data_mutex)); + pool->shm->busy_count -= 1; + pthread_mutex_unlock(&(pool->shm->data_mutex)); + + if (WIFSIGNALED(status)) { + // TODO: Get current_job based on PID + const char *job_filepath = "TODO"; + + LOG_FATALF_NO_EXIT( + "tpool.c", + "Child process was terminated by signal (%s).\n" + BLANK_STR "The process was working on %s", + strsignal(WTERMSIG(status)), + job_filepath + ); + } + break; + } + } + + } else { + worker_proc_init(pool, ((start_thread_arg_t *) arg)->thread_id); + + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_signal(&pool->shm->workers_initialized_cond); + pool->shm->initialized_count += 1; + pthread_mutex_unlock(&pool->shm->mutex); worker_thread_loop(pool); - if (pool->cleanup_func != NULL) { - LOG_INFO("tpool.c", "Executing cleanup function") - pool->cleanup_func(); - LOG_DEBUG("tpool.c", "Done executing cleanup function") - } + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_signal(&pool->shm->done_working_cond); + pthread_mutex_unlock(&pool->shm->mutex); - pthread_cond_signal(&(pool->working_cond)); - pthread_mutex_unlock(&(pool->work_mutex)); - exit(0); - - } else { - int status; - // TODO: On crash, print debug info and resume thread - waitpid(pid, &status, 0); - - LOG_DEBUGF("tpool.c", "Child process terminated with status code %d", WEXITSTATUS(status)) - - pthread_mutex_lock(&(pool->work_mutex)); - pool->busy_cnt -= 1; - pool->done_cnt++; - pthread_mutex_unlock(&(pool->work_mutex)); - - if (WIFSIGNALED(status)) { -// parse_job_t *job = g_hash_table_lookup(ScanCtx.dbg_current_files, GINT_TO_POINTER(pthread_self())); - const char *job_filepath = "TODO"; - - LOG_FATALF_NO_EXIT( - "tpool.c", - "Child process was terminated by signal (%s).\n" - BLANK_STR "The process was working on %s", - strsignal(WTERMSIG(status)), - job_filepath - ) - } + return NULL; } return NULL; } void tpool_wait(tpool_t *pool) { - LOG_DEBUG("tpool.c", "Waiting for worker threads to finish") - pthread_mutex_lock(&(pool->work_mutex)); + LOG_DEBUG("tpool.c", "Waiting for worker threads to finish"); + pthread_mutex_lock(&pool->shm->mutex); - pool->waiting = TRUE; + pool->shm->waiting = TRUE; + pool->shm->ipc_ctx.no_more_jobs = TRUE; while (TRUE) { - if (pool->done_cnt < pool->work_cnt) { - pthread_cond_wait(&(pool->working_cond), &(pool->work_mutex)); + if (pool->shm->ipc_ctx.job_count > 0) { + pthread_cond_wait(&(pool->shm->done_working_cond), &pool->shm->mutex); } else { - LOG_INFOF("tpool.c", "Received head=NULL signal, busy_cnt=%d", pool->busy_cnt); - - if (pool->done_cnt == pool->work_cnt && pool->busy_cnt == 0) { - pool->stop = TRUE; + if (pool->shm->ipc_ctx.job_count == 0 && pool->shm->busy_count == 0) { + pool->shm->stop = TRUE; break; } } @@ -262,34 +255,25 @@ void tpool_wait(tpool_t *pool) { if (pool->print_progress && !LogCtx.json_logs) { progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size); } - pthread_mutex_unlock(&(pool->work_mutex)); + pthread_mutex_unlock(&pool->shm->mutex); - LOG_INFO("tpool.c", "Worker threads finished") + LOG_INFO("tpool.c", "Worker threads finished"); } void tpool_destroy(tpool_t *pool) { - if (pool == NULL) { - return; - } + LOG_INFO("tpool.c", "Destroying thread pool"); - LOG_INFO("tpool.c", "Destroying thread pool") + database_close(ProcData.ipc_db, FALSE); - pthread_mutex_lock(&(pool->work_mutex)); - tpool_work_t *work = pool->work_head; int count = 0; - while (work != NULL) { - tpool_work_t *tmp = work->next; - free(work); - work = tmp; - count += 1; - } LOG_DEBUGF("tpool.c", "Destroyed %d jobs", count); - pthread_cond_broadcast(&(pool->has_work_cond)); - pthread_mutex_unlock(&(pool->work_mutex)); + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_broadcast(&pool->shm->ipc_ctx.has_work_cond); + pthread_mutex_unlock(&pool->shm->mutex); - for (size_t i = 0; i < pool->thread_cnt; i++) { + for (size_t i = 0; i < pool->num_threads; i++) { pthread_t thread = pool->threads[i]; if (thread != 0) { void *_; @@ -297,42 +281,33 @@ void tpool_destroy(tpool_t *pool) { } } - LOG_INFO("tpool.c", "Final cleanup") + pthread_mutex_destroy(&pool->shm->ipc_ctx.mutex); + pthread_mutex_destroy(&pool->shm->mutex); + pthread_cond_destroy(&pool->shm->ipc_ctx.has_work_cond); + pthread_cond_destroy(&pool->shm->done_working_cond); - pthread_mutex_destroy(&(pool->work_mutex)); - pthread_cond_destroy(&(pool->has_work_cond)); - pthread_cond_destroy(&(pool->working_cond)); - - munmap(pool->shared_memory, pool->shared_memory_size); + munmap(pool->shm, sizeof(*pool->shm)); } /** * Create a thread pool * @param thread_cnt Worker threads count */ -tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int print_progress) { +tpool_t *tpool_create(int thread_cnt, int print_progress) { - size_t shm_size = 1024 * 1024 * 2000; + int fork = FALSE; - void *shared_memory = mmap(NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + tpool_t *pool = malloc(sizeof(tpool_t)); - tpool_t *pool = (tpool_t *) shared_memory; - pool->shared_memory = shared_memory; - pool->shared_memory_size = shm_size; - pool->mempool = (ncx_slab_pool_t *) (pool->shared_memory + sizeof(tpool_t)); - pool->mempool->addr = pool->mempool; - pool->mempool->min_shift = 4; - pool->mempool->end = pool->shared_memory + shm_size; + pool->shm = mmap(NULL, sizeof(*pool->shm), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - ncx_slab_init(pool->mempool); - - pool->thread_cnt = thread_cnt; - pool->work_cnt = 0; - pool->done_cnt = 0; - pool->busy_cnt = 0; - pool->stop = FALSE; - pool->waiting = FALSE; - pool->cleanup_func = cleanup_func; + pool->fork = fork; + pool->num_threads = thread_cnt; + pool->shm->ipc_ctx.job_count = 0; + pool->shm->ipc_ctx.no_more_jobs = FALSE; + pool->shm->stop = FALSE; + pool->shm->waiting = FALSE; + pool->shm->job_type = JOB_UNDEFINED; memset(pool->threads, 0, sizeof(pool->threads)); pool->print_progress = print_progress; @@ -340,27 +315,50 @@ tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int print_progress) { pthread_mutexattr_init(&mutexattr); pthread_mutexattr_setpshared(&mutexattr, TRUE); - pthread_mutex_init(&(pool->work_mutex), &mutexattr); - pthread_mutex_init(&(pool->mem_mutex), &mutexattr); + pthread_mutex_init(&(pool->shm->mutex), &mutexattr); + pthread_mutex_init(&(pool->shm->data_mutex), &mutexattr); + pthread_mutex_init(&(pool->shm->ipc_ctx.mutex), &mutexattr); + pthread_mutex_init(&(pool->shm->ipc_ctx.db_mutex), &mutexattr); + pthread_mutex_init(&(pool->shm->ipc_ctx.index_db_mutex), &mutexattr); pthread_condattr_t condattr; pthread_condattr_init(&condattr); pthread_condattr_setpshared(&condattr, TRUE); - pthread_cond_init(&(pool->has_work_cond), &condattr); - pthread_cond_init(&(pool->working_cond), &condattr); + pthread_cond_init(&(pool->shm->ipc_ctx.has_work_cond), &condattr); + pthread_cond_init(&(pool->shm->done_working_cond), &condattr); + pthread_cond_init(&(pool->shm->workers_initialized_cond), &condattr); - pool->work_head = NULL; - pool->work_tail = NULL; + remove("/dev/shm/ipc.sist2"); + remove("/dev/shm/ipc.sist2-wal"); + remove("/dev/shm/ipc.sist2-shm"); + ProcData.ipc_db = database_create("/dev/shm/ipc.sist2", IPC_PRODUCER_DATABASE); + ProcData.ipc_db->ipc_ctx = &pool->shm->ipc_ctx; + database_initialize(ProcData.ipc_db); return pool; } void tpool_start(tpool_t *pool) { - LOG_INFOF("tpool.c", "Starting thread pool with %d threads", pool->thread_cnt) + LOG_INFOF("tpool.c", "Starting thread pool with %d threads", pool->num_threads); - for (size_t i = 0; i < pool->thread_cnt; i++) { - pthread_create(&pool->threads[i], NULL, tpool_worker, pool); + pthread_mutex_lock(&pool->shm->mutex); + + for (int i = 0; i < pool->num_threads; i++) { + + start_thread_arg_t *arg = malloc(sizeof(start_thread_arg_t)); + arg->thread_id = i + 1; + arg->pool = pool; + + pthread_create(&pool->threads[i], NULL, tpool_worker, arg); } + + // Only open the database when all workers are done initializing + while (pool->shm->initialized_count != pool->num_threads) { + pthread_cond_wait(&pool->shm->workers_initialized_cond, &pool->shm->mutex); + } + pthread_mutex_unlock(&pool->shm->mutex); + + database_open(ProcData.ipc_db); } diff --git a/src/tpool.h b/src/tpool.h index 8f73449..71742d8 100644 --- a/src/tpool.h +++ b/src/tpool.h @@ -2,34 +2,27 @@ #define SIST2_TPOOL_H #include "sist.h" +#include "third-party/libscan/libscan/scan.h" +#include "index/elastic.h" +#include "src/database/database.h" struct tpool; typedef struct tpool tpool_t; -typedef struct { - size_t arg_size; - void *arg; -} tpool_work_arg_t; - -typedef struct { - size_t arg_size; - char arg[0]; -} tpool_work_arg_shm_t; - -typedef void (*thread_func_t)(tpool_work_arg_shm_t *arg); - -tpool_t *tpool_create(int num, void (*cleanup_func)(), int print_progress); +tpool_t *tpool_create(int num, int print_progress); void tpool_start(tpool_t *pool); void tpool_destroy(tpool_t *pool); -int tpool_add_work(tpool_t *pool, thread_func_t func, tpool_work_arg_t *arg); +int tpool_add_work(tpool_t *pool, job_t *job); void tpool_wait(tpool_t *pool); void tpool_dump_debug_info(tpool_t *pool); +void job_destroy(job_t *job); + #endif diff --git a/src/types.h b/src/types.h index 1ba8a66..f4528dd 100644 --- a/src/types.h +++ b/src/types.h @@ -1,24 +1,26 @@ #ifndef SIST2_TYPES_H #define SIST2_TYPES_H -#define INDEX_TYPE_NDJSON "ndjson" +typedef struct database database_t; typedef struct index_descriptor { char id[SIST_INDEX_ID_LEN]; char version[64]; + int version_major; + int version_minor; + int version_patch; long timestamp; char root[PATH_MAX]; char rewrite_url[8192]; - short root_len; + int root_len; char name[1024]; - char type[64]; } index_descriptor_t; typedef struct index_t { struct index_descriptor desc; - struct store_t *store; - struct store_t *tag_store; - struct store_t *meta_store; + + database_t *db; + char path[PATH_MAX]; } index_t; diff --git a/src/util.c b/src/util.c index 79d5bd2..5f6220c 100644 --- a/src/util.c +++ b/src/util.c @@ -25,7 +25,6 @@ dyn_buffer_t url_escape(char *str) { } char *abspath(const char *path) { - char *expanded = expandpath(path); char *abs = realpath(expanded, NULL); @@ -34,8 +33,7 @@ char *abspath(const char *path) { return NULL; } if (strlen(abs) > 1) { - abs = realloc(abs, strlen(abs) + 2); - strcat(abs, "/"); + abs = realloc(abs, strlen(abs) + 1); } return abs; @@ -76,9 +74,8 @@ char *expandpath(const char *path) { } } - char *expanded = malloc(strlen(tmp) + 2); + char *expanded = malloc(strlen(tmp) + 1); strcpy(expanded, tmp); - strcat(expanded, "/"); wordfree(&w); return expanded; @@ -103,6 +100,10 @@ void progress_bar_print_json(size_t done, size_t count, size_t tn_size, size_t i void progress_bar_print(double percentage, size_t tn_size, size_t index_size) { + if (isnan(percentage)) { + return; + } + // TODO: Fix this with shm/ctx static int last_val = -1; @@ -150,10 +151,6 @@ void progress_bar_print(double percentage, size_t tn_size, size_t index_size) { PrintingProgressBar = TRUE; } -GHashTable *incremental_get_table() { - GHashTable *file_table = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL); - return file_table; -} const char *find_file_in_paths(const char *paths[], const char *filename) { @@ -167,7 +164,7 @@ const char *find_file_in_paths(const char *paths[], const char *filename) { char path[PATH_MAX]; snprintf(path, sizeof(path), "%s%s", apath, filename); - LOG_DEBUGF("util.c", "Looking for '%s' in folder '%s'", filename, apath) + LOG_DEBUGF("util.c", "Looking for '%s' in folder '%s'", filename, apath); free(apath); struct stat info; @@ -269,3 +266,39 @@ void str_unescape(char *dst, const char *str) { } *cur = '\0'; } + +#define NSEC_PER_SEC 1000000000 + +struct timespec timespec_normalise(struct timespec ts) { + while (ts.tv_nsec >= NSEC_PER_SEC) { + ts.tv_sec += 1; + ts.tv_nsec -= NSEC_PER_SEC; + } + + while (ts.tv_nsec <= -NSEC_PER_SEC) { + ts.tv_sec -= 1; + ts.tv_nsec += NSEC_PER_SEC; + } + + if (ts.tv_nsec < 0) { + ts.tv_sec -= 1; + ts.tv_nsec = (NSEC_PER_SEC + ts.tv_nsec); + } + + return ts; +} + +struct timespec timespec_add(struct timespec ts1, long usec) { + ts1 = timespec_normalise(ts1); + + struct timespec ts2 = timespec_normalise((struct timespec) { + .tv_sec = 0, + .tv_nsec = usec * 1000 + }); + + ts1.tv_sec += ts2.tv_sec; + ts1.tv_nsec += ts2.tv_nsec; + + return timespec_normalise(ts1); +} + diff --git a/src/util.h b/src/util.h index 75f494e..91a5855 100644 --- a/src/util.h +++ b/src/util.h @@ -5,8 +5,6 @@ #include #include -#include - #include "third-party/utf8.h/utf8.h" #include "libscan/scan.h" @@ -22,9 +20,6 @@ extern int PrintingProgressBar; void progress_bar_print_json(size_t done, size_t count, size_t tn_size, size_t index_size, int waiting); void progress_bar_print(double percentage, size_t tn_size, size_t index_size); -GHashTable *incremental_get_table(); - - const char *find_file_in_paths(const char **paths, const char *filename); @@ -100,31 +95,23 @@ static void generate_doc_id(const char *rel_path, char *doc_id) { buf2hex(md, sizeof(md), doc_id); } -__always_inline -static void incremental_put(GHashTable *table, const char doc_id[SIST_DOC_ID_LEN], int mtime) { - char *ptr = malloc(SIST_DOC_ID_LEN); - strcpy(ptr, doc_id); - g_hash_table_insert(table, ptr, GINT_TO_POINTER(mtime)); -} +#define MILLISECOND 1000 -__always_inline -static int incremental_get(GHashTable *table, const char doc_id[SIST_DOC_ID_LEN]) { - if (table != NULL) { - return GPOINTER_TO_INT(g_hash_table_lookup(table, doc_id)); - } else { - return 0; - } -} +struct timespec timespec_add(struct timespec ts1, long usec); -/** - * Marks a file by adding it to a table. - * !!Not thread safe. - */ -__always_inline -static int incremental_mark_file(GHashTable *table, const char doc_id[SIST_DOC_ID_LEN]) { - char *ptr = malloc(SIST_DOC_ID_LEN); - strcpy(ptr, doc_id); - return g_hash_table_insert(table, ptr, GINT_TO_POINTER(1)); -} +#define TIMER_INIT() struct timespec timer_begin +#define TIMER_START() clock_gettime(CLOCK_REALTIME, &timer_begin) +#define TIMER_END(x) do { \ + struct timespec timer_end; \ + clock_gettime(CLOCK_REALTIME, &timer_end); \ + x = (timer_end.tv_sec - timer_begin.tv_sec) * 1000000 + (timer_end.tv_nsec - timer_begin.tv_nsec) / 1000; \ +} while (0) + +#define pthread_cond_timedwait_ms(cond, mutex, delay_ms) do {\ + struct timespec now; \ + clock_gettime(CLOCK_REALTIME, &now); \ + struct timespec end_time = timespec_add(now, MILLISECOND * delay_ms); \ + pthread_cond_timedwait(cond, mutex, &end_time); \ + } while (0) #endif diff --git a/src/web/serve.c b/src/web/serve.c index 5302a59..b2b63d7 100644 --- a/src/web/serve.c +++ b/src/web/serve.c @@ -1,15 +1,14 @@ #include "serve.h" #include "src/sist.h" -#include "src/io/store.h" -#include "static_generated.c" +//#include "src/io/store.h" #include "src/index/elastic.h" #include "src/index/web.h" #include "src/auth0/auth0_c_api.h" +#include "src/web/web_util.h" #include -#define HTTP_SERVER_HEADER "Server: sist2/" VERSION "\r\n" #define HTTP_TEXT_TYPE_HEADER "Content-Type: text/plain;charset=utf-8\r\n" #define HTTP_REPLY_NOT_FOUND mg_http_reply(nc, 404, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Not found"); @@ -20,62 +19,6 @@ static struct mg_http_serve_opts DefaultServeOpts = { .mime_types = "" }; - -__always_inline -static char *address_to_string(struct mg_addr *addr) { - static char address_to_string_buf[INET6_ADDRSTRLEN]; - - return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf)); -} - -static void send_response_line(struct mg_connection *nc, int status_code, size_t length, char *extra_headers) { - mg_printf( - nc, - "HTTP/1.1 %d %s\r\n" - HTTP_SERVER_HEADER - "Content-Length: %d\r\n" - "%s\r\n\r\n", - status_code, "OK", - length, - extra_headers - ); -} - - -index_t *get_index_by_id(const char *index_id) { - for (int i = WebCtx.index_count; i >= 0; i--) { - if (strncmp(index_id, WebCtx.indices[i].desc.id, SIST_INDEX_ID_LEN) == 0) { - return &WebCtx.indices[i]; - } - } - return NULL; -} - -store_t *get_store(const char *index_id) { - index_t *idx = get_index_by_id(index_id); - if (idx != NULL) { - return idx->store; - } - return NULL; -} - -store_t *get_tag_store(const char *index_id) { - index_t *idx = get_index_by_id(index_id); - if (idx != NULL) { - return idx->tag_store; - } - return NULL; -} - -void search_index(struct mg_connection *nc, struct mg_http_message *hm) { - if (WebCtx.dev) { - mg_http_serve_file(nc, hm, "sist2-vue/dist/index.html", &DefaultServeOpts); - } else { - send_response_line(nc, 200, sizeof(index_html), "Content-Type: text/html"); - mg_send(nc, index_html, sizeof(index_html)); - } -} - void stats_files(struct mg_connection *nc, struct mg_http_message *hm) { if (hm->uri.len != SIST_INDEX_ID_LEN + 4) { @@ -87,7 +30,7 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) { memcpy(arg_index_id, hm->uri.ptr + 3, SIST_INDEX_ID_LEN); *(arg_index_id + SIST_INDEX_ID_LEN - 1) = '\0'; - index_t *index = get_index_by_id(arg_index_id); + index_t *index = web_get_index_by_id(arg_index_id); if (index == NULL) { HTTP_REPLY_NOT_FOUND return; @@ -123,87 +66,58 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) { mg_http_serve_file(nc, hm, full_path, &opts); } -void javascript(struct mg_connection *nc, struct mg_http_message *hm) { +void serve_index_html(struct mg_connection *nc, struct mg_http_message *hm) { + if (WebCtx.dev) { + mg_http_serve_file(nc, hm, "sist2-vue/dist/index.html", &DefaultServeOpts); + } else { + web_serve_asset_index_html(nc); + } +} + +void serve_index_js(struct mg_connection *nc, struct mg_http_message *hm) { if (WebCtx.dev) { mg_http_serve_file(nc, hm, "sist2-vue/dist/js/index.js", &DefaultServeOpts); } else { - send_response_line(nc, 200, sizeof(index_js), "Content-Type: application/javascript"); - mg_send(nc, index_js, sizeof(index_js)); + web_serve_asset_index_js(nc); } } -void javascript_vendor(struct mg_connection *nc, struct mg_http_message *hm) { +void serve_chunk_vendors_js(struct mg_connection *nc, struct mg_http_message *hm) { if (WebCtx.dev) { mg_http_serve_file(nc, hm, "sist2-vue/dist/js/chunk-vendors.js", &DefaultServeOpts); } else { - send_response_line(nc, 200, sizeof(chunk_vendors_js), "Content-Type: application/javascript"); - mg_send(nc, chunk_vendors_js, sizeof(chunk_vendors_js)); + web_serve_asset_chunk_vendors_js(nc); } } -void favicon(struct mg_connection *nc, struct mg_http_message *hm) { - send_response_line(nc, 200, sizeof(favicon_ico), "Content-Type: image/x-icon"); - mg_send(nc, favicon_ico, sizeof(favicon_ico)); +void serve_favicon_ico(struct mg_connection *nc, struct mg_http_message *hm) { + web_serve_asset_favicon_ico(nc); } -void style(struct mg_connection *nc, struct mg_http_message *hm) { - send_response_line(nc, 200, sizeof(index_css), "Content-Type: text/css"); - mg_send(nc, index_css, sizeof(index_css)); +void serve_style_css(struct mg_connection *nc, struct mg_http_message *hm) { + web_serve_asset_style_css(nc); } -void style_vendor(struct mg_connection *nc, struct mg_http_message *hm) { - send_response_line(nc, 200, sizeof(chunk_vendors_css), "Content-Type: text/css"); - mg_send(nc, chunk_vendors_css, sizeof(chunk_vendors_css)); +void serve_chunk_vendors_css(struct mg_connection *nc, struct mg_http_message *hm) { + web_serve_asset_chunk_vendors_css(nc); } -void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) { +void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, const char *arg_index, + const char *arg_doc_id, int arg_num) { - int has_thumbnail_index = FALSE; - - if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2) { - - if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2 + 4) { - LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr) - HTTP_REPLY_NOT_FOUND - return; - } - has_thumbnail_index = TRUE; - } - - char arg_doc_id[SIST_DOC_ID_LEN]; - char arg_index[SIST_INDEX_ID_LEN]; - - memcpy(arg_index, hm->uri.ptr + 3, SIST_INDEX_ID_LEN); - *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0'; - memcpy(arg_doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN); - *(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0'; - - store_t *store = get_store(arg_index); - if (store == NULL) { - LOG_DEBUGF("serve.c", "Could not get store for index: %s", arg_index) + database_t *db = web_get_database(arg_index); + if (db == NULL) { + LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index); HTTP_REPLY_NOT_FOUND return; } - char *data; size_t data_len = 0; - if (has_thumbnail_index) { - const char *tn_index = hm->uri.ptr + SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2; - - char tn_key[sizeof(arg_doc_id) + sizeof(char) * 4]; - - memcpy(tn_key, arg_doc_id, sizeof(arg_doc_id)); - memcpy(tn_key + sizeof(arg_doc_id) - 1, tn_index, sizeof(char) * 4); - *(tn_key + sizeof(tn_key) - 1) = '\0'; - - data = store_read(store, (char *) tn_key, sizeof(tn_key), &data_len); - } else { - data = store_read(store, (char *) arg_doc_id, sizeof(arg_doc_id), &data_len); - } + void *data = database_read_thumbnail(db, arg_doc_id, arg_num, &data_len); if (data_len != 0) { - send_response_line( + web_send_headers( nc, 200, data_len, "Content-Type: image/jpeg\r\n" "Cache-Control: max-age=31536000" @@ -216,10 +130,50 @@ void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) { } } -void search(struct mg_connection *nc, struct mg_http_message *hm) { +void thumbnail_with_num(struct mg_connection *nc, struct mg_http_message *hm) { + if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2 + 5) { + LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr); + HTTP_REPLY_NOT_FOUND + return; + } + char arg_doc_id[SIST_DOC_ID_LEN]; + char arg_index[SIST_INDEX_ID_LEN]; + char arg_num[5] = {0}; + + memcpy(arg_index, hm->uri.ptr + 3, SIST_INDEX_ID_LEN); + *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0'; + memcpy(arg_doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN); + *(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0'; + memcpy(arg_num, hm->uri.ptr + SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2, 4); + + int num = (int) strtol(arg_num, NULL, 10); + + serve_thumbnail(nc, hm, arg_index, arg_doc_id, num); +} + +void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) { + + if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2) { + LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr); + HTTP_REPLY_NOT_FOUND + return; + } + + char arg_doc_id[SIST_DOC_ID_LEN]; + char arg_index[SIST_INDEX_ID_LEN]; + + memcpy(arg_index, hm->uri.ptr + 3, SIST_INDEX_ID_LEN); + *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0'; + memcpy(arg_doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN); + *(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0'; + + serve_thumbnail(nc, hm, arg_index, arg_doc_id, 0); +} + +void search(struct mg_connection *nc, struct mg_http_message *hm) { if (hm->body.len == 0) { - LOG_DEBUG("serve.c", "Client sent empty body, ignoring request") + LOG_DEBUG("serve.c", "Client sent empty body, ignoring request"); mg_http_reply(nc, 400, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Invalid request"); return; } @@ -266,7 +220,7 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s if (strcmp(MG_VERSION, EXPECTED_MONGOOSE_VERSION) != 0) { LOG_WARNING("serve.c", "sist2 was not linked with latest mongoose version, " - "serving file from disk might not work as expected.") + "serving file from disk might not work as expected."); } const char *path = cJSON_GetObjectItem(json, "path")->valuestring; @@ -285,7 +239,7 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s idx->desc.root, path_unescaped, strlen(path_unescaped) == 0 ? "" : "/", name_unescaped, strlen(ext) == 0 ? "" : ".", ext); - LOG_DEBUGF("serve.c", "Serving file from disk: %s", full_path) + LOG_DEBUGF("serve.c", "Serving file from disk: %s", full_path); char disposition[8192]; snprintf(disposition, sizeof(disposition), @@ -372,7 +326,7 @@ void index_info(struct mg_connection *nc) { char *json_str = cJSON_PrintUnformatted(json); - send_response_line(nc, 200, strlen(json_str), "Content-Type: application/json"); + web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json"); mg_send(nc, json_str, strlen(json_str)); free(json_str); cJSON_Delete(json); @@ -382,7 +336,7 @@ void index_info(struct mg_connection *nc) { void file(struct mg_connection *nc, struct mg_http_message *hm) { if (hm->uri.len != SIST_DOC_ID_LEN + 2) { - LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr) + LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr); HTTP_REPLY_NOT_FOUND return; } @@ -412,7 +366,7 @@ void file(struct mg_connection *nc, struct mg_http_message *hm) { next = parent->valuestring; } - index_t *idx = get_index_by_id(index_id->valuestring); + index_t *idx = web_get_index_by_id(index_id->valuestring); if (idx == NULL) { cJSON_Delete(doc); @@ -431,9 +385,9 @@ void file(struct mg_connection *nc, struct mg_http_message *hm) { void status(struct mg_connection *nc) { char *status = elastic_get_status(); if (strcmp(status, "open") == 0) { - send_response_line(nc, 204, 0, "Content-Type: application/json"); + web_send_headers(nc, 204, 0, "Content-Type: application/json"); } else { - send_response_line(nc, 500, 0, "Content-Type: application/json"); + web_send_headers(nc, 500, 0, "Content-Type: application/json"); } free(status); @@ -475,114 +429,114 @@ tag_req_t *parse_tag_request(cJSON *json) { } void tag(struct mg_connection *nc, struct mg_http_message *hm) { - if (hm->uri.len != SIST_INDEX_ID_LEN + 4) { - LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr) - HTTP_REPLY_NOT_FOUND - return; - } - - char arg_index[SIST_INDEX_ID_LEN]; - memcpy(arg_index, hm->uri.ptr + 5, SIST_INDEX_ID_LEN); - *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0'; - - if (hm->body.len < 2 || hm->method.len != 4 || memcmp(&hm->method, "POST", 4) == 0) { - LOG_DEBUG("serve.c", "Invalid tag request") - HTTP_REPLY_NOT_FOUND - return; - } - - store_t *store = get_tag_store(arg_index); - if (store == NULL) { - LOG_DEBUGF("serve.c", "Could not get tag store for index: %s", arg_index) - HTTP_REPLY_NOT_FOUND - return; - } - - char *body = malloc(hm->body.len + 1); - memcpy(body, hm->body.ptr, hm->body.len); - *(body + hm->body.len) = '\0'; - cJSON *json = cJSON_Parse(body); - - tag_req_t *arg_req = parse_tag_request(json); - if (arg_req == NULL) { - LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index) - cJSON_Delete(json); - free(body); - mg_http_reply(nc, 400, "", "Invalid request"); - return; - } - - cJSON *arr = NULL; - - size_t data_len = 0; - const char *data = store_read(store, arg_req->doc_id, SIST_DOC_ID_LEN, &data_len); - if (data_len == 0) { - arr = cJSON_CreateArray(); - } else { - arr = cJSON_Parse(data); - } - - if (arg_req->delete) { - - if (data_len > 0) { - cJSON *element = NULL; - int i = 0; - cJSON_ArrayForEach(element, arr) { - if (strcmp(element->valuestring, arg_req->name) == 0) { - cJSON_DeleteItemFromArray(arr, i); - break; - } - i++; - } - } - - char *buf = malloc(sizeof(char) * 8192); - snprintf(buf, 8192, - "{" - " \"script\" : {" - " \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\"," - " \"lang\": \"painless\"," - " \"params\" : {" - " \"tag\" : \"%s\"" - " }" - " }" - "}", arg_req->name - ); - - char url[4096]; - snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id); - nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl); - - } else { - cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name)); - - char *buf = malloc(sizeof(char) * 8192); - snprintf(buf, 8192, - "{" - " \"script\" : {" - " \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\"," - " \"lang\": \"painless\"," - " \"params\" : {" - " \"tag\" : \"%s\"" - " }" - " }" - "}", arg_req->name - ); - - char url[4096]; - snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id); - nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl); - } - - char *json_str = cJSON_PrintUnformatted(arr); - store_write(store, arg_req->doc_id, SIST_DOC_ID_LEN, json_str, strlen(json_str) + 1); - store_flush(store); - - free(arg_req); - free(json_str); - cJSON_Delete(json); - cJSON_Delete(arr); - free(body); +// if (hm->uri.len != SIST_INDEX_ID_LEN + 4) { +// LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr) +// HTTP_REPLY_NOT_FOUND +// return; +// } +// +// char arg_index[SIST_INDEX_ID_LEN]; +// memcpy(arg_index, hm->uri.ptr + 5, SIST_INDEX_ID_LEN); +// *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0'; +// +// if (hm->body.len < 2 || hm->method.len != 4 || memcmp(&hm->method, "POST", 4) == 0) { +// LOG_DEBUG("serve.c", "Invalid tag request") +// HTTP_REPLY_NOT_FOUND +// return; +// } +// +// store_t *store = get_tag_store(arg_index); +// if (store == NULL) { +// LOG_DEBUGF("serve.c", "Could not get tag store for index: %s", arg_index) +// HTTP_REPLY_NOT_FOUND +// return; +// } +// +// char *body = malloc(hm->body.len + 1); +// memcpy(body, hm->body.ptr, hm->body.len); +// *(body + hm->body.len) = '\0'; +// cJSON *json = cJSON_Parse(body); +// +// tag_req_t *arg_req = parse_tag_request(json); +// if (arg_req == NULL) { +// LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index) +// cJSON_Delete(json); +// free(body); +// mg_http_reply(nc, 400, "", "Invalid request"); +// return; +// } +// +// cJSON *arr = NULL; +// +// size_t data_len = 0; +// const char *data = store_read(store, arg_req->doc_id, SIST_DOC_ID_LEN, &data_len); +// if (data_len == 0) { +// arr = cJSON_CreateArray(); +// } else { +// arr = cJSON_Parse(data); +// } +// +// if (arg_req->delete) { +// +// if (data_len > 0) { +// cJSON *element = NULL; +// int i = 0; +// cJSON_ArrayForEach(element, arr) { +// if (strcmp(element->valuestring, arg_req->name) == 0) { +// cJSON_DeleteItemFromArray(arr, i); +// break; +// } +// i++; +// } +// } +// +// char *buf = malloc(sizeof(char) * 8192); +// snprintf(buf, 8192, +// "{" +// " \"script\" : {" +// " \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\"," +// " \"lang\": \"painless\"," +// " \"params\" : {" +// " \"tag\" : \"%s\"" +// " }" +// " }" +// "}", arg_req->name +// ); +// +// char url[4096]; +// snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id); +// nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl); +// +// } else { +// cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name)); +// +// char *buf = malloc(sizeof(char) * 8192); +// snprintf(buf, 8192, +// "{" +// " \"script\" : {" +// " \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\"," +// " \"lang\": \"painless\"," +// " \"params\" : {" +// " \"tag\" : \"%s\"" +// " }" +// " }" +// "}", arg_req->name +// ); +// +// char url[4096]; +// snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id); +// nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl); +// } +// +// char *json_str = cJSON_PrintUnformatted(arr); +// store_write(store, arg_req->doc_id, SIST_DOC_ID_LEN, json_str, strlen(json_str) + 1); +// store_flush(store); +// +// free(arg_req); +// free(json_str); +// cJSON_Delete(json); +// cJSON_Delete(arr); +// free(body); } int validate_auth(struct mg_connection *nc, struct mg_http_message *hm) { @@ -601,7 +555,7 @@ int check_auth0(struct mg_http_message *hm) { struct mg_str *cookie = mg_http_get_header(hm, "Cookie"); if (cookie == NULL) { - LOG_WARNING("serve.c", "Unauthorized request (no auth cookie)") + LOG_WARNING("serve.c", "Unauthorized request (no auth cookie)"); return FALSE; } @@ -610,7 +564,7 @@ int check_auth0(struct mg_http_message *hm) { token = mg_http_get_header_var(*cookie, mg_str("sist2-auth0")); if (token.len == 0) { - LOG_WARNING("serve.c", "Unauthorized request (no auth cookie)") + LOG_WARNING("serve.c", "Unauthorized request (no auth cookie)"); return FALSE; } @@ -644,28 +598,31 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo } } + char uri[256]; + memcpy(uri, hm->uri.ptr, hm->uri.len); + *(uri + hm->uri.len) = '\0'; LOG_DEBUGF("serve.c", "<%s> GET %s", - address_to_string(&(nc->rem)), - hm->uri - ) + web_address_to_string(&(nc->rem)), + uri + ); if (mg_http_match_uri(hm, "/")) { - search_index(nc, hm); + serve_index_html(nc, hm); return; } else if (mg_http_match_uri(hm, "/favicon.ico")) { - favicon(nc, hm); + serve_favicon_ico(nc, hm); return; } else if (mg_http_match_uri(hm, "/css/index.css")) { - style(nc, hm); + serve_style_css(nc, hm); return; } else if (mg_http_match_uri(hm, "/css/chunk-vendors.css")) { - style_vendor(nc, hm); + serve_chunk_vendors_css(nc, hm); return; } else if (mg_http_match_uri(hm, "/js/index.js")) { - javascript(nc, hm); + serve_index_js(nc, hm); return; } else if (mg_http_match_uri(hm, "/js/chunk-vendors.js")) { - javascript_vendor(nc, hm); + serve_chunk_vendors_js(nc, hm); return; } else if (mg_http_match_uri(hm, "/i")) { index_info(nc); @@ -683,6 +640,8 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo status(nc); } else if (mg_http_match_uri(hm, "/f/*")) { file(nc, hm); + } else if (mg_http_match_uri(hm, "/t/*/*/*")) { + thumbnail_with_num(nc, hm); } else if (mg_http_match_uri(hm, "/t/*/*")) { thumbnail(nc, hm); } else if (mg_http_match_uri(hm, "/s/*/*")) { @@ -706,7 +665,7 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo response_t *r = ctx->response; if (r->status_code == 200) { - send_response_line(nc, 200, r->size, "Content-Type: application/json"); + web_send_headers(nc, 200, r->size, "Content-Type: application/json"); mg_send(nc, r->body, r->size); } else if (r->status_code == 0) { sist_log("serve.c", LOG_SIST_ERROR, "Could not connect to elasticsearch!"); @@ -738,7 +697,7 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo void serve(const char *listen_address) { - LOG_INFOF("serve.c", "Starting web server @ http://%s", listen_address) + LOG_INFOF("serve.c", "Starting web server @ http://%s", listen_address); struct mg_mgr mgr; mg_mgr_init(&mgr); @@ -747,12 +706,12 @@ void serve(const char *listen_address) { struct mg_connection *nc = mg_http_listen(&mgr, listen_address, ev_router, NULL); if (nc == NULL) { - LOG_FATALF("serve.c", "Couldn't bind web server on address %s", listen_address) + LOG_FATALF("serve.c", "Couldn't bind web server on address %s", listen_address); } while (ok) { mg_mgr_poll(&mgr, 10); } mg_mgr_free(&mgr); - LOG_INFO("serve.c", "Finished web event loop") + LOG_INFO("serve.c", "Finished web event loop"); } diff --git a/src/web/web_util.c b/src/web/web_util.c new file mode 100644 index 0000000..8cd86f4 --- /dev/null +++ b/src/web/web_util.c @@ -0,0 +1,63 @@ +#include "web_util.h" +#include "static_generated.c" + + +void web_serve_asset_index_html(struct mg_connection *nc) { + web_send_headers(nc, 200, sizeof(index_html), "Content-Type: text/html"); + mg_send(nc, index_html, sizeof(index_html)); +} + +void web_serve_asset_index_js(struct mg_connection *nc) { + web_send_headers(nc, 200, sizeof(index_js), "Content-Type: application/javascript"); + mg_send(nc, index_js, sizeof(index_js)); +} + +void web_serve_asset_chunk_vendors_js(struct mg_connection *nc) { + web_send_headers(nc, 200, sizeof(chunk_vendors_js), "Content-Type: application/javascript"); + mg_send(nc, chunk_vendors_js, sizeof(chunk_vendors_js)); +} + +void web_serve_asset_favicon_ico(struct mg_connection *nc) { + web_send_headers(nc, 200, sizeof(favicon_ico), "Content-Type: image/x-icon"); + mg_send(nc, favicon_ico, sizeof(favicon_ico)); +} + +void web_serve_asset_style_css(struct mg_connection *nc) { + web_send_headers(nc, 200, sizeof(index_css), "Content-Type: text/css"); + mg_send(nc, index_css, sizeof(index_css)); +} + +void web_serve_asset_chunk_vendors_css(struct mg_connection *nc) { + web_send_headers(nc, 200, sizeof(chunk_vendors_css), "Content-Type: text/css"); + mg_send(nc, chunk_vendors_css, sizeof(chunk_vendors_css)); +} + +index_t *web_get_index_by_id(const char *index_id) { + for (int i = WebCtx.index_count; i >= 0; i--) { + if (strncmp(index_id, WebCtx.indices[i].desc.id, SIST_INDEX_ID_LEN) == 0) { + return &WebCtx.indices[i]; + } + } + return NULL; +} + +database_t *web_get_database(const char *index_id) { + index_t *idx = web_get_index_by_id(index_id); + if (idx != NULL) { + return idx->db; + } + return NULL; +} + +void web_send_headers(struct mg_connection *nc, int status_code, size_t length, char *extra_headers) { + mg_printf( + nc, + "HTTP/1.1 %d %s\r\n" + HTTP_SERVER_HEADER + "Content-Length: %d\r\n" + "%s\r\n\r\n", + status_code, "OK", + length, + extra_headers + ); +} diff --git a/src/web/web_util.h b/src/web/web_util.h new file mode 100644 index 0000000..a470dba --- /dev/null +++ b/src/web/web_util.h @@ -0,0 +1,32 @@ +#ifndef SIST2_WEB_UTIL_H +#define SIST2_WEB_UTIL_H + +#include "src/sist.h" +#include "src/index/elastic.h" +#include "src/ctx.h" +#include + +#define HTTP_SERVER_HEADER "Server: sist2/" VERSION "\r\n" + +index_t *web_get_index_by_id(const char *index_id); + +database_t *web_get_database(const char *index_id); + +__always_inline +static char *web_address_to_string(struct mg_addr *addr) { + return "TODO"; +// static char address_to_string_buf[INET6_ADDRSTRLEN]; +// +// return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf)); +} + +void web_send_headers(struct mg_connection *nc, int status_code, size_t length, char *extra_headers); + +void web_serve_asset_index_html(struct mg_connection *nc); +void web_serve_asset_index_js(struct mg_connection *nc); +void web_serve_asset_chunk_vendors_js(struct mg_connection *nc); +void web_serve_asset_favicon_ico(struct mg_connection *nc); +void web_serve_asset_style_css(struct mg_connection *nc); +void web_serve_asset_chunk_vendors_css(struct mg_connection *nc); + +#endif //SIST2_WEB_UTIL_H diff --git a/third-party/libscan/CMakeLists.txt b/third-party/libscan/CMakeLists.txt index ea8042b..3cc065d 100644 --- a/third-party/libscan/CMakeLists.txt +++ b/third-party/libscan/CMakeLists.txt @@ -97,7 +97,6 @@ find_package(LibLZMA REQUIRED) find_package(ZLIB REQUIRED) find_package(unofficial-pcre CONFIG REQUIRED) - find_library(JBIG2DEC_LIB NAMES jbig2decd jbig2dec) find_library(HARFBUZZ_LIB NAMES harfbuzz harfbuzzd) find_library(FREETYPE_LIB NAMES freetype freetyped) @@ -110,6 +109,7 @@ find_library(CMS_LIB NAMES lcms2) find_library(JAS_LIB NAMES jasper) find_library(GUMBO_LIB NAMES gumbo) find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/) +find_package(Leptonica CONFIG REQUIRED) target_compile_options( @@ -231,6 +231,7 @@ target_link_libraries( antiword mobi unofficial::pcre::pcre unofficial::pcre::pcre16 unofficial::pcre::pcre32 unofficial::pcre::pcrecpp + leptonica ) target_include_directories( diff --git a/third-party/libscan/libscan/arc/arc.c b/third-party/libscan/libscan/arc/arc.c index 092bc8c..d56f538 100644 --- a/third-party/libscan/libscan/arc/arc.c +++ b/third-party/libscan/libscan/arc/arc.c @@ -9,27 +9,13 @@ #define MAX_DECOMPRESSED_SIZE_RATIO 40.0 -int should_parse_filtered_file(const char *filepath, int ext) { - char tmp[PATH_MAX * 2]; +int should_parse_filtered_file(const char *filepath) { - if (ext == 0) { - return FALSE; - } - - if (strncmp(filepath + ext, "tgz", 3) == 0) { + if (strstr(filepath, ".tgz")) { return TRUE; } - memcpy(tmp, filepath, ext - 1); - *(tmp + ext - 1) = '\0'; - - char *idx = strrchr(tmp, '.'); - - if (idx == NULL) { - return FALSE; - } - - if (strcmp(idx, ".tar") == 0) { + if (strstr(filepath, ".tar.")) { return TRUE; } @@ -206,18 +192,10 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre while (archive_read_next_header(a, &entry) == ARCHIVE_OK) { struct stat entry_stat = *archive_entry_stat(entry); - sub_job->vfile.st_mode = entry_stat.st_mode; sub_job->vfile.st_size = entry_stat.st_size; sub_job->vfile.mtime = (int) entry_stat.st_mtim.tv_sec; - double decompressed_size_ratio = (double) sub_job->vfile.st_size / (double) f->st_size; - if (decompressed_size_ratio > MAX_DECOMPRESSED_SIZE_RATIO) { - CTX_LOG_DEBUGF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)", sub_job->filepath, - decompressed_size_ratio) - continue; - } - - if (S_ISREG(sub_job->vfile.st_mode)) { + if (S_ISREG(entry_stat.st_mode)) { const char *utf8_name = archive_entry_pathname_utf8(entry); @@ -231,6 +209,13 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre } sub_job->base = (int) (strrchr(sub_job->filepath, '/') - sub_job->filepath) + 1; + double decompressed_size_ratio = (double) sub_job->vfile.st_size / (double) f->st_size; + if (decompressed_size_ratio > MAX_DECOMPRESSED_SIZE_RATIO) { + CTX_LOG_DEBUGF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)", sub_job->filepath, + decompressed_size_ratio) + break; + } + // Handle excludes if (exclude != NULL && EXCLUDED(sub_job->filepath)) { CTX_LOG_DEBUGF("arc.c", "Excluded: %s", sub_job->filepath) diff --git a/third-party/libscan/libscan/arc/arc.h b/third-party/libscan/libscan/arc/arc.h index ea9d570..1ea5e49 100644 --- a/third-party/libscan/libscan/arc/arc.h +++ b/third-party/libscan/libscan/arc/arc.h @@ -67,7 +67,7 @@ static int vfile_close_callback(struct archive *a, void *user_data) { int arc_open(scan_arc_ctx_t *ctx, vfile_t *f, struct archive **a, arc_data_t *arc_data, int allow_recurse); -int should_parse_filtered_file(const char *filepath, int ext); +int should_parse_filtered_file(const char *filepath); scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre *exclude, pcre_extra *exclude_extra); diff --git a/third-party/libscan/libscan/ebook/ebook.c b/third-party/libscan/libscan/ebook/ebook.c index acf70fd..95fb4a4 100644 --- a/third-party/libscan/libscan/ebook/ebook.c +++ b/third-party/libscan/libscan/ebook/ebook.c @@ -162,7 +162,7 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d avcodec_receive_packet(jpeg_encoder, &jpeg_packet); APPEND_LONG_META(doc, MetaThumbnail, 1) - ctx->store(doc->doc_id, sizeof(doc->doc_id), (char *) jpeg_packet.data, jpeg_packet.size); + ctx->store(doc->doc_id, 0, (char *) jpeg_packet.data, jpeg_packet.size); free(samples); av_packet_unref(&jpeg_packet); diff --git a/third-party/libscan/libscan/font/font.c b/third-party/libscan/libscan/font/font.c index 6092b19..d734134 100644 --- a/third-party/libscan/libscan/font/font.c +++ b/third-party/libscan/libscan/font/font.c @@ -232,7 +232,7 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) { bmp_format(&bmp_data, dimensions, bitmap); APPEND_LONG_META(doc, MetaThumbnail, 1) - ctx->store(doc->doc_id, sizeof(doc->doc_id), (char *) bmp_data.buf, bmp_data.cur); + ctx->store(doc->doc_id, 0, bmp_data.buf, bmp_data.cur); dyn_buffer_destroy(&bmp_data); free(bitmap); diff --git a/third-party/libscan/libscan/media/media.c b/third-party/libscan/libscan/media/media.c index fe9360e..38cb421 100644 --- a/third-party/libscan/libscan/media/media.c +++ b/third-party/libscan/libscan/media/media.c @@ -468,8 +468,7 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor if (scaled_frame == STORE_AS_IS) { return_value = SAVE_THUMBNAIL_OK; - ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) frame_and_packet->packet->data, - frame_and_packet->packet->size); + ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size); } else { // Encode frame to jpeg AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height, @@ -482,19 +481,17 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor // Save thumbnail if (thumbnail_index == 0) { - ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) jpeg_packet.data, jpeg_packet.size); + ctx->store(doc->doc_id, 0, jpeg_packet.data, jpeg_packet.size); return_value = SAVE_THUMBNAIL_OK; } else if (thumbnail_index > 1) { - return_value = SAVE_THUMBNAIL_OK; // TO FIX: the 2nd rendered frame is always broken, just skip it until // I figure out a better fix. thumbnail_index -= 1; - char tn_key[sizeof(doc->doc_id) + sizeof(char) * 4]; - snprintf(tn_key, sizeof(tn_key), "%s%04d", doc->doc_id, thumbnail_index); + ctx->store(doc->doc_id, thumbnail_index, jpeg_packet.data, jpeg_packet.size); - ctx->store((char *) tn_key, sizeof(tn_key), (char *) jpeg_packet.data, jpeg_packet.size); + return_value = SAVE_THUMBNAIL_OK; } else { return_value = SAVE_THUMBNAIL_SKIPPED; } @@ -854,8 +851,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu if (scaled_frame == STORE_AS_IS) { APPEND_LONG_META(doc, MetaThumbnail, 1) - ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) frame_and_packet->packet->data, - frame_and_packet->packet->size); + ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size); } else { // Encode frame to jpeg AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height, @@ -868,7 +864,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu // Save thumbnail APPEND_LONG_META(doc, MetaThumbnail, 1) - ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) jpeg_packet.data, jpeg_packet.size); + ctx->store(doc->doc_id, 0, jpeg_packet.data, jpeg_packet.size); av_packet_unref(&jpeg_packet); avcodec_free_context(&jpeg_encoder); diff --git a/third-party/libscan/libscan/ooxml/ooxml.c b/third-party/libscan/libscan/ooxml/ooxml.c index 61a496b..49010e1 100644 --- a/third-party/libscan/libscan/ooxml/ooxml.c +++ b/third-party/libscan/libscan/ooxml/ooxml.c @@ -191,7 +191,7 @@ void read_thumbnail(scan_ooxml_ctx_t *ctx, document_t *doc, struct archive *a, s archive_read_data(a, buf, entry_size); APPEND_LONG_META(doc, MetaThumbnail, 1) - ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), buf, entry_size); + ctx->store(doc->doc_id, 1, buf, entry_size); free(buf); } diff --git a/third-party/libscan/libscan/scan.h b/third-party/libscan/libscan/scan.h index fd3fd1f..9d09016 100644 --- a/third-party/libscan/libscan/scan.h +++ b/third-party/libscan/libscan/scan.h @@ -6,6 +6,7 @@ #endif #include +#include #include #include #include @@ -16,7 +17,7 @@ #define UNUSED(x) __attribute__((__unused__)) x -typedef void (*store_callback_t)(char *key, size_t key_len, char *buf, size_t buf_len); +typedef void (*store_callback_t)(char *key, int num, void *buf, size_t buf_len); typedef void (*logf_callback_t)(const char *filepath, int level, char *format, ...); @@ -111,8 +112,8 @@ typedef struct document { unsigned long size; unsigned int mime; int mtime; - short base; - short ext; + int base; + int ext; meta_line_t *meta_head; meta_line_t *meta_tail; char filepath[PATH_MAX * 2 + 1]; @@ -144,7 +145,6 @@ typedef struct vfile { int mtime; size_t st_size; - unsigned int st_mode; SHA_CTX sha1_ctx; unsigned char sha1_digest[SHA1_DIGEST_LENGTH]; @@ -161,7 +161,7 @@ typedef struct vfile { logf_callback_t logf; } vfile_t; -typedef struct parse_job_t { +typedef struct { int base; int ext; struct vfile vfile; diff --git a/third-party/libscan/libscan/util.h b/third-party/libscan/libscan/util.h index 09f7ad5..11092e2 100644 --- a/third-party/libscan/libscan/util.h +++ b/third-party/libscan/libscan/util.h @@ -358,4 +358,37 @@ static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) { } } +static parse_job_t *create_parse_job(const char *filepath, int mtime, size_t st_size) { + parse_job_t *job = (parse_job_t *) malloc(sizeof(parse_job_t)); + + job->parent[0] = '\0'; + + strcpy(job->filepath, filepath); + strcpy(job->vfile.filepath, filepath); + job->vfile.st_size = st_size; + job->vfile.mtime = mtime; + + const char *slash = strrchr(filepath, '/'); + if (slash == NULL) { + job->base = 0; + } else { + job->base = (int) (slash - filepath + 1); + } + + const char *dot = strrchr(filepath + job->base, '.'); + if (dot == NULL) { + job->ext = (int) strlen(filepath); + } else { + job->ext = (int) (dot - filepath + 1); + } + + job->vfile.fd = -1; + job->vfile.is_fs_file = TRUE; + job->vfile.has_checksum = FALSE; + job->vfile.rewind_buffer_size = 0; + job->vfile.rewind_buffer = NULL; + + return job; +} + #endif diff --git a/third-party/libscan/test/test_util.cpp b/third-party/libscan/test/test_util.cpp index fb9aa5c..6ed8998 100644 --- a/third-party/libscan/test/test_util.cpp +++ b/third-party/libscan/test/test_util.cpp @@ -55,7 +55,6 @@ void load_file(const char *filepath, vfile_t *f) { f->mtime = (int)info.st_mtim.tv_sec; f->st_size = info.st_size; - f->st_mode = info.st_mode; f->fd = open(filepath, O_RDONLY); diff --git a/third-party/libscan/test/test_util.h b/third-party/libscan/test/test_util.h index e388b49..98243d2 100644 --- a/third-party/libscan/test/test_util.h +++ b/third-party/libscan/test/test_util.h @@ -21,7 +21,7 @@ static void noop_log(const char *filepath, int level, char *str) { static size_t store_size = 0; -static void counter_store(char* key, size_t key_len, char *value, size_t value_len) { +static void counter_store(char* key, int num, void *value, size_t value_len) { store_size += value_len; // char id[37]; // char tmp[PATH_MAX]; diff --git a/third-party/libscan/third-party/antiword b/third-party/libscan/third-party/antiword index ddb0421..badfdac 160000 --- a/third-party/libscan/third-party/antiword +++ b/third-party/libscan/third-party/antiword @@ -1 +1 @@ -Subproject commit ddb042143e72a8b789e06f09dbc897dfa9f15b82 +Subproject commit badfdac84586511d4f2b626516162d62a3625349 From 300c70883d33ce338cb2f32bd3ce84ab7c67a6ae Mon Sep 17 00:00:00 2001 From: simon987 Date: Mon, 10 Apr 2023 11:04:16 -0400 Subject: [PATCH 5/7] Fixes and cleanup --- scripts/mime.csv | 3 + src/cli.c | 2 +- src/cli.h | 1 + src/ctx.h | 7 - src/database/database.c | 68 ++- src/database/database.h | 10 +- src/io/walk.c | 10 - src/main.c | 212 +++---- src/parsing/fs_util.h | 1 - src/parsing/magic_util.c | 2 +- src/parsing/mime_generated.c | 567 ++++++++++--------- src/parsing/parse.c | 34 +- src/sist.h | 2 + src/tpool.c | 153 ++--- src/tpool.h | 2 - third-party/libscan/libscan/arc/arc.c | 8 +- third-party/libscan/libscan/comic/comic.c | 4 +- third-party/libscan/libscan/ebook/ebook.c | 38 +- third-party/libscan/libscan/font/font.c | 12 +- third-party/libscan/libscan/json/json.c | 2 +- third-party/libscan/libscan/macros.h | 16 +- third-party/libscan/libscan/media/media.c | 92 +-- third-party/libscan/libscan/mobi/scan_mobi.c | 18 +- third-party/libscan/libscan/msdoc/msdoc.c | 10 +- third-party/libscan/libscan/ooxml/ooxml.c | 30 +- third-party/libscan/libscan/raw/raw.c | 42 +- third-party/libscan/libscan/scan.h | 20 +- third-party/libscan/libscan/text/text.c | 12 +- third-party/libscan/libscan/wpd/wpd.c | 8 +- 29 files changed, 678 insertions(+), 708 deletions(-) diff --git a/scripts/mime.csv b/scripts/mime.csv index 4b32f32..e1b25df 100644 --- a/scripts/mime.csv +++ b/scripts/mime.csv @@ -1,3 +1,4 @@ +application/x-matlab-data,mat application/arj, arj application/base64, mme application/binhex, hqx @@ -346,6 +347,8 @@ text/mcf, mcf text/pascal, pas text/PGP, text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml|make|log|markdown|yaml +text/x-script.python, pyx +text/csv, application/vnd.coffeescript, coffee text/richtext, rt|rtf|rtx text/rtf, diff --git a/src/cli.c b/src/cli.c index e2540ba..9546a27 100644 --- a/src/cli.c +++ b/src/cli.c @@ -142,7 +142,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { char *abs_output = abspath(args->output); if (args->incremental && abs_output == NULL) { - LOG_WARNINGF("main.c", "Could not open original index for incremental scan: %s. Will not perform incremental scan.", abs_output); + LOG_WARNINGF("main.c", "Could not open original index for incremental scan: %s. Will not perform incremental scan.", args->output); args->incremental = FALSE; } else if (!args->incremental && abs_output != NULL) { LOG_FATALF("main.c", "Index already exists: %s. If you wish to perform incremental scan, you must specify --incremental", abs_output); diff --git a/src/cli.h b/src/cli.h index e1e039c..10d48c3 100644 --- a/src/cli.h +++ b/src/cli.h @@ -14,6 +14,7 @@ typedef struct scan_args { int content_size; int threads; int incremental; + int optimize_database; char *output; char *rewrite_url; char *name; diff --git a/src/ctx.h b/src/ctx.h index f56afd8..9e54cb0 100644 --- a/src/ctx.h +++ b/src/ctx.h @@ -38,13 +38,6 @@ typedef struct { pcre_extra *exclude_extra; int fast; - pthread_mutex_t dbg_current_files_mu; - - int dbg_failed_files_count; - int dbg_skipped_files_count; - int dbg_excluded_files_count; - pthread_mutex_t dbg_file_counts_mu; - scan_arc_ctx_t arc_ctx; scan_comic_ctx_t comic_ctx; scan_ebook_ctx_t ebook_ctx; diff --git a/src/database/database.c b/src/database/database.c index 741187c..bf867c7 100644 --- a/src/database/database.c +++ b/src/database/database.c @@ -8,7 +8,6 @@ #include - database_t *database_create(const char *filename, database_type_t type) { database_t *db = malloc(sizeof(database_t)); @@ -81,7 +80,7 @@ void database_initialize(database_t *db) { } void database_open(database_t *db) { - LOG_DEBUGF("tpool.c", "Opening database %s (%d)", db->filename, db->type); + LOG_DEBUGF("database.c", "Opening database %s (%d)", db->filename, db->type); CRASH_IF_NOT_SQLITE_OK(sqlite3_open(db->filename, &db->db)); @@ -113,7 +112,8 @@ void database_open(database_t *db) { &db->write_document_stmt, NULL)); CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( db->db, - "INSERT INTO thumbnail (id, num, data) VALUES (?,?,?) ON CONFLICT DO UPDATE SET data=excluded.data;", -1, + "INSERT INTO thumbnail (id, num, data) VALUES (?,?,?) ON CONFLICT DO UPDATE SET data=excluded.data;", + -1, &db->write_thumbnail_stmt, NULL)); // Create functions @@ -186,12 +186,16 @@ void database_close(database_t *db, int optimize) { if (optimize) { LOG_DEBUG("database.c", "Optimizing database"); - // TODO: This should be an optional argument -// CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "VACUUM;", NULL, NULL, NULL)); + CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "VACUUM;", NULL, NULL, NULL)); CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA optimize;", NULL, NULL, NULL)); } sqlite3_close(db->db); + + if (db->type == IPC_PRODUCER_DATABASE) { + remove(db->filename); + } + free(db); db = NULL; } @@ -202,11 +206,14 @@ void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *r int ret = sqlite3_step(db->select_thumbnail_stmt); - // TODO: if row not found, return null - if (ret != SQLITE_ROW) { - LOG_FATALF("database.c", "FIXME: tn step returned %d", ret); + if (ret == SQLITE_DONE) { + CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->select_thumbnail_stmt)); + *return_value_len = 0; + return NULL; } + CRASH_IF_STMT_FAIL(ret); + const void *blob = sqlite3_column_blob(db->select_thumbnail_stmt, 0); const int blob_size = sqlite3_column_bytes(db->select_thumbnail_stmt, 0); @@ -275,11 +282,47 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) { return desc; } +database_iterator_t *database_create_delete_list_iterator(database_t *db) { + + sqlite3_stmt *stmt; + sqlite3_prepare_v2(db->db, "SELECT id FROM delete_list;", -1, &stmt, NULL); + + database_iterator_t *iter = malloc(sizeof(database_iterator_t)); + + iter->stmt = stmt; + iter->db = db; + + return iter; +} + +char *database_delete_list_iter(database_iterator_t *iter) { + int ret = sqlite3_step(iter->stmt); + + if (ret == SQLITE_ROW) { + const char *id = (const char *) sqlite3_column_text(iter->stmt, 0); + char *id_heap = malloc(strlen(id) + 1); + strcpy(id_heap, id); + return id_heap; + } + + if (ret != SQLITE_DONE) { + LOG_FATALF("database.c", "FIXME: delete iter returned %s", sqlite3_errmsg(iter->db->db)); + } + + if (sqlite3_finalize(iter->stmt) != SQLITE_OK) { + LOG_FATALF("database.c", "FIXME: delete iter returned %s", sqlite3_errmsg(iter->db->db)); + } + + iter->stmt = NULL; + + return NULL; +} + database_iterator_t *database_create_document_iterator(database_t *db) { sqlite3_stmt *stmt; - // TODO: remove mtime, size, _id from json_data + // TODO optimization: remove mtime, size, _id from json_data sqlite3_prepare_v2(db->db, "WITH doc (j) AS (SELECT CASE" " WHEN sc.json_data IS NULL THEN" @@ -494,10 +537,10 @@ job_t *database_get_work(database_t *db, job_type_t job_type) { CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_index_job_stmt)); pthread_mutex_unlock(&db->ipc_ctx->db_mutex); return NULL; - } else { - CRASH_IF_STMT_FAIL(ret); } + CRASH_IF_STMT_FAIL(ret); + job = malloc(sizeof(*job)); const char *line = (const char *) sqlite3_column_text(db->pop_index_job_stmt, 2); @@ -511,9 +554,6 @@ job_t *database_get_work(database_t *db, job_type_t job_type) { job->bulk_line->type = sqlite3_column_int(db->pop_index_job_stmt, 1); job->bulk_line->next = NULL; - // TODO CRASH IF NOT OK - sqlite3_step(db->pop_parse_job_stmt); - CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_index_job_stmt)); } diff --git a/src/database/database.h b/src/database/database.h index d36f802..3c61661 100644 --- a/src/database/database.h +++ b/src/database/database.h @@ -41,7 +41,7 @@ typedef struct { pthread_mutex_t db_mutex; pthread_mutex_t index_db_mutex; pthread_cond_t has_work_cond; - char current_job[256][PATH_MAX * 2]; + char current_job[MAX_THREADS][PATH_MAX * 2]; } database_ipc_ctx_t; typedef struct database { @@ -106,6 +106,14 @@ cJSON *database_document_iter(database_iterator_t *); #define database_document_iter_foreach(element, iter) \ for (cJSON *element = database_document_iter(iter); element != NULL; element = database_document_iter(iter)) +database_iterator_t *database_create_delete_list_iterator(database_t *db); + +char * database_delete_list_iter(database_iterator_t *iter); + +#define database_delete_list_iter_foreach(element, iter) \ + for (char *element = database_delete_list_iter(iter); element != NULL; element = database_delete_list_iter(iter)) + + cJSON *database_incremental_scan_begin(database_t *db); cJSON *database_incremental_scan_end(database_t *db); diff --git a/src/io/walk.c b/src/io/walk.c index c9fa8b0..dfeb2b3 100644 --- a/src/io/walk.c +++ b/src/io/walk.c @@ -24,9 +24,6 @@ int handle_entry(const char *filepath, const struct stat *info, int typeflag, st LOG_DEBUGF("walk.c", "Excluded: %s", filepath); if (typeflag == FTW_F && S_ISREG(info->st_mode)) { - pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu); - ScanCtx.dbg_excluded_files_count += 1; - pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu); } else if (typeflag == FTW_D) { return FTW_SKIP_SUBTREE; } @@ -83,13 +80,6 @@ int iterate_file_list(void *input_file) { if (ScanCtx.exclude != NULL && EXCLUDED(absolute_path)) { LOG_DEBUGF("walk.c", "Excluded: %s", absolute_path); - - if (S_ISREG(info.st_mode)) { - pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu); - ScanCtx.dbg_excluded_files_count += 1; - pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu); - } - continue; } diff --git a/src/main.c b/src/main.c index 72bdc89..7b02d10 100644 --- a/src/main.c +++ b/src/main.c @@ -18,10 +18,6 @@ #include "src/database/database.h" -#define DESCRIPTION "Lightning-fast file system indexer and search tool." - -#define EPILOG "Made by simon987 . Released under GPL-3.0" - static const char *const usage[] = { "sist2 scan [OPTION]... PATH", @@ -32,77 +28,6 @@ static const char *const usage[] = { }; -static __sighandler_t sigsegv_handler = NULL; -static __sighandler_t sigabrt_handler = NULL; - -void sig_handler(int signum) { - - LogCtx.verbose = TRUE; - LogCtx.very_verbose = TRUE; - - LOG_ERROR("*SIGNAL HANDLER*", "=============================================\n\n"); - LOG_ERRORF("*SIGNAL HANDLER*", "Uh oh! Caught fatal signal: %s", strsignal(signum)); - - // TODO: Print debug info -// if (ScanCtx.dbg_current_files != NULL) { -// GHashTableIter iter; -// g_hash_table_iter_init(&iter, ScanCtx.dbg_current_files); -// -// void *key; -// void *value; -// while (g_hash_table_iter_next(&iter, &key, &value)) { -// parse_job_t *job = value; -// -// if (isatty(STDERR_FILENO)) { -// LOG_DEBUGF( -// "*SIGNAL HANDLER*", -// "Thread \033[%dm[%04llX]\033[0m was working on job '%s'", -// 31 + ((unsigned int) key) % 7, key, job->filepath -// ); -// } else { -// LOG_DEBUGF( -// "*SIGNAL HANDLER*", -// "THREAD [%04llX] was working on job %s", -// key, job->filepath -// ); -// } -// } -// } - - if (ScanCtx.pool != NULL) { - tpool_dump_debug_info(ScanCtx.pool); - } - - if (IndexCtx.pool != NULL) { - tpool_dump_debug_info(IndexCtx.pool); - } - - LOG_INFO( - "*SIGNAL HANDLER*", - "Please consider creating a bug report at https://github.com/simon987/sist2/issues !" - ); - LOG_INFO( - "*SIGNAL HANDLER*", - "sist2 is an open source project and relies on the collaboration of its users to diagnose and fix bugs" - ); - -#ifndef SIST_DEBUG - LOG_WARNING( - "*SIGNAL HANDLER*", - "You are running sist2 in release mode! Please consider downloading the debug binary from the Github " - "releases page to provide additionnal information when submitting a bug report." - ); -#endif - - if (signum == SIGSEGV && sigsegv_handler != NULL) { - sigsegv_handler(signum); - } else if (signum == SIGABRT && sigabrt_handler != NULL) { - sigabrt_handler(signum); - } - - exit(-1); -} - void database_scan_begin(scan_args_t *args) { index_descriptor_t *desc = &ScanCtx.index.desc; @@ -158,7 +83,7 @@ void write_thumbnail_callback(char *key, int num, void *buf, size_t buf_len) { database_write_thumbnail(ProcData.index_db, key, num, buf, buf_len); } -void _log(const char *filepath, int level, char *str) { +void log_callback(const char *filepath, int level, char *str) { if (level == LEVEL_FATAL) { sist_log(filepath, level, str); exit(-1); @@ -175,7 +100,7 @@ void _log(const char *filepath, int level, char *str) { } } -void _logf(const char *filepath, int level, char *format, ...) { +void logf_callback(const char *filepath, int level, char *format, ...) { va_list args; @@ -198,15 +123,13 @@ void _logf(const char *filepath, int level, char *format, ...) { } void initialize_scan_context(scan_args_t *args) { - // TODO: shared - pthread_mutex_init(&ScanCtx.dbg_file_counts_mu, NULL); ScanCtx.calculate_checksums = args->calculate_checksums; // Archive ScanCtx.arc_ctx.mode = args->archive_mode; - ScanCtx.arc_ctx.log = _log; - ScanCtx.arc_ctx.logf = _logf; + ScanCtx.arc_ctx.log = log_callback; + ScanCtx.arc_ctx.logf = logf_callback; ScanCtx.arc_ctx.parse = (parse_callback_t) parse; if (args->archive_passphrase != NULL) { strcpy(ScanCtx.arc_ctx.passphrase, args->archive_passphrase); @@ -215,8 +138,8 @@ void initialize_scan_context(scan_args_t *args) { } // Comic - ScanCtx.comic_ctx.log = _log; - ScanCtx.comic_ctx.logf = _logf; + ScanCtx.comic_ctx.log = log_callback; + ScanCtx.comic_ctx.logf = logf_callback; ScanCtx.comic_ctx.store = write_thumbnail_callback; ScanCtx.comic_ctx.enable_tn = args->tn_count > 0; ScanCtx.comic_ctx.tn_size = args->tn_size; @@ -232,24 +155,24 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.ebook_ctx.tesseract_lang = args->tesseract_lang; ScanCtx.ebook_ctx.tesseract_path = args->tesseract_path; } - ScanCtx.ebook_ctx.log = _log; - ScanCtx.ebook_ctx.logf = _logf; + ScanCtx.ebook_ctx.log = log_callback; + ScanCtx.ebook_ctx.logf = logf_callback; ScanCtx.ebook_ctx.store = write_thumbnail_callback; ScanCtx.ebook_ctx.fast_epub_parse = args->fast_epub; ScanCtx.ebook_ctx.tn_qscale = args->tn_quality; // Font ScanCtx.font_ctx.enable_tn = args->tn_count > 0; - ScanCtx.font_ctx.log = _log; - ScanCtx.font_ctx.logf = _logf; + ScanCtx.font_ctx.log = log_callback; + ScanCtx.font_ctx.logf = logf_callback; ScanCtx.font_ctx.store = write_thumbnail_callback; // Media ScanCtx.media_ctx.tn_qscale = args->tn_quality; ScanCtx.media_ctx.tn_size = args->tn_size; ScanCtx.media_ctx.tn_count = args->tn_count; - ScanCtx.media_ctx.log = _log; - ScanCtx.media_ctx.logf = _logf; + ScanCtx.media_ctx.log = log_callback; + ScanCtx.media_ctx.logf = logf_callback; ScanCtx.media_ctx.store = write_thumbnail_callback; ScanCtx.media_ctx.max_media_buffer = (long) args->max_memory_buffer_mib * 1024 * 1024; ScanCtx.media_ctx.read_subtitles = args->read_subtitles; @@ -264,24 +187,24 @@ void initialize_scan_context(scan_args_t *args) { // OOXML ScanCtx.ooxml_ctx.enable_tn = args->tn_count > 0; ScanCtx.ooxml_ctx.content_size = args->content_size; - ScanCtx.ooxml_ctx.log = _log; - ScanCtx.ooxml_ctx.logf = _logf; + ScanCtx.ooxml_ctx.log = log_callback; + ScanCtx.ooxml_ctx.logf = logf_callback; ScanCtx.ooxml_ctx.store = write_thumbnail_callback; // MOBI ScanCtx.mobi_ctx.content_size = args->content_size; - ScanCtx.mobi_ctx.log = _log; - ScanCtx.mobi_ctx.logf = _logf; + ScanCtx.mobi_ctx.log = log_callback; + ScanCtx.mobi_ctx.logf = logf_callback; // TEXT ScanCtx.text_ctx.content_size = args->content_size; - ScanCtx.text_ctx.log = _log; - ScanCtx.text_ctx.logf = _logf; + ScanCtx.text_ctx.log = log_callback; + ScanCtx.text_ctx.logf = logf_callback; // MSDOC ScanCtx.msdoc_ctx.content_size = args->content_size; - ScanCtx.msdoc_ctx.log = _log; - ScanCtx.msdoc_ctx.logf = _logf; + ScanCtx.msdoc_ctx.log = log_callback; + ScanCtx.msdoc_ctx.logf = logf_callback; ScanCtx.msdoc_ctx.store = write_thumbnail_callback; ScanCtx.msdoc_ctx.msdoc_mime = mime_get_mime_by_string("application/msword"); @@ -299,20 +222,20 @@ void initialize_scan_context(scan_args_t *args) { ScanCtx.raw_ctx.tn_qscale = args->tn_quality; ScanCtx.raw_ctx.enable_tn = args->tn_count > 0; ScanCtx.raw_ctx.tn_size = args->tn_size; - ScanCtx.raw_ctx.log = _log; - ScanCtx.raw_ctx.logf = _logf; + ScanCtx.raw_ctx.log = log_callback; + ScanCtx.raw_ctx.logf = logf_callback; ScanCtx.raw_ctx.store = write_thumbnail_callback; // Wpd ScanCtx.wpd_ctx.content_size = args->content_size; - ScanCtx.wpd_ctx.log = _log; - ScanCtx.wpd_ctx.logf = _logf; + ScanCtx.wpd_ctx.log = log_callback; + ScanCtx.wpd_ctx.logf = logf_callback; ScanCtx.wpd_ctx.wpd_mime = mime_get_mime_by_string("application/wordperfect"); // Json ScanCtx.json_ctx.content_size = args->content_size; - ScanCtx.json_ctx.log = _log; - ScanCtx.json_ctx.logf = _logf; + ScanCtx.json_ctx.log = log_callback; + ScanCtx.json_ctx.logf = logf_callback; ScanCtx.json_ctx.json_mime = mime_get_mime_by_string("application/json"); ScanCtx.json_ctx.ndjson_mime = mime_get_mime_by_string("application/ndjson"); } @@ -344,9 +267,6 @@ void sist2_scan(scan_args_t *args) { tpool_wait(ScanCtx.pool); tpool_destroy(ScanCtx.pool); - LOG_DEBUGF("main.c", "Skipped files: %d", ScanCtx.dbg_skipped_files_count); - LOG_DEBUGF("main.c", "Excluded files: %d", ScanCtx.dbg_excluded_files_count); - LOG_DEBUGF("main.c", "Failed files: %d", ScanCtx.dbg_failed_files_count); LOG_DEBUGF("main.c", "Thumbnail store size: %lu", ScanCtx.stat_tn_size); LOG_DEBUGF("main.c", "Index size: %lu", ScanCtx.stat_index_size); @@ -358,7 +278,7 @@ void sist2_scan(scan_args_t *args) { } database_generate_stats(db, args->treemap_threshold); - database_close(db, TRUE); + database_close(db, args->optimize_database); } void sist2_index(index_args_t *args) { @@ -397,16 +317,19 @@ void sist2_index(index_args_t *args) { print_json(json, doc_id); } else { index_json(json, doc_id); - cnt +=1; + cnt += 1; } } free(iterator); database_close(db, FALSE); - // Only read the _delete index if we're sending data to ES if (!args->print) { - // TODO: (delete_list iterator) + database_iterator_t *del_iter = database_create_delete_list_iterator(db); + database_delete_list_iter_foreach(id, del_iter) { + delete_document(id); + free(id); + } } tpool_wait(IndexCtx.pool); @@ -496,12 +419,7 @@ int set_to_negative_if_value_is_zero(UNUSED(struct argparse *self), const struct } } -#include - int main(int argc, const char *argv[]) { -// sigsegv_handler = signal(SIGSEGV, sig_handler); -// sigabrt_handler = signal(SIGABRT, sig_handler); - setlocale(LC_ALL, ""); scan_args_t *scan_args = scan_args_create(); @@ -521,36 +439,37 @@ int main(int argc, const char *argv[]) { struct argparse_option options[] = { OPT_HELP(), - OPT_BOOLEAN('v', "version", &arg_version, "Show version and exit"), - OPT_BOOLEAN(0, "verbose", &LogCtx.verbose, "Turn on logging"), - OPT_BOOLEAN(0, "very-verbose", &LogCtx.very_verbose, "Turn on debug messages"), + OPT_BOOLEAN('v', "version", &arg_version, "Print version and exit."), + OPT_BOOLEAN(0, "verbose", &LogCtx.verbose, "Turn on logging."), + OPT_BOOLEAN(0, "very-verbose", &LogCtx.very_verbose, "Turn on debug messages."), OPT_BOOLEAN(0, "json-logs", &LogCtx.json_logs, "Output logs in JSON format."), OPT_GROUP("Scan options"), - OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"), + OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT: 1"), OPT_INTEGER('q', "thumbnail-quality", &scan_args->tn_quality, - "Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT=2", + "Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2", set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_quality), OPT_INTEGER(0, "thumbnail-size", &scan_args->tn_size, - "Thumbnail size, in pixels. DEFAULT=500", + "Thumbnail size, in pixels. DEFAULT: 552", set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_size), OPT_INTEGER(0, "thumbnail-count", &scan_args->tn_count, - "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT=1", + "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1", set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_count), OPT_INTEGER(0, "content-size", &scan_args->content_size, - "Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT=32768", + "Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768", set_to_negative_if_value_is_zero, (intptr_t) &scan_args->content_size), + OPT_STRING('o', "output", &scan_args->output, "Output index file path. DEFAULT: index.sist2"), OPT_BOOLEAN(0, "incremental", &scan_args->incremental, - // TODO: Update help string - "Reuse an existing index and only scan modified files."), - OPT_STRING('o', "output", &scan_args->output, "Output directory. DEFAULT=index.sist2/"), + "If the output file path exists, only scan new or modified files."), + OPT_BOOLEAN(0, "optimize-index", &scan_args->optimize_database, + "Defragment index file after scan to reduce its file size."), OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."), - OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: (name of the directory)"), + OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: index"), OPT_INTEGER(0, "depth", &scan_args->depth, "Scan up to DEPTH subdirectories deep. " "Use 0 to only scan files in PATH. DEFAULT: -1"), OPT_STRING(0, "archive", &scan_args->archive, "Archive file mode (skip|list|shallow|recurse). " - "skip: Don't parse, list: only get file names as text, " - "shallow: Don't parse archives inside archives. DEFAULT: recurse"), + "skip: don't scan, list: only save file names as text, " + "shallow: don't scan archives inside archives. DEFAULT: recurse"), OPT_STRING(0, "archive-passphrase", &scan_args->archive_passphrase, "Passphrase for encrypted archive files"), @@ -559,8 +478,8 @@ int main(int argc, const char *argv[]) { "which are installed on your machine)"), OPT_BOOLEAN(0, "ocr-images", &scan_args->ocr_images, "Enable OCR'ing of image files."), OPT_BOOLEAN(0, "ocr-ebooks", &scan_args->ocr_ebooks, "Enable OCR'ing of ebook files."), - OPT_STRING('e', "exclude", &scan_args->exclude_regex, "Files that match this regex will not be scanned"), - OPT_BOOLEAN(0, "fast", &scan_args->fast, "Only index file names & mime type"), + OPT_STRING('e', "exclude", &scan_args->exclude_regex, "Files that match this regex will not be scanned."), + OPT_BOOLEAN(0, "fast", &scan_args->fast, "Only index file names & mime type."), OPT_STRING(0, "treemap-threshold", &scan_args->treemap_threshold_str, "Relative size threshold for treemap " "(see USAGE.md). DEFAULT: 0.0005"), OPT_INTEGER(0, "mem-buffer", &scan_args->max_memory_buffer_mib, @@ -568,19 +487,20 @@ int main(int argc, const char *argv[]) { "(see USAGE.md). DEFAULT: 2000"), OPT_BOOLEAN(0, "read-subtitles", &scan_args->read_subtitles, "Read subtitles from media files."), OPT_BOOLEAN(0, "fast-epub", &scan_args->fast_epub, - "Faster but less accurate EPUB parsing (no thumbnails, metadata)"), + "Faster but less accurate EPUB parsing (no thumbnails, metadata)."), OPT_BOOLEAN(0, "checksums", &scan_args->calculate_checksums, "Calculate file checksums when scanning."), OPT_STRING(0, "list-file", &scan_args->list_path, "Specify a list of newline-delimited paths to be scanned" " instead of normal directory traversal. Use '-' to read" " from stdin."), OPT_GROUP("Index options"), - OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"), - OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"), + OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT: 1"), + OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT: http://localhost:9200"), OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."), - OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), - OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."), + OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT: sist2"), + OPT_BOOLEAN('p', "print", &index_args->print, + "Print JSON documents to stdout instead of indexing to elasticsearch."), OPT_BOOLEAN(0, "incremental-index", &index_args->incremental, "Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch."), OPT_STRING(0, "script-file", &common_script_path, "Path to user script."), @@ -588,15 +508,15 @@ int main(int argc, const char *argv[]) { OPT_STRING(0, "settings-file", &index_args->es_settings_path, "Path to Elasticsearch settings."), OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."), OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 70"), - OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. " - "(You must use this option the first time you use the index command)"), + OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings."), OPT_GROUP("Web options"), - OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"), + OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT: http://localhost:9200"), OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."), - OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), - OPT_STRING(0, "bind", &web_args->listen_address, "Listen on this address. DEFAULT=localhost:4090"), + OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT: sist2"), + OPT_STRING(0, "bind", &web_args->listen_address, + "Listen for connections on this address. DEFAULT: localhost:4090"), OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"), OPT_STRING(0, "auth0-audience", &web_args->auth0_audience, "API audience/identifier"), OPT_STRING(0, "auth0-domain", &web_args->auth0_domain, "Application domain"), @@ -609,10 +529,10 @@ int main(int argc, const char *argv[]) { OPT_STRING(0, "lang", &web_args->lang, "Default UI language. Can be changed by the user"), OPT_GROUP("Exec-script options"), - OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"), + OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT: http://localhost:9200"), OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."), - OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), + OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT: sist2"), OPT_STRING(0, "script-file", &common_script_path, "Path to user script."), OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."), @@ -621,7 +541,11 @@ int main(int argc, const char *argv[]) { struct argparse argparse; argparse_init(&argparse, options, usage, 0); - argparse_describe(&argparse, DESCRIPTION, EPILOG); + argparse_describe( + &argparse, + "\nLightning-fast file system indexer and search tool.", + "\nMade by simon987 . Released under GPL-3.0" + ); argc = argparse_parse(&argparse, argc, argv); if (arg_version) { diff --git a/src/parsing/fs_util.h b/src/parsing/fs_util.h index a3b257e..3b7c1a7 100644 --- a/src/parsing/fs_util.h +++ b/src/parsing/fs_util.h @@ -6,7 +6,6 @@ #define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));}; static int fs_read(struct vfile *f, void *buf, size_t size) { - if (f->fd == -1) { SHA1_Init(&f->sha1_ctx); diff --git a/src/parsing/magic_util.c b/src/parsing/magic_util.c index e5443a8..d8539a1 100644 --- a/src/parsing/magic_util.c +++ b/src/parsing/magic_util.c @@ -12,7 +12,7 @@ char *magic_buffer_embedded(void *buffer, size_t buffer_size) { const char *magic_buffers[1] = {magic_database_buffer,}; size_t sizes[1] = {sizeof(magic_database_buffer),}; - // TODO: check if we can reuse the magic instance + // TODO optimisation: check if we can reuse the magic instance int load_ret = magic_load_buffers(magic, (void **) &magic_buffers, sizes, 1); if (load_ret != 0) { diff --git a/src/parsing/mime_generated.c b/src/parsing/mime_generated.c index 6eedeae..c68c52c 100644 --- a/src/parsing/mime_generated.c +++ b/src/parsing/mime_generated.c @@ -174,287 +174,291 @@ application_x_mach_binary=655526, application_x_mach_executable=655527, application_x_magic_cap_package_1_0=655528, application_x_mathcad=655529, -application_x_maxis_dbpf=655530, -application_x_meme=655531, -application_x_midi=655532, -application_x_mif=655533, -application_x_mix_transfer=655534, -application_x_mobipocket_ebook=655535 | 0x02000000, -application_x_ms_compress_szdd=655536, -application_x_ms_pdb=655537, -application_x_ms_reader=655538, -application_x_msaccess=655539, -application_x_n64_rom=655540, -application_x_navi_animation=655541, -application_x_navidoc=655542, -application_x_navimap=655543, -application_x_navistyle=655544, -application_x_nes_rom=655545, -application_x_netcdf=655546, -application_x_newton_compatible_pkg=655547, -application_x_nintendo_ds_rom=655548, -application_x_object=655549, -application_x_omc=655550, -application_x_omcdatamaker=655551, -application_x_omcregerator=655552, -application_x_pagemaker=655553, -application_x_pcl=655554, -application_x_pgp_keyring=655555, -application_x_pixclscript=655556, -application_x_pkcs7_certreqresp=655557, -application_x_pkcs7_signature=655558, -application_x_project=655559, -application_x_qpro=655560, -application_x_rar=655561 | 0x10000000, -application_x_rpm=655562, -application_x_sdp=655563, -application_x_sea=655564, -application_x_seelogo=655565, -application_x_setupscript=655566, -application_x_shar=655567, -application_x_sharedlib=655568, -application_x_shockwave_flash=655569, -application_x_snappy_framed=655570, -application_x_sprite=655571, -application_x_sqlite3=655572, -application_x_stargallery_thm=655573, -application_x_stuffit=655574, -application_x_sv4cpio=655575, -application_x_sv4crc=655576, -application_x_tar=655577 | 0x10000000, -application_x_tbook=655578, -application_x_terminfo=655579, -application_x_terminfo2=655580, -application_x_tex_tfm=655581, -application_x_texinfo=655582, -application_x_ustar=655583, -application_x_visio=655584, -application_x_vnd_audioexplosion_mzz=655585, -application_x_vnd_ls_xpix=655586, -application_x_vrml=655587, -application_x_wais_source=655588, -application_x_wine_extension_ini=655589, -application_x_wintalk=655590, -application_x_world=655591, -application_x_wri=655592, -application_x_x509_ca_cert=655593, -application_x_xz=655594 | 0x08000000, -application_x_zip=655595, -application_x_zstd=655596 | 0x08000000, -application_x_zstd_dictionary=655597, -application_xml=655598, -application_zip=655599 | 0x10000000, -application_zlib=655600, -audio_basic=458993 | 0x80000000, -audio_it=458994, -audio_make=458995, -audio_mid=458996, -audio_midi=458997, -audio_mp4=458998, -audio_mpeg=458999, -audio_ogg=459000, -audio_s3m=459001, -audio_tsp_audio=459002, -audio_tsplayer=459003, -audio_vnd_qcelp=459004, -audio_voxware=459005, -audio_x_aiff=459006, -audio_x_flac=459007, -audio_x_gsm=459008, -audio_x_hx_aac_adts=459009, -audio_x_jam=459010, -audio_x_liveaudio=459011, -audio_x_m4a=459012, -audio_x_midi=459013, -audio_x_mod=459014, -audio_x_mp4a_latm=459015, -audio_x_mpeg_3=459016, -audio_x_mpequrl=459017, -audio_x_nspaudio=459018, -audio_x_pn_realaudio=459019, -audio_x_psid=459020, -audio_x_realaudio=459021, -audio_x_s3m=459022, -audio_x_twinvq=459023, -audio_x_twinvq_plugin=459024, -audio_x_voc=459025, -audio_x_wav=459026, -audio_x_xbox_executable=459027 | 0x80000000, -audio_x_xbox360_executable=459028 | 0x80000000, -audio_xm=459029, -font_otf=327958 | 0x20000000, -font_sfnt=327959 | 0x20000000, -font_woff=327960 | 0x20000000, -font_woff2=327961 | 0x20000000, -image_bmp=524570, -image_cmu_raster=524571, -image_fif=524572, -image_florian=524573, -image_g3fax=524574, -image_gif=524575, -image_heic=524576, -image_ief=524577, -image_jpeg=524578, -image_jutvision=524579, -image_naplps=524580, -image_pict=524581, -image_png=524582, -image_svg=524583 | 0x80000000, -image_svg_xml=524584 | 0x80000000, -image_tiff=524585, -image_vnd_adobe_photoshop=524586 | 0x80000000, -image_vnd_djvu=524587 | 0x80000000, -image_vnd_fpx=524588, -image_vnd_microsoft_icon=524589, -image_vnd_rn_realflash=524590, -image_vnd_rn_realpix=524591, -image_vnd_wap_wbmp=524592, -image_vnd_xiff=524593, -image_webp=524594, -image_wmf=524595, -image_x_3ds=524596, -image_x_adobe_dng=524597 | 0x00800000, -image_x_award_bioslogo=524598, -image_x_canon_cr2=524599 | 0x00800000, -image_x_canon_crw=524600 | 0x00800000, -image_x_cmu_raster=524601, -image_x_cur=524602, -image_x_dcraw=524603 | 0x00800000, -image_x_dwg=524604, -image_x_eps=524605, -image_x_epson_erf=524606 | 0x00800000, -image_x_exr=524607, -image_x_fuji_raf=524608 | 0x00800000, -image_x_gem=524609, -image_x_icns=524610, -image_x_icon=524611 | 0x80000000, -image_x_jg=524612, -image_x_jps=524613, -image_x_kodak_dcr=524614 | 0x00800000, -image_x_kodak_k25=524615 | 0x00800000, -image_x_kodak_kdc=524616 | 0x00800000, -image_x_minolta_mrw=524617 | 0x00800000, -image_x_ms_bmp=524618, -image_x_niff=524619, -image_x_nikon_nef=524620 | 0x00800000, -image_x_olympus_orf=524621 | 0x00800000, -image_x_panasonic_raw=524622 | 0x00800000, -image_x_pcx=524623, -image_x_pentax_pef=524624 | 0x00800000, -image_x_pict=524625, -image_x_portable_bitmap=524626, -image_x_portable_graymap=524627, -image_x_portable_pixmap=524628, -image_x_quicktime=524629, -image_x_rgb=524630, -image_x_sigma_x3f=524631 | 0x00800000, -image_x_sony_arw=524632 | 0x00800000, -image_x_sony_sr2=524633 | 0x00800000, -image_x_sony_srf=524634 | 0x00800000, -image_x_tga=524635, -image_x_tiff=524636, -image_x_win_bitmap=524637, -image_x_xcf=524638 | 0x80000000, -image_x_xpixmap=524639 | 0x80000000, -image_x_xwindowdump=524640, -message_news=196961, -message_rfc822=196962, -model_vnd_dwf=65891, -model_vnd_gdl=65892, -model_vnd_gs_gdl=65893, -model_vrml=65894, -model_x_pov=65895, +application_x_matlab_data=655530, +application_x_maxis_dbpf=655531, +application_x_meme=655532, +application_x_midi=655533, +application_x_mif=655534, +application_x_mix_transfer=655535, +application_x_mobipocket_ebook=655536 | 0x02000000, +application_x_ms_compress_szdd=655537, +application_x_ms_pdb=655538, +application_x_ms_reader=655539, +application_x_msaccess=655540, +application_x_n64_rom=655541, +application_x_navi_animation=655542, +application_x_navidoc=655543, +application_x_navimap=655544, +application_x_navistyle=655545, +application_x_nes_rom=655546, +application_x_netcdf=655547, +application_x_newton_compatible_pkg=655548, +application_x_nintendo_ds_rom=655549, +application_x_object=655550, +application_x_omc=655551, +application_x_omcdatamaker=655552, +application_x_omcregerator=655553, +application_x_pagemaker=655554, +application_x_pcl=655555, +application_x_pgp_keyring=655556, +application_x_pixclscript=655557, +application_x_pkcs7_certreqresp=655558, +application_x_pkcs7_signature=655559, +application_x_project=655560, +application_x_qpro=655561, +application_x_rar=655562 | 0x10000000, +application_x_rpm=655563, +application_x_sdp=655564, +application_x_sea=655565, +application_x_seelogo=655566, +application_x_setupscript=655567, +application_x_shar=655568, +application_x_sharedlib=655569, +application_x_shockwave_flash=655570, +application_x_snappy_framed=655571, +application_x_sprite=655572, +application_x_sqlite3=655573, +application_x_stargallery_thm=655574, +application_x_stuffit=655575, +application_x_sv4cpio=655576, +application_x_sv4crc=655577, +application_x_tar=655578 | 0x10000000, +application_x_tbook=655579, +application_x_terminfo=655580, +application_x_terminfo2=655581, +application_x_tex_tfm=655582, +application_x_texinfo=655583, +application_x_ustar=655584, +application_x_visio=655585, +application_x_vnd_audioexplosion_mzz=655586, +application_x_vnd_ls_xpix=655587, +application_x_vrml=655588, +application_x_wais_source=655589, +application_x_wine_extension_ini=655590, +application_x_wintalk=655591, +application_x_world=655592, +application_x_wri=655593, +application_x_x509_ca_cert=655594, +application_x_xz=655595 | 0x08000000, +application_x_zip=655596, +application_x_zstd=655597 | 0x08000000, +application_x_zstd_dictionary=655598, +application_xml=655599, +application_zip=655600 | 0x10000000, +application_zlib=655601, +audio_basic=458994 | 0x80000000, +audio_it=458995, +audio_make=458996, +audio_mid=458997, +audio_midi=458998, +audio_mp4=458999, +audio_mpeg=459000, +audio_ogg=459001, +audio_s3m=459002, +audio_tsp_audio=459003, +audio_tsplayer=459004, +audio_vnd_qcelp=459005, +audio_voxware=459006, +audio_x_aiff=459007, +audio_x_flac=459008, +audio_x_gsm=459009, +audio_x_hx_aac_adts=459010, +audio_x_jam=459011, +audio_x_liveaudio=459012, +audio_x_m4a=459013, +audio_x_midi=459014, +audio_x_mod=459015, +audio_x_mp4a_latm=459016, +audio_x_mpeg_3=459017, +audio_x_mpequrl=459018, +audio_x_nspaudio=459019, +audio_x_pn_realaudio=459020, +audio_x_psid=459021, +audio_x_realaudio=459022, +audio_x_s3m=459023, +audio_x_twinvq=459024, +audio_x_twinvq_plugin=459025, +audio_x_voc=459026, +audio_x_wav=459027, +audio_x_xbox_executable=459028 | 0x80000000, +audio_x_xbox360_executable=459029 | 0x80000000, +audio_xm=459030, +font_otf=327959 | 0x20000000, +font_sfnt=327960 | 0x20000000, +font_woff=327961 | 0x20000000, +font_woff2=327962 | 0x20000000, +image_bmp=524571, +image_cmu_raster=524572, +image_fif=524573, +image_florian=524574, +image_g3fax=524575, +image_gif=524576, +image_heic=524577, +image_ief=524578, +image_jpeg=524579, +image_jutvision=524580, +image_naplps=524581, +image_pict=524582, +image_png=524583, +image_svg=524584 | 0x80000000, +image_svg_xml=524585 | 0x80000000, +image_tiff=524586, +image_vnd_adobe_photoshop=524587 | 0x80000000, +image_vnd_djvu=524588 | 0x80000000, +image_vnd_fpx=524589, +image_vnd_microsoft_icon=524590, +image_vnd_rn_realflash=524591, +image_vnd_rn_realpix=524592, +image_vnd_wap_wbmp=524593, +image_vnd_xiff=524594, +image_webp=524595, +image_wmf=524596, +image_x_3ds=524597, +image_x_adobe_dng=524598 | 0x00800000, +image_x_award_bioslogo=524599, +image_x_canon_cr2=524600 | 0x00800000, +image_x_canon_crw=524601 | 0x00800000, +image_x_cmu_raster=524602, +image_x_cur=524603, +image_x_dcraw=524604 | 0x00800000, +image_x_dwg=524605, +image_x_eps=524606, +image_x_epson_erf=524607 | 0x00800000, +image_x_exr=524608, +image_x_fuji_raf=524609 | 0x00800000, +image_x_gem=524610, +image_x_icns=524611, +image_x_icon=524612 | 0x80000000, +image_x_jg=524613, +image_x_jps=524614, +image_x_kodak_dcr=524615 | 0x00800000, +image_x_kodak_k25=524616 | 0x00800000, +image_x_kodak_kdc=524617 | 0x00800000, +image_x_minolta_mrw=524618 | 0x00800000, +image_x_ms_bmp=524619, +image_x_niff=524620, +image_x_nikon_nef=524621 | 0x00800000, +image_x_olympus_orf=524622 | 0x00800000, +image_x_panasonic_raw=524623 | 0x00800000, +image_x_pcx=524624, +image_x_pentax_pef=524625 | 0x00800000, +image_x_pict=524626, +image_x_portable_bitmap=524627, +image_x_portable_graymap=524628, +image_x_portable_pixmap=524629, +image_x_quicktime=524630, +image_x_rgb=524631, +image_x_sigma_x3f=524632 | 0x00800000, +image_x_sony_arw=524633 | 0x00800000, +image_x_sony_sr2=524634 | 0x00800000, +image_x_sony_srf=524635 | 0x00800000, +image_x_tga=524636, +image_x_tiff=524637, +image_x_win_bitmap=524638, +image_x_xcf=524639 | 0x80000000, +image_x_xpixmap=524640 | 0x80000000, +image_x_xwindowdump=524641, +message_news=196962, +message_rfc822=196963, +model_vnd_dwf=65892, +model_vnd_gdl=65893, +model_vnd_gs_gdl=65894, +model_vrml=65895, +model_x_pov=65896, sist2_sidecar=2, -text_PGP=590184, -text_asp=590185, -text_css=590186, -text_html=590187 | 0x01000000, -text_javascript=590188, -text_mcf=590189, -text_pascal=590190, -text_plain=590191, -text_richtext=590192, -text_rtf=590193, -text_scriplet=590194, -text_tab_separated_values=590195, -text_troff=590196, -text_uri_list=590197, -text_vnd_abc=590198, -text_vnd_fmi_flexstor=590199, -text_vnd_wap_wml=590200, -text_vnd_wap_wmlscript=590201, -text_webviewhtml=590202, -text_x_Algol68=590203, -text_x_asm=590204, -text_x_audiosoft_intra=590205, -text_x_awk=590206, -text_x_bcpl=590207, -text_x_c=590208, -text_x_c__=590209, -text_x_component=590210, -text_x_diff=590211, -text_x_fortran=590212, -text_x_java=590213, -text_x_la_asf=590214, -text_x_lisp=590215, -text_x_m=590216, -text_x_m4=590217, -text_x_makefile=590218, -text_x_ms_regedit=590219, -text_x_msdos_batch=590220, -text_x_objective_c=590221, -text_x_pascal=590222, -text_x_perl=590223, -text_x_php=590224, -text_x_po=590225, -text_x_python=590226, -text_x_ruby=590227, -text_x_sass=590228, -text_x_scss=590229, -text_x_server_parsed_html=590230, -text_x_setext=590231, -text_x_sgml=590232 | 0x01000000, -text_x_shellscript=590233, -text_x_speech=590234, -text_x_tcl=590235, -text_x_tex=590236, -text_x_uil=590237, -text_x_uuencode=590238, -text_x_vcalendar=590239, -text_x_vcard=590240, -text_xml=590241 | 0x01000000, -video_MP2T=393634, -video_animaflex=393635, -video_avi=393636, -video_avs_video=393637, -video_mp4=393638, -video_mpeg=393639, -video_quicktime=393640, -video_vdo=393641, -video_vivo=393642, -video_vnd_rn_realvideo=393643, -video_vosaic=393644, -video_webm=393645, -video_x_amt_demorun=393646, -video_x_amt_showrun=393647, -video_x_atomic3d_feature=393648, -video_x_dl=393649, -video_x_dv=393650, -video_x_fli=393651, -video_x_flv=393652, -video_x_isvideo=393653, -video_x_jng=393654 | 0x80000000, -video_x_m4v=393655, -video_x_matroska=393656, -video_x_mng=393657, -video_x_motion_jpeg=393658, -video_x_ms_asf=393659, -video_x_msvideo=393660, -video_x_qtc=393661, -video_x_sgi_movie=393662, -x_epoc_x_sisx_app=721343, +text_PGP=590185, +text_asp=590186, +text_css=590187, +text_csv=590188, +text_html=590189 | 0x01000000, +text_javascript=590190, +text_mcf=590191, +text_pascal=590192, +text_plain=590193, +text_richtext=590194, +text_rtf=590195, +text_scriplet=590196, +text_tab_separated_values=590197, +text_troff=590198, +text_uri_list=590199, +text_vnd_abc=590200, +text_vnd_fmi_flexstor=590201, +text_vnd_wap_wml=590202, +text_vnd_wap_wmlscript=590203, +text_webviewhtml=590204, +text_x_Algol68=590205, +text_x_asm=590206, +text_x_audiosoft_intra=590207, +text_x_awk=590208, +text_x_bcpl=590209, +text_x_c=590210, +text_x_c__=590211, +text_x_component=590212, +text_x_diff=590213, +text_x_fortran=590214, +text_x_java=590215, +text_x_la_asf=590216, +text_x_lisp=590217, +text_x_m=590218, +text_x_m4=590219, +text_x_makefile=590220, +text_x_ms_regedit=590221, +text_x_msdos_batch=590222, +text_x_objective_c=590223, +text_x_pascal=590224, +text_x_perl=590225, +text_x_php=590226, +text_x_po=590227, +text_x_python=590228, +text_x_ruby=590229, +text_x_sass=590230, +text_x_script_python=590231, +text_x_scss=590232, +text_x_server_parsed_html=590233, +text_x_setext=590234, +text_x_sgml=590235 | 0x01000000, +text_x_shellscript=590236, +text_x_speech=590237, +text_x_tcl=590238, +text_x_tex=590239, +text_x_uil=590240, +text_x_uuencode=590241, +text_x_vcalendar=590242, +text_x_vcard=590243, +text_xml=590244 | 0x01000000, +video_MP2T=393637, +video_animaflex=393638, +video_avi=393639, +video_avs_video=393640, +video_mp4=393641, +video_mpeg=393642, +video_quicktime=393643, +video_vdo=393644, +video_vivo=393645, +video_vnd_rn_realvideo=393646, +video_vosaic=393647, +video_webm=393648, +video_x_amt_demorun=393649, +video_x_amt_showrun=393650, +video_x_atomic3d_feature=393651, +video_x_dl=393652, +video_x_dv=393653, +video_x_fli=393654, +video_x_flv=393655, +video_x_isvideo=393656, +video_x_jng=393657 | 0x80000000, +video_x_m4v=393658, +video_x_matroska=393659, +video_x_mng=393660, +video_x_motion_jpeg=393661, +video_x_ms_asf=393662, +video_x_msvideo=393663, +video_x_qtc=393664, +video_x_sgi_movie=393665, +x_epoc_x_sisx_app=721346, }; char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) { +case application_x_matlab_data: return "application/x-matlab-data"; case application_arj: return "application/arj"; case application_base64: return "application/base64"; case application_binhex: return "application/binhex"; @@ -802,6 +806,8 @@ case text_mcf: return "text/mcf"; case text_pascal: return "text/pascal"; case text_PGP: return "text/PGP"; case text_plain: return "text/plain"; +case text_x_script_python: return "text/x-script.python"; +case text_csv: return "text/csv"; case application_vnd_coffeescript: return "application/vnd.coffeescript"; case text_richtext: return "text/richtext"; case text_rtf: return "text/rtf"; @@ -906,6 +912,7 @@ case image_x_epson_erf: return "image/x-epson-erf"; case sist2_sidecar: return "sist2/sidecar"; default: return NULL;}} unsigned int mime_extension_lookup(unsigned long extension_crc32) {switch (extension_crc32) { +case 2495639202:return application_x_matlab_data; case 104524599:return application_arj; case 1388642652:return application_base64; case 3514823219:return application_binhex; @@ -1194,6 +1201,7 @@ case 398963028:return text_javascript; case 1431272808:return text_mcf; case 509266722:return text_pascal; case 1689700070:case 794565824:case 351504808:case 214229345:case 30677878:case 1835907068:case 1154021400:case 3992351814:case 2107886487:case 2202503947:case 999008199:case 473390917:case 3679822420:case 1465078094:case 1466496025:case 2277716423:case 157353380:case 2002237032:case 4216257084:case 590894066:case 987584319:case 2268432115:case 3551958239:case 1436306077:case 3060306774:case 808890964:case 2564639436:case 3322219037:case 3334425408:case 3818365258:case 1403162576:case 590812979:case 1800036834:case 144986711:case 621471808:case 449607278:case 2403297477:case 2529069283:case 3929123204:return text_plain; +case 194218739:return text_x_script_python; case 1401235891:return application_vnd_coffeescript; case 196656302:case 1203117491:case 3183026384:return text_richtext; case 2119613712:return text_scriplet; @@ -1288,6 +1296,7 @@ case 142938048:return image_x_epson_erf; case 287571459:return sist2_sidecar; default: return 0;}} unsigned int mime_name_lookup(unsigned long mime_crc32) {switch (mime_crc32) { +case 3272851765: return application_x_matlab_data; case 3812269631: return application_arj; case 2479484568: return application_base64; case 3891182180: return application_binhex; @@ -1635,6 +1644,8 @@ case 768274928: return text_mcf; case 3970938585: return text_pascal; case 1059844876: return text_PGP; case 1152832851: return text_plain; +case 3112468514: return text_x_script_python; +case 1881267919: return text_csv; case 2809123822: return application_vnd_coffeescript; case 4000659158: return text_richtext; case 1060344107: return text_rtf; diff --git a/src/parsing/parse.c b/src/parsing/parse.c index 4786407..e680a03 100644 --- a/src/parsing/parse.c +++ b/src/parsing/parse.c @@ -46,17 +46,13 @@ file_type_t get_file_type(unsigned int mime, size_t size, const char *filepath) return FILETYPE_MEDIA; } else if (IS_PDF(mime)) { return FILETYPE_EBOOK; - } else if (major_mime == MimeText && ScanCtx.text_ctx.content_size > 0) { - if (IS_MARKUP(mime)) { - return FILETYPE_MARKUP; - } else { - return FILETYPE_TEXT; - } - + } else if (IS_MARKUP(mime)) { + return FILETYPE_MARKUP; + } else if (major_mime == MimeText) { + return FILETYPE_TEXT; } else if (IS_FONT(mime)) { return FILETYPE_FONT; - } else if ( - ScanCtx.arc_ctx.mode != ARC_MODE_SKIP && ( + } else if (ScanCtx.arc_ctx.mode != ARC_MODE_SKIP && ( IS_ARC(mime) || (IS_ARC_FILTER(mime) && should_parse_filtered_file(filepath)) )) { @@ -98,10 +94,6 @@ int get_mime(parse_job_t *job) { } } - if (strlen(extension) == 0 && strlen(job->filepath + job->base) == 40) { - fprintf(stderr, "GIT? %s", job->filepath); - } - if (ScanCtx.fast) { return 0; } @@ -122,7 +114,6 @@ int get_mime(parse_job_t *job) { LOG_ERRORF(job->filepath, "(virtual) read(): [%d] %s", bytes_read, archive_error_string(job->vfile.arc)); } - return GET_MIME_ERROR_FATAL; } @@ -130,12 +121,13 @@ int get_mime(parse_job_t *job) { if (magic_mime_str != NULL) { mime = (int) mime_get_mime_by_string(magic_mime_str); - free(magic_mime_str); if (mime == 0) { LOG_WARNINGF(job->filepath, "Couldn't find mime %s", magic_mime_str); + free(magic_mime_str); return 0; } + free(magic_mime_str); } if (job->vfile.reset != NULL) { @@ -163,14 +155,11 @@ void parse(parse_job_t *job) { doc->meta_head = NULL; doc->meta_tail = NULL; doc->size = job->vfile.st_size; - doc->mtime = (int) job->vfile.mtime; + doc->mtime = job->vfile.mtime; doc->mime = get_mime(job); generate_doc_id(doc->filepath + ScanCtx.index.desc.root_len, doc->doc_id); if (doc->mime == GET_MIME_ERROR_FATAL) { - pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu); - ScanCtx.dbg_failed_files_count += 1; - pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu); CLOSE_FILE(job->vfile) free(doc); @@ -178,9 +167,6 @@ void parse(parse_job_t *job) { } if (database_mark_document(ProcData.index_db, doc->doc_id, doc->mtime)) { - pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu); - ScanCtx.dbg_skipped_files_count += 1; - pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu); CLOSE_FILE(job->vfile) free(doc); @@ -246,7 +232,7 @@ void parse(parse_job_t *job) { meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + SIST_INDEX_ID_LEN); meta_parent->key = MetaParent; strcpy(meta_parent->str_val, job->parent); - APPEND_META((doc), meta_parent) + APPEND_META((doc), meta_parent); } CLOSE_FILE(job->vfile) @@ -254,7 +240,7 @@ void parse(parse_job_t *job) { if (job->vfile.has_checksum) { char sha1_digest_str[SHA1_STR_LENGTH]; buf2hex((unsigned char *) job->vfile.sha1_digest, SHA1_DIGEST_LENGTH, (char *) sha1_digest_str); - APPEND_STR_META(doc, MetaChecksum, (const char *) sha1_digest_str) + APPEND_STR_META(doc, MetaChecksum, (const char *) sha1_digest_str); } write_document(doc); diff --git a/src/sist.h b/src/sist.h index 8e6eb5a..3598cb6 100644 --- a/src/sist.h +++ b/src/sist.h @@ -27,6 +27,8 @@ #define UNUSED(x) __attribute__((__unused__)) x +#define MAX_THREADS (256) + #include "util.h" #include "log.h" #include "types.h" diff --git a/src/tpool.c b/src/tpool.c index 8f665f8..6cdf31e 100644 --- a/src/tpool.c +++ b/src/tpool.c @@ -6,7 +6,7 @@ #include #include "parsing/parse.h" -#define BLANK_STR " " +#define BLANK_STR " " typedef struct { int thread_id; @@ -17,7 +17,6 @@ typedef struct { typedef struct tpool { pthread_t threads[256]; int num_threads; - int fork; int print_progress; @@ -32,6 +31,8 @@ typedef struct tpool { pthread_cond_t workers_initialized_cond; int busy_count; int initialized_count; + int thread_id_to_pid_mapping[MAX_THREADS]; + char ipc_database_filepath[128]; } *shm; } tpool_t; @@ -43,11 +44,6 @@ void job_destroy(job_t *job) { free(job); } -void tpool_dump_debug_info(tpool_t *pool) { - // TODO - LOG_DEBUGF("tpool.c", "pool->num_threads = %d", pool->num_threads); -} - /** * Push work object to thread pool */ @@ -130,108 +126,124 @@ static void worker_thread_loop(tpool_t *pool) { } static void worker_proc_init(tpool_t *pool, int thread_id) { - // TODO create PID -> thread_id mapping for signal handler + pthread_mutex_lock(&pool->shm->data_mutex); + pool->shm->thread_id_to_pid_mapping[thread_id] = getpid(); + pthread_mutex_unlock(&pool->shm->data_mutex); ProcData.thread_id = thread_id; if (ScanCtx.index.path[0] != '\0') { - // TODO This should be closed in proc cleanup function ProcData.index_db = database_create(ScanCtx.index.path, INDEX_DATABASE); ProcData.index_db->ipc_ctx = &pool->shm->ipc_ctx; database_open(ProcData.index_db); } - // TODO /dev/shm pthread_mutex_lock(&pool->shm->mutex); - ProcData.ipc_db = database_create("/dev/shm/ipc.sist2", IPC_CONSUMER_DATABASE); + ProcData.ipc_db = database_create(pool->shm->ipc_database_filepath, IPC_CONSUMER_DATABASE); ProcData.ipc_db->ipc_ctx = &pool->shm->ipc_ctx; database_open(ProcData.ipc_db); pthread_mutex_unlock(&pool->shm->mutex); } -void worker_proc_cleanup(tpool_t* pool) { +void worker_proc_cleanup(tpool_t *pool) { if (ProcData.index_db != NULL) { database_close(ProcData.index_db, FALSE); } database_close(ProcData.ipc_db, FALSE); } +#ifndef SIST_DEBUG +#define TPOOL_FORK +#endif + /** * Thread worker function */ static void *tpool_worker(void *arg) { tpool_t *pool = ((start_thread_arg_t *) arg)->pool; - if (pool->fork) { - while (TRUE) { - int pid = fork(); +#ifdef TPOOL_FORK + while (TRUE) { + int pid = fork(); - if (pid == 0) { - worker_proc_init(pool, ((start_thread_arg_t *) arg)->thread_id); + if (pid == 0) { + worker_proc_init(pool, ((start_thread_arg_t *) arg)->thread_id); - pthread_mutex_lock(&pool->shm->mutex); - pthread_cond_signal(&pool->shm->workers_initialized_cond); - pool->shm->initialized_count += 1; - pthread_mutex_unlock(&pool->shm->mutex); + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_signal(&pool->shm->workers_initialized_cond); + pool->shm->initialized_count += 1; + pthread_mutex_unlock(&pool->shm->mutex); - worker_thread_loop(pool); + worker_thread_loop(pool); - pthread_mutex_lock(&pool->shm->mutex); - pthread_cond_signal(&pool->shm->done_working_cond); - pthread_mutex_unlock(&pool->shm->mutex); + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_signal(&pool->shm->done_working_cond); + pthread_mutex_unlock(&pool->shm->mutex); - worker_proc_cleanup(pool); + worker_proc_cleanup(pool); - exit(0); + exit(0); - } else { - int status; - // TODO: On crash, print debug info and resume thread - waitpid(pid, &status, 0); + } else { + int status; + waitpid(pid, &status, 0); - LOG_DEBUGF("tpool.c", "Child process terminated with status code %d", WEXITSTATUS(status)); + LOG_DEBUGF("tpool.c", "Child process terminated with status code %d", WEXITSTATUS(status)); - pthread_mutex_lock(&(pool->shm->ipc_ctx.mutex)); - pool->shm->ipc_ctx.completed_job_count += 1; - pthread_mutex_unlock(&(pool->shm->ipc_ctx.mutex)); + pthread_mutex_lock(&(pool->shm->ipc_ctx.mutex)); + pool->shm->ipc_ctx.completed_job_count += 1; + pthread_mutex_unlock(&(pool->shm->ipc_ctx.mutex)); - pthread_mutex_lock(&(pool->shm->data_mutex)); - pool->shm->busy_count -= 1; - pthread_mutex_unlock(&(pool->shm->data_mutex)); + pthread_mutex_lock(&(pool->shm->data_mutex)); + pool->shm->busy_count -= 1; + pthread_mutex_unlock(&(pool->shm->data_mutex)); - if (WIFSIGNALED(status)) { - // TODO: Get current_job based on PID - const char *job_filepath = "TODO"; - - LOG_FATALF_NO_EXIT( - "tpool.c", - "Child process was terminated by signal (%s).\n" - BLANK_STR "The process was working on %s", - strsignal(WTERMSIG(status)), - job_filepath - ); + if (WIFSIGNALED(status)) { + int crashed_thread_id = -1; + for (int i = 0; i < MAX_THREADS; i++) { + if (pool->shm->thread_id_to_pid_mapping[i] == pid) { + crashed_thread_id = i; + break; + } } - break; + + const char *job_filepath; + if (crashed_thread_id != -1) { + job_filepath = pool->shm->ipc_ctx.current_job[crashed_thread_id]; + } else { + job_filepath = "unknown"; + } + + LOG_FATALF_NO_EXIT( + "tpool.c", + "Child process crashed (%s).\n" + BLANK_STR "The process was working on %s\n" + BLANK_STR "Please consider creating a bug report at https://github.com/simon987/sist2/issues !\n" + BLANK_STR "sist2 is an open source project and relies on the collaboration of its users to diagnose and fix bugs.\n", + strsignal(WTERMSIG(status)), + job_filepath + ); + continue; } + break; } - - } else { - worker_proc_init(pool, ((start_thread_arg_t *) arg)->thread_id); - - pthread_mutex_lock(&pool->shm->mutex); - pthread_cond_signal(&pool->shm->workers_initialized_cond); - pool->shm->initialized_count += 1; - pthread_mutex_unlock(&pool->shm->mutex); - - worker_thread_loop(pool); - - pthread_mutex_lock(&pool->shm->mutex); - pthread_cond_signal(&pool->shm->done_working_cond); - pthread_mutex_unlock(&pool->shm->mutex); - - return NULL; } +#else + worker_proc_init(pool, ((start_thread_arg_t *) arg)->thread_id); + + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_signal(&pool->shm->workers_initialized_cond); + pool->shm->initialized_count += 1; + pthread_mutex_unlock(&pool->shm->mutex); + + worker_thread_loop(pool); + + pthread_mutex_lock(&pool->shm->mutex); + pthread_cond_signal(&pool->shm->done_working_cond); + pthread_mutex_unlock(&pool->shm->mutex); +#endif + return NULL; } @@ -295,13 +307,10 @@ void tpool_destroy(tpool_t *pool) { */ tpool_t *tpool_create(int thread_cnt, int print_progress) { - int fork = FALSE; - tpool_t *pool = malloc(sizeof(tpool_t)); pool->shm = mmap(NULL, sizeof(*pool->shm), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - pool->fork = fork; pool->num_threads = thread_cnt; pool->shm->ipc_ctx.job_count = 0; pool->shm->ipc_ctx.no_more_jobs = FALSE; @@ -310,6 +319,7 @@ tpool_t *tpool_create(int thread_cnt, int print_progress) { pool->shm->job_type = JOB_UNDEFINED; memset(pool->threads, 0, sizeof(pool->threads)); pool->print_progress = print_progress; + sprintf(pool->shm->ipc_database_filepath, "/dev/shm/sist2-ipc-%d.sqlite", getpid()); pthread_mutexattr_t mutexattr; pthread_mutexattr_init(&mutexattr); @@ -329,10 +339,7 @@ tpool_t *tpool_create(int thread_cnt, int print_progress) { pthread_cond_init(&(pool->shm->done_working_cond), &condattr); pthread_cond_init(&(pool->shm->workers_initialized_cond), &condattr); - remove("/dev/shm/ipc.sist2"); - remove("/dev/shm/ipc.sist2-wal"); - remove("/dev/shm/ipc.sist2-shm"); - ProcData.ipc_db = database_create("/dev/shm/ipc.sist2", IPC_PRODUCER_DATABASE); + ProcData.ipc_db = database_create(pool->shm->ipc_database_filepath, IPC_PRODUCER_DATABASE); ProcData.ipc_db->ipc_ctx = &pool->shm->ipc_ctx; database_initialize(ProcData.ipc_db); diff --git a/src/tpool.h b/src/tpool.h index 71742d8..b4350bc 100644 --- a/src/tpool.h +++ b/src/tpool.h @@ -19,8 +19,6 @@ int tpool_add_work(tpool_t *pool, job_t *job); void tpool_wait(tpool_t *pool); -void tpool_dump_debug_info(tpool_t *pool); - void job_destroy(job_t *job); #endif diff --git a/third-party/libscan/libscan/arc/arc.c b/third-party/libscan/libscan/arc/arc.c index d56f538..bb5ed32 100644 --- a/third-party/libscan/libscan/arc/arc.c +++ b/third-party/libscan/libscan/arc/arc.c @@ -147,7 +147,7 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre } if (ret != ARCHIVE_OK) { - CTX_LOG_ERRORF(f->filepath, "(arc.c) [%d] %s", ret, archive_error_string(a)) + CTX_LOG_ERRORF(f->filepath, "(arc.c) [%d] %s", ret, archive_error_string(a)); archive_read_free(a); return SCAN_ERR_READ; } @@ -169,7 +169,7 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre meta_line_t *meta_list = malloc(sizeof(meta_line_t) + buf.cur); meta_list->key = MetaContent; strcpy(meta_list->str_val, buf.buf); - APPEND_META(doc, meta_list) + APPEND_META(doc, meta_list); dyn_buffer_destroy(&buf); } else { @@ -212,13 +212,13 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre double decompressed_size_ratio = (double) sub_job->vfile.st_size / (double) f->st_size; if (decompressed_size_ratio > MAX_DECOMPRESSED_SIZE_RATIO) { CTX_LOG_DEBUGF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)", sub_job->filepath, - decompressed_size_ratio) + decompressed_size_ratio); break; } // Handle excludes if (exclude != NULL && EXCLUDED(sub_job->filepath)) { - CTX_LOG_DEBUGF("arc.c", "Excluded: %s", sub_job->filepath) + CTX_LOG_DEBUGF("arc.c", "Excluded: %s", sub_job->filepath); continue; } diff --git a/third-party/libscan/libscan/comic/comic.c b/third-party/libscan/libscan/comic/comic.c index aeb0baf..263ff70 100644 --- a/third-party/libscan/libscan/comic/comic.c +++ b/third-party/libscan/libscan/comic/comic.c @@ -18,7 +18,7 @@ void parse_comic(scan_comic_ctx_t *ctx, vfile_t *f, document_t *doc) { int ret = arc_open(&arc_ctx, f, &a, &arc_data, TRUE); if (ret != ARCHIVE_OK) { - CTX_LOG_ERRORF(f->filepath, "(cbr.c) [%d] %s", ret, archive_error_string(a)) + CTX_LOG_ERRORF(f->filepath, "(cbr.c) [%d] %s", ret, archive_error_string(a)); archive_read_free(a); return; } @@ -38,7 +38,7 @@ void parse_comic(scan_comic_ctx_t *ctx, vfile_t *f, document_t *doc) { if (read != entry_size) { const char *err_str = archive_error_string(a); if (err_str) { - CTX_LOG_ERRORF("comic.c", "Error while reading entry: %s", err_str) + CTX_LOG_ERRORF("comic.c", "Error while reading entry: %s", err_str); } free(buf); break; diff --git a/third-party/libscan/libscan/ebook/ebook.c b/third-party/libscan/libscan/ebook/ebook.c index 95fb4a4..b53d937 100644 --- a/third-party/libscan/libscan/ebook/ebook.c +++ b/third-party/libscan/libscan/ebook/ebook.c @@ -54,7 +54,7 @@ load_pixmap(scan_ebook_ctx_t *ctx, int page, fz_context *fzctx, fz_document *fzd fz_catch(fzctx)err = 1; if (err != 0) { - CTX_LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, fzctx->error.message) + CTX_LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, fzctx->error.message); return NULL; } @@ -86,14 +86,14 @@ load_pixmap(scan_ebook_ctx_t *ctx, int page, fz_context *fzctx, fz_document *fzd } fz_catch(fzctx)err = fzctx->error.errcode; if (err != 0) { - CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message) + CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message); fz_drop_page(fzctx, *cover); fz_drop_pixmap(fzctx, pixmap); return NULL; } if (pixmap->n != 3) { - CTX_LOG_ERRORF(doc->filepath, "Got unexpected pixmap depth: %d", pixmap->n) + CTX_LOG_ERRORF(doc->filepath, "Got unexpected pixmap depth: %d", pixmap->n); fz_drop_page(fzctx, *cover); fz_drop_pixmap(fzctx, pixmap); return NULL; @@ -113,7 +113,7 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d if (pixmap_is_blank(pixmap)) { fz_drop_page(fzctx, cover); fz_drop_pixmap(fzctx, pixmap); - CTX_LOG_DEBUG(doc->filepath, "Cover page is blank, using page 1 instead") + CTX_LOG_DEBUG(doc->filepath, "Cover page is blank, using page 1 instead"); pixmap = load_pixmap(ctx, 1, fzctx, fzdoc, doc, &cover); if (pixmap == NULL) { return FALSE; @@ -161,7 +161,7 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d av_init_packet(&jpeg_packet); avcodec_receive_packet(jpeg_encoder, &jpeg_packet); - APPEND_LONG_META(doc, MetaThumbnail, 1) + APPEND_LONG_META(doc, MetaThumbnail, 1); ctx->store(doc->doc_id, 0, (char *) jpeg_packet.data, jpeg_packet.size); free(samples); @@ -180,14 +180,14 @@ void fz_err_callback(void *user, const char *message) { document_t *doc = (document_t *) user; const scan_ebook_ctx_t *ctx = &thread_ctx; - CTX_LOG_WARNINGF(doc->filepath, "FZ: %s", message) + CTX_LOG_WARNINGF(doc->filepath, "FZ: %s", message); } void fz_warn_callback(void *user, const char *message) { document_t *doc = (document_t *) user; const scan_ebook_ctx_t *ctx = &thread_ctx; - CTX_LOG_DEBUGF(doc->filepath, "FZ: %s", message) + CTX_LOG_DEBUGF(doc->filepath, "FZ: %s", message); } static void init_fzctx(fz_context *fzctx, document_t *doc) { @@ -243,7 +243,7 @@ void fill_image(fz_context *fzctx, UNUSED(fz_device *dev), if (img->w >= MIN_OCR_WIDTH && img->h >= MIN_OCR_HEIGHT && OCR_IS_VALID_BPP(img->n)) { fz_pixmap *pix = img->get_pixmap(fzctx, img, NULL, img->w, img->h, &l2factor); - ocr_extract_text(thread_ctx.tesseract_path, thread_ctx.tesseract_lang, pix->samples, pix->w, pix->h, pix->n, pix->stride, pix->xres, fill_image_ocr_cb); + ocr_extract_text(thread_ctx.tesseract_path, thread_ctx.tesseract_lang, pix->samples, pix->w, pix->h, pix->n, (int)pix->stride, pix->xres, fill_image_ocr_cb); fz_drop_pixmap(fzctx, pix); } } @@ -282,14 +282,14 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi fz_catch(fzctx)err = fzctx->error.errcode; if (err) { - CTX_LOG_WARNINGF(doc->filepath, "fz_count_pages() returned error code [%d] %s", err, fzctx->error.message) + CTX_LOG_WARNINGF(doc->filepath, "fz_count_pages() returned error code [%d] %s", err, fzctx->error.message); fz_drop_stream(fzctx, stream); fz_drop_document(fzctx, fzdoc); fz_drop_context(fzctx); return; } - APPEND_LONG_META(doc, MetaPages, page_count) + APPEND_LONG_META(doc, MetaPages, page_count); if (ctx->enable_tn) { if (render_cover(ctx, fzctx, doc, fzdoc) == FALSE) { @@ -312,7 +312,7 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi fz_catch(fzctx); if (strlen(title) > 0) { - APPEND_UTF8_META(doc, MetaTitle, title) + APPEND_UTF8_META(doc, MetaTitle, title); } char author[4096] = {'\0',}; @@ -320,7 +320,7 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi fz_catch(fzctx); if (strlen(author) > 0) { - APPEND_UTF8_META(doc, MetaAuthor, author) + APPEND_UTF8_META(doc, MetaAuthor, author); } @@ -334,7 +334,7 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi fz_try(fzctx)page = fz_load_page(fzctx, fzdoc, current_page); fz_catch(fzctx)err = fzctx->error.errcode; if (err != 0) { - CTX_LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, fzctx->error.message) + CTX_LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, fzctx->error.message); text_buffer_destroy(&thread_buffer); fz_drop_page(fzctx, page); fz_drop_stream(fzctx, stream); @@ -363,7 +363,7 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi } fz_catch(fzctx)err = fzctx->error.errcode; if (err != 0) { - CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message) + CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message); text_buffer_destroy(&thread_buffer); fz_drop_page(fzctx, page); fz_drop_stext_page(fzctx, stext); @@ -393,7 +393,7 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi meta_line_t *meta_content = malloc(sizeof(meta_line_t) + thread_buffer.dyn_buffer.cur); meta_content->key = MetaContent; memcpy(meta_content->str_val, thread_buffer.dyn_buffer.buf, thread_buffer.dyn_buffer.cur); - APPEND_META(doc, meta_content) + APPEND_META(doc, meta_content); text_buffer_destroy(&thread_buffer); } @@ -418,7 +418,7 @@ void parse_epub_fast(scan_ebook_ctx_t *ctx, vfile_t *f, document_t *doc) { int ret = arc_open(&arc_ctx, f, &a, &arc_data, TRUE); if (ret != ARCHIVE_OK) { - CTX_LOG_ERRORF(f->filepath, "(ebook.c) [%d] %s", ret, archive_error_string(a)) + CTX_LOG_ERRORF(f->filepath, "(ebook.c) [%d] %s", ret, archive_error_string(a)); archive_read_free(a); return; } @@ -439,7 +439,7 @@ void parse_epub_fast(scan_ebook_ctx_t *ctx, vfile_t *f, document_t *doc) { if (read != entry_size) { const char *err_str = archive_error_string(a); if (err_str) { - CTX_LOG_ERRORF("ebook.c", "Error while reading entry: %s", err_str) + CTX_LOG_ERRORF("ebook.c", "Error while reading entry: %s", err_str); } free(buf); break; @@ -460,7 +460,7 @@ void parse_epub_fast(scan_ebook_ctx_t *ctx, vfile_t *f, document_t *doc) { meta_line_t *meta_content = malloc(sizeof(meta_line_t) + content_buffer.dyn_buffer.cur); meta_content->key = MetaContent; memcpy(meta_content->str_val, content_buffer.dyn_buffer.buf, content_buffer.dyn_buffer.cur); - APPEND_META(doc, meta_content) + APPEND_META(doc, meta_content); text_buffer_destroy(&content_buffer); @@ -477,7 +477,7 @@ void parse_ebook(scan_ebook_ctx_t *ctx, vfile_t *f, const char *mime_str, docume size_t buf_len; void *buf = read_all(f, &buf_len); if (buf == NULL) { - CTX_LOG_ERROR(f->filepath, "read_all() failed") + CTX_LOG_ERROR(f->filepath, "read_all() failed"); return; } diff --git a/third-party/libscan/libscan/font/font.c b/third-party/libscan/libscan/font/font.c index d734134..7c37665 100644 --- a/third-party/libscan/libscan/font/font.c +++ b/third-party/libscan/libscan/font/font.c @@ -146,7 +146,7 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) { size_t buf_len = 0; void *buf = read_all(f, &buf_len); if (buf == NULL) { - CTX_LOG_ERROR(f->filepath, "read_all() failed") + CTX_LOG_ERROR(f->filepath, "read_all() failed"); return; } @@ -154,7 +154,7 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) { FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, (int) buf_len, 0, &face); if (err != 0) { CTX_LOG_ERRORF(doc->filepath, "(font.c) FT_New_Memory_Face() returned error code [%d] %s", err, - FT_Error_String(err)) + FT_Error_String(err)); free(buf); return; } @@ -174,7 +174,7 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) { meta_line_t *meta_name = malloc(sizeof(meta_line_t) + strlen(font_name)); meta_name->key = MetaFontName; strcpy(meta_name->str_val, font_name); - APPEND_META(doc, meta_name) + APPEND_META(doc, meta_name); if (!ctx->enable_tn) { FT_Done_Face(face); @@ -188,7 +188,7 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) { err = FT_Set_Pixel_Sizes(face, 0, pixel); if (err != 0) { CTX_LOG_WARNINGF(doc->filepath, "(font.c) FT_Set_Pixel_Sizes() returned error code [%d] %s", err, - FT_Error_String(err)) + FT_Error_String(err)); FT_Done_Face(face); free(buf); return; @@ -210,7 +210,7 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) { err = FT_Load_Char(face, c, FT_LOAD_NO_HINTING | FT_LOAD_RENDER); if (err != 0) { CTX_LOG_WARNINGF(doc->filepath, "(font.c) FT_Load_Char() returned error code [%d] %s", err, - FT_Error_String(err)) + FT_Error_String(err)); continue; } } @@ -231,7 +231,7 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) { dyn_buffer_t bmp_data = dyn_buffer_create(); bmp_format(&bmp_data, dimensions, bitmap); - APPEND_LONG_META(doc, MetaThumbnail, 1) + APPEND_LONG_META(doc, MetaThumbnail, 1); ctx->store(doc->doc_id, 0, bmp_data.buf, bmp_data.cur); dyn_buffer_destroy(&bmp_data); diff --git a/third-party/libscan/libscan/json/json.c b/third-party/libscan/libscan/json/json.c index ef93405..1bf590f 100644 --- a/third-party/libscan/libscan/json/json.c +++ b/third-party/libscan/libscan/json/json.c @@ -33,7 +33,7 @@ int json_extract_text(cJSON *json, text_buffer_t *tex) { scan_code_t parse_json(scan_json_ctx_t *ctx, vfile_t *f, document_t *doc) { if (f->st_size > JSON_MAX_FILE_SIZE) { - CTX_LOG_WARNINGF("json.c", "File larger than maximum allowed [%s]", f->filepath) + CTX_LOG_WARNINGF("json.c", "File larger than maximum allowed [%s]", f->filepath); return SCAN_ERR_SKIP; } diff --git a/third-party/libscan/libscan/macros.h b/third-party/libscan/libscan/macros.h index 944225c..00084b2 100644 --- a/third-party/libscan/libscan/macros.h +++ b/third-party/libscan/libscan/macros.h @@ -25,20 +25,20 @@ #define SHA1_STR_LENGTH (SHA1_DIGEST_LENGTH * 2 + 1) #define MD5_STR_LENGTH (MD5_DIGEST_LENGTH * 2 + 1) -#define APPEND_STR_META(doc, keyname, value) \ +#define APPEND_STR_META(doc, keyname, value) do {\ {meta_line_t *meta_str = malloc(sizeof(meta_line_t) + strlen(value)); \ meta_str->key = keyname; \ strcpy(meta_str->str_val, value); \ - APPEND_META(doc, meta_str)} + APPEND_META(doc, meta_str);}} while(0) -#define APPEND_LONG_META(doc, keyname, value) \ +#define APPEND_LONG_META(doc, keyname, value) do{\ {meta_line_t *meta_long = malloc(sizeof(meta_line_t)); \ meta_long->key = keyname; \ meta_long->long_val = value; \ - APPEND_META(doc, meta_long)} + APPEND_META(doc, meta_long);}} while(0) -#define APPEND_META(doc, meta) \ +#define APPEND_META(doc, meta) do {\ meta->next = NULL;\ if (doc->meta_head == NULL) {\ doc->meta_head = meta;\ @@ -46,7 +46,7 @@ } else {\ doc->meta_tail->next = meta;\ doc->meta_tail = meta;\ - } + }}while(0) #define APPEND_UTF8_META(doc, keyname, str) \ text_buffer_t tex = text_buffer_create(-1); \ @@ -55,5 +55,5 @@ meta_line_t *meta_tag = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); \ meta_tag->key = keyname; \ strcpy(meta_tag->str_val, tex.dyn_buffer.buf); \ - APPEND_META(doc, meta_tag) \ - text_buffer_destroy(&tex); + APPEND_META(doc, meta_tag); \ + text_buffer_destroy(&tex) diff --git a/third-party/libscan/libscan/media/media.c b/third-party/libscan/libscan/media/media.c index 38cb421..fa62724 100644 --- a/third-party/libscan/libscan/media/media.c +++ b/third-party/libscan/libscan/media/media.c @@ -163,7 +163,7 @@ static void read_subtitles(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, i text_buffer_terminate_string(&tex); - APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf) + APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf); text_buffer_destroy(&tex); avcodec_free_context(&decoder); } @@ -190,7 +190,7 @@ read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *d CTX_LOG_WARNINGF(doc->filepath, "(media.c) avcodec_read_frame() returned error code [%d] %s", read_frame_ret, av_err2str(read_frame_ret) - ) + ); } frame_and_packet_free(result); return NULL; @@ -210,7 +210,7 @@ read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *d CTX_LOG_ERRORF(doc->filepath, "(media.c) avcodec_send_packet() returned error code [%d] %s", decode_ret, av_err2str(decode_ret) - ) + ); frame_and_packet_free(result); return NULL; } @@ -230,7 +230,7 @@ void append_tag_meta_if_not_exists(scan_media_ctx_t *ctx, document_t *doc, AVDic while (meta != NULL) { if (meta->key == key) { CTX_LOG_DEBUGF(doc->filepath, "Ignoring duplicate tag: '%02x=%s' and '%02x=%s'", - key, meta->str_val, key, tag->value) + key, meta->str_val, key, tag->value); return; } meta = meta->next; @@ -243,7 +243,7 @@ void append_tag_meta_if_not_exists(scan_media_ctx_t *ctx, document_t *doc, AVDic meta_tag->key = key; strcpy(meta_tag->str_val, tex.dyn_buffer.buf); - APPEND_META(doc, meta_tag) + APPEND_META(doc, meta_tag); text_buffer_destroy(&tex); } @@ -253,7 +253,7 @@ void append_tag_meta_if_not_exists(scan_media_ctx_t *ctx, document_t *doc, AVDic #define STRCPY_TOLOWER(dst, str) \ strncpy(dst, str, sizeof(dst)); \ char *ptr = dst; \ - for (; *ptr; ++ptr) *ptr = (char) tolower(*ptr); + for (; *ptr; ++ptr) *ptr = (char) tolower(*ptr) __always_inline static void append_audio_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, document_t *doc) { @@ -261,18 +261,18 @@ static void append_audio_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx AVDictionaryEntry *tag = NULL; while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { char key[256]; - STRCPY_TOLOWER(key, tag->key) + STRCPY_TOLOWER(key, tag->key); if (strcmp(key, "artist") == 0) { - APPEND_TAG_META(MetaArtist) + APPEND_TAG_META(MetaArtist); } else if (strcmp(key, "genre") == 0) { - APPEND_TAG_META(MetaGenre) + APPEND_TAG_META(MetaGenre); } else if (strcmp(key, "title") == 0) { - APPEND_TAG_META(MetaTitle) + APPEND_TAG_META(MetaTitle); } else if (strcmp(key, "album_artist") == 0) { - APPEND_TAG_META(MetaAlbumArtist) + APPEND_TAG_META(MetaAlbumArtist); } else if (strcmp(key, "album") == 0) { - APPEND_TAG_META(MetaAlbum) + APPEND_TAG_META(MetaAlbum); } else if (strcmp(key, "comment") == 0) { append_tag_meta_if_not_exists(ctx, doc, tag, MetaContent); } @@ -291,14 +291,14 @@ append_video_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVFrame *f if (meta_duration->long_val > INT32_MAX) { meta_duration->long_val = 0; } - APPEND_META(doc, meta_duration) + APPEND_META(doc, meta_duration); } if (pFormatCtx->bit_rate != 0) { meta_line_t *meta_bitrate = malloc(sizeof(meta_line_t)); meta_bitrate->key = MetaMediaBitrate; meta_bitrate->long_val = pFormatCtx->bit_rate; - APPEND_META(doc, meta_bitrate) + APPEND_META(doc, meta_bitrate); } } @@ -306,7 +306,7 @@ append_video_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVFrame *f if (is_video) { while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { char key[256]; - STRCPY_TOLOWER(key, tag->key) + STRCPY_TOLOWER(key, tag->key); if (strcmp(key, "title") == 0) { append_tag_meta_if_not_exists(ctx, doc, tag, MetaTitle); @@ -320,38 +320,38 @@ append_video_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVFrame *f // EXIF metadata while ((tag = av_dict_get(frame->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { char key[256]; - STRCPY_TOLOWER(key, tag->key) + STRCPY_TOLOWER(key, tag->key); if (strcmp(key, "artist") == 0) { append_tag_meta_if_not_exists(ctx, doc, tag, MetaArtist); } else if (strcmp(key, "imagedescription") == 0) { append_tag_meta_if_not_exists(ctx, doc, tag, MetaContent); } else if (strcmp(key, "make") == 0) { - APPEND_TAG_META(MetaExifMake) + APPEND_TAG_META(MetaExifMake); } else if (strcmp(key, "model") == 0) { - APPEND_TAG_META(MetaExifModel) + APPEND_TAG_META(MetaExifModel); } else if (strcmp(key, "software") == 0) { - APPEND_TAG_META(MetaExifSoftware) + APPEND_TAG_META(MetaExifSoftware); } else if (strcmp(key, "fnumber") == 0) { - APPEND_TAG_META(MetaExifFNumber) + APPEND_TAG_META(MetaExifFNumber); } else if (strcmp(key, "focallength") == 0) { - APPEND_TAG_META(MetaExifFocalLength) + APPEND_TAG_META(MetaExifFocalLength); } else if (strcmp(key, "usercomment") == 0) { - APPEND_TAG_META(MetaExifUserComment) + APPEND_TAG_META(MetaExifUserComment); } else if (strcmp(key, "isospeedratings") == 0) { - APPEND_TAG_META(MetaExifIsoSpeedRatings) + APPEND_TAG_META(MetaExifIsoSpeedRatings); } else if (strcmp(key, "exposuretime") == 0) { - APPEND_TAG_META(MetaExifExposureTime) + APPEND_TAG_META(MetaExifExposureTime); } else if (strcmp(key, "datetime") == 0) { - APPEND_TAG_META(MetaExifDateTime) + APPEND_TAG_META(MetaExifDateTime); } else if (strcmp(key, "gpslatitude") == 0) { - APPEND_TAG_META(MetaExifGpsLatitudeDMS) + APPEND_TAG_META(MetaExifGpsLatitudeDMS); } else if (strcmp(key, "gpslatituderef") == 0) { - APPEND_TAG_META(MetaExifGpsLatitudeRef) + APPEND_TAG_META(MetaExifGpsLatitudeRef); } else if (strcmp(key, "gpslongitude") == 0) { - APPEND_TAG_META(MetaExifGpsLongitudeDMS) + APPEND_TAG_META(MetaExifGpsLongitudeDMS); } else if (strcmp(key, "gpslongituderef") == 0) { - APPEND_TAG_META(MetaExifGpsLongitudeRef) + APPEND_TAG_META(MetaExifGpsLongitudeRef); } } } @@ -432,11 +432,11 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor CTX_LOG_DEBUGF( doc->filepath, "(media.c) Could not seek media file: %s", av_err2str(seek_ret) - ) + ); } if (seek_ok == FALSE && thumbnail_index != 0) { - CTX_LOG_WARNING(doc->filepath, "(media.c) Could not seek media file. Can't generate additional thumbnails.") + CTX_LOG_WARNING(doc->filepath, "(media.c) Could not seek media file. Can't generate additional thumbnails."); return SAVE_THUMBNAIL_FAILED; } } @@ -522,7 +522,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, const AVCodecDescriptor *desc = avcodec_descriptor_get(stream->codecpar->codec_id); if (desc != NULL) { - APPEND_STR_META(doc, MetaMediaAudioCodec, desc->name) + APPEND_STR_META(doc, MetaMediaAudioCodec, desc->name); } audio_stream = i; @@ -533,18 +533,18 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, const AVCodecDescriptor *desc = avcodec_descriptor_get(stream->codecpar->codec_id); if (desc != NULL) { - APPEND_STR_META(doc, MetaMediaVideoCodec, desc->name) + APPEND_STR_META(doc, MetaMediaVideoCodec, desc->name); } meta_line_t *meta_w = malloc(sizeof(meta_line_t)); meta_w->key = MetaWidth; meta_w->long_val = stream->codecpar->width; - APPEND_META(doc, meta_w) + APPEND_META(doc, meta_w); meta_line_t *meta_h = malloc(sizeof(meta_line_t)); meta_h->key = MetaHeight; meta_h->long_val = stream->codecpar->height; - APPEND_META(doc, meta_h) + APPEND_META(doc, meta_h); video_stream = i; } @@ -611,7 +611,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, } if (number_of_thumbnails_generated > 0) { - APPEND_LONG_META(doc, MetaThumbnail, number_of_thumbnails_generated) + APPEND_LONG_META(doc, MetaThumbnail, number_of_thumbnails_generated); } avcodec_free_context(&decoder); @@ -625,12 +625,12 @@ void parse_media_filename(scan_media_ctx_t *ctx, const char *filepath, document_ AVFormatContext *pFormatCtx = avformat_alloc_context(); if (pFormatCtx == NULL) { - CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()") + CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()"); return; } int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL); if (res < 0) { - CTX_LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res)) + CTX_LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res)); avformat_close_input(&pFormatCtx); avformat_free_context(pFormatCtx); return; @@ -724,7 +724,7 @@ void parse_media_vfile(scan_media_ctx_t *ctx, struct vfile *f, document_t *doc, AVFormatContext *pFormatCtx = avformat_alloc_context(); if (pFormatCtx == NULL) { - CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()") + CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()"); return; } @@ -737,13 +737,13 @@ void parse_media_vfile(scan_media_ctx_t *ctx, struct vfile *f, document_t *doc, if (f->st_size <= ctx->max_media_buffer) { int ret = memfile_open(f, &memfile); if (ret == 0) { - CTX_LOG_DEBUGF(f->filepath, "Loading media file in memory (%ldB)", f->st_size) + CTX_LOG_DEBUGF(f->filepath, "Loading media file in memory (%ldB)", f->st_size); io_ctx = avio_alloc_context(buffer, AVIO_BUF_SIZE, 0, &memfile, memfile_read, NULL, memfile_seek); } } if (io_ctx == NULL) { - CTX_LOG_DEBUGF(f->filepath, "Reading media file without seek support", f->st_size) + CTX_LOG_DEBUGF(f->filepath, "Reading media file without seek support", f->st_size); io_ctx = avio_alloc_context(buffer, AVIO_BUF_SIZE, 0, f, vfile_read, NULL, NULL); } @@ -752,7 +752,7 @@ void parse_media_vfile(scan_media_ctx_t *ctx, struct vfile *f, document_t *doc, int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL); if (res < 0) { if (res != -5) { - CTX_LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res)) + CTX_LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res)); } av_free(io_ctx->buffer); memfile_close(&memfile); @@ -787,7 +787,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu AVFormatContext *pFormatCtx = avformat_alloc_context(); if (pFormatCtx == NULL) { - CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()") + CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()"); return FALSE; } @@ -795,7 +795,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu int ret = memfile_open_buf(buf, buf_len, &memfile); if (ret == 0) { - CTX_LOG_DEBUGF(doc->filepath, "Loading media file in memory (%ldB)", buf_len) + CTX_LOG_DEBUGF(doc->filepath, "Loading media file in memory (%ldB)", buf_len); io_ctx = avio_alloc_context(buffer, AVIO_BUF_SIZE, 0, &memfile, memfile_read, NULL, memfile_seek); } else { avformat_close_input(&pFormatCtx); @@ -850,7 +850,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu } if (scaled_frame == STORE_AS_IS) { - APPEND_LONG_META(doc, MetaThumbnail, 1) + APPEND_LONG_META(doc, MetaThumbnail, 1); ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size); } else { // Encode frame to jpeg @@ -863,7 +863,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu avcodec_receive_packet(jpeg_encoder, &jpeg_packet); // Save thumbnail - APPEND_LONG_META(doc, MetaThumbnail, 1) + APPEND_LONG_META(doc, MetaThumbnail, 1); ctx->store(doc->doc_id, 0, jpeg_packet.data, jpeg_packet.size); av_packet_unref(&jpeg_packet); diff --git a/third-party/libscan/libscan/mobi/scan_mobi.c b/third-party/libscan/libscan/mobi/scan_mobi.c index 1a42fab..9d2c5d9 100644 --- a/third-party/libscan/libscan/mobi/scan_mobi.c +++ b/third-party/libscan/libscan/mobi/scan_mobi.c @@ -8,7 +8,7 @@ void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) { MOBIData *m = mobi_init(); if (m == NULL) { - CTX_LOG_ERROR(f->filepath, "mobi_init() failed") + CTX_LOG_ERROR(f->filepath, "mobi_init() failed"); return; } @@ -16,7 +16,7 @@ void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) { char* buf = read_all(f, &buf_len); if (buf == NULL) { mobi_free(m); - CTX_LOG_ERROR(f->filepath, "read_all() failed") + CTX_LOG_ERROR(f->filepath, "read_all() failed"); return; } @@ -24,7 +24,7 @@ void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) { if (file == NULL) { mobi_free(m); free(buf); - CTX_LOG_ERRORF(f->filepath, "fmemopen() failed (%d)", errno) + CTX_LOG_ERRORF(f->filepath, "fmemopen() failed (%d)", errno); return; } @@ -33,25 +33,25 @@ void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) { if (mobi_ret != MOBI_SUCCESS) { mobi_free(m); free(buf); - CTX_LOG_ERRORF(f->filepath, "mobi_laod_file() returned error code [%d]", mobi_ret) + CTX_LOG_ERRORF(f->filepath, "mobi_laod_file() returned error code [%d]", mobi_ret); return; } char *author = mobi_meta_get_author(m); if (author != NULL) { - APPEND_STR_META(doc, MetaAuthor, author) + APPEND_STR_META(doc, MetaAuthor, author); free(author); } char *title = mobi_meta_get_title(m); if (title != NULL) { - APPEND_STR_META(doc, MetaTitle, title) + APPEND_STR_META(doc, MetaTitle, title); free(title); } const size_t maxlen = mobi_get_text_maxsize(m); if (maxlen == MOBI_NOTSET) { free(buf); - CTX_LOG_DEBUGF("%s", "Invalid text maxsize: %zu", maxlen) + CTX_LOG_DEBUGF("%s", "Invalid text maxsize: %zu", maxlen); return; } @@ -62,7 +62,7 @@ void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) { mobi_free(m); free(content_str); free(buf); - CTX_LOG_ERRORF(f->filepath, "mobi_get_rawml() returned error code [%d]", mobi_ret) + CTX_LOG_ERRORF(f->filepath, "mobi_get_rawml() returned error code [%d]", mobi_ret); return; } @@ -70,7 +70,7 @@ void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) { text_buffer_append_markup(&tex, content_str); text_buffer_terminate_string(&tex); - APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf) + APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf); free(content_str); free(buf); diff --git a/third-party/libscan/libscan/msdoc/msdoc.c b/third-party/libscan/libscan/msdoc/msdoc.c index a628ea6..526b1c1 100644 --- a/third-party/libscan/libscan/msdoc/msdoc.c +++ b/third-party/libscan/libscan/msdoc/msdoc.c @@ -39,12 +39,12 @@ void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, voi iInitDocument(file_in, (int) buf_len); const char *author = szGetAuthor(); if (author != NULL) { - APPEND_UTF8_META(doc, MetaAuthor, author) + APPEND_UTF8_META(doc, MetaAuthor, author); } const char *title = szGetTitle(); if (title != NULL) { - APPEND_UTF8_META(doc, MetaTitle, title) + APPEND_UTF8_META(doc, MetaTitle, title); } vFreeDocument(); @@ -60,7 +60,7 @@ void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, voi meta_line_t *meta_content = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); meta_content->key = MetaContent; memcpy(meta_content->str_val, tex.dyn_buffer.buf, tex.dyn_buffer.cur); - APPEND_META(doc, meta_content) + APPEND_META(doc, meta_content); text_buffer_destroy(&tex); } @@ -74,14 +74,14 @@ void parse_msdoc(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc) { size_t buf_len; char *buf = read_all(f, &buf_len); if (buf == NULL) { - CTX_LOG_ERROR(f->filepath, "read_all() failed") + CTX_LOG_ERROR(f->filepath, "read_all() failed"); return; } FILE *file = fmemopen(buf, buf_len, "rb"); if (file == NULL) { free(buf); - CTX_LOG_ERRORF(f->filepath, "fmemopen() failed (%d)", errno) + CTX_LOG_ERRORF(f->filepath, "fmemopen() failed (%d)", errno); return; } diff --git a/third-party/libscan/libscan/ooxml/ooxml.c b/third-party/libscan/libscan/ooxml/ooxml.c index 49010e1..da23683 100644 --- a/third-party/libscan/libscan/ooxml/ooxml.c +++ b/third-party/libscan/libscan/ooxml/ooxml.c @@ -39,7 +39,7 @@ int extract_text(scan_ooxml_ctx_t *ctx, xmlDoc *xml, xmlNode *node, text_buffer_ xmlErrorPtr err = xmlGetLastError(); if (err != NULL) { if (err->level == XML_ERR_FATAL) { - CTX_LOG_ERRORF("ooxml.c", "Got fatal XML error while parsing document: %s", err->message) + CTX_LOG_ERRORF("ooxml.c", "Got fatal XML error while parsing document: %s", err->message); return -1; } } @@ -85,13 +85,13 @@ static int read_part(scan_ooxml_ctx_t *ctx, struct archive *a, text_buffer_t *bu XML_PARSE_RECOVER | XML_PARSE_NOWARNING | XML_PARSE_NOERROR | XML_PARSE_NONET); if (xml == NULL) { - CTX_LOG_ERROR(doc->filepath, "Could not parse XML") + CTX_LOG_ERROR(doc->filepath, "Could not parse XML"); return READ_PART_ERR; } xmlNode *root = xmlDocGetRootElement(xml); if (root == NULL) { - CTX_LOG_ERROR(doc->filepath, "Empty document") + CTX_LOG_ERROR(doc->filepath, "Empty document"); xmlFreeDoc(xml); return READ_PART_ERR; } @@ -108,13 +108,13 @@ static int read_doc_props_app(scan_ooxml_ctx_t *ctx, struct archive *a, document XML_PARSE_RECOVER | XML_PARSE_NOWARNING | XML_PARSE_NOERROR | XML_PARSE_NONET); if (xml == NULL) { - CTX_LOG_ERROR(doc->filepath, "Could not parse XML") + CTX_LOG_ERROR(doc->filepath, "Could not parse XML"); return -1; } xmlNode *root = xmlDocGetRootElement(xml); if (root == NULL) { - CTX_LOG_ERROR(doc->filepath, "Empty document") + CTX_LOG_ERROR(doc->filepath, "Empty document"); xmlFreeDoc(xml); return -1; } @@ -127,7 +127,7 @@ static int read_doc_props_app(scan_ooxml_ctx_t *ctx, struct archive *a, document } if (xmlStrEqual(child->name, _X("Pages"))) { - APPEND_LONG_META(doc, MetaPages, strtol((char *) text, NULL, 10)) + APPEND_LONG_META(doc, MetaPages, strtol((char *) text, NULL, 10)); } xmlFree(text); @@ -144,13 +144,13 @@ static int read_doc_props(scan_ooxml_ctx_t *ctx, struct archive *a, document_t * XML_PARSE_RECOVER | XML_PARSE_NOWARNING | XML_PARSE_NOERROR | XML_PARSE_NONET); if (xml == NULL) { - CTX_LOG_ERROR(doc->filepath, "Could not parse XML") + CTX_LOG_ERROR(doc->filepath, "Could not parse XML"); return -1; } xmlNode *root = xmlDocGetRootElement(xml); if (root == NULL) { - CTX_LOG_ERROR(doc->filepath, "Empty document") + CTX_LOG_ERROR(doc->filepath, "Empty document"); xmlFreeDoc(xml); return -1; } @@ -163,11 +163,11 @@ static int read_doc_props(scan_ooxml_ctx_t *ctx, struct archive *a, document_t * } if (xmlStrEqual(child->name, _X("title"))) { - APPEND_STR_META(doc, MetaTitle, (char *) text) + APPEND_STR_META(doc, MetaTitle, (char *) text); } else if (xmlStrEqual(child->name, _X("creator"))) { - APPEND_STR_META(doc, MetaAuthor, (char *) text) + APPEND_STR_META(doc, MetaAuthor, (char *) text); } else if (xmlStrEqual(child->name, _X("lastModifiedBy"))) { - APPEND_STR_META(doc, MetaModifiedBy, (char *) text) + APPEND_STR_META(doc, MetaModifiedBy, (char *) text); } xmlFree(text); @@ -190,7 +190,7 @@ void read_thumbnail(scan_ooxml_ctx_t *ctx, document_t *doc, struct archive *a, s char *buf = malloc(entry_size); archive_read_data(a, buf, entry_size); - APPEND_LONG_META(doc, MetaThumbnail, 1) + APPEND_LONG_META(doc, MetaThumbnail, 1); ctx->store(doc->doc_id, 1, buf, entry_size); free(buf); } @@ -200,7 +200,7 @@ void parse_ooxml(scan_ooxml_ctx_t *ctx, vfile_t *f, document_t *doc) { size_t buf_len; void *buf = read_all(f, &buf_len); if (buf == NULL) { - CTX_LOG_ERROR(f->filepath, "read_all() failed") + CTX_LOG_ERROR(f->filepath, "read_all() failed"); return; } @@ -209,7 +209,7 @@ void parse_ooxml(scan_ooxml_ctx_t *ctx, vfile_t *f, document_t *doc) { int ret = archive_read_open_memory(a, buf, buf_len); if (ret != ARCHIVE_OK) { - CTX_LOG_ERRORF(doc->filepath, "Could not read archive: %s", archive_error_string(a)) + CTX_LOG_ERRORF(doc->filepath, "Could not read archive: %s", archive_error_string(a)); archive_read_free(a); free(buf); return; @@ -250,7 +250,7 @@ void parse_ooxml(scan_ooxml_ctx_t *ctx, vfile_t *f, document_t *doc) { meta_line_t *meta = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); meta->key = MetaContent; strcpy(meta->str_val, tex.dyn_buffer.buf); - APPEND_META(doc, meta) + APPEND_META(doc, meta); } archive_read_close(a); diff --git a/third-party/libscan/libscan/raw/raw.c b/third-party/libscan/libscan/raw/raw.c index c4b07da..7991d7c 100644 --- a/third-party/libscan/libscan/raw/raw.c +++ b/third-party/libscan/libscan/raw/raw.c @@ -83,7 +83,7 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do av_init_packet(&jpeg_packet); avcodec_receive_packet(jpeg_encoder, &jpeg_packet); - APPEND_LONG_META(doc, MetaThumbnail, 1) + APPEND_LONG_META(doc, MetaThumbnail, 1); ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) jpeg_packet.data, jpeg_packet.size); av_packet_unref(&jpeg_packet); @@ -100,76 +100,76 @@ void parse_raw(scan_raw_ctx_t *ctx, vfile_t *f, document_t *doc) { libraw_data_t *libraw_lib = libraw_init(0); if (!libraw_lib) { - CTX_LOG_ERROR("raw.c", "Cannot create libraw handle") + CTX_LOG_ERROR("raw.c", "Cannot create libraw handle"); return; } size_t buf_len = 0; void *buf = read_all(f, &buf_len); if (buf == NULL) { - CTX_LOG_ERROR(f->filepath, "read_all() failed") + CTX_LOG_ERROR(f->filepath, "read_all() failed"); return; } int ret = libraw_open_buffer(libraw_lib, buf, buf_len); if (ret != 0) { - CTX_LOG_ERROR(f->filepath, "Could not open raw file") + CTX_LOG_ERROR(f->filepath, "Could not open raw file"); free(buf); libraw_close(libraw_lib); return; } if (*libraw_lib->idata.model != '\0') { - APPEND_STR_META(doc, MetaExifModel, libraw_lib->idata.model) + APPEND_STR_META(doc, MetaExifModel, libraw_lib->idata.model); } if (*libraw_lib->idata.make != '\0') { - APPEND_STR_META(doc, MetaExifMake, libraw_lib->idata.make) + APPEND_STR_META(doc, MetaExifMake, libraw_lib->idata.make); } if (*libraw_lib->idata.software != '\0') { - APPEND_STR_META(doc, MetaExifSoftware, libraw_lib->idata.software) + APPEND_STR_META(doc, MetaExifSoftware, libraw_lib->idata.software); } - APPEND_LONG_META(doc, MetaWidth, libraw_lib->sizes.width) - APPEND_LONG_META(doc, MetaHeight, libraw_lib->sizes.height) + APPEND_LONG_META(doc, MetaWidth, libraw_lib->sizes.width); + APPEND_LONG_META(doc, MetaHeight, libraw_lib->sizes.height); char tmp[1024]; snprintf(tmp, sizeof(tmp), "%g", libraw_lib->other.iso_speed); - APPEND_STR_META(doc, MetaExifIsoSpeedRatings, tmp) + APPEND_STR_META(doc, MetaExifIsoSpeedRatings, tmp); if (*libraw_lib->other.desc != '\0') { - APPEND_STR_META(doc, MetaContent, libraw_lib->other.desc) + APPEND_STR_META(doc, MetaContent, libraw_lib->other.desc); } if (*libraw_lib->other.artist != '\0') { - APPEND_STR_META(doc, MetaArtist, libraw_lib->other.artist) + APPEND_STR_META(doc, MetaArtist, libraw_lib->other.artist); } struct tm *time = localtime(&libraw_lib->other.timestamp); strftime(tmp, sizeof(tmp), "%Y:%m:%d %H:%M:%S", time); - APPEND_STR_META(doc, MetaExifDateTime, tmp) + APPEND_STR_META(doc, MetaExifDateTime, tmp); snprintf(tmp, sizeof(tmp), "%.1f", libraw_lib->other.focal_len); - APPEND_STR_META(doc, MetaExifFocalLength, tmp) + APPEND_STR_META(doc, MetaExifFocalLength, tmp); snprintf(tmp, sizeof(tmp), "%.1f", libraw_lib->other.aperture); - APPEND_STR_META(doc, MetaExifFNumber, tmp) + APPEND_STR_META(doc, MetaExifFNumber, tmp); int denominator = (int) roundf(1 / libraw_lib->other.shutter); snprintf(tmp, sizeof(tmp), "1/%d", denominator); - APPEND_STR_META(doc, MetaExifExposureTime, tmp) + APPEND_STR_META(doc, MetaExifExposureTime, tmp); libraw_gps_info_t gps = libraw_lib->other.parsed_gps; double gps_longitude_dec = (gps.longitude[0] + gps.longitude[1] / 60 + gps.longitude[2] / 3600) * DMS_REF(gps.longref); snprintf(tmp, sizeof(tmp), "%.15f", gps_longitude_dec); if (gps_longitude_dec != 0.0) { - APPEND_STR_META(doc, MetaExifGpsLongitudeDec, tmp) + APPEND_STR_META(doc, MetaExifGpsLongitudeDec, tmp); } double gps_latitude_dec = (gps.latitude[0] + gps.latitude[1] / 60 + gps.latitude[2] / 3600) * DMS_REF(gps.latref); snprintf(tmp, sizeof(tmp), "%.15f", gps_latitude_dec); if (gps_latitude_dec != 0.0) { - APPEND_STR_META(doc, MetaExifGpsLatitudeDec, tmp) + APPEND_STR_META(doc, MetaExifGpsLatitudeDec, tmp); } - APPEND_STR_META(doc, MetaMediaVideoCodec, "raw") + APPEND_STR_META(doc, MetaMediaVideoCodec, "raw"); if (!ctx->enable_tn) { free(buf); @@ -179,7 +179,7 @@ void parse_raw(scan_raw_ctx_t *ctx, vfile_t *f, document_t *doc) { int unpack_ret = libraw_unpack_thumb(libraw_lib); if (unpack_ret != 0) { - CTX_LOG_ERRORF(f->filepath, "libraw_unpack_thumb returned error code %d", unpack_ret) + CTX_LOG_ERRORF(f->filepath, "libraw_unpack_thumb returned error code %d", unpack_ret); free(buf); libraw_close(libraw_lib); return; @@ -212,7 +212,7 @@ void parse_raw(scan_raw_ctx_t *ctx, vfile_t *f, document_t *doc) { ret = libraw_unpack(libraw_lib); if (ret != 0) { - CTX_LOG_ERROR(f->filepath, "Could not unpack raw file") + CTX_LOG_ERROR(f->filepath, "Could not unpack raw file"); free(buf); libraw_close(libraw_lib); return; diff --git a/third-party/libscan/libscan/scan.h b/third-party/libscan/libscan/scan.h index 9d09016..60cf21f 100644 --- a/third-party/libscan/libscan/scan.h +++ b/third-party/libscan/libscan/scan.h @@ -34,20 +34,20 @@ typedef int scan_code_t; #define LEVEL_ERROR 3 #define LEVEL_FATAL 4 -#define CTX_LOG_DEBUGF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_DEBUG, fmt, __VA_ARGS__); -#define CTX_LOG_DEBUG(filepath, str) ctx->log(filepath, LEVEL_DEBUG, str); +#define CTX_LOG_DEBUGF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_DEBUG, fmt, __VA_ARGS__) +#define CTX_LOG_DEBUG(filepath, str) ctx->log(filepath, LEVEL_DEBUG, str) -#define CTX_LOG_INFOF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_INFO, fmt, __VA_ARGS__); -#define CTX_LOG_INFO(filepath, str) ctx->log(filepath, LEVEL_INFO, str); +#define CTX_LOG_INFOF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_INFO, fmt, __VA_ARGS__) +#define CTX_LOG_INFO(filepath, str) ctx->log(filepath, LEVEL_INFO, str) -#define CTX_LOG_WARNINGF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_WARNING, fmt, __VA_ARGS__); -#define CTX_LOG_WARNING(filepath, str) ctx->log(filepath, LEVEL_WARNING, str); +#define CTX_LOG_WARNINGF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_WARNING, fmt, __VA_ARGS__) +#define CTX_LOG_WARNING(filepath, str) ctx->log(filepath, LEVEL_WARNING, str) -#define CTX_LOG_ERRORF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_ERROR, fmt, __VA_ARGS__); -#define CTX_LOG_ERROR(filepath, str) ctx->log(filepath, LEVEL_ERROR, str); +#define CTX_LOG_ERRORF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_ERROR, fmt, __VA_ARGS__) +#define CTX_LOG_ERROR(filepath, str) ctx->log(filepath, LEVEL_ERROR, str) -#define CTX_LOG_FATALF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_FATAL, fmt, __VA_ARGS__); exit(-1); -#define CTX_LOG_FATAL(filepath, str) ctx->log(filepath, LEVEL_FATAL, str); exit(-1); +#define CTX_LOG_FATALF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_FATAL, fmt, __VA_ARGS__); exit(-1) +#define CTX_LOG_FATAL(filepath, str) ctx->log(filepath, LEVEL_FATAL, str); exit(-1) #define SIST_DOC_ID_LEN MD5_STR_LENGTH #define SIST_INDEX_ID_LEN MD5_STR_LENGTH diff --git a/third-party/libscan/libscan/text/text.c b/third-party/libscan/libscan/text/text.c index b4ffe33..663253d 100644 --- a/third-party/libscan/libscan/text/text.c +++ b/third-party/libscan/libscan/text/text.c @@ -2,6 +2,10 @@ scan_code_t parse_text(scan_text_ctx_t *ctx, vfile_t *f, document_t *doc) { + if (ctx->content_size <= 0) { + return SCAN_OK; + } + int to_read = MIN(ctx->content_size, f->st_size); if (to_read <= 2) { @@ -11,7 +15,7 @@ scan_code_t parse_text(scan_text_ctx_t *ctx, vfile_t *f, document_t *doc) { char *buf = malloc(to_read); int ret = f->read(f, buf, to_read); if (ret < 0) { - CTX_LOG_ERRORF(doc->filepath, "read() returned error code: [%d]", ret) + CTX_LOG_ERRORF(doc->filepath, "read() returned error code: [%d]", ret); free(buf); return SCAN_ERR_READ; } @@ -39,12 +43,16 @@ scan_code_t parse_text(scan_text_ctx_t *ctx, vfile_t *f, document_t *doc) { scan_code_t parse_markup(scan_text_ctx_t *ctx, vfile_t *f, document_t *doc) { + if (ctx->content_size <= 0) { + return SCAN_OK; + } + int to_read = MIN(MAX_MARKUP_SIZE, f->st_size); char *buf = malloc(to_read + 1); int ret = f->read(f, buf, to_read); if (ret < 0) { - CTX_LOG_ERRORF(doc->filepath, "read() returned error code: [%d]", ret) + CTX_LOG_ERRORF(doc->filepath, "read() returned error code: [%d]", ret); free(buf); return SCAN_ERR_READ; } diff --git a/third-party/libscan/libscan/wpd/wpd.c b/third-party/libscan/libscan/wpd/wpd.c index d1c13b6..e71f0d1 100644 --- a/third-party/libscan/libscan/wpd/wpd.c +++ b/third-party/libscan/libscan/wpd/wpd.c @@ -10,14 +10,14 @@ scan_code_t parse_wpd(scan_wpd_ctx_t *ctx, vfile_t *f, document_t *doc) { wpd_confidence_t conf = wpd_is_file_format_supported(stream); if (conf == C_WPD_CONFIDENCE_SUPPORTED_ENCRYPTION || conf == C_WPD_CONFIDENCE_UNSUPPORTED_ENCRYPTION) { - CTX_LOG_DEBUGF("wpd.c", "File is encrypted! Password-protected WPD files are not supported yet (conf=%d)", conf) + CTX_LOG_DEBUGF("wpd.c", "File is encrypted! Password-protected WPD files are not supported yet (conf=%d)", conf); wpd_memory_stream_destroy(stream); free(buf); return SCAN_ERR_READ; } if (conf != C_WPD_CONFIDENCE_EXCELLENT) { - CTX_LOG_ERRORF("wpd.c", "Unsupported file format! [%s] (conf=%d)", doc->filepath, conf) + CTX_LOG_ERRORF("wpd.c", "Unsupported file format! [%s] (conf=%d)", doc->filepath, conf); wpd_memory_stream_destroy(stream); free(buf); return SCAN_ERR_READ; @@ -28,11 +28,11 @@ scan_code_t parse_wpd(scan_wpd_ctx_t *ctx, vfile_t *f, document_t *doc) { if (res != C_WPD_OK) { CTX_LOG_ERRORF("wpd.c", "Error while parsing WPD file [%s] (%d)", - doc->filepath, res) + doc->filepath, res); } if (tex.dyn_buffer.cur != 0) { - APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf) + APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf); } text_buffer_destroy(&tex); From 6182338f296f85fdcdfee1a0ee21cece116ac0a1 Mon Sep 17 00:00:00 2001 From: simon987 Date: Mon, 10 Apr 2023 15:10:56 -0400 Subject: [PATCH 6/7] Update dependencies, fix some build issues --- .dockerignore | 8 +++- .gitignore | 3 +- CMakeLists.txt | 10 +---- Dockerfile | 8 ++-- README.md | 3 +- third-party/libscan/CMakeLists.txt | 63 ++++++------------------------ 6 files changed, 26 insertions(+), 69 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4020560..6806de8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,7 +15,6 @@ Makefile **/*.cbp VERSION **/node_modules/ -.git/ sist2-*-linux-debug sist2-*-linux sist2_debug @@ -33,4 +32,9 @@ tmp_scan/ Dockerfile Dockerfile.arm64 docker-compose.yml -state.db \ No newline at end of file +state.db +*-journal +build/ +__pycache__/ +sist2-vue/dist +sist2-admin/frontend/dist \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8f9f8a3..0ca141d 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,5 @@ src/web/static_generated.c src/magic_generated.c src/index/static_generated.c *.sist2 -*-shm \ No newline at end of file +*-shm +*-journal \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index d3869f0..cb92ba3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,6 @@ set(CMAKE_C_STANDARD 11) option(SIST_DEBUG "Build a debug executable" on) option(SIST_FAST "Enable more optimisation flags" off) -option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0) add_compile_definitions( "SIST_PLATFORM=${SIST_PLATFORM}" @@ -56,14 +55,10 @@ set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib) find_package(PkgConfig REQUIRED) -find_package(lmdb CONFIG REQUIRED) find_package(cJSON CONFIG REQUIRED) find_package(unofficial-mongoose CONFIG REQUIRED) find_package(CURL CONFIG REQUIRED) -find_library(MAGIC_LIB - NAMES libmagic.so.1 magic - PATHS /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/ -) +find_library(MAGIC_LIB NAMES libmagic.a REQUIRED) find_package(unofficial-sqlite3 CONFIG REQUIRED) @@ -91,7 +86,6 @@ if (SIST_DEBUG) -fsanitize=address -fno-inline # -O2 - -w ) target_link_options( sist2 @@ -137,8 +131,6 @@ target_link_libraries( sist2 z - lmdb - cjson argparse unofficial::mongoose::mongoose CURL::libcurl diff --git a/Dockerfile b/Dockerfile index 1ca0e01..ec69796 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,9 @@ COPY sist2-admin sist2-admin RUN cd sist2-vue/ && npm install && npm run build RUN cd sist2-admin/frontend/ && npm install && npm run build -RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake . -RUN make -j$(nproc) -RUN strip sist2 || mv sist2_debug sist2 +RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .. +RUN cd build && make -j$(nproc) +RUN strip build/sist2 || mv sist2_debug build/sist2 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea @@ -49,7 +49,7 @@ RUN mkdir -p /usr/share/tessdata && \ curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata # sist2 -COPY --from=build /build/sist2 /root/sist2 +COPY --from=build /build/build/sist2 /root/sist2 # sist2-admin COPY sist2-admin/requirements.txt sist2-admin/ diff --git a/README.md b/README.md index 39d0783..fa89016 100644 --- a/README.md +++ b/README.md @@ -149,8 +149,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux 1. Install vcpkg dependencies ```bash - vcpkg install curl[core,openssl] - vcpkg install lmdb sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw jasper lcms gumbo + vcpkg install curl[core,openssl] sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample] ``` 1. Build diff --git a/third-party/libscan/CMakeLists.txt b/third-party/libscan/CMakeLists.txt index 3cc065d..5bb23af 100644 --- a/third-party/libscan/CMakeLists.txt +++ b/third-party/libscan/CMakeLists.txt @@ -78,6 +78,7 @@ else() -fno-stack-protector -fomit-frame-pointer #-freciprocal-math + -w ) endif() @@ -99,17 +100,17 @@ find_package(unofficial-pcre CONFIG REQUIRED) find_library(JBIG2DEC_LIB NAMES jbig2decd jbig2dec) find_library(HARFBUZZ_LIB NAMES harfbuzz harfbuzzd) -find_library(FREETYPE_LIB NAMES freetype freetyped) -find_package(unofficial-brotli CONFIG REQUIRED) find_library(LZO2_LIB NAMES lzo2) -find_library(RAW_LIB NAMES libraw.a) find_library(MUPDF_LIB NAMES liblibmupdf.a) find_library(CMS_LIB NAMES lcms2) find_library(JAS_LIB NAMES jasper) find_library(GUMBO_LIB NAMES gumbo) find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/) find_package(Leptonica CONFIG REQUIRED) +find_package(FFMPEG REQUIRED) +find_package(libraw CONFIG REQUIRED) +find_package(Freetype REQUIRED) target_compile_options( @@ -118,39 +119,7 @@ target_compile_options( -g ) -if (SIST_DEBUG) - SET(FFMPEG_DEBUG "--enable-debug=3" "--disable-optimizations") -else() - SET(FFMPEG_DEBUG "") -endif() - include(ExternalProject) -find_program(MAKE_EXE NAMES gmake nmake make) - -ExternalProject_Add( - ffmpeg - GIT_REPOSITORY https://git.ffmpeg.org/ffmpeg.git - GIT_TAG "n4.4" - - UPDATE_COMMAND "" - PATCH_COMMAND "" - TEST_COMMAND "" - CONFIGURE_COMMAND ./configure --disable-shared --enable-static --disable-ffmpeg --disable-ffplay - --disable-ffprobe --disable-doc --disable-manpages --disable-postproc --disable-avfilter --disable-alsa - --disable-lzma --disable-xlib --disable-vdpau --disable-vaapi --disable-sdl2 - --disable-network ${FFMPEG_DEBUG} - INSTALL_COMMAND "" - - PREFIX "third-party/ext_ffmpeg" - SOURCE_DIR "third-party/ext_ffmpeg/src/ffmpeg" - BINARY_DIR "third-party/ext_ffmpeg/src/ffmpeg" - - BUILD_COMMAND ${MAKE_EXE} -j33 --silent -) - -SET(FFMPEG_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_ffmpeg/src/ffmpeg) -SET(FFMPEG_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_ffmpeg/src/ffmpeg) - ExternalProject_Add( libwpd URL http://prdownloads.sourceforge.net/libwpd/libwpd-0.9.9.tar.gz @@ -165,19 +134,20 @@ ExternalProject_Add( SOURCE_DIR "third-party/ext_libwpd/src/libwpd" BINARY_DIR "third-party/ext_libwpd/src/libwpd" - BUILD_COMMAND ${MAKE_EXE} -j33 + BUILD_COMMAND make -j33 ) SET(WPD_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwpd/src/lib/.libs/) SET(WPD_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwpd/inc/) add_dependencies( scan - ffmpeg antiword libwpd mobi ) + +target_link_directories(scan PUBLIC ${FFMPEG_LIBRARY_DIRS}) target_link_libraries( scan PUBLIC @@ -202,36 +172,26 @@ target_link_libraries( stdc++ - -Wl,--whole-archive - m - -Wl,--no-whole-archive - ${JPEG_LIBRARIES} ${Tesseract_LIBRARIES} ${LIBXML2_LIBRARIES} - ${FREETYPE_LIB} - unofficial::brotli::brotlidec-static - - ${FFMPEG_LIB_DIR}/libavformat/libavformat.a - ${FFMPEG_LIB_DIR}/libavcodec/libavcodec.a - ${FFMPEG_LIB_DIR}/libavutil/libavutil.a - ${FFMPEG_LIB_DIR}/libswresample/libswresample.a - ${FFMPEG_LIB_DIR}/libswscale/libswscale.a + Freetype::Freetype z ${CMAKE_THREAD_LIBS_INIT} - ${RAW_LIB} ${GOMP_LIB} ${CMS_LIB} ${JAS_LIB} ${GUMBO_LIB} - dl + antiword mobi unofficial::pcre::pcre unofficial::pcre::pcre16 unofficial::pcre::pcre32 unofficial::pcre::pcrecpp leptonica + libraw::raw + ${FFMPEG_LIBRARIES} ) target_include_directories( @@ -243,6 +203,7 @@ target_include_directories( ${FFMPEG_INCLUDE_DIR} ${MOBI_INCLUDE_DIR} ${WPD_INCLUDE_DIR} + ${FFMPEG_INCLUDE_DIRS} ) if (BUILD_TESTS) From 01490d1cbf06230925afff2b07dfb85bb1b6cf0a Mon Sep 17 00:00:00 2001 From: simon987 Date: Mon, 10 Apr 2023 19:45:08 -0400 Subject: [PATCH 7/7] Update sist2-admin for 3.x.x, more fixes --- Dockerfile | 4 +- Dockerfile.arm64 | 34 +- .../frontend/src/components/JobOptions.vue | 12 +- .../frontend/src/components/ScanOptions.vue | 8 +- sist2-admin/frontend/src/i18n/messages.js | 7 +- sist2-admin/frontend/yarn.lock | 781 +++++++++++++++++- sist2-admin/sist2_admin/app.py | 5 +- sist2-admin/sist2_admin/jobs.py | 52 +- sist2-admin/sist2_admin/sist2.py | 34 +- sist2-admin/sist2_admin/state.py | 4 +- sist2-vue/src/components/FullThumbnail.vue | 2 +- src/tpool.c | 4 - src/web/serve.c | 3 +- third-party/libscan/CMakeLists.txt | 2 +- 14 files changed, 874 insertions(+), 78 deletions(-) diff --git a/Dockerfile b/Dockerfile index ec69796..af69ff0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN cd sist2-admin/frontend/ && npm install && npm run build RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .. RUN cd build && make -j$(nproc) -RUN strip build/sist2 || mv sist2_debug build/sist2 +RUN strip build/sist2 || mv build/sist2_debug build/sist2 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea @@ -33,7 +33,7 @@ ENV LC_ALL C.UTF-8 ENTRYPOINT ["/root/sist2"] RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 libmagic1 python3 \ - python3-pip git tesseract-ocr libpq-dev && rm -rf /var/lib/apt/lists/* + python3-pip git tesseract-ocr && rm -rf /var/lib/apt/lists/* RUN mkdir -p /usr/share/tessdata && \ cd /usr/share/tessdata/ && \ diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 0008ec8..66c2e4f 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -3,13 +3,20 @@ MAINTAINER simon987 WORKDIR /build/ ADD . /build/ -RUN cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake . -RUN make -j$(nproc) -RUN strip sist2 +RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .. +RUN cd build && make -j$(nproc) +RUN strip build/sist2 || mv build/sist2_debug build/sist2 -FROM --platform="linux/arm64/v8" ubuntu:20.04 +FROM --platform=linux/arm64/v8 ubuntu@sha256:537da24818633b45fcb65e5285a68c3ec1f3db25f5ae5476a7757bc8dfae92a3 -RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/* +WORKDIR /root + +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +ENTRYPOINT ["/root/sist2"] + +RUN apt update && apt install -y curl libasan5 libmagic1 tesseract-ocr python3-pip python3 git && rm -rf /var/lib/apt/lists/* RUN mkdir -p /usr/share/tessdata && \ cd /usr/share/tessdata/ && \ @@ -18,11 +25,16 @@ RUN mkdir -p /usr/share/tessdata && \ curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\ curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\ curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\ - curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata + curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\ + curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\ + curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\ + curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\ + curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 +# sist2 +COPY --from=build /build/build/sist2 /root/sist2 -ENTRYPOINT ["/root/sist2"] - -COPY --from=build /build/sist2 /root/sist2 \ No newline at end of file +# sist2-admin +COPY sist2-admin/requirements.txt sist2-admin/ +RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt +COPY --from=build /build/sist2-admin/ sist2-admin/ diff --git a/sist2-admin/frontend/src/components/JobOptions.vue b/sist2-admin/frontend/src/components/JobOptions.vue index f40991e..84ed8e7 100644 --- a/sist2-admin/frontend/src/components/JobOptions.vue +++ b/sist2-admin/frontend/src/components/JobOptions.vue @@ -28,16 +28,22 @@ export default { return this.$store.state.jobDesktopNotificationMap[this.job.name]; } }, - methods: { + mounted() { + this.cronValid = this.checkCron(this.job.cron_expression) + }, + methods: { + checkCron(expression) { + return /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(expression); + }, updateNotifications(value) { this.$store.dispatch("setJobDesktopNotification", { job: this.job.name, enabled: value - }) + }); }, update() { if (this.job.schedule_enabled) { - this.cronValid = /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(this.job.cron_expression); + this.cronValid = this.checkCron(this.job.cron_expression); } else { this.cronValid = undefined; } diff --git a/sist2-admin/frontend/src/components/ScanOptions.vue b/sist2-admin/frontend/src/components/ScanOptions.vue index 2f4598a..a000156 100644 --- a/sist2-admin/frontend/src/components/ScanOptions.vue +++ b/sist2-admin/frontend/src/components/ScanOptions.vue @@ -6,9 +6,6 @@ - - - @@ -70,8 +67,9 @@ {{ $t("scanOptions.readSubtitles") }} - - + + {{ $t("scanOptions.optimizeIndex") }} + diff --git a/sist2-admin/frontend/src/i18n/messages.js b/sist2-admin/frontend/src/i18n/messages.js index 9483cc1..aa652f2 100644 --- a/sist2-admin/frontend/src/i18n/messages.js +++ b/sist2-admin/frontend/src/i18n/messages.js @@ -56,6 +56,10 @@ export default { tagline: "Tagline in navbar", auth: "Basic auth in user:password format", tagAuth: "Basic auth in user:password format for tagging", + auth0Audience: "Auth0 audience", + auth0Domain: "Auth0 domain", + auth0ClientId: "Auth0 client ID", + auth0PublicKey: "Auth0 public key", }, scanOptions: { title: "Scanning options", @@ -80,7 +84,8 @@ export default { checksums: "Calculate file checksums when scanning", readSubtitles: "Read subtitles from media files", memBuffer: "Maximum memory buffer size per thread in MiB for files inside archives", - treemapThreshold: "Relative size threshold for treemap" + treemapThreshold: "Relative size threshold for treemap", + optimizeIndex: "Defragment index file after scan to reduce its file size." }, indexOptions: { title: "Indexing options", diff --git a/sist2-admin/frontend/yarn.lock b/sist2-admin/frontend/yarn.lock index eec90de..d7fa72d 100644 --- a/sist2-admin/frontend/yarn.lock +++ b/sist2-admin/frontend/yarn.lock @@ -5,6 +5,7 @@ "@achrinza/node-ipc@^9.2.5": version "9.2.6" resolved "https://registry.npmjs.org/@achrinza/node-ipc/-/node-ipc-9.2.6.tgz" + integrity sha512-ULSIYPy4ZPM301dfCxRz0l2GJjOwIo/PqmWonIu1bLml7UmnVQmH+juJcoyXp6E8gIRRNAjGYftJnNQlfy4vPg== dependencies: "@node-ipc/js-queue" "2.0.3" event-pubsub "4.3.0" @@ -13,6 +14,7 @@ "@ampproject/remapping@^2.1.0": version "2.2.0" resolved "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz" + integrity sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w== dependencies: "@jridgewell/gen-mapping" "^0.1.0" "@jridgewell/trace-mapping" "^0.3.9" @@ -20,16 +22,19 @@ "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.16.7", "@babel/code-frame@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz" + integrity sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q== dependencies: "@babel/highlight" "^7.18.6" "@babel/compat-data@^7.17.7", "@babel/compat-data@^7.20.0", "@babel/compat-data@^7.20.1": version "7.20.5" resolved "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.20.5.tgz" + integrity sha512-KZXo2t10+/jxmkhNXc7pZTqRvSOIvVv/+lJwHS+B2rErwOyjuVRh60yVpb7liQ1U5t7lLJ1bz+t8tSypUZdm0g== "@babel/core@^7.12.16": version "7.17.10" resolved "https://registry.npmjs.org/@babel/core/-/core-7.17.10.tgz" + integrity sha512-liKoppandF3ZcBnIYFjfSDHZLKdLHGJRkoWtG8zQyGJBQfIYobpnVGI5+pLBNtS6psFLDzyq8+h5HiVljW9PNA== dependencies: "@ampproject/remapping" "^2.1.0" "@babel/code-frame" "^7.16.7" @@ -50,6 +55,7 @@ "@babel/generator@^7.17.10", "@babel/generator@^7.20.5": version "7.20.5" resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.20.5.tgz" + integrity sha512-jl7JY2Ykn9S0yj4DQP82sYvPU+T3g0HFcWTqDLqiuA9tGRNIj9VfbtXGAYTTkyNEnQk1jkMGOdYka8aG/lulCA== dependencies: "@babel/types" "^7.20.5" "@jridgewell/gen-mapping" "^0.3.2" @@ -58,12 +64,14 @@ "@babel/helper-annotate-as-pure@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz" + integrity sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA== dependencies: "@babel/types" "^7.18.6" "@babel/helper-builder-binary-assignment-operator-visitor@^7.18.6": version "7.18.9" resolved "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.18.9.tgz" + integrity sha512-yFQ0YCHoIqarl8BCRwBL8ulYUaZpz3bNsA7oFepAzee+8/+ImtADXNOmO5vJvsPff3qi+hvpkY/NYBTrBQgdNw== dependencies: "@babel/helper-explode-assignable-expression" "^7.18.6" "@babel/types" "^7.18.9" @@ -71,6 +79,7 @@ "@babel/helper-compilation-targets@^7.12.16", "@babel/helper-compilation-targets@^7.17.10", "@babel/helper-compilation-targets@^7.17.7", "@babel/helper-compilation-targets@^7.18.9", "@babel/helper-compilation-targets@^7.20.0": version "7.20.0" resolved "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.0.tgz" + integrity sha512-0jp//vDGp9e8hZzBc6N/KwA5ZK3Wsm/pfm4CrY7vzegkVxc65SgSn6wYOnwHe9Js9HRQ1YTCKLGPzDtaS3RoLQ== dependencies: "@babel/compat-data" "^7.20.0" "@babel/helper-validator-option" "^7.18.6" @@ -80,6 +89,7 @@ "@babel/helper-create-class-features-plugin@^7.18.6", "@babel/helper-create-class-features-plugin@^7.20.5": version "7.20.5" resolved "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.20.5.tgz" + integrity sha512-3RCdA/EmEaikrhayahwToF0fpweU/8o2p8vhc1c/1kftHOdTKuC65kik/TLc+qfbS8JKw4qqJbne4ovICDhmww== dependencies: "@babel/helper-annotate-as-pure" "^7.18.6" "@babel/helper-environment-visitor" "^7.18.9" @@ -92,6 +102,7 @@ "@babel/helper-create-regexp-features-plugin@^7.18.6", "@babel/helper-create-regexp-features-plugin@^7.20.5": version "7.20.5" resolved "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.20.5.tgz" + integrity sha512-m68B1lkg3XDGX5yCvGO0kPx3v9WIYLnzjKfPcQiwntEQa5ZeRkPmo2X/ISJc8qxWGfwUr+kvZAeEzAwLec2r2w== dependencies: "@babel/helper-annotate-as-pure" "^7.18.6" regexpu-core "^5.2.1" @@ -99,6 +110,7 @@ "@babel/helper-define-polyfill-provider@^0.3.3": version "0.3.3" resolved "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.3.tgz" + integrity sha512-z5aQKU4IzbqCC1XH0nAqfsFLMVSo22SBKUc0BxGrLkolTdPTructy0ToNnlO2zA4j9Q/7pjMZf0DSY+DSTYzww== dependencies: "@babel/helper-compilation-targets" "^7.17.7" "@babel/helper-plugin-utils" "^7.16.7" @@ -110,16 +122,19 @@ "@babel/helper-environment-visitor@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz" + integrity sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg== "@babel/helper-explode-assignable-expression@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz" + integrity sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg== dependencies: "@babel/types" "^7.18.6" "@babel/helper-function-name@^7.18.9", "@babel/helper-function-name@^7.19.0": version "7.19.0" resolved "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.19.0.tgz" + integrity sha512-WAwHBINyrpqywkUH0nTnNgI5ina5TFn85HKS0pbPDfxFfhyR/aNQEn4hGi1P1JyT//I0t4OgXUlofzWILRvS5w== dependencies: "@babel/template" "^7.18.10" "@babel/types" "^7.19.0" @@ -127,24 +142,28 @@ "@babel/helper-hoist-variables@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz" + integrity sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q== dependencies: "@babel/types" "^7.18.6" "@babel/helper-member-expression-to-functions@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.18.9.tgz" + integrity sha512-RxifAh2ZoVU67PyKIO4AMi1wTenGfMR/O/ae0CCRqwgBAt5v7xjdtRw7UoSbsreKrQn5t7r89eruK/9JjYHuDg== dependencies: "@babel/types" "^7.18.9" "@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz" + integrity sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA== dependencies: "@babel/types" "^7.18.6" "@babel/helper-module-transforms@^7.17.7", "@babel/helper-module-transforms@^7.18.6", "@babel/helper-module-transforms@^7.19.6": version "7.20.2" resolved "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.20.2.tgz" + integrity sha512-zvBKyJXRbmK07XhMuujYoJ48B5yvvmM6+wcpv6Ivj4Yg6qO7NOZOSnvZN9CRl1zz1Z4cKf8YejmCMh8clOoOeA== dependencies: "@babel/helper-environment-visitor" "^7.18.9" "@babel/helper-module-imports" "^7.18.6" @@ -158,16 +177,19 @@ "@babel/helper-optimise-call-expression@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.18.6.tgz" + integrity sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA== dependencies: "@babel/types" "^7.18.6" "@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.14.5", "@babel/helper-plugin-utils@^7.16.7", "@babel/helper-plugin-utils@^7.18.6", "@babel/helper-plugin-utils@^7.18.9", "@babel/helper-plugin-utils@^7.19.0", "@babel/helper-plugin-utils@^7.20.2", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3": version "7.20.2" resolved "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.20.2.tgz" + integrity sha512-8RvlJG2mj4huQ4pZ+rU9lqKi9ZKiRmuvGuM2HlWmkmgOhbs6zEAw6IEiJ5cQqGbDzGZOhwuOQNtZMi/ENLjZoQ== "@babel/helper-remap-async-to-generator@^7.18.6", "@babel/helper-remap-async-to-generator@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.18.9.tgz" + integrity sha512-dI7q50YKd8BAv3VEfgg7PS7yD3Rtbi2J1XMXaalXO0W0164hYLnh8zpjRS0mte9MfVp/tltvr/cfdXPvJr1opA== dependencies: "@babel/helper-annotate-as-pure" "^7.18.6" "@babel/helper-environment-visitor" "^7.18.9" @@ -177,6 +199,7 @@ "@babel/helper-replace-supers@^7.18.6", "@babel/helper-replace-supers@^7.19.1": version "7.19.1" resolved "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.19.1.tgz" + integrity sha512-T7ahH7wV0Hfs46SFh5Jz3s0B6+o8g3c+7TMxu7xKfmHikg7EAZ3I2Qk9LFhjxXq8sL7UkP5JflezNwoZa8WvWw== dependencies: "@babel/helper-environment-visitor" "^7.18.9" "@babel/helper-member-expression-to-functions" "^7.18.9" @@ -187,36 +210,43 @@ "@babel/helper-simple-access@^7.19.4", "@babel/helper-simple-access@^7.20.2": version "7.20.2" resolved "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.20.2.tgz" + integrity sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA== dependencies: "@babel/types" "^7.20.2" "@babel/helper-skip-transparent-expression-wrappers@^7.18.9": version "7.20.0" resolved "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.20.0.tgz" + integrity sha512-5y1JYeNKfvnT8sZcK9DVRtpTbGiomYIHviSP3OQWmDPU3DeH4a1ZlT/N2lyQ5P8egjcRaT/Y9aNqUxK0WsnIIg== dependencies: "@babel/types" "^7.20.0" "@babel/helper-split-export-declaration@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz" + integrity sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA== dependencies: "@babel/types" "^7.18.6" "@babel/helper-string-parser@^7.19.4": version "7.19.4" resolved "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz" + integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw== "@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": version "7.19.1" resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz" + integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== "@babel/helper-validator-option@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz" + integrity sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw== "@babel/helper-wrap-function@^7.18.9": version "7.20.5" resolved "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.20.5.tgz" + integrity sha512-bYMxIWK5mh+TgXGVqAtnu5Yn1un+v8DDZtqyzKRLUzrh70Eal2O3aZ7aPYiMADO4uKlkzOiRiZ6GX5q3qxvW9Q== dependencies: "@babel/helper-function-name" "^7.19.0" "@babel/template" "^7.18.10" @@ -226,6 +256,7 @@ "@babel/helpers@^7.17.9": version "7.17.9" resolved "https://registry.npmjs.org/@babel/helpers/-/helpers-7.17.9.tgz" + integrity sha512-cPCt915ShDWUEzEp3+UNRktO2n6v49l5RSnG9M5pS24hA+2FAc5si+Pn1i4VVbQQ+jh+bIZhPFQOJOzbrOYY1Q== dependencies: "@babel/template" "^7.16.7" "@babel/traverse" "^7.17.9" @@ -234,6 +265,7 @@ "@babel/highlight@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz" + integrity sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g== dependencies: "@babel/helper-validator-identifier" "^7.18.6" chalk "^2.0.0" @@ -242,16 +274,19 @@ "@babel/parser@^7.17.10", "@babel/parser@^7.18.10", "@babel/parser@^7.20.5", "@babel/parser@^7.7.0": version "7.20.5" resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.20.5.tgz" + integrity sha512-r27t/cy/m9uKLXQNWWebeCUHgnAZq0CpG1OwKRxzJMP1vpSU4bSIK2hq+/cp0bQxetkXx38n09rNu8jVkcK/zA== "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz" + integrity sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.18.9.tgz" + integrity sha512-AHrP9jadvH7qlOj6PINbgSuphjQUAK7AOT7DPjBo9EHoLhQTnnK5u45e1Hd4DbSQEO9nqPWtQ89r+XEOWFScKg== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/helper-skip-transparent-expression-wrappers" "^7.18.9" @@ -260,6 +295,7 @@ "@babel/plugin-proposal-async-generator-functions@^7.20.1": version "7.20.1" resolved "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.20.1.tgz" + integrity sha512-Gh5rchzSwE4kC+o/6T8waD0WHEQIsDmjltY8WnWRXHUdH8axZhuH86Ov9M72YhJfDrZseQwuuWaaIT/TmePp3g== dependencies: "@babel/helper-environment-visitor" "^7.18.9" "@babel/helper-plugin-utils" "^7.19.0" @@ -269,6 +305,7 @@ "@babel/plugin-proposal-class-properties@^7.12.13", "@babel/plugin-proposal-class-properties@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz" + integrity sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ== dependencies: "@babel/helper-create-class-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" @@ -276,6 +313,7 @@ "@babel/plugin-proposal-class-static-block@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.18.6.tgz" + integrity sha512-+I3oIiNxrCpup3Gi8n5IGMwj0gOCAjcJUSQEcotNnCCPMEnixawOQ+KeJPlgfjzx+FKQ1QSyZOWe7wmoJp7vhw== dependencies: "@babel/helper-create-class-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" @@ -284,6 +322,7 @@ "@babel/plugin-proposal-decorators@^7.12.13": version "7.20.5" resolved "https://registry.npmjs.org/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.20.5.tgz" + integrity sha512-Lac7PpRJXcC3s9cKsBfl+uc+DYXU5FD06BrTFunQO6QIQT+DwyzDPURAowI3bcvD1dZF/ank1Z5rstUJn3Hn4Q== dependencies: "@babel/helper-create-class-features-plugin" "^7.20.5" "@babel/helper-plugin-utils" "^7.20.2" @@ -294,6 +333,7 @@ "@babel/plugin-proposal-dynamic-import@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.18.6.tgz" + integrity sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-syntax-dynamic-import" "^7.8.3" @@ -301,6 +341,7 @@ "@babel/plugin-proposal-export-namespace-from@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.9.tgz" + integrity sha512-k1NtHyOMvlDDFeb9G5PhUXuGj8m/wiwojgQVEhJ/fsVsMCpLyOP4h0uGEjYJKrRI+EVPlb5Jk+Gt9P97lOGwtA== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/plugin-syntax-export-namespace-from" "^7.8.3" @@ -308,6 +349,7 @@ "@babel/plugin-proposal-json-strings@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.18.6.tgz" + integrity sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-syntax-json-strings" "^7.8.3" @@ -315,6 +357,7 @@ "@babel/plugin-proposal-logical-assignment-operators@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.18.9.tgz" + integrity sha512-128YbMpjCrP35IOExw2Fq+x55LMP42DzhOhX2aNNIdI9avSWl2PI0yuBWarr3RYpZBSPtabfadkH2yeRiMD61Q== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4" @@ -322,6 +365,7 @@ "@babel/plugin-proposal-nullish-coalescing-operator@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz" + integrity sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3" @@ -329,6 +373,7 @@ "@babel/plugin-proposal-numeric-separator@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz" + integrity sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-syntax-numeric-separator" "^7.10.4" @@ -336,6 +381,7 @@ "@babel/plugin-proposal-object-rest-spread@^7.20.2": version "7.20.2" resolved "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.20.2.tgz" + integrity sha512-Ks6uej9WFK+fvIMesSqbAto5dD8Dz4VuuFvGJFKgIGSkJuRGcrwGECPA1fDgQK3/DbExBJpEkTeYeB8geIFCSQ== dependencies: "@babel/compat-data" "^7.20.1" "@babel/helper-compilation-targets" "^7.20.0" @@ -346,6 +392,7 @@ "@babel/plugin-proposal-optional-catch-binding@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.18.6.tgz" + integrity sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-syntax-optional-catch-binding" "^7.8.3" @@ -353,6 +400,7 @@ "@babel/plugin-proposal-optional-chaining@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.18.9.tgz" + integrity sha512-v5nwt4IqBXihxGsW2QmCWMDS3B3bzGIk/EQVZz2ei7f3NJl8NzAJVvUmpDW5q1CRNY+Beb/k58UAH1Km1N411w== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/helper-skip-transparent-expression-wrappers" "^7.18.9" @@ -361,6 +409,7 @@ "@babel/plugin-proposal-private-methods@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz" + integrity sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA== dependencies: "@babel/helper-create-class-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" @@ -368,6 +417,7 @@ "@babel/plugin-proposal-private-property-in-object@^7.18.6": version "7.20.5" resolved "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.20.5.tgz" + integrity sha512-Vq7b9dUA12ByzB4EjQTPo25sFhY+08pQDBSZRtUAkj7lb7jahaHR5igera16QZ+3my1nYR4dKsNdYj5IjPHilQ== dependencies: "@babel/helper-annotate-as-pure" "^7.18.6" "@babel/helper-create-class-features-plugin" "^7.20.5" @@ -377,6 +427,7 @@ "@babel/plugin-proposal-unicode-property-regex@^7.18.6", "@babel/plugin-proposal-unicode-property-regex@^7.4.4": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.18.6.tgz" + integrity sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w== dependencies: "@babel/helper-create-regexp-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" @@ -384,114 +435,133 @@ "@babel/plugin-syntax-async-generators@^7.8.4": version "7.8.4" resolved "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz" + integrity sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw== dependencies: "@babel/helper-plugin-utils" "^7.8.0" "@babel/plugin-syntax-class-properties@^7.12.13": version "7.12.13" resolved "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz" + integrity sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA== dependencies: "@babel/helper-plugin-utils" "^7.12.13" "@babel/plugin-syntax-class-static-block@^7.14.5": version "7.14.5" resolved "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz" + integrity sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw== dependencies: "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-syntax-decorators@^7.19.0": version "7.19.0" resolved "https://registry.npmjs.org/@babel/plugin-syntax-decorators/-/plugin-syntax-decorators-7.19.0.tgz" + integrity sha512-xaBZUEDntt4faL1yN8oIFlhfXeQAWJW7CLKYsHTUqriCUbj8xOra8bfxxKGi/UwExPFBuPdH4XfHc9rGQhrVkQ== dependencies: "@babel/helper-plugin-utils" "^7.19.0" "@babel/plugin-syntax-dynamic-import@^7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz" + integrity sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ== dependencies: "@babel/helper-plugin-utils" "^7.8.0" "@babel/plugin-syntax-export-namespace-from@^7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz" + integrity sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q== dependencies: "@babel/helper-plugin-utils" "^7.8.3" "@babel/plugin-syntax-import-assertions@^7.20.0": version "7.20.0" resolved "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.20.0.tgz" + integrity sha512-IUh1vakzNoWalR8ch/areW7qFopR2AEw03JlG7BbrDqmQ4X3q9uuipQwSGrUn7oGiemKjtSLDhNtQHzMHr1JdQ== dependencies: "@babel/helper-plugin-utils" "^7.19.0" "@babel/plugin-syntax-json-strings@^7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz" + integrity sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA== dependencies: "@babel/helper-plugin-utils" "^7.8.0" "@babel/plugin-syntax-jsx@^7.0.0", "@babel/plugin-syntax-jsx@^7.12.13", "@babel/plugin-syntax-jsx@^7.2.0": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.18.6.tgz" + integrity sha512-6mmljtAedFGTWu2p/8WIORGwy+61PLgOMPOdazc7YoJ9ZCWUyFy3A6CpPkRKLKD1ToAesxX8KGEViAiLo9N+7Q== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-syntax-logical-assignment-operators@^7.10.4": version "7.10.4" resolved "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz" + integrity sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig== dependencies: "@babel/helper-plugin-utils" "^7.10.4" "@babel/plugin-syntax-nullish-coalescing-operator@^7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz" + integrity sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ== dependencies: "@babel/helper-plugin-utils" "^7.8.0" "@babel/plugin-syntax-numeric-separator@^7.10.4": version "7.10.4" resolved "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz" + integrity sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug== dependencies: "@babel/helper-plugin-utils" "^7.10.4" "@babel/plugin-syntax-object-rest-spread@^7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz" + integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA== dependencies: "@babel/helper-plugin-utils" "^7.8.0" "@babel/plugin-syntax-optional-catch-binding@^7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz" + integrity sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q== dependencies: "@babel/helper-plugin-utils" "^7.8.0" "@babel/plugin-syntax-optional-chaining@^7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz" + integrity sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg== dependencies: "@babel/helper-plugin-utils" "^7.8.0" "@babel/plugin-syntax-private-property-in-object@^7.14.5": version "7.14.5" resolved "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz" + integrity sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg== dependencies: "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-syntax-top-level-await@^7.14.5": version "7.14.5" resolved "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz" + integrity sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw== dependencies: "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-transform-arrow-functions@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.18.6.tgz" + integrity sha512-9S9X9RUefzrsHZmKMbDXxweEH+YlE8JJEuat9FdvW9Qh1cw7W64jELCtWNkPBPX5En45uy28KGvA/AySqUh8CQ== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-transform-async-to-generator@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.18.6.tgz" + integrity sha512-ARE5wZLKnTgPW7/1ftQmSi1CmkqqHo2DNmtztFhvgtOWSDfq0Cq9/9L+KnZNYSNrydBekhW3rwShduf59RoXag== dependencies: "@babel/helper-module-imports" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" @@ -500,18 +570,21 @@ "@babel/plugin-transform-block-scoped-functions@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.18.6.tgz" + integrity sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-transform-block-scoping@^7.20.2": version "7.20.5" resolved "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.20.5.tgz" + integrity sha512-WvpEIW9Cbj9ApF3yJCjIEEf1EiNJLtXagOrL5LNWEZOo3jv8pmPoYTSNJQvqej8OavVlgOoOPw6/htGZro6IkA== dependencies: "@babel/helper-plugin-utils" "^7.20.2" "@babel/plugin-transform-classes@^7.20.2": version "7.20.2" resolved "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.20.2.tgz" + integrity sha512-9rbPp0lCVVoagvtEyQKSo5L8oo0nQS/iif+lwlAz29MccX2642vWDlSZK+2T2buxbopotId2ld7zZAzRfz9j1g== dependencies: "@babel/helper-annotate-as-pure" "^7.18.6" "@babel/helper-compilation-targets" "^7.20.0" @@ -526,18 +599,21 @@ "@babel/plugin-transform-computed-properties@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.18.9.tgz" + integrity sha512-+i0ZU1bCDymKakLxn5srGHrsAPRELC2WIbzwjLhHW9SIE1cPYkLCL0NlnXMZaM1vhfgA2+M7hySk42VBvrkBRw== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/plugin-transform-destructuring@^7.20.2": version "7.20.2" resolved "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.20.2.tgz" + integrity sha512-mENM+ZHrvEgxLTBXUiQ621rRXZes3KWUv6NdQlrnr1TkWVw+hUjQBZuP2X32qKlrlG2BzgR95gkuCRSkJl8vIw== dependencies: "@babel/helper-plugin-utils" "^7.20.2" "@babel/plugin-transform-dotall-regex@^7.18.6", "@babel/plugin-transform-dotall-regex@^7.4.4": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.18.6.tgz" + integrity sha512-6S3jpun1eEbAxq7TdjLotAsl4WpQI9DxfkycRcKrjhQYzU87qpXdknpBg/e+TdcMehqGnLFi7tnFUBR02Vq6wg== dependencies: "@babel/helper-create-regexp-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" @@ -545,12 +621,14 @@ "@babel/plugin-transform-duplicate-keys@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.18.9.tgz" + integrity sha512-d2bmXCtZXYc59/0SanQKbiWINadaJXqtvIQIzd4+hNwkWBgyCd5F/2t1kXoUdvPMrxzPvhK6EMQRROxsue+mfw== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/plugin-transform-exponentiation-operator@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.18.6.tgz" + integrity sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw== dependencies: "@babel/helper-builder-binary-assignment-operator-visitor" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" @@ -558,12 +636,14 @@ "@babel/plugin-transform-for-of@^7.18.8": version "7.18.8" resolved "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.18.8.tgz" + integrity sha512-yEfTRnjuskWYo0k1mHUqrVWaZwrdq8AYbfrpqULOJOaucGSp4mNMVps+YtA8byoevxS/urwU75vyhQIxcCgiBQ== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-transform-function-name@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.18.9.tgz" + integrity sha512-WvIBoRPaJQ5yVHzcnJFor7oS5Ls0PYixlTYE63lCj2RtdQEl15M68FXQlxnG6wdraJIXRdR7KI+hQ7q/9QjrCQ== dependencies: "@babel/helper-compilation-targets" "^7.18.9" "@babel/helper-function-name" "^7.18.9" @@ -572,18 +652,21 @@ "@babel/plugin-transform-literals@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.18.9.tgz" + integrity sha512-IFQDSRoTPnrAIrI5zoZv73IFeZu2dhu6irxQjY9rNjTT53VmKg9fenjvoiOWOkJ6mm4jKVPtdMzBY98Fp4Z4cg== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/plugin-transform-member-expression-literals@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.18.6.tgz" + integrity sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-transform-modules-amd@^7.19.6": version "7.19.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.19.6.tgz" + integrity sha512-uG3od2mXvAtIFQIh0xrpLH6r5fpSQN04gIVovl+ODLdUMANokxQLZnPBHcjmv3GxRjnqwLuHvppjjcelqUFZvg== dependencies: "@babel/helper-module-transforms" "^7.19.6" "@babel/helper-plugin-utils" "^7.19.0" @@ -591,6 +674,7 @@ "@babel/plugin-transform-modules-commonjs@^7.19.6": version "7.19.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.19.6.tgz" + integrity sha512-8PIa1ym4XRTKuSsOUXqDG0YaOlEuTVvHMe5JCfgBMOtHvJKw/4NGovEGN33viISshG/rZNVrACiBmPQLvWN8xQ== dependencies: "@babel/helper-module-transforms" "^7.19.6" "@babel/helper-plugin-utils" "^7.19.0" @@ -599,6 +683,7 @@ "@babel/plugin-transform-modules-systemjs@^7.19.6": version "7.19.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.19.6.tgz" + integrity sha512-fqGLBepcc3kErfR9R3DnVpURmckXP7gj7bAlrTQyBxrigFqszZCkFkcoxzCp2v32XmwXLvbw+8Yq9/b+QqksjQ== dependencies: "@babel/helper-hoist-variables" "^7.18.6" "@babel/helper-module-transforms" "^7.19.6" @@ -608,6 +693,7 @@ "@babel/plugin-transform-modules-umd@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.18.6.tgz" + integrity sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ== dependencies: "@babel/helper-module-transforms" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" @@ -615,6 +701,7 @@ "@babel/plugin-transform-named-capturing-groups-regex@^7.19.1": version "7.20.5" resolved "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.20.5.tgz" + integrity sha512-mOW4tTzi5iTLnw+78iEq3gr8Aoq4WNRGpmSlrogqaiCBoR1HFhpU4JkpQFOHfeYx3ReVIFWOQJS4aZBRvuZ6mA== dependencies: "@babel/helper-create-regexp-features-plugin" "^7.20.5" "@babel/helper-plugin-utils" "^7.20.2" @@ -622,12 +709,14 @@ "@babel/plugin-transform-new-target@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.18.6.tgz" + integrity sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-transform-object-super@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.18.6.tgz" + integrity sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/helper-replace-supers" "^7.18.6" @@ -635,18 +724,21 @@ "@babel/plugin-transform-parameters@^7.20.1": version "7.20.5" resolved "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.20.5.tgz" + integrity sha512-h7plkOmcndIUWXZFLgpbrh2+fXAi47zcUX7IrOQuZdLD0I0KvjJ6cvo3BEcAOsDOcZhVKGJqv07mkSqK0y2isQ== dependencies: "@babel/helper-plugin-utils" "^7.20.2" "@babel/plugin-transform-property-literals@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.18.6.tgz" + integrity sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-transform-regenerator@^7.18.6": version "7.20.5" resolved "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.20.5.tgz" + integrity sha512-kW/oO7HPBtntbsahzQ0qSE3tFvkFwnbozz3NWFhLGqH75vLEg+sCGngLlhVkePlCs3Jv0dBBHDzCHxNiFAQKCQ== dependencies: "@babel/helper-plugin-utils" "^7.20.2" regenerator-transform "^0.15.1" @@ -654,12 +746,14 @@ "@babel/plugin-transform-reserved-words@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.18.6.tgz" + integrity sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-transform-runtime@^7.12.15": version "7.19.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.19.6.tgz" + integrity sha512-PRH37lz4JU156lYFW1p8OxE5i7d6Sl/zV58ooyr+q1J1lnQPyg5tIiXlIwNVhJaY4W3TmOtdc8jqdXQcB1v5Yw== dependencies: "@babel/helper-module-imports" "^7.18.6" "@babel/helper-plugin-utils" "^7.19.0" @@ -671,12 +765,14 @@ "@babel/plugin-transform-shorthand-properties@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.18.6.tgz" + integrity sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-transform-spread@^7.19.0": version "7.19.0" resolved "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.19.0.tgz" + integrity sha512-RsuMk7j6n+r752EtzyScnWkQyuJdli6LdO5Klv8Yx0OfPVTcQkIUfS8clx5e9yHXzlnhOZF3CbQ8C2uP5j074w== dependencies: "@babel/helper-plugin-utils" "^7.19.0" "@babel/helper-skip-transparent-expression-wrappers" "^7.18.9" @@ -684,30 +780,35 @@ "@babel/plugin-transform-sticky-regex@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.18.6.tgz" + integrity sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q== dependencies: "@babel/helper-plugin-utils" "^7.18.6" "@babel/plugin-transform-template-literals@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.18.9.tgz" + integrity sha512-S8cOWfT82gTezpYOiVaGHrCbhlHgKhQt8XH5ES46P2XWmX92yisoZywf5km75wv5sYcXDUCLMmMxOLCtthDgMA== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/plugin-transform-typeof-symbol@^7.18.9": version "7.18.9" resolved "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.18.9.tgz" + integrity sha512-SRfwTtF11G2aemAZWivL7PD+C9z52v9EvMqH9BuYbabyPuKUvSWks3oCg6041pT925L4zVFqaVBeECwsmlguEw== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/plugin-transform-unicode-escapes@^7.18.10": version "7.18.10" resolved "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.18.10.tgz" + integrity sha512-kKAdAI+YzPgGY/ftStBFXTI1LZFju38rYThnfMykS+IXy8BVx+res7s2fxf1l8I35DV2T97ezo6+SGrXz6B3iQ== dependencies: "@babel/helper-plugin-utils" "^7.18.9" "@babel/plugin-transform-unicode-regex@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.18.6.tgz" + integrity sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA== dependencies: "@babel/helper-create-regexp-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" @@ -715,6 +816,7 @@ "@babel/preset-env@^7.12.16": version "7.20.2" resolved "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.20.2.tgz" + integrity sha512-1G0efQEWR1EHkKvKHqbG+IN/QdgwfByUpM5V5QroDzGV2t3S/WXNQd693cHiHTlCFMpr9B6FkPFXDA2lQcKoDg== dependencies: "@babel/compat-data" "^7.20.1" "@babel/helper-compilation-targets" "^7.20.0" @@ -795,6 +897,7 @@ "@babel/preset-modules@^0.1.5": version "0.1.5" resolved "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.5.tgz" + integrity sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA== dependencies: "@babel/helper-plugin-utils" "^7.0.0" "@babel/plugin-proposal-unicode-property-regex" "^7.4.4" @@ -805,12 +908,14 @@ "@babel/runtime@^7.12.13", "@babel/runtime@^7.8.4": version "7.20.6" resolved "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.6.tgz" + integrity sha512-Q+8MqP7TiHMWzSfwiJwXCjyf4GYA4Dgw3emg/7xmwsdLJOZUp+nMqcOwOzzYheuM1rhDu8FSj2l0aoMygEuXuA== dependencies: regenerator-runtime "^0.13.11" "@babel/template@^7.0.0", "@babel/template@^7.16.7", "@babel/template@^7.18.10": version "7.18.10" resolved "https://registry.npmjs.org/@babel/template/-/template-7.18.10.tgz" + integrity sha512-TI+rCtooWHr3QJ27kJxfjutghu44DLnasDMwpDqCXVTal9RLp3RSYNh4NdBrRP2cQAoG9A8juOQl6P6oZG4JxA== dependencies: "@babel/code-frame" "^7.18.6" "@babel/parser" "^7.18.10" @@ -819,6 +924,7 @@ "@babel/traverse@^7.0.0", "@babel/traverse@^7.17.10", "@babel/traverse@^7.17.9", "@babel/traverse@^7.19.1", "@babel/traverse@^7.20.1", "@babel/traverse@^7.20.5", "@babel/traverse@^7.7.0": version "7.20.5" resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.20.5.tgz" + integrity sha512-WM5ZNN3JITQIq9tFZaw1ojLU3WgWdtkxnhM1AegMS+PvHjkM5IXjmYEGY7yukz5XS4sJyEf2VzWjI8uAavhxBQ== dependencies: "@babel/code-frame" "^7.18.6" "@babel/generator" "^7.20.5" @@ -834,6 +940,7 @@ "@babel/types@^7.0.0", "@babel/types@^7.17.0", "@babel/types@^7.17.10", "@babel/types@^7.18.10", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.19.0", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5", "@babel/types@^7.4.4", "@babel/types@^7.7.0": version "7.20.5" resolved "https://registry.npmjs.org/@babel/types/-/types-7.20.5.tgz" + integrity sha512-c9fst/h2/dcF7H+MJKZ2T0KjEQ8hY/BNnDk/H3XY8C4Aw/eWQXWn/lWntHF9ooUBnGmEvbfGrTgLWc+um0YDUg== dependencies: "@babel/helper-string-parser" "^7.19.4" "@babel/helper-validator-identifier" "^7.19.1" @@ -842,16 +949,19 @@ "@hapi/hoek@^9.0.0": version "9.3.0" resolved "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz" + integrity sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ== "@hapi/topo@^5.0.0": version "5.1.0" resolved "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz" + integrity sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg== dependencies: "@hapi/hoek" "^9.0.0" "@jridgewell/gen-mapping@^0.1.0": version "0.1.1" resolved "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz" + integrity sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w== dependencies: "@jridgewell/set-array" "^1.0.0" "@jridgewell/sourcemap-codec" "^1.4.10" @@ -859,6 +969,7 @@ "@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2": version "0.3.2" resolved "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz" + integrity sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A== dependencies: "@jridgewell/set-array" "^1.0.1" "@jridgewell/sourcemap-codec" "^1.4.10" @@ -867,14 +978,17 @@ "@jridgewell/resolve-uri@3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz" + integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== "@jridgewell/set-array@^1.0.0", "@jridgewell/set-array@^1.0.1": version "1.1.0" resolved "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.0.tgz" + integrity sha512-SfJxIxNVYLTsKwzB3MoOQ1yxf4w/E6MdkvTgrgAt1bfxjSrLUoHMKrDOykwN14q65waezZIdqDneUIPh4/sKxg== "@jridgewell/source-map@^0.3.2": version "0.3.2" resolved "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz" + integrity sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw== dependencies: "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" @@ -882,10 +996,12 @@ "@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10": version "1.4.14" resolved "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz" + integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== "@jridgewell/trace-mapping@^0.3.14", "@jridgewell/trace-mapping@^0.3.9": version "0.3.17" resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz" + integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g== dependencies: "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" @@ -893,16 +1009,19 @@ "@leichtgewicht/ip-codec@^2.0.1": version "2.0.4" resolved "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz" + integrity sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A== "@node-ipc/js-queue@2.0.3": version "2.0.3" resolved "https://registry.npmjs.org/@node-ipc/js-queue/-/js-queue-2.0.3.tgz" + integrity sha512-fL1wpr8hhD5gT2dA1qifeVaoDFlQR5es8tFuKqjHX+kdOtdNHnxkVZbtIrR2rxnMFvehkjaZRNV2H/gPXlb0hw== dependencies: easy-stack "1.0.1" "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz" + integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== dependencies: "@nodelib/fs.stat" "2.0.5" run-parallel "^1.1.9" @@ -910,10 +1029,12 @@ "@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": version "2.0.5" resolved "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz" + integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== "@nodelib/fs.walk@^1.2.3": version "1.2.8" resolved "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz" + integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== dependencies: "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" @@ -921,6 +1042,7 @@ "@nuxt/opencollective@^0.3.2": version "0.3.2" resolved "https://registry.npmjs.org/@nuxt/opencollective/-/opencollective-0.3.2.tgz" + integrity sha512-XG7rUdXG9fcafu9KTDIYjJSkRO38EwjlKYIb5TQ/0WDbiTUTtUtgncMscKOYzfsY86kGs05pAuMOR+3Fi0aN3A== dependencies: chalk "^4.1.0" consola "^2.15.0" @@ -929,28 +1051,34 @@ "@polka/url@^1.0.0-next.20": version "1.0.0-next.21" resolved "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.21.tgz" + integrity sha512-a5Sab1C4/icpTZVzZc5Ghpz88yQtGOyNqYXcZgOssB2uuAr+wF/MvN6bgtW32q7HHrvBki+BsZ0OuNv6EV3K9g== "@sideway/address@^4.1.3": version "4.1.4" resolved "https://registry.npmjs.org/@sideway/address/-/address-4.1.4.tgz" + integrity sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw== dependencies: "@hapi/hoek" "^9.0.0" "@sideway/formula@^3.0.0": version "3.0.1" - resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.1.tgz#80fcbcbaf7ce031e0ef2dd29b1bfc7c3f583611f" + resolved "https://registry.npmjs.org/@sideway/formula/-/formula-3.0.1.tgz" + integrity sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg== "@sideway/pinpoint@^2.0.0": version "2.0.0" resolved "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz" + integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== "@socket.io/component-emitter@~3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz" + integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg== "@soda/friendly-errors-webpack-plugin@^1.8.0": version "1.8.1" resolved "https://registry.npmjs.org/@soda/friendly-errors-webpack-plugin/-/friendly-errors-webpack-plugin-1.8.1.tgz" + integrity sha512-h2ooWqP8XuFqTXT+NyAFbrArzfQA7R6HTezADrvD9Re8fxMLTPPniLdqVTdDaO0eIoLaAwKT+d6w+5GeTk7Vbg== dependencies: chalk "^3.0.0" error-stack-parser "^2.0.6" @@ -960,14 +1088,17 @@ "@soda/get-current-script@^1.0.2": version "1.0.2" resolved "https://registry.npmjs.org/@soda/get-current-script/-/get-current-script-1.0.2.tgz" + integrity sha512-T7VNNlYVM1SgQ+VsMYhnDkcGmWhQdL0bDyGm5TlQ3GBXnJscEClUUOKduWTmm2zCnvNLC1hc3JpuXjs/nFOc5w== "@trysound/sax@0.2.0": version "0.2.0" resolved "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz" + integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA== "@types/body-parser@*": version "1.19.2" resolved "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz" + integrity sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g== dependencies: "@types/connect" "*" "@types/node" "*" @@ -975,12 +1106,14 @@ "@types/bonjour@^3.5.9": version "3.5.10" resolved "https://registry.npmjs.org/@types/bonjour/-/bonjour-3.5.10.tgz" + integrity sha512-p7ienRMiS41Nu2/igbJxxLDWrSZ0WxM8UQgCeO9KhoVF7cOVFkrKsiDr1EsJIla8vV3oEEjGcz11jc5yimhzZw== dependencies: "@types/node" "*" "@types/connect-history-api-fallback@^1.3.5": version "1.3.5" resolved "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.3.5.tgz" + integrity sha512-h8QJa8xSb1WD4fpKBDcATDNGXghFj6/3GRWG6dhmRcu0RX1Ubasur2Uvx5aeEwlf0MwblEC2bMzzMQntxnw/Cw== dependencies: "@types/express-serve-static-core" "*" "@types/node" "*" @@ -988,12 +1121,14 @@ "@types/connect@*": version "3.4.35" resolved "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz" + integrity sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ== dependencies: "@types/node" "*" "@types/eslint-scope@^3.7.3": version "3.7.4" resolved "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.4.tgz" + integrity sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA== dependencies: "@types/eslint" "*" "@types/estree" "*" @@ -1001,6 +1136,7 @@ "@types/eslint@*": version "8.4.10" resolved "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.10.tgz" + integrity sha512-Sl/HOqN8NKPmhWo2VBEPm0nvHnu2LL3v9vKo8MEq0EtbJ4eVzGPl41VNPvn5E1i5poMk4/XD8UriLHpJvEP/Nw== dependencies: "@types/estree" "*" "@types/json-schema" "*" @@ -1008,10 +1144,12 @@ "@types/estree@*", "@types/estree@^0.0.51": version "0.0.51" resolved "https://registry.npmjs.org/@types/estree/-/estree-0.0.51.tgz" + integrity sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ== "@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.18": version "4.17.31" resolved "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz" + integrity sha512-DxMhY+NAsTwMMFHBTtJFNp5qiHKJ7TeqOo23zVEM9alT1Ml27Q3xcTH0xwxn7Q0BbMcVEJOs/7aQtUWupUQN3Q== dependencies: "@types/node" "*" "@types/qs" "*" @@ -1020,6 +1158,7 @@ "@types/express@*", "@types/express@^4.17.13": version "4.17.14" resolved "https://registry.npmjs.org/@types/express/-/express-4.17.14.tgz" + integrity sha512-TEbt+vaPFQ+xpxFLFssxUDXj5cWCxZJjIcB7Yg0k0GMHGtgtQgpvx/MUQUeAkNbA9AAGrwkAsoeItdTgS7FMyg== dependencies: "@types/body-parser" "*" "@types/express-serve-static-core" "^4.17.18" @@ -1029,58 +1168,71 @@ "@types/html-minifier-terser@^6.0.0": version "6.1.0" resolved "https://registry.npmjs.org/@types/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz" + integrity sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg== "@types/http-proxy@^1.17.8": version "1.17.9" resolved "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.9.tgz" + integrity sha512-QsbSjA/fSk7xB+UXlCT3wHBy5ai9wOcNDWwZAtud+jXhwOM3l+EYZh8Lng4+/6n8uar0J7xILzqftJdJ/Wdfkw== dependencies: "@types/node" "*" "@types/json-schema@*", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": version "7.0.11" resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz" + integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== "@types/mime@*": version "3.0.1" resolved "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz" + integrity sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA== "@types/minimist@^1.2.0": version "1.2.2" resolved "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz" + integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== "@types/node@*": version "18.11.10" resolved "https://registry.npmjs.org/@types/node/-/node-18.11.10.tgz" + integrity sha512-juG3RWMBOqcOuXC643OAdSA525V44cVgGV6dUDuiFtss+8Fk5x1hI93Rsld43VeJVIeqlP9I7Fn9/qaVqoEAuQ== "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz" + integrity sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw== "@types/parse-json@^4.0.0": version "4.0.0" resolved "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz" + integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== "@types/qs@*": version "6.9.7" resolved "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz" + integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== "@types/range-parser@*": version "1.2.4" resolved "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz" + integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== "@types/retry@0.12.0": version "0.12.0" resolved "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz" + integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA== "@types/serve-index@^1.9.1": version "1.9.1" resolved "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.1.tgz" + integrity sha512-d/Hs3nWDxNL2xAczmOVZNj92YZCS6RGxfBPjKzuu/XirCgXdpKEb88dYNbrYGint6IVWLNP+yonwVAuRC0T2Dg== dependencies: "@types/express" "*" "@types/serve-static@*", "@types/serve-static@^1.13.10": version "1.15.0" resolved "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz" + integrity sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg== dependencies: "@types/mime" "*" "@types/node" "*" @@ -1088,26 +1240,31 @@ "@types/sockjs@^0.3.33": version "0.3.33" resolved "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.33.tgz" + integrity sha512-f0KEEe05NvUnat+boPTZ0dgaLZ4SfSouXUgv5noUiefG2ajgKjmETo9ZJyuqsl7dfl2aHlLJUiki6B4ZYldiiw== dependencies: "@types/node" "*" "@types/ws@^8.5.1": version "8.5.3" resolved "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz" + integrity sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w== dependencies: "@types/node" "*" "@vue/babel-helper-vue-jsx-merge-props@^1.4.0": version "1.4.0" resolved "https://registry.npmjs.org/@vue/babel-helper-vue-jsx-merge-props/-/babel-helper-vue-jsx-merge-props-1.4.0.tgz" + integrity sha512-JkqXfCkUDp4PIlFdDQ0TdXoIejMtTHP67/pvxlgeY+u5k3LEdKuWZ3LK6xkxo52uDoABIVyRwqVkfLQJhk7VBA== "@vue/babel-helper-vue-transform-on@^1.0.2": version "1.0.2" resolved "https://registry.npmjs.org/@vue/babel-helper-vue-transform-on/-/babel-helper-vue-transform-on-1.0.2.tgz" + integrity sha512-hz4R8tS5jMn8lDq6iD+yWL6XNB699pGIVLk7WSJnn1dbpjaazsjZQkieJoRX6gW5zpYSCFqQ7jUquPNY65tQYA== "@vue/babel-plugin-jsx@^1.0.3": version "1.1.1" resolved "https://registry.npmjs.org/@vue/babel-plugin-jsx/-/babel-plugin-jsx-1.1.1.tgz" + integrity sha512-j2uVfZjnB5+zkcbc/zsOc0fSNGCMMjaEXP52wdwdIfn0qjFfEYpYZBFKFg+HHnQeJCVrjOeO0YxgaL7DMrym9w== dependencies: "@babel/helper-module-imports" "^7.0.0" "@babel/plugin-syntax-jsx" "^7.0.0" @@ -1122,6 +1279,7 @@ "@vue/babel-plugin-transform-vue-jsx@^1.4.0": version "1.4.0" resolved "https://registry.npmjs.org/@vue/babel-plugin-transform-vue-jsx/-/babel-plugin-transform-vue-jsx-1.4.0.tgz" + integrity sha512-Fmastxw4MMx0vlgLS4XBX0XiBbUFzoMGeVXuMV08wyOfXdikAFqBTuYPR0tlk+XskL19EzHc39SgjrPGY23JnA== dependencies: "@babel/helper-module-imports" "^7.0.0" "@babel/plugin-syntax-jsx" "^7.2.0" @@ -1133,6 +1291,7 @@ "@vue/babel-preset-app@^5.0.8": version "5.0.8" resolved "https://registry.npmjs.org/@vue/babel-preset-app/-/babel-preset-app-5.0.8.tgz" + integrity sha512-yl+5qhpjd8e1G4cMXfORkkBlvtPCIgmRf3IYCWYDKIQ7m+PPa5iTm4feiNmCMD6yGqQWMhhK/7M3oWGL9boKwg== dependencies: "@babel/core" "^7.12.16" "@babel/helper-compilation-targets" "^7.12.16" @@ -1154,6 +1313,7 @@ "@vue/babel-preset-jsx@^1.1.2": version "1.4.0" resolved "https://registry.npmjs.org/@vue/babel-preset-jsx/-/babel-preset-jsx-1.4.0.tgz" + integrity sha512-QmfRpssBOPZWL5xw7fOuHNifCQcNQC1PrOo/4fu6xlhlKJJKSA3HqX92Nvgyx8fqHZTUGMPHmFA+IDqwXlqkSA== dependencies: "@vue/babel-helper-vue-jsx-merge-props" "^1.4.0" "@vue/babel-plugin-transform-vue-jsx" "^1.4.0" @@ -1167,30 +1327,35 @@ "@vue/babel-sugar-composition-api-inject-h@^1.4.0": version "1.4.0" resolved "https://registry.npmjs.org/@vue/babel-sugar-composition-api-inject-h/-/babel-sugar-composition-api-inject-h-1.4.0.tgz" + integrity sha512-VQq6zEddJHctnG4w3TfmlVp5FzDavUSut/DwR0xVoe/mJKXyMcsIibL42wPntozITEoY90aBV0/1d2KjxHU52g== dependencies: "@babel/plugin-syntax-jsx" "^7.2.0" "@vue/babel-sugar-composition-api-render-instance@^1.4.0": version "1.4.0" resolved "https://registry.npmjs.org/@vue/babel-sugar-composition-api-render-instance/-/babel-sugar-composition-api-render-instance-1.4.0.tgz" + integrity sha512-6ZDAzcxvy7VcnCjNdHJ59mwK02ZFuP5CnucloidqlZwVQv5CQLijc3lGpR7MD3TWFi78J7+a8J56YxbCtHgT9Q== dependencies: "@babel/plugin-syntax-jsx" "^7.2.0" "@vue/babel-sugar-functional-vue@^1.4.0": version "1.4.0" resolved "https://registry.npmjs.org/@vue/babel-sugar-functional-vue/-/babel-sugar-functional-vue-1.4.0.tgz" + integrity sha512-lTEB4WUFNzYt2In6JsoF9sAYVTo84wC4e+PoZWSgM6FUtqRJz7wMylaEhSRgG71YF+wfLD6cc9nqVeXN2rwBvw== dependencies: "@babel/plugin-syntax-jsx" "^7.2.0" "@vue/babel-sugar-inject-h@^1.4.0": version "1.4.0" resolved "https://registry.npmjs.org/@vue/babel-sugar-inject-h/-/babel-sugar-inject-h-1.4.0.tgz" + integrity sha512-muwWrPKli77uO2fFM7eA3G1lAGnERuSz2NgAxuOLzrsTlQl8W4G+wwbM4nB6iewlKbwKRae3nL03UaF5ffAPMA== dependencies: "@babel/plugin-syntax-jsx" "^7.2.0" "@vue/babel-sugar-v-model@^1.4.0": version "1.4.0" resolved "https://registry.npmjs.org/@vue/babel-sugar-v-model/-/babel-sugar-v-model-1.4.0.tgz" + integrity sha512-0t4HGgXb7WHYLBciZzN5s0Hzqan4Ue+p/3FdQdcaHAb7s5D9WZFGoSxEZHrR1TFVZlAPu1bejTKGeAzaaG3NCQ== dependencies: "@babel/plugin-syntax-jsx" "^7.2.0" "@vue/babel-helper-vue-jsx-merge-props" "^1.4.0" @@ -1202,6 +1367,7 @@ "@vue/babel-sugar-v-on@^1.4.0": version "1.4.0" resolved "https://registry.npmjs.org/@vue/babel-sugar-v-on/-/babel-sugar-v-on-1.4.0.tgz" + integrity sha512-m+zud4wKLzSKgQrWwhqRObWzmTuyzl6vOP7024lrpeJM4x2UhQtRDLgYjXAw9xBXjCwS0pP9kXjg91F9ZNo9JA== dependencies: "@babel/plugin-syntax-jsx" "^7.2.0" "@vue/babel-plugin-transform-vue-jsx" "^1.4.0" @@ -1210,10 +1376,12 @@ "@vue/cli-overlay@^5.0.8": version "5.0.8" resolved "https://registry.npmjs.org/@vue/cli-overlay/-/cli-overlay-5.0.8.tgz" + integrity sha512-KmtievE/B4kcXp6SuM2gzsnSd8WebkQpg3XaB6GmFh1BJGRqa1UiW9up7L/Q67uOdTigHxr5Ar2lZms4RcDjwQ== "@vue/cli-plugin-babel@~5.0.8": version "5.0.8" resolved "https://registry.npmjs.org/@vue/cli-plugin-babel/-/cli-plugin-babel-5.0.8.tgz" + integrity sha512-a4qqkml3FAJ3auqB2kN2EMPocb/iu0ykeELwed+9B1c1nQ1HKgslKMHMPavYx3Cd/QAx2mBD4hwKBqZXEI/CsQ== dependencies: "@babel/core" "^7.12.16" "@vue/babel-preset-app" "^5.0.8" @@ -1225,24 +1393,27 @@ "@vue/cli-plugin-router@~5.0.8": version "5.0.8" resolved "https://registry.npmjs.org/@vue/cli-plugin-router/-/cli-plugin-router-5.0.8.tgz" + integrity sha512-Gmv4dsGdAsWPqVijz3Ux2OS2HkMrWi1ENj2cYL75nUeL+Xj5HEstSqdtfZ0b1q9NCce+BFB6QnHfTBXc/fCvMg== dependencies: "@vue/cli-shared-utils" "^5.0.8" "@vue/cli-plugin-vuex@~5.0.8": version "5.0.8" resolved "https://registry.npmjs.org/@vue/cli-plugin-vuex/-/cli-plugin-vuex-5.0.8.tgz" + integrity sha512-HSYWPqrunRE5ZZs8kVwiY6oWcn95qf/OQabwLfprhdpFWAGtLStShjsGED2aDpSSeGAskQETrtR/5h7VqgIlBA== "@vue/cli-service@~5.0.8": version "5.0.8" resolved "https://registry.npmjs.org/@vue/cli-service/-/cli-service-5.0.8.tgz" + integrity sha512-nV7tYQLe7YsTtzFrfOMIHc5N2hp5lHG2rpYr0aNja9rNljdgcPZLyQRb2YRivTHqTv7lI962UXFURcpStHgyFw== dependencies: "@babel/helper-compilation-targets" "^7.12.16" "@soda/friendly-errors-webpack-plugin" "^1.8.0" "@soda/get-current-script" "^1.0.2" "@types/minimist" "^1.2.0" "@vue/cli-overlay" "^5.0.8" - "@vue/cli-plugin-router" "~5.0.8" - "@vue/cli-plugin-vuex" "~5.0.8" + "@vue/cli-plugin-router" "^5.0.8" + "@vue/cli-plugin-vuex" "^5.0.8" "@vue/cli-shared-utils" "^5.0.8" "@vue/component-compiler-utils" "^3.3.0" "@vue/vue-loader-v15" "npm:vue-loader@^15.9.7" @@ -1295,6 +1466,7 @@ "@vue/cli-shared-utils@^5.0.8": version "5.0.8" resolved "https://registry.npmjs.org/@vue/cli-shared-utils/-/cli-shared-utils-5.0.8.tgz" + integrity sha512-uK2YB7bBVuQhjOJF+O52P9yFMXeJVj7ozqJkwYE9PlMHL1LMHjtCYm4cSdOebuPzyP+/9p0BimM/OqxsevIopQ== dependencies: "@achrinza/node-ipc" "^9.2.5" chalk "^4.1.2" @@ -1312,6 +1484,7 @@ "@vue/component-compiler-utils@^3.1.0", "@vue/component-compiler-utils@^3.3.0": version "3.3.0" resolved "https://registry.npmjs.org/@vue/component-compiler-utils/-/component-compiler-utils-3.3.0.tgz" + integrity sha512-97sfH2mYNU+2PzGrmK2haqffDpVASuib9/w2/noxiFi31Z54hW+q3izKQXXQZSNhtiUpAI36uSuYepeBe4wpHQ== dependencies: consolidate "^0.15.1" hash-sum "^1.0.2" @@ -1327,6 +1500,7 @@ "@vue/vue-loader-v15@npm:vue-loader@^15.9.7": version "15.10.1" resolved "https://registry.npmjs.org/vue-loader/-/vue-loader-15.10.1.tgz" + integrity sha512-SaPHK1A01VrNthlix6h1hq4uJu7S/z0kdLUb6klubo738NeQoLbS6V9/d8Pv19tU0XdQKju3D1HSKuI8wJ5wMA== dependencies: "@vue/component-compiler-utils" "^3.1.0" hash-sum "^1.0.2" @@ -1337,10 +1511,12 @@ "@vue/web-component-wrapper@^1.3.0": version "1.3.0" resolved "https://registry.npmjs.org/@vue/web-component-wrapper/-/web-component-wrapper-1.3.0.tgz" + integrity sha512-Iu8Tbg3f+emIIMmI2ycSI8QcEuAUgPTgHwesDU1eKMLE4YC/c/sFbGc70QgMq31ijRftV0R7vCm9co6rldCeOA== "@webassemblyjs/ast@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz" + integrity sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw== dependencies: "@webassemblyjs/helper-numbers" "1.11.1" "@webassemblyjs/helper-wasm-bytecode" "1.11.1" @@ -1348,18 +1524,22 @@ "@webassemblyjs/floating-point-hex-parser@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz" + integrity sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ== "@webassemblyjs/helper-api-error@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz" + integrity sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg== "@webassemblyjs/helper-buffer@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz" + integrity sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA== "@webassemblyjs/helper-numbers@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz" + integrity sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ== dependencies: "@webassemblyjs/floating-point-hex-parser" "1.11.1" "@webassemblyjs/helper-api-error" "1.11.1" @@ -1368,10 +1548,12 @@ "@webassemblyjs/helper-wasm-bytecode@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz" + integrity sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q== "@webassemblyjs/helper-wasm-section@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz" + integrity sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg== dependencies: "@webassemblyjs/ast" "1.11.1" "@webassemblyjs/helper-buffer" "1.11.1" @@ -1381,22 +1563,26 @@ "@webassemblyjs/ieee754@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz" + integrity sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ== dependencies: "@xtuc/ieee754" "^1.2.0" "@webassemblyjs/leb128@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz" + integrity sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw== dependencies: "@xtuc/long" "4.2.2" "@webassemblyjs/utf8@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz" + integrity sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ== "@webassemblyjs/wasm-edit@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz" + integrity sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA== dependencies: "@webassemblyjs/ast" "1.11.1" "@webassemblyjs/helper-buffer" "1.11.1" @@ -1410,6 +1596,7 @@ "@webassemblyjs/wasm-gen@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz" + integrity sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA== dependencies: "@webassemblyjs/ast" "1.11.1" "@webassemblyjs/helper-wasm-bytecode" "1.11.1" @@ -1420,6 +1607,7 @@ "@webassemblyjs/wasm-opt@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz" + integrity sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw== dependencies: "@webassemblyjs/ast" "1.11.1" "@webassemblyjs/helper-buffer" "1.11.1" @@ -1429,6 +1617,7 @@ "@webassemblyjs/wasm-parser@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz" + integrity sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA== dependencies: "@webassemblyjs/ast" "1.11.1" "@webassemblyjs/helper-api-error" "1.11.1" @@ -1440,6 +1629,7 @@ "@webassemblyjs/wast-printer@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz" + integrity sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg== dependencies: "@webassemblyjs/ast" "1.11.1" "@xtuc/long" "4.2.2" @@ -1447,14 +1637,17 @@ "@xtuc/ieee754@^1.2.0": version "1.2.0" resolved "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz" + integrity sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA== "@xtuc/long@4.2.2": version "4.2.2" resolved "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz" + integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8: version "1.3.8" resolved "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz" + integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== dependencies: mime-types "~2.1.34" negotiator "0.6.3" @@ -1462,38 +1655,46 @@ accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8: acorn-import-assertions@^1.7.6: version "1.8.0" resolved "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz" + integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw== acorn-walk@^8.0.0, acorn-walk@^8.0.2: version "8.2.0" resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz" + integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== acorn@^8.0.4, acorn@^8.0.5, acorn@^8.5.0, acorn@^8.7.1: version "8.8.1" resolved "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz" + integrity sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA== address@^1.1.2: version "1.2.0" resolved "https://registry.npmjs.org/address/-/address-1.2.0.tgz" + integrity sha512-tNEZYz5G/zYunxFm7sfhAxkXEuLj3K6BKwv6ZURlsF6yiUQ65z0Q2wZW9L5cPUl9ocofGvXOdFYbFHp0+6MOig== ajv-formats@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz" + integrity sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA== dependencies: ajv "^8.0.0" ajv-keywords@^3.5.2: version "3.5.2" resolved "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz" + integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== ajv-keywords@^5.0.0: version "5.1.0" resolved "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz" + integrity sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw== dependencies: fast-deep-equal "^3.1.3" ajv@^6.12.4, ajv@^6.12.5: version "6.12.6" resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz" + integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== dependencies: fast-deep-equal "^3.1.1" fast-json-stable-stringify "^2.0.0" @@ -1503,6 +1704,7 @@ ajv@^6.12.4, ajv@^6.12.5: ajv@^8.0.0, ajv@^8.8.0: version "8.11.2" resolved "https://registry.npmjs.org/ajv/-/ajv-8.11.2.tgz" + integrity sha512-E4bfmKAhGiSTvMfL1Myyycaub+cUEU2/IvpylXkUu7CHBkBj1f/ikdzbD7YQ6FKUbixDxeYvB/xY4fvyroDlQg== dependencies: fast-deep-equal "^3.1.1" json-schema-traverse "^1.0.0" @@ -1512,38 +1714,46 @@ ajv@^8.0.0, ajv@^8.8.0: ansi-escapes@^3.0.0: version "3.2.0" resolved "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz" + integrity sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ== ansi-html-community@^0.0.8: version "0.0.8" resolved "https://registry.npmjs.org/ansi-html-community/-/ansi-html-community-0.0.8.tgz" + integrity sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw== ansi-regex@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz" + integrity sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw== ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz" + integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== dependencies: color-convert "^1.9.0" ansi-styles@^4.0.0, ansi-styles@^4.1.0: version "4.3.0" resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== dependencies: color-convert "^2.0.1" any-promise@^1.0.0: version "1.3.0" resolved "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz" + integrity sha1-q8av7tzqUugJzcA3au0845Y10X8= anymatch@~3.1.2: version "3.1.3" resolved "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz" + integrity sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw== dependencies: normalize-path "^3.0.0" picomatch "^2.0.4" @@ -1551,36 +1761,44 @@ anymatch@~3.1.2: arch@^2.1.1: version "2.2.0" resolved "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz" + integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== array-flatten@1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" + integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== array-flatten@^2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz" + integrity sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ== array-union@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz" + integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== async@^2.6.4: version "2.6.4" resolved "https://registry.npmjs.org/async/-/async-2.6.4.tgz" + integrity sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA== dependencies: lodash "^4.17.14" asynckit@^0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz" + integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= at-least-node@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz" + integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== autoprefixer@^10.2.4: version "10.4.13" resolved "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz" + integrity sha512-49vKpMqcZYsJjwotvt4+h/BCjJVnhGwcLpDt5xkcaOG3eLrG/HUYLagrihYsQ+qrIBgIzX1Rw7a6L8I/ZA1Atg== dependencies: browserslist "^4.21.4" caniuse-lite "^1.0.30001426" @@ -1592,6 +1810,7 @@ autoprefixer@^10.2.4: axios@^0.27.2: version "0.27.2" resolved "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz" + integrity sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ== dependencies: follow-redirects "^1.14.9" form-data "^4.0.0" @@ -1599,6 +1818,7 @@ axios@^0.27.2: babel-eslint@^10.1.0: version "10.1.0" resolved "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz" + integrity sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg== dependencies: "@babel/code-frame" "^7.0.0" "@babel/parser" "^7.7.0" @@ -1610,6 +1830,7 @@ babel-eslint@^10.1.0: babel-loader@^8.2.2: version "8.2.5" resolved "https://registry.npmjs.org/babel-loader/-/babel-loader-8.2.5.tgz" + integrity sha512-OSiFfH89LrEMiWd4pLNqGz4CwJDtbs2ZVc+iGu2HrkRfPxId9F2anQj38IxWpmRfsUY0aBZYi1EFcd3mhtRMLQ== dependencies: find-cache-dir "^3.3.1" loader-utils "^2.0.0" @@ -1619,12 +1840,14 @@ babel-loader@^8.2.2: babel-plugin-dynamic-import-node@^2.3.3: version "2.3.3" resolved "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz" + integrity sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ== dependencies: object.assign "^4.1.0" babel-plugin-polyfill-corejs2@^0.3.3: version "0.3.3" resolved "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.3.tgz" + integrity sha512-8hOdmFYFSZhqg2C/JgLUQ+t52o5nirNwaWM2B9LWteozwIvM14VSwdsCAUET10qT+kmySAlseadmfeeSWFCy+Q== dependencies: "@babel/compat-data" "^7.17.7" "@babel/helper-define-polyfill-provider" "^0.3.3" @@ -1633,6 +1856,7 @@ babel-plugin-polyfill-corejs2@^0.3.3: babel-plugin-polyfill-corejs3@^0.6.0: version "0.6.0" resolved "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.6.0.tgz" + integrity sha512-+eHqR6OPcBhJOGgsIar7xoAB1GcSwVUA3XjAd7HJNzOXT4wv6/H7KIdA/Nc60cvUlDbKApmqNvD1B1bzOt4nyA== dependencies: "@babel/helper-define-polyfill-provider" "^0.3.3" core-js-compat "^3.25.1" @@ -1640,32 +1864,39 @@ babel-plugin-polyfill-corejs3@^0.6.0: babel-plugin-polyfill-regenerator@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.4.1.tgz" + integrity sha512-NtQGmyQDXjQqQ+IzRkBVwEOz9lQ4zxAQZgoAYEtU9dJjnl1Oc98qnN7jcp+bE7O7aYzVpavXE3/VKXNzUbh7aw== dependencies: "@babel/helper-define-polyfill-provider" "^0.3.3" balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== base64-js@^1.3.1: version "1.5.1" resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== batch@0.6.1: version "0.6.1" resolved "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz" + integrity sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw== big.js@^5.2.2: version "5.2.2" resolved "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz" + integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ== binary-extensions@^2.0.0: version "2.2.0" resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz" + integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== bl@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz" + integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== dependencies: buffer "^5.5.0" inherits "^2.0.4" @@ -1674,10 +1905,12 @@ bl@^4.1.0: bluebird@^3.1.1: version "3.7.2" resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz" + integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== body-parser@1.20.1: version "1.20.1" resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz" + integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== dependencies: bytes "3.1.2" content-type "~1.0.4" @@ -1695,6 +1928,7 @@ body-parser@1.20.1: bonjour-service@^1.0.11: version "1.0.14" resolved "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.0.14.tgz" + integrity sha512-HIMbgLnk1Vqvs6B4Wq5ep7mxvj9sGz5d1JJyDNSGNIdA/w2MCz6GTjWTdjqOJV1bEPj+6IkxDvWNFKEBxNt4kQ== dependencies: array-flatten "^2.1.2" dns-equal "^1.0.0" @@ -1704,10 +1938,12 @@ bonjour-service@^1.0.11: boolbase@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz" + integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== bootstrap-vue@^2.21.2: version "2.22.0" resolved "https://registry.npmjs.org/bootstrap-vue/-/bootstrap-vue-2.22.0.tgz" + integrity sha512-denjR/ae0K7Jrcqud3TrZWw0p/crtyigeGUNunWQ4t+KFi+7rzJ6j6lx1W5/gpUtSSUgNbWrXcHH4lIWXzXOOQ== dependencies: "@nuxt/opencollective" "^0.3.2" bootstrap "^4.6.1" @@ -1718,10 +1954,12 @@ bootstrap-vue@^2.21.2: bootstrap@^4.5.2, bootstrap@^4.6.1: version "4.6.1" resolved "https://registry.npmjs.org/bootstrap/-/bootstrap-4.6.1.tgz" + integrity sha512-0dj+VgI9Ecom+rvvpNZ4MUZJz8dcX7WCX+eTID9+/8HgOkv3dsRzi8BGeZJCQU6flWQVYxwTQnEZFrmJSEO7og== brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== dependencies: balanced-match "^1.0.0" concat-map "0.0.1" @@ -1729,12 +1967,14 @@ brace-expansion@^1.1.7: braces@^3.0.2, braces@~3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz" + integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== dependencies: fill-range "^7.0.1" browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.16.3, browserslist@^4.16.6, browserslist@^4.21.3, browserslist@^4.21.4: version "4.21.4" resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz" + integrity sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw== dependencies: caniuse-lite "^1.0.30001400" electron-to-chromium "^1.4.251" @@ -1744,10 +1984,12 @@ browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.16.3, browserslist@^4 buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" + integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== buffer@^5.5.0: version "5.7.1" resolved "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz" + integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== dependencies: base64-js "^1.3.1" ieee754 "^1.1.13" @@ -1755,14 +1997,17 @@ buffer@^5.5.0: bytes@3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz" + integrity sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw== bytes@3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz" + integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== call-bind@^1.0.0, call-bind@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== dependencies: function-bind "^1.1.1" get-intrinsic "^1.0.2" @@ -1770,10 +2015,12 @@ call-bind@^1.0.0, call-bind@^1.0.2: callsites@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz" + integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== camel-case@^4.1.2: version "4.1.2" resolved "https://registry.npmjs.org/camel-case/-/camel-case-4.1.2.tgz" + integrity sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw== dependencies: pascal-case "^3.1.2" tslib "^2.0.3" @@ -1781,14 +2028,17 @@ camel-case@^4.1.2: camelcase@^5.0.0: version "5.3.1" resolved "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz" + integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== camelcase@^6.0.0: version "6.3.0" resolved "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz" + integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== caniuse-api@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/caniuse-api/-/caniuse-api-3.0.0.tgz" + integrity sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw== dependencies: browserslist "^4.0.0" caniuse-lite "^1.0.0" @@ -1798,14 +2048,17 @@ caniuse-api@^3.0.0: caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001400, caniuse-lite@^1.0.30001426: version "1.0.30001436" resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001436.tgz" + integrity sha512-ZmWkKsnC2ifEPoWUvSAIGyOYwT+keAaaWPHiQ9DfMqS1t6tfuyFYoWR78TeZtznkEQ64+vGXH9cZrElwR2Mrxg== case-sensitive-paths-webpack-plugin@^2.3.0: version "2.4.0" resolved "https://registry.npmjs.org/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz" + integrity sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw== chalk@^2.0.0, chalk@^2.1.0: version "2.4.2" resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== dependencies: ansi-styles "^3.2.1" escape-string-regexp "^1.0.5" @@ -1814,6 +2067,7 @@ chalk@^2.0.0, chalk@^2.1.0: chalk@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz" + integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== dependencies: ansi-styles "^4.1.0" supports-color "^7.1.0" @@ -1821,6 +2075,7 @@ chalk@^3.0.0: chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== dependencies: ansi-styles "^4.1.0" supports-color "^7.1.0" @@ -1828,6 +2083,7 @@ chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: chokidar@^3.5.3: version "3.5.3" resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz" + integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== dependencies: anymatch "~3.1.2" braces "~3.0.2" @@ -1842,28 +2098,33 @@ chokidar@^3.5.3: chrome-trace-event@^1.0.2: version "1.0.3" resolved "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz" + integrity sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg== clean-css@^5.2.2: version "5.3.1" resolved "https://registry.npmjs.org/clean-css/-/clean-css-5.3.1.tgz" + integrity sha512-lCr8OHhiWCTw4v8POJovCoh4T7I9U11yVsPjMWWnnMmp9ZowCxyad1Pathle/9HjaDp+fdQKjO9fQydE6RHTZg== dependencies: source-map "~0.6.0" cli-cursor@^2.0.0: version "2.1.0" resolved "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz" + integrity sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw== dependencies: restore-cursor "^2.0.0" cli-cursor@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz" + integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== dependencies: restore-cursor "^3.1.0" cli-highlight@^2.1.10: version "2.1.11" resolved "https://registry.npmjs.org/cli-highlight/-/cli-highlight-2.1.11.tgz" + integrity sha512-9KDcoEVwyUXrjcJNvHD0NFc/hiwe/WPVYIleQh2O1N2Zro5gWJZ/K+3DGn8w8P/F6FxOgzyC5bxDyHIgCSPhGg== dependencies: chalk "^4.0.0" highlight.js "^10.7.1" @@ -1875,10 +2136,12 @@ cli-highlight@^2.1.10: cli-spinners@^2.5.0: version "2.7.0" resolved "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.7.0.tgz" + integrity sha512-qu3pN8Y3qHNgE2AFweciB1IfMnmZ/fsNTEE+NOFjmGB2F/7rLhnhzppvpCnN4FovtP26k8lHyy9ptEbNwWFLzw== clipboardy@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/clipboardy/-/clipboardy-2.3.0.tgz" + integrity sha512-mKhiIL2DrQIsuXMgBgnfEHOZOryC7kY7YO//TN6c63wlEm3NG5tz+YgY5rVi29KCmq/QQjKYvM7a19+MDOTHOQ== dependencies: arch "^2.1.1" execa "^1.0.0" @@ -1887,6 +2150,7 @@ clipboardy@^2.3.0: cliui@^7.0.2, cliui@^7.0.4: version "7.0.4" resolved "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz" + integrity sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ== dependencies: string-width "^4.2.0" strip-ansi "^6.0.0" @@ -1895,6 +2159,7 @@ cliui@^7.0.2, cliui@^7.0.4: clone-deep@^4.0.1: version "4.0.1" resolved "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz" + integrity sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ== dependencies: is-plain-object "^2.0.4" kind-of "^6.0.2" @@ -1903,66 +2168,80 @@ clone-deep@^4.0.1: clone@^1.0.2: version "1.0.4" resolved "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz" + integrity sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg== color-convert@^1.9.0: version "1.9.3" resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== dependencies: color-name "1.1.3" color-convert@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== dependencies: color-name "~1.1.4" color-name@1.1.3: version "1.1.3" resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" + integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= color-name@~1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== colord@^2.9.1: version "2.9.3" resolved "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz" + integrity sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw== colorette@^2.0.10: version "2.0.19" resolved "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz" + integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ== combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== dependencies: delayed-stream "~1.0.0" commander@^2.20.0: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" + integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== commander@^7.2.0: version "7.2.0" resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz" + integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== commander@^8.3.0: version "8.3.0" resolved "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz" + integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww== commondir@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz" + integrity sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs= compressible@~2.0.16: version "2.0.18" resolved "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz" + integrity sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg== dependencies: mime-db ">= 1.43.0 < 2" compression@^1.7.4: version "1.7.4" resolved "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz" + integrity sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ== dependencies: accepts "~1.3.5" bytes "3.0.0" @@ -1975,48 +2254,58 @@ compression@^1.7.4: concat-map@0.0.1: version "0.0.1" resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" + integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= connect-history-api-fallback@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz" + integrity sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA== consola@^2.15.0: version "2.15.3" resolved "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz" + integrity sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw== consolidate@^0.15.1: version "0.15.1" resolved "https://registry.npmjs.org/consolidate/-/consolidate-0.15.1.tgz" + integrity sha512-DW46nrsMJgy9kqAbPt5rKaCr7uFtpo4mSUvLHIUbJEjm0vo+aY5QLwBUq3FK4tRnJr/X0Psc0C4jf/h+HtXSMw== dependencies: bluebird "^3.1.1" content-disposition@0.5.4: version "0.5.4" resolved "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz" + integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== dependencies: safe-buffer "5.2.1" content-type@~1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz" + integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== convert-source-map@^1.7.0: version "1.8.0" resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz" + integrity sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA== dependencies: safe-buffer "~5.1.1" cookie-signature@1.0.6: version "1.0.6" resolved "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz" + integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== cookie@0.5.0: version "0.5.0" resolved "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz" + integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== copy-webpack-plugin@^9.0.1: version "9.1.0" resolved "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-9.1.0.tgz" + integrity sha512-rxnR7PaGigJzhqETHGmAcxKnLZSR5u1Y3/bcIv/1FnqXedcL/E2ewK7ZCNrArJKCiSv8yVXhTqetJh8inDvfsA== dependencies: fast-glob "^3.2.7" glob-parent "^6.0.1" @@ -2028,20 +2317,24 @@ copy-webpack-plugin@^9.0.1: core-js-compat@^3.25.1, core-js-compat@^3.8.3: version "3.26.1" resolved "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.26.1.tgz" + integrity sha512-622/KzTudvXCDLRw70iHW4KKs1aGpcRcowGWyYJr2DEBfRrd6hNJybxSWJFuZYD4ma86xhrwDDHxmDaIq4EA8A== dependencies: browserslist "^4.21.4" core-js@^3.6.5, core-js@^3.8.3: version "3.22.4" resolved "https://registry.npmjs.org/core-js/-/core-js-3.22.4.tgz" + integrity sha512-1uLykR+iOfYja+6Jn/57743gc9n73EWiOnSJJ4ba3B4fOEYDBv25MagmEZBxTp5cWq4b/KPx/l77zgsp28ju4w== core-util-is@~1.0.0: version "1.0.3" resolved "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz" + integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== cosmiconfig@^7.0.0: version "7.1.0" resolved "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz" + integrity sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA== dependencies: "@types/parse-json" "^4.0.0" import-fresh "^3.2.1" @@ -2052,6 +2345,7 @@ cosmiconfig@^7.0.0: cross-spawn@^6.0.0: version "6.0.5" resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz" + integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== dependencies: nice-try "^1.0.4" path-key "^2.0.1" @@ -2062,6 +2356,7 @@ cross-spawn@^6.0.0: cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" @@ -2070,10 +2365,12 @@ cross-spawn@^7.0.3: css-declaration-sorter@^6.3.1: version "6.3.1" resolved "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-6.3.1.tgz" + integrity sha512-fBffmak0bPAnyqc/HO8C3n2sHrp9wcqQz6ES9koRF2/mLOVAx9zIQ3Y7R29sYCteTPqMCwns4WYQoCX91Xl3+w== css-loader@^6.5.0: version "6.7.2" resolved "https://registry.npmjs.org/css-loader/-/css-loader-6.7.2.tgz" + integrity sha512-oqGbbVcBJkm8QwmnNzrFrWTnudnRZC+1eXikLJl0n4ljcfotgRifpg2a1lKy8jTrc4/d9A/ap1GFq1jDKG7J+Q== dependencies: icss-utils "^5.1.0" postcss "^8.4.18" @@ -2087,6 +2384,7 @@ css-loader@^6.5.0: css-minimizer-webpack-plugin@^3.0.2: version "3.4.1" resolved "https://registry.npmjs.org/css-minimizer-webpack-plugin/-/css-minimizer-webpack-plugin-3.4.1.tgz" + integrity sha512-1u6D71zeIfgngN2XNRJefc/hY7Ybsxd74Jm4qngIXyUEk7fss3VUzuHxLAq/R8NAba4QU9OUSaMZlbpRc7bM4Q== dependencies: cssnano "^5.0.6" jest-worker "^27.0.2" @@ -2098,6 +2396,7 @@ css-minimizer-webpack-plugin@^3.0.2: css-select@^4.1.3: version "4.3.0" resolved "https://registry.npmjs.org/css-select/-/css-select-4.3.0.tgz" + integrity sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ== dependencies: boolbase "^1.0.0" css-what "^6.0.1" @@ -2108,6 +2407,7 @@ css-select@^4.1.3: css-tree@^1.1.2, css-tree@^1.1.3: version "1.1.3" resolved "https://registry.npmjs.org/css-tree/-/css-tree-1.1.3.tgz" + integrity sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q== dependencies: mdn-data "2.0.14" source-map "^0.6.1" @@ -2115,14 +2415,17 @@ css-tree@^1.1.2, css-tree@^1.1.3: css-what@^6.0.1: version "6.1.0" resolved "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz" + integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== cssesc@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz" + integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg== cssnano-preset-default@^5.2.13: version "5.2.13" resolved "https://registry.npmjs.org/cssnano-preset-default/-/cssnano-preset-default-5.2.13.tgz" + integrity sha512-PX7sQ4Pb+UtOWuz8A1d+Rbi+WimBIxJTRyBdgGp1J75VU0r/HFQeLnMYgHiCAp6AR4rqrc7Y4R+1Rjk3KJz6DQ== dependencies: css-declaration-sorter "^6.3.1" cssnano-utils "^3.1.0" @@ -2157,10 +2460,12 @@ cssnano-preset-default@^5.2.13: cssnano-utils@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/cssnano-utils/-/cssnano-utils-3.1.0.tgz" + integrity sha512-JQNR19/YZhz4psLX/rQ9M83e3z2Wf/HdJbryzte4a3NSuafyp9w/I4U+hx5C2S9g41qlstH7DEWnZaaj83OuEA== cssnano@^5.0.0, cssnano@^5.0.6: version "5.1.14" resolved "https://registry.npmjs.org/cssnano/-/cssnano-5.1.14.tgz" + integrity sha512-Oou7ihiTocbKqi0J1bB+TRJIQX5RMR3JghA8hcWSw9mjBLQ5Y3RWqEDoYG3sRNlAbCIXpqMoZGbq5KDR3vdzgw== dependencies: cssnano-preset-default "^5.2.13" lilconfig "^2.0.3" @@ -2169,54 +2474,64 @@ cssnano@^5.0.0, cssnano@^5.0.6: csso@^4.2.0: version "4.2.0" resolved "https://registry.npmjs.org/csso/-/csso-4.2.0.tgz" + integrity sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA== dependencies: css-tree "^1.1.2" de-indent@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz" + integrity sha1-sgOOhG3DO6pXlhKNCAS0VbjB4h0= debug@2.6.9: version "2.6.9" resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== dependencies: ms "2.0.0" debug@^3.2.7: version "3.2.7" resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== dependencies: ms "^2.1.1" debug@^4.1.0, debug@^4.1.1, debug@~4.3.1, debug@~4.3.2: version "4.3.4" resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" deepmerge@^1.5.2: version "1.5.2" resolved "https://registry.npmjs.org/deepmerge/-/deepmerge-1.5.2.tgz" + integrity sha512-95k0GDqvBjZavkuvzx/YqVLv/6YYa17fz6ILMSf7neqQITCPbnfEnQvEgMPNjH4kgobe7+WIL0yJEHku+H3qtQ== default-gateway@^6.0.3: version "6.0.3" resolved "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz" + integrity sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg== dependencies: execa "^5.0.0" defaults@^1.0.3: version "1.0.4" resolved "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz" + integrity sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A== dependencies: clone "^1.0.2" define-lazy-prop@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz" + integrity sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og== define-properties@^1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz" + integrity sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA== dependencies: has-property-descriptors "^1.0.0" object-keys "^1.1.1" @@ -2224,48 +2539,58 @@ define-properties@^1.1.4: delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" + integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= depd@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" + integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== depd@~1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" + integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== destroy@1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz" + integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== detect-node@^2.0.4: version "2.1.0" resolved "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz" + integrity sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g== dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz" + integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== dependencies: path-type "^4.0.0" dns-equal@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz" + integrity sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg== dns-packet@^5.2.2: version "5.4.0" resolved "https://registry.npmjs.org/dns-packet/-/dns-packet-5.4.0.tgz" + integrity sha512-EgqGeaBB8hLiHLZtp/IbaDQTL8pZ0+IvwzSHA6d7VyMDM+B9hgddEMa9xjK5oYnw0ci0JQ6g2XCD7/f6cafU6g== dependencies: "@leichtgewicht/ip-codec" "^2.0.1" dom-converter@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz" + integrity sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA== dependencies: utila "~0.4" dom-serializer@^1.0.1: version "1.4.1" resolved "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz" + integrity sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag== dependencies: domelementtype "^2.0.1" domhandler "^4.2.0" @@ -2274,16 +2599,19 @@ dom-serializer@^1.0.1: domelementtype@^2.0.1, domelementtype@^2.2.0: version "2.3.0" resolved "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz" + integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== domhandler@^4.0.0, domhandler@^4.2.0, domhandler@^4.3.1: version "4.3.1" resolved "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz" + integrity sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ== dependencies: domelementtype "^2.2.0" domutils@^2.5.2, domutils@^2.8.0: version "2.8.0" resolved "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz" + integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A== dependencies: dom-serializer "^1.0.1" domelementtype "^2.2.0" @@ -2292,6 +2620,7 @@ domutils@^2.5.2, domutils@^2.8.0: dot-case@^3.0.4: version "3.0.4" resolved "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz" + integrity sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w== dependencies: no-case "^3.0.4" tslib "^2.0.3" @@ -2299,48 +2628,59 @@ dot-case@^3.0.4: dotenv-expand@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz" + integrity sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA== dotenv@^10.0.0: version "10.0.0" resolved "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz" + integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== duplexer@^0.1.2: version "0.1.2" resolved "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz" + integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== easy-stack@1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/easy-stack/-/easy-stack-1.0.1.tgz" + integrity sha512-wK2sCs4feiiJeFXn3zvY0p41mdU5VUgbgs1rNsc/y5ngFUijdWd+iIN8eoyuZHKB8xN6BL4PdWmzqFmxNg6V2w== ee-first@1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== electron-to-chromium@^1.4.251: version "1.4.284" resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz" + integrity sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA== emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" + integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== emojis-list@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz" + integrity sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q== encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz" + integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== end-of-stream@^1.1.0: version "1.4.4" resolved "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== dependencies: once "^1.4.0" engine.io-client@~6.2.1: version "6.2.2" resolved "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.2.2.tgz" + integrity sha512-8ZQmx0LQGRTYkHuogVZuGSpDqYZtCM/nv8zQ68VZ+JkOpazJ7ICdsSpaO6iXwvaU30oFg5QJOJWj8zWqhbKjkQ== dependencies: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.1" @@ -2351,10 +2691,12 @@ engine.io-client@~6.2.1: engine.io-parser@~5.0.3: version "5.0.4" resolved "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.0.4.tgz" + integrity sha512-+nVFp+5z1E3HcToEnO7ZIj3g+3k9389DvWtvJZz0T6/eOCPIyyxehFcedoYrZQrp0LgQbD9pPXhpMBKMd5QURg== enhanced-resolve@^5.10.0: version "5.12.0" resolved "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.12.0.tgz" + integrity sha512-QHTXI/sZQmko1cbDoNAa3mJ5qhWUUNAq3vR0/YiD379fWQrcfuoX1+HW2S0MTt7XmoPLapdaDKUtelUSPic7hQ== dependencies: graceful-fs "^4.2.4" tapable "^2.2.0" @@ -2362,38 +2704,46 @@ enhanced-resolve@^5.10.0: entities@^2.0.0: version "2.2.0" resolved "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz" + integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== error-ex@^1.3.1: version "1.3.2" resolved "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz" + integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g== dependencies: is-arrayish "^0.2.1" error-stack-parser@^2.0.6: version "2.0.7" resolved "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.0.7.tgz" + integrity sha512-chLOW0ZGRf4s8raLrDxa5sdkvPec5YdvwbFnqJme4rk0rFajP8mPtrDL1+I+CwrQDCjswDA5sREX7jYQDQs9vA== dependencies: stackframe "^1.1.1" es-module-lexer@^0.9.0: version "0.9.3" resolved "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.9.3.tgz" + integrity sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ== escalade@^3.1.1: version "3.1.1" resolved "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz" + integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== escape-html@~1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz" + integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow== escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" + integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= eslint-scope@5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz" + integrity sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw== dependencies: esrecurse "^4.3.0" estraverse "^4.1.1" @@ -2401,44 +2751,54 @@ eslint-scope@5.1.1: eslint-visitor-keys@^1.0.0: version "1.3.0" resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz" + integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== esrecurse@^4.3.0: version "4.3.0" resolved "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz" + integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== dependencies: estraverse "^5.2.0" estraverse@^4.1.1: version "4.3.0" resolved "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz" + integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== estraverse@^5.2.0: version "5.3.0" resolved "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz" + integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== esutils@^2.0.2: version "2.0.3" resolved "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz" + integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== etag@~1.8.1: version "1.8.1" resolved "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" + integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== event-pubsub@4.3.0: version "4.3.0" resolved "https://registry.npmjs.org/event-pubsub/-/event-pubsub-4.3.0.tgz" + integrity sha512-z7IyloorXvKbFx9Bpie2+vMJKKx1fH1EN5yiTfp8CiLOTptSYy1g8H4yDpGlEdshL1PBiFtBHepF2cNsqeEeFQ== eventemitter3@^4.0.0: version "4.0.7" resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz" + integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== events@^3.2.0: version "3.3.0" resolved "https://registry.npmjs.org/events/-/events-3.3.0.tgz" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== execa@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz" + integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA== dependencies: cross-spawn "^6.0.0" get-stream "^4.0.0" @@ -2451,6 +2811,7 @@ execa@^1.0.0: execa@^5.0.0: version "5.1.1" resolved "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz" + integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== dependencies: cross-spawn "^7.0.3" get-stream "^6.0.0" @@ -2465,6 +2826,7 @@ execa@^5.0.0: express@^4.17.3: version "4.18.2" resolved "https://registry.npmjs.org/express/-/express-4.18.2.tgz" + integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== dependencies: accepts "~1.3.8" array-flatten "1.1.1" @@ -2501,10 +2863,12 @@ express@^4.17.3: fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz" + integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== fast-glob@^3.2.7, fast-glob@^3.2.9: version "3.2.12" resolved "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz" + integrity sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w== dependencies: "@nodelib/fs.stat" "^2.0.2" "@nodelib/fs.walk" "^1.2.3" @@ -2515,34 +2879,40 @@ fast-glob@^3.2.7, fast-glob@^3.2.9: fast-json-stable-stringify@^2.0.0: version "2.1.0" resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== fastq@^1.6.0: version "1.14.0" resolved "https://registry.npmjs.org/fastq/-/fastq-1.14.0.tgz" + integrity sha512-eR2D+V9/ExcbF9ls441yIuN6TI2ED1Y2ZcA5BmMtJsOkWOFRJQ0Jt0g1UwqXJJVAb+V+umH5Dfr8oh4EVP7VVg== dependencies: reusify "^1.0.4" faye-websocket@^0.11.3: version "0.11.4" resolved "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz" + integrity sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g== dependencies: websocket-driver ">=0.5.1" figures@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz" + integrity sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA== dependencies: escape-string-regexp "^1.0.5" fill-range@^7.0.1: version "7.0.1" resolved "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz" + integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== dependencies: to-regex-range "^5.0.1" finalhandler@1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz" + integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== dependencies: debug "2.6.9" encodeurl "~1.0.2" @@ -2555,6 +2925,7 @@ finalhandler@1.2.0: find-cache-dir@^3.3.1: version "3.3.2" resolved "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz" + integrity sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig== dependencies: commondir "^1.0.1" make-dir "^3.0.2" @@ -2563,6 +2934,7 @@ find-cache-dir@^3.3.1: find-up@^4.0.0, find-up@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" + integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== dependencies: locate-path "^5.0.0" path-exists "^4.0.0" @@ -2570,10 +2942,12 @@ find-up@^4.0.0, find-up@^4.1.0: follow-redirects@^1.0.0, follow-redirects@^1.14.9: version "1.15.0" resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.0.tgz" + integrity sha512-aExlJShTV4qOUOL7yF1U5tvLCB0xQuudbf6toyYA0E/acBNw71mvjFTnLaRp50aQaYocMR0a/RMMBIHeZnGyjQ== form-data@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" @@ -2582,18 +2956,22 @@ form-data@^4.0.0: forwarded@0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz" + integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== fraction.js@^4.2.0: version "4.2.0" resolved "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz" + integrity sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA== fresh@0.5.2: version "0.5.2" resolved "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz" + integrity sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q== fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz" + integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== dependencies: at-least-node "^1.0.0" graceful-fs "^4.2.0" @@ -2603,30 +2981,37 @@ fs-extra@^9.1.0: fs-monkey@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.3.tgz" + integrity sha512-cybjIfiiE+pTWicSCLFHSrXZ6EilF30oh91FDP9S2B051prEa7QWfrVTQm10/dDpswBDXZugPa1Ogu8Yh+HV0Q== fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" + integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= fsevents@~2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== function-bind@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz" + integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz" + integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== get-caller-file@^2.0.5: version "2.0.5" resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz" + integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== get-intrinsic@^1.0.2, get-intrinsic@^1.1.1: version "1.1.3" resolved "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz" + integrity sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A== dependencies: function-bind "^1.1.1" has "^1.0.3" @@ -2635,32 +3020,38 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.1: get-stream@^4.0.0: version "4.1.0" resolved "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz" + integrity sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w== dependencies: pump "^3.0.0" get-stream@^6.0.0: version "6.0.1" resolved "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz" + integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== glob-parent@^5.1.2, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== dependencies: is-glob "^4.0.1" glob-parent@^6.0.1: version "6.0.2" resolved "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz" + integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A== dependencies: is-glob "^4.0.3" glob-to-regexp@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz" + integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== glob@^7.1.3: version "7.2.0" resolved "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz" + integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" @@ -2672,10 +3063,12 @@ glob@^7.1.3: globals@^11.1.0: version "11.12.0" resolved "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== globby@^11.0.2, globby@^11.0.3: version "11.1.0" resolved "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz" + integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== dependencies: array-union "^2.1.0" dir-glob "^3.0.1" @@ -2687,64 +3080,78 @@ globby@^11.0.2, globby@^11.0.3: graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9: version "4.2.10" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz" + integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== gzip-size@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz" + integrity sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q== dependencies: duplexer "^0.1.2" handle-thing@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz" + integrity sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg== has-flag@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz" + integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= has-flag@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== has-property-descriptors@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz" + integrity sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ== dependencies: get-intrinsic "^1.1.1" has-symbols@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz" + integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== has@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/has/-/has-1.0.3.tgz" + integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== dependencies: function-bind "^1.1.1" hash-sum@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/hash-sum/-/hash-sum-1.0.2.tgz" + integrity sha512-fUs4B4L+mlt8/XAtSOGMUO1TXmAelItBPtJG7CyHJfYTdDjwisntGO2JQz7oUsatOY9o68+57eziUVNw/mRHmA== hash-sum@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/hash-sum/-/hash-sum-2.0.0.tgz" + integrity sha512-WdZTbAByD+pHfl/g9QSsBIIwy8IT+EsPiKDs0KNX+zSHhdDLFKdZu0BQHljvO+0QI/BasbMSUa8wYNCZTvhslg== he@^1.1.0, he@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/he/-/he-1.2.0.tgz" + integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== highlight.js@^10.7.1: version "10.7.3" resolved "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz" + integrity sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A== hosted-git-info@^2.1.4: version "2.8.9" resolved "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz" + integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== hpack.js@^2.1.6: version "2.1.6" resolved "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz" + integrity sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ== dependencies: inherits "^2.0.1" obuf "^1.0.0" @@ -2754,10 +3161,12 @@ hpack.js@^2.1.6: html-entities@^2.3.2: version "2.3.3" resolved "https://registry.npmjs.org/html-entities/-/html-entities-2.3.3.tgz" + integrity sha512-DV5Ln36z34NNTDgnz0EWGBLZENelNAtkiFA4kyNOG2tDI6Mz1uSWiq1wAKdyjnJwyDiDO7Fa2SO1CTxPXL8VxA== html-minifier-terser@^6.0.2: version "6.1.0" resolved "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz" + integrity sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw== dependencies: camel-case "^4.1.2" clean-css "^5.2.2" @@ -2770,14 +3179,17 @@ html-minifier-terser@^6.0.2: html-tags@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/html-tags/-/html-tags-2.0.0.tgz" + integrity sha512-+Il6N8cCo2wB/Vd3gqy/8TZhTD3QvcVeQLCnZiGkGCH3JP28IgGAY41giccp2W4R3jfyJPAP318FQTa1yU7K7g== html-tags@^3.1.0: version "3.2.0" resolved "https://registry.npmjs.org/html-tags/-/html-tags-3.2.0.tgz" + integrity sha512-vy7ClnArOZwCnqZgvv+ddgHgJiAFXe3Ge9ML5/mBctVJoUoYPCdxVucOywjDARn6CVoh3dRSFdPHy2sX80L0Wg== html-webpack-plugin@^5.1.0: version "5.5.0" resolved "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-5.5.0.tgz" + integrity sha512-sy88PC2cRTVxvETRgUHFrL4No3UxvcH8G1NepGhqaTT+GXN2kTamqasot0inS5hXeg1cMbFDt27zzo9p35lZVw== dependencies: "@types/html-minifier-terser" "^6.0.0" html-minifier-terser "^6.0.2" @@ -2788,6 +3200,7 @@ html-webpack-plugin@^5.1.0: htmlparser2@^6.1.0: version "6.1.0" resolved "https://registry.npmjs.org/htmlparser2/-/htmlparser2-6.1.0.tgz" + integrity sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A== dependencies: domelementtype "^2.0.1" domhandler "^4.0.0" @@ -2797,10 +3210,12 @@ htmlparser2@^6.1.0: http-deceiver@^1.2.7: version "1.2.7" resolved "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz" + integrity sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw== http-errors@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz" + integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== dependencies: depd "2.0.0" inherits "2.0.4" @@ -2811,6 +3226,7 @@ http-errors@2.0.0: http-errors@~1.6.2: version "1.6.3" resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz" + integrity sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A== dependencies: depd "~1.1.2" inherits "2.0.3" @@ -2820,10 +3236,12 @@ http-errors@~1.6.2: http-parser-js@>=0.5.1: version "0.5.8" resolved "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz" + integrity sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q== http-proxy-middleware@^2.0.3: version "2.0.6" resolved "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz" + integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw== dependencies: "@types/http-proxy" "^1.17.8" http-proxy "^1.18.1" @@ -2834,6 +3252,7 @@ http-proxy-middleware@^2.0.3: http-proxy@^1.18.1: version "1.18.1" resolved "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz" + integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ== dependencies: eventemitter3 "^4.0.0" follow-redirects "^1.0.0" @@ -2842,28 +3261,34 @@ http-proxy@^1.18.1: human-signals@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz" + integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz" + integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== dependencies: safer-buffer ">= 2.1.2 < 3" icss-utils@^5.0.0, icss-utils@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz" + integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== ieee754@^1.1.13: version "1.2.1" resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== ignore@^5.2.0: version "5.2.1" resolved "https://registry.npmjs.org/ignore/-/ignore-5.2.1.tgz" + integrity sha512-d2qQLzTJ9WxQftPAuEQpSPmKqzxePjzVbpAVv62AQ64NTL+wR4JkrVqR/LqFsFEUsHDAiId52mJteHDFuDkElA== import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz" + integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== dependencies: parent-module "^1.0.0" resolve-from "^4.0.0" @@ -2871,6 +3296,7 @@ import-fresh@^3.2.1: inflight@^1.0.4: version "1.0.6" resolved "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" + integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= dependencies: once "^1.3.0" wrappy "1" @@ -2878,118 +3304,144 @@ inflight@^1.0.4: inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== inherits@2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + integrity sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw== ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz" + integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== ipaddr.js@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.0.1.tgz" + integrity sha512-1qTgH9NG+IIJ4yfKs2e6Pp1bZg8wbDbKHT21HrLIeYBTRLgMYKnMTPAuI3Lcs61nfx5h1xlXnbJtH1kX5/d/ng== is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz" + integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== is-binary-path@~2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz" + integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw== dependencies: binary-extensions "^2.0.0" is-core-module@^2.8.1: version "2.9.0" resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz" + integrity sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A== dependencies: has "^1.0.3" is-docker@^2.0.0, is-docker@^2.1.1: version "2.2.1" resolved "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz" + integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== is-extglob@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz" + integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= is-file-esm@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/is-file-esm/-/is-file-esm-1.0.0.tgz" + integrity sha512-rZlaNKb4Mr8WlRu2A9XdeoKgnO5aA53XdPHgCKVyCrQ/rWi89RET1+bq37Ru46obaQXeiX4vmFIm1vks41hoSA== dependencies: read-pkg-up "^7.0.1" is-fullwidth-code-point@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz" + integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8= is-fullwidth-code-point@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz" + integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: version "4.0.3" resolved "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz" + integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== dependencies: is-extglob "^2.1.1" is-interactive@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz" + integrity sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w== is-number@^7.0.0: version "7.0.0" resolved "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz" + integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== is-plain-obj@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz" + integrity sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA== is-plain-object@^2.0.4: version "2.0.4" resolved "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz" + integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== dependencies: isobject "^3.0.1" is-stream@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz" + integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= is-stream@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz" + integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== is-unicode-supported@^0.1.0: version "0.1.0" resolved "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz" + integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw== is-wsl@^2.1.1, is-wsl@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz" + integrity sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww== dependencies: is-docker "^2.0.0" isarray@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== isexe@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" + integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= isobject@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz" + integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== javascript-stringify@^2.0.1: version "2.1.0" resolved "https://registry.npmjs.org/javascript-stringify/-/javascript-stringify-2.1.0.tgz" + integrity sha512-JVAfqNPTvNq3sB/VHQJAFxN/sPgKnsKrCwyRt15zwNCdrMMJDdcEOdubuy+DuJYYdm0ox1J4uzEuYKkN+9yhVg== jest-worker@^27.0.2, jest-worker@^27.4.5: version "27.5.1" resolved "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz" + integrity sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg== dependencies: "@types/node" "*" merge-stream "^2.0.0" @@ -2998,6 +3450,7 @@ jest-worker@^27.0.2, jest-worker@^27.4.5: joi@^17.4.0: version "17.7.0" resolved "https://registry.npmjs.org/joi/-/joi-17.7.0.tgz" + integrity sha512-1/ugc8djfn93rTE3WRKdCzGGt/EtiYKxITMO4Wiv6q5JL1gl9ePt4kBsl1S499nbosspfctIQTpYIhSmHA3WAg== dependencies: "@hapi/hoek" "^9.0.0" "@hapi/topo" "^5.0.0" @@ -3008,48 +3461,59 @@ joi@^17.4.0: js-message@1.0.7: version "1.0.7" resolved "https://registry.npmjs.org/js-message/-/js-message-1.0.7.tgz" + integrity sha512-efJLHhLjIyKRewNS9EGZ4UpI8NguuL6fKkhRxVuMmrGV2xN/0APGdQYwLFky5w9naebSZ0OwAGp0G6/2Cg90rA== js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz" + integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== jsesc@^2.5.1: version "2.5.2" resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz" + integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== jsesc@~0.5.0: version "0.5.0" resolved "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz" + integrity sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA== json-parse-better-errors@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz" + integrity sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw== json-parse-even-better-errors@^2.3.0, json-parse-even-better-errors@^2.3.1: version "2.3.1" resolved "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz" + integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz" + integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== json-schema-traverse@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz" + integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== json5@^1.0.1: version "1.0.2" resolved "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz" + integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA== dependencies: minimist "^1.2.0" json5@^2.1.2, json5@^2.2.1: version "2.2.3" resolved "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== jsonfile@^6.0.1: version "6.1.0" resolved "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz" + integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== dependencies: universalify "^2.0.0" optionalDependencies: @@ -3058,20 +3522,24 @@ jsonfile@^6.0.1: kind-of@^6.0.2: version "6.0.3" resolved "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz" + integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== klona@^2.0.5: version "2.0.5" resolved "https://registry.npmjs.org/klona/-/klona-2.0.5.tgz" + integrity sha512-pJiBpiXMbt7dkzXe8Ghj/u4FfXOOa98fPW+bihOJ4SjnoijweJrNThJfd3ifXpXhREjpoF2mZVH1GfS9LV3kHQ== launch-editor-middleware@^2.2.1: version "2.3.0" resolved "https://registry.npmjs.org/launch-editor-middleware/-/launch-editor-middleware-2.3.0.tgz" + integrity sha512-GJR64trLdFFwCoL9DMn/d1SZX0OzTDPixu4mcfWTShQ4tIqCHCGvlg9fOEYQXyBlrSMQwylsJfUWncheShfV2w== dependencies: launch-editor "^2.3.0" launch-editor@^2.2.1, launch-editor@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/launch-editor/-/launch-editor-2.3.0.tgz" + integrity sha512-3QrsCXejlWYHjBPFXTyGNhPj4rrQdB+5+r5r3wArpLH201aR+nWUgw/zKKkTmilCfY/sv6u8qo98pNvtg8LUTA== dependencies: picocolors "^1.0.0" shell-quote "^1.6.1" @@ -3079,18 +3547,22 @@ launch-editor@^2.2.1, launch-editor@^2.3.0: lilconfig@^2.0.3: version "2.0.6" resolved "https://registry.npmjs.org/lilconfig/-/lilconfig-2.0.6.tgz" + integrity sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg== lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz" + integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== loader-runner@^4.1.0, loader-runner@^4.2.0: version "4.3.0" resolved "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz" + integrity sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg== loader-utils@^1.0.2, loader-utils@^1.1.0: version "1.4.2" resolved "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.2.tgz" + integrity sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg== dependencies: big.js "^5.2.2" emojis-list "^3.0.0" @@ -3099,6 +3571,7 @@ loader-utils@^1.0.2, loader-utils@^1.1.0: loader-utils@^2.0.0: version "2.0.4" resolved "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz" + integrity sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw== dependencies: big.js "^5.2.2" emojis-list "^3.0.0" @@ -3107,40 +3580,49 @@ loader-utils@^2.0.0: locate-path@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz" + integrity sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g== dependencies: p-locate "^4.1.0" lodash.debounce@^4.0.8: version "4.0.8" resolved "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz" + integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow== lodash.defaultsdeep@^4.6.1: version "4.6.1" resolved "https://registry.npmjs.org/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.1.tgz" + integrity sha512-3j8wdDzYuWO3lM3Reg03MuQR957t287Rpcxp1njpEa8oDrikb+FwGdW3n+FELh/A6qib6yPit0j/pv9G/yeAqA== lodash.kebabcase@^4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/lodash.kebabcase/-/lodash.kebabcase-4.1.1.tgz" + integrity sha512-N8XRTIMMqqDgSy4VLKPnJ/+hpGZN+PHQiJnSenYqPaVV/NCqEogTnAdZLQiGKhxX+JCs8waWq2t1XHWKOmlY8g== lodash.mapvalues@^4.6.0: version "4.6.0" resolved "https://registry.npmjs.org/lodash.mapvalues/-/lodash.mapvalues-4.6.0.tgz" + integrity sha1-G6+lAF3p3W9PJmaMMMo3IwzJaJw= lodash.memoize@^4.1.2: version "4.1.2" resolved "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz" + integrity sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag== lodash.uniq@^4.5.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz" + integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== lodash@^4.17.14, lodash@^4.17.20, lodash@^4.17.21: version "4.17.21" resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== log-symbols@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz" + integrity sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg== dependencies: chalk "^4.1.0" is-unicode-supported "^0.1.0" @@ -3148,6 +3630,7 @@ log-symbols@^4.1.0: log-update@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/log-update/-/log-update-2.3.0.tgz" + integrity sha512-vlP11XfFGyeNQlmEn9tJ66rEW1coA/79m5z6BCkudjbAGE83uhAcGYrBFwfs3AdLiLzGRusRPAbSPK9xZteCmg== dependencies: ansi-escapes "^3.0.0" cli-cursor "^2.0.0" @@ -3156,12 +3639,14 @@ log-update@^2.3.0: lower-case@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz" + integrity sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg== dependencies: tslib "^2.0.3" lru-cache@^4.1.2: version "4.1.5" resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz" + integrity sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g== dependencies: pseudomap "^1.0.2" yallist "^2.1.2" @@ -3169,54 +3654,65 @@ lru-cache@^4.1.2: lru-cache@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz" + integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== dependencies: yallist "^4.0.0" make-dir@^3.0.2, make-dir@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz" + integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== dependencies: semver "^6.0.0" mdn-data@2.0.14: version "2.0.14" resolved "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.14.tgz" + integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow== media-typer@0.3.0: version "0.3.0" resolved "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== memfs@^3.4.3: version "3.4.12" resolved "https://registry.npmjs.org/memfs/-/memfs-3.4.12.tgz" + integrity sha512-BcjuQn6vfqP+k100e0E9m61Hyqa//Brp+I3f0OBmN0ATHlFA8vx3Lt8z57R3u2bPqe3WGDBC+nF72fTH7isyEw== dependencies: fs-monkey "^1.0.3" merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz" + integrity sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w== merge-source-map@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/merge-source-map/-/merge-source-map-1.1.0.tgz" + integrity sha512-Qkcp7P2ygktpMPh2mCQZaf3jhN6D3Z/qVZHSdWvQ+2Ef5HgRAPBO57A77+ENm0CPx2+1Ce/MYKi3ymqdfuqibw== dependencies: source-map "^0.6.1" merge-stream@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz" + integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== merge2@^1.3.0, merge2@^1.4.1: version "1.4.1" resolved "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz" + integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== methods@~1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" + integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== micromatch@^4.0.2, micromatch@^4.0.4: version "4.0.5" resolved "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz" + integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== dependencies: braces "^3.0.2" picomatch "^2.3.1" @@ -3224,84 +3720,102 @@ micromatch@^4.0.2, micromatch@^4.0.4: mime-db@1.52.0, "mime-db@>= 1.43.0 < 2": version "1.52.0" resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.17, mime-types@~2.1.24, mime-types@~2.1.34: version "2.1.35" resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== dependencies: mime-db "1.52.0" mime@1.6.0: version "1.6.0" resolved "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== mimic-fn@^1.0.0: version "1.2.0" resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz" + integrity sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ== mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" + integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== mini-css-extract-plugin@^2.5.3: version "2.7.1" resolved "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.7.1.tgz" + integrity sha512-viOoaUFy+Z2w43VsGPbtfwFrr0tKwDctK9dUofG5MBViYhD1noGFUzzDIVw0KPwCGUP+c7zqLxm+acuQs7zLzw== dependencies: schema-utils "^4.0.0" minimalistic-assert@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz" + integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A== minimatch@^3.0.4: version "3.1.2" resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== dependencies: brace-expansion "^1.1.7" minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: version "1.2.6" resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz" + integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== minipass@^3.1.1: version "3.1.6" resolved "https://registry.npmjs.org/minipass/-/minipass-3.1.6.tgz" + integrity sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ== dependencies: yallist "^4.0.0" mkdirp@^0.5.6: version "0.5.6" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz" + integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== dependencies: minimist "^1.2.6" module-alias@^2.2.2: version "2.2.2" resolved "https://registry.npmjs.org/module-alias/-/module-alias-2.2.2.tgz" + integrity sha512-A/78XjoX2EmNvppVWEhM2oGk3x4lLxnkEA4jTbaK97QKSDjkIoOsKQlfylt/d3kKKi596Qy3NP5XrXJ6fZIC9Q== moment@^2.29.3: version "2.29.4" resolved "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz" + integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== mrmime@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/mrmime/-/mrmime-1.0.1.tgz" + integrity sha512-hzzEagAgDyoU1Q6yg5uI+AorQgdvMCur3FcKf7NhMKWsaYg+RnbTyHRa/9IlLF9rf455MOCtcqqrQQ83pPP7Uw== ms@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz" + integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== ms@2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== ms@2.1.3, ms@^2.1.1: version "2.1.3" resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== multicast-dns@^7.2.5: version "7.2.5" resolved "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.5.tgz" + integrity sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg== dependencies: dns-packet "^5.2.2" thunky "^1.0.2" @@ -3309,6 +3823,7 @@ multicast-dns@^7.2.5: mz@^2.4.0: version "2.7.0" resolved "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz" + integrity sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q== dependencies: any-promise "^1.0.0" object-assign "^4.0.1" @@ -3317,22 +3832,27 @@ mz@^2.4.0: nanoid@^3.3.4: version "3.3.4" resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz" + integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== negotiator@0.6.3: version "0.6.3" resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz" + integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== neo-async@^2.6.2: version "2.6.2" resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz" + integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== nice-try@^1.0.4: version "1.0.5" resolved "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz" + integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== no-case@^3.0.4: version "3.0.4" resolved "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz" + integrity sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg== dependencies: lower-case "^2.0.2" tslib "^2.0.3" @@ -3340,20 +3860,24 @@ no-case@^3.0.4: node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" node-forge@^1: version "1.3.1" resolved "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz" + integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== node-releases@^2.0.6: version "2.0.6" resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz" + integrity sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg== normalize-package-data@^2.5.0: version "2.5.0" resolved "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz" + integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== dependencies: hosted-git-info "^2.1.4" resolve "^1.10.0" @@ -3363,48 +3887,58 @@ normalize-package-data@^2.5.0: normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz" + integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== normalize-range@^0.1.2: version "0.1.2" resolved "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz" + integrity sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA== normalize-url@^6.0.1: version "6.1.0" resolved "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz" + integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== npm-run-path@^2.0.0: version "2.0.2" resolved "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz" + integrity sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8= dependencies: path-key "^2.0.0" npm-run-path@^4.0.1: version "4.0.1" resolved "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz" + integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== dependencies: path-key "^3.0.0" nth-check@^2.0.1: version "2.1.1" resolved "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz" + integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w== dependencies: boolbase "^1.0.0" object-assign@^4.0.1: version "4.1.1" resolved "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz" + integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= object-inspect@^1.9.0: version "1.12.2" resolved "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz" + integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== object-keys@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz" + integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== object.assign@^4.1.0: version "4.1.4" resolved "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz" + integrity sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ== dependencies: call-bind "^1.0.2" define-properties "^1.1.4" @@ -3414,38 +3948,45 @@ object.assign@^4.1.0: obuf@^1.0.0, obuf@^1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz" + integrity sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg== on-finished@2.4.1: version "2.4.1" resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz" + integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== dependencies: ee-first "1.1.1" on-headers@~1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz" + integrity sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA== once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.npmjs.org/once/-/once-1.4.0.tgz" + integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= dependencies: wrappy "1" onetime@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz" + integrity sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ== dependencies: mimic-fn "^1.0.0" onetime@^5.1.0, onetime@^5.1.2: version "5.1.2" resolved "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz" + integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== dependencies: mimic-fn "^2.1.0" open@^8.0.2, open@^8.0.9: version "8.4.0" resolved "https://registry.npmjs.org/open/-/open-8.4.0.tgz" + integrity sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q== dependencies: define-lazy-prop "^2.0.0" is-docker "^2.1.1" @@ -3454,10 +3995,12 @@ open@^8.0.2, open@^8.0.9: opener@^1.5.2: version "1.5.2" resolved "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz" + integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A== ora@^5.3.0: version "5.4.1" resolved "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz" + integrity sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ== dependencies: bl "^4.1.0" chalk "^4.1.0" @@ -3472,22 +4015,26 @@ ora@^5.3.0: p-finally@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz" + integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4= p-limit@^2.2.0: version "2.3.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz" + integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== dependencies: p-try "^2.0.0" p-locate@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz" + integrity sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A== dependencies: p-limit "^2.2.0" p-retry@^4.5.0: version "4.6.2" resolved "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz" + integrity sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ== dependencies: "@types/retry" "0.12.0" retry "^0.13.1" @@ -3495,10 +4042,12 @@ p-retry@^4.5.0: p-try@^2.0.0: version "2.2.0" resolved "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" + integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== param-case@^3.0.4: version "3.0.4" resolved "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz" + integrity sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A== dependencies: dot-case "^3.0.4" tslib "^2.0.3" @@ -3506,12 +4055,14 @@ param-case@^3.0.4: parent-module@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz" + integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== dependencies: callsites "^3.0.0" parse-json@^5.0.0: version "5.2.0" resolved "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz" + integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== dependencies: "@babel/code-frame" "^7.0.0" error-ex "^1.3.1" @@ -3521,24 +4072,29 @@ parse-json@^5.0.0: parse5-htmlparser2-tree-adapter@^6.0.0: version "6.0.1" resolved "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz" + integrity sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA== dependencies: parse5 "^6.0.1" parse5@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz" + integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug== parse5@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz" + integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== parseurl@~1.3.2, parseurl@~1.3.3: version "1.3.3" resolved "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz" + integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== pascal-case@^3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz" + integrity sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g== dependencies: no-case "^3.0.4" tslib "^2.0.3" @@ -3546,60 +4102,74 @@ pascal-case@^3.1.2: path-exists@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz" + integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" + integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= path-key@^2.0.0, path-key@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz" + integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A= path-key@^3.0.0, path-key@^3.1.0: version "3.1.1" resolved "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== path-parse@^1.0.7: version "1.0.7" resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" + integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== path-type@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz" + integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== picocolors@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/picocolors/-/picocolors-0.2.1.tgz" + integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA== picocolors@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz" + integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: version "2.3.1" resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz" + integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== pkg-dir@^4.1.0: version "4.2.0" resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz" + integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== dependencies: find-up "^4.0.0" popper.js@^1.16.1: version "1.16.1" resolved "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz" + integrity sha512-Wb4p1J4zyFTbM+u6WuO4XstYx4Ky9Cewe4DWrel7B0w6VVICvPwdOpotjzcf6eD8TsckVnIMNONQyPIUFOUbCQ== portal-vue@^2.1.7: version "2.1.7" resolved "https://registry.npmjs.org/portal-vue/-/portal-vue-2.1.7.tgz" + integrity sha512-+yCno2oB3xA7irTt0EU5Ezw22L2J51uKAacE/6hMPMoO/mx3h4rXFkkBkT4GFsMDv/vEe8TNKC3ujJJ0PTwb6g== portfinder@^1.0.26: version "1.0.32" resolved "https://registry.npmjs.org/portfinder/-/portfinder-1.0.32.tgz" + integrity sha512-on2ZJVVDXRADWE6jnQaX0ioEylzgBpQk8r55NE4wjXW1ZxO+BgDlY6DXwj20i0V8eB4SenDQ00WEaxfiIQPcxg== dependencies: async "^2.6.4" debug "^3.2.7" @@ -3608,6 +4178,7 @@ portfinder@^1.0.26: postcss-calc@^8.2.3: version "8.2.4" resolved "https://registry.npmjs.org/postcss-calc/-/postcss-calc-8.2.4.tgz" + integrity sha512-SmWMSJmB8MRnnULldx0lQIyhSNvuDl9HfrZkaqqE/WHAhToYsAvDq+yAsA/kIyINDszOp3Rh0GFoNuH5Ypsm3Q== dependencies: postcss-selector-parser "^6.0.9" postcss-value-parser "^4.2.0" @@ -3615,6 +4186,7 @@ postcss-calc@^8.2.3: postcss-colormin@^5.3.0: version "5.3.0" resolved "https://registry.npmjs.org/postcss-colormin/-/postcss-colormin-5.3.0.tgz" + integrity sha512-WdDO4gOFG2Z8n4P8TWBpshnL3JpmNmJwdnfP2gbk2qBA8PWwOYcmjmI/t3CmMeL72a7Hkd+x/Mg9O2/0rD54Pg== dependencies: browserslist "^4.16.6" caniuse-api "^3.0.0" @@ -3624,6 +4196,7 @@ postcss-colormin@^5.3.0: postcss-convert-values@^5.1.3: version "5.1.3" resolved "https://registry.npmjs.org/postcss-convert-values/-/postcss-convert-values-5.1.3.tgz" + integrity sha512-82pC1xkJZtcJEfiLw6UXnXVXScgtBrjlO5CBmuDQc+dlb88ZYheFsjTn40+zBVi3DkfF7iezO0nJUPLcJK3pvA== dependencies: browserslist "^4.21.4" postcss-value-parser "^4.2.0" @@ -3631,22 +4204,27 @@ postcss-convert-values@^5.1.3: postcss-discard-comments@^5.1.2: version "5.1.2" resolved "https://registry.npmjs.org/postcss-discard-comments/-/postcss-discard-comments-5.1.2.tgz" + integrity sha512-+L8208OVbHVF2UQf1iDmRcbdjJkuBF6IS29yBDSiWUIzpYaAhtNl6JYnYm12FnkeCwQqF5LeklOu6rAqgfBZqQ== postcss-discard-duplicates@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-discard-duplicates/-/postcss-discard-duplicates-5.1.0.tgz" + integrity sha512-zmX3IoSI2aoenxHV6C7plngHWWhUOV3sP1T8y2ifzxzbtnuhk1EdPwm0S1bIUNaJ2eNbWeGLEwzw8huPD67aQw== postcss-discard-empty@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/postcss-discard-empty/-/postcss-discard-empty-5.1.1.tgz" + integrity sha512-zPz4WljiSuLWsI0ir4Mcnr4qQQ5e1Ukc3i7UfE2XcrwKK2LIPIqE5jxMRxO6GbI3cv//ztXDsXwEWT3BHOGh3A== postcss-discard-overridden@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-discard-overridden/-/postcss-discard-overridden-5.1.0.tgz" + integrity sha512-21nOL7RqWR1kasIVdKs8HNqQJhFxLsyRfAnUDm4Fe4t4mCWL9OJiHvlHPjcd8zc5Myu89b/7wZDnOSjFgeWRtw== postcss-loader@^6.1.1: version "6.2.1" resolved "https://registry.npmjs.org/postcss-loader/-/postcss-loader-6.2.1.tgz" + integrity sha512-WbbYpmAaKcux/P66bZ40bpWsBucjx/TTgVVzRZ9yUO8yQfVBlameJ0ZGVaPfH64hNSBh63a+ICP5nqOpBA0w+Q== dependencies: cosmiconfig "^7.0.0" klona "^2.0.5" @@ -3655,6 +4233,7 @@ postcss-loader@^6.1.1: postcss-merge-longhand@^5.1.7: version "5.1.7" resolved "https://registry.npmjs.org/postcss-merge-longhand/-/postcss-merge-longhand-5.1.7.tgz" + integrity sha512-YCI9gZB+PLNskrK0BB3/2OzPnGhPkBEwmwhfYk1ilBHYVAZB7/tkTHFBAnCrvBBOmeYyMYw3DMjT55SyxMBzjQ== dependencies: postcss-value-parser "^4.2.0" stylehacks "^5.1.1" @@ -3662,6 +4241,7 @@ postcss-merge-longhand@^5.1.7: postcss-merge-rules@^5.1.3: version "5.1.3" resolved "https://registry.npmjs.org/postcss-merge-rules/-/postcss-merge-rules-5.1.3.tgz" + integrity sha512-LbLd7uFC00vpOuMvyZop8+vvhnfRGpp2S+IMQKeuOZZapPRY4SMq5ErjQeHbHsjCUgJkRNrlU+LmxsKIqPKQlA== dependencies: browserslist "^4.21.4" caniuse-api "^3.0.0" @@ -3671,12 +4251,14 @@ postcss-merge-rules@^5.1.3: postcss-minify-font-values@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-minify-font-values/-/postcss-minify-font-values-5.1.0.tgz" + integrity sha512-el3mYTgx13ZAPPirSVsHqFzl+BBBDrXvbySvPGFnQcTI4iNslrPaFq4muTkLZmKlGk4gyFAYUBMH30+HurREyA== dependencies: postcss-value-parser "^4.2.0" postcss-minify-gradients@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/postcss-minify-gradients/-/postcss-minify-gradients-5.1.1.tgz" + integrity sha512-VGvXMTpCEo4qHTNSa9A0a3D+dxGFZCYwR6Jokk+/3oB6flu2/PnPXAh2x7x52EkY5xlIHLm+Le8tJxe/7TNhzw== dependencies: colord "^2.9.1" cssnano-utils "^3.1.0" @@ -3685,6 +4267,7 @@ postcss-minify-gradients@^5.1.1: postcss-minify-params@^5.1.4: version "5.1.4" resolved "https://registry.npmjs.org/postcss-minify-params/-/postcss-minify-params-5.1.4.tgz" + integrity sha512-+mePA3MgdmVmv6g+30rn57USjOGSAyuxUmkfiWpzalZ8aiBkdPYjXWtHuwJGm1v5Ojy0Z0LaSYhHaLJQB0P8Jw== dependencies: browserslist "^4.21.4" cssnano-utils "^3.1.0" @@ -3693,16 +4276,19 @@ postcss-minify-params@^5.1.4: postcss-minify-selectors@^5.2.1: version "5.2.1" resolved "https://registry.npmjs.org/postcss-minify-selectors/-/postcss-minify-selectors-5.2.1.tgz" + integrity sha512-nPJu7OjZJTsVUmPdm2TcaiohIwxP+v8ha9NehQ2ye9szv4orirRU3SDdtUmKH+10nzn0bAyOXZ0UEr7OpvLehg== dependencies: postcss-selector-parser "^6.0.5" postcss-modules-extract-imports@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.0.0.tgz" + integrity sha512-bdHleFnP3kZ4NYDhuGlVK+CMrQ/pqUm8bx/oGL93K6gVwiclvX5x0n76fYMKuIGKzlABOy13zsvqjb0f92TEXw== postcss-modules-local-by-default@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.0.0.tgz" + integrity sha512-sT7ihtmGSF9yhm6ggikHdV0hlziDTX7oFoXtuVWeDd3hHObNkcHRo9V3yg7vCAY7cONyxJC/XXCmmiHHcvX7bQ== dependencies: icss-utils "^5.0.0" postcss-selector-parser "^6.0.2" @@ -3711,52 +4297,61 @@ postcss-modules-local-by-default@^4.0.0: postcss-modules-scope@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-3.0.0.tgz" + integrity sha512-hncihwFA2yPath8oZ15PZqvWGkWf+XUfQgUGamS4LqoP1anQLOsOJw0vr7J7IwLpoY9fatA2qiGUGmuZL0Iqlg== dependencies: postcss-selector-parser "^6.0.4" postcss-modules-values@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz" + integrity sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ== dependencies: icss-utils "^5.0.0" postcss-normalize-charset@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-normalize-charset/-/postcss-normalize-charset-5.1.0.tgz" + integrity sha512-mSgUJ+pd/ldRGVx26p2wz9dNZ7ji6Pn8VWBajMXFf8jk7vUoSrZ2lt/wZR7DtlZYKesmZI680qjr2CeFF2fbUg== postcss-normalize-display-values@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-normalize-display-values/-/postcss-normalize-display-values-5.1.0.tgz" + integrity sha512-WP4KIM4o2dazQXWmFaqMmcvsKmhdINFblgSeRgn8BJ6vxaMyaJkwAzpPpuvSIoG/rmX3M+IrRZEz2H0glrQNEA== dependencies: postcss-value-parser "^4.2.0" postcss-normalize-positions@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/postcss-normalize-positions/-/postcss-normalize-positions-5.1.1.tgz" + integrity sha512-6UpCb0G4eofTCQLFVuI3EVNZzBNPiIKcA1AKVka+31fTVySphr3VUgAIULBhxZkKgwLImhzMR2Bw1ORK+37INg== dependencies: postcss-value-parser "^4.2.0" postcss-normalize-repeat-style@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-5.1.1.tgz" + integrity sha512-mFpLspGWkQtBcWIRFLmewo8aC3ImN2i/J3v8YCFUwDnPu3Xz4rLohDO26lGjwNsQxB3YF0KKRwspGzE2JEuS0g== dependencies: postcss-value-parser "^4.2.0" postcss-normalize-string@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-normalize-string/-/postcss-normalize-string-5.1.0.tgz" + integrity sha512-oYiIJOf4T9T1N4i+abeIc7Vgm/xPCGih4bZz5Nm0/ARVJ7K6xrDlLwvwqOydvyL3RHNf8qZk6vo3aatiw/go3w== dependencies: postcss-value-parser "^4.2.0" postcss-normalize-timing-functions@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-5.1.0.tgz" + integrity sha512-DOEkzJ4SAXv5xkHl0Wa9cZLF3WCBhF3o1SKVxKQAa+0pYKlueTpCgvkFAHfk+Y64ezX9+nITGrDZeVGgITJXjg== dependencies: postcss-value-parser "^4.2.0" postcss-normalize-unicode@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/postcss-normalize-unicode/-/postcss-normalize-unicode-5.1.1.tgz" + integrity sha512-qnCL5jzkNUmKVhZoENp1mJiGNPcsJCs1aaRmURmeJGES23Z/ajaln+EPTD+rBeNkSryI+2WTdW+lwcVdOikrpA== dependencies: browserslist "^4.21.4" postcss-value-parser "^4.2.0" @@ -3764,6 +4359,7 @@ postcss-normalize-unicode@^5.1.1: postcss-normalize-url@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-normalize-url/-/postcss-normalize-url-5.1.0.tgz" + integrity sha512-5upGeDO+PVthOxSmds43ZeMeZfKH+/DKgGRD7TElkkyS46JXAUhMzIKiCa7BabPeIy3AQcTkXwVVN7DbqsiCew== dependencies: normalize-url "^6.0.1" postcss-value-parser "^4.2.0" @@ -3771,12 +4367,14 @@ postcss-normalize-url@^5.1.0: postcss-normalize-whitespace@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/postcss-normalize-whitespace/-/postcss-normalize-whitespace-5.1.1.tgz" + integrity sha512-83ZJ4t3NUDETIHTa3uEg6asWjSBYL5EdkVB0sDncx9ERzOKBVJIUeDO9RyA9Zwtig8El1d79HBp0JEi8wvGQnA== dependencies: postcss-value-parser "^4.2.0" postcss-ordered-values@^5.1.3: version "5.1.3" resolved "https://registry.npmjs.org/postcss-ordered-values/-/postcss-ordered-values-5.1.3.tgz" + integrity sha512-9UO79VUhPwEkzbb3RNpqqghc6lcYej1aveQteWY+4POIwlqkYE21HKWaLDF6lWNuqCobEAyTovVhtI32Rbv2RQ== dependencies: cssnano-utils "^3.1.0" postcss-value-parser "^4.2.0" @@ -3784,6 +4382,7 @@ postcss-ordered-values@^5.1.3: postcss-reduce-initial@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/postcss-reduce-initial/-/postcss-reduce-initial-5.1.1.tgz" + integrity sha512-//jeDqWcHPuXGZLoolFrUXBDyuEGbr9S2rMo19bkTIjBQ4PqkaO+oI8wua5BOUxpfi97i3PCoInsiFIEBfkm9w== dependencies: browserslist "^4.21.4" caniuse-api "^3.0.0" @@ -3791,12 +4390,14 @@ postcss-reduce-initial@^5.1.1: postcss-reduce-transforms@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-reduce-transforms/-/postcss-reduce-transforms-5.1.0.tgz" + integrity sha512-2fbdbmgir5AvpW9RLtdONx1QoYG2/EtqpNQbFASDlixBbAYuTcJ0dECwlqNqH7VbaUnEnh8SrxOe2sRIn24XyQ== dependencies: postcss-value-parser "^4.2.0" postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5, postcss-selector-parser@^6.0.9: version "6.0.11" resolved "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz" + integrity sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g== dependencies: cssesc "^3.0.0" util-deprecate "^1.0.2" @@ -3804,6 +4405,7 @@ postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector postcss-svgo@^5.1.0: version "5.1.0" resolved "https://registry.npmjs.org/postcss-svgo/-/postcss-svgo-5.1.0.tgz" + integrity sha512-D75KsH1zm5ZrHyxPakAxJWtkyXew5qwS70v56exwvw542d9CRtTo78K0WeFxZB4G7JXKKMbEZtZayTGdIky/eA== dependencies: postcss-value-parser "^4.2.0" svgo "^2.7.0" @@ -3811,16 +4413,19 @@ postcss-svgo@^5.1.0: postcss-unique-selectors@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/postcss-unique-selectors/-/postcss-unique-selectors-5.1.1.tgz" + integrity sha512-5JiODlELrz8L2HwxfPnhOWZYWDxVHWL83ufOv84NrcgipI7TaeRsatAhK4Tr2/ZiYldpK/wBvw5BD3qfaK96GA== dependencies: postcss-selector-parser "^6.0.5" postcss-value-parser@^4.1.0, postcss-value-parser@^4.2.0: version "4.2.0" resolved "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz" + integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== postcss@^7.0.36: version "7.0.39" resolved "https://registry.npmjs.org/postcss/-/postcss-7.0.39.tgz" + integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA== dependencies: picocolors "^0.2.1" source-map "^0.6.1" @@ -3828,6 +4433,7 @@ postcss@^7.0.36: postcss@^8.2.6, postcss@^8.3.5, postcss@^8.4.18: version "8.4.19" resolved "https://registry.npmjs.org/postcss/-/postcss-8.4.19.tgz" + integrity sha512-h+pbPsyhlYj6N2ozBmHhHrs9DzGmbaarbLvWipMRO7RLS+v4onj26MPFXA5OBYFxyqYhUJK456SwDcY9H2/zsA== dependencies: nanoid "^3.3.4" picocolors "^1.0.0" @@ -3836,10 +4442,12 @@ postcss@^8.2.6, postcss@^8.3.5, postcss@^8.4.18: "prettier@^1.18.2 || ^2.0.0": version "2.8.0" resolved "https://registry.npmjs.org/prettier/-/prettier-2.8.0.tgz" + integrity sha512-9Lmg8hTFZKG0Asr/kW9Bp8tJjRVluO8EJQVfY2T7FMw9T5jy4I/Uvx0Rca/XWf50QQ1/SS48+6IJWnrb+2yemA== pretty-error@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/pretty-error/-/pretty-error-4.0.0.tgz" + integrity sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw== dependencies: lodash "^4.17.20" renderkid "^3.0.0" @@ -3847,10 +4455,12 @@ pretty-error@^4.0.0: process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz" + integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== progress-webpack-plugin@^1.0.12: version "1.0.16" resolved "https://registry.npmjs.org/progress-webpack-plugin/-/progress-webpack-plugin-1.0.16.tgz" + integrity sha512-sdiHuuKOzELcBANHfrupYo+r99iPRyOnw15qX+rNlVUqXGfjXdH4IgxriKwG1kNJwVswKQHMdj1hYZMcb9jFaA== dependencies: chalk "^2.1.0" figures "^2.0.0" @@ -3859,6 +4469,7 @@ progress-webpack-plugin@^1.0.12: proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz" + integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== dependencies: forwarded "0.2.0" ipaddr.js "1.9.1" @@ -3866,10 +4477,12 @@ proxy-addr@~2.0.7: pseudomap@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz" + integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= pump@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz" + integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== dependencies: end-of-stream "^1.1.0" once "^1.3.1" @@ -3877,30 +4490,36 @@ pump@^3.0.0: punycode@^2.1.0: version "2.1.1" resolved "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== qs@6.11.0: version "6.11.0" resolved "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz" + integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== dependencies: side-channel "^1.0.4" queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz" + integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== randombytes@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz" + integrity sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ== dependencies: safe-buffer "^5.1.0" range-parser@^1.2.1, range-parser@~1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== raw-body@2.5.1: version "2.5.1" resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz" + integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== dependencies: bytes "3.1.2" http-errors "2.0.0" @@ -3910,6 +4529,7 @@ raw-body@2.5.1: read-pkg-up@^7.0.1: version "7.0.1" resolved "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-7.0.1.tgz" + integrity sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg== dependencies: find-up "^4.1.0" read-pkg "^5.2.0" @@ -3918,6 +4538,7 @@ read-pkg-up@^7.0.1: read-pkg@^5.1.1, read-pkg@^5.2.0: version "5.2.0" resolved "https://registry.npmjs.org/read-pkg/-/read-pkg-5.2.0.tgz" + integrity sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg== dependencies: "@types/normalize-package-data" "^2.4.0" normalize-package-data "^2.5.0" @@ -3927,6 +4548,7 @@ read-pkg@^5.1.1, read-pkg@^5.2.0: readable-stream@^2.0.1: version "2.3.7" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz" + integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== dependencies: core-util-is "~1.0.0" inherits "~2.0.3" @@ -3939,6 +4561,7 @@ readable-stream@^2.0.1: readable-stream@^3.0.6, readable-stream@^3.4.0: version "3.6.0" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz" + integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== dependencies: inherits "^2.0.3" string_decoder "^1.1.1" @@ -3947,32 +4570,38 @@ readable-stream@^3.0.6, readable-stream@^3.4.0: readdirp@~3.6.0: version "3.6.0" resolved "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz" + integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== dependencies: picomatch "^2.2.1" regenerate-unicode-properties@^10.1.0: version "10.1.0" resolved "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.1.0.tgz" + integrity sha512-d1VudCLoIGitcU/hEg2QqvyGZQmdC0Lf8BqdOMXGFSvJP4bNV1+XqbPQeHHLD51Jh4QJJ225dlIFvY4Ly6MXmQ== dependencies: regenerate "^1.4.2" regenerate@^1.4.2: version "1.4.2" resolved "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz" + integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A== regenerator-runtime@^0.13.11: version "0.13.11" resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz" + integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== regenerator-transform@^0.15.1: version "0.15.1" resolved "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.1.tgz" + integrity sha512-knzmNAcuyxV+gQCufkYcvOqX/qIIfHLv0u5x79kRxuGojfYVky1f15TzZEu2Avte8QGepvUNTnLskf8E6X6Vyg== dependencies: "@babel/runtime" "^7.8.4" regexpu-core@^5.2.1: version "5.2.2" resolved "https://registry.npmjs.org/regexpu-core/-/regexpu-core-5.2.2.tgz" + integrity sha512-T0+1Zp2wjF/juXMrMxHxidqGYn8U4R+zleSJhX9tQ1PUsS8a9UtYfbsF9LdiVgNX3kiX8RNaKM42nfSgvFJjmw== dependencies: regenerate "^1.4.2" regenerate-unicode-properties "^10.1.0" @@ -3984,20 +4613,24 @@ regexpu-core@^5.2.1: regjsgen@^0.7.1: version "0.7.1" resolved "https://registry.npmjs.org/regjsgen/-/regjsgen-0.7.1.tgz" + integrity sha512-RAt+8H2ZEzHeYWxZ3H2z6tF18zyyOnlcdaafLrm21Bguj7uZy6ULibiAFdXEtKQY4Sy7wDTwDiOazasMLc4KPA== regjsparser@^0.9.1: version "0.9.1" resolved "https://registry.npmjs.org/regjsparser/-/regjsparser-0.9.1.tgz" + integrity sha512-dQUtn90WanSNl+7mQKcXAgZxvUe7Z0SqXlgzv0za4LwiUhyzBC58yQO3liFoUgu8GiJVInAhJjkj1N0EtQ5nkQ== dependencies: jsesc "~0.5.0" relateurl@^0.2.7: version "0.2.7" resolved "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz" + integrity sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog== renderkid@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/renderkid/-/renderkid-3.0.0.tgz" + integrity sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg== dependencies: css-select "^4.1.3" dom-converter "^0.2.0" @@ -4008,22 +4641,27 @@ renderkid@^3.0.0: require-directory@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz" + integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= require-from-string@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz" + integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== requires-port@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz" + integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== resolve-from@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz" + integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== resolve@^1.10.0, resolve@^1.12.0, resolve@^1.14.2: version "1.22.0" resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.0.tgz" + integrity sha512-Hhtrw0nLeSrFQ7phPp4OOcVjLPIeMnRlr5mcnVuMe7M/7eBn98A3hmFRLoFo3DLZkivSYwhRUJTyPyWAk56WLw== dependencies: is-core-module "^2.8.1" path-parse "^1.0.7" @@ -4032,6 +4670,7 @@ resolve@^1.10.0, resolve@^1.12.0, resolve@^1.14.2: restore-cursor@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz" + integrity sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q== dependencies: onetime "^2.0.0" signal-exit "^3.0.2" @@ -4039,6 +4678,7 @@ restore-cursor@^2.0.0: restore-cursor@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz" + integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== dependencies: onetime "^5.1.0" signal-exit "^3.0.2" @@ -4046,38 +4686,46 @@ restore-cursor@^3.1.0: retry@^0.13.1: version "0.13.1" resolved "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz" + integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg== reusify@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz" + integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== rimraf@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== dependencies: glob "^7.1.3" run-parallel@^1.1.9: version "1.2.0" resolved "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz" + integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== dependencies: queue-microtask "^1.2.2" safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.1.0, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== "safer-buffer@>= 2.1.2 < 3": version "2.1.2" resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== schema-utils@^2.6.5: version "2.7.1" resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz" + integrity sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg== dependencies: "@types/json-schema" "^7.0.5" ajv "^6.12.4" @@ -4086,6 +4734,7 @@ schema-utils@^2.6.5: schema-utils@^3.0.0, schema-utils@^3.1.0, schema-utils@^3.1.1: version "3.1.1" resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz" + integrity sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw== dependencies: "@types/json-schema" "^7.0.8" ajv "^6.12.5" @@ -4094,6 +4743,7 @@ schema-utils@^3.0.0, schema-utils@^3.1.0, schema-utils@^3.1.1: schema-utils@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-4.0.0.tgz" + integrity sha512-1edyXKgh6XnJsJSQ8mKWXnN/BVaIbFMLpouRUrXgVq7WYne5kw3MW7UPhO44uRXQSIpTSXoJbmrR2X0w9kUTyg== dependencies: "@types/json-schema" "^7.0.9" ajv "^8.8.0" @@ -4103,30 +4753,36 @@ schema-utils@^4.0.0: select-hose@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz" + integrity sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg== selfsigned@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/selfsigned/-/selfsigned-2.1.1.tgz" + integrity sha512-GSL3aowiF7wa/WtSFwnUrludWFoNhftq8bUkH9pkzjpN2XSPOAYEgg6e0sS9s0rZwgJzJiQRPU18A6clnoW5wQ== dependencies: node-forge "^1" "semver@2 || 3 || 4 || 5", semver@^5.5.0: version "5.7.1" resolved "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz" + integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: version "6.3.0" resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" + integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== semver@^7.3.4, semver@^7.3.5, semver@^7.3.8: version "7.3.8" resolved "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz" + integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A== dependencies: lru-cache "^6.0.0" send@0.18.0: version "0.18.0" resolved "https://registry.npmjs.org/send/-/send-0.18.0.tgz" + integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== dependencies: debug "2.6.9" depd "2.0.0" @@ -4145,12 +4801,14 @@ send@0.18.0: serialize-javascript@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz" + integrity sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag== dependencies: randombytes "^2.1.0" serve-index@^1.9.1: version "1.9.1" resolved "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz" + integrity sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw== dependencies: accepts "~1.3.4" batch "0.6.1" @@ -4163,6 +4821,7 @@ serve-index@^1.9.1: serve-static@1.15.0: version "1.15.0" resolved "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz" + integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== dependencies: encodeurl "~1.0.2" escape-html "~1.0.3" @@ -4172,44 +4831,53 @@ serve-static@1.15.0: setprototypeof@1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz" + integrity sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ== setprototypeof@1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz" + integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== shallow-clone@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz" + integrity sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA== dependencies: kind-of "^6.0.2" shebang-command@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz" + integrity sha1-RKrGW2lbAzmJaMOfNj/uXer98eo= dependencies: shebang-regex "^1.0.0" shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== dependencies: shebang-regex "^3.0.0" shebang-regex@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz" + integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM= shebang-regex@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== shell-quote@^1.6.1: version "1.7.3" resolved "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.3.tgz" + integrity sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw== side-channel@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== dependencies: call-bind "^1.0.0" get-intrinsic "^1.0.2" @@ -4218,10 +4886,12 @@ side-channel@^1.0.4: signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3: version "3.0.7" resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz" + integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== sirv@^1.0.7: version "1.0.19" resolved "https://registry.npmjs.org/sirv/-/sirv-1.0.19.tgz" + integrity sha512-JuLThK3TnZG1TAKDwNIqNq6QA2afLOCcm+iE8D1Kj3GA40pSPsxQjjJl0J8X3tsR7T+CP1GavpzLwYkgVLWrZQ== dependencies: "@polka/url" "^1.0.0-next.20" mrmime "^1.0.0" @@ -4230,10 +4900,12 @@ sirv@^1.0.7: slash@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz" + integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== socket.io-client@^4.5.1: version "4.5.1" resolved "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.5.1.tgz" + integrity sha512-e6nLVgiRYatS+AHXnOnGi4ocOpubvOUCGhyWw8v+/FxW8saHkinG6Dfhi9TU0Kt/8mwJIAASxvw6eujQmjdZVA== dependencies: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.2" @@ -4243,6 +4915,7 @@ socket.io-client@^4.5.1: socket.io-parser@~4.2.0: version "4.2.2" resolved "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.2.tgz" + integrity sha512-DJtziuKypFkMMHCm2uIshOYC7QaylbtzQwiMYDuCKy3OPkjLzu4B2vAhTlqipRHHzrI0NJeBAizTK7X+6m1jVw== dependencies: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.1" @@ -4250,6 +4923,7 @@ socket.io-parser@~4.2.0: sockjs@^0.3.24: version "0.3.24" resolved "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz" + integrity sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ== dependencies: faye-websocket "^0.11.3" uuid "^8.3.2" @@ -4258,10 +4932,12 @@ sockjs@^0.3.24: source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz" + integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== source-map-support@~0.5.20: version "0.5.21" resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz" + integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== dependencies: buffer-from "^1.0.0" source-map "^0.6.0" @@ -4269,10 +4945,12 @@ source-map-support@~0.5.20: source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1: version "0.6.1" resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== spdx-correct@^3.0.0: version "3.1.1" resolved "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz" + integrity sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w== dependencies: spdx-expression-parse "^3.0.0" spdx-license-ids "^3.0.0" @@ -4280,10 +4958,12 @@ spdx-correct@^3.0.0: spdx-exceptions@^2.1.0: version "2.3.0" resolved "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz" + integrity sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A== spdx-expression-parse@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz" + integrity sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q== dependencies: spdx-exceptions "^2.1.0" spdx-license-ids "^3.0.0" @@ -4291,10 +4971,12 @@ spdx-expression-parse@^3.0.0: spdx-license-ids@^3.0.0: version "3.0.12" resolved "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.12.tgz" + integrity sha512-rr+VVSXtRhO4OHbXUiAF7xW3Bo9DuuF6C5jH+q/x15j2jniycgKbxU09Hr0WqlSLUs4i4ltHGXqTe7VHclYWyA== spdy-transport@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz" + integrity sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw== dependencies: debug "^4.1.0" detect-node "^2.0.4" @@ -4306,6 +4988,7 @@ spdy-transport@^3.0.0: spdy@^4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/spdy/-/spdy-4.0.2.tgz" + integrity sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA== dependencies: debug "^4.1.0" handle-thing "^2.0.0" @@ -4316,28 +4999,34 @@ spdy@^4.0.2: ssri@^8.0.1: version "8.0.1" resolved "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz" + integrity sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ== dependencies: minipass "^3.1.1" stable@^0.1.8: version "0.1.8" resolved "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz" + integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w== stackframe@^1.1.1: version "1.2.1" resolved "https://registry.npmjs.org/stackframe/-/stackframe-1.2.1.tgz" + integrity sha512-h88QkzREN/hy8eRdyNhhsO7RSJ5oyTqxxmmn0dzBIMUclZsjpfmrsg81vp8mjjAs2vAZ72nyWxRUwSwmh0e4xg== statuses@2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" + integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== "statuses@>= 1.4.0 < 2": version "1.5.0" resolved "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" + integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA== string-width@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz" + integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw== dependencies: is-fullwidth-code-point "^2.0.0" strip-ansi "^4.0.0" @@ -4345,6 +5034,7 @@ string-width@^2.1.1: string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== dependencies: emoji-regex "^8.0.0" is-fullwidth-code-point "^3.0.0" @@ -4353,38 +5043,45 @@ string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: string_decoder@^1.1.1: version "1.3.0" resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== dependencies: safe-buffer "~5.2.0" string_decoder@~1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== dependencies: safe-buffer "~5.1.0" strip-ansi@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz" + integrity sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow== dependencies: ansi-regex "^3.0.0" strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: ansi-regex "^5.0.1" strip-eof@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz" + integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8= strip-final-newline@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz" + integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== stylehacks@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/stylehacks/-/stylehacks-5.1.1.tgz" + integrity sha512-sBpcd5Hx7G6seo7b1LkpttvTz7ikD0LlH5RmdcBNb6fFR0Fl7LQwHDFr300q4cwUqi+IYrFGmsIHieMBfnN/Bw== dependencies: browserslist "^4.21.4" postcss-selector-parser "^6.0.4" @@ -4392,32 +5089,38 @@ stylehacks@^5.1.1: supports-color@^5.3.0: version "5.5.0" resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz" + integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== dependencies: has-flag "^3.0.0" supports-color@^7.1.0: version "7.2.0" resolved "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== dependencies: has-flag "^4.0.0" supports-color@^8.0.0: version "8.1.1" resolved "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz" + integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== dependencies: has-flag "^4.0.0" supports-preserve-symlinks-flag@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== svg-tags@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz" + integrity sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA== svgo@^2.7.0: version "2.8.0" resolved "https://registry.npmjs.org/svgo/-/svgo-2.8.0.tgz" + integrity sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg== dependencies: "@trysound/sax" "0.2.0" commander "^7.2.0" @@ -4430,10 +5133,12 @@ svgo@^2.7.0: tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0: version "2.2.1" resolved "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz" + integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== terser-webpack-plugin@^5.1.1, terser-webpack-plugin@^5.1.3: version "5.3.6" resolved "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.6.tgz" + integrity sha512-kfLFk+PoLUQIbLmB1+PZDMRSZS99Mp+/MHqDNmMA6tOItzRt+Npe3E+fsMs5mfcM0wCtrrdU387UnV+vnSffXQ== dependencies: "@jridgewell/trace-mapping" "^0.3.14" jest-worker "^27.4.5" @@ -4444,6 +5149,7 @@ terser-webpack-plugin@^5.1.1, terser-webpack-plugin@^5.1.3: terser@^5.10.0, terser@^5.14.1: version "5.16.1" resolved "https://registry.npmjs.org/terser/-/terser-5.16.1.tgz" + integrity sha512-xvQfyfA1ayT0qdK47zskQgRZeWLoOQ8JQ6mIgRGVNwZKdQMU+5FkCBjmv4QjcrTzyZquRw2FVtlJSRUmMKQslw== dependencies: "@jridgewell/source-map" "^0.3.2" acorn "^8.5.0" @@ -4453,18 +5159,21 @@ terser@^5.10.0, terser@^5.14.1: thenify-all@^1.0.0: version "1.6.0" resolved "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz" + integrity sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY= dependencies: thenify ">= 3.1.0 < 4" "thenify@>= 3.1.0 < 4": version "3.3.1" resolved "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz" + integrity sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw== dependencies: any-promise "^1.0.0" thread-loader@^3.0.0: version "3.0.4" resolved "https://registry.npmjs.org/thread-loader/-/thread-loader-3.0.4.tgz" + integrity sha512-ByaL2TPb+m6yArpqQUZvP+5S1mZtXsEP7nWKKlAUTm7fCml8kB5s1uI3+eHRP2bk5mVYfRSBI7FFf+tWEyLZwA== dependencies: json-parse-better-errors "^1.0.2" loader-runner "^4.1.0" @@ -4475,44 +5184,54 @@ thread-loader@^3.0.0: thunky@^1.0.2: version "1.1.0" resolved "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz" + integrity sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA== to-fast-properties@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz" + integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4= to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz" + integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== dependencies: is-number "^7.0.0" toidentifier@1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz" + integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== totalist@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/totalist/-/totalist-1.1.0.tgz" + integrity sha512-gduQwd1rOdDMGxFG1gEvhV88Oirdo2p+KjoYFU7k2g+i7n6AFFbDQ5kMPUsW0pNbfQsB/cwXvT1i4Bue0s9g5g== tr46@~0.0.3: version "0.0.3" resolved "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz" + integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= tslib@^2.0.3: version "2.4.1" resolved "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz" + integrity sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA== type-fest@^0.6.0: version "0.6.0" resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.6.0.tgz" + integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg== type-fest@^0.8.1: version "0.8.1" resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz" + integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== type-is@~1.6.18: version "1.6.18" resolved "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz" + integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== dependencies: media-typer "0.3.0" mime-types "~2.1.24" @@ -4520,10 +5239,12 @@ type-is@~1.6.18: unicode-canonical-property-names-ecmascript@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz" + integrity sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ== unicode-match-property-ecmascript@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz" + integrity sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q== dependencies: unicode-canonical-property-names-ecmascript "^2.0.0" unicode-property-aliases-ecmascript "^2.0.0" @@ -4531,22 +5252,27 @@ unicode-match-property-ecmascript@^2.0.0: unicode-match-property-value-ecmascript@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.1.0.tgz" + integrity sha512-qxkjQt6qjg/mYscYMC0XKRn3Rh0wFPlfxB0xkt9CfyTvpX1Ra0+rAmdX2QyAobptSEvuy4RtpPRui6XkV+8wjA== unicode-property-aliases-ecmascript@^2.0.0: version "2.1.0" resolved "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz" + integrity sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w== universalify@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz" + integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" + integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== update-browserslist-db@^1.0.9: version "1.0.10" resolved "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz" + integrity sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ== dependencies: escalade "^3.1.1" picocolors "^1.0.0" @@ -4554,28 +5280,34 @@ update-browserslist-db@^1.0.9: uri-js@^4.2.2: version "4.4.1" resolved "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== dependencies: punycode "^2.1.0" util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" + integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== utila@~0.4: version "0.4.0" resolved "https://registry.npmjs.org/utila/-/utila-0.4.0.tgz" + integrity sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA== utils-merge@1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" + integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== uuid@^8.3.2: version "8.3.2" resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== validate-npm-package-license@^3.0.1: version "3.0.4" resolved "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz" + integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew== dependencies: spdx-correct "^3.0.0" spdx-expression-parse "^3.0.0" @@ -4583,22 +5315,27 @@ validate-npm-package-license@^3.0.1: vary@~1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" + integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== vue-functional-data-merge@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/vue-functional-data-merge/-/vue-functional-data-merge-3.1.0.tgz" + integrity sha512-leT4kdJVQyeZNY1kmnS1xiUlQ9z1B/kdBFCILIjYYQDqZgLqCLa0UhjSSeRX6c3mUe6U5qYeM8LrEqkHJ1B4LA== vue-hot-reload-api@^2.3.0: version "2.3.4" resolved "https://registry.npmjs.org/vue-hot-reload-api/-/vue-hot-reload-api-2.3.4.tgz" + integrity sha512-BXq3jwIagosjgNVae6tkHzzIk6a8MHFtzAdwhnV5VlvPTFxDCvIttgSiHWjdGoTJvXtmRu5HacExfdarRcFhog== vue-i18n@^8.24.4: version "8.24.4" resolved "https://registry.npmjs.org/vue-i18n/-/vue-i18n-8.24.4.tgz" + integrity sha512-RZE94WUAGxEiBAANxQ0pptbRwDkNKNSXl3fnJslpFOxVMF6UkUtMDSuYGuW2blDrVgweIXVpethOVkYoNNT9xw== vue-loader@^17.0.0: version "17.0.1" resolved "https://registry.npmjs.org/vue-loader/-/vue-loader-17.0.1.tgz" + integrity sha512-/OOyugJnImKCkAKrAvdsWMuwoCqGxWT5USLsjohzWbMgOwpA5wQmzQiLMzZd7DjhIfunzAGIApTOgIylz/kwcg== dependencies: chalk "^4.1.0" hash-sum "^2.0.0" @@ -4607,10 +5344,12 @@ vue-loader@^17.0.0: vue-router@^3.5.4: version "3.5.4" resolved "https://registry.npmjs.org/vue-router/-/vue-router-3.5.4.tgz" + integrity sha512-x+/DLAJZv2mcQ7glH2oV9ze8uPwcI+H+GgTgTmb5I55bCgY3+vXWIsqbYUzbBSZnwFHEJku4eoaH/x98veyymQ== vue-style-loader@^4.1.0, vue-style-loader@^4.1.3: version "4.1.3" resolved "https://registry.npmjs.org/vue-style-loader/-/vue-style-loader-4.1.3.tgz" + integrity sha512-sFuh0xfbtpRlKfm39ss/ikqs9AbKCoXZBpHeVZ8Tx650o0k0q/YCM7FRvigtxpACezfq6af+a7JeqVTWvncqDg== dependencies: hash-sum "^1.0.2" loader-utils "^1.0.2" @@ -4618,6 +5357,7 @@ vue-style-loader@^4.1.0, vue-style-loader@^4.1.3: vue-template-compiler@^2.6.11: version "2.6.14" resolved "https://registry.npmjs.org/vue-template-compiler/-/vue-template-compiler-2.6.14.tgz" + integrity sha512-ODQS1SyMbjKoO1JBJZojSw6FE4qnh9rIpUZn2EUT86FKizx9uH5z6uXiIrm4/Nb/gwxTi/o17ZDEGWAXHvtC7g== dependencies: de-indent "^1.0.2" he "^1.1.0" @@ -4625,18 +5365,22 @@ vue-template-compiler@^2.6.11: vue-template-es2015-compiler@^1.9.0: version "1.9.1" resolved "https://registry.npmjs.org/vue-template-es2015-compiler/-/vue-template-es2015-compiler-1.9.1.tgz" + integrity sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw== vue@^2.6.14: version "2.6.14" resolved "https://registry.npmjs.org/vue/-/vue-2.6.14.tgz" + integrity sha512-x2284lgYvjOMj3Za7kqzRcUSxBboHqtgRE2zlos1qWaOye5yUmHn42LB1250NJBLRwEcdrB0JRwyPTEPhfQjiQ== vuex@^3.4.0: version "3.6.2" resolved "https://registry.npmjs.org/vuex/-/vuex-3.6.2.tgz" + integrity sha512-ETW44IqCgBpVomy520DT5jf8n0zoCac+sxWnn+hMe/CzaSejb/eVw2YToiXYX+Ex/AuHHia28vWTq4goAexFbw== watchpack@^2.4.0: version "2.4.0" resolved "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz" + integrity sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg== dependencies: glob-to-regexp "^0.4.1" graceful-fs "^4.1.2" @@ -4644,22 +5388,26 @@ watchpack@^2.4.0: wbuf@^1.1.0, wbuf@^1.7.3: version "1.7.3" resolved "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz" + integrity sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA== dependencies: minimalistic-assert "^1.0.0" wcwidth@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz" + integrity sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg== dependencies: defaults "^1.0.3" webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz" + integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= webpack-bundle-analyzer@^4.4.0: version "4.7.0" resolved "https://registry.npmjs.org/webpack-bundle-analyzer/-/webpack-bundle-analyzer-4.7.0.tgz" + integrity sha512-j9b8ynpJS4K+zfO5GGwsAcQX4ZHpWV+yRiHDiL+bE0XHJ8NiPYLTNVQdlFYWxtpg9lfAQNlwJg16J9AJtFSXRg== dependencies: acorn "^8.0.4" acorn-walk "^8.0.0" @@ -4674,6 +5422,7 @@ webpack-bundle-analyzer@^4.4.0: webpack-chain@^6.5.1: version "6.5.1" resolved "https://registry.npmjs.org/webpack-chain/-/webpack-chain-6.5.1.tgz" + integrity sha512-7doO/SRtLu8q5WM0s7vPKPWX580qhi0/yBHkOxNkv50f6qB76Zy9o2wRTrrPULqYTvQlVHuvbA8v+G5ayuUDsA== dependencies: deepmerge "^1.5.2" javascript-stringify "^2.0.1" @@ -4681,6 +5430,7 @@ webpack-chain@^6.5.1: webpack-dev-middleware@^5.3.1: version "5.3.3" resolved "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz" + integrity sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA== dependencies: colorette "^2.0.10" memfs "^3.4.3" @@ -4691,6 +5441,7 @@ webpack-dev-middleware@^5.3.1: webpack-dev-server@^4.7.3: version "4.11.1" resolved "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.11.1.tgz" + integrity sha512-lILVz9tAUy1zGFwieuaQtYiadImb5M3d+H+L1zDYalYoDl0cksAB1UNyuE5MMWJrG6zR1tXkCP2fitl7yoUJiw== dependencies: "@types/bonjour" "^3.5.9" "@types/connect-history-api-fallback" "^1.3.5" @@ -4725,6 +5476,7 @@ webpack-dev-server@^4.7.3: webpack-merge@^5.7.3: version "5.8.0" resolved "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.8.0.tgz" + integrity sha512-/SaI7xY0831XwP6kzuwhKWVKDP9t1QY1h65lAFLbZqMPIuYcD9QAW4u9STIbU9kaJbPBB/geU/gLr1wDjOhQ+Q== dependencies: clone-deep "^4.0.1" wildcard "^2.0.0" @@ -4732,14 +5484,17 @@ webpack-merge@^5.7.3: webpack-sources@^3.2.3: version "3.2.3" resolved "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz" + integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== webpack-virtual-modules@^0.4.2: version "0.4.6" resolved "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.4.6.tgz" + integrity sha512-5tyDlKLqPfMqjT3Q9TAqf2YqjwmnUleZwzJi1A5qXnlBCdj2AtOJ6wAWdglTIDOPgOiOrXeBeFcsQ8+aGQ6QbA== webpack@^5.54.0: version "5.75.0" resolved "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz" + integrity sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ== dependencies: "@types/eslint-scope" "^3.7.3" "@types/estree" "^0.0.51" @@ -4769,6 +5524,7 @@ webpack@^5.54.0: websocket-driver@>=0.5.1, websocket-driver@^0.7.4: version "0.7.4" resolved "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz" + integrity sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg== dependencies: http-parser-js ">=0.5.1" safe-buffer ">=5.1.0" @@ -4777,14 +5533,17 @@ websocket-driver@>=0.5.1, websocket-driver@^0.7.4: websocket-extensions@>=0.1.1: version "0.1.4" resolved "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz" + integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== whatwg-fetch@^3.6.2: version "3.6.2" resolved "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.2.tgz" + integrity sha512-bJlen0FcuU/0EMLrdbJ7zOnW6ITZLrZMIarMUVmdKtsGvZna8vxKYaexICWPfZ8qwf9fzNq+UEIZrnSaApt6RA== whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz" + integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= dependencies: tr46 "~0.0.3" webidl-conversions "^3.0.0" @@ -4792,22 +5551,26 @@ whatwg-url@^5.0.0: which@^1.2.9: version "1.3.1" resolved "https://registry.npmjs.org/which/-/which-1.3.1.tgz" + integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== dependencies: isexe "^2.0.0" which@^2.0.1: version "2.0.2" resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz" + integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== dependencies: isexe "^2.0.0" wildcard@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz" + integrity sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw== wrap-ansi@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-3.0.1.tgz" + integrity sha512-iXR3tDXpbnTpzjKSylUJRkLuOrEC7hwEB221cgn6wtF8wpmz28puFXAEfPT5zrjM3wahygB//VuWEr1vTkDcNQ== dependencies: string-width "^2.1.1" strip-ansi "^4.0.0" @@ -4815,6 +5578,7 @@ wrap-ansi@^3.0.1: wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== dependencies: ansi-styles "^4.0.0" string-width "^4.1.0" @@ -4823,46 +5587,57 @@ wrap-ansi@^7.0.0: wrappy@1: version "1.0.2" resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" + integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= ws@^7.3.1: version "7.5.9" resolved "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz" + integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== ws@^8.4.2: version "8.11.0" resolved "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz" + integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg== ws@~8.2.3: version "8.2.3" resolved "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz" + integrity sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA== xmlhttprequest-ssl@~2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz" + integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A== y18n@^5.0.5: version "5.0.8" resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz" + integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== yallist@^2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz" + integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI= yallist@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== yaml@^1.10.0, yaml@^1.10.2: version "1.10.2" resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz" + integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== yargs-parser@^20.2.2: version "20.2.9" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz" + integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== yargs@^16.0.0: version "16.2.0" resolved "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz" + integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== dependencies: cliui "^7.0.2" escalade "^3.1.1" diff --git a/sist2-admin/sist2_admin/app.py b/sist2-admin/sist2_admin/app.py index ef254d6..360be0b 100644 --- a/sist2-admin/sist2_admin/app.py +++ b/sist2-admin/sist2_admin/app.py @@ -251,7 +251,7 @@ def check_es_version(es_url: str, insecure: bool): def start_frontend_(frontend: Sist2Frontend): - frontend.web_options.indices = list(map(lambda j: db["jobs"][j].last_index, frontend.jobs)) + frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs)) pid = sist2.web(frontend.web_options, frontend.name) RUNNING_FRONTENDS[frontend.name] = pid @@ -378,6 +378,9 @@ if __name__ == '__main__': if db["sist2_admin"]["info"]["version"] == "1": logger.info("Migrating to v2 database schema") migrate_v1_to_v2(db) + if db["sist2_admin"]["info"]["version"] == "2": + logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.") + exit(-1) start_frontends() cron.initialize(db, _run_job) diff --git a/sist2-admin/sist2_admin/jobs.py b/sist2-admin/sist2_admin/jobs.py index cbf29b1..639612e 100644 --- a/sist2-admin/sist2_admin/jobs.py +++ b/sist2-admin/sist2_admin/jobs.py @@ -1,23 +1,21 @@ import json import logging import os.path -import shutil import signal import uuid from datetime import datetime from enum import Enum -from hashlib import md5 from logging import FileHandler from threading import Lock, Thread from time import sleep from uuid import uuid4, UUID from hexlib.db import PersistentState -from pydantic import BaseModel, validator +from pydantic import BaseModel from config import logger, LOG_FOLDER from notifications import Notifications -from sist2 import ScanOptions, IndexOptions, Sist2, Sist2Index +from sist2 import ScanOptions, IndexOptions, Sist2 from state import RUNNING_FRONTENDS from web import Sist2Frontend @@ -38,7 +36,8 @@ class Sist2Job(BaseModel): schedule_enabled: bool = False previous_index: str = None - last_index: str = None + index_path: str = None + previous_index_path: str = None last_index_date: datetime = None status: JobStatus = JobStatus("created") last_modified: datetime @@ -124,10 +123,10 @@ class Sist2ScanTask(Sist2Task): self.job.scan_options.name = self.job.name - if self.job.last_index and os.path.exists(self.job.last_index) and not self.job.do_full_scan: - self.job.scan_options.incremental = self.job.last_index + if self.job.index_path is not None and not self.job.do_full_scan: + self.job.scan_options.output = self.job.index_path else: - self.job.scan_options.incremental = None + self.job.scan_options.output = None def set_pid(pid): self.pid = pid @@ -139,19 +138,26 @@ class Sist2ScanTask(Sist2Task): self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"})) logger.info(f"Task {self.display_name} failed ({return_code})") else: - index = Sist2Index(self.job.scan_options.output) - - # Save latest index - self.job.previous_index = self.job.last_index - - self.job.last_index = index.path + self.job.index_path = self.job.scan_options.output self.job.last_index_date = datetime.now() self.job.do_full_scan = False db["jobs"][self.job.name] = self.job - self._logger.info(json.dumps({"sist2-admin": f"Save last_index={self.job.last_index}"})) + self._logger.info(json.dumps({"sist2-admin": f"Save last_index_date={self.job.last_index_date}"})) logger.info(f"Completed {self.display_name} ({return_code=})") + # Remove old index + if return_code == 0: + if self.job.previous_index_path is not None and self.job.previous_index_path != self.job.index_path: + self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index_path=}"})) + try: + os.remove(self.job.previous_index_path) + except FileNotFoundError: + pass + + self.job.previous_index_path = self.job.index_path + db["jobs"][self.job.name] = self.job + return return_code @@ -173,18 +179,11 @@ class Sist2IndexTask(Sist2Task): ok = return_code == 0 if ok: - # Remove old index - if self.job.previous_index is not None: - self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index=}"})) - try: - shutil.rmtree(self.job.previous_index) - except FileNotFoundError: - pass - self.restart_running_frontends(db, sist2) # Update status self.job.status = JobStatus("indexed") if ok else JobStatus("failed") + self.job.previous_index_path = self.job.index_path db["jobs"][self.job.name] = self.job self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"})) @@ -198,13 +197,16 @@ class Sist2IndexTask(Sist2Task): frontend = db["frontends"][frontend_name] frontend: Sist2Frontend - os.kill(pid, signal.SIGTERM) + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass try: os.wait() except ChildProcessError: pass - frontend.web_options.indices = map(lambda j: db["jobs"][j].last_index, frontend.jobs) + frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs) pid = sist2.web(frontend.web_options, frontend.name) RUNNING_FRONTENDS[frontend_name] = pid diff --git a/sist2-admin/sist2_admin/sist2.py b/sist2-admin/sist2_admin/sist2.py index d5eebde..ae368c4 100644 --- a/sist2-admin/sist2_admin/sist2.py +++ b/sist2-admin/sist2_admin/sist2.py @@ -2,7 +2,6 @@ import datetime import json import logging import os.path -import traceback from datetime import datetime from io import TextIOWrapper from logging import FileHandler @@ -78,10 +77,10 @@ class IndexOptions(BaseModel): es_url: str = "http://elasticsearch:9200" es_insecure_ssl: bool = False es_index: str = "sist2" - incremental_index: bool = False + incremental_index: bool = True script: str = "" script_file: str = None - batch_size: int = 100 + batch_size: int = 70 def __init__(self, **kwargs): super().__init__(**kwargs) @@ -110,9 +109,8 @@ ARCHIVE_RECURSE = "recurse" class ScanOptions(BaseModel): path: str threads: int = 1 - mem_throttle: int = 0 thumbnail_quality: int = 2 - thumbnail_size: int = 500 + thumbnail_size: int = 552 thumbnail_count: int = 1 content_size: int = 32768 depth: int = -1 @@ -128,7 +126,8 @@ class ScanOptions(BaseModel): read_subtitles: bool = False fast_epub: bool = False checksums: bool = False - incremental: str = None + incremental: bool = True + optimize_index: bool = False output: str = None name: str = None rewrite_url: str = None @@ -138,14 +137,15 @@ class ScanOptions(BaseModel): super().__init__(**kwargs) def args(self): - args = ["scan", self.path, f"--threads={self.threads}", f"--mem-throttle={self.mem_throttle}", - f"--thumbnail-quality={self.thumbnail_quality}", f"--thumbnail-count={self.thumbnail_count}", - f"--thumbnail-size={self.thumbnail_size}", f"--content-size={self.content_size}", - f"--output={self.output}", f"--depth={self.depth}", f"--archive={self.archive}", - f"--mem-buffer={self.mem_buffer}"] + args = ["scan", self.path, f"--threads={self.threads}", f"--thumbnail-quality={self.thumbnail_quality}", + f"--thumbnail-count={self.thumbnail_count}", f"--thumbnail-size={self.thumbnail_size}", + f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}", + f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"] if self.incremental: - args.append(f"--incremental={self.incremental}") + args.append(f"--incremental") + if self.optimize_index: + args.append(f"--optimize-index") if self.rewrite_url: args.append(f"--rewrite-url={self.rewrite_url}") if self.name: @@ -235,11 +235,11 @@ class Sist2: def scan(self, options: ScanOptions, logs_cb, set_pid_cb): - output_dir = os.path.join( - self._data_dir, - f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2" - ) - options.output = output_dir + if options.output is None: + options.output = os.path.join( + self._data_dir, + f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2" + ) args = [ self._bin_path, diff --git a/sist2-admin/sist2_admin/state.py b/sist2-admin/sist2_admin/state.py index f8b46a8..12dbff7 100644 --- a/sist2-admin/sist2_admin/state.py +++ b/sist2-admin/sist2_admin/state.py @@ -10,7 +10,7 @@ RUNNING_FRONTENDS: Dict[str, int] = {} TESSERACT_LANGS = get_tesseract_langs() -DB_SCHEMA_VERSION = "2" +DB_SCHEMA_VERSION = "3" from pydantic import BaseModel @@ -76,4 +76,4 @@ def migrate_v1_to_v2(db: PersistentState): db["sist2_admin"]["info"] = { "version": "2" - } \ No newline at end of file + } diff --git a/sist2-vue/src/components/FullThumbnail.vue b/sist2-vue/src/components/FullThumbnail.vue index 5c9bbcf..50836af 100644 --- a/sist2-vue/src/components/FullThumbnail.vue +++ b/sist2-vue/src/components/FullThumbnail.vue @@ -75,7 +75,7 @@ export default { } return (this.currentThumbnailNum === 0) ? `t/${doc._source.index}/${doc._id}` - : `t/${doc._source.index}/${doc._id}${String(thumbnailNum).padStart(4, "0")}`; + : `t/${doc._source.index}/${doc._id}/${String(thumbnailNum).padStart(4, "0")}`; }, humanTime: humanTime, onThumbnailClick() { diff --git a/src/tpool.c b/src/tpool.c index 6cdf31e..5f3b679 100644 --- a/src/tpool.c +++ b/src/tpool.c @@ -277,10 +277,6 @@ void tpool_destroy(tpool_t *pool) { database_close(ProcData.ipc_db, FALSE); - int count = 0; - - LOG_DEBUGF("tpool.c", "Destroyed %d jobs", count); - pthread_mutex_lock(&pool->shm->mutex); pthread_cond_broadcast(&pool->shm->ipc_ctx.has_work_cond); pthread_mutex_unlock(&pool->shm->mutex); diff --git a/src/web/serve.c b/src/web/serve.c index b2b63d7..2c867b2 100644 --- a/src/web/serve.c +++ b/src/web/serve.c @@ -1,7 +1,6 @@ #include "serve.h" #include "src/sist.h" -//#include "src/io/store.h" #include "src/index/elastic.h" #include "src/index/web.h" #include "src/auth0/auth0_c_api.h" @@ -145,7 +144,7 @@ void thumbnail_with_num(struct mg_connection *nc, struct mg_http_message *hm) { *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0'; memcpy(arg_doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN); *(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0'; - memcpy(arg_num, hm->uri.ptr + SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2, 4); + memcpy(arg_num, hm->uri.ptr + SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 3, 4); int num = (int) strtol(arg_num, NULL, 10); diff --git a/third-party/libscan/CMakeLists.txt b/third-party/libscan/CMakeLists.txt index 5bb23af..ee7ca73 100644 --- a/third-party/libscan/CMakeLists.txt +++ b/third-party/libscan/CMakeLists.txt @@ -106,7 +106,7 @@ find_library(MUPDF_LIB NAMES liblibmupdf.a) find_library(CMS_LIB NAMES lcms2) find_library(JAS_LIB NAMES jasper) find_library(GUMBO_LIB NAMES gumbo) -find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/) +find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/11/) find_package(Leptonica CONFIG REQUIRED) find_package(FFMPEG REQUIRED) find_package(libraw CONFIG REQUIRED)