diff --git a/src/cli.c b/src/cli.c index 18ded85..c1eb59a 100644 --- a/src/cli.c +++ b/src/cli.c @@ -20,6 +20,29 @@ scan_args_t *scan_args_create() { return args; } +void scan_args_destroy(scan_args_t *args) { + if (args->name != NULL) { + free(args->name); + } + if (args->path != NULL) { + free(args->path); + } + if (args->output != NULL) { + free(args->output); + } + free(args); +} + +void index_args_destroy(index_args_t *args) { + //todo + free(args); +} + +void web_args_destroy(web_args_t *args) { + //todo + free(args); +} + int scan_args_validate(scan_args_t *args, int argc, const char **argv) { if (argc < 2) { fprintf(stderr, "Required positional argument: PATH.\n"); @@ -113,6 +136,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) { return 1; } else { args->index_path = argv[1]; + free(index_path); } if (args->es_url == NULL) { diff --git a/src/cli.h b/src/cli.h index 0b9a3ef..c1c5aab 100644 --- a/src/cli.h +++ b/src/cli.h @@ -17,6 +17,7 @@ typedef struct scan_args { } scan_args_t; scan_args_t *scan_args_create(); +void scan_args_destroy(scan_args_t *args); int scan_args_validate(scan_args_t *args, int argc, const char **argv); #ifndef SIST_SCAN_ONLY @@ -40,7 +41,10 @@ typedef struct web_args { } web_args_t; index_args_t *index_args_create(); +void index_args_destroy(index_args_t *args); + web_args_t *web_args_create(); +void web_args_destroy(web_args_t *args); int index_args_validate(index_args_t *args, int argc, const char **argv); int web_args_validate(web_args_t *args, int argc, const char **argv); diff --git a/src/ctx.h b/src/ctx.h index a283894..2205804 100644 --- a/src/ctx.h +++ b/src/ctx.h @@ -35,7 +35,7 @@ struct { struct { char *es_url; int index_count; - char* b64credentials; + char *b64credentials; struct index_t indices[16]; } WebCtx; #endif diff --git a/src/io/serialize.c b/src/io/serialize.c index bb992df..ccc52d1 100644 --- a/src/io/serialize.c +++ b/src/io/serialize.c @@ -1,7 +1,7 @@ #include "src/ctx.h" #include "serialize.h" -static __thread int IndexFd = -1; +static __thread int index_fd = -1; typedef struct { unsigned char uuid[16]; @@ -119,13 +119,13 @@ char *get_meta_key_text(enum metakey meta_key) { void write_document(document_t *doc) { - if (IndexFd == -1) { + if (index_fd == -1) { char dstfile[PATH_MAX]; pthread_t self = pthread_self(); snprintf(dstfile, PATH_MAX, "%s_index_%lu", ScanCtx.index.path, self); - IndexFd = open(dstfile, O_CREAT | O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR); + index_fd = open(dstfile, O_CREAT | O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR); - if (IndexFd == -1) { + if (index_fd == -1) { perror("open"); } } @@ -158,13 +158,16 @@ void write_document(document_t *doc) { } dyn_buffer_write_char(&buf, '\n'); - write(IndexFd, buf.buf, buf.cur); + int res = write(index_fd, buf.buf, buf.cur); + if (res == -1) { + perror("write"); + } ScanCtx.stat_index_size += buf.cur; dyn_buffer_destroy(&buf); } -void serializer_cleanup() { - close(IndexFd); +void thread_cleanup() { + close(index_fd); } void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func func) { diff --git a/src/io/serialize.h b/src/io/serialize.h index 2635efd..24a8481 100644 --- a/src/io/serialize.h +++ b/src/io/serialize.h @@ -18,7 +18,7 @@ void incremental_read(GHashTable *table, const char *filepath); /** * Must be called after write_document */ -void serializer_cleanup(); +void thread_cleanup(); void write_index_descriptor(char *path, index_descriptor_t *desc); diff --git a/src/main.c b/src/main.c index 0c00f7c..393fd31 100644 --- a/src/main.c +++ b/src/main.c @@ -19,9 +19,9 @@ static const char *const usage[] = { }; void global_init() { - #ifndef SIST_SCAN_ONLY +#ifndef SIST_SCAN_ONLY curl_global_init(CURL_GLOBAL_NOTHING); - #endif +#endif av_log_set_level(AV_LOG_QUIET); } @@ -93,7 +93,7 @@ void sist2_scan(scan_args_t *args) { printf("Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table)); } - ScanCtx.pool = tpool_create(args->threads, serializer_cleanup); + ScanCtx.pool = tpool_create(args->threads, thread_cleanup); tpool_start(ScanCtx.pool); walk_directory_tree(ScanCtx.index.desc.root); tpool_wait(ScanCtx.pool); @@ -126,6 +126,7 @@ void sist2_scan(scan_args_t *args) { } #ifndef SIST_SCAN_ONLY + void sist2_index(index_args_t *args) { IndexCtx.es_url = args->es_url; @@ -198,6 +199,7 @@ void sist2_web(web_args_t *args) { serve(args->bind, args->port); } + #endif @@ -206,14 +208,14 @@ int main(int argc, const char *argv[]) { global_init(); scan_args_t *scan_args = scan_args_create(); - #ifndef SIST_SCAN_ONLY +#ifndef SIST_SCAN_ONLY index_args_t *index_args = index_args_create(); web_args_t *web_args = web_args_create(); - #endif +#endif int arg_version = 0; - char * common_es_url = NULL; + char *common_es_url = NULL; struct argparse_option options[] = { OPT_HELP(), @@ -233,22 +235,22 @@ int main(int argc, const char *argv[]) { OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."), OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: (name of the directory)"), OPT_INTEGER(0, "depth", &scan_args->depth, "Scan up to DEPTH subdirectories deep. " - "Use 0 to only scan files in PATH. DEFAULT: -1"), + "Use 0 to only scan files in PATH. DEFAULT: -1"), - #ifndef SIST_SCAN_ONLY +#ifndef SIST_SCAN_ONLY OPT_GROUP("Index options"), OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"), OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."), OPT_STRING(0, "script-file", &index_args->script_path, "Path to user script."), OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. " - "(You must use this option the first time you use the index command)"), + "(You must use this option the first time you use the index command)"), OPT_GROUP("Web options"), OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"), OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"), OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"), OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"), - #endif +#endif OPT_END(), }; @@ -263,10 +265,10 @@ int main(int argc, const char *argv[]) { exit(0); } - #ifndef SIST_SCAN_ONLY +#ifndef SIST_SCAN_ONLY web_args->es_url = common_es_url; index_args->es_url = common_es_url; - #endif +#endif if (argc == 0) { argparse_usage(&argparse); @@ -281,7 +283,7 @@ int main(int argc, const char *argv[]) { } - #ifndef SIST_SCAN_ONLY +#ifndef SIST_SCAN_ONLY else if (strcmp(argv[0], "index") == 0) { int err = index_args_validate(index_args, argc, argv); @@ -299,12 +301,20 @@ int main(int argc, const char *argv[]) { sist2_web(web_args); } - #endif +#endif else { fprintf(stderr, "Invalid command: '%s'\n", argv[0]); argparse_usage(&argparse); return 1; } printf("\n"); + + scan_args_destroy(scan_args); + +#ifndef SIST_SCAN_ONLY + index_args_destroy(index_args); + web_args_destroy(web_args); +#endif + return 0; } diff --git a/src/parsing/font.c b/src/parsing/font.c index d2f61dc..69fb980 100644 --- a/src/parsing/font.c +++ b/src/parsing/font.c @@ -1,11 +1,9 @@ #include "font.h" -#include "ft2build.h" -#include "freetype/freetype.h" #include "src/ctx.h" -__thread FT_Library library = NULL; +__thread FT_Library ft_lib = NULL; typedef struct text_dimensions { @@ -139,15 +137,15 @@ void bmp_format(dyn_buffer_t *buf, text_dimensions_t dimensions, const unsigned } void parse_font(const char *buf, size_t buf_len, document_t *doc) { - if (library == NULL) { - FT_Init_FreeType(&library); + if (ft_lib == NULL) { + FT_Init_FreeType(&ft_lib); } if (buf == NULL) { return; } FT_Face face; - FT_Error err = FT_New_Memory_Face(library, (unsigned char *) buf, buf_len, 0, &face); + FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, buf_len, 0, &face); if (err != 0) { return; } diff --git a/src/parsing/parse.c b/src/parsing/parse.c index 571cb81..527586b 100644 --- a/src/parsing/parse.c +++ b/src/parsing/parse.c @@ -44,7 +44,6 @@ void parse(void *arg) { if (Magic == NULL) { Magic = magic_open(MAGIC_MIME_TYPE); - magic_load(Magic, NULL); } doc.filepath = job->filepath; @@ -98,31 +97,33 @@ void parse(void *arg) { int mmime = MAJOR_MIME(doc.mime); - if (!(SHOULD_PARSE(doc.mime))) { + parse_text(bytes_read, &fd, (char *) buf, &doc); - } else if ((mmime == MimeVideo && doc.size >= MIN_VIDEO_SIZE) || - (mmime == MimeImage && doc.size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) { - parse_media(job->filepath, &doc); - - } else if (IS_PDF(doc.mime)) { - void *pdf_buf = read_all(job, (char *) buf, bytes_read, &fd); - parse_pdf(pdf_buf, doc.size, &doc); - - if (pdf_buf != buf && pdf_buf != NULL) { - free(pdf_buf); - } - - } else if (mmime == MimeText && ScanCtx.content_size > 0) { - parse_text(bytes_read, &fd, (char *) buf, &doc); - - } else if (IS_FONT(doc.mime)) { - void *font_buf = read_all(job, (char *) buf, bytes_read, &fd); - parse_font(font_buf, doc.size, &doc); - - if (font_buf != buf && font_buf != NULL) { - free(font_buf); - } - } +// if (!(SHOULD_PARSE(doc.mime))) { +// +// } else if ((mmime == MimeVideo && doc.size >= MIN_VIDEO_SIZE) || +// (mmime == MimeImage && doc.size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) { +// parse_media(job->filepath, &doc); +// +// } else if (IS_PDF(doc.mime)) { +// void *pdf_buf = read_all(job, (char *) buf, bytes_read, &fd); +// parse_pdf(pdf_buf, doc.size, &doc); +// +// if (pdf_buf != buf && pdf_buf != NULL) { +// free(pdf_buf); +// } +// +// } else if (mmime == MimeText && ScanCtx.content_size > 0) { +// parse_text(bytes_read, &fd, (char *) buf, &doc); +// +// } else if (IS_FONT(doc.mime)) { +// void *font_buf = read_all(job, (char *) buf, bytes_read, &fd); +// parse_font(font_buf, doc.size, &doc); +// +// if (font_buf != buf && font_buf != NULL) { +// free(font_buf); +// } +// } write_document(&doc); diff --git a/src/sist.h b/src/sist.h index ef91196..5b46733 100644 --- a/src/sist.h +++ b/src/sist.h @@ -26,6 +26,8 @@ #include #include #include +#include "ft2build.h" +#include "freetype/freetype.h" #ifndef SIST_SCAN_ONLY #include diff --git a/src/tpool.c b/src/tpool.c index 1e7770b..354ad7e 100644 --- a/src/tpool.c +++ b/src/tpool.c @@ -151,6 +151,7 @@ void tpool_wait(tpool_t *pool) { usleep(500000); if (pool->done_cnt == pool->work_cnt) { pool->stop = 1; + usleep(1000000); break; } } @@ -178,7 +179,8 @@ void tpool_destroy(tpool_t *pool) { for (size_t i = 0; i < pool->thread_cnt; i++) { pthread_t thread = pool->threads[i]; if (thread != 0) { - pthread_cancel(thread); + void *_; + pthread_join(thread, &_); } } @@ -218,8 +220,6 @@ tpool_t *tpool_create(size_t thread_cnt, void cleanup_func()) { void tpool_start(tpool_t *pool) { for (size_t i = 0; i < pool->thread_cnt; i++) { - pthread_t thread = pool->threads[i]; - pthread_create(&thread, NULL, tpool_worker, pool); - pthread_detach(thread); + pthread_create(&pool->threads[i], NULL, tpool_worker, pool); } }