simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-16 08:56:45 +00:00
Code Issues Projects Releases Wiki Activity

Cleaner shutdown

Browse Source
This commit is contained in:
simon 2019-11-30 19:59:11 -05:00
parent 4ab2ba1a02
commit 1a1032a8a7
10 changed files with 100 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/cli.c
View File

@ -20,6 +20,29 @@ scan_args_t *scan_args_create() {
return args;
}
void scan_args_destroy(scan_args_t *args) {
if (args->name != NULL) {
free(args->name);
}
if (args->path != NULL) {
free(args->path);
}
if (args->output != NULL) {
free(args->output);
}
free(args);
}
void index_args_destroy(index_args_t *args) {
//todo
free(args);
}
void web_args_destroy(web_args_t *args) {
//todo
free(args);
}
int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n");
@ -113,6 +136,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
return 1;
} else {
args->index_path = argv[1];
free(index_path);
}
if (args->es_url == NULL) {

4
src/cli.h
View File

@ -17,6 +17,7 @@ typedef struct scan_args {
} scan_args_t;
scan_args_t *scan_args_create();
void scan_args_destroy(scan_args_t *args);
int scan_args_validate(scan_args_t *args, int argc, const char **argv);
#ifndef SIST_SCAN_ONLY
@ -40,7 +41,10 @@ typedef struct web_args {
} web_args_t;
index_args_t *index_args_create();
void index_args_destroy(index_args_t *args);
web_args_t *web_args_create();
void web_args_destroy(web_args_t *args);
int index_args_validate(index_args_t *args, int argc, const char **argv);
int web_args_validate(web_args_t *args, int argc, const char **argv);

2
src/ctx.h
View File

@ -35,7 +35,7 @@ struct {
struct {
char *es_url;
int index_count;
char* b64credentials;
char *b64credentials;
struct index_t indices[16];
} WebCtx;
#endif

17
src/io/serialize.c
View File

@ -1,7 +1,7 @@
#include "src/ctx.h"
#include "serialize.h"
static __thread int IndexFd = -1;
static __thread int index_fd = -1;
typedef struct {
unsigned char uuid[16];
@ -119,13 +119,13 @@ char *get_meta_key_text(enum metakey meta_key) {
void write_document(document_t *doc) {
if (IndexFd == -1) {
if (index_fd == -1) {
char dstfile[PATH_MAX];
pthread_t self = pthread_self();
snprintf(dstfile, PATH_MAX, "%s_index_%lu", ScanCtx.index.path, self);
IndexFd = open(dstfile, O_CREAT | O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR);
index_fd = open(dstfile, O_CREAT | O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR);
if (IndexFd == -1) {
if (index_fd == -1) {
perror("open");
}
}
@ -158,13 +158,16 @@ void write_document(document_t *doc) {
}
dyn_buffer_write_char(&buf, '\n');
write(IndexFd, buf.buf, buf.cur);
int res = write(index_fd, buf.buf, buf.cur);
if (res == -1) {
perror("write");
}
ScanCtx.stat_index_size += buf.cur;
dyn_buffer_destroy(&buf);
}
void serializer_cleanup() {
close(IndexFd);
void thread_cleanup() {
close(index_fd);
}
void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func func) {

2
src/io/serialize.h
View File

@ -18,7 +18,7 @@ void incremental_read(GHashTable *table, const char *filepath);
/**
* Must be called after write_document
*/
void serializer_cleanup();
void thread_cleanup();
void write_index_descriptor(char *path, index_descriptor_t *desc);

38
src/main.c
View File

@ -19,9 +19,9 @@ static const char *const usage[] = {
};
void global_init() {
#ifndef SIST_SCAN_ONLY
#ifndef SIST_SCAN_ONLY
curl_global_init(CURL_GLOBAL_NOTHING);
#endif
#endif
av_log_set_level(AV_LOG_QUIET);
}
@ -93,7 +93,7 @@ void sist2_scan(scan_args_t *args) {
printf("Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table));
}
ScanCtx.pool = tpool_create(args->threads, serializer_cleanup);
ScanCtx.pool = tpool_create(args->threads, thread_cleanup);
tpool_start(ScanCtx.pool);
walk_directory_tree(ScanCtx.index.desc.root);
tpool_wait(ScanCtx.pool);
@ -126,6 +126,7 @@ void sist2_scan(scan_args_t *args) {
}
#ifndef SIST_SCAN_ONLY
void sist2_index(index_args_t *args) {
IndexCtx.es_url = args->es_url;
@ -198,6 +199,7 @@ void sist2_web(web_args_t *args) {
serve(args->bind, args->port);
}
#endif
@ -206,14 +208,14 @@ int main(int argc, const char *argv[]) {
global_init();
scan_args_t *scan_args = scan_args_create();
#ifndef SIST_SCAN_ONLY
#ifndef SIST_SCAN_ONLY
index_args_t *index_args = index_args_create();
web_args_t *web_args = web_args_create();
#endif
#endif
int arg_version = 0;
char * common_es_url = NULL;
char *common_es_url = NULL;
struct argparse_option options[] = {
OPT_HELP(),
@ -233,22 +235,22 @@ int main(int argc, const char *argv[]) {
OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."),
OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: (name of the directory)"),
OPT_INTEGER(0, "depth", &scan_args->depth, "Scan up to DEPTH subdirectories deep. "
"Use 0 to only scan files in PATH. DEFAULT: -1"),
"Use 0 to only scan files in PATH. DEFAULT: -1"),
#ifndef SIST_SCAN_ONLY
#ifndef SIST_SCAN_ONLY
OPT_GROUP("Index options"),
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
OPT_STRING(0, "script-file", &index_args->script_path, "Path to user script."),
OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. "
"(You must use this option the first time you use the index command)"),
"(You must use this option the first time you use the index command)"),
OPT_GROUP("Web options"),
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"),
OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"),
OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"),
#endif
#endif
OPT_END(),
};
@ -263,10 +265,10 @@ int main(int argc, const char *argv[]) {
exit(0);
}
#ifndef SIST_SCAN_ONLY
#ifndef SIST_SCAN_ONLY
web_args->es_url = common_es_url;
index_args->es_url = common_es_url;
#endif
#endif
if (argc == 0) {
argparse_usage(&argparse);
@ -281,7 +283,7 @@ int main(int argc, const char *argv[]) {
}
#ifndef SIST_SCAN_ONLY
#ifndef SIST_SCAN_ONLY
else if (strcmp(argv[0], "index") == 0) {
int err = index_args_validate(index_args, argc, argv);
@ -299,12 +301,20 @@ int main(int argc, const char *argv[]) {
sist2_web(web_args);
}
#endif
#endif
else {
fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
argparse_usage(&argparse);
return 1;
}
printf("\n");
scan_args_destroy(scan_args);
#ifndef SIST_SCAN_ONLY
index_args_destroy(index_args);
web_args_destroy(web_args);
#endif
return 0;
}

10
src/parsing/font.c
View File

@ -1,11 +1,9 @@
#include "font.h"
#include "ft2build.h"
#include "freetype/freetype.h"
#include "src/ctx.h"
__thread FT_Library library = NULL;
__thread FT_Library ft_lib = NULL;
typedef struct text_dimensions {
@ -139,15 +137,15 @@ void bmp_format(dyn_buffer_t *buf, text_dimensions_t dimensions, const unsigned
}
void parse_font(const char *buf, size_t buf_len, document_t *doc) {
if (library == NULL) {
FT_Init_FreeType(&library);
if (ft_lib == NULL) {
FT_Init_FreeType(&ft_lib);
}
if (buf == NULL) {
return;
}
FT_Face face;
FT_Error err = FT_New_Memory_Face(library, (unsigned char *) buf, buf_len, 0, &face);
FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, buf_len, 0, &face);
if (err != 0) {
return;
}

51
src/parsing/parse.c
View File

@ -44,7 +44,6 @@ void parse(void *arg) {
if (Magic == NULL) {
Magic = magic_open(MAGIC_MIME_TYPE);
magic_load(Magic, NULL);
}
doc.filepath = job->filepath;
@ -98,31 +97,33 @@ void parse(void *arg) {
int mmime = MAJOR_MIME(doc.mime);
if (!(SHOULD_PARSE(doc.mime))) {
parse_text(bytes_read, &fd, (char *) buf, &doc);
} else if ((mmime == MimeVideo && doc.size >= MIN_VIDEO_SIZE) ||
(mmime == MimeImage && doc.size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) {
parse_media(job->filepath, &doc);
} else if (IS_PDF(doc.mime)) {
void *pdf_buf = read_all(job, (char *) buf, bytes_read, &fd);
parse_pdf(pdf_buf, doc.size, &doc);
if (pdf_buf != buf && pdf_buf != NULL) {
free(pdf_buf);
}
} else if (mmime == MimeText && ScanCtx.content_size > 0) {
parse_text(bytes_read, &fd, (char *) buf, &doc);
} else if (IS_FONT(doc.mime)) {
void *font_buf = read_all(job, (char *) buf, bytes_read, &fd);
parse_font(font_buf, doc.size, &doc);
if (font_buf != buf && font_buf != NULL) {
free(font_buf);
}
}
// if (!(SHOULD_PARSE(doc.mime))) {
//
// } else if ((mmime == MimeVideo && doc.size >= MIN_VIDEO_SIZE) ||
// (mmime == MimeImage && doc.size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) {
// parse_media(job->filepath, &doc);
//
// } else if (IS_PDF(doc.mime)) {
// void *pdf_buf = read_all(job, (char *) buf, bytes_read, &fd);
// parse_pdf(pdf_buf, doc.size, &doc);
//
// if (pdf_buf != buf && pdf_buf != NULL) {
// free(pdf_buf);
// }
//
// } else if (mmime == MimeText && ScanCtx.content_size > 0) {
// parse_text(bytes_read, &fd, (char *) buf, &doc);
//
// } else if (IS_FONT(doc.mime)) {
// void *font_buf = read_all(job, (char *) buf, bytes_read, &fd);
// parse_font(font_buf, doc.size, &doc);
//
// if (font_buf != buf && font_buf != NULL) {
// free(font_buf);
// }
// }
write_document(&doc);

2
src/sist.h
View File

@ -26,6 +26,8 @@
#include <pthread.h>
#include <sys/stat.h>
#include <wordexp.h>
#include "ft2build.h"
#include "freetype/freetype.h"
#ifndef SIST_SCAN_ONLY
#include <onion/onion.h>

8
src/tpool.c
View File

@ -151,6 +151,7 @@ void tpool_wait(tpool_t *pool) {
usleep(500000);
if (pool->done_cnt == pool->work_cnt) {
pool->stop = 1;
usleep(1000000);
break;
}
}
@ -178,7 +179,8 @@ void tpool_destroy(tpool_t *pool) {
for (size_t i = 0; i < pool->thread_cnt; i++) {
pthread_t thread = pool->threads[i];
if (thread != 0) {
pthread_cancel(thread);
void *_;
pthread_join(thread, &_);
}
}
@ -218,8 +220,6 @@ tpool_t *tpool_create(size_t thread_cnt, void cleanup_func()) {
void tpool_start(tpool_t *pool) {
for (size_t i = 0; i < pool->thread_cnt; i++) {
pthread_t thread = pool->threads[i];
pthread_create(&thread, NULL, tpool_worker, pool);
pthread_detach(thread);
pthread_create(&pool->threads[i], NULL, tpool_worker, pool);
}
}