simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-12-17 09:19:05 +00:00
Code Issues Projects Releases Wiki Activity

bugfixes & refactoring

Browse Source
This commit is contained in:
simon
2019-10-26 12:35:01 -04:00
parent 564a17a8fa
commit c3b7a05dde
21 changed files with 440 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
src/main.c
View File

@@ -1,8 +1,11 @@
#include "sist.h"
#include "ctx.h"
static const char *const Version = "1.0.0";
#define DESCRIPTION "Lightning-fast file system indexer and search tool."
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
static const char *const Version = "1.0.0";
static const char *const usage[] = {
"sist2 scan [OPTION]... PATH",
"sist2 index [OPTION]... INDEX",
@@ -10,6 +13,11 @@ static const char *const usage[] = {
NULL,
};
void global_init() {
curl_global_init(CURL_GLOBAL_NOTHING);
av_log_set_level(AV_LOG_QUIET);
}
void init_dir(const char *dirpath) {
char path[PATH_MAX];
snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
@@ -32,11 +40,16 @@ void scan_print_header() {
printf("output\t\t%s\n", ScanCtx.index.path);
}
void sist2_scan(const char *path, const char *incremental_from) {
void sist2_scan(scan_args_t *args) {
av_log_set_level(AV_LOG_QUIET);
strcpy(ScanCtx.index.desc.root, abspath(path));
ScanCtx.tn_qscale = args->quality;
ScanCtx.tn_size = args->size;
ScanCtx.content_size = args->content_size;
ScanCtx.pool = tpool_create(args->threads, serializer_cleanup);
ScanCtx.threads = args->threads;
strncpy(ScanCtx.index.path, args->output, sizeof(ScanCtx.index.path));
strncpy(ScanCtx.index.desc.name, args->name, sizeof(ScanCtx.index.desc.name));
strcpy(ScanCtx.index.desc.root, args->path);
ScanCtx.index.desc.root_len = (short) strlen(ScanCtx.index.desc.root);
init_dir(ScanCtx.index.path);
@@ -51,12 +64,11 @@ void sist2_scan(const char *path, const char *incremental_from) {
scan_print_header();
if (incremental_from != NULL) {
incremental_from = abspath(incremental_from);
if (args->incremental != NULL) {
ScanCtx.original_table = incremental_get_table();
ScanCtx.copy_table = incremental_get_table();
DIR *dir = opendir(incremental_from);
DIR *dir = opendir(args->incremental);
if (dir == NULL) {
perror("opendir");
return;
@@ -65,7 +77,7 @@ void sist2_scan(const char *path, const char *incremental_from) {
while ((de = readdir(dir)) != NULL) {
if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
char file_path[PATH_MAX];
snprintf(file_path, PATH_MAX, "%s/%s", incremental_from, de->d_name);
snprintf(file_path, PATH_MAX, "%s/%s", args->incremental, de->d_name);
incremental_read(ScanCtx.original_table, file_path);
}
}
@@ -76,14 +88,15 @@ void sist2_scan(const char *path, const char *incremental_from) {
walk_directory_tree(ScanCtx.index.desc.root);
tpool_wait(ScanCtx.pool);
tpool_destroy(ScanCtx.pool);
if (incremental_from != NULL) {
if (args->incremental != NULL) {
char dst_path[PATH_MAX];
snprintf(store_path, PATH_MAX, "%sthumbs", incremental_from);
snprintf(store_path, PATH_MAX, "%sthumbs", args->incremental);
snprintf(dst_path, PATH_MAX, "%s_index_original", ScanCtx.index.path);
store_t *source = store_create(store_path);
DIR *dir = opendir(incremental_from);
DIR *dir = opendir(args->incremental);
if (dir == NULL) {
perror("opendir");
return;
@@ -92,7 +105,7 @@ void sist2_scan(const char *path, const char *incremental_from) {
while ((de = readdir(dir)) != NULL) {
if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
char file_path[PATH_MAX];
snprintf(file_path, PATH_MAX, "%s/%s", incremental_from, de->d_name);
snprintf(file_path, PATH_MAX, "%s/%s", args->incremental, de->d_name);
incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table);
}
}
@@ -101,16 +114,18 @@ void sist2_scan(const char *path, const char *incremental_from) {
}
store_destroy(ScanCtx.index.store);
tpool_destroy(ScanCtx.pool);
}
void sist2_index(const char *path, int print_index, int arg_force_reset) {
if (!print_index) {
elastic_init(arg_force_reset);
void sist2_index(index_args_t *args) {
IndexCtx.es_url = args->es_url;
if (!args->print) {
elastic_init(args->force_reset);
}
char *index_path = abspath(path);
char descriptor_path[PATH_MAX];
snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", index_path);
snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->index_path);
index_descriptor_t desc = read_index_descriptor(descriptor_path);
if (strcmp(desc.version, Version) != 0) {
@@ -118,14 +133,14 @@ void sist2_index(const char *path, int print_index, int arg_force_reset) {
return;
}
DIR *dir = opendir(index_path);
DIR *dir = opendir(args->index_path);
if (dir == NULL) {
perror("opendir");
return;
}
index_func f;
if (print_index) {
if (args->print) {
f = print_json;
} else {
f = index_json;
@@ -135,22 +150,27 @@ void sist2_index(const char *path, int print_index, int arg_force_reset) {
while ((de = readdir(dir)) != NULL) {
if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
char file_path[PATH_MAX];
snprintf(file_path, PATH_MAX, "%s/%s", index_path, de->d_name);
snprintf(file_path, PATH_MAX, "%s/%s", args->index_path, de->d_name);
read_index(file_path, desc.uuid, f);
}
}
if (!print_index) {
if (!args->print) {
elastic_flush();
destroy_indexer();
}
}
void sist2_web(const char *indices[], int index_count, const char *host, const char *port) {
void sist2_web(web_args_t *args) {
for (int i = 0; i < index_count; i++) {
char *abs_path = abspath(indices[i]);
WebCtx.es_url = args->es_url;
WebCtx.index_count = args->index_count;
for (int i = 0; i < args->index_count; i++) {
char *abs_path = abspath(args->indices[i]);
if (abs_path == NULL) {
return;
}
char path_tmp[PATH_MAX];
snprintf(path_tmp, PATH_MAX, "%sthumbs", abs_path);
@@ -161,168 +181,88 @@ void sist2_web(const char *indices[], int index_count, const char *host, const c
strcpy(WebCtx.indices[i].path, abs_path);
printf("Loaded index: %s\n", WebCtx.indices[i].desc.name);
free(abs_path);
}
WebCtx.index_count = index_count;
serve(host, port);
serve(args->bind, args->port);
}
int main(int argc, const char *argv[]) {
curl_global_init(CURL_GLOBAL_NOTHING);
global_init();
float arg_quality = 0;
int arg_size = 0;
int arg_content_size = 0;
int arg_threads = 0;
char *arg_incremental = NULL;
char *arg_output = NULL;
char *arg_rewrite_url = NULL;
char *arg_name = NULL;
scan_args_t *scan_args = scan_args_create();
index_args_t *index_args = index_args_create();
web_args_t *web_args = web_args_create();
char *arg_es_url = NULL;
int arg_print_index = 0;
int arg_force_reset = 0;
char *arg_web_host = NULL;
char *arg_web_port = NULL;
char * common_es_url;
struct argparse_option options[] = {
OPT_HELP(),
OPT_GROUP("Scan options"),
OPT_INTEGER('t', "threads", &arg_threads, "Number of threads. DEFAULT=1"),
OPT_FLOAT('q', "quality", &arg_quality,
OPT_INTEGER('t', "threads", &scan_args->threads, "Number of threads. DEFAULT=1"),
OPT_FLOAT('q', "quality", &scan_args->quality,
"Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=15"),
OPT_INTEGER(0, "size", &arg_size, "Thumbnail size, in pixels. DEFAULT=200"),
OPT_INTEGER(0, "content-size", &arg_content_size,
OPT_INTEGER(0, "size", &scan_args->size, "Thumbnail size, in pixels. DEFAULT=200"),
OPT_INTEGER(0, "content-size", &scan_args->content_size,
"Number of bytes to be extracted from text documents. DEFAULT=4096"),
OPT_STRING(0, "incremental", &arg_incremental, "Reuse an existing index and only scan modified files."),
OPT_STRING('o', "output", &arg_output, "Output directory. DEFAULT=index.sist2/"),
OPT_STRING(0, "rewrite-url", &arg_rewrite_url, "Serve files from this url instead of from disk."),
OPT_STRING(0, "name", &arg_name, "Index display name. DEFAULT: (name of the directory)"),
OPT_STRING(0, "incremental", &scan_args->incremental,
"Reuse an existing index and only scan modified files."),
OPT_STRING('o', "output", &scan_args->output, "Output directory. DEFAULT=index.sist2/"),
OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."),
OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: (name of the directory)"),
OPT_GROUP("Index options"),
OPT_STRING(0, "es-url", &arg_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
OPT_BOOLEAN('p', "print", &arg_print_index, "Just print JSON documents to stdout."),
OPT_BOOLEAN('f', "force-reset", &arg_force_reset, "Reset Elasticsearch mappings and settings. "
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. "
"(You must use this option the first time you use the index command)"),
OPT_GROUP("Web options"),
OPT_STRING(0, "es-url", &arg_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
OPT_STRING(0, "bind", &arg_web_host, "Listen on this address. DEFAULT=localhost"),
OPT_STRING(0, "port", &arg_web_port, "Listen on this port. DEFAULT=4090"),
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"),
OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"),
OPT_END(),
};
struct argparse argparse;
argparse_init(&argparse, options, usage, 0);
argparse_describe(
&argparse,
"\nLightning-fast file system indexer and search tool.",
"\nMade by simon987 <me@simon987.net>. Released under GPL-3.0"
);
argparse_describe(&argparse, DESCRIPTION, EPILOG);
argc = argparse_parse(&argparse, argc, argv);
//Set defaults
if (arg_quality == 0) {
arg_quality = 15;
} else if (arg_quality < 1 || arg_quality > 31) {
fprintf(stderr, "Invalid quality: %f\n", arg_quality);
return 1;
}
web_args->es_url = common_es_url;
index_args->es_url = common_es_url;
if (arg_size == 0) {
arg_size = 200;
} else if (arg_size <= 0) {
fprintf(stderr, "Invalid size: %d\n", arg_size);
return 1;
}
if (arg_content_size == 0) {
arg_content_size = 4096;
} else if (arg_content_size <= 0) {
fprintf(stderr, "Invalid content-size: %d\n", arg_content_size);
return 1;
}
if (arg_threads == 0) {
arg_threads = 1;
} else if (arg_threads < 0) {
fprintf(stderr, "Invalid threads: %d\n", arg_threads);
return 1;
}
if (arg_output == NULL) {
arg_output = "index.sist2/";
}
if (arg_es_url == NULL) {
arg_es_url = "http://localhost:9200";
}
if (arg_web_host == NULL) {
arg_web_host = "localhost";
}
if (arg_web_port == NULL) {
arg_web_port = "4090";
}
// Commands
if (argc == 0) {
argparse_usage(&argparse);
return 1;
} else if (strcmp(argv[0], "scan") == 0) {
if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n");
argparse_usage(&argparse);
return 1;
}
if (arg_name == NULL) {
arg_name = g_path_get_basename(argv[1]);
int err = scan_args_validate(scan_args, argc, argv);
if (err != 0) {
return err;
}
sist2_scan(scan_args);
int ret = mkdir(arg_output, S_IRUSR | S_IWUSR | S_IXUSR);
if (ret != 0) {
fprintf(stderr, "Invalid output: '%s' (%s).\n", arg_output, strerror(errno));
return 1;
}
ScanCtx.tn_qscale = arg_quality;
ScanCtx.tn_size = arg_size;
ScanCtx.content_size = arg_content_size;
ScanCtx.pool = tpool_create(arg_threads, serializer_cleanup);
ScanCtx.threads = arg_threads;
strncpy(ScanCtx.index.path, arg_output, sizeof(ScanCtx.index.path));
strncpy(ScanCtx.index.desc.name, arg_name, sizeof(ScanCtx.index.desc.name));
if (arg_rewrite_url == NULL) {
strcpy(ScanCtx.index.desc.rewrite_url, "");
} else {
strcpy(ScanCtx.index.desc.rewrite_url, arg_rewrite_url);
}
sist2_scan(argv[1], arg_incremental);
} else if (strcmp(argv[0], "index") == 0) {
if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n");
argparse_usage(&argparse);
return 1;
}
IndexCtx.es_url = arg_es_url;
sist2_index(argv[1], arg_print_index, arg_force_reset);
int err = index_args_validate(index_args, argc, argv);
if (err != 0) {
return err;
}
sist2_index(index_args);
} else if (strcmp(argv[0], "web") == 0) {
if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n");
argparse_usage(&argparse);
return 1;
}
WebCtx.es_url = arg_es_url;
sist2_web(argv + 1, argc - 1, arg_web_host, arg_web_port);
int err = web_args_validate(web_args, argc, argv);
if (err != 0) {
return err;
}
sist2_web(web_args);
} else {
fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
argparse_usage(&argparse);