#include "cli.h" #include "ctx.h" #include #define DEFAULT_OUTPUT "index.sist2/" #define DEFAULT_CONTENT_SIZE 32768 #define DEFAULT_QUALITY 2 #define DEFAULT_THUMBNAIL_SIZE 500 #define DEFAULT_THUMBNAIL_COUNT 1 #define DEFAULT_REWRITE_URL "" #define DEFAULT_ES_URL "http://localhost:9200" #define DEFAULT_ES_INDEX "sist2" #define DEFAULT_BATCH_SIZE 100 #define DEFAULT_TAGLINE "Lightning-fast file system indexer and search tool" #define DEFAULT_LANG "en" #define DEFAULT_LISTEN_ADDRESS "localhost:4090" #define DEFAULT_TREEMAP_THRESHOLD 0.0005 #define DEFAULT_MAX_MEM_BUFFER 2000 #define DEFAULT_THROTTLE_MEMORY_THRESHOLD 0 const char *TESS_DATAPATHS[] = { "/usr/share/tessdata/", "/usr/share/tesseract-ocr/tessdata/", "/usr/share/tesseract-ocr/4.00/tessdata/", "./", NULL }; scan_args_t *scan_args_create() { scan_args_t *args = calloc(sizeof(scan_args_t), 1); args->depth = -1; return args; } exec_args_t *exec_args_create() { exec_args_t *args = calloc(sizeof(exec_args_t), 1); return args; } void scan_args_destroy(scan_args_t *args) { if (args->name != NULL) { free(args->name); } if (args->incremental != NULL) { free(args->incremental); } if (args->path != NULL) { free(args->path); } if (args->output != NULL) { free(args->output); } free(args); } void index_args_destroy(index_args_t *args) { //todo if (args->es_mappings_path) { free(args->es_mappings); } if (args->es_settings_path) { free(args->es_settings); } if (args->index_path != NULL) { free(args->index_path); } free(args); } void web_args_destroy(web_args_t *args) { //todo free(args); } void exec_args_destroy(exec_args_t *args) { if (args->index_path != NULL) { free(args->index_path); } free(args); } int scan_args_validate(scan_args_t *args, int argc, const char **argv) { if (argc < 2) { fprintf(stderr, "Required positional argument: PATH.\n"); return 1; } char *abs_path = abspath(argv[1]); if (abs_path == NULL) { LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]) } else { args->path = abs_path; } if (args->incremental != OPTION_VALUE_UNSPECIFIED) { args->incremental = abspath(args->incremental); if (abs_path == NULL) { sist_log("main.c", LOG_SIST_WARNING, "Could not open original index! Disabled incremental scan feature."); args->incremental = NULL; } } if (args->tn_quality == OPTION_VALUE_UNSPECIFIED) { args->tn_quality = DEFAULT_QUALITY; } else if (args->tn_quality < 2 || args->tn_quality > 31) { fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [2, 31].\n", args->tn_quality); return 1; } if (args->tn_size == OPTION_VALUE_UNSPECIFIED) { args->tn_size = DEFAULT_THUMBNAIL_SIZE; } else if (args->tn_size < 32) { printf("Invalid value --thumbnail-size argument: %d. Must be greater than 32 pixels.\n", args->tn_size); return 1; } if (args->tn_count == OPTION_VALUE_UNSPECIFIED) { args->tn_count = DEFAULT_THUMBNAIL_COUNT; } else if (args->tn_count == OPTION_VALUE_DISABLE) { args->tn_count = 0; } else if (args->tn_count > 1000) { printf("Invalid value --thumbnail-count argument: %d. Must be <= 1000.\n", args->tn_size); return 1; } if (args->content_size == OPTION_VALUE_UNSPECIFIED) { args->content_size = DEFAULT_CONTENT_SIZE; } if (args->threads == 0) { args->threads = 1; } else if (args->threads < 0) { fprintf(stderr, "Invalid value for --threads: %d. Must be a positive number\n", args->threads); return 1; } if (args->output == OPTION_VALUE_UNSPECIFIED) { args->output = malloc(strlen(DEFAULT_OUTPUT) + 1); strcpy(args->output, DEFAULT_OUTPUT); } else { args->output = expandpath(args->output); } int ret = mkdir(args->output, S_IRUSR | S_IWUSR | S_IXUSR); if (ret != 0) { fprintf(stderr, "Invalid output: '%s' (%s).\n", args->output, strerror(errno)); return 1; } if (args->depth <= 0) { args->depth = G_MAXINT32; } else { args->depth += 1; } if (args->name == OPTION_VALUE_UNSPECIFIED) { args->name = g_path_get_basename(args->output); } else { char *tmp = malloc(strlen(args->name) + 1); strcpy(tmp, args->name); args->name = tmp; } if (args->rewrite_url == OPTION_VALUE_UNSPECIFIED) { args->rewrite_url = DEFAULT_REWRITE_URL; } if (args->archive == OPTION_VALUE_UNSPECIFIED || strcmp(args->archive, "recurse") == 0) { args->archive_mode = ARC_MODE_RECURSE; } else if (strcmp(args->archive, "list") == 0) { args->archive_mode = ARC_MODE_LIST; } else if (strcmp(args->archive, "shallow") == 0) { args->archive_mode = ARC_MODE_SHALLOW; } else if (strcmp(args->archive, "skip") == 0) { args->archive_mode = ARC_MODE_SKIP; } else { fprintf(stderr, "Archive mode must be one of (skip, list, shallow, recurse), got '%s'", args->archive); return 1; } if (args->ocr_images && args->tesseract_lang == OPTION_VALUE_UNSPECIFIED) { fprintf(stderr, "You must specify --ocr-lang to use --ocr-images"); return 1; } if (args->ocr_ebooks && args->tesseract_lang == OPTION_VALUE_UNSPECIFIED) { fprintf(stderr, "You must specify --ocr-lang to use --ocr-ebooks"); return 1; } if (args->tesseract_lang != OPTION_VALUE_UNSPECIFIED) { if (!args->ocr_ebooks && !args->ocr_images) { fprintf(stderr, "You must specify at least one of --ocr-ebooks, --ocr-images"); return 1; } TessBaseAPI *api = TessBaseAPICreate(); const char *trained_data_path = NULL; char *lang = malloc(strlen(args->tesseract_lang) + 1); strcpy(lang, args->tesseract_lang); lang = strtok(lang, "+"); while (lang != NULL) { char filename[128]; sprintf(filename, "%s.traineddata", lang); const char *path = find_file_in_paths(TESS_DATAPATHS, filename); if (path == NULL) { LOG_FATALF("cli.c", "Could not find tesseract language file: %s!", filename); } if (trained_data_path != NULL && path != trained_data_path) { LOG_FATAL("cli.c", "When specifying more than one tesseract language, all the traineddata " "files must be in the same folder") } trained_data_path = path; lang = strtok(NULL, "+"); } free(lang); ret = TessBaseAPIInit3(api, trained_data_path, args->tesseract_lang); if (ret != 0) { fprintf(stderr, "Could not initialize tesseract with lang '%s'\n", args->tesseract_lang); return 1; } TessBaseAPIEnd(api); TessBaseAPIDelete(api); args->tesseract_path = trained_data_path; } if (args->exclude_regex != OPTION_VALUE_UNSPECIFIED) { const char *error; int error_offset; pcre *re = pcre_compile(args->exclude_regex, 0, &error, &error_offset, 0); if (error != NULL) { LOG_FATALF("cli.c", "pcre_compile returned error: %s (offset:%d)", error, error_offset) } pcre_extra *re_extra = pcre_study(re, 0, &error); if (error != NULL) { LOG_FATALF("cli.c", "pcre_study returned error: %s", error) } ScanCtx.exclude = re; ScanCtx.exclude_extra = re_extra; } else { ScanCtx.exclude = NULL; } if (args->treemap_threshold_str == OPTION_VALUE_UNSPECIFIED) { args->treemap_threshold = DEFAULT_TREEMAP_THRESHOLD; } else { args->treemap_threshold = atof(args->treemap_threshold_str); } if (args->max_memory_buffer_mib == OPTION_VALUE_UNSPECIFIED) { args->max_memory_buffer_mib = DEFAULT_MAX_MEM_BUFFER; } if (args->scan_mem_limit_mib == OPTION_VALUE_UNSPECIFIED || args->scan_mem_limit_mib == OPTION_VALUE_DISABLE) { args->scan_mem_limit_mib = DEFAULT_THROTTLE_MEMORY_THRESHOLD; } if (args->list_path != OPTION_VALUE_UNSPECIFIED) { if (strcmp(args->list_path, "-") == 0) { args->list_file = stdin; LOG_DEBUG("cli.c", "Using stdin as list file") } else { args->list_file = fopen(args->list_path, "r"); if (args->list_file == NULL) { LOG_FATALF("main.c", "List file could not be opened: %s (%s)", args->list_path, errno); } } } LOG_DEBUGF("cli.c", "arg tn_quality=%f", args->tn_quality) LOG_DEBUGF("cli.c", "arg tn_size=%d", args->tn_size) LOG_DEBUGF("cli.c", "arg tn_count=%d", args->tn_count) LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size) LOG_DEBUGF("cli.c", "arg threads=%d", args->threads) LOG_DEBUGF("cli.c", "arg incremental=%s", args->incremental) LOG_DEBUGF("cli.c", "arg output=%s", args->output) LOG_DEBUGF("cli.c", "arg rewrite_url=%s", args->rewrite_url) LOG_DEBUGF("cli.c", "arg name=%s", args->name) LOG_DEBUGF("cli.c", "arg depth=%d", args->depth) LOG_DEBUGF("cli.c", "arg path=%s", args->path) LOG_DEBUGF("cli.c", "arg archive=%s", args->archive) LOG_DEBUGF("cli.c", "arg archive_passphrase=%s", args->archive_passphrase) LOG_DEBUGF("cli.c", "arg tesseract_lang=%s", args->tesseract_lang) LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path) LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex) LOG_DEBUGF("cli.c", "arg fast=%d", args->fast) LOG_DEBUGF("cli.c", "arg fast_epub=%d", args->fast_epub) LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold) LOG_DEBUGF("cli.c", "arg max_memory_buffer_mib=%d", args->max_memory_buffer_mib) LOG_DEBUGF("cli.c", "arg list_path=%s", args->list_path) return 0; } int load_external_file(const char *file_path, char **dst) { struct stat info; int res = stat(file_path, &info); if (res == -1) { LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno)) return 1; } int fd = open(file_path, O_RDONLY); if (fd == -1) { LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno)) return 1; } *dst = malloc(info.st_size + 1); res = read(fd, *dst, info.st_size); if (res < 0) { LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno)) return 1; } *(*dst + info.st_size) = '\0'; close(fd); return 0; } int index_args_validate(index_args_t *args, int argc, const char **argv) { LogCtx.verbose = 1; if (argc < 2) { fprintf(stderr, "Required positional argument: PATH.\n"); return 1; } if (args->threads == 0) { args->threads = 1; } else if (args->threads < 0) { fprintf(stderr, "Invalid threads: %d\n", args->threads); return 1; } char *index_path = abspath(argv[1]); if (index_path == NULL) { LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]) } else { args->index_path = index_path; } if (args->es_url == NULL) { args->es_url = DEFAULT_ES_URL; } if (args->es_index == NULL) { args->es_index = DEFAULT_ES_INDEX; } if (args->script_path != NULL) { if (load_external_file(args->script_path, &args->script) != 0) { return 1; } } if (args->es_settings_path != NULL) { if (load_external_file(args->es_settings_path, &args->es_settings) != 0) { return 1; } } if (args->es_mappings_path != NULL) { if (load_external_file(args->es_mappings_path, &args->es_mappings) != 0) { return 1; } } if (args->batch_size == 0) { args->batch_size = DEFAULT_BATCH_SIZE; } LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url) LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index) LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl) LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path) LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path) LOG_DEBUGF("cli.c", "arg async_script=%d", args->async_script) if (args->script) { char log_buf[5000]; strncpy(log_buf, args->script, sizeof(log_buf)); *(log_buf + sizeof(log_buf) - 1) = '\0'; LOG_DEBUGF("cli.c", "arg script=%s", log_buf) } LOG_DEBUGF("cli.c", "arg print=%d", args->print) LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path) LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings) LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path) LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings) LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size) LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset) return 0; } int web_args_validate(web_args_t *args, int argc, const char **argv) { LogCtx.verbose = 1; if (argc < 2) { fprintf(stderr, "Required positional argument: PATH.\n"); return 1; } if (args->es_url == NULL) { args->es_url = DEFAULT_ES_URL; } if (args->listen_address == NULL) { args->listen_address = DEFAULT_LISTEN_ADDRESS; } if (args->es_index == NULL) { args->es_index = DEFAULT_ES_INDEX; } if (args->tagline == NULL) { args->tagline = DEFAULT_TAGLINE; } if (args->lang == NULL) { args->lang = DEFAULT_LANG; } if (strlen(args->lang) != 2 && strlen(args->lang) != 5) { fprintf(stderr, "Invalid --lang value, see usage\n"); return 1; } if (args->credentials != NULL) { char *ptr = strstr(args->credentials, ":"); if (ptr == NULL) { fprintf(stderr, "Invalid --auth format, see usage\n"); return 1; } strncpy(args->auth_user, args->credentials, (ptr - args->credentials)); strcpy(args->auth_pass, ptr + 1); if (strlen(args->auth_user) == 0) { fprintf(stderr, "--auth username must be at least one character long"); return 1; } args->auth_enabled = TRUE; } else { args->auth_enabled = FALSE; } if (args->tag_credentials != NULL && args->credentials != NULL) { fprintf(stderr, "--auth and --tag-auth are mutually exclusive"); return 1; } if (args->tag_credentials != NULL) { char *ptr = strstr(args->tag_credentials, ":"); if (ptr == NULL) { fprintf(stderr, "Invalid --tag-auth format, see usage\n"); return 1; } strncpy(args->auth_user, args->tag_credentials, (ptr - args->tag_credentials)); strcpy(args->auth_pass, ptr + 1); if (strlen(args->auth_user) == 0) { fprintf(stderr, "--tag-auth username must be at least one character long"); return 1; } args->tag_auth_enabled = TRUE; } else { args->tag_auth_enabled = FALSE; } if (args->auth0_public_key_path != NULL || args->auth0_audience != NULL || args->auth0_client_id || args->auth0_domain) { if (args->auth0_public_key_path == NULL) { fprintf(stderr, "Missing --auth0-public-key-file argument"); return 1; } if (args->auth0_audience == NULL) { fprintf(stderr, "Missing --auth0-audience argument"); return 1; } if (args->auth0_client_id == NULL) { fprintf(stderr, "Missing --auth0-client-id argument"); return 1; } if (args->auth0_domain == NULL) { fprintf(stderr, "Missing --auth0-domain argument"); return 1; } } if (args->auth0_public_key_path != NULL) { if (load_external_file(args->auth0_public_key_path, &args->auth0_public_key) != 0) { return 1; } args->auth0_enabled = TRUE; } else { args->auth0_enabled = FALSE; } args->index_count = argc - 1; args->indices = argv + 1; for (int i = 0; i < args->index_count; i++) { char *abs_path = abspath(args->indices[i]); if (abs_path == NULL) { LOG_FATALF("cli.c", "Index not found: %s", args->indices[i]) } } LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url) LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index) LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl) LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline) LOG_DEBUGF("cli.c", "arg dev=%d", args->dev) LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address) LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials) LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials) LOG_DEBUGF("cli.c", "arg auth_user=%s", args->auth_user) LOG_DEBUGF("cli.c", "arg auth_pass=%s", args->auth_pass) LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count) for (int i = 0; i < args->index_count; i++) { LOG_DEBUGF("cli.c", "arg indices[%d]=%s", i, args->indices[i]) } return 0; } index_args_t *index_args_create() { index_args_t *args = calloc(sizeof(index_args_t), 1); return args; } web_args_t *web_args_create() { web_args_t *args = calloc(sizeof(web_args_t), 1); return args; } int exec_args_validate(exec_args_t *args, int argc, const char **argv) { if (argc < 2) { fprintf(stderr, "Required positional argument: PATH.\n"); return 1; } char *index_path = abspath(argv[1]); if (index_path == NULL) { LOG_FATALF("cli.c", "Invalid index PATH argument. File not found: %s", argv[1]) } else { args->index_path = index_path; } if (args->es_url == NULL) { args->es_url = DEFAULT_ES_URL; } if (args->es_index == NULL) { args->es_index = DEFAULT_ES_INDEX; } if (args->script_path == NULL) { LOG_FATAL("cli.c", "--script-file argument is required"); } if (load_external_file(args->script_path, &args->script) != 0) { return 1; } LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path) char log_buf[5000]; strncpy(log_buf, args->script, sizeof(log_buf)); *(log_buf + sizeof(log_buf) - 1) = '\0'; LOG_DEBUGF("cli.c", "arg script=%s", log_buf) return 0; }