simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-16 00:46:43 +00:00
Code Issues Projects Releases Wiki Activity

unify READ_INDICES to reduce clutter

Browse Source
This commit is contained in:
Yatao Li 2022-01-19 21:30:35 +08:00
parent 291d307689
commit 679e12f786
4 changed files with 31 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/cli.c
View File

@ -336,8 +336,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
if (index_path == NULL) {
LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1])
} else {
args->index_path = argv[1];
free(index_path);
args->index_path = index_path;
}
if (args->es_url == NULL) {
@ -522,8 +521,7 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
if (index_path == NULL) {
LOG_FATALF("cli.c", "Invalid index PATH argument. File not found: %s", argv[1])
} else {
args->index_path = argv[1];
free(index_path);
args->index_path = index_path;
}
if (args->es_url == NULL) {

14
src/io/serialize.h
View File

@ -38,4 +38,18 @@ void write_index_descriptor(char *path, index_descriptor_t *desc);
index_descriptor_t read_index_descriptor(char *path);
// caller ensures char file_path[PATH_MAX]
#define READ_INDICES(file_path, index_path, action_ok, action_main_fail, cond_original) \
snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", index_path); \
if (0 == access(file_path, R_OK)) { \
action_ok; \
} else { \
action_main_fail; \
} \
snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", index_path); \
if ((cond_original) && 0 == access(file_path, R_OK)) { \
action_ok; \
} \
#endif

49
src/main.c
View File

@ -305,23 +305,15 @@ void load_incremental_index(const scan_args_t *args) {
}
char descriptor_path[PATH_MAX];
snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->incremental);
snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental);
index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
if (strcmp(original_desc.version, Version) != 0) {
LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc.version, Version)
}
snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", args->incremental);
if (0 == access(file_path, R_OK)) {
incremental_read(ScanCtx.original_table, file_path, &original_desc);
} else {
LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno));
}
snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", args->incremental);
if (0 == access(file_path, R_OK)) {
incremental_read(ScanCtx.original_table, file_path, &original_desc);
}
READ_INDICES(file_path, args->incremental, incremental_read(ScanCtx.original_table, file_path, &original_desc),
LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)), 1);
closedir(dir);
@ -351,17 +343,8 @@ void save_incremental_index(scan_args_t* args) {
snprintf(file_path, PATH_MAX, "%s_index_delete.list.zst", ScanCtx.index.path);
incremental_delete(file_path, ScanCtx.original_table, ScanCtx.new_table);
snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", args->incremental);
if (0 == access(file_path, R_OK)) {
incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table);
} else {
perror("incremental_copy");
return;
}
snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", args->incremental);
if (0 == access(file_path, R_OK)) {
incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table);
}
READ_INDICES(file_path, args->incremental, incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table),
perror("incremental_copy"), 1);
closedir(dir);
store_destroy(source);
@ -458,7 +441,7 @@ void sist2_index(index_args_t *args) {
}
char descriptor_path[PATH_MAX];
snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->index_path);
snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->index_path);
index_descriptor_t desc = read_index_descriptor(descriptor_path);
@ -474,11 +457,11 @@ void sist2_index(index_args_t *args) {
}
char path_tmp[PATH_MAX];
snprintf(path_tmp, sizeof(path_tmp), "%s/tags", args->index_path);
snprintf(path_tmp, sizeof(path_tmp), "%stags", args->index_path);
IndexCtx.tag_store = store_create(path_tmp, STORE_SIZE_TAG);
IndexCtx.tags = store_read_all(IndexCtx.tag_store);
snprintf(path_tmp, sizeof(path_tmp), "%s/meta", args->index_path);
snprintf(path_tmp, sizeof(path_tmp), "%smeta", args->index_path);
IndexCtx.meta_store = store_create(path_tmp, STORE_SIZE_META);
IndexCtx.meta = store_read_all(IndexCtx.meta_store);
@ -499,17 +482,11 @@ void sist2_index(index_args_t *args) {
IndexCtx.pool = tpool_create(args->threads, cleanup, FALSE, args->print == 0);
tpool_start(IndexCtx.pool);
snprintf(file_path, PATH_MAX, "%s/_index_original.ndjson.zst", args->index_path);
if (!args->incremental && 0 == access(file_path, R_OK)) {
READ_INDICES(file_path, args->index_path, {
read_index(file_path, desc.id, desc.type, f);
LOG_DEBUGF("main.c", "Read index file %s (%s)", file_path, desc.type)
}
snprintf(file_path, PATH_MAX, "%s/_index_main.ndjson.zst", args->index_path);
if (0 == access(file_path, R_OK)) {
read_index(file_path, desc.id, desc.type, f);
LOG_DEBUGF("main.c", "Read index file %s (%s)", file_path, desc.type)
}
snprintf(file_path, PATH_MAX, "%s/_index_delete.list.zst", args->index_path);
LOG_DEBUGF("main.c", "Read index file %s (%s)", file_path, desc.type);
}, {}, !args->incremental);
snprintf(file_path, PATH_MAX, "%s_index_delete.list.zst", args->index_path);
if (0 == access(file_path, R_OK)) {
read_lines(file_path, (line_processor_t) {
.data = desc.id,
@ -539,7 +516,7 @@ void sist2_exec_script(exec_args_t *args) {
LogCtx.verbose = TRUE;
char descriptor_path[PATH_MAX];
snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->index_path);
snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->index_path);
index_descriptor_t desc = read_index_descriptor(descriptor_path);
IndexCtx.es_url = args->es_url;

12
src/stats.c
View File

@ -96,16 +96,8 @@ void fill_tables(cJSON *document, UNUSED(const char index_id[MD5_STR_LENGTH])) {
}
void read_index_into_tables(index_t *index) {
DIR *dir = opendir(index->path);
struct dirent *de;
while ((de = readdir(dir)) != NULL) {
if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
char file_path[PATH_MAX];
snprintf(file_path, PATH_MAX, "%s%s", index->path, de->d_name);
read_index(file_path, index->desc.id, index->desc.type, fill_tables);
}
}
closedir(dir);
char file_path[PATH_MAX];
READ_INDICES(file_path, index->path, read_index(file_path, index->desc.id, index->desc.type, fill_tables), {}, 1);
}
static size_t rfind(const char *str, int c) {