simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-19 18:26:43 +00:00
Code Issues Projects Releases Wiki Activity

unify READ_INDICES to reduce clutter

Browse Source
This commit is contained in:
Yatao Li 2022-01-19 21:30:35 +08:00
parent 291d307689
commit 679e12f786
4 changed files with 31 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/cli.c
View File

@ -336,8 +336,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
if (index_path == NULL) { if (index_path == NULL) {
LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]) LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1])
} else { } else {
args->index_path = argv[1]; args->index_path = index_path;
free(index_path);
} }
if (args->es_url == NULL) { if (args->es_url == NULL) {
@ -522,8 +521,7 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
if (index_path == NULL) { if (index_path == NULL) {
LOG_FATALF("cli.c", "Invalid index PATH argument. File not found: %s", argv[1]) LOG_FATALF("cli.c", "Invalid index PATH argument. File not found: %s", argv[1])
} else { } else {
args->index_path = argv[1]; args->index_path = index_path;
free(index_path);
} }
if (args->es_url == NULL) { if (args->es_url == NULL) {

14
src/io/serialize.h
View File

@ -38,4 +38,18 @@ void write_index_descriptor(char *path, index_descriptor_t *desc);
index_descriptor_t read_index_descriptor(char *path); index_descriptor_t read_index_descriptor(char *path);
// caller ensures char file_path[PATH_MAX]
#define READ_INDICES(file_path, index_path, action_ok, action_main_fail, cond_original) \
snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", index_path); \
if (0 == access(file_path, R_OK)) { \
action_ok; \
} else { \
action_main_fail; \
} \
snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", index_path); \
if ((cond_original) && 0 == access(file_path, R_OK)) { \
action_ok; \
} \
#endif #endif

49
src/main.c
View File

@ -305,23 +305,15 @@ void load_incremental_index(const scan_args_t *args) {
} }
char descriptor_path[PATH_MAX]; char descriptor_path[PATH_MAX];
snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->incremental); snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental);
index_descriptor_t original_desc = read_index_descriptor(descriptor_path); index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
if (strcmp(original_desc.version, Version) != 0) { if (strcmp(original_desc.version, Version) != 0) {
LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc.version, Version) LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc.version, Version)
} }
snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", args->incremental); READ_INDICES(file_path, args->incremental, incremental_read(ScanCtx.original_table, file_path, &original_desc),
if (0 == access(file_path, R_OK)) { LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)), 1);
incremental_read(ScanCtx.original_table, file_path, &original_desc);
} else {
LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno));
}
snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", args->incremental);
if (0 == access(file_path, R_OK)) {
incremental_read(ScanCtx.original_table, file_path, &original_desc);
}
closedir(dir); closedir(dir);
@ -351,17 +343,8 @@ void save_incremental_index(scan_args_t* args) {
snprintf(file_path, PATH_MAX, "%s_index_delete.list.zst", ScanCtx.index.path); snprintf(file_path, PATH_MAX, "%s_index_delete.list.zst", ScanCtx.index.path);
incremental_delete(file_path, ScanCtx.original_table, ScanCtx.new_table); incremental_delete(file_path, ScanCtx.original_table, ScanCtx.new_table);
snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", args->incremental); READ_INDICES(file_path, args->incremental, incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table),
if (0 == access(file_path, R_OK)) { perror("incremental_copy"), 1);
incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table);
} else {
perror("incremental_copy");
return;
}
snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", args->incremental);
if (0 == access(file_path, R_OK)) {
incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table);
}
closedir(dir); closedir(dir);
store_destroy(source); store_destroy(source);
@ -458,7 +441,7 @@ void sist2_index(index_args_t *args) {
} }
char descriptor_path[PATH_MAX]; char descriptor_path[PATH_MAX];
snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->index_path); snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->index_path);
index_descriptor_t desc = read_index_descriptor(descriptor_path); index_descriptor_t desc = read_index_descriptor(descriptor_path);
@ -474,11 +457,11 @@ void sist2_index(index_args_t *args) {
} }
char path_tmp[PATH_MAX]; char path_tmp[PATH_MAX];
snprintf(path_tmp, sizeof(path_tmp), "%s/tags", args->index_path); snprintf(path_tmp, sizeof(path_tmp), "%stags", args->index_path);
IndexCtx.tag_store = store_create(path_tmp, STORE_SIZE_TAG); IndexCtx.tag_store = store_create(path_tmp, STORE_SIZE_TAG);
IndexCtx.tags = store_read_all(IndexCtx.tag_store); IndexCtx.tags = store_read_all(IndexCtx.tag_store);
snprintf(path_tmp, sizeof(path_tmp), "%s/meta", args->index_path); snprintf(path_tmp, sizeof(path_tmp), "%smeta", args->index_path);
IndexCtx.meta_store = store_create(path_tmp, STORE_SIZE_META); IndexCtx.meta_store = store_create(path_tmp, STORE_SIZE_META);
IndexCtx.meta = store_read_all(IndexCtx.meta_store); IndexCtx.meta = store_read_all(IndexCtx.meta_store);
@ -499,17 +482,11 @@ void sist2_index(index_args_t *args) {
IndexCtx.pool = tpool_create(args->threads, cleanup, FALSE, args->print == 0); IndexCtx.pool = tpool_create(args->threads, cleanup, FALSE, args->print == 0);
tpool_start(IndexCtx.pool); tpool_start(IndexCtx.pool);
snprintf(file_path, PATH_MAX, "%s/_index_original.ndjson.zst", args->index_path); READ_INDICES(file_path, args->index_path, {
if (!args->incremental && 0 == access(file_path, R_OK)) {
read_index(file_path, desc.id, desc.type, f); read_index(file_path, desc.id, desc.type, f);
LOG_DEBUGF("main.c", "Read index file %s (%s)", file_path, desc.type) LOG_DEBUGF("main.c", "Read index file %s (%s)", file_path, desc.type);
} }, {}, !args->incremental);
snprintf(file_path, PATH_MAX, "%s/_index_main.ndjson.zst", args->index_path); snprintf(file_path, PATH_MAX, "%s_index_delete.list.zst", args->index_path);
if (0 == access(file_path, R_OK)) {
read_index(file_path, desc.id, desc.type, f);
LOG_DEBUGF("main.c", "Read index file %s (%s)", file_path, desc.type)
}
snprintf(file_path, PATH_MAX, "%s/_index_delete.list.zst", args->index_path);
if (0 == access(file_path, R_OK)) { if (0 == access(file_path, R_OK)) {
read_lines(file_path, (line_processor_t) { read_lines(file_path, (line_processor_t) {
.data = desc.id, .data = desc.id,
@ -539,7 +516,7 @@ void sist2_exec_script(exec_args_t *args) {
LogCtx.verbose = TRUE; LogCtx.verbose = TRUE;
char descriptor_path[PATH_MAX]; char descriptor_path[PATH_MAX];
snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->index_path); snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->index_path);
index_descriptor_t desc = read_index_descriptor(descriptor_path); index_descriptor_t desc = read_index_descriptor(descriptor_path);
IndexCtx.es_url = args->es_url; IndexCtx.es_url = args->es_url;

10
src/stats.c
View File

@ -96,16 +96,8 @@ void fill_tables(cJSON *document, UNUSED(const char index_id[MD5_STR_LENGTH])) {
} }
void read_index_into_tables(index_t *index) { void read_index_into_tables(index_t *index) {
DIR *dir = opendir(index->path);
struct dirent *de;
while ((de = readdir(dir)) != NULL) {
if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
char file_path[PATH_MAX]; char file_path[PATH_MAX];
snprintf(file_path, PATH_MAX, "%s%s", index->path, de->d_name); READ_INDICES(file_path, index->path, read_index(file_path, index->desc.id, index->desc.type, fill_tables), {}, 1);
read_index(file_path, index->desc.id, index->desc.type, fill_tables);
}
}
closedir(dir);
} }
static size_t rfind(const char *str, int c) { static size_t rfind(const char *str, int c) {