simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-16 00:46:43 +00:00
Code Issues Projects Releases Wiki Activity

only add new entries into new_table to save memory

Browse Source
This commit is contained in:
Yatao Li 2022-01-20 23:39:38 +08:00
parent 2cb57f3634
commit e65905a165
4 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/io/serialize.c
View File

@ -535,7 +535,7 @@ void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
read_index(filepath, "", INDEX_TYPE_NDJSON, incremental_copy_handle_doc);
}
void incremental_delete(const char *del_filepath, GHashTable *orig_table, GHashTable *new_table) {
void incremental_delete(const char *del_filepath, GHashTable *orig_table, GHashTable *copy_table, GHashTable *new_table) {
GHashTableIter iter;
gpointer key, UNUSED(value);
char path_md5[MD5_STR_LENGTH + 1];
@ -544,7 +544,8 @@ void incremental_delete(const char *del_filepath, GHashTable *orig_table, GHashT
initialize_writer_ctx(del_filepath);
g_hash_table_iter_init(&iter, orig_table);
while(g_hash_table_iter_next(&iter, &key, &value)) {
if (NULL == g_hash_table_lookup(new_table, key)) {
if (NULL == g_hash_table_lookup(new_table, key) &&
NULL == g_hash_table_lookup(copy_table, key)) {
memcpy(path_md5, key, MD5_STR_LENGTH - 1);
zstd_write_string(path_md5, MD5_STR_LENGTH);
}

2
src/io/serialize.h
View File

@ -17,7 +17,7 @@ typedef void(*index_func)(cJSON *, const char[MD5_STR_LENGTH]);
void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
const char *dst_filepath, GHashTable *copy_table);
void incremental_delete(const char *del_filepath, GHashTable *orig_table, GHashTable *new_table);
void incremental_delete(const char *del_filepath, GHashTable *orig_table, GHashTable *copy_table, GHashTable *new_table);
void write_document(document_t *doc);

2
src/main.c
View File

@ -341,7 +341,7 @@ void save_incremental_index(scan_args_t* args) {
}
snprintf(file_path, PATH_MAX, "%s_index_delete.list.zst", ScanCtx.index.path);
incremental_delete(file_path, ScanCtx.original_table, ScanCtx.new_table);
incremental_delete(file_path, ScanCtx.original_table, ScanCtx.copy_table, ScanCtx.new_table);
READ_INDICES(file_path, args->incremental, incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table),
perror("incremental_copy"), 1);

1
src/parsing/parse.c
View File

@ -81,7 +81,6 @@ void parse(void *arg) {
if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) {
pthread_mutex_lock(&ScanCtx.copy_table_mu);
incremental_mark_file(ScanCtx.copy_table, doc->path_md5);
incremental_mark_file(ScanCtx.new_table, doc->path_md5);
pthread_mutex_unlock(&ScanCtx.copy_table_mu);
pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu);