From a75675eceaf708d73640bd6c449e806d2d891186 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 16 Apr 2022 11:48:43 -0400 Subject: [PATCH] Fix thumbnail copy bug, update tests --- src/io/serialize.c | 4 ++-- src/io/serialize.h | 4 ++-- src/main.c | 33 ++++++++++++++++++--------------- tests/test_scan.py | 42 +++++++++++++++++++++++++++++++++++++----- 4 files changed, 59 insertions(+), 24 deletions(-) diff --git a/src/io/serialize.c b/src/io/serialize.c index 40880fc..8c77da2 100644 --- a/src/io/serialize.c +++ b/src/io/serialize.c @@ -505,9 +505,9 @@ void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[SIST_ // Copy tn store contents size_t buf_len; - char *buf = store_read(IncrementalCopySourceStore, (char *) doc_id, sizeof(doc_id), &buf_len); + char *buf = store_read(IncrementalCopySourceStore, (char *) doc_id, SIST_DOC_ID_LEN, &buf_len); if (buf_len != 0) { - store_write(IncrementalCopyDestinationStore, (char *) doc_id, sizeof(doc_id), buf, buf_len); + store_write(IncrementalCopyDestinationStore, (char *) doc_id, SIST_DOC_ID_LEN, buf, buf_len); free(buf); } } diff --git a/src/io/serialize.h b/src/io/serialize.h index c51d262..2da3cb3 100644 --- a/src/io/serialize.h +++ b/src/io/serialize.h @@ -42,13 +42,13 @@ index_descriptor_t read_index_descriptor(char *path); // caller ensures char file_path[PATH_MAX] #define READ_INDICES(file_path, index_path, action_ok, action_main_fail, cond_original) \ snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", index_path); \ - if (0 == access(file_path, R_OK)) { \ + if (access(file_path, R_OK) == 0) { \ action_ok; \ } else { \ action_main_fail; \ } \ snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", index_path); \ - if ((cond_original) && (0 == access(file_path, R_OK))) { \ + if ((cond_original) && access(file_path, R_OK) == 0) { \ action_ok; \ } \ diff --git a/src/main.c b/src/main.c index c88923e..c95ccb1 100644 --- a/src/main.c +++ b/src/main.c @@ -103,7 +103,7 @@ void sig_handler(int signum) { exit(-1); } -void init_dir(const char *dirpath, scan_args_t* args) { +void init_dir(const char *dirpath, scan_args_t *args) { char path[PATH_MAX]; snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath); @@ -112,16 +112,16 @@ void init_dir(const char *dirpath, scan_args_t* args) { strcpy(ScanCtx.index.desc.type, INDEX_TYPE_NDJSON); if (args->incremental != NULL) { - // copy old index id - char descriptor_path[PATH_MAX]; - snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental); - index_descriptor_t original_desc = read_index_descriptor(descriptor_path); - memcpy(ScanCtx.index.desc.id, original_desc.id, sizeof(original_desc.id)); + // copy old index id + char descriptor_path[PATH_MAX]; + snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental); + index_descriptor_t original_desc = read_index_descriptor(descriptor_path); + memcpy(ScanCtx.index.desc.id, original_desc.id, sizeof(original_desc.id)); } else { - // generate new index id based on timestamp - unsigned char index_md5[MD5_DIGEST_LENGTH]; - MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5); - buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id); + // generate new index id based on timestamp + unsigned char index_md5[MD5_DIGEST_LENGTH]; + MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5); + buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id); } write_index_descriptor(path, &ScanCtx.index.desc); @@ -324,9 +324,13 @@ void load_incremental_index(const scan_args_t *args) { LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc.version, Version) } - READ_INDICES(file_path, args->incremental, incremental_read(ScanCtx.original_table, file_path, &original_desc), - LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)), - 1); + READ_INDICES( + file_path, + args->incremental, + incremental_read(ScanCtx.original_table, file_path, &original_desc), + LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)), + TRUE + ); LOG_INFOF("main.c", "Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table)) } @@ -777,9 +781,8 @@ int main(int argc, const char *argv[]) { sist2_exec_script(exec_args); } else { - fprintf(stderr, "Invalid command: '%s'\n", argv[0]); argparse_usage(&argparse); - goto end; + LOG_FATALF("main.c", "Invalid command: '%s'\n", argv[0]) } printf("\n"); diff --git a/tests/test_scan.py b/tests/test_scan.py index 5d4bf1f..f7a8b16 100644 --- a/tests/test_scan.py +++ b/tests/test_scan.py @@ -35,10 +35,20 @@ def sist2_index(files, *args): path = copy_files(files) shutil.rmtree("test_i", ignore_errors=True) - sist2("scan", path, "-o", "test_i", *args) + sist2("scan", path, "-o", "test_i", "-t12", *args) return iter(sist2_index_to_dict("test_i")) +def get_lmdb_contents(path): + import lmdb + + env = lmdb.open(path) + + txn = env.begin(write=False) + + return dict((k, v) for k, v in txn.cursor()) + + def sist2_incremental_index(files, func=None, incremental_index=False, *args): path = copy_files(files) @@ -46,7 +56,7 @@ def sist2_incremental_index(files, func=None, incremental_index=False, *args): func(path) shutil.rmtree("test_i_inc", ignore_errors=True) - sist2("scan", path, "-o", "test_i_inc", "--incremental", "test_i", *args) + sist2("scan", path, "-o", "test_i_inc", "--incremental", "test_i", "-t12", *args) return iter(sist2_index_to_dict("test_i_inc", incremental_index)) @@ -76,9 +86,31 @@ class ScanTest(unittest.TestCase): pass file_count = sum(1 for _ in sist2_index(TEST_FILES)) - self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, remove_files)), file_count - 2) - self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, add_files, incremental_index=True)), 3) - self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, add_files)), file_count + 3) + lmdb_full = get_lmdb_contents("test_i/thumbs") + + # Remove files + num_files_rm1 = len(list(sist2_incremental_index(TEST_FILES, remove_files))) + lmdb_rm1 = get_lmdb_contents("test_i_inc/thumbs") + self.assertEqual(num_files_rm1, file_count - 2) + self.assertEqual(len(set(lmdb_full.keys() - set(lmdb_rm1.keys()))), 2) + + # add files (incremental_index=True) + num_files_add_inc = len(list(sist2_incremental_index(TEST_FILES, add_files, incremental_index=True))) + lmdb_add_inc = get_lmdb_contents("test_i_inc/thumbs") + self.assertEqual(num_files_add_inc, 3) + self.assertEqual(set(lmdb_full.keys()), set(lmdb_add_inc.keys())) + + # add files + num_files_add = len(list(sist2_incremental_index(TEST_FILES, add_files))) + lmdb_add = get_lmdb_contents("test_i_inc/thumbs") + self.assertEqual(num_files_add, file_count + 3) + self.assertEqual(set(lmdb_full.keys()), set(lmdb_add.keys())) + + # (No action) + sist2_incremental_index(TEST_FILES) + lmdb_inc = get_lmdb_contents("test_i_inc/thumbs") + + self.assertEqual(set(lmdb_full.keys()), set(lmdb_inc.keys())) if __name__ == "__main__":