Fix thumbnail copy bug, update tests

src/io/serialize.c

@@ -505,9 +505,9 @@ void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[SIST_
 
         // Copy tn store contents
         size_t buf_len;
-        char *buf = store_read(IncrementalCopySourceStore, (char *) doc_id, sizeof(doc_id), &buf_len);
+        char *buf = store_read(IncrementalCopySourceStore, (char *) doc_id, SIST_DOC_ID_LEN, &buf_len);
         if (buf_len != 0) {
-            store_write(IncrementalCopyDestinationStore, (char *) doc_id, sizeof(doc_id), buf, buf_len);
+            store_write(IncrementalCopyDestinationStore, (char *) doc_id, SIST_DOC_ID_LEN, buf, buf_len);
             free(buf);
         }
     }

src/io/serialize.h

@@ -42,13 +42,13 @@ index_descriptor_t read_index_descriptor(char *path);
 // caller ensures char file_path[PATH_MAX]
 #define READ_INDICES(file_path, index_path, action_ok, action_main_fail, cond_original) \
     snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", index_path);              \
-    if (0 == access(file_path, R_OK)) {                                                 \
+    if (access(file_path, R_OK) == 0) {                                                 \
         action_ok;                                                                      \
     } else {                                                                            \
         action_main_fail;                                                               \
     }                                                                                   \
     snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", index_path);          \
-    if ((cond_original) && (0 == access(file_path, R_OK))) {                            \
+    if ((cond_original) && access(file_path, R_OK) == 0) {                              \
         action_ok;                                                                      \
     }                                                                                   \
 

src/main.c

@@ -103,7 +103,7 @@ void sig_handler(int signum) {
     exit(-1);
 }
 
-void init_dir(const char *dirpath, scan_args_t* args) {
+void init_dir(const char *dirpath, scan_args_t *args) {
     char path[PATH_MAX];
     snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
 
@@ -112,16 +112,16 @@ void init_dir(const char *dirpath, scan_args_t* args) {
     strcpy(ScanCtx.index.desc.type, INDEX_TYPE_NDJSON);
 
     if (args->incremental != NULL) {
-      // copy old index id
+        // copy old index id
-      char descriptor_path[PATH_MAX];
+        char descriptor_path[PATH_MAX];
-      snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental);
+        snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental);
-      index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
+        index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
-      memcpy(ScanCtx.index.desc.id, original_desc.id, sizeof(original_desc.id));
+        memcpy(ScanCtx.index.desc.id, original_desc.id, sizeof(original_desc.id));
     } else {
-      // generate new index id based on timestamp
+        // generate new index id based on timestamp
-      unsigned char index_md5[MD5_DIGEST_LENGTH];
+        unsigned char index_md5[MD5_DIGEST_LENGTH];
-      MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
+        MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
-      buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
+        buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
     }
 
     write_index_descriptor(path, &ScanCtx.index.desc);
@@ -324,9 +324,13 @@ void load_incremental_index(const scan_args_t *args) {
         LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc.version, Version)
     }
 
-    READ_INDICES(file_path, args->incremental, incremental_read(ScanCtx.original_table, file_path, &original_desc),
+    READ_INDICES(
-                 LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)),
+            file_path,
-                 1);
+            args->incremental,
+            incremental_read(ScanCtx.original_table, file_path, &original_desc),
+            LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)),
+            TRUE
+    );
 
     LOG_INFOF("main.c", "Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table))
 }
@@ -777,9 +781,8 @@ int main(int argc, const char *argv[]) {
         sist2_exec_script(exec_args);
 
     } else {
-        fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
         argparse_usage(&argparse);
-        goto end;
+        LOG_FATALF("main.c", "Invalid command: '%s'\n", argv[0])
     }
     printf("\n");
 

tests/test_scan.py

@@ -35,10 +35,20 @@ def sist2_index(files, *args):
     path = copy_files(files)
 
     shutil.rmtree("test_i", ignore_errors=True)
-    sist2("scan", path, "-o", "test_i", *args)
+    sist2("scan", path, "-o", "test_i", "-t12", *args)
     return iter(sist2_index_to_dict("test_i"))
 
 
+def get_lmdb_contents(path):
+    import lmdb
+
+    env = lmdb.open(path)
+
+    txn = env.begin(write=False)
+
+    return dict((k, v) for k, v in txn.cursor())
+
+
 def sist2_incremental_index(files, func=None, incremental_index=False, *args):
     path = copy_files(files)
 
@@ -46,7 +56,7 @@ def sist2_incremental_index(files, func=None, incremental_index=False, *args):
         func(path)
 
     shutil.rmtree("test_i_inc", ignore_errors=True)
-    sist2("scan", path, "-o", "test_i_inc", "--incremental", "test_i", *args)
+    sist2("scan", path, "-o", "test_i_inc", "--incremental", "test_i", "-t12", *args)
     return iter(sist2_index_to_dict("test_i_inc", incremental_index))
 
 
@@ -76,9 +86,31 @@ class ScanTest(unittest.TestCase):
                 pass
 
         file_count = sum(1 for _ in sist2_index(TEST_FILES))
-        self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, remove_files)), file_count - 2)
+        lmdb_full = get_lmdb_contents("test_i/thumbs")
-        self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, add_files, incremental_index=True)), 3)
+
-        self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, add_files)), file_count + 3)
+        # Remove files
+        num_files_rm1 = len(list(sist2_incremental_index(TEST_FILES, remove_files)))
+        lmdb_rm1 = get_lmdb_contents("test_i_inc/thumbs")
+        self.assertEqual(num_files_rm1, file_count - 2)
+        self.assertEqual(len(set(lmdb_full.keys() - set(lmdb_rm1.keys()))), 2)
+
+        # add files (incremental_index=True)
+        num_files_add_inc = len(list(sist2_incremental_index(TEST_FILES, add_files, incremental_index=True)))
+        lmdb_add_inc = get_lmdb_contents("test_i_inc/thumbs")
+        self.assertEqual(num_files_add_inc, 3)
+        self.assertEqual(set(lmdb_full.keys()), set(lmdb_add_inc.keys()))
+
+        # add files
+        num_files_add = len(list(sist2_incremental_index(TEST_FILES, add_files)))
+        lmdb_add = get_lmdb_contents("test_i_inc/thumbs")
+        self.assertEqual(num_files_add, file_count + 3)
+        self.assertEqual(set(lmdb_full.keys()), set(lmdb_add.keys()))
+
+        # (No action)
+        sist2_incremental_index(TEST_FILES)
+        lmdb_inc = get_lmdb_contents("test_i_inc/thumbs")
+
+        self.assertEqual(set(lmdb_full.keys()), set(lmdb_inc.keys()))
 
 
 if __name__ == "__main__":