Fix #415, fix sqlite-index error

sist2-admin/frontend/src/views/Tasks.vue

@@ -81,7 +81,7 @@ function humanDuration(sec_num) {
         return `${seconds}s`;
     }
 
-    return "<0s";
+    return "<1s";
 }
 
 export default {
@@ -134,7 +134,7 @@ export default {
                     duration: this.taskDuration(row),
                     time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"),
                     logs: null,
-                    status: [0,1].includes(row.return_code) ? "ok" : "failed",
+                    status: row.return_code === 0 ? "ok" : "failed",
                     _row: row
                 }));
             });

sist2-admin/sist2_admin/jobs.py

@@ -120,6 +120,10 @@ class Sist2Task:
 
         logger.info(f"Started task {self.display_name}")
 
+    def set_pid(self, pid):
+        self.pid = pid
+
+
 
 class Sist2ScanTask(Sist2Task):
 
@@ -133,13 +137,10 @@ class Sist2ScanTask(Sist2Task):
         else:
             self.job.scan_options.output = None
 
-        def set_pid(pid):
+        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
-            self.pid = pid
-
-        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
         self.ended = datetime.utcnow()
 
-        is_ok = return_code in (0, 1)
+        is_ok = (return_code in (0, 1)) if "debug" in sist2.bin_path else (return_code == 0)
 
         if not is_ok:
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
@@ -165,6 +166,9 @@ class Sist2ScanTask(Sist2Task):
             self.job.previous_index_path = self.job.index_path
             db["jobs"][self.job.name] = self.job
 
+        if is_ok:
+            return 0
+
         return return_code
 
 
@@ -185,7 +189,7 @@ class Sist2IndexTask(Sist2Task):
 
         logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
 
-        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback)
+        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
         self.ended = datetime.utcnow()
 
         duration = self.ended - self.started
@@ -249,7 +253,7 @@ class Sist2UserScriptTask(Sist2Task):
         super().run(sist2, db)
 
         try:
-            self.user_script.setup(self.log_callback)
+            self.user_script.setup(self.log_callback, self.set_pid)
         except Exception as e:
             logger.error(f"Setup for {self.user_script.name} failed: ")
             logger.exception(e)
@@ -269,7 +273,7 @@ class Sist2UserScriptTask(Sist2Task):
         self.log_callback({"sist2-admin": f"Starting user script with {executable=}, {index_path=}, {extra_args=}"})
 
         proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.user_script.script_dir())
-        self.pid = proc.pid
+        self.set_pid(proc.pid)
 
         t_stderr = Thread(target=self._consume_logs, args=(self.log_callback, proc, "stderr", False))
         t_stderr.start()
@@ -316,7 +320,7 @@ class TaskQueue:
     def _tasks_failed(self):
         done = set()
 
-        for row in self._db["task_done"].sql("WHERE return_code NOT IN (0,1)"):
+        for row in self._db["task_done"].sql("WHERE return_code != 0"):
             done.add(uuid.UUID(row["id"]))
 
         return done

sist2-admin/sist2_admin/script.py

@@ -20,7 +20,7 @@ def set_executable(file):
     os.chmod(file, os.stat(file).st_mode | stat.S_IEXEC)
 
 
-def _initialize_git_repository(url, path, log_cb, force_clone):
+def _initialize_git_repository(url, path, log_cb, force_clone, set_pid_cb):
     log_cb({"sist2-admin": f"Cloning {url}"})
 
     if force_clone or not os.path.exists(os.path.join(path, ".git")):
@@ -36,14 +36,18 @@ def _initialize_git_repository(url, path, log_cb, force_clone):
         log_cb({"sist2-admin": f"Executing setup script {setup_script}"})
 
         set_executable(setup_script)
-        result = subprocess.run([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        proc = subprocess.Popen([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        for line in result.stdout.split(b"\n"):
+        set_pid_cb(proc.pid)
+        proc.wait()
+        stdout = proc.stdout.read()
+
+        for line in stdout.split(b"\n"):
             if line:
                 log_cb({"stdout": line.decode()})
 
-        log_cb({"stdout": f"Executed setup script {setup_script}, return code = {result.returncode}"})
+        log_cb({"stdout": f"Executed setup script {setup_script}, return code = {proc.returncode}"})
 
-        if result.returncode != 0:
+        if proc.returncode != 0:
             raise Exception("Error when running setup script!")
 
     log_cb({"sist2-admin": f"Initialized git repository in {path}"})
@@ -60,11 +64,11 @@ class UserScript(BaseModel):
     def script_dir(self):
         return os.path.join(SCRIPT_FOLDER, self.name)
 
-    def setup(self, log_cb):
+    def setup(self, log_cb, set_pid_cb):
         os.makedirs(self.script_dir(), exist_ok=True)
 
         if self.type == ScriptType.GIT:
-            _initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone)
+            _initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone, set_pid_cb)
             self.force_clone = False
         elif self.type == ScriptType.SIMPLE:
             self._setup_simple()

sist2-admin/sist2_admin/sist2.py

@@ -243,7 +243,7 @@ class Sist2:
         self.bin_path = bin_path
         self._data_dir = data_directory
 
-    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
+    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb, set_pid_cb):
 
         args = [
             self.bin_path,
@@ -255,6 +255,8 @@ class Sist2:
         logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
+        set_pid_cb(proc.pid)
+
         t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
         t_stderr.start()
 

src/ctx.c

@@ -1,8 +1,6 @@
 #include "ctx.h"
 
 ScanCtx_t ScanCtx = {
-        .stat_index_size = 0,
-        .stat_tn_size = 0,
         .pool = NULL,
         .index.path = {0,},
 };

src/ctx.h

@@ -31,9 +31,6 @@ typedef struct {
     int depth;
     int calculate_checksums;
 
-    size_t stat_tn_size;
-    size_t stat_index_size;
-
     pcre *exclude;
     pcre_extra *exclude_extra;
     int fast;

src/database/database.c

@@ -149,7 +149,7 @@ void database_open(database_t *db) {
     }
 
 #ifdef SIST_DEBUG
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA foreign_keys = ON;", NULL, NULL, NULL));
+        //    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA foreign_keys = ON;", NULL, NULL, NULL));
 #else
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA ignore_check_constraints = ON;", NULL, NULL, NULL));
 #endif
@@ -373,7 +373,7 @@ void database_open(database_t *db) {
 }
 
 void database_close(database_t *db, int optimize) {
-    LOG_DEBUGF("database.c", "Closing database %s", db->filename);
+    LOG_DEBUGF("database.c", "Closing database %s (%p)", db->filename, db->db);
 
     if (optimize) {
         LOG_DEBUG("database.c", "Optimizing database");
@@ -594,8 +594,9 @@ cJSON *database_document_iter(database_iterator_t *iter) {
 cJSON *database_incremental_scan_begin(database_t *db) {
     LOG_DEBUG("database.c", "Preparing database for incremental scan");
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "DELETE FROM marked;", NULL, NULL, NULL));
+    LOG_DEBUG("database.c", "Preparing database for incremental scan (create marked table)");
     CRASH_IF_NOT_SQLITE_OK(
-            sqlite3_exec(db->db, "INSERT INTO marked SELECT ROWID, 0, mtime FROM document;", NULL, NULL, NULL));
+            sqlite3_exec(db->db, "INSERT INTO marked SELECT id, 0, mtime FROM document;", NULL, NULL, NULL));
 }
 
 cJSON *database_incremental_scan_end(database_t *db) {

src/main.c

@@ -260,9 +260,6 @@ void sist2_scan(scan_args_t *args) {
     tpool_wait(ScanCtx.pool);
     tpool_destroy(ScanCtx.pool);
 
-    LOG_DEBUGF("main.c", "Thumbnail store size: %lu", ScanCtx.stat_tn_size);
-    LOG_DEBUGF("main.c", "Index size: %lu", ScanCtx.stat_index_size);
-
     database_t *db = database_create(args->output, INDEX_DATABASE);
     database_open(db);
 
@@ -356,7 +353,6 @@ void sist2_sqlite_index(sqlite_index_args_t *args) {
     database_fts_optimize(db);
 
     database_close(db, FALSE);
-    database_close(search_db, FALSE);
 }
 
 void sist2_web(web_args_t *args) {

src/sist.h

@@ -55,7 +55,7 @@
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
 static const int VersionMinor = 3;
-static const int VersionPatch = 0;
+static const int VersionPatch = 1;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown

src/tpool.c

@@ -110,11 +110,11 @@ static void worker_thread_loop(tpool_t *pool) {
             if (LogCtx.json_logs) {
                 progress_bar_print_json(done,
                                         count,
-                                        ScanCtx.stat_tn_size,
+                                        0,
-                                        ScanCtx.stat_index_size, pool->shm->waiting);
+                                        0, pool->shm->waiting);
             } else {
                 progress_bar_print((double) done / count,
-                                   ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
+                                   0, 0);
             }
         }
 
@@ -272,7 +272,7 @@ void tpool_wait(tpool_t *pool) {
         }
     }
     if (pool->print_progress && !LogCtx.json_logs) {
-        progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
+        progress_bar_print(1.0, 0, 0);
     }
     pthread_mutex_unlock(&pool->shm->mutex);