diff --git a/sist2-admin/frontend/src/views/Tasks.vue b/sist2-admin/frontend/src/views/Tasks.vue index 15a3dd3..2961df4 100644 --- a/sist2-admin/frontend/src/views/Tasks.vue +++ b/sist2-admin/frontend/src/views/Tasks.vue @@ -81,7 +81,7 @@ function humanDuration(sec_num) { return `${seconds}s`; } - return "<0s"; + return "<1s"; } export default { @@ -134,7 +134,7 @@ export default { duration: this.taskDuration(row), time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"), logs: null, - status: [0,1].includes(row.return_code) ? "ok" : "failed", + status: row.return_code === 0 ? "ok" : "failed", _row: row })); }); diff --git a/sist2-admin/sist2_admin/jobs.py b/sist2-admin/sist2_admin/jobs.py index 1244d7a..7016b1a 100644 --- a/sist2-admin/sist2_admin/jobs.py +++ b/sist2-admin/sist2_admin/jobs.py @@ -120,6 +120,10 @@ class Sist2Task: logger.info(f"Started task {self.display_name}") + def set_pid(self, pid): + self.pid = pid + + class Sist2ScanTask(Sist2Task): @@ -133,13 +137,10 @@ class Sist2ScanTask(Sist2Task): else: self.job.scan_options.output = None - def set_pid(pid): - self.pid = pid - - return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid) + return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=self.set_pid) self.ended = datetime.utcnow() - is_ok = return_code in (0, 1) + is_ok = (return_code in (0, 1)) if "debug" in sist2.bin_path else (return_code == 0) if not is_ok: self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"})) @@ -165,6 +166,9 @@ class Sist2ScanTask(Sist2Task): self.job.previous_index_path = self.job.index_path db["jobs"][self.job.name] = self.job + if is_ok: + return 0 + return return_code @@ -185,7 +189,7 @@ class Sist2IndexTask(Sist2Task): logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}") - return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback) + return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback, set_pid_cb=self.set_pid) self.ended = datetime.utcnow() duration = self.ended - self.started @@ -249,7 +253,7 @@ class Sist2UserScriptTask(Sist2Task): super().run(sist2, db) try: - self.user_script.setup(self.log_callback) + self.user_script.setup(self.log_callback, self.set_pid) except Exception as e: logger.error(f"Setup for {self.user_script.name} failed: ") logger.exception(e) @@ -269,7 +273,7 @@ class Sist2UserScriptTask(Sist2Task): self.log_callback({"sist2-admin": f"Starting user script with {executable=}, {index_path=}, {extra_args=}"}) proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.user_script.script_dir()) - self.pid = proc.pid + self.set_pid(proc.pid) t_stderr = Thread(target=self._consume_logs, args=(self.log_callback, proc, "stderr", False)) t_stderr.start() @@ -316,7 +320,7 @@ class TaskQueue: def _tasks_failed(self): done = set() - for row in self._db["task_done"].sql("WHERE return_code NOT IN (0,1)"): + for row in self._db["task_done"].sql("WHERE return_code != 0"): done.add(uuid.UUID(row["id"])) return done diff --git a/sist2-admin/sist2_admin/script.py b/sist2-admin/sist2_admin/script.py index 1c6c2ca..10d699c 100644 --- a/sist2-admin/sist2_admin/script.py +++ b/sist2-admin/sist2_admin/script.py @@ -20,7 +20,7 @@ def set_executable(file): os.chmod(file, os.stat(file).st_mode | stat.S_IEXEC) -def _initialize_git_repository(url, path, log_cb, force_clone): +def _initialize_git_repository(url, path, log_cb, force_clone, set_pid_cb): log_cb({"sist2-admin": f"Cloning {url}"}) if force_clone or not os.path.exists(os.path.join(path, ".git")): @@ -36,14 +36,18 @@ def _initialize_git_repository(url, path, log_cb, force_clone): log_cb({"sist2-admin": f"Executing setup script {setup_script}"}) set_executable(setup_script) - result = subprocess.run([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - for line in result.stdout.split(b"\n"): + proc = subprocess.Popen([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + set_pid_cb(proc.pid) + proc.wait() + stdout = proc.stdout.read() + + for line in stdout.split(b"\n"): if line: log_cb({"stdout": line.decode()}) - log_cb({"stdout": f"Executed setup script {setup_script}, return code = {result.returncode}"}) + log_cb({"stdout": f"Executed setup script {setup_script}, return code = {proc.returncode}"}) - if result.returncode != 0: + if proc.returncode != 0: raise Exception("Error when running setup script!") log_cb({"sist2-admin": f"Initialized git repository in {path}"}) @@ -60,11 +64,11 @@ class UserScript(BaseModel): def script_dir(self): return os.path.join(SCRIPT_FOLDER, self.name) - def setup(self, log_cb): + def setup(self, log_cb, set_pid_cb): os.makedirs(self.script_dir(), exist_ok=True) if self.type == ScriptType.GIT: - _initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone) + _initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone, set_pid_cb) self.force_clone = False elif self.type == ScriptType.SIMPLE: self._setup_simple() diff --git a/sist2-admin/sist2_admin/sist2.py b/sist2-admin/sist2_admin/sist2.py index 8a5ecfd..57b6609 100644 --- a/sist2-admin/sist2_admin/sist2.py +++ b/sist2-admin/sist2_admin/sist2.py @@ -243,7 +243,7 @@ class Sist2: self.bin_path = bin_path self._data_dir = data_directory - def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb): + def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb, set_pid_cb): args = [ self.bin_path, @@ -255,6 +255,8 @@ class Sist2: logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"}) proc = Popen(args, stdout=PIPE, stderr=PIPE) + set_pid_cb(proc.pid) + t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc)) t_stderr.start() diff --git a/src/ctx.c b/src/ctx.c index fe6e8a7..5f3a733 100644 --- a/src/ctx.c +++ b/src/ctx.c @@ -1,8 +1,6 @@ #include "ctx.h" ScanCtx_t ScanCtx = { - .stat_index_size = 0, - .stat_tn_size = 0, .pool = NULL, .index.path = {0,}, }; diff --git a/src/ctx.h b/src/ctx.h index ea32c77..809ca26 100644 --- a/src/ctx.h +++ b/src/ctx.h @@ -31,9 +31,6 @@ typedef struct { int depth; int calculate_checksums; - size_t stat_tn_size; - size_t stat_index_size; - pcre *exclude; pcre_extra *exclude_extra; int fast; diff --git a/src/database/database.c b/src/database/database.c index 1962cb2..62e409d 100644 --- a/src/database/database.c +++ b/src/database/database.c @@ -149,7 +149,7 @@ void database_open(database_t *db) { } #ifdef SIST_DEBUG - CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA foreign_keys = ON;", NULL, NULL, NULL)); + // CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA foreign_keys = ON;", NULL, NULL, NULL)); #else CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA ignore_check_constraints = ON;", NULL, NULL, NULL)); #endif @@ -373,7 +373,7 @@ void database_open(database_t *db) { } void database_close(database_t *db, int optimize) { - LOG_DEBUGF("database.c", "Closing database %s", db->filename); + LOG_DEBUGF("database.c", "Closing database %s (%p)", db->filename, db->db); if (optimize) { LOG_DEBUG("database.c", "Optimizing database"); @@ -594,8 +594,9 @@ cJSON *database_document_iter(database_iterator_t *iter) { cJSON *database_incremental_scan_begin(database_t *db) { LOG_DEBUG("database.c", "Preparing database for incremental scan"); CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "DELETE FROM marked;", NULL, NULL, NULL)); + LOG_DEBUG("database.c", "Preparing database for incremental scan (create marked table)"); CRASH_IF_NOT_SQLITE_OK( - sqlite3_exec(db->db, "INSERT INTO marked SELECT ROWID, 0, mtime FROM document;", NULL, NULL, NULL)); + sqlite3_exec(db->db, "INSERT INTO marked SELECT id, 0, mtime FROM document;", NULL, NULL, NULL)); } cJSON *database_incremental_scan_end(database_t *db) { diff --git a/src/main.c b/src/main.c index e75e9fd..a0a7ac1 100644 --- a/src/main.c +++ b/src/main.c @@ -260,9 +260,6 @@ void sist2_scan(scan_args_t *args) { tpool_wait(ScanCtx.pool); tpool_destroy(ScanCtx.pool); - LOG_DEBUGF("main.c", "Thumbnail store size: %lu", ScanCtx.stat_tn_size); - LOG_DEBUGF("main.c", "Index size: %lu", ScanCtx.stat_index_size); - database_t *db = database_create(args->output, INDEX_DATABASE); database_open(db); @@ -356,7 +353,6 @@ void sist2_sqlite_index(sqlite_index_args_t *args) { database_fts_optimize(db); database_close(db, FALSE); - database_close(search_db, FALSE); } void sist2_web(web_args_t *args) { diff --git a/src/sist.h b/src/sist.h index 74b54ec..b7d78d6 100644 --- a/src/sist.h +++ b/src/sist.h @@ -55,7 +55,7 @@ static const char *const Version = VERSION; static const int VersionMajor = 3; static const int VersionMinor = 3; -static const int VersionPatch = 0; +static const int VersionPatch = 1; #ifndef SIST_PLATFORM #define SIST_PLATFORM unknown diff --git a/src/tpool.c b/src/tpool.c index 430c0b2..1b9eebd 100644 --- a/src/tpool.c +++ b/src/tpool.c @@ -110,11 +110,11 @@ static void worker_thread_loop(tpool_t *pool) { if (LogCtx.json_logs) { progress_bar_print_json(done, count, - ScanCtx.stat_tn_size, - ScanCtx.stat_index_size, pool->shm->waiting); + 0, + 0, pool->shm->waiting); } else { progress_bar_print((double) done / count, - ScanCtx.stat_tn_size, ScanCtx.stat_index_size); + 0, 0); } } @@ -272,7 +272,7 @@ void tpool_wait(tpool_t *pool) { } } if (pool->print_progress && !LogCtx.json_logs) { - progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size); + progress_bar_print(1.0, 0, 0); } pthread_mutex_unlock(&pool->shm->mutex);