simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-04 07:52:59 +00:00
Code Issues Projects Releases Wiki Activity

Fix #415, fix sqlite-index error

Browse Source
This commit is contained in:
simon987 2023-09-16 09:43:55 -04:00
parent 8fdb832c85
commit a19ec3305a
10 changed files with 38 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sist2-admin/frontend/src/views/Tasks.vue
View File

@ -81,7 +81,7 @@ function humanDuration(sec_num) {
return `${seconds}s`;
}
return "<0s";
return "<1s";
}
export default {
@ -134,7 +134,7 @@ export default {
duration: this.taskDuration(row),
time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"),
logs: null,
status: [0,1].includes(row.return_code) ? "ok" : "failed",
status: row.return_code === 0 ? "ok" : "failed",
_row: row
}));
});

22
sist2-admin/sist2_admin/jobs.py
View File

@ -120,6 +120,10 @@ class Sist2Task:
logger.info(f"Started task {self.display_name}")
def set_pid(self, pid):
self.pid = pid
class Sist2ScanTask(Sist2Task):
@ -133,13 +137,10 @@ class Sist2ScanTask(Sist2Task):
else:
self.job.scan_options.output = None
def set_pid(pid):
self.pid = pid
return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
self.ended = datetime.utcnow()
is_ok = return_code in (0, 1)
is_ok = (return_code in (0, 1)) if "debug" in sist2.bin_path else (return_code == 0)
if not is_ok:
self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
@ -165,6 +166,9 @@ class Sist2ScanTask(Sist2Task):
self.job.previous_index_path = self.job.index_path
db["jobs"][self.job.name] = self.job
if is_ok:
return 0
return return_code
@ -185,7 +189,7 @@ class Sist2IndexTask(Sist2Task):
logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback)
return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
self.ended = datetime.utcnow()
duration = self.ended - self.started
@ -249,7 +253,7 @@ class Sist2UserScriptTask(Sist2Task):
super().run(sist2, db)
try:
self.user_script.setup(self.log_callback)
self.user_script.setup(self.log_callback, self.set_pid)
except Exception as e:
logger.error(f"Setup for {self.user_script.name} failed: ")
logger.exception(e)
@ -269,7 +273,7 @@ class Sist2UserScriptTask(Sist2Task):
self.log_callback({"sist2-admin": f"Starting user script with {executable=}, {index_path=}, {extra_args=}"})
proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.user_script.script_dir())
self.pid = proc.pid
self.set_pid(proc.pid)
t_stderr = Thread(target=self._consume_logs, args=(self.log_callback, proc, "stderr", False))
t_stderr.start()
@ -316,7 +320,7 @@ class TaskQueue:
def _tasks_failed(self):
done = set()
for row in self._db["task_done"].sql("WHERE return_code NOT IN (0,1)"):
for row in self._db["task_done"].sql("WHERE return_code != 0"):
done.add(uuid.UUID(row["id"]))
return done

18
sist2-admin/sist2_admin/script.py
View File

@ -20,7 +20,7 @@ def set_executable(file):
os.chmod(file, os.stat(file).st_mode | stat.S_IEXEC)
def _initialize_git_repository(url, path, log_cb, force_clone):
def _initialize_git_repository(url, path, log_cb, force_clone, set_pid_cb):
log_cb({"sist2-admin": f"Cloning {url}"})
if force_clone or not os.path.exists(os.path.join(path, ".git")):
@ -36,14 +36,18 @@ def _initialize_git_repository(url, path, log_cb, force_clone):
log_cb({"sist2-admin": f"Executing setup script {setup_script}"})
set_executable(setup_script)
result = subprocess.run([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
for line in result.stdout.split(b"\n"):
proc = subprocess.Popen([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
set_pid_cb(proc.pid)
proc.wait()
stdout = proc.stdout.read()
for line in stdout.split(b"\n"):
if line:
log_cb({"stdout": line.decode()})
log_cb({"stdout": f"Executed setup script {setup_script}, return code = {result.returncode}"})
log_cb({"stdout": f"Executed setup script {setup_script}, return code = {proc.returncode}"})
if result.returncode != 0:
if proc.returncode != 0:
raise Exception("Error when running setup script!")
log_cb({"sist2-admin": f"Initialized git repository in {path}"})
@ -60,11 +64,11 @@ class UserScript(BaseModel):
def script_dir(self):
return os.path.join(SCRIPT_FOLDER, self.name)
def setup(self, log_cb):
def setup(self, log_cb, set_pid_cb):
os.makedirs(self.script_dir(), exist_ok=True)
if self.type == ScriptType.GIT:
_initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone)
_initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone, set_pid_cb)
self.force_clone = False
elif self.type == ScriptType.SIMPLE:
self._setup_simple()

4
sist2-admin/sist2_admin/sist2.py
View File

@ -243,7 +243,7 @@ class Sist2:
self.bin_path = bin_path
self._data_dir = data_directory
def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb, set_pid_cb):
args = [
self.bin_path,
@ -255,6 +255,8 @@ class Sist2:
logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
proc = Popen(args, stdout=PIPE, stderr=PIPE)
set_pid_cb(proc.pid)
t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
t_stderr.start()

2
src/ctx.c
View File

@ -1,8 +1,6 @@
#include "ctx.h"
ScanCtx_t ScanCtx = {
.stat_index_size = 0,
.stat_tn_size = 0,
.pool = NULL,
.index.path = {0,},
};

3
src/ctx.h
View File

@ -31,9 +31,6 @@ typedef struct {
int depth;
int calculate_checksums;
size_t stat_tn_size;
size_t stat_index_size;
pcre *exclude;
pcre_extra *exclude_extra;
int fast;

7
src/database/database.c
View File

@ -149,7 +149,7 @@ void database_open(database_t *db) {
}
#ifdef SIST_DEBUG
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA foreign_keys = ON;", NULL, NULL, NULL));
// CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA foreign_keys = ON;", NULL, NULL, NULL));
#else
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA ignore_check_constraints = ON;", NULL, NULL, NULL));
#endif
@ -373,7 +373,7 @@ void database_open(database_t *db) {
}
void database_close(database_t *db, int optimize) {
LOG_DEBUGF("database.c", "Closing database %s", db->filename);
LOG_DEBUGF("database.c", "Closing database %s (%p)", db->filename, db->db);
if (optimize) {
LOG_DEBUG("database.c", "Optimizing database");
@ -594,8 +594,9 @@ cJSON *database_document_iter(database_iterator_t *iter) {
cJSON *database_incremental_scan_begin(database_t *db) {
LOG_DEBUG("database.c", "Preparing database for incremental scan");
CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "DELETE FROM marked;", NULL, NULL, NULL));
LOG_DEBUG("database.c", "Preparing database for incremental scan (create marked table)");
CRASH_IF_NOT_SQLITE_OK(
sqlite3_exec(db->db, "INSERT INTO marked SELECT ROWID, 0, mtime FROM document;", NULL, NULL, NULL));
sqlite3_exec(db->db, "INSERT INTO marked SELECT id, 0, mtime FROM document;", NULL, NULL, NULL));
}
cJSON *database_incremental_scan_end(database_t *db) {

4
src/main.c
View File

@ -260,9 +260,6 @@ void sist2_scan(scan_args_t *args) {
tpool_wait(ScanCtx.pool);
tpool_destroy(ScanCtx.pool);
LOG_DEBUGF("main.c", "Thumbnail store size: %lu", ScanCtx.stat_tn_size);
LOG_DEBUGF("main.c", "Index size: %lu", ScanCtx.stat_index_size);
database_t *db = database_create(args->output, INDEX_DATABASE);
database_open(db);
@ -356,7 +353,6 @@ void sist2_sqlite_index(sqlite_index_args_t *args) {
database_fts_optimize(db);
database_close(db, FALSE);
database_close(search_db, FALSE);
}
void sist2_web(web_args_t *args) {

2
src/sist.h
View File

@ -55,7 +55,7 @@
static const char *const Version = VERSION;
static const int VersionMajor = 3;
static const int VersionMinor = 3;
static const int VersionPatch = 0;
static const int VersionPatch = 1;
#ifndef SIST_PLATFORM
#define SIST_PLATFORM unknown

8
src/tpool.c
View File

@ -110,11 +110,11 @@ static void worker_thread_loop(tpool_t *pool) {
if (LogCtx.json_logs) {
progress_bar_print_json(done,
count,
ScanCtx.stat_tn_size,
ScanCtx.stat_index_size, pool->shm->waiting);
0,
0, pool->shm->waiting);
} else {
progress_bar_print((double) done / count,
ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
0, 0);
}
}
@ -272,7 +272,7 @@ void tpool_wait(tpool_t *pool) {
}
}
if (pool->print_progress && !LogCtx.json_logs) {
progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
progress_bar_print(1.0, 0, 0);
}
pthread_mutex_unlock(&pool->shm->mutex);