diff --git a/sist2-admin/frontend/src/Sist2AdminApi.js b/sist2-admin/frontend/src/Sist2AdminApi.js index 0a68fe0..6bfca50 100644 --- a/sist2-admin/frontend/src/Sist2AdminApi.js +++ b/sist2-admin/frontend/src/Sist2AdminApi.js @@ -112,6 +112,16 @@ class Sist2AdminApi { getSist2AdminInfo() { return axios.get(`${this.baseUrl}/api/`); } + + getLogsToDelete(jobName, n) { + return axios.get(`${this.baseUrl}/api/job/${jobName}/logs_to_delete`, { + params: {n: n} + }); + } + + deleteTaskLogs(taskId) { + return axios.post(`${this.baseUrl}/api/task/${taskId}/delete_logs`); + } } export default new Sist2AdminApi() \ No newline at end of file diff --git a/sist2-admin/frontend/src/components/JobOptions.vue b/sist2-admin/frontend/src/components/JobOptions.vue index 84ed8e7..eced466 100644 --- a/sist2-admin/frontend/src/components/JobOptions.vue +++ b/sist2-admin/frontend/src/components/JobOptions.vue @@ -1,57 +1,94 @@ \ No newline at end of file diff --git a/sist2-admin/frontend/src/i18n/messages.js b/sist2-admin/frontend/src/i18n/messages.js index a46b6a3..f1102ad 100644 --- a/sist2-admin/frontend/src/i18n/messages.js +++ b/sist2-admin/frontend/src/i18n/messages.js @@ -5,10 +5,13 @@ export default { go: "Go", online: "online", offline: "offline", + view: "View", delete: "Delete", runNow: "Index now", create: "Create", + cancel: "Cancel", test: "Test", + confirmation: "Confirmation", jobTitle: "job configuration", tasks: "Tasks", @@ -99,6 +102,8 @@ export default { jobOptions: { title: "Job options", cron: "Job schedule", + keepNLogs: "Keep last N log files. Set to -1 to keep all logs.", + deleteNow: "Delete now", scheduleEnabled: "Enable scheduled re-scan", noJobAvailable: "No jobs available.", desktopNotifications: "Desktop notifications" diff --git a/sist2-admin/frontend/src/views/Tasks.vue b/sist2-admin/frontend/src/views/Tasks.vue index 7fe92e8..766500a 100644 --- a/sist2-admin/frontend/src/views/Tasks.vue +++ b/sist2-admin/frontend/src/views/Tasks.vue @@ -1,38 +1,49 @@ diff --git a/sist2-admin/sist2_admin/app.py b/sist2-admin/sist2_admin/app.py index a63d460..db339e5 100644 --- a/sist2-admin/sist2_admin/app.py +++ b/sist2-admin/sist2_admin/app.py @@ -21,7 +21,8 @@ from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus from notifications import Subscribe, Notifications from sist2 import Sist2 -from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION +from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION, migrate_v3_to_v4, \ + get_log_files_to_remove, delete_log_file from web import Sist2Frontend sist2 = Sist2(SIST2_BINARY, DATA_FOLDER) @@ -80,7 +81,6 @@ async def get_jobs(): @app.put("/api/job/{name:str}") async def update_job(name: str, new_job: Sist2Job): - new_job.last_modified = datetime.utcnow() job = db["jobs"][name] if not job: @@ -133,6 +133,16 @@ async def kill_job(task_id: str): return task_queue.kill_task(task_id) +@app.post("/api/task/{task_id:str}/delete_logs") +async def delete_task_logs(task_id: str): + if not db["task_done"][task_id]: + raise HTTPException(status_code=404) + + delete_log_file(db, task_id) + + return "ok" + + def _run_job(job: Sist2Job): job.last_modified = datetime.utcnow() if job.status == JobStatus("created"): @@ -157,6 +167,11 @@ async def run_job(name: str): return "ok" +@app.get("/api/job/{name:str}/logs_to_delete") +async def task_history(n: int, name: str): + return get_log_files_to_remove(db, name, n) + + @app.delete("/api/job/{name:str}") async def delete_job(name: str): job = db["jobs"][name] @@ -320,7 +335,6 @@ async def ws_tail_log(websocket: WebSocket): async with Subscribe(notifications) as ob: async for notification in ob.notifications(): await websocket.send_json(notification) - print(notification) except ConnectionClosed: return @@ -380,6 +394,9 @@ if __name__ == '__main__': if db["sist2_admin"]["info"]["version"] == "2": logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.") exit(-1) + if db["sist2_admin"]["info"]["version"] == "3": + logger.info("Migrating to v4 database schema") + migrate_v3_to_v4(db) start_frontends() cron.initialize(db, _run_job) diff --git a/sist2-admin/sist2_admin/jobs.py b/sist2-admin/sist2_admin/jobs.py index 09032b2..bbc2f7d 100644 --- a/sist2-admin/sist2_admin/jobs.py +++ b/sist2-admin/sist2_admin/jobs.py @@ -16,7 +16,7 @@ from pydantic import BaseModel from config import logger, LOG_FOLDER from notifications import Notifications from sist2 import ScanOptions, IndexOptions, Sist2 -from state import RUNNING_FRONTENDS +from state import RUNNING_FRONTENDS, get_log_files_to_remove, delete_log_file from web import Sist2Frontend @@ -35,6 +35,8 @@ class Sist2Job(BaseModel): cron_expression: str schedule_enabled: bool = False + keep_last_n_logs: int = -1 + previous_index: str = None index_path: str = None previous_index_path: str = None @@ -301,8 +303,14 @@ class TaskQueue: "ended": task.ended, "started": task.started, "name": task.display_name, - "return_code": task_result + "return_code": task_result, + "has_logs": 1 } + + logs_to_delete = get_log_files_to_remove(self._db, task.job.name, task.job.keep_last_n_logs) + for row in logs_to_delete: + delete_log_file(self._db, row["id"]) + if isinstance(task, Sist2IndexTask): self._notifications.notify({ "message": "notifications.indexCompleted", diff --git a/sist2-admin/sist2_admin/state.py b/sist2-admin/sist2_admin/state.py index 12dbff7..a42b06b 100644 --- a/sist2-admin/sist2_admin/state.py +++ b/sist2-admin/sist2_admin/state.py @@ -1,16 +1,19 @@ from typing import Dict +import os import shutil from hexlib.db import Table, PersistentState import pickle from tesseract import get_tesseract_langs +import sqlite3 +from config import LOG_FOLDER RUNNING_FRONTENDS: Dict[str, int] = {} TESSERACT_LANGS = get_tesseract_langs() -DB_SCHEMA_VERSION = "3" +DB_SCHEMA_VERSION = "4" from pydantic import BaseModel @@ -50,8 +53,35 @@ class PickleTable(Table): yield dict((k, _deserialize(v)) for k, v in row.items()) -def migrate_v1_to_v2(db: PersistentState): +def get_log_files_to_remove(db: PersistentState, job_name: str, n: int): + if n < 0: + return [] + counter = 0 + to_remove = [] + + for row in db["task_done"].sql("WHERE has_logs=1 ORDER BY started DESC"): + if row["name"].endswith(f"[{job_name}]"): + counter += 1 + + if counter > n: + to_remove.append(row) + + return to_remove + + +def delete_log_file(db: PersistentState, task_id: str): + db["task_done"][task_id] = { + "has_logs": 0 + } + + try: + os.remove(os.path.join(LOG_FOLDER, f"sist2-{task_id}.log")) + except: + pass + + +def migrate_v1_to_v2(db: PersistentState): shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v2.bak") # Frontends @@ -77,3 +107,16 @@ def migrate_v1_to_v2(db: PersistentState): db["sist2_admin"]["info"] = { "version": "2" } + + +def migrate_v3_to_v4(db: PersistentState): + shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v4.bak") + + conn = sqlite3.connect(db.dbfile) + conn.execute("ALTER TABLE task_done ADD COLUMN has_logs INTEGER DEFAULT 1") + conn.commit() + conn.close() + + db["sist2_admin"]["info"] = { + "version": "4" + }