Add log cleanup features

sist2-admin/sist2_admin/app.py

@@ -21,7 +21,8 @@ from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
 from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
 from notifications import Subscribe, Notifications
 from sist2 import Sist2
-from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
+from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION, migrate_v3_to_v4, \
+    get_log_files_to_remove, delete_log_file
 from web import Sist2Frontend
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
@@ -80,7 +81,6 @@ async def get_jobs():
 
 @app.put("/api/job/{name:str}")
 async def update_job(name: str, new_job: Sist2Job):
-
     new_job.last_modified = datetime.utcnow()
     job = db["jobs"][name]
     if not job:
@@ -133,6 +133,16 @@ async def kill_job(task_id: str):
     return task_queue.kill_task(task_id)
 
 
+@app.post("/api/task/{task_id:str}/delete_logs")
+async def delete_task_logs(task_id: str):
+    if not db["task_done"][task_id]:
+        raise HTTPException(status_code=404)
+
+    delete_log_file(db, task_id)
+
+    return "ok"
+
+
 def _run_job(job: Sist2Job):
     job.last_modified = datetime.utcnow()
     if job.status == JobStatus("created"):
@@ -157,6 +167,11 @@ async def run_job(name: str):
     return "ok"
 
 
+@app.get("/api/job/{name:str}/logs_to_delete")
+async def task_history(n: int, name: str):
+    return get_log_files_to_remove(db, name, n)
+
+
 @app.delete("/api/job/{name:str}")
 async def delete_job(name: str):
     job = db["jobs"][name]
@@ -320,7 +335,6 @@ async def ws_tail_log(websocket: WebSocket):
         async with Subscribe(notifications) as ob:
             async for notification in ob.notifications():
                 await websocket.send_json(notification)
-                print(notification)
 
     except ConnectionClosed:
         return
@@ -380,6 +394,9 @@ if __name__ == '__main__':
     if db["sist2_admin"]["info"]["version"] == "2":
         logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.")
         exit(-1)
+    if db["sist2_admin"]["info"]["version"] == "3":
+        logger.info("Migrating to v4 database schema")
+        migrate_v3_to_v4(db)
 
     start_frontends()
     cron.initialize(db, _run_job)

sist2-admin/sist2_admin/jobs.py

@@ -16,7 +16,7 @@ from pydantic import BaseModel
 from config import logger, LOG_FOLDER
 from notifications import Notifications
 from sist2 import ScanOptions, IndexOptions, Sist2
-from state import RUNNING_FRONTENDS
+from state import RUNNING_FRONTENDS, get_log_files_to_remove, delete_log_file
 from web import Sist2Frontend
 
 
@@ -35,6 +35,8 @@ class Sist2Job(BaseModel):
     cron_expression: str
     schedule_enabled: bool = False
 
+    keep_last_n_logs: int = -1
+
     previous_index: str = None
     index_path: str = None
     previous_index_path: str = None
@@ -301,8 +303,14 @@ class TaskQueue:
                 "ended": task.ended,
                 "started": task.started,
                 "name": task.display_name,
-                "return_code": task_result
+                "return_code": task_result,
+                "has_logs": 1
             }
+
+            logs_to_delete = get_log_files_to_remove(self._db, task.job.name, task.job.keep_last_n_logs)
+            for row in logs_to_delete:
+                delete_log_file(self._db, row["id"])
+
         if isinstance(task, Sist2IndexTask):
             self._notifications.notify({
                 "message": "notifications.indexCompleted",

sist2-admin/sist2_admin/state.py

@@ -1,16 +1,19 @@
 from typing import Dict
+import os
 import shutil
 
 from hexlib.db import Table, PersistentState
 import pickle
 
 from tesseract import get_tesseract_langs
+import sqlite3
+from config import LOG_FOLDER
 
 RUNNING_FRONTENDS: Dict[str, int] = {}
 
 TESSERACT_LANGS = get_tesseract_langs()
 
-DB_SCHEMA_VERSION = "3"
+DB_SCHEMA_VERSION = "4"
 
 from pydantic import BaseModel
 
@@ -50,8 +53,35 @@ class PickleTable(Table):
             yield dict((k, _deserialize(v)) for k, v in row.items())
 
 
-def migrate_v1_to_v2(db: PersistentState):
+def get_log_files_to_remove(db: PersistentState, job_name: str, n: int):
+    if n < 0:
+        return []
 
+    counter = 0
+    to_remove = []
+
+    for row in db["task_done"].sql("WHERE has_logs=1 ORDER BY started DESC"):
+        if row["name"].endswith(f"[{job_name}]"):
+            counter += 1
+
+        if counter > n:
+            to_remove.append(row)
+
+    return to_remove
+
+
+def delete_log_file(db: PersistentState, task_id: str):
+    db["task_done"][task_id] = {
+        "has_logs": 0
+    }
+
+    try:
+        os.remove(os.path.join(LOG_FOLDER, f"sist2-{task_id}.log"))
+    except:
+        pass
+
+
+def migrate_v1_to_v2(db: PersistentState):
     shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v2.bak")
 
     # Frontends
@@ -77,3 +107,16 @@ def migrate_v1_to_v2(db: PersistentState):
     db["sist2_admin"]["info"] = {
         "version": "2"
     }
+
+
+def migrate_v3_to_v4(db: PersistentState):
+    shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v4.bak")
+
+    conn = sqlite3.connect(db.dbfile)
+    conn.execute("ALTER TABLE task_done ADD COLUMN has_logs INTEGER DEFAULT 1")
+    conn.commit()
+    conn.close()
+
+    db["sist2_admin"]["info"] = {
+        "version": "4"
+    }