Update sist2-admin for 3.x.x, more fixes

sist2-admin/sist2_admin/app.py

@@ -251,7 +251,7 @@ def check_es_version(es_url: str, insecure: bool):
 
 
 def start_frontend_(frontend: Sist2Frontend):
-    frontend.web_options.indices = list(map(lambda j: db["jobs"][j].last_index, frontend.jobs))
+    frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
 
     pid = sist2.web(frontend.web_options, frontend.name)
     RUNNING_FRONTENDS[frontend.name] = pid
@@ -378,6 +378,9 @@ if __name__ == '__main__':
     if db["sist2_admin"]["info"]["version"] == "1":
         logger.info("Migrating to v2 database schema")
         migrate_v1_to_v2(db)
+    if db["sist2_admin"]["info"]["version"] == "2":
+        logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.")
+        exit(-1)
 
     start_frontends()
     cron.initialize(db, _run_job)

sist2-admin/sist2_admin/jobs.py

@@ -1,23 +1,21 @@
 import json
 import logging
 import os.path
-import shutil
 import signal
 import uuid
 from datetime import datetime
 from enum import Enum
-from hashlib import md5
 from logging import FileHandler
 from threading import Lock, Thread
 from time import sleep
 from uuid import uuid4, UUID
 
 from hexlib.db import PersistentState
-from pydantic import BaseModel, validator
+from pydantic import BaseModel
 
 from config import logger, LOG_FOLDER
 from notifications import Notifications
-from sist2 import ScanOptions, IndexOptions, Sist2, Sist2Index
+from sist2 import ScanOptions, IndexOptions, Sist2
 from state import RUNNING_FRONTENDS
 from web import Sist2Frontend
 
@@ -38,7 +36,8 @@ class Sist2Job(BaseModel):
     schedule_enabled: bool = False
 
     previous_index: str = None
-    last_index: str = None
+    index_path: str = None
+    previous_index_path: str = None
     last_index_date: datetime = None
     status: JobStatus = JobStatus("created")
     last_modified: datetime
@@ -124,10 +123,10 @@ class Sist2ScanTask(Sist2Task):
 
         self.job.scan_options.name = self.job.name
 
-        if self.job.last_index and os.path.exists(self.job.last_index) and not self.job.do_full_scan:
-            self.job.scan_options.incremental = self.job.last_index
+        if self.job.index_path is not None and not self.job.do_full_scan:
+            self.job.scan_options.output = self.job.index_path
         else:
-            self.job.scan_options.incremental = None
+            self.job.scan_options.output = None
 
         def set_pid(pid):
             self.pid = pid
@@ -139,19 +138,26 @@ class Sist2ScanTask(Sist2Task):
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
             logger.info(f"Task {self.display_name} failed ({return_code})")
         else:
-            index = Sist2Index(self.job.scan_options.output)
-
-            # Save latest index
-            self.job.previous_index = self.job.last_index
-
-            self.job.last_index = index.path
+            self.job.index_path = self.job.scan_options.output
             self.job.last_index_date = datetime.now()
             self.job.do_full_scan = False
             db["jobs"][self.job.name] = self.job
-            self._logger.info(json.dumps({"sist2-admin": f"Save last_index={self.job.last_index}"}))
+            self._logger.info(json.dumps({"sist2-admin": f"Save last_index_date={self.job.last_index_date}"}))
 
         logger.info(f"Completed {self.display_name} ({return_code=})")
 
+        # Remove old index
+        if return_code == 0:
+            if self.job.previous_index_path is not None and self.job.previous_index_path != self.job.index_path:
+                self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index_path=}"}))
+                try:
+                    os.remove(self.job.previous_index_path)
+                except FileNotFoundError:
+                    pass
+
+            self.job.previous_index_path = self.job.index_path
+            db["jobs"][self.job.name] = self.job
+
         return return_code
 
 
@@ -173,18 +179,11 @@ class Sist2IndexTask(Sist2Task):
         ok = return_code == 0
 
         if ok:
-            # Remove old index
-            if self.job.previous_index is not None:
-                self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index=}"}))
-                try:
-                    shutil.rmtree(self.job.previous_index)
-                except FileNotFoundError:
-                    pass
-
             self.restart_running_frontends(db, sist2)
 
         # Update status
         self.job.status = JobStatus("indexed") if ok else JobStatus("failed")
+        self.job.previous_index_path = self.job.index_path
         db["jobs"][self.job.name] = self.job
 
         self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
@@ -198,13 +197,16 @@ class Sist2IndexTask(Sist2Task):
             frontend = db["frontends"][frontend_name]
             frontend: Sist2Frontend
 
-            os.kill(pid, signal.SIGTERM)
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
             try:
                 os.wait()
             except ChildProcessError:
                 pass
 
-            frontend.web_options.indices = map(lambda j: db["jobs"][j].last_index, frontend.jobs)
+            frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
 
             pid = sist2.web(frontend.web_options, frontend.name)
             RUNNING_FRONTENDS[frontend_name] = pid

sist2-admin/sist2_admin/sist2.py

@@ -2,7 +2,6 @@ import datetime
 import json
 import logging
 import os.path
-import traceback
 from datetime import datetime
 from io import TextIOWrapper
 from logging import FileHandler
@@ -78,10 +77,10 @@ class IndexOptions(BaseModel):
     es_url: str = "http://elasticsearch:9200"
     es_insecure_ssl: bool = False
     es_index: str = "sist2"
-    incremental_index: bool = False
+    incremental_index: bool = True
     script: str = ""
     script_file: str = None
-    batch_size: int = 100
+    batch_size: int = 70
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -110,9 +109,8 @@ ARCHIVE_RECURSE = "recurse"
 class ScanOptions(BaseModel):
     path: str
     threads: int = 1
-    mem_throttle: int = 0
     thumbnail_quality: int = 2
-    thumbnail_size: int = 500
+    thumbnail_size: int = 552
     thumbnail_count: int = 1
     content_size: int = 32768
     depth: int = -1
@@ -128,7 +126,8 @@ class ScanOptions(BaseModel):
     read_subtitles: bool = False
     fast_epub: bool = False
     checksums: bool = False
-    incremental: str = None
+    incremental: bool = True
+    optimize_index: bool = False
     output: str = None
     name: str = None
     rewrite_url: str = None
@@ -138,14 +137,15 @@ class ScanOptions(BaseModel):
         super().__init__(**kwargs)
 
     def args(self):
-        args = ["scan", self.path, f"--threads={self.threads}", f"--mem-throttle={self.mem_throttle}",
-                f"--thumbnail-quality={self.thumbnail_quality}", f"--thumbnail-count={self.thumbnail_count}",
-                f"--thumbnail-size={self.thumbnail_size}", f"--content-size={self.content_size}",
-                f"--output={self.output}", f"--depth={self.depth}", f"--archive={self.archive}",
-                f"--mem-buffer={self.mem_buffer}"]
+        args = ["scan", self.path, f"--threads={self.threads}", f"--thumbnail-quality={self.thumbnail_quality}",
+                f"--thumbnail-count={self.thumbnail_count}", f"--thumbnail-size={self.thumbnail_size}",
+                f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
+                f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
 
         if self.incremental:
-            args.append(f"--incremental={self.incremental}")
+            args.append(f"--incremental")
+        if self.optimize_index:
+            args.append(f"--optimize-index")
         if self.rewrite_url:
             args.append(f"--rewrite-url={self.rewrite_url}")
         if self.name:
@@ -235,11 +235,11 @@ class Sist2:
 
     def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
 
-        output_dir = os.path.join(
-            self._data_dir,
-            f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2"
-        )
-        options.output = output_dir
+        if options.output is None:
+            options.output = os.path.join(
+                self._data_dir,
+                f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2"
+            )
 
         args = [
             self._bin_path,

sist2-admin/sist2_admin/state.py

@@ -10,7 +10,7 @@ RUNNING_FRONTENDS: Dict[str, int] = {}
 
 TESSERACT_LANGS = get_tesseract_langs()
 
-DB_SCHEMA_VERSION = "2"
+DB_SCHEMA_VERSION = "3"
 
 from pydantic import BaseModel
 
@@ -76,4 +76,4 @@ def migrate_v1_to_v2(db: PersistentState):
 
     db["sist2_admin"]["info"] = {
         "version": "2"
-    }
+    }