Version bump

Bump semver from 5.7.1 to 5.7.2 in /sist2-admin/frontend

Dockerfile

@@ -1,11 +1,6 @@
 FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
-RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
-
 WORKDIR /build/
 
 COPY scripts scripts
@@ -25,7 +20,6 @@ RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
 
-
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 

Dockerfile.arm64

@@ -1,6 +1,19 @@
 FROM simon987/sist2-build-arm64 as build
 MAINTAINER simon987 <me@simon987.net>
 
+WORKDIR /build/
+
+COPY scripts scripts
+COPY schema schema
+COPY CMakeLists.txt .
+COPY third-party third-party
+COPY src src
+COPY sist2-vue sist2-vue
+COPY sist2-admin sist2-admin
+
+RUN cd sist2-vue/ && npm install && npm run build
+RUN cd sist2-admin/frontend/ && npm install && npm run build
+
 WORKDIR /build/
 ADD . /build/
 RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..

README.md

@@ -206,7 +206,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp]
+    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,ffprobe,zlib]
     ```
 
 4. Build

scripts/build.sh

@@ -4,6 +4,20 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
+(
+  cd sist2-vue/
+  npm install
+  npm run build
+) &
+
+(
+  cd sist2-admin/frontend/
+  npm install
+  npm run build
+) &
+
+wait
+
 mkdir build
 (
   cd build

scripts/build_arm64.sh

@@ -4,6 +4,20 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
+(
+  cd sist2-vue/
+  npm install
+  npm run build
+) &
+
+(
+  cd sist2-admin/frontend/
+  npm install
+  npm run build
+) &
+
+wait
+
 mkdir build
 (
   cd build

sist2-admin/frontend/src/App.vue

@@ -4,7 +4,8 @@
         <b-container class="pt-4">
             <b-alert show dismissible variant="info">
                 This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
-        to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on Github</a>. Thank you!
+                to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
+                Github</a>. Thank you!
             </b-alert>
             <router-view/>
         </b-container>
@@ -31,7 +32,11 @@ export default {
     },
     methods: {
         connectNotifications() {
+            if (window.location.protocol === "https:") {
+                this.socket = new WebSocket(`wss://${window.location.host}/notifications`);
+            } else {
                 this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
+            }
             this.socket.onopen = () => {
                 this.socket.send("Hello from client");
             }

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -7,15 +7,15 @@ class Sist2AdminApi {
     }
 
     getJobs() {
-        return axios.get(`${this.baseUrl}/api/job/`);
+        return axios.get(`${this.baseUrl}/api/job`);
     }
 
     getFrontends() {
-        return axios.get(`${this.baseUrl}/api/frontend/`);
+        return axios.get(`${this.baseUrl}/api/frontend`);
     }
 
     getTasks() {
-        return axios.get(`${this.baseUrl}/api/task/`);
+        return axios.get(`${this.baseUrl}/api/task`);
     }
 
     killTask(taskId) {
@@ -42,7 +42,7 @@ class Sist2AdminApi {
     }
 
     getSearchBackends() {
-        return axios.get(`${this.baseUrl}/api/search_backend/`);
+        return axios.get(`${this.baseUrl}/api/search_backend`);
     }
 
     deleteBackend(name) {
@@ -127,7 +127,7 @@ class Sist2AdminApi {
     }
 
     getSist2AdminInfo() {
-        return axios.get(`${this.baseUrl}/api/`);
+        return axios.get(`${this.baseUrl}/api`);
     }
 
     getLogsToDelete(jobName, n) {

sist2-admin/frontend/src/views/Tail.vue

@@ -65,7 +65,11 @@ export default {
             }
 
             const n = this.mode === "follow" ? 32 : 9999999999;
+            if (window.location.protocol === "https:") {
+                this.socket = new WebSocket(`wss://${window.location.host}/log/${this.taskId}?n=${n}`);
+            } else {
                 this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
+            }
             this.socket.onopen = () => {
                 this.socket.send("Hello from client");
             }

sist2-admin/frontend/src/views/Tasks.vue

@@ -134,7 +134,7 @@ export default {
                     duration: this.taskDuration(row),
                     time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"),
                     logs: null,
-                    status: row.return_code === 0 ? "ok" : "failed",
+                    status: [0,1].includes(row.return_code) ? "ok" : "failed",
                     _row: row
                 }));
             });

sist2-admin/sist2_admin/app.py

@@ -275,7 +275,10 @@ def check_es_version(es_url: str, insecure: bool):
 
 
 def start_frontend_(frontend: Sist2Frontend):
-    frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
+    frontend.web_options.indices = [
+        os.path.join(DATA_FOLDER, db["jobs"][j].index_path)
+        for j in frontend.jobs
+    ]
 
     backend_name = frontend.web_options.search_backend
     search_backend = db["search_backends"][backend_name]
@@ -354,7 +357,7 @@ def delete_search_backend(name: str):
     del db["search_backends"][name]
 
     try:
-        os.remove(backend.search_index)
+        os.remove(os.path.join(DATA_FOLDER, backend.search_index))
     except:
         pass
 

sist2-admin/sist2_admin/cron.py

@@ -10,7 +10,9 @@ from jobs import Sist2Job
 
 
 def _check_schedule(db: PersistentState, run_job):
-    for job in db["jobs"]:
+    jobs = list(db["jobs"])
+
+    for job in jobs:
         job: Sist2Job
 
         if job.schedule_enabled:

sist2-admin/sist2_admin/jobs.py

@@ -13,7 +13,7 @@ from uuid import uuid4, UUID
 from hexlib.db import PersistentState
 from pydantic import BaseModel
 
-from config import logger, LOG_FOLDER
+from config import logger, LOG_FOLDER, DATA_FOLDER
 from notifications import Notifications
 from sist2 import ScanOptions, IndexOptions, Sist2
 from state import RUNNING_FRONTENDS, get_log_files_to_remove, delete_log_file
@@ -131,7 +131,9 @@ class Sist2ScanTask(Sist2Task):
         return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
         self.ended = datetime.utcnow()
 
-        if return_code != 0:
+        is_ok = return_code in (0, 1)
+
+        if not is_ok:
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
             logger.info(f"Task {self.display_name} failed ({return_code})")
         else:
@@ -144,7 +146,7 @@ class Sist2ScanTask(Sist2Task):
         logger.info(f"Completed {self.display_name} ({return_code=})")
 
         # Remove old index
-        if return_code == 0:
+        if is_ok:
             if self.job.previous_index_path is not None and self.job.previous_index_path != self.job.index_path:
                 self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index_path=}"}))
                 try:
@@ -218,7 +220,10 @@ class Sist2IndexTask(Sist2Task):
 
             logger.debug(f"Fetched search backend options for {backend_name}")
 
-            frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
+            frontend.web_options.indices = [
+                os.path.join(DATA_FOLDER, db["jobs"][j].index_path)
+                for j in frontend.jobs
+            ]
 
             pid = sist2.web(frontend.web_options, search_backend, frontend.name)
             RUNNING_FRONTENDS[frontend_name] = pid
@@ -244,7 +249,7 @@ class TaskQueue:
     def _tasks_failed(self):
         done = set()
 
-        for row in self._db["task_done"].sql("WHERE return_code != 0"):
+        for row in self._db["task_done"].sql("WHERE return_code NOT IN (0,1)"):
             done.add(uuid.UUID(row["id"]))
 
         return done

sist2-admin/sist2_admin/sist2.py

@@ -49,7 +49,7 @@ class Sist2SearchBackend(BaseModel):
     def create_default(name: str, backend_type: SearchBackendType = SearchBackendType("elasticsearch")):
         return Sist2SearchBackend(
             name=name,
-            search_index=os.path.join(DATA_FOLDER, f"search-index-{name.replace('/', '_')}.sist2"),
+            search_index=f"search-index-{name.replace('/', '_')}.sist2",
             backend_type=backend_type
         )
 
@@ -63,10 +63,13 @@ class IndexOptions(BaseModel):
         super().__init__(**kwargs)
 
     def args(self, search_backend):
+        absolute_path = os.path.join(DATA_FOLDER, self.path)
+
         if search_backend.backend_type == SearchBackendType("sqlite"):
-            args = ["sqlite-index", self.path, "--search-index", search_backend.search_index]
+            search_index_absolute = os.path.join(DATA_FOLDER, search_backend.search_index)
+            args = ["sqlite-index", absolute_path, "--search-index", search_index_absolute]
         else:
-            args = ["index", self.path, f"--threads={search_backend.threads}",
+            args = ["index", absolute_path, f"--threads={search_backend.threads}",
                     f"--es-url={search_backend.es_url}",
                     f"--es-index={search_backend.es_index}",
                     f"--batch-size={search_backend.batch_size}"]
@@ -118,9 +121,12 @@ class ScanOptions(BaseModel):
         super().__init__(**kwargs)
 
     def args(self):
+
+        output_path = os.path.join(DATA_FOLDER, self.output)
+
         args = ["scan", self.path, f"--threads={self.threads}", f"--thumbnail-quality={self.thumbnail_quality}",
                 f"--thumbnail-count={self.thumbnail_count}", f"--thumbnail-size={self.thumbnail_size}",
-                f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
+                f"--content-size={self.content_size}", f"--output={output_path}", f"--depth={self.depth}",
                 f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
 
         if self.incremental:
@@ -181,6 +187,7 @@ class Sist2Index:
     def name(self) -> str:
         return self._descriptor["name"]
 
+
 class WebOptions(BaseModel):
     indices: List[str] = []
 
@@ -206,7 +213,8 @@ class WebOptions(BaseModel):
                 f"--lang={self.lang}"]
 
         if search_backend.backend_type == SearchBackendType("sqlite"):
-            args.append(f"--search-index={search_backend.search_index}")
+            search_index_absolute = os.path.join(DATA_FOLDER, search_backend.search_index)
+            args.append(f"--search-index={search_index_absolute}")
         else:
             args.append(f"--es-url={search_backend.es_url}")
             args.append(f"--es-index={search_backend.es_index}")
@@ -232,10 +240,11 @@ class WebOptions(BaseModel):
 
         return args
 
+
 class Sist2:
 
     def __init__(self, bin_path: str, data_directory: str):
-        self._bin_path = bin_path
+        self.bin_path = bin_path
         self._data_dir = data_directory
 
     def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
@@ -248,7 +257,7 @@ class Sist2:
             search_backend.script_file = None
 
         args = [
-            self._bin_path,
+            self.bin_path,
             *options.args(search_backend),
             "--json-logs",
             "--very-verbose"
@@ -269,13 +278,10 @@ class Sist2:
     def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
 
         if options.output is None:
-            options.output = os.path.join(
-                self._data_dir,
-                f"scan-{options.name.replace('/', '_')}-{datetime.utcnow()}.sist2"
-            )
+            options.output = f"scan-{options.name.replace('/', '_')}-{datetime.utcnow()}.sist2"
 
         args = [
-            self._bin_path,
+            self.bin_path,
             *options.args(),
             "--json-logs",
             "--very-verbose"
@@ -333,7 +339,7 @@ class Sist2:
             options.auth0_public_key_file = None
 
         args = [
-            self._bin_path,
+            self.bin_path,
             *options.args(search_backend)
         ]
 
@@ -354,5 +360,3 @@ class Sist2:
         t_stdout.start()
 
         return proc.pid
-
-

sist2-vue/package-lock.json

@@ -2613,9 +2613,9 @@
       }
     },
     "node_modules/@vue/babel-preset-app/node_modules/semver": {
-      "version": "7.3.8",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -2948,9 +2948,9 @@
       }
     },
     "node_modules/@vue/cli-plugin-typescript/node_modules/semver": {
-      "version": "7.3.8",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -3462,9 +3462,9 @@
       }
     },
     "node_modules/@vue/cli-service/node_modules/semver": {
-      "version": "7.3.8",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -3603,9 +3603,9 @@
       }
     },
     "node_modules/@vue/cli-shared-utils/node_modules/semver": {
-      "version": "7.3.8",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -4548,9 +4548,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001444",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001444.tgz",
-      "integrity": "sha512-ecER9xgJQVMqcrxThKptsW0pPxSae8R2RB87LNa+ivW9ppNWRHEplXcDzkCOP4LYWGj8hunXLqaiC41iBATNyg==",
+      "version": "1.0.30001515",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001515.tgz",
+      "integrity": "sha512-eEFDwUOZbE24sb+Ecsx3+OvNETqjWIdabMy52oOkIgcUtAsQifjUG9q4U9dgTHJM2mfk4uEPxc0+xuFdJ629QA==",
       "dev": true,
       "funding": [
         {
@@ -4560,6 +4560,10 @@
         {
           "type": "tidelift",
           "url": "https://tidelift.com/funding/github/npm/caniuse-lite"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
         }
       ]
     },
@@ -5157,9 +5161,9 @@
       }
     },
     "node_modules/cross-spawn/node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
       "dev": true,
       "bin": {
         "semver": "bin/semver"
@@ -6154,9 +6158,9 @@
       }
     },
     "node_modules/editorconfig/node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
       "dev": true,
       "bin": {
         "semver": "bin/semver"
@@ -6829,9 +6833,9 @@
       }
     },
     "node_modules/fork-ts-checker-webpack-plugin/node_modules/semver": {
-      "version": "7.3.8",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -8559,9 +8563,9 @@
       }
     },
     "node_modules/normalize-package-data/node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
       "dev": true,
       "bin": {
         "semver": "bin/semver"
@@ -10186,9 +10190,9 @@
       }
     },
     "node_modules/sass-loader/node_modules/semver": {
-      "version": "7.3.5",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
-      "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -10244,9 +10248,9 @@
       }
     },
     "node_modules/semver": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-      "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
       "dev": true,
       "bin": {
         "semver": "bin/semver.js"
@@ -13965,9 +13969,9 @@
           }
         },
         "semver": {
-          "version": "7.3.8",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-          "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+          "version": "7.5.4",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+          "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
           "dev": true,
           "requires": {
             "lru-cache": "^6.0.0"
@@ -14210,9 +14214,9 @@
           }
         },
         "semver": {
-          "version": "7.3.8",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-          "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+          "version": "7.5.4",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+          "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
           "dev": true,
           "requires": {
             "lru-cache": "^6.0.0"
@@ -14543,9 +14547,9 @@
           }
         },
         "semver": {
-          "version": "7.3.8",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-          "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+          "version": "7.5.4",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+          "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
           "dev": true,
           "requires": {
             "lru-cache": "^6.0.0"
@@ -14650,9 +14654,9 @@
           }
         },
         "semver": {
-          "version": "7.3.8",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-          "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+          "version": "7.5.4",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+          "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
           "dev": true,
           "requires": {
             "lru-cache": "^6.0.0"
@@ -15422,9 +15426,9 @@
       }
     },
     "caniuse-lite": {
-      "version": "1.0.30001444",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001444.tgz",
-      "integrity": "sha512-ecER9xgJQVMqcrxThKptsW0pPxSae8R2RB87LNa+ivW9ppNWRHEplXcDzkCOP4LYWGj8hunXLqaiC41iBATNyg==",
+      "version": "1.0.30001515",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001515.tgz",
+      "integrity": "sha512-eEFDwUOZbE24sb+Ecsx3+OvNETqjWIdabMy52oOkIgcUtAsQifjUG9q4U9dgTHJM2mfk4uEPxc0+xuFdJ629QA==",
       "dev": true
     },
     "case-sensitive-paths-webpack-plugin": {
@@ -15893,9 +15897,9 @@
       },
       "dependencies": {
         "semver": {
-          "version": "5.7.1",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+          "version": "5.7.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+          "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
           "dev": true
         }
       }
@@ -16617,9 +16621,9 @@
           }
         },
         "semver": {
-          "version": "5.7.1",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+          "version": "5.7.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+          "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
           "dev": true
         },
         "yallist": {
@@ -17137,9 +17141,9 @@
           }
         },
         "semver": {
-          "version": "7.3.8",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
-          "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+          "version": "7.5.4",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+          "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
           "dev": true,
           "requires": {
             "lru-cache": "^6.0.0"
@@ -18469,9 +18473,9 @@
       },
       "dependencies": {
         "semver": {
-          "version": "5.7.1",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+          "version": "5.7.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+          "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
           "dev": true
         }
       }
@@ -19640,9 +19644,9 @@
           }
         },
         "semver": {
-          "version": "7.3.5",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
-          "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+          "version": "7.5.4",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+          "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
           "dev": true,
           "requires": {
             "lru-cache": "^6.0.0"
@@ -19688,9 +19692,9 @@
       }
     },
     "semver": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-      "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
       "dev": true
     },
     "send": {

sist2-vue/src/Sist2Api.ts

@@ -531,8 +531,8 @@ class Sist2Api {
             size: 0
         }).then(res => {
             const range = {
-                min: res.aggregations.dateMin.value,
-                max: res.aggregations.dateMax.value,
+                min: res.aggregations.dateMin.value / 1000,
+                max: res.aggregations.dateMax.value / 1000,
             }
 
             if (range.min == null) {

src/cli.c

@@ -83,6 +83,10 @@ void exec_args_destroy(exec_args_t *args) {
     free(args);
 }
 
+void sqlite_index_args_destroy(sqlite_index_args_t *args) {
+    // TODO
+}
+
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (argc < 2) {
         fprintf(stderr, "Required positional argument: PATH.\n");

src/cli.h

@@ -134,5 +134,7 @@ void exec_args_destroy(exec_args_t *args);
 
 int exec_args_validate(exec_args_t *args, int argc, const char **argv);
 
+void sqlite_index_args_destroy(sqlite_index_args_t *args);
+
 
 #endif

src/database/database_fts.c

@@ -447,14 +447,18 @@ database_summary_stats_t database_fts_get_date_range(database_t *db) {
     return stats;
 }
 
-char *get_after_where(char **after, fts_sort_t sort) {
+char *get_after_where(char **after, fts_sort_t sort, int sort_asc) {
     if (after == NULL) {
         return NULL;
     }
 
+    if (sort_asc) {
         return "(sort_var, doc.ROWID) > (?3, ?4)";
     }
 
+    return "(sort_var, doc.ROWID) < (?3, ?4)";
+}
+
 cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
                            long size_max, long date_min, long date_max, int page_size,
                            char **index_ids, char **mime_types, char **tags, int sort_asc,
@@ -469,7 +473,7 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
     char *index_id_where = index_ids_where_clause(index_ids);
     char *mime_where = mime_types_where_clause(mime_types);
     const char *query_where = match_where(query);
-    const char *after_where = get_after_where(after, sort);
+    const char *after_where = get_after_where(after, sort, sort_asc);
     const char *tags_where = tags_where_clause(tags);
 
     if (!query_where && sort == FTS_SORT_SCORE) {

src/database/database_stats.c

@@ -149,6 +149,7 @@ void database_generate_stats(database_t *db, double treemap_threshold) {
 
             merged_rows += 1;
         }
+        free(iter);
     } while (merged_rows > TREEMAP_MINIMUM_MERGES_TO_CONTINUE);
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db,

src/main.c

@@ -68,9 +68,7 @@ void database_scan_begin(scan_args_t *args) {
         desc->version_patch = VersionPatch;
 
         // generate new index id based on timestamp
-        unsigned char index_md5[MD5_DIGEST_LENGTH];
-        MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
-        buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
+        md5_hexdigest(&ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), ScanCtx.index.desc.id);
 
         database_initialize(db);
         database_open(db);
@@ -683,6 +681,7 @@ int main(int argc, const char *argv[]) {
     index_args_destroy(index_args);
     web_args_destroy(web_args);
     exec_args_destroy(exec_args);
+    sqlite_index_args_destroy(sqlite_index_args);
 
     return 0;
 }

src/parsing/fs_util.h

@@ -2,15 +2,18 @@
 #define SIST2_FS_UTIL_H
 
 #include "src/sist.h"
+#include <openssl/evp.h>
 
 #define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));};
 
 static int fs_read(struct vfile *f, void *buf, size_t size) {
     if (f->fd == -1) {
-        SHA1_Init(&f->sha1_ctx);
+        f->sha1_ctx = EVP_MD_CTX_new();
+        EVP_DigestInit_ex(f->sha1_ctx, EVP_sha1(), NULL);
 
         f->fd = open(f->filepath, O_RDONLY);
         if (f->fd == -1) {
+            EVP_MD_CTX_free(f->sha1_ctx);
             return -1;
         }
     }
@@ -19,7 +22,7 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
 
     if (ret != 0 && f->calculate_checksum) {
         f->has_checksum = TRUE;
-        safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret);
+        safe_digest_update(f->sha1_ctx, (unsigned char *) buf, ret);
     }
 
     return ret;
@@ -27,8 +30,11 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
 
 static void fs_close(struct vfile *f) {
     if (f->fd != -1) {
-        SHA1_Final(f->sha1_digest, &f->sha1_ctx);
+        EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
+        EVP_MD_CTX_free(f->sha1_ctx);
+        f->sha1_ctx = NULL;
         close(f->fd);
+        f->fd = -1;
     }
 }
 

src/parsing/parse.c

@@ -155,19 +155,17 @@ void parse(parse_job_t *job) {
     doc->meta_head = NULL;
     doc->meta_tail = NULL;
     doc->size = job->vfile.st_size;
-    doc->mtime = job->vfile.mtime;
+    doc->mtime = MAX(job->vfile.mtime, 0);
     doc->mime = get_mime(job);
     generate_doc_id(doc->filepath + ScanCtx.index.desc.root_len, doc->doc_id);
 
     if (doc->mime == GET_MIME_ERROR_FATAL) {
-
         CLOSE_FILE(job->vfile)
         free(doc);
         return;
     }
 
     if (database_mark_document(ProcData.index_db, doc->doc_id, doc->mtime)) {
-
         CLOSE_FILE(job->vfile)
         free(doc);
         return;

src/sist.h

@@ -51,11 +51,11 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.1.1"
+#define VERSION "3.1.3"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
 static const int VersionMinor = 1;
-static const int VersionPatch = 1;
+static const int VersionPatch = 3;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown

src/util.h

@@ -7,6 +7,7 @@
 
 #include "third-party/utf8.h/utf8.h"
 #include "libscan/scan.h"
+#include <openssl/evp.h>
 
 
 char *abspath(const char *path);
@@ -86,13 +87,22 @@ static void buf2hex(const unsigned char *buf, size_t buflen, char *hex_string) {
     *s = '\0';
 }
 
+static void md5_hexdigest(void *data, size_t size, char *output) {
+    EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
+    EVP_DigestInit_ex(md_ctx, EVP_md5(), NULL);
+
+    EVP_DigestUpdate(md_ctx, data, size);
+
+    unsigned char digest[MD5_DIGEST_LENGTH];
+    EVP_DigestFinal_ex(md_ctx, digest, NULL);
+    EVP_MD_CTX_free(md_ctx);
+
+    buf2hex(digest, MD5_DIGEST_LENGTH, output);
+}
 
 __always_inline
 static void generate_doc_id(const char *rel_path, char *doc_id) {
-    unsigned char md[MD5_DIGEST_LENGTH];
-
-    MD5((unsigned char *) rel_path, strlen(rel_path), md);
-    buf2hex(md, sizeof(md), doc_id);
+    md5_hexdigest(rel_path, strlen(rel_path), doc_id);
 }
 
 #define MILLISECOND 1000

src/web/serve.c

@@ -248,9 +248,11 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
 
     char mime_mapping[8192];
     if (strlen(ext) == 0) {
-        snprintf(mime_mapping, sizeof(mime_mapping), "%s=%s", full_path, mime);
+        snprintf(mime_mapping, sizeof(mime_mapping), "%s=%s%s",
+                 full_path, mime, STR_STARTS_WITH_CONSTANT(mime, "text/") ? "; charset=utf8" : "");
     } else {
-        snprintf(mime_mapping, sizeof(mime_mapping), "%s=%s", ext, mime);
+        snprintf(mime_mapping, sizeof(mime_mapping), "%s=%s%s",
+                 ext, mime, STR_STARTS_WITH_CONSTANT(mime, "text/") ? "; charset=utf8" : "");
     }
 
     struct mg_http_serve_opts opts = {
@@ -526,9 +528,9 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) {
     }
 
     tag_req_t *req = parse_tag_request(json);
-    cJSON_Delete(json);
     if (req == NULL) {
         LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index);
+        cJSON_Delete(json);
         HTTP_REPLY_BAD_REQUEST
         return;
     }
@@ -552,6 +554,7 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) {
     }
 
     free(req);
+    cJSON_Delete(json);
 }
 
 int validate_auth(struct mg_connection *nc, struct mg_http_message *hm) {

third-party/libscan/libscan/arc/arc.c

@@ -22,7 +22,11 @@ int should_parse_filtered_file(const char *filepath) {
 }
 
 void arc_close(struct vfile *f) {
-    SHA1_Final(f->sha1_digest, &f->sha1_ctx);
+    if (f->sha1_ctx != NULL) {
+        EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
+        EVP_MD_CTX_free(f->sha1_ctx);
+        f->sha1_ctx = NULL;
+    }
 
     if (f->rewind_buffer != NULL) {
         free(f->rewind_buffer);
@@ -59,7 +63,7 @@ int arc_read(struct vfile *f, void *buf, size_t size) {
     if (bytes_read != 0 && bytes_read <= size && f->calculate_checksum) {
         f->has_checksum = TRUE;
 
-        safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, bytes_read);
+        safe_digest_update(f->sha1_ctx, (unsigned char *) buf, bytes_read);
     }
 
     if (bytes_read != size && archive_errno(f->arc) != 0) {
@@ -237,9 +241,12 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
                     sub_job->ext = (int) strlen(sub_job->filepath);
                 }
 
-                SHA1_Init(&sub_job->vfile.sha1_ctx);
+                sub_job->vfile.sha1_ctx = EVP_MD_CTX_new();
+                EVP_DigestInit(sub_job->vfile.sha1_ctx, EVP_sha1());
 
                 ctx->parse(sub_job);
+
+                sub_job->vfile.close(&sub_job->vfile);
             }
         }
 

third-party/libscan/libscan/arc/arc.h

@@ -35,7 +35,8 @@ static int vfile_open_callback(struct archive *a, void *user_data) {
     arc_data_t *data = (arc_data_t *) user_data;
 
     if (!data->f->is_fs_file) {
-        SHA1_Init(&data->f->sha1_ctx);
+        data->f->sha1_ctx = EVP_MD_CTX_new();
+        EVP_DigestInit(data->f->sha1_ctx, EVP_md5());
     }
 
     return ARCHIVE_OK;
@@ -49,7 +50,7 @@ static long vfile_read_callback(struct archive *a, void *user_data, const void *
 
     if (!data->f->is_fs_file && ret > 0) {
         data->f->has_checksum = TRUE;
-        safe_sha1_update(&data->f->sha1_ctx, (unsigned char*)data->buf, ret);
+        safe_digest_update(data->f->sha1_ctx, (unsigned char *) data->buf, ret);
     }
 
     return ret;
@@ -59,7 +60,9 @@ static int vfile_close_callback(struct archive *a, void *user_data) {
     arc_data_t *data = (arc_data_t *) user_data;
 
     if (!data->f->is_fs_file) {
-        SHA1_Final((unsigned char *) data->f->sha1_digest, &data->f->sha1_ctx);
+        EVP_DigestFinal_ex(data->f->sha1_ctx, data->f->sha1_digest, NULL);
+        EVP_MD_CTX_free(data->f->sha1_ctx);
+        data->f->sha1_ctx = NULL;
     }
 
     return ARCHIVE_OK;

third-party/libscan/libscan/ebook/ebook.c

@@ -11,8 +11,6 @@
 pthread_mutex_t Mutex;
 #endif
 
-/* fill_image callback doesn't let us pass opaque pointers unless I create my own device */
-__thread text_buffer_t thread_buffer;
 __thread scan_ebook_ctx_t thread_ctx;
 
 static void my_fz_lock(UNUSED(void *user), int lock) {
@@ -232,21 +230,47 @@ static int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
     return 0;
 }
 
-static void fill_image_ocr_cb(const char* text, size_t len) {
-  text_buffer_append_string(&thread_buffer, text, len - 1);
+int read_stext(text_buffer_t *tex, fz_stext_page *stext) {
+
+    int count = 0;
+
+    fz_stext_block *block = stext->first_block;
+
+    while (block != NULL) {
+        int ret = read_stext_block(block, tex);
+        count += 1;
+        if (ret == TEXT_BUF_FULL) {
+            break;
+        }
+        block = block->next;
     }
 
-void fill_image(fz_context *fzctx, UNUSED(fz_device *dev),
-                fz_image *img, UNUSED(fz_matrix ctm), UNUSED(float alpha),
-                UNUSED(fz_color_params color_params)) {
-
-    int l2factor = 0;
-
-    if (img->w >= MIN_OCR_WIDTH && img->h >= MIN_OCR_HEIGHT && OCR_IS_VALID_BPP(img->n)) {
-        fz_pixmap *pix = img->get_pixmap(fzctx, img, NULL, img->w, img->h, &l2factor);
-        ocr_extract_text(thread_ctx.tesseract_path, thread_ctx.tesseract_lang, pix->samples, pix->w, pix->h, pix->n, (int)pix->stride, pix->xres, fill_image_ocr_cb);
-        fz_drop_pixmap(fzctx, pix);
+    return count;
 }
+
+int load_page(fz_context *fzctx, fz_document *fzdoc, int current_page, fz_page **page) {
+    int err = 0;
+
+    fz_var(err);
+    fz_try(fzctx)(*page) = fz_load_page(fzctx, fzdoc, current_page);
+    fz_catch(fzctx)err = fzctx->error.errcode;
+
+    return err;
+}
+
+fz_device *new_stext_dev(fz_context *fzctx, fz_stext_page *stext) {
+    fz_stext_options opts = {
+            .flags = FZ_STEXT_DEHYPHENATE,
+            .scale = 0
+    };
+
+    fz_device *stext_dev = fz_new_stext_device(fzctx, stext, &opts);
+    stext_dev->stroke_path = NULL;
+    stext_dev->stroke_text = NULL;
+    stext_dev->clip_text = NULL;
+    stext_dev->clip_stroke_path = NULL;
+    stext_dev->clip_stroke_text = NULL;
+    return stext_dev;
 }
 
 void
@@ -326,46 +350,37 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi
 
 
     if (ctx->content_size > 0) {
-        fz_stext_options opts = {0};
-        thread_buffer = text_buffer_create(ctx->content_size);
+        text_buffer_t tex = text_buffer_create(ctx->content_size);
 
         for (int current_page = 0; current_page < page_count; current_page++) {
             fz_page *page = NULL;
-            fz_var(err);
-            fz_try(fzctx)page = fz_load_page(fzctx, fzdoc, current_page);
-            fz_catch(fzctx)err = fzctx->error.errcode;
+            err = load_page(fzctx, fzdoc, current_page, &page);
+
             if (err != 0) {
-                CTX_LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, fzctx->error.message);
-                text_buffer_destroy(&thread_buffer);
+                CTX_LOG_WARNINGF(doc->filepath,
+                                 "fz_load_page() returned error code [%d] %s", err, fzctx->error.message);
+                text_buffer_destroy(&tex);
                 fz_drop_page(fzctx, page);
                 fz_drop_stream(fzctx, stream);
                 fz_drop_document(fzctx, fzdoc);
                 fz_drop_context(fzctx);
                 return;
             }
+            fz_rect page_mediabox = fz_bound_page(fzctx, page);
 
-            fz_stext_page *stext = fz_new_stext_page(fzctx, fz_bound_page(fzctx, page));
-            fz_device *dev = fz_new_stext_device(fzctx, stext, &opts);
-            dev->stroke_path = NULL;
-            dev->stroke_text = NULL;
-            dev->clip_text = NULL;
-            dev->clip_stroke_path = NULL;
-            dev->clip_stroke_text = NULL;
-
-            if (ctx->tesseract_lang != NULL) {
-                dev->fill_image = fill_image;
-            }
+            fz_stext_page *stext = fz_new_stext_page(fzctx, page_mediabox);
+            fz_device *stext_dev = new_stext_dev(fzctx, stext);
 
             fz_var(err);
-            fz_try(fzctx)fz_run_page(fzctx, page, dev, fz_identity, NULL);
+            fz_try(fzctx)fz_run_page(fzctx, page, stext_dev, fz_identity, NULL);
             fz_always(fzctx) {
-                    fz_close_device(fzctx, dev);
-                    fz_drop_device(fzctx, dev);
+                    fz_close_device(fzctx, stext_dev);
+                    fz_drop_device(fzctx, stext_dev);
                 } fz_catch(fzctx) err = fzctx->error.errcode;
 
             if (err != 0) {
                 CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message);
-                text_buffer_destroy(&thread_buffer);
+                text_buffer_destroy(&tex);
                 fz_drop_page(fzctx, page);
                 fz_drop_stext_page(fzctx, stext);
                 fz_drop_stream(fzctx, stream);
@@ -374,29 +389,63 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi
                 return;
             }
 
-            fz_stext_block *block = stext->first_block;
-            while (block != NULL) {
-                int ret = read_stext_block(block, &thread_buffer);
-                if (ret == TEXT_BUF_FULL) {
-                    break;
-                }
-                block = block->next;
-            }
+            int num_blocks_read = read_stext(&tex, stext);
+
             fz_drop_stext_page(fzctx, stext);
-            fz_drop_page(fzctx, page);
 
-            if (thread_buffer.dyn_buffer.cur >= ctx->content_size) {
+            if (tex.dyn_buffer.cur >= ctx->content_size) {
+                fz_drop_page(fzctx, page);
                 break;
             }
-        }
-        text_buffer_terminate_string(&thread_buffer);
 
-        meta_line_t *meta_content = malloc(sizeof(meta_line_t) + thread_buffer.dyn_buffer.cur);
+            // If OCR is enabled and no text is found on the page
+            if (ctx->tesseract_lang != NULL && num_blocks_read == 0) {
+                stext = fz_new_stext_page(fzctx, page_mediabox);
+                stext_dev = new_stext_dev(fzctx, stext);
+
+                fz_device *ocr_dev = fz_new_ocr_device(fzctx, stext_dev, fz_identity,
+                                                       page_mediabox, TRUE,
+                                                       ctx->tesseract_lang,
+                                                       ctx->tesseract_path,
+                                                       NULL, NULL);
+
+                fz_var(err);
+                fz_try(fzctx)fz_run_page(fzctx, page, ocr_dev, fz_identity, NULL);
+                fz_always(fzctx) {
+                        fz_close_device(fzctx, ocr_dev);
+                        fz_drop_device(fzctx, ocr_dev);
+                    } fz_catch(fzctx) err = fzctx->error.errcode;
+
+                if (err != 0) {
+                    CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message);
+                    fz_close_device(fzctx, stext_dev);
+                    fz_drop_device(fzctx, stext_dev);
+                    text_buffer_destroy(&tex);
+                    fz_drop_page(fzctx, page);
+                    fz_drop_stext_page(fzctx, stext);
+                    fz_drop_stream(fzctx, stream);
+                    fz_drop_document(fzctx, fzdoc);
+                    fz_drop_context(fzctx);
+                    return;
+                }
+
+                fz_close_device(fzctx, stext_dev);
+                fz_drop_device(fzctx, stext_dev);
+
+                read_stext(&tex, stext);
+                fz_drop_stext_page(fzctx, stext);
+            }
+
+            fz_drop_page(fzctx, page);
+        }
+        text_buffer_terminate_string(&tex);
+
+        meta_line_t *meta_content = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur);
         meta_content->key = MetaContent;
-        memcpy(meta_content->str_val, thread_buffer.dyn_buffer.buf, thread_buffer.dyn_buffer.cur);
+        memcpy(meta_content->str_val, tex.dyn_buffer.buf, tex.dyn_buffer.cur);
         APPEND_META(doc, meta_content);
 
-        text_buffer_destroy(&thread_buffer);
+        text_buffer_destroy(&tex);
     }
 
     fz_drop_stream(fzctx, stream);

third-party/libscan/libscan/media/media.c

@@ -118,13 +118,12 @@ static void read_subtitles(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, i
     AVPacket packet;
     AVSubtitle subtitle;
 
-    AVCodec *subtitle_codec = avcodec_find_decoder(pFormatCtx->streams[stream_idx]->codecpar->codec_id);
+    const AVCodec *subtitle_codec = avcodec_find_decoder(pFormatCtx->streams[stream_idx]->codecpar->codec_id);
     AVCodecContext *decoder = avcodec_alloc_context3(subtitle_codec);
+    decoder->thread_count = 1;
     avcodec_parameters_to_context(decoder, pFormatCtx->streams[stream_idx]->codecpar);
     avcodec_open2(decoder, subtitle_codec, NULL);
 
-    decoder->sub_text_format = FF_SUB_TEXT_FMT_ASS;
-
     int got_sub;
 
     while (1) {
@@ -177,8 +176,6 @@ read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *d
     result->packet = av_packet_alloc();
     result->frame = av_frame_alloc();
 
-    av_init_packet(result->packet);
-
     int receive_ret = -EAGAIN;
     while (receive_ret == -EAGAIN) {
         // Get video frame
@@ -477,13 +474,12 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
         avcodec_send_frame(thumbnail_encoder, scaled_frame);
         avcodec_send_frame(thumbnail_encoder, NULL); // send EOF
 
-        AVPacket thumbnail_packet;
-        av_init_packet(&thumbnail_packet);
-        avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
+        AVPacket *thumbnail_packet = av_packet_alloc();
+        avcodec_receive_packet(thumbnail_encoder, thumbnail_packet);
 
         // Save thumbnail
         if (thumbnail_index == 0) {
-            ctx->store(doc->doc_id, 0, thumbnail_packet.data, thumbnail_packet.size);
+            ctx->store(doc->doc_id, 0, thumbnail_packet->data, thumbnail_packet->size);
             return_value = SAVE_THUMBNAIL_OK;
 
         } else if (thumbnail_index > 1) {
@@ -491,7 +487,7 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
             //  I figure out a better fix.
             thumbnail_index -= 1;
 
-            ctx->store(doc->doc_id, thumbnail_index, thumbnail_packet.data, thumbnail_packet.size);
+            ctx->store(doc->doc_id, thumbnail_index, thumbnail_packet->data, thumbnail_packet->size);
 
             return_value = SAVE_THUMBNAIL_OK;
         } else {
@@ -499,7 +495,7 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
         }
 
         avcodec_free_context(&thumbnail_encoder);
-        av_packet_unref(&thumbnail_packet);
+        av_packet_free(&thumbnail_packet);
         av_free(*scaled_frame->data);
         av_frame_free(&scaled_frame);
     }
@@ -578,8 +574,9 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
         }
 
         // Decoder
-        AVCodec *video_codec = avcodec_find_decoder(stream->codecpar->codec_id);
+        const AVCodec *video_codec = avcodec_find_decoder(stream->codecpar->codec_id);
         AVCodecContext *decoder = avcodec_alloc_context3(video_codec);
+        decoder->thread_count = 1;
         avcodec_parameters_to_context(decoder, stream->codecpar);
         avcodec_open2(decoder, video_codec, NULL);
 
@@ -630,6 +627,9 @@ void parse_media_filename(scan_media_ctx_t *ctx, const char *filepath, document_
         CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()");
         return;
     }
+    pFormatCtx->max_analyze_duration = 100000000;
+    pFormatCtx->probesize = 100000000;
+
     int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL);
     if (res < 0) {
         CTX_LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res));
@@ -697,9 +697,10 @@ int memfile_open(vfile_t *f, memfile_t *mem) {
     mem->file = fmemopen(mem->buf, mem->size, "rb");
 
     if (f->calculate_checksum) {
-        SHA1_Init(&f->sha1_ctx);
-        safe_sha1_update(&f->sha1_ctx, mem->buf, mem->size);
-        SHA1_Final(f->sha1_digest, &f->sha1_ctx);
+        safe_digest_update(f->sha1_ctx, mem->buf, mem->size);
+        EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
+        EVP_MD_CTX_free(f->sha1_ctx);
+        f->sha1_ctx = NULL;
         f->has_checksum = TRUE;
     }
 
@@ -729,6 +730,9 @@ void parse_media_vfile(scan_media_ctx_t *ctx, struct vfile *f, document_t *doc,
         CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()");
         return;
     }
+    pFormatCtx->max_analyze_duration = 100000000;
+    pFormatCtx->probesize = 100000000;
+
 
     unsigned char *buffer = (unsigned char *) av_malloc(AVIO_BUF_SIZE);
     AVIOContext *io_ctx = NULL;
@@ -792,6 +796,8 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
         CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()");
         return FALSE;
     }
+    pFormatCtx->max_analyze_duration = 100000000;
+    pFormatCtx->probesize = 100000000;
 
     unsigned char *buffer = (unsigned char *) av_malloc(AVIO_BUF_SIZE);
 
@@ -823,6 +829,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
     // Decoder
     const AVCodec *video_codec = avcodec_find_decoder(stream->codecpar->codec_id);
     AVCodecContext *decoder = avcodec_alloc_context3(video_codec);
+    decoder->thread_count = 1;
     avcodec_parameters_to_context(decoder, stream->codecpar);
     avcodec_open2(decoder, video_codec, NULL);
 
@@ -861,15 +868,14 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
         avcodec_send_frame(jpeg_encoder, scaled_frame);
         avcodec_send_frame(jpeg_encoder, NULL); // Send EOF
 
-        AVPacket jpeg_packet;
-        av_init_packet(&jpeg_packet);
-        avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
+        AVPacket *jpeg_packet = av_packet_alloc();
+        avcodec_receive_packet(jpeg_encoder, jpeg_packet);
 
         // Save thumbnail
         APPEND_LONG_META(doc, MetaThumbnail, 1);
-        ctx->store(doc->doc_id, 0, jpeg_packet.data, jpeg_packet.size);
+        ctx->store(doc->doc_id, 0, jpeg_packet->data, jpeg_packet->size);
 
-        av_packet_unref(&jpeg_packet);
+        av_packet_free(&jpeg_packet);
         avcodec_free_context(&jpeg_encoder);
         av_free(*scaled_frame->data);
         av_frame_free(&scaled_frame);

third-party/libscan/libscan/scan.h

@@ -8,6 +8,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <openssl/evp.h>
 #include <openssl/md5.h>
 #include <openssl/sha.h>
 
@@ -146,7 +147,7 @@ typedef struct vfile {
     int mtime;
     size_t st_size;
 
-    SHA_CTX sha1_ctx;
+    EVP_MD_CTX *sha1_ctx;
     unsigned char sha1_digest[SHA1_DIGEST_LENGTH];
 
     void *rewind_buffer;

third-party/libscan/libscan/util.h

@@ -6,6 +6,7 @@
 #include "string.h"
 #include "../third-party/utf8.h/utf8.h"
 #include "macros.h"
+#include <openssl/evp.h>
 
 #define STR_STARTS_WITH_CONSTANT(x, y) (strncmp(y, x, sizeof(y) - 1) == 0)
 
@@ -339,7 +340,7 @@ static void *read_all(vfile_t *f, size_t *size) {
 #define STACK_BUFFER_SIZE (size_t)(4096 * 8)
 
 __always_inline
-static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) {
+static void safe_digest_update(EVP_MD_CTX *ctx, void *buf, size_t size) {
     unsigned char stack_buf[STACK_BUFFER_SIZE];
 
     void *sha1_buf;
@@ -351,7 +352,7 @@ static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) {
     }
 
     memcpy(sha1_buf, buf, size);
-    SHA1_Update(ctx, (const void *) sha1_buf, size);
+    EVP_DigestUpdate(ctx, sha1_buf, size);
 
     if (sha1_buf != stack_buf) {
         free(sha1_buf);