version bump

Bump semver from 5.7.1 to 5.7.2 in /sist2-admin/frontend

.dockerignore

@@ -37,4 +37,5 @@ state.db
 build/
 __pycache__/
 sist2-vue/dist
-sist2-admin/frontend/dist
+sist2-admin/frontend/dist
+*.fts

Dockerfile

@@ -1,11 +1,6 @@
 FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
-RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
-
 WORKDIR /build/
 
 COPY scripts scripts
@@ -25,7 +20,6 @@ RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
 
-
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 
@@ -45,6 +39,7 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2

Dockerfile.arm64

@@ -1,6 +1,19 @@
 FROM simon987/sist2-build-arm64 as build
 MAINTAINER simon987 <me@simon987.net>
 
+WORKDIR /build/
+
+COPY scripts scripts
+COPY schema schema
+COPY CMakeLists.txt .
+COPY third-party third-party
+COPY src src
+COPY sist2-vue sist2-vue
+COPY sist2-admin sist2-admin
+
+RUN cd sist2-vue/ && npm install && npm run build
+RUN cd sist2-admin/frontend/ && npm install && npm run build
+
 WORKDIR /build/
 ADD . /build/
 RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
@@ -20,16 +33,17 @@ RUN apt update && apt install -y curl libasan5 libmagic1 tesseract-ocr python3-p
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
-    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
-    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
-    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
 COPY --from=build /build/build/sist2 /root/sist2

README.md

@@ -28,7 +28,7 @@ sist2 (Simple incremental search tool)
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
-\*\*\* See [OCR](#ocr)
+\*\*\* See [OCR](#ocr)    
 \*\*\*\* See [Named-Entity Recognition](#NER)
 
 ## Getting Started
@@ -46,7 +46,7 @@ services:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
   sist2-admin:
-    image: simon987/sist2:3.0.7-x64-linux
+    image: simon987/sist2:3.1.0-x64-linux
     restart: unless-stopped
     volumes:
       - ./sist2-admin-data/:/sist2-admin/
@@ -126,7 +126,7 @@ Download the language data files with your package manager (`apt install tessera
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
 The `simon987/sist2` image comes with common languages
-(hin, jpn, eng, fra, rus, spa, chi_sim, deu) pre-installed.
+(hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
 name must be identical to the `*.traineddata` file installed on your system
@@ -206,7 +206,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
+    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,ffprobe,zlib]
     ```
 
 4. Build

docs/USAGE.md

@@ -17,7 +17,7 @@ Lightning-fast file system indexer and search tool.
 
 Scan options
     -t, --threads=<int>               Number of threads. DEFAULT: 1
-    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2
+    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
     --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
     --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
     --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
@@ -88,8 +88,8 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 
 See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
 
-For example, `--thumbnail-size=500`, `--thumbnail-quality=2` for a directory with 8 million images will create a thumbnail database 
+For example, `--thumbnail-size=500`, `--thumbnail-quality=50` for a directory with 8 million images will create a thumbnail database 
-that is about `8000000 * 36kB = 288GB`.
+that is about `8000000 * 11.8kB = 94.4GB`.
 
 ![thumbnail_size](thumbnail_size.png)
 

schema/mappings.json

@@ -68,7 +68,7 @@
     },
     "mtime": {
       "type": "date",
-      "format": "epoch_millis"
+      "format": "epoch_second"
     },
     "size": {
       "type": "long"

scripts/build.sh

@@ -4,6 +4,20 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
+(
+  cd sist2-vue/
+  npm install
+  npm run build
+) &
+
+(
+  cd sist2-admin/frontend/
+  npm install
+  npm run build
+) &
+
+wait
+
 mkdir build
 (
   cd build

scripts/build_arm64.sh

@@ -4,6 +4,20 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
+(
+  cd sist2-vue/
+  npm install
+  npm run build
+) &
+
+(
+  cd sist2-admin/frontend/
+  npm install
+  npm run build
+) &
+
+wait
+
 mkdir build
 (
   cd build

scripts/test_in_docker.sh

@@ -0,0 +1,7 @@
+docker build . -t tmp
+
+docker run --rm -it\
+  -v $(pwd):/host \
+  tmp \
+  scan --ocr-lang eng --ocr-ebooks -t6 --incremental --very-verbose \
+  -o /host/docker.sist2 /host/third-party/libscan/libscan-test-files/test_files/

sist2-admin/frontend/src/App.vue

@@ -1,14 +1,15 @@
 <template>
-  <div id="app">
+    <div id="app">
-    <NavBar></NavBar>
+        <NavBar></NavBar>
-    <b-container class="pt-4">
+        <b-container class="pt-4">
-      <b-alert show dismissible variant="info">
+            <b-alert show dismissible variant="info">
-        This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
+                This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
-        to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on Github</a>. Thank you!
+                to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
-      </b-alert>
+                Github</a>. Thank you!
-      <router-view/>
+            </b-alert>
-    </b-container>
+            <router-view/>
-  </div>
+        </b-container>
+    </div>
 </template>
 
 <script>
@@ -16,83 +17,87 @@ import NavBar from "@/components/NavBar";
 import Sist2AdminApi from "@/Sist2AdminApi";
 
 export default {
-  components: {NavBar},
+    components: {NavBar},
-  data() {
+    data() {
-    return {
+        return {
-      socket: null
+            socket: null
-    }
-  },
-  mounted() {
-    Sist2AdminApi.getSist2AdminInfo()
-        .then(resp => this.$store.commit("setSist2AdminInfo", resp.data));
-    this.$store.dispatch("loadBrowserSettings");
-    this.connectNotifications();
-    // this.socket.onclose = this.connectNotifications;
-  },
-  methods: {
-    connectNotifications() {
-      this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
-      this.socket.onopen = () => {
-        this.socket.send("Hello from client");
-      }
-
-      this.socket.onmessage = e => {
-        const notification = JSON.parse(e.data);
-        if (notification.message) {
-          notification.messageString = this.$t(notification.message).toString();
         }
+    },
+    mounted() {
+        Sist2AdminApi.getSist2AdminInfo()
+            .then(resp => this.$store.commit("setSist2AdminInfo", resp.data));
+        this.$store.dispatch("loadBrowserSettings");
+        this.connectNotifications();
+        // this.socket.onclose = this.connectNotifications;
+    },
+    methods: {
+        connectNotifications() {
+            if (window.location.protocol === "https:") {
+                this.socket = new WebSocket(`wss://${window.location.host}/notifications`);
+            } else {
+                this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
+            }
+            this.socket.onopen = () => {
+                this.socket.send("Hello from client");
+            }
 
-        this.$store.dispatch("notify", notification)
+            this.socket.onmessage = e => {
-      }
+                const notification = JSON.parse(e.data);
+                if (notification.message) {
+                    notification.messageString = this.$t(notification.message).toString();
+                }
+
+                this.$store.dispatch("notify", notification)
+            }
+        }
     }
-  }
 }
 </script>
 
 <style>
 html, body {
-  height: 100%;
+    height: 100%;
 }
 
 #app {
-  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+    /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
-  -webkit-font-smoothing: antialiased;
+    -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
+    -moz-osx-font-smoothing: grayscale;
-  /*text-align: center;*/
+    /*text-align: center;*/
-  color: #2c3e50;
+    color: #2c3e50;
-  padding-bottom: 1em;
+    padding-bottom: 1em;
-  min-height: 100%;
+    min-height: 100%;
 }
 
 .info-icon {
-  width: 1rem;
+    width: 1rem;
-  margin-right: 0.2rem;
+    margin-right: 0.2rem;
-  cursor: pointer;
+    cursor: pointer;
-  line-height: 1rem;
+    line-height: 1rem;
-  height: 1rem;
+    height: 1rem;
-  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
-  filter: brightness(45%);
+    filter: brightness(45%);
-  display: block;
+    display: block;
 }
 
 .tabs {
-  margin-top: 10px;
+    margin-top: 10px;
 }
 
 .modal-title {
-  text-overflow: ellipsis;
+    text-overflow: ellipsis;
-  overflow: hidden;
+    overflow: hidden;
-  white-space: nowrap;
+    white-space: nowrap;
 }
 
 @media screen and (min-width: 1500px) {
-  .container {
+    .container {
-    max-width: 1440px;
+        max-width: 1440px;
-  }
+    }
 }
 
 label {
-  margin-top: 0.5rem;
+    margin-top: 0.5rem;
-  margin-bottom: 0;
+    margin-bottom: 0;
 }
 </style>

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -7,15 +7,15 @@ class Sist2AdminApi {
     }
 
     getJobs() {
-        return axios.get(`${this.baseUrl}/api/job/`);
+        return axios.get(`${this.baseUrl}/api/job`);
     }
 
     getFrontends() {
-        return axios.get(`${this.baseUrl}/api/frontend/`);
+        return axios.get(`${this.baseUrl}/api/frontend`);
     }
 
     getTasks() {
-        return axios.get(`${this.baseUrl}/api/task/`);
+        return axios.get(`${this.baseUrl}/api/task`);
     }
 
     killTask(taskId) {
@@ -33,9 +33,26 @@ class Sist2AdminApi {
         return axios.get(`${this.baseUrl}/api/job/${name}`);
     }
 
-    /**
+    getSearchBackend(name) {
-     * @param {string} name
+        return axios.get(`${this.baseUrl}/api/search_backend/${name}`);
-     */
+    }
+
+    updateSearchBackend(name, data) {
+        return axios.put(`${this.baseUrl}/api/search_backend/${name}`, data);
+    }
+
+    getSearchBackends() {
+        return axios.get(`${this.baseUrl}/api/search_backend`);
+    }
+
+    deleteBackend(name) {
+        return axios.delete(`${this.baseUrl}/api/search_backend/${name}`)
+    }
+
+    createBackend(name) {
+        return axios.post(`${this.baseUrl}/api/search_backend/${name}`);
+    }
+
     getFrontend(name) {
         return axios.get(`${this.baseUrl}/api/frontend/${name}`);
     }
@@ -110,7 +127,17 @@ class Sist2AdminApi {
     }
 
     getSist2AdminInfo() {
-        return axios.get(`${this.baseUrl}/api/`);
+        return axios.get(`${this.baseUrl}/api`);
+    }
+
+    getLogsToDelete(jobName, n) {
+        return axios.get(`${this.baseUrl}/api/job/${jobName}/logs_to_delete`, {
+            params: {n: n}
+        });
+    }
+
+    deleteTaskLogs(taskId) {
+        return axios.post(`${this.baseUrl}/api/task/${taskId}/delete_logs`);
     }
 }
 

sist2-admin/frontend/src/components/IndexOptions.vue

@@ -1,64 +0,0 @@
-<template>
-  <div>
-    <label>{{ $t("indexOptions.threads") }}</label>
-    <b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.esUrl") }}</label>
-    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
-      {{ esTestMessage }}
-    </b-alert>
-    <b-input-group>
-      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
-      <b-input-group-append>
-        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
-      </b-input-group-append>
-    </b-input-group>
-
-    <label>{{ $t("indexOptions.esIndex") }}</label>
-    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
-
-    <br>
-    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
-      {{ $t("webOptions.esInsecure") }}
-    </b-form-checkbox>
-
-    <label>{{ $t("indexOptions.batchSize") }}</label>
-    <b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
-
-    <label>{{ $t("indexOptions.script") }}</label>
-    <b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
-  </div>
-
-</template>
-
-<script>
-import sist2AdminApi from "@/Sist2AdminApi";
-
-export default {
-  name: "IndexOptions",
-  props: ["options"],
-  data() {
-    return {
-      showEsTestAlert: false,
-      esTestOk: false,
-      esTestMessage: "",
-    }
-  },
-  methods: {
-    update() {
-      this.$emit("change", this.options);
-    },
-    testEs() {
-      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
-        this.showEsTestAlert = true;
-        this.esTestOk = resp.data.ok;
-        this.esTestMessage = resp.data.message;
-      });
-    }
-  },
-}
-</script>
-
-<style scoped>
-
-</style>

sist2-admin/frontend/src/components/JobListItem.vue

@@ -44,8 +44,7 @@ export default {
         return "";
       }
 
-      const date = Date.parse(dateString);
+      return moment.utc(dateString).local().fromNow();
-      return moment(date).fromNow();
     }
   }
 }

sist2-admin/frontend/src/components/JobOptions.vue

@@ -1,57 +1,94 @@
 <template>
-  <div>
+    <div>
-    <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
+        <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
-      {{ $t("jobOptions.desktopNotifications") }}
+            {{ $t("jobOptions.desktopNotifications") }}
-    </b-form-checkbox>
+        </b-form-checkbox>
 
-    <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
+        <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
-      {{ $t("jobOptions.scheduleEnabled") }}
+            {{ $t("jobOptions.scheduleEnabled") }}
-    </b-form-checkbox>
+        </b-form-checkbox>
 
-    <label>{{ $t("jobOptions.cron") }}</label>
+        <label>{{ $t("jobOptions.cron") }}</label>
-    <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression" :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
+        <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression"
-  </div>
+                      :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
+
+        <label>{{ $t("jobOptions.keepNLogs") }}</label>
+        <b-input-group>
+            <b-form-input type="number" v-model="job.keep_last_n_logs" @change="update()"></b-form-input>
+            <b-input-group-append>
+                <b-button variant="danger" @click="onDeleteNowClick()">{{ $t("jobOptions.deleteNow") }}</b-button>
+            </b-input-group-append>
+        </b-input-group>
+
+    </div>
 </template>
 
 <script>
 
+import Sist2AdminApi from "@/Sist2AdminApi";
+
 export default {
-  name: "JobOptions",
+    name: "JobOptions",
-  props: ["job"],
+    props: ["job"],
-  data() {
+    data() {
-    return {
+        return {
-      cronValid: undefined
+            cronValid: undefined,
-    }
+            logsToDelete: null
-  },
+        }
-  computed: {
+    },
-    desktopNotificationsEnabled() {
+    computed: {
-      return this.$store.state.jobDesktopNotificationMap[this.job.name];
+        desktopNotificationsEnabled() {
-    }
+            return this.$store.state.jobDesktopNotificationMap[this.job.name];
-  },
+        }
+    },
     mounted() {
-      this.cronValid = this.checkCron(this.job.cron_expression)
+        this.cronValid = this.checkCron(this.job.cron_expression)
     },
     methods: {
-      checkCron(expression) {
+        checkCron(expression) {
-          return /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(expression);
+            return /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(expression);
-      },
+        },
-    updateNotifications(value) {
+        updateNotifications(value) {
-      this.$store.dispatch("setJobDesktopNotification", {
+            this.$store.dispatch("setJobDesktopNotification", {
-        job: this.job.name,
+                job: this.job.name,
-        enabled: value
+                enabled: value
-      });
+            });
-    },
+        },
-    update() {
+        update() {
-      if (this.job.schedule_enabled) {
+            if (this.job.schedule_enabled) {
-        this.cronValid = this.checkCron(this.job.cron_expression);
+                this.cronValid = this.checkCron(this.job.cron_expression);
-      } else {
+            } else {
-        this.cronValid = undefined;
+                this.cronValid = undefined;
-      }
+            }
 
-      if (this.cronValid !== false) {
+            if (this.cronValid !== false) {
-        this.$emit("change", this.job);
+                this.$emit("change", this.job);
-      }
+            }
+        },
+        onDeleteNowClick() {
+            Sist2AdminApi.getLogsToDelete(this.job.name, this.job.keep_last_n_logs).then(resp => {
+                const toDelete = resp.data;
+                const message = `Delete ${toDelete.length} log files?`;
+
+                this.$bvModal.msgBoxConfirm(message, {
+                    title: this.$t("confirmation"),
+                    size: "sm",
+                    buttonSize: "sm",
+                    okVariant: "danger",
+                    okTitle: this.$t("delete"),
+                    cancelTitle: this.$t("cancel"),
+                    footerClass: "p-2",
+                    hideHeaderClose: false,
+                    centered: true
+                }).then(value => {
+                    if (value) {
+                        toDelete.forEach(row => {
+                            Sist2AdminApi.deleteTaskLogs(row["id"]);
+                        });
+                    }
+                });
+            })
+        }
     },
-  },
 }
 </script>

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -7,7 +7,7 @@
     <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
-    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+    <b-form-input type="number" min="0" max="100" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailCount") }}</label>
     <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>

sist2-admin/frontend/src/components/SearchBackendListItem.vue

@@ -0,0 +1,24 @@
+<template>
+    <b-list-group-item action :to="`/searchBackend/${backend.name}`">
+
+        <div class="d-flex w-100 justify-content-between">
+            <h5 class="mb-1">
+                {{ backend.name }}
+            </h5>
+
+            <div>
+                <b-badge v-if="backend.backend_type === 'sqlite'" variant="info">SQLite</b-badge>
+                <b-badge v-else variant="info">Elasticsearch</b-badge>
+            </div>
+        </div>
+
+    </b-list-group-item>
+</template>
+
+<script>
+
+export default {
+    name: "SearchBackendListItem",
+    props: ["backend"],
+}
+</script>

sist2-admin/frontend/src/components/SearchBackendSelect.vue

@@ -0,0 +1,37 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <div v-else>
+        <label>{{$t("backendOptions.searchBackend")}}</label>
+        <b-select :options="options" :value="value" @change="$emit('change', $event)"></b-select>
+    </div>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "SearchBackendSelect",
+    props: ["value"],
+    data() {
+        return {
+            loading: true,
+            backends: null,
+        }
+    },
+    computed: {
+        options() {
+            return this.backends.map(backend => backend.name)
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getSearchBackends().then(resp => {
+            this.loading = false;
+            this.backends = resp.data
+        })
+    }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -1,56 +1,35 @@
 <template>
-  <div>
+    <div>
-    <label>{{ $t("webOptions.esUrl") }}</label>
-    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
-      {{ esTestMessage }}
-    </b-alert>
 
-    <b-input-group>
+        <label>{{ $t("webOptions.lang") }}</label>
-      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+        <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
-      <b-input-group-append>
-        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
-      </b-input-group-append>
-    </b-input-group>
 
-    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
+        <label>{{ $t("webOptions.bind") }}</label>
-      {{ $t("webOptions.esInsecure") }}
+        <b-form-input v-model="options.bind" @change="update()"></b-form-input>
-    </b-form-checkbox>
 
-    <label>{{ $t("webOptions.esIndex") }}</label>
+        <label>{{ $t("webOptions.tagline") }}</label>
-    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+        <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
 
-    <label>{{ $t("webOptions.lang") }}</label>
+        <label>{{ $t("webOptions.auth") }}</label>
-    <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
+        <b-form-input v-model="options.auth" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.bind") }}</label>
+        <label>{{ $t("webOptions.tagAuth") }}</label>
-    <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+        <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.tagline") }}</label>
+        <br>
-    <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+        <h5>Auth0 options</h5>
+        <label>{{ $t("webOptions.auth0Audience") }}</label>
+        <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.auth") }}</label>
+        <label>{{ $t("webOptions.auth0Domain") }}</label>
-    <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+        <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.tagAuth") }}</label>
+        <label>{{ $t("webOptions.auth0ClientId") }}</label>
-    <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+        <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
 
-    <br>
+        <label>{{ $t("webOptions.auth0PublicKey") }}</label>
-    <h5>Auth0 options</h5>
+        <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
-    <label>{{ $t("webOptions.auth0Audience") }}</label>
+    </div>
-    <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.auth0Domain") }}</label>
-    <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.auth0ClientId") }}</label>
-    <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.auth0PublicKey") }}</label>
-    <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
-
-
-
-  </div>
 </template>
 
 <script>
@@ -58,31 +37,24 @@
 import sist2AdminApi from "@/Sist2AdminApi";
 
 export default {
-  name: "WebOptions",
+    name: "WebOptions",
-  props: ["options", "frontendName"],
+    props: ["options", "frontendName"],
-  data() {
+    data() {
-    return {
+        return {
-      showEsTestAlert: false,
+            showEsTestAlert: false,
-      esTestOk: false,
+            esTestOk: false,
-      esTestMessage: "",
+            esTestMessage: "",
-    }
+        }
-  },
-  methods: {
-    update() {
-      if (!this.options.es_url.startsWith("https")) {
-        this.options.es_insecure_ssl = false;
-      }
-
-      this.$emit("change", this.options);
     },
-    testEs() {
+    methods: {
-      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        update() {
-        this.showEsTestAlert = true;
+            if (!this.options.es_url.startsWith("https")) {
-        this.esTestOk = resp.data.ok;
+                this.options.es_insecure_ssl = false;
-        this.esTestMessage = resp.data.message;
+            }
-      });
+
+            this.$emit("change", this.options);
+        },
     }
-  }
 }
 </script>
 

sist2-admin/frontend/src/i18n/messages.js

@@ -5,10 +5,13 @@ export default {
         go: "Go",
         online: "online",
         offline: "offline",
+        view: "View",
         delete: "Delete",
         runNow: "Index now",
         create: "Create",
+        cancel: "Cancel",
         test: "Test",
+        confirmation: "Confirmation",
 
         jobTitle: "job configuration",
         tasks: "Tasks",
@@ -45,12 +48,13 @@ export default {
         extraQueryArgs: "Extra query arguments when launching from sist2-admin",
         customUrl: "Custom URL when launching from sist2-admin",
 
+        searchBackends: "Search backends",
+        searchBackendTitle: "search backend configuration",
+        newBackendName: "New search backend name",
+
         selectJobs: "Select jobs",
         webOptions: {
             title: "Web options",
-            esUrl: "Elasticsearch URL",
-            esIndex: "Elasticsearch index name",
-            esInsecure: "Do not verify SSL connections to Elasticsearch.",
             lang: "UI Language",
             bind: "Listen address",
             tagline: "Tagline in navbar",
@@ -61,12 +65,24 @@ export default {
             auth0ClientId: "Auth0 client ID",
             auth0PublicKey: "Auth0 public key",
         },
+        backendOptions: {
+            title: "Search backend options",
+            searchBackend: "Search backend",
+            type: "Search backend type",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            threads: "Number of threads",
+            batchSize: "Index batch size",
+            script: "User script",
+            searchIndex: "Search index file location"
+        },
         scanOptions: {
             title: "Scanning options",
             path: "Path",
             threads: "Number of threads",
             memThrottle: "Total memory threshold in MiB for scan throttling",
-            thumbnailQuality: "Thumbnail quality, on a scale of 2 to 32, 2 being the best",
+            thumbnailQuality: "Thumbnail quality, on a scale of 0 to 100, 100 being the best",
             thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
             thumbnailSize: "Thumbnail size, in pixels",
             contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",
@@ -87,20 +103,14 @@ export default {
             treemapThreshold: "Relative size threshold for treemap",
             optimizeIndex: "Defragment index file after scan to reduce its file size."
         },
-        indexOptions: {
-            title: "Indexing options",
-            threads: "Number of threads",
-            esUrl: "Elasticsearch URL",
-            esIndex: "Elasticsearch index name",
-            esInsecure: "Do not verify SSL connections to Elasticsearch.",
-            batchSize: "Index batch size",
-            script: "User script"
-        },
         jobOptions: {
             title: "Job options",
             cron: "Job schedule",
+            keepNLogs: "Keep last N log files. Set to -1 to keep all logs.",
+            deleteNow: "Delete now",
             scheduleEnabled: "Enable scheduled re-scan",
             noJobAvailable: "No jobs available.",
+            noBackendError: "You must select a search backend to run this job",
             desktopNotifications: "Desktop notifications"
         },
         frontendOptions: {

sist2-admin/frontend/src/router/index.js

@@ -5,6 +5,7 @@ import Job from "@/views/Job";
 import Tasks from "@/views/Tasks";
 import Frontend from "@/views/Frontend";
 import Tail from "@/views/Tail";
+import SearchBackend from "@/views/SearchBackend.vue";
 
 Vue.use(VueRouter);
 
@@ -29,6 +30,11 @@ const routes = [
     name: "Frontend",
     component: Frontend
   },
+  {
+    path: "/searchBackend/:name",
+    name: "SearchBackend",
+    component: SearchBackend
+  },
   {
     path: "/log/:taskId",
     name: "Tail",

sist2-admin/frontend/src/views/Frontend.vue

@@ -1,60 +1,70 @@
 <template>
-  <b-card>
+    <b-card>
-    <b-card-title>
+        <b-card-title>
-      {{ name }}
+            {{ name }}
-      <small style="vertical-align: top">
+            <small style="vertical-align: top">
-        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+                <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
-        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+                <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
-      </small>
+            </small>
-    </b-card-title>
+        </b-card-title>
 
-    <div class="mb-3" v-if="!loading">
+        <div class="mb-3" v-if="!loading">
-      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+            <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
-          $t("start")
+                $t("start")
-        }}
+                }}
-      </b-button>
+            </b-button>
-      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+            <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
-          $t("stop")
+                $t("stop")
-        }}
+                }}
-      </b-button>
+            </b-button>
-      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+            <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
-        {{ $t("go") }}
+                {{ $t("go") }}
-      </b-button>
+            </b-button>
-      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+            <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
-    </div>
+        </div>
 
 
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <b-card-body v-else>
+        <b-card-body v-else>
 
-      <h4>{{ $t("frontendOptions.title") }}</h4>
+            <h4>{{ $t("frontendOptions.title") }}</h4>
-      <b-card>
+            <b-card>
-        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+                <b-form-checkbox v-model="frontend.auto_start" @change="update()">
-          {{ $t("autoStart") }}
+                    {{ $t("autoStart") }}
-        </b-form-checkbox>
+                </b-form-checkbox>
 
-        <label>{{ $t("extraQueryArgs") }}</label>
+                <label>{{ $t("extraQueryArgs") }}</label>
-        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+                <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
 
-        <label>{{ $t("customUrl") }}</label>
+                <label>{{ $t("customUrl") }}</label>
-        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+                <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
 
-        <br/>
+                <br/>
 
-        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+                <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
 
-        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+                <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
-      </b-card>
+            </b-card>
 
-      <br/>
+            <br/>
 
-      <h4>{{ $t("webOptions.title") }}</h4>
+            <h4>{{ $t("webOptions.title") }}</h4>
-      <b-card>
+            <b-card>
-        <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
+                <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
-      </b-card>
+                            @change="update()"></WebOptions>
-    </b-card-body>
+            </b-card>
 
-  </b-card>
+            <br>
+
+            <h4>{{ $t("backendOptions.title") }}</h4>
+            <b-card>
+                <SearchBackendSelect :value="frontend.web_options.search_backend"
+                                     @change="onBackendSelect($event)"></SearchBackendSelect>
+            </b-card>
+        </b-card-body>
+
+
+    </b-card>
 </template>
 
 <script>
@@ -62,68 +72,73 @@
 import Sist2AdminApi from "@/Sist2AdminApi";
 import JobCheckboxGroup from "@/components/JobCheckboxGroup";
 import WebOptions from "@/components/WebOptions";
+import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
-  name: 'Frontend',
+    name: 'Frontend',
-  components: {JobCheckboxGroup, WebOptions},
+    components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
-  data() {
+    data() {
-    return {
+        return {
-      loading: true,
+            loading: true,
-      frontend: null,
+            frontend: null,
-    }
+        }
-  },
-  computed: {
-    valid() {
-      return !this.loading && this.frontend.jobs.length > 0;
     },
-    frontendUrl() {
+    computed: {
-      if (this.frontend.custom_url) {
+        valid() {
-        return this.frontend.custom_url + this.args;
+            return !this.loading && this.frontend.jobs.length > 0;
-      }
+        },
+        frontendUrl() {
+            if (this.frontend.custom_url) {
+                return this.frontend.custom_url + this.args;
+            }
 
-      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+            if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
-        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+                return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
-      }
+            }
 
-      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+            return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+        },
+        name() {
+            return this.$route.params.name;
+        },
+        port() {
+            return this.frontend.web_options.bind.split(":")[1]
+        },
+        args() {
+            const args = this.frontend.extra_query_args;
+            if (args !== "") {
+                return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+            }
+            return "";
+        }
     },
-    name() {
+    mounted() {
-      return this.$route.params.name;
+        Sist2AdminApi.getFrontend(this.name).then(resp => {
+            this.frontend = resp.data;
+            this.loading = false;
+        });
     },
-    port() {
+    methods: {
-      return this.frontend.web_options.bind.split(":")[1]
+        start() {
-    },
+            this.frontend.running = true;
-    args() {
+            Sist2AdminApi.startFrontend(this.name)
-      const args = this.frontend.extra_query_args;
+        },
-      if (args !== "") {
+        stop() {
-        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+            this.frontend.running = false;
-      }
+            Sist2AdminApi.stopFrontend(this.name)
-      return "";
+        },
+        deleteFrontend() {
+            Sist2AdminApi.deleteFrontend(this.name).then(() => {
+                this.$router.push("/");
+            });
+        },
+        update() {
+            Sist2AdminApi.updateFrontend(this.name, this.frontend);
+        },
+        onBackendSelect(backend) {
+            this.frontend.web_options.search_backend = backend;
+            this.update();
+        }
     }
-  },
-  mounted() {
-    Sist2AdminApi.getFrontend(this.name).then(resp => {
-      this.frontend = resp.data;
-      this.loading = false;
-    });
-  },
-  methods: {
-    start() {
-      this.frontend.running = true;
-      Sist2AdminApi.startFrontend(this.name)
-    },
-    stop() {
-      this.frontend.running = false;
-      Sist2AdminApi.stopFrontend(this.name)
-    },
-    deleteFrontend() {
-      Sist2AdminApi.deleteFrontend(this.name).then(() => {
-        this.$router.push("/frontends");
-      });
-    },
-    update() {
-      Sist2AdminApi.updateFrontend(this.name, this.frontend);
-    },
-  }
 }
 </script>

sist2-admin/frontend/src/views/Home.vue

@@ -1,60 +1,89 @@
 <template>
-  <div>
+    <div>
-    <b-card>
+        <b-card>
-      <b-card-title>{{ $t("jobs") }}</b-card-title>
+            <b-card-title>{{ $t("jobs") }}</b-card-title>
-      <b-row>
+            <b-row>
-        <b-col>
+                <b-col>
-          <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
+                    <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
-          <b-popover
+                    <b-popover
-              :show.sync="showHelp"
+                            :show.sync="showHelp"
-              target="new-job"
+                            target="new-job"
-              placement="top"
+                            placement="top"
-              triggers="manual"
+                            triggers="manual"
-              variant="primary"
+                            variant="primary"
-              :content="$t('newJobHelp')"
+                            :content="$t('newJobHelp')"
-          ></b-popover>
+                    ></b-popover>
-        </b-col>
+                </b-col>
-        <b-col>
+                <b-col>
-          <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
+                    <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">
-          </b-button>
+                        {{ $t("create") }}
-        </b-col>
+                    </b-button>
-      </b-row>
+                </b-col>
+            </b-row>
 
-      <hr/>
+            <hr/>
 
-      <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
+            <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
-      <b-list-group v-else>
+            <b-list-group v-else>
-        <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
+                <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
-      </b-list-group>
+            </b-list-group>
-    </b-card>
+        </b-card>
 
-    <br/>
+        <br/>
 
-    <b-card>
+        <b-card>
 
-      <b-card-title>{{ $t("frontends") }}</b-card-title>
+            <b-card-title>{{ $t("frontends") }}</b-card-title>
 
-      <b-row>
+            <b-row>
-        <b-col>
+                <b-col>
-          <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
+                    <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
-        </b-col>
+                </b-col>
-        <b-col>
+                <b-col>
-          <b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
+                    <b-button variant="primary" @click="createFrontend()"
-            {{ $t("create") }}
+                              :disabled="!frontendNameValid(newFrontendName)">
-          </b-button>
+                        {{ $t("create") }}
-        </b-col>
+                    </b-button>
-      </b-row>
+                </b-col>
+            </b-row>
 
-      <hr/>
+            <hr/>
 
-      <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
+            <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
-      <b-list-group v-else>
+            <b-list-group v-else>
-        <FrontendListItem v-for="frontend in frontends"
+                <FrontendListItem v-for="frontend in frontends"
-                          :key="frontend.name" :frontend="frontend"></FrontendListItem>
+                                  :key="frontend.name" :frontend="frontend"></FrontendListItem>
-      </b-list-group>
+            </b-list-group>
 
-    </b-card>
+        </b-card>
-  </div>
+
+        <br/>
+
+        <b-card>
+            <b-card-title>{{ $t("searchBackends") }}</b-card-title>
+
+            <b-row>
+                <b-col>
+                    <b-input v-model="newBackendName" :placeholder="$t('newBackendName')"></b-input>
+                </b-col>
+                <b-col>
+                    <b-button variant="primary" @click="createBackend()"
+                              :disabled="!backendNameValid(newBackendName)">
+                        {{ $t("create") }}
+                    </b-button>
+                </b-col>
+            </b-row>
+
+            <hr/>
+
+            <b-progress v-if="backendsLoading" striped animated value="100"></b-progress>
+            <b-list-group v-else>
+                <SearchBackendListItem v-for="backend in backends"
+                                  :key="backend.name" :backend="backend"></SearchBackendListItem>
+            </b-list-group>
+
+        </b-card>
+    </div>
 </template>
 
 <script>
@@ -62,61 +91,80 @@ import JobListItem from "@/components/JobListItem";
 import {formatBindAddress} from "@/util";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import FrontendListItem from "@/components/FrontendListItem";
+import SearchBackendListItem from "@/components/SearchBackendListItem.vue";
 
 export default {
-  name: "Jobs",
+    name: "Jobs",
-  components: {JobListItem, FrontendListItem},
+    components: {SearchBackendListItem, JobListItem, FrontendListItem},
-  data() {
+    data() {
-    return {
+        return {
-      jobsLoading: true,
+            jobsLoading: true,
-      newJobName: "",
+            newJobName: "",
-      jobs: [],
+            jobs: [],
 
-      frontendsLoading: true,
+            frontendsLoading: true,
-      frontends: [],
+            frontends: [],
-      formatBindAddress,
+            formatBindAddress,
-      newFrontendName: "",
+            newFrontendName: "",
 
-      showHelp: false
+            backends: [],
+            backendsLoading: true,
+            newBackendName: "",
+
+            showHelp: false
+        }
+    },
+    mounted() {
+        this.loading = true;
+        this.reload();
+    },
+    methods: {
+        jobNameValid(name) {
+            if (this.jobs.some(job => job.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        frontendNameValid(name) {
+            if (this.frontends.some(frontend => frontend.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        backendNameValid(name) {
+            if (this.backends.some(backend => backend.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        reload() {
+            Sist2AdminApi.getJobs().then(resp => {
+                this.jobs = resp.data;
+                this.jobsLoading = false;
+
+                this.showHelp = this.jobs.length === 0;
+            });
+            Sist2AdminApi.getFrontends().then(resp => {
+                this.frontends = resp.data;
+                this.frontendsLoading = false;
+            });
+            Sist2AdminApi.getSearchBackends().then(resp => {
+                this.backends = resp.data;
+                this.backendsLoading = false;
+            })
+        },
+        createJob() {
+            Sist2AdminApi.createJob(this.newJobName).then(this.reload);
+        },
+        createFrontend() {
+            Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
+        },
+        createBackend() {
+            Sist2AdminApi.createBackend(this.newBackendName).then(this.reload);
+        }
     }
-  },
-  mounted() {
-    this.loading = true;
-    this.reload();
-  },
-  methods: {
-    jobNameValid(name) {
-      if (this.jobs.some(job => job.name === name)) {
-        return false;
-      }
-
-      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
-    },
-    frontendNameValid(name) {
-      if (this.frontends.some(job => job.name === name)) {
-        return false;
-      }
-
-      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
-    },
-    reload() {
-      Sist2AdminApi.getJobs().then(resp => {
-        this.jobs = resp.data;
-        this.jobsLoading = false;
-
-        this.showHelp = this.jobs.length === 0;
-      });
-      Sist2AdminApi.getFrontends().then(resp => {
-        this.frontends = resp.data;
-        this.frontendsLoading = false;
-      });
-    },
-    createJob() {
-      Sist2AdminApi.createJob(this.newJobName).then(this.reload);
-    },
-    createFrontend() {
-      Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
-    }
-  }
 }
 </script>

sist2-admin/frontend/src/views/Job.vue

@@ -1,92 +1,112 @@
 <template>
-  <b-card>
+    <b-card>
-    <b-card-title>
+        <b-card-title>
-      [{{ getName() }}]
+            [{{ getName() }}]
-      {{ $t("jobTitle") }}
+            {{ $t("jobTitle") }}
-    </b-card-title>
+        </b-card-title>
 
-    <div class="mb-3">
+        <div class="mb-3">
-      <b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
+            <b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
-      <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
+            <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
-    </div>
+        </div>
 
-    <div v-if="job">
+        <div v-if="job">
-      {{ $t("status") }}: <code>{{ job.status }}</code>
+            {{ $t("status") }}: <code>{{ job.status }}</code>
-    </div>
+        </div>
 
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <b-card-body v-else>
+        <b-card-body v-else>
 
-      <h4>{{ $t("jobOptions.title") }}</h4>
+            <h4>{{ $t("jobOptions.title") }}</h4>
-      <b-card>
+            <b-card>
-        <JobOptions :job="job" @change="update"></JobOptions>
+                <JobOptions :job="job" @change="update"></JobOptions>
-      </b-card>
+            </b-card>
 
-      <br/>
+            <br/>
 
-      <h4>{{ $t("scanOptions.title") }}</h4>
+            <h4>{{ $t("scanOptions.title") }}</h4>
-      <b-card>
+            <b-card>
-        <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
+                <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
-      </b-card>
+            </b-card>
 
-      <br/>
+            <br/>
 
-      <h4>{{ $t("indexOptions.title") }}</h4>
+            <h4>{{ $t("backendOptions.title") }}</h4>
-      <b-card>
+            <b-card>
-        <IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
+                <b-alert v-if="!valid" variant="warning" show>{{ $t("jobOptions.noBackendError") }}</b-alert>
-      </b-card>
+                <SearchBackendSelect :value="job.index_options.search_backend"
+                                     @change="onBackendSelect($event)"></SearchBackendSelect>
+            </b-card>
 
-    </b-card-body>
+        </b-card-body>
 
-  </b-card>
+    </b-card>
 </template>
 
 <script>
 import ScanOptions from "@/components/ScanOptions";
 import Sist2AdminApi from "@/Sist2AdminApi";
-import IndexOptions from "@/components/IndexOptions";
 import JobOptions from "@/components/JobOptions";
+import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
-  name: "Job",
+    name: "Job",
-  components: {
+    components: {
-    IndexOptions,
+        SearchBackendSelect,
-    ScanOptions,
+        ScanOptions,
-    JobOptions
+        JobOptions
-  },
+    },
-  data() {
+    data() {
-    return {
+        return {
-      loading: true,
+            loading: true,
-      job: null
+            job: null,
+        }
+    },
+    methods: {
+        getName() {
+            return this.$route.params.name;
+        },
+        update() {
+            Sist2AdminApi.updateJob(this.getName(), this.job);
+        },
+        runJob() {
+            Sist2AdminApi.runJob(this.getName()).then(() => {
+                this.$bvToast.toast(this.$t("runJobConfirmation"), {
+                    title: this.$t("runJobConfirmationTitle"),
+                    variant: "success",
+                    toaster: "b-toaster-bottom-right"
+                });
+            });
+        },
+        deleteJob() {
+            Sist2AdminApi.deleteJob(this.getName())
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete job " +
+                        "because it is referenced by a frontend", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        },
+        onBackendSelect(backend) {
+            this.job.index_options.search_backend = backend;
+            this.update();
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getJob(this.getName()).then(resp => {
+            this.loading = false;
+            this.job = resp.data;
+        })
+    },
+    computed: {
+        valid() {
+            return this.job?.index_options.search_backend != null;
+        }
     }
-  },
-  methods: {
-    getName() {
-      return this.$route.params.name;
-    },
-    update() {
-      Sist2AdminApi.updateJob(this.getName(), this.job);
-    },
-    runJob() {
-      Sist2AdminApi.runJob(this.getName()).then(() => {
-        this.$bvToast.toast(this.$t("runJobConfirmation"), {
-          title: this.$t("runJobConfirmationTitle"),
-          variant: "success",
-          toaster: "b-toaster-bottom-right"
-        });
-      });
-    },
-    deleteJob() {
-      Sist2AdminApi.deleteJob(this.getName()).then(() => {
-        this.$router.push("/");
-      })
-    }
-  },
-  mounted() {
-    Sist2AdminApi.getJob(this.getName()).then(resp => {
-      this.loading = false;
-      this.job = resp.data;
-    })
-  }
 }
 </script>

sist2-admin/frontend/src/views/SearchBackend.vue

@@ -0,0 +1,126 @@
+<template>
+
+    <b-card>
+        <b-card-title>
+            <span class="text-monospace">{{ getName() }}</span>
+            {{ $t("searchBackendTitle") }}
+        </b-card-title>
+
+        <div class="mb-3">
+            <b-button variant="danger" @click="deleteBackend()">{{ $t("delete") }}</b-button>
+        </div>
+
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-card-body v-else>
+
+            <label>{{ $t("backendOptions.type") }}</label>
+            <b-select :options="backendTypeOptions" v-model="backend.backend_type" @change="update()"></b-select>
+
+            <hr/>
+
+            <template v-if="backend.backend_type === 'elasticsearch'">
+                <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+                    {{ esTestMessage }}
+                </b-alert>
+
+                <label>{{ $t("backendOptions.esUrl") }}</label>
+                <b-input-group>
+                    <b-form-input v-model="backend.es_url" @change="update()"></b-form-input>
+                    <b-input-group-append>
+                        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+                    </b-input-group-append>
+                </b-input-group>
+
+                <b-form-checkbox v-model="backend.es_insecure_ssl" :disabled="!this.backend.es_url.startsWith('https')"
+                                 @change="update()">
+                    {{ $t("backendOptions.esInsecure") }}
+                </b-form-checkbox>
+
+                <label>{{ $t("backendOptions.esIndex") }}</label>
+                <b-form-input v-model="backend.es_index" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.threads") }}</label>
+                <b-form-input v-model="backend.threads" type="number" min="1" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.batchSize") }}</label>
+                <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.script") }}</label>
+                <b-form-textarea v-model="backend.script" rows="6" @change="update()"></b-form-textarea>
+            </template>
+            <template v-else>
+                <label>{{ $t("backendOptions.searchIndex") }}</label>
+                <b-form-input v-model="backend.search_index" disabled></b-form-input>
+            </template>
+        </b-card-body>
+
+    </b-card>
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "SearchBackend",
+    data() {
+        return {
+            showEsTestAlert: false,
+            esTestOk: false,
+            esTestMessage: "",
+            loading: true,
+            backend: null,
+            backendTypeOptions: [
+                {
+                    text: "Elasticsearch",
+                    value: "elasticsearch"
+                },
+                {
+                    text: "SQLite",
+                    value: "sqlite"
+                }
+            ]
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getSearchBackend(this.getName()).then(resp => {
+            this.backend = resp.data;
+            this.loading = false;
+        });
+    },
+    methods: {
+        getName() {
+            return this.$route.params.name;
+        },
+        testEs() {
+            sist2AdminApi.pingEs(this.backend.es_url, this.backend.es_insecure_ssl)
+                .then((resp) => {
+                    this.showEsTestAlert = true;
+                    this.esTestOk = resp.data.ok;
+                    this.esTestMessage = resp.data.message;
+                });
+        },
+        update() {
+            Sist2AdminApi.updateSearchBackend(this.getName(), this.backend);
+        },
+        deleteBackend() {
+            Sist2AdminApi.deleteBackend(this.getName())
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete search backend " +
+                        "because it is referenced by a job or frontend", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        }
+    }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/views/Tail.vue

@@ -1,168 +1,172 @@
 <template>
-  <b-card>
+    <b-card>
-    <b-card-body>
+        <b-card-body>
 
-      <h4 class="mb-3">{{ taskId }} {{ $t("logs") }}</h4>
+            <h4 class="mb-3">{{ taskId }} {{ $t("logs") }}</h4>
 
-      <div v-if="$store.state.sist2AdminInfo">
+            <div v-if="$store.state.sist2AdminInfo">
-        {{ $t("logFile") }}
+                {{ $t("logFile") }}
-        <code>{{ $store.state.sist2AdminInfo.logs_folder }}/sist2-{{ taskId }}.log</code>
+                <code>{{ $store.state.sist2AdminInfo.logs_folder }}/sist2-{{ taskId }}.log</code>
-        <br/>
+                <br/>
-        <br/>
+                <br/>
-      </div>
+            </div>
 
-      <b-row>
+            <b-row>
-        <b-col>
+                <b-col>
-          <span>{{ $t("logLevel") }}</span>
+                    <span>{{ $t("logLevel") }}</span>
-          <b-select :options="levels.slice(0, -1)" v-model="logLevel" @input="connect()"></b-select>
+                    <b-select :options="levels.slice(0, -1)" v-model="logLevel" @input="connect()"></b-select>
-        </b-col>
+                </b-col>
-        <b-col>
+                <b-col>
-          <span>{{ $t("logMode") }}</span>
+                    <span>{{ $t("logMode") }}</span>
-          <b-select :options="modeOptions" v-model="mode" @input="connect()"></b-select>
+                    <b-select :options="modeOptions" v-model="mode" @input="connect()"></b-select>
-        </b-col>
+                </b-col>
-      </b-row>
+            </b-row>
 
-      <div id="log-tail-output" class="mt-3 ml-1"></div>
+            <div id="log-tail-output" class="mt-3 ml-1"></div>
 
-    </b-card-body>
+        </b-card-body>
-  </b-card>
+    </b-card>
 </template>
 
 <script>
 
 export default {
-  name: "Tail",
+    name: "Tail",
-  data() {
+    data() {
-    return {
+        return {
-      logLevel: "DEBUG",
+            logLevel: "DEBUG",
-      levels: ["DEBUG", "INFO", "WARNING", "ERROR", "ADMIN", "FATAL"],
+            levels: ["DEBUG", "INFO", "WARNING", "ERROR", "ADMIN", "FATAL"],
-      socket: null,
+            socket: null,
-      mode: "follow",
+            mode: "follow",
-      modeOptions: [
+            modeOptions: [
-        {
+                {
-          "text": this.$t('follow'),
+                    "text": this.$t('follow'),
-          "value": "follow"
+                    "value": "follow"
-        },
+                },
-        {
+                {
-          "text": this.$t('wholeFile'),
+                    "text": this.$t('wholeFile'),
-          "value": "wholeFile"
+                    "value": "wholeFile"
+                }
+            ]
         }
-      ]
+    },
+    computed: {
+        taskId: function () {
+            return this.$route.params.taskId;
+        }
+    },
+    methods: {
+        connect() {
+            let lineCount = 0;
+            const outputElem = document.getElementById("log-tail-output")
+            outputElem.replaceChildren();
+            if (this.socket !== null) {
+                this.socket.close();
+            }
+
+            const n = this.mode === "follow" ? 32 : 9999999999;
+            if (window.location.protocol === "https:") {
+                this.socket = new WebSocket(`wss://${window.location.host}/log/${this.taskId}?n=${n}`);
+            } else {
+                this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
+            }
+            this.socket.onopen = () => {
+                this.socket.send("Hello from client");
+            }
+
+            this.socket.onmessage = e => {
+                let message;
+                try {
+                    message = JSON.parse(e.data);
+                } catch {
+                    console.error(e.data)
+                    return;
+                }
+
+                if ("ping" in message) {
+                    return;
+                }
+
+                if (message.level === undefined) {
+
+                    if ("stderr" in message) {
+                        message.level = "ERROR";
+                        message.message = message["stderr"];
+                    } else {
+                        message.level = "ADMIN";
+                        message.message = message["sist2-admin"];
+                    }
+                    message.datetime = ""
+                    message.filepath = ""
+                }
+
+                if (this.levels.indexOf(message.level) < this.levels.indexOf(this.logLevel)) {
+                    return;
+                }
+
+                const logLine = `${message.datetime} [${message.level} ${message.filepath}] ${message.message}`;
+
+                const span = document.createElement("span");
+                span.setAttribute("class", message.level);
+                span.appendChild(document.createTextNode(logLine));
+
+                outputElem.appendChild(span);
+                lineCount += 1;
+
+                if (this.mode === "follow" && lineCount >= n) {
+                    outputElem.firstChild.remove();
+                }
+            }
+        }
+    },
+    mounted() {
+        this.connect()
     }
-  },
-  computed: {
-    taskId: function () {
-      return this.$route.params.taskId;
-    }
-  },
-  methods: {
-    connect() {
-      let lineCount = 0;
-      const outputElem = document.getElementById("log-tail-output")
-      outputElem.replaceChildren();
-      if (this.socket !== null) {
-        this.socket.close();
-      }
-
-      const n = this.mode === "follow" ? 32 : 9999999999;
-      this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
-      this.socket.onopen = () => {
-        this.socket.send("Hello from client");
-      }
-
-      this.socket.onmessage = e => {
-        let message;
-        try {
-          message = JSON.parse(e.data);
-        } catch {
-          console.error(e.data)
-          return;
-        }
-
-        if ("ping" in message) {
-          return;
-        }
-
-        if (message.level === undefined) {
-
-          if ("stderr" in message) {
-            message.level = "ERROR";
-            message.message = message["stderr"];
-          } else {
-            message.level = "ADMIN";
-            message.message = message["sist2-admin"];
-          }
-          message.datetime = ""
-          message.filepath = ""
-        }
-
-        if (this.levels.indexOf(message.level) < this.levels.indexOf(this.logLevel)) {
-          return;
-        }
-
-        const logLine = `${message.datetime} [${message.level} ${message.filepath}] ${message.message}`;
-
-        const span = document.createElement("span");
-        span.setAttribute("class", message.level);
-        span.appendChild(document.createTextNode(logLine));
-
-        outputElem.appendChild(span);
-        lineCount += 1;
-
-        if (this.mode === "follow" && lineCount >= n) {
-          outputElem.firstChild.remove();
-        }
-      }
-    }
-  },
-  mounted() {
-    this.connect()
-  }
 }
 
 </script>
 
 <style>
 #log-tail-output span {
-  display: block;
+    display: block;
 }
 
 span.DEBUG {
-  color: #9E9E9E;
+    color: #9E9E9E;
 }
 
 span.WARNING {
-  color: #FFB300;
+    color: #FFB300;
 }
 
 span.INFO {
-  color: #039BE5;
+    color: #039BE5;
 }
 
 span.ERROR {
-  color: #F4511E;
+    color: #F4511E;
 }
 
 span.FATAL {
-  color: #F4511E;
+    color: #F4511E;
 }
 
 span.ADMIN {
-  color: #ee05ff;
+    color: #ee05ff;
 }
 
 
 #log-tail-output {
-  font-size: 13px;
+    font-size: 13px;
-  font-family: monospace;
+    font-family: monospace;
 
-  padding: 6px;
+    padding: 6px;
-  background-color: #f5f5f5;
+    background-color: #f5f5f5;
-  border: 1px solid #ccc;
+    border: 1px solid #ccc;
-  border-radius: 4px;
+    border-radius: 4px;
-  margin: 3px;
+    margin: 3px;
-  white-space: pre;
+    white-space: pre;
-  color: #000;
+    color: #000;
-  overflow: hidden;
+    overflow: hidden;
 }
 </style>

sist2-admin/frontend/src/views/Tasks.vue

@@ -1,38 +1,49 @@
 <template>
-  <div>
+    <div>
 
-    <b-card v-if="tasks.length > 0">
+        <b-card v-if="tasks.length > 0">
-      <h2>{{ $t("runningTasks") }}</h2>
+            <h2>{{ $t("runningTasks") }}</h2>
-      <b-list-group>
+            <b-list-group>
-        <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
+                <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
-      </b-list-group>
+            </b-list-group>
-    </b-card>
+        </b-card>
 
-    <b-card class="mt-4">
+        <b-card class="mt-4">
 
-      <b-card-title>{{ $t("taskHistory") }}</b-card-title>
+            <b-card-title>{{ $t("taskHistory") }}</b-card-title>
 
-      <br/>
+            <br/>
 
-      <b-table
+            <b-table
-          id="task-history"
+                    id="task-history"
-          :items="historyItems"
+                    :items="historyItems"
-          :fields="historyFields"
+                    :fields="historyFields"
-          :current-page="historyCurrentPage"
+                    :current-page="historyCurrentPage"
-          :tbody-tr-class="rowClass"
+                    :tbody-tr-class="rowClass"
-          :per-page="10"
+                    :per-page="10"
-      >
+            >
-        <template #cell(logs)="data">
+                <template #cell(logs)="data">
-          <router-link :to="`/log/${data.item.logs}`">{{ $t("logs") }}</router-link>
+                    <template v-if="data.item._row.has_logs">
-        </template>
+                        <b-button variant="link" size="sm" :to="`/log/${data.item.id}`">
+                            {{ $t("view") }}
+                        </b-button>
+                        /
+                        <b-button variant="link" size="sm" @click="deleteLogs(data.item.id)">
+                            {{ $t("delete") }}
+                        </b-button>
+                    </template>
+                </template>
 
-      </b-table>
+                <template #cell(delete)="data">
+                </template>
 
-      <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
+            </b-table>
-                    :per-page="10"></b-pagination>
 
-    </b-card>
+            <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
-  </div>
+                          :per-page="10"></b-pagination>
+
+        </b-card>
+    </div>
 </template>
 
 <script>
@@ -45,106 +56,116 @@ const HOUR = 3600;
 const MINUTE = 60;
 
 function humanDuration(sec_num) {
-  sec_num = sec_num / 1000;
+    sec_num = sec_num / 1000;
-  const days = Math.floor(sec_num / DAY);
+    const days = Math.floor(sec_num / DAY);
-  sec_num -= days * DAY;
+    sec_num -= days * DAY;
-  const hours = Math.floor(sec_num / HOUR);
+    const hours = Math.floor(sec_num / HOUR);
-  sec_num -= hours * HOUR;
+    sec_num -= hours * HOUR;
-  const minutes = Math.floor(sec_num / MINUTE);
+    const minutes = Math.floor(sec_num / MINUTE);
-  sec_num -= minutes * MINUTE;
+    sec_num -= minutes * MINUTE;
-  const seconds = Math.floor(sec_num);
+    const seconds = Math.floor(sec_num);
 
-  if (days > 0) {
+    if (days > 0) {
-    return `${days} days ${hours}h ${minutes}m ${seconds}s`;
+        return `${days} days ${hours}h ${minutes}m ${seconds}s`;
-  }
+    }
 
-  if (hours > 0) {
+    if (hours > 0) {
-    return `${hours}h ${minutes}m ${seconds}s`;
+        return `${hours}h ${minutes}m ${seconds}s`;
-  }
+    }
 
-  if (minutes > 0) {
+    if (minutes > 0) {
-    return `${minutes}m ${seconds}s`;
+        return `${minutes}m ${seconds}s`;
-  }
+    }
 
-  if (seconds > 0) {
+    if (seconds > 0) {
-    return `${seconds}s`;
+        return `${seconds}s`;
-  }
+    }
 
-  return "<0s";
+    return "<0s";
 }
 
 export default {
-  name: 'Tasks',
+    name: 'Tasks',
-  components: {TaskListItem},
+    components: {TaskListItem},
-  data() {
+    data() {
-    return {
+        return {
-      loading: true,
+            loading: true,
-      tasks: [],
+            tasks: [],
-      taskHistory: [],
+            taskHistory: [],
-      timerId: null,
+            timerId: null,
-      historyFields: [
+            historyFields: [
-        {key: "name", label: this.$t("taskName")},
+                {key: "name", label: this.$t("taskName")},
-        {key: "time", label: this.$t("taskStarted")},
+                {key: "time", label: this.$t("taskStarted")},
-        {key: "duration", label: this.$t("taskDuration")},
+                {key: "duration", label: this.$t("taskDuration")},
-        {key: "status", label: this.$t("taskStatus")},
+                {key: "status", label: this.$t("taskStatus")},
-        {key: "logs", label: this.$t("logs")},
+                {key: "logs", label: this.$t("logs")},
-      ],
+            ],
-      historyCurrentPage: 1,
+            historyCurrentPage: 1,
-      historyItems: []
+            historyItems: []
-    }
+        }
-  },
+    },
-  props: {
+    props: {
-    msg: String
+        msg: String
-  },
+    },
-  mounted() {
+    mounted() {
-    this.loading = true;
+        this.loading = true;
-    this.update().then(() => this.loading = false);
+        this.update().then(() => this.loading = false);
 
-    this.timerId = window.setInterval(this.update, 1000);
+        this.timerId = window.setInterval(this.update, 1000);
-    this.updateHistory();
+        this.updateHistory();
-  },
-  destroyed() {
-    if (this.timerId) {
-      window.clearInterval(this.timerId);
-    }
-  },
-  methods: {
-    rowClass(row) {
-      if (row.status === "failed") {
-        return "table-danger";
-      }
-      return null;
     },
-    updateHistory() {
+    destroyed() {
-      Sist2AdminApi.getTaskHistory().then(resp => {
+        if (this.timerId) {
-        this.historyItems = resp.data.map(row => ({
+            window.clearInterval(this.timerId);
-          id: row.id,
+        }
-          name: row.name,
-          duration: this.taskDuration(row),
-          time: moment(row.started).format("dd, MMM Do YYYY, HH:mm:ss"),
-          logs: row.id,
-          status: row.return_code === 0 ? "ok" : "failed"
-        }));
-      });
     },
-    update() {
+    methods: {
-      return Sist2AdminApi.getTasks().then(resp => {
+        rowClass(row) {
-        this.tasks = resp.data;
+            if (row.status === "failed") {
-      })
+                return "table-danger";
-    },
+            }
-    taskDuration(task) {
+            return null;
-      const start = moment.utc(task.started);
+        },
-      const end = moment.utc(task.ended);
+        updateHistory() {
+            Sist2AdminApi.getTaskHistory().then(resp => {
+                this.historyItems = resp.data.map(row => ({
+                    id: row.id,
+                    name: row.name,
+                    duration: this.taskDuration(row),
+                    time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"),
+                    logs: null,
+                    status: [0,1].includes(row.return_code) ? "ok" : "failed",
+                    _row: row
+                }));
+            });
+        },
+        update() {
+            return Sist2AdminApi.getTasks().then(resp => {
+                this.tasks = resp.data;
+            })
+        },
+        taskDuration(task) {
+            const start = moment.utc(task.started);
+            const end = moment.utc(task.ended);
 
-      return humanDuration(end.diff(start))
+            return humanDuration(end.diff(start))
+        },
+        deleteLogs(taskId) {
+            Sist2AdminApi.deleteTaskLogs(taskId).then(() => {
+                this.updateHistory();
+            })
+        }
     }
-  }
 }
 </script>
 
 <style scoped>
 #task-history {
-  font-family: monospace;
+    font-family: monospace;
-  font-size: 12px;
+    font-size: 12px;
+}
+
+.btn-link {
+    padding: 0;
 }
 </style>

sist2-admin/sist2_admin/app.py

@@ -20,8 +20,9 @@ import cron
 from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
 from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
 from notifications import Subscribe, Notifications
-from sist2 import Sist2
+from sist2 import Sist2, Sist2SearchBackend
-from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
+from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION, migrate_v3_to_v4, \
+    get_log_files_to_remove, delete_log_file, create_default_search_backends
 from web import Sist2Frontend
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
@@ -80,9 +81,7 @@ async def get_jobs():
 
 @app.put("/api/job/{name:str}")
 async def update_job(name: str, new_job: Sist2Job):
-    # TODO: Check etag
+    new_job.last_modified = datetime.utcnow()
-
-    new_job.last_modified = datetime.now()
     job = db["jobs"][name]
     if not job:
         raise HTTPException(status_code=404)
@@ -134,8 +133,18 @@ async def kill_job(task_id: str):
     return task_queue.kill_task(task_id)
 
 
+@app.post("/api/task/{task_id:str}/delete_logs")
+async def delete_task_logs(task_id: str):
+    if not db["task_done"][task_id]:
+        raise HTTPException(status_code=404)
+
+    delete_log_file(db, task_id)
+
+    return "ok"
+
+
 def _run_job(job: Sist2Job):
-    job.last_modified = datetime.now()
+    job.last_modified = datetime.utcnow()
     if job.status == JobStatus("created"):
         job.status = JobStatus("started")
     db["jobs"][job.name] = job
@@ -158,14 +167,29 @@ async def run_job(name: str):
     return "ok"
 
 
+@app.get("/api/job/{name:str}/logs_to_delete")
+async def task_history(n: int, name: str):
+    return get_log_files_to_remove(db, name, n)
+
+
 @app.delete("/api/job/{name:str}")
 async def delete_job(name: str):
-    job = db["jobs"][name]
+    job: Sist2Job = db["jobs"][name]
-    if job:
+    if not job:
-        del db["jobs"][name]
-    else:
         raise HTTPException(status_code=404)
 
+    if any(name in frontend.jobs for frontend in db["frontends"]):
+        raise HTTPException(status_code=400, detail="in use (frontend)")
+
+    try:
+        os.remove(job.previous_index)
+    except:
+        pass
+
+    del db["jobs"][name]
+
+    return "ok"
+
 
 @app.delete("/api/frontend/{name:str}")
 async def delete_frontend(name: str):
@@ -251,9 +275,21 @@ def check_es_version(es_url: str, insecure: bool):
 
 
 def start_frontend_(frontend: Sist2Frontend):
-    frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
+    frontend.web_options.indices = [
+        os.path.join(DATA_FOLDER, db["jobs"][j].index_path)
+        for j in frontend.jobs
+    ]
 
-    pid = sist2.web(frontend.web_options, frontend.name)
+    backend_name = frontend.web_options.search_backend
+    search_backend = db["search_backends"][backend_name]
+    if search_backend is None:
+        logger.error(
+            f"Error while running task: search backend not found: {backend_name}")
+        return -1
+
+    logger.debug(f"Fetched search backend options for {backend_name}")
+
+    pid = sist2.web(frontend.web_options, search_backend, frontend.name)
     RUNNING_FRONTENDS[frontend.name] = pid
 
 
@@ -283,6 +319,62 @@ async def get_frontends():
     return res
 
 
+@app.get("/api/search_backend/")
+async def get_search_backends():
+    return list(db["search_backends"])
+
+
+@app.put("/api/search_backend/{name:str}")
+async def update_search_backend(name: str, backend: Sist2SearchBackend):
+    if not db["search_backends"][name]:
+        raise HTTPException(status_code=404)
+
+    db["search_backends"][name] = backend
+    return "ok"
+
+
+@app.get("/api/search_backend/{name:str}")
+def get_search_backend(name: str):
+    backend = db["search_backends"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    return backend
+
+
+@app.delete("/api/search_backend/{name:str}")
+def delete_search_backend(name: str):
+    backend: Sist2SearchBackend = db["search_backends"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    if any(frontend.web_options.search_backend == name for frontend in db["frontends"]):
+        raise HTTPException(status_code=400, detail="in use (frontend)")
+
+    if any(job.index_options.search_backend == name for job in db["jobs"]):
+        raise HTTPException(status_code=400, detail="in use (job)")
+
+    del db["search_backends"][name]
+
+    try:
+        os.remove(os.path.join(DATA_FOLDER, backend.search_index))
+    except:
+        pass
+
+    return "ok"
+
+
+@app.post("/api/search_backend/{name:str}")
+def create_search_backend(name: str):
+    if db["search_backends"][name] is not None:
+        return HTTPException(status_code=400, detail="already exists")
+
+    backend = Sist2SearchBackend.create_default(name)
+    db["search_backends"][name] = backend
+
+    return backend
+
+
 def tail(filepath: str, n: int):
     with open(filepath) as file:
 
@@ -321,7 +413,6 @@ async def ws_tail_log(websocket: WebSocket):
         async with Subscribe(notifications) as ob:
             async for notification in ob.notifications():
                 await websocket.send_json(notification)
-                print(notification)
 
     except ConnectionClosed:
         return
@@ -352,7 +443,7 @@ async def ws_tail_log(websocket: WebSocket, task_id: str, n: int):
 
 
 def main():
-    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0")
+    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0", timeout_graceful_shutdown=0)
 
 
 def initialize_db():
@@ -361,6 +452,8 @@ def initialize_db():
     frontend = Sist2Frontend.create_default("default")
     db["frontends"]["default"] = frontend
 
+    create_default_search_backends(db)
+
     logger.info("Initialized database.")
 
 
@@ -381,6 +474,12 @@ if __name__ == '__main__':
     if db["sist2_admin"]["info"]["version"] == "2":
         logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.")
         exit(-1)
+    if db["sist2_admin"]["info"]["version"] == "3":
+        logger.info("Migrating to v4 database schema")
+        migrate_v3_to_v4(db)
+
+    if db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
+        raise Exception(f"Incompatible database version for {db.dbfile}")
 
     start_frontends()
     cron.initialize(db, _run_job)

sist2-admin/sist2_admin/cron.py

@@ -10,7 +10,9 @@ from jobs import Sist2Job
 
 
 def _check_schedule(db: PersistentState, run_job):
-    for job in db["jobs"]:
+    jobs = list(db["jobs"])
+
+    for job in jobs:
         job: Sist2Job
 
         if job.schedule_enabled:

sist2-admin/sist2_admin/jobs.py

@@ -13,10 +13,10 @@ from uuid import uuid4, UUID
 from hexlib.db import PersistentState
 from pydantic import BaseModel
 
-from config import logger, LOG_FOLDER
+from config import logger, LOG_FOLDER, DATA_FOLDER
 from notifications import Notifications
 from sist2 import ScanOptions, IndexOptions, Sist2
-from state import RUNNING_FRONTENDS
+from state import RUNNING_FRONTENDS, get_log_files_to_remove, delete_log_file
 from web import Sist2Frontend
 
 
@@ -35,6 +35,8 @@ class Sist2Job(BaseModel):
     cron_expression: str
     schedule_enabled: bool = False
 
+    keep_last_n_logs: int = -1
+
     previous_index: str = None
     index_path: str = None
     previous_index_path: str = None
@@ -53,15 +55,10 @@ class Sist2Job(BaseModel):
             name=name,
             scan_options=ScanOptions(path="/"),
             index_options=IndexOptions(),
-            last_modified=datetime.now(),
+            last_modified=datetime.utcnow(),
             cron_expression="0 0 * * *"
         )
 
-    # @validator("etag", always=True)
-    # def validate_etag(cls, value, values):
-    #     s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
-    #     return md5(s.encode()).hexdigest()
-
 
 class Sist2TaskProgress:
 
@@ -111,7 +108,7 @@ class Sist2Task:
             self._logger.info(json.dumps(log_json))
 
     def run(self, sist2: Sist2, db: PersistentState):
-        self.started = datetime.now()
+        self.started = datetime.utcnow()
 
         logger.info(f"Started task {self.display_name}")
 
@@ -132,14 +129,16 @@ class Sist2ScanTask(Sist2Task):
             self.pid = pid
 
         return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
-        self.ended = datetime.now()
+        self.ended = datetime.utcnow()
 
-        if return_code != 0:
+        is_ok = return_code in (0, 1)
+
+        if not is_ok:
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
             logger.info(f"Task {self.display_name} failed ({return_code})")
         else:
             self.job.index_path = self.job.scan_options.output
-            self.job.last_index_date = datetime.now()
+            self.job.last_index_date = datetime.utcnow()
             self.job.do_full_scan = False
             db["jobs"][self.job.name] = self.job
             self._logger.info(json.dumps({"sist2-admin": f"Save last_index_date={self.job.last_index_date}"}))
@@ -147,7 +146,7 @@ class Sist2ScanTask(Sist2Task):
         logger.info(f"Completed {self.display_name} ({return_code=})")
 
         # Remove old index
-        if return_code == 0:
+        if is_ok:
             if self.job.previous_index_path is not None and self.job.previous_index_path != self.job.index_path:
                 self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index_path=}"}))
                 try:
@@ -171,8 +170,15 @@ class Sist2IndexTask(Sist2Task):
 
         self.job.index_options.path = self.job.scan_options.output
 
-        return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
+        search_backend = db["search_backends"][self.job.index_options.search_backend]
-        self.ended = datetime.now()
+        if search_backend is None:
+            logger.error(f"Error while running task: search backend not found: {self.job.index_options.search_backend}")
+            return -1
+
+        logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
+
+        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback)
+        self.ended = datetime.utcnow()
 
         duration = self.ended - self.started
 
@@ -206,9 +212,20 @@ class Sist2IndexTask(Sist2Task):
             except ChildProcessError:
                 pass
 
-            frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
+            backend_name = frontend.web_options.search_backend
+            search_backend = db["search_backends"][backend_name]
+            if search_backend is None:
+                logger.error(f"Error while running task: search backend not found: {backend_name}")
+                return -1
 
-            pid = sist2.web(frontend.web_options, frontend.name)
+            logger.debug(f"Fetched search backend options for {backend_name}")
+
+            frontend.web_options.indices = [
+                os.path.join(DATA_FOLDER, db["jobs"][j].index_path)
+                for j in frontend.jobs
+            ]
+
+            pid = sist2.web(frontend.web_options, search_backend, frontend.name)
             RUNNING_FRONTENDS[frontend_name] = pid
 
             self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))
@@ -232,7 +249,7 @@ class TaskQueue:
     def _tasks_failed(self):
         done = set()
 
-        for row in self._db["task_done"].sql("WHERE return_code != 0"):
+        for row in self._db["task_done"].sql("WHERE return_code NOT IN (0,1)"):
             done.add(uuid.UUID(row["id"]))
 
         return done
@@ -301,8 +318,14 @@ class TaskQueue:
                 "ended": task.ended,
                 "started": task.started,
                 "name": task.display_name,
-                "return_code": task_result
+                "return_code": task_result,
+                "has_logs": 1
             }
+
+            logs_to_delete = get_log_files_to_remove(self._db, task.job.name, task.job.keep_last_n_logs)
+            for row in logs_to_delete:
+                delete_log_file(self._db, row["id"])
+
         if isinstance(task, Sist2IndexTask):
             self._notifications.notify({
                 "message": "notifications.indexCompleted",

sist2-admin/sist2_admin/sist2.py

@@ -3,6 +3,7 @@ import json
 import logging
 import os.path
 from datetime import datetime
+from enum import Enum
 from io import TextIOWrapper
 from logging import FileHandler
 from subprocess import Popen, PIPE
@@ -12,7 +13,7 @@ from typing import List
 
 from pydantic import BaseModel
 
-from config import logger, LOG_FOLDER
+from config import logger, LOG_FOLDER, DATA_FOLDER
 
 
 class Sist2Version:
@@ -25,77 +26,60 @@ class Sist2Version:
         return f"{self.major}.{self.minor}.{self.patch}"
 
 
-class WebOptions(BaseModel):
+class SearchBackendType(Enum):
-    indices: List[str] = []
+    SQLITE = "sqlite"
+    ELASTICSEARCH = "elasticsearch"
+
+
+class Sist2SearchBackend(BaseModel):
+    backend_type: SearchBackendType = SearchBackendType("elasticsearch")
+    name: str
+
+    search_index: str = ""
+
     es_url: str = "http://elasticsearch:9200"
     es_insecure_ssl: bool = False
     es_index: str = "sist2"
-    bind: str = "0.0.0.0:4090"
-    auth: str = None
-    tag_auth: str = None
-    tagline: str = "Lightning-fast file system indexer and search tool"
-    dev: bool = False
-    lang: str = "en"
-    auth0_audience: str = None
-    auth0_domain: str = None
-    auth0_client_id: str = None
-    auth0_public_key: str = None
-    auth0_public_key_file: str = None
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def args(self):
-        args = ["web", f"--es-url={self.es_url}", f"--es-index={self.es_index}", f"--bind={self.bind}",
-                f"--tagline={self.tagline}", f"--lang={self.lang}"]
-
-        if self.auth0_audience:
-            args.append(f"--auth0-audience={self.auth0_audience}")
-        if self.auth0_domain:
-            args.append(f"--auth0-domain={self.auth0_domain}")
-        if self.auth0_client_id:
-            args.append(f"--auth0-client-id={self.auth0_client_id}")
-        if self.auth0_public_key_file:
-            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
-        if self.es_insecure_ssl:
-            args.append(f"--es-insecure-ssl")
-        if self.auth:
-            args.append(f"--auth={self.auth}")
-        if self.tag_auth:
-            args.append(f"--tag-auth={self.tag_auth}")
-        if self.dev:
-            args.append(f"--dev")
-
-        args.extend(self.indices)
-
-        return args
-
-
-class IndexOptions(BaseModel):
-    path: str = None
     threads: int = 1
-    es_url: str = "http://elasticsearch:9200"
-    es_insecure_ssl: bool = False
-    es_index: str = "sist2"
-    incremental_index: bool = True
     script: str = ""
     script_file: str = None
     batch_size: int = 70
 
+    @staticmethod
+    def create_default(name: str, backend_type: SearchBackendType = SearchBackendType("elasticsearch")):
+        return Sist2SearchBackend(
+            name=name,
+            search_index=f"search-index-{name.replace('/', '_')}.sist2",
+            backend_type=backend_type
+        )
+
+
+class IndexOptions(BaseModel):
+    path: str = None
+    incremental_index: bool = True
+    search_backend: str = None
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def args(self):
+    def args(self, search_backend):
+        absolute_path = os.path.join(DATA_FOLDER, self.path)
 
-        args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
+        if search_backend.backend_type == SearchBackendType("sqlite"):
-                f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
+            search_index_absolute = os.path.join(DATA_FOLDER, search_backend.search_index)
+            args = ["sqlite-index", absolute_path, "--search-index", search_index_absolute]
+        else:
+            args = ["index", absolute_path, f"--threads={search_backend.threads}",
+                    f"--es-url={search_backend.es_url}",
+                    f"--es-index={search_backend.es_index}",
+                    f"--batch-size={search_backend.batch_size}"]
 
-        if self.script_file:
+            if search_backend.script_file:
-            args.append(f"--script-file={self.script_file}")
+                args.append(f"--script-file={search_backend.script_file}")
-        if self.es_insecure_ssl:
+            if search_backend.es_insecure_ssl:
-            args.append(f"--es-insecure-ssl")
+                args.append(f"--es-insecure-ssl")
-        if self.incremental_index:
+            if self.incremental_index:
-            args.append(f"--incremental-index")
+                args.append(f"--incremental-index")
 
         return args
 
@@ -109,7 +93,7 @@ ARCHIVE_RECURSE = "recurse"
 class ScanOptions(BaseModel):
     path: str
     threads: int = 1
-    thumbnail_quality: int = 2
+    thumbnail_quality: int = 50
     thumbnail_size: int = 552
     thumbnail_count: int = 1
     content_size: int = 32768
@@ -137,9 +121,12 @@ class ScanOptions(BaseModel):
         super().__init__(**kwargs)
 
     def args(self):
+
+        output_path = os.path.join(DATA_FOLDER, self.output)
+
         args = ["scan", self.path, f"--threads={self.threads}", f"--thumbnail-quality={self.thumbnail_quality}",
                 f"--thumbnail-count={self.thumbnail_count}", f"--thumbnail-size={self.thumbnail_size}",
-                f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
+                f"--content-size={self.content_size}", f"--output={output_path}", f"--depth={self.depth}",
                 f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
 
         if self.incremental:
@@ -201,27 +188,82 @@ class Sist2Index:
         return self._descriptor["name"]
 
 
+class WebOptions(BaseModel):
+    indices: List[str] = []
+
+    search_backend: str = "elasticsearch"
+
+    bind: str = "0.0.0.0:4090"
+    auth: str = None
+    tag_auth: str = None
+    tagline: str = "Lightning-fast file system indexer and search tool"
+    dev: bool = False
+    lang: str = "en"
+    auth0_audience: str = None
+    auth0_domain: str = None
+    auth0_client_id: str = None
+    auth0_public_key: str = None
+    auth0_public_key_file: str = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self, search_backend: Sist2SearchBackend):
+        args = ["web", f"--bind={self.bind}", f"--tagline={self.tagline}",
+                f"--lang={self.lang}"]
+
+        if search_backend.backend_type == SearchBackendType("sqlite"):
+            search_index_absolute = os.path.join(DATA_FOLDER, search_backend.search_index)
+            args.append(f"--search-index={search_index_absolute}")
+        else:
+            args.append(f"--es-url={search_backend.es_url}")
+            args.append(f"--es-index={search_backend.es_index}")
+            if search_backend.es_insecure_ssl:
+                args.append(f"--es-insecure-ssl")
+
+        if self.auth0_audience:
+            args.append(f"--auth0-audience={self.auth0_audience}")
+        if self.auth0_domain:
+            args.append(f"--auth0-domain={self.auth0_domain}")
+        if self.auth0_client_id:
+            args.append(f"--auth0-client-id={self.auth0_client_id}")
+        if self.auth0_public_key_file:
+            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
+        if self.auth:
+            args.append(f"--auth={self.auth}")
+        if self.tag_auth:
+            args.append(f"--tag-auth={self.tag_auth}")
+        if self.dev:
+            args.append(f"--dev")
+
+        args.extend(self.indices)
+
+        return args
+
+
 class Sist2:
 
     def __init__(self, bin_path: str, data_directory: str):
-        self._bin_path = bin_path
+        self.bin_path = bin_path
         self._data_dir = data_directory
 
-    def index(self, options: IndexOptions, logs_cb):
+    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
 
-        if options.script:
+        if search_backend.script and search_backend.backend_type == SearchBackendType("elasticsearch"):
             with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
-                f.write(options.script)
+                f.write(search_backend.script)
-            options.script_file = f.name
+            search_backend.script_file = f.name
         else:
-            options.script_file = None
+            search_backend.script_file = None
 
         args = [
-            self._bin_path,
+            self.bin_path,
-            *options.args(),
+            *options.args(search_backend),
             "--json-logs",
             "--very-verbose"
         ]
+
+        logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
         t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
@@ -236,13 +278,10 @@ class Sist2:
     def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
 
         if options.output is None:
-            options.output = os.path.join(
+            options.output = f"scan-{options.name.replace('/', '_')}-{datetime.utcnow()}.sist2"
-                self._data_dir,
-                f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2"
-            )
 
         args = [
-            self._bin_path,
+            self.bin_path,
             *options.args(),
             "--json-logs",
             "--very-verbose"
@@ -290,7 +329,7 @@ class Sist2:
                 except NameError:
                     pass
 
-    def web(self, options: WebOptions, name: str):
+    def web(self, options: WebOptions, search_backend: Sist2SearchBackend, name: str):
 
         if options.auth0_public_key:
             with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
@@ -300,8 +339,8 @@ class Sist2:
             options.auth0_public_key_file = None
 
         args = [
-            self._bin_path,
+            self.bin_path,
-            *options.args()
+            *options.args(search_backend)
         ]
 
         web_logger = logging.Logger(name=f"sist2-frontend-{name}")

sist2-admin/sist2_admin/state.py

@@ -1,16 +1,20 @@
 from typing import Dict
+import os
 import shutil
 
 from hexlib.db import Table, PersistentState
 import pickle
 
 from tesseract import get_tesseract_langs
+import sqlite3
+from config import LOG_FOLDER, logger
+from sist2 import SearchBackendType, Sist2SearchBackend
 
 RUNNING_FRONTENDS: Dict[str, int] = {}
 
 TESSERACT_LANGS = get_tesseract_langs()
 
-DB_SCHEMA_VERSION = "3"
+DB_SCHEMA_VERSION = "4"
 
 from pydantic import BaseModel
 
@@ -50,8 +54,35 @@ class PickleTable(Table):
             yield dict((k, _deserialize(v)) for k, v in row.items())
 
 
-def migrate_v1_to_v2(db: PersistentState):
+def get_log_files_to_remove(db: PersistentState, job_name: str, n: int):
+    if n < 0:
+        return []
 
+    counter = 0
+    to_remove = []
+
+    for row in db["task_done"].sql("WHERE has_logs=1 ORDER BY started DESC"):
+        if row["name"].endswith(f"[{job_name}]"):
+            counter += 1
+
+            if counter > n:
+                to_remove.append(row)
+
+    return to_remove
+
+
+def delete_log_file(db: PersistentState, task_id: str):
+    db["task_done"][task_id] = {
+        "has_logs": 0
+    }
+
+    try:
+        os.remove(os.path.join(LOG_FOLDER, f"sist2-{task_id}.log"))
+    except:
+        pass
+
+
+def migrate_v1_to_v2(db: PersistentState):
     shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v2.bak")
 
     # Frontends
@@ -77,3 +108,29 @@ def migrate_v1_to_v2(db: PersistentState):
     db["sist2_admin"]["info"] = {
         "version": "2"
     }
+
+
+def create_default_search_backends(db: PersistentState):
+    es_backend = Sist2SearchBackend.create_default(name="elasticsearch",
+                                                   backend_type=SearchBackendType("elasticsearch"))
+    db["search_backends"]["elasticsearch"] = es_backend
+    sqlite_backend = Sist2SearchBackend.create_default(name="sqlite", backend_type=SearchBackendType("sqlite"))
+    db["search_backends"]["sqlite"] = sqlite_backend
+
+
+def migrate_v3_to_v4(db: PersistentState):
+    shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v4.bak")
+
+    create_default_search_backends(db)
+
+    try:
+        conn = sqlite3.connect(db.dbfile)
+        conn.execute("ALTER TABLE task_done ADD COLUMN has_logs INTEGER DEFAULT 1")
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        logger.exception(e)
+
+    db["sist2_admin"]["info"] = {
+        "version": "4"
+    }

sist2-vue/package.json

@@ -13,7 +13,7 @@
     "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "d3": "^7.8.4",
+    "d3": "^5.6.1",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",
     "fslightbox-vue": "fslightbox-vue.tgz",

sist2-vue/public/index.html

@@ -19,12 +19,6 @@
             We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled.
             Please enable it to continue.
         </strong>
-        <br/>
-        <strong>
-            Nous sommes désolés mais <%= htmlWebpackPlugin.options.title %> ne fonctionne pas correctement
-            si JavaScript est activé.
-            Veuillez l'activer pour continuer.
-        </strong>
     </div>
 </noscript>
 <div id="app"></div>

sist2-vue/src/App.vue

@@ -10,7 +10,7 @@
             <b-spinner type="grow" variant="primary"></b-spinner>
         </div>
         <div class="loading-text">
-            Loading • Chargement • 装载 • Wird geladen
+            Loading • Chargement • 装载 • Wird geladen • Ładowanie
         </div>
     </div>
 </template>

sist2-vue/src/Sist2Api.ts

@@ -531,8 +531,8 @@ class Sist2Api {
             size: 0
         }).then(res => {
             const range = {
-                min: res.aggregations.dateMin.value,
+                min: res.aggregations.dateMin.value / 1000,
-                max: res.aggregations.dateMax.value,
+                max: res.aggregations.dateMax.value / 1000,
             }
 
             if (range.min == null) {

sist2-vue/src/Sist2ElasticsearchQuery.ts

@@ -170,14 +170,16 @@ class Sist2ElasticsearchQuery {
                 }
             },
             sort: SORT_MODES[getters.sortMode].mode,
-            aggs:
-                {
-                    total_size: {"sum": {"field": "size"}},
-                    total_count: {"value_count": {"field": "size"}}
-                },
             size: size,
         } as any;
 
+        if (!after) {
+            q.aggs = {
+                total_size: {"sum": {"field": "size"}},
+                total_count: {"value_count": {"field": "size"}}
+            };
+        }
+
         if (!empty && !blankSearch) {
             q.query.bool.must = query;
         }

sist2-vue/src/components/D3Treemap.vue

@@ -1,10 +1,10 @@
 <template>
-  <div>
+    <div>
-    <b-btn style="float:right;margin-bottom: 10px" @click="downloadTreemap()" variant="primary">
+        <b-btn style="float:right;margin-bottom: 10px" @click="downloadTreemap()" variant="primary">
-      {{ $t("download") }}
+            {{ $t("download") }}
-    </b-btn>
+        </b-btn>
-    <svg id="treemap"></svg>
+        <svg id="treemap"></svg>
-  </div>
+    </div>
 </template>
 
 <script>
@@ -16,252 +16,252 @@ import domtoimage from "dom-to-image";
 
 
 const TILING_MODES = {
-  "squarify": d3.treemapSquarify,
+    "squarify": d3.treemapSquarify,
-  "binary": d3.treemapBinary,
+    "binary": d3.treemapBinary,
-  "sliceDice": d3.treemapSliceDice,
+    "sliceDice": d3.treemapSliceDice,
-  "slice": d3.treemapSlice,
+    "slice": d3.treemapSlice,
-  "dice": d3.treemapDice,
+    "dice": d3.treemapDice,
 };
 
 const COLORS = {
-  "PuBuGn": d3.interpolatePuBuGn,
+    "PuBuGn": d3.interpolatePuBuGn,
-  "PuRd": d3.interpolatePuRd,
+    "PuRd": d3.interpolatePuRd,
-  "PuBu": d3.interpolatePuBu,
+    "PuBu": d3.interpolatePuBu,
-  "YlOrBr": d3.interpolateYlOrBr,
+    "YlOrBr": d3.interpolateYlOrBr,
-  "YlOrRd": d3.interpolateYlOrRd,
+    "YlOrRd": d3.interpolateYlOrRd,
-  "YlGn": d3.interpolateYlGn,
+    "YlGn": d3.interpolateYlGn,
-  "YlGnBu": d3.interpolateYlGnBu,
+    "YlGnBu": d3.interpolateYlGnBu,
-  "Plasma": d3.interpolatePlasma,
+    "Plasma": d3.interpolatePlasma,
-  "Magma": d3.interpolateMagma,
+    "Magma": d3.interpolateMagma,
-  "Inferno": d3.interpolateInferno,
+    "Inferno": d3.interpolateInferno,
-  "Viridis": d3.interpolateViridis,
+    "Viridis": d3.interpolateViridis,
-  "Turbo": d3.interpolateTurbo,
+    "Turbo": d3.interpolateTurbo,
 };
 
 const SIZES = {
-  "small": [800, 600],
+    "small": [800, 600],
-  "medium": [1300, 750],
+    "medium": [1300, 750],
-  "large": [1900, 900],
+    "large": [1900, 900],
-  "x-large": [2800, 1700],
+    "x-large": [2800, 1700],
-  "xx-large": [3600, 2000],
+    "xx-large": [3600, 2000],
 };
 
 
 const uids = {};
 
 function uid(name) {
-  let id = uids[name] || 0;
+    let id = uids[name] || 0;
-  uids[name] = id + 1;
+    uids[name] = id + 1;
-  return name + id;
+    return name + id;
 }
 
 function cascade(root, offset) {
-  const x = new Map;
+    const x = new Map;
-  const y = new Map;
+    const y = new Map;
-  return root.eachAfter(d => {
+    return root.eachAfter(d => {
-    if (d.children && d.children.length !== 0) {
+        if (d.children && d.children.length !== 0) {
-      x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
+            x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
-      y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
+            y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
-    } else {
+        } else {
-      x.set(d, 0);
+            x.set(d, 0);
-      y.set(d, 0);
+            y.set(d, 0);
-    }
+        }
-  }).eachBefore(d => {
+    }).eachBefore(d => {
-    d.x1 -= 2 * offset * x.get(d);
+        d.x1 -= 2 * offset * x.get(d);
-    d.y1 -= 2 * offset * y.get(d);
+        d.y1 -= 2 * offset * y.get(d);
-  });
+    });
 }
 
 function cascadeTreemap(data, svg, width, height, tilingMode, treemapColor) {
-  const root = cascade(
+    const root = cascade(
-      d3.treemap()
+        d3.treemap()
-          .size([width, height])
+            .size([width, height])
-          .tile(TILING_MODES[tilingMode])
+            .tile(TILING_MODES[tilingMode])
-          .paddingOuter(3)
+            .paddingOuter(3)
-          .paddingTop(16)
+            .paddingTop(16)
-          .paddingInner(1)
+            .paddingInner(1)
-          .round(true)(
+            .round(true)(
-              d3.hierarchy(data)
+                d3.hierarchy(data)
-                  .sum(d => d.value)
+                    .sum(d => d.value)
-                  .sort((a, b) => b.value - a.value)
+                    .sort((a, b) => b.value - a.value)
-          ),
+            ),
-      3 // treemap.paddingOuter
+        3 // treemap.paddingOuter
-  );
+    );
 
-  const maxDepth = Math.max(...root.descendants().map(d => d.depth));
+    const maxDepth = Math.max(...root.descendants().map(d => d.depth));
-  const color = d3.scaleSequential([maxDepth, -1], COLORS[treemapColor]);
+    const color = d3.scaleSequential([maxDepth, -1], COLORS[treemapColor]);
 
-  svg.append("filter")
+    svg.append("filter")
-      .attr("id", "shadow")
+        .attr("id", "shadow")
-      .append("feDropShadow")
+        .append("feDropShadow")
-      .attr("flood-opacity", 0.3)
+        .attr("flood-opacity", 0.3)
-      .attr("dx", 0)
+        .attr("dx", 0)
-      .attr("stdDeviation", 3);
+        .attr("stdDeviation", 3);
 
-  const node = svg.selectAll("g")
+    const node = svg.selectAll("g")
-      .data(
+        .data(
-          d3.nest()
+            d3.nest()
-              .key(d => d.depth).sortKeys(d3.ascending)
+                .key(d => d.depth).sortKeys(d3.ascending)
-              .entries(root.descendants())
+                .entries(root.descendants())
-      )
+        )
-      .join("g")
+        .join("g")
-      .attr("filter", "url(#shadow)")
+        .attr("filter", "url(#shadow)")
-      .selectAll("g")
+        .selectAll("g")
-      .data(d => d.values)
+        .data(d => d.values)
-      .join("g")
+        .join("g")
-      .attr("transform", d => `translate(${d.x0},${d.y0})`);
+        .attr("transform", d => `translate(${d.x0},${d.y0})`);
 
-  node.append("title")
+    node.append("title")
-      .text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+        .text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
 
-  node.append("rect")
+    node.append("rect")
-      .attr("id", d => (d.nodeUid = uid("node")))
+        .attr("id", d => (d.nodeUid = uid("node")))
-      .attr("fill", d => color(d.depth))
+        .attr("fill", d => color(d.depth))
-      .attr("width", d => d.x1 - d.x0)
+        .attr("width", d => d.x1 - d.x0)
-      .attr("height", d => d.y1 - d.y0);
+        .attr("height", d => d.y1 - d.y0);
 
-  node.append("clipPath")
+    node.append("clipPath")
-      .attr("id", d => (d.clipUid = uid("clip")))
+        .attr("id", d => (d.clipUid = uid("clip")))
-      .append("use")
+        .append("use")
-      .attr("href", d => `#${d.nodeUid}`);
+        .attr("href", d => `#${d.nodeUid}`);
 
-  node.append("text")
+    node.append("text")
-      .attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
+        .attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
-      .attr("clip-path", d => `url(#${d.clipUid})`)
+        .attr("clip-path", d => `url(#${d.clipUid})`)
-      .selectAll("tspan")
+        .selectAll("tspan")
-      .data(d => [d.data.name, humanFileSize(d.value)])
+        .data(d => [d.data.name, humanFileSize(d.value)])
-      .join("tspan")
+        .join("tspan")
-      .text(d => d);
+        .text(d => d);
 
-  node.filter(d => d.children).selectAll("tspan")
+    node.filter(d => d.children).selectAll("tspan")
-      .attr("dx", 3)
+        .attr("dx", 3)
-      .attr("y", 13);
+        .attr("y", 13);
 
-  node.filter(d => !d.children).selectAll("tspan")
+    node.filter(d => !d.children).selectAll("tspan")
-      .attr("x", 3)
+        .attr("x", 3)
-      .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`);
+        .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`);
 }
 
 function flatTreemap(data, svg, width, height, groupingDepth, tilingMode, fillOpacity) {
-  const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
+    const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
 
-  const root = d3.treemap()
+    const root = d3.treemap()
-      .tile(TILING_MODES[tilingMode])
+        .tile(TILING_MODES[tilingMode])
-      .size([width, height])
+        .size([width, height])
-      .padding(1)
+        .padding(1)
-      .round(true)(
+        .round(true)(
-          d3.hierarchy(data)
+            d3.hierarchy(data)
-              .sum(d => d.value)
+                .sum(d => d.value)
-              .sort((a, b) => b.value - a.value)
+                .sort((a, b) => b.value - a.value)
-      );
+        );
 
-  const leaf = svg.selectAll("g")
+    const leaf = svg.selectAll("g")
-      .data(root.leaves())
+        .data(root.leaves())
-      .join("g")
+        .join("g")
-      .attr("transform", d => `translate(${d.x0},${d.y0})`);
+        .attr("transform", d => `translate(${d.x0},${d.y0})`);
 
-  leaf.append("title")
+    leaf.append("title")
-      .text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+        .text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
 
-  leaf.append("rect")
+    leaf.append("rect")
-      .attr("id", d => (d.leafUid = uid("leaf")))
+        .attr("id", d => (d.leafUid = uid("leaf")))
-      .attr("fill", d => {
+        .attr("fill", d => {
-        while (d.depth > groupingDepth) d = d.parent;
+            while (d.depth > groupingDepth) d = d.parent;
-        return ordinalColor(d.data.name);
+            return ordinalColor(d.data.name);
-      })
+        })
-      .attr("fill-opacity", fillOpacity)
+        .attr("fill-opacity", fillOpacity)
-      .attr("width", d => d.x1 - d.x0)
+        .attr("width", d => d.x1 - d.x0)
-      .attr("height", d => d.y1 - d.y0);
+        .attr("height", d => d.y1 - d.y0);
 
-  leaf.append("clipPath")
+    leaf.append("clipPath")
-      .attr("id", d => (d.clipUid = uid("clip")))
+        .attr("id", d => (d.clipUid = uid("clip")))
-      .append("use")
+        .append("use")
-      .attr("href", d => `#${d.leafUid}`);
+        .attr("href", d => `#${d.leafUid}`);
 
-  leaf.append("text")
+    leaf.append("text")
-      .attr("clip-path", d => `url(#${d.clipUid})`)
+        .attr("clip-path", d => `url(#${d.clipUid})`)
-      .selectAll("tspan")
+        .selectAll("tspan")
-      .data(d => {
+        .data(d => {
-        if (d.data.name === ".") {
+            if (d.data.name === ".") {
-          d = d.parent;
+                d = d.parent;
-        }
+            }
-        return [d.data.name, humanFileSize(d.value)]
+            return [d.data.name, humanFileSize(d.value)]
-      })
+        })
-      .join("tspan")
+        .join("tspan")
-      .attr("x", 2)
+        .attr("x", 2)
-      .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`)
+        .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`)
-      .text(d => d);
+        .text(d => d);
 }
 
 function exportTreemap(indexName, width, height) {
-  domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
+    domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
-      .then(function (blob) {
+        .then(function (blob) {
-        let a = document.createElement("a");
+            let a = document.createElement("a");
-        let url = URL.createObjectURL(blob);
+            let url = URL.createObjectURL(blob);
 
-        a.href = url;
+            a.href = url;
-        a.download = `${indexName}_treemap.png`;
+            a.download = `${indexName}_treemap.png`;
-        document.body.appendChild(a);
+            document.body.appendChild(a);
-        a.click();
+            a.click();
-        setTimeout(function () {
+            setTimeout(function () {
-          document.body.removeChild(a);
+                document.body.removeChild(a);
-          window.URL.revokeObjectURL(url);
+                window.URL.revokeObjectURL(url);
-        }, 0);
+            }, 0);
-      });
+        });
 }
 
 export default {
-  name: "D3Treemap",
+    name: "D3Treemap",
-  props: ["indexId"],
+    props: ["indexId"],
-  watch: {
+    watch: {
-    indexId: function () {
+        indexId: function () {
-      this.update(this.indexId);
+            this.update(this.indexId);
-    }
-  },
-  mounted() {
-    this.update(this.indexId);
-  },
-  methods: {
-    update(indexId) {
-      const width = SIZES[this.$store.state.optTreemapSize][0];
-      const height = SIZES[this.$store.state.optTreemapSize][1];
-      const tilingMode = this.$store.state.optTreemapTiling;
-      const groupingDepth = this.$store.state.optTreemapColorGroupingDepth;
-      const treemapColor = this.$store.state.optTreemapColor;
-      const treemapType = this.$store.state.optTreemapType;
-
-      const treemapSvg = d3.select("#treemap");
-
-      treemapSvg.selectAll("*").remove();
-      treemapSvg.attr("viewBox", [0, 0, width, height])
-          .attr("xmlns", "http://www.w3.org/2000/svg")
-          .attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
-          .attr("version", "1.1")
-          .style("overflow", "visible")
-          .style("font", "10px sans-serif");
-
-      d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
-        tabularData.forEach(row => {
-          row.taxonomy = row.path.split("/");
-          row.size = Number(row.size);
-        });
-
-        if (treemapType === "cascaded") {
-          const data = burrow(tabularData, false);
-          cascadeTreemap(data, treemapSvg, width, height, tilingMode, treemapColor);
-        } else {
-          const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
-          const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
-          flatTreemap(data, treemapSvg, width, height, groupingDepth, tilingMode, fillOpacity);
         }
-      });
     },
-    downloadTreemap() {
+    mounted() {
-      const width = SIZES[this.$store.state.optTreemapSize][0];
+        this.update(this.indexId);
-      const height = SIZES[this.$store.state.optTreemapSize][1];
+    },
+    methods: {
+        update(indexId) {
+            const width = SIZES[this.$store.state.optTreemapSize][0];
+            const height = SIZES[this.$store.state.optTreemapSize][1];
+            const tilingMode = this.$store.state.optTreemapTiling;
+            const groupingDepth = this.$store.state.optTreemapColorGroupingDepth;
+            const treemapColor = this.$store.state.optTreemapColor;
+            const treemapType = this.$store.state.optTreemapType;
 
-      exportTreemap(this.indexId, width, height);
+            const treemapSvg = d3.select("#treemap");
+
+            treemapSvg.selectAll("*").remove();
+            treemapSvg.attr("viewBox", [0, 0, width, height])
+                .attr("xmlns", "http://www.w3.org/2000/svg")
+                .attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
+                .attr("version", "1.1")
+                .style("overflow", "visible")
+                .style("font", "10px sans-serif");
+
+            d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
+                tabularData.forEach(row => {
+                    row.taxonomy = row.path.split("/");
+                    row.size = Number(row.size);
+                });
+
+                if (treemapType === "cascaded") {
+                    const data = burrow(tabularData, false);
+                    cascadeTreemap(data, treemapSvg, width, height, tilingMode, treemapColor);
+                } else {
+                    const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
+                    const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
+                    flatTreemap(data, treemapSvg, width, height, groupingDepth, tilingMode, fillOpacity);
+                }
+            });
+        },
+        downloadTreemap() {
+            const width = SIZES[this.$store.state.optTreemapSize][0];
+            const height = SIZES[this.$store.state.optTreemapSize][1];
+
+            exportTreemap(this.indexId, width, height);
+        }
     }
-  }
 }
 </script>

sist2-vue/src/i18n/messages.ts

@@ -1,7 +1,7 @@
 export default {
     en: {
         filePage: {
-          notFound: "Not found"
+            notFound: "Not found"
         },
         searchBar: {
             simple: "Search",
@@ -92,6 +92,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -184,7 +185,7 @@ export default {
     },
     de: {
         filePage: {
-          notFound: "Nicht gefunden"
+            notFound: "Nicht gefunden"
         },
         searchBar: {
             simple: "Suche",
@@ -271,6 +272,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -445,6 +447,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -619,6 +622,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -703,4 +707,188 @@ export default {
             selectedIndices: "选中索引",
         },
     },
+    pl: {
+        filePage: {
+            notFound: "Nie znaleziono"
+        },
+        searchBar: {
+            simple: "Szukaj",
+            advanced: "Zaawansowane szukanie",
+            fuzzy: "Również podobne"
+        },
+        addTag: "Tag",
+        deleteTag: "Usuń",
+        download: "Pobierz",
+        and: "i",
+        page: "strona",
+        pages: "stron",
+        mimeTypes: "Typy danych",
+        tags: "Tagi",
+        tagFilter: "Filtruj tagi",
+        forExample: "Na przykład:",
+        help: {
+            simpleSearch: "Proste szukanie",
+            advancedSearch: "Zaawansowane szukanie",
+            help: "Pomoc",
+            term: "<WYRAZ>",
+            and: "operator I",
+            or: "operator LUB",
+            not: "zabrania danego wyrazu",
+            quotes: "znajdzie objętą sekwencję wyrazów w podanej kolejności",
+            prefix: "znajdzie dowolny wyraz rozpoczynający się na takie litery, jeśli zastosowane na końcu wyrazu",
+            parens: "używane do grupowania wyrażeń",
+            tildeTerm: "znajdzie wyraz w podanej odległości",
+            tildePhrase: "znajdzie frazę przeplecioną podaną liczbą niepasujących wyrazów",
+            example1:
+                "Na przykład: <code>\"pieczone jajko\" +(kiełbasa | ziemniak) -frytki</code> znajdzie frazę " +
+                "<i>pieczone jajko</i> gdzie występuje też: <i>kiełbasa</i> albo <i>ziemniak</i>, ale zignoruje rezultat " +
+                "zawierający <i>frytki</i>.",
+            defaultOperator:
+                "Kiedy nie podano ani <code>+</code>, ani <code>|</code>, to domyślnym operatorem jest " +
+                "<code>+</code> (i).",
+            fuzzy:
+                "Kiedy opcja <b>Również podobne</b> jest zaznaczona, częściowo zgodne wyrazy są również znajdywane.",
+            moreInfoSimple: "Po więcej informacji sięgnij do <a target=\"_blank\" " +
+                "rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">dokumentacji Elasticsearch</a>",
+            moreInfoAdvanced: "Aby uzyskać więcej informacji o zaawansowanym szukaniu, przeczytaj <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">dokumentację Elasticsearch</a>"
+        },
+        config: "Ustawienia",
+        configDescription: "Ustawienia są zapisywane na żywo w tej przeglądarce.",
+        configReset: "Zresetuj ustawienia",
+        searchOptions: "Opcje szukania",
+        treemapOptions: "Opcje mapy",
+        mlOptions: "Opcje uczenia maszynowego",
+        displayOptions: "Opcje wyświetlania",
+        opt: {
+            lang: "Język",
+            highlight: "Zaznaczaj znalezione fragmenty",
+            fuzzy: "Ustaw szukanie również podobnych jako domyślne",
+            searchInPath: "Włącz szukanie również w ścieżce dokumentu",
+            suggestPath: "Włącz auto-uzupełnianie w filtrze ścieżek",
+            fragmentSize: "Podświetl wielkość kontekstu w znakach",
+            queryMode: "Tryb szukania",
+            displayMode: "Wyświetlanie",
+            columns: "Liczba kolumn",
+            treemapType: "Typ mapy",
+            treemapTiling: "Układanie mapy",
+            treemapColorGroupingDepth: "Jak głęboko grupować kolory mapy (na płasko)",
+            treemapColor: "Kolor mapy (kaskadowo)",
+            treemapSize: "Wielkość mapy",
+            theme: "Styl graficzny",
+            lightboxLoadOnlyCurrent: "Nie pobieraj od razu obrazów w pełnej wielkości dla sąsiednich obrazów podglądu.",
+            slideDuration: "Czas trwania jednego slajdu w pokazie slajdów",
+            resultSize: "Liczba wyników na stronę",
+            tagOrOperator: "Użyj operatora LUB przy wyborze kilku tagów",
+            hideDuplicates: "Ukryj zduplikowane wyniki (według sumy kontrolnej)",
+            hideLegacy: "Ukryj powiadomienie Elasticsearch 'legacyES'",
+            updateMimeMap: "Uaktualniaj drzewo typów mediów na żywo",
+            useDatePicker: "Używaj kalendarza do wyboru dat, zamiast suwaka",
+            vidPreviewInterval: "Czas trwania jednej klatki w podglądzie wideo (w ms)",
+            simpleLightbox: "Wyłącz animacje w podglądzie obrazów",
+            showTagPickerFilter: "Pokazuj pole filtrowania tagów",
+            featuredFields: "Wybrane pola szablonu Javascript. Będą pojawiać się przy wynikach wyszukiwania.",
+            featuredFieldsList: "Dostępne zmienne",
+            autoAnalyze: "Automatycznie analizuj tekst",
+            defaultModel: "Domyślny model",
+            mlRepositories: "Repozytoria modeli (każde w osobnej linii)"
+        },
+        queryMode: {
+            simple: "Proste",
+            advanced: "Zaawansowane",
+        },
+        lang: {
+            en: "English",
+            de: "Deutsch",
+            fr: "Français",
+            pl: "Polski",
+            "zh-CN": "简体中文",
+        },
+        displayMode: {
+            grid: "Siatka",
+            list: "Lista",
+        },
+        columns: {
+            auto: "Automatyczna"
+        },
+        treemapType: {
+            cascaded: "Kaskadowa",
+            flat: "Płaska (kompaktowa)"
+        },
+        treemapSize: {
+            small: "Mała",
+            medium: "Średnia",
+            large: "Duża",
+            xLarge: "Bardzo duża",
+            xxLarge: "Ogromna",
+            custom: "Inna",
+        },
+        treemapTiling: {
+            binary: "Binarnie",
+            squarify: "Kwadratowo",
+            slice: "Wycinek",
+            dice: "Kostka",
+            sliceDice: "Wycinek i kostka",
+        },
+        theme: {
+            light: "Jasny",
+            black: "Czarny"
+        },
+        hit: "traf",
+        hits: "trafień",
+        details: "Szczegóły",
+        stats: "Statystyki",
+        queryTime: "Czas szukania",
+        totalSize: "Całkowita wielkość",
+        pathBar: {
+            placeholder: "Filtruj ścieżki",
+            modalTitle: "Wybierz ścieżkę"
+        },
+        debug: "Informacje dla programistów",
+        debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
+            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
+        tagline: "Slogan",
+        toast: {
+            esConnErrTitle: "Problem z połączeniem z Elasticsearch",
+            esConnErr: "Moduł strony internetowej sist2 napotkał problem przy połączeniu z Elasticsearch." +
+                " Zobacz logi serwera, aby uzyskać więcej informacji.",
+            esQueryErrTitle: "Problem z kwerendą",
+            esQueryErr: "Kwerenda szukania jest niezrozumiała albo nie udało się jej przesłać. Sprawdź dokumentację zaawansowanego szukania. " +
+                "Zobacz logi serwera, aby uzyskać więcej informacji.",
+            dupeTagTitle: "Zduplikowany tag",
+            dupeTag: "Ten dokument już ma taki tag.",
+            copiedToClipboard: "Skopiowano do schowka"
+        },
+        saveTagModalTitle: "Dodaj tag",
+        saveTagPlaceholder: "Nazwa",
+        confirm: "Zatwierdź",
+        indexPickerPlaceholder: "Wybierz indeks",
+        sort: {
+            relevance: "Zgodność z szukanym",
+            dateAsc: "Data (najpierw starsze)",
+            dateDesc: "Data (najpierw nowsze)",
+            sizeAsc: "Wielkość (najpierw mniejsze)",
+            sizeDesc: "Wielkość (najpierw większe)",
+            nameAsc: "Nazwa (A-z)",
+            nameDesc: "Nazwa (Z-a)",
+            random: "Losowo",
+        },
+        d3: {
+            mimeCount: "Dystrybucja liczby plików według typów mediów",
+            mimeSize: "Dystrybucja wielkości plików według typów mediów",
+            dateHistogram: "Dystrybucja dat modyfikacji plików",
+            sizeHistogram: "Dystrybucja wielkości plików",
+        },
+        indexPicker: {
+            selectNone: "Zaznacz nic",
+            selectAll: "Zaznacz wszystko",
+            selectedIndex: "wybrany indeks",
+            selectedIndices: "wybrane indeksy",
+        },
+        ml: {
+            analyzeText: "Analizuj",
+            auto: "Automatycznie",
+            repoFetchError: "Nie udało się uzyskać listy modeli. Zobacz konsolę przeglądarki, aby uzyskać więcej informacji.",
+            repoFetchErrorTitle: "Nie udało się pobrać repozytoriów modeli",
+        }
+    }
 }

sist2-vue/src/views/Configuration.vue

@@ -237,6 +237,7 @@ export default {
                 {value: "fr", text: this.$t("lang.fr")},
                 {value: "zh-CN", text: this.$t("lang.zh-CN")},
                 {value: "de", text: this.$t("lang.de")},
+                {value: "pl", text: this.$t("lang.pl")},
             ],
             queryModeOptions: [
                 {value: "simple", text: this.$t("queryMode.simple")},

sist2-vue/src/views/StatsPage.vue

@@ -57,7 +57,9 @@ export default {
                 };
             })
         },
-        indices: () => this.$store.state.indices
+        indices() {
+            return this.$store.state.indices;
+        }
     }
 }
 </script>

src/cli.c

@@ -5,7 +5,7 @@
 #define DEFAULT_OUTPUT "index.sist2"
 #define DEFAULT_NAME "index"
 #define DEFAULT_CONTENT_SIZE 32768
-#define DEFAULT_QUALITY 2
+#define DEFAULT_QUALITY 50
 #define DEFAULT_THUMBNAIL_SIZE 552
 #define DEFAULT_THUMBNAIL_COUNT 1
 #define DEFAULT_REWRITE_URL ""
@@ -83,6 +83,10 @@ void exec_args_destroy(exec_args_t *args) {
     free(args);
 }
 
+void sqlite_index_args_destroy(sqlite_index_args_t *args) {
+    // TODO
+}
+
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (argc < 2) {
         fprintf(stderr, "Required positional argument: PATH.\n");
@@ -93,15 +97,18 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (abs_path == NULL) {
         LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
     } else {
-        abs_path = realloc(abs_path, strlen(abs_path) + 2);
+        char *new_abs_path = realloc(abs_path, strlen(abs_path) + 2);
-        strcat(abs_path, "/");
+        if (new_abs_path == NULL) {
-        args->path = abs_path;
+            LOG_FATALF("cli.c", "FIXME: realloc() failed for argv[1]=%s, abs_path=%s", argv[1], abs_path);
+        }
+        strcat(new_abs_path, "/");
+        args->path = new_abs_path;
     }
 
     if (args->tn_quality == OPTION_VALUE_UNSPECIFIED) {
         args->tn_quality = DEFAULT_QUALITY;
-    } else if (args->tn_quality < 2 || args->tn_quality > 31) {
+    } else if (args->tn_quality < 0 || args->tn_quality > 100) {
-        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [2, 31].\n",
+        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [0, 100].\n",
                 args->tn_quality);
         return 1;
     }
@@ -109,7 +116,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (args->tn_size == OPTION_VALUE_UNSPECIFIED) {
         args->tn_size = DEFAULT_THUMBNAIL_SIZE;
     } else if (args->tn_size < 32) {
-        printf("Invalid value --thumbnail-size argument: %d. Must be greater than 32 pixels.\n", args->tn_size);
+        printf("Invalid value --thumbnail-size argument: %d. Must be >= 32 pixels.\n", args->tn_size);
         return 1;
     }
 
@@ -142,10 +149,14 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     char *abs_output = abspath(args->output);
     if (args->incremental && abs_output == NULL) {
-        LOG_WARNINGF("main.c", "Could not open original index for incremental scan: %s. Will not perform incremental scan.", args->output);
+        LOG_WARNINGF("main.c",
+                     "Could not open original index for incremental scan: %s. Will not perform incremental scan.",
+                     args->output);
         args->incremental = FALSE;
     } else if (!args->incremental && abs_output != NULL) {
-        LOG_FATALF("main.c", "Index already exists: %s. If you wish to perform incremental scan, you must specify --incremental", abs_output);
+        LOG_FATALF("main.c",
+                   "Index already exists: %s. If you wish to perform incremental scan, you must specify --incremental",
+                   abs_output);
     }
     free(abs_output);
 

src/cli.h

@@ -134,5 +134,7 @@ void exec_args_destroy(exec_args_t *args);
 
 int exec_args_validate(exec_args_t *args, int argc, const char **argv);
 
+void sqlite_index_args_destroy(sqlite_index_args_t *args);
+
 
 #endif

src/database/database_fts.c

@@ -447,12 +447,16 @@ database_summary_stats_t database_fts_get_date_range(database_t *db) {
     return stats;
 }
 
-char *get_after_where(char **after, fts_sort_t sort) {
+char *get_after_where(char **after, fts_sort_t sort, int sort_asc) {
     if (after == NULL) {
         return NULL;
     }
 
-    return "(sort_var, doc.ROWID) > (?3, ?4)";
+    if (sort_asc) {
+        return "(sort_var, doc.ROWID) > (?3, ?4)";
+    }
+
+    return "(sort_var, doc.ROWID) < (?3, ?4)";
 }
 
 cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
@@ -466,10 +470,10 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
     const char *path_where = path_where_clause(path);
     const char *size_where = size_where_clause(size_min, size_max);
     const char *date_where = date_where_clause(date_min, date_max);
-    const char *index_id_where = index_ids_where_clause(index_ids);
+    char *index_id_where = index_ids_where_clause(index_ids);
-    const char *mime_where = mime_types_where_clause(mime_types);
+    char *mime_where = mime_types_where_clause(mime_types);
     const char *query_where = match_where(query);
-    const char *after_where = get_after_where(after, sort);
+    const char *after_where = get_after_where(after, sort, sort_asc);
     const char *tags_where = tags_where_clause(tags);
 
     if (!query_where && sort == FTS_SORT_SCORE) {

src/database/database_stats.c

@@ -149,6 +149,7 @@ void database_generate_stats(database_t *db, double treemap_threshold) {
 
             merged_rows += 1;
         }
+        free(iter);
     } while (merged_rows > TREEMAP_MINIMUM_MERGES_TO_CONTINUE);
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db,

src/log.h

@@ -3,6 +3,7 @@
 
 
 #include <signal.h>
+
 #define LOG_MAX_LENGTH 8192
 
 #define LOG_SIST_DEBUG 0
@@ -33,11 +34,12 @@
 
 #define LOG_FATALF(filepath, fmt, ...)\
     sist_logf(filepath, LOG_SIST_FATAL, fmt, __VA_ARGS__);\
-    raise(SIGUSR1)
+    raise(SIGUSR1);                   \
+    exit(-1)
 #define LOG_FATAL(filepath, str) \
     sist_log(filepath, LOG_SIST_FATAL, str);\
-    exit(SIGUSR1)
+    raise(SIGUSR1);                   \
-
+    exit(-1)
 #define LOG_FATALF_NO_EXIT(filepath, fmt, ...) \
     sist_logf(filepath, LOG_SIST_FATAL, fmt, __VA_ARGS__)
 #define LOG_FATAL_NO_EXIT(filepath, str) \
@@ -46,6 +48,7 @@
 #include "sist.h"
 
 void sist_logf(const char *filepath, int level, char *format, ...);
+
 void vsist_logf(const char *filepath, int level, char *format, va_list ap);
 
 void sist_log(const char *filepath, int level, char *str);

src/main.c

@@ -68,9 +68,7 @@ void database_scan_begin(scan_args_t *args) {
         desc->version_patch = VersionPatch;
 
         // generate new index id based on timestamp
-        unsigned char index_md5[MD5_DIGEST_LENGTH];
+        md5_hexdigest(&ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), ScanCtx.index.desc.id);
-        MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
-        buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
 
         database_initialize(db);
         database_open(db);
@@ -490,7 +488,7 @@ int main(int argc, const char *argv[]) {
             OPT_GROUP("Scan options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT: 1"),
             OPT_INTEGER('q', "thumbnail-quality", &scan_args->tn_quality,
-                        "Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2",
+                        "Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50",
                         set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_quality),
             OPT_INTEGER(0, "thumbnail-size", &scan_args->tn_size,
                         "Thumbnail size, in pixels. DEFAULT: 552",
@@ -683,6 +681,7 @@ int main(int argc, const char *argv[]) {
     index_args_destroy(index_args);
     web_args_destroy(web_args);
     exec_args_destroy(exec_args);
+    sqlite_index_args_destroy(sqlite_index_args);
 
     return 0;
 }

src/parsing/fs_util.h

@@ -2,15 +2,18 @@
 #define SIST2_FS_UTIL_H
 
 #include "src/sist.h"
+#include <openssl/evp.h>
 
 #define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));};
 
 static int fs_read(struct vfile *f, void *buf, size_t size) {
     if (f->fd == -1) {
-        SHA1_Init(&f->sha1_ctx);
+        f->sha1_ctx = EVP_MD_CTX_new();
+        EVP_DigestInit_ex(f->sha1_ctx, EVP_sha1(), NULL);
 
         f->fd = open(f->filepath, O_RDONLY);
         if (f->fd == -1) {
+            EVP_MD_CTX_free(f->sha1_ctx);
             return -1;
         }
     }
@@ -19,7 +22,7 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
 
     if (ret != 0 && f->calculate_checksum) {
         f->has_checksum = TRUE;
-        safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret);
+        safe_digest_update(f->sha1_ctx, (unsigned char *) buf, ret);
     }
 
     return ret;
@@ -27,8 +30,11 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
 
 static void fs_close(struct vfile *f) {
     if (f->fd != -1) {
-        SHA1_Final(f->sha1_digest, &f->sha1_ctx);
+        EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
+        EVP_MD_CTX_free(f->sha1_ctx);
+        f->sha1_ctx = NULL;
         close(f->fd);
+        f->fd = -1;
     }
 }
 

src/parsing/parse.c

@@ -53,9 +53,9 @@ file_type_t get_file_type(unsigned int mime, size_t size, const char *filepath)
     } else if (IS_FONT(mime)) {
         return FILETYPE_FONT;
     } else if (ScanCtx.arc_ctx.mode != ARC_MODE_SKIP && (
-                    IS_ARC(mime) ||
+            IS_ARC(mime) ||
-                    (IS_ARC_FILTER(mime) && should_parse_filtered_file(filepath))
+            (IS_ARC_FILTER(mime) && should_parse_filtered_file(filepath))
-            )) {
+    )) {
         return FILETYPE_ARCHIVE;
     } else if ((ScanCtx.ooxml_ctx.content_size > 0 || ScanCtx.media_ctx.tn_size > 0) && IS_DOC(mime)) {
         return FILETYPE_OOXML;
@@ -155,19 +155,17 @@ void parse(parse_job_t *job) {
     doc->meta_head = NULL;
     doc->meta_tail = NULL;
     doc->size = job->vfile.st_size;
-    doc->mtime = job->vfile.mtime;
+    doc->mtime = MAX(job->vfile.mtime, 0);
     doc->mime = get_mime(job);
     generate_doc_id(doc->filepath + ScanCtx.index.desc.root_len, doc->doc_id);
 
     if (doc->mime == GET_MIME_ERROR_FATAL) {
-
         CLOSE_FILE(job->vfile)
         free(doc);
         return;
     }
 
     if (database_mark_document(ProcData.index_db, doc->doc_id, doc->mtime)) {
-
         CLOSE_FILE(job->vfile)
         free(doc);
         return;

src/sist.h

@@ -51,11 +51,11 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.0.7"
+#define VERSION "3.1.4"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
-static const int VersionMinor = 0;
+static const int VersionMinor = 1;
-static const int VersionPatch = 7;
+static const int VersionPatch = 4;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown

src/util.h

@@ -7,6 +7,7 @@
 
 #include "third-party/utf8.h/utf8.h"
 #include "libscan/scan.h"
+#include <openssl/evp.h>
 
 
 char *abspath(const char *path);
@@ -86,13 +87,22 @@ static void buf2hex(const unsigned char *buf, size_t buflen, char *hex_string) {
     *s = '\0';
 }
 
+static void md5_hexdigest(void *data, size_t size, char *output) {
+    EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
+    EVP_DigestInit_ex(md_ctx, EVP_md5(), NULL);
+
+    EVP_DigestUpdate(md_ctx, data, size);
+
+    unsigned char digest[MD5_DIGEST_LENGTH];
+    EVP_DigestFinal_ex(md_ctx, digest, NULL);
+    EVP_MD_CTX_free(md_ctx);
+
+    buf2hex(digest, MD5_DIGEST_LENGTH, output);
+}
 
 __always_inline
 static void generate_doc_id(const char *rel_path, char *doc_id) {
-    unsigned char md[MD5_DIGEST_LENGTH];
+    md5_hexdigest(rel_path, strlen(rel_path), doc_id);
-
-    MD5((unsigned char *) rel_path, strlen(rel_path), md);
-    buf2hex(md, sizeof(md), doc_id);
 }
 
 #define MILLISECOND 1000

src/web/serve.c

@@ -248,9 +248,11 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
 
     char mime_mapping[8192];
     if (strlen(ext) == 0) {
-        snprintf(mime_mapping, sizeof(mime_mapping), "%s=%s", full_path, mime);
+        snprintf(mime_mapping, sizeof(mime_mapping), "%s=%s%s",
+                 full_path, mime, STR_STARTS_WITH_CONSTANT(mime, "text/") ? "; charset=utf8" : "");
     } else {
-        snprintf(mime_mapping, sizeof(mime_mapping), "%s=%s", ext, mime);
+        snprintf(mime_mapping, sizeof(mime_mapping), "%s=%s%s",
+                 ext, mime, STR_STARTS_WITH_CONSTANT(mime, "text/") ? "; charset=utf8" : "");
     }
 
     struct mg_http_serve_opts opts = {
@@ -526,9 +528,9 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) {
     }
 
     tag_req_t *req = parse_tag_request(json);
-    cJSON_Delete(json);
     if (req == NULL) {
         LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index);
+        cJSON_Delete(json);
         HTTP_REPLY_BAD_REQUEST
         return;
     }
@@ -552,6 +554,7 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) {
     }
 
     free(req);
+    cJSON_Delete(json);
 }
 
 int validate_auth(struct mg_connection *nc, struct mg_http_message *hm) {

src/web/web_fts.c

@@ -170,7 +170,7 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
         cJSON_Delete(json);
         return NULL;
     }
-    if (req_path.val && (strstr(req_path.val->valuestring, "*") || strlen(req_path.val) >= PATH_MAX)) {
+    if (req_path.val && (strstr(req_path.val->valuestring, "*") || strlen(req_path.val->valuestring) >= PATH_MAX)) {
         cJSON_Delete(json);
         return NULL;
     }

third-party/libscan/libscan/arc/arc.c

@@ -4,7 +4,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <fcntl.h>
-#include <openssl/evp.h>
 #include <pcre.h>
 
 #define MAX_DECOMPRESSED_SIZE_RATIO 40.0
@@ -23,7 +22,11 @@ int should_parse_filtered_file(const char *filepath) {
 }
 
 void arc_close(struct vfile *f) {
-    SHA1_Final(f->sha1_digest, &f->sha1_ctx);
+    if (f->sha1_ctx != NULL) {
+        EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
+        EVP_MD_CTX_free(f->sha1_ctx);
+        f->sha1_ctx = NULL;
+    }
 
     if (f->rewind_buffer != NULL) {
         free(f->rewind_buffer);
@@ -60,7 +63,7 @@ int arc_read(struct vfile *f, void *buf, size_t size) {
     if (bytes_read != 0 && bytes_read <= size && f->calculate_checksum) {
         f->has_checksum = TRUE;
 
-        safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, bytes_read);
+        safe_digest_update(f->sha1_ctx, (unsigned char *) buf, bytes_read);
     }
 
     if (bytes_read != size && archive_errno(f->arc) != 0) {
@@ -211,11 +214,20 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
 
                 double decompressed_size_ratio = (double) sub_job->vfile.st_size / (double) f->st_size;
                 if (decompressed_size_ratio > MAX_DECOMPRESSED_SIZE_RATIO) {
-                    CTX_LOG_DEBUGF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)", sub_job->filepath,
+                    CTX_LOG_ERRORF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)",
+                                   sub_job->filepath,
                                    decompressed_size_ratio);
                     break;
                 }
 
+                if ((archive_entry_is_encrypted(entry) || archive_entry_is_data_encrypted(entry) ||
+                     archive_entry_is_metadata_encrypted(entry)) && ctx->passphrase[0] == 0) {
+                    // Is encrypted but no password is specified, skip
+                    CTX_LOG_ERRORF("arc.c", "Skipped %s, archive is encrypted but no passphrase is supplied",
+                                   doc->filepath);
+                    break;
+                }
+
                 // Handle excludes
                 if (exclude != NULL && EXCLUDED(sub_job->filepath)) {
                     CTX_LOG_DEBUGF("arc.c", "Excluded: %s", sub_job->filepath);
@@ -229,9 +241,12 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
                     sub_job->ext = (int) strlen(sub_job->filepath);
                 }
 
-                SHA1_Init(&sub_job->vfile.sha1_ctx);
+                sub_job->vfile.sha1_ctx = EVP_MD_CTX_new();
+                EVP_DigestInit(sub_job->vfile.sha1_ctx, EVP_sha1());
 
                 ctx->parse(sub_job);
+
+                sub_job->vfile.close(&sub_job->vfile);
             }
         }
 

third-party/libscan/libscan/arc/arc.h

@@ -35,7 +35,8 @@ static int vfile_open_callback(struct archive *a, void *user_data) {
     arc_data_t *data = (arc_data_t *) user_data;
 
     if (!data->f->is_fs_file) {
-        SHA1_Init(&data->f->sha1_ctx);
+        data->f->sha1_ctx = EVP_MD_CTX_new();
+        EVP_DigestInit(data->f->sha1_ctx, EVP_md5());
     }
 
     return ARCHIVE_OK;
@@ -49,7 +50,7 @@ static long vfile_read_callback(struct archive *a, void *user_data, const void *
 
     if (!data->f->is_fs_file && ret > 0) {
         data->f->has_checksum = TRUE;
-        safe_sha1_update(&data->f->sha1_ctx, (unsigned char*)data->buf, ret);
+        safe_digest_update(data->f->sha1_ctx, (unsigned char *) data->buf, ret);
     }
 
     return ret;
@@ -59,7 +60,9 @@ static int vfile_close_callback(struct archive *a, void *user_data) {
     arc_data_t *data = (arc_data_t *) user_data;
 
     if (!data->f->is_fs_file) {
-        SHA1_Final((unsigned char *) data->f->sha1_digest, &data->f->sha1_ctx);
+        EVP_DigestFinal_ex(data->f->sha1_ctx, data->f->sha1_digest, NULL);
+        EVP_MD_CTX_free(data->f->sha1_ctx);
+        data->f->sha1_ctx = NULL;
     }
 
     return ARCHIVE_OK;

third-party/libscan/libscan/ebook/ebook.c

@@ -11,8 +11,6 @@
 pthread_mutex_t Mutex;
 #endif
 
-/* fill_image callback doesn't let us pass opaque pointers unless I create my own device */
-__thread text_buffer_t thread_buffer;
 __thread scan_ebook_ctx_t thread_ctx;
 
 static void my_fz_lock(UNUSED(void *user), int lock) {
@@ -153,22 +151,23 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
 
     sws_freeContext(sws_ctx);
 
-    // YUV420p -> JPEG
+    // YUV420p -> JPEG/WEBP
-    AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(pixmap->w, pixmap->h, ctx->tn_qscale);
+    AVCodecContext *thumbnail_encoder = alloc_webp_encoder(pixmap->w, pixmap->h, ctx->tn_qscale);
-    avcodec_send_frame(jpeg_encoder, scaled_frame);
+    avcodec_send_frame(thumbnail_encoder, scaled_frame);
+    avcodec_send_frame(thumbnail_encoder, NULL); // Send EOF
 
-    AVPacket jpeg_packet;
+    AVPacket thumbnail_packet;
-    av_init_packet(&jpeg_packet);
+    av_init_packet(&thumbnail_packet);
-    avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
+    avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
 
     APPEND_LONG_META(doc, MetaThumbnail, 1);
-    ctx->store(doc->doc_id, 0, (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store(doc->doc_id, 0, (char *) thumbnail_packet.data, thumbnail_packet.size);
 
     free(samples);
-    av_packet_unref(&jpeg_packet);
+    av_packet_unref(&thumbnail_packet);
     av_free(*scaled_frame->data);
     av_frame_free(&scaled_frame);
-    avcodec_free_context(&jpeg_encoder);
+    avcodec_free_context(&thumbnail_encoder);
 
     fz_drop_pixmap(fzctx, pixmap);
     fz_drop_page(fzctx, cover);
@@ -231,21 +230,47 @@ static int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
     return 0;
 }
 
-static void fill_image_ocr_cb(const char* text, size_t len) {
+int read_stext(text_buffer_t *tex, fz_stext_page *stext) {
-  text_buffer_append_string(&thread_buffer, text, len - 1);
+
+    int count = 0;
+
+    fz_stext_block *block = stext->first_block;
+
+    while (block != NULL) {
+        int ret = read_stext_block(block, tex);
+        count += 1;
+        if (ret == TEXT_BUF_FULL) {
+            break;
+        }
+        block = block->next;
+    }
+
+    return count;
 }
 
-void fill_image(fz_context *fzctx, UNUSED(fz_device *dev),
+int load_page(fz_context *fzctx, fz_document *fzdoc, int current_page, fz_page **page) {
-                fz_image *img, UNUSED(fz_matrix ctm), UNUSED(float alpha),
+    int err = 0;
-                UNUSED(fz_color_params color_params)) {
 
-    int l2factor = 0;
+    fz_var(err);
+    fz_try(fzctx)(*page) = fz_load_page(fzctx, fzdoc, current_page);
+    fz_catch(fzctx)err = fzctx->error.errcode;
 
-    if (img->w >= MIN_OCR_WIDTH && img->h >= MIN_OCR_HEIGHT && OCR_IS_VALID_BPP(img->n)) {
+    return err;
-        fz_pixmap *pix = img->get_pixmap(fzctx, img, NULL, img->w, img->h, &l2factor);
+}
-        ocr_extract_text(thread_ctx.tesseract_path, thread_ctx.tesseract_lang, pix->samples, pix->w, pix->h, pix->n, (int)pix->stride, pix->xres, fill_image_ocr_cb);
+
-        fz_drop_pixmap(fzctx, pix);
+fz_device *new_stext_dev(fz_context *fzctx, fz_stext_page *stext) {
-    }
+    fz_stext_options opts = {
+            .flags = FZ_STEXT_DEHYPHENATE,
+            .scale = 0
+    };
+
+    fz_device *stext_dev = fz_new_stext_device(fzctx, stext, &opts);
+    stext_dev->stroke_path = NULL;
+    stext_dev->stroke_text = NULL;
+    stext_dev->clip_text = NULL;
+    stext_dev->clip_stroke_path = NULL;
+    stext_dev->clip_stroke_text = NULL;
+    return stext_dev;
 }
 
 void
@@ -325,46 +350,37 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi
 
 
     if (ctx->content_size > 0) {
-        fz_stext_options opts = {0};
+        text_buffer_t tex = text_buffer_create(ctx->content_size);
-        thread_buffer = text_buffer_create(ctx->content_size);
 
         for (int current_page = 0; current_page < page_count; current_page++) {
             fz_page *page = NULL;
-            fz_var(err);
+            err = load_page(fzctx, fzdoc, current_page, &page);
-            fz_try(fzctx)page = fz_load_page(fzctx, fzdoc, current_page);
+
-            fz_catch(fzctx)err = fzctx->error.errcode;
             if (err != 0) {
-                CTX_LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, fzctx->error.message);
+                CTX_LOG_WARNINGF(doc->filepath,
-                text_buffer_destroy(&thread_buffer);
+                                 "fz_load_page() returned error code [%d] %s", err, fzctx->error.message);
+                text_buffer_destroy(&tex);
                 fz_drop_page(fzctx, page);
                 fz_drop_stream(fzctx, stream);
                 fz_drop_document(fzctx, fzdoc);
                 fz_drop_context(fzctx);
                 return;
             }
+            fz_rect page_mediabox = fz_bound_page(fzctx, page);
 
-            fz_stext_page *stext = fz_new_stext_page(fzctx, fz_bound_page(fzctx, page));
+            fz_stext_page *stext = fz_new_stext_page(fzctx, page_mediabox);
-            fz_device *dev = fz_new_stext_device(fzctx, stext, &opts);
+            fz_device *stext_dev = new_stext_dev(fzctx, stext);
-            dev->stroke_path = NULL;
-            dev->stroke_text = NULL;
-            dev->clip_text = NULL;
-            dev->clip_stroke_path = NULL;
-            dev->clip_stroke_text = NULL;
-
-            if (ctx->tesseract_lang != NULL) {
-                dev->fill_image = fill_image;
-            }
 
             fz_var(err);
-            fz_try(fzctx)fz_run_page(fzctx, page, dev, fz_identity, NULL);
+            fz_try(fzctx)fz_run_page(fzctx, page, stext_dev, fz_identity, NULL);
             fz_always(fzctx) {
-                    fz_close_device(fzctx, dev);
+                    fz_close_device(fzctx, stext_dev);
-                    fz_drop_device(fzctx, dev);
+                    fz_drop_device(fzctx, stext_dev);
-                } fz_catch(fzctx)err = fzctx->error.errcode;
+                } fz_catch(fzctx) err = fzctx->error.errcode;
 
             if (err != 0) {
                 CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message);
-                text_buffer_destroy(&thread_buffer);
+                text_buffer_destroy(&tex);
                 fz_drop_page(fzctx, page);
                 fz_drop_stext_page(fzctx, stext);
                 fz_drop_stream(fzctx, stream);
@@ -373,29 +389,63 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi
                 return;
             }
 
-            fz_stext_block *block = stext->first_block;
+            int num_blocks_read = read_stext(&tex, stext);
-            while (block != NULL) {
-                int ret = read_stext_block(block, &thread_buffer);
-                if (ret == TEXT_BUF_FULL) {
-                    break;
-                }
-                block = block->next;
-            }
-            fz_drop_stext_page(fzctx, stext);
-            fz_drop_page(fzctx, page);
 
-            if (thread_buffer.dyn_buffer.cur >= ctx->content_size) {
+            fz_drop_stext_page(fzctx, stext);
+
+            if (tex.dyn_buffer.cur >= ctx->content_size) {
+                fz_drop_page(fzctx, page);
                 break;
             }
-        }
-        text_buffer_terminate_string(&thread_buffer);
 
-        meta_line_t *meta_content = malloc(sizeof(meta_line_t) + thread_buffer.dyn_buffer.cur);
+            // If OCR is enabled and no text is found on the page
+            if (ctx->tesseract_lang != NULL && num_blocks_read == 0) {
+                stext = fz_new_stext_page(fzctx, page_mediabox);
+                stext_dev = new_stext_dev(fzctx, stext);
+
+                fz_device *ocr_dev = fz_new_ocr_device(fzctx, stext_dev, fz_identity,
+                                                       page_mediabox, TRUE,
+                                                       ctx->tesseract_lang,
+                                                       ctx->tesseract_path,
+                                                       NULL, NULL);
+
+                fz_var(err);
+                fz_try(fzctx)fz_run_page(fzctx, page, ocr_dev, fz_identity, NULL);
+                fz_always(fzctx) {
+                        fz_close_device(fzctx, ocr_dev);
+                        fz_drop_device(fzctx, ocr_dev);
+                    } fz_catch(fzctx) err = fzctx->error.errcode;
+
+                if (err != 0) {
+                    CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message);
+                    fz_close_device(fzctx, stext_dev);
+                    fz_drop_device(fzctx, stext_dev);
+                    text_buffer_destroy(&tex);
+                    fz_drop_page(fzctx, page);
+                    fz_drop_stext_page(fzctx, stext);
+                    fz_drop_stream(fzctx, stream);
+                    fz_drop_document(fzctx, fzdoc);
+                    fz_drop_context(fzctx);
+                    return;
+                }
+
+                fz_close_device(fzctx, stext_dev);
+                fz_drop_device(fzctx, stext_dev);
+
+                read_stext(&tex, stext);
+                fz_drop_stext_page(fzctx, stext);
+            }
+
+            fz_drop_page(fzctx, page);
+        }
+        text_buffer_terminate_string(&tex);
+
+        meta_line_t *meta_content = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur);
         meta_content->key = MetaContent;
-        memcpy(meta_content->str_val, thread_buffer.dyn_buffer.buf, thread_buffer.dyn_buffer.cur);
+        memcpy(meta_content->str_val, tex.dyn_buffer.buf, tex.dyn_buffer.cur);
         APPEND_META(doc, meta_content);
 
-        text_buffer_destroy(&thread_buffer);
+        text_buffer_destroy(&tex);
     }
 
     fz_drop_stream(fzctx, stream);

third-party/libscan/libscan/media/media.c

@@ -68,7 +68,7 @@ void *scale_frame(const AVCodecContext *decoder, const AVFrame *frame, int size)
 
     struct SwsContext *sws_ctx = sws_getContext(
             decoder->width, decoder->height, decoder->pix_fmt,
-            dstW, dstH, AV_PIX_FMT_YUVJ420P,
+            dstW, dstH, AV_PIX_FMT_YUV420P,
             SIST_SWS_ALGO, 0, 0, 0
     );
 
@@ -118,13 +118,12 @@ static void read_subtitles(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, i
     AVPacket packet;
     AVSubtitle subtitle;
 
-    AVCodec *subtitle_codec = avcodec_find_decoder(pFormatCtx->streams[stream_idx]->codecpar->codec_id);
+    const AVCodec *subtitle_codec = avcodec_find_decoder(pFormatCtx->streams[stream_idx]->codecpar->codec_id);
     AVCodecContext *decoder = avcodec_alloc_context3(subtitle_codec);
+    decoder->thread_count = 1;
     avcodec_parameters_to_context(decoder, pFormatCtx->streams[stream_idx]->codecpar);
     avcodec_open2(decoder, subtitle_codec, NULL);
 
-    decoder->sub_text_format = FF_SUB_TEXT_FMT_ASS;
-
     int got_sub;
 
     while (1) {
@@ -177,8 +176,6 @@ read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *d
     result->packet = av_packet_alloc();
     result->frame = av_frame_alloc();
 
-    av_init_packet(result->packet);
-
     int receive_ret = -EAGAIN;
     while (receive_ret == -EAGAIN) {
         // Get video frame
@@ -436,7 +433,8 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
         }
 
         if (seek_ok == FALSE && thumbnail_index != 0) {
-            CTX_LOG_WARNING(doc->filepath, "(media.c) Could not seek media file. Can't generate additional thumbnails.");
+            CTX_LOG_WARNING(doc->filepath,
+                            "(media.c) Could not seek media file. Can't generate additional thumbnails.");
             return SAVE_THUMBNAIL_FAILED;
         }
     }
@@ -470,18 +468,18 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
 
         ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size);
     } else {
-        // Encode frame to jpeg
+        // Encode frame
-        AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
+        AVCodecContext *thumbnail_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height,
-                                                          ctx->tn_qscale);
+                                                               ctx->tn_qscale);
-        avcodec_send_frame(jpeg_encoder, scaled_frame);
+        avcodec_send_frame(thumbnail_encoder, scaled_frame);
+        avcodec_send_frame(thumbnail_encoder, NULL); // send EOF
 
-        AVPacket jpeg_packet;
+        AVPacket *thumbnail_packet = av_packet_alloc();
-        av_init_packet(&jpeg_packet);
+        avcodec_receive_packet(thumbnail_encoder, thumbnail_packet);
-        avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
 
         // Save thumbnail
         if (thumbnail_index == 0) {
-            ctx->store(doc->doc_id, 0, jpeg_packet.data, jpeg_packet.size);
+            ctx->store(doc->doc_id, 0, thumbnail_packet->data, thumbnail_packet->size);
             return_value = SAVE_THUMBNAIL_OK;
 
         } else if (thumbnail_index > 1) {
@@ -489,15 +487,15 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
             //  I figure out a better fix.
             thumbnail_index -= 1;
 
-            ctx->store(doc->doc_id, thumbnail_index, jpeg_packet.data, jpeg_packet.size);
+            ctx->store(doc->doc_id, thumbnail_index, thumbnail_packet->data, thumbnail_packet->size);
 
             return_value = SAVE_THUMBNAIL_OK;
         } else {
             return_value = SAVE_THUMBNAIL_SKIPPED;
         }
 
-        avcodec_free_context(&jpeg_encoder);
+        avcodec_free_context(&thumbnail_encoder);
-        av_packet_unref(&jpeg_packet);
+        av_packet_free(&thumbnail_packet);
         av_free(*scaled_frame->data);
         av_frame_free(&scaled_frame);
     }
@@ -576,8 +574,9 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
         }
 
         // Decoder
-        AVCodec *video_codec = avcodec_find_decoder(stream->codecpar->codec_id);
+        const AVCodec *video_codec = avcodec_find_decoder(stream->codecpar->codec_id);
         AVCodecContext *decoder = avcodec_alloc_context3(video_codec);
+        decoder->thread_count = 1;
         avcodec_parameters_to_context(decoder, stream->codecpar);
         avcodec_open2(decoder, video_codec, NULL);
 
@@ -628,6 +627,9 @@ void parse_media_filename(scan_media_ctx_t *ctx, const char *filepath, document_
         CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()");
         return;
     }
+    pFormatCtx->max_analyze_duration = 100000000;
+    pFormatCtx->probesize = 100000000;
+
     int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL);
     if (res < 0) {
         CTX_LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res));
@@ -695,9 +697,10 @@ int memfile_open(vfile_t *f, memfile_t *mem) {
     mem->file = fmemopen(mem->buf, mem->size, "rb");
 
     if (f->calculate_checksum) {
-        SHA1_Init(&f->sha1_ctx);
+        safe_digest_update(f->sha1_ctx, mem->buf, mem->size);
-        safe_sha1_update(&f->sha1_ctx, mem->buf, mem->size);
+        EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
-        SHA1_Final(f->sha1_digest, &f->sha1_ctx);
+        EVP_MD_CTX_free(f->sha1_ctx);
+        f->sha1_ctx = NULL;
         f->has_checksum = TRUE;
     }
 
@@ -727,6 +730,9 @@ void parse_media_vfile(scan_media_ctx_t *ctx, struct vfile *f, document_t *doc,
         CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()");
         return;
     }
+    pFormatCtx->max_analyze_duration = 100000000;
+    pFormatCtx->probesize = 100000000;
+
 
     unsigned char *buffer = (unsigned char *) av_malloc(AVIO_BUF_SIZE);
     AVIOContext *io_ctx = NULL;
@@ -790,6 +796,8 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
         CTX_LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()");
         return FALSE;
     }
+    pFormatCtx->max_analyze_duration = 100000000;
+    pFormatCtx->probesize = 100000000;
 
     unsigned char *buffer = (unsigned char *) av_malloc(AVIO_BUF_SIZE);
 
@@ -821,6 +829,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
     // Decoder
     const AVCodec *video_codec = avcodec_find_decoder(stream->codecpar->codec_id);
     AVCodecContext *decoder = avcodec_alloc_context3(video_codec);
+    decoder->thread_count = 1;
     avcodec_parameters_to_context(decoder, stream->codecpar);
     avcodec_open2(decoder, video_codec, NULL);
 
@@ -854,19 +863,19 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
         ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size);
     } else {
         // Encode frame to jpeg
-        AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
+        AVCodecContext *jpeg_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height,
                                                           ctx->tn_qscale);
         avcodec_send_frame(jpeg_encoder, scaled_frame);
+        avcodec_send_frame(jpeg_encoder, NULL); // Send EOF
 
-        AVPacket jpeg_packet;
+        AVPacket *jpeg_packet = av_packet_alloc();
-        av_init_packet(&jpeg_packet);
+        avcodec_receive_packet(jpeg_encoder, jpeg_packet);
-        avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
 
         // Save thumbnail
         APPEND_LONG_META(doc, MetaThumbnail, 1);
-        ctx->store(doc->doc_id, 0, jpeg_packet.data, jpeg_packet.size);
+        ctx->store(doc->doc_id, 0, jpeg_packet->data, jpeg_packet->size);
 
-        av_packet_unref(&jpeg_packet);
+        av_packet_free(&jpeg_packet);
         avcodec_free_context(&jpeg_encoder);
         av_free(*scaled_frame->data);
         av_frame_free(&scaled_frame);
@@ -883,4 +892,4 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
     fclose(memfile.file);
 
     return TRUE;
-}
+}

third-party/libscan/libscan/media/media.h

@@ -48,6 +48,28 @@ static AVCodecContext *alloc_jpeg_encoder(int w, int h, int qscale) {
     return jpeg;
 }
 
+static AVCodecContext *alloc_webp_encoder(int w, int h, int qscale) {
+
+    const AVCodec *webp_codec = avcodec_find_encoder(AV_CODEC_ID_WEBP);
+    AVCodecContext *webp = avcodec_alloc_context3(webp_codec);
+    webp->width = w;
+    webp->height = h;
+    webp->time_base.den = 1000000;
+    webp->time_base.num = 1;
+    webp->compression_level = 6;
+    webp->global_quality = FF_QP2LAMBDA * qscale;
+
+    webp->pix_fmt = AV_PIX_FMT_YUV420P;
+    webp->color_range = AVCOL_RANGE_JPEG;
+    int ret = avcodec_open2(webp, webp_codec, NULL);
+
+    if (ret != 0) {
+        return NULL;
+    }
+
+    return webp;
+}
+
 
 void parse_media(scan_media_ctx_t *ctx, vfile_t *f, document_t *doc, const char *mime_str);
 

third-party/libscan/libscan/raw/raw.c

@@ -52,7 +52,7 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
 
     struct SwsContext *sws_ctx = sws_getContext(
             img->width, img->height, AV_PIX_FMT_RGB24,
-            dstW, dstH, AV_PIX_FMT_YUVJ420P,
+            dstW, dstH, AV_PIX_FMT_YUV420P,
             SIST_SWS_ALGO, 0, 0, 0
     );
 
@@ -76,20 +76,21 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
 
     sws_freeContext(sws_ctx);
 
-    AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height, 1.0f);
+    AVCodecContext *thumbnail_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height, ctx->tn_qscale);
-    avcodec_send_frame(jpeg_encoder, scaled_frame);
+    avcodec_send_frame(thumbnail_encoder, scaled_frame);
+    avcodec_send_frame(thumbnail_encoder, NULL); // Send EOF
 
-    AVPacket jpeg_packet;
+    AVPacket thumbnail_packet;
-    av_init_packet(&jpeg_packet);
+    av_init_packet(&thumbnail_packet);
-    avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
+    avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
 
     APPEND_LONG_META(doc, MetaThumbnail, 1);
-    ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) thumbnail_packet.data, thumbnail_packet.size);
 
-    av_packet_unref(&jpeg_packet);
+    av_packet_unref(&thumbnail_packet);
     av_free(*scaled_frame->data);
     av_frame_free(&scaled_frame);
-    avcodec_free_context(&jpeg_encoder);
+    avcodec_free_context(&thumbnail_encoder);
 
     return TRUE;
 }

third-party/libscan/libscan/scan.h

@@ -8,6 +8,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <openssl/evp.h>
 #include <openssl/md5.h>
 #include <openssl/sha.h>
 
@@ -146,7 +147,7 @@ typedef struct vfile {
     int mtime;
     size_t st_size;
 
-    SHA_CTX sha1_ctx;
+    EVP_MD_CTX *sha1_ctx;
     unsigned char sha1_digest[SHA1_DIGEST_LENGTH];
 
     void *rewind_buffer;

third-party/libscan/libscan/util.h

@@ -6,6 +6,7 @@
 #include "string.h"
 #include "../third-party/utf8.h/utf8.h"
 #include "macros.h"
+#include <openssl/evp.h>
 
 #define STR_STARTS_WITH_CONSTANT(x, y) (strncmp(y, x, sizeof(y) - 1) == 0)
 
@@ -339,7 +340,7 @@ static void *read_all(vfile_t *f, size_t *size) {
 #define STACK_BUFFER_SIZE (size_t)(4096 * 8)
 
 __always_inline
-static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) {
+static void safe_digest_update(EVP_MD_CTX *ctx, void *buf, size_t size) {
     unsigned char stack_buf[STACK_BUFFER_SIZE];
 
     void *sha1_buf;
@@ -351,7 +352,7 @@ static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) {
     }
 
     memcpy(sha1_buf, buf, size);
-    SHA1_Update(ctx, (const void *) sha1_buf, size);
+    EVP_DigestUpdate(ctx, sha1_buf, size);
 
     if (sha1_buf != stack_buf) {
         free(sha1_buf);