cleanup

This reverts commit 13e7ea188b.

.dockerignore

@@ -28,4 +28,8 @@ sist2
 **/ext_libwpd
 **/core
 *.a
-tmp_scan/
+tmp_scan/
+Dockerfile
+Dockerfile.arm64
+docker-compose.yml
+state.db

.gitignore

@@ -10,6 +10,9 @@ Makefile
 LOG
 sist2*
 !sist2-vue/
+!sist2-admin
+!sist2_admin
+!sist2.py
 *.sist2/
 bundle*.css
 bundle.js
@@ -25,4 +28,8 @@ test_i
 test_i_inc
 node_modules/
 .cmake/
-i_inc/
+i_inc/
+state.db
+*.pyc
+!sist2-admin/frontend/dist
+*.js.map

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "third-party/libscan/third-party/antiword"]
 	path = third-party/libscan/third-party/antiword
 	url = https://github.com/simon987/antiword
+[submodule "third-party/libscan/third-party/libmobi"]
+	path = third-party/libscan/third-party/libmobi
+	url = https://github.com/bfabiszewski/libmobi

CMakeLists.txt

@@ -4,6 +4,7 @@ set(CMAKE_C_STANDARD 11)
 project(sist2 C)
 
 option(SIST_DEBUG "Build a debug executable" on)
+option(SIST_FAST "Enable more optimisation flags" off)
 option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0)
 
 add_compile_definitions(
@@ -54,6 +55,10 @@ find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
+find_library(MAGIC_LIB
+        NAMES libmagic.so.1 magic
+        PATHS /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/
+)
 
 
 target_include_directories(
@@ -93,10 +98,22 @@ if (SIST_DEBUG)
             PROPERTIES
             OUTPUT_NAME sist2_debug
     )
+elseif (SIST_FAST)
+    target_compile_options(
+            sist2
+            PRIVATE
+
+            -Ofast
+            -march=native
+            -fno-stack-protector
+            -fomit-frame-pointer
+            -freciprocal-math
+    )
 else ()
     target_compile_options(
             sist2
             PRIVATE
+
             -Ofast
             -fno-stack-protector
             -fomit-frame-pointer
@@ -121,11 +138,12 @@ target_link_libraries(
         CURL::libcurl
 
         pthread
-        magic
 
         c
 
         scan
+
+        ${MAGIC_LIB}
 )
 
 add_custom_target(

Dockerfile

@@ -2,14 +2,29 @@ FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
-COPY . .
+
+COPY scripts scripts
+COPY schema schema
+COPY CMakeLists.txt .
+COPY third-party third-party
+COPY src src
+COPY sist2-vue sist2-vue
+
 RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
 RUN make -j$(nproc)
 RUN strip sist2 || mv sist2_debug sist2
 
-FROM --platform="linux/amd64" ubuntu:21.10
+FROM --platform="linux/amd64" ubuntu:20.04
 
-RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
+WORKDIR /root
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["sist2"]
+
+RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 libmagic1 python3  \
+    python3-pip git tesseract-ocr && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -20,9 +35,10 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
     curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
 
-ENTRYPOINT ["/root/sist2"]
+# sist2
+COPY --from=build /build/sist2 sist2
 
-ENV LANG C.UTF-8
+# sist2-admin
-ENV LC_ALL C.UTF-8
+COPY sist2-admin/requirements.txt sist2-admin/
-
+RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
-COPY --from=build /build/sist2 /root/sist2
+COPY sist2-admin/ sist2-admin/

README.md

@@ -52,7 +52,7 @@ sist2 (Simple incremental search tool)
 Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x` *
     2. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
        recommended!)*
-    3. *(or)* `docker pull simon987/sist2:2.11.7-x64-linux`
+    3. *(or)* `docker pull simon987/sist2:2.12.1-x64-linux`
 
 1. See [Usage guide](docs/USAGE.md)
 

docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3"
+
+services:
+  elasticsearch:
+    image: elasticsearch:7.14.0
+    container_name: sist2-es
+    environment:
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+  sist2-admin:
+    build:
+      context: .
+    container_name: sist2-admin
+    volumes:
+      - /mnt/array/sist2-admin-data/:/sist2-admin/
+      - /:/host
+    ports:
+      # NOTE: Don't export this port publicly!
+      - 8080:8080
+      - 4090:4090
+    working_dir: /root/sist2-admin/
+    entrypoint: python3
+    command:
+      - /root/sist2-admin/sist2_admin/app.py

docs/USAGE.md

@@ -103,7 +103,7 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 * `--thumbnail-count`
     Maximum number of thumbnails to generate. When set to a value >= 2, thumbnails for video previews
     will be generated. The actual number of thumbnails generated depends on the length of the video (maximum 1 image 
-    every ~5s). Set to 0 to completely disable thumbnails.
+    every ~7s). Set to 0 to completely disable thumbnails.
 * `--content-size` 
     Number of bytes of text to be extracted from the content of files (plain text, PDFs etc.).
     Repeated whitespace and special characters do not count toward this limit.
@@ -292,7 +292,7 @@ Both the `root` and `rewrite_url` fields are safe to manually modify from the
 
 # Elasticsearch
 
-Elasticsearch versions >=6.8.0, <8.0.0 are supported by sist2. 
+Elasticsearch versions >=6.8.0, 7.X.X and 8.X.X are supported by sist2. 
 
 Using a version >=7.14.0 is recommended to enable the following features:
 

schema/mappings.json

@@ -67,7 +67,8 @@
       "index": false
     },
     "mtime": {
-      "type": "integer"
+      "type": "date",
+      "format": "epoch_millis"
     },
     "size": {
       "type": "long"

schema/settings.json

@@ -3,7 +3,7 @@
     "refresh_interval": "30s",
     "codec": "best_compression",
     "number_of_replicas": 0,
-    "highlight.max_analyzed_offset": 10000000
+    "highlight.max_analyzed_offset": 1000000
   },
   "analysis": {
     "tokenizer": {
@@ -16,7 +16,7 @@
         "delimiter": "."
       },
       "my_nGram_tokenizer": {
-        "type": "nGram",
+        "type": "ngram",
         "min_gram": 3,
         "max_gram": 3
       }

scripts/before_build.sh

@@ -5,5 +5,6 @@ rm -rf index.sist2/
 python3 scripts/mime.py > src/parsing/mime_generated.c
 python3 scripts/serve_static.py > src/web/static_generated.c
 python3 scripts/index_static.py > src/index/static_generated.c
+python3 scripts/magic_static.py > src/magic_generated.c
 
 printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h

scripts/magic_static.py

@@ -0,0 +1,8 @@
+
+try:
+    with open("/usr/lib/file/magic.mgc", "rb") as f:
+        data = f.read()
+except:
+    data = bytes([])
+
+print("char magic_database_buffer[%d] = {%s};" % (len(data), ",".join(str(int(b)) for b in data)))

scripts/reset.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-make clean
-rm -rf CMakeFiles/ CMakeCache.txt Makefile \
-  third-party/libscan/CMakeFiles third-party/libscan/CMakeCache.txt third-party/libscan/third-party/ext_ffmpeg \
-  third-party/libscan/third-party/ext_libmobi third-party/libscan/Makefile

scripts/start_dev_es.sh

@@ -1,2 +1,3 @@
-docker run --rm -it -p 9200:9200 -e "discovery.type=single-node" \
+docker run --rm -it --name "sist2-dev-es"\
+       	-p 9200:9200 -e "discovery.type=single-node" \
 	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.14.0

scripts/start_dev_es_6.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es-6"\
+       	-p 9202:9200 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:6.8.0

scripts/start_dev_es_8.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es"\
+       	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.1.2

sist2-admin/frontend/babel.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  presets: [
+    '@vue/cli-plugin-babel/preset'
+  ]
+}

sist2-admin/frontend/dist/css/app.0f0b676b.css

@@ -0,0 +1 @@
+.navbar[data-v-27bc1d68]{box-shadow:0 .125rem .25rem rgba(0,0,0,.08)!important;border-radius:0}.theme-black .navbar[data-v-27bc1d68]{background:rgba(84,107,122,.18823529411764706);border-bottom:none}.navbar-brand[data-v-27bc1d68]{color:#222!important;font-size:1.75rem;padding:0}.navbar-brand[data-v-27bc1d68]:hover{color:#000!important}.version[data-v-27bc1d68]{color:#222!important;margin-left:-18px;margin-top:-14px;font-size:11px;font-family:monospace}.btn-link[data-v-27bc1d68]{color:#222}body,html{height:100%}#app{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;color:#2c3e50;padding-bottom:1em;min-height:100%}.info-icon{width:1rem;margin-right:.2rem;cursor:pointer;line-height:1rem;height:1rem;background-image:url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjYuNjY3IDQyNi42NjciIGZpbGw9IiNmZmYiPjxwYXRoIGQ9Ik0xOTIgMTkyaDQyLjY2N3YxMjhIMTkyeiIvPjxwYXRoIGQ9Ik0yMTMuMzMzIDBDOTUuNDY3IDAgMCA5NS40NjcgMCAyMTMuMzMzczk1LjQ2NyAyMTMuMzMzIDIxMy4zMzMgMjEzLjMzM1M0MjYuNjY3IDMzMS4yIDQyNi42NjcgMjEzLjMzMyAzMzEuMiAwIDIxMy4zMzMgMHptMCAzODRjLTk0LjA4IDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMgNDIuNjY3IDIxMy4zMzMgNDIuNjY3IDM4NCAxMTkuMjUzIDM4NCAyMTMuMzMzIDMwNy40MTMgMzg0IDIxMy4zMzMgMzg0eiIvPjxwYXRoIGQ9Ik0xOTIgMTA2LjY2N2g0Mi42Njd2NDIuNjY3SDE5MnoiLz48L3N2Zz4=);filter:brightness(45%);display:block}.tabs{margin-top:10px}.modal-title{text-overflow:ellipsis;overflow:hidden;white-space:nowrap}@media screen and (min-width:1500px){.container{max-width:1440px}}label{margin-top:.5rem;margin-bottom:0}.shrink[data-v-9b017c42]{flex-grow:inherit}#task-history[data-v-46960281]{font-family:monospace;font-size:12px}#log-tail-output span{display:block}span.DEBUG{color:#9e9e9e}span.WARNING{color:#ffb300}span.INFO{color:#039be5}span.ERROR,span.FATAL{color:#f4511e}span.ADMIN{color:#ee05ff}#log-tail-output{font-size:13px;font-family:monospace;padding:6px;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px;margin:3px;white-space:pre;color:#000;overflow:hidden}

sist2-admin/frontend/dist/index.html

@@ -0,0 +1 @@
+<!DOCTYPE html><html lang=""><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="favicon.ico"><title>sist2-admin</title><link href="css/app.0f0b676b.css" rel="preload" as="style"><link href="css/chunk-vendors.aa66c7e8.css" rel="preload" as="style"><link href="js/app.b34f501e.js" rel="preload" as="script"><link href="js/chunk-vendors.fad0ee6a.js" rel="preload" as="script"><link href="css/chunk-vendors.aa66c7e8.css" rel="stylesheet"><link href="css/app.0f0b676b.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but sist2-admin-vue doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="js/chunk-vendors.fad0ee6a.js"></script><script src="js/app.b34f501e.js"></script></body></html>

sist2-admin/frontend/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "sist2-admin-vue",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "serve": "vue-cli-service serve",
+    "build": "vue-cli-service build",
+    "watch": "vue-cli-service build --watch"
+  },
+  "dependencies": {
+    "axios": "^0.27.2",
+    "bootstrap-vue": "^2.21.2",
+    "core-js": "^3.6.5",
+    "moment": "^2.29.3",
+    "socket.io-client": "^4.5.1",
+    "vue": "^2.6.14",
+    "vue-i18n": "^8.24.4",
+    "vue-router": "^3.5.4",
+    "vuex": "^3.4.0"
+  },
+  "devDependencies": {
+    "@vue/cli-plugin-babel": "~4.5.13",
+    "@vue/cli-plugin-eslint": "~4.5.13",
+    "@vue/cli-plugin-router": "~4.5.13",
+    "@vue/cli-plugin-vuex": "~4.5.13",
+    "@vue/cli-service": "~4.5.13",
+    "babel-eslint": "^10.1.0",
+    "bootstrap": "^4.5.2",
+    "eslint": "^6.7.2",
+    "eslint-plugin-vue": "^6.2.2",
+    "vue-template-compiler": "^2.6.11"
+  },
+  "eslintConfig": {
+    "root": true,
+    "env": {
+      "node": true
+    },
+    "extends": [
+      "plugin:vue/essential",
+      "eslint:recommended"
+    ],
+    "parserOptions": {
+      "parser": "babel-eslint"
+    },
+    "rules": {}
+  },
+  "browserslist": [
+    "> 1%",
+    "last 2 versions",
+    "not dead"
+  ]
+}

sist2-admin/frontend/public/index.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width,initial-scale=1.0">
+    <link rel="icon" href="<%= BASE_URL %>favicon.ico">
+    <title>sist2-admin</title>
+  </head>
+  <body>
+    <noscript>
+      <strong>We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled. Please enable it to continue.</strong>
+    </noscript>
+    <div id="app"></div>
+    <!-- built files will be auto injected -->
+  </body>
+</html>

sist2-admin/frontend/src/App.vue

@@ -0,0 +1,98 @@
+<template>
+  <div id="app">
+    <NavBar></NavBar>
+    <b-container class="pt-4">
+      <b-alert show dismissible variant="info">
+        This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
+        to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on Github</a>. Thank you!
+      </b-alert>
+      <router-view/>
+    </b-container>
+  </div>
+</template>
+
+<script>
+import NavBar from "@/components/NavBar";
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  components: {NavBar},
+  data() {
+    return {
+      socket: null
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getSist2AdminInfo()
+        .then(resp => this.$store.commit("setSist2AdminInfo", resp.data));
+    this.$store.dispatch("loadBrowserSettings");
+    this.connectNotifications();
+    // this.socket.onclose = this.connectNotifications;
+  },
+  methods: {
+    connectNotifications() {
+      this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
+      this.socket.onopen = () => {
+        this.socket.send("Hello from client");
+      }
+
+      this.socket.onmessage = e => {
+        const notification = JSON.parse(e.data);
+        if (notification.message) {
+          notification.messageString = this.$t(notification.message).toString();
+        }
+
+        this.$store.dispatch("notify", notification)
+      }
+    }
+  }
+}
+</script>
+
+<style>
+html, body {
+  height: 100%;
+}
+
+#app {
+  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  /*text-align: center;*/
+  color: #2c3e50;
+  padding-bottom: 1em;
+  min-height: 100%;
+}
+
+.info-icon {
+  width: 1rem;
+  margin-right: 0.2rem;
+  cursor: pointer;
+  line-height: 1rem;
+  height: 1rem;
+  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+  filter: brightness(45%);
+  display: block;
+}
+
+.tabs {
+  margin-top: 10px;
+}
+
+.modal-title {
+  text-overflow: ellipsis;
+  overflow: hidden;
+  white-space: nowrap;
+}
+
+@media screen and (min-width: 1500px) {
+  .container {
+    max-width: 1440px;
+  }
+}
+
+label {
+  margin-top: 0.5rem;
+  margin-bottom: 0;
+}
+</style>

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -0,0 +1,117 @@
+import axios from "axios";
+
+class Sist2AdminApi {
+
+    constructor() {
+        this.baseUrl = window.location.protocol + "//" + window.location.host;
+    }
+
+    getJobs() {
+        return axios.get(`${this.baseUrl}/api/job/`);
+    }
+
+    getFrontends() {
+        return axios.get(`${this.baseUrl}/api/frontend/`);
+    }
+
+    getTasks() {
+        return axios.get(`${this.baseUrl}/api/task/`);
+    }
+
+    killTask(taskId) {
+        return axios.post(`${this.baseUrl}/api/task/${taskId}/kill`)
+    }
+
+    getTaskHistory() {
+        return axios.get(`${this.baseUrl}/api/task/history`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    getJob(name) {
+        return axios.get(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    getFrontend(name) {
+        return axios.get(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    startFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}/start`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    stopFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}/stop`);
+    }
+
+    /**
+     * @param {string} name
+     * @param job
+     */
+    updateJob(name, job) {
+        return axios.put(`${this.baseUrl}/api/job/${name}`, job);
+    }
+
+    /**
+     * @param {string} name
+     * @param frontend
+     */
+    updateFrontend(name, frontend) {
+        return axios.put(`${this.baseUrl}/api/frontend/${name}`, frontend);
+    }
+
+    /**
+     * @param {string} name
+     */
+    runJob(name) {
+        return axios.get(`${this.baseUrl}/api/job/${name}/run`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    deleteJob(name) {
+        return axios.delete(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    deleteFrontend(name) {
+        return axios.delete(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    createJob(name) {
+        return axios.post(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    createFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    pingEs(url, insecure) {
+        return axios.get(`${this.baseUrl}/api/ping_es`, {params: {url, insecure}});
+    }
+
+    getSist2AdminInfo() {
+        return axios.get(`${this.baseUrl}/api/`);
+    }
+}
+
+export default new Sist2AdminApi()

sist2-admin/frontend/src/components/FrontendListItem.vue

@@ -0,0 +1,31 @@
+<template>
+  <b-list-group-item action :to="`/frontend/${frontend.name}`">
+
+    <div class="d-flex w-100 justify-content-between">
+      <h5 class="mb-1" style="display: block">
+        {{ frontend.name }}
+        <b-badge variant="light">{{ formatBindAddress(frontend.web_options.bind) }}</b-badge>
+      </h5>
+
+      <div>
+        <b-badge v-if="frontend.running" variant="success">{{$t("online")}}</b-badge>
+        <b-badge v-else variant="secondary">{{$t("offline")}}</b-badge>
+      </div>
+    </div>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import {formatBindAddress} from "@/util";
+
+export default {
+  name: "FrontendListItem",
+  props: ["frontend"],
+  data() {
+    return {
+      formatBindAddress
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/components/IndexOptions.vue

@@ -0,0 +1,64 @@
+<template>
+  <div>
+    <label>{{ $t("indexOptions.threads") }}</label>
+    <b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.esUrl") }}</label>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <label>{{ $t("indexOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
+    <br>
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("indexOptions.batchSize") }}</label>
+    <b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("indexOptions.script") }}</label>
+    <b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
+  </div>
+
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "IndexOptions",
+  props: ["options"],
+  data() {
+    return {
+      showEsTestAlert: false,
+      esTestOk: false,
+      esTestMessage: "",
+    }
+  },
+  methods: {
+    update() {
+      this.$emit("change", this.options);
+    },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
+  },
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/JobCheckboxGroup.vue

@@ -0,0 +1,42 @@
+<template>
+  <div>
+    <h5>{{ $t("selectJobs") }}</h5>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-form-group v-else>
+      <b-form-checkbox-group
+          v-if="jobs.length > 0"
+          :checked="frontend.jobs"
+          @input="frontend.jobs = $event; $emit('input')"
+      >
+        <div v-for="job in jobs" :key="job.name">
+          <b-form-checkbox :disabled="job.status !== 'indexed'" :value="job.name">[{{ job.name }}]</b-form-checkbox>
+          <br/>
+        </div>
+      </b-form-checkbox-group>
+      <div v-else>
+        <span class="text-muted">{{ $t('jobOptions.noJobAvailable') }}</span>
+        &nbsp;<router-link to="/">{{$t("create")}}</router-link>
+      </div>
+    </b-form-group>
+  </div>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "JobCheckboxGroup",
+  props: ["frontend"],
+  mounted() {
+    Sist2AdminApi.getJobs().then(resp => {
+      this.jobs = resp.data;
+      this.loading = false;
+    });
+  },
+  data() {
+    return {
+      loading: true,
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/components/JobListItem.vue

@@ -0,0 +1,56 @@
+<template>
+  <b-list-group-item class="flex-column align-items-start" action :to="`job/${job.name}`">
+
+    <div class="d-flex w-100 justify-content-between">
+      <div>
+        <h5 class="mb-1">
+          {{ job.name }}
+        </h5>
+      </div>
+      <div>
+        <b-row>
+          <b-col>
+            <small v-if="job.last_index_date">
+              {{ $t("scanned") }} {{ formatLastIndexDate(job.last_index_date) }}</small>
+            <div v-else>&nbsp;</div>
+          </b-col>
+        </b-row>
+        <b-row v-if="job.schedule_enabled">
+          <b-col>
+            <small><code>{{job.cron_expression }}</code></small>
+          </b-col>
+        </b-row>
+        <b-row v-else>
+          <b-col>
+            &nbsp;
+          </b-col>
+        </b-row>
+
+      </div>
+    </div>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import moment from "moment";
+
+export default {
+  name: "JobListItem",
+  props: ["job"],
+  methods: {
+    formatLastIndexDate(dateString) {
+      if (dateString === null) {
+        return "";
+      }
+
+      const date = Date.parse(dateString);
+      return moment(date).fromNow();
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/JobOptions.vue

@@ -0,0 +1,51 @@
+<template>
+  <div>
+    <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
+      {{ $t("jobOptions.desktopNotifications") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
+      {{ $t("jobOptions.scheduleEnabled") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("jobOptions.cron") }}</label>
+    <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression" :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
+  </div>
+</template>
+
+<script>
+
+export default {
+  name: "JobOptions",
+  props: ["job"],
+  data() {
+    return {
+      cronValid: undefined
+    }
+  },
+  computed: {
+    desktopNotificationsEnabled() {
+      return this.$store.state.jobDesktopNotificationMap[this.job.name];
+    }
+  },
+  methods: {
+    updateNotifications(value) {
+      this.$store.dispatch("setJobDesktopNotification", {
+        job: this.job.name,
+        enabled: value
+      })
+    },
+    update() {
+      if (this.job.schedule_enabled) {
+        this.cronValid = /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(this.job.cron_expression);
+      } else {
+        this.cronValid = undefined;
+      }
+
+      if (this.cronValid !== false) {
+        this.$emit("change", this.job);
+      }
+    },
+  },
+}
+</script>

sist2-admin/frontend/src/components/NavBar.vue

@@ -0,0 +1,69 @@
+<template>
+  <b-navbar>
+    <b-navbar-brand to="/">
+      <Sist2Icon></Sist2Icon>
+    </b-navbar-brand>
+
+    <b-button class="ml-auto" to="/task" variant="link">{{ $t("tasks") }}</b-button>
+  </b-navbar>
+</template>
+
+<script>
+import Sist2Icon from "@/components/icons/Sist2Icon";
+
+export default {
+  name: "NavBar",
+  components: {Sist2Icon},
+  methods: {
+    tagline() {
+      return this.$store.state.sist2Info.tagline;
+    },
+    sist2Version() {
+      return this.$store.state.sist2Info.version;
+    },
+    isDebug() {
+      return this.$store.state.sist2Info.debug;
+    },
+    isLegacy() {
+      return this.$store.state.sist2Info.esVersionLegacy;
+    },
+    hideLegacy() {
+      return this.$store.state.optHideLegacy;
+    }
+  }
+}
+</script>
+
+<style scoped>
+.navbar {
+  box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
+  border-radius: 0;
+}
+
+.theme-black .navbar {
+  background: #546b7a30;
+  border-bottom: none;
+}
+
+.navbar-brand {
+  color: #222 !important;
+  font-size: 1.75rem;
+  padding: 0;
+}
+
+.navbar-brand:hover {
+  color: #000 !important;
+}
+
+.version {
+  color: #222 !important;
+  margin-left: -18px;
+  margin-top: -14px;
+  font-size: 11px;
+  font-family: monospace;
+}
+
+.btn-link {
+  color: #222;
+}
+</style>

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -0,0 +1,111 @@
+<template>
+  <div>
+    <label>{{ $t("scanOptions.path") }}</label>
+    <b-form-input v-model="options.path" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.threads") }}</label>
+    <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.memThrottle") }}</label>
+    <b-form-input type="number" min="0" v-model="options.mem_throttle" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
+    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailCount") }}</label>
+    <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailSize") }}</label>
+    <b-form-input type="number" min="100" v-model="options.thumbnail_size" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.contentSize") }}</label>
+    <b-form-input type="number" min="0" v-model="options.content_size" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.rewriteUrl") }}</label>
+    <b-form-input v-model="options.rewrite_url" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.depth") }}</label>
+    <b-form-input type="number" min="0" v-model="options.depth" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.archive") }}</label>
+    <b-form-select :options="['skip', 'list', 'shallow', 'recurse']" v-model="options.archive"
+                   @change="update()"></b-form-select>
+
+    <label>{{ $t("scanOptions.archivePassphrase") }}</label>
+    <b-form-input v-model="options.archive_passphrase" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.ocrLang") }}</label>
+    <b-alert variant="danger" show v-if="selectedOcrLangs.length === 0 && !disableOcrLang">{{ $t("scanOptions.ocrLangAlert") }}</b-alert>
+    <b-checkbox-group :disabled="disableOcrLang" v-model="selectedOcrLangs" @input="onOcrLangChange">
+      <b-checkbox v-for="lang in ocrLangs" :key="lang" :value="lang">{{ lang }}</b-checkbox>
+    </b-checkbox-group>
+
+    <!--    <b-form-input readonly v-model="options.ocr_lang" @change="update()"></b-form-input>-->
+
+    <div style="height: 10px"></div>
+
+    <b-form-checkbox v-model="options.ocr_images" @change="update()">
+      {{ $t("scanOptions.ocrImages") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.ocr_ebooks" @change="update()">
+      {{ $t("scanOptions.ocrEbooks") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("scanOptions.exclude") }}</label>
+    <b-form-input v-model="options.exclude" @change="update()"
+                  :placeholder="$t('scanOptions.excludePlaceholder')"></b-form-input>
+
+    <div style="height: 10px"></div>
+
+    <b-form-checkbox v-model="options.fast" @change="update()">
+      {{ $t("scanOptions.fast") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.checksums" @change="update()">
+      {{ $t("scanOptions.checksums") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.read_subtitles" @change="update()">
+      {{ $t("scanOptions.readSubtitles") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("scanOptions.memBuffer") }}</label>
+    <b-form-input type="number" min="0" v-model="options.mem_buffer" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.treemapThreshold") }}</label>
+    <b-form-input type="number" min="0" v-model="options.treemap_threshold" @change="update()"></b-form-input>
+  </div>
+</template>
+
+<script>
+
+export default {
+  name: "ScanOptions",
+  props: ["options"],
+  data() {
+    return {
+      disableOcrLang: false,
+      selectedOcrLangs: []
+    }
+  },
+  computed: {
+    ocrLangs() {
+      return this.$store.state.sist2AdminInfo?.tesseract_langs || [];
+    }
+  },
+  methods: {
+    onOcrLangChange() {
+      this.options.ocr_lang = this.selectedOcrLangs.join("+");
+    },
+    update() {
+      this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;
+      this.$emit("change", this.options);
+    },
+  },
+  mounted() {
+    this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;
+    this.selectedOcrLangs = this.options.ocr_lang ? this.options.ocr_lang.split("+") : [];
+  }
+}
+</script>

sist2-admin/frontend/src/components/TaskListItem.vue

@@ -0,0 +1,57 @@
+<template>
+  <b-list-group-item>
+    <b-row style="height: 50px">
+      <b-col><h5>{{ task.display_name }}</h5></b-col>
+      <b-col class="shrink">
+        <router-link class="btn btn-link" :to="`/log/${task.id}`">{{ $t("logs") }}</router-link>
+      </b-col>
+      <b-col class="shrink">
+        <b-btn variant="link" @click="killTask(task.id)">{{ $t("kill") }}</b-btn>
+      </b-col>
+    </b-row>
+
+    <b-row>
+      <b-col>
+        <b-progress :max="task.progress.count">
+          <b-progress-bar :value="task.progress.done" :label-html="label" :striped="!task.progress.waiting"/>
+        </b-progress>
+      </b-col>
+    </b-row>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "TaskListItem",
+  props: ["task"],
+  computed: {
+    label() {
+
+      const count = this.task.progress.count;
+      const done = this.task.progress.done;
+
+      return `<span>${done}/${count}</span>`
+    }
+  },
+  methods: {
+    killTask(taskId) {
+      sist2AdminApi.killTask(taskId).then(() => {
+        this.$bvToast.toast(this.$t("killConfirmation"), {
+          title: this.$t("killConfirmationTitle"),
+          variant: "success",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    }
+  }
+}
+</script>
+
+<style scoped>
+.shrink {
+  flex-grow: inherit;
+}
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -0,0 +1,75 @@
+<template>
+  <div>
+    <label>{{ $t("webOptions.esUrl") }}</label>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("webOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.lang") }}</label>
+    <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
+
+    <label>{{ $t("webOptions.bind") }}</label>
+    <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.tagline") }}</label>
+    <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+
+    <label>{{ $t("webOptions.auth") }}</label>
+    <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.tagAuth") }}</label>
+    <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+
+  </div>
+</template>
+
+<script>
+
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "WebOptions",
+  props: ["options", "frontendName"],
+  data() {
+    return {
+      showEsTestAlert: false,
+      esTestOk: false,
+      esTestMessage: "",
+    }
+  },
+  methods: {
+    update() {
+      if (!this.options.es_url.startsWith("https")) {
+        this.options.es_insecure_ssl = false;
+      }
+
+      this.$emit("change", this.options);
+    },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/icons/Sist2Icon.vue

@@ -0,0 +1,40 @@
+<template>
+  <svg
+      xmlns="http://www.w3.org/2000/svg"
+      width="27.868069mm"
+      height="7.6446671mm"
+      viewBox="0 0 27.868069 7.6446671"
+  >
+    <g transform="translate(-4.5018313,-4.1849793)">
+      <g
+          style="fill: currentColor;fill-opacity:1;stroke:none;stroke-width:0.26458332">
+        <path
+            d="m 6.3153296,11.829646 q -0.7717014,0 -1.8134983,-0.337619 v -0.916395 q 1.0128581,0.511252 1.803852,0.511252 0.5643067,0 0.901926,-0.236334 0.3376194,-0.236333 0.3376194,-0.63183 0,-0.3424428 -0.2845649,-0.5498376 Q 6.980922,9.4566645 6.3635609,9.3264399 L 5.9921796,9.2492698 Q 5.2301245,9.0949295 4.8732126,8.7428407 4.5211238,8.3859288 4.5211238,7.7733908 q 0,-0.7765245 0.5305447,-1.1961372 0.5305447,-0.4196126 1.5096409,-0.4196126 0.829579,0 1.6061036,0.3183268 V 7.3441319 Q 7.4101809,6.9004036 6.5854251,6.9004036 q -1.1671984,0 -1.1671984,0.7958171 0,0.2604492 0.1012858,0.4147895 0.1012858,0.1495171 0.3858507,0.2556261 0.2845649,0.1012858 0.8392253,0.2122179 l 0.3569119,0.067524 q 1.3408312,0.2652724 1.3408312,1.4614098 0,0.80064 -0.5691298,1.263661 -0.5691298,0.458197 -1.5578722,0.458197 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 11.943927,5.3087694 q -0.144694,0 -0.144694,-0.144694 V 4.3296733 q 0,-0.144694 0.144694,-0.144694 h 0.694531 q 0.144694,0 0.144694,0.144694 v 0.8344021 q 0,0.144694 -0.144694,0.144694 z M 13.5645,11.728361 q -0.795817,0 -1.234722,-0.511253 -0.434082,-0.516075 -0.434082,-1.4469398 V 6.9823969 H 10.714028 V 6.2878656 h 2.069124 v 3.4823026 q 0,0.5884228 0.221864,0.8971028 0.221865,0.308681 0.6463,0.308681 h 1.036974 v 0.752409 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 18.209178,11.829646 q -0.771701,0 -1.813498,-0.337619 v -0.916395 q 1.012858,0.511252 1.803852,0.511252 0.564306,0 0.901926,-0.236334 0.337619,-0.236333 0.337619,-0.63183 0,-0.3424428 -0.284565,-0.5498376 Q 18.87477,9.4566645 18.257409,9.3264399 l -0.371381,-0.07717 Q 17.123973,9.0949295 16.767061,8.7428407 16.414972,8.3859288 16.414972,7.7733908 q 0,-0.7765245 0.530545,-1.1961372 0.530545,-0.4196126 1.509641,-0.4196126 0.829579,0 1.606103,0.3183268 v 0.8681641 q -0.757232,-0.4437283 -1.581988,-0.4437283 -1.167198,0 -1.167198,0.7958171 0,0.2604492 0.101286,0.4147895 0.101286,0.1495171 0.385851,0.2556261 0.284565,0.1012858 0.839225,0.2122179 l 0.356912,0.067524 q 1.340831,0.2652724 1.340831,1.4614098 0,0.80064 -0.56913,1.263661 -0.56913,0.458197 -1.557872,0.458197 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 25.207545,11.709068 q -0.993565,0 -1.408355,-0.40032 -0.409966,-0.405143 -0.409966,-1.3794164 V 6.9775737 H 21.947107 V 6.2878656 h 1.442117 V 4.8746874 l 0.887457,-0.3858507 v 1.7990289 h 2.016069 v 0.6897081 h -2.016069 v 2.9517579 q 0,0.5932454 0.226687,0.8344024 0.226687,0.236333 0.790994,0.236333 h 0.998388 v 0.709001 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 27.995317,11.043476 q 0,-0.178456 0.120578,-0.299035 0.274919,-0.289388 0.651123,-0.684885 0.376205,-0.4003199 0.805464,-0.8681638 0.327973,-0.356912 0.491959,-0.5353679 0.16881,-0.1832791 0.255626,-0.2845649 0.09164,-0.1012858 0.178456,-0.2073948 0.255626,-0.3086805 0.405144,-0.5257215 0.15434,-0.2170411 0.250803,-0.4292589 0.168809,-0.3762045 0.168809,-0.7524089 0,-0.5980686 -0.352089,-0.935688 -0.356911,-0.3424425 -0.979096,-0.3424425 -0.863341,0 -1.938899,0.6414768 V 4.8361023 q 0.491959,-0.2363335 0.979096,-0.3569119 0.47749,-0.1205783 0.945334,-0.1205783 0.501606,0 0.940511,0.1350477 0.438905,0.1350478 0.766878,0.4244358 0.289388,0.2556261 0.463021,0.6270074 0.173633,0.3665582 0.173633,0.829579 0,0.4726671 -0.212218,0.9501574 -0.106109,0.2411567 -0.274919,0.4726671 -0.163986,0.2266873 -0.424435,0.540191 Q 31.270225,8.501684 31.077299,8.718725 30.884374,8.9357661 30.628748,9.2106847 30.445469,9.4084332 30.286305,9.5675966 30.131965,9.72676 29.958332,9.9003928 29.7847,10.069203 29.558012,10.300713 29.336148,10.5274 29.012998,10.869843 h 3.356901 v 0.819932 h -4.374582 z"
+            style="stroke-width:0.26458332"
+        />
+      </g>
+    </g>
+  </svg>
+</template>
+
+<script>
+export default {
+  name: "Sist2Icon"
+}
+</script>

sist2-admin/frontend/src/i18n/messages.js

@@ -0,0 +1,110 @@
+export default {
+    en: {
+        start: "Start",
+        stop: "Stop",
+        go: "Go",
+        online: "online",
+        offline: "offline",
+        delete: "Delete",
+        runNow: "Index now",
+        create: "Create",
+        test: "Test",
+
+        jobTitle: "job configuration",
+        tasks: "Tasks",
+        runningTasks: "Running tasks",
+        frontends: "Frontends",
+        jobDisabled: "There is no valid index for this job",
+        status: "Status",
+
+        taskHistory: "Task history",
+        taskName: "Task name",
+        taskStarted: "Started",
+        taskDuration: "Duration",
+        taskStatus: "Status",
+        logs: "Logs",
+        kill: "Kill",
+        killConfirmation: "SIGTERM signal sent to sist2 process",
+        killConfirmationTitle: "Confirmation",
+        follow: "Follow",
+        wholeFile: "Whole file",
+        logLevel: "Log level",
+        logMode: "Follow mode",
+        logFile: "Reading log file",
+
+        jobs: "Jobs",
+        newJobName: "New job name",
+        newJobHelp: "Create a new job to get started!",
+        newFrontendName: "New frontend name",
+        scanned: "last scan",
+        autoStart: "Start automatically",
+
+        runJobConfirmationTitle: "Task queued",
+        runJobConfirmation: "Check the Tasks page to monitor the status.",
+
+        monitoring: "Monitoring",
+        enableMonitoring: "Enable monitoring",
+        extraQueryArgs: "Extra query arguments when launching from sist2-admin",
+        customUrl: "Custom URL when launching from sist2-admin",
+
+        selectJobs: "Select jobs",
+        webOptions: {
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            lang: "UI Language",
+            bind: "Listen address",
+            tagline: "Tagline in navbar",
+            auth: "Basic auth in user:password format",
+            tagAuth: "Basic auth in user:password format for tagging",
+        },
+        scanOptions: {
+            title: "Scanning options",
+            path: "Path",
+            threads: "Number of threads",
+            memThrottle: "Total memory threshold in MiB for scan throttling",
+            thumbnailQuality: "Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best",
+            thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
+            thumbnailSize: "Thumbnail size, in pixels",
+            contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",
+            rewriteUrl: "Serve files from this url instead of from disk",
+            depth: "Scan up to this many subdirectories deep",
+            archive: "Archive file mode",
+            archivePassphrase: "Passphrase for encrypted archive files",
+            ocrLang: "Tesseract language",
+            ocrLangAlert: "You must select at least one language",
+            ocrEbooks: "Enable OCR'ing of ebook files",
+            ocrImages: "Enable OCR'ing of image files",
+            exclude: "Files that match this regex will not be scanned",
+            excludePlaceholder: "Exclude",
+            fast: "Only index file names & mime type",
+            checksums: "Calculate file checksums when scanning",
+            readSubtitles: "Read subtitles from media files",
+            memBuffer: "Maximum memory buffer size per thread in MiB for files inside archives",
+            treemapThreshold: "Relative size threshold for treemap"
+        },
+        indexOptions: {
+            title: "Indexing options",
+            threads: "Number of threads",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            batchSize: "Index batch size",
+            script: "User script"
+        },
+        jobOptions: {
+            title: "Job options",
+            cron: "Job schedule",
+            scheduleEnabled: "Enable scheduled re-scan",
+            noJobAvailable: "No jobs available.",
+            desktopNotifications: "Desktop notifications"
+        },
+        frontendOptions: {
+            title: "Frontend options",
+            noJobSelectedWarning: "You must select at least one job to start this frontend"
+        },
+        notifications: {
+            indexCompleted: "Task completed for [$JOB$]"
+        }
+    }
+}

sist2-admin/frontend/src/main.js

@@ -0,0 +1,31 @@
+import Vue from 'vue'
+
+import { BootstrapVue, IconsPlugin } from 'bootstrap-vue'
+
+import "bootstrap/dist/css/bootstrap.min.css"
+import "bootstrap-vue/dist/bootstrap-vue.min.css"
+
+Vue.use(BootstrapVue);
+Vue.use(IconsPlugin);
+
+import App from './App.vue';
+import router from './router';
+import store from './store';
+import VueI18n from "vue-i18n";
+import messages from "@/i18n/messages";
+
+Vue.use(VueI18n);
+
+const i18n = new VueI18n({
+  locale: "en",
+  messages: messages
+});
+
+Vue.config.productionTip = false
+
+new Vue({
+  router,
+  store,
+  i18n,
+  render: h => h(App)
+}).$mount('#app')

sist2-admin/frontend/src/router/index.js

@@ -0,0 +1,45 @@
+import Vue from 'vue'
+import VueRouter from 'vue-router'
+import Home from '../views/Home.vue'
+import Job from "@/views/Job";
+import Tasks from "@/views/Tasks";
+import Frontend from "@/views/Frontend";
+import Tail from "@/views/Tail";
+
+Vue.use(VueRouter);
+
+const routes = [
+  {
+    path: "/",
+    name: "Home",
+    component: Home
+  },
+  {
+    path: "/job/:name",
+    name: "Job",
+    component: Job
+  },
+  {
+    path: "/task/",
+    name: "Tasks",
+    component: Tasks
+  },
+  {
+    path: "/frontend/:name",
+    name: "Frontend",
+    component: Frontend
+  },
+  {
+    path: "/log/:taskId",
+    name: "Tail",
+    component: Tail
+  },
+]
+
+const router = new VueRouter({
+  mode: "hash",
+  base: process.env.BASE_URL,
+  routes
+})
+
+export default router

sist2-admin/frontend/src/store/index.js

@@ -0,0 +1,63 @@
+import Vue from "vue";
+import Vuex from "vuex";
+
+Vue.use(Vuex);
+
+function saveBrowserSettings(state) {
+    const settings = {
+        jobDesktopNotificationMap: state.jobDesktopNotificationMap
+    };
+    localStorage.setItem("sist2-admin-settings", JSON.stringify(settings));
+
+    console.log("SAVED");
+    console.log(settings);
+}
+
+export default new Vuex.Store({
+    state: {
+        sist2AdminInfo: null,
+        jobDesktopNotificationMap: {}
+    },
+    mutations: {
+        setSist2AdminInfo: (state, payload) => state.sist2AdminInfo = payload,
+        setJobDesktopNotificationMap: (state, payload) => state.jobDesktopNotificationMap = payload,
+    },
+    actions: {
+        notify: async ({state}, notification) => {
+
+            if (!state.jobDesktopNotificationMap[notification.job]) {
+                console.log("pass");
+                return;
+            }
+
+            new Notification(notification.messageString.replace("$JOB$", notification.job));
+        },
+        setJobDesktopNotification: async ({state}, {job, enabled}) => {
+
+            if (enabled === true) {
+                const permission = await Notification.requestPermission()
+
+                if (permission !== "granted") {
+                    return false;
+                }
+            }
+
+            state.jobDesktopNotificationMap[job] = enabled;
+            saveBrowserSettings(state);
+
+            return true;
+        },
+        loadBrowserSettings({commit}) {
+            const settingString = localStorage.getItem("sist2-admin-settings");
+
+            if (!settingString) {
+                return;
+            }
+
+            const settings = JSON.parse(settingString);
+
+            commit("setJobDesktopNotificationMap", settings["jobDesktopNotificationMap"]);
+        }
+    },
+    modules: {}
+})

sist2-admin/frontend/src/util.js

@@ -0,0 +1,8 @@
+
+export function formatBindAddress(address) {
+    if (address.startsWith("0.0.0.0")) {
+        return address.slice("0.0.0.0".length)
+    }
+
+    return address
+}

sist2-admin/frontend/src/views/Frontend.vue

@@ -0,0 +1,133 @@
+<template>
+  <b-card>
+    <b-card-title>
+      {{ name }}
+      <small style="vertical-align: top">
+        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+      </small>
+    </b-card-title>
+
+    <div class="mb-3" v-if="!loading">
+      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+          $t("start")
+        }}
+      </b-button>
+      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+          $t("stop")
+        }}
+      </b-button>
+      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+        {{ $t("go") }}
+      </b-button>
+      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+    </div>
+
+
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card-body v-else>
+
+      <h4>{{ $t("frontendOptions.title") }}</h4>
+      <b-card>
+        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+          {{ $t("autoStart") }}
+        </b-form-checkbox>
+
+        <b-form-checkbox v-model="frontend.enable_monitoring" @change="update()">
+          {{ $t("enableMonitoring") }}
+        </b-form-checkbox>
+
+        <label>{{ $t("extraQueryArgs") }}</label>
+        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+
+        <label>{{ $t("customUrl") }}</label>
+        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+
+        <br/>
+
+        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+
+        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("jobOptions.title") }}</h4>
+      <b-card>
+        <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
+      </b-card>
+    </b-card-body>
+
+  </b-card>
+</template>
+
+<script>
+
+import Sist2AdminApi from "@/Sist2AdminApi";
+import JobCheckboxGroup from "@/components/JobCheckboxGroup";
+import WebOptions from "@/components/WebOptions";
+
+export default {
+  name: 'Frontend',
+  components: {JobCheckboxGroup, WebOptions},
+  data() {
+    return {
+      loading: true,
+      frontend: null,
+    }
+  },
+  computed: {
+    valid() {
+      return !this.loading && this.frontend.jobs.length > 0;
+    },
+    frontendUrl() {
+      if (this.frontend.custom_url) {
+        return this.frontend.custom_url + this.args;
+      }
+
+      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+      }
+
+      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+    },
+    name() {
+      return this.$route.params.name;
+    },
+    port() {
+      return this.frontend.web_options.bind.split(":")[1]
+    },
+    args() {
+      const args = this.frontend.extra_query_args;
+      if (args !== "") {
+        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+      }
+      return "";
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getFrontend(this.name).then(resp => {
+      this.frontend = resp.data;
+      this.loading = false;
+    });
+  },
+  methods: {
+    start() {
+      this.frontend.running = true;
+      Sist2AdminApi.startFrontend(this.name)
+    },
+    stop() {
+      this.frontend.running = false;
+      Sist2AdminApi.stopFrontend(this.name)
+    },
+    deleteFrontend() {
+      Sist2AdminApi.deleteFrontend(this.name).then(() => {
+        this.$router.push("/frontends");
+      });
+    },
+    update() {
+      Sist2AdminApi.updateFrontend(this.name, this.frontend);
+    },
+  }
+}
+</script>

sist2-admin/frontend/src/views/Home.vue

@@ -0,0 +1,122 @@
+<template>
+  <div>
+    <b-card>
+      <b-card-title>{{ $t("jobs") }}</b-card-title>
+      <b-row>
+        <b-col>
+          <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
+          <b-popover
+              :show.sync="showHelp"
+              target="new-job"
+              placement="top"
+              triggers="manual"
+              variant="primary"
+              :content="$t('newJobHelp')"
+          ></b-popover>
+        </b-col>
+        <b-col>
+          <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
+          </b-button>
+        </b-col>
+      </b-row>
+
+      <hr/>
+
+      <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
+      <b-list-group v-else>
+        <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
+      </b-list-group>
+    </b-card>
+
+    <br/>
+
+    <b-card>
+
+      <b-card-title>{{ $t("frontends") }}</b-card-title>
+
+      <b-row>
+        <b-col>
+          <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
+        </b-col>
+        <b-col>
+          <b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
+            {{ $t("create") }}
+          </b-button>
+        </b-col>
+      </b-row>
+
+      <hr/>
+
+      <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
+      <b-list-group v-else>
+        <FrontendListItem v-for="frontend in frontends"
+                          :key="frontend.name" :frontend="frontend"></FrontendListItem>
+      </b-list-group>
+
+    </b-card>
+  </div>
+</template>
+
+<script>
+import JobListItem from "@/components/JobListItem";
+import {formatBindAddress} from "@/util";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import FrontendListItem from "@/components/FrontendListItem";
+
+export default {
+  name: "Jobs",
+  components: {JobListItem, FrontendListItem},
+  data() {
+    return {
+      jobsLoading: true,
+      newJobName: "",
+      jobs: [],
+
+      frontendsLoading: true,
+      frontends: [],
+      formatBindAddress,
+      newFrontendName: "",
+
+      showHelp: false
+    }
+  },
+  mounted() {
+    this.loading = true;
+    this.reload();
+  },
+  methods: {
+    jobNameValid(name) {
+      if (this.jobs.some(job => job.name === name)) {
+        return false;
+      }
+
+      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+    },
+    frontendNameValid(name) {
+      if (this.frontends.some(job => job.name === name)) {
+        return false;
+      }
+
+      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+    },
+    reload() {
+      Sist2AdminApi.getJobs().then(resp => {
+        this.jobs = resp.data;
+        this.jobsLoading = false;
+
+        this.showHelp = this.jobs.length === 0;
+      });
+      Sist2AdminApi.getFrontends().then(resp => {
+        this.frontends = resp.data;
+        this.frontendsLoading = false;
+      });
+    },
+    createJob() {
+      Sist2AdminApi.createJob(this.newJobName).then(this.reload);
+    },
+    createFrontend() {
+      Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/views/Job.vue

@@ -0,0 +1,92 @@
+<template>
+  <b-card>
+    <b-card-title>
+      [{{ getName() }}]
+      {{ $t("jobTitle") }}
+    </b-card-title>
+
+    <div class="mb-3">
+      <b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
+      <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
+    </div>
+
+    <div v-if="job">
+      {{ $t("status") }}: <code>{{ job.status }}</code>
+    </div>
+
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card-body v-else>
+
+      <h4>{{ $t("jobOptions.title") }}</h4>
+      <b-card>
+        <JobOptions :job="job" @change="update"></JobOptions>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("scanOptions.title") }}</h4>
+      <b-card>
+        <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("indexOptions.title") }}</h4>
+      <b-card>
+        <IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
+      </b-card>
+
+    </b-card-body>
+
+  </b-card>
+</template>
+
+<script>
+import ScanOptions from "@/components/ScanOptions";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import IndexOptions from "@/components/IndexOptions";
+import JobOptions from "@/components/JobOptions";
+
+export default {
+  name: "Job",
+  components: {
+    IndexOptions,
+    ScanOptions,
+    JobOptions
+  },
+  data() {
+    return {
+      loading: true,
+      job: null
+    }
+  },
+  methods: {
+    getName() {
+      return this.$route.params.name;
+    },
+    update() {
+      Sist2AdminApi.updateJob(this.getName(), this.job);
+    },
+    runJob() {
+      Sist2AdminApi.runJob(this.getName()).then(() => {
+        this.$bvToast.toast(this.$t("runJobConfirmation"), {
+          title: this.$t("runJobConfirmationTitle"),
+          variant: "success",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    },
+    deleteJob() {
+      Sist2AdminApi.deleteJob(this.getName()).then(() => {
+        this.$router.push("/");
+      })
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getJob(this.getName()).then(resp => {
+      this.loading = false;
+      this.job = resp.data;
+    })
+  }
+}
+</script>

sist2-admin/frontend/src/views/Tail.vue

@@ -0,0 +1,168 @@
+<template>
+  <b-card>
+    <b-card-body>
+
+      <h4 class="mb-3">{{ taskId }} {{ $t("logs") }}</h4>
+
+      <div v-if="$store.state.sist2AdminInfo">
+        {{ $t("logFile") }}
+        <code>{{ $store.state.sist2AdminInfo.logs_folder }}/sist2-{{ taskId }}.log</code>
+        <br/>
+        <br/>
+      </div>
+
+      <b-row>
+        <b-col>
+          <span>{{ $t("logLevel") }}</span>
+          <b-select :options="levels.slice(0, -1)" v-model="logLevel" @input="connect()"></b-select>
+        </b-col>
+        <b-col>
+          <span>{{ $t("logMode") }}</span>
+          <b-select :options="modeOptions" v-model="mode" @input="connect()"></b-select>
+        </b-col>
+      </b-row>
+
+      <div id="log-tail-output" class="mt-3 ml-1"></div>
+
+    </b-card-body>
+  </b-card>
+</template>
+
+<script>
+
+export default {
+  name: "Tail",
+  data() {
+    return {
+      logLevel: "DEBUG",
+      levels: ["DEBUG", "INFO", "WARNING", "ERROR", "ADMIN", "FATAL"],
+      socket: null,
+      mode: "follow",
+      modeOptions: [
+        {
+          "text": this.$t('follow'),
+          "value": "follow"
+        },
+        {
+          "text": this.$t('wholeFile'),
+          "value": "wholeFile"
+        }
+      ]
+    }
+  },
+  computed: {
+    taskId: function () {
+      return this.$route.params.taskId;
+    }
+  },
+  methods: {
+    connect() {
+      let lineCount = 0;
+      const outputElem = document.getElementById("log-tail-output")
+      outputElem.replaceChildren();
+      if (this.socket !== null) {
+        this.socket.close();
+      }
+
+      const n = this.mode === "follow" ? 32 : 9999999999;
+      this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
+      this.socket.onopen = () => {
+        this.socket.send("Hello from client");
+      }
+
+      this.socket.onmessage = e => {
+        let message;
+        try {
+          message = JSON.parse(e.data);
+        } catch {
+          console.error(e.data)
+          return;
+        }
+
+        if ("ping" in message) {
+          return;
+        }
+
+        if (message.level === undefined) {
+
+          if ("stderr" in message) {
+            message.level = "ERROR";
+            message.message = message["stderr"];
+          } else {
+            message.level = "ADMIN";
+            message.message = message["sist2-admin"];
+          }
+          message.datetime = ""
+          message.filepath = ""
+        }
+
+        if (this.levels.indexOf(message.level) < this.levels.indexOf(this.logLevel)) {
+          return;
+        }
+
+        const logLine = `${message.datetime} [${message.level} ${message.filepath}] ${message.message}`;
+
+        const span = document.createElement("span");
+        span.setAttribute("class", message.level);
+        span.appendChild(document.createTextNode(logLine));
+
+        outputElem.appendChild(span);
+        lineCount += 1;
+
+        if (this.mode === "follow" && lineCount >= n) {
+          outputElem.firstChild.remove();
+        }
+      }
+    }
+  },
+  mounted() {
+    this.connect()
+  }
+}
+
+</script>
+
+<style>
+#log-tail-output span {
+  display: block;
+}
+
+span.DEBUG {
+  color: #9E9E9E;
+}
+
+span.WARNING {
+  color: #FFB300;
+}
+
+span.INFO {
+  color: #039BE5;
+}
+
+span.ERROR {
+  color: #F4511E;
+}
+
+span.FATAL {
+  color: #F4511E;
+}
+
+span.ADMIN {
+  color: #ee05ff;
+}
+
+
+#log-tail-output {
+  font-size: 13px;
+  font-family: monospace;
+
+  padding: 6px;
+  background-color: #f5f5f5;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  margin: 3px;
+  white-space: pre;
+  color: #000;
+  overflow: hidden;
+}
+</style>

sist2-admin/frontend/src/views/Tasks.vue

@@ -0,0 +1,124 @@
+<template>
+  <div>
+
+    <b-card v-if="tasks.length > 0">
+      <h2>{{ $t("runningTasks") }}</h2>
+      <b-list-group>
+        <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
+      </b-list-group>
+    </b-card>
+
+    <b-card class="mt-4">
+
+      <b-card-title>{{ $t("taskHistory") }}</b-card-title>
+
+      <br/>
+
+      <b-table
+          id="task-history"
+          :items="historyItems"
+          :fields="historyFields"
+          :current-page="historyCurrentPage"
+          :tbody-tr-class="rowClass"
+          :per-page="10"
+      >
+        <template #cell(logs)="data">
+          <router-link :to="`/log/${data.item.logs}`">{{ $t("logs") }}</router-link>
+        </template>
+
+      </b-table>
+
+      <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
+                    :per-page="10"></b-pagination>
+
+    </b-card>
+  </div>
+</template>
+
+<script>
+import TaskListItem from "@/components/TaskListItem";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import moment from "moment";
+
+export default {
+  name: 'Tasks',
+  components: {TaskListItem},
+  data() {
+    return {
+      loading: true,
+      tasks: [],
+      taskHistory: [],
+      timerId: null,
+      historyFields: [
+        {key: "name", label: this.$t("taskName")},
+        {key: "time", label: this.$t("taskStarted")},
+        {key: "duration", label: this.$t("taskDuration")},
+        {key: "status", label: this.$t("taskStatus")},
+        {key: "logs", label: this.$t("logs")},
+      ],
+      historyCurrentPage: 1,
+      historyItems: []
+    }
+  },
+  props: {
+    msg: String
+  },
+  mounted() {
+    this.loading = true;
+    this.update().then(() => this.loading = false);
+
+    this.timerId = window.setInterval(this.update, 1000);
+    this.updateHistory();
+  },
+  destroyed() {
+    if (this.timerId) {
+      window.clearInterval(this.timerId);
+    }
+  },
+  methods: {
+    rowClass(row) {
+      if (row.status === "failed") {
+        return "table-danger";
+      }
+      return null;
+    },
+    updateHistory() {
+      Sist2AdminApi.getTaskHistory().then(resp => {
+        this.historyItems = resp.data.map(row => ({
+          id: row.id,
+          name: row.name,
+          duration: this.taskDuration(row),
+          time: moment(row.started).format("dd, MMM Do YYYY, HH:mm:ss"),
+          logs: row.id,
+          status: row.return_code === 0 ? "ok" : "failed"
+        }));
+      });
+    },
+    update() {
+      return Sist2AdminApi.getTasks().then(resp => {
+        this.tasks = resp.data;
+      })
+    },
+    taskDuration(task) {
+      const start = moment(task.started);
+      const end = moment(task.ended);
+
+      let duration = moment.utc(end.diff(start)).format("HH[h] mm[m] ss[s]");
+
+      duration = duration.replace("00h ", "");
+      duration = duration.replace(/^00m /, "");
+      duration = duration.replace(/00s/, "<1s");
+      duration = duration.replace(/^0/, "");
+
+      return duration;
+    }
+  }
+}
+</script>
+
+<style scoped>
+#task-history {
+  font-family: monospace;
+  font-size: 12px;
+}
+</style>

sist2-admin/frontend/vue.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+    publicPath: ""
+};

sist2-admin/requirements.txt

@@ -0,0 +1,5 @@
+fastapi
+git+https://github.com/simon987/hexlib.git
+uvicorn
+websockets
+pycron

sist2-admin/sist2_admin/app.py

@@ -0,0 +1,392 @@
+import asyncio
+import os
+import signal
+from datetime import datetime
+from urllib.parse import urlparse
+
+import requests
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from hexlib.db import PersistentState
+from requests import ConnectionError
+from requests.exceptions import SSLError
+from starlette.middleware.cors import CORSMiddleware
+from starlette.responses import RedirectResponse
+from starlette.staticfiles import StaticFiles
+from starlette.websockets import WebSocket
+from websockets.exceptions import ConnectionClosed
+
+import cron
+from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
+from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
+from notifications import Subscribe, Notifications
+from sist2 import Sist2
+from state import PickleTable, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
+from web import Sist2Frontend
+
+VERSION = "1.0"
+
+sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
+db = PersistentState(table_factory=PickleTable, dbfile=os.path.join(DATA_FOLDER, "state.db"))
+notifications = Notifications()
+task_queue = TaskQueue(sist2, db, notifications)
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_credentials=True,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.mount("/ui/", StaticFiles(directory="./frontend/dist", html=True), name="static")
+
+
+@app.get("/")
+async def home():
+    return RedirectResponse("ui")
+
+
+@app.get("/api")
+async def api():
+    return {
+        "version": VERSION,
+        "tesseract_langs": TESSERACT_LANGS,
+        "logs_folder": LOG_FOLDER
+    }
+
+
+@app.get("/api/job/{name:str}")
+async def get_job(name: str):
+    row = db["jobs"][name]
+    if row:
+        return row["job"]
+    raise HTTPException(status_code=404)
+
+
+@app.get("/api/frontend/{name:str}")
+async def get_frontend(name: str):
+    row = db["frontends"][name]
+    if row:
+        frontend = row["frontend"]
+        frontend: Sist2Frontend
+        frontend.running = frontend.name in RUNNING_FRONTENDS
+        return frontend
+    raise HTTPException(status_code=404)
+
+
+@app.get("/api/job/")
+async def get_jobs():
+    return [row["job"] for row in db["jobs"]]
+
+
+@app.put("/api/job/{name:str}")
+async def update_job(name: str, job: Sist2Job):
+    # TODO: Check etag
+
+    job.last_modified = datetime.now()
+    row = db["jobs"][name]
+    if not row:
+        raise HTTPException(status_code=404)
+
+    args_that_trigger_full_scan = [
+        "path",
+        "thumbnail_count",
+        "thumbnail_quality",
+        "thumbnail_size",
+        "content_size",
+        "depth",
+        "archive",
+        "archive_passphrase",
+        "ocr_lang",
+        "ocr_images",
+        "ocr_ebooks",
+        "fast",
+        "checksums",
+        "read_subtitles",
+    ]
+    for arg in args_that_trigger_full_scan:
+        if getattr(row["job"].scan_options, arg) != getattr(job.scan_options, arg):
+            job.do_full_scan = True
+
+    db["jobs"][name] = {"job": job}
+
+
+@app.put("/api/frontend/{name:str}")
+async def update_frontend(name: str, frontend: Sist2Frontend):
+    db["frontends"][name] = {"frontend": frontend}
+
+    # TODO: Check etag
+
+    return "ok"
+
+
+@app.get("/api/task/")
+async def get_tasks():
+    return list(map(lambda t: t.json(), task_queue.tasks()))
+
+
+@app.get("/api/task/history")
+async def task_history():
+    return list(db["task_done"].sql("ORDER BY started DESC"))
+
+
+@app.post("/api/task/{task_id:str}/kill")
+async def kill_job(task_id: str):
+    return task_queue.kill_task(task_id)
+
+
+def _run_job(job: Sist2Job):
+    job.last_modified = datetime.now()
+    if job.status == JobStatus("created"):
+        job.status = JobStatus("started")
+    db["jobs"][job.name] = {"job": job}
+
+    scan_task = Sist2ScanTask(job, f"Scan [{job.name}]")
+    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=scan_task)
+
+    task_queue.submit(scan_task)
+    task_queue.submit(index_task)
+
+
+@app.get("/api/job/{name:str}/run")
+async def run_job(name: str):
+    row = db["jobs"][name]
+    if not row:
+        raise HTTPException(status_code=404)
+
+    _run_job(row["job"])
+
+    return "ok"
+
+
+@app.delete("/api/job/{name:str}")
+async def delete_job(name: str):
+    row = db["jobs"][name]
+    if row:
+        del db["jobs"][name]
+    else:
+        raise HTTPException(status_code=404)
+
+
+@app.delete("/api/frontend/{name:str}")
+async def delete_frontend(name: str):
+    if name in RUNNING_FRONTENDS:
+        os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        del RUNNING_FRONTENDS[name]
+
+    row = db["frontends"][name]
+    if row:
+        del db["frontends"][name]
+    else:
+        raise HTTPException(status_code=404)
+
+
+@app.post("/api/job/{name:str}")
+async def create_job(name: str):
+    if db["jobs"][name]:
+        raise ValueError("Job with the same name already exists")
+
+    job = Sist2Job.create_default(name)
+    db["jobs"][name] = {"job": job}
+
+    return job
+
+
+@app.post("/api/frontend/{name:str}")
+async def create_frontend(name: str):
+    if db["frontend"][name]:
+        raise ValueError("Frontend with the same name already exists")
+
+    frontend = Sist2Frontend.create_default(name)
+    db["frontends"][name] = {"frontend": frontend}
+
+    return frontend
+
+
+@app.get("/api/ping_es")
+async def ping_es(url: str, insecure: bool):
+    return check_es_version(url, insecure)
+
+
+def check_es_version(es_url: str, insecure: bool):
+    try:
+        url = urlparse(es_url)
+        if url.username:
+            auth = (url.username, url.password)
+            es_url = f"{url.scheme}://{url.hostname}:{url.port}"
+        else:
+            auth = None
+        r = requests.get(es_url, verify=insecure, auth=auth)
+    except SSLError:
+        return {
+            "ok": False,
+            "message": "Invalid SSL certificate"
+        }
+    except ConnectionError as e:
+        return {
+            "ok": False,
+            "message": "Connection refused"
+        }
+    except ValueError as e:
+        return {
+            "ok": False,
+            "message": str(e)
+        }
+
+    if r.status_code == 401:
+        return {
+            "ok": False,
+            "message": "Authentication failure"
+        }
+
+    try:
+        return {
+            "ok": True,
+            "message": "Elasticsearch version " + r.json()["version"]["number"]
+        }
+    except:
+        return {
+            "ok": False,
+            "message": "Could not read version"
+        }
+
+
+def start_frontend_(frontend: Sist2Frontend):
+    frontend.web_options.indices = list(map(lambda j: db["jobs"][j]["job"].last_index, frontend.jobs))
+
+    pid = sist2.web(frontend.web_options, frontend.name)
+    RUNNING_FRONTENDS[frontend.name] = pid
+
+
+@app.post("/api/frontend/{name:str}/start")
+async def start_frontend(name: str):
+    row = db["frontends"][name]
+    if not row:
+        raise HTTPException(status_code=404)
+
+    start_frontend_(row["frontend"])
+
+
+@app.post("/api/frontend/{name:str}/stop")
+async def stop_frontend(name: str):
+    if name in RUNNING_FRONTENDS:
+        os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        del RUNNING_FRONTENDS[name]
+
+
+@app.get("/api/frontend/")
+async def get_frontends():
+    res = []
+    for row in db["frontends"]:
+        frontend = row["frontend"]
+        frontend: Sist2Frontend
+        frontend.running = frontend.name in RUNNING_FRONTENDS
+        res.append(frontend)
+    return res
+
+
+def tail(filepath: str, n: int):
+    with open(filepath) as file:
+
+        reached_eof = False
+        buffer = []
+
+        line = ""
+        while True:
+            tmp = file.readline()
+            if tmp:
+                line += tmp
+
+                if line.endswith("\n"):
+
+                    if reached_eof:
+                        yield line
+                    else:
+                        if len(buffer) > n:
+                            buffer.pop(0)
+                        buffer.append(line)
+                line = ""
+            else:
+                if not reached_eof:
+                    reached_eof = True
+                    yield from buffer
+                yield None
+
+
+@app.websocket("/notifications")
+async def ws_tail_log(websocket: WebSocket):
+    await websocket.accept()
+
+    try:
+        await websocket.receive_text()
+
+        async with Subscribe(notifications) as ob:
+            async for notification in ob.notifications():
+                await websocket.send_json(notification)
+                print(notification)
+
+    except ConnectionClosed:
+        return
+
+
+@app.websocket("/log/{task_id}")
+async def ws_tail_log(websocket: WebSocket, task_id: str, n: int):
+    log_file = os.path.join(LOG_FOLDER, f"sist2-{task_id}.log")
+
+    await websocket.accept()
+
+    try:
+        await websocket.receive_text()
+    except ConnectionClosed:
+        return
+
+    while True:
+        for line in tail(log_file, n):
+
+            try:
+                if line:
+                    await websocket.send_text(line)
+                else:
+                    await websocket.send_json({"ping": ""})
+                    await asyncio.sleep(0.1)
+            except ConnectionClosed:
+                return
+
+
+def main():
+    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0")
+
+
+def initialize_db():
+    db["sist2_admin"]["info"] = {"version": DB_SCHEMA_VERSION}
+
+    frontend = Sist2Frontend.create_default("default")
+    db["frontends"]["default"] = {"frontend": frontend}
+
+    logger.info("Initialized database.")
+
+
+def start_frontends():
+    for row in db["frontends"]:
+        frontend: Sist2Frontend = row["frontend"]
+        if frontend.auto_start and len(frontend.jobs) > 0:
+            start_frontend_(frontend)
+
+
+if __name__ == '__main__':
+
+    if not db["sist2_admin"]["info"]:
+        initialize_db()
+    elif db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
+        print("Database has incompatible schema version! Delete state.db to continue.")
+        exit(-1)
+
+    start_frontends()
+    cron.initialize(db, _run_job)
+
+    logger.info("Started sist2-admin. Hello!")
+
+    main()

sist2-admin/sist2_admin/config.py

@@ -0,0 +1,30 @@
+import os
+import logging
+import sys
+from logging import StreamHandler
+from logging.handlers import RotatingFileHandler
+
+MAX_LOG_SIZE = 1 * 1024 * 1024
+
+SIST2_BINARY = os.environ.get("SIST2_BINARY", "/root/sist2")
+DATA_FOLDER = os.environ.get("DATA_FOLDER", "/sist2-admin/")
+LOG_FOLDER = os.path.join(DATA_FOLDER, "logs")
+WEBSERVER_PORT = 8080
+
+os.makedirs(LOG_FOLDER, exist_ok=True)
+os.makedirs(DATA_FOLDER, exist_ok=True)
+
+logger = logging.Logger("sist2-admin")
+
+_log_file = os.path.join(LOG_FOLDER, "sist2-admin.log")
+_log_fmt = "%(asctime)s [%(levelname)s] %(message)s"
+_log_formatter = logging.Formatter(_log_fmt, datefmt='%Y-%m-%d %H:%M:%S')
+
+console_handler = StreamHandler(sys.stdout)
+console_handler.setFormatter(_log_formatter)
+
+file_handler = RotatingFileHandler(_log_file, mode="a", maxBytes=MAX_LOG_SIZE, backupCount=1)
+file_handler.setFormatter(_log_formatter)
+
+logger.addHandler(console_handler)
+logger.addHandler(file_handler)

sist2-admin/sist2_admin/cron.py

@@ -0,0 +1,33 @@
+from threading import Thread
+
+import pycron
+import time
+
+from hexlib.db import PersistentState
+
+from config import logger
+from jobs import Sist2Job
+
+
+def _check_schedule(db: PersistentState, run_job):
+    for job in (row["job"] for row in db["jobs"]):
+        job: Sist2Job
+
+        if job.schedule_enabled:
+            if pycron.is_now(job.cron_expression):
+                logger.info(f"Submit scan task to queue for [{job.name}]")
+                run_job(job)
+
+
+def _cron_thread(db, run_job):
+    time.sleep(60 - (time.time() % 60))
+    start = time.time()
+
+    while True:
+        _check_schedule(db, run_job)
+        time.sleep(60 - ((time.time() - start) % 60))
+
+
+def initialize(db, run_job):
+    t = Thread(target=_cron_thread, args=(db, run_job), daemon=True, name="timer")
+    t.start()

sist2-admin/sist2_admin/jobs.py

@@ -0,0 +1,315 @@
+import json
+import logging
+import os.path
+import shutil
+import signal
+import uuid
+from datetime import datetime
+from enum import Enum
+from hashlib import md5
+from logging import FileHandler
+from threading import Lock, Thread
+from time import sleep
+from uuid import uuid4, UUID
+
+from hexlib.db import PersistentState
+from pydantic import BaseModel, validator
+
+from config import logger, LOG_FOLDER
+from notifications import Notifications
+from sist2 import ScanOptions, IndexOptions, Sist2, Sist2Index
+from state import RUNNING_FRONTENDS
+from web import Sist2Frontend
+
+
+class JobStatus(Enum):
+    CREATED = "created"
+    STARTED = "started"
+    INDEXED = "indexed"
+    FAILED = "failed"
+
+
+class Sist2Job(BaseModel):
+    name: str
+    scan_options: ScanOptions
+    index_options: IndexOptions
+
+    cron_expression: str
+    schedule_enabled: bool = False
+
+    previous_index: str = None
+    last_index: str = None
+    last_index_date: datetime = None
+    status: JobStatus = JobStatus("created")
+    last_modified: datetime
+    etag: str = None
+    do_full_scan: bool = False
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @staticmethod
+    def create_default(name: str):
+        return Sist2Job(
+            name=name,
+            scan_options=ScanOptions(path="/"),
+            index_options=IndexOptions(),
+            last_modified=datetime.now(),
+            cron_expression="0 0 * * *"
+        )
+
+    @validator("etag", always=True)
+    def validate_etag(cls, value, values):
+        s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
+        return md5(s.encode()).hexdigest()
+
+
+class Sist2TaskProgress:
+
+    def __init__(self, done: int = 0, count: int = 0, index_size: int = 0, tn_size: int = 0, waiting: bool = False):
+        self.done = done
+        self.count = count
+        self.index_size = index_size
+        self.store_size = tn_size
+        self.waiting = waiting
+
+    def percent(self):
+        return (self.done / self.count) if self.count else 0
+
+
+class Sist2Task:
+
+    def __init__(self, job: Sist2Job, display_name: str, depends_on: uuid.UUID = None):
+        self.job = job
+        self.display_name = display_name
+
+        self.progress = Sist2TaskProgress()
+        self.id = uuid4()
+        self.pid = None
+        self.started = None
+        self.ended = None
+        self.depends_on = depends_on
+
+        self._logger = logging.Logger(name=f"{self.id}")
+        self._logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"sist2-{self.id}.log")))
+
+    def json(self):
+        return {
+            "id": self.id,
+            "job": self.job,
+            "display_name": self.display_name,
+            "progress": self.progress,
+            "started": self.started,
+            "ended": self.ended,
+            "depends_on": self.depends_on,
+        }
+
+    def log_callback(self, log_json):
+
+        if "progress" in log_json:
+            self.progress = Sist2TaskProgress(**log_json["progress"])
+        elif self._logger:
+            self._logger.info(json.dumps(log_json))
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        self.started = datetime.now()
+
+        logger.info(f"Started task {self.display_name}")
+
+
+class Sist2ScanTask(Sist2Task):
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        self.job.scan_options.name = self.job.name
+
+        if self.job.last_index and os.path.exists(self.job.last_index) and not self.job.do_full_scan:
+            self.job.scan_options.incremental = self.job.last_index
+        else:
+            self.job.scan_options.incremental = None
+
+        def set_pid(pid):
+            self.pid = pid
+
+        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
+        self.ended = datetime.now()
+
+        if return_code != 0:
+            self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
+            logger.info(f"Task {self.display_name} failed ({return_code})")
+        else:
+            index = Sist2Index(self.job.scan_options.output)
+
+            # Save latest index
+            self.job.previous_index = self.job.last_index
+
+            self.job.last_index = index.path
+            self.job.last_index_date = datetime.now()
+            self.job.do_full_scan = False
+            db["jobs"][self.job.name] = {"job": self.job}
+            self._logger.info(json.dumps({"sist2-admin": f"Save last_index={self.job.last_index}"}))
+
+        logger.info(f"Completed {self.display_name} ({return_code=})")
+
+        return return_code
+
+
+class Sist2IndexTask(Sist2Task):
+
+    def __init__(self, job: Sist2Job, display_name: str, depends_on: Sist2Task):
+        super().__init__(job, display_name, depends_on=depends_on.id)
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        self.job.index_options.path = self.job.scan_options.output
+
+        return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
+        self.ended = datetime.now()
+
+        duration = self.ended - self.started
+
+        ok = return_code == 0
+
+        if ok:
+            # Remove old index
+            if self.job.previous_index is not None:
+                self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index=}"}))
+                try:
+                    shutil.rmtree(self.job.previous_index)
+                except FileNotFoundError:
+                    pass
+
+            self.restart_running_frontends(db, sist2)
+
+        # Update status
+        self.job.status = JobStatus("indexed") if ok else JobStatus("failed")
+        db["jobs"][self.job.name] = {"job": self.job}
+
+        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
+
+        logger.info(f"Completed {self.display_name} ({return_code=})")
+
+        return return_code
+
+    def restart_running_frontends(self, db: PersistentState, sist2: Sist2):
+        for frontend_name, pid in RUNNING_FRONTENDS.items():
+            frontend = db["frontends"][frontend_name]["frontend"]
+            frontend: Sist2Frontend
+
+            os.kill(pid, signal.SIGTERM)
+            try:
+                os.wait()
+            except ChildProcessError:
+                pass
+
+            frontend.web_options.indices = map(lambda j: db["jobs"][j]["job"].last_index, frontend.jobs)
+
+            pid = sist2.web(frontend.web_options, frontend.name)
+            RUNNING_FRONTENDS[frontend_name] = pid
+
+            self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))
+
+
+class TaskQueue:
+    def __init__(self, sist2: Sist2, db: PersistentState, notifications: Notifications):
+        self._lock = Lock()
+
+        self._sist2 = sist2
+        self._db = db
+        self._notifications = notifications
+
+        self._tasks = {}
+        self._queue = []
+        self._sem = 0
+
+        self._thread = Thread(target=self._check_new_task, daemon=True)
+        self._thread.start()
+
+    def _tasks_failed(self):
+        done = set()
+
+        for row in self._db["task_done"].sql("WHERE return_code != 0"):
+            done.add(uuid.UUID(row["id"]))
+
+        return done
+
+    def _tasks_done(self):
+
+        done = set()
+
+        for row in self._db["task_done"]:
+            done.add(uuid.UUID(row["id"]))
+
+        return done
+
+    def _check_new_task(self):
+        while True:
+            with self._lock:
+                for task in list(self._queue):
+                    task: Sist2Task
+
+                    if self._sem >= 1:
+                        break
+
+                    if not task.depends_on or task.depends_on in self._tasks_done():
+                        self._queue.remove(task)
+
+                        if task.depends_on in self._tasks_failed():
+                            # The task which we depend on failed, continue
+                            continue
+
+                        self._sem += 1
+
+                        t = Thread(target=self._run_task, args=(task,))
+
+                        self._tasks[task.id] = {
+                            "task": task,
+                            "thread": t,
+                        }
+
+                        t.start()
+                    break
+            sleep(1)
+
+    def tasks(self):
+        return list(map(lambda t: t["task"], self._tasks.values()))
+
+    def kill_task(self, task_id):
+
+        task = self._tasks.get(UUID(task_id))
+
+        if task:
+            pid = task["task"].pid
+            logger.info(f"Killing task {task_id} (pid={pid})")
+            os.kill(pid, signal.SIGTERM)
+            return True
+
+        return False
+
+    def _run_task(self, task: Sist2Task):
+        task_result = task.run(self._sist2, self._db)
+
+        with self._lock:
+            del self._tasks[task.id]
+            self._sem -= 1
+
+            self._db["task_done"][task.id] = {
+                "ended": task.ended,
+                "started": task.started,
+                "name": task.display_name,
+                "return_code": task_result
+            }
+        if isinstance(task, Sist2IndexTask):
+            self._notifications.notify({
+                "message": "notifications.indexCompleted",
+                "job": task.job.name
+            })
+
+    def submit(self, task: Sist2Task):
+
+        logger.info(f"Submitted task to queue {task.display_name}")
+
+        with self._lock:
+            self._queue.append(task)

sist2-admin/sist2_admin/notifications.py

@@ -0,0 +1,40 @@
+import asyncio
+from typing import List
+
+
+class Notifications:
+    def __init__(self):
+        self._subscribers: List[Subscribe] = []
+
+    def subscribe(self, ob):
+        self._subscribers.append(ob)
+
+    def unsubscribe(self, ob):
+        self._subscribers.remove(ob)
+
+    def notify(self, notification: dict):
+        for ob in self._subscribers:
+            ob.notify(notification)
+
+
+class Subscribe:
+    def __init__(self, notifications: Notifications):
+        self._queue = []
+        self._notifications = notifications
+
+    async def __aenter__(self):
+        self._notifications.subscribe(self)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        self._notifications.unsubscribe(self)
+
+    def notify(self, notification: dict):
+        self._queue.append(notification)
+
+    async def notifications(self):
+        while True:
+            try:
+                yield self._queue.pop(0)
+            except IndexError:
+                await asyncio.sleep(0.1)

sist2-admin/sist2_admin/sist2.py

@@ -0,0 +1,307 @@
+import datetime
+import json
+import logging
+import os.path
+import traceback
+from datetime import datetime
+from io import TextIOWrapper
+from logging import FileHandler
+from subprocess import Popen, PIPE
+from tempfile import NamedTemporaryFile
+from threading import Thread
+from typing import List
+
+from pydantic import BaseModel
+
+from config import logger, LOG_FOLDER
+
+
+class Sist2Version:
+    def __init__(self, version: str):
+        self._version = version
+
+        self.major, self.minor, self.patch = [int(x) for x in version.split(".")]
+
+    def __str__(self):
+        return f"{self.major}.{self.minor}.{self.patch}"
+
+
+class WebOptions(BaseModel):
+    indices: List[str] = []
+    es_url: str = "http://elasticsearch:9200"
+    es_insecure_ssl: bool = False
+    es_index: str = "sist2"
+    bind: str = "0.0.0.0:4090"
+    auth: str = None
+    tag_auth: str = None
+    tagline: str = "Lightning-fast file system indexer and search tool"
+    dev: bool = False
+    lang: str = "en"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+        args = ["web", f"--es-url={self.es_url}", f"--bind={self.bind}",
+                f"--tagline={self.tagline}", f"--lang={self.lang}"]
+
+        if self.es_insecure_ssl:
+            args.append(f"--es-insecure-ssl")
+        if self.auth:
+            args.append(f"--auth={self.auth}")
+        if self.tag_auth:
+            args.append(f"--tag_auth={self.tag_auth}")
+        if self.dev:
+            args.append(f"--dev")
+
+        args.extend(self.indices)
+
+        return args
+
+
+class IndexOptions(BaseModel):
+    path: str = None
+    threads: int = 1
+    es_url: str = "http://elasticsearch:9200"
+    es_insecure_ssl: bool = False
+    es_index: str = "sist2"
+    incremental_index: bool = False
+    script: str = ""
+    script_file: str = None
+    batch_size: int = 100
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+
+        args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
+                f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
+
+        if self.script_file:
+            args.append(f"--script-file={self.script_file}")
+        if self.es_insecure_ssl:
+            args.append(f"--es-insecure-ssl")
+        if self.incremental_index:
+            args.append(f"--incremental-index")
+
+        return args
+
+
+ARCHIVE_SKIP = "skip"
+ARCHIVE_LIST = "list"
+ARCHIVE_SHALLOW = "shallow"
+ARCHIVE_RECURSE = "recurse"
+
+
+class ScanOptions(BaseModel):
+    path: str
+    threads: int = 1
+    mem_throttle: int = 0
+    thumbnail_quality: float = 1.0
+    thumbnail_size: int = 500
+    thumbnail_count: int = 1
+    content_size: int = 32768
+    depth: int = -1
+    archive: str = ARCHIVE_RECURSE
+    archive_passphrase: str = None
+    ocr_lang: bool = None
+    ocr_images: bool = False
+    ocr_ebooks: bool = False
+    exclude: str = None
+    fast: bool = False
+    treemap_threshold: float = 0.0005
+    mem_buffer: int = 2000
+    read_subtitles: bool = False
+    fast_epub: bool = False
+    checksums: bool = False
+    incremental: str = None
+    output: str = None
+    name: str = None
+    rewrite_url: str = None
+    list_file: str = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+        args = ["scan", self.path, f"--threads={self.threads}", f"--mem-throttle={self.mem_throttle}",
+                f"--thumbnail-quality={self.thumbnail_quality}", f"--thumbnail-count={self.thumbnail_count}",
+                f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
+                f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
+
+        if self.incremental:
+            args.append(f"--incremental={self.incremental}")
+        if self.rewrite_url:
+            args.append(f"--rewrite-url={self.rewrite_url}")
+        if self.name:
+            args.append(f"--name={self.name}")
+        if self.archive_passphrase:
+            args.append(f"--archive-passphrase={self.archive_passphrase}")
+        if self.ocr_lang:
+            args.append(f"--ocr-lang={self.ocr_lang}")
+        if self.ocr_ebooks:
+            args.append(f"--ocr-ebooks")
+        if self.ocr_images:
+            args.append(f"--ocr-images")
+        if self.exclude:
+            args.append(f"--exclude={self.exclude}")
+        if self.fast:
+            args.append(f"--fast")
+        if self.treemap_threshold:
+            args.append(f"--treemap-threshold={self.treemap_threshold}")
+        if self.read_subtitles:
+            args.append(f"--read-subtitles")
+        if self.fast_epub:
+            args.append(f"--fast-epub")
+        if self.checksums:
+            args.append(f"--checksums")
+        if self.list_file:
+            args.append(f"--list_file={self.list_file}")
+
+        return args
+
+
+class Sist2Index:
+    def __init__(self, path):
+        self.path = path
+
+        with open(os.path.join(path, "descriptor.json")) as f:
+            self._descriptor = json.load(f)
+
+    def to_json(self):
+        return {
+            "path": self.path,
+            "version": self.version(),
+            "timestamp": self.timestamp(),
+            "name": self.name()
+        }
+
+    def version(self) -> Sist2Version:
+        return Sist2Version(self._descriptor["version"])
+
+    def timestamp(self) -> datetime:
+        return datetime.fromtimestamp(self._descriptor["timestamp"])
+
+    def name(self) -> str:
+        return self._descriptor["name"]
+
+
+class Sist2:
+
+    def __init__(self, bin_path: str, data_directory: str):
+        self._bin_path = bin_path
+        self._data_dir = data_directory
+
+    def index(self, options: IndexOptions, logs_cb):
+
+        if options.script:
+            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
+                f.write(options.script)
+            options.script_file = f.name
+        else:
+            options.script_file = None
+
+        args = [
+            self._bin_path,
+            *options.args(),
+            "--json-logs",
+            "--very-verbose"
+        ]
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        self._consume_logs_stdout(logs_cb, proc)
+
+        t_stderr.join()
+
+        return proc.returncode
+
+    def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
+
+        output_dir = os.path.join(
+            self._data_dir,
+            f"scan-{datetime.now()}.sist2"
+        )
+        options.output = output_dir
+
+        args = [
+            self._bin_path,
+            *options.args(),
+            "--json-logs",
+            "--very-verbose"
+        ]
+
+        logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
+
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        set_pid_cb(proc.pid)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        self._consume_logs_stdout(logs_cb, proc)
+
+        t_stderr.join()
+
+        return proc.returncode
+
+    @staticmethod
+    def _consume_logs_stderr(logs_cb, proc):
+        pipe_wrapper = TextIOWrapper(proc.stderr, encoding="utf8")
+        try:
+            for line in pipe_wrapper:
+                if line.strip() == "":
+                    continue
+                logs_cb({"stderr": line})
+        finally:
+            proc.wait()
+            pipe_wrapper.close()
+
+    @staticmethod
+    def _consume_logs_stdout(logs_cb, proc):
+        pipe_wrapper = TextIOWrapper(proc.stdout, encoding="utf8")
+        try:
+            for line in pipe_wrapper:
+                if line.strip() == "":
+                    continue
+                log_object = json.loads(line)
+                logs_cb(log_object)
+        except Exception as e:
+            proc.kill()
+            try:
+                print(line)
+            except NameError:
+                pass
+            print(traceback.format_exc())
+        finally:
+            pass
+            # proc.wait()
+            # pipe_wrapper.close()
+
+    def web(self, options: WebOptions, name: str):
+        args = [
+            self._bin_path,
+            *options.args()
+        ]
+
+        web_logger = logging.Logger(name=f"sist2-frontend-{name}")
+        web_logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"frontend-{name}.log")))
+
+        def logs_cb(message):
+            web_logger.info(json.dumps(message))
+
+        logger.info(f"Starting frontend {' '.join(args)}")
+
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        t_stdout = Thread(target=self._consume_logs_stdout, args=(logs_cb, proc))
+        t_stdout.start()
+
+        return proc.pid

sist2-admin/sist2_admin/state.py

@@ -0,0 +1,50 @@
+from typing import Dict
+
+from hexlib.db import Table
+import pickle
+
+from tesseract import get_tesseract_langs
+
+RUNNING_FRONTENDS: Dict[str, int] = {}
+
+TESSERACT_LANGS = get_tesseract_langs()
+
+DB_SCHEMA_VERSION = "1"
+
+from pydantic import BaseModel
+
+
+def _serialize(item):
+    if isinstance(item, BaseModel):
+        return pickle.dumps(item)
+    if isinstance(item, bytes):
+        raise Exception("FIXME: bytes in PickleTable")
+    return item
+
+
+def _deserialize(item):
+    if isinstance(item, bytes):
+        return pickle.loads(item)
+    return item
+
+
+class PickleTable(Table):
+
+    def __getitem__(self, item):
+        row = super().__getitem__(item)
+        if row:
+            return dict((k, _deserialize(v)) for k, v in row.items())
+        return row
+
+    def __setitem__(self, key, value):
+        value = dict((k, _serialize(v)) for k, v in value.items())
+        super().__setitem__(key, value)
+
+    def __iter__(self):
+        for row in super().__iter__():
+            yield dict((k, _deserialize(v)) for k, v in row.items())
+
+    def sql(self, where_clause, *params):
+        for row in super().sql(where_clause, *params):
+            yield dict((k, _deserialize(v)) for k, v in row.items())
+

sist2-admin/sist2_admin/tesseract.py

@@ -0,0 +1,14 @@
+import subprocess
+
+
+def get_tesseract_langs():
+
+    res = subprocess.check_output([
+        "tesseract",
+        "--list-langs"
+    ]).decode()
+
+    languages = res.split("\n")[1:]
+
+    return list(filter(lambda lang: lang and lang != "osd", languages))
+

sist2-admin/sist2_admin/web.py

@@ -0,0 +1,29 @@
+import os.path
+from typing import List
+
+from pydantic import BaseModel
+
+from sist2 import WebOptions
+
+
+class Sist2Frontend(BaseModel):
+    name: str
+    jobs: List[str]
+    web_options: WebOptions
+    running: bool = False
+
+    auto_start: bool = False
+    enable_monitoring: bool = True
+    extra_query_args: str = ""
+    custom_url: str = None
+
+    def get_log_path(self, log_folder: str):
+        return os.path.join(log_folder, f"frontend-{self.name}.log")
+
+    @staticmethod
+    def create_default(name: str):
+        return Sist2Frontend(
+            name=name,
+            web_options=WebOptions(),
+            jobs=[]
+        )

sist2-vue/dist/index.html

@@ -1,3 +1,32 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>sist2</title><link href="css/chunk-vendors.css" rel="preload" as="style"><link href="css/index.css" rel="preload" as="style"><link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/index.css" rel="stylesheet"></head><body><noscript><style>body {
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
+
+    <title>sist2</title>
+<link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"></head>
+<body>
+<noscript>
+    <style>
+        body {
             height: initial;
-        }</style><div style="text-align: center; margin-top: 100px"><strong>We're sorry but sist2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong><br><strong>Nous sommes désolés mais sist2 ne fonctionne pas correctement si JavaScript est activé. Veuillez l'activer pour continuer.</strong></div></noscript><div id="app"></div><script src="js/chunk-vendors.js"></script><script src="js/index.js"></script></body></html>
+        }
+    </style>
+    <div style="text-align: center; margin-top: 100px">
+        <strong>
+            We're sorry but sist2 doesn't work properly without JavaScript enabled.
+            Please enable it to continue.
+        </strong>
+        <br/>
+        <strong>
+            Nous sommes désolés mais sist2 ne fonctionne pas correctement
+            si JavaScript est activé.
+            Veuillez l'activer pour continuer.
+        </strong>
+    </div>
+</noscript>
+<div id="app"></div>
+<script type="text/javascript" src="js/chunk-vendors.js"></script><script type="text/javascript" src="js/index.js"></script></body>
+</html>

sist2-vue/package-lock.json

@@ -12,7 +12,6 @@
         "axios": "^0.25.0",
         "bootstrap-vue": "^2.21.2",
         "core-js": "^3.6.5",
-        "crypto-es": "^1.2.7",
         "d3": "^5.16.0",
         "date-fns": "^2.21.3",
         "dom-to-image": "^2.6.0",
@@ -5261,11 +5260,6 @@
         "node": "*"
       }
     },
-    "node_modules/crypto-es": {
-      "version": "1.2.7",
-      "resolved": "https://registry.npmjs.org/crypto-es/-/crypto-es-1.2.7.tgz",
-      "integrity": "sha512-UUqiVJ2gUuZFmbFsKmud3uuLcNP2+Opt+5ysmljycFCyhA0+T16XJmo1ev/t5kMChMqWh7IEvURNCqsg+SjZGQ=="
-    },
     "node_modules/css-color-names": {
       "version": "0.0.4",
       "resolved": "https://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz",
@@ -9742,9 +9736,9 @@
       }
     },
     "node_modules/minimist": {
-      "version": "1.2.5",
+      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
-      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
+      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
       "dev": true
     },
     "node_modules/minipass": {
@@ -14098,9 +14092,9 @@
       }
     },
     "node_modules/url-parse": {
-      "version": "1.5.4",
+      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.4.tgz",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-ITeAByWWoqutFClc/lRZnFplgXgEZr3WJ6XngMM/N9DMIm4K8zXPCZ1Jdu0rERwO84w1WC5wkle2ubwTA4NTBg==",
+      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
       "dev": true,
       "dependencies": {
         "querystringify": "^2.1.1",
@@ -19621,11 +19615,6 @@
         "randomfill": "^1.0.3"
       }
     },
-    "crypto-es": {
-      "version": "1.2.7",
-      "resolved": "https://registry.npmjs.org/crypto-es/-/crypto-es-1.2.7.tgz",
-      "integrity": "sha512-UUqiVJ2gUuZFmbFsKmud3uuLcNP2+Opt+5ysmljycFCyhA0+T16XJmo1ev/t5kMChMqWh7IEvURNCqsg+SjZGQ=="
-    },
     "css-color-names": {
       "version": "0.0.4",
       "resolved": "https://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz",
@@ -23335,9 +23324,9 @@
       }
     },
     "minimist": {
-      "version": "1.2.5",
+      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
-      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
+      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
       "dev": true
     },
     "minipass": {
@@ -27019,9 +27008,9 @@
       }
     },
     "url-parse": {
-      "version": "1.5.4",
+      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.4.tgz",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-ITeAByWWoqutFClc/lRZnFplgXgEZr3WJ6XngMM/N9DMIm4K8zXPCZ1Jdu0rERwO84w1WC5wkle2ubwTA4NTBg==",
+      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
       "dev": true,
       "requires": {
         "querystringify": "^2.1.1",

sist2-vue/package.json

@@ -11,7 +11,6 @@
     "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "crypto-es": "^1.2.7",
     "d3": "^5.16.0",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",

sist2-vue/src/Sist2Api.ts

@@ -1,6 +1,5 @@
 import axios from "axios";
 import {ext, strUnescape, lum} from "./util";
-import CryptoES from 'crypto-es';
 
 export interface EsTag {
     id: string
@@ -30,7 +29,6 @@ export interface EsHit {
     _index: string
     _id: string
     _score: number
-    _path_md5: string
     _type: string
     _tags: Tag[]
     _seq: number
@@ -249,11 +247,6 @@ class Sist2Api {
                 res.hits.hits.forEach((hit: EsHit) => {
                     hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
                     hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
-                    hit["_path_md5"] = CryptoES.MD5(
-                        hit["_source"]["path"] +
-                        (hit["_source"]["path"] ? "/" : "") +
-                        hit["_source"]["name"] + ext(hit)
-                    ).toString();
 
                     this.setHitProps(hit);
                     this.setHitTags(hit);
@@ -343,10 +336,6 @@ class Sist2Api {
         };
     }
 
-    getDocInfo(docId: string) {
-        return axios.get(`${this.baseUrl}d/${docId}`);
-    }
-
     getTags() {
         return this.esQuery({
             aggs: {
@@ -380,8 +369,7 @@ class Sist2Api {
         return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
             delete: false,
             name: tag,
-            doc_id: hit["_id"],
+            doc_id: hit["_id"]
-            path_md5: hit._path_md5
         });
     }
 
@@ -389,8 +377,7 @@ class Sist2Api {
         return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
             delete: true,
             name: tag,
-            doc_id: hit["_id"],
+            doc_id: hit["_id"]
-            path_md5: hit._path_md5
         });
     }
 

sist2-vue/src/Sist2Query.ts

@@ -69,7 +69,7 @@ interface SortMode {
 
 class Sist2Query {
 
-    searchQuery(): any {
+    searchQuery(blankSearch: boolean = false): any {
 
         const getters = store.getters;
 
@@ -93,22 +93,6 @@ class Sist2Query {
             {terms: {index: selectedIndexIds}}
         ] as any[];
 
-        if (sizeMin && sizeMax) {
-            filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
-        } else if (sizeMin) {
-            filters.push({range: {size: {gte: sizeMin}}})
-        } else if (sizeMax) {
-            filters.push({range: {size: {lte: sizeMax}}})
-        }
-
-        if (dateMin && dateMax) {
-            filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
-        } else if (dateMin) {
-            filters.push({range: {mtime: {gte: dateMin}}})
-        } else if (dateMax) {
-            filters.push({range: {mtime: {lte: dateMax}}})
-        }
-
         const fields = [
             "name^8",
             "content^3",
@@ -128,20 +112,39 @@ class Sist2Query {
             fields.push("name.nGram^3");
         }
 
-        const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
+        if (!blankSearch) {
-        if (path !== "") {
+            if (sizeMin && sizeMax) {
-            filters.push({term: {path: path}})
+                filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
-        }
+            } else if (sizeMin) {
+                filters.push({range: {size: {gte: sizeMin}}})
+            } else if (sizeMax) {
+                filters.push({range: {size: {lte: sizeMax}}})
+            }
 
-        if (selectedMimeTypes.length > 0) {
+            if (dateMin && dateMax) {
-            filters.push({terms: {"mime": selectedMimeTypes}});
+                filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
-        }
+            } else if (dateMin) {
+                filters.push({range: {mtime: {gte: dateMin}}})
+            } else if (dateMax) {
+                filters.push({range: {mtime: {lte: dateMax}}})
+            }
 
-        if (selectedTags.length > 0) {
+            const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
-            if (getters.optTagOrOperator) {
+
-                filters.push({terms: {"tag": selectedTags}});
+            if (path !== "") {
-            } else {
+                filters.push({term: {path: path}})
-                selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
+            }
+
+            if (selectedMimeTypes.length > 0) {
+                filters.push({terms: {"mime": selectedMimeTypes}});
+            }
+
+            if (selectedTags.length > 0) {
+                if (getters.optTagOrOperator) {
+                    filters.push({terms: {"tag": selectedTags}});
+                } else {
+                    selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
+                }
             }
         }
 
@@ -182,7 +185,7 @@ class Sist2Query {
             size: size,
         } as any;
 
-        if (!empty) {
+        if (!empty && !blankSearch) {
             q.query.bool.must = query;
         }
 
@@ -207,7 +210,7 @@ class Sist2Query {
             };
 
             if (!legacyES) {
-                q.highlight.max_analyzed_offset = 9_999_999;
+                q.highlight.max_analyzed_offset = 999_999;
             }
 
             if (getters.optSearchInPath) {
@@ -237,7 +240,7 @@ class Sist2Query {
                 }
             }
 
-            if (!empty) {
+            if (!empty && !blankSearch) {
                 q.query.function_score.query.bool.must.push(query);
             }
         }

sist2-vue/src/components/InfoTable.vue

@@ -72,6 +72,12 @@ export default {
         }
       });
 
+      Object.keys(src).forEach(key => {
+        if (key.startsWith("mt_") || key.startsWith("int_")) {
+          items.push({key: key, value: src[key]});
+        }
+      });
+
       // Exif GPS
       if ("exif_gps_longitude_dec" in src) {
         items.push({

sist2-vue/src/components/LazyContentDiv.vue

@@ -1,11 +1,13 @@
 <template>
   <Preloader v-if="loading"></Preloader>
-  <div v-else-if="content" class="content-div">{{ content }}</div>
+  <div v-else-if="content" class="content-div" v-html="content"></div>
 </template>
 
 <script>
 import Sist2Api from "@/Sist2Api";
 import Preloader from "@/components/Preloader";
+import Sist2Query from "@/Sist2Query";
+import store from "@/store";
 
 export default {
   name: "LazyContentDiv",
@@ -18,10 +20,72 @@ export default {
     }
   },
   mounted() {
-    Sist2Api.getDocInfo(this.docId).then(src => {
+    const query = Sist2Query.searchQuery();
-      this.content = src.data.content;
+
+    if (this.$store.state.optHighlight) {
+
+      const fields = this.$store.state.fuzzy
+          ? {"content.nGram": {}}
+          : {content: {}};
+
+      query.highlight = {
+        pre_tags: ["<mark>"],
+        post_tags: ["</mark>"],
+        number_of_fragments: 0,
+        fields,
+      };
+
+      if (!store.state.sist2Info.esVersionLegacy) {
+        query.highlight.max_analyzed_offset = 999_999;
+      }
+    }
+
+    if ("function_score" in query.query) {
+      query.query = query.query.function_score.query;
+    }
+
+    if (!("must" in query.query.bool)) {
+      query.query.bool.must = [];
+    } else if (!Array.isArray(query.query.bool.must)) {
+      query.query.bool.must = [query.query.bool.must];
+    }
+
+    query.query.bool.must.push({match: {_id: this.docId}});
+
+    delete query["sort"];
+    delete query["aggs"];
+    delete query["search_after"];
+    delete query.query["function_score"];
+
+    query._source = {
+      includes: ["content", "name", "path", "extension"]
+    }
+
+    query.size = 1;
+
+    Sist2Api.esQuery(query).then(resp => {
       this.loading = false;
-    })
+      if (resp.hits.hits.length === 1) {
+        this.content = this.getContent(resp.hits.hits[0]);
+      } else {
+        console.log("FIXME: could not get content")
+        console.log(resp)
+      }
+    });
+  },
+  methods: {
+    getContent(doc) {
+      if (!doc.highlight) {
+        return doc._source.content;
+      }
+
+      if (doc.highlight["content.nGram"]) {
+        return doc.highlight["content.nGram"][0];
+      }
+      if (doc.highlight.content) {
+        return doc.highlight.content[0];
+      }
+    }
   }
 }
 </script>

sist2-vue/src/components/Lightbox.vue

@@ -1,6 +1,7 @@
 <template>
-  <div>
+  <div :class="{'disable-animations': $store.state.optSimpleLightbox}">
     <FsLightbox
+        ref="lightbox"
         :key="lightboxKey"
         :toggler="showLightbox"
         :sources="lightboxSources"
@@ -10,7 +11,7 @@
         :source-index="lightboxSlide"
         :custom-toolbar-buttons="customButtons"
         :slideshow-time="$store.getters.optLightboxSlideDuration * 1000"
-        :zoom-increment="0.5"
+        :zoom-increment="0.25"
         :load-only-current-source="$store.getters.optLightboxLoadOnlyCurrent"
         :on-close="onClose"
         :on-open="onShow"
@@ -29,6 +30,7 @@ export default {
   components: {FsLightbox},
   data() {
     return {
+      disableAnimations: true,
       customButtons: [
         {
           viewBox: "0 0 384.928 384.928",
@@ -64,7 +66,84 @@ export default {
       return this.$store.getters["uiLightboxTypes"];
     }
   },
+  mounted() {
+    const listener = document.onkeydown;
+
+    document.onkeydown = (e) => {
+
+      const ret = this.keyDownListener(e)
+
+      if (listener && ret) {
+        return listener(e);
+      }
+    };
+  },
   methods: {
+    keyDownListener(e) {
+
+      const isLightboxOpen = this.$refs.lightbox === undefined || this.$refs.lightbox.$el.tagName === undefined;
+
+      if (isLightboxOpen) {
+        return true;
+      }
+
+      const lightboxStore = this.$refs.lightbox.fsLightboxStore.slice(-1)[0];
+
+      switch (e.key) {
+        case " ": {
+          e.preventDefault();
+          e.stopPropagation();
+          e.stopImmediatePropagation();
+
+          // Find video at current slide, toggle play/pause
+          [...document.getElementsByClassName("fslightbox-absoluted")].forEach(elem => {
+            if (elem.style.transform === "translate(0px)" || elem.style.transform === "translate(0px, 0px)") {
+              const vid = elem.getElementsByTagName("video")[0];
+
+              if (vid) {
+                if (vid.paused) {
+                  vid.play();
+                } else {
+                  vid.pause()
+                }
+              }
+            }
+
+            return false;
+          });
+
+          return false;
+        }
+        case "ArrowUp":
+        case "k": {
+          if (!lightboxStore.data.isThumbing && lightboxStore.core.thumbsToggler) {
+            lightboxStore.core.thumbsToggler.toggleThumbs();
+          }
+          return false;
+        }
+        case "ArrowDown":
+        case "j": {
+          if (lightboxStore.data.isThumbing && lightboxStore.core.thumbsToggler) {
+            lightboxStore.core.thumbsToggler.toggleThumbs();
+          }
+          return false;
+        }
+        case "h": {
+          if (lightboxStore.core.stageManager.getPreviousSlideIndex) {
+            lightboxStore.core.slideIndexChanger.jumpTo(lightboxStore.core.stageManager.getPreviousSlideIndex());
+          }
+          return false;
+        }
+        case "l": {
+          if (lightboxStore.core.stageManager.getNextSlideIndex) {
+            lightboxStore.core.slideIndexChanger.jumpTo(lightboxStore.core.stageManager.getNextSlideIndex());
+          }
+          return false;
+        }
+      }
+
+      return true;
+    },
     onDownloadClick() {
       const url = this.lightboxSources[this.lightboxSlide];
 
@@ -125,4 +204,20 @@ export default {
 .fslightbox-toolbar-button:nth-child(7) {
   order: 7;
 }
+
+.disable-animations .fslightbox-container {
+  background: rgba(30,30,30,.9);
+}
+
+.disable-animations .fslightbox-transform-transition {
+  transition: none;
+}
+
+.disable-animations .fslightbox-fade-in-strong {
+  animation: none;
+}
+
+.fslightbox-container video, .fslightbox-container img {
+  cursor: unset !important;
+}
 </style>

sist2-vue/src/components/NavBar.vue

@@ -60,7 +60,6 @@ export default {
   color: #222 !important;
   font-size: 1.75rem;
   padding: 0;
-  font-family: Hack;
 }
 
 .navbar-brand:hover {

sist2-vue/src/components/SortSelect.vue

@@ -42,6 +42,8 @@
 </template>
 
 <script>
+import {randomSeed} from "@/util";
+
 export default {
   name: "SortSelect",
   computed: {
@@ -52,7 +54,7 @@ export default {
   methods: {
     onSelect(sortMode) {
       if (sortMode === "random") {
-        this.$store.commit("setSeed", Math.round(Math.random() * 100000));
+        this.$store.commit("setSeed", randomSeed());
       }
       this.$store.commit("setSortMode", sortMode);
     }

sist2-vue/src/components/TagPicker.vue

@@ -1,5 +1,13 @@
 <template>
-  <div id="tagTree"></div>
+  <div>
+    <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
+      <b-form-input :value="filter"
+                    :placeholder="$t('tagFilter')"
+                    @input="onFilter($event)"></b-form-input>
+    </b-input-group>
+
+    <div id="tagTree"></div>
+  </div>
 </template>
 
 <script>
@@ -112,10 +120,12 @@ function addTag(map, tag, id, count) {
 
 export default {
   name: "TagPicker",
+  props: ["showSearchBar"],
   data() {
     return {
       tagTree: null,
       loadedFromArgs: false,
+      filter: ""
     }
   },
   mounted() {
@@ -129,6 +139,10 @@ export default {
     });
   },
   methods: {
+    onFilter(value) {
+      this.filter = value;
+      this.tagTree.search(value);
+    },
     initializeTree() {
       const tagMap = [];
       this.tagTree = new InspireTree({
@@ -163,7 +177,8 @@ export default {
       });
     },
     handleTreeClick(node, e) {
-      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused") {
+      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
+          || e === "matched" || e === "hidden") {
         return;
       }
 
@@ -180,7 +195,15 @@ export default {
 }
 </style>
 <style>
-.inspire-tree .focused>.wholerow {
+.inspire-tree .focused > .wholerow {
   border: none;
 }
+
+#tag-picker-filter-bar {
+  padding: 10px 4px 4px;
+}
+
+.theme-black .inspire-tree .matched > .wholerow {
+  background: rgba(251, 191, 41, 0.25);
+}
 </style>

sist2-vue/src/i18n/messages.ts

@@ -16,6 +16,7 @@ export default {
         pages: "pages",
         mimeTypes: "Media types",
         tags: "Tags",
+        tagFilter: "Filter tags",
         help: {
             simpleSearch: "Simple search",
             advancedSearch: "Advanced search",
@@ -72,7 +73,9 @@ export default {
             hideLegacy: "Hide the 'legacyES' Elasticsearch notice",
             updateMimeMap: "Update the Media Types tree in real time",
             useDatePicker: "Use a Date Picker component rather than a slider",
-            vidPreviewInterval: "Video preview frame duration in ms"
+            vidPreviewInterval: "Video preview frame duration in ms",
+            simpleLightbox: "Disable animations in image viewer",
+            showTagPickerFilter: "Display the tag filter bar"
         },
         queryMode: {
             simple: "Simple",
@@ -182,6 +185,7 @@ export default {
         pages: "pages",
         mimeTypes: "Types de médias",
         tags: "Tags",
+        tagFilter: "Filtrer les tags",
         help: {
             simpleSearch: "Recherche simple",
             advancedSearch: "Recherche avancée",
@@ -239,7 +243,9 @@ export default {
             hideLegacy: "Masquer la notice 'legacyES' Elasticsearch",
             updateMimeMap: "Mettre à jour l'arbre de Types de médias en temps réel",
             useDatePicker: "Afficher un composant « Date Picker » plutôt qu'un slider",
-            vidPreviewInterval: "Durée des images d'aperçu video en millisecondes"
+            vidPreviewInterval: "Durée des images d'aperçu video en millisecondes",
+            simpleLightbox: "Désactiver les animations du visualiseur d'images",
+            showTagPickerFilter: "Afficher le filtre dans l'onglet Tags"
         },
         queryMode: {
             simple: "Simple",
@@ -350,6 +356,7 @@ export default {
         pages: "页",
         mimeTypes: "文件类型",
         tags: "标签",
+        tagFilter: "筛选标签",
         help: {
             simpleSearch: "简易搜索",
             advancedSearch: "高级搜索",
@@ -406,7 +413,9 @@ export default {
             hideLegacy: "隐藏'legacyES' Elasticsearch 通知",
             updateMimeMap: "媒体类型树的实时更新",
             useDatePicker: "使用日期选择器组件而不是滑块",
-            vidPreviewInterval: "视频预览帧的持续时间，以毫秒为单位"
+            vidPreviewInterval: "视频预览帧的持续时间，以毫秒为单位",
+            simpleLightbox: "在图片查看器中，禁用动画",
+            showTagPickerFilter: "显示标签过滤栏"
         },
         queryMode: {
             simple: "简单",

sist2-vue/src/plugins/bootstrap-vue.js

@@ -2,6 +2,6 @@ import Vue from "vue"
 
 import BootstrapVue from "bootstrap-vue"
 import "bootstrap/dist/css/bootstrap.min.css"
-import "bootstrap-vue/dist/bootstrap-vue.css"
+import "bootstrap-vue/dist/bootstrap-vue.min.css"
 
 Vue.use(BootstrapVue)

sist2-vue/src/store/index.ts

@@ -2,7 +2,9 @@ import Vue from "vue"
 import Vuex from "vuex"
 import VueRouter, {Route} from "vue-router";
 import {EsHit, EsResult, EsTag, Index, Tag} from "@/Sist2Api";
-import {deserializeMimes, serializeMimes} from "@/util";
+import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
+
+const CONF_VERSION = 2;
 
 Vue.use(Vuex)
 
@@ -24,7 +26,6 @@ export default new Vuex.Store({
         sortMode: "score",
 
         fuzzy: false,
-        size: 60,
 
         optLang: "en",
         optLangIsDefault: true,
@@ -32,6 +33,7 @@ export default new Vuex.Store({
         optTheme: "light",
         optDisplay: "grid",
 
+        optSize: 60,
         optHighlight: true,
         optTagOrOperator: false,
         optFuzzy: true,
@@ -51,6 +53,8 @@ export default new Vuex.Store({
         optUpdateMimeMap: false,
         optUseDatePicker: false,
         optVidPreviewInterval: 700,
+        optSimpleLightbox: true,
+        optShowTagPickerFilter: true,
 
         _onLoadSelectedIndices: [] as string[],
         _onLoadSelectedMimeTypes: [] as string[],
@@ -149,7 +153,7 @@ export default new Vuex.Store({
         setOptSuggestPath: (state, val) => state.optSuggestPath = val,
         setOptFragmentSize: (state, val) => state.optFragmentSize = val,
         setOptQueryMode: (state, val) => state.optQueryMode = val,
-        setOptResultSize: (state, val) => state.size = val,
+        setOptResultSize: (state, val) => state.optSize = val,
         setOptTagOrOperator: (state, val) => state.optTagOrOperator = val,
 
         setOptTreemapType: (state, val) => state.optTreemapType = val,
@@ -161,6 +165,8 @@ export default new Vuex.Store({
         setOptUpdateMimeMap: (state, val) => state.optUpdateMimeMap = val,
         setOptUseDatePicker: (state, val) => state.optUseDatePicker = val,
         setOptVidPreviewInterval: (state, val) => state.optVidPreviewInterval = val,
+        setOptSimpleLightbox: (state, val) => state.optSimpleLightbox = val,
+        setOptShowTagPickerFilter: (state, val) => state.optShowTagPickerFilter = val,
 
         setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
         setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val,
@@ -235,10 +241,18 @@ export default new Vuex.Store({
 
             if (route.query.sort) {
                 commit("setSortMode", route.query.sort);
+                if (route.query.sort === "random" && route.query.seed === undefined) {
+                    route.query.seed = randomSeed().toString();
+                }
                 commit("setSeed", Number(route.query.seed));
             }
         },
         async updateArgs({state}, router: VueRouter) {
+
+            if (router.currentRoute.path !== "/") {
+                return;
+            }
+
             await router.push({
                 query: {
                     q: state.searchText.trim() ? state.searchText.trim().replace(/\s+/g, " ") : undefined,
@@ -267,6 +281,8 @@ export default new Vuex.Store({
                 }
             });
 
+            conf["version"] = CONF_VERSION;
+
             localStorage.setItem("sist2_configuration", JSON.stringify(conf));
         },
         loadConfiguration({state}) {
@@ -274,6 +290,11 @@ export default new Vuex.Store({
             if (confString) {
                 const conf = JSON.parse(confString);
 
+                if (!("version" in conf) || conf["version"] != CONF_VERSION) {
+                    localStorage.removeItem("sist2_configuration");
+                    window.location.reload();
+                }
+
                 Object.keys(state).forEach((key) => {
                     if (key.startsWith("opt")) {
                         (state as any)[key] = conf[key];
@@ -335,7 +356,7 @@ export default new Vuex.Store({
         searchText: state => state.searchText,
         pathText: state => state.pathText,
         fuzzy: state => state.fuzzy,
-        size: state => state.size,
+        size: state => state.optSize,
         sortMode: state => state.sortMode,
         lastQueryResult: state => state.lastQueryResults,
         lastDoc: function (state): EsHit | null {
@@ -373,10 +394,12 @@ export default new Vuex.Store({
         optTreemapColor: state => state.optTreemapColor,
         optLightboxLoadOnlyCurrent: state => state.optLightboxLoadOnlyCurrent,
         optLightboxSlideDuration: state => state.optLightboxSlideDuration,
-        optResultSize: state => state.size,
+        optResultSize: state => state.optSize,
         optHideLegacy: state => state.optHideLegacy,
         optUpdateMimeMap: state => state.optUpdateMimeMap,
         optUseDatePicker: state => state.optUseDatePicker,
         optVidPreviewInterval: state => state.optVidPreviewInterval,
+        optSimpleLightbox: state => state.optSimpleLightbox,
+        optShowTagPickerFilter: state => state.optShowTagPickerFilter,
     }
 })

sist2-vue/src/util.ts

@@ -162,4 +162,8 @@ export function decompressMime(mime: string): string {
         .replace("F", "font/")
         .replace(",", "+")
         .replace("X", "x-")
+}
+
+export function randomSeed(): number {
+    return Math.round(Math.random() * 100000);
 }

sist2-vue/src/views/Configuration.vue

@@ -45,6 +45,16 @@
           <b-form-checkbox :checked="optUseDatePicker" @input="setOptUseDatePicker">
             {{ $t("opt.useDatePicker") }}
           </b-form-checkbox>
+
+          <b-form-checkbox :checked="optSimpleLightbox" @input="setOptSimpleLightbox">{{
+              $t("opt.simpleLightbox")
+            }}
+          </b-form-checkbox>
+
+          <b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{
+              $t("opt.showTagPickerFilter")
+            }}
+          </b-form-checkbox>
         </b-card>
 
         <br/>
@@ -239,6 +249,8 @@ export default {
       "optUpdateMimeMap",
       "optUseDatePicker",
       "optVidPreviewInterval",
+      "optSimpleLightbox",
+      "optShowTagPickerFilter",
     ]),
     clientWidth() {
       return window.innerWidth;
@@ -285,6 +297,8 @@ export default {
       "setOptUpdateMimeMap",
       "setOptUseDatePicker",
       "setOptVidPreviewInterval",
+      "setOptSimpleLightbox",
+      "setOptShowTagPickerFilter",
     ]),
     onResetClick() {
       localStorage.removeItem("sist2_configuration");

sist2-vue/src/views/FilePage.vue

@@ -56,6 +56,22 @@ export default Vue.extend({
     onThumbnailClick() {
       window.open(`/f/${this.doc._id}`, "_blank");
     },
+    findByCustomField(field, id) {
+      return {
+        query: {
+          bool: {
+            must: [
+              {
+                match: {
+                  [field]: id
+                }
+              }
+            ]
+          }
+        },
+        size: 1
+      }
+    },
     findById(id) {
       return {
         query: {
@@ -103,6 +119,8 @@ export default Vue.extend({
       query = this.findById(this.$route.query.byId);
     } else if (this.$route.query.byName) {
       query = this.findByName(this.$route.query.byName);
+    } else if (this.$route.query.by && this.$route.query.q) {
+      query = this.findByCustomField(this.$route.query.by, this.$route.query.q)
     }
 
     if (query) {

sist2-vue/src/views/SearchPage.vue

@@ -32,7 +32,7 @@
               <MimePicker></MimePicker>
             </b-tab>
             <b-tab :title="$t('tags')">
-              <TagPicker></TagPicker>
+              <TagPicker :show-search-bar="$store.state.optShowTagPickerFilter"></TagPicker>
             </b-tab>
           </b-tabs>
         </b-col>
@@ -139,7 +139,9 @@ export default Vue.extend({
         this.setSist2Info(data);
         this.setIndices(data.indices);
 
-        Sist2Api.getMimeTypes(Sist2Query.searchQuery()).then(({mimeMap}) => {
+        const doBlankSearch = !this.$store.state.optUpdateMimeMap;
+
+        Sist2Api.getMimeTypes(Sist2Query.searchQuery(doBlankSearch)).then(({mimeMap}) => {
           this.$store.commit("setUiMimeMap", mimeMap);
           this.uiLoading = false;
           this.search(true);
@@ -206,7 +208,7 @@ export default Vue.extend({
       this.$store.commit("setUiReachedScrollEnd", false);
     },
     async handleSearch(resp: EsResult) {
-      if (resp.hits.hits.length == 0) {
+      if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
         this.$store.commit("setUiReachedScrollEnd", true);
       }
 
@@ -246,6 +248,8 @@ export default Vue.extend({
       this.$store.commit("setLastQueryResult", resp);
 
       this.docs.push(...resp.hits.hits);
+
+      resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
     },
     getDateRange(): Promise<{ min: number, max: number }> {
       return sist2.esQuery({

src/cli.c

@@ -81,6 +81,11 @@ void web_args_destroy(web_args_t *args) {
 }
 
 void exec_args_destroy(exec_args_t *args) {
+
+    if (args->index_path != NULL) {
+        free(args->index_path);
+    }
+
     free(args);
 }
 
@@ -124,6 +129,9 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         args->tn_count = DEFAULT_THUMBNAIL_COUNT;
     } else if (args->tn_count == OPTION_VALUE_DISABLE) {
         args->tn_count = 0;
+    } else if (args->tn_count > 1000) {
+        printf("Invalid value --thumbnail-count argument: %d. Must be <= 1000.\n", args->tn_size);
+        return 1;
     }
 
     if (args->content_size == OPTION_VALUE_UNSPECIFIED) {
@@ -390,6 +398,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
 
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
     LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
+    LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl)
     LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path)
     LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
     LOG_DEBUGF("cli.c", "arg async_script=%d", args->async_script)
@@ -504,6 +513,7 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
 
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
     LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
+    LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl)
     LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline)
     LOG_DEBUGF("cli.c", "arg dev=%d", args->dev)
     LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address)

src/cli.h

@@ -50,6 +50,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv);
 typedef struct index_args {
     char *es_url;
     char *es_index;
+    int es_insecure_ssl;
     char *index_path;
     const char *script_path;
     char *script;
@@ -68,6 +69,7 @@ typedef struct index_args {
 typedef struct web_args {
     char *es_url;
     char *es_index;
+    int es_insecure_ssl;
     char *listen_address;
     char *credentials;
     char *tag_credentials;
@@ -85,7 +87,8 @@ typedef struct web_args {
 typedef struct exec_args {
     char *es_url;
     char *es_index;
-    const char *index_path;
+    int es_insecure_ssl;
+    char *index_path;
     const char *script_path;
     int async_script;
     char *script;

src/ctx.h

@@ -75,10 +75,12 @@ typedef struct {
     int verbose;
     int very_verbose;
     int no_color;
+    int json_logs;
 } LogCtx_t;
 
 typedef struct {
     char *es_url;
+    int es_insecure_ssl;
     es_version_t *es_version;
     char *es_index;
     int batch_size;
@@ -97,6 +99,7 @@ typedef struct {
     char *es_url;
     es_version_t *es_version;
     char *es_index;
+    int es_insecure_ssl;
     int index_count;
     char *auth_user;
     char *auth_pass;

src/index/elastic.c

@@ -21,6 +21,8 @@ void free_queue(int max);
 
 void elastic_flush();
 
+void print_error(response_t *r);
+
 void destroy_indexer(es_indexer_t *indexer) {
 
     if (indexer == NULL) {
@@ -45,13 +47,13 @@ void elastic_cleanup() {
     destroy_indexer(Indexer);
 }
 
-void print_json(cJSON *document, const char id_str[MD5_STR_LENGTH]) {
+void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) {
 
     cJSON *line = cJSON_CreateObject();
 
     cJSON_AddStringToObject(line, "_id", id_str);
     cJSON_AddStringToObject(line, "_index", IndexCtx.es_index);
-    cJSON_AddStringToObject(line, "_type", "_doc");
+//    cJSON_AddStringToObject(line, "_type", "_doc");
     cJSON_AddItemReferenceToObject(line, "_source", document);
 
     char *json = cJSON_PrintUnformatted(line);
@@ -72,19 +74,19 @@ void delete_document(const char* document_id_str, void* UNUSED(_data)) {
     bulk_line->type = ES_BULK_LINE_DELETE;
     bulk_line->next = NULL;
 
-    memcpy(bulk_line->path_md5_str, document_id_str, MD5_STR_LENGTH);
+    strcpy(bulk_line->doc_id, document_id_str);
     tpool_add_work(IndexCtx.pool, index_json_func, bulk_line);
 }
 
 
-void index_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]) {
+void index_json(cJSON *document, const char doc_id[SIST_DOC_ID_LEN]) {
     char *json = cJSON_PrintUnformatted(document);
 
     size_t json_len = strlen(json);
     es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t) + json_len + 2);
     bulk_line->type = ES_BULK_LINE_INDEX;
     memcpy(bulk_line->line, json, json_len);
-    memcpy(bulk_line->path_md5_str, index_id_str, MD5_STR_LENGTH);
+    strcpy(bulk_line->doc_id, doc_id);
     *(bulk_line->line + json_len) = '\n';
     *(bulk_line->line + json_len + 1) = '\0';
     bulk_line->next = NULL;
@@ -93,7 +95,7 @@ void index_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]) {
     tpool_add_work(IndexCtx.pool, index_json_func, bulk_line);
 }
 
-void execute_update_script(const char *script, int async, const char index_id[MD5_STR_LENGTH]) {
+void execute_update_script(const char *script, int async, const char index_id[SIST_INDEX_ID_LEN]) {
 
     if (Indexer == NULL) {
         Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
@@ -108,16 +110,16 @@ void execute_update_script(const char *script, int async, const char index_id[MD
     cJSON *term_obj = cJSON_AddObjectToObject(query, "term");
     cJSON_AddStringToObject(term_obj, "index", index_id);
 
-    char *str = cJSON_Print(body);
+    char *str = cJSON_PrintUnformatted(body);
 
-    char bulk_url[4096];
+    char url[4096];
     if (async) {
-        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query?wait_for_completion=false", Indexer->es_url,
+        snprintf(url, sizeof(url), "%s/%s/_update_by_query?wait_for_completion=false", Indexer->es_url,
                  Indexer->es_index);
     } else {
-        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query", Indexer->es_url, Indexer->es_index);
+        snprintf(url, sizeof(url), "%s/%s/_update_by_query", Indexer->es_url, Indexer->es_index);
     }
-    response_t *r = web_post(bulk_url, str);
+    response_t *r = web_post(url, str, IndexCtx.es_insecure_ssl);
     if (!async) {
         LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
     }
@@ -137,13 +139,18 @@ void execute_update_script(const char *script, int async, const char index_id[MD
 
     if (async) {
         cJSON *task = cJSON_GetObjectItem(resp, "task");
+
+        if (task == NULL) {
+            LOG_FATALF("elastic.c", "FIXME: Could not get task id: %s", r->body);
+        }
+
         LOG_INFOF("elastic.c", "User script queued: %s/_tasks/%s", Indexer->es_url, task->valuestring);
     }
 
     cJSON_Delete(resp);
 }
 
-void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
+void *create_bulk_buffer(int max, int *count, size_t *buf_len, int legacy) {
     es_bulk_line_t *line = Indexer->line_head;
     *count = 0;
 
@@ -164,11 +171,20 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
     while (line != NULL && *count < max) {
         char action_str[256];
         if (line->type == ES_BULK_LINE_INDEX) {
-            snprintf(
+
-                    action_str, sizeof(action_str),
+            if (legacy) {
-                    "{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n",
+                snprintf(
-                    line->path_md5_str, Indexer->es_index
+                        action_str, sizeof(action_str),
-            );
+                        "{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n",
+                        line->doc_id, Indexer->es_index
+                );
+            } else {
+                snprintf(
+                        action_str, sizeof(action_str),
+                        "{\"index\":{\"_id\":\"%s\",\"_index\":\"%s\"}}\n",
+                        line->doc_id, Indexer->es_index
+                );
+            }
 
             size_t action_str_len = strlen(action_str);
             size_t line_len = strlen(line->line);
@@ -184,7 +200,7 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
             snprintf(
                     action_str, sizeof(action_str),
                     "{\"delete\":{\"_id\":\"%s\",\"_index\":\"%s\"}}\n",
-                    line->path_md5_str, Indexer->es_index
+                    line->doc_id, Indexer->es_index
             );
 
             size_t action_str_len = strlen(action_str);
@@ -212,7 +228,13 @@ void print_errors(response_t *r) {
     *(tmp + r->size) = '\0';
 
     cJSON *ret_json = cJSON_Parse(tmp);
-    if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) {
+    cJSON *errors = cJSON_GetObjectItem(ret_json, "errors");
+
+    if (errors == NULL) {
+        char *str = cJSON_Print(ret_json);
+        LOG_ERRORF("elastic.c", "%s\n", str);
+        cJSON_free(str);
+    } else if (errors->valueint != 0) {
         cJSON *err;
         cJSON_ArrayForEach(err, cJSON_GetObjectItem(ret_json, "items")) {
             if (cJSON_GetObjectItem(cJSON_GetObjectItem(err, "index"), "status")->valueint != 201) {
@@ -250,11 +272,11 @@ void _elastic_flush(int max) {
 
     size_t buf_len;
     int count;
-    void *buf = create_bulk_buffer(max, &count, &buf_len);
+    void *buf = create_bulk_buffer(max, &count, &buf_len, IS_LEGACY_VERSION(IndexCtx.es_version));
 
     char bulk_url[4096];
     snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_bulk?pipeline=tie", Indexer->es_url, Indexer->es_index);
-    response_t *r = web_post(bulk_url, buf);
+    response_t *r = web_post(bulk_url, buf, IndexCtx.es_insecure_ssl);
 
     if (r->status_code == 0) {
         LOG_FATALF("elastic.c", "Could not connect to %s, make sure that elasticsearch is running!\n", IndexCtx.es_url)
@@ -263,7 +285,7 @@ void _elastic_flush(int max) {
     if (r->status_code == 413) {
 
         if (max <= 1) {
-            LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->path_md5_str)
+            LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->doc_id)
             free_response(r);
             free(buf);
             free_queue(1);
@@ -380,7 +402,7 @@ void finish_indexer(char *script, int async_script, char *index_id) {
     char url[4096];
 
     snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index);
-    response_t *r = web_post(url, "");
+    response_t *r = web_post(url, "", IndexCtx.es_insecure_ssl);
     LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
     free_response(r);
 
@@ -389,36 +411,44 @@ void finish_indexer(char *script, int async_script, char *index_id) {
         free(script);
 
         snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_post(url, "");
+        r = web_post(url, "", IndexCtx.es_insecure_ssl);
         LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
         free_response(r);
     }
 
     snprintf(url, sizeof(url), "%s/%s/_forcemerge", IndexCtx.es_url, IndexCtx.es_index);
-    r = web_post(url, "");
+    r = web_post(url, "", IndexCtx.es_insecure_ssl);
     LOG_INFOF("elastic.c", "Merge index <%d>", r->status_code);
     free_response(r);
 
     snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
-    r = web_put(url, "{\"index\":{\"refresh_interval\":\"1s\"}}");
+    r = web_put(url, "{\"index\":{\"refresh_interval\":\"1s\"}}", IndexCtx.es_insecure_ssl);
     LOG_INFOF("elastic.c", "Set refresh interval <%d>", r->status_code);
     free_response(r);
 }
 
-es_version_t *elastic_get_version(const char *es_url) {
+es_version_t *elastic_get_version(const char *es_url, int insecure) {
-    response_t *r = web_get(es_url, 30);
+    response_t *r = web_get(es_url, 30, insecure);
 
     char *tmp = malloc(r->size + 1);
     memcpy(tmp, r->body, r->size);
     *(tmp + r->size) = '\0';
     cJSON *response = cJSON_Parse(tmp);
     free(tmp);
-    free_response(r);
 
     if (response == NULL) {
         return NULL;
     }
 
+    if (cJSON_GetObjectItem(response, "error") != NULL) {
+        LOG_WARNING("elastic.c", "Could not get Elasticsearch version")
+        print_error(r);
+        free_response(r);
+        return NULL;
+    }
+
+    free_response(r);
+
     if (cJSON_GetObjectItem(response, "version") == NULL ||
         cJSON_GetObjectItem(cJSON_GetObjectItem(response, "version"), "number") == NULL) {
         cJSON_Delete(response);
@@ -443,7 +473,7 @@ es_version_t *elastic_get_version(const char *es_url) {
 
 void elastic_init(int force_reset, const char *user_mappings, const char *user_settings) {
 
-    es_version_t *es_version = elastic_get_version(IndexCtx.es_url);
+    es_version_t *es_version = elastic_get_version(IndexCtx.es_url, IndexCtx.es_insecure_ssl);
     IndexCtx.es_version = es_version;
 
     if (es_version == NULL) {
@@ -452,33 +482,33 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s
 
     LOG_INFOF("elastic.c",
               "Elasticsearch version is %s (supported=%d, legacy=%d)",
-              format_es_version(es_version), IS_SUPPORTED_ES_VERSION(es_version), USE_LEGACY_ES_SETTINGS(es_version));
+              format_es_version(es_version), IS_SUPPORTED_ES_VERSION(es_version), IS_LEGACY_VERSION(es_version));
 
     if (!IS_SUPPORTED_ES_VERSION(es_version)) {
-        LOG_FATAL("elastic.c", "sist2 only supports Elasticsearch v6.8 or newer")
+        LOG_FATAL("elastic.c", "This elasticsearch version is not supported!")
     }
 
     char *settings = NULL;
-    if (USE_LEGACY_ES_SETTINGS(es_version)) {
+    if (IS_LEGACY_VERSION(es_version)) {
-        settings = settings_json;
-    } else {
         settings = settings_legacy_json;
+    } else {
+        settings = settings_json;
     }
 
     // Check if index exists
     char url[4096];
     snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index);
-    response_t *r = web_get(url, 30);
+    response_t *r = web_get(url, 30, IndexCtx.es_insecure_ssl);
     int index_exists = r->status_code == 200;
     free_response(r);
 
     if (!index_exists || force_reset) {
-        r = web_delete(url);
+        r = web_delete(url, IndexCtx.es_insecure_ssl);
         LOG_INFOF("elastic.c", "Delete index <%d>", r->status_code);
         free_response(r);
 
         snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_put(url, "");
+        r = web_put(url, "", IndexCtx.es_insecure_ssl);
 
         if (r->status_code != 200) {
             print_error(r);
@@ -489,17 +519,17 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s
         free_response(r);
 
         snprintf(url, sizeof(url), "%s/%s/_close", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_post(url, "");
+        r = web_post(url, "", IndexCtx.es_insecure_ssl);
         LOG_INFOF("elastic.c", "Close index <%d>", r->status_code);
         free_response(r);
 
         snprintf(url, sizeof(url), "%s/_ingest/pipeline/tie", IndexCtx.es_url);
-        r = web_put(url, pipeline_json);
+        r = web_put(url, pipeline_json, IndexCtx.es_insecure_ssl);
         LOG_INFOF("elastic.c", "Create pipeline <%d>", r->status_code);
         free_response(r);
 
         snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_put(url, user_settings ? user_settings : settings);
+        r = web_put(url, user_settings ? user_settings : settings, IndexCtx.es_insecure_ssl);
         LOG_INFOF("elastic.c", "Update ES settings <%d>", r->status_code);
         if (r->status_code != 200) {
             print_error(r);
@@ -507,8 +537,13 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s
         }
         free_response(r);
 
-        snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index);
+        if (IS_LEGACY_VERSION(es_version)) {
-        r = web_put(url, user_mappings ? user_mappings : mappings_json);
+            snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index);
+        } else {
+            snprintf(url, sizeof(url), "%s/%s/_mappings", IndexCtx.es_url, IndexCtx.es_index);
+        }
+
+        r = web_put(url, user_mappings ? user_mappings : mappings_json, IndexCtx.es_insecure_ssl);
         LOG_INFOF("elastic.c", "Update ES mappings <%d>", r->status_code);
         if (r->status_code != 200) {
             print_error(r);
@@ -517,7 +552,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s
         free_response(r);
 
         snprintf(url, sizeof(url), "%s/%s/_open", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_post(url, "");
+        r = web_post(url, "", IndexCtx.es_insecure_ssl);
         LOG_INFOF("elastic.c", "Open index <%d>", r->status_code);
         free_response(r);
     }
@@ -527,7 +562,7 @@ cJSON *elastic_get_document(const char *id_str) {
     char url[4096];
     snprintf(url, sizeof(url), "%s/%s/_doc/%s", WebCtx.es_url, WebCtx.es_index, id_str);
 
-    response_t *r = web_get(url, 3);
+    response_t *r = web_get(url, 3, WebCtx.es_insecure_ssl);
     cJSON *json = NULL;
     if (r->status_code == 200) {
         char *tmp = malloc(r->size + 1);
@@ -545,7 +580,7 @@ char *elastic_get_status() {
     snprintf(url, sizeof(url),
              "%s/_cluster/state/metadata/%s?filter_path=metadata.indices.*.state", WebCtx.es_url, WebCtx.es_index);
 
-    response_t *r = web_get(url, 30);
+    response_t *r = web_get(url, 30, IndexCtx.es_insecure_ssl);
     cJSON *json = NULL;
     char *status = malloc(128 * sizeof(char));
     status[0] = '\0';

src/index/elastic.h

@@ -8,7 +8,7 @@
 
 typedef struct es_bulk_line {
     struct es_bulk_line *next;
-    char path_md5_str[MD5_STR_LENGTH];
+    char doc_id[SIST_DOC_ID_LEN];
     int type;
     char line[0];
 } es_bulk_line_t;
@@ -20,8 +20,10 @@ typedef struct {
 } es_version_t;
 
 #define VERSION_GE(version, maj, min) ((version)->major > (maj) || ((version)->major == (maj) && (version)->minor >= (min)))
-#define IS_SUPPORTED_ES_VERSION(es_version) VERSION_GE((es_version), 6, 8)
+#define VERSION_LT(version, maj, min) (!VERSION_GE(version, maj, min))
-#define USE_LEGACY_ES_SETTINGS(es_version) (!VERSION_GE((es_version), 7, 14))
+
+#define IS_SUPPORTED_ES_VERSION(es_version) ((es_version) != NULL && VERSION_GE((es_version), 6, 8) && VERSION_LT((es_version), 9, 0))
+#define IS_LEGACY_VERSION(es_version) ((es_version) != NULL && VERSION_LT((es_version), 7, 14))
 
 __always_inline
 static const char *format_es_version(es_version_t *version) {
@@ -40,9 +42,9 @@ typedef struct es_indexer es_indexer_t;
 
 void elastic_index_line(es_bulk_line_t *line);
 
-void print_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]);
+void print_json(cJSON *document, const char index_id_str[SIST_INDEX_ID_LEN]);
 
-void index_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]);
+void index_json(cJSON *document, const char doc_id[SIST_INDEX_ID_LEN]);
 
 void delete_document(const char *document_id_str, void* data);
 
@@ -57,8 +59,8 @@ cJSON *elastic_get_document(const char *id_str);
 
 char *elastic_get_status();
 
-es_version_t *elastic_get_version(const char *es_url);
+es_version_t *elastic_get_version(const char *es_url, int insecure);
 
-void execute_update_script(const char *script, int async, const char index_id[MD5_STR_LENGTH]);
+void execute_update_script(const char *script, int async, const char index_id[SIST_INDEX_ID_LEN]);
 
 #endif

src/index/web.c

@@ -22,7 +22,7 @@ void free_response(response_t *resp) {
     free(resp);
 }
 
-void web_post_async_poll(subreq_ctx_t* req) {
+void web_post_async_poll(subreq_ctx_t *req) {
     fd_set fdread;
     fd_set fdwrite;
     fd_set fdexcep;
@@ -34,7 +34,7 @@ void web_post_async_poll(subreq_ctx_t* req) {
 
     CURLMcode mc = curl_multi_fdset(req->multi, &fdread, &fdwrite, &fdexcep, &maxfd);
 
-    if(mc != CURLM_OK) {
+    if (mc != CURLM_OK) {
         req->done = TRUE;
         return;
     }
@@ -47,7 +47,7 @@ void web_post_async_poll(subreq_ctx_t* req) {
     struct timeval timeout = {1, 0};
     int rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
 
-    switch(rc) {
+    switch (rc) {
         case -1:
             req->done = TRUE;
             break;
@@ -64,6 +64,10 @@ void web_post_async_poll(subreq_ctx_t* req) {
         req->response->size = req->response_buf.cur;
         curl_easy_getinfo(req->handle, CURLINFO_RESPONSE_CODE, &req->response->status_code);
 
+        if (req->response->status_code == 0) {
+            LOG_ERRORF("web.c", "CURL Error: %s", req->curl_err_buffer)
+        }
+
         curl_multi_cleanup(req->multi);
         curl_easy_cleanup(req->handle);
         curl_slist_free_all(req->headers);
@@ -71,7 +75,7 @@ void web_post_async_poll(subreq_ctx_t* req) {
     }
 }
 
-subreq_ctx_t *web_post_async(const char *url, char *data) {
+subreq_ctx_t *web_post_async(const char *url, char *data, int insecure) {
     subreq_ctx_t *req = calloc(1, sizeof(subreq_ctx_t));
     req->response = calloc(1, sizeof(response_t));
     req->data = data;
@@ -84,6 +88,11 @@ subreq_ctx_t *web_post_async(const char *url, char *data) {
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
     curl_easy_setopt(curl, CURLOPT_POST, 1);
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+    if (insecure) {
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+    }
+
+    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, req->curl_err_buffer);
 
     struct curl_slist *headers = NULL;
     headers = curl_slist_append(headers, "Content-Type: application/json");
@@ -100,7 +109,7 @@ subreq_ctx_t *web_post_async(const char *url, char *data) {
     return req;
 }
 
-response_t *web_get(const char *url, int timeout) {
+response_t *web_get(const char *url, int timeout, int insecure) {
     response_t *resp = malloc(sizeof(response_t));
 
     CURL *curl;
@@ -112,14 +121,24 @@ response_t *web_get(const char *url, int timeout) {
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
+    if (insecure) {
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+    }
 
     struct curl_slist *headers = NULL;
     headers = curl_slist_append(headers, "Content-Type: application/json");
     curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
 
+    char err_buffer[CURL_ERROR_SIZE + 1] = {};
+    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer);
+
     curl_easy_perform(curl);
     curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);
 
+    if (resp->status_code == 0) {
+        LOG_ERRORF("web.c", "CURL Error: %s", err_buffer)
+    }
+
     curl_easy_cleanup(curl);
     curl_slist_free_all(headers);
 
@@ -128,7 +147,7 @@ response_t *web_get(const char *url, int timeout) {
     return resp;
 }
 
-response_t *web_post(const char *url, const char *data) {
+response_t *web_post(const char *url, const char *data, int insecure) {
 
     response_t *resp = malloc(sizeof(response_t));
 
@@ -141,6 +160,12 @@ response_t *web_post(const char *url, const char *data) {
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
     curl_easy_setopt(curl, CURLOPT_POST, 1);
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+    if (insecure) {
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+    }
+
+    char err_buffer[CURL_ERROR_SIZE + 1] = {};
+    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer);
 
     struct curl_slist *headers = NULL;
     headers = curl_slist_append(headers, "Content-Type: application/json");
@@ -151,17 +176,21 @@ response_t *web_post(const char *url, const char *data) {
     curl_easy_perform(curl);
     curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);
 
-    curl_easy_cleanup(curl);
-    curl_slist_free_all(headers);
-
     resp->body = buffer.buf;
     resp->size = buffer.cur;
 
+    if (resp->status_code == 0) {
+        LOG_ERRORF("web.c", "CURL Error: %s", err_buffer)
+    }
+
+    curl_easy_cleanup(curl);
+    curl_slist_free_all(headers);
+
     return resp;
 }
 
 
-response_t *web_put(const char *url, const char *data) {
+response_t *web_put(const char *url, const char *data, int insecure) {
 
     response_t *resp = malloc(sizeof(response_t));
 
@@ -175,7 +204,10 @@ response_t *web_put(const char *url, const char *data) {
     curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
-    curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4 );
+    curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4);
+    if (insecure) {
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+    }
 
     struct curl_slist *headers = NULL;
     headers = curl_slist_append(headers, "Content-Type: application/json");
@@ -194,7 +226,7 @@ response_t *web_put(const char *url, const char *data) {
     return resp;
 }
 
-response_t *web_delete(const char *url) {
+response_t *web_delete(const char *url, int insecure) {
 
     response_t *resp = malloc(sizeof(response_t));
 
@@ -207,6 +239,9 @@ response_t *web_delete(const char *url) {
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
     curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+    if (insecure) {
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+    }
 
     curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "");
     struct curl_slist *headers = NULL;

src/index/web.h

@@ -25,14 +25,15 @@ typedef struct {
     response_t *response;
     int running_handles;
     int done;
+    char curl_err_buffer[CURL_ERROR_SIZE + 1];
 } subreq_ctx_t;
 
-response_t *web_get(const char *url, int timeout);
+response_t *web_get(const char *url, int timeout, int insecure);
-response_t *web_post(const char * url, const char * data);
+response_t *web_post(const char * url, const char * data, int insecure);
 void web_post_async_poll(subreq_ctx_t* req);
-subreq_ctx_t *web_post_async(const char *url, char *data);
+subreq_ctx_t *web_post_async(const char *url, char *data, int insecure);
-response_t *web_put(const char *url, const char *data);
+response_t *web_put(const char *url, const char *data, int insecure);
-response_t *web_delete(const char *url);
+response_t *web_delete(const char *url, int insecure);
 
 void free_response(response_t *resp);
 

src/io/serialize.c

@@ -124,9 +124,7 @@ char *build_json_string(document_t *doc) {
         cJSON_AddStringToObject(json, "path", "");
     }
 
-    char md5_str[MD5_STR_LENGTH];
+    cJSON_AddStringToObject(json, "_id", doc->doc_id);
-    buf2hex(doc->path_md5, MD5_DIGEST_LENGTH, md5_str);
-    cJSON_AddStringToObject(json, "_id", md5_str);
 
     // Metadata
     meta_line_t *meta = doc->meta_head;
@@ -452,32 +450,31 @@ void read_lines(const char *path, const line_processor_t processor) {
 
     dyn_buffer_destroy(&buf);
     fclose(file);
-
 }
 
-void read_index_ndjson(const char *line, void* _data) {
+void read_index_ndjson(const char *line, void *_data) {
-    void** data = _data;
+    void **data = _data;
-    const char* index_id = data[0];
+    const char *index_id = data[0];
     index_func func = data[1];
     read_index_bin_handle_line(line, index_id, func);
 }
 
-void read_index(const char *path, const char index_id[MD5_STR_LENGTH], const char *type, index_func func) {
+void read_index(const char *path, const char index_id[SIST_INDEX_ID_LEN], const char *type, index_func func) {
     if (strcmp(type, INDEX_TYPE_NDJSON) == 0) {
         read_lines(path, (line_processor_t) {
-            .data = (void*[2]){(void*)index_id, func} ,
+                .data = (void *[2]) {(void *) index_id, func},
-            .func = read_index_ndjson,
+                .func = read_index_ndjson,
         });
     }
 }
 
 static __thread GHashTable *IncrementalReadTable = NULL;
 
-void json_put_incremental(cJSON *document, UNUSED(const char id_str[MD5_STR_LENGTH])) {
+void json_put_incremental(cJSON *document, UNUSED(const char doc_id[SIST_DOC_ID_LEN])) {
     const char *path_md5_str = cJSON_GetObjectItem(document, "_id")->valuestring;
     const int mtime = cJSON_GetObjectItem(document, "mtime")->valueint;
 
-    incremental_put_str(IncrementalReadTable, path_md5_str, mtime);
+    incremental_put(IncrementalReadTable, path_md5_str, mtime);
 }
 
 void incremental_read(GHashTable *table, const char *filepath, index_descriptor_t *desc) {
@@ -490,13 +487,11 @@ static __thread GHashTable *IncrementalNewTable = NULL;
 static __thread store_t *IncrementalCopySourceStore = NULL;
 static __thread store_t *IncrementalCopyDestinationStore = NULL;
 
-void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[MD5_STR_LENGTH])) {
+void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[SIST_DOC_ID_LEN])) {
 
-    const char *path_md5_str = cJSON_GetObjectItem(document, "_id")->valuestring;
+    const char *doc_id = cJSON_GetObjectItem(document, "_id")->valuestring;
-    unsigned char path_md5[MD5_DIGEST_LENGTH];
-    hex2buf(path_md5_str, MD5_STR_LENGTH - 1, path_md5);
 
-    if (cJSON_GetObjectItem(document, "parent") != NULL || incremental_get_str(IncrementalCopyTable, path_md5_str)) {
+    if (cJSON_GetObjectItem(document, "parent") != NULL || incremental_get(IncrementalCopyTable, doc_id)) {
         // Copy index line
         cJSON_DeleteItemFromObject(document, "index");
         char *json_str = cJSON_PrintUnformatted(document);
@@ -505,16 +500,33 @@ void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[MD5_S
         json_str = realloc(json_str, json_str_len + 1);
         *(json_str + json_str_len) = '\n';
 
-        zstd_write_string(json_str, json_str_len + 1);
-        free(json_str);
-
         // Copy tn store contents
         size_t buf_len;
-        char *buf = store_read(IncrementalCopySourceStore, (char *) path_md5, sizeof(path_md5), &buf_len);
+        char *buf = store_read(IncrementalCopySourceStore, (char *) doc_id, SIST_DOC_ID_LEN, &buf_len);
         if (buf_len != 0) {
-            store_write(IncrementalCopyDestinationStore, (char *) path_md5, sizeof(path_md5), buf, buf_len);
+            store_write(IncrementalCopyDestinationStore, (char *) doc_id, SIST_DOC_ID_LEN, buf, buf_len);
             free(buf);
         }
+
+        // Also copy additional thumbnails
+        if (cJSON_GetObjectItem(document, "thumbnail") != NULL) {
+            const int thumbnail_count = cJSON_GetObjectItem(document, "thumbnail")->valueint;
+
+            for (int i = 1; i < thumbnail_count; i++) {
+                char tn_key[SIST_DOC_ID_LEN + sizeof(char) * 4];
+
+                snprintf(tn_key, sizeof(tn_key), "%s%04d", doc_id, i);
+
+                buf = store_read(IncrementalCopySourceStore, tn_key, sizeof(tn_key), &buf_len);
+                if (buf_len != 0) {
+                    store_write(IncrementalCopyDestinationStore, tn_key, sizeof(tn_key), buf, buf_len);
+                    free(buf);
+                }
+            }
+        }
+
+        zstd_write_string(json_str, json_str_len + 1);
+        free(json_str);
     }
 }
 
@@ -536,24 +548,24 @@ void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
     read_index(filepath, "", INDEX_TYPE_NDJSON, incremental_copy_handle_doc);
 }
 
-void incremental_delete_handle_doc(cJSON *document, UNUSED(const char id_str[MD5_STR_LENGTH])) {
+void incremental_delete_handle_doc(cJSON *document, UNUSED(const char id_str[SIST_DOC_ID_LEN])) {
 
-    char path_md5_n[MD5_STR_LENGTH + 1];
+    char doc_id_n[SIST_DOC_ID_LEN + 1];
-    path_md5_n[MD5_STR_LENGTH] = '\0';
+    doc_id_n[SIST_DOC_ID_LEN] = '\0';
-    path_md5_n[MD5_STR_LENGTH - 1] = '\n';
+    doc_id_n[SIST_DOC_ID_LEN - 1] = '\n';
-    const char *path_md5_str = cJSON_GetObjectItem(document, "_id")->valuestring;
+    const char *doc_id = cJSON_GetObjectItem(document, "_id")->valuestring;
 
     // do not delete archive virtual entries
     if (cJSON_GetObjectItem(document, "parent") == NULL 
-        && !incremental_get_str(IncrementalCopyTable, path_md5_str)
+        && !incremental_get(IncrementalCopyTable, doc_id)
-        && !incremental_get_str(IncrementalNewTable, path_md5_str)
+        && !incremental_get(IncrementalNewTable, doc_id)
         ) {
-        memcpy(path_md5_n, path_md5_str, MD5_STR_LENGTH - 1);
+        memcpy(doc_id_n, doc_id, SIST_DOC_ID_LEN - 1);
-        zstd_write_string(path_md5_n, MD5_STR_LENGTH);
+        zstd_write_string(doc_id, sizeof(doc_id_n));
     }
 }
 
-void incremental_delete(const char *del_filepath, const char* index_filepath, 
+void incremental_delete(const char *del_filepath, const char *index_filepath,
                         GHashTable *copy_table, GHashTable *new_table) {
 
     if (WriterCtx.out_file == NULL) {

src/io/serialize.h

@@ -12,7 +12,7 @@ typedef struct line_processor {
   void (*func)(const char*, void*);
 } line_processor_t;
 
-typedef void(*index_func)(cJSON *, const char[MD5_STR_LENGTH]);
+typedef void(*index_func)(cJSON *, const char[SIST_DOC_ID_LEN]);
 
 void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
                       const char *dst_filepath, GHashTable *copy_table);
@@ -24,7 +24,7 @@ void write_document(document_t *doc);
 
 void read_lines(const char *path, const line_processor_t processor);
 
-void read_index(const char *path, const char[MD5_STR_LENGTH], const char *type, index_func);
+void read_index(const char *path, const char index_id[SIST_INDEX_ID_LEN], const char *type, index_func);
 
 void incremental_read(GHashTable *table, const char *filepath, index_descriptor_t *desc);
 
@@ -42,13 +42,13 @@ index_descriptor_t read_index_descriptor(char *path);
 // caller ensures char file_path[PATH_MAX]
 #define READ_INDICES(file_path, index_path, action_ok, action_main_fail, cond_original) \
     snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", index_path);              \
-    if (0 == access(file_path, R_OK)) {                                                 \
+    if (access(file_path, R_OK) == 0) {                                                 \
         action_ok;                                                                      \
     } else {                                                                            \
         action_main_fail;                                                               \
     }                                                                                   \
     snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", index_path);          \
-    if ((cond_original) && (0 == access(file_path, R_OK))) {                            \
+    if ((cond_original) && access(file_path, R_OK) == 0) {                              \
         action_ok;                                                                      \
     }                                                                                   \
 

src/io/store.c

@@ -52,22 +52,7 @@ void store_flush(store_t *store) {
 void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len) {
 
     if (LogCtx.very_verbose) {
-        if (key_len == MD5_DIGEST_LENGTH) {
+        LOG_DEBUGF("store.c", "Store write %s@{%s} %lu bytes", store->path, key, buf_len)
-            char path_md5_str[MD5_STR_LENGTH];
-            buf2hex((unsigned char *) key, MD5_DIGEST_LENGTH, path_md5_str);
-
-            LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", path_md5_str, buf_len)
-
-        } else if (key_len == MD5_DIGEST_LENGTH + sizeof(int)) {
-            char path_md5_str[MD5_STR_LENGTH];
-            buf2hex((unsigned char *) key, MD5_DIGEST_LENGTH, path_md5_str);
-
-            LOG_DEBUGF("store.c", "Store write {%s/%d} %lu bytes",
-                       path_md5_str, *(int *) (key + MD5_DIGEST_LENGTH), buf_len);
-
-        } else {
-            LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", key, buf_len)
-        }
     }
 
 #if (SIST_FAKE_STORE != 1)
@@ -102,7 +87,7 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
     }
 
     if (db_full) {
-        LOG_INFOF("store.c", "Updating mdb mapsize to %lu bytes", store->size)
+        LOG_DEBUGF("store.c", "Updating mdb mapsize to %lu bytes", store->size)
 
         if (should_abort_transaction) {
             mdb_txn_abort(txn);
@@ -131,7 +116,7 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
                        store->path, mdb_strerror(ret), ret,
                        put_ret, put_ret_retry);
         }
-        LOG_INFOF("store.c", "Updated mdb mapsize to %lu bytes", store->size)
+        LOG_DEBUGF("store.c", "Updated mdb mapsize to %lu bytes", store->size)
     } else if (put_ret != 0) {
         LOG_ERROR("store.c", mdb_strerror(put_ret))
     }

src/io/walk.c

@@ -22,7 +22,7 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info,
 
     job->vfile.info = *info;
 
-    memset(job->parent, 0, MD5_DIGEST_LENGTH);
+    job->parent[0] = '\0';
 
     job->vfile.filepath = job->filepath;
     job->vfile.read = fs_read;

src/log.c

@@ -1,4 +1,5 @@
 #include "log.h"
+#include "ctx.h"
 
 #include <pthread.h>
 #include <stdarg.h>
@@ -30,6 +31,30 @@ void vsist_logf(const char *filepath, int level, char *format, va_list ap) {
     strftime(datetime, sizeof(datetime), "%Y-%m-%d %H:%M:%S", &result);
 
     int log_len;
+    if (LogCtx.json_logs) {
+        vsnprintf(log_str, sizeof(log_str), format, ap);
+
+        cJSON *log_str_json = cJSON_CreateString(log_str);
+        char *log_str_json_str = cJSON_PrintUnformatted(log_str_json);
+
+        cJSON *filepath_json = cJSON_CreateString(filepath);
+        char *filepath_json_str = cJSON_PrintUnformatted(filepath_json);
+
+        log_len = snprintf(
+                log_str, sizeof(log_str),
+                "{\"thread\":\"%04llX\",\"datetime\":\"%s\",\"level\":\"%s\",\"filepath\":%s,\"message\":%s}\n",
+                pid, datetime, log_levels[level], filepath_json_str, log_str_json_str
+        );
+
+        cJSON_Delete(filepath_json);
+        cJSON_Delete(log_str_json);
+        free(log_str_json_str);
+        free(filepath_json_str);
+
+        write(STDOUT_FILENO, log_str, log_len);
+        return;
+    }
+
     if (is_tty) {
         log_len = snprintf(
                 log_str, sizeof(log_str),
@@ -97,6 +122,28 @@ void sist_log(const char *filepath, int level, char *str) {
     strftime(datetime, sizeof(datetime), "%Y-%m-%d %H:%M:%S", &result);
 
     int log_len;
+
+    if (LogCtx.json_logs) {
+        cJSON *log_str_json = cJSON_CreateString(str);
+        char *log_str_json_str = cJSON_PrintUnformatted(log_str_json);
+
+        cJSON *filepath_json = cJSON_CreateString(filepath);
+        char *filepath_json_str = cJSON_PrintUnformatted(filepath_json);
+
+        log_len = snprintf(
+                log_str, sizeof(log_str),
+                "{\"thread\":\"%04llX\",\"datetime\":\"%s\",\"level\":\"%s\",\"filepath\":%s,\"message\":%s}\n",
+                pid, datetime, log_levels[level], filepath_json_str, log_str_json_str
+        );
+
+        cJSON_Delete(log_str_json);
+        cJSON_Delete(filepath_json);
+        free(log_str_json_str);
+        free(filepath_json_str);
+
+        write(STDOUT_FILENO, log_str, log_len);
+        return;
+    }
     if (is_tty) {
         log_len = snprintf(
                 log_str, sizeof(log_str),

src/main.c

@@ -38,8 +38,8 @@ static __sighandler_t sigabrt_handler = NULL;
 
 void sig_handler(int signum) {
 
-    LogCtx.verbose = 1;
+    LogCtx.verbose = TRUE;
-    LogCtx.very_verbose = 1;
+    LogCtx.very_verbose = TRUE;
 
     LOG_ERROR("*SIGNAL HANDLER*", "=============================================\n\n");
     LOG_ERRORF("*SIGNAL HANDLER*", "Uh oh! Caught fatal signal: %s", strsignal(signum));
@@ -103,7 +103,7 @@ void sig_handler(int signum) {
     exit(-1);
 }
 
-void init_dir(const char *dirpath) {
+void init_dir(const char *dirpath, scan_args_t *args) {
     char path[PATH_MAX];
     snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
 
@@ -111,9 +111,18 @@ void init_dir(const char *dirpath) {
     strcpy(ScanCtx.index.desc.version, Version);
     strcpy(ScanCtx.index.desc.type, INDEX_TYPE_NDJSON);
 
-    unsigned char index_md5[MD5_DIGEST_LENGTH];
+    if (args->incremental != NULL) {
-    MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
+        // copy old index id
-    buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
+        char descriptor_path[PATH_MAX];
+        snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental);
+        index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
+        memcpy(ScanCtx.index.desc.id, original_desc.id, sizeof(original_desc.id));
+    } else {
+        // generate new index id based on timestamp
+        unsigned char index_md5[MD5_DIGEST_LENGTH];
+        MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
+        buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
+    }
 
     write_index_descriptor(path, &ScanCtx.index.desc);
 }
@@ -315,9 +324,13 @@ void load_incremental_index(const scan_args_t *args) {
         LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc.version, Version)
     }
 
-    READ_INDICES(file_path, args->incremental, incremental_read(ScanCtx.original_table, file_path, &original_desc),
+    READ_INDICES(
-                 LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)),
+            file_path,
-                 1);
+            args->incremental,
+            incremental_read(ScanCtx.original_table, file_path, &original_desc),
+            LOG_DEBUG("main.c", "The base index for incremental scan does not have a main index"),
+            TRUE
+    );
 
     LOG_INFOF("main.c", "Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table))
 }
@@ -378,7 +391,7 @@ void sist2_scan(scan_args_t *args) {
 
     initialize_scan_context(args);
 
-    init_dir(ScanCtx.index.path);
+    init_dir(ScanCtx.index.path, args);
 
     char store_path[PATH_MAX];
     snprintf(store_path, PATH_MAX, "%sthumbs", ScanCtx.index.path);
@@ -422,8 +435,8 @@ void sist2_scan(scan_args_t *args) {
     LOG_DEBUGF("main.c", "Skipped files: %d", ScanCtx.dbg_skipped_files_count)
     LOG_DEBUGF("main.c", "Excluded files: %d", ScanCtx.dbg_excluded_files_count)
     LOG_DEBUGF("main.c", "Failed files: %d", ScanCtx.dbg_failed_files_count)
-    LOG_DEBUGF("main.c", "Thumbnail store size: %d", ScanCtx.stat_tn_size)
+    LOG_DEBUGF("main.c", "Thumbnail store size: %lu", ScanCtx.stat_tn_size)
-    LOG_DEBUGF("main.c", "Index size: %d", ScanCtx.stat_index_size)
+    LOG_DEBUGF("main.c", "Index size: %lu", ScanCtx.stat_index_size)
 
     if (args->incremental != NULL) {
         save_incremental_index(args);
@@ -440,6 +453,7 @@ void sist2_index(index_args_t *args) {
 
     IndexCtx.es_url = args->es_url;
     IndexCtx.es_index = args->es_index;
+    IndexCtx.es_insecure_ssl = args->es_insecure_ssl;
     IndexCtx.batch_size = args->batch_size;
     IndexCtx.needs_es_connection = !args->print;
 
@@ -525,6 +539,8 @@ void sist2_exec_script(exec_args_t *args) {
 
     IndexCtx.es_url = args->es_url;
     IndexCtx.es_index = args->es_index;
+    IndexCtx.es_insecure_ssl = args->es_insecure_ssl;
+    IndexCtx.needs_es_connection = TRUE;
 
     LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type)
 
@@ -536,6 +552,7 @@ void sist2_web(web_args_t *args) {
 
     WebCtx.es_url = args->es_url;
     WebCtx.es_index = args->es_index;
+    WebCtx.es_insecure_ssl = args->es_insecure_ssl;
     WebCtx.index_count = args->index_count;
     WebCtx.auth_user = args->auth_user;
     WebCtx.auth_pass = args->auth_pass;
@@ -606,6 +623,7 @@ int main(int argc, const char *argv[]) {
     int arg_version = 0;
 
     char *common_es_url = NULL;
+    int common_es_insecure_ssl = 0;
     char *common_es_index = NULL;
     char *common_script_path = NULL;
     int common_async_script = 0;
@@ -617,6 +635,7 @@ int main(int argc, const char *argv[]) {
             OPT_BOOLEAN('v', "version", &arg_version, "Show version and exit"),
             OPT_BOOLEAN(0, "verbose", &LogCtx.verbose, "Turn on logging"),
             OPT_BOOLEAN(0, "very-verbose", &LogCtx.very_verbose, "Turn on debug messages"),
+            OPT_BOOLEAN(0, "json-logs", &LogCtx.json_logs, "Output logs in JSON format."),
 
             OPT_GROUP("Scan options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),
@@ -671,6 +690,7 @@ int main(int argc, const char *argv[]) {
             OPT_GROUP("Index options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"),
+            OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."),
             OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
             OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
             OPT_BOOLEAN(0, "incremental-index", &index_args->incremental,
@@ -685,6 +705,7 @@ int main(int argc, const char *argv[]) {
 
             OPT_GROUP("Web options"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
+            OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."),
             OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
             OPT_STRING(0, "bind", &web_args->listen_address, "Listen on this address. DEFAULT=localhost:4090"),
             OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"),
@@ -695,6 +716,7 @@ int main(int argc, const char *argv[]) {
 
             OPT_GROUP("Exec-script options"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
+            OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."),
             OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
             OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
             OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),
@@ -724,6 +746,10 @@ int main(int argc, const char *argv[]) {
     index_args->es_index = common_es_index;
     exec_args->es_index = common_es_index;
 
+    web_args->es_insecure_ssl = common_es_insecure_ssl;
+    index_args->es_insecure_ssl = common_es_insecure_ssl;
+    exec_args->es_insecure_ssl = common_es_insecure_ssl;
+
     index_args->script_path = common_script_path;
     exec_args->script_path = common_script_path;
     index_args->threads = common_threads;
@@ -767,9 +793,8 @@ int main(int argc, const char *argv[]) {
         sist2_exec_script(exec_args);
 
     } else {
-        fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
         argparse_usage(&argparse);
-        goto end;
+        LOG_FATALF("main.c", "Invalid command: '%s'\n", argv[0])
     }
     printf("\n");
 

src/parsing/parse.c

@@ -5,6 +5,7 @@
 #include "mime.h"
 #include "src/io/serialize.h"
 #include "src/parsing/sidecar.h"
+#include "src/magic_generated.c"
 
 #include <magic.h>
 
@@ -69,7 +70,7 @@ void parse(void *arg) {
     doc->base = (short) job->base;
 
     char *rel_path = doc->filepath + ScanCtx.index.desc.root_len;
-    MD5((unsigned char *) rel_path, strlen(rel_path), doc->path_md5);
+    generate_doc_id(rel_path, doc->doc_id);
 
     doc->meta_head = NULL;
     doc->meta_tail = NULL;
@@ -77,10 +78,10 @@ void parse(void *arg) {
     doc->size = job->vfile.info.st_size;
     doc->mtime = (int) job->vfile.info.st_mtim.tv_sec;
 
-    int inc_ts = incremental_get(ScanCtx.original_table, doc->path_md5);
+    int inc_ts = incremental_get(ScanCtx.original_table, doc->doc_id);
     if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) {
         pthread_mutex_lock(&ScanCtx.copy_table_mu);
-        incremental_mark_file(ScanCtx.copy_table, doc->path_md5);
+        incremental_mark_file(ScanCtx.copy_table, doc->doc_id);
         pthread_mutex_unlock(&ScanCtx.copy_table_mu);
 
         pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu);
@@ -96,16 +97,14 @@ void parse(void *arg) {
 
     if (ScanCtx.new_table != NULL) {
         pthread_mutex_lock(&ScanCtx.copy_table_mu);
-        incremental_mark_file(ScanCtx.new_table, doc->path_md5);
+        incremental_mark_file(ScanCtx.new_table, doc->doc_id);
         pthread_mutex_unlock(&ScanCtx.copy_table_mu);
     }
 
     char *buf[MAGIC_BUF_SIZE];
 
     if (LogCtx.very_verbose) {
-        char path_md5_str[MD5_STR_LENGTH];
+        LOG_DEBUGF(job->filepath, "Starting parse job {%s}", doc->doc_id)
-        buf2hex(doc->path_md5, MD5_DIGEST_LENGTH, path_md5_str);
-        LOG_DEBUGF(job->filepath, "Starting parse job {%s}", path_md5_str)
     }
 
     if (job->vfile.info.st_size == 0) {
@@ -145,7 +144,15 @@ void parse(void *arg) {
         }
 
         magic_t magic = magic_open(MAGIC_MIME_TYPE);
-        magic_load(magic, NULL);
+
+        const char *magic_buffers[1] = {magic_database_buffer,};
+        size_t sizes[1] = {sizeof(magic_database_buffer),};
+
+        int load_ret = magic_load_buffers(magic, (void **) &magic_buffers, sizes, 1);
+
+        if (load_ret != 0) {
+            LOG_FATALF("parse.c", "Could not load libmagic database: (%d)", load_ret)
+        }
 
         const char *magic_mime_str = magic_buffer(magic, buf, bytes_read);
         if (magic_mime_str != NULL) {
@@ -218,10 +225,10 @@ void parse(void *arg) {
     abort:
 
     //Parent meta
-    if (!md5_digest_is_null(job->parent)) {
+    if (job->parent[0] != '\0') {
-        meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + MD5_STR_LENGTH);
+        meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + SIST_INDEX_ID_LEN);
         meta_parent->key = MetaParent;
-        buf2hex(job->parent, MD5_DIGEST_LENGTH, meta_parent->str_val);
+        strcpy(meta_parent->str_val, job->parent);
         APPEND_META((doc), meta_parent)
 
         doc->has_parent = TRUE;

src/parsing/sidecar.c

@@ -23,16 +23,19 @@ void parse_sidecar(vfile_t *vfile, document_t *doc) {
     }
     char *json_str = cJSON_PrintUnformatted(json);
 
-    unsigned char path_md5[MD5_DIGEST_LENGTH];
+    char assoc_doc_id[SIST_DOC_ID_LEN];
-    MD5((unsigned char *) vfile->filepath + ScanCtx.index.desc.root_len, doc->ext - 1 - ScanCtx.index.desc.root_len,
-        path_md5);
 
-    char path_md5_str[MD5_STR_LENGTH];
+    char rel_path[PATH_MAX];
-    buf2hex(path_md5, MD5_DIGEST_LENGTH, path_md5_str);
+    size_t rel_path_len = doc->ext - 1 - ScanCtx.index.desc.root_len;
+    memcpy(rel_path, vfile->filepath + ScanCtx.index.desc.root_len, rel_path_len);
+    *(rel_path + rel_path_len) = '\0';
 
-    store_write(ScanCtx.index.meta_store, path_md5_str, MD5_STR_LENGTH, json_str, strlen(json_str) + 1);
+    generate_doc_id(rel_path, assoc_doc_id);
+
+    store_write(ScanCtx.index.meta_store, assoc_doc_id, sizeof(assoc_doc_id), json_str,
+                strlen(json_str) + 1);
 
     cJSON_Delete(json);
     free(json_str);
     free(buf);
-}
+}