cleanup

This reverts commit 13e7ea188b6556a3c1072e9241e49587ef1cece0.

.dockerignore

@@ -29,3 +29,7 @@ sist2
 **/core
 *.a
 tmp_scan/
+Dockerfile
+Dockerfile.arm64
+docker-compose.yml
+state.db

.gitignore

@@ -10,6 +10,9 @@ Makefile
 LOG
 sist2*
 !sist2-vue/
+!sist2-admin
+!sist2_admin
+!sist2.py
 *.sist2/
 bundle*.css
 bundle.js
@@ -26,3 +29,7 @@ test_i_inc
 node_modules/
 .cmake/
 i_inc/
+state.db
+*.pyc
+!sist2-admin/frontend/dist
+*.js.map

Dockerfile

@@ -2,14 +2,29 @@ FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
-COPY . .
+
+COPY scripts scripts
+COPY schema schema
+COPY CMakeLists.txt .
+COPY third-party third-party
+COPY src src
+COPY sist2-vue sist2-vue
+
 RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
 RUN make -j$(nproc)
 RUN strip sist2 || mv sist2_debug sist2
 
-FROM --platform="linux/amd64" ubuntu:21.10
+FROM --platform="linux/amd64" ubuntu:20.04
 
-RUN apt update && apt install -y curl libasan5 libmagic1 && rm -rf /var/lib/apt/lists/*
+WORKDIR /root
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["sist2"]
+
+RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 libmagic1 python3  \
+    python3-pip git tesseract-ocr && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -20,9 +35,10 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
     curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
 
-ENTRYPOINT ["/root/sist2"]
+# sist2
+COPY --from=build /build/sist2 sist2
 
-ENV LANG C.UTF-8
+# sist2-admin
-ENV LC_ALL C.UTF-8
+COPY sist2-admin/requirements.txt sist2-admin/
-
+RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
-COPY --from=build /build/sist2 /root/sist2
+COPY sist2-admin/ sist2-admin/

docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3"
+
+services:
+  elasticsearch:
+    image: elasticsearch:7.14.0
+    container_name: sist2-es
+    environment:
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+  sist2-admin:
+    build:
+      context: .
+    container_name: sist2-admin
+    volumes:
+      - /mnt/array/sist2-admin-data/:/sist2-admin/
+      - /:/host
+    ports:
+      # NOTE: Don't export this port publicly!
+      - 8080:8080
+      - 4090:4090
+    working_dir: /root/sist2-admin/
+    entrypoint: python3
+    command:
+      - /root/sist2-admin/sist2_admin/app.py

schema/mappings.json

@@ -67,7 +67,8 @@
       "index": false
     },
     "mtime": {
-      "type": "integer"
+      "type": "date",
+      "format": "epoch_millis"
     },
     "size": {
       "type": "long"

scripts/reset.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-make clean
-rm -rf CMakeFiles/ CMakeCache.txt Makefile \
-  third-party/libscan/CMakeFiles third-party/libscan/CMakeCache.txt third-party/libscan/third-party/ext_ffmpeg \
-  third-party/libscan/third-party/ext_libmobi third-party/libscan/Makefile

sist2-admin/frontend/babel.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  presets: [
+    '@vue/cli-plugin-babel/preset'
+  ]
+}

sist2-admin/frontend/dist/css/app.0f0b676b.css

@@ -0,0 +1 @@
+.navbar[data-v-27bc1d68]{box-shadow:0 .125rem .25rem rgba(0,0,0,.08)!important;border-radius:0}.theme-black .navbar[data-v-27bc1d68]{background:rgba(84,107,122,.18823529411764706);border-bottom:none}.navbar-brand[data-v-27bc1d68]{color:#222!important;font-size:1.75rem;padding:0}.navbar-brand[data-v-27bc1d68]:hover{color:#000!important}.version[data-v-27bc1d68]{color:#222!important;margin-left:-18px;margin-top:-14px;font-size:11px;font-family:monospace}.btn-link[data-v-27bc1d68]{color:#222}body,html{height:100%}#app{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;color:#2c3e50;padding-bottom:1em;min-height:100%}.info-icon{width:1rem;margin-right:.2rem;cursor:pointer;line-height:1rem;height:1rem;background-image:url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjYuNjY3IDQyNi42NjciIGZpbGw9IiNmZmYiPjxwYXRoIGQ9Ik0xOTIgMTkyaDQyLjY2N3YxMjhIMTkyeiIvPjxwYXRoIGQ9Ik0yMTMuMzMzIDBDOTUuNDY3IDAgMCA5NS40NjcgMCAyMTMuMzMzczk1LjQ2NyAyMTMuMzMzIDIxMy4zMzMgMjEzLjMzM1M0MjYuNjY3IDMzMS4yIDQyNi42NjcgMjEzLjMzMyAzMzEuMiAwIDIxMy4zMzMgMHptMCAzODRjLTk0LjA4IDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMgNDIuNjY3IDIxMy4zMzMgNDIuNjY3IDM4NCAxMTkuMjUzIDM4NCAyMTMuMzMzIDMwNy40MTMgMzg0IDIxMy4zMzMgMzg0eiIvPjxwYXRoIGQ9Ik0xOTIgMTA2LjY2N2g0Mi42Njd2NDIuNjY3SDE5MnoiLz48L3N2Zz4=);filter:brightness(45%);display:block}.tabs{margin-top:10px}.modal-title{text-overflow:ellipsis;overflow:hidden;white-space:nowrap}@media screen and (min-width:1500px){.container{max-width:1440px}}label{margin-top:.5rem;margin-bottom:0}.shrink[data-v-9b017c42]{flex-grow:inherit}#task-history[data-v-46960281]{font-family:monospace;font-size:12px}#log-tail-output span{display:block}span.DEBUG{color:#9e9e9e}span.WARNING{color:#ffb300}span.INFO{color:#039be5}span.ERROR,span.FATAL{color:#f4511e}span.ADMIN{color:#ee05ff}#log-tail-output{font-size:13px;font-family:monospace;padding:6px;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px;margin:3px;white-space:pre;color:#000;overflow:hidden}

sist2-admin/frontend/dist/index.html

@@ -0,0 +1 @@
+<!DOCTYPE html><html lang=""><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="favicon.ico"><title>sist2-admin</title><link href="css/app.0f0b676b.css" rel="preload" as="style"><link href="css/chunk-vendors.aa66c7e8.css" rel="preload" as="style"><link href="js/app.b34f501e.js" rel="preload" as="script"><link href="js/chunk-vendors.fad0ee6a.js" rel="preload" as="script"><link href="css/chunk-vendors.aa66c7e8.css" rel="stylesheet"><link href="css/app.0f0b676b.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but sist2-admin-vue doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="js/chunk-vendors.fad0ee6a.js"></script><script src="js/app.b34f501e.js"></script></body></html>

sist2-admin/frontend/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "sist2-admin-vue",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "serve": "vue-cli-service serve",
+    "build": "vue-cli-service build",
+    "watch": "vue-cli-service build --watch"
+  },
+  "dependencies": {
+    "axios": "^0.27.2",
+    "bootstrap-vue": "^2.21.2",
+    "core-js": "^3.6.5",
+    "moment": "^2.29.3",
+    "socket.io-client": "^4.5.1",
+    "vue": "^2.6.14",
+    "vue-i18n": "^8.24.4",
+    "vue-router": "^3.5.4",
+    "vuex": "^3.4.0"
+  },
+  "devDependencies": {
+    "@vue/cli-plugin-babel": "~4.5.13",
+    "@vue/cli-plugin-eslint": "~4.5.13",
+    "@vue/cli-plugin-router": "~4.5.13",
+    "@vue/cli-plugin-vuex": "~4.5.13",
+    "@vue/cli-service": "~4.5.13",
+    "babel-eslint": "^10.1.0",
+    "bootstrap": "^4.5.2",
+    "eslint": "^6.7.2",
+    "eslint-plugin-vue": "^6.2.2",
+    "vue-template-compiler": "^2.6.11"
+  },
+  "eslintConfig": {
+    "root": true,
+    "env": {
+      "node": true
+    },
+    "extends": [
+      "plugin:vue/essential",
+      "eslint:recommended"
+    ],
+    "parserOptions": {
+      "parser": "babel-eslint"
+    },
+    "rules": {}
+  },
+  "browserslist": [
+    "> 1%",
+    "last 2 versions",
+    "not dead"
+  ]
+}

sist2-admin/frontend/public/index.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width,initial-scale=1.0">
+    <link rel="icon" href="<%= BASE_URL %>favicon.ico">
+    <title>sist2-admin</title>
+  </head>
+  <body>
+    <noscript>
+      <strong>We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled. Please enable it to continue.</strong>
+    </noscript>
+    <div id="app"></div>
+    <!-- built files will be auto injected -->
+  </body>
+</html>

sist2-admin/frontend/src/App.vue

@@ -0,0 +1,98 @@
+<template>
+  <div id="app">
+    <NavBar></NavBar>
+    <b-container class="pt-4">
+      <b-alert show dismissible variant="info">
+        This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
+        to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on Github</a>. Thank you!
+      </b-alert>
+      <router-view/>
+    </b-container>
+  </div>
+</template>
+
+<script>
+import NavBar from "@/components/NavBar";
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  components: {NavBar},
+  data() {
+    return {
+      socket: null
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getSist2AdminInfo()
+        .then(resp => this.$store.commit("setSist2AdminInfo", resp.data));
+    this.$store.dispatch("loadBrowserSettings");
+    this.connectNotifications();
+    // this.socket.onclose = this.connectNotifications;
+  },
+  methods: {
+    connectNotifications() {
+      this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
+      this.socket.onopen = () => {
+        this.socket.send("Hello from client");
+      }
+
+      this.socket.onmessage = e => {
+        const notification = JSON.parse(e.data);
+        if (notification.message) {
+          notification.messageString = this.$t(notification.message).toString();
+        }
+
+        this.$store.dispatch("notify", notification)
+      }
+    }
+  }
+}
+</script>
+
+<style>
+html, body {
+  height: 100%;
+}
+
+#app {
+  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  /*text-align: center;*/
+  color: #2c3e50;
+  padding-bottom: 1em;
+  min-height: 100%;
+}
+
+.info-icon {
+  width: 1rem;
+  margin-right: 0.2rem;
+  cursor: pointer;
+  line-height: 1rem;
+  height: 1rem;
+  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+  filter: brightness(45%);
+  display: block;
+}
+
+.tabs {
+  margin-top: 10px;
+}
+
+.modal-title {
+  text-overflow: ellipsis;
+  overflow: hidden;
+  white-space: nowrap;
+}
+
+@media screen and (min-width: 1500px) {
+  .container {
+    max-width: 1440px;
+  }
+}
+
+label {
+  margin-top: 0.5rem;
+  margin-bottom: 0;
+}
+</style>

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -0,0 +1,117 @@
+import axios from "axios";
+
+class Sist2AdminApi {
+
+    constructor() {
+        this.baseUrl = window.location.protocol + "//" + window.location.host;
+    }
+
+    getJobs() {
+        return axios.get(`${this.baseUrl}/api/job/`);
+    }
+
+    getFrontends() {
+        return axios.get(`${this.baseUrl}/api/frontend/`);
+    }
+
+    getTasks() {
+        return axios.get(`${this.baseUrl}/api/task/`);
+    }
+
+    killTask(taskId) {
+        return axios.post(`${this.baseUrl}/api/task/${taskId}/kill`)
+    }
+
+    getTaskHistory() {
+        return axios.get(`${this.baseUrl}/api/task/history`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    getJob(name) {
+        return axios.get(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    getFrontend(name) {
+        return axios.get(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    startFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}/start`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    stopFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}/stop`);
+    }
+
+    /**
+     * @param {string} name
+     * @param job
+     */
+    updateJob(name, job) {
+        return axios.put(`${this.baseUrl}/api/job/${name}`, job);
+    }
+
+    /**
+     * @param {string} name
+     * @param frontend
+     */
+    updateFrontend(name, frontend) {
+        return axios.put(`${this.baseUrl}/api/frontend/${name}`, frontend);
+    }
+
+    /**
+     * @param {string} name
+     */
+    runJob(name) {
+        return axios.get(`${this.baseUrl}/api/job/${name}/run`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    deleteJob(name) {
+        return axios.delete(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    deleteFrontend(name) {
+        return axios.delete(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    createJob(name) {
+        return axios.post(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    createFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    pingEs(url, insecure) {
+        return axios.get(`${this.baseUrl}/api/ping_es`, {params: {url, insecure}});
+    }
+
+    getSist2AdminInfo() {
+        return axios.get(`${this.baseUrl}/api/`);
+    }
+}
+
+export default new Sist2AdminApi()

sist2-admin/frontend/src/components/FrontendListItem.vue

@@ -0,0 +1,31 @@
+<template>
+  <b-list-group-item action :to="`/frontend/${frontend.name}`">
+
+    <div class="d-flex w-100 justify-content-between">
+      <h5 class="mb-1" style="display: block">
+        {{ frontend.name }}
+        <b-badge variant="light">{{ formatBindAddress(frontend.web_options.bind) }}</b-badge>
+      </h5>
+
+      <div>
+        <b-badge v-if="frontend.running" variant="success">{{$t("online")}}</b-badge>
+        <b-badge v-else variant="secondary">{{$t("offline")}}</b-badge>
+      </div>
+    </div>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import {formatBindAddress} from "@/util";
+
+export default {
+  name: "FrontendListItem",
+  props: ["frontend"],
+  data() {
+    return {
+      formatBindAddress
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/components/IndexOptions.vue

@@ -0,0 +1,64 @@
+<template>
+  <div>
+    <label>{{ $t("indexOptions.threads") }}</label>
+    <b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.esUrl") }}</label>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <label>{{ $t("indexOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
+    <br>
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("indexOptions.batchSize") }}</label>
+    <b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("indexOptions.script") }}</label>
+    <b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
+  </div>
+
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "IndexOptions",
+  props: ["options"],
+  data() {
+    return {
+      showEsTestAlert: false,
+      esTestOk: false,
+      esTestMessage: "",
+    }
+  },
+  methods: {
+    update() {
+      this.$emit("change", this.options);
+    },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
+  },
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/JobCheckboxGroup.vue

@@ -0,0 +1,42 @@
+<template>
+  <div>
+    <h5>{{ $t("selectJobs") }}</h5>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-form-group v-else>
+      <b-form-checkbox-group
+          v-if="jobs.length > 0"
+          :checked="frontend.jobs"
+          @input="frontend.jobs = $event; $emit('input')"
+      >
+        <div v-for="job in jobs" :key="job.name">
+          <b-form-checkbox :disabled="job.status !== 'indexed'" :value="job.name">[{{ job.name }}]</b-form-checkbox>
+          <br/>
+        </div>
+      </b-form-checkbox-group>
+      <div v-else>
+        <span class="text-muted">{{ $t('jobOptions.noJobAvailable') }}</span>
+        &nbsp;<router-link to="/">{{$t("create")}}</router-link>
+      </div>
+    </b-form-group>
+  </div>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "JobCheckboxGroup",
+  props: ["frontend"],
+  mounted() {
+    Sist2AdminApi.getJobs().then(resp => {
+      this.jobs = resp.data;
+      this.loading = false;
+    });
+  },
+  data() {
+    return {
+      loading: true,
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/components/JobListItem.vue

@@ -0,0 +1,56 @@
+<template>
+  <b-list-group-item class="flex-column align-items-start" action :to="`job/${job.name}`">
+
+    <div class="d-flex w-100 justify-content-between">
+      <div>
+        <h5 class="mb-1">
+          {{ job.name }}
+        </h5>
+      </div>
+      <div>
+        <b-row>
+          <b-col>
+            <small v-if="job.last_index_date">
+              {{ $t("scanned") }} {{ formatLastIndexDate(job.last_index_date) }}</small>
+            <div v-else>&nbsp;</div>
+          </b-col>
+        </b-row>
+        <b-row v-if="job.schedule_enabled">
+          <b-col>
+            <small><code>{{job.cron_expression }}</code></small>
+          </b-col>
+        </b-row>
+        <b-row v-else>
+          <b-col>
+            &nbsp;
+          </b-col>
+        </b-row>
+
+      </div>
+    </div>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import moment from "moment";
+
+export default {
+  name: "JobListItem",
+  props: ["job"],
+  methods: {
+    formatLastIndexDate(dateString) {
+      if (dateString === null) {
+        return "";
+      }
+
+      const date = Date.parse(dateString);
+      return moment(date).fromNow();
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/JobOptions.vue

@@ -0,0 +1,51 @@
+<template>
+  <div>
+    <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
+      {{ $t("jobOptions.desktopNotifications") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
+      {{ $t("jobOptions.scheduleEnabled") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("jobOptions.cron") }}</label>
+    <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression" :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
+  </div>
+</template>
+
+<script>
+
+export default {
+  name: "JobOptions",
+  props: ["job"],
+  data() {
+    return {
+      cronValid: undefined
+    }
+  },
+  computed: {
+    desktopNotificationsEnabled() {
+      return this.$store.state.jobDesktopNotificationMap[this.job.name];
+    }
+  },
+  methods: {
+    updateNotifications(value) {
+      this.$store.dispatch("setJobDesktopNotification", {
+        job: this.job.name,
+        enabled: value
+      })
+    },
+    update() {
+      if (this.job.schedule_enabled) {
+        this.cronValid = /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(this.job.cron_expression);
+      } else {
+        this.cronValid = undefined;
+      }
+
+      if (this.cronValid !== false) {
+        this.$emit("change", this.job);
+      }
+    },
+  },
+}
+</script>

sist2-admin/frontend/src/components/NavBar.vue

@@ -0,0 +1,69 @@
+<template>
+  <b-navbar>
+    <b-navbar-brand to="/">
+      <Sist2Icon></Sist2Icon>
+    </b-navbar-brand>
+
+    <b-button class="ml-auto" to="/task" variant="link">{{ $t("tasks") }}</b-button>
+  </b-navbar>
+</template>
+
+<script>
+import Sist2Icon from "@/components/icons/Sist2Icon";
+
+export default {
+  name: "NavBar",
+  components: {Sist2Icon},
+  methods: {
+    tagline() {
+      return this.$store.state.sist2Info.tagline;
+    },
+    sist2Version() {
+      return this.$store.state.sist2Info.version;
+    },
+    isDebug() {
+      return this.$store.state.sist2Info.debug;
+    },
+    isLegacy() {
+      return this.$store.state.sist2Info.esVersionLegacy;
+    },
+    hideLegacy() {
+      return this.$store.state.optHideLegacy;
+    }
+  }
+}
+</script>
+
+<style scoped>
+.navbar {
+  box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
+  border-radius: 0;
+}
+
+.theme-black .navbar {
+  background: #546b7a30;
+  border-bottom: none;
+}
+
+.navbar-brand {
+  color: #222 !important;
+  font-size: 1.75rem;
+  padding: 0;
+}
+
+.navbar-brand:hover {
+  color: #000 !important;
+}
+
+.version {
+  color: #222 !important;
+  margin-left: -18px;
+  margin-top: -14px;
+  font-size: 11px;
+  font-family: monospace;
+}
+
+.btn-link {
+  color: #222;
+}
+</style>

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -0,0 +1,111 @@
+<template>
+  <div>
+    <label>{{ $t("scanOptions.path") }}</label>
+    <b-form-input v-model="options.path" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.threads") }}</label>
+    <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.memThrottle") }}</label>
+    <b-form-input type="number" min="0" v-model="options.mem_throttle" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
+    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailCount") }}</label>
+    <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailSize") }}</label>
+    <b-form-input type="number" min="100" v-model="options.thumbnail_size" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.contentSize") }}</label>
+    <b-form-input type="number" min="0" v-model="options.content_size" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.rewriteUrl") }}</label>
+    <b-form-input v-model="options.rewrite_url" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.depth") }}</label>
+    <b-form-input type="number" min="0" v-model="options.depth" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.archive") }}</label>
+    <b-form-select :options="['skip', 'list', 'shallow', 'recurse']" v-model="options.archive"
+                   @change="update()"></b-form-select>
+
+    <label>{{ $t("scanOptions.archivePassphrase") }}</label>
+    <b-form-input v-model="options.archive_passphrase" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.ocrLang") }}</label>
+    <b-alert variant="danger" show v-if="selectedOcrLangs.length === 0 && !disableOcrLang">{{ $t("scanOptions.ocrLangAlert") }}</b-alert>
+    <b-checkbox-group :disabled="disableOcrLang" v-model="selectedOcrLangs" @input="onOcrLangChange">
+      <b-checkbox v-for="lang in ocrLangs" :key="lang" :value="lang">{{ lang }}</b-checkbox>
+    </b-checkbox-group>
+
+    <!--    <b-form-input readonly v-model="options.ocr_lang" @change="update()"></b-form-input>-->
+
+    <div style="height: 10px"></div>
+
+    <b-form-checkbox v-model="options.ocr_images" @change="update()">
+      {{ $t("scanOptions.ocrImages") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.ocr_ebooks" @change="update()">
+      {{ $t("scanOptions.ocrEbooks") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("scanOptions.exclude") }}</label>
+    <b-form-input v-model="options.exclude" @change="update()"
+                  :placeholder="$t('scanOptions.excludePlaceholder')"></b-form-input>
+
+    <div style="height: 10px"></div>
+
+    <b-form-checkbox v-model="options.fast" @change="update()">
+      {{ $t("scanOptions.fast") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.checksums" @change="update()">
+      {{ $t("scanOptions.checksums") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.read_subtitles" @change="update()">
+      {{ $t("scanOptions.readSubtitles") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("scanOptions.memBuffer") }}</label>
+    <b-form-input type="number" min="0" v-model="options.mem_buffer" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.treemapThreshold") }}</label>
+    <b-form-input type="number" min="0" v-model="options.treemap_threshold" @change="update()"></b-form-input>
+  </div>
+</template>
+
+<script>
+
+export default {
+  name: "ScanOptions",
+  props: ["options"],
+  data() {
+    return {
+      disableOcrLang: false,
+      selectedOcrLangs: []
+    }
+  },
+  computed: {
+    ocrLangs() {
+      return this.$store.state.sist2AdminInfo?.tesseract_langs || [];
+    }
+  },
+  methods: {
+    onOcrLangChange() {
+      this.options.ocr_lang = this.selectedOcrLangs.join("+");
+    },
+    update() {
+      this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;
+      this.$emit("change", this.options);
+    },
+  },
+  mounted() {
+    this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;
+    this.selectedOcrLangs = this.options.ocr_lang ? this.options.ocr_lang.split("+") : [];
+  }
+}
+</script>

sist2-admin/frontend/src/components/TaskListItem.vue

@@ -0,0 +1,57 @@
+<template>
+  <b-list-group-item>
+    <b-row style="height: 50px">
+      <b-col><h5>{{ task.display_name }}</h5></b-col>
+      <b-col class="shrink">
+        <router-link class="btn btn-link" :to="`/log/${task.id}`">{{ $t("logs") }}</router-link>
+      </b-col>
+      <b-col class="shrink">
+        <b-btn variant="link" @click="killTask(task.id)">{{ $t("kill") }}</b-btn>
+      </b-col>
+    </b-row>
+
+    <b-row>
+      <b-col>
+        <b-progress :max="task.progress.count">
+          <b-progress-bar :value="task.progress.done" :label-html="label" :striped="!task.progress.waiting"/>
+        </b-progress>
+      </b-col>
+    </b-row>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "TaskListItem",
+  props: ["task"],
+  computed: {
+    label() {
+
+      const count = this.task.progress.count;
+      const done = this.task.progress.done;
+
+      return `<span>${done}/${count}</span>`
+    }
+  },
+  methods: {
+    killTask(taskId) {
+      sist2AdminApi.killTask(taskId).then(() => {
+        this.$bvToast.toast(this.$t("killConfirmation"), {
+          title: this.$t("killConfirmationTitle"),
+          variant: "success",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    }
+  }
+}
+</script>
+
+<style scoped>
+.shrink {
+  flex-grow: inherit;
+}
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -0,0 +1,75 @@
+<template>
+  <div>
+    <label>{{ $t("webOptions.esUrl") }}</label>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("webOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.lang") }}</label>
+    <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
+
+    <label>{{ $t("webOptions.bind") }}</label>
+    <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.tagline") }}</label>
+    <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+
+    <label>{{ $t("webOptions.auth") }}</label>
+    <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.tagAuth") }}</label>
+    <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+
+  </div>
+</template>
+
+<script>
+
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "WebOptions",
+  props: ["options", "frontendName"],
+  data() {
+    return {
+      showEsTestAlert: false,
+      esTestOk: false,
+      esTestMessage: "",
+    }
+  },
+  methods: {
+    update() {
+      if (!this.options.es_url.startsWith("https")) {
+        this.options.es_insecure_ssl = false;
+      }
+
+      this.$emit("change", this.options);
+    },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/icons/Sist2Icon.vue

@@ -0,0 +1,40 @@
+<template>
+  <svg
+      xmlns="http://www.w3.org/2000/svg"
+      width="27.868069mm"
+      height="7.6446671mm"
+      viewBox="0 0 27.868069 7.6446671"
+  >
+    <g transform="translate(-4.5018313,-4.1849793)">
+      <g
+          style="fill: currentColor;fill-opacity:1;stroke:none;stroke-width:0.26458332">
+        <path
+            d="m 6.3153296,11.829646 q -0.7717014,0 -1.8134983,-0.337619 v -0.916395 q 1.0128581,0.511252 1.803852,0.511252 0.5643067,0 0.901926,-0.236334 0.3376194,-0.236333 0.3376194,-0.63183 0,-0.3424428 -0.2845649,-0.5498376 Q 6.980922,9.4566645 6.3635609,9.3264399 L 5.9921796,9.2492698 Q 5.2301245,9.0949295 4.8732126,8.7428407 4.5211238,8.3859288 4.5211238,7.7733908 q 0,-0.7765245 0.5305447,-1.1961372 0.5305447,-0.4196126 1.5096409,-0.4196126 0.829579,0 1.6061036,0.3183268 V 7.3441319 Q 7.4101809,6.9004036 6.5854251,6.9004036 q -1.1671984,0 -1.1671984,0.7958171 0,0.2604492 0.1012858,0.4147895 0.1012858,0.1495171 0.3858507,0.2556261 0.2845649,0.1012858 0.8392253,0.2122179 l 0.3569119,0.067524 q 1.3408312,0.2652724 1.3408312,1.4614098 0,0.80064 -0.5691298,1.263661 -0.5691298,0.458197 -1.5578722,0.458197 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 11.943927,5.3087694 q -0.144694,0 -0.144694,-0.144694 V 4.3296733 q 0,-0.144694 0.144694,-0.144694 h 0.694531 q 0.144694,0 0.144694,0.144694 v 0.8344021 q 0,0.144694 -0.144694,0.144694 z M 13.5645,11.728361 q -0.795817,0 -1.234722,-0.511253 -0.434082,-0.516075 -0.434082,-1.4469398 V 6.9823969 H 10.714028 V 6.2878656 h 2.069124 v 3.4823026 q 0,0.5884228 0.221864,0.8971028 0.221865,0.308681 0.6463,0.308681 h 1.036974 v 0.752409 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 18.209178,11.829646 q -0.771701,0 -1.813498,-0.337619 v -0.916395 q 1.012858,0.511252 1.803852,0.511252 0.564306,0 0.901926,-0.236334 0.337619,-0.236333 0.337619,-0.63183 0,-0.3424428 -0.284565,-0.5498376 Q 18.87477,9.4566645 18.257409,9.3264399 l -0.371381,-0.07717 Q 17.123973,9.0949295 16.767061,8.7428407 16.414972,8.3859288 16.414972,7.7733908 q 0,-0.7765245 0.530545,-1.1961372 0.530545,-0.4196126 1.509641,-0.4196126 0.829579,0 1.606103,0.3183268 v 0.8681641 q -0.757232,-0.4437283 -1.581988,-0.4437283 -1.167198,0 -1.167198,0.7958171 0,0.2604492 0.101286,0.4147895 0.101286,0.1495171 0.385851,0.2556261 0.284565,0.1012858 0.839225,0.2122179 l 0.356912,0.067524 q 1.340831,0.2652724 1.340831,1.4614098 0,0.80064 -0.56913,1.263661 -0.56913,0.458197 -1.557872,0.458197 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 25.207545,11.709068 q -0.993565,0 -1.408355,-0.40032 -0.409966,-0.405143 -0.409966,-1.3794164 V 6.9775737 H 21.947107 V 6.2878656 h 1.442117 V 4.8746874 l 0.887457,-0.3858507 v 1.7990289 h 2.016069 v 0.6897081 h -2.016069 v 2.9517579 q 0,0.5932454 0.226687,0.8344024 0.226687,0.236333 0.790994,0.236333 h 0.998388 v 0.709001 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 27.995317,11.043476 q 0,-0.178456 0.120578,-0.299035 0.274919,-0.289388 0.651123,-0.684885 0.376205,-0.4003199 0.805464,-0.8681638 0.327973,-0.356912 0.491959,-0.5353679 0.16881,-0.1832791 0.255626,-0.2845649 0.09164,-0.1012858 0.178456,-0.2073948 0.255626,-0.3086805 0.405144,-0.5257215 0.15434,-0.2170411 0.250803,-0.4292589 0.168809,-0.3762045 0.168809,-0.7524089 0,-0.5980686 -0.352089,-0.935688 -0.356911,-0.3424425 -0.979096,-0.3424425 -0.863341,0 -1.938899,0.6414768 V 4.8361023 q 0.491959,-0.2363335 0.979096,-0.3569119 0.47749,-0.1205783 0.945334,-0.1205783 0.501606,0 0.940511,0.1350477 0.438905,0.1350478 0.766878,0.4244358 0.289388,0.2556261 0.463021,0.6270074 0.173633,0.3665582 0.173633,0.829579 0,0.4726671 -0.212218,0.9501574 -0.106109,0.2411567 -0.274919,0.4726671 -0.163986,0.2266873 -0.424435,0.540191 Q 31.270225,8.501684 31.077299,8.718725 30.884374,8.9357661 30.628748,9.2106847 30.445469,9.4084332 30.286305,9.5675966 30.131965,9.72676 29.958332,9.9003928 29.7847,10.069203 29.558012,10.300713 29.336148,10.5274 29.012998,10.869843 h 3.356901 v 0.819932 h -4.374582 z"
+            style="stroke-width:0.26458332"
+        />
+      </g>
+    </g>
+  </svg>
+</template>
+
+<script>
+export default {
+  name: "Sist2Icon"
+}
+</script>

sist2-admin/frontend/src/i18n/messages.js

@@ -0,0 +1,110 @@
+export default {
+    en: {
+        start: "Start",
+        stop: "Stop",
+        go: "Go",
+        online: "online",
+        offline: "offline",
+        delete: "Delete",
+        runNow: "Index now",
+        create: "Create",
+        test: "Test",
+
+        jobTitle: "job configuration",
+        tasks: "Tasks",
+        runningTasks: "Running tasks",
+        frontends: "Frontends",
+        jobDisabled: "There is no valid index for this job",
+        status: "Status",
+
+        taskHistory: "Task history",
+        taskName: "Task name",
+        taskStarted: "Started",
+        taskDuration: "Duration",
+        taskStatus: "Status",
+        logs: "Logs",
+        kill: "Kill",
+        killConfirmation: "SIGTERM signal sent to sist2 process",
+        killConfirmationTitle: "Confirmation",
+        follow: "Follow",
+        wholeFile: "Whole file",
+        logLevel: "Log level",
+        logMode: "Follow mode",
+        logFile: "Reading log file",
+
+        jobs: "Jobs",
+        newJobName: "New job name",
+        newJobHelp: "Create a new job to get started!",
+        newFrontendName: "New frontend name",
+        scanned: "last scan",
+        autoStart: "Start automatically",
+
+        runJobConfirmationTitle: "Task queued",
+        runJobConfirmation: "Check the Tasks page to monitor the status.",
+
+        monitoring: "Monitoring",
+        enableMonitoring: "Enable monitoring",
+        extraQueryArgs: "Extra query arguments when launching from sist2-admin",
+        customUrl: "Custom URL when launching from sist2-admin",
+
+        selectJobs: "Select jobs",
+        webOptions: {
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            lang: "UI Language",
+            bind: "Listen address",
+            tagline: "Tagline in navbar",
+            auth: "Basic auth in user:password format",
+            tagAuth: "Basic auth in user:password format for tagging",
+        },
+        scanOptions: {
+            title: "Scanning options",
+            path: "Path",
+            threads: "Number of threads",
+            memThrottle: "Total memory threshold in MiB for scan throttling",
+            thumbnailQuality: "Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best",
+            thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
+            thumbnailSize: "Thumbnail size, in pixels",
+            contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",
+            rewriteUrl: "Serve files from this url instead of from disk",
+            depth: "Scan up to this many subdirectories deep",
+            archive: "Archive file mode",
+            archivePassphrase: "Passphrase for encrypted archive files",
+            ocrLang: "Tesseract language",
+            ocrLangAlert: "You must select at least one language",
+            ocrEbooks: "Enable OCR'ing of ebook files",
+            ocrImages: "Enable OCR'ing of image files",
+            exclude: "Files that match this regex will not be scanned",
+            excludePlaceholder: "Exclude",
+            fast: "Only index file names & mime type",
+            checksums: "Calculate file checksums when scanning",
+            readSubtitles: "Read subtitles from media files",
+            memBuffer: "Maximum memory buffer size per thread in MiB for files inside archives",
+            treemapThreshold: "Relative size threshold for treemap"
+        },
+        indexOptions: {
+            title: "Indexing options",
+            threads: "Number of threads",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            batchSize: "Index batch size",
+            script: "User script"
+        },
+        jobOptions: {
+            title: "Job options",
+            cron: "Job schedule",
+            scheduleEnabled: "Enable scheduled re-scan",
+            noJobAvailable: "No jobs available.",
+            desktopNotifications: "Desktop notifications"
+        },
+        frontendOptions: {
+            title: "Frontend options",
+            noJobSelectedWarning: "You must select at least one job to start this frontend"
+        },
+        notifications: {
+            indexCompleted: "Task completed for [$JOB$]"
+        }
+    }
+}

sist2-admin/frontend/src/main.js

@@ -0,0 +1,31 @@
+import Vue from 'vue'
+
+import { BootstrapVue, IconsPlugin } from 'bootstrap-vue'
+
+import "bootstrap/dist/css/bootstrap.min.css"
+import "bootstrap-vue/dist/bootstrap-vue.min.css"
+
+Vue.use(BootstrapVue);
+Vue.use(IconsPlugin);
+
+import App from './App.vue';
+import router from './router';
+import store from './store';
+import VueI18n from "vue-i18n";
+import messages from "@/i18n/messages";
+
+Vue.use(VueI18n);
+
+const i18n = new VueI18n({
+  locale: "en",
+  messages: messages
+});
+
+Vue.config.productionTip = false
+
+new Vue({
+  router,
+  store,
+  i18n,
+  render: h => h(App)
+}).$mount('#app')

sist2-admin/frontend/src/router/index.js

@@ -0,0 +1,45 @@
+import Vue from 'vue'
+import VueRouter from 'vue-router'
+import Home from '../views/Home.vue'
+import Job from "@/views/Job";
+import Tasks from "@/views/Tasks";
+import Frontend from "@/views/Frontend";
+import Tail from "@/views/Tail";
+
+Vue.use(VueRouter);
+
+const routes = [
+  {
+    path: "/",
+    name: "Home",
+    component: Home
+  },
+  {
+    path: "/job/:name",
+    name: "Job",
+    component: Job
+  },
+  {
+    path: "/task/",
+    name: "Tasks",
+    component: Tasks
+  },
+  {
+    path: "/frontend/:name",
+    name: "Frontend",
+    component: Frontend
+  },
+  {
+    path: "/log/:taskId",
+    name: "Tail",
+    component: Tail
+  },
+]
+
+const router = new VueRouter({
+  mode: "hash",
+  base: process.env.BASE_URL,
+  routes
+})
+
+export default router

sist2-admin/frontend/src/store/index.js

@@ -0,0 +1,63 @@
+import Vue from "vue";
+import Vuex from "vuex";
+
+Vue.use(Vuex);
+
+function saveBrowserSettings(state) {
+    const settings = {
+        jobDesktopNotificationMap: state.jobDesktopNotificationMap
+    };
+    localStorage.setItem("sist2-admin-settings", JSON.stringify(settings));
+
+    console.log("SAVED");
+    console.log(settings);
+}
+
+export default new Vuex.Store({
+    state: {
+        sist2AdminInfo: null,
+        jobDesktopNotificationMap: {}
+    },
+    mutations: {
+        setSist2AdminInfo: (state, payload) => state.sist2AdminInfo = payload,
+        setJobDesktopNotificationMap: (state, payload) => state.jobDesktopNotificationMap = payload,
+    },
+    actions: {
+        notify: async ({state}, notification) => {
+
+            if (!state.jobDesktopNotificationMap[notification.job]) {
+                console.log("pass");
+                return;
+            }
+
+            new Notification(notification.messageString.replace("$JOB$", notification.job));
+        },
+        setJobDesktopNotification: async ({state}, {job, enabled}) => {
+
+            if (enabled === true) {
+                const permission = await Notification.requestPermission()
+
+                if (permission !== "granted") {
+                    return false;
+                }
+            }
+
+            state.jobDesktopNotificationMap[job] = enabled;
+            saveBrowserSettings(state);
+
+            return true;
+        },
+        loadBrowserSettings({commit}) {
+            const settingString = localStorage.getItem("sist2-admin-settings");
+
+            if (!settingString) {
+                return;
+            }
+
+            const settings = JSON.parse(settingString);
+
+            commit("setJobDesktopNotificationMap", settings["jobDesktopNotificationMap"]);
+        }
+    },
+    modules: {}
+})

sist2-admin/frontend/src/util.js

@@ -0,0 +1,8 @@
+
+export function formatBindAddress(address) {
+    if (address.startsWith("0.0.0.0")) {
+        return address.slice("0.0.0.0".length)
+    }
+
+    return address
+}

sist2-admin/frontend/src/views/Frontend.vue

@@ -0,0 +1,133 @@
+<template>
+  <b-card>
+    <b-card-title>
+      {{ name }}
+      <small style="vertical-align: top">
+        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+      </small>
+    </b-card-title>
+
+    <div class="mb-3" v-if="!loading">
+      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+          $t("start")
+        }}
+      </b-button>
+      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+          $t("stop")
+        }}
+      </b-button>
+      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+        {{ $t("go") }}
+      </b-button>
+      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+    </div>
+
+
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card-body v-else>
+
+      <h4>{{ $t("frontendOptions.title") }}</h4>
+      <b-card>
+        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+          {{ $t("autoStart") }}
+        </b-form-checkbox>
+
+        <b-form-checkbox v-model="frontend.enable_monitoring" @change="update()">
+          {{ $t("enableMonitoring") }}
+        </b-form-checkbox>
+
+        <label>{{ $t("extraQueryArgs") }}</label>
+        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+
+        <label>{{ $t("customUrl") }}</label>
+        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+
+        <br/>
+
+        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+
+        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("jobOptions.title") }}</h4>
+      <b-card>
+        <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
+      </b-card>
+    </b-card-body>
+
+  </b-card>
+</template>
+
+<script>
+
+import Sist2AdminApi from "@/Sist2AdminApi";
+import JobCheckboxGroup from "@/components/JobCheckboxGroup";
+import WebOptions from "@/components/WebOptions";
+
+export default {
+  name: 'Frontend',
+  components: {JobCheckboxGroup, WebOptions},
+  data() {
+    return {
+      loading: true,
+      frontend: null,
+    }
+  },
+  computed: {
+    valid() {
+      return !this.loading && this.frontend.jobs.length > 0;
+    },
+    frontendUrl() {
+      if (this.frontend.custom_url) {
+        return this.frontend.custom_url + this.args;
+      }
+
+      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+      }
+
+      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+    },
+    name() {
+      return this.$route.params.name;
+    },
+    port() {
+      return this.frontend.web_options.bind.split(":")[1]
+    },
+    args() {
+      const args = this.frontend.extra_query_args;
+      if (args !== "") {
+        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+      }
+      return "";
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getFrontend(this.name).then(resp => {
+      this.frontend = resp.data;
+      this.loading = false;
+    });
+  },
+  methods: {
+    start() {
+      this.frontend.running = true;
+      Sist2AdminApi.startFrontend(this.name)
+    },
+    stop() {
+      this.frontend.running = false;
+      Sist2AdminApi.stopFrontend(this.name)
+    },
+    deleteFrontend() {
+      Sist2AdminApi.deleteFrontend(this.name).then(() => {
+        this.$router.push("/frontends");
+      });
+    },
+    update() {
+      Sist2AdminApi.updateFrontend(this.name, this.frontend);
+    },
+  }
+}
+</script>

sist2-admin/frontend/src/views/Home.vue

@@ -0,0 +1,122 @@
+<template>
+  <div>
+    <b-card>
+      <b-card-title>{{ $t("jobs") }}</b-card-title>
+      <b-row>
+        <b-col>
+          <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
+          <b-popover
+              :show.sync="showHelp"
+              target="new-job"
+              placement="top"
+              triggers="manual"
+              variant="primary"
+              :content="$t('newJobHelp')"
+          ></b-popover>
+        </b-col>
+        <b-col>
+          <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
+          </b-button>
+        </b-col>
+      </b-row>
+
+      <hr/>
+
+      <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
+      <b-list-group v-else>
+        <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
+      </b-list-group>
+    </b-card>
+
+    <br/>
+
+    <b-card>
+
+      <b-card-title>{{ $t("frontends") }}</b-card-title>
+
+      <b-row>
+        <b-col>
+          <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
+        </b-col>
+        <b-col>
+          <b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
+            {{ $t("create") }}
+          </b-button>
+        </b-col>
+      </b-row>
+
+      <hr/>
+
+      <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
+      <b-list-group v-else>
+        <FrontendListItem v-for="frontend in frontends"
+                          :key="frontend.name" :frontend="frontend"></FrontendListItem>
+      </b-list-group>
+
+    </b-card>
+  </div>
+</template>
+
+<script>
+import JobListItem from "@/components/JobListItem";
+import {formatBindAddress} from "@/util";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import FrontendListItem from "@/components/FrontendListItem";
+
+export default {
+  name: "Jobs",
+  components: {JobListItem, FrontendListItem},
+  data() {
+    return {
+      jobsLoading: true,
+      newJobName: "",
+      jobs: [],
+
+      frontendsLoading: true,
+      frontends: [],
+      formatBindAddress,
+      newFrontendName: "",
+
+      showHelp: false
+    }
+  },
+  mounted() {
+    this.loading = true;
+    this.reload();
+  },
+  methods: {
+    jobNameValid(name) {
+      if (this.jobs.some(job => job.name === name)) {
+        return false;
+      }
+
+      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+    },
+    frontendNameValid(name) {
+      if (this.frontends.some(job => job.name === name)) {
+        return false;
+      }
+
+      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+    },
+    reload() {
+      Sist2AdminApi.getJobs().then(resp => {
+        this.jobs = resp.data;
+        this.jobsLoading = false;
+
+        this.showHelp = this.jobs.length === 0;
+      });
+      Sist2AdminApi.getFrontends().then(resp => {
+        this.frontends = resp.data;
+        this.frontendsLoading = false;
+      });
+    },
+    createJob() {
+      Sist2AdminApi.createJob(this.newJobName).then(this.reload);
+    },
+    createFrontend() {
+      Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/views/Job.vue

@@ -0,0 +1,92 @@
+<template>
+  <b-card>
+    <b-card-title>
+      [{{ getName() }}]
+      {{ $t("jobTitle") }}
+    </b-card-title>
+
+    <div class="mb-3">
+      <b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
+      <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
+    </div>
+
+    <div v-if="job">
+      {{ $t("status") }}: <code>{{ job.status }}</code>
+    </div>
+
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card-body v-else>
+
+      <h4>{{ $t("jobOptions.title") }}</h4>
+      <b-card>
+        <JobOptions :job="job" @change="update"></JobOptions>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("scanOptions.title") }}</h4>
+      <b-card>
+        <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("indexOptions.title") }}</h4>
+      <b-card>
+        <IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
+      </b-card>
+
+    </b-card-body>
+
+  </b-card>
+</template>
+
+<script>
+import ScanOptions from "@/components/ScanOptions";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import IndexOptions from "@/components/IndexOptions";
+import JobOptions from "@/components/JobOptions";
+
+export default {
+  name: "Job",
+  components: {
+    IndexOptions,
+    ScanOptions,
+    JobOptions
+  },
+  data() {
+    return {
+      loading: true,
+      job: null
+    }
+  },
+  methods: {
+    getName() {
+      return this.$route.params.name;
+    },
+    update() {
+      Sist2AdminApi.updateJob(this.getName(), this.job);
+    },
+    runJob() {
+      Sist2AdminApi.runJob(this.getName()).then(() => {
+        this.$bvToast.toast(this.$t("runJobConfirmation"), {
+          title: this.$t("runJobConfirmationTitle"),
+          variant: "success",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    },
+    deleteJob() {
+      Sist2AdminApi.deleteJob(this.getName()).then(() => {
+        this.$router.push("/");
+      })
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getJob(this.getName()).then(resp => {
+      this.loading = false;
+      this.job = resp.data;
+    })
+  }
+}
+</script>

sist2-admin/frontend/src/views/Tail.vue

@@ -0,0 +1,168 @@
+<template>
+  <b-card>
+    <b-card-body>
+
+      <h4 class="mb-3">{{ taskId }} {{ $t("logs") }}</h4>
+
+      <div v-if="$store.state.sist2AdminInfo">
+        {{ $t("logFile") }}
+        <code>{{ $store.state.sist2AdminInfo.logs_folder }}/sist2-{{ taskId }}.log</code>
+        <br/>
+        <br/>
+      </div>
+
+      <b-row>
+        <b-col>
+          <span>{{ $t("logLevel") }}</span>
+          <b-select :options="levels.slice(0, -1)" v-model="logLevel" @input="connect()"></b-select>
+        </b-col>
+        <b-col>
+          <span>{{ $t("logMode") }}</span>
+          <b-select :options="modeOptions" v-model="mode" @input="connect()"></b-select>
+        </b-col>
+      </b-row>
+
+      <div id="log-tail-output" class="mt-3 ml-1"></div>
+
+    </b-card-body>
+  </b-card>
+</template>
+
+<script>
+
+export default {
+  name: "Tail",
+  data() {
+    return {
+      logLevel: "DEBUG",
+      levels: ["DEBUG", "INFO", "WARNING", "ERROR", "ADMIN", "FATAL"],
+      socket: null,
+      mode: "follow",
+      modeOptions: [
+        {
+          "text": this.$t('follow'),
+          "value": "follow"
+        },
+        {
+          "text": this.$t('wholeFile'),
+          "value": "wholeFile"
+        }
+      ]
+    }
+  },
+  computed: {
+    taskId: function () {
+      return this.$route.params.taskId;
+    }
+  },
+  methods: {
+    connect() {
+      let lineCount = 0;
+      const outputElem = document.getElementById("log-tail-output")
+      outputElem.replaceChildren();
+      if (this.socket !== null) {
+        this.socket.close();
+      }
+
+      const n = this.mode === "follow" ? 32 : 9999999999;
+      this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
+      this.socket.onopen = () => {
+        this.socket.send("Hello from client");
+      }
+
+      this.socket.onmessage = e => {
+        let message;
+        try {
+          message = JSON.parse(e.data);
+        } catch {
+          console.error(e.data)
+          return;
+        }
+
+        if ("ping" in message) {
+          return;
+        }
+
+        if (message.level === undefined) {
+
+          if ("stderr" in message) {
+            message.level = "ERROR";
+            message.message = message["stderr"];
+          } else {
+            message.level = "ADMIN";
+            message.message = message["sist2-admin"];
+          }
+          message.datetime = ""
+          message.filepath = ""
+        }
+
+        if (this.levels.indexOf(message.level) < this.levels.indexOf(this.logLevel)) {
+          return;
+        }
+
+        const logLine = `${message.datetime} [${message.level} ${message.filepath}] ${message.message}`;
+
+        const span = document.createElement("span");
+        span.setAttribute("class", message.level);
+        span.appendChild(document.createTextNode(logLine));
+
+        outputElem.appendChild(span);
+        lineCount += 1;
+
+        if (this.mode === "follow" && lineCount >= n) {
+          outputElem.firstChild.remove();
+        }
+      }
+    }
+  },
+  mounted() {
+    this.connect()
+  }
+}
+
+</script>
+
+<style>
+#log-tail-output span {
+  display: block;
+}
+
+span.DEBUG {
+  color: #9E9E9E;
+}
+
+span.WARNING {
+  color: #FFB300;
+}
+
+span.INFO {
+  color: #039BE5;
+}
+
+span.ERROR {
+  color: #F4511E;
+}
+
+span.FATAL {
+  color: #F4511E;
+}
+
+span.ADMIN {
+  color: #ee05ff;
+}
+
+
+#log-tail-output {
+  font-size: 13px;
+  font-family: monospace;
+
+  padding: 6px;
+  background-color: #f5f5f5;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  margin: 3px;
+  white-space: pre;
+  color: #000;
+  overflow: hidden;
+}
+</style>

sist2-admin/frontend/src/views/Tasks.vue

@@ -0,0 +1,124 @@
+<template>
+  <div>
+
+    <b-card v-if="tasks.length > 0">
+      <h2>{{ $t("runningTasks") }}</h2>
+      <b-list-group>
+        <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
+      </b-list-group>
+    </b-card>
+
+    <b-card class="mt-4">
+
+      <b-card-title>{{ $t("taskHistory") }}</b-card-title>
+
+      <br/>
+
+      <b-table
+          id="task-history"
+          :items="historyItems"
+          :fields="historyFields"
+          :current-page="historyCurrentPage"
+          :tbody-tr-class="rowClass"
+          :per-page="10"
+      >
+        <template #cell(logs)="data">
+          <router-link :to="`/log/${data.item.logs}`">{{ $t("logs") }}</router-link>
+        </template>
+
+      </b-table>
+
+      <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
+                    :per-page="10"></b-pagination>
+
+    </b-card>
+  </div>
+</template>
+
+<script>
+import TaskListItem from "@/components/TaskListItem";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import moment from "moment";
+
+export default {
+  name: 'Tasks',
+  components: {TaskListItem},
+  data() {
+    return {
+      loading: true,
+      tasks: [],
+      taskHistory: [],
+      timerId: null,
+      historyFields: [
+        {key: "name", label: this.$t("taskName")},
+        {key: "time", label: this.$t("taskStarted")},
+        {key: "duration", label: this.$t("taskDuration")},
+        {key: "status", label: this.$t("taskStatus")},
+        {key: "logs", label: this.$t("logs")},
+      ],
+      historyCurrentPage: 1,
+      historyItems: []
+    }
+  },
+  props: {
+    msg: String
+  },
+  mounted() {
+    this.loading = true;
+    this.update().then(() => this.loading = false);
+
+    this.timerId = window.setInterval(this.update, 1000);
+    this.updateHistory();
+  },
+  destroyed() {
+    if (this.timerId) {
+      window.clearInterval(this.timerId);
+    }
+  },
+  methods: {
+    rowClass(row) {
+      if (row.status === "failed") {
+        return "table-danger";
+      }
+      return null;
+    },
+    updateHistory() {
+      Sist2AdminApi.getTaskHistory().then(resp => {
+        this.historyItems = resp.data.map(row => ({
+          id: row.id,
+          name: row.name,
+          duration: this.taskDuration(row),
+          time: moment(row.started).format("dd, MMM Do YYYY, HH:mm:ss"),
+          logs: row.id,
+          status: row.return_code === 0 ? "ok" : "failed"
+        }));
+      });
+    },
+    update() {
+      return Sist2AdminApi.getTasks().then(resp => {
+        this.tasks = resp.data;
+      })
+    },
+    taskDuration(task) {
+      const start = moment(task.started);
+      const end = moment(task.ended);
+
+      let duration = moment.utc(end.diff(start)).format("HH[h] mm[m] ss[s]");
+
+      duration = duration.replace("00h ", "");
+      duration = duration.replace(/^00m /, "");
+      duration = duration.replace(/00s/, "<1s");
+      duration = duration.replace(/^0/, "");
+
+      return duration;
+    }
+  }
+}
+</script>
+
+<style scoped>
+#task-history {
+  font-family: monospace;
+  font-size: 12px;
+}
+</style>

sist2-admin/frontend/vue.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+    publicPath: ""
+};

sist2-admin/requirements.txt

@@ -0,0 +1,5 @@
+fastapi
+git+https://github.com/simon987/hexlib.git
+uvicorn
+websockets
+pycron

sist2-admin/sist2_admin/app.py

@@ -0,0 +1,392 @@
+import asyncio
+import os
+import signal
+from datetime import datetime
+from urllib.parse import urlparse
+
+import requests
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from hexlib.db import PersistentState
+from requests import ConnectionError
+from requests.exceptions import SSLError
+from starlette.middleware.cors import CORSMiddleware
+from starlette.responses import RedirectResponse
+from starlette.staticfiles import StaticFiles
+from starlette.websockets import WebSocket
+from websockets.exceptions import ConnectionClosed
+
+import cron
+from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
+from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
+from notifications import Subscribe, Notifications
+from sist2 import Sist2
+from state import PickleTable, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
+from web import Sist2Frontend
+
+VERSION = "1.0"
+
+sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
+db = PersistentState(table_factory=PickleTable, dbfile=os.path.join(DATA_FOLDER, "state.db"))
+notifications = Notifications()
+task_queue = TaskQueue(sist2, db, notifications)
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_credentials=True,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.mount("/ui/", StaticFiles(directory="./frontend/dist", html=True), name="static")
+
+
+@app.get("/")
+async def home():
+    return RedirectResponse("ui")
+
+
+@app.get("/api")
+async def api():
+    return {
+        "version": VERSION,
+        "tesseract_langs": TESSERACT_LANGS,
+        "logs_folder": LOG_FOLDER
+    }
+
+
+@app.get("/api/job/{name:str}")
+async def get_job(name: str):
+    row = db["jobs"][name]
+    if row:
+        return row["job"]
+    raise HTTPException(status_code=404)
+
+
+@app.get("/api/frontend/{name:str}")
+async def get_frontend(name: str):
+    row = db["frontends"][name]
+    if row:
+        frontend = row["frontend"]
+        frontend: Sist2Frontend
+        frontend.running = frontend.name in RUNNING_FRONTENDS
+        return frontend
+    raise HTTPException(status_code=404)
+
+
+@app.get("/api/job/")
+async def get_jobs():
+    return [row["job"] for row in db["jobs"]]
+
+
+@app.put("/api/job/{name:str}")
+async def update_job(name: str, job: Sist2Job):
+    # TODO: Check etag
+
+    job.last_modified = datetime.now()
+    row = db["jobs"][name]
+    if not row:
+        raise HTTPException(status_code=404)
+
+    args_that_trigger_full_scan = [
+        "path",
+        "thumbnail_count",
+        "thumbnail_quality",
+        "thumbnail_size",
+        "content_size",
+        "depth",
+        "archive",
+        "archive_passphrase",
+        "ocr_lang",
+        "ocr_images",
+        "ocr_ebooks",
+        "fast",
+        "checksums",
+        "read_subtitles",
+    ]
+    for arg in args_that_trigger_full_scan:
+        if getattr(row["job"].scan_options, arg) != getattr(job.scan_options, arg):
+            job.do_full_scan = True
+
+    db["jobs"][name] = {"job": job}
+
+
+@app.put("/api/frontend/{name:str}")
+async def update_frontend(name: str, frontend: Sist2Frontend):
+    db["frontends"][name] = {"frontend": frontend}
+
+    # TODO: Check etag
+
+    return "ok"
+
+
+@app.get("/api/task/")
+async def get_tasks():
+    return list(map(lambda t: t.json(), task_queue.tasks()))
+
+
+@app.get("/api/task/history")
+async def task_history():
+    return list(db["task_done"].sql("ORDER BY started DESC"))
+
+
+@app.post("/api/task/{task_id:str}/kill")
+async def kill_job(task_id: str):
+    return task_queue.kill_task(task_id)
+
+
+def _run_job(job: Sist2Job):
+    job.last_modified = datetime.now()
+    if job.status == JobStatus("created"):
+        job.status = JobStatus("started")
+    db["jobs"][job.name] = {"job": job}
+
+    scan_task = Sist2ScanTask(job, f"Scan [{job.name}]")
+    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=scan_task)
+
+    task_queue.submit(scan_task)
+    task_queue.submit(index_task)
+
+
+@app.get("/api/job/{name:str}/run")
+async def run_job(name: str):
+    row = db["jobs"][name]
+    if not row:
+        raise HTTPException(status_code=404)
+
+    _run_job(row["job"])
+
+    return "ok"
+
+
+@app.delete("/api/job/{name:str}")
+async def delete_job(name: str):
+    row = db["jobs"][name]
+    if row:
+        del db["jobs"][name]
+    else:
+        raise HTTPException(status_code=404)
+
+
+@app.delete("/api/frontend/{name:str}")
+async def delete_frontend(name: str):
+    if name in RUNNING_FRONTENDS:
+        os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        del RUNNING_FRONTENDS[name]
+
+    row = db["frontends"][name]
+    if row:
+        del db["frontends"][name]
+    else:
+        raise HTTPException(status_code=404)
+
+
+@app.post("/api/job/{name:str}")
+async def create_job(name: str):
+    if db["jobs"][name]:
+        raise ValueError("Job with the same name already exists")
+
+    job = Sist2Job.create_default(name)
+    db["jobs"][name] = {"job": job}
+
+    return job
+
+
+@app.post("/api/frontend/{name:str}")
+async def create_frontend(name: str):
+    if db["frontend"][name]:
+        raise ValueError("Frontend with the same name already exists")
+
+    frontend = Sist2Frontend.create_default(name)
+    db["frontends"][name] = {"frontend": frontend}
+
+    return frontend
+
+
+@app.get("/api/ping_es")
+async def ping_es(url: str, insecure: bool):
+    return check_es_version(url, insecure)
+
+
+def check_es_version(es_url: str, insecure: bool):
+    try:
+        url = urlparse(es_url)
+        if url.username:
+            auth = (url.username, url.password)
+            es_url = f"{url.scheme}://{url.hostname}:{url.port}"
+        else:
+            auth = None
+        r = requests.get(es_url, verify=insecure, auth=auth)
+    except SSLError:
+        return {
+            "ok": False,
+            "message": "Invalid SSL certificate"
+        }
+    except ConnectionError as e:
+        return {
+            "ok": False,
+            "message": "Connection refused"
+        }
+    except ValueError as e:
+        return {
+            "ok": False,
+            "message": str(e)
+        }
+
+    if r.status_code == 401:
+        return {
+            "ok": False,
+            "message": "Authentication failure"
+        }
+
+    try:
+        return {
+            "ok": True,
+            "message": "Elasticsearch version " + r.json()["version"]["number"]
+        }
+    except:
+        return {
+            "ok": False,
+            "message": "Could not read version"
+        }
+
+
+def start_frontend_(frontend: Sist2Frontend):
+    frontend.web_options.indices = list(map(lambda j: db["jobs"][j]["job"].last_index, frontend.jobs))
+
+    pid = sist2.web(frontend.web_options, frontend.name)
+    RUNNING_FRONTENDS[frontend.name] = pid
+
+
+@app.post("/api/frontend/{name:str}/start")
+async def start_frontend(name: str):
+    row = db["frontends"][name]
+    if not row:
+        raise HTTPException(status_code=404)
+
+    start_frontend_(row["frontend"])
+
+
+@app.post("/api/frontend/{name:str}/stop")
+async def stop_frontend(name: str):
+    if name in RUNNING_FRONTENDS:
+        os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        del RUNNING_FRONTENDS[name]
+
+
+@app.get("/api/frontend/")
+async def get_frontends():
+    res = []
+    for row in db["frontends"]:
+        frontend = row["frontend"]
+        frontend: Sist2Frontend
+        frontend.running = frontend.name in RUNNING_FRONTENDS
+        res.append(frontend)
+    return res
+
+
+def tail(filepath: str, n: int):
+    with open(filepath) as file:
+
+        reached_eof = False
+        buffer = []
+
+        line = ""
+        while True:
+            tmp = file.readline()
+            if tmp:
+                line += tmp
+
+                if line.endswith("\n"):
+
+                    if reached_eof:
+                        yield line
+                    else:
+                        if len(buffer) > n:
+                            buffer.pop(0)
+                        buffer.append(line)
+                line = ""
+            else:
+                if not reached_eof:
+                    reached_eof = True
+                    yield from buffer
+                yield None
+
+
+@app.websocket("/notifications")
+async def ws_tail_log(websocket: WebSocket):
+    await websocket.accept()
+
+    try:
+        await websocket.receive_text()
+
+        async with Subscribe(notifications) as ob:
+            async for notification in ob.notifications():
+                await websocket.send_json(notification)
+                print(notification)
+
+    except ConnectionClosed:
+        return
+
+
+@app.websocket("/log/{task_id}")
+async def ws_tail_log(websocket: WebSocket, task_id: str, n: int):
+    log_file = os.path.join(LOG_FOLDER, f"sist2-{task_id}.log")
+
+    await websocket.accept()
+
+    try:
+        await websocket.receive_text()
+    except ConnectionClosed:
+        return
+
+    while True:
+        for line in tail(log_file, n):
+
+            try:
+                if line:
+                    await websocket.send_text(line)
+                else:
+                    await websocket.send_json({"ping": ""})
+                    await asyncio.sleep(0.1)
+            except ConnectionClosed:
+                return
+
+
+def main():
+    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0")
+
+
+def initialize_db():
+    db["sist2_admin"]["info"] = {"version": DB_SCHEMA_VERSION}
+
+    frontend = Sist2Frontend.create_default("default")
+    db["frontends"]["default"] = {"frontend": frontend}
+
+    logger.info("Initialized database.")
+
+
+def start_frontends():
+    for row in db["frontends"]:
+        frontend: Sist2Frontend = row["frontend"]
+        if frontend.auto_start and len(frontend.jobs) > 0:
+            start_frontend_(frontend)
+
+
+if __name__ == '__main__':
+
+    if not db["sist2_admin"]["info"]:
+        initialize_db()
+    elif db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
+        print("Database has incompatible schema version! Delete state.db to continue.")
+        exit(-1)
+
+    start_frontends()
+    cron.initialize(db, _run_job)
+
+    logger.info("Started sist2-admin. Hello!")
+
+    main()

sist2-admin/sist2_admin/config.py

@@ -0,0 +1,30 @@
+import os
+import logging
+import sys
+from logging import StreamHandler
+from logging.handlers import RotatingFileHandler
+
+MAX_LOG_SIZE = 1 * 1024 * 1024
+
+SIST2_BINARY = os.environ.get("SIST2_BINARY", "/root/sist2")
+DATA_FOLDER = os.environ.get("DATA_FOLDER", "/sist2-admin/")
+LOG_FOLDER = os.path.join(DATA_FOLDER, "logs")
+WEBSERVER_PORT = 8080
+
+os.makedirs(LOG_FOLDER, exist_ok=True)
+os.makedirs(DATA_FOLDER, exist_ok=True)
+
+logger = logging.Logger("sist2-admin")
+
+_log_file = os.path.join(LOG_FOLDER, "sist2-admin.log")
+_log_fmt = "%(asctime)s [%(levelname)s] %(message)s"
+_log_formatter = logging.Formatter(_log_fmt, datefmt='%Y-%m-%d %H:%M:%S')
+
+console_handler = StreamHandler(sys.stdout)
+console_handler.setFormatter(_log_formatter)
+
+file_handler = RotatingFileHandler(_log_file, mode="a", maxBytes=MAX_LOG_SIZE, backupCount=1)
+file_handler.setFormatter(_log_formatter)
+
+logger.addHandler(console_handler)
+logger.addHandler(file_handler)

sist2-admin/sist2_admin/cron.py

@@ -0,0 +1,33 @@
+from threading import Thread
+
+import pycron
+import time
+
+from hexlib.db import PersistentState
+
+from config import logger
+from jobs import Sist2Job
+
+
+def _check_schedule(db: PersistentState, run_job):
+    for job in (row["job"] for row in db["jobs"]):
+        job: Sist2Job
+
+        if job.schedule_enabled:
+            if pycron.is_now(job.cron_expression):
+                logger.info(f"Submit scan task to queue for [{job.name}]")
+                run_job(job)
+
+
+def _cron_thread(db, run_job):
+    time.sleep(60 - (time.time() % 60))
+    start = time.time()
+
+    while True:
+        _check_schedule(db, run_job)
+        time.sleep(60 - ((time.time() - start) % 60))
+
+
+def initialize(db, run_job):
+    t = Thread(target=_cron_thread, args=(db, run_job), daemon=True, name="timer")
+    t.start()

sist2-admin/sist2_admin/jobs.py

@@ -0,0 +1,315 @@
+import json
+import logging
+import os.path
+import shutil
+import signal
+import uuid
+from datetime import datetime
+from enum import Enum
+from hashlib import md5
+from logging import FileHandler
+from threading import Lock, Thread
+from time import sleep
+from uuid import uuid4, UUID
+
+from hexlib.db import PersistentState
+from pydantic import BaseModel, validator
+
+from config import logger, LOG_FOLDER
+from notifications import Notifications
+from sist2 import ScanOptions, IndexOptions, Sist2, Sist2Index
+from state import RUNNING_FRONTENDS
+from web import Sist2Frontend
+
+
+class JobStatus(Enum):
+    CREATED = "created"
+    STARTED = "started"
+    INDEXED = "indexed"
+    FAILED = "failed"
+
+
+class Sist2Job(BaseModel):
+    name: str
+    scan_options: ScanOptions
+    index_options: IndexOptions
+
+    cron_expression: str
+    schedule_enabled: bool = False
+
+    previous_index: str = None
+    last_index: str = None
+    last_index_date: datetime = None
+    status: JobStatus = JobStatus("created")
+    last_modified: datetime
+    etag: str = None
+    do_full_scan: bool = False
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @staticmethod
+    def create_default(name: str):
+        return Sist2Job(
+            name=name,
+            scan_options=ScanOptions(path="/"),
+            index_options=IndexOptions(),
+            last_modified=datetime.now(),
+            cron_expression="0 0 * * *"
+        )
+
+    @validator("etag", always=True)
+    def validate_etag(cls, value, values):
+        s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
+        return md5(s.encode()).hexdigest()
+
+
+class Sist2TaskProgress:
+
+    def __init__(self, done: int = 0, count: int = 0, index_size: int = 0, tn_size: int = 0, waiting: bool = False):
+        self.done = done
+        self.count = count
+        self.index_size = index_size
+        self.store_size = tn_size
+        self.waiting = waiting
+
+    def percent(self):
+        return (self.done / self.count) if self.count else 0
+
+
+class Sist2Task:
+
+    def __init__(self, job: Sist2Job, display_name: str, depends_on: uuid.UUID = None):
+        self.job = job
+        self.display_name = display_name
+
+        self.progress = Sist2TaskProgress()
+        self.id = uuid4()
+        self.pid = None
+        self.started = None
+        self.ended = None
+        self.depends_on = depends_on
+
+        self._logger = logging.Logger(name=f"{self.id}")
+        self._logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"sist2-{self.id}.log")))
+
+    def json(self):
+        return {
+            "id": self.id,
+            "job": self.job,
+            "display_name": self.display_name,
+            "progress": self.progress,
+            "started": self.started,
+            "ended": self.ended,
+            "depends_on": self.depends_on,
+        }
+
+    def log_callback(self, log_json):
+
+        if "progress" in log_json:
+            self.progress = Sist2TaskProgress(**log_json["progress"])
+        elif self._logger:
+            self._logger.info(json.dumps(log_json))
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        self.started = datetime.now()
+
+        logger.info(f"Started task {self.display_name}")
+
+
+class Sist2ScanTask(Sist2Task):
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        self.job.scan_options.name = self.job.name
+
+        if self.job.last_index and os.path.exists(self.job.last_index) and not self.job.do_full_scan:
+            self.job.scan_options.incremental = self.job.last_index
+        else:
+            self.job.scan_options.incremental = None
+
+        def set_pid(pid):
+            self.pid = pid
+
+        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
+        self.ended = datetime.now()
+
+        if return_code != 0:
+            self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
+            logger.info(f"Task {self.display_name} failed ({return_code})")
+        else:
+            index = Sist2Index(self.job.scan_options.output)
+
+            # Save latest index
+            self.job.previous_index = self.job.last_index
+
+            self.job.last_index = index.path
+            self.job.last_index_date = datetime.now()
+            self.job.do_full_scan = False
+            db["jobs"][self.job.name] = {"job": self.job}
+            self._logger.info(json.dumps({"sist2-admin": f"Save last_index={self.job.last_index}"}))
+
+        logger.info(f"Completed {self.display_name} ({return_code=})")
+
+        return return_code
+
+
+class Sist2IndexTask(Sist2Task):
+
+    def __init__(self, job: Sist2Job, display_name: str, depends_on: Sist2Task):
+        super().__init__(job, display_name, depends_on=depends_on.id)
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        self.job.index_options.path = self.job.scan_options.output
+
+        return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
+        self.ended = datetime.now()
+
+        duration = self.ended - self.started
+
+        ok = return_code == 0
+
+        if ok:
+            # Remove old index
+            if self.job.previous_index is not None:
+                self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index=}"}))
+                try:
+                    shutil.rmtree(self.job.previous_index)
+                except FileNotFoundError:
+                    pass
+
+            self.restart_running_frontends(db, sist2)
+
+        # Update status
+        self.job.status = JobStatus("indexed") if ok else JobStatus("failed")
+        db["jobs"][self.job.name] = {"job": self.job}
+
+        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
+
+        logger.info(f"Completed {self.display_name} ({return_code=})")
+
+        return return_code
+
+    def restart_running_frontends(self, db: PersistentState, sist2: Sist2):
+        for frontend_name, pid in RUNNING_FRONTENDS.items():
+            frontend = db["frontends"][frontend_name]["frontend"]
+            frontend: Sist2Frontend
+
+            os.kill(pid, signal.SIGTERM)
+            try:
+                os.wait()
+            except ChildProcessError:
+                pass
+
+            frontend.web_options.indices = map(lambda j: db["jobs"][j]["job"].last_index, frontend.jobs)
+
+            pid = sist2.web(frontend.web_options, frontend.name)
+            RUNNING_FRONTENDS[frontend_name] = pid
+
+            self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))
+
+
+class TaskQueue:
+    def __init__(self, sist2: Sist2, db: PersistentState, notifications: Notifications):
+        self._lock = Lock()
+
+        self._sist2 = sist2
+        self._db = db
+        self._notifications = notifications
+
+        self._tasks = {}
+        self._queue = []
+        self._sem = 0
+
+        self._thread = Thread(target=self._check_new_task, daemon=True)
+        self._thread.start()
+
+    def _tasks_failed(self):
+        done = set()
+
+        for row in self._db["task_done"].sql("WHERE return_code != 0"):
+            done.add(uuid.UUID(row["id"]))
+
+        return done
+
+    def _tasks_done(self):
+
+        done = set()
+
+        for row in self._db["task_done"]:
+            done.add(uuid.UUID(row["id"]))
+
+        return done
+
+    def _check_new_task(self):
+        while True:
+            with self._lock:
+                for task in list(self._queue):
+                    task: Sist2Task
+
+                    if self._sem >= 1:
+                        break
+
+                    if not task.depends_on or task.depends_on in self._tasks_done():
+                        self._queue.remove(task)
+
+                        if task.depends_on in self._tasks_failed():
+                            # The task which we depend on failed, continue
+                            continue
+
+                        self._sem += 1
+
+                        t = Thread(target=self._run_task, args=(task,))
+
+                        self._tasks[task.id] = {
+                            "task": task,
+                            "thread": t,
+                        }
+
+                        t.start()
+                    break
+            sleep(1)
+
+    def tasks(self):
+        return list(map(lambda t: t["task"], self._tasks.values()))
+
+    def kill_task(self, task_id):
+
+        task = self._tasks.get(UUID(task_id))
+
+        if task:
+            pid = task["task"].pid
+            logger.info(f"Killing task {task_id} (pid={pid})")
+            os.kill(pid, signal.SIGTERM)
+            return True
+
+        return False
+
+    def _run_task(self, task: Sist2Task):
+        task_result = task.run(self._sist2, self._db)
+
+        with self._lock:
+            del self._tasks[task.id]
+            self._sem -= 1
+
+            self._db["task_done"][task.id] = {
+                "ended": task.ended,
+                "started": task.started,
+                "name": task.display_name,
+                "return_code": task_result
+            }
+        if isinstance(task, Sist2IndexTask):
+            self._notifications.notify({
+                "message": "notifications.indexCompleted",
+                "job": task.job.name
+            })
+
+    def submit(self, task: Sist2Task):
+
+        logger.info(f"Submitted task to queue {task.display_name}")
+
+        with self._lock:
+            self._queue.append(task)

sist2-admin/sist2_admin/notifications.py

@@ -0,0 +1,40 @@
+import asyncio
+from typing import List
+
+
+class Notifications:
+    def __init__(self):
+        self._subscribers: List[Subscribe] = []
+
+    def subscribe(self, ob):
+        self._subscribers.append(ob)
+
+    def unsubscribe(self, ob):
+        self._subscribers.remove(ob)
+
+    def notify(self, notification: dict):
+        for ob in self._subscribers:
+            ob.notify(notification)
+
+
+class Subscribe:
+    def __init__(self, notifications: Notifications):
+        self._queue = []
+        self._notifications = notifications
+
+    async def __aenter__(self):
+        self._notifications.subscribe(self)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        self._notifications.unsubscribe(self)
+
+    def notify(self, notification: dict):
+        self._queue.append(notification)
+
+    async def notifications(self):
+        while True:
+            try:
+                yield self._queue.pop(0)
+            except IndexError:
+                await asyncio.sleep(0.1)

sist2-admin/sist2_admin/sist2.py

@@ -0,0 +1,307 @@
+import datetime
+import json
+import logging
+import os.path
+import traceback
+from datetime import datetime
+from io import TextIOWrapper
+from logging import FileHandler
+from subprocess import Popen, PIPE
+from tempfile import NamedTemporaryFile
+from threading import Thread
+from typing import List
+
+from pydantic import BaseModel
+
+from config import logger, LOG_FOLDER
+
+
+class Sist2Version:
+    def __init__(self, version: str):
+        self._version = version
+
+        self.major, self.minor, self.patch = [int(x) for x in version.split(".")]
+
+    def __str__(self):
+        return f"{self.major}.{self.minor}.{self.patch}"
+
+
+class WebOptions(BaseModel):
+    indices: List[str] = []
+    es_url: str = "http://elasticsearch:9200"
+    es_insecure_ssl: bool = False
+    es_index: str = "sist2"
+    bind: str = "0.0.0.0:4090"
+    auth: str = None
+    tag_auth: str = None
+    tagline: str = "Lightning-fast file system indexer and search tool"
+    dev: bool = False
+    lang: str = "en"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+        args = ["web", f"--es-url={self.es_url}", f"--bind={self.bind}",
+                f"--tagline={self.tagline}", f"--lang={self.lang}"]
+
+        if self.es_insecure_ssl:
+            args.append(f"--es-insecure-ssl")
+        if self.auth:
+            args.append(f"--auth={self.auth}")
+        if self.tag_auth:
+            args.append(f"--tag_auth={self.tag_auth}")
+        if self.dev:
+            args.append(f"--dev")
+
+        args.extend(self.indices)
+
+        return args
+
+
+class IndexOptions(BaseModel):
+    path: str = None
+    threads: int = 1
+    es_url: str = "http://elasticsearch:9200"
+    es_insecure_ssl: bool = False
+    es_index: str = "sist2"
+    incremental_index: bool = False
+    script: str = ""
+    script_file: str = None
+    batch_size: int = 100
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+
+        args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
+                f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
+
+        if self.script_file:
+            args.append(f"--script-file={self.script_file}")
+        if self.es_insecure_ssl:
+            args.append(f"--es-insecure-ssl")
+        if self.incremental_index:
+            args.append(f"--incremental-index")
+
+        return args
+
+
+ARCHIVE_SKIP = "skip"
+ARCHIVE_LIST = "list"
+ARCHIVE_SHALLOW = "shallow"
+ARCHIVE_RECURSE = "recurse"
+
+
+class ScanOptions(BaseModel):
+    path: str
+    threads: int = 1
+    mem_throttle: int = 0
+    thumbnail_quality: float = 1.0
+    thumbnail_size: int = 500
+    thumbnail_count: int = 1
+    content_size: int = 32768
+    depth: int = -1
+    archive: str = ARCHIVE_RECURSE
+    archive_passphrase: str = None
+    ocr_lang: bool = None
+    ocr_images: bool = False
+    ocr_ebooks: bool = False
+    exclude: str = None
+    fast: bool = False
+    treemap_threshold: float = 0.0005
+    mem_buffer: int = 2000
+    read_subtitles: bool = False
+    fast_epub: bool = False
+    checksums: bool = False
+    incremental: str = None
+    output: str = None
+    name: str = None
+    rewrite_url: str = None
+    list_file: str = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+        args = ["scan", self.path, f"--threads={self.threads}", f"--mem-throttle={self.mem_throttle}",
+                f"--thumbnail-quality={self.thumbnail_quality}", f"--thumbnail-count={self.thumbnail_count}",
+                f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
+                f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
+
+        if self.incremental:
+            args.append(f"--incremental={self.incremental}")
+        if self.rewrite_url:
+            args.append(f"--rewrite-url={self.rewrite_url}")
+        if self.name:
+            args.append(f"--name={self.name}")
+        if self.archive_passphrase:
+            args.append(f"--archive-passphrase={self.archive_passphrase}")
+        if self.ocr_lang:
+            args.append(f"--ocr-lang={self.ocr_lang}")
+        if self.ocr_ebooks:
+            args.append(f"--ocr-ebooks")
+        if self.ocr_images:
+            args.append(f"--ocr-images")
+        if self.exclude:
+            args.append(f"--exclude={self.exclude}")
+        if self.fast:
+            args.append(f"--fast")
+        if self.treemap_threshold:
+            args.append(f"--treemap-threshold={self.treemap_threshold}")
+        if self.read_subtitles:
+            args.append(f"--read-subtitles")
+        if self.fast_epub:
+            args.append(f"--fast-epub")
+        if self.checksums:
+            args.append(f"--checksums")
+        if self.list_file:
+            args.append(f"--list_file={self.list_file}")
+
+        return args
+
+
+class Sist2Index:
+    def __init__(self, path):
+        self.path = path
+
+        with open(os.path.join(path, "descriptor.json")) as f:
+            self._descriptor = json.load(f)
+
+    def to_json(self):
+        return {
+            "path": self.path,
+            "version": self.version(),
+            "timestamp": self.timestamp(),
+            "name": self.name()
+        }
+
+    def version(self) -> Sist2Version:
+        return Sist2Version(self._descriptor["version"])
+
+    def timestamp(self) -> datetime:
+        return datetime.fromtimestamp(self._descriptor["timestamp"])
+
+    def name(self) -> str:
+        return self._descriptor["name"]
+
+
+class Sist2:
+
+    def __init__(self, bin_path: str, data_directory: str):
+        self._bin_path = bin_path
+        self._data_dir = data_directory
+
+    def index(self, options: IndexOptions, logs_cb):
+
+        if options.script:
+            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
+                f.write(options.script)
+            options.script_file = f.name
+        else:
+            options.script_file = None
+
+        args = [
+            self._bin_path,
+            *options.args(),
+            "--json-logs",
+            "--very-verbose"
+        ]
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        self._consume_logs_stdout(logs_cb, proc)
+
+        t_stderr.join()
+
+        return proc.returncode
+
+    def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
+
+        output_dir = os.path.join(
+            self._data_dir,
+            f"scan-{datetime.now()}.sist2"
+        )
+        options.output = output_dir
+
+        args = [
+            self._bin_path,
+            *options.args(),
+            "--json-logs",
+            "--very-verbose"
+        ]
+
+        logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
+
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        set_pid_cb(proc.pid)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        self._consume_logs_stdout(logs_cb, proc)
+
+        t_stderr.join()
+
+        return proc.returncode
+
+    @staticmethod
+    def _consume_logs_stderr(logs_cb, proc):
+        pipe_wrapper = TextIOWrapper(proc.stderr, encoding="utf8")
+        try:
+            for line in pipe_wrapper:
+                if line.strip() == "":
+                    continue
+                logs_cb({"stderr": line})
+        finally:
+            proc.wait()
+            pipe_wrapper.close()
+
+    @staticmethod
+    def _consume_logs_stdout(logs_cb, proc):
+        pipe_wrapper = TextIOWrapper(proc.stdout, encoding="utf8")
+        try:
+            for line in pipe_wrapper:
+                if line.strip() == "":
+                    continue
+                log_object = json.loads(line)
+                logs_cb(log_object)
+        except Exception as e:
+            proc.kill()
+            try:
+                print(line)
+            except NameError:
+                pass
+            print(traceback.format_exc())
+        finally:
+            pass
+            # proc.wait()
+            # pipe_wrapper.close()
+
+    def web(self, options: WebOptions, name: str):
+        args = [
+            self._bin_path,
+            *options.args()
+        ]
+
+        web_logger = logging.Logger(name=f"sist2-frontend-{name}")
+        web_logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"frontend-{name}.log")))
+
+        def logs_cb(message):
+            web_logger.info(json.dumps(message))
+
+        logger.info(f"Starting frontend {' '.join(args)}")
+
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        t_stdout = Thread(target=self._consume_logs_stdout, args=(logs_cb, proc))
+        t_stdout.start()
+
+        return proc.pid

sist2-admin/sist2_admin/state.py

@@ -0,0 +1,50 @@
+from typing import Dict
+
+from hexlib.db import Table
+import pickle
+
+from tesseract import get_tesseract_langs
+
+RUNNING_FRONTENDS: Dict[str, int] = {}
+
+TESSERACT_LANGS = get_tesseract_langs()
+
+DB_SCHEMA_VERSION = "1"
+
+from pydantic import BaseModel
+
+
+def _serialize(item):
+    if isinstance(item, BaseModel):
+        return pickle.dumps(item)
+    if isinstance(item, bytes):
+        raise Exception("FIXME: bytes in PickleTable")
+    return item
+
+
+def _deserialize(item):
+    if isinstance(item, bytes):
+        return pickle.loads(item)
+    return item
+
+
+class PickleTable(Table):
+
+    def __getitem__(self, item):
+        row = super().__getitem__(item)
+        if row:
+            return dict((k, _deserialize(v)) for k, v in row.items())
+        return row
+
+    def __setitem__(self, key, value):
+        value = dict((k, _serialize(v)) for k, v in value.items())
+        super().__setitem__(key, value)
+
+    def __iter__(self):
+        for row in super().__iter__():
+            yield dict((k, _deserialize(v)) for k, v in row.items())
+
+    def sql(self, where_clause, *params):
+        for row in super().sql(where_clause, *params):
+            yield dict((k, _deserialize(v)) for k, v in row.items())
+

sist2-admin/sist2_admin/tesseract.py

@@ -0,0 +1,14 @@
+import subprocess
+
+
+def get_tesseract_langs():
+
+    res = subprocess.check_output([
+        "tesseract",
+        "--list-langs"
+    ]).decode()
+
+    languages = res.split("\n")[1:]
+
+    return list(filter(lambda lang: lang and lang != "osd", languages))
+

sist2-admin/sist2_admin/web.py

@@ -0,0 +1,29 @@
+import os.path
+from typing import List
+
+from pydantic import BaseModel
+
+from sist2 import WebOptions
+
+
+class Sist2Frontend(BaseModel):
+    name: str
+    jobs: List[str]
+    web_options: WebOptions
+    running: bool = False
+
+    auto_start: bool = False
+    enable_monitoring: bool = True
+    extra_query_args: str = ""
+    custom_url: str = None
+
+    def get_log_path(self, log_folder: str):
+        return os.path.join(log_folder, f"frontend-{self.name}.log")
+
+    @staticmethod
+    def create_default(name: str):
+        return Sist2Frontend(
+            name=name,
+            web_options=WebOptions(),
+            jobs=[]
+        )

sist2-vue/dist/index.html

@@ -1,3 +1,32 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>sist2</title><link href="css/chunk-vendors.css" rel="preload" as="style"><link href="css/index.css" rel="preload" as="style"><link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/index.css" rel="stylesheet"></head><body><noscript><style>body {
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
+
+    <title>sist2</title>
+<link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"></head>
+<body>
+<noscript>
+    <style>
+        body {
             height: initial;
-        }</style><div style="text-align: center; margin-top: 100px"><strong>We're sorry but sist2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong><br><strong>Nous sommes désolés mais sist2 ne fonctionne pas correctement si JavaScript est activé. Veuillez l'activer pour continuer.</strong></div></noscript><div id="app"></div><script src="js/chunk-vendors.js"></script><script src="js/index.js"></script></body></html>
+        }
+    </style>
+    <div style="text-align: center; margin-top: 100px">
+        <strong>
+            We're sorry but sist2 doesn't work properly without JavaScript enabled.
+            Please enable it to continue.
+        </strong>
+        <br/>
+        <strong>
+            Nous sommes désolés mais sist2 ne fonctionne pas correctement
+            si JavaScript est activé.
+            Veuillez l'activer pour continuer.
+        </strong>
+    </div>
+</noscript>
+<div id="app"></div>
+<script type="text/javascript" src="js/chunk-vendors.js"></script><script type="text/javascript" src="js/index.js"></script></body>
+</html>

sist2-vue/src/components/NavBar.vue

@@ -60,7 +60,6 @@ export default {
   color: #222 !important;
   font-size: 1.75rem;
   padding: 0;
-  font-family: Hack;
 }
 
 .navbar-brand:hover {

sist2-vue/src/components/SortSelect.vue

@@ -42,6 +42,8 @@
 </template>
 
 <script>
+import {randomSeed} from "@/util";
+
 export default {
   name: "SortSelect",
   computed: {
@@ -52,7 +54,7 @@ export default {
   methods: {
     onSelect(sortMode) {
       if (sortMode === "random") {
-        this.$store.commit("setSeed", Math.round(Math.random() * 100000));
+        this.$store.commit("setSeed", randomSeed());
       }
       this.$store.commit("setSortMode", sortMode);
     }

sist2-vue/src/plugins/bootstrap-vue.js

@@ -2,6 +2,6 @@ import Vue from "vue"
 
 import BootstrapVue from "bootstrap-vue"
 import "bootstrap/dist/css/bootstrap.min.css"
-import "bootstrap-vue/dist/bootstrap-vue.css"
+import "bootstrap-vue/dist/bootstrap-vue.min.css"
 
 Vue.use(BootstrapVue)

sist2-vue/src/store/index.ts

@@ -2,7 +2,7 @@ import Vue from "vue"
 import Vuex from "vuex"
 import VueRouter, {Route} from "vue-router";
 import {EsHit, EsResult, EsTag, Index, Tag} from "@/Sist2Api";
-import {deserializeMimes, serializeMimes} from "@/util";
+import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
 
 const CONF_VERSION = 2;
 
@@ -241,6 +241,9 @@ export default new Vuex.Store({
 
             if (route.query.sort) {
                 commit("setSortMode", route.query.sort);
+                if (route.query.sort === "random" && route.query.seed === undefined) {
+                    route.query.seed = randomSeed().toString();
+                }
                 commit("setSeed", Number(route.query.seed));
             }
         },

sist2-vue/src/util.ts

@@ -163,3 +163,7 @@ export function decompressMime(mime: string): string {
         .replace(",", "+")
         .replace("X", "x-")
 }
+
+export function randomSeed(): number {
+    return Math.round(Math.random() * 100000);
+}

src/ctx.h

@@ -75,6 +75,7 @@ typedef struct {
     int verbose;
     int very_verbose;
     int no_color;
+    int json_logs;
 } LogCtx_t;
 
 typedef struct {

src/io/serialize.c

@@ -500,9 +500,6 @@ void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[SIST_
         json_str = realloc(json_str, json_str_len + 1);
         *(json_str + json_str_len) = '\n';
 
-        zstd_write_string(json_str, json_str_len + 1);
-        free(json_str);
-
         // Copy tn store contents
         size_t buf_len;
         char *buf = store_read(IncrementalCopySourceStore, (char *) doc_id, SIST_DOC_ID_LEN, &buf_len);
@@ -510,6 +507,26 @@ void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[SIST_
             store_write(IncrementalCopyDestinationStore, (char *) doc_id, SIST_DOC_ID_LEN, buf, buf_len);
             free(buf);
         }
+
+        // Also copy additional thumbnails
+        if (cJSON_GetObjectItem(document, "thumbnail") != NULL) {
+            const int thumbnail_count = cJSON_GetObjectItem(document, "thumbnail")->valueint;
+
+            for (int i = 1; i < thumbnail_count; i++) {
+                char tn_key[SIST_DOC_ID_LEN + sizeof(char) * 4];
+
+                snprintf(tn_key, sizeof(tn_key), "%s%04d", doc_id, i);
+
+                buf = store_read(IncrementalCopySourceStore, tn_key, sizeof(tn_key), &buf_len);
+                if (buf_len != 0) {
+                    store_write(IncrementalCopyDestinationStore, tn_key, sizeof(tn_key), buf, buf_len);
+                    free(buf);
+                }
+            }
+        }
+
+        zstd_write_string(json_str, json_str_len + 1);
+        free(json_str);
     }
 }
 

src/io/store.c

@@ -87,7 +87,7 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
     }
 
     if (db_full) {
-        LOG_INFOF("store.c", "Updating mdb mapsize to %lu bytes", store->size)
+        LOG_DEBUGF("store.c", "Updating mdb mapsize to %lu bytes", store->size)
 
         if (should_abort_transaction) {
             mdb_txn_abort(txn);
@@ -116,7 +116,7 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
                        store->path, mdb_strerror(ret), ret,
                        put_ret, put_ret_retry);
         }
-        LOG_INFOF("store.c", "Updated mdb mapsize to %lu bytes", store->size)
+        LOG_DEBUGF("store.c", "Updated mdb mapsize to %lu bytes", store->size)
     } else if (put_ret != 0) {
         LOG_ERROR("store.c", mdb_strerror(put_ret))
     }

src/log.c

@@ -1,4 +1,5 @@
 #include "log.h"
+#include "ctx.h"
 
 #include <pthread.h>
 #include <stdarg.h>
@@ -30,6 +31,30 @@ void vsist_logf(const char *filepath, int level, char *format, va_list ap) {
     strftime(datetime, sizeof(datetime), "%Y-%m-%d %H:%M:%S", &result);
 
     int log_len;
+    if (LogCtx.json_logs) {
+        vsnprintf(log_str, sizeof(log_str), format, ap);
+
+        cJSON *log_str_json = cJSON_CreateString(log_str);
+        char *log_str_json_str = cJSON_PrintUnformatted(log_str_json);
+
+        cJSON *filepath_json = cJSON_CreateString(filepath);
+        char *filepath_json_str = cJSON_PrintUnformatted(filepath_json);
+
+        log_len = snprintf(
+                log_str, sizeof(log_str),
+                "{\"thread\":\"%04llX\",\"datetime\":\"%s\",\"level\":\"%s\",\"filepath\":%s,\"message\":%s}\n",
+                pid, datetime, log_levels[level], filepath_json_str, log_str_json_str
+        );
+
+        cJSON_Delete(filepath_json);
+        cJSON_Delete(log_str_json);
+        free(log_str_json_str);
+        free(filepath_json_str);
+
+        write(STDOUT_FILENO, log_str, log_len);
+        return;
+    }
+
     if (is_tty) {
         log_len = snprintf(
                 log_str, sizeof(log_str),
@@ -97,6 +122,28 @@ void sist_log(const char *filepath, int level, char *str) {
     strftime(datetime, sizeof(datetime), "%Y-%m-%d %H:%M:%S", &result);
 
     int log_len;
+
+    if (LogCtx.json_logs) {
+        cJSON *log_str_json = cJSON_CreateString(str);
+        char *log_str_json_str = cJSON_PrintUnformatted(log_str_json);
+
+        cJSON *filepath_json = cJSON_CreateString(filepath);
+        char *filepath_json_str = cJSON_PrintUnformatted(filepath_json);
+
+        log_len = snprintf(
+                log_str, sizeof(log_str),
+                "{\"thread\":\"%04llX\",\"datetime\":\"%s\",\"level\":\"%s\",\"filepath\":%s,\"message\":%s}\n",
+                pid, datetime, log_levels[level], filepath_json_str, log_str_json_str
+        );
+
+        cJSON_Delete(log_str_json);
+        cJSON_Delete(filepath_json);
+        free(log_str_json_str);
+        free(filepath_json_str);
+
+        write(STDOUT_FILENO, log_str, log_len);
+        return;
+    }
     if (is_tty) {
         log_len = snprintf(
                 log_str, sizeof(log_str),

src/main.c

@@ -328,7 +328,7 @@ void load_incremental_index(const scan_args_t *args) {
             file_path,
             args->incremental,
             incremental_read(ScanCtx.original_table, file_path, &original_desc),
-            LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)),
+            LOG_DEBUG("main.c", "The base index for incremental scan does not have a main index"),
             TRUE
     );
 
@@ -635,6 +635,7 @@ int main(int argc, const char *argv[]) {
             OPT_BOOLEAN('v', "version", &arg_version, "Show version and exit"),
             OPT_BOOLEAN(0, "verbose", &LogCtx.verbose, "Turn on logging"),
             OPT_BOOLEAN(0, "very-verbose", &LogCtx.very_verbose, "Turn on debug messages"),
+            OPT_BOOLEAN(0, "json-logs", &LogCtx.json_logs, "Output logs in JSON format."),
 
             OPT_GROUP("Scan options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),

src/sist.h

@@ -49,7 +49,7 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "2.12.2"
+#define VERSION "2.13.0"
 static const char *const Version = VERSION;
 
 #ifndef SIST_PLATFORM

src/tpool.c

@@ -34,6 +34,7 @@ typedef struct tpool {
 
     int free_arg;
     int stop;
+    int waiting;
 
     int print_progress;
 
@@ -215,7 +216,13 @@ static void *tpool_worker(void *arg) {
         }
 
         if (pool->print_progress) {
-            progress_bar_print((double) pool->done_cnt / pool->work_cnt, ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
+            if (LogCtx.json_logs) {
+                progress_bar_print_json(pool->done_cnt, pool->work_cnt, ScanCtx.stat_tn_size,
+                                        ScanCtx.stat_index_size, pool->waiting);
+            } else {
+                progress_bar_print((double) pool->done_cnt / pool->work_cnt, ScanCtx.stat_tn_size,
+                                   ScanCtx.stat_index_size);
+            }
         }
 
         if (pool->work_head == NULL) {
@@ -238,6 +245,9 @@ static void *tpool_worker(void *arg) {
 void tpool_wait(tpool_t *pool) {
     LOG_DEBUG("tpool.c", "Waiting for worker threads to finish")
     pthread_mutex_lock(&(pool->work_mutex));
+
+    pool->waiting = TRUE;
+
     while (TRUE) {
         if (pool->done_cnt < pool->work_cnt) {
             pthread_cond_wait(&(pool->working_cond), &(pool->work_mutex));
@@ -250,7 +260,7 @@ void tpool_wait(tpool_t *pool) {
             }
         }
     }
-    if (pool->print_progress) {
+    if (pool->print_progress && !LogCtx.json_logs) {
         progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
     }
     pthread_mutex_unlock(&(pool->work_mutex));
@@ -312,6 +322,7 @@ tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int free_arg, int pri
     pool->throttle_stuck_cnt = 0;
     pool->mem_limit = mem_limit;
     pool->stop = FALSE;
+    pool->waiting = FALSE;
     pool->free_arg = free_arg;
     pool->cleanup_func = cleanup_func;
     pool->threads = calloc(sizeof(pthread_t), thread_cnt);

src/util.c

@@ -86,6 +86,21 @@ char *expandpath(const char *path) {
 
 int PrintingProgressBar = 0;
 
+#define BOOLEAN_STRING(x) ((x) == 0  ? "false" : "true")
+
+void progress_bar_print_json(size_t done, size_t count, size_t tn_size, size_t index_size, int waiting) {
+
+    char log_str[1024];
+
+    size_t log_len = snprintf(
+            log_str, sizeof(log_str),
+            "{\"progress\": {\"done\":%lu,\"count\":%lu,\"tn_size\":%lu,\"index_size\":%lu,\"waiting\":%s}}\n",
+            done, count, tn_size, index_size, BOOLEAN_STRING(waiting)
+    );
+
+    write(STDOUT_FILENO, log_str, log_len);
+}
+
 void progress_bar_print(double percentage, size_t tn_size, size_t index_size) {
 
     static int last_val = -1;

src/util.h

@@ -19,6 +19,7 @@ dyn_buffer_t url_escape(char *str);
 
 extern int PrintingProgressBar;
 
+void progress_bar_print_json(size_t done, size_t count,  size_t tn_size, size_t index_size, int waiting);
 void progress_bar_print(double percentage, size_t tn_size, size_t index_size);
 
 GHashTable *incremental_get_table();

src/web/serve.c

@@ -20,6 +20,13 @@ static struct mg_http_serve_opts DefaultServeOpts = {
 };
 
 
+__always_inline
+static char *address_to_string(struct mg_addr *addr) {
+    static char address_to_string_buf[INET6_ADDRSTRLEN];
+
+    return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf));
+}
+
 static void send_response_line(struct mg_connection *nc, int status_code, size_t length, char *extra_headers) {
     mg_printf(
             nc,
@@ -592,6 +599,11 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
             }
         }
 
+        LOG_DEBUGF("serve.c", "<%s> GET %s",
+                   address_to_string(&(nc->rem)),
+                   hm->uri
+        )
+
         if (mg_http_match_uri(hm, "/")) {
             search_index(nc, hm);
         } else if (mg_http_match_uri(hm, "/favicon.ico")) {

third-party/libscan/CMakeLists.txt

@@ -109,7 +109,7 @@ find_library(MUPDF_LIB NAMES liblibmupdf.a)
 find_library(CMS_LIB NAMES lcms2)
 find_library(JAS_LIB NAMES jasper)
 find_library(GUMBO_LIB NAMES gumbo)
-find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/)
+find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/)
 
 
 target_compile_options(

third-party/libscan/libscan/ocr/ocr.h

@@ -28,6 +28,8 @@ ocr_extract_text(const char *tesseract_path, const char *tesseract_lang,
     TessBaseAPI *api = TessBaseAPICreate();
     TessBaseAPIInit3(api, tesseract_path, tesseract_lang);
 
+    TessBaseAPISetPageSegMode(api, PSM_AUTO_OSD);
+
     TessBaseAPISetImage(api, img_buf, img_w, img_h, img_bpp, img_stride);
     TessBaseAPISetSourceResolution(api, img_xres);