cleanup

This reverts commit 13e7ea188b.

.dockerignore

@@ -27,4 +27,9 @@ sist2
 **/ext_libmobi
 **/ext_libwpd
 **/core
-*.a
+*.a
+tmp_scan/
+Dockerfile
+Dockerfile.arm64
+docker-compose.yml
+state.db

.drone.yml

@@ -11,21 +11,6 @@ steps:
     image: simon987/sist2-build
     commands:
       - ./scripts/build.sh
-  - name: docker
-    image: plugins/docker
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: simon987/sist2
-      context: ./
-      dockerfile: ./Dockerfile
-      auto_tag: true
-      auto_tag_suffix: x64-linux
-      when:
-        event:
-          - tag
   - name: scp files
     image: appleboy/drone-scp
     settings:
@@ -42,6 +27,21 @@ steps:
         - ./VERSION
         - ./sist2-x64-linux
         - ./sist2-x64-linux-debug
+  - name: docker
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: simon987/sist2
+      context: ./
+      dockerfile: ./Dockerfile
+      auto_tag: true
+      auto_tag_suffix: x64-linux
+      when:
+        event:
+          - tag
 
 ---
 kind: pipeline

.gitignore

@@ -10,6 +10,9 @@ Makefile
 LOG
 sist2*
 !sist2-vue/
+!sist2-admin
+!sist2_admin
+!sist2.py
 *.sist2/
 bundle*.css
 bundle.js
@@ -23,4 +26,10 @@ git_hash.h
 Testing/
 test_i
 test_i_inc
-node_modules/
+node_modules/
+.cmake/
+i_inc/
+state.db
+*.pyc
+!sist2-admin/frontend/dist
+*.js.map

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "third-party/libscan"]
-	path = third-party/libscan
-	url = https://github.com/simon987/libscan
 [submodule "third-party/argparse"]
 	path = third-party/argparse
 	url = https://github.com/simon987/argparse
@@ -10,3 +7,6 @@
 [submodule "third-party/libscan/third-party/antiword"]
 	path = third-party/libscan/third-party/antiword
 	url = https://github.com/simon987/antiword
+[submodule "third-party/libscan/third-party/libmobi"]
+	path = third-party/libscan/third-party/libmobi
+	url = https://github.com/bfabiszewski/libmobi

CMakeLists.txt

@@ -4,6 +4,7 @@ set(CMAKE_C_STANDARD 11)
 project(sist2 C)
 
 option(SIST_DEBUG "Build a debug executable" on)
+option(SIST_FAST "Enable more optimisation flags" off)
 option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0)
 
 add_compile_definitions(
@@ -21,10 +22,6 @@ set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
 add_executable(sist2
-
-        # argparse
-        third-party/argparse/argparse.h third-party/argparse/argparse.c
-
         src/main.c
         src/sist.h
         src/io/walk.h src/io/walk.c
@@ -41,7 +38,11 @@ add_executable(sist2
         src/log.c src/log.h
         src/cli.c src/cli.h
         src/stats.c src/stats.h src/ctx.c
-        src/parsing/sidecar.c src/parsing/sidecar.h)
+        src/parsing/sidecar.c src/parsing/sidecar.h
+
+        # argparse
+        third-party/argparse/argparse.h third-party/argparse/argparse.c
+        )
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
 set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
@@ -54,6 +55,10 @@ find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
+find_library(MAGIC_LIB
+        NAMES libmagic.so.1 magic
+        PATHS /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/
+)
 
 
 target_include_directories(
@@ -86,16 +91,29 @@ if (SIST_DEBUG)
             sist2
             PRIVATE
             -fsanitize=address
+            -static-libasan
     )
     set_target_properties(
             sist2
             PROPERTIES
             OUTPUT_NAME sist2_debug
     )
+elseif (SIST_FAST)
+    target_compile_options(
+            sist2
+            PRIVATE
+
+            -Ofast
+            -march=native
+            -fno-stack-protector
+            -fomit-frame-pointer
+            -freciprocal-math
+    )
 else ()
     target_compile_options(
             sist2
             PRIVATE
+
             -Ofast
             -fno-stack-protector
             -fomit-frame-pointer
@@ -120,11 +138,12 @@ target_link_libraries(
         CURL::libcurl
 
         pthread
-        magic
 
         c
 
         scan
+
+        ${MAGIC_LIB}
 )
 
 add_custom_target(

Dockerfile

@@ -2,14 +2,29 @@ FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
-COPY . .
+
+COPY scripts scripts
+COPY schema schema
+COPY CMakeLists.txt .
+COPY third-party third-party
+COPY src src
+COPY sist2-vue sist2-vue
+
 RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
 RUN make -j$(nproc)
-RUN strip sist2
+RUN strip sist2 || mv sist2_debug sist2
 
-FROM ubuntu:21.10
+FROM --platform="linux/amd64" ubuntu:20.04
 
-RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
+WORKDIR /root
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["sist2"]
+
+RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 libmagic1 python3  \
+    python3-pip git tesseract-ocr && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -20,9 +35,10 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
     curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
 
-ENTRYPOINT ["/root/sist2"]
+# sist2
+COPY --from=build /build/sist2 sist2
 
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-
-COPY --from=build /build/sist2 /root/sist2
+# sist2-admin
+COPY sist2-admin/requirements.txt sist2-admin/
+RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
+COPY sist2-admin/ sist2-admin/

Dockerfile.arm64

@@ -7,7 +7,7 @@ RUN cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE
 RUN make -j$(nproc)
 RUN strip sist2
 
-FROM --platform linux/arm64/v8 ubuntu:21.10
+FROM --platform="linux/arm64/v8" ubuntu:21.10
 
 RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
 

README.md

@@ -48,10 +48,11 @@ sist2 (Simple incremental search tool)
               - "ES_JAVA_OPTS=-Xms1G -Xmx2G"
         ```
 1. Download sist2 executable
-    1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) *
-    1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
+    1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases). 
+Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x` *
+    2. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
        recommended!)*
-    1. *(or)* `docker pull simon987/sist2:2.11.4-x64-linux`
+    3. *(or)* `docker pull simon987/sist2:2.12.1-x64-linux`
 
 1. See [Usage guide](docs/USAGE.md)
 
@@ -67,23 +68,23 @@ See [Usage guide](docs/USAGE.md) for more details
 
 ## Format support
 
-File type | Library | Content | Thumbnail | Metadata
-:---|:---|:---|:---|:---
-pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title |
-cbz,cbr | [libscan](https://github.com/simon987/libscan) | - | yes | - |
-`audio/*` | ffmpeg | - | yes | ID3 tags |
-`video/*` | ffmpeg | - | yes | title, comment, artist |
-`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
-raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags, GPS tags |
-ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
-`text/plain` | [libscan](https://github.com/simon987/libscan) | yes | no | - |
-html, xml | [libscan](https://github.com/simon987/libscan) | yes | no | - |
-tar, zip, rar, 7z, ar ...  | Libarchive | yes\* | - | no |
-docx, xlsx, pptx | [libscan](https://github.com/simon987/libscan) | yes | if embedded | creator, modified_by, title |
-doc (MS Word 97-2003) | antiword | yes | yes | author, title |
-mobi, azw, azw3 | libmobi | yes | no | author, title |
-wpd (WordPerfect) | libwpd | yes | no | *planned* |
-json, jsonl, ndjson | [libscan](https://github.com/simon987/libscan) | yes | - | - |
+| File type                                                                 | Library                                                                      | Content  | Thumbnail   | Metadata                                                                                                                               |
+|:--------------------------------------------------------------------------|:-----------------------------------------------------------------------------|:---------|:------------|:---------------------------------------------------------------------------------------------------------------------------------------|
+| pdf,xps,fb2,epub                                                          | MuPDF                                                                        | text+ocr | yes         | author, title                                                                                                                          |
+| cbz,cbr                                                                   | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | -        | yes         | -                                                                                                                                      |
+| `audio/*`                                                                 | ffmpeg                                                                       | -        | yes         | ID3 tags                                                                                                                               |
+| `video/*`                                                                 | ffmpeg                                                                       | -        | yes         | title, comment, artist                                                                                                                 |
+| `image/*`                                                                 | ffmpeg                                                                       | ocr      | yes         | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
+| raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf | LibRaw                                                                       | no       | yes         | Common EXIF tags, GPS tags                                                                                                             |
+| ttf,ttc,cff,woff,fnt,otf                                                  | Freetype2                                                                    | -        | yes, `bmp`  | Name & style                                                                                                                           |
+| `text/plain`                                                              | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| html, xml                                                                 | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
+| docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
+| doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | yes         | author, title                                                                                                                          |
+| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | no          | author, title                                                                                                                          |
+| wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
+| json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
 \* *See [Archive files](#archive-files)*
 
@@ -102,18 +103,24 @@ scan is also supported.
 
 ### OCR
 
-You can enable OCR support for pdf,xps,fb2,epub file types with the
-`--ocr <lang>` option. Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
+You can enable OCR support for ebook (pdf,xps,fb2,epub) or image file types with the
+`--ocr-lang <lang>` option in combination with `--ocr-images` and/or `--ocr-ebooks`.
+Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
 The `simon987/sist2` image comes with common languages
 (hin, jpn, eng, fra, rus, spa) pre-installed.
 
-Examples
+You can use the `+` separator to specify multiple languages. The language
+name must be identical to the `*.traineddata` file installed on your system 
+(use `chi_sim` rather than `chi-sim`).
+
+Examples:
 
 ```bash
-sist2 scan --ocr jpn ~/Books/Manga/
-sist2 scan --ocr eng ~/Books/Textbooks/
+sist2 scan --ocr-ebooks --ocr-lang jpn ~/Books/Manga/
+sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
+sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
 ```
 
 ## Build from source
@@ -126,7 +133,7 @@ You can compile **sist2** by yourself if you don't want to use the pre-compiled
 git clone --recursive https://github.com/simon987/sist2/
 cd sist2
 docker build . -f ./Dockerfile -t my-sist2-image
-docker run --rm my-sist2-image cat /root/sist2 > sist2-x64-linux
+docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 ```
 
 ### On a linux computer
@@ -143,7 +150,7 @@ docker run --rm my-sist2-image cat /root/sist2 > sist2-x64-linux
 
     ```bash
     vcpkg install curl[core,openssl]
-    vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libuuid libmagic libraw jasper lcms gumbo
+    vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw jasper lcms gumbo
     ```
 
 1. Build

contrib/systemd/Makefile

@@ -0,0 +1,7 @@
+install:
+	install sist2-update-all.sh /usr/bin/sist2-update-all.sh
+	install sist2-update-files.sh /usr/bin/sist2-update-files.sh
+	install sist2-update-nextcloud.sh /usr/bin/sist2-update-nextcloud.sh
+	install sist2-update.service /etc/systemd/system/sist2-update.service
+	install sist2-update.timer /etc/systemd/system/sist2-update.timer
+	systemctl daemon-reload

contrib/systemd/README.md

@@ -0,0 +1,31 @@
+# Systemd integration example
+
+This example contains my (yatli) personal configuration for sist2 auto-updating.
+The following indices are involved in this configuration:
+
+| Index     | Path             | Description                                |
+|-----------|------------------|--------------------------------------------|
+| files     | /zpool/files     | Main file repository                       |
+| nextcloud | /zpool/nextcloud | Externally synchronized to a cloud account |
+
+The systemd integration achieves automatic sist2 scanning & indexing everyday at 3:00AM.
+
+### Tailoring the configuration for yourself
+
+`sist2-update-all.sh` calls update scripts for each sist2 index. Add or remove
+update scripts accordingly to suit your need. Each update script (e.g.
+`sist2-update-files.sh`) has important parameters laid down at the beginning so
+make sure to edit them to point to your files and index locations.
+
+### Installation
+
+```bash
+# install the services and scripts
+sudo make install
+# enable & start the timer
+sudo systemctl enable sist2-update.timer
+sudo systemctl start sist2-update.timer
+# verify that the timer has been enabled
+systemctl list-timers --all
+```
+

contrib/systemd/sist2-update-all.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+echo "Update index: Files"
+source ${__dir}/sist2-update-files.sh
+echo "Update index: Nextcloud"
+source ${__dir}/sist2-update-nextcloud.sh
+echo "Done. Restarting sist2."
+docker restart sist2-sist2-1

contrib/systemd/sist2-update-files.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+DATE=$(date +%Y_%m_%d)
+CONTENT=/zpool/files
+ORIG=/mnt/ssd/sist-index/files.idx
+NEW=/mnt/ssd/sist-index/files_$DATE.idx
+EXCLUDE='ZArchives|TorrentStore|TorrentDownload|624f0c59-1fef-44f6-95e9-7483296f2833|ubuntu-full-2021-12-07'
+NAME=Files
+#REWRITE_URL="http://localhost:33333/activate?collection=$NAME&path="
+REWRITE_URL=""
+
+sist2 scan \
+  --threads 14 \
+  --mem-throttle 32768 \
+  --quality 1.0 \
+  --name $NAME \
+  --ocr-lang=eng+chi_sim \
+  --ocr-ebooks \
+  --ocr-images \
+  --exclude=$EXCLUDE \
+  --rewrite-url=$REWRITE_URL \
+  --incremental=$ORIG \
+  --output=$NEW \
+  $CONTENT
+echo ">>> Scan complete"
+rm -rf $ORIG
+mv $NEW $ORIG 
+
+unset http_proxy
+unset https_proxy
+unset HTTP_PROXY
+unset HTTPS_PROXY
+sist2 index $ORIG --incremental-index
+echo ">>> Index complete"

contrib/systemd/sist2-update-nextcloud.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+DATE=$(date +%Y_%m_%d)
+CONTENT=/zpool/nextcloud/v-yadli
+ORIG=/mnt/ssd/sist-index/nextcloud.idx
+NEW=/mnt/ssd/sist-index/nextcloud_$DATE.idx
+EXCLUDE='Yatao|.*263418493\\/Image\\/.*'
+NAME=NextCloud
+# REWRITE_URL="http://localhost:33333/activate?collection=$NAME&path="
+REWRITE_URL=""
+
+sist2 scan \
+  --threads 14 \
+  --mem-throttle 32768 \
+  --quality 1.0 \
+  --name $NAME \
+  --ocr-lang=eng+chi_sim \
+  --ocr-ebooks \
+  --ocr-images \
+  --exclude=$EXCLUDE \
+  --rewrite-url=$REWRITE_URL \
+  --incremental=$ORIG \
+  --output=$NEW \
+  $CONTENT
+echo ">>> Scan complete"
+rm -rf $ORIG
+mv $NEW $ORIG 
+
+unset http_proxy
+unset https_proxy
+unset HTTP_PROXY
+unset HTTPS_PROXY
+sist2 index $ORIG --incremental-index

contrib/systemd/sist2-update.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=sist2-update
+
+[Service]
+User=yatli
+ExecStart=/bin/bash /usr/bin/sist2-update-all.sh

contrib/systemd/sist2-update.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=sist2-update
+
+[Timer]
+OnCalendar=*-*-* 3:00:00
+Persistent=true
+Unit=sist2-update.service
+
+[Install]
+WantedBy=timers.target

docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3"
+
+services:
+  elasticsearch:
+    image: elasticsearch:7.14.0
+    container_name: sist2-es
+    environment:
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+  sist2-admin:
+    build:
+      context: .
+    container_name: sist2-admin
+    volumes:
+      - /mnt/array/sist2-admin-data/:/sist2-admin/
+      - /:/host
+    ports:
+      # NOTE: Don't export this port publicly!
+      - 8080:8080
+      - 4090:4090
+    working_dir: /root/sist2-admin/
+    entrypoint: python3
+    command:
+      - /root/sist2-admin/sist2_admin/app.py

docs/USAGE.md

@@ -13,7 +13,6 @@
     * [options](#web-options)
     * [examples](#web-examples)
     * [rewrite_url](#rewrite_url)
-    * [link to specific indices](#link-to-specific-indices)
 * [elasticsearch](#elasticsearch)
 * [exec-script](#exec-script)
 * [tagging](#tagging)
@@ -26,58 +25,66 @@ Usage: sist2 scan [OPTION]... PATH
    or: sist2 exec-script [OPTION]... INDEX
 Lightning-fast file system indexer and search tool.
 
-    -h, --help                    show this help message and exit
-    -v, --version                 Show version and exit
-    --verbose                     Turn on logging
-    --very-verbose                Turn on debug messages
+    -h, --help                        show this help message and exit
+    -v, --version                     Show version and exit
+    --verbose                         Turn on logging
+    --very-verbose                    Turn on debug messages
 
 Scan options
-    -t, --threads=<int>           Number of threads. DEFAULT=1
-    -q, --quality=<flt>           Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=3
-    --size=<int>                  Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
-    --content-size=<int>          Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
-    --incremental=<str>           Reuse an existing index and only scan modified files.
-    -o, --output=<str>            Output directory. DEFAULT=index.sist2/
-    --rewrite-url=<str>           Serve files from this url instead of from disk.
-    --name=<str>                  Index display name. DEFAULT: (name of the directory)
-    --depth=<int>                 Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --archive=<str>               Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
-    --archive-passphrase=<str>    Passphrase for encrypted archive files
-    --ocr=<str>                   Tesseract language (use tesseract --list-langs to see which are installed on your machine)
-    -e, --exclude=<str>           Files that match this regex will not be scanned
-    --fast                        Only index file names & mime type
-    --treemap-threshold=<str>     Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
-    --mem-buffer=<int>            Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000
-    --read-subtitles              Read subtitles from media files.
-    --fast-epub                   Faster but less accurate EPUB parsing (no thumbnails, metadata)
-    --checksums                   Calculate file checksums when scanning.
+    -t, --threads=<int>               Number of threads. DEFAULT=1
+    --mem-throttle=<int>              Total memory threshold in MiB for scan throttling. DEFAULT=0
+    -q, --thumbnail-quality=<flt>     Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=1
+    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT=500
+    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT=1
+    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT=32768
+    --incremental=<str>               Reuse an existing index and only scan modified files.
+    -o, --output=<str>                Output directory. DEFAULT=index.sist2/
+    --rewrite-url=<str>               Serve files from this url instead of from disk.
+    --name=<str>                      Index display name. DEFAULT: (name of the directory)
+    --depth=<int>                     Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
+    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
+    --archive-passphrase=<str>        Passphrase for encrypted archive files
+    --ocr-lang=<str>                  Tesseract language (use 'tesseract --list-langs' to see which are installed on your machine)
+    --ocr-images                      Enable OCR'ing of image files.
+    --ocr-ebooks                      Enable OCR'ing of ebook files.
+    -e, --exclude=<str>               Files that match this regex will not be scanned
+    --fast                            Only index file names & mime type
+    --treemap-threshold=<str>         Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
+    --mem-buffer=<int>                Maximum memory buffer size per thread in MiB for files inside archives (see USAGE.md). DEFAULT: 2000
+    --read-subtitles                  Read subtitles from media files.
+    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata)
+    --checksums                       Calculate file checksums when scanning.
+    --list-file=<str>                 Specify a list of newline-delimited paths to be scanned instead of normal directory traversal. Use '-' to read from stdin.
 
 Index options
-    -t, --threads=<int>           Number of threads. DEFAULT=1
-    --es-url=<str>                Elasticsearch url with port. DEFAULT=http://localhost:9200
-    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
-    -p, --print                   Just print JSON documents to stdout.
-    --script-file=<str>           Path to user script.
-    --mappings-file=<str>         Path to Elasticsearch mappings.
-    --settings-file=<str>         Path to Elasticsearch settings.
-    --async-script                Execute user script asynchronously.
-    --batch-size=<int>            Index batch size. DEFAULT: 100
-    -f, --force-reset             Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
+    -t, --threads=<int>               Number of threads. DEFAULT=1
+    --es-url=<str>                    Elasticsearch url with port. DEFAULT=http://localhost:9200
+    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    -p, --print                       Just print JSON documents to stdout.
+    --incremental-index               Conduct incremental indexing, assumes that the old index is already digested by Elasticsearch.
+    --script-file=<str>               Path to user script.
+    --mappings-file=<str>             Path to Elasticsearch mappings.
+    --settings-file=<str>             Path to Elasticsearch settings.
+    --async-script                    Execute user script asynchronously.
+    --batch-size=<int>                Index batch size. DEFAULT: 100
+    -f, --force-reset                 Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
 
 Web options
-    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
-    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
-    --bind=<str>                  Listen on this address. DEFAULT=localhost:4090
-    --auth=<str>                  Basic auth in user:password format
-    --tag-auth=<str>              Basic auth in user:password format for tagging
-    --tagline=<str>               Tagline in navbar
-    --dev                         Serve html & js files from disk (for development)
+    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --bind=<str>                      Listen on this address. DEFAULT=localhost:4090
+    --auth=<str>                      Basic auth in user:password format
+    --tag-auth=<str>                  Basic auth in user:password format for tagging
+    --tagline=<str>                   Tagline in navbar
+    --dev                             Serve html & js files from disk (for development)
+    --lang=<str>                      Default UI language. Can be changed by the user
 
 Exec-script options
-    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
-    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
-    --script-file=<str>           Path to user script.
-    --async-script                Execute user script asynchronously.
+    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --script-file=<str>               Path to user script.
+    --async-script                    Execute user script asynchronously.
+Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
 ## Scan
@@ -85,14 +92,22 @@ Exec-script options
 ### Scan options
 
 * `-t, --threads` 
-      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-WmiObject Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
-* `-q, --quality` 
+      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
+* `--mem-throttle`
+    Total memory threshold in MiB for scan throttling. Worker threads will not start a new parse job
+    until the total memory usage of sist2 is below this threshold. Set to 0 to disable. DEFAULT=0
+* `-q, --thumbnail-quality` 
     Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best.
-* `--size` 
+* `--thumbnail-size` 
     Thumbnail size in pixels.
+* `--thumbnail-count`
+    Maximum number of thumbnails to generate. When set to a value >= 2, thumbnails for video previews
+    will be generated. The actual number of thumbnails generated depends on the length of the video (maximum 1 image 
+    every ~7s). Set to 0 to completely disable thumbnails.
 * `--content-size` 
-    Number of bytes of text to be extracted from the content of files (plain text and PDFs).
+    Number of bytes of text to be extracted from the content of files (plain text, PDFs etc.).
     Repeated whitespace and special characters do not count toward this limit.
+    Set to 0 to completely disable content parsing.
 * `--incremental`
     Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
     will be copied to the new index and will not be parsed again.
@@ -105,7 +120,7 @@ Exec-script options
     * list: Only get file names as text
     * shallow: Don't parse archives inside archives.
     * recurse: Scan archives recursively (default)
-* `--ocr` See [OCR](../README.md#OCR)
+* `--ocr-lang`, `--ocr-ebooks`, `--ocr-images` See [OCR](../README.md#OCR)
 * `-e, --exclude` Regex pattern to exclude files. A file is excluded if the pattern matches any 
     part of the full absolute path.
     
@@ -125,13 +140,13 @@ Exec-script options
     In effect, smaller `treemap-threshold` values will yield a more detailed 
     (but also a more cluttered and harder to read) visualization. 
     
-* `--mem-buffer` Maximum memory buffer size in MB (per thread) for files inside archives. Media files 
+* `--mem-buffer` Maximum memory buffer size in MiB (per thread) for files inside archives. Media files 
     larger than this number will be read sequentially and no *seek* operations will be supported.
 
     To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
 * `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
 * `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
-* `--checksums` Calculate file checksums (sha1) when scanning files. This option does not cause any additional read 
+* `--checksums` Calculate file checksums (SHA1) when scanning files. This option does not cause any additional read 
   operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
   files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
 
@@ -201,6 +216,9 @@ and values are raw image bytes.
     Elasticsearch index name. DEFAULT=sist2
  * `-p, --print` 
     Print index in JSON format to stdout.
+ * `--incremental-index`
+   Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
+   Only the new changes since the last scan will be sent.
  * `--script-file` 
     Path to user script. See [Scripting](scripting.md).
  * `--mappings-file`
@@ -248,7 +266,9 @@ sist2 index --print ./my_index/ | jq | less
     `--auth` argument, but authentication is only applied the `/tag/` endpoint.
  * `--tagline=<str>` When specified, will replace the default tagline in the navbar.
  * `--dev` Serve html & js files from disk (for development, used to modify frontend files without having to recompile)
- 
+ * `--lang=<str>` Set the default web UI language (See #180 for a list of supported languages, default
+   is `en`). The user can change the language in the configuration page
+
 ### Web examples
 
 **Single index**
@@ -272,7 +292,7 @@ Both the `root` and `rewrite_url` fields are safe to manually modify from the
 
 # Elasticsearch
 
-Elasticsearch versions >=6.8.0, <8.0.0 are supported by sist2. 
+Elasticsearch versions >=6.8.0, 7.X.X and 8.X.X are supported by sist2. 
 
 Using a version >=7.14.0 is recommended to enable the following features:
 

schema/mappings.json

@@ -39,7 +39,7 @@
       "index": false
     },
     "thumbnail": {
-      "type": "keyword",
+      "type": "integer",
       "index": false
     },
     "videoc": {
@@ -67,7 +67,8 @@
       "index": false
     },
     "mtime": {
-      "type": "integer"
+      "type": "date",
+      "format": "epoch_millis"
     },
     "size": {
       "type": "long"

schema/settings.json

@@ -3,7 +3,7 @@
     "refresh_interval": "30s",
     "codec": "best_compression",
     "number_of_replicas": 0,
-    "highlight.max_analyzed_offset": 10000000
+    "highlight.max_analyzed_offset": 1000000
   },
   "analysis": {
     "tokenizer": {
@@ -16,7 +16,7 @@
         "delimiter": "."
       },
       "my_nGram_tokenizer": {
-        "type": "nGram",
+        "type": "ngram",
         "min_gram": 3,
         "max_gram": 3
       }

scripts/before_build.sh

@@ -5,6 +5,6 @@ rm -rf index.sist2/
 python3 scripts/mime.py > src/parsing/mime_generated.c
 python3 scripts/serve_static.py > src/web/static_generated.c
 python3 scripts/index_static.py > src/index/static_generated.c
+python3 scripts/magic_static.py > src/magic_generated.c
 
-printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h
-printf "static const char *const LibScanCommitHash = \"%s\";\n" $(cd third-party/libscan/ && git rev-parse HEAD) >> src/git_hash.h
+printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h

scripts/magic_static.py

@@ -0,0 +1,8 @@
+
+try:
+    with open("/usr/lib/file/magic.mgc", "rb") as f:
+        data = f.read()
+except:
+    data = bytes([])
+
+print("char magic_database_buffer[%d] = {%s};" % (len(data), ",".join(str(int(b)) for b in data)))

scripts/reset.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-make clean
-rm -rf CMakeFiles/ CMakeCache.txt Makefile \
-  third-party/libscan/CMakeFiles third-party/libscan/CMakeCache.txt third-party/libscan/third-party/ext_ffmpeg \
-  third-party/libscan/third-party/ext_libmobi third-party/libscan/Makefile

scripts/start_dev_es.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es"\
+       	-p 9200:9200 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.14.0

scripts/start_dev_es_6.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es-6"\
+       	-p 9202:9200 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:6.8.0

scripts/start_dev_es_8.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es"\
+       	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.1.2

sist2-admin/frontend/babel.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  presets: [
+    '@vue/cli-plugin-babel/preset'
+  ]
+}

sist2-admin/frontend/dist/css/app.0f0b676b.css

@@ -0,0 +1 @@
+.navbar[data-v-27bc1d68]{box-shadow:0 .125rem .25rem rgba(0,0,0,.08)!important;border-radius:0}.theme-black .navbar[data-v-27bc1d68]{background:rgba(84,107,122,.18823529411764706);border-bottom:none}.navbar-brand[data-v-27bc1d68]{color:#222!important;font-size:1.75rem;padding:0}.navbar-brand[data-v-27bc1d68]:hover{color:#000!important}.version[data-v-27bc1d68]{color:#222!important;margin-left:-18px;margin-top:-14px;font-size:11px;font-family:monospace}.btn-link[data-v-27bc1d68]{color:#222}body,html{height:100%}#app{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;color:#2c3e50;padding-bottom:1em;min-height:100%}.info-icon{width:1rem;margin-right:.2rem;cursor:pointer;line-height:1rem;height:1rem;background-image:url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjYuNjY3IDQyNi42NjciIGZpbGw9IiNmZmYiPjxwYXRoIGQ9Ik0xOTIgMTkyaDQyLjY2N3YxMjhIMTkyeiIvPjxwYXRoIGQ9Ik0yMTMuMzMzIDBDOTUuNDY3IDAgMCA5NS40NjcgMCAyMTMuMzMzczk1LjQ2NyAyMTMuMzMzIDIxMy4zMzMgMjEzLjMzM1M0MjYuNjY3IDMzMS4yIDQyNi42NjcgMjEzLjMzMyAzMzEuMiAwIDIxMy4zMzMgMHptMCAzODRjLTk0LjA4IDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMgNDIuNjY3IDIxMy4zMzMgNDIuNjY3IDM4NCAxMTkuMjUzIDM4NCAyMTMuMzMzIDMwNy40MTMgMzg0IDIxMy4zMzMgMzg0eiIvPjxwYXRoIGQ9Ik0xOTIgMTA2LjY2N2g0Mi42Njd2NDIuNjY3SDE5MnoiLz48L3N2Zz4=);filter:brightness(45%);display:block}.tabs{margin-top:10px}.modal-title{text-overflow:ellipsis;overflow:hidden;white-space:nowrap}@media screen and (min-width:1500px){.container{max-width:1440px}}label{margin-top:.5rem;margin-bottom:0}.shrink[data-v-9b017c42]{flex-grow:inherit}#task-history[data-v-46960281]{font-family:monospace;font-size:12px}#log-tail-output span{display:block}span.DEBUG{color:#9e9e9e}span.WARNING{color:#ffb300}span.INFO{color:#039be5}span.ERROR,span.FATAL{color:#f4511e}span.ADMIN{color:#ee05ff}#log-tail-output{font-size:13px;font-family:monospace;padding:6px;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px;margin:3px;white-space:pre;color:#000;overflow:hidden}

sist2-admin/frontend/dist/index.html

@@ -0,0 +1 @@
+<!DOCTYPE html><html lang=""><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="favicon.ico"><title>sist2-admin</title><link href="css/app.0f0b676b.css" rel="preload" as="style"><link href="css/chunk-vendors.aa66c7e8.css" rel="preload" as="style"><link href="js/app.b34f501e.js" rel="preload" as="script"><link href="js/chunk-vendors.fad0ee6a.js" rel="preload" as="script"><link href="css/chunk-vendors.aa66c7e8.css" rel="stylesheet"><link href="css/app.0f0b676b.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but sist2-admin-vue doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="js/chunk-vendors.fad0ee6a.js"></script><script src="js/app.b34f501e.js"></script></body></html>

sist2-admin/frontend/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "sist2-admin-vue",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "serve": "vue-cli-service serve",
+    "build": "vue-cli-service build",
+    "watch": "vue-cli-service build --watch"
+  },
+  "dependencies": {
+    "axios": "^0.27.2",
+    "bootstrap-vue": "^2.21.2",
+    "core-js": "^3.6.5",
+    "moment": "^2.29.3",
+    "socket.io-client": "^4.5.1",
+    "vue": "^2.6.14",
+    "vue-i18n": "^8.24.4",
+    "vue-router": "^3.5.4",
+    "vuex": "^3.4.0"
+  },
+  "devDependencies": {
+    "@vue/cli-plugin-babel": "~4.5.13",
+    "@vue/cli-plugin-eslint": "~4.5.13",
+    "@vue/cli-plugin-router": "~4.5.13",
+    "@vue/cli-plugin-vuex": "~4.5.13",
+    "@vue/cli-service": "~4.5.13",
+    "babel-eslint": "^10.1.0",
+    "bootstrap": "^4.5.2",
+    "eslint": "^6.7.2",
+    "eslint-plugin-vue": "^6.2.2",
+    "vue-template-compiler": "^2.6.11"
+  },
+  "eslintConfig": {
+    "root": true,
+    "env": {
+      "node": true
+    },
+    "extends": [
+      "plugin:vue/essential",
+      "eslint:recommended"
+    ],
+    "parserOptions": {
+      "parser": "babel-eslint"
+    },
+    "rules": {}
+  },
+  "browserslist": [
+    "> 1%",
+    "last 2 versions",
+    "not dead"
+  ]
+}

sist2-admin/frontend/public/index.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width,initial-scale=1.0">
+    <link rel="icon" href="<%= BASE_URL %>favicon.ico">
+    <title>sist2-admin</title>
+  </head>
+  <body>
+    <noscript>
+      <strong>We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled. Please enable it to continue.</strong>
+    </noscript>
+    <div id="app"></div>
+    <!-- built files will be auto injected -->
+  </body>
+</html>

sist2-admin/frontend/src/App.vue

@@ -0,0 +1,98 @@
+<template>
+  <div id="app">
+    <NavBar></NavBar>
+    <b-container class="pt-4">
+      <b-alert show dismissible variant="info">
+        This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
+        to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on Github</a>. Thank you!
+      </b-alert>
+      <router-view/>
+    </b-container>
+  </div>
+</template>
+
+<script>
+import NavBar from "@/components/NavBar";
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  components: {NavBar},
+  data() {
+    return {
+      socket: null
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getSist2AdminInfo()
+        .then(resp => this.$store.commit("setSist2AdminInfo", resp.data));
+    this.$store.dispatch("loadBrowserSettings");
+    this.connectNotifications();
+    // this.socket.onclose = this.connectNotifications;
+  },
+  methods: {
+    connectNotifications() {
+      this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
+      this.socket.onopen = () => {
+        this.socket.send("Hello from client");
+      }
+
+      this.socket.onmessage = e => {
+        const notification = JSON.parse(e.data);
+        if (notification.message) {
+          notification.messageString = this.$t(notification.message).toString();
+        }
+
+        this.$store.dispatch("notify", notification)
+      }
+    }
+  }
+}
+</script>
+
+<style>
+html, body {
+  height: 100%;
+}
+
+#app {
+  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  /*text-align: center;*/
+  color: #2c3e50;
+  padding-bottom: 1em;
+  min-height: 100%;
+}
+
+.info-icon {
+  width: 1rem;
+  margin-right: 0.2rem;
+  cursor: pointer;
+  line-height: 1rem;
+  height: 1rem;
+  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+  filter: brightness(45%);
+  display: block;
+}
+
+.tabs {
+  margin-top: 10px;
+}
+
+.modal-title {
+  text-overflow: ellipsis;
+  overflow: hidden;
+  white-space: nowrap;
+}
+
+@media screen and (min-width: 1500px) {
+  .container {
+    max-width: 1440px;
+  }
+}
+
+label {
+  margin-top: 0.5rem;
+  margin-bottom: 0;
+}
+</style>

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -0,0 +1,117 @@
+import axios from "axios";
+
+class Sist2AdminApi {
+
+    constructor() {
+        this.baseUrl = window.location.protocol + "//" + window.location.host;
+    }
+
+    getJobs() {
+        return axios.get(`${this.baseUrl}/api/job/`);
+    }
+
+    getFrontends() {
+        return axios.get(`${this.baseUrl}/api/frontend/`);
+    }
+
+    getTasks() {
+        return axios.get(`${this.baseUrl}/api/task/`);
+    }
+
+    killTask(taskId) {
+        return axios.post(`${this.baseUrl}/api/task/${taskId}/kill`)
+    }
+
+    getTaskHistory() {
+        return axios.get(`${this.baseUrl}/api/task/history`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    getJob(name) {
+        return axios.get(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    getFrontend(name) {
+        return axios.get(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    startFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}/start`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    stopFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}/stop`);
+    }
+
+    /**
+     * @param {string} name
+     * @param job
+     */
+    updateJob(name, job) {
+        return axios.put(`${this.baseUrl}/api/job/${name}`, job);
+    }
+
+    /**
+     * @param {string} name
+     * @param frontend
+     */
+    updateFrontend(name, frontend) {
+        return axios.put(`${this.baseUrl}/api/frontend/${name}`, frontend);
+    }
+
+    /**
+     * @param {string} name
+     */
+    runJob(name) {
+        return axios.get(`${this.baseUrl}/api/job/${name}/run`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    deleteJob(name) {
+        return axios.delete(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    deleteFrontend(name) {
+        return axios.delete(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    createJob(name) {
+        return axios.post(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    createFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    pingEs(url, insecure) {
+        return axios.get(`${this.baseUrl}/api/ping_es`, {params: {url, insecure}});
+    }
+
+    getSist2AdminInfo() {
+        return axios.get(`${this.baseUrl}/api/`);
+    }
+}
+
+export default new Sist2AdminApi()

sist2-admin/frontend/src/components/FrontendListItem.vue

@@ -0,0 +1,31 @@
+<template>
+  <b-list-group-item action :to="`/frontend/${frontend.name}`">
+
+    <div class="d-flex w-100 justify-content-between">
+      <h5 class="mb-1" style="display: block">
+        {{ frontend.name }}
+        <b-badge variant="light">{{ formatBindAddress(frontend.web_options.bind) }}</b-badge>
+      </h5>
+
+      <div>
+        <b-badge v-if="frontend.running" variant="success">{{$t("online")}}</b-badge>
+        <b-badge v-else variant="secondary">{{$t("offline")}}</b-badge>
+      </div>
+    </div>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import {formatBindAddress} from "@/util";
+
+export default {
+  name: "FrontendListItem",
+  props: ["frontend"],
+  data() {
+    return {
+      formatBindAddress
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/components/IndexOptions.vue

@@ -0,0 +1,64 @@
+<template>
+  <div>
+    <label>{{ $t("indexOptions.threads") }}</label>
+    <b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.esUrl") }}</label>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <label>{{ $t("indexOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
+    <br>
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("indexOptions.batchSize") }}</label>
+    <b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("indexOptions.script") }}</label>
+    <b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
+  </div>
+
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "IndexOptions",
+  props: ["options"],
+  data() {
+    return {
+      showEsTestAlert: false,
+      esTestOk: false,
+      esTestMessage: "",
+    }
+  },
+  methods: {
+    update() {
+      this.$emit("change", this.options);
+    },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
+  },
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/JobCheckboxGroup.vue

@@ -0,0 +1,42 @@
+<template>
+  <div>
+    <h5>{{ $t("selectJobs") }}</h5>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-form-group v-else>
+      <b-form-checkbox-group
+          v-if="jobs.length > 0"
+          :checked="frontend.jobs"
+          @input="frontend.jobs = $event; $emit('input')"
+      >
+        <div v-for="job in jobs" :key="job.name">
+          <b-form-checkbox :disabled="job.status !== 'indexed'" :value="job.name">[{{ job.name }}]</b-form-checkbox>
+          <br/>
+        </div>
+      </b-form-checkbox-group>
+      <div v-else>
+        <span class="text-muted">{{ $t('jobOptions.noJobAvailable') }}</span>
+        &nbsp;<router-link to="/">{{$t("create")}}</router-link>
+      </div>
+    </b-form-group>
+  </div>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "JobCheckboxGroup",
+  props: ["frontend"],
+  mounted() {
+    Sist2AdminApi.getJobs().then(resp => {
+      this.jobs = resp.data;
+      this.loading = false;
+    });
+  },
+  data() {
+    return {
+      loading: true,
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/components/JobListItem.vue

@@ -0,0 +1,56 @@
+<template>
+  <b-list-group-item class="flex-column align-items-start" action :to="`job/${job.name}`">
+
+    <div class="d-flex w-100 justify-content-between">
+      <div>
+        <h5 class="mb-1">
+          {{ job.name }}
+        </h5>
+      </div>
+      <div>
+        <b-row>
+          <b-col>
+            <small v-if="job.last_index_date">
+              {{ $t("scanned") }} {{ formatLastIndexDate(job.last_index_date) }}</small>
+            <div v-else>&nbsp;</div>
+          </b-col>
+        </b-row>
+        <b-row v-if="job.schedule_enabled">
+          <b-col>
+            <small><code>{{job.cron_expression }}</code></small>
+          </b-col>
+        </b-row>
+        <b-row v-else>
+          <b-col>
+            &nbsp;
+          </b-col>
+        </b-row>
+
+      </div>
+    </div>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import moment from "moment";
+
+export default {
+  name: "JobListItem",
+  props: ["job"],
+  methods: {
+    formatLastIndexDate(dateString) {
+      if (dateString === null) {
+        return "";
+      }
+
+      const date = Date.parse(dateString);
+      return moment(date).fromNow();
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/JobOptions.vue

@@ -0,0 +1,51 @@
+<template>
+  <div>
+    <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
+      {{ $t("jobOptions.desktopNotifications") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
+      {{ $t("jobOptions.scheduleEnabled") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("jobOptions.cron") }}</label>
+    <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression" :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
+  </div>
+</template>
+
+<script>
+
+export default {
+  name: "JobOptions",
+  props: ["job"],
+  data() {
+    return {
+      cronValid: undefined
+    }
+  },
+  computed: {
+    desktopNotificationsEnabled() {
+      return this.$store.state.jobDesktopNotificationMap[this.job.name];
+    }
+  },
+  methods: {
+    updateNotifications(value) {
+      this.$store.dispatch("setJobDesktopNotification", {
+        job: this.job.name,
+        enabled: value
+      })
+    },
+    update() {
+      if (this.job.schedule_enabled) {
+        this.cronValid = /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(this.job.cron_expression);
+      } else {
+        this.cronValid = undefined;
+      }
+
+      if (this.cronValid !== false) {
+        this.$emit("change", this.job);
+      }
+    },
+  },
+}
+</script>

sist2-admin/frontend/src/components/NavBar.vue

@@ -0,0 +1,69 @@
+<template>
+  <b-navbar>
+    <b-navbar-brand to="/">
+      <Sist2Icon></Sist2Icon>
+    </b-navbar-brand>
+
+    <b-button class="ml-auto" to="/task" variant="link">{{ $t("tasks") }}</b-button>
+  </b-navbar>
+</template>
+
+<script>
+import Sist2Icon from "@/components/icons/Sist2Icon";
+
+export default {
+  name: "NavBar",
+  components: {Sist2Icon},
+  methods: {
+    tagline() {
+      return this.$store.state.sist2Info.tagline;
+    },
+    sist2Version() {
+      return this.$store.state.sist2Info.version;
+    },
+    isDebug() {
+      return this.$store.state.sist2Info.debug;
+    },
+    isLegacy() {
+      return this.$store.state.sist2Info.esVersionLegacy;
+    },
+    hideLegacy() {
+      return this.$store.state.optHideLegacy;
+    }
+  }
+}
+</script>
+
+<style scoped>
+.navbar {
+  box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
+  border-radius: 0;
+}
+
+.theme-black .navbar {
+  background: #546b7a30;
+  border-bottom: none;
+}
+
+.navbar-brand {
+  color: #222 !important;
+  font-size: 1.75rem;
+  padding: 0;
+}
+
+.navbar-brand:hover {
+  color: #000 !important;
+}
+
+.version {
+  color: #222 !important;
+  margin-left: -18px;
+  margin-top: -14px;
+  font-size: 11px;
+  font-family: monospace;
+}
+
+.btn-link {
+  color: #222;
+}
+</style>

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -0,0 +1,111 @@
+<template>
+  <div>
+    <label>{{ $t("scanOptions.path") }}</label>
+    <b-form-input v-model="options.path" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.threads") }}</label>
+    <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.memThrottle") }}</label>
+    <b-form-input type="number" min="0" v-model="options.mem_throttle" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
+    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailCount") }}</label>
+    <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailSize") }}</label>
+    <b-form-input type="number" min="100" v-model="options.thumbnail_size" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.contentSize") }}</label>
+    <b-form-input type="number" min="0" v-model="options.content_size" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.rewriteUrl") }}</label>
+    <b-form-input v-model="options.rewrite_url" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.depth") }}</label>
+    <b-form-input type="number" min="0" v-model="options.depth" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.archive") }}</label>
+    <b-form-select :options="['skip', 'list', 'shallow', 'recurse']" v-model="options.archive"
+                   @change="update()"></b-form-select>
+
+    <label>{{ $t("scanOptions.archivePassphrase") }}</label>
+    <b-form-input v-model="options.archive_passphrase" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.ocrLang") }}</label>
+    <b-alert variant="danger" show v-if="selectedOcrLangs.length === 0 && !disableOcrLang">{{ $t("scanOptions.ocrLangAlert") }}</b-alert>
+    <b-checkbox-group :disabled="disableOcrLang" v-model="selectedOcrLangs" @input="onOcrLangChange">
+      <b-checkbox v-for="lang in ocrLangs" :key="lang" :value="lang">{{ lang }}</b-checkbox>
+    </b-checkbox-group>
+
+    <!--    <b-form-input readonly v-model="options.ocr_lang" @change="update()"></b-form-input>-->
+
+    <div style="height: 10px"></div>
+
+    <b-form-checkbox v-model="options.ocr_images" @change="update()">
+      {{ $t("scanOptions.ocrImages") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.ocr_ebooks" @change="update()">
+      {{ $t("scanOptions.ocrEbooks") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("scanOptions.exclude") }}</label>
+    <b-form-input v-model="options.exclude" @change="update()"
+                  :placeholder="$t('scanOptions.excludePlaceholder')"></b-form-input>
+
+    <div style="height: 10px"></div>
+
+    <b-form-checkbox v-model="options.fast" @change="update()">
+      {{ $t("scanOptions.fast") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.checksums" @change="update()">
+      {{ $t("scanOptions.checksums") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.read_subtitles" @change="update()">
+      {{ $t("scanOptions.readSubtitles") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("scanOptions.memBuffer") }}</label>
+    <b-form-input type="number" min="0" v-model="options.mem_buffer" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.treemapThreshold") }}</label>
+    <b-form-input type="number" min="0" v-model="options.treemap_threshold" @change="update()"></b-form-input>
+  </div>
+</template>
+
+<script>
+
+export default {
+  name: "ScanOptions",
+  props: ["options"],
+  data() {
+    return {
+      disableOcrLang: false,
+      selectedOcrLangs: []
+    }
+  },
+  computed: {
+    ocrLangs() {
+      return this.$store.state.sist2AdminInfo?.tesseract_langs || [];
+    }
+  },
+  methods: {
+    onOcrLangChange() {
+      this.options.ocr_lang = this.selectedOcrLangs.join("+");
+    },
+    update() {
+      this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;
+      this.$emit("change", this.options);
+    },
+  },
+  mounted() {
+    this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;
+    this.selectedOcrLangs = this.options.ocr_lang ? this.options.ocr_lang.split("+") : [];
+  }
+}
+</script>

sist2-admin/frontend/src/components/TaskListItem.vue

@@ -0,0 +1,57 @@
+<template>
+  <b-list-group-item>
+    <b-row style="height: 50px">
+      <b-col><h5>{{ task.display_name }}</h5></b-col>
+      <b-col class="shrink">
+        <router-link class="btn btn-link" :to="`/log/${task.id}`">{{ $t("logs") }}</router-link>
+      </b-col>
+      <b-col class="shrink">
+        <b-btn variant="link" @click="killTask(task.id)">{{ $t("kill") }}</b-btn>
+      </b-col>
+    </b-row>
+
+    <b-row>
+      <b-col>
+        <b-progress :max="task.progress.count">
+          <b-progress-bar :value="task.progress.done" :label-html="label" :striped="!task.progress.waiting"/>
+        </b-progress>
+      </b-col>
+    </b-row>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "TaskListItem",
+  props: ["task"],
+  computed: {
+    label() {
+
+      const count = this.task.progress.count;
+      const done = this.task.progress.done;
+
+      return `<span>${done}/${count}</span>`
+    }
+  },
+  methods: {
+    killTask(taskId) {
+      sist2AdminApi.killTask(taskId).then(() => {
+        this.$bvToast.toast(this.$t("killConfirmation"), {
+          title: this.$t("killConfirmationTitle"),
+          variant: "success",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    }
+  }
+}
+</script>
+
+<style scoped>
+.shrink {
+  flex-grow: inherit;
+}
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -0,0 +1,75 @@
+<template>
+  <div>
+    <label>{{ $t("webOptions.esUrl") }}</label>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("webOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.lang") }}</label>
+    <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
+
+    <label>{{ $t("webOptions.bind") }}</label>
+    <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.tagline") }}</label>
+    <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+
+    <label>{{ $t("webOptions.auth") }}</label>
+    <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.tagAuth") }}</label>
+    <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+
+  </div>
+</template>
+
+<script>
+
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "WebOptions",
+  props: ["options", "frontendName"],
+  data() {
+    return {
+      showEsTestAlert: false,
+      esTestOk: false,
+      esTestMessage: "",
+    }
+  },
+  methods: {
+    update() {
+      if (!this.options.es_url.startsWith("https")) {
+        this.options.es_insecure_ssl = false;
+      }
+
+      this.$emit("change", this.options);
+    },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/i18n/messages.js

@@ -0,0 +1,110 @@
+export default {
+    en: {
+        start: "Start",
+        stop: "Stop",
+        go: "Go",
+        online: "online",
+        offline: "offline",
+        delete: "Delete",
+        runNow: "Index now",
+        create: "Create",
+        test: "Test",
+
+        jobTitle: "job configuration",
+        tasks: "Tasks",
+        runningTasks: "Running tasks",
+        frontends: "Frontends",
+        jobDisabled: "There is no valid index for this job",
+        status: "Status",
+
+        taskHistory: "Task history",
+        taskName: "Task name",
+        taskStarted: "Started",
+        taskDuration: "Duration",
+        taskStatus: "Status",
+        logs: "Logs",
+        kill: "Kill",
+        killConfirmation: "SIGTERM signal sent to sist2 process",
+        killConfirmationTitle: "Confirmation",
+        follow: "Follow",
+        wholeFile: "Whole file",
+        logLevel: "Log level",
+        logMode: "Follow mode",
+        logFile: "Reading log file",
+
+        jobs: "Jobs",
+        newJobName: "New job name",
+        newJobHelp: "Create a new job to get started!",
+        newFrontendName: "New frontend name",
+        scanned: "last scan",
+        autoStart: "Start automatically",
+
+        runJobConfirmationTitle: "Task queued",
+        runJobConfirmation: "Check the Tasks page to monitor the status.",
+
+        monitoring: "Monitoring",
+        enableMonitoring: "Enable monitoring",
+        extraQueryArgs: "Extra query arguments when launching from sist2-admin",
+        customUrl: "Custom URL when launching from sist2-admin",
+
+        selectJobs: "Select jobs",
+        webOptions: {
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            lang: "UI Language",
+            bind: "Listen address",
+            tagline: "Tagline in navbar",
+            auth: "Basic auth in user:password format",
+            tagAuth: "Basic auth in user:password format for tagging",
+        },
+        scanOptions: {
+            title: "Scanning options",
+            path: "Path",
+            threads: "Number of threads",
+            memThrottle: "Total memory threshold in MiB for scan throttling",
+            thumbnailQuality: "Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best",
+            thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
+            thumbnailSize: "Thumbnail size, in pixels",
+            contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",
+            rewriteUrl: "Serve files from this url instead of from disk",
+            depth: "Scan up to this many subdirectories deep",
+            archive: "Archive file mode",
+            archivePassphrase: "Passphrase for encrypted archive files",
+            ocrLang: "Tesseract language",
+            ocrLangAlert: "You must select at least one language",
+            ocrEbooks: "Enable OCR'ing of ebook files",
+            ocrImages: "Enable OCR'ing of image files",
+            exclude: "Files that match this regex will not be scanned",
+            excludePlaceholder: "Exclude",
+            fast: "Only index file names & mime type",
+            checksums: "Calculate file checksums when scanning",
+            readSubtitles: "Read subtitles from media files",
+            memBuffer: "Maximum memory buffer size per thread in MiB for files inside archives",
+            treemapThreshold: "Relative size threshold for treemap"
+        },
+        indexOptions: {
+            title: "Indexing options",
+            threads: "Number of threads",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            batchSize: "Index batch size",
+            script: "User script"
+        },
+        jobOptions: {
+            title: "Job options",
+            cron: "Job schedule",
+            scheduleEnabled: "Enable scheduled re-scan",
+            noJobAvailable: "No jobs available.",
+            desktopNotifications: "Desktop notifications"
+        },
+        frontendOptions: {
+            title: "Frontend options",
+            noJobSelectedWarning: "You must select at least one job to start this frontend"
+        },
+        notifications: {
+            indexCompleted: "Task completed for [$JOB$]"
+        }
+    }
+}

sist2-admin/frontend/src/main.js

@@ -0,0 +1,31 @@
+import Vue from 'vue'
+
+import { BootstrapVue, IconsPlugin } from 'bootstrap-vue'
+
+import "bootstrap/dist/css/bootstrap.min.css"
+import "bootstrap-vue/dist/bootstrap-vue.min.css"
+
+Vue.use(BootstrapVue);
+Vue.use(IconsPlugin);
+
+import App from './App.vue';
+import router from './router';
+import store from './store';
+import VueI18n from "vue-i18n";
+import messages from "@/i18n/messages";
+
+Vue.use(VueI18n);
+
+const i18n = new VueI18n({
+  locale: "en",
+  messages: messages
+});
+
+Vue.config.productionTip = false
+
+new Vue({
+  router,
+  store,
+  i18n,
+  render: h => h(App)
+}).$mount('#app')

sist2-admin/frontend/src/router/index.js

@@ -0,0 +1,45 @@
+import Vue from 'vue'
+import VueRouter from 'vue-router'
+import Home from '../views/Home.vue'
+import Job from "@/views/Job";
+import Tasks from "@/views/Tasks";
+import Frontend from "@/views/Frontend";
+import Tail from "@/views/Tail";
+
+Vue.use(VueRouter);
+
+const routes = [
+  {
+    path: "/",
+    name: "Home",
+    component: Home
+  },
+  {
+    path: "/job/:name",
+    name: "Job",
+    component: Job
+  },
+  {
+    path: "/task/",
+    name: "Tasks",
+    component: Tasks
+  },
+  {
+    path: "/frontend/:name",
+    name: "Frontend",
+    component: Frontend
+  },
+  {
+    path: "/log/:taskId",
+    name: "Tail",
+    component: Tail
+  },
+]
+
+const router = new VueRouter({
+  mode: "hash",
+  base: process.env.BASE_URL,
+  routes
+})
+
+export default router

sist2-admin/frontend/src/store/index.js

@@ -0,0 +1,63 @@
+import Vue from "vue";
+import Vuex from "vuex";
+
+Vue.use(Vuex);
+
+function saveBrowserSettings(state) {
+    const settings = {
+        jobDesktopNotificationMap: state.jobDesktopNotificationMap
+    };
+    localStorage.setItem("sist2-admin-settings", JSON.stringify(settings));
+
+    console.log("SAVED");
+    console.log(settings);
+}
+
+export default new Vuex.Store({
+    state: {
+        sist2AdminInfo: null,
+        jobDesktopNotificationMap: {}
+    },
+    mutations: {
+        setSist2AdminInfo: (state, payload) => state.sist2AdminInfo = payload,
+        setJobDesktopNotificationMap: (state, payload) => state.jobDesktopNotificationMap = payload,
+    },
+    actions: {
+        notify: async ({state}, notification) => {
+
+            if (!state.jobDesktopNotificationMap[notification.job]) {
+                console.log("pass");
+                return;
+            }
+
+            new Notification(notification.messageString.replace("$JOB$", notification.job));
+        },
+        setJobDesktopNotification: async ({state}, {job, enabled}) => {
+
+            if (enabled === true) {
+                const permission = await Notification.requestPermission()
+
+                if (permission !== "granted") {
+                    return false;
+                }
+            }
+
+            state.jobDesktopNotificationMap[job] = enabled;
+            saveBrowserSettings(state);
+
+            return true;
+        },
+        loadBrowserSettings({commit}) {
+            const settingString = localStorage.getItem("sist2-admin-settings");
+
+            if (!settingString) {
+                return;
+            }
+
+            const settings = JSON.parse(settingString);
+
+            commit("setJobDesktopNotificationMap", settings["jobDesktopNotificationMap"]);
+        }
+    },
+    modules: {}
+})

sist2-admin/frontend/src/util.js

@@ -0,0 +1,8 @@
+
+export function formatBindAddress(address) {
+    if (address.startsWith("0.0.0.0")) {
+        return address.slice("0.0.0.0".length)
+    }
+
+    return address
+}

sist2-admin/frontend/src/views/Frontend.vue

@@ -0,0 +1,133 @@
+<template>
+  <b-card>
+    <b-card-title>
+      {{ name }}
+      <small style="vertical-align: top">
+        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+      </small>
+    </b-card-title>
+
+    <div class="mb-3" v-if="!loading">
+      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+          $t("start")
+        }}
+      </b-button>
+      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+          $t("stop")
+        }}
+      </b-button>
+      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+        {{ $t("go") }}
+      </b-button>
+      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+    </div>
+
+
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card-body v-else>
+
+      <h4>{{ $t("frontendOptions.title") }}</h4>
+      <b-card>
+        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+          {{ $t("autoStart") }}
+        </b-form-checkbox>
+
+        <b-form-checkbox v-model="frontend.enable_monitoring" @change="update()">
+          {{ $t("enableMonitoring") }}
+        </b-form-checkbox>
+
+        <label>{{ $t("extraQueryArgs") }}</label>
+        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+
+        <label>{{ $t("customUrl") }}</label>
+        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+
+        <br/>
+
+        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+
+        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("jobOptions.title") }}</h4>
+      <b-card>
+        <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
+      </b-card>
+    </b-card-body>
+
+  </b-card>
+</template>
+
+<script>
+
+import Sist2AdminApi from "@/Sist2AdminApi";
+import JobCheckboxGroup from "@/components/JobCheckboxGroup";
+import WebOptions from "@/components/WebOptions";
+
+export default {
+  name: 'Frontend',
+  components: {JobCheckboxGroup, WebOptions},
+  data() {
+    return {
+      loading: true,
+      frontend: null,
+    }
+  },
+  computed: {
+    valid() {
+      return !this.loading && this.frontend.jobs.length > 0;
+    },
+    frontendUrl() {
+      if (this.frontend.custom_url) {
+        return this.frontend.custom_url + this.args;
+      }
+
+      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+      }
+
+      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+    },
+    name() {
+      return this.$route.params.name;
+    },
+    port() {
+      return this.frontend.web_options.bind.split(":")[1]
+    },
+    args() {
+      const args = this.frontend.extra_query_args;
+      if (args !== "") {
+        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+      }
+      return "";
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getFrontend(this.name).then(resp => {
+      this.frontend = resp.data;
+      this.loading = false;
+    });
+  },
+  methods: {
+    start() {
+      this.frontend.running = true;
+      Sist2AdminApi.startFrontend(this.name)
+    },
+    stop() {
+      this.frontend.running = false;
+      Sist2AdminApi.stopFrontend(this.name)
+    },
+    deleteFrontend() {
+      Sist2AdminApi.deleteFrontend(this.name).then(() => {
+        this.$router.push("/frontends");
+      });
+    },
+    update() {
+      Sist2AdminApi.updateFrontend(this.name, this.frontend);
+    },
+  }
+}
+</script>

sist2-admin/frontend/src/views/Home.vue

@@ -0,0 +1,122 @@
+<template>
+  <div>
+    <b-card>
+      <b-card-title>{{ $t("jobs") }}</b-card-title>
+      <b-row>
+        <b-col>
+          <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
+          <b-popover
+              :show.sync="showHelp"
+              target="new-job"
+              placement="top"
+              triggers="manual"
+              variant="primary"
+              :content="$t('newJobHelp')"
+          ></b-popover>
+        </b-col>
+        <b-col>
+          <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
+          </b-button>
+        </b-col>
+      </b-row>
+
+      <hr/>
+
+      <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
+      <b-list-group v-else>
+        <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
+      </b-list-group>
+    </b-card>
+
+    <br/>
+
+    <b-card>
+
+      <b-card-title>{{ $t("frontends") }}</b-card-title>
+
+      <b-row>
+        <b-col>
+          <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
+        </b-col>
+        <b-col>
+          <b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
+            {{ $t("create") }}
+          </b-button>
+        </b-col>
+      </b-row>
+
+      <hr/>
+
+      <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
+      <b-list-group v-else>
+        <FrontendListItem v-for="frontend in frontends"
+                          :key="frontend.name" :frontend="frontend"></FrontendListItem>
+      </b-list-group>
+
+    </b-card>
+  </div>
+</template>
+
+<script>
+import JobListItem from "@/components/JobListItem";
+import {formatBindAddress} from "@/util";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import FrontendListItem from "@/components/FrontendListItem";
+
+export default {
+  name: "Jobs",
+  components: {JobListItem, FrontendListItem},
+  data() {
+    return {
+      jobsLoading: true,
+      newJobName: "",
+      jobs: [],
+
+      frontendsLoading: true,
+      frontends: [],
+      formatBindAddress,
+      newFrontendName: "",
+
+      showHelp: false
+    }
+  },
+  mounted() {
+    this.loading = true;
+    this.reload();
+  },
+  methods: {
+    jobNameValid(name) {
+      if (this.jobs.some(job => job.name === name)) {
+        return false;
+      }
+
+      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+    },
+    frontendNameValid(name) {
+      if (this.frontends.some(job => job.name === name)) {
+        return false;
+      }
+
+      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+    },
+    reload() {
+      Sist2AdminApi.getJobs().then(resp => {
+        this.jobs = resp.data;
+        this.jobsLoading = false;
+
+        this.showHelp = this.jobs.length === 0;
+      });
+      Sist2AdminApi.getFrontends().then(resp => {
+        this.frontends = resp.data;
+        this.frontendsLoading = false;
+      });
+    },
+    createJob() {
+      Sist2AdminApi.createJob(this.newJobName).then(this.reload);
+    },
+    createFrontend() {
+      Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/views/Job.vue

@@ -0,0 +1,92 @@
+<template>
+  <b-card>
+    <b-card-title>
+      [{{ getName() }}]
+      {{ $t("jobTitle") }}
+    </b-card-title>
+
+    <div class="mb-3">
+      <b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
+      <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
+    </div>
+
+    <div v-if="job">
+      {{ $t("status") }}: <code>{{ job.status }}</code>
+    </div>
+
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card-body v-else>
+
+      <h4>{{ $t("jobOptions.title") }}</h4>
+      <b-card>
+        <JobOptions :job="job" @change="update"></JobOptions>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("scanOptions.title") }}</h4>
+      <b-card>
+        <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("indexOptions.title") }}</h4>
+      <b-card>
+        <IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
+      </b-card>
+
+    </b-card-body>
+
+  </b-card>
+</template>
+
+<script>
+import ScanOptions from "@/components/ScanOptions";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import IndexOptions from "@/components/IndexOptions";
+import JobOptions from "@/components/JobOptions";
+
+export default {
+  name: "Job",
+  components: {
+    IndexOptions,
+    ScanOptions,
+    JobOptions
+  },
+  data() {
+    return {
+      loading: true,
+      job: null
+    }
+  },
+  methods: {
+    getName() {
+      return this.$route.params.name;
+    },
+    update() {
+      Sist2AdminApi.updateJob(this.getName(), this.job);
+    },
+    runJob() {
+      Sist2AdminApi.runJob(this.getName()).then(() => {
+        this.$bvToast.toast(this.$t("runJobConfirmation"), {
+          title: this.$t("runJobConfirmationTitle"),
+          variant: "success",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    },
+    deleteJob() {
+      Sist2AdminApi.deleteJob(this.getName()).then(() => {
+        this.$router.push("/");
+      })
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getJob(this.getName()).then(resp => {
+      this.loading = false;
+      this.job = resp.data;
+    })
+  }
+}
+</script>

sist2-admin/frontend/src/views/Tail.vue

@@ -0,0 +1,168 @@
+<template>
+  <b-card>
+    <b-card-body>
+
+      <h4 class="mb-3">{{ taskId }} {{ $t("logs") }}</h4>
+
+      <div v-if="$store.state.sist2AdminInfo">
+        {{ $t("logFile") }}
+        <code>{{ $store.state.sist2AdminInfo.logs_folder }}/sist2-{{ taskId }}.log</code>
+        <br/>
+        <br/>
+      </div>
+
+      <b-row>
+        <b-col>
+          <span>{{ $t("logLevel") }}</span>
+          <b-select :options="levels.slice(0, -1)" v-model="logLevel" @input="connect()"></b-select>
+        </b-col>
+        <b-col>
+          <span>{{ $t("logMode") }}</span>
+          <b-select :options="modeOptions" v-model="mode" @input="connect()"></b-select>
+        </b-col>
+      </b-row>
+
+      <div id="log-tail-output" class="mt-3 ml-1"></div>
+
+    </b-card-body>
+  </b-card>
+</template>
+
+<script>
+
+export default {
+  name: "Tail",
+  data() {
+    return {
+      logLevel: "DEBUG",
+      levels: ["DEBUG", "INFO", "WARNING", "ERROR", "ADMIN", "FATAL"],
+      socket: null,
+      mode: "follow",
+      modeOptions: [
+        {
+          "text": this.$t('follow'),
+          "value": "follow"
+        },
+        {
+          "text": this.$t('wholeFile'),
+          "value": "wholeFile"
+        }
+      ]
+    }
+  },
+  computed: {
+    taskId: function () {
+      return this.$route.params.taskId;
+    }
+  },
+  methods: {
+    connect() {
+      let lineCount = 0;
+      const outputElem = document.getElementById("log-tail-output")
+      outputElem.replaceChildren();
+      if (this.socket !== null) {
+        this.socket.close();
+      }
+
+      const n = this.mode === "follow" ? 32 : 9999999999;
+      this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
+      this.socket.onopen = () => {
+        this.socket.send("Hello from client");
+      }
+
+      this.socket.onmessage = e => {
+        let message;
+        try {
+          message = JSON.parse(e.data);
+        } catch {
+          console.error(e.data)
+          return;
+        }
+
+        if ("ping" in message) {
+          return;
+        }
+
+        if (message.level === undefined) {
+
+          if ("stderr" in message) {
+            message.level = "ERROR";
+            message.message = message["stderr"];
+          } else {
+            message.level = "ADMIN";
+            message.message = message["sist2-admin"];
+          }
+          message.datetime = ""
+          message.filepath = ""
+        }
+
+        if (this.levels.indexOf(message.level) < this.levels.indexOf(this.logLevel)) {
+          return;
+        }
+
+        const logLine = `${message.datetime} [${message.level} ${message.filepath}] ${message.message}`;
+
+        const span = document.createElement("span");
+        span.setAttribute("class", message.level);
+        span.appendChild(document.createTextNode(logLine));
+
+        outputElem.appendChild(span);
+        lineCount += 1;
+
+        if (this.mode === "follow" && lineCount >= n) {
+          outputElem.firstChild.remove();
+        }
+      }
+    }
+  },
+  mounted() {
+    this.connect()
+  }
+}
+
+</script>
+
+<style>
+#log-tail-output span {
+  display: block;
+}
+
+span.DEBUG {
+  color: #9E9E9E;
+}
+
+span.WARNING {
+  color: #FFB300;
+}
+
+span.INFO {
+  color: #039BE5;
+}
+
+span.ERROR {
+  color: #F4511E;
+}
+
+span.FATAL {
+  color: #F4511E;
+}
+
+span.ADMIN {
+  color: #ee05ff;
+}
+
+
+#log-tail-output {
+  font-size: 13px;
+  font-family: monospace;
+
+  padding: 6px;
+  background-color: #f5f5f5;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  margin: 3px;
+  white-space: pre;
+  color: #000;
+  overflow: hidden;
+}
+</style>

sist2-admin/frontend/src/views/Tasks.vue

@@ -0,0 +1,124 @@
+<template>
+  <div>
+
+    <b-card v-if="tasks.length > 0">
+      <h2>{{ $t("runningTasks") }}</h2>
+      <b-list-group>
+        <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
+      </b-list-group>
+    </b-card>
+
+    <b-card class="mt-4">
+
+      <b-card-title>{{ $t("taskHistory") }}</b-card-title>
+
+      <br/>
+
+      <b-table
+          id="task-history"
+          :items="historyItems"
+          :fields="historyFields"
+          :current-page="historyCurrentPage"
+          :tbody-tr-class="rowClass"
+          :per-page="10"
+      >
+        <template #cell(logs)="data">
+          <router-link :to="`/log/${data.item.logs}`">{{ $t("logs") }}</router-link>
+        </template>
+
+      </b-table>
+
+      <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
+                    :per-page="10"></b-pagination>
+
+    </b-card>
+  </div>
+</template>
+
+<script>
+import TaskListItem from "@/components/TaskListItem";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import moment from "moment";
+
+export default {
+  name: 'Tasks',
+  components: {TaskListItem},
+  data() {
+    return {
+      loading: true,
+      tasks: [],
+      taskHistory: [],
+      timerId: null,
+      historyFields: [
+        {key: "name", label: this.$t("taskName")},
+        {key: "time", label: this.$t("taskStarted")},
+        {key: "duration", label: this.$t("taskDuration")},
+        {key: "status", label: this.$t("taskStatus")},
+        {key: "logs", label: this.$t("logs")},
+      ],
+      historyCurrentPage: 1,
+      historyItems: []
+    }
+  },
+  props: {
+    msg: String
+  },
+  mounted() {
+    this.loading = true;
+    this.update().then(() => this.loading = false);
+
+    this.timerId = window.setInterval(this.update, 1000);
+    this.updateHistory();
+  },
+  destroyed() {
+    if (this.timerId) {
+      window.clearInterval(this.timerId);
+    }
+  },
+  methods: {
+    rowClass(row) {
+      if (row.status === "failed") {
+        return "table-danger";
+      }
+      return null;
+    },
+    updateHistory() {
+      Sist2AdminApi.getTaskHistory().then(resp => {
+        this.historyItems = resp.data.map(row => ({
+          id: row.id,
+          name: row.name,
+          duration: this.taskDuration(row),
+          time: moment(row.started).format("dd, MMM Do YYYY, HH:mm:ss"),
+          logs: row.id,
+          status: row.return_code === 0 ? "ok" : "failed"
+        }));
+      });
+    },
+    update() {
+      return Sist2AdminApi.getTasks().then(resp => {
+        this.tasks = resp.data;
+      })
+    },
+    taskDuration(task) {
+      const start = moment(task.started);
+      const end = moment(task.ended);
+
+      let duration = moment.utc(end.diff(start)).format("HH[h] mm[m] ss[s]");
+
+      duration = duration.replace("00h ", "");
+      duration = duration.replace(/^00m /, "");
+      duration = duration.replace(/00s/, "<1s");
+      duration = duration.replace(/^0/, "");
+
+      return duration;
+    }
+  }
+}
+</script>
+
+<style scoped>
+#task-history {
+  font-family: monospace;
+  font-size: 12px;
+}
+</style>

sist2-admin/frontend/vue.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+    publicPath: ""
+};

sist2-admin/requirements.txt

@@ -0,0 +1,5 @@
+fastapi
+git+https://github.com/simon987/hexlib.git
+uvicorn
+websockets
+pycron

sist2-admin/sist2_admin/app.py

@@ -0,0 +1,392 @@
+import asyncio
+import os
+import signal
+from datetime import datetime
+from urllib.parse import urlparse
+
+import requests
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from hexlib.db import PersistentState
+from requests import ConnectionError
+from requests.exceptions import SSLError
+from starlette.middleware.cors import CORSMiddleware
+from starlette.responses import RedirectResponse
+from starlette.staticfiles import StaticFiles
+from starlette.websockets import WebSocket
+from websockets.exceptions import ConnectionClosed
+
+import cron
+from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
+from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
+from notifications import Subscribe, Notifications
+from sist2 import Sist2
+from state import PickleTable, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
+from web import Sist2Frontend
+
+VERSION = "1.0"
+
+sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
+db = PersistentState(table_factory=PickleTable, dbfile=os.path.join(DATA_FOLDER, "state.db"))
+notifications = Notifications()
+task_queue = TaskQueue(sist2, db, notifications)
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_credentials=True,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.mount("/ui/", StaticFiles(directory="./frontend/dist", html=True), name="static")
+
+
+@app.get("/")
+async def home():
+    return RedirectResponse("ui")
+
+
+@app.get("/api")
+async def api():
+    return {
+        "version": VERSION,
+        "tesseract_langs": TESSERACT_LANGS,
+        "logs_folder": LOG_FOLDER
+    }
+
+
+@app.get("/api/job/{name:str}")
+async def get_job(name: str):
+    row = db["jobs"][name]
+    if row:
+        return row["job"]
+    raise HTTPException(status_code=404)
+
+
+@app.get("/api/frontend/{name:str}")
+async def get_frontend(name: str):
+    row = db["frontends"][name]
+    if row:
+        frontend = row["frontend"]
+        frontend: Sist2Frontend
+        frontend.running = frontend.name in RUNNING_FRONTENDS
+        return frontend
+    raise HTTPException(status_code=404)
+
+
+@app.get("/api/job/")
+async def get_jobs():
+    return [row["job"] for row in db["jobs"]]
+
+
+@app.put("/api/job/{name:str}")
+async def update_job(name: str, job: Sist2Job):
+    # TODO: Check etag
+
+    job.last_modified = datetime.now()
+    row = db["jobs"][name]
+    if not row:
+        raise HTTPException(status_code=404)
+
+    args_that_trigger_full_scan = [
+        "path",
+        "thumbnail_count",
+        "thumbnail_quality",
+        "thumbnail_size",
+        "content_size",
+        "depth",
+        "archive",
+        "archive_passphrase",
+        "ocr_lang",
+        "ocr_images",
+        "ocr_ebooks",
+        "fast",
+        "checksums",
+        "read_subtitles",
+    ]
+    for arg in args_that_trigger_full_scan:
+        if getattr(row["job"].scan_options, arg) != getattr(job.scan_options, arg):
+            job.do_full_scan = True
+
+    db["jobs"][name] = {"job": job}
+
+
+@app.put("/api/frontend/{name:str}")
+async def update_frontend(name: str, frontend: Sist2Frontend):
+    db["frontends"][name] = {"frontend": frontend}
+
+    # TODO: Check etag
+
+    return "ok"
+
+
+@app.get("/api/task/")
+async def get_tasks():
+    return list(map(lambda t: t.json(), task_queue.tasks()))
+
+
+@app.get("/api/task/history")
+async def task_history():
+    return list(db["task_done"].sql("ORDER BY started DESC"))
+
+
+@app.post("/api/task/{task_id:str}/kill")
+async def kill_job(task_id: str):
+    return task_queue.kill_task(task_id)
+
+
+def _run_job(job: Sist2Job):
+    job.last_modified = datetime.now()
+    if job.status == JobStatus("created"):
+        job.status = JobStatus("started")
+    db["jobs"][job.name] = {"job": job}
+
+    scan_task = Sist2ScanTask(job, f"Scan [{job.name}]")
+    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=scan_task)
+
+    task_queue.submit(scan_task)
+    task_queue.submit(index_task)
+
+
+@app.get("/api/job/{name:str}/run")
+async def run_job(name: str):
+    row = db["jobs"][name]
+    if not row:
+        raise HTTPException(status_code=404)
+
+    _run_job(row["job"])
+
+    return "ok"
+
+
+@app.delete("/api/job/{name:str}")
+async def delete_job(name: str):
+    row = db["jobs"][name]
+    if row:
+        del db["jobs"][name]
+    else:
+        raise HTTPException(status_code=404)
+
+
+@app.delete("/api/frontend/{name:str}")
+async def delete_frontend(name: str):
+    if name in RUNNING_FRONTENDS:
+        os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        del RUNNING_FRONTENDS[name]
+
+    row = db["frontends"][name]
+    if row:
+        del db["frontends"][name]
+    else:
+        raise HTTPException(status_code=404)
+
+
+@app.post("/api/job/{name:str}")
+async def create_job(name: str):
+    if db["jobs"][name]:
+        raise ValueError("Job with the same name already exists")
+
+    job = Sist2Job.create_default(name)
+    db["jobs"][name] = {"job": job}
+
+    return job
+
+
+@app.post("/api/frontend/{name:str}")
+async def create_frontend(name: str):
+    if db["frontend"][name]:
+        raise ValueError("Frontend with the same name already exists")
+
+    frontend = Sist2Frontend.create_default(name)
+    db["frontends"][name] = {"frontend": frontend}
+
+    return frontend
+
+
+@app.get("/api/ping_es")
+async def ping_es(url: str, insecure: bool):
+    return check_es_version(url, insecure)
+
+
+def check_es_version(es_url: str, insecure: bool):
+    try:
+        url = urlparse(es_url)
+        if url.username:
+            auth = (url.username, url.password)
+            es_url = f"{url.scheme}://{url.hostname}:{url.port}"
+        else:
+            auth = None
+        r = requests.get(es_url, verify=insecure, auth=auth)
+    except SSLError:
+        return {
+            "ok": False,
+            "message": "Invalid SSL certificate"
+        }
+    except ConnectionError as e:
+        return {
+            "ok": False,
+            "message": "Connection refused"
+        }
+    except ValueError as e:
+        return {
+            "ok": False,
+            "message": str(e)
+        }
+
+    if r.status_code == 401:
+        return {
+            "ok": False,
+            "message": "Authentication failure"
+        }
+
+    try:
+        return {
+            "ok": True,
+            "message": "Elasticsearch version " + r.json()["version"]["number"]
+        }
+    except:
+        return {
+            "ok": False,
+            "message": "Could not read version"
+        }
+
+
+def start_frontend_(frontend: Sist2Frontend):
+    frontend.web_options.indices = list(map(lambda j: db["jobs"][j]["job"].last_index, frontend.jobs))
+
+    pid = sist2.web(frontend.web_options, frontend.name)
+    RUNNING_FRONTENDS[frontend.name] = pid
+
+
+@app.post("/api/frontend/{name:str}/start")
+async def start_frontend(name: str):
+    row = db["frontends"][name]
+    if not row:
+        raise HTTPException(status_code=404)
+
+    start_frontend_(row["frontend"])
+
+
+@app.post("/api/frontend/{name:str}/stop")
+async def stop_frontend(name: str):
+    if name in RUNNING_FRONTENDS:
+        os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        del RUNNING_FRONTENDS[name]
+
+
+@app.get("/api/frontend/")
+async def get_frontends():
+    res = []
+    for row in db["frontends"]:
+        frontend = row["frontend"]
+        frontend: Sist2Frontend
+        frontend.running = frontend.name in RUNNING_FRONTENDS
+        res.append(frontend)
+    return res
+
+
+def tail(filepath: str, n: int):
+    with open(filepath) as file:
+
+        reached_eof = False
+        buffer = []
+
+        line = ""
+        while True:
+            tmp = file.readline()
+            if tmp:
+                line += tmp
+
+                if line.endswith("\n"):
+
+                    if reached_eof:
+                        yield line
+                    else:
+                        if len(buffer) > n:
+                            buffer.pop(0)
+                        buffer.append(line)
+                line = ""
+            else:
+                if not reached_eof:
+                    reached_eof = True
+                    yield from buffer
+                yield None
+
+
+@app.websocket("/notifications")
+async def ws_tail_log(websocket: WebSocket):
+    await websocket.accept()
+
+    try:
+        await websocket.receive_text()
+
+        async with Subscribe(notifications) as ob:
+            async for notification in ob.notifications():
+                await websocket.send_json(notification)
+                print(notification)
+
+    except ConnectionClosed:
+        return
+
+
+@app.websocket("/log/{task_id}")
+async def ws_tail_log(websocket: WebSocket, task_id: str, n: int):
+    log_file = os.path.join(LOG_FOLDER, f"sist2-{task_id}.log")
+
+    await websocket.accept()
+
+    try:
+        await websocket.receive_text()
+    except ConnectionClosed:
+        return
+
+    while True:
+        for line in tail(log_file, n):
+
+            try:
+                if line:
+                    await websocket.send_text(line)
+                else:
+                    await websocket.send_json({"ping": ""})
+                    await asyncio.sleep(0.1)
+            except ConnectionClosed:
+                return
+
+
+def main():
+    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0")
+
+
+def initialize_db():
+    db["sist2_admin"]["info"] = {"version": DB_SCHEMA_VERSION}
+
+    frontend = Sist2Frontend.create_default("default")
+    db["frontends"]["default"] = {"frontend": frontend}
+
+    logger.info("Initialized database.")
+
+
+def start_frontends():
+    for row in db["frontends"]:
+        frontend: Sist2Frontend = row["frontend"]
+        if frontend.auto_start and len(frontend.jobs) > 0:
+            start_frontend_(frontend)
+
+
+if __name__ == '__main__':
+
+    if not db["sist2_admin"]["info"]:
+        initialize_db()
+    elif db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
+        print("Database has incompatible schema version! Delete state.db to continue.")
+        exit(-1)
+
+    start_frontends()
+    cron.initialize(db, _run_job)
+
+    logger.info("Started sist2-admin. Hello!")
+
+    main()

sist2-admin/sist2_admin/config.py

@@ -0,0 +1,30 @@
+import os
+import logging
+import sys
+from logging import StreamHandler
+from logging.handlers import RotatingFileHandler
+
+MAX_LOG_SIZE = 1 * 1024 * 1024
+
+SIST2_BINARY = os.environ.get("SIST2_BINARY", "/root/sist2")
+DATA_FOLDER = os.environ.get("DATA_FOLDER", "/sist2-admin/")
+LOG_FOLDER = os.path.join(DATA_FOLDER, "logs")
+WEBSERVER_PORT = 8080
+
+os.makedirs(LOG_FOLDER, exist_ok=True)
+os.makedirs(DATA_FOLDER, exist_ok=True)
+
+logger = logging.Logger("sist2-admin")
+
+_log_file = os.path.join(LOG_FOLDER, "sist2-admin.log")
+_log_fmt = "%(asctime)s [%(levelname)s] %(message)s"
+_log_formatter = logging.Formatter(_log_fmt, datefmt='%Y-%m-%d %H:%M:%S')
+
+console_handler = StreamHandler(sys.stdout)
+console_handler.setFormatter(_log_formatter)
+
+file_handler = RotatingFileHandler(_log_file, mode="a", maxBytes=MAX_LOG_SIZE, backupCount=1)
+file_handler.setFormatter(_log_formatter)
+
+logger.addHandler(console_handler)
+logger.addHandler(file_handler)

sist2-admin/sist2_admin/cron.py

@@ -0,0 +1,33 @@
+from threading import Thread
+
+import pycron
+import time
+
+from hexlib.db import PersistentState
+
+from config import logger
+from jobs import Sist2Job
+
+
+def _check_schedule(db: PersistentState, run_job):
+    for job in (row["job"] for row in db["jobs"]):
+        job: Sist2Job
+
+        if job.schedule_enabled:
+            if pycron.is_now(job.cron_expression):
+                logger.info(f"Submit scan task to queue for [{job.name}]")
+                run_job(job)
+
+
+def _cron_thread(db, run_job):
+    time.sleep(60 - (time.time() % 60))
+    start = time.time()
+
+    while True:
+        _check_schedule(db, run_job)
+        time.sleep(60 - ((time.time() - start) % 60))
+
+
+def initialize(db, run_job):
+    t = Thread(target=_cron_thread, args=(db, run_job), daemon=True, name="timer")
+    t.start()

sist2-admin/sist2_admin/jobs.py

@@ -0,0 +1,315 @@
+import json
+import logging
+import os.path
+import shutil
+import signal
+import uuid
+from datetime import datetime
+from enum import Enum
+from hashlib import md5
+from logging import FileHandler
+from threading import Lock, Thread
+from time import sleep
+from uuid import uuid4, UUID
+
+from hexlib.db import PersistentState
+from pydantic import BaseModel, validator
+
+from config import logger, LOG_FOLDER
+from notifications import Notifications
+from sist2 import ScanOptions, IndexOptions, Sist2, Sist2Index
+from state import RUNNING_FRONTENDS
+from web import Sist2Frontend
+
+
+class JobStatus(Enum):
+    CREATED = "created"
+    STARTED = "started"
+    INDEXED = "indexed"
+    FAILED = "failed"
+
+
+class Sist2Job(BaseModel):
+    name: str
+    scan_options: ScanOptions
+    index_options: IndexOptions
+
+    cron_expression: str
+    schedule_enabled: bool = False
+
+    previous_index: str = None
+    last_index: str = None
+    last_index_date: datetime = None
+    status: JobStatus = JobStatus("created")
+    last_modified: datetime
+    etag: str = None
+    do_full_scan: bool = False
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @staticmethod
+    def create_default(name: str):
+        return Sist2Job(
+            name=name,
+            scan_options=ScanOptions(path="/"),
+            index_options=IndexOptions(),
+            last_modified=datetime.now(),
+            cron_expression="0 0 * * *"
+        )
+
+    @validator("etag", always=True)
+    def validate_etag(cls, value, values):
+        s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
+        return md5(s.encode()).hexdigest()
+
+
+class Sist2TaskProgress:
+
+    def __init__(self, done: int = 0, count: int = 0, index_size: int = 0, tn_size: int = 0, waiting: bool = False):
+        self.done = done
+        self.count = count
+        self.index_size = index_size
+        self.store_size = tn_size
+        self.waiting = waiting
+
+    def percent(self):
+        return (self.done / self.count) if self.count else 0
+
+
+class Sist2Task:
+
+    def __init__(self, job: Sist2Job, display_name: str, depends_on: uuid.UUID = None):
+        self.job = job
+        self.display_name = display_name
+
+        self.progress = Sist2TaskProgress()
+        self.id = uuid4()
+        self.pid = None
+        self.started = None
+        self.ended = None
+        self.depends_on = depends_on
+
+        self._logger = logging.Logger(name=f"{self.id}")
+        self._logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"sist2-{self.id}.log")))
+
+    def json(self):
+        return {
+            "id": self.id,
+            "job": self.job,
+            "display_name": self.display_name,
+            "progress": self.progress,
+            "started": self.started,
+            "ended": self.ended,
+            "depends_on": self.depends_on,
+        }
+
+    def log_callback(self, log_json):
+
+        if "progress" in log_json:
+            self.progress = Sist2TaskProgress(**log_json["progress"])
+        elif self._logger:
+            self._logger.info(json.dumps(log_json))
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        self.started = datetime.now()
+
+        logger.info(f"Started task {self.display_name}")
+
+
+class Sist2ScanTask(Sist2Task):
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        self.job.scan_options.name = self.job.name
+
+        if self.job.last_index and os.path.exists(self.job.last_index) and not self.job.do_full_scan:
+            self.job.scan_options.incremental = self.job.last_index
+        else:
+            self.job.scan_options.incremental = None
+
+        def set_pid(pid):
+            self.pid = pid
+
+        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
+        self.ended = datetime.now()
+
+        if return_code != 0:
+            self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
+            logger.info(f"Task {self.display_name} failed ({return_code})")
+        else:
+            index = Sist2Index(self.job.scan_options.output)
+
+            # Save latest index
+            self.job.previous_index = self.job.last_index
+
+            self.job.last_index = index.path
+            self.job.last_index_date = datetime.now()
+            self.job.do_full_scan = False
+            db["jobs"][self.job.name] = {"job": self.job}
+            self._logger.info(json.dumps({"sist2-admin": f"Save last_index={self.job.last_index}"}))
+
+        logger.info(f"Completed {self.display_name} ({return_code=})")
+
+        return return_code
+
+
+class Sist2IndexTask(Sist2Task):
+
+    def __init__(self, job: Sist2Job, display_name: str, depends_on: Sist2Task):
+        super().__init__(job, display_name, depends_on=depends_on.id)
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        self.job.index_options.path = self.job.scan_options.output
+
+        return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
+        self.ended = datetime.now()
+
+        duration = self.ended - self.started
+
+        ok = return_code == 0
+
+        if ok:
+            # Remove old index
+            if self.job.previous_index is not None:
+                self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index=}"}))
+                try:
+                    shutil.rmtree(self.job.previous_index)
+                except FileNotFoundError:
+                    pass
+
+            self.restart_running_frontends(db, sist2)
+
+        # Update status
+        self.job.status = JobStatus("indexed") if ok else JobStatus("failed")
+        db["jobs"][self.job.name] = {"job": self.job}
+
+        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
+
+        logger.info(f"Completed {self.display_name} ({return_code=})")
+
+        return return_code
+
+    def restart_running_frontends(self, db: PersistentState, sist2: Sist2):
+        for frontend_name, pid in RUNNING_FRONTENDS.items():
+            frontend = db["frontends"][frontend_name]["frontend"]
+            frontend: Sist2Frontend
+
+            os.kill(pid, signal.SIGTERM)
+            try:
+                os.wait()
+            except ChildProcessError:
+                pass
+
+            frontend.web_options.indices = map(lambda j: db["jobs"][j]["job"].last_index, frontend.jobs)
+
+            pid = sist2.web(frontend.web_options, frontend.name)
+            RUNNING_FRONTENDS[frontend_name] = pid
+
+            self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))
+
+
+class TaskQueue:
+    def __init__(self, sist2: Sist2, db: PersistentState, notifications: Notifications):
+        self._lock = Lock()
+
+        self._sist2 = sist2
+        self._db = db
+        self._notifications = notifications
+
+        self._tasks = {}
+        self._queue = []
+        self._sem = 0
+
+        self._thread = Thread(target=self._check_new_task, daemon=True)
+        self._thread.start()
+
+    def _tasks_failed(self):
+        done = set()
+
+        for row in self._db["task_done"].sql("WHERE return_code != 0"):
+            done.add(uuid.UUID(row["id"]))
+
+        return done
+
+    def _tasks_done(self):
+
+        done = set()
+
+        for row in self._db["task_done"]:
+            done.add(uuid.UUID(row["id"]))
+
+        return done
+
+    def _check_new_task(self):
+        while True:
+            with self._lock:
+                for task in list(self._queue):
+                    task: Sist2Task
+
+                    if self._sem >= 1:
+                        break
+
+                    if not task.depends_on or task.depends_on in self._tasks_done():
+                        self._queue.remove(task)
+
+                        if task.depends_on in self._tasks_failed():
+                            # The task which we depend on failed, continue
+                            continue
+
+                        self._sem += 1
+
+                        t = Thread(target=self._run_task, args=(task,))
+
+                        self._tasks[task.id] = {
+                            "task": task,
+                            "thread": t,
+                        }
+
+                        t.start()
+                    break
+            sleep(1)
+
+    def tasks(self):
+        return list(map(lambda t: t["task"], self._tasks.values()))
+
+    def kill_task(self, task_id):
+
+        task = self._tasks.get(UUID(task_id))
+
+        if task:
+            pid = task["task"].pid
+            logger.info(f"Killing task {task_id} (pid={pid})")
+            os.kill(pid, signal.SIGTERM)
+            return True
+
+        return False
+
+    def _run_task(self, task: Sist2Task):
+        task_result = task.run(self._sist2, self._db)
+
+        with self._lock:
+            del self._tasks[task.id]
+            self._sem -= 1
+
+            self._db["task_done"][task.id] = {
+                "ended": task.ended,
+                "started": task.started,
+                "name": task.display_name,
+                "return_code": task_result
+            }
+        if isinstance(task, Sist2IndexTask):
+            self._notifications.notify({
+                "message": "notifications.indexCompleted",
+                "job": task.job.name
+            })
+
+    def submit(self, task: Sist2Task):
+
+        logger.info(f"Submitted task to queue {task.display_name}")
+
+        with self._lock:
+            self._queue.append(task)

sist2-admin/sist2_admin/notifications.py

@@ -0,0 +1,40 @@
+import asyncio
+from typing import List
+
+
+class Notifications:
+    def __init__(self):
+        self._subscribers: List[Subscribe] = []
+
+    def subscribe(self, ob):
+        self._subscribers.append(ob)
+
+    def unsubscribe(self, ob):
+        self._subscribers.remove(ob)
+
+    def notify(self, notification: dict):
+        for ob in self._subscribers:
+            ob.notify(notification)
+
+
+class Subscribe:
+    def __init__(self, notifications: Notifications):
+        self._queue = []
+        self._notifications = notifications
+
+    async def __aenter__(self):
+        self._notifications.subscribe(self)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        self._notifications.unsubscribe(self)
+
+    def notify(self, notification: dict):
+        self._queue.append(notification)
+
+    async def notifications(self):
+        while True:
+            try:
+                yield self._queue.pop(0)
+            except IndexError:
+                await asyncio.sleep(0.1)

sist2-admin/sist2_admin/sist2.py

@@ -0,0 +1,307 @@
+import datetime
+import json
+import logging
+import os.path
+import traceback
+from datetime import datetime
+from io import TextIOWrapper
+from logging import FileHandler
+from subprocess import Popen, PIPE
+from tempfile import NamedTemporaryFile
+from threading import Thread
+from typing import List
+
+from pydantic import BaseModel
+
+from config import logger, LOG_FOLDER
+
+
+class Sist2Version:
+    def __init__(self, version: str):
+        self._version = version
+
+        self.major, self.minor, self.patch = [int(x) for x in version.split(".")]
+
+    def __str__(self):
+        return f"{self.major}.{self.minor}.{self.patch}"
+
+
+class WebOptions(BaseModel):
+    indices: List[str] = []
+    es_url: str = "http://elasticsearch:9200"
+    es_insecure_ssl: bool = False
+    es_index: str = "sist2"
+    bind: str = "0.0.0.0:4090"
+    auth: str = None
+    tag_auth: str = None
+    tagline: str = "Lightning-fast file system indexer and search tool"
+    dev: bool = False
+    lang: str = "en"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+        args = ["web", f"--es-url={self.es_url}", f"--bind={self.bind}",
+                f"--tagline={self.tagline}", f"--lang={self.lang}"]
+
+        if self.es_insecure_ssl:
+            args.append(f"--es-insecure-ssl")
+        if self.auth:
+            args.append(f"--auth={self.auth}")
+        if self.tag_auth:
+            args.append(f"--tag_auth={self.tag_auth}")
+        if self.dev:
+            args.append(f"--dev")
+
+        args.extend(self.indices)
+
+        return args
+
+
+class IndexOptions(BaseModel):
+    path: str = None
+    threads: int = 1
+    es_url: str = "http://elasticsearch:9200"
+    es_insecure_ssl: bool = False
+    es_index: str = "sist2"
+    incremental_index: bool = False
+    script: str = ""
+    script_file: str = None
+    batch_size: int = 100
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+
+        args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
+                f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
+
+        if self.script_file:
+            args.append(f"--script-file={self.script_file}")
+        if self.es_insecure_ssl:
+            args.append(f"--es-insecure-ssl")
+        if self.incremental_index:
+            args.append(f"--incremental-index")
+
+        return args
+
+
+ARCHIVE_SKIP = "skip"
+ARCHIVE_LIST = "list"
+ARCHIVE_SHALLOW = "shallow"
+ARCHIVE_RECURSE = "recurse"
+
+
+class ScanOptions(BaseModel):
+    path: str
+    threads: int = 1
+    mem_throttle: int = 0
+    thumbnail_quality: float = 1.0
+    thumbnail_size: int = 500
+    thumbnail_count: int = 1
+    content_size: int = 32768
+    depth: int = -1
+    archive: str = ARCHIVE_RECURSE
+    archive_passphrase: str = None
+    ocr_lang: bool = None
+    ocr_images: bool = False
+    ocr_ebooks: bool = False
+    exclude: str = None
+    fast: bool = False
+    treemap_threshold: float = 0.0005
+    mem_buffer: int = 2000
+    read_subtitles: bool = False
+    fast_epub: bool = False
+    checksums: bool = False
+    incremental: str = None
+    output: str = None
+    name: str = None
+    rewrite_url: str = None
+    list_file: str = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+        args = ["scan", self.path, f"--threads={self.threads}", f"--mem-throttle={self.mem_throttle}",
+                f"--thumbnail-quality={self.thumbnail_quality}", f"--thumbnail-count={self.thumbnail_count}",
+                f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
+                f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
+
+        if self.incremental:
+            args.append(f"--incremental={self.incremental}")
+        if self.rewrite_url:
+            args.append(f"--rewrite-url={self.rewrite_url}")
+        if self.name:
+            args.append(f"--name={self.name}")
+        if self.archive_passphrase:
+            args.append(f"--archive-passphrase={self.archive_passphrase}")
+        if self.ocr_lang:
+            args.append(f"--ocr-lang={self.ocr_lang}")
+        if self.ocr_ebooks:
+            args.append(f"--ocr-ebooks")
+        if self.ocr_images:
+            args.append(f"--ocr-images")
+        if self.exclude:
+            args.append(f"--exclude={self.exclude}")
+        if self.fast:
+            args.append(f"--fast")
+        if self.treemap_threshold:
+            args.append(f"--treemap-threshold={self.treemap_threshold}")
+        if self.read_subtitles:
+            args.append(f"--read-subtitles")
+        if self.fast_epub:
+            args.append(f"--fast-epub")
+        if self.checksums:
+            args.append(f"--checksums")
+        if self.list_file:
+            args.append(f"--list_file={self.list_file}")
+
+        return args
+
+
+class Sist2Index:
+    def __init__(self, path):
+        self.path = path
+
+        with open(os.path.join(path, "descriptor.json")) as f:
+            self._descriptor = json.load(f)
+
+    def to_json(self):
+        return {
+            "path": self.path,
+            "version": self.version(),
+            "timestamp": self.timestamp(),
+            "name": self.name()
+        }
+
+    def version(self) -> Sist2Version:
+        return Sist2Version(self._descriptor["version"])
+
+    def timestamp(self) -> datetime:
+        return datetime.fromtimestamp(self._descriptor["timestamp"])
+
+    def name(self) -> str:
+        return self._descriptor["name"]
+
+
+class Sist2:
+
+    def __init__(self, bin_path: str, data_directory: str):
+        self._bin_path = bin_path
+        self._data_dir = data_directory
+
+    def index(self, options: IndexOptions, logs_cb):
+
+        if options.script:
+            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
+                f.write(options.script)
+            options.script_file = f.name
+        else:
+            options.script_file = None
+
+        args = [
+            self._bin_path,
+            *options.args(),
+            "--json-logs",
+            "--very-verbose"
+        ]
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        self._consume_logs_stdout(logs_cb, proc)
+
+        t_stderr.join()
+
+        return proc.returncode
+
+    def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
+
+        output_dir = os.path.join(
+            self._data_dir,
+            f"scan-{datetime.now()}.sist2"
+        )
+        options.output = output_dir
+
+        args = [
+            self._bin_path,
+            *options.args(),
+            "--json-logs",
+            "--very-verbose"
+        ]
+
+        logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
+
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        set_pid_cb(proc.pid)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        self._consume_logs_stdout(logs_cb, proc)
+
+        t_stderr.join()
+
+        return proc.returncode
+
+    @staticmethod
+    def _consume_logs_stderr(logs_cb, proc):
+        pipe_wrapper = TextIOWrapper(proc.stderr, encoding="utf8")
+        try:
+            for line in pipe_wrapper:
+                if line.strip() == "":
+                    continue
+                logs_cb({"stderr": line})
+        finally:
+            proc.wait()
+            pipe_wrapper.close()
+
+    @staticmethod
+    def _consume_logs_stdout(logs_cb, proc):
+        pipe_wrapper = TextIOWrapper(proc.stdout, encoding="utf8")
+        try:
+            for line in pipe_wrapper:
+                if line.strip() == "":
+                    continue
+                log_object = json.loads(line)
+                logs_cb(log_object)
+        except Exception as e:
+            proc.kill()
+            try:
+                print(line)
+            except NameError:
+                pass
+            print(traceback.format_exc())
+        finally:
+            pass
+            # proc.wait()
+            # pipe_wrapper.close()
+
+    def web(self, options: WebOptions, name: str):
+        args = [
+            self._bin_path,
+            *options.args()
+        ]
+
+        web_logger = logging.Logger(name=f"sist2-frontend-{name}")
+        web_logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"frontend-{name}.log")))
+
+        def logs_cb(message):
+            web_logger.info(json.dumps(message))
+
+        logger.info(f"Starting frontend {' '.join(args)}")
+
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        t_stdout = Thread(target=self._consume_logs_stdout, args=(logs_cb, proc))
+        t_stdout.start()
+
+        return proc.pid

sist2-admin/sist2_admin/state.py

@@ -0,0 +1,50 @@
+from typing import Dict
+
+from hexlib.db import Table
+import pickle
+
+from tesseract import get_tesseract_langs
+
+RUNNING_FRONTENDS: Dict[str, int] = {}
+
+TESSERACT_LANGS = get_tesseract_langs()
+
+DB_SCHEMA_VERSION = "1"
+
+from pydantic import BaseModel
+
+
+def _serialize(item):
+    if isinstance(item, BaseModel):
+        return pickle.dumps(item)
+    if isinstance(item, bytes):
+        raise Exception("FIXME: bytes in PickleTable")
+    return item
+
+
+def _deserialize(item):
+    if isinstance(item, bytes):
+        return pickle.loads(item)
+    return item
+
+
+class PickleTable(Table):
+
+    def __getitem__(self, item):
+        row = super().__getitem__(item)
+        if row:
+            return dict((k, _deserialize(v)) for k, v in row.items())
+        return row
+
+    def __setitem__(self, key, value):
+        value = dict((k, _serialize(v)) for k, v in value.items())
+        super().__setitem__(key, value)
+
+    def __iter__(self):
+        for row in super().__iter__():
+            yield dict((k, _deserialize(v)) for k, v in row.items())
+
+    def sql(self, where_clause, *params):
+        for row in super().sql(where_clause, *params):
+            yield dict((k, _deserialize(v)) for k, v in row.items())
+

sist2-admin/sist2_admin/tesseract.py

@@ -0,0 +1,14 @@
+import subprocess
+
+
+def get_tesseract_langs():
+
+    res = subprocess.check_output([
+        "tesseract",
+        "--list-langs"
+    ]).decode()
+
+    languages = res.split("\n")[1:]
+
+    return list(filter(lambda lang: lang and lang != "osd", languages))
+

sist2-admin/sist2_admin/web.py

@@ -0,0 +1,29 @@
+import os.path
+from typing import List
+
+from pydantic import BaseModel
+
+from sist2 import WebOptions
+
+
+class Sist2Frontend(BaseModel):
+    name: str
+    jobs: List[str]
+    web_options: WebOptions
+    running: bool = False
+
+    auto_start: bool = False
+    enable_monitoring: bool = True
+    extra_query_args: str = ""
+    custom_url: str = None
+
+    def get_log_path(self, log_folder: str):
+        return os.path.join(log_folder, f"frontend-{self.name}.log")
+
+    @staticmethod
+    def create_default(name: str):
+        return Sist2Frontend(
+            name=name,
+            web_options=WebOptions(),
+            jobs=[]
+        )

sist2-vue/dist/index.html

@@ -1,3 +1,32 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>sist2</title><link href="css/chunk-vendors.css" rel="preload" as="style"><link href="css/index.css" rel="preload" as="style"><link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/index.css" rel="stylesheet"></head><body><noscript><style>body {
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
+
+    <title>sist2</title>
+<link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"></head>
+<body>
+<noscript>
+    <style>
+        body {
             height: initial;
-        }</style><div style="text-align: center; margin-top: 100px"><strong>We're sorry but sist2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong><br><strong>Nous sommes désolés mais sist2 ne fonctionne pas correctement si JavaScript est activé. Veuillez l'activer pour continuer.</strong></div></noscript><div id="app"></div><script src="js/chunk-vendors.js"></script><script src="js/index.js"></script></body></html>
+        }
+    </style>
+    <div style="text-align: center; margin-top: 100px">
+        <strong>
+            We're sorry but sist2 doesn't work properly without JavaScript enabled.
+            Please enable it to continue.
+        </strong>
+        <br/>
+        <strong>
+            Nous sommes désolés mais sist2 ne fonctionne pas correctement
+            si JavaScript est activé.
+            Veuillez l'activer pour continuer.
+        </strong>
+    </div>
+</noscript>
+<div id="app"></div>
+<script type="text/javascript" src="js/chunk-vendors.js"></script><script type="text/javascript" src="js/index.js"></script></body>
+</html>

sist2-vue/package.json

@@ -8,10 +8,9 @@
   },
   "dependencies": {
     "@egjs/vue-infinitegrid": "3.3.0",
-    "axios": "^0.21.1",
+    "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "crypto-es": "^1.2.7",
     "d3": "^5.16.0",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",

sist2-vue/src/App.vue

@@ -146,6 +146,7 @@ html, body {
 
 .theme-black .nav-tabs .nav-link {
   color: #e0e0e0;
+  border-radius: 0;
 }
 
 .theme-black .nav-tabs .nav-item.show .nav-link, .theme-black .nav-tabs .nav-link.active {
@@ -309,4 +310,8 @@ mark {
   display: inline-block;
   width: 40%;
 }
+
+.pointer {
+  cursor: pointer;
+}
 </style>

sist2-vue/src/Sist2Api.ts

@@ -1,6 +1,5 @@
 import axios from "axios";
 import {ext, strUnescape, lum} from "./util";
-import CryptoES from 'crypto-es';
 
 export interface EsTag {
     id: string
@@ -30,7 +29,6 @@ export interface EsHit {
     _index: string
     _id: string
     _score: number
-    _path_md5: string
     _type: string
     _tags: Tag[]
     _seq: number
@@ -62,8 +60,9 @@ export interface EsHit {
         isPlayableImage: boolean
         isAudio: boolean
         hasThumbnail: boolean
-        tnW: number
-        tnH: number
+        hasVidPreview: boolean
+        /** Number of thumbnails available */
+        tnNum: number
     }
     highlight: {
         name: string[] | undefined,
@@ -134,8 +133,15 @@ class Sist2Api {
 
         if ("thumbnail" in hit._source) {
             hit._props.hasThumbnail = true;
-            hit._props.tnW = Number(hit._source.thumbnail.split(",")[0]);
-            hit._props.tnH = Number(hit._source.thumbnail.split(",")[1]);
+
+            if (Number.isNaN(Number(hit._source.thumbnail))) {
+                // Backwards compatibility
+                hit._props.tnNum = 1;
+                hit._props.hasVidPreview = false;
+            } else {
+                hit._props.tnNum = Number(hit._source.thumbnail);
+                hit._props.hasVidPreview = hit._props.tnNum > 1;
+            }
         }
 
         switch (mimeCategory) {
@@ -241,11 +247,6 @@ class Sist2Api {
                 res.hits.hits.forEach((hit: EsHit) => {
                     hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
                     hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
-                    hit["_path_md5"] = CryptoES.MD5(
-                        hit["_source"]["path"] +
-                        (hit["_source"]["path"] ? "/" : "") +
-                        hit["_source"]["name"] + ext(hit)
-                    ).toString();
 
                     this.setHitProps(hit);
                     this.setHitTags(hit);
@@ -256,20 +257,31 @@ class Sist2Api {
         });
     }
 
-    getMimeTypes() {
-        return this.esQuery({
-            aggs: {
-                mimeTypes: {
-                    terms: {
-                        field: "mime",
-                        size: 10000
-                    }
+    getMimeTypes(query = undefined) {
+        const AGGS = {
+            mimeTypes: {
+                terms: {
+                    field: "mime",
+                    size: 10000
                 }
-            },
-            size: 0,
-        }).then(resp => {
+            }
+        };
+
+        if (!query) {
+            query = {
+                aggs: AGGS,
+                size: 0,
+            };
+        } else {
+            query.size = 0;
+            query.aggs = AGGS;
+        }
+
+        return this.esQuery(query).then(resp => {
             const mimeMap: any[] = [];
-            resp["aggregations"]["mimeTypes"]["buckets"].sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
+            const buckets = resp["aggregations"]["mimeTypes"]["buckets"];
+
+            buckets.sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
                 const tmp = bucket["key"].split("/");
                 const category = tmp[0];
                 const mime = tmp[1];
@@ -289,11 +301,18 @@ class Sist2Api {
                 });
 
                 if (!category_exists) {
-                    mimeMap.push({"text": category, children: [child]});
+                    mimeMap.push({text: category, children: [child], id: category});
                 }
             })
 
-            return mimeMap;
+            mimeMap.forEach(node => {
+                if (node.children) {
+                    node.children.sort((a, b) => a.id.localeCompare(b.id));
+                }
+            })
+            mimeMap.sort((a, b) => a.id.localeCompare(b.id))
+
+            return {buckets, mimeMap};
         });
     }
 
@@ -317,10 +336,6 @@ class Sist2Api {
         };
     }
 
-    getDocInfo(docId: string) {
-        return axios.get(`${this.baseUrl}d/${docId}`);
-    }
-
     getTags() {
         return this.esQuery({
             aggs: {
@@ -354,8 +369,7 @@ class Sist2Api {
         return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
             delete: false,
             name: tag,
-            doc_id: hit["_id"],
-            path_md5: hit._path_md5
+            doc_id: hit["_id"]
         });
     }
 
@@ -363,8 +377,7 @@ class Sist2Api {
         return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
             delete: true,
             name: tag,
-            doc_id: hit["_id"],
-            path_md5: hit._path_md5
+            doc_id: hit["_id"]
         });
     }
 

sist2-vue/src/Sist2Query.ts

@@ -69,7 +69,7 @@ interface SortMode {
 
 class Sist2Query {
 
-    searchQuery(): any {
+    searchQuery(blankSearch: boolean = false): any {
 
         const getters = store.getters;
 
@@ -93,22 +93,6 @@ class Sist2Query {
             {terms: {index: selectedIndexIds}}
         ] as any[];
 
-        if (sizeMin && sizeMax) {
-            filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
-        } else if (sizeMin) {
-            filters.push({range: {size: {gte: sizeMin}}})
-        } else if (sizeMax) {
-            filters.push({range: {size: {lte: sizeMax}}})
-        }
-
-        if (dateMin && dateMax) {
-            filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
-        } else if (dateMin) {
-            filters.push({range: {mtime: {gte: dateMin}}})
-        } else if (dateMax) {
-            filters.push({range: {mtime: {lte: dateMax}}})
-        }
-
         const fields = [
             "name^8",
             "content^3",
@@ -128,20 +112,39 @@ class Sist2Query {
             fields.push("name.nGram^3");
         }
 
-        const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
-        if (path !== "") {
-            filters.push({term: {path: path}})
-        }
+        if (!blankSearch) {
+            if (sizeMin && sizeMax) {
+                filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
+            } else if (sizeMin) {
+                filters.push({range: {size: {gte: sizeMin}}})
+            } else if (sizeMax) {
+                filters.push({range: {size: {lte: sizeMax}}})
+            }
 
-        if (selectedMimeTypes.length > 0) {
-            filters.push({terms: {"mime": selectedMimeTypes}});
-        }
+            if (dateMin && dateMax) {
+                filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
+            } else if (dateMin) {
+                filters.push({range: {mtime: {gte: dateMin}}})
+            } else if (dateMax) {
+                filters.push({range: {mtime: {lte: dateMax}}})
+            }
 
-        if (selectedTags.length > 0) {
-            if (getters.optTagOrOperator) {
-                filters.push({terms: {"tag": selectedTags}});
-            } else {
-                selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
+            const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
+
+            if (path !== "") {
+                filters.push({term: {path: path}})
+            }
+
+            if (selectedMimeTypes.length > 0) {
+                filters.push({terms: {"mime": selectedMimeTypes}});
+            }
+
+            if (selectedTags.length > 0) {
+                if (getters.optTagOrOperator) {
+                    filters.push({terms: {"tag": selectedTags}});
+                } else {
+                    selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
+                }
             }
         }
 
@@ -182,7 +185,7 @@ class Sist2Query {
             size: size,
         } as any;
 
-        if (!empty) {
+        if (!empty && !blankSearch) {
             q.query.bool.must = query;
         }
 
@@ -207,7 +210,7 @@ class Sist2Query {
             };
 
             if (!legacyES) {
-                q.highlight.max_analyzed_offset = 9_999_999;
+                q.highlight.max_analyzed_offset = 999_999;
             }
 
             if (getters.optSearchInPath) {
@@ -237,7 +240,7 @@ class Sist2Query {
                 }
             }
 
-            if (!empty) {
+            if (!empty && !blankSearch) {
                 q.query.function_score.query.bool.must.push(query);
             }
         }

sist2-vue/src/components/DateSlider.vue

@@ -1,5 +1,31 @@
 <template>
-  <div id="dateSlider"></div>
+  <div v-if="$store.state.optUseDatePicker">
+    <b-row>
+      <b-col sm="6">
+        <b-form-datepicker
+            value-as-date
+            :date-format-options="{ year: 'numeric', month: '2-digit', day: '2-digit' }"
+            :locale="$store.state.optLang"
+            class="mb-2"
+            :value="dateMin" @input="setDateMin"></b-form-datepicker>
+      </b-col>
+      <b-col sm="6">
+        <b-form-datepicker
+            value-as-date
+            :date-format-options="{ year: 'numeric', month: '2-digit', day: '2-digit' }"
+            :locale="$store.state.optLang"
+            class="mb-2"
+            :value="dateMax" @input="setDateMax"></b-form-datepicker>
+      </b-col>
+    </b-row>
+  </div>
+  <div v-else>
+    <b-row>
+      <b-col style="height: 70px;">
+        <div id="dateSlider"></div>
+      </b-col>
+    </b-row>
+  </div>
 </template>
 
 <script>
@@ -10,11 +36,36 @@ import {mergeTooltips} from "@/util-js";
 
 export default {
   name: "DateSlider",
+  methods: {
+    setDateMin(val) {
+      const epochDate = Math.ceil(+val / 1000);
+      this.$store.commit("setDateMin", epochDate);
+    },
+    setDateMax(val) {
+      const epochDate = Math.ceil(+val / 1000);
+      this.$store.commit("setDateMax", epochDate);
+    },
+  },
+  computed: {
+    dateMin() {
+      const dateMin = this.$store.state.dateMin ? this.$store.state.dateMin : this.$store.state.dateBoundsMin;
+      return new Date(dateMin * 1000)
+    },
+    dateMax() {
+      const dateMax = this.$store.state.dateMax ? this.$store.state.dateMax : this.$store.state.dateBoundsMax;
+      return new Date(dateMax * 1000)
+    }
+  },
   mounted() {
     this.$store.subscribe((mutation) => {
       if (mutation.type === "setDateBoundsMax") {
         const elem = document.getElementById("dateSlider");
 
+        if (elem === null) {
+          // Using b-form-datepicker, skip initialisation of slider
+          return
+        }
+
         if (elem.children.length > 0) {
           return;
         }

sist2-vue/src/components/DebugInfo.vue

@@ -15,7 +15,7 @@
 
 <script>
 import IndexDebugInfo from "@/components/IndexDebugInfo";
-import DebugIcon from "@/components/DebugIcon";
+import DebugIcon from "@/components/icons/DebugIcon";
 
 export default {
   name: "DebugInfo.vue",
@@ -27,10 +27,10 @@ export default {
         {key: "platform", value: this.$store.state.sist2Info.platform},
         {key: "debugBinary", value: this.$store.state.sist2Info.debug},
         {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
-        {key: "libscanCommitHash", value: this.$store.state.sist2Info.libscanHash},
         {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
         {key: "tagline", value: this.$store.state.sist2Info.tagline},
         {key: "dev", value: this.$store.state.sist2Info.dev},
+        {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
         {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
         {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
         {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},

sist2-vue/src/components/DocCard.vue

@@ -1,5 +1,6 @@
 <template>
-  <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`">
+  <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
+       @click="$store.commit('busTnTouchStart', null)">
     <b-card
         no-body
         img-top
@@ -10,37 +11,7 @@
       <ContentDiv :doc="doc"></ContentDiv>
 
       <!-- Thumbnail-->
-      <div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
-        <div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
-          <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
-        </div>
-
-        <div
-            v-if="doc._props.isImage && !hover && doc._props.tnW / doc._props.tnH < 5"
-            class="card-img-overlay"
-            :class="{'small-badge': smallBadge}">
-          <span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
-        </div>
-
-        <div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0 && !hover"
-             class="card-img-overlay"
-             :class="{'small-badge': smallBadge}">
-          <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
-        </div>
-
-        <div v-if="doc._props.isPlayableVideo" class="play">
-          <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
-            <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
-          </svg>
-        </div>
-
-        <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-             :src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
-             alt=""
-             class="pointer fit card-img-top" @click="onThumbnailClick()">
-        <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
-             class="fit card-img-top">
-      </div>
+      <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
 
       <!-- Audio player-->
       <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
@@ -71,31 +42,19 @@ import TagContainer from "@/components/TagContainer.vue";
 import DocFileTitle from "@/components/DocFileTitle.vue";
 import DocInfoModal from "@/components/DocInfoModal.vue";
 import ContentDiv from "@/components/ContentDiv.vue";
+import FullThumbnail from "@/components/FullThumbnail";
 
 
 export default {
-  components: {ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+  components: {FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
   props: ["doc", "width"],
   data() {
     return {
       ext: ext,
       showInfo: false,
-      hover: false
     }
   },
   computed: {
-    placeHolderStyle() {
-
-      const tokens = this.doc._source.thumbnail.split(",");
-      const w = Number(tokens[0]);
-      const h = Number(tokens[1]);
-
-      const MAX_HEIGHT = 400;
-
-      return {
-        height: `${Math.min((h / w) * this.width, MAX_HEIGHT)}px`,
-      }
-    },
     smallBadge() {
       return this.width < 150;
     }
@@ -117,28 +76,10 @@ export default {
         }
       });
     },
-    onTnEnter() {
-      this.hover = true;
-    },
-    onTnLeave() {
-      this.hover = false;
-    }
   },
 }
 </script>
 <style>
-.img-wrapper {
-  position: relative;
-}
-
-.img-wrapper:hover svg {
-  fill: rgba(0, 0, 0, 1);
-}
-
-.pointer {
-  cursor: pointer;
-}
-
 .fit {
   display: block;
   min-width: 64px;
@@ -148,15 +89,17 @@ export default {
   width: auto;
   height: auto;
 }
+
+.audio-fit {
+  height: 39px;
+  vertical-align: bottom;
+  display: inline;
+  width: 100%;
+}
 </style>
 
 <style scoped>
 
-.card-img-top {
-  border-top-left-radius: 0;
-  border-top-right-radius: 0;
-}
-
 .padding-03 {
   padding: 0.3rem;
 }
@@ -174,55 +117,11 @@ export default {
   padding: 0.3rem;
 }
 
-.thumbnail-placeholder {
-
-}
-
-.card-img-overlay {
-  pointer-events: none;
-  padding: 0.75rem;
-  bottom: unset;
-  top: 0;
-  left: unset;
-  right: unset;
-}
-
-.badge-resolution {
-  color: #212529;
-  background-color: #FFC107;
-}
-
-.play {
-  position: absolute;
-  width: 25px;
-  height: 25px;
-  left: 50%;
-  top: 50%;
-  transform: translate(-50%, -50%);
-  pointer-events: none;
-}
-
-.play svg {
-  fill: rgba(0, 0, 0, 0.7);
-}
-
 .doc-card {
   padding-left: 3px;
   padding-right: 3px;
 }
 
-.small-badge {
-  padding: 1px 3px;
-  font-size: 70%;
-}
-
-.audio-fit {
-  height: 39px;
-  vertical-align: bottom;
-  display: inline;
-  width: 100%;
-}
-
 .sub-document .card {
   background: #AB47BC1F !important;
 }

sist2-vue/src/components/DocInfoModal.vue

@@ -2,7 +2,10 @@
   <b-modal :visible="show" size="lg" :hide-footer="true" static lazy @close="$emit('close')" @hide="$emit('close')"
   >
     <template #modal-title>
-      <h5 class="modal-title" :title="doc._source.name + ext(doc)">{{ doc._source.name + ext(doc) }}</h5>
+      <h5 class="modal-title" :title="doc._source.name + ext(doc)">
+        {{ doc._source.name + ext(doc) }}
+        <router-link :to="`/file?byId=${doc._id}`">#</router-link>
+      </h5>
     </template>
 
     <img v-if="doc._props.hasThumbnail" :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">

sist2-vue/src/components/DocListItem.vue

@@ -1,10 +1,13 @@
 <template>
-  <b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}">
+  <b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}"
+                     @mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
 
     <!-- Info modal-->
     <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
 
     <div class="media ml-2">
+
+      <!-- Thumbnail-->
       <div v-if="doc._props.hasThumbnail" class="align-self-start mr-2 wrapper-sm">
         <div class="img-wrapper">
           <div v-if="doc._props.isPlayableVideo" class="play">
@@ -25,6 +28,7 @@
         <FileIcon></FileIcon>
       </div>
 
+      <!-- Doc line-->
       <div class="doc-line ml-3">
         <div style="display: flex">
           <span class="info-icon" @click="showInfo = true"></span>
@@ -56,7 +60,7 @@ import TagContainer from "@/components/TagContainer";
 import DocFileTitle from "@/components/DocFileTitle";
 import DocInfoModal from "@/components/DocInfoModal";
 import ContentDiv from "@/components/ContentDiv";
-import FileIcon from "@/components/FileIcon";
+import FileIcon from "@/components/icons/FileIcon";
 
 export default {
   name: "DocListItem",
@@ -85,7 +89,13 @@ export default {
         return this.doc.highlight["path.nGram"] + "/"
       }
       return this.doc._source.path + "/"
-    }
+    },
+    onTnEnter() {
+      this.hover = true;
+    },
+    onTnLeave() {
+      this.hover = false;
+    },
   }
 }
 </script>
@@ -147,6 +157,7 @@ export default {
 .list-group-item .img-wrapper {
   width: 88px;
   height: 88px;
+  position: relative;
 }
 
 .fit-sm {

sist2-vue/src/components/FullThumbnail.vue

@@ -0,0 +1,173 @@
+<template>
+  <div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()"
+       @touchstart="onTouchStart()">
+    <div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
+      <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
+    </div>
+
+    <div
+        v-if="doc._props.isImage && !hover && doc._props.tnW / doc._props.tnH < 5"
+        class="card-img-overlay"
+        :class="{'small-badge': smallBadge}">
+      <span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
+    </div>
+
+    <div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0 && !hover"
+         class="card-img-overlay"
+         :class="{'small-badge': smallBadge}">
+      <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
+    </div>
+
+    <div v-if="doc._props.isPlayableVideo" class="play">
+      <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
+        <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
+      </svg>
+    </div>
+
+    <img ref="tn"
+         v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
+         :src="tnSrc"
+         alt=""
+         :style="{height: (doc._props.isGif && hover) ? `${tnHeight()}px` : undefined}"
+         class="pointer fit card-img-top" @click="onThumbnailClick()">
+    <img v-else :src="tnSrc" alt=""
+         class="fit card-img-top">
+
+    <ThumbnailProgressBar v-if="hover && doc._props.hasVidPreview"
+                          :progress="(currentThumbnailNum + 1) / (doc._props.tnNum)"
+    ></ThumbnailProgressBar>
+  </div>
+</template>
+
+<script>
+import {humanTime} from "@/util";
+import ThumbnailProgressBar from "@/components/ThumbnailProgressBar";
+
+export default {
+  name: "FullThumbnail",
+  props: ["doc", "smallBadge"],
+  components: {ThumbnailProgressBar},
+  data() {
+    return {
+      hover: false,
+      currentThumbnailNum: 0,
+      timeoutId: null
+    }
+  },
+  created() {
+    this.$store.subscribe((mutation) => {
+      if (mutation.type === "busTnTouchStart" && mutation.payload !== this.doc._id) {
+        this.onTnLeave();
+      }
+    });
+  },
+  computed: {
+    tnSrc() {
+      const doc = this.doc;
+      const props = doc._props;
+      if (props.isGif && this.hover) {
+        return `f/${doc._id}`;
+      }
+      return (this.currentThumbnailNum === 0)
+          ? `t/${doc._source.index}/${doc._id}`
+          : `t/${doc._source.index}/${doc._id}${String(this.currentThumbnailNum).padStart(4, "0")}`;
+    },
+  },
+  methods: {
+    humanTime: humanTime,
+    onThumbnailClick() {
+      this.$emit("onThumbnailClick");
+    },
+    tnHeight() {
+      return this.$refs.tn.height;
+    },
+    tnWidth() {
+      return this.$refs.tn.width;
+    },
+    onTnEnter() {
+      this.hover = true;
+      if (this.doc._props.hasVidPreview) {
+        this.currentThumbnailNum += 1;
+        this.scheduleNextTnNum();
+      }
+    },
+    onTnLeave() {
+      this.currentThumbnailNum = 0;
+      this.hover = false;
+      if (this.timeoutId !== null) {
+        window.clearTimeout(this.timeoutId);
+        this.timeoutId = null;
+      }
+    },
+    scheduleNextTnNum() {
+      const INTERVAL = this.$store.state.optVidPreviewInterval ?? 700;
+      this.timeoutId = window.setTimeout(() => {
+        if (!this.hover) {
+          return;
+        }
+        this.scheduleNextTnNum();
+        if (this.currentThumbnailNum === this.doc._props.tnNum - 1) {
+          this.currentThumbnailNum = 0;
+        } else {
+          this.currentThumbnailNum += 1;
+        }
+      }, INTERVAL);
+    },
+    onTouchStart() {
+      this.$store.commit("busTnTouchStart", this.doc._id);
+      if (!this.hover) {
+        this.onTnEnter()
+      }
+    },
+  }
+}
+</script>
+
+<style scoped>
+.img-wrapper {
+  position: relative;
+}
+
+.img-wrapper:hover svg {
+  fill: rgba(0, 0, 0, 1);
+}
+
+.card-img-top {
+  border-top-left-radius: 0;
+  border-top-right-radius: 0;
+}
+
+.play {
+  position: absolute;
+  width: 25px;
+  height: 25px;
+  left: 50%;
+  top: 50%;
+  transform: translate(-50%, -50%);
+  pointer-events: none;
+}
+
+.play svg {
+  fill: rgba(0, 0, 0, 0.7);
+}
+
+.badge-resolution {
+  color: #212529;
+  background-color: #FFC107;
+}
+
+.card-img-overlay {
+  pointer-events: none;
+  padding: 0.75rem;
+  bottom: unset;
+  top: 0;
+  left: unset;
+  right: unset;
+}
+
+.small-badge {
+  padding: 1px 3px;
+  font-size: 70%;
+}
+
+</style>

sist2-vue/src/components/IndexPicker.vue

@@ -29,7 +29,7 @@
           :class="{active: lastClickIndex === idx}"
       >
         <div class="d-flex">
-          <b-checkbox @change="toggleIndex(idx)" :checked="isSelected(idx)"></b-checkbox>
+          <b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
           {{ idx.name }}
           <span class="text-muted timestamp-text ml-2">{{ formatIdxDate(idx.timestamp) }}</span>
         </div>
@@ -133,6 +133,11 @@ export default Vue.extend({
   font-size: 80%;
 }
 
+.theme-black .version-badge {
+  color: #eee !important;
+  background: none;
+}
+
 .version-badge {
   color: #222 !important;
   background: none;
@@ -163,4 +168,24 @@ export default Vue.extend({
   background-color: inherit;
   color: inherit;
 }
+
+.theme-black .list-group-item {
+  border: 1px solid rgba(255,255,255, 0.1);
+}
+
+.theme-black .list-group-item:first-child {
+  border: 1px solid rgba(255,255,255, 0.05);
+}
+
+.theme-black .list-group-item.active {
+  z-index: 2;
+  background-color: inherit;
+  color: inherit;
+  border: 1px solid rgba(255,255,255, 0.3);
+  border-radius: 0;
+}
+
+.theme-black .list-group {
+  border-radius: 0;
+}
 </style>

sist2-vue/src/components/InfoTable.vue

@@ -1,6 +1,5 @@
 <template>
   <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0 mt-4">
-
     <template #cell(value)="data">
       <span v-if="'html' in data.item" v-html="data.item.html"></span>
       <span v-else>{{ data.value }}</span>
@@ -33,12 +32,18 @@ function dmsToDecimal(dms, ref) {
 export default {
   name: "InfoTable",
   props: ["doc"],
+  data() {
+    return {
+      indexName: "loading..."
+    }
+  },
   computed: {
     tableItems() {
+      this.indexName;
       const src = this.doc._source;
 
       const items = [
-        {key: "index", value: `[${this.$store.getters.indexMap[src.index].name}]`},
+        {key: "index", value: `[${this.indexName}]`},
         {key: "mtime", value: humanDate(src.mtime)},
         {key: "mime", value: src.mime},
         {key: "size", value: humanFileSize(src.size)},
@@ -67,6 +72,12 @@ export default {
         }
       });
 
+      Object.keys(src).forEach(key => {
+        if (key.startsWith("mt_") || key.startsWith("int_")) {
+          items.push({key: key, value: src[key]});
+        }
+      });
+
       // Exif GPS
       if ("exif_gps_longitude_dec" in src) {
         items.push({
@@ -85,7 +96,16 @@ export default {
 
       return items;
     }
-  }
+  },
+  mounted() {
+    if (this.$store.getters.indexMap[this.doc.index]) {
+      this.indexName = this.$store.getters.indexMap[this.doc._source.index].name
+    }
+
+    window.setTimeout(() => {
+      this.indexName = this.$store.getters.indexMap[this.doc._source.index].name
+    }, 500)
+  },
 }
 </script>
 

sist2-vue/src/components/LazyContentDiv.vue

@@ -1,11 +1,13 @@
 <template>
   <Preloader v-if="loading"></Preloader>
-  <div v-else-if="content" class="content-div">{{ content }}</div>
+  <div v-else-if="content" class="content-div" v-html="content"></div>
 </template>
 
 <script>
 import Sist2Api from "@/Sist2Api";
 import Preloader from "@/components/Preloader";
+import Sist2Query from "@/Sist2Query";
+import store from "@/store";
 
 export default {
   name: "LazyContentDiv",
@@ -18,10 +20,72 @@ export default {
     }
   },
   mounted() {
-    Sist2Api.getDocInfo(this.docId).then(src => {
-      this.content = src.data.content;
+    const query = Sist2Query.searchQuery();
+
+    if (this.$store.state.optHighlight) {
+
+      const fields = this.$store.state.fuzzy
+          ? {"content.nGram": {}}
+          : {content: {}};
+
+      query.highlight = {
+        pre_tags: ["<mark>"],
+        post_tags: ["</mark>"],
+        number_of_fragments: 0,
+        fields,
+      };
+
+      if (!store.state.sist2Info.esVersionLegacy) {
+        query.highlight.max_analyzed_offset = 999_999;
+      }
+    }
+
+    if ("function_score" in query.query) {
+      query.query = query.query.function_score.query;
+    }
+
+    if (!("must" in query.query.bool)) {
+      query.query.bool.must = [];
+    } else if (!Array.isArray(query.query.bool.must)) {
+      query.query.bool.must = [query.query.bool.must];
+    }
+
+    query.query.bool.must.push({match: {_id: this.docId}});
+
+    delete query["sort"];
+    delete query["aggs"];
+    delete query["search_after"];
+    delete query.query["function_score"];
+
+    query._source = {
+      includes: ["content", "name", "path", "extension"]
+    }
+
+    query.size = 1;
+
+    Sist2Api.esQuery(query).then(resp => {
       this.loading = false;
-    })
+      if (resp.hits.hits.length === 1) {
+        this.content = this.getContent(resp.hits.hits[0]);
+      } else {
+        console.log("FIXME: could not get content")
+        console.log(resp)
+      }
+    });
+  },
+  methods: {
+    getContent(doc) {
+      if (!doc.highlight) {
+        return doc._source.content;
+      }
+
+      if (doc.highlight["content.nGram"]) {
+        return doc.highlight["content.nGram"][0];
+      }
+      if (doc.highlight.content) {
+        return doc.highlight.content[0];
+      }
+    }
   }
 }
 </script>

sist2-vue/src/components/Lightbox.vue

@@ -1,7 +1,7 @@
 <template>
-  <div>
-    <!-- TODO: Set slideshowTime as a configurable option-->
+  <div :class="{'disable-animations': $store.state.optSimpleLightbox}">
     <FsLightbox
+        ref="lightbox"
         :key="lightboxKey"
         :toggler="showLightbox"
         :sources="lightboxSources"
@@ -10,8 +10,8 @@
         :types="lightboxTypes"
         :source-index="lightboxSlide"
         :custom-toolbar-buttons="customButtons"
-        :slideshow-time="1000 * 10"
-        :zoom-increment="0.5"
+        :slideshow-time="$store.getters.optLightboxSlideDuration * 1000"
+        :zoom-increment="0.25"
         :load-only-current-source="$store.getters.optLightboxLoadOnlyCurrent"
         :on-close="onClose"
         :on-open="onShow"
@@ -30,6 +30,7 @@ export default {
   components: {FsLightbox},
   data() {
     return {
+      disableAnimations: true,
       customButtons: [
         {
           viewBox: "0 0 384.928 384.928",
@@ -65,7 +66,84 @@ export default {
       return this.$store.getters["uiLightboxTypes"];
     }
   },
+  mounted() {
+    const listener = document.onkeydown;
+
+    document.onkeydown = (e) => {
+
+      const ret = this.keyDownListener(e)
+
+      if (listener && ret) {
+        return listener(e);
+      }
+    };
+  },
   methods: {
+    keyDownListener(e) {
+
+      const isLightboxOpen = this.$refs.lightbox === undefined || this.$refs.lightbox.$el.tagName === undefined;
+
+      if (isLightboxOpen) {
+        return true;
+      }
+
+      const lightboxStore = this.$refs.lightbox.fsLightboxStore.slice(-1)[0];
+
+      switch (e.key) {
+        case " ": {
+          e.preventDefault();
+          e.stopPropagation();
+          e.stopImmediatePropagation();
+
+          // Find video at current slide, toggle play/pause
+          [...document.getElementsByClassName("fslightbox-absoluted")].forEach(elem => {
+            if (elem.style.transform === "translate(0px)" || elem.style.transform === "translate(0px, 0px)") {
+              const vid = elem.getElementsByTagName("video")[0];
+
+              if (vid) {
+                if (vid.paused) {
+                  vid.play();
+                } else {
+                  vid.pause()
+                }
+              }
+            }
+
+            return false;
+          });
+
+          return false;
+        }
+        case "ArrowUp":
+        case "k": {
+          if (!lightboxStore.data.isThumbing && lightboxStore.core.thumbsToggler) {
+            lightboxStore.core.thumbsToggler.toggleThumbs();
+          }
+          return false;
+        }
+        case "ArrowDown":
+        case "j": {
+          if (lightboxStore.data.isThumbing && lightboxStore.core.thumbsToggler) {
+            lightboxStore.core.thumbsToggler.toggleThumbs();
+          }
+          return false;
+        }
+        case "h": {
+          if (lightboxStore.core.stageManager.getPreviousSlideIndex) {
+            lightboxStore.core.slideIndexChanger.jumpTo(lightboxStore.core.stageManager.getPreviousSlideIndex());
+          }
+          return false;
+        }
+        case "l": {
+          if (lightboxStore.core.stageManager.getNextSlideIndex) {
+            lightboxStore.core.slideIndexChanger.jumpTo(lightboxStore.core.stageManager.getNextSlideIndex());
+          }
+          return false;
+        }
+      }
+
+      return true;
+    },
     onDownloadClick() {
       const url = this.lightboxSources[this.lightboxSlide];
 
@@ -126,4 +204,20 @@ export default {
 .fslightbox-toolbar-button:nth-child(7) {
   order: 7;
 }
+
+.disable-animations .fslightbox-container {
+  background: rgba(30,30,30,.9);
+}
+
+.disable-animations .fslightbox-transform-transition {
+  transition: none;
+}
+
+.disable-animations .fslightbox-fade-in-strong {
+  animation: none;
+}
+
+.fslightbox-container video, .fslightbox-container img {
+  cursor: unset !important;
+}
 </style>

sist2-vue/src/components/LightboxCaption.vue

@@ -3,7 +3,7 @@
     <p>
       <b>{{
           `[${$store.getters.indices.find(i => i.id === hit._source.index).name}]`
-        }}</b>{{ `/${hit._source.path}/${hit._source.name}${ext(hit)}` }}
+        }}</b>{{ `${hit._source.path === '' ? '' : '/'}${hit._source.path}/${hit._source.name}${ext(hit)}` }}
     </p>
     <p style="margin-top: -1em">
       <span v-if="hit._source.width">{{ `${hit._source.width}x${hit._source.height}`}}</span>

sist2-vue/src/components/MimePicker.vue

@@ -7,40 +7,25 @@ import InspireTree from "inspire-tree";
 import InspireTreeDOM from "inspire-tree-dom";
 
 import "inspire-tree-dom/dist/inspire-tree-light.min.css";
-import {getSelectedTreeNodes} from "@/util";
+import {getSelectedTreeNodes, getTreeNodeAttributes} from "@/util";
+import Sist2Api from "@/Sist2Api";
+import Sist2Query from "@/Sist2Query";
 
 export default {
   name: "MimePicker",
   data() {
     return {
       mimeTree: null,
+      stashedMimeTreeAttributes: null,
+      updateBusy: false
     }
   },
   mounted() {
     this.$store.subscribe((mutation) => {
-      if (mutation.type === "setUiMimeMap") {
-        const mimeMap = mutation.payload.slice();
-
-        const elem = document.getElementById("mimeTree");
-        console.log(elem);
-
-        this.mimeTree = new InspireTree({
-          selection: {
-            mode: 'checkbox'
-          },
-          data: mimeMap
-        });
-        new InspireTreeDOM(this.mimeTree, {
-          target: '#mimeTree'
-        });
-        this.mimeTree.on("node.state.changed", this.handleTreeClick);
-        this.mimeTree.deselect();
-
-        if (this.$store.state._onLoadSelectedMimeTypes.length > 0) {
-          this.$store.state._onLoadSelectedMimeTypes.forEach(mime => {
-            this.mimeTree.node(mime).select();
-          });
-        }
+      if (mutation.type === "setUiMimeMap" && this.mimeTree === null) {
+        this.initializeTree();
+      } else if (mutation.type === "busSearch") {
+        this.updateTree();
       }
     });
   },
@@ -50,8 +35,85 @@ export default {
         return;
       }
 
+      if (this.updateBusy) {
+        return;
+      }
+
       this.$store.commit("setSelectedMimeTypes", getSelectedTreeNodes(this.mimeTree));
     },
+    updateTree() {
+
+      if (this.$store.getters.optUpdateMimeMap === false) {
+        return;
+      }
+
+      if (this.updateBusy) {
+        return
+      }
+      this.updateBusy = true;
+
+      if (this.stashedMimeTreeAttributes === null) {
+        this.stashedMimeTreeAttributes = getTreeNodeAttributes(this.mimeTree);
+      }
+
+      const query = Sist2Query.searchQuery();
+
+      Sist2Api.getMimeTypes(query).then(({buckets, mimeMap}) => {
+        this.$store.commit("setUiMimeMap", mimeMap);
+        this.$store.commit("setUiDetailsMimeAgg", buckets);
+
+        this.mimeTree.removeAll();
+        this.mimeTree.addNodes(mimeMap);
+
+        // Restore selected mimes
+        if (this.stashedMimeTreeAttributes === null) {
+          // NOTE: This happens when successive fast searches are triggered
+          this.stashedMimeTreeAttributes = {};
+          // Always add the selected mime types
+          this.$store.state.selectedMimeTypes.forEach(mime => {
+            this.stashedMimeTreeAttributes[mime] = {
+              checked: true
+            }
+          });
+        }
+
+        Object.entries(this.stashedMimeTreeAttributes).forEach(([mime, attributes]) => {
+          if (this.mimeTree.node(mime)) {
+            if (attributes.checked) {
+              this.mimeTree.node(mime).select();
+            }
+            if (attributes.collapsed === false) {
+              this.mimeTree.node(mime).expand();
+            }
+          }
+        });
+        this.stashedMimeTreeAttributes = null;
+        this.updateBusy = false;
+      });
+    },
+
+    initializeTree() {
+      const mimeMap = this.$store.state.uiMimeMap;
+
+      this.mimeTree = new InspireTree({
+        selection: {
+          mode: "checkbox"
+        },
+        data: mimeMap
+      });
+
+      new InspireTreeDOM(this.mimeTree, {
+        target: "#mimeTree"
+      });
+      this.mimeTree.on("node.state.changed", this.handleTreeClick);
+      this.mimeTree.deselect();
+
+      if (this.$store.state._onLoadSelectedMimeTypes.length > 0) {
+        this.$store.state._onLoadSelectedMimeTypes.forEach(mime => {
+          this.mimeTree.node(mime).select();
+        });
+      }
+    }
   }
 }
 </script>

sist2-vue/src/components/NavBar.vue

@@ -20,7 +20,7 @@
 </template>
 
 <script>
-import Sist2Icon from "@/components/Sist2Icon";
+import Sist2Icon from "@/components/icons/Sist2Icon";
 
 export default {
   name: "NavBar",
@@ -60,7 +60,6 @@ export default {
   color: #222 !important;
   font-size: 1.75rem;
   padding: 0;
-  font-family: Hack;
 }
 
 .navbar-brand:hover {

sist2-vue/src/components/ResultsCard.vue

@@ -3,7 +3,10 @@
     <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
 
     <div style="float: right">
-      <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile">{{ $t("details") }}</b-button>
+      <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
+          $t("details")
+        }}
+      </b-button>
 
       <template v-if="hitCount !== 0">
         <SortSelect class="ml-2"></SortSelect>
@@ -14,22 +17,42 @@
 
     <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
       <b-card>
-        <b-table :items="tableItems" small borderless thead-class="hidden" class="mb-0"></b-table>
+        <b-table :items="tableItems" small borderless bordered thead-class="hidden" class="mb-0"></b-table>
+
+        <br/>
+        <h4>
+          {{$t("mimeTypes")}}
+          <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick"><ClipboardIcon/></b-button>
+        </h4>
+        <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
+        <b-table
+            v-else
+            sort-by="doc_count"
+            :sort-desc="true"
+            thead-class="hidden"
+            :items="$store.state.uiDetailsMimeAgg" small bordered class="mb-0"
+        ></b-table>
       </b-card>
     </b-collapse>
   </b-card>
 </template>
 
 <script lang="ts">
-import {EsResult} from "@/Sist2Api";
+import Sist2Api, {EsResult} from "@/Sist2Api";
 import Vue from "vue";
 import {humanFileSize} from "@/util";
 import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
 import SortSelect from "@/components/SortSelect.vue";
+import Preloader from "@/components/Preloader.vue";
+import Sist2Query from "@/Sist2Query";
+import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
 
 export default Vue.extend({
   name: "ResultsCard",
-  components: {SortSelect, DisplayModeToggle},
+  components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
+  created() {
+
+  },
   computed: {
     lastResultsLoaded() {
       return this.$store.state.lastQueryResults != null;
@@ -54,6 +77,39 @@ export default Vue.extend({
     totalSize() {
       return humanFileSize((this.$store.state.lastQueryResults as EsResult).aggregations.total_size.value);
     },
+    onToggle() {
+      const show = !document.getElementById("collapse-1").classList.contains("show");
+      this.$store.commit("setUiShowDetails", show);
+
+      if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
+        // Mime aggs are not updated automatically, update now
+        this.forceUpdateMimeAgg();
+      }
+    },
+    onCopyClick() {
+      let tsvString = "";
+      this.$store.state.uiDetailsMimeAgg.slice().sort((a,b) => b["doc_count"] - a["doc_count"]).forEach(row => {
+        tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
+      });
+
+      navigator.clipboard.writeText(tsvString);
+
+      this.$bvToast.toast(
+          this.$t("toast.copiedToClipboard"),
+          {
+            title: null,
+            noAutoHide: false,
+            toaster: "b-toaster-bottom-right",
+            headerClass: "hidden",
+            bodyClass: "toast-body-info",
+          });
+    },
+    forceUpdateMimeAgg() {
+      const query = Sist2Query.searchQuery();
+      Sist2Api.getMimeTypes(query).then(({buckets}) => {
+        this.$store.commit("setUiDetailsMimeAgg", buckets);
+      });
+    }
   },
 });
 

sist2-vue/src/components/SortSelect.vue

@@ -42,6 +42,8 @@
 </template>
 
 <script>
+import {randomSeed} from "@/util";
+
 export default {
   name: "SortSelect",
   computed: {
@@ -52,7 +54,7 @@ export default {
   methods: {
     onSelect(sortMode) {
       if (sortMode === "random") {
-        this.$store.commit("setSeed", Math.round(Math.random() * 100000));
+        this.$store.commit("setSeed", randomSeed());
       }
       this.$store.commit("setSortMode", sortMode);
     }

sist2-vue/src/components/TagContainer.vue

@@ -51,7 +51,7 @@
         >{{ tag.text.split(".").pop() }}</span>
 
         <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
-          <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">Delete</b-button>
+          <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{$t("deleteTag")}}</b-button>
         </b-popover>
       </div>
 
@@ -63,7 +63,7 @@
     </template>
 
     <!-- Add button -->
-    <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">Add</small>
+    <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{$t("addTag")}}</small>
 
     <!-- Size tag-->
     <small v-else class="text-muted badge-size">{{

sist2-vue/src/components/TagPicker.vue

@@ -1,5 +1,13 @@
 <template>
-  <div id="tagTree"></div>
+  <div>
+    <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
+      <b-form-input :value="filter"
+                    :placeholder="$t('tagFilter')"
+                    @input="onFilter($event)"></b-form-input>
+    </b-input-group>
+
+    <div id="tagTree"></div>
+  </div>
 </template>
 
 <script>
@@ -112,15 +120,17 @@ function addTag(map, tag, id, count) {
 
 export default {
   name: "TagPicker",
+  props: ["showSearchBar"],
   data() {
     return {
       tagTree: null,
       loadedFromArgs: false,
+      filter: ""
     }
   },
   mounted() {
     this.$store.subscribe((mutation) => {
-      if (mutation.type === "setUiMimeMap") {
+      if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
         this.initializeTree();
         this.updateTree();
       } else if (mutation.type === "busUpdateTags") {
@@ -129,6 +139,10 @@ export default {
     });
   },
   methods: {
+    onFilter(value) {
+      this.filter = value;
+      this.tagTree.search(value);
+    },
     initializeTree() {
       const tagMap = [];
       this.tagTree = new InspireTree({
@@ -147,6 +161,7 @@ export default {
       this.tagTree.on("node.state.changed", this.handleTreeClick);
     },
     updateTree() {
+      // TODO: remember which tags are selected and restore?
       const tagMap = [];
       Sist2Api.getTags().then(tags => {
         tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
@@ -162,7 +177,8 @@ export default {
       });
     },
     handleTreeClick(node, e) {
-      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused") {
+      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
+          || e === "matched" || e === "hidden") {
         return;
       }
 
@@ -179,7 +195,15 @@ export default {
 }
 </style>
 <style>
-.inspire-tree .focused>.wholerow {
+.inspire-tree .focused > .wholerow {
   border: none;
 }
+
+#tag-picker-filter-bar {
+  padding: 10px 4px 4px;
+}
+
+.theme-black .inspire-tree .matched > .wholerow {
+  background: rgba(251, 191, 41, 0.25);
+}
 </style>

sist2-vue/src/components/ThumbnailProgressBar.vue

@@ -0,0 +1,40 @@
+<template>
+  <div class="thumbnail-progress-bar" :style="{width: `${percentProgress}%`}"></div>
+</template>
+
+<script>
+export default {
+  name: "ThumbnailProgressBar",
+  props: ["doc", "progress"],
+  computed: {
+    percentProgress() {
+      return Math.min(Math.max(this.progress * 100, 0), 100);
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+.thumbnail-progress-bar {
+  position: absolute;
+  left: 0;
+  bottom: 0;
+
+  height: 4px;
+  background: #2196f3AA;
+
+  z-index: 9;
+}
+
+.theme-black .thumbnail-progress-bar {
+  background: rgba(0, 188, 212, 0.95);
+}
+
+.sub-document .thumbnail-progress-bar {
+  max-width: calc(100% - 8px);
+  left: 4px;
+}
+
+
+</style>

sist2-vue/src/components/icons/ClipboardIcon.vue

@@ -0,0 +1,21 @@
+<template>
+  <svg style="width:24px;height:24px" viewBox="0 0 24 24">
+    <path
+        fill="currentColor"
+        d="M17,9H7V7H17M17,13H7V11H17M14,17H7V15H14M12,3A1,1 0 0,1 13,4A1,1 0 0,1 12,5A1,1 0 0,1 11,4A1,1 0 0,1 12,3M19,3H14.82C14.4,1.84 13.3,1 12,1C10.7,1 9.6,1.84 9.18,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3Z"/>
+  </svg>
+</template>
+
+<script>
+export default {
+  name: "ClipboardIcon"
+}
+</script>
+
+<style scoped>
+svg {
+  display: inline-block;
+  width: 20px;
+  height: 20px;
+}
+</style>