Bug fixes

Bump d3-color and d3 in /sist2-vue

.dockerignore

@@ -8,13 +8,13 @@ Testing/
 **/cmake_install.cmake
 **/CMakeCache.txt
 **/CMakeFiles/
+.cmake
 LICENSE
 Makefile
 **/*.md
 **/*.cbp
 VERSION
 **/node_modules/
-.git/
 sist2-*-linux-debug
 sist2-*-linux
 sist2_debug
@@ -27,4 +27,14 @@ sist2
 **/ext_libmobi
 **/ext_libwpd
 **/core
-*.a
+*.a
+tmp_scan/
+Dockerfile
+Dockerfile.arm64
+docker-compose.yml
+state.db
+*-journal
+build/
+__pycache__/
+sist2-vue/dist
+sist2-admin/frontend/dist

.drone.yml

@@ -11,21 +11,6 @@ steps:
     image: simon987/sist2-build
     commands:
       - ./scripts/build.sh
-  - name: docker
-    image: plugins/docker
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: simon987/sist2
-      context: ./
-      dockerfile: ./Dockerfile
-      auto_tag: true
-      auto_tag_suffix: x64-linux
-      when:
-        event:
-          - tag
   - name: scp files
     image: appleboy/drone-scp
     settings:
@@ -42,6 +27,21 @@ steps:
         - ./VERSION
         - ./sist2-x64-linux
         - ./sist2-x64-linux-debug
+  - name: docker
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: simon987/sist2
+      context: ./
+      dockerfile: ./Dockerfile
+      auto_tag: true
+      auto_tag_suffix: x64-linux
+      when:
+        event:
+          - tag
 
 ---
 kind: pipeline

.gitattributes

@@ -1,3 +0,0 @@
-CMakeModules/* linguist-vendored
-**/*_generated.c linguist-vendored
-**/*_generated.h linguist-vendored

.gitignore

@@ -10,6 +10,9 @@ Makefile
 LOG
 sist2*
 !sist2-vue/
+!sist2-admin
+!sist2_admin
+!sist2.py
 *.sist2/
 bundle*.css
 bundle.js
@@ -23,4 +26,21 @@ git_hash.h
 Testing/
 test_i
 test_i_inc
-node_modules/
+node_modules/
+.cmake/
+i_inc/
+state.db
+*.pyc
+!sist2-admin/frontend/dist
+*.js.map
+sist2-vue/dist
+sist2-admin/frontend/dist
+.ninja_deps
+.ninja_log
+build.ninja
+src/web/static_generated.c
+src/magic_generated.c
+src/index/static_generated.c
+*.sist2
+*-shm
+*-journal

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "third-party/libscan"]
-	path = third-party/libscan
-	url = https://github.com/simon987/libscan
 [submodule "third-party/argparse"]
 	path = third-party/argparse
 	url = https://github.com/simon987/argparse
@@ -10,3 +7,9 @@
 [submodule "third-party/libscan/third-party/antiword"]
 	path = third-party/libscan/third-party/antiword
 	url = https://github.com/simon987/antiword
+[submodule "third-party/libscan/third-party/libmobi"]
+	path = third-party/libscan/third-party/libmobi
+	url = https://github.com/bfabiszewski/libmobi
+[submodule "third-party/libscan/libscan-test-files"]
+	path = third-party/libscan/libscan-test-files
+	url = https://github.com/simon987/libscan-test-files

CMakeLists.txt

@@ -1,10 +1,11 @@
 cmake_minimum_required(VERSION 3.7)
+
+project(sist2)
 set(CMAKE_C_STANDARD 11)
 
-project(sist2 C)
-
 option(SIST_DEBUG "Build a debug executable" on)
-option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0)
+option(SIST_FAST "Enable more optimisation flags" off)
+option(SIST_DEBUG_INFO "Turn on debug information in web interface" on)
 
 add_compile_definitions(
         "SIST_PLATFORM=${SIST_PLATFORM}"
@@ -14,46 +15,62 @@ if (SIST_DEBUG)
     add_compile_definitions(
             "SIST_DEBUG=${SIST_DEBUG}"
     )
-endif()
+    set(VCPKG_BUILD_TYPE debug)
+else ()
+    set(VCPKG_BUILD_TYPE release)
+endif ()
+
+if (SIST_DEBUG_INFO)
+    add_compile_definitions(
+            "SIST_DEBUG_INFO=${SIST_DEBUG_INFO}"
+    )
+endif ()
+
 
 add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
 add_executable(sist2
-
         # argparse
         third-party/argparse/argparse.h third-party/argparse/argparse.c
 
         src/main.c
         src/sist.h
         src/io/walk.h src/io/walk.c
-        src/io/store.h src/io/store.c
         src/tpool.h src/tpool.c
         src/parsing/parse.h src/parsing/parse.c
+        src/parsing/magic_util.c src/parsing/magic_util.h
         src/io/serialize.h src/io/serialize.c
         src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
         src/index/web.c src/index/web.h
         src/web/serve.c src/web/serve.h
+        src/web/web_util.c src/web/web_util.h
         src/index/elastic.c src/index/elastic.h
         src/util.c src/util.h
-        src/ctx.h src/types.h
+        src/ctx.c src/ctx.h
+        src/types.h
         src/log.c src/log.h
         src/cli.c src/cli.h
-        src/stats.c src/stats.h src/ctx.c
+        src/parsing/sidecar.c src/parsing/sidecar.h
-        src/parsing/sidecar.c src/parsing/sidecar.h)
+        src/database/database.c src/database/database.h
+        src/parsing/fs_util.h
+
+        src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp
+
+        src/database/database_stats.c src/database/database_schema.c)
+set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
 set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
 
 find_package(PkgConfig REQUIRED)
 
-pkg_search_module(GLIB REQUIRED glib-2.0)
-
-find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
+find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
+find_package(unofficial-sqlite3 CONFIG REQUIRED)
 
 
 target_include_directories(
@@ -62,7 +79,6 @@ target_include_directories(
         ${CMAKE_SOURCE_DIR}/third-party/utf8.h/
         ${CMAKE_SOURCE_DIR}/third-party/libscan/
         ${CMAKE_SOURCE_DIR}/
-        ${GLIB_INCLUDE_DIRS}
 )
 
 target_compile_options(
@@ -80,25 +96,39 @@ if (SIST_DEBUG)
             -fno-omit-frame-pointer
             -fsanitize=address
             -fno-inline
-#            -O2
+            #            -O2
     )
     target_link_options(
             sist2
             PRIVATE
             -fsanitize=address
+            -static-libasan
     )
     set_target_properties(
             sist2
             PROPERTIES
             OUTPUT_NAME sist2_debug
     )
+elseif (SIST_FAST)
+    target_compile_options(
+            sist2
+            PRIVATE
+
+            -Ofast
+            -march=native
+            -fno-stack-protector
+            -fomit-frame-pointer
+            -freciprocal-math
+    )
 else ()
     target_compile_options(
             sist2
             PRIVATE
+
             -Ofast
             -fno-stack-protector
             -fomit-frame-pointer
+            -w
     )
 endif ()
 
@@ -112,19 +142,16 @@ target_link_libraries(
         sist2
 
         z
-        lmdb
-        cjson
         argparse
-        ${GLIB_LDFLAGS}
         unofficial::mongoose::mongoose
         CURL::libcurl
 
         pthread
-        magic
-
-        c
 
         scan
+
+        ${MAGIC_LIB}
+        unofficial::sqlite3::sqlite3
 )
 
 add_custom_target(

Dockerfile

@@ -1,15 +1,38 @@
 FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
+RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /build/
-COPY . .
-RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
-RUN make -j$(nproc)
-RUN strip sist2
 
-FROM ubuntu:21.10
+COPY scripts scripts
+COPY schema schema
+COPY CMakeLists.txt .
+COPY third-party third-party
+COPY src src
+COPY sist2-vue sist2-vue
+COPY sist2-admin sist2-admin
 
-RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
+RUN cd sist2-vue/ && npm install && npm run build
+RUN cd sist2-admin/frontend/ && npm install && npm run build
+
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN cd build && make -j$(nproc)
+RUN strip build/sist2 || mv build/sist2_debug build/sist2
+
+FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
+
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["/root/sist2"]
+
+RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 libmagic1 python3  \
+    python3-pip git tesseract-ocr && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -18,11 +41,17 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
     curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
+    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
-ENTRYPOINT ["/root/sist2"]
+# sist2
+COPY --from=build /build/build/sist2 /root/sist2
 
-ENV LANG C.UTF-8
+# sist2-admin
-ENV LC_ALL C.UTF-8
+WORKDIR /root/sist2-admin
-
+COPY sist2-admin/requirements.txt /root/sist2-admin/
-COPY --from=build /build/sist2 /root/sist2
+RUN python3 -m pip install --no-cache -r /root/sist2-admin/requirements.txt
+COPY --from=build /build/sist2-admin/ /root/sist2-admin/

Dockerfile.arm64

@@ -3,13 +3,20 @@ MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 ADD . /build/
-RUN cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
-RUN make -j$(nproc)
+RUN cd build && make -j$(nproc)
-RUN strip sist2
+RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
-FROM --platform="linux/arm64/v8" ubuntu:21.10
+FROM --platform=linux/arm64/v8 ubuntu@sha256:537da24818633b45fcb65e5285a68c3ec1f3db25f5ae5476a7757bc8dfae92a3
 
-RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
+WORKDIR /root
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["/root/sist2"]
+
+RUN apt update && apt install -y curl libasan5 libmagic1 tesseract-ocr python3-pip python3 git && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -18,11 +25,16 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
     curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
+    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
-ENV LANG C.UTF-8
+# sist2
-ENV LC_ALL C.UTF-8
+COPY --from=build /build/build/sist2 /root/sist2
 
-ENTRYPOINT ["/root/sist2"]
+# sist2-admin
-
+COPY sist2-admin/requirements.txt sist2-admin/
-COPY --from=build /build/sist2 /root/sist2
+RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
+COPY --from=build /build/sist2-admin/ sist2-admin/

README.md

@@ -10,13 +10,13 @@ sist2 (Simple incremental search tool)
 
 *Warning: sist2 is in early development*
 
-![search panel](docs/sist2.png)
+![search panel](docs/sist2.gif)
 
 ## Features
 
 * Fast, low memory usage, multi-threaded
+* Manage & schedule scan jobs with simple web interface (Docker only)
 * Mobile-friendly Web interface
-* Portable (all its features are packaged in a single executable)
 * Extracts text and metadata from common file types \*
 * Generates thumbnails \*
 * Incremental scanning
@@ -24,66 +24,80 @@ sist2 (Simple incremental search tool)
 * Recursive scan inside archive files \*\*
 * OCR support with tesseract \*\*\*
 * Stats page & disk utilisation visualization
+* Named-entity recognition (client-side) \*\*\*\*
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
 \*\*\* See [OCR](#ocr)
-
+\*\*\*\* See [Named-Entity Recognition](#NER)
-![stats](docs/stats.png)
 
 ## Getting Started
 
+### Using Docker Compose *(Windows/Linux/Mac)*
+
+```yaml
+version: "3"
+
+services:
+  elasticsearch:
+    image: elasticsearch:7.17.9
+    restart: unless-stopped
+    environment:
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+  sist2-admin:
+    image: simon987/sist2:3.0.3
+    restart: unless-stopped
+    volumes:
+      - ./sist2-admin-data/:/sist2-admin/
+      - /:/host
+    ports:
+      - 4090:4090 # sist2
+      - 8080:8080 # sist2-admin
+    working_dir: /root/sist2-admin/
+    entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
+```
+
+Navigate to http://localhost:8080/ to configure sist2-admin.
+
+### Using the executable file *(Linux/WSL only)*
+
 1. Have an Elasticsearch (>= 6.8.X, ideally >=7.14.0) instance running
     1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
-    1. *(or)* Run using docker:
+    2. *(or)* Run using docker:
         ```bash
-        docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.14.0
+        docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
         ```
-    1. *(or)* Run using docker-compose:
-        ```yaml
-          elasticsearch:
-            image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0
-            environment:
-              - discovery.type=single-node
-              - "ES_JAVA_OPTS=-Xms1G -Xmx2G"
-        ```
-1. Download sist2 executable
-    1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) *
-    1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
-       recommended!)*
-    1. *(or)* `docker pull simon987/sist2:2.11.4-x64-linux`
 
-1. See [Usage guide](docs/USAGE.md)
+2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
+   Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
+3. See [usage guide](docs/USAGE.md) for command line usage.
 
-\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
+Example usage:
 
-## Example usage
+1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
-
+2. Push index to Elasticsearch: `sist2 index ./documents.sist2`
-See [Usage guide](docs/USAGE.md) for more details
+3. Start web interface: `sist2 web ./documents.sist2`
-
-1. Scan a directory: `sist2 scan ~/Documents -o ./docs_idx`
-1. Push index to Elasticsearch: `sist2 index ./docs_idx`
-1. Start web interface: `sist2 web ./docs_idx`
 
 ## Format support
 
-File type | Library | Content | Thumbnail | Metadata
+| File type                                                                 | Library                                                                      | Content  | Thumbnail   | Metadata                                                                                                                               |
-:---|:---|:---|:---|:---
+|:--------------------------------------------------------------------------|:-----------------------------------------------------------------------------|:---------|:------------|:---------------------------------------------------------------------------------------------------------------------------------------|
-pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title |
+| pdf,xps,fb2,epub                                                          | MuPDF                                                                        | text+ocr | yes         | author, title                                                                                                                          |
-cbz,cbr | [libscan](https://github.com/simon987/libscan) | - | yes | - |
+| cbz,cbr                                                                   | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | -        | yes         | -                                                                                                                                      |
-`audio/*` | ffmpeg | - | yes | ID3 tags |
+| `audio/*`                                                                 | ffmpeg                                                                       | -        | yes         | ID3 tags                                                                                                                               |
-`video/*` | ffmpeg | - | yes | title, comment, artist |
+| `video/*`                                                                 | ffmpeg                                                                       | -        | yes         | title, comment, artist                                                                                                                 |
-`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
+| `image/*`                                                                 | ffmpeg                                                                       | ocr      | yes         | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
-raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags, GPS tags |
+| raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf | LibRaw                                                                       | no       | yes         | Common EXIF tags, GPS tags                                                                                                             |
-ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
+| ttf,ttc,cff,woff,fnt,otf                                                  | Freetype2                                                                    | -        | yes, `bmp`  | Name & style                                                                                                                           |
-`text/plain` | [libscan](https://github.com/simon987/libscan) | yes | no | - |
+| `text/plain`                                                              | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
-html, xml | [libscan](https://github.com/simon987/libscan) | yes | no | - |
+| html, xml                                                                 | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
-tar, zip, rar, 7z, ar ...  | Libarchive | yes\* | - | no |
+| tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
-docx, xlsx, pptx | [libscan](https://github.com/simon987/libscan) | yes | if embedded | creator, modified_by, title |
+| docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
-doc (MS Word 97-2003) | antiword | yes | yes | author, title |
+| doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
-mobi, azw, azw3 | libmobi | yes | no | author, title |
+| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | yes         | author, title                                                                                                                          |
-wpd (WordPerfect) | libwpd | yes | no | *planned* |
+| wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
-json, jsonl, ndjson | [libscan](https://github.com/simon987/libscan) | yes | - | - |
+| json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
 \* *See [Archive files](#archive-files)*
 
@@ -102,53 +116,83 @@ scan is also supported.
 
 ### OCR
 
-You can enable OCR support for pdf,xps,fb2,epub file types with the
+You can enable OCR support for ebook (pdf,xps,fb2,epub) or image file types with the
-`--ocr <lang>` option. Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
+`--ocr-lang <lang>` option in combination with `--ocr-images` and/or `--ocr-ebooks`.
+Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
 The `simon987/sist2` image comes with common languages
-(hin, jpn, eng, fra, rus, spa) pre-installed.
+(hin, jpn, eng, fra, rus, spa, chi_sim, deu) pre-installed.
 
-Examples
+You can use the `+` separator to specify multiple languages. The language
+name must be identical to the `*.traineddata` file installed on your system
+(use `chi_sim` rather than `chi-sim`).
+
+Examples:
 
 ```bash
-sist2 scan --ocr jpn ~/Books/Manga/
+sist2 scan --ocr-ebooks --ocr-lang jpn ~/Books/Manga/
-sist2 scan --ocr eng ~/Books/Textbooks/
+sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
+sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
 ```
 
+### NER
+
+sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to 
+**Configuration** > **Machine learning options** > **Model repositories**
+to enable it.
+
+The text processing is done in your browser, no data is sent to any third-party services.
+See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) for more details.
+
+#### List of available repositories:
+
+| URL                                                                                                     | Maintainer                              | Purpose |
+|---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
+| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
+
+
+<details>
+  <summary>Screenshot</summary>
+
+![ner](docs/ner.png)
+
+</details>
+
 ## Build from source
 
 You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries
 
-### With docker (recommended)
+### Using docker
 
 ```bash
 git clone --recursive https://github.com/simon987/sist2/
 cd sist2
-docker build . -f ./Dockerfile -t my-sist2-image
+docker build . -t my-sist2-image
-docker run --rm my-sist2-image cat /root/sist2 > sist2-x64-linux
+# Copy sist2 executable from docker image
+docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 ```
 
-### On a linux computer
+### Using a linux computer
 
 1. Install compile-time dependencies
 
    ```bash
-   apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git
+   apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
    ```
 
-1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
+2. Install vcpkg using my fork: https://github.com/simon987/vcpkg
-
+3. Install vcpkg dependencies
-1. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl]
+    vcpkg install curl[core,openssl] sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
-    vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libuuid libmagic libraw jasper lcms gumbo
     ```
 
-1. Build
+4. Build
     ```bash
     git clone --recursive https://github.com/simon987/sist2/
+    (cd sist2-vue; npm install; npm run build)
+    (cd sist2-admin/frontend; npm install; npm run build)
     cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

contrib/systemd/Makefile

@@ -0,0 +1,7 @@
+install:
+	install sist2-update-all.sh /usr/bin/sist2-update-all.sh
+	install sist2-update-files.sh /usr/bin/sist2-update-files.sh
+	install sist2-update-nextcloud.sh /usr/bin/sist2-update-nextcloud.sh
+	install sist2-update.service /etc/systemd/system/sist2-update.service
+	install sist2-update.timer /etc/systemd/system/sist2-update.timer
+	systemctl daemon-reload

contrib/systemd/README.md

@@ -0,0 +1,31 @@
+# Systemd integration example
+
+This example contains my (yatli) personal configuration for sist2 auto-updating.
+The following indices are involved in this configuration:
+
+| Index     | Path             | Description                                |
+|-----------|------------------|--------------------------------------------|
+| files     | /zpool/files     | Main file repository                       |
+| nextcloud | /zpool/nextcloud | Externally synchronized to a cloud account |
+
+The systemd integration achieves automatic sist2 scanning & indexing everyday at 3:00AM.
+
+### Tailoring the configuration for yourself
+
+`sist2-update-all.sh` calls update scripts for each sist2 index. Add or remove
+update scripts accordingly to suit your need. Each update script (e.g.
+`sist2-update-files.sh`) has important parameters laid down at the beginning so
+make sure to edit them to point to your files and index locations.
+
+### Installation
+
+```bash
+# install the services and scripts
+sudo make install
+# enable & start the timer
+sudo systemctl enable sist2-update.timer
+sudo systemctl start sist2-update.timer
+# verify that the timer has been enabled
+systemctl list-timers --all
+```
+

contrib/systemd/sist2-update-all.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+echo "Update index: Files"
+source ${__dir}/sist2-update-files.sh
+echo "Update index: Nextcloud"
+source ${__dir}/sist2-update-nextcloud.sh
+echo "Done. Restarting sist2."
+docker restart sist2-sist2-1

contrib/systemd/sist2-update-files.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+DATE=$(date +%Y_%m_%d)
+CONTENT=/zpool/files
+ORIG=/mnt/ssd/sist-index/files.idx
+NEW=/mnt/ssd/sist-index/files_$DATE.idx
+EXCLUDE='ZArchives|TorrentStore|TorrentDownload|624f0c59-1fef-44f6-95e9-7483296f2833|ubuntu-full-2021-12-07'
+NAME=Files
+#REWRITE_URL="http://localhost:33333/activate?collection=$NAME&path="
+REWRITE_URL=""
+
+sist2 scan \
+  --threads 14 \
+  --mem-throttle 32768 \
+  --thumbnail-quality 2 \
+  --name $NAME \
+  --ocr-lang=eng+chi_sim \
+  --ocr-ebooks \
+  --ocr-images \
+  --exclude=$EXCLUDE \
+  --rewrite-url=$REWRITE_URL \
+  --incremental=$ORIG \
+  --output=$NEW \
+  $CONTENT
+echo ">>> Scan complete"
+rm -rf $ORIG
+mv $NEW $ORIG 
+
+unset http_proxy
+unset https_proxy
+unset HTTP_PROXY
+unset HTTPS_PROXY
+sist2 index $ORIG --incremental-index
+echo ">>> Index complete"

contrib/systemd/sist2-update-nextcloud.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+DATE=$(date +%Y_%m_%d)
+CONTENT=/zpool/nextcloud/v-yadli
+ORIG=/mnt/ssd/sist-index/nextcloud.idx
+NEW=/mnt/ssd/sist-index/nextcloud_$DATE.idx
+EXCLUDE='Yatao|.*263418493\\/Image\\/.*'
+NAME=NextCloud
+# REWRITE_URL="http://localhost:33333/activate?collection=$NAME&path="
+REWRITE_URL=""
+
+sist2 scan \
+  --threads 14 \
+  --mem-throttle 32768 \
+  --thumbnail-quality 2 \
+  --name $NAME \
+  --ocr-lang=eng+chi_sim \
+  --ocr-ebooks \
+  --ocr-images \
+  --exclude=$EXCLUDE \
+  --rewrite-url=$REWRITE_URL \
+  --incremental=$ORIG \
+  --output=$NEW \
+  $CONTENT
+echo ">>> Scan complete"
+rm -rf $ORIG
+mv $NEW $ORIG 
+
+unset http_proxy
+unset https_proxy
+unset HTTP_PROXY
+unset HTTPS_PROXY
+sist2 index $ORIG --incremental-index

contrib/systemd/sist2-update.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=sist2-update
+
+[Service]
+User=yatli
+ExecStart=/bin/bash /usr/bin/sist2-update-all.sh

contrib/systemd/sist2-update.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=sist2-update
+
+[Timer]
+OnCalendar=*-*-* 3:00:00
+Persistent=true
+Unit=sist2-update.service
+
+[Install]
+WantedBy=timers.target

docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3"
+
+services:
+  elasticsearch:
+    image: elasticsearch:7.17.9
+    container_name: sist2-es
+    environment:
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+  sist2-admin:
+    build:
+      context: .
+    container_name: sist2-admin
+    volumes:
+      - /mnt/array/sist2-admin-data/:/sist2-admin/
+      - /:/host
+    ports:
+      - 4090:4090
+      # NOTE: Don't export this port publicly!
+      - 8080:8080
+    working_dir: /root/sist2-admin/
+    entrypoint: python3
+    command:
+      - /root/sist2-admin/sist2_admin/app.py

docs/USAGE.md

@@ -1,139 +1,92 @@
 # Usage
 
-*More examples (specifically with docker/compose) are in progress*
-
-* [scan](#scan)
-    * [options](#scan-options)
-    * [examples](#scan-examples)
-    * [index format](#index-format)
-* [index](#index)
-    * [options](#index-options)
-    * [examples](#index-examples)
-* [web](#web)
-    * [options](#web-options)
-    * [examples](#web-examples)
-    * [rewrite_url](#rewrite_url)
-    * [link to specific indices](#link-to-specific-indices)
-* [elasticsearch](#elasticsearch)
-* [exec-script](#exec-script)
-* [tagging](#tagging)
-* [sidecar files](#sidecar-files)
-
 ```
 Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
    or: sist2 exec-script [OPTION]... INDEX
+
 Lightning-fast file system indexer and search tool.
 
-    -h, --help                    show this help message and exit
+    -h, --help                        show this help message and exit
-    -v, --version                 Show version and exit
+    -v, --version                     Print version and exit.
-    --verbose                     Turn on logging
+    --verbose                         Turn on logging.
-    --very-verbose                Turn on debug messages
+    --very-verbose                    Turn on debug messages.
+    --json-logs                       Output logs in JSON format.
 
 Scan options
-    -t, --threads=<int>           Number of threads. DEFAULT=1
+    -t, --threads=<int>               Number of threads. DEFAULT: 1
-    -q, --quality=<flt>           Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=3
+    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2
-    --size=<int>                  Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
+    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
-    --content-size=<int>          Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
+    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
-    --incremental=<str>           Reuse an existing index and only scan modified files.
+    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
-    -o, --output=<str>            Output directory. DEFAULT=index.sist2/
+    -o, --output=<str>                Output index file path. DEFAULT: index.sist2
-    --rewrite-url=<str>           Serve files from this url instead of from disk.
+    --incremental                     If the output file path exists, only scan new or modified files.
-    --name=<str>                  Index display name. DEFAULT: (name of the directory)
+    --optimize-index                  Defragment index file after scan to reduce its file size.
-    --depth=<int>                 Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
+    --rewrite-url=<str>               Serve files from this url instead of from disk.
-    --archive=<str>               Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
+    --name=<str>                      Index display name. DEFAULT: index
-    --archive-passphrase=<str>    Passphrase for encrypted archive files
+    --depth=<int>                     Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --ocr=<str>                   Tesseract language (use tesseract --list-langs to see which are installed on your machine)
+    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: don't scan, list: only save file names as text, shallow: don't scan archives inside archives. DEFAULT: recurse
-    -e, --exclude=<str>           Files that match this regex will not be scanned
+    --archive-passphrase=<str>        Passphrase for encrypted archive files
-    --fast                        Only index file names & mime type
+    --ocr-lang=<str>                  Tesseract language (use 'tesseract --list-langs' to see which are installed on your machine)
-    --treemap-threshold=<str>     Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
+    --ocr-images                      Enable OCR'ing of image files.
-    --mem-buffer=<int>            Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000
+    --ocr-ebooks                      Enable OCR'ing of ebook files.
-    --read-subtitles              Read subtitles from media files.
+    -e, --exclude=<str>               Files that match this regex will not be scanned.
-    --fast-epub                   Faster but less accurate EPUB parsing (no thumbnails, metadata)
+    --fast                            Only index file names & mime type.
-    --checksums                   Calculate file checksums when scanning.
+    --treemap-threshold=<str>         Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
+    --mem-buffer=<int>                Maximum memory buffer size per thread in MiB for files inside archives (see USAGE.md). DEFAULT: 2000
+    --read-subtitles                  Read subtitles from media files.
+    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata).
+    --checksums                       Calculate file checksums when scanning.
+    --list-file=<str>                 Specify a list of newline-delimited paths to be scanned instead of normal directory traversal. Use '-' to read from stdin.
 
 Index options
-    -t, --threads=<int>           Number of threads. DEFAULT=1
+    -t, --threads=<int>               Number of threads. DEFAULT: 1
-    --es-url=<str>                Elasticsearch url with port. DEFAULT=http://localhost:9200
+    --es-url=<str>                    Elasticsearch url with port. DEFAULT: http://localhost:9200
-    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    -p, --print                   Just print JSON documents to stdout.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
-    --script-file=<str>           Path to user script.
+    -p, --print                       Print JSON documents to stdout instead of indexing to elasticsearch.
-    --mappings-file=<str>         Path to Elasticsearch mappings.
+    --incremental-index               Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
-    --settings-file=<str>         Path to Elasticsearch settings.
+    --script-file=<str>               Path to user script.
-    --async-script                Execute user script asynchronously.
+    --mappings-file=<str>             Path to Elasticsearch mappings.
-    --batch-size=<int>            Index batch size. DEFAULT: 100
+    --settings-file=<str>             Path to Elasticsearch settings.
-    -f, --force-reset             Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
+    --async-script                    Execute user script asynchronously.
+    --batch-size=<int>                Index batch size. DEFAULT: 70
+    -f, --force-reset                 Reset Elasticsearch mappings and settings.
 
 Web options
-    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
-    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    --bind=<str>                  Listen on this address. DEFAULT=localhost:4090
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
-    --auth=<str>                  Basic auth in user:password format
+    --bind=<str>                      Listen for connections on this address. DEFAULT: localhost:4090
-    --tag-auth=<str>              Basic auth in user:password format for tagging
+    --auth=<str>                      Basic auth in user:password format
-    --tagline=<str>               Tagline in navbar
+    --auth0-audience=<str>            API audience/identifier
-    --dev                         Serve html & js files from disk (for development)
+    --auth0-domain=<str>              Application domain
+    --auth0-client-id=<str>           Application client ID
+    --auth0-public-key-file=<str>     Path to Auth0 public key file extracted from <domain>/pem
+    --tag-auth=<str>                  Basic auth in user:password format for tagging
+    --tagline=<str>                   Tagline in navbar
+    --dev                             Serve html & js files from disk (for development)
+    --lang=<str>                      Default UI language. Can be changed by the user
 
 Exec-script options
-    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
-    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    --script-file=<str>           Path to user script.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
-    --async-script                Execute user script asynchronously.
+    --script-file=<str>               Path to user script.
+    --async-script                    Execute user script asynchronously.
+
+Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
-## Scan
+#### Thumbnail database size estimation
 
-### Scan options
+See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
 
-* `-t, --threads` 
+For example, `--thumbnail-size=500`, `--thumbnail-quality=2` for a directory with 8 million images will create a thumbnail database 
-      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-WmiObject Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
+that is about `8000000 * 36kB = 288GB`.
-* `-q, --quality` 
-    Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best.
-* `--size` 
-    Thumbnail size in pixels.
-* `--content-size` 
-    Number of bytes of text to be extracted from the content of files (plain text and PDFs).
-    Repeated whitespace and special characters do not count toward this limit.
-* `--incremental`
-    Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
-    will be copied to the new index and will not be parsed again.
-* `-o, --output` Output directory. 
-* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
-* `--name` Set the `name` option for the web module
-* `--depth` Maximum scan dept. Set to 0 only scan files directly in the root directory, set to -1 for infinite depth
-* `--archive` Archive file mode.
-    * skip: Don't parse
-    * list: Only get file names as text
-    * shallow: Don't parse archives inside archives.
-    * recurse: Scan archives recursively (default)
-* `--ocr` See [OCR](../README.md#OCR)
-* `-e, --exclude` Regex pattern to exclude files. A file is excluded if the pattern matches any 
-    part of the full absolute path.
-    
-    Examples: 
-    * `-e ".*\.ttf"`: Ignore ttf files
-    * `-e ".*\.(ttf|rar)"`: Ignore ttf and rar files
-    * `-e "^/mnt/backups/"`: Ignore all files in the `/mnt/backups/` directory
-    * `-e "^/mnt/Data[12]/"`: Ignore all files in the `/mnt/Data1/` and `/mnt/Data2/` directory
-    * `-e "(^/usr/)|(^/var/)|(^/media/DRIVE-A/tmp/)|(^/media/DRIVE-B/Trash/)"` Exclude the
-     `/usr`, `/var`, `/media/DRIVE-A/tmp`, `/media/DRIVE-B/Trash` directories
-* `--fast` Only index file names and mime type
-* `--treemap-threshold` Directories smaller than (`treemap-threshold` * `<total size of the index>`)
-    will not be considered for the disk utilisation visualization; their size will be added to
-    the parent directory. If the parent directory is still smaller than the threshold, it will also be "merged upwards"
-    and so on.
-    
-    In effect, smaller `treemap-threshold` values will yield a more detailed 
-    (but also a more cluttered and harder to read) visualization. 
-    
-* `--mem-buffer` Maximum memory buffer size in MB (per thread) for files inside archives. Media files 
-    larger than this number will be read sequentially and no *seek* operations will be supported.
 
-    To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
+![thumbnail_size](thumbnail_size.png)
-* `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
-* `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
-* `--checksums` Calculate file checksums (sha1) when scanning files. This option does not cause any additional read 
-  operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
-  files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
 
 ### Scan examples
 
@@ -142,82 +95,22 @@ Simple scan
 sist2 scan ~/Documents
 
 sist2 scan \
-    --threads 4 --content-size 16000000 --quality 1.0 --archive shallow \
+    --threads 4 --content-size 16000000 --thumbnail-quality 2 --archive shallow \
     --name "My Documents" --rewrite-url "http://nas.domain.local/My Documents/" \
-    ~/Documents -o ./documents.idx/
+    ~/Documents -o ./documents.sist2
 ```
 
 Incremental scan
-```
+
-sist2 scan --incremental ./orig_idx/ -o ./updated_idx/ ~/Documents
+If the index file does not exist, `--incremental` has no effect.
+```bash
+sist scan ~/Documents -o ./documents.sist2
+sist scan ~/Documents -o ./documents.sist2 --incremental
+# or
+sist scan ~/Documents -o ./documents.sist2 --incremental
+sist scan ~/Documents -o ./documents.sist2 --incremental
 ```
 
-### Index format
-
-A typical `ndjson` type index structure looks like this:
-```
-documents.idx/
-├── descriptor.json
-├── _index_main.ndjson.zst
-├── treemap.csv
-├── agg_mime.csv
-├── agg_date.csv
-├── add_size.csv
-├── thumbs/
-|   ├── data.mdb
-|   └── lock.mdb
-├── tags/
-|   ├── data.mdb
-|   └── lock.mdb
-└── meta/
-    ├── data.mdb
-    └── lock.mdb
-```
-
-The `_index_*.ndjson.zst` files contain the document data in JSON format, in a compressed newline-delemited file.
-
-The `thumbs/` folder is a [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database)
-database containing the thumbnails.
-
-The `descriptor.json` file contains general information about the index. The 
-following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rewrite_url) and `timestamp`.
-
-The `.csv` are pre-computed aggregations necessary for the stats page.
-
-*thumbs/*:
-
-LMDB key-value store. Keys are **binary** 16-byte md5 hash* (`_id` field)
-and values are raw image bytes.
-
-*\* Hash is calculated from the full path of the file, including the extension, relative to the index root*
-
-
-## Index
-### Index options
- * `--es-url` 
- Elasticsearch url and port. If you are using docker, make sure that both containers are on the
- same network.
- * `--es-index` 
-    Elasticsearch index name. DEFAULT=sist2
- * `-p, --print` 
-    Print index in JSON format to stdout.
- * `--script-file` 
-    Path to user script. See [Scripting](scripting.md).
- * `--mappings-file`
-    Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used.
- * `--settings-file`
-    Path to custom Elasticsearch settings. *(See above)*
- * `--async-script` 
-    Use `wait_for_completion=false` elasticsearch option while executing user script.
-     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
- * `--batch-size=<int>` 
-    Index batch size. Indexing is generally faster with larger batches, but payloads that
-    are too large will fail and additional overhead for retrying with smaller sizes may slow
-    down the process.
- * `-f, --force-reset` 
-    Reset Elasticsearch mappings and settings.
- * `-t, --threads` Number of threads to use. Ideally, choose a number equal to the number of logical cores of the machine hosting Elasticsearch.
-    
 ### Index examples
 
 **Push to elasticsearch**
@@ -248,7 +141,10 @@ sist2 index --print ./my_index/ | jq | less
     `--auth` argument, but authentication is only applied the `/tag/` endpoint.
  * `--tagline=<str>` When specified, will replace the default tagline in the navbar.
  * `--dev` Serve html & js files from disk (for development, used to modify frontend files without having to recompile)
- 
+ * `--lang=<str>` Set the default web UI language (See #180 for a list of supported languages, default
+   is `en`). The user can change the language in the configuration page
+ * `--auth0-audience`, `--auth0-domain`, `--auth0-client-id`, `--auth0-public-key-file` See [Authentication with Auth0](auth0.md)
+
 ### Web examples
 
 **Single index**
@@ -272,7 +168,7 @@ Both the `root` and `rewrite_url` fields are safe to manually modify from the
 
 # Elasticsearch
 
-Elasticsearch versions >=6.8.0, <8.0.0 are supported by sist2. 
+Elasticsearch versions >=6.8.0, 7.X.X and 8.X.X are supported by sist2. 
 
 Using a version >=7.14.0 is recommended to enable the following features:
 
@@ -343,8 +239,8 @@ The sidecar file must have exactly the same file path and the `.s2meta` suffix.
 ```
 
 ```
-sist2 scan ~/Documents -o ./docs.idx
+sist2 scan ~/Documents -o ./docs.sist2
-sist2 index ./docs.idx
+sist2 index ./docs.sist2
 ```
 
 *NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,

docs/auth0.md

@@ -0,0 +1,19 @@
+
+# Authentication with Auth0
+
+1. Create a new Auth0 application (Single page app)
+2. Create a new Auth0 API:
+   1. Choose `RS256` signing algorithm
+   2. Set identifier (audience) to `https://sist2`
+3. Download the Auth0 certificate from https://<domain>.auth0.com/pem (you can find the domain Applications->Basic information)
+4. Extract the public key from the certificate using `openssl x509 -pubkey -noout -in cert.pem > pubkey.txt`
+5. Start the sist2 web server
+
+Example options:
+```bash
+sist2 web \
+  --auth0-client-id XXX \
+  --auth0-audience https://sist2 \
+  --auth0-domain YYY.auth0.com \
+  --auth0-public-key-file /ZZZ/pubkey.txt
+```

schema/mappings.json

@@ -39,7 +39,7 @@
       "index": false
     },
     "thumbnail": {
-      "type": "keyword",
+      "type": "integer",
       "index": false
     },
     "videoc": {
@@ -67,7 +67,8 @@
       "index": false
     },
     "mtime": {
-      "type": "integer"
+      "type": "date",
+      "format": "epoch_millis"
     },
     "size": {
       "type": "long"

schema/settings.json

@@ -3,7 +3,7 @@
     "refresh_interval": "30s",
     "codec": "best_compression",
     "number_of_replicas": 0,
-    "highlight.max_analyzed_offset": 10000000
+    "highlight.max_analyzed_offset": 1000000
   },
   "analysis": {
     "tokenizer": {
@@ -16,7 +16,7 @@
         "delimiter": "."
       },
       "my_nGram_tokenizer": {
-        "type": "nGram",
+        "type": "ngram",
         "min_gram": 3,
         "max_gram": 3
       }

scripts/before_build.sh

@@ -1,10 +1,13 @@
 #!/usr/bin/env bash
 
-rm -rf index.sist2/
+(
+  cd ..
+  rm -rf index.sist2
 
-python3 scripts/mime.py > src/parsing/mime_generated.c
+  python3 scripts/mime.py > src/parsing/mime_generated.c
-python3 scripts/serve_static.py > src/web/static_generated.c
+  python3 scripts/serve_static.py > src/web/static_generated.c
-python3 scripts/index_static.py > src/index/static_generated.c
+  python3 scripts/index_static.py > src/index/static_generated.c
+  python3 scripts/magic_static.py > src/magic_generated.c
 
-printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h
+  printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h
-printf "static const char *const LibScanCommitHash = \"%s\";\n" $(cd third-party/libscan/ && git rev-parse HEAD) >> src/git_hash.h
+)

scripts/build.sh

@@ -4,14 +4,20 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
-rm -rf CMakeFiles CMakeCache.txt
+mkdir build
-cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+(
-make -j $(nproc)
+  cd build
-strip sist2
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
-./sist2 -v > VERSION
+  make -j $(nproc)
-mv sist2 sist2-x64-linux
+  strip sist2
+  ./sist2 -v > VERSION
+)
+mv build/sist2 sist2-x64-linux
 
-rm -rf CMakeFiles CMakeCache.txt
+(
-cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+  cd build
-make -j  $(nproc)
+  rm -rf CMakeFiles CMakeCache.txt
-mv sist2_debug sist2-x64-linux-debug
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  make -j  $(nproc)
+)
+mv build/sist2_debug sist2-x64-linux-debug

scripts/build_arm64.sh

@@ -4,14 +4,19 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
-rm -rf CMakeFiles CMakeCache.txt
+mkdir build
-cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+(
-make -j $(nproc)
+  cd build
-strip sist2
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
-mv sist2 sist2-arm64-linux
+  make -j $(nproc)
+  strip sist2
+)
+mv build/sist2 sist2-arm64-linux
 
 rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+(
-make -j $(nproc)
+  cd build
-strip sist2
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
-mv sist2_debug sist2-arm64-linux-debug
+  make -j $(nproc)
+)
+mv build/sist2_debug sist2-arm64-linux-debug

scripts/magic_static.py

@@ -0,0 +1,8 @@
+
+try:
+    with open("/usr/lib/file/magic.mgc", "rb") as f:
+        data = f.read()
+except:
+    data = bytes([])
+
+print("char magic_database_buffer[%d] = {%s};" % (len(data), ",".join(str(int(b)) for b in data)))

scripts/mime.csv

@@ -1,3 +1,4 @@
+application/x-matlab-data,mat
 application/arj, arj
 application/base64, mme
 application/binhex, hqx
@@ -29,7 +30,7 @@ application/mime, aps
 application/mspowerpoint, ppz
 application/msword, doc|dot|w6w|wiz|word
 application/netmc, mcp
-application/octet-stream, bin|dump|gpg
+application/octet-stream, bin|dump|gpg|pack|idx
 application/oda, oda
 application/ogg, ogv
 application/pdf, pdf
@@ -243,7 +244,7 @@ audio/make, funk|my|pfunk
 audio/midi, kar
 audio/mid, rmi
 audio/mp4, m4b
-audio/mpeg, m2a|mpa
+audio/mpeg, m2a|mpa|mpga
 audio/ogg, ogg
 audio/s3m, s3m
 audio/tsp-audio, tsi
@@ -346,6 +347,8 @@ text/mcf, mcf
 text/pascal, pas
 text/PGP,
 text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml|make|log|markdown|yaml
+text/x-script.python, pyx
+text/csv,
 application/vnd.coffeescript, coffee
 text/richtext, rt|rtf|rtx
 text/rtf,
@@ -382,7 +385,7 @@ text/x-pascal, p
 text/x-perl, pl
 text/x-php, php
 text/x-po, po
-text/x-python, py
+text/x-python, py|pyi
 text/x-ruby, rb
 text/x-sass, sass
 text/x-scss, scss

scripts/mime.py

@@ -1,3 +1,5 @@
+import zlib
+
 mimes = {}
 noparse = set()
 ext_in_hash = set()
@@ -135,24 +137,40 @@ def clean(t):
     return t.replace("/", "_").replace(".", "_").replace("+", "_").replace("-", "_")
 
 
+def crc(s):
+    return zlib.crc32(s.encode()) & 0xffffffff
+
+
 with open("scripts/mime.csv") as f:
     for l in f:
         mime, ext_list = l.split(",")
         if l.startswith("!"):
             mime = mime[1:]
             noparse.add(mime)
-        ext = [x.strip() for x in ext_list.split("|")]
+        ext = [x.strip() for x in ext_list.split("|") if x.strip() != ""]
         mimes[mime] = ext
 
+    seen_crc = set()
+    for ext in mimes.values():
+        for e in ext:
+            if crc(e) in seen_crc:
+                raise Exception("CRC32 collision")
+            seen_crc.add(crc(e))
+
+    seen_crc = set()
+    for mime in mimes.keys():
+        if crc(mime) in seen_crc:
+            raise Exception("CRC32 collision")
+        seen_crc.add(crc(mime))
+
     print("// **Generated by mime.py**")
     print("#ifndef MIME_GENERATED_C")
     print("#define MIME_GENERATED_C")
-    print("#include <glib.h>\n")
     print("#include <stdlib.h>\n")
     # Enum
     print("enum mime {")
     for mime, ext in sorted(mimes.items()):
-        print("    " + clean(mime) + "=" + mime_id(mime) + ",")
+        print(f"{clean(mime)}={mime_id(mime)},")
     print("};")
 
     # Enum -> string
@@ -163,20 +181,20 @@ with open("scripts/mime.csv") as f:
     print("default: return NULL;}}")
 
     # Ext -> Enum
-    print("GHashTable *mime_get_ext_table() {"
+    print("unsigned int mime_extension_lookup(unsigned long extension_crc32) {"
-          "GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);")
+          "switch (extension_crc32) {")
     for mime, ext in mimes.items():
-        for e in [e for e in ext if e]:
+        if len(ext) > 0:
-            print("g_hash_table_insert(ext_table, \"" + e + "\", (gpointer)" + clean(mime) + ");")
+            for e in ext:
-            if e in ext_in_hash:
+                print(f"case {crc(e)}:", end="")
-                raise Exception("extension already in hash: " + e)
+            print(f"return {clean(mime)};")
-            ext_in_hash.add(e)
+    print("default: return 0;}}")
-    print("return ext_table;}")
 
     # string -> Enum
-    print("GHashTable *mime_get_mime_table() {"
+    print("unsigned int mime_name_lookup(unsigned long mime_crc32) {"
-          "GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);")
+          "switch (mime_crc32) {")
-    for mime, ext in mimes.items():
+    for mime in mimes.keys():
-        print("g_hash_table_insert(mime_table, \"" + mime + "\", (gpointer)" + clean(mime) + ");")
+        print(f"case {crc(mime)}: return {clean(mime)};")
-    print("return mime_table;}")
+
+    print("default: return 0;}}")
     print("#endif")

scripts/reset.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-make clean
-rm -rf CMakeFiles/ CMakeCache.txt Makefile \
-  third-party/libscan/CMakeFiles third-party/libscan/CMakeCache.txt third-party/libscan/third-party/ext_ffmpeg \
-  third-party/libscan/third-party/ext_libmobi third-party/libscan/Makefile

scripts/start_dev_es.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es"\
+       	-p 9200:9200 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.17.9

scripts/start_dev_es_6.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es-6"\
+       	-p 9202:9200 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:6.8.0

scripts/start_dev_es_8.sh

@@ -0,0 +1,3 @@
+docker run --rm -it --name "sist2-dev-es"\
+       	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.7.0

sist2-admin/frontend/babel.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  presets: [
+    '@vue/cli-plugin-babel/preset'
+  ]
+}

sist2-admin/frontend/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "sist2-admin-vue",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "serve": "vue-cli-service serve",
+    "build": "vue-cli-service build",
+    "watch": "vue-cli-service build --watch"
+  },
+  "dependencies": {
+    "axios": "^0.27.2",
+    "bootstrap-vue": "^2.21.2",
+    "core-js": "^3.6.5",
+    "moment": "^2.29.3",
+    "socket.io-client": "^4.5.1",
+    "vue": "^2.6.14",
+    "vue-i18n": "^8.24.4",
+    "vue-router": "^3.5.4",
+    "vuex": "^3.4.0"
+  },
+  "devDependencies": {
+    "@vue/cli-plugin-babel": "~5.0.8",
+    "@vue/cli-plugin-router": "~5.0.8",
+    "@vue/cli-plugin-vuex": "~5.0.8",
+    "@vue/cli-service": "~5.0.8",
+    "babel-eslint": "^10.1.0",
+    "bootstrap": "^4.5.2",
+    "vue-template-compiler": "^2.6.11"
+  },
+  "eslintConfig": {
+    "root": true,
+    "env": {
+      "node": true
+    },
+    "extends": [
+      "plugin:vue/essential",
+      "eslint:recommended"
+    ],
+    "parserOptions": {
+      "parser": "babel-eslint"
+    },
+    "rules": {}
+  },
+  "browserslist": [
+    "> 1%",
+    "last 2 versions",
+    "not dead"
+  ]
+}

sist2-admin/frontend/public/index.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width,initial-scale=1.0">
+    <link rel="icon" href="<%= BASE_URL %>serve_favicon_ico.ico">
+    <title>sist2-admin</title>
+  </head>
+  <body>
+    <noscript>
+      <strong>We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled. Please enable it to continue.</strong>
+    </noscript>
+    <div id="app"></div>
+    <!-- built files will be auto injected -->
+  </body>
+</html>

sist2-admin/frontend/src/App.vue

@@ -0,0 +1,98 @@
+<template>
+  <div id="app">
+    <NavBar></NavBar>
+    <b-container class="pt-4">
+      <b-alert show dismissible variant="info">
+        This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
+        to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on Github</a>. Thank you!
+      </b-alert>
+      <router-view/>
+    </b-container>
+  </div>
+</template>
+
+<script>
+import NavBar from "@/components/NavBar";
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  components: {NavBar},
+  data() {
+    return {
+      socket: null
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getSist2AdminInfo()
+        .then(resp => this.$store.commit("setSist2AdminInfo", resp.data));
+    this.$store.dispatch("loadBrowserSettings");
+    this.connectNotifications();
+    // this.socket.onclose = this.connectNotifications;
+  },
+  methods: {
+    connectNotifications() {
+      this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
+      this.socket.onopen = () => {
+        this.socket.send("Hello from client");
+      }
+
+      this.socket.onmessage = e => {
+        const notification = JSON.parse(e.data);
+        if (notification.message) {
+          notification.messageString = this.$t(notification.message).toString();
+        }
+
+        this.$store.dispatch("notify", notification)
+      }
+    }
+  }
+}
+</script>
+
+<style>
+html, body {
+  height: 100%;
+}
+
+#app {
+  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  /*text-align: center;*/
+  color: #2c3e50;
+  padding-bottom: 1em;
+  min-height: 100%;
+}
+
+.info-icon {
+  width: 1rem;
+  margin-right: 0.2rem;
+  cursor: pointer;
+  line-height: 1rem;
+  height: 1rem;
+  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+  filter: brightness(45%);
+  display: block;
+}
+
+.tabs {
+  margin-top: 10px;
+}
+
+.modal-title {
+  text-overflow: ellipsis;
+  overflow: hidden;
+  white-space: nowrap;
+}
+
+@media screen and (min-width: 1500px) {
+  .container {
+    max-width: 1440px;
+  }
+}
+
+label {
+  margin-top: 0.5rem;
+  margin-bottom: 0;
+}
+</style>

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -0,0 +1,117 @@
+import axios from "axios";
+
+class Sist2AdminApi {
+
+    constructor() {
+        this.baseUrl = window.location.protocol + "//" + window.location.host;
+    }
+
+    getJobs() {
+        return axios.get(`${this.baseUrl}/api/job/`);
+    }
+
+    getFrontends() {
+        return axios.get(`${this.baseUrl}/api/frontend/`);
+    }
+
+    getTasks() {
+        return axios.get(`${this.baseUrl}/api/task/`);
+    }
+
+    killTask(taskId) {
+        return axios.post(`${this.baseUrl}/api/task/${taskId}/kill`)
+    }
+
+    getTaskHistory() {
+        return axios.get(`${this.baseUrl}/api/task/history`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    getJob(name) {
+        return axios.get(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    getFrontend(name) {
+        return axios.get(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    startFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}/start`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    stopFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}/stop`);
+    }
+
+    /**
+     * @param {string} name
+     * @param job
+     */
+    updateJob(name, job) {
+        return axios.put(`${this.baseUrl}/api/job/${name}`, job);
+    }
+
+    /**
+     * @param {string} name
+     * @param frontend
+     */
+    updateFrontend(name, frontend) {
+        return axios.put(`${this.baseUrl}/api/frontend/${name}`, frontend);
+    }
+
+    /**
+     * @param {string} name
+     */
+    runJob(name) {
+        return axios.get(`${this.baseUrl}/api/job/${name}/run`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    deleteJob(name) {
+        return axios.delete(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    deleteFrontend(name) {
+        return axios.delete(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    createJob(name) {
+        return axios.post(`${this.baseUrl}/api/job/${name}`);
+    }
+
+    /**
+     * @param {string} name
+     */
+    createFrontend(name) {
+        return axios.post(`${this.baseUrl}/api/frontend/${name}`);
+    }
+
+    pingEs(url, insecure) {
+        return axios.get(`${this.baseUrl}/api/ping_es`, {params: {url, insecure}});
+    }
+
+    getSist2AdminInfo() {
+        return axios.get(`${this.baseUrl}/api/`);
+    }
+}
+
+export default new Sist2AdminApi()

sist2-admin/frontend/src/components/FrontendListItem.vue

@@ -0,0 +1,31 @@
+<template>
+  <b-list-group-item action :to="`/frontend/${frontend.name}`">
+
+    <div class="d-flex w-100 justify-content-between">
+      <h5 class="mb-1" style="display: block">
+        {{ frontend.name }}
+        <b-badge variant="light">{{ formatBindAddress(frontend.web_options.bind) }}</b-badge>
+      </h5>
+
+      <div>
+        <b-badge v-if="frontend.running" variant="success">{{$t("online")}}</b-badge>
+        <b-badge v-else variant="secondary">{{$t("offline")}}</b-badge>
+      </div>
+    </div>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import {formatBindAddress} from "@/util";
+
+export default {
+  name: "FrontendListItem",
+  props: ["frontend"],
+  data() {
+    return {
+      formatBindAddress
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/components/IndexOptions.vue

@@ -0,0 +1,64 @@
+<template>
+  <div>
+    <label>{{ $t("indexOptions.threads") }}</label>
+    <b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.esUrl") }}</label>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <label>{{ $t("indexOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
+    <br>
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("indexOptions.batchSize") }}</label>
+    <b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("indexOptions.script") }}</label>
+    <b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
+  </div>
+
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "IndexOptions",
+  props: ["options"],
+  data() {
+    return {
+      showEsTestAlert: false,
+      esTestOk: false,
+      esTestMessage: "",
+    }
+  },
+  methods: {
+    update() {
+      this.$emit("change", this.options);
+    },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
+  },
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/JobCheckboxGroup.vue

@@ -0,0 +1,42 @@
+<template>
+  <div>
+    <h5>{{ $t("selectJobs") }}</h5>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-form-group v-else>
+      <b-form-checkbox-group
+          v-if="jobs.length > 0"
+          :checked="frontend.jobs"
+          @input="frontend.jobs = $event; $emit('input')"
+      >
+        <div v-for="job in jobs" :key="job.name">
+          <b-form-checkbox :disabled="job.status !== 'indexed'" :value="job.name">[{{ job.name }}]</b-form-checkbox>
+          <br/>
+        </div>
+      </b-form-checkbox-group>
+      <div v-else>
+        <span class="text-muted">{{ $t('jobOptions.noJobAvailable') }}</span>
+        &nbsp;<router-link to="/">{{$t("create")}}</router-link>
+      </div>
+    </b-form-group>
+  </div>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "JobCheckboxGroup",
+  props: ["frontend"],
+  mounted() {
+    Sist2AdminApi.getJobs().then(resp => {
+      this.jobs = resp.data;
+      this.loading = false;
+    });
+  },
+  data() {
+    return {
+      loading: true,
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/components/JobListItem.vue

@@ -0,0 +1,56 @@
+<template>
+  <b-list-group-item class="flex-column align-items-start" action :to="`job/${job.name}`">
+
+    <div class="d-flex w-100 justify-content-between">
+      <div>
+        <h5 class="mb-1">
+          {{ job.name }}
+        </h5>
+      </div>
+      <div>
+        <b-row>
+          <b-col>
+            <small v-if="job.last_index_date">
+              {{ $t("scanned") }} {{ formatLastIndexDate(job.last_index_date) }}</small>
+            <div v-else>&nbsp;</div>
+          </b-col>
+        </b-row>
+        <b-row v-if="job.schedule_enabled">
+          <b-col>
+            <small><code>{{job.cron_expression }}</code></small>
+          </b-col>
+        </b-row>
+        <b-row v-else>
+          <b-col>
+            &nbsp;
+          </b-col>
+        </b-row>
+
+      </div>
+    </div>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import moment from "moment";
+
+export default {
+  name: "JobListItem",
+  props: ["job"],
+  methods: {
+    formatLastIndexDate(dateString) {
+      if (dateString === null) {
+        return "";
+      }
+
+      const date = Date.parse(dateString);
+      return moment(date).fromNow();
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/JobOptions.vue

@@ -0,0 +1,57 @@
+<template>
+  <div>
+    <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
+      {{ $t("jobOptions.desktopNotifications") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
+      {{ $t("jobOptions.scheduleEnabled") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("jobOptions.cron") }}</label>
+    <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression" :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
+  </div>
+</template>
+
+<script>
+
+export default {
+  name: "JobOptions",
+  props: ["job"],
+  data() {
+    return {
+      cronValid: undefined
+    }
+  },
+  computed: {
+    desktopNotificationsEnabled() {
+      return this.$store.state.jobDesktopNotificationMap[this.job.name];
+    }
+  },
+    mounted() {
+      this.cronValid = this.checkCron(this.job.cron_expression)
+    },
+    methods: {
+      checkCron(expression) {
+          return /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(expression);
+      },
+    updateNotifications(value) {
+      this.$store.dispatch("setJobDesktopNotification", {
+        job: this.job.name,
+        enabled: value
+      });
+    },
+    update() {
+      if (this.job.schedule_enabled) {
+        this.cronValid = this.checkCron(this.job.cron_expression);
+      } else {
+        this.cronValid = undefined;
+      }
+
+      if (this.cronValid !== false) {
+        this.$emit("change", this.job);
+      }
+    },
+  },
+}
+</script>

sist2-admin/frontend/src/components/NavBar.vue

@@ -0,0 +1,69 @@
+<template>
+  <b-navbar>
+    <b-navbar-brand to="/">
+      <Sist2Icon></Sist2Icon>
+    </b-navbar-brand>
+
+    <b-button class="ml-auto" to="/task" variant="link">{{ $t("tasks") }}</b-button>
+  </b-navbar>
+</template>
+
+<script>
+import Sist2Icon from "@/components/icons/Sist2Icon";
+
+export default {
+  name: "NavBar",
+  components: {Sist2Icon},
+  methods: {
+    tagline() {
+      return this.$store.state.sist2Info.tagline;
+    },
+    sist2Version() {
+      return this.$store.state.sist2Info.version;
+    },
+    isDebug() {
+      return this.$store.state.sist2Info.debug;
+    },
+    isLegacy() {
+      return this.$store.state.sist2Info.esVersionLegacy;
+    },
+    hideLegacy() {
+      return this.$store.state.optHideLegacy;
+    }
+  }
+}
+</script>
+
+<style scoped>
+.navbar {
+  box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
+  border-radius: 0;
+}
+
+.theme-black .navbar {
+  background: #546b7a30;
+  border-bottom: none;
+}
+
+.navbar-brand {
+  color: #222 !important;
+  font-size: 1.75rem;
+  padding: 0;
+}
+
+.navbar-brand:hover {
+  color: #000 !important;
+}
+
+.version {
+  color: #222 !important;
+  margin-left: -18px;
+  margin-top: -14px;
+  font-size: 11px;
+  font-family: monospace;
+}
+
+.btn-link {
+  color: #222;
+}
+</style>

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -0,0 +1,109 @@
+<template>
+  <div>
+    <label>{{ $t("scanOptions.path") }}</label>
+    <b-form-input v-model="options.path" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.threads") }}</label>
+    <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
+    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailCount") }}</label>
+    <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.thumbnailSize") }}</label>
+    <b-form-input type="number" min="100" v-model="options.thumbnail_size" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.contentSize") }}</label>
+    <b-form-input type="number" min="0" v-model="options.content_size" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.rewriteUrl") }}</label>
+    <b-form-input v-model="options.rewrite_url" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.depth") }}</label>
+    <b-form-input type="number" min="0" v-model="options.depth" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.archive") }}</label>
+    <b-form-select :options="['skip', 'list', 'shallow', 'recurse']" v-model="options.archive"
+                   @change="update()"></b-form-select>
+
+    <label>{{ $t("scanOptions.archivePassphrase") }}</label>
+    <b-form-input v-model="options.archive_passphrase" @change="update()"></b-form-input>
+
+    <label>{{ $t("scanOptions.ocrLang") }}</label>
+    <b-alert variant="danger" show v-if="selectedOcrLangs.length === 0 && !disableOcrLang">{{ $t("scanOptions.ocrLangAlert") }}</b-alert>
+    <b-checkbox-group :disabled="disableOcrLang" v-model="selectedOcrLangs" @input="onOcrLangChange">
+      <b-checkbox v-for="lang in ocrLangs" :key="lang" :value="lang">{{ lang }}</b-checkbox>
+    </b-checkbox-group>
+
+    <!--    <b-form-input readonly v-model="options.ocr_lang" @change="update()"></b-form-input>-->
+
+    <div style="height: 10px"></div>
+
+    <b-form-checkbox v-model="options.ocr_images" @change="update()">
+      {{ $t("scanOptions.ocrImages") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.ocr_ebooks" @change="update()">
+      {{ $t("scanOptions.ocrEbooks") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("scanOptions.exclude") }}</label>
+    <b-form-input v-model="options.exclude" @change="update()"
+                  :placeholder="$t('scanOptions.excludePlaceholder')"></b-form-input>
+
+    <div style="height: 10px"></div>
+
+    <b-form-checkbox v-model="options.fast" @change="update()">
+      {{ $t("scanOptions.fast") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.checksums" @change="update()">
+      {{ $t("scanOptions.checksums") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.read_subtitles" @change="update()">
+      {{ $t("scanOptions.readSubtitles") }}
+    </b-form-checkbox>
+
+    <b-form-checkbox v-model="options.optimize_index" @change="update()">
+        {{ $t("scanOptions.optimizeIndex") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("scanOptions.treemapThreshold") }}</label>
+    <b-form-input type="number" min="0" v-model="options.treemap_threshold" @change="update()"></b-form-input>
+  </div>
+</template>
+
+<script>
+
+export default {
+  name: "ScanOptions",
+  props: ["options"],
+  data() {
+    return {
+      disableOcrLang: false,
+      selectedOcrLangs: []
+    }
+  },
+  computed: {
+    ocrLangs() {
+      return this.$store.state.sist2AdminInfo?.tesseract_langs || [];
+    }
+  },
+  methods: {
+    onOcrLangChange() {
+      this.options.ocr_lang = this.selectedOcrLangs.join("+");
+    },
+    update() {
+      this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;
+      this.$emit("change", this.options);
+    },
+  },
+  mounted() {
+    this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;
+    this.selectedOcrLangs = this.options.ocr_lang ? this.options.ocr_lang.split("+") : [];
+  }
+}
+</script>

sist2-admin/frontend/src/components/TaskListItem.vue

@@ -0,0 +1,57 @@
+<template>
+  <b-list-group-item>
+    <b-row style="height: 50px">
+      <b-col><h5>{{ task.display_name }}</h5></b-col>
+      <b-col class="shrink">
+        <router-link class="btn btn-link" :to="`/log/${task.id}`">{{ $t("logs") }}</router-link>
+      </b-col>
+      <b-col class="shrink">
+        <b-btn variant="link" @click="killTask(task.id)">{{ $t("kill") }}</b-btn>
+      </b-col>
+    </b-row>
+
+    <b-row>
+      <b-col>
+        <b-progress :max="task.progress.count">
+          <b-progress-bar :value="task.progress.done" :label-html="label" :striped="!task.progress.waiting"/>
+        </b-progress>
+      </b-col>
+    </b-row>
+
+  </b-list-group-item>
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "TaskListItem",
+  props: ["task"],
+  computed: {
+    label() {
+
+      const count = this.task.progress.count;
+      const done = this.task.progress.done;
+
+      return `<span>${done}/${count}</span>`
+    }
+  },
+  methods: {
+    killTask(taskId) {
+      sist2AdminApi.killTask(taskId).then(() => {
+        this.$bvToast.toast(this.$t("killConfirmation"), {
+          title: this.$t("killConfirmationTitle"),
+          variant: "success",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    }
+  }
+}
+</script>
+
+<style scoped>
+.shrink {
+  flex-grow: inherit;
+}
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -0,0 +1,91 @@
+<template>
+  <div>
+    <label>{{ $t("webOptions.esUrl") }}</label>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("webOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.lang") }}</label>
+    <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
+
+    <label>{{ $t("webOptions.bind") }}</label>
+    <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.tagline") }}</label>
+    <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+
+    <label>{{ $t("webOptions.auth") }}</label>
+    <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.tagAuth") }}</label>
+    <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+
+    <br>
+    <h5>Auth0 options</h5>
+    <label>{{ $t("webOptions.auth0Audience") }}</label>
+    <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.auth0Domain") }}</label>
+    <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.auth0ClientId") }}</label>
+    <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.auth0PublicKey") }}</label>
+    <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
+
+
+
+  </div>
+</template>
+
+<script>
+
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "WebOptions",
+  props: ["options", "frontendName"],
+  data() {
+    return {
+      showEsTestAlert: false,
+      esTestOk: false,
+      esTestMessage: "",
+    }
+  },
+  methods: {
+    update() {
+      if (!this.options.es_url.startsWith("https")) {
+        this.options.es_insecure_ssl = false;
+      }
+
+      this.$emit("change", this.options);
+    },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/i18n/messages.js

@@ -0,0 +1,114 @@
+export default {
+    en: {
+        start: "Start",
+        stop: "Stop",
+        go: "Go",
+        online: "online",
+        offline: "offline",
+        delete: "Delete",
+        runNow: "Index now",
+        create: "Create",
+        test: "Test",
+
+        jobTitle: "job configuration",
+        tasks: "Tasks",
+        runningTasks: "Running tasks",
+        frontends: "Frontends",
+        jobDisabled: "There is no valid index for this job",
+        status: "Status",
+
+        taskHistory: "Task history",
+        taskName: "Task name",
+        taskStarted: "Started",
+        taskDuration: "Duration",
+        taskStatus: "Status",
+        logs: "Logs",
+        kill: "Kill",
+        killConfirmation: "SIGTERM signal sent to sist2 process",
+        killConfirmationTitle: "Confirmation",
+        follow: "Follow",
+        wholeFile: "Whole file",
+        logLevel: "Log level",
+        logMode: "Follow mode",
+        logFile: "Reading log file",
+
+        jobs: "Jobs",
+        newJobName: "New job name",
+        newJobHelp: "Create a new job to get started!",
+        newFrontendName: "New frontend name",
+        scanned: "last scan",
+        autoStart: "Start automatically",
+
+        runJobConfirmationTitle: "Task queued",
+        runJobConfirmation: "Check the Tasks page to monitor the status.",
+
+        extraQueryArgs: "Extra query arguments when launching from sist2-admin",
+        customUrl: "Custom URL when launching from sist2-admin",
+
+        selectJobs: "Select jobs",
+        webOptions: {
+            title: "Web options",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            lang: "UI Language",
+            bind: "Listen address",
+            tagline: "Tagline in navbar",
+            auth: "Basic auth in user:password format",
+            tagAuth: "Basic auth in user:password format for tagging",
+            auth0Audience: "Auth0 audience",
+            auth0Domain: "Auth0 domain",
+            auth0ClientId: "Auth0 client ID",
+            auth0PublicKey: "Auth0 public key",
+        },
+        scanOptions: {
+            title: "Scanning options",
+            path: "Path",
+            threads: "Number of threads",
+            memThrottle: "Total memory threshold in MiB for scan throttling",
+            thumbnailQuality: "Thumbnail quality, on a scale of 2 to 32, 2 being the best",
+            thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
+            thumbnailSize: "Thumbnail size, in pixels",
+            contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",
+            rewriteUrl: "Serve files from this url instead of from disk",
+            depth: "Scan up to this many subdirectories deep",
+            archive: "Archive file mode",
+            archivePassphrase: "Passphrase for encrypted archive files",
+            ocrLang: "Tesseract language",
+            ocrLangAlert: "You must select at least one language",
+            ocrEbooks: "Enable OCR'ing of ebook files",
+            ocrImages: "Enable OCR'ing of image files",
+            exclude: "Files that match this regex will not be scanned",
+            excludePlaceholder: "Exclude",
+            fast: "Only index file names & mime type",
+            checksums: "Calculate file checksums when scanning",
+            readSubtitles: "Read subtitles from media files",
+            memBuffer: "Maximum memory buffer size per thread in MiB for files inside archives",
+            treemapThreshold: "Relative size threshold for treemap",
+            optimizeIndex: "Defragment index file after scan to reduce its file size."
+        },
+        indexOptions: {
+            title: "Indexing options",
+            threads: "Number of threads",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            batchSize: "Index batch size",
+            script: "User script"
+        },
+        jobOptions: {
+            title: "Job options",
+            cron: "Job schedule",
+            scheduleEnabled: "Enable scheduled re-scan",
+            noJobAvailable: "No jobs available.",
+            desktopNotifications: "Desktop notifications"
+        },
+        frontendOptions: {
+            title: "Frontend options",
+            noJobSelectedWarning: "You must select at least one job to start this frontend"
+        },
+        notifications: {
+            indexCompleted: "Task completed for [$JOB$]"
+        }
+    }
+}

sist2-admin/frontend/src/main.js

@@ -0,0 +1,31 @@
+import Vue from 'vue'
+
+import { BootstrapVue, IconsPlugin } from 'bootstrap-vue'
+
+import "bootstrap/dist/css/bootstrap.min.css"
+import "bootstrap-vue/dist/bootstrap-vue.min.css"
+
+Vue.use(BootstrapVue);
+Vue.use(IconsPlugin);
+
+import App from './App.vue';
+import router from './router';
+import store from './store';
+import VueI18n from "vue-i18n";
+import messages from "@/i18n/messages";
+
+Vue.use(VueI18n);
+
+const i18n = new VueI18n({
+  locale: "en",
+  messages: messages
+});
+
+Vue.config.productionTip = false
+
+new Vue({
+  router,
+  store,
+  i18n,
+  render: h => h(App)
+}).$mount('#app')

sist2-admin/frontend/src/router/index.js

@@ -0,0 +1,45 @@
+import Vue from 'vue'
+import VueRouter from 'vue-router'
+import Home from '../views/Home.vue'
+import Job from "@/views/Job";
+import Tasks from "@/views/Tasks";
+import Frontend from "@/views/Frontend";
+import Tail from "@/views/Tail";
+
+Vue.use(VueRouter);
+
+const routes = [
+  {
+    path: "/",
+    name: "Home",
+    component: Home
+  },
+  {
+    path: "/job/:name",
+    name: "Job",
+    component: Job
+  },
+  {
+    path: "/task/",
+    name: "Tasks",
+    component: Tasks
+  },
+  {
+    path: "/frontend/:name",
+    name: "Frontend",
+    component: Frontend
+  },
+  {
+    path: "/log/:taskId",
+    name: "Tail",
+    component: Tail
+  },
+]
+
+const router = new VueRouter({
+  mode: "hash",
+  base: process.env.BASE_URL,
+  routes
+})
+
+export default router

sist2-admin/frontend/src/store/index.js

@@ -0,0 +1,63 @@
+import Vue from "vue";
+import Vuex from "vuex";
+
+Vue.use(Vuex);
+
+function saveBrowserSettings(state) {
+    const settings = {
+        jobDesktopNotificationMap: state.jobDesktopNotificationMap
+    };
+    localStorage.setItem("sist2-admin-settings", JSON.stringify(settings));
+
+    console.log("SAVED");
+    console.log(settings);
+}
+
+export default new Vuex.Store({
+    state: {
+        sist2AdminInfo: null,
+        jobDesktopNotificationMap: {}
+    },
+    mutations: {
+        setSist2AdminInfo: (state, payload) => state.sist2AdminInfo = payload,
+        setJobDesktopNotificationMap: (state, payload) => state.jobDesktopNotificationMap = payload,
+    },
+    actions: {
+        notify: async ({state}, notification) => {
+
+            if (!state.jobDesktopNotificationMap[notification.job]) {
+                console.log("pass");
+                return;
+            }
+
+            new Notification(notification.messageString.replace("$JOB$", notification.job));
+        },
+        setJobDesktopNotification: async ({state}, {job, enabled}) => {
+
+            if (enabled === true) {
+                const permission = await Notification.requestPermission()
+
+                if (permission !== "granted") {
+                    return false;
+                }
+            }
+
+            state.jobDesktopNotificationMap[job] = enabled;
+            saveBrowserSettings(state);
+
+            return true;
+        },
+        loadBrowserSettings({commit}) {
+            const settingString = localStorage.getItem("sist2-admin-settings");
+
+            if (!settingString) {
+                return;
+            }
+
+            const settings = JSON.parse(settingString);
+
+            commit("setJobDesktopNotificationMap", settings["jobDesktopNotificationMap"]);
+        }
+    },
+    modules: {}
+})

sist2-admin/frontend/src/util.js

@@ -0,0 +1,8 @@
+
+export function formatBindAddress(address) {
+    if (address.startsWith("0.0.0.0")) {
+        return address.slice("0.0.0.0".length)
+    }
+
+    return address
+}

sist2-admin/frontend/src/views/Frontend.vue

@@ -0,0 +1,129 @@
+<template>
+  <b-card>
+    <b-card-title>
+      {{ name }}
+      <small style="vertical-align: top">
+        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+      </small>
+    </b-card-title>
+
+    <div class="mb-3" v-if="!loading">
+      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+          $t("start")
+        }}
+      </b-button>
+      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+          $t("stop")
+        }}
+      </b-button>
+      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+        {{ $t("go") }}
+      </b-button>
+      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+    </div>
+
+
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card-body v-else>
+
+      <h4>{{ $t("frontendOptions.title") }}</h4>
+      <b-card>
+        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+          {{ $t("autoStart") }}
+        </b-form-checkbox>
+
+        <label>{{ $t("extraQueryArgs") }}</label>
+        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+
+        <label>{{ $t("customUrl") }}</label>
+        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+
+        <br/>
+
+        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+
+        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("webOptions.title") }}</h4>
+      <b-card>
+        <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
+      </b-card>
+    </b-card-body>
+
+  </b-card>
+</template>
+
+<script>
+
+import Sist2AdminApi from "@/Sist2AdminApi";
+import JobCheckboxGroup from "@/components/JobCheckboxGroup";
+import WebOptions from "@/components/WebOptions";
+
+export default {
+  name: 'Frontend',
+  components: {JobCheckboxGroup, WebOptions},
+  data() {
+    return {
+      loading: true,
+      frontend: null,
+    }
+  },
+  computed: {
+    valid() {
+      return !this.loading && this.frontend.jobs.length > 0;
+    },
+    frontendUrl() {
+      if (this.frontend.custom_url) {
+        return this.frontend.custom_url + this.args;
+      }
+
+      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+      }
+
+      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+    },
+    name() {
+      return this.$route.params.name;
+    },
+    port() {
+      return this.frontend.web_options.bind.split(":")[1]
+    },
+    args() {
+      const args = this.frontend.extra_query_args;
+      if (args !== "") {
+        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+      }
+      return "";
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getFrontend(this.name).then(resp => {
+      this.frontend = resp.data;
+      this.loading = false;
+    });
+  },
+  methods: {
+    start() {
+      this.frontend.running = true;
+      Sist2AdminApi.startFrontend(this.name)
+    },
+    stop() {
+      this.frontend.running = false;
+      Sist2AdminApi.stopFrontend(this.name)
+    },
+    deleteFrontend() {
+      Sist2AdminApi.deleteFrontend(this.name).then(() => {
+        this.$router.push("/frontends");
+      });
+    },
+    update() {
+      Sist2AdminApi.updateFrontend(this.name, this.frontend);
+    },
+  }
+}
+</script>

sist2-admin/frontend/src/views/Home.vue

@@ -0,0 +1,122 @@
+<template>
+  <div>
+    <b-card>
+      <b-card-title>{{ $t("jobs") }}</b-card-title>
+      <b-row>
+        <b-col>
+          <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
+          <b-popover
+              :show.sync="showHelp"
+              target="new-job"
+              placement="top"
+              triggers="manual"
+              variant="primary"
+              :content="$t('newJobHelp')"
+          ></b-popover>
+        </b-col>
+        <b-col>
+          <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
+          </b-button>
+        </b-col>
+      </b-row>
+
+      <hr/>
+
+      <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
+      <b-list-group v-else>
+        <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
+      </b-list-group>
+    </b-card>
+
+    <br/>
+
+    <b-card>
+
+      <b-card-title>{{ $t("frontends") }}</b-card-title>
+
+      <b-row>
+        <b-col>
+          <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
+        </b-col>
+        <b-col>
+          <b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
+            {{ $t("create") }}
+          </b-button>
+        </b-col>
+      </b-row>
+
+      <hr/>
+
+      <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
+      <b-list-group v-else>
+        <FrontendListItem v-for="frontend in frontends"
+                          :key="frontend.name" :frontend="frontend"></FrontendListItem>
+      </b-list-group>
+
+    </b-card>
+  </div>
+</template>
+
+<script>
+import JobListItem from "@/components/JobListItem";
+import {formatBindAddress} from "@/util";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import FrontendListItem from "@/components/FrontendListItem";
+
+export default {
+  name: "Jobs",
+  components: {JobListItem, FrontendListItem},
+  data() {
+    return {
+      jobsLoading: true,
+      newJobName: "",
+      jobs: [],
+
+      frontendsLoading: true,
+      frontends: [],
+      formatBindAddress,
+      newFrontendName: "",
+
+      showHelp: false
+    }
+  },
+  mounted() {
+    this.loading = true;
+    this.reload();
+  },
+  methods: {
+    jobNameValid(name) {
+      if (this.jobs.some(job => job.name === name)) {
+        return false;
+      }
+
+      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+    },
+    frontendNameValid(name) {
+      if (this.frontends.some(job => job.name === name)) {
+        return false;
+      }
+
+      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+    },
+    reload() {
+      Sist2AdminApi.getJobs().then(resp => {
+        this.jobs = resp.data;
+        this.jobsLoading = false;
+
+        this.showHelp = this.jobs.length === 0;
+      });
+      Sist2AdminApi.getFrontends().then(resp => {
+        this.frontends = resp.data;
+        this.frontendsLoading = false;
+      });
+    },
+    createJob() {
+      Sist2AdminApi.createJob(this.newJobName).then(this.reload);
+    },
+    createFrontend() {
+      Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
+    }
+  }
+}
+</script>

sist2-admin/frontend/src/views/Job.vue

@@ -0,0 +1,92 @@
+<template>
+  <b-card>
+    <b-card-title>
+      [{{ getName() }}]
+      {{ $t("jobTitle") }}
+    </b-card-title>
+
+    <div class="mb-3">
+      <b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
+      <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
+    </div>
+
+    <div v-if="job">
+      {{ $t("status") }}: <code>{{ job.status }}</code>
+    </div>
+
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card-body v-else>
+
+      <h4>{{ $t("jobOptions.title") }}</h4>
+      <b-card>
+        <JobOptions :job="job" @change="update"></JobOptions>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("scanOptions.title") }}</h4>
+      <b-card>
+        <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("indexOptions.title") }}</h4>
+      <b-card>
+        <IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
+      </b-card>
+
+    </b-card-body>
+
+  </b-card>
+</template>
+
+<script>
+import ScanOptions from "@/components/ScanOptions";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import IndexOptions from "@/components/IndexOptions";
+import JobOptions from "@/components/JobOptions";
+
+export default {
+  name: "Job",
+  components: {
+    IndexOptions,
+    ScanOptions,
+    JobOptions
+  },
+  data() {
+    return {
+      loading: true,
+      job: null
+    }
+  },
+  methods: {
+    getName() {
+      return this.$route.params.name;
+    },
+    update() {
+      Sist2AdminApi.updateJob(this.getName(), this.job);
+    },
+    runJob() {
+      Sist2AdminApi.runJob(this.getName()).then(() => {
+        this.$bvToast.toast(this.$t("runJobConfirmation"), {
+          title: this.$t("runJobConfirmationTitle"),
+          variant: "success",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    },
+    deleteJob() {
+      Sist2AdminApi.deleteJob(this.getName()).then(() => {
+        this.$router.push("/");
+      })
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getJob(this.getName()).then(resp => {
+      this.loading = false;
+      this.job = resp.data;
+    })
+  }
+}
+</script>

sist2-admin/frontend/src/views/Tail.vue

@@ -0,0 +1,168 @@
+<template>
+  <b-card>
+    <b-card-body>
+
+      <h4 class="mb-3">{{ taskId }} {{ $t("logs") }}</h4>
+
+      <div v-if="$store.state.sist2AdminInfo">
+        {{ $t("logFile") }}
+        <code>{{ $store.state.sist2AdminInfo.logs_folder }}/sist2-{{ taskId }}.log</code>
+        <br/>
+        <br/>
+      </div>
+
+      <b-row>
+        <b-col>
+          <span>{{ $t("logLevel") }}</span>
+          <b-select :options="levels.slice(0, -1)" v-model="logLevel" @input="connect()"></b-select>
+        </b-col>
+        <b-col>
+          <span>{{ $t("logMode") }}</span>
+          <b-select :options="modeOptions" v-model="mode" @input="connect()"></b-select>
+        </b-col>
+      </b-row>
+
+      <div id="log-tail-output" class="mt-3 ml-1"></div>
+
+    </b-card-body>
+  </b-card>
+</template>
+
+<script>
+
+export default {
+  name: "Tail",
+  data() {
+    return {
+      logLevel: "DEBUG",
+      levels: ["DEBUG", "INFO", "WARNING", "ERROR", "ADMIN", "FATAL"],
+      socket: null,
+      mode: "follow",
+      modeOptions: [
+        {
+          "text": this.$t('follow'),
+          "value": "follow"
+        },
+        {
+          "text": this.$t('wholeFile'),
+          "value": "wholeFile"
+        }
+      ]
+    }
+  },
+  computed: {
+    taskId: function () {
+      return this.$route.params.taskId;
+    }
+  },
+  methods: {
+    connect() {
+      let lineCount = 0;
+      const outputElem = document.getElementById("log-tail-output")
+      outputElem.replaceChildren();
+      if (this.socket !== null) {
+        this.socket.close();
+      }
+
+      const n = this.mode === "follow" ? 32 : 9999999999;
+      this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
+      this.socket.onopen = () => {
+        this.socket.send("Hello from client");
+      }
+
+      this.socket.onmessage = e => {
+        let message;
+        try {
+          message = JSON.parse(e.data);
+        } catch {
+          console.error(e.data)
+          return;
+        }
+
+        if ("ping" in message) {
+          return;
+        }
+
+        if (message.level === undefined) {
+
+          if ("stderr" in message) {
+            message.level = "ERROR";
+            message.message = message["stderr"];
+          } else {
+            message.level = "ADMIN";
+            message.message = message["sist2-admin"];
+          }
+          message.datetime = ""
+          message.filepath = ""
+        }
+
+        if (this.levels.indexOf(message.level) < this.levels.indexOf(this.logLevel)) {
+          return;
+        }
+
+        const logLine = `${message.datetime} [${message.level} ${message.filepath}] ${message.message}`;
+
+        const span = document.createElement("span");
+        span.setAttribute("class", message.level);
+        span.appendChild(document.createTextNode(logLine));
+
+        outputElem.appendChild(span);
+        lineCount += 1;
+
+        if (this.mode === "follow" && lineCount >= n) {
+          outputElem.firstChild.remove();
+        }
+      }
+    }
+  },
+  mounted() {
+    this.connect()
+  }
+}
+
+</script>
+
+<style>
+#log-tail-output span {
+  display: block;
+}
+
+span.DEBUG {
+  color: #9E9E9E;
+}
+
+span.WARNING {
+  color: #FFB300;
+}
+
+span.INFO {
+  color: #039BE5;
+}
+
+span.ERROR {
+  color: #F4511E;
+}
+
+span.FATAL {
+  color: #F4511E;
+}
+
+span.ADMIN {
+  color: #ee05ff;
+}
+
+
+#log-tail-output {
+  font-size: 13px;
+  font-family: monospace;
+
+  padding: 6px;
+  background-color: #f5f5f5;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  margin: 3px;
+  white-space: pre;
+  color: #000;
+  overflow: hidden;
+}
+</style>

sist2-admin/frontend/src/views/Tasks.vue

@@ -0,0 +1,150 @@
+<template>
+  <div>
+
+    <b-card v-if="tasks.length > 0">
+      <h2>{{ $t("runningTasks") }}</h2>
+      <b-list-group>
+        <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
+      </b-list-group>
+    </b-card>
+
+    <b-card class="mt-4">
+
+      <b-card-title>{{ $t("taskHistory") }}</b-card-title>
+
+      <br/>
+
+      <b-table
+          id="task-history"
+          :items="historyItems"
+          :fields="historyFields"
+          :current-page="historyCurrentPage"
+          :tbody-tr-class="rowClass"
+          :per-page="10"
+      >
+        <template #cell(logs)="data">
+          <router-link :to="`/log/${data.item.logs}`">{{ $t("logs") }}</router-link>
+        </template>
+
+      </b-table>
+
+      <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
+                    :per-page="10"></b-pagination>
+
+    </b-card>
+  </div>
+</template>
+
+<script>
+import TaskListItem from "@/components/TaskListItem";
+import Sist2AdminApi from "@/Sist2AdminApi";
+import moment from "moment";
+
+const DAY = 3600 * 24;
+const HOUR = 3600;
+const MINUTE = 60;
+
+function humanDuration(sec_num) {
+  sec_num = sec_num / 1000;
+  const days = Math.floor(sec_num / DAY);
+  sec_num -= days * DAY;
+  const hours = Math.floor(sec_num / HOUR);
+  sec_num -= hours * HOUR;
+  const minutes = Math.floor(sec_num / MINUTE);
+  sec_num -= minutes * MINUTE;
+  const seconds = Math.floor(sec_num);
+
+  if (days > 0) {
+    return `${days} days ${hours}h ${minutes}m ${seconds}s`;
+  }
+
+  if (hours > 0) {
+    return `${hours}h ${minutes}m ${seconds}s`;
+  }
+
+  if (minutes > 0) {
+    return `${minutes}m ${seconds}s`;
+  }
+
+  if (seconds > 0) {
+    return `${seconds}s`;
+  }
+
+  return "<0s";
+}
+
+export default {
+  name: 'Tasks',
+  components: {TaskListItem},
+  data() {
+    return {
+      loading: true,
+      tasks: [],
+      taskHistory: [],
+      timerId: null,
+      historyFields: [
+        {key: "name", label: this.$t("taskName")},
+        {key: "time", label: this.$t("taskStarted")},
+        {key: "duration", label: this.$t("taskDuration")},
+        {key: "status", label: this.$t("taskStatus")},
+        {key: "logs", label: this.$t("logs")},
+      ],
+      historyCurrentPage: 1,
+      historyItems: []
+    }
+  },
+  props: {
+    msg: String
+  },
+  mounted() {
+    this.loading = true;
+    this.update().then(() => this.loading = false);
+
+    this.timerId = window.setInterval(this.update, 1000);
+    this.updateHistory();
+  },
+  destroyed() {
+    if (this.timerId) {
+      window.clearInterval(this.timerId);
+    }
+  },
+  methods: {
+    rowClass(row) {
+      if (row.status === "failed") {
+        return "table-danger";
+      }
+      return null;
+    },
+    updateHistory() {
+      Sist2AdminApi.getTaskHistory().then(resp => {
+        this.historyItems = resp.data.map(row => ({
+          id: row.id,
+          name: row.name,
+          duration: this.taskDuration(row),
+          time: moment(row.started).format("dd, MMM Do YYYY, HH:mm:ss"),
+          logs: row.id,
+          status: row.return_code === 0 ? "ok" : "failed"
+        }));
+      });
+    },
+    update() {
+      return Sist2AdminApi.getTasks().then(resp => {
+        this.tasks = resp.data;
+      })
+    },
+    taskDuration(task) {
+      const start = moment.utc(task.started);
+      const end = moment.utc(task.ended);
+
+      return humanDuration(end.diff(start))
+    }
+  }
+}
+</script>
+
+<style scoped>
+#task-history {
+  font-family: monospace;
+  font-size: 12px;
+}
+</style>

sist2-admin/frontend/vue.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+    publicPath: "",
+    filenameHashing: false,
+    productionSourceMap: false,
+};

sist2-admin/requirements.txt

@@ -0,0 +1,5 @@
+fastapi
+git+https://github.com/simon987/hexlib.git
+uvicorn
+websockets
+pycron

sist2-admin/sist2_admin/app.py

@@ -0,0 +1,390 @@
+import asyncio
+import os
+import signal
+from datetime import datetime
+from urllib.parse import urlparse
+
+import requests
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from hexlib.db import PersistentState
+from requests import ConnectionError
+from requests.exceptions import SSLError
+from starlette.middleware.cors import CORSMiddleware
+from starlette.responses import RedirectResponse
+from starlette.staticfiles import StaticFiles
+from starlette.websockets import WebSocket
+from websockets.exceptions import ConnectionClosed
+
+import cron
+from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
+from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
+from notifications import Subscribe, Notifications
+from sist2 import Sist2
+from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
+from web import Sist2Frontend
+
+sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
+db = PersistentState(dbfile=os.path.join(DATA_FOLDER, "state.db"))
+notifications = Notifications()
+task_queue = TaskQueue(sist2, db, notifications)
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_credentials=True,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.mount("/ui/", StaticFiles(directory="./frontend/dist", html=True), name="static")
+
+
+@app.get("/")
+async def home():
+    return RedirectResponse("ui")
+
+
+@app.get("/api")
+async def api():
+    return {
+        "tesseract_langs": TESSERACT_LANGS,
+        "logs_folder": LOG_FOLDER
+    }
+
+
+@app.get("/api/job/{name:str}")
+async def get_job(name: str):
+    job = db["jobs"][name]
+    if not job:
+        raise HTTPException(status_code=404)
+    return job
+
+
+@app.get("/api/frontend/{name:str}")
+async def get_frontend(name: str):
+    frontend = db["frontends"][name]
+    frontend: Sist2Frontend
+    if frontend:
+        frontend.running = frontend.name in RUNNING_FRONTENDS
+        return frontend
+    raise HTTPException(status_code=404)
+
+
+@app.get("/api/job/")
+async def get_jobs():
+    return list(db["jobs"])
+
+
+@app.put("/api/job/{name:str}")
+async def update_job(name: str, new_job: Sist2Job):
+    # TODO: Check etag
+
+    new_job.last_modified = datetime.now()
+    job = db["jobs"][name]
+    if not job:
+        raise HTTPException(status_code=404)
+
+    args_that_trigger_full_scan = [
+        "path",
+        "thumbnail_count",
+        "thumbnail_quality",
+        "thumbnail_size",
+        "content_size",
+        "depth",
+        "archive",
+        "archive_passphrase",
+        "ocr_lang",
+        "ocr_images",
+        "ocr_ebooks",
+        "fast",
+        "checksums",
+        "read_subtitles",
+    ]
+    for arg in args_that_trigger_full_scan:
+        if getattr(new_job.scan_options, arg) != getattr(job.scan_options, arg):
+            new_job.do_full_scan = True
+
+    db["jobs"][name] = new_job
+
+
+@app.put("/api/frontend/{name:str}")
+async def update_frontend(name: str, frontend: Sist2Frontend):
+    db["frontends"][name] = frontend
+
+    # TODO: Check etag
+
+    return "ok"
+
+
+@app.get("/api/task/")
+async def get_tasks():
+    return list(map(lambda t: t.json(), task_queue.tasks()))
+
+
+@app.get("/api/task/history")
+async def task_history():
+    return list(db["task_done"].sql("ORDER BY started DESC"))
+
+
+@app.post("/api/task/{task_id:str}/kill")
+async def kill_job(task_id: str):
+    return task_queue.kill_task(task_id)
+
+
+def _run_job(job: Sist2Job):
+    job.last_modified = datetime.now()
+    if job.status == JobStatus("created"):
+        job.status = JobStatus("started")
+    db["jobs"][job.name] = job
+
+    scan_task = Sist2ScanTask(job, f"Scan [{job.name}]")
+    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=scan_task)
+
+    task_queue.submit(scan_task)
+    task_queue.submit(index_task)
+
+
+@app.get("/api/job/{name:str}/run")
+async def run_job(name: str):
+    job = db["jobs"][name]
+    if not job:
+        raise HTTPException(status_code=404)
+
+    _run_job(job)
+
+    return "ok"
+
+
+@app.delete("/api/job/{name:str}")
+async def delete_job(name: str):
+    job = db["jobs"][name]
+    if job:
+        del db["jobs"][name]
+    else:
+        raise HTTPException(status_code=404)
+
+
+@app.delete("/api/frontend/{name:str}")
+async def delete_frontend(name: str):
+    if name in RUNNING_FRONTENDS:
+        os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        del RUNNING_FRONTENDS[name]
+
+    frontend = db["frontends"][name]
+    if frontend:
+        del db["frontends"][name]
+    else:
+        raise HTTPException(status_code=404)
+
+
+@app.post("/api/job/{name:str}")
+async def create_job(name: str):
+    if db["jobs"][name]:
+        raise ValueError("Job with the same name already exists")
+
+    job = Sist2Job.create_default(name)
+    db["jobs"][name] = job
+
+    return job
+
+
+@app.post("/api/frontend/{name:str}")
+async def create_frontend(name: str):
+    if db["frontends"][name]:
+        raise ValueError("Frontend with the same name already exists")
+
+    frontend = Sist2Frontend.create_default(name)
+    db["frontends"][name] = frontend
+
+    return frontend
+
+
+@app.get("/api/ping_es")
+async def ping_es(url: str, insecure: bool):
+    return check_es_version(url, insecure)
+
+
+def check_es_version(es_url: str, insecure: bool):
+    try:
+        url = urlparse(es_url)
+        if url.username:
+            auth = (url.username, url.password)
+            es_url = f"{url.scheme}://{url.hostname}:{url.port}"
+        else:
+            auth = None
+        r = requests.get(es_url, verify=insecure, auth=auth)
+    except SSLError:
+        return {
+            "ok": False,
+            "message": "Invalid SSL certificate"
+        }
+    except ConnectionError as e:
+        return {
+            "ok": False,
+            "message": "Connection refused"
+        }
+    except ValueError as e:
+        return {
+            "ok": False,
+            "message": str(e)
+        }
+
+    if r.status_code == 401:
+        return {
+            "ok": False,
+            "message": "Authentication failure"
+        }
+
+    try:
+        return {
+            "ok": True,
+            "message": "Elasticsearch version " + r.json()["version"]["number"]
+        }
+    except:
+        return {
+            "ok": False,
+            "message": "Could not read version"
+        }
+
+
+def start_frontend_(frontend: Sist2Frontend):
+    frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
+
+    pid = sist2.web(frontend.web_options, frontend.name)
+    RUNNING_FRONTENDS[frontend.name] = pid
+
+
+@app.post("/api/frontend/{name:str}/start")
+async def start_frontend(name: str):
+    frontend = db["frontends"][name]
+    if not frontend:
+        raise HTTPException(status_code=404)
+
+    start_frontend_(frontend)
+
+
+@app.post("/api/frontend/{name:str}/stop")
+async def stop_frontend(name: str):
+    if name in RUNNING_FRONTENDS:
+        os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        del RUNNING_FRONTENDS[name]
+
+
+@app.get("/api/frontend/")
+async def get_frontends():
+    res = []
+    for frontend in db["frontends"]:
+        frontend: Sist2Frontend
+        frontend.running = frontend.name in RUNNING_FRONTENDS
+        res.append(frontend)
+    return res
+
+
+def tail(filepath: str, n: int):
+    with open(filepath) as file:
+
+        reached_eof = False
+        buffer = []
+
+        line = ""
+        while True:
+            tmp = file.readline()
+            if tmp:
+                line += tmp
+
+                if line.endswith("\n"):
+
+                    if reached_eof:
+                        yield line
+                    else:
+                        if len(buffer) > n:
+                            buffer.pop(0)
+                        buffer.append(line)
+                line = ""
+            else:
+                if not reached_eof:
+                    reached_eof = True
+                    yield from buffer
+                yield None
+
+
+@app.websocket("/notifications")
+async def ws_tail_log(websocket: WebSocket):
+    await websocket.accept()
+
+    try:
+        await websocket.receive_text()
+
+        async with Subscribe(notifications) as ob:
+            async for notification in ob.notifications():
+                await websocket.send_json(notification)
+                print(notification)
+
+    except ConnectionClosed:
+        return
+
+
+@app.websocket("/log/{task_id}")
+async def ws_tail_log(websocket: WebSocket, task_id: str, n: int):
+    log_file = os.path.join(LOG_FOLDER, f"sist2-{task_id}.log")
+
+    await websocket.accept()
+
+    try:
+        await websocket.receive_text()
+    except ConnectionClosed:
+        return
+
+    while True:
+        for line in tail(log_file, n):
+
+            try:
+                if line:
+                    await websocket.send_text(line)
+                else:
+                    await websocket.send_json({"ping": ""})
+                    await asyncio.sleep(0.1)
+            except ConnectionClosed:
+                return
+
+
+def main():
+    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0")
+
+
+def initialize_db():
+    db["sist2_admin"]["info"] = {"version": DB_SCHEMA_VERSION}
+
+    frontend = Sist2Frontend.create_default("default")
+    db["frontends"]["default"] = frontend
+
+    logger.info("Initialized database.")
+
+
+def start_frontends():
+    for frontend in db["frontends"]:
+        frontend: Sist2Frontend
+        if frontend.auto_start and len(frontend.jobs) > 0:
+            start_frontend_(frontend)
+
+
+if __name__ == '__main__':
+
+    if not db["sist2_admin"]["info"]:
+        initialize_db()
+    if db["sist2_admin"]["info"]["version"] == "1":
+        logger.info("Migrating to v2 database schema")
+        migrate_v1_to_v2(db)
+    if db["sist2_admin"]["info"]["version"] == "2":
+        logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.")
+        exit(-1)
+
+    start_frontends()
+    cron.initialize(db, _run_job)
+
+    logger.info("Started sist2-admin. Hello!")
+
+    main()

sist2-admin/sist2_admin/config.py

@@ -0,0 +1,30 @@
+import os
+import logging
+import sys
+from logging import StreamHandler
+from logging.handlers import RotatingFileHandler
+
+MAX_LOG_SIZE = 1 * 1024 * 1024
+
+SIST2_BINARY = os.environ.get("SIST2_BINARY", "/root/sist2")
+DATA_FOLDER = os.environ.get("DATA_FOLDER", "/sist2-admin/")
+LOG_FOLDER = os.path.join(DATA_FOLDER, "logs")
+WEBSERVER_PORT = 8080
+
+os.makedirs(LOG_FOLDER, exist_ok=True)
+os.makedirs(DATA_FOLDER, exist_ok=True)
+
+logger = logging.Logger("sist2-admin")
+
+_log_file = os.path.join(LOG_FOLDER, "sist2-admin.log")
+_log_fmt = "%(asctime)s [%(levelname)s] %(message)s"
+_log_formatter = logging.Formatter(_log_fmt, datefmt='%Y-%m-%d %H:%M:%S')
+
+console_handler = StreamHandler(sys.stdout)
+console_handler.setFormatter(_log_formatter)
+
+file_handler = RotatingFileHandler(_log_file, mode="a", maxBytes=MAX_LOG_SIZE, backupCount=1)
+file_handler.setFormatter(_log_formatter)
+
+logger.addHandler(console_handler)
+logger.addHandler(file_handler)

sist2-admin/sist2_admin/cron.py

@@ -0,0 +1,33 @@
+from threading import Thread
+
+import pycron
+import time
+
+from hexlib.db import PersistentState
+
+from config import logger
+from jobs import Sist2Job
+
+
+def _check_schedule(db: PersistentState, run_job):
+    for job in db["jobs"]:
+        job: Sist2Job
+
+        if job.schedule_enabled:
+            if pycron.is_now(job.cron_expression):
+                logger.info(f"Submit scan task to queue for [{job.name}]")
+                run_job(job)
+
+
+def _cron_thread(db, run_job):
+    time.sleep(60 - (time.time() % 60))
+    start = time.time()
+
+    while True:
+        _check_schedule(db, run_job)
+        time.sleep(60 - ((time.time() - start) % 60))
+
+
+def initialize(db, run_job):
+    t = Thread(target=_cron_thread, args=(db, run_job), daemon=True, name="timer")
+    t.start()

sist2-admin/sist2_admin/jobs.py

@@ -0,0 +1,317 @@
+import json
+import logging
+import os.path
+import signal
+import uuid
+from datetime import datetime
+from enum import Enum
+from logging import FileHandler
+from threading import Lock, Thread
+from time import sleep
+from uuid import uuid4, UUID
+
+from hexlib.db import PersistentState
+from pydantic import BaseModel
+
+from config import logger, LOG_FOLDER
+from notifications import Notifications
+from sist2 import ScanOptions, IndexOptions, Sist2
+from state import RUNNING_FRONTENDS
+from web import Sist2Frontend
+
+
+class JobStatus(Enum):
+    CREATED = "created"
+    STARTED = "started"
+    INDEXED = "indexed"
+    FAILED = "failed"
+
+
+class Sist2Job(BaseModel):
+    name: str
+    scan_options: ScanOptions
+    index_options: IndexOptions
+
+    cron_expression: str
+    schedule_enabled: bool = False
+
+    previous_index: str = None
+    index_path: str = None
+    previous_index_path: str = None
+    last_index_date: datetime = None
+    status: JobStatus = JobStatus("created")
+    last_modified: datetime
+    etag: str = None
+    do_full_scan: bool = False
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @staticmethod
+    def create_default(name: str):
+        return Sist2Job(
+            name=name,
+            scan_options=ScanOptions(path="/"),
+            index_options=IndexOptions(),
+            last_modified=datetime.now(),
+            cron_expression="0 0 * * *"
+        )
+
+    # @validator("etag", always=True)
+    # def validate_etag(cls, value, values):
+    #     s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
+    #     return md5(s.encode()).hexdigest()
+
+
+class Sist2TaskProgress:
+
+    def __init__(self, done: int = 0, count: int = 0, index_size: int = 0, tn_size: int = 0, waiting: bool = False):
+        self.done = done
+        self.count = count
+        self.index_size = index_size
+        self.store_size = tn_size
+        self.waiting = waiting
+
+    def percent(self):
+        return (self.done / self.count) if self.count else 0
+
+
+class Sist2Task:
+
+    def __init__(self, job: Sist2Job, display_name: str, depends_on: uuid.UUID = None):
+        self.job = job
+        self.display_name = display_name
+
+        self.progress = Sist2TaskProgress()
+        self.id = uuid4()
+        self.pid = None
+        self.started = None
+        self.ended = None
+        self.depends_on = depends_on
+
+        self._logger = logging.Logger(name=f"{self.id}")
+        self._logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"sist2-{self.id}.log")))
+
+    def json(self):
+        return {
+            "id": self.id,
+            "job": self.job,
+            "display_name": self.display_name,
+            "progress": self.progress,
+            "started": self.started,
+            "ended": self.ended,
+            "depends_on": self.depends_on,
+        }
+
+    def log_callback(self, log_json):
+
+        if "progress" in log_json:
+            self.progress = Sist2TaskProgress(**log_json["progress"])
+        elif self._logger:
+            self._logger.info(json.dumps(log_json))
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        self.started = datetime.now()
+
+        logger.info(f"Started task {self.display_name}")
+
+
+class Sist2ScanTask(Sist2Task):
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        self.job.scan_options.name = self.job.name
+
+        if self.job.index_path is not None and not self.job.do_full_scan:
+            self.job.scan_options.output = self.job.index_path
+        else:
+            self.job.scan_options.output = None
+
+        def set_pid(pid):
+            self.pid = pid
+
+        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
+        self.ended = datetime.now()
+
+        if return_code != 0:
+            self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
+            logger.info(f"Task {self.display_name} failed ({return_code})")
+        else:
+            self.job.index_path = self.job.scan_options.output
+            self.job.last_index_date = datetime.now()
+            self.job.do_full_scan = False
+            db["jobs"][self.job.name] = self.job
+            self._logger.info(json.dumps({"sist2-admin": f"Save last_index_date={self.job.last_index_date}"}))
+
+        logger.info(f"Completed {self.display_name} ({return_code=})")
+
+        # Remove old index
+        if return_code == 0:
+            if self.job.previous_index_path is not None and self.job.previous_index_path != self.job.index_path:
+                self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index_path=}"}))
+                try:
+                    os.remove(self.job.previous_index_path)
+                except FileNotFoundError:
+                    pass
+
+            self.job.previous_index_path = self.job.index_path
+            db["jobs"][self.job.name] = self.job
+
+        return return_code
+
+
+class Sist2IndexTask(Sist2Task):
+
+    def __init__(self, job: Sist2Job, display_name: str, depends_on: Sist2Task):
+        super().__init__(job, display_name, depends_on=depends_on.id)
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        self.job.index_options.path = self.job.scan_options.output
+
+        return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
+        self.ended = datetime.now()
+
+        duration = self.ended - self.started
+
+        ok = return_code == 0
+
+        if ok:
+            self.restart_running_frontends(db, sist2)
+
+        # Update status
+        self.job.status = JobStatus("indexed") if ok else JobStatus("failed")
+        self.job.previous_index_path = self.job.index_path
+        db["jobs"][self.job.name] = self.job
+
+        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
+
+        logger.info(f"Completed {self.display_name} ({return_code=})")
+
+        return return_code
+
+    def restart_running_frontends(self, db: PersistentState, sist2: Sist2):
+        for frontend_name, pid in RUNNING_FRONTENDS.items():
+            frontend = db["frontends"][frontend_name]
+            frontend: Sist2Frontend
+
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+            try:
+                os.wait()
+            except ChildProcessError:
+                pass
+
+            frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
+
+            pid = sist2.web(frontend.web_options, frontend.name)
+            RUNNING_FRONTENDS[frontend_name] = pid
+
+            self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))
+
+
+class TaskQueue:
+    def __init__(self, sist2: Sist2, db: PersistentState, notifications: Notifications):
+        self._lock = Lock()
+
+        self._sist2 = sist2
+        self._db = db
+        self._notifications = notifications
+
+        self._tasks = {}
+        self._queue = []
+        self._sem = 0
+
+        self._thread = Thread(target=self._check_new_task, daemon=True)
+        self._thread.start()
+
+    def _tasks_failed(self):
+        done = set()
+
+        for row in self._db["task_done"].sql("WHERE return_code != 0"):
+            done.add(uuid.UUID(row["id"]))
+
+        return done
+
+    def _tasks_done(self):
+
+        done = set()
+
+        for row in self._db["task_done"]:
+            done.add(uuid.UUID(row["id"]))
+
+        return done
+
+    def _check_new_task(self):
+        while True:
+            with self._lock:
+                for task in list(self._queue):
+                    task: Sist2Task
+
+                    if self._sem >= 1:
+                        break
+
+                    if not task.depends_on or task.depends_on in self._tasks_done():
+                        self._queue.remove(task)
+
+                        if task.depends_on in self._tasks_failed():
+                            # The task which we depend on failed, continue
+                            continue
+
+                        self._sem += 1
+
+                        t = Thread(target=self._run_task, args=(task,))
+
+                        self._tasks[task.id] = {
+                            "task": task,
+                            "thread": t,
+                        }
+
+                        t.start()
+                    break
+            sleep(1)
+
+    def tasks(self):
+        return list(map(lambda t: t["task"], self._tasks.values()))
+
+    def kill_task(self, task_id):
+
+        task = self._tasks.get(UUID(task_id))
+
+        if task:
+            pid = task["task"].pid
+            logger.info(f"Killing task {task_id} (pid={pid})")
+            os.kill(pid, signal.SIGTERM)
+            return True
+
+        return False
+
+    def _run_task(self, task: Sist2Task):
+        task_result = task.run(self._sist2, self._db)
+
+        with self._lock:
+            del self._tasks[task.id]
+            self._sem -= 1
+
+            self._db["task_done"][task.id] = {
+                "ended": task.ended,
+                "started": task.started,
+                "name": task.display_name,
+                "return_code": task_result
+            }
+        if isinstance(task, Sist2IndexTask):
+            self._notifications.notify({
+                "message": "notifications.indexCompleted",
+                "job": task.job.name
+            })
+
+    def submit(self, task: Sist2Task):
+
+        logger.info(f"Submitted task to queue {task.display_name}")
+
+        with self._lock:
+            self._queue.append(task)

sist2-admin/sist2_admin/notifications.py

@@ -0,0 +1,40 @@
+import asyncio
+from typing import List
+
+
+class Notifications:
+    def __init__(self):
+        self._subscribers: List[Subscribe] = []
+
+    def subscribe(self, ob):
+        self._subscribers.append(ob)
+
+    def unsubscribe(self, ob):
+        self._subscribers.remove(ob)
+
+    def notify(self, notification: dict):
+        for ob in self._subscribers:
+            ob.notify(notification)
+
+
+class Subscribe:
+    def __init__(self, notifications: Notifications):
+        self._queue = []
+        self._notifications = notifications
+
+    async def __aenter__(self):
+        self._notifications.subscribe(self)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        self._notifications.unsubscribe(self)
+
+    def notify(self, notification: dict):
+        self._queue.append(notification)
+
+    async def notifications(self):
+        while True:
+            try:
+                yield self._queue.pop(0)
+            except IndexError:
+                await asyncio.sleep(0.1)

sist2-admin/sist2_admin/sist2.py

@@ -0,0 +1,323 @@
+import datetime
+import json
+import logging
+import os.path
+from datetime import datetime
+from io import TextIOWrapper
+from logging import FileHandler
+from subprocess import Popen, PIPE
+from tempfile import NamedTemporaryFile
+from threading import Thread
+from typing import List
+
+from pydantic import BaseModel
+
+from config import logger, LOG_FOLDER
+
+
+class Sist2Version:
+    def __init__(self, version: str):
+        self._version = version
+
+        self.major, self.minor, self.patch = [int(x) for x in version.split(".")]
+
+    def __str__(self):
+        return f"{self.major}.{self.minor}.{self.patch}"
+
+
+class WebOptions(BaseModel):
+    indices: List[str] = []
+    es_url: str = "http://elasticsearch:9200"
+    es_insecure_ssl: bool = False
+    es_index: str = "sist2"
+    bind: str = "0.0.0.0:4090"
+    auth: str = None
+    tag_auth: str = None
+    tagline: str = "Lightning-fast file system indexer and search tool"
+    dev: bool = False
+    lang: str = "en"
+    auth0_audience: str = None
+    auth0_domain: str = None
+    auth0_client_id: str = None
+    auth0_public_key: str = None
+    auth0_public_key_file: str = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+        args = ["web", f"--es-url={self.es_url}", f"--bind={self.bind}",
+                f"--tagline={self.tagline}", f"--lang={self.lang}"]
+
+        if self.auth0_audience:
+            args.append(f"--auth0-audience={self.auth0_audience}")
+        if self.auth0_domain:
+            args.append(f"--auth0-domain={self.auth0_domain}")
+        if self.auth0_client_id:
+            args.append(f"--auth0-client-id={self.auth0_client_id}")
+        if self.auth0_public_key_file:
+            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
+        if self.es_insecure_ssl:
+            args.append(f"--es-insecure-ssl")
+        if self.auth:
+            args.append(f"--auth={self.auth}")
+        if self.tag_auth:
+            args.append(f"--tag-auth={self.tag_auth}")
+        if self.dev:
+            args.append(f"--dev")
+
+        args.extend(self.indices)
+
+        return args
+
+
+class IndexOptions(BaseModel):
+    path: str = None
+    threads: int = 1
+    es_url: str = "http://elasticsearch:9200"
+    es_insecure_ssl: bool = False
+    es_index: str = "sist2"
+    incremental_index: bool = True
+    script: str = ""
+    script_file: str = None
+    batch_size: int = 70
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+
+        args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
+                f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
+
+        if self.script_file:
+            args.append(f"--script-file={self.script_file}")
+        if self.es_insecure_ssl:
+            args.append(f"--es-insecure-ssl")
+        if self.incremental_index:
+            args.append(f"--incremental-index")
+
+        return args
+
+
+ARCHIVE_SKIP = "skip"
+ARCHIVE_LIST = "list"
+ARCHIVE_SHALLOW = "shallow"
+ARCHIVE_RECURSE = "recurse"
+
+
+class ScanOptions(BaseModel):
+    path: str
+    threads: int = 1
+    thumbnail_quality: int = 2
+    thumbnail_size: int = 552
+    thumbnail_count: int = 1
+    content_size: int = 32768
+    depth: int = -1
+    archive: str = ARCHIVE_RECURSE
+    archive_passphrase: str = None
+    ocr_lang: str = None
+    ocr_images: bool = False
+    ocr_ebooks: bool = False
+    exclude: str = None
+    fast: bool = False
+    treemap_threshold: float = 0.0005
+    mem_buffer: int = 2000
+    read_subtitles: bool = False
+    fast_epub: bool = False
+    checksums: bool = False
+    incremental: bool = True
+    optimize_index: bool = False
+    output: str = None
+    name: str = None
+    rewrite_url: str = None
+    list_file: str = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+        args = ["scan", self.path, f"--threads={self.threads}", f"--thumbnail-quality={self.thumbnail_quality}",
+                f"--thumbnail-count={self.thumbnail_count}", f"--thumbnail-size={self.thumbnail_size}",
+                f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
+                f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
+
+        if self.incremental:
+            args.append(f"--incremental")
+        if self.optimize_index:
+            args.append(f"--optimize-index")
+        if self.rewrite_url:
+            args.append(f"--rewrite-url={self.rewrite_url}")
+        if self.name:
+            args.append(f"--name={self.name}")
+        if self.archive_passphrase:
+            args.append(f"--archive-passphrase={self.archive_passphrase}")
+        if self.ocr_lang:
+            args.append(f"--ocr-lang={self.ocr_lang}")
+        if self.ocr_ebooks:
+            args.append(f"--ocr-ebooks")
+        if self.ocr_images:
+            args.append(f"--ocr-images")
+        if self.exclude:
+            args.append(f"--exclude={self.exclude}")
+        if self.fast:
+            args.append(f"--fast")
+        if self.treemap_threshold:
+            args.append(f"--treemap-threshold={self.treemap_threshold}")
+        if self.read_subtitles:
+            args.append(f"--read-subtitles")
+        if self.fast_epub:
+            args.append(f"--fast-epub")
+        if self.checksums:
+            args.append(f"--checksums")
+        if self.list_file:
+            args.append(f"--list_file={self.list_file}")
+
+        return args
+
+
+class Sist2Index:
+    def __init__(self, path):
+        self.path = path
+
+        with open(os.path.join(path, "descriptor.json")) as f:
+            self._descriptor = json.load(f)
+
+    def to_json(self):
+        return {
+            "path": self.path,
+            "version": self.version(),
+            "timestamp": self.timestamp(),
+            "name": self.name()
+        }
+
+    def version(self) -> Sist2Version:
+        return Sist2Version(self._descriptor["version"])
+
+    def timestamp(self) -> datetime:
+        return datetime.fromtimestamp(self._descriptor["timestamp"])
+
+    def name(self) -> str:
+        return self._descriptor["name"]
+
+
+class Sist2:
+
+    def __init__(self, bin_path: str, data_directory: str):
+        self._bin_path = bin_path
+        self._data_dir = data_directory
+
+    def index(self, options: IndexOptions, logs_cb):
+
+        if options.script:
+            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
+                f.write(options.script)
+            options.script_file = f.name
+        else:
+            options.script_file = None
+
+        args = [
+            self._bin_path,
+            *options.args(),
+            "--json-logs",
+            "--very-verbose"
+        ]
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        self._consume_logs_stdout(logs_cb, proc)
+
+        t_stderr.join()
+
+        return proc.returncode
+
+    def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
+
+        if options.output is None:
+            options.output = os.path.join(
+                self._data_dir,
+                f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2"
+            )
+
+        args = [
+            self._bin_path,
+            *options.args(),
+            "--json-logs",
+            "--very-verbose"
+        ]
+
+        logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
+
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        set_pid_cb(proc.pid)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        self._consume_logs_stdout(logs_cb, proc)
+
+        t_stderr.join()
+
+        return proc.returncode
+
+    @staticmethod
+    def _consume_logs_stderr(logs_cb, proc):
+        pipe_wrapper = TextIOWrapper(proc.stderr, encoding="utf8", errors="ignore")
+        try:
+            for line in pipe_wrapper:
+                if line.strip() == "":
+                    continue
+                logs_cb({"stderr": line})
+        finally:
+            proc.wait()
+            pipe_wrapper.close()
+
+    @staticmethod
+    def _consume_logs_stdout(logs_cb, proc):
+        pipe_wrapper = TextIOWrapper(proc.stdout, encoding="utf8", errors="ignore")
+        for line in pipe_wrapper:
+            try:
+                if line.strip() == "":
+                    continue
+                log_object = json.loads(line)
+                logs_cb(log_object)
+            except Exception as e:
+                try:
+                    logs_cb({"sist2-admin": f"Could not decode log line: {line}; {e}"})
+                except NameError:
+                    pass
+
+    def web(self, options: WebOptions, name: str):
+
+        if options.auth0_public_key:
+            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
+                f.write(options.auth0_public_key)
+            options.auth0_public_key_file = f.name
+        else:
+            options.auth0_public_key_file = None
+
+        args = [
+            self._bin_path,
+            *options.args()
+        ]
+
+        web_logger = logging.Logger(name=f"sist2-frontend-{name}")
+        web_logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"frontend-{name}.log")))
+
+        def logs_cb(message):
+            web_logger.info(json.dumps(message))
+
+        logger.info(f"Starting frontend {' '.join(args)}")
+
+        proc = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr.start()
+
+        t_stdout = Thread(target=self._consume_logs_stdout, args=(logs_cb, proc))
+        t_stdout.start()
+
+        return proc.pid

sist2-admin/sist2_admin/state.py

@@ -0,0 +1,79 @@
+from typing import Dict
+import shutil
+
+from hexlib.db import Table, PersistentState
+import pickle
+
+from tesseract import get_tesseract_langs
+
+RUNNING_FRONTENDS: Dict[str, int] = {}
+
+TESSERACT_LANGS = get_tesseract_langs()
+
+DB_SCHEMA_VERSION = "3"
+
+from pydantic import BaseModel
+
+
+def _serialize(item):
+    if isinstance(item, BaseModel):
+        return pickle.dumps(item)
+    if isinstance(item, bytes):
+        raise Exception("FIXME: bytes in PickleTable")
+    return item
+
+
+def _deserialize(item):
+    if isinstance(item, bytes):
+        return pickle.loads(item)
+    return item
+
+
+class PickleTable(Table):
+
+    def __getitem__(self, item):
+        row = super().__getitem__(item)
+        if row:
+            return dict((k, _deserialize(v)) for k, v in row.items())
+        return row
+
+    def __setitem__(self, key, value):
+        value = dict((k, _serialize(v)) for k, v in value.items())
+        super().__setitem__(key, value)
+
+    def __iter__(self):
+        for row in super().__iter__():
+            yield dict((k, _deserialize(v)) for k, v in row.items())
+
+    def sql(self, where_clause, *params):
+        for row in super().sql(where_clause, *params):
+            yield dict((k, _deserialize(v)) for k, v in row.items())
+
+
+def migrate_v1_to_v2(db: PersistentState):
+
+    shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v2.bak")
+
+    # Frontends
+    db._table_factory = PickleTable
+    frontends = [row["frontend"] for row in db["frontends"]]
+    del db["frontends"]
+
+    db._table_factory = Table
+    for frontend in frontends:
+        db["frontends"][frontend.name] = frontend
+    list(db["frontends"])
+
+    # Jobs
+    db._table_factory = PickleTable
+    jobs = [row["job"] for row in db["jobs"]]
+    del db["jobs"]
+
+    db._table_factory = Table
+    for job in jobs:
+        db["jobs"][job.name] = job
+    list(db["jobs"])
+
+    db["sist2_admin"]["info"] = {
+        "version": "2"
+    }

sist2-admin/sist2_admin/tesseract.py

@@ -0,0 +1,14 @@
+import subprocess
+
+
+def get_tesseract_langs():
+
+    res = subprocess.check_output([
+        "tesseract",
+        "--list-langs"
+    ]).decode()
+
+    languages = res.split("\n")[1:]
+
+    return list(filter(lambda lang: lang and lang != "osd", languages))
+

sist2-admin/sist2_admin/web.py

@@ -0,0 +1,28 @@
+import os.path
+from typing import List
+
+from pydantic import BaseModel
+
+from sist2 import WebOptions
+
+
+class Sist2Frontend(BaseModel):
+    name: str
+    jobs: List[str]
+    web_options: WebOptions
+    running: bool = False
+
+    auto_start: bool = False
+    extra_query_args: str = ""
+    custom_url: str = None
+
+    def get_log_path(self, log_folder: str):
+        return os.path.join(log_folder, f"frontend-{self.name}.log")
+
+    @staticmethod
+    def create_default(name: str):
+        return Sist2Frontend(
+            name=name,
+            web_options=WebOptions(),
+            jobs=[]
+        )

sist2-vue/dist/index.html

@@ -1,3 +0,0 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>sist2</title><link href="css/chunk-vendors.css" rel="preload" as="style"><link href="css/index.css" rel="preload" as="style"><link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/index.css" rel="stylesheet"></head><body><noscript><style>body {
-            height: initial;
-        }</style><div style="text-align: center; margin-top: 100px"><strong>We're sorry but sist2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong><br><strong>Nous sommes désolés mais sist2 ne fonctionne pas correctement si JavaScript est activé. Veuillez l'activer pour continuer.</strong></div></noscript><div id="app"></div><script src="js/chunk-vendors.js"></script><script src="js/index.js"></script></body></html>

sist2-vue/package.json

@@ -7,15 +7,16 @@
     "build": "vue-cli-service build --mode production"
   },
   "dependencies": {
+    "@auth0/auth0-spa-js": "^2.0.2",
     "@egjs/vue-infinitegrid": "3.3.0",
-    "axios": "^0.21.1",
+    "@tensorflow/tfjs": "^4.4.0",
+    "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "crypto-es": "^1.2.7",
+    "d3": "^7.8.4",
-    "d3": "^5.16.0",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",
-    "fslightbox-vue": "file:../../../mnt/Hatchery/main/projects/sist2/fslightbox-vue-pro-1.3.1.tgz",
+    "fslightbox-vue": "fslightbox-vue.tgz",
     "nouislider": "^15.2.0",
     "underscore": "^1.13.1",
     "vue": "^2.6.12",
@@ -27,12 +28,12 @@
     "vuex": "^3.4.0"
   },
   "devDependencies": {
-    "@babel/polyfill": "^7.11.5",
+    "@babel/polyfill": "^7.12.1",
-    "@vue/cli-plugin-babel": "~4.5.0",
+    "@vue/cli-plugin-babel": "~5.0.8",
-    "@vue/cli-plugin-router": "~4.5.0",
+    "@vue/cli-plugin-router": "~5.0.8",
-    "@vue/cli-plugin-typescript": "~4.5.0",
+    "@vue/cli-plugin-typescript": "^5.0.8",
-    "@vue/cli-plugin-vuex": "~4.5.0",
+    "@vue/cli-plugin-vuex": "~5.0.8",
-    "@vue/cli-service": "~4.5.0",
+    "@vue/cli-service": "^5.0.8",
     "@vue/test-utils": "^1.0.3",
     "bootstrap": "^4.5.2",
     "inspire-tree": "^4.3.1",

sist2-vue/src/App.vue

@@ -1,312 +1,395 @@
 <template>
-  <div id="app" :class="getClass()">
+    <div id="app" :class="getClass()" v-if="!authLoading">
-    <NavBar></NavBar>
+        <NavBar></NavBar>
-    <router-view v-if="!configLoading"/>
+        <router-view v-if="!configLoading"/>
-  </div>
+    </div>
+    <div class="loading-page" v-else>
+        <div class="loading-spinners">
+            <b-spinner type="grow" variant="primary"></b-spinner>
+            <b-spinner type="grow" variant="primary"></b-spinner>
+            <b-spinner type="grow" variant="primary"></b-spinner>
+        </div>
+        <div class="loading-text">
+            Loading • Chargement • 装载 • Wird geladen
+        </div>
+    </div>
 </template>
 
 <script>
 import NavBar from "@/components/NavBar";
-import {mapGetters} from "vuex";
+import {mapActions, mapGetters, mapMutations} from "vuex";
+import Sist2Api from "@/Sist2Api";
+import ModelsRepo from "@/ml/modelsRepo";
+import {setupAuth0} from "@/main";
 
 export default {
-  components: {NavBar},
+    components: {NavBar},
-  data() {
+    data() {
-    return {
+        return {
-      configLoading: false
+            configLoading: false,
-    }
+            authLoading: true,
-  },
+            sist2InfoLoading: true
-  computed: {
+        }
-    ...mapGetters(["optTheme"]),
+    },
-  },
+    computed: {
-  mounted() {
+        ...mapGetters(["optTheme"]),
-    this.$store.dispatch("loadConfiguration").then(() => {
+    },
-      this.$root.$i18n.locale = this.$store.state.optLang;
+    mounted() {
-    });
+        this.$store.dispatch("loadConfiguration").then(() => {
+            this.$root.$i18n.locale = this.$store.state.optLang;
+            ModelsRepo.init(this.$store.getters.mlRepositoryList).catch(err => {
+                this.$bvToast.toast(
+                    this.$t("ml.repoFetchError"),
+                    {
+                        title: this.$t("ml.repoFetchErrorTitle"),
+                        noAutoHide: true,
+                        toaster: "b-toaster-bottom-right",
+                        headerClass: "toast-header-warning",
+                        bodyClass: "toast-body-warning",
+                    });
+            });
+        });
 
-    this.$store.subscribe((mutation) => {
+        this.$store.subscribe((mutation) => {
-      if (mutation.type === "setOptLang") {
+            if (mutation.type === "setOptLang") {
-        this.$root.$i18n.locale = mutation.payload;
+                this.$root.$i18n.locale = mutation.payload;
-        this.configLoading = true;
+                this.configLoading = true;
-        window.setTimeout(() => this.configLoading = false, 10);
+                window.setTimeout(() => this.configLoading = false, 10);
-      }
+            }
-    });
+
-  },
+            if (mutation.type === "setAuth0Token") {
-  methods: {
+                this.authLoading = false;
-    getClass() {
+            }
-      return {
+        });
-        "theme-light": this.optTheme === "light",
+
-        "theme-black": this.optTheme === "black",
+        Sist2Api.getSist2Info().then(data => {
-      }
+
+            if (data.auth0Enabled) {
+                this.authLoading = true;
+                setupAuth0(data.auth0Domain, data.auth0ClientId, data.auth0Audience)
+
+                this.$auth.$watch("loading", loading => {
+                    if (loading === false) {
+
+                        if (!this.$auth.isAuthenticated) {
+                            this.$auth.loginWithRedirect();
+                            return;
+                        }
+
+                        // Remove "code" param
+                        window.history.replaceState({}, "", "/" + window.location.hash);
+
+                        this.$store.dispatch("loadAuth0Token");
+                    }
+                });
+            } else {
+                this.authLoading = false;
+            }
+
+            this.setSist2Info(data);
+            this.setIndices(data.indices)
+        });
+    },
+    methods: {
+        ...mapActions(["setSist2Info",]),
+        ...mapMutations(["setIndices",]),
+        getClass() {
+            return {
+                "theme-light": this.optTheme === "light",
+                "theme-black": this.optTheme === "black",
+            }
+        }
     }
-  }
+    ,
-  ,
 }
 </script>
 
 <style>
 html, body {
-  height: 100%;
+    height: 100%;
 }
 
 #app {
-  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+    /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
-  -webkit-font-smoothing: antialiased;
+    -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
+    -moz-osx-font-smoothing: grayscale;
-  /*text-align: center;*/
+    /*text-align: center;*/
-  color: #2c3e50;
+    color: #2c3e50;
-  padding-bottom: 1em;
+    padding-bottom: 1em;
-  min-height: 100%;
+    min-height: 100%;
 }
 
 /*Black theme*/
 .theme-black {
-  background-color: #000;
+    background-color: #000;
 }
 
 .theme-black .card, .theme-black .modal-content {
-  background: #212121;
+    background: #212121;
-  color: #e0e0e0;
+    color: #e0e0e0;
-  border-radius: 1px;
+    border-radius: 1px;
-  border: none;
+    border: none;
 }
 
 
 .theme-black .table {
-  color: #e0e0e0;
+    color: #e0e0e0;
 }
 
 .theme-black .table td, .theme-black .table th {
-  border: none;
+    border: none;
 }
 
 .theme-black .table thead th {
-  border-bottom: 1px solid #646464;
+    border-bottom: 1px solid #646464;
 }
 
 .theme-black .custom-select {
-  overflow: auto;
+    overflow: auto;
-  background-color: #37474F;
+    background-color: #37474F;
-  border: 1px solid #616161;
+    border: 1px solid #616161;
-  color: #bdbdbd;
+    color: #bdbdbd;
 }
 
 .theme-black .custom-select:focus {
-  border-color: #757575;
+    border-color: #757575;
-  outline: 0;
+    outline: 0;
-  box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
+    box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
 }
 
 .theme-black .inspire-tree .selected > .wholerow, .theme-black .inspire-tree .selected > .title-wrap:hover + .wholerow {
-  background: none !important;
+    background: none !important;
 }
 
 .theme-black .inspire-tree .icon-expand::before, .theme-black .inspire-tree .icon-collapse::before {
-  background-color: black !important;
+    background-color: black !important;
 }
 
 .theme-black .inspire-tree .title {
-  color: #eee;
+    color: #eee;
 }
 
 .theme-black .inspire-tree {
-  font-weight: 400;
+    font-weight: 400;
-  font-size: 14px;
+    font-size: 14px;
-  font-family: Helvetica, Nueue, Verdana, sans-serif;
+    font-family: Helvetica, Nueue, Verdana, sans-serif;
-  max-height: 350px;
+    max-height: 350px;
-  overflow: auto;
+    overflow: auto;
 }
 
 .inspire-tree [type=checkbox] {
-  left: 22px !important;
+    left: 22px !important;
-  top: 7px !important;
+    top: 7px !important;
 }
 
 .theme-black .form-control {
-  background-color: #37474F;
+    background-color: #37474F;
-  border: 1px solid #616161;
+    border: 1px solid #616161;
-  color: #dbdbdb !important;
+    color: #dbdbdb !important;
 }
 
 .theme-black .form-control:focus {
-  background-color: #546E7A;
+    background-color: #546E7A;
-  color: #fff;
+    color: #fff;
 }
 
 .theme-black .input-group-text, .theme-black .default-input {
-  background: #37474F !important;
+    background: #37474F !important;
-  border: 1px solid #616161 !important;
+    border: 1px solid #616161 !important;
-  color: #dbdbdb !important;
+    color: #dbdbdb !important;
 }
 
 .theme-black ::placeholder {
-  color: #BDBDBD !important;
+    color: #BDBDBD !important;
-  opacity: 1;
+    opacity: 1;
 }
 
 .theme-black .nav-tabs .nav-link {
-  color: #e0e0e0;
+    color: #e0e0e0;
+    border-radius: 0;
 }
 
 .theme-black .nav-tabs .nav-item.show .nav-link, .theme-black .nav-tabs .nav-link.active {
-  background-color: #212121;
+    background-color: #212121;
-  border-color: #616161 #616161 #212121;
+    border-color: #616161 #616161 #212121;
-  color: #e0e0e0;
+    color: #e0e0e0;
 }
 
 .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:focus {
-  border-color: #616161 #616161 #212121;
+    border-color: #616161 #616161 #212121;
-  color: #e0e0e0;
+    color: #e0e0e0;
 }
 
 .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:hover {
-  border-color: #e0e0e0 #e0e0e0 #212121;
+    border-color: #e0e0e0 #e0e0e0 #212121;
-  color: #e0e0e0;
+    color: #e0e0e0;
 }
 
 .theme-black .nav-tabs {
-  border-bottom: #616161;
+    border-bottom: #616161;
 }
 
 .theme-black a:hover, .theme-black .btn:hover {
-  color: #fff;
+    color: #fff;
 }
 
 .theme-black .b-dropdown a:hover {
-  color: inherit;
+    color: inherit;
 }
 
 .theme-black .btn {
-  color: #eee;
+    color: #eee;
 }
 
 .theme-black .modal-header .close {
-  color: #e0e0e0;
+    color: #e0e0e0;
-  text-shadow: none;
+    text-shadow: none;
 }
 
 .theme-black .modal-header {
-  border-bottom: 1px solid #646464;
+    border-bottom: 1px solid #646464;
 }
 
 /* -------------------------- */
 
 #nav {
-  padding: 30px;
+    padding: 30px;
 }
 
 #nav a {
-  font-weight: bold;
+    font-weight: bold;
-  color: #2c3e50;
+    color: #2c3e50;
 }
 
 #nav a.router-link-exact-active {
-  color: #42b983;
+    color: #42b983;
 }
 
 .mobile {
-  display: none;
+    display: none;
 }
 
 .container {
-  padding-top: 1em;
+    padding-top: 1em;
 }
 
 @media (max-width: 650px) {
-  .mobile {
+    .mobile {
-    display: initial;
+        display: initial;
-  }
+    }
 
-  .not-mobile {
+    .not-mobile {
-    display: none;
+        display: none;
-  }
+    }
 
-  .grid-single-column .fit {
+    .grid-single-column .fit {
-    max-height: none !important;
+        max-height: none !important;
-  }
+    }
 
-  .container {
+    .container {
-    padding-left: 0;
+        padding-left: 0;
-    padding-right: 0;
+        padding-right: 0;
-    padding-top: 0
+        padding-top: 0
-  }
+    }
 
-  .lightbox-caption {
+    .lightbox-caption {
-    display: none;
+        display: none;
-  }
+    }
 }
 
 .info-icon {
-  width: 1rem;
+    width: 1rem;
-  margin-right: 0.2rem;
+    margin-right: 0.2rem;
-  cursor: pointer;
+    cursor: pointer;
-  line-height: 1rem;
+    line-height: 1rem;
-  height: 1rem;
+    height: 1rem;
-  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
-  filter: brightness(45%);
+    filter: brightness(45%);
-  display: block;
+    display: block;
 }
 
 .tabs {
-  margin-top: 10px;
+    margin-top: 10px;
 }
 
 .modal-title {
-  text-overflow: ellipsis;
+    text-overflow: ellipsis;
-  overflow: hidden;
+    overflow: hidden;
-  white-space: nowrap;
+    white-space: nowrap;
 }
 
 @media screen and (min-width: 1500px) {
-  .container {
+    .container {
-    max-width: 1440px;
+        max-width: 1440px;
-  }
+    }
 }
 
 .noUi-connects {
-  border-radius: 1px !important;
+    border-radius: 1px !important;
 }
 
 mark {
-  background: #fff217;
+    background: #fff217;
-  border-radius: 0;
+    border-radius: 0;
-  padding: 1px 0;
+    padding: 1px 0;
-  color: inherit;
+    color: inherit;
 }
 
 .theme-black mark {
-  background: rgba(251, 191, 41, 0.25);
+    background: rgba(251, 191, 41, 0.25);
-  border-radius: 0;
+    border-radius: 0;
-  padding: 1px 0;
+    padding: 1px 0;
-  color: inherit;
+    color: inherit;
 }
 
 .theme-black .content-div mark {
-  background: rgba(251, 191, 41, 0.40);
+    background: rgba(251, 191, 41, 0.40);
-  color: white;
+    color: white;
 }
 
 .content-div {
-  font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+    font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
-  font-size: 13px;
+    font-size: 13px;
-  padding: 1em;
+    padding: 1em;
-  background-color: #f5f5f5;
+    background-color: #f5f5f5;
-  border: 1px solid #ccc;
+    border: 1px solid #ccc;
-  border-radius: 4px;
+    border-radius: 4px;
-  margin: 3px;
+    margin: 3px;
-  white-space: normal;
+    white-space: normal;
-  color: #000;
+    color: #000;
-  overflow: hidden;
+    overflow: hidden;
 }
 
 .theme-black .content-div {
-  background-color: #37474F;
+    background-color: #37474F;
-  border: 1px solid #616161;
+    border: 1px solid #616161;
-  color: #E0E0E0FF;
+    color: #E0E0E0FF;
 }
 
 .graph {
-  display: inline-block;
+    display: inline-block;
-  width: 40%;
+    width: 40%;
+}
+
+.pointer {
+    cursor: pointer;
+}
+
+.loading-page {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    flex-direction: column;
+    height: 100%;
+    gap: 15px
+}
+
+.loading-spinners {
+    display: flex;
+    gap: 10px;
+}
+
+.loading-text {
+    text-align: center;
 }
 </style>

sist2-vue/src/Sist2Api.ts

@@ -1,6 +1,5 @@
 import axios from "axios";
 import {ext, strUnescape, lum} from "./util";
-import CryptoES from 'crypto-es';
 
 export interface EsTag {
     id: string
@@ -30,7 +29,6 @@ export interface EsHit {
     _index: string
     _id: string
     _score: number
-    _path_md5: string
     _type: string
     _tags: Tag[]
     _seq: number
@@ -62,8 +60,10 @@ export interface EsHit {
         isPlayableImage: boolean
         isAudio: boolean
         hasThumbnail: boolean
-        tnW: number
+        hasVidPreview: boolean
-        tnH: number
+        imageAspectRatio: number
+        /** Number of thumbnails available */
+        tnNum: number
     }
     highlight: {
         name: string[] | undefined,
@@ -134,8 +134,15 @@ class Sist2Api {
 
         if ("thumbnail" in hit._source) {
             hit._props.hasThumbnail = true;
-            hit._props.tnW = Number(hit._source.thumbnail.split(",")[0]);
+
-            hit._props.tnH = Number(hit._source.thumbnail.split(",")[1]);
+            if (Number.isNaN(Number(hit._source.thumbnail))) {
+                // Backwards compatibility
+                hit._props.tnNum = 1;
+                hit._props.hasVidPreview = false;
+            } else {
+                hit._props.tnNum = Number(hit._source.thumbnail);
+                hit._props.hasVidPreview = hit._props.tnNum > 1;
+            }
         }
 
         switch (mimeCategory) {
@@ -149,6 +156,9 @@ class Sist2Api {
                     && hit._source.videoc !== "raw" && hit._source.videoc !== "ppm") {
                     hit._props.isPlayableImage = true;
                 }
+                if ("width" in hit._source && "height" in hit._source) {
+                    hit._props.imageAspectRatio = hit._source.width / hit._source.height;
+                }
                 break;
             case "video":
                 if ("videoc" in hit._source) {
@@ -181,30 +191,6 @@ class Sist2Api {
     setHitTags(hit: EsHit): void {
         const tags = [] as Tag[];
 
-        const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
-
-        switch (mimeCategory) {
-            case "image":
-            case "video":
-                if ("videoc" in hit._source && hit._source.videoc) {
-                    tags.push({
-                        style: "video",
-                        text: hit._source.videoc.replace(" ", ""),
-                        userTag: false
-                    } as Tag);
-                }
-                break
-            case "audio":
-                if ("audioc" in hit._source && hit._source.audioc) {
-                    tags.push({
-                        style: "audio",
-                        text: hit._source.audioc,
-                        userTag: false
-                    } as Tag);
-                }
-                break;
-        }
-
         // User tags
         if ("tag" in hit._source) {
             hit._source.tag.forEach(tag => {
@@ -241,11 +227,6 @@ class Sist2Api {
                 res.hits.hits.forEach((hit: EsHit) => {
                     hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
                     hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
-                    hit["_path_md5"] = CryptoES.MD5(
-                        hit["_source"]["path"] +
-                        (hit["_source"]["path"] ? "/" : "") +
-                        hit["_source"]["name"] + ext(hit)
-                    ).toString();
 
                     this.setHitProps(hit);
                     this.setHitTags(hit);
@@ -256,20 +237,31 @@ class Sist2Api {
         });
     }
 
-    getMimeTypes() {
+    getMimeTypes(query = undefined) {
-        return this.esQuery({
+        const AGGS = {
-            aggs: {
+            mimeTypes: {
-                mimeTypes: {
+                terms: {
-                    terms: {
+                    field: "mime",
-                        field: "mime",
+                    size: 10000
-                        size: 10000
-                    }
                 }
-            },
+            }
-            size: 0,
+        };
-        }).then(resp => {
+
+        if (!query) {
+            query = {
+                aggs: AGGS,
+                size: 0,
+            };
+        } else {
+            query.size = 0;
+            query.aggs = AGGS;
+        }
+
+        return this.esQuery(query).then(resp => {
             const mimeMap: any[] = [];
-            resp["aggregations"]["mimeTypes"]["buckets"].sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
+            const buckets = resp["aggregations"]["mimeTypes"]["buckets"];
+
+            buckets.sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
                 const tmp = bucket["key"].split("/");
                 const category = tmp[0];
                 const mime = tmp[1];
@@ -289,11 +281,18 @@ class Sist2Api {
                 });
 
                 if (!category_exists) {
-                    mimeMap.push({"text": category, children: [child]});
+                    mimeMap.push({text: category, children: [child], id: category});
                 }
             })
 
-            return mimeMap;
+            mimeMap.forEach(node => {
+                if (node.children) {
+                    node.children.sort((a, b) => a.id.localeCompare(b.id));
+                }
+            })
+            mimeMap.sort((a, b) => a.id.localeCompare(b.id))
+
+            return {buckets, mimeMap};
         });
     }
 
@@ -317,10 +316,6 @@ class Sist2Api {
         };
     }
 
-    getDocInfo(docId: string) {
-        return axios.get(`${this.baseUrl}d/${docId}`);
-    }
-
     getTags() {
         return this.esQuery({
             aggs: {
@@ -354,8 +349,7 @@ class Sist2Api {
         return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
             delete: false,
             name: tag,
-            doc_id: hit["_id"],
+            doc_id: hit["_id"]
-            path_md5: hit._path_md5
         });
     }
 
@@ -363,25 +357,24 @@ class Sist2Api {
         return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
             delete: true,
             name: tag,
-            doc_id: hit["_id"],
+            doc_id: hit["_id"]
-            path_md5: hit._path_md5
         });
     }
 
-    getTreemapCsvUrl(indexId: string) {
+    getTreemapStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/1`;
+        return `${this.baseUrl}s/${indexId}/TMAP`;
     }
 
-    getMimeCsvUrl(indexId: string) {
+    getMimeStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/2`;
+        return `${this.baseUrl}s/${indexId}/MAGG`;
     }
 
-    getSizeCsv(indexId: string) {
+    getSizeStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/3`;
+        return `${this.baseUrl}s/${indexId}/SAGG`;
     }
 
-    getDateCsv(indexId: string) {
+    getDateStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/4`;
+        return `${this.baseUrl}s/${indexId}/DAGG`;
     }
 }
 

sist2-vue/src/Sist2Query.ts

@@ -69,7 +69,7 @@ interface SortMode {
 
 class Sist2Query {
 
-    searchQuery(): any {
+    searchQuery(blankSearch: boolean = false): any {
 
         const getters = store.getters;
 
@@ -93,22 +93,6 @@ class Sist2Query {
             {terms: {index: selectedIndexIds}}
         ] as any[];
 
-        if (sizeMin && sizeMax) {
-            filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
-        } else if (sizeMin) {
-            filters.push({range: {size: {gte: sizeMin}}})
-        } else if (sizeMax) {
-            filters.push({range: {size: {lte: sizeMax}}})
-        }
-
-        if (dateMin && dateMax) {
-            filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
-        } else if (dateMin) {
-            filters.push({range: {mtime: {gte: dateMin}}})
-        } else if (dateMax) {
-            filters.push({range: {mtime: {lte: dateMax}}})
-        }
-
         const fields = [
             "name^8",
             "content^3",
@@ -128,20 +112,39 @@ class Sist2Query {
             fields.push("name.nGram^3");
         }
 
-        const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
+        if (!blankSearch) {
-        if (path !== "") {
+            if (sizeMin && sizeMax) {
-            filters.push({term: {path: path}})
+                filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
-        }
+            } else if (sizeMin) {
+                filters.push({range: {size: {gte: sizeMin}}})
+            } else if (sizeMax) {
+                filters.push({range: {size: {lte: sizeMax}}})
+            }
 
-        if (selectedMimeTypes.length > 0) {
+            if (dateMin && dateMax) {
-            filters.push({terms: {"mime": selectedMimeTypes}});
+                filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
-        }
+            } else if (dateMin) {
+                filters.push({range: {mtime: {gte: dateMin}}})
+            } else if (dateMax) {
+                filters.push({range: {mtime: {lte: dateMax}}})
+            }
 
-        if (selectedTags.length > 0) {
+            const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
-            if (getters.optTagOrOperator) {
+
-                filters.push({terms: {"tag": selectedTags}});
+            if (path !== "") {
-            } else {
+                filters.push({term: {path: path}})
-                selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
+            }
+
+            if (selectedMimeTypes.length > 0) {
+                filters.push({terms: {"mime": selectedMimeTypes}});
+            }
+
+            if (selectedTags.length > 0) {
+                if (getters.optTagOrOperator) {
+                    filters.push({terms: {"tag": selectedTags}});
+                } else {
+                    selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
+                }
             }
         }
 
@@ -182,7 +185,7 @@ class Sist2Query {
             size: size,
         } as any;
 
-        if (!empty) {
+        if (!empty && !blankSearch) {
             q.query.bool.must = query;
         }
 
@@ -207,7 +210,7 @@ class Sist2Query {
             };
 
             if (!legacyES) {
-                q.highlight.max_analyzed_offset = 9_999_999;
+                q.highlight.max_analyzed_offset = 999_999;
             }
 
             if (getters.optSearchInPath) {
@@ -237,7 +240,7 @@ class Sist2Query {
                 }
             }
 
-            if (!empty) {
+            if (!empty && !blankSearch) {
                 q.query.function_score.query.bool.must.push(query);
             }
         }

sist2-vue/src/components/AnalyzedContentSpan.vue

@@ -0,0 +1,21 @@
+<template>
+    <span :style="getStyle()">{{span.text}}</span>
+</template>
+
+<script>
+
+
+import ModelsRepo from "@/ml/modelsRepo";
+
+export default {
+    name: "AnalyzedContentSpan",
+    props: ["span", "text"],
+    methods: {
+        getStyle() {
+            return ModelsRepo.data[this.$store.getters.mlModel.name].labelStyles[this.span.label];
+        }
+    }
+}
+</script>
+
+<style scoped></style>

sist2-vue/src/components/AnalyzedContentSpanContainer.vue

@@ -0,0 +1,75 @@
+<template>
+    <div>
+        <b-card class="mb-2">
+            <AnalyzedContentSpan v-for="span of legend" :key="span.id" :span="span"
+                                 class="mr-2"></AnalyzedContentSpan>
+        </b-card>
+        <div class="content-div">
+            <AnalyzedContentSpan v-for="span of mergedSpans" :key="span.id" :span="span"></AnalyzedContentSpan>
+        </div>
+    </div>
+</template>
+
+<script>
+
+
+import AnalyzedContentSpan from "@/components/AnalyzedContentSpan.vue";
+import ModelsRepo from "@/ml/modelsRepo";
+
+export default {
+    name: "AnalyzedContentSpanContainer",
+    components: {AnalyzedContentSpan},
+    props: ["spans", "text"],
+    computed: {
+        legend() {
+            return Object.entries(ModelsRepo.data[this.$store.state.mlModel.name].legend)
+                .map(([label, name]) => ({
+                    text: name,
+                    id: label,
+                    label: label
+                }));
+        },
+        mergedSpans() {
+            const spans = this.spans;
+
+            const merged = [];
+
+            let lastLabel = null;
+            let fixSpace = false;
+            for (let i = 0; i < spans.length; i++) {
+
+                if (spans[i].label !== lastLabel) {
+                    let start = spans[i].wordIndex;
+                    const nextSpan = spans.slice(i + 1).find(s => s.label !== spans[i].label)
+                    let end = nextSpan ? nextSpan.wordIndex : undefined;
+
+                    if (end !== undefined && this.text[end - 1] === " ") {
+                        end -= 1;
+                        fixSpace = true;
+                    }
+
+                    merged.push({
+                        text: this.text.slice(start, end),
+                        label: spans[i].label,
+                        id: spans[i].wordIndex
+                    });
+
+                    if (fixSpace) {
+                        merged.push({
+                            text: " ",
+                            label: "O",
+                            id: end
+                        });
+                        fixSpace = false;
+                    }
+                    lastLabel = spans[i].label;
+                }
+            }
+
+            return merged;
+        },
+    },
+}
+</script>
+
+<style scoped></style>

sist2-vue/src/components/D3DateHistogram.vue

@@ -120,7 +120,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#date-histogram");
 
-      d3.csv(Sist2Api.getDateCsv(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getDateStat(indexId)).then(tabularData => {
         dateHistogram(tabularData.slice(), svg, this.$t("d3.dateHistogram"));
       });
     }

sist2-vue/src/components/D3MimeBarCount.vue

@@ -91,7 +91,7 @@ export default {
       const mimeSvgCount = d3.select("#agg-mime-count");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
         mimeBarCount(tabularData.slice(), mimeSvgCount, fillOpacity, this.$t("d3.mimeCount"));
       });
     }

sist2-vue/src/components/D3MimeBarSize.vue

@@ -90,7 +90,7 @@ export default {
       const mimeSvgSize = d3.select("#agg-mime-size");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
         mimeBarSize(tabularData.slice(), mimeSvgSize, fillOpacity, this.$t("d3.mimeSize"));
       });
     }

sist2-vue/src/components/D3SizeHistogram.vue

@@ -117,7 +117,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#size-histogram");
 
-      d3.csv(Sist2Api.getSizeCsv(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getSizeStat(indexId)).then(tabularData => {
         sizeHistogram(tabularData.slice(), svg, this.$t("d3.sizeHistogram"));
       });
     }

sist2-vue/src/components/D3Treemap.vue

@@ -240,7 +240,7 @@ export default {
           .style("overflow", "visible")
           .style("font", "10px sans-serif");
 
-      d3.csv(Sist2Api.getTreemapCsvUrl(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
         tabularData.forEach(row => {
           row.taxonomy = row.path.split("/");
           row.size = Number(row.size);

sist2-vue/src/components/DateSlider.vue

@@ -1,5 +1,31 @@
 <template>
-  <div id="dateSlider"></div>
+  <div v-if="$store.state.optUseDatePicker">
+    <b-row>
+      <b-col sm="6">
+        <b-form-datepicker
+            value-as-date
+            :date-format-options="{ year: 'numeric', month: '2-digit', day: '2-digit' }"
+            :locale="$store.state.optLang"
+            class="mb-2"
+            :value="dateMin" @input="setDateMin"></b-form-datepicker>
+      </b-col>
+      <b-col sm="6">
+        <b-form-datepicker
+            value-as-date
+            :date-format-options="{ year: 'numeric', month: '2-digit', day: '2-digit' }"
+            :locale="$store.state.optLang"
+            class="mb-2"
+            :value="dateMax" @input="setDateMax"></b-form-datepicker>
+      </b-col>
+    </b-row>
+  </div>
+  <div v-else>
+    <b-row>
+      <b-col style="height: 70px;">
+        <div id="dateSlider"></div>
+      </b-col>
+    </b-row>
+  </div>
 </template>
 
 <script>
@@ -10,11 +36,36 @@ import {mergeTooltips} from "@/util-js";
 
 export default {
   name: "DateSlider",
+  methods: {
+    setDateMin(val) {
+      const epochDate = Math.ceil(+val / 1000);
+      this.$store.commit("setDateMin", epochDate);
+    },
+    setDateMax(val) {
+      const epochDate = Math.ceil(+val / 1000);
+      this.$store.commit("setDateMax", epochDate);
+    },
+  },
+  computed: {
+    dateMin() {
+      const dateMin = this.$store.state.dateMin ? this.$store.state.dateMin : this.$store.state.dateBoundsMin;
+      return new Date(dateMin * 1000)
+    },
+    dateMax() {
+      const dateMax = this.$store.state.dateMax ? this.$store.state.dateMax : this.$store.state.dateBoundsMax;
+      return new Date(dateMax * 1000)
+    }
+  },
   mounted() {
     this.$store.subscribe((mutation) => {
       if (mutation.type === "setDateBoundsMax") {
         const elem = document.getElementById("dateSlider");
 
+        if (elem === null) {
+          // Using b-form-datepicker, skip initialisation of slider
+          return
+        }
+
         if (elem.children.length > 0) {
           return;
         }

sist2-vue/src/components/DebugInfo.vue

@@ -1,5 +1,5 @@
 <template>
-  <b-card class="mb-4 mt-4">
+  <b-card v-if="$store.state.sist2Info.showDebugInfo" class="mb-4 mt-4">
     <b-card-title><DebugIcon class="mr-1"></DebugIcon>{{ $t("debug") }}</b-card-title>
     <p v-html="$t('debugDescription')"></p>
 
@@ -15,7 +15,7 @@
 
 <script>
 import IndexDebugInfo from "@/components/IndexDebugInfo";
-import DebugIcon from "@/components/DebugIcon";
+import DebugIcon from "@/components/icons/DebugIcon";
 
 export default {
   name: "DebugInfo.vue",
@@ -27,10 +27,10 @@ export default {
         {key: "platform", value: this.$store.state.sist2Info.platform},
         {key: "debugBinary", value: this.$store.state.sist2Info.debug},
         {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
-        {key: "libscanCommitHash", value: this.$store.state.sist2Info.libscanHash},
         {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
         {key: "tagline", value: this.$store.state.sist2Info.tagline},
         {key: "dev", value: this.$store.state.sist2Info.dev},
+        {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
         {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
         {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
         {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},

sist2-vue/src/components/DocCard.vue

@@ -1,5 +1,6 @@
 <template>
-  <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`">
+  <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
+       @click="$store.commit('busTnTouchStart', null)">
     <b-card
         no-body
         img-top
@@ -10,37 +11,7 @@
       <ContentDiv :doc="doc"></ContentDiv>
 
       <!-- Thumbnail-->
-      <div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
+      <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
-        <div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
-          <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
-        </div>
-
-        <div
-            v-if="doc._props.isImage && !hover && doc._props.tnW / doc._props.tnH < 5"
-            class="card-img-overlay"
-            :class="{'small-badge': smallBadge}">
-          <span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
-        </div>
-
-        <div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0 && !hover"
-             class="card-img-overlay"
-             :class="{'small-badge': smallBadge}">
-          <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
-        </div>
-
-        <div v-if="doc._props.isPlayableVideo" class="play">
-          <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
-            <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
-          </svg>
-        </div>
-
-        <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-             :src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
-             alt=""
-             class="pointer fit card-img-top" @click="onThumbnailClick()">
-        <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
-             class="fit card-img-top">
-      </div>
 
       <!-- Audio player-->
       <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
@@ -56,6 +27,11 @@
           <DocFileTitle :doc="doc"></DocFileTitle>
         </div>
 
+        <!-- Featured line -->
+        <div style="display: flex">
+          <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
+        </div>
+
         <!-- Tags -->
         <div class="card-text">
           <TagContainer :hit="doc"></TagContainer>
@@ -71,31 +47,20 @@ import TagContainer from "@/components/TagContainer.vue";
 import DocFileTitle from "@/components/DocFileTitle.vue";
 import DocInfoModal from "@/components/DocInfoModal.vue";
 import ContentDiv from "@/components/ContentDiv.vue";
+import FullThumbnail from "@/components/FullThumbnail";
+import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
 
 
 export default {
-  components: {ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+  components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
   props: ["doc", "width"],
   data() {
     return {
       ext: ext,
       showInfo: false,
-      hover: false
     }
   },
   computed: {
-    placeHolderStyle() {
-
-      const tokens = this.doc._source.thumbnail.split(",");
-      const w = Number(tokens[0]);
-      const h = Number(tokens[1]);
-
-      const MAX_HEIGHT = 400;
-
-      return {
-        height: `${Math.min((h / w) * this.width, MAX_HEIGHT)}px`,
-      }
-    },
     smallBadge() {
       return this.width < 150;
     }
@@ -117,28 +82,10 @@ export default {
         }
       });
     },
-    onTnEnter() {
-      this.hover = true;
-    },
-    onTnLeave() {
-      this.hover = false;
-    }
   },
 }
 </script>
 <style>
-.img-wrapper {
-  position: relative;
-}
-
-.img-wrapper:hover svg {
-  fill: rgba(0, 0, 0, 1);
-}
-
-.pointer {
-  cursor: pointer;
-}
-
 .fit {
   display: block;
   min-width: 64px;
@@ -148,15 +95,17 @@ export default {
   width: auto;
   height: auto;
 }
+
+.audio-fit {
+  height: 39px;
+  vertical-align: bottom;
+  display: inline;
+  width: 100%;
+}
 </style>
 
 <style scoped>
 
-.card-img-top {
-  border-top-left-radius: 0;
-  border-top-right-radius: 0;
-}
-
 .padding-03 {
   padding: 0.3rem;
 }
@@ -174,55 +123,11 @@ export default {
   padding: 0.3rem;
 }
 
-.thumbnail-placeholder {
-
-}
-
-.card-img-overlay {
-  pointer-events: none;
-  padding: 0.75rem;
-  bottom: unset;
-  top: 0;
-  left: unset;
-  right: unset;
-}
-
-.badge-resolution {
-  color: #212529;
-  background-color: #FFC107;
-}
-
-.play {
-  position: absolute;
-  width: 25px;
-  height: 25px;
-  left: 50%;
-  top: 50%;
-  transform: translate(-50%, -50%);
-  pointer-events: none;
-}
-
-.play svg {
-  fill: rgba(0, 0, 0, 0.7);
-}
-
 .doc-card {
   padding-left: 3px;
   padding-right: 3px;
 }
 
-.small-badge {
-  padding: 1px 3px;
-  font-size: 70%;
-}
-
-.audio-fit {
-  height: 39px;
-  vertical-align: bottom;
-  display: inline;
-  width: 100%;
-}
-
 .sub-document .card {
   background: #AB47BC1F !important;
 }

sist2-vue/src/components/DocInfoModal.vue

@@ -2,7 +2,10 @@
   <b-modal :visible="show" size="lg" :hide-footer="true" static lazy @close="$emit('close')" @hide="$emit('close')"
   >
     <template #modal-title>
-      <h5 class="modal-title" :title="doc._source.name + ext(doc)">{{ doc._source.name + ext(doc) }}</h5>
+      <h5 class="modal-title" :title="doc._source.name + ext(doc)">
+        {{ doc._source.name + ext(doc) }}
+        <router-link :to="`/file?byId=${doc._id}`">#</router-link>
+      </h5>
     </template>
 
     <img v-if="doc._props.hasThumbnail" :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">

sist2-vue/src/components/DocListItem.vue

@@ -1,10 +1,13 @@
 <template>
-  <b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}">
+  <b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}"
+                     @mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
 
     <!-- Info modal-->
     <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
 
     <div class="media ml-2">
+
+      <!-- Thumbnail-->
       <div v-if="doc._props.hasThumbnail" class="align-self-start mr-2 wrapper-sm">
         <div class="img-wrapper">
           <div v-if="doc._props.isPlayableVideo" class="play">
@@ -25,6 +28,7 @@
         <FileIcon></FileIcon>
       </div>
 
+      <!-- Doc line-->
       <div class="doc-line ml-3">
         <div style="display: flex">
           <span class="info-icon" @click="showInfo = true"></span>
@@ -46,6 +50,11 @@
           <span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
           <span v-if="doc._source.author">{{ doc._source.author }}</span>
         </div>
+
+        <!-- Featured line -->
+        <div style="display: flex">
+          <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
+        </div>
       </div>
     </div>
   </b-list-group-item>
@@ -56,11 +65,12 @@ import TagContainer from "@/components/TagContainer";
 import DocFileTitle from "@/components/DocFileTitle";
 import DocInfoModal from "@/components/DocInfoModal";
 import ContentDiv from "@/components/ContentDiv";
-import FileIcon from "@/components/FileIcon";
+import FileIcon from "@/components/icons/FileIcon";
+import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
 
 export default {
   name: "DocListItem",
-  components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+  components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
   props: ["doc"],
   data() {
     return {
@@ -85,7 +95,13 @@ export default {
         return this.doc.highlight["path.nGram"] + "/"
       }
       return this.doc._source.path + "/"
-    }
+    },
+    onTnEnter() {
+      this.hover = true;
+    },
+    onTnLeave() {
+      this.hover = false;
+    },
   }
 }
 </script>
@@ -147,6 +163,7 @@ export default {
 .list-group-item .img-wrapper {
   width: 88px;
   height: 88px;
+  position: relative;
 }
 
 .fit-sm {

sist2-vue/src/components/FeaturedFieldsLine.vue

@@ -0,0 +1,46 @@
+<template>
+    <div class="featured-line" v-html="featuredLineHtml"></div>
+</template>
+
+<script>
+import {humanDate, humanFileSize} from "@/util";
+
+function scopedEval(context, expr) {
+    const evaluator = Function.apply(null, [...Object.keys(context), "expr", "return eval(expr)"]);
+    return evaluator.apply(null, [...Object.values(context), expr]);
+}
+
+
+export default {
+    name: "FeaturedFieldsLine",
+    props: ["doc"],
+    computed: {
+        featuredLineHtml() {
+            if (this.$store.getters.optFeaturedFields === undefined) {
+                return "";
+            }
+
+            const scope = {doc: this.doc._source, humanDate: humanDate, humanFileSize: humanFileSize};
+
+            return this.$store.getters.optFeaturedFields
+                .replaceAll(/\$\{([^}]*)}/g, (match, g1) => {
+                    return scopedEval(scope, g1);
+                });
+        }
+    }
+}
+</script>
+
+<style scoped>
+
+.featured-line {
+    font-size: 90%;
+    font-family: 'Source Sans Pro', 'Helvetica Neue', Arial, sans-serif;
+    color: #424242;
+    padding-left: 2px;
+}
+
+.theme-black .featured-line {
+    color: #bebebe;
+}
+</style>

sist2-vue/src/components/FullThumbnail.vue

@@ -0,0 +1,188 @@
+<template>
+  <div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()"
+       @touchstart="onTouchStart()">
+    <div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
+      <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
+    </div>
+
+    <div
+        v-if="doc._props.isImage && doc._props.imageAspectRatio < 5"
+        class="card-img-overlay"
+        :class="{'small-badge': smallBadge}">
+      <span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
+    </div>
+
+    <div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0"
+         class="card-img-overlay"
+         :class="{'small-badge': smallBadge}">
+      <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
+    </div>
+
+    <div v-if="doc._props.isPlayableVideo" class="play">
+      <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
+        <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
+      </svg>
+    </div>
+
+    <img ref="tn"
+         v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
+         :src="tnSrc"
+         alt=""
+         :style="{height: (doc._props.isGif && hover) ? `${tnHeight()}px` : undefined}"
+         class="pointer fit card-img-top" @click="onThumbnailClick()">
+    <img v-else :src="tnSrc" alt=""
+         class="fit card-img-top">
+
+    <ThumbnailProgressBar v-if="hover && doc._props.hasVidPreview"
+                          :progress="(currentThumbnailNum + 1) / (doc._props.tnNum)"
+    ></ThumbnailProgressBar>
+  </div>
+</template>
+
+<script>
+import {humanTime} from "@/util";
+import ThumbnailProgressBar from "@/components/ThumbnailProgressBar";
+
+export default {
+  name: "FullThumbnail",
+  props: ["doc", "smallBadge"],
+  components: {ThumbnailProgressBar},
+  data() {
+    return {
+      hover: false,
+      currentThumbnailNum: 0,
+      timeoutId: null
+    }
+  },
+  created() {
+    this.$store.subscribe((mutation) => {
+      if (mutation.type === "busTnTouchStart" && mutation.payload !== this.doc._id) {
+        this.onTnLeave();
+      }
+    });
+  },
+  computed: {
+    tnSrc() {
+      return this.getThumbnailSrc(this.currentThumbnailNum);
+    },
+  },
+  methods: {
+    getThumbnailSrc(thumbnailNum) {
+      const doc = this.doc;
+      const props = doc._props;
+      if (props.isGif && this.hover) {
+        return `f/${doc._id}`;
+      }
+      return (this.currentThumbnailNum === 0)
+          ? `t/${doc._source.index}/${doc._id}`
+          : `t/${doc._source.index}/${doc._id}/${String(thumbnailNum).padStart(4, "0")}`;
+    },
+    humanTime: humanTime,
+    onThumbnailClick() {
+      this.$emit("onThumbnailClick");
+    },
+    tnHeight() {
+      return this.$refs.tn.height;
+    },
+    tnWidth() {
+      return this.$refs.tn.width;
+    },
+    onTnEnter() {
+      this.hover = true;
+      const start = Date.now()
+      if (this.doc._props.hasVidPreview) {
+        let img = new Image();
+        img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
+        img.onload = () => {
+          this.currentThumbnailNum += 1;
+          this.scheduleNextTnNum(Date.now() - start);
+        }
+      }
+    },
+    onTnLeave() {
+      this.currentThumbnailNum = 0;
+      this.hover = false;
+      if (this.timeoutId !== null) {
+        window.clearTimeout(this.timeoutId);
+        this.timeoutId = null;
+      }
+    },
+    scheduleNextTnNum(offset = 0) {
+      const INTERVAL = (this.$store.state.optVidPreviewInterval ?? 700) - offset;
+      this.timeoutId = window.setTimeout(() => {
+        const start = Date.now();
+        if (!this.hover) {
+          return;
+        }
+        if (this.currentThumbnailNum === this.doc._props.tnNum - 1) {
+          this.currentThumbnailNum = 0;
+          this.scheduleNextTnNum();
+        } else {
+          let img = new Image();
+          img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
+          img.onload = () => {
+            this.currentThumbnailNum += 1;
+            this.scheduleNextTnNum(Date.now() - start);
+          }
+        }
+      }, INTERVAL);
+    },
+    onTouchStart() {
+      this.$store.commit("busTnTouchStart", this.doc._id);
+      if (!this.hover) {
+        this.onTnEnter()
+      }
+    },
+  }
+}
+</script>
+
+<style scoped>
+.img-wrapper {
+  position: relative;
+}
+
+.img-wrapper:hover svg {
+  fill: rgba(0, 0, 0, 1);
+}
+
+.card-img-top {
+  border-top-left-radius: 0;
+  border-top-right-radius: 0;
+}
+
+.play {
+  position: absolute;
+  width: 25px;
+  height: 25px;
+  left: 50%;
+  top: 50%;
+  transform: translate(-50%, -50%);
+  pointer-events: none;
+}
+
+.play svg {
+  fill: rgba(0, 0, 0, 0.7);
+}
+
+.badge-resolution {
+  color: #c6c6c6;
+  background-color: #272727CC;
+  padding: 2px 3px;
+}
+
+.card-img-overlay {
+  pointer-events: none;
+  padding: 2px 6px;
+  bottom: 4px;
+  top: unset;
+  left: unset;
+  right: 0;
+}
+
+.small-badge {
+  padding: 1px 3px;
+  font-size: 70%;
+}
+
+</style>

sist2-vue/src/components/IndexPicker.vue

@@ -29,7 +29,7 @@
           :class="{active: lastClickIndex === idx}"
       >
         <div class="d-flex">
-          <b-checkbox @change="toggleIndex(idx)" :checked="isSelected(idx)"></b-checkbox>
+          <b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
           {{ idx.name }}
           <span class="text-muted timestamp-text ml-2">{{ formatIdxDate(idx.timestamp) }}</span>
         </div>
@@ -133,6 +133,11 @@ export default Vue.extend({
   font-size: 80%;
 }
 
+.theme-black .version-badge {
+  color: #eee !important;
+  background: none;
+}
+
 .version-badge {
   color: #222 !important;
   background: none;
@@ -163,4 +168,24 @@ export default Vue.extend({
   background-color: inherit;
   color: inherit;
 }
+
+.theme-black .list-group-item {
+  border: 1px solid rgba(255,255,255, 0.1);
+}
+
+.theme-black .list-group-item:first-child {
+  border: 1px solid rgba(255,255,255, 0.05);
+}
+
+.theme-black .list-group-item.active {
+  z-index: 2;
+  background-color: inherit;
+  color: inherit;
+  border: 1px solid rgba(255,255,255, 0.3);
+  border-radius: 0;
+}
+
+.theme-black .list-group {
+  border-radius: 0;
+}
 </style>