Update CI build script

.dockerignore

@@ -38,6 +38,3 @@ build/
 __pycache__/
 sist2-vue/dist
 sist2-admin/frontend/dist
-*.fts
-.git
-third-party/libscan/third-party/ext_*/*

.gitattributes

@@ -0,0 +1,3 @@
+CMakeModules/* linguist-vendored
+**/*_generated.c linguist-vendored
+**/*_generated.h linguist-vendored

.gitignore

@@ -3,7 +3,6 @@ thumbs
 *.cbp
 CMakeCache.txt
 CMakeFiles
-cmake-build-default-event-trace
 cmake-build-debug
 cmake_install.cmake
 Makefile
@@ -45,5 +44,3 @@ src/index/static_generated.c
 *.sist2
 *-shm
 *-journal
-.vscode
-*.fts

CMakeLists.txt

@@ -5,7 +5,6 @@ set(CMAKE_C_STANDARD 11)
 
 option(SIST_DEBUG "Build a debug executable" on)
 option(SIST_FAST "Enable more optimisation flags" off)
-option(SIST_DEBUG_INFO "Turn on debug information in web interface" on)
 
 add_compile_definitions(
         "SIST_PLATFORM=${SIST_PLATFORM}"
@@ -15,24 +14,13 @@ if (SIST_DEBUG)
     add_compile_definitions(
             "SIST_DEBUG=${SIST_DEBUG}"
     )
-    set(VCPKG_BUILD_TYPE debug)
+endif()
-else ()
-    set(VCPKG_BUILD_TYPE release)
-endif ()
-
-if (SIST_DEBUG_INFO)
-    add_compile_definitions(
-            "SIST_DEBUG_INFO=${SIST_DEBUG_INFO}"
-    )
-endif ()
-
 
 add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
-add_executable(
+add_executable(sist2
-        sist2
         # argparse
         third-party/argparse/argparse.h third-party/argparse/argparse.c
 
@@ -53,16 +41,13 @@ add_executable(
         src/types.h
         src/log.c src/log.h
         src/cli.c src/cli.h
+        src/parsing/sidecar.c src/parsing/sidecar.h
         src/database/database.c src/database/database.h
         src/parsing/fs_util.h
 
         src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp
 
-        src/database/database_stats.c
+        src/database/database_stats.c src/database/database_stats.h src/database/database_schema.c)
-        src/database/database_schema.c
-        src/database/database_fts.c
-        src/web/web_fts.c
-        src/database/database_embeddings.c)
 set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
@@ -75,7 +60,6 @@ find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
 find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
 find_package(unofficial-sqlite3 CONFIG REQUIRED)
-find_package(OpenBLAS CONFIG REQUIRED)
 
 
 target_include_directories(
@@ -101,7 +85,7 @@ if (SIST_DEBUG)
             -fno-omit-frame-pointer
             -fsanitize=address
             -fno-inline
-            #            -O2
+#            -O2
     )
     target_link_options(
             sist2
@@ -131,7 +115,6 @@ else ()
             PRIVATE
 
             -Ofast
-#            -g
             -fno-stack-protector
             -fomit-frame-pointer
             -w
@@ -158,7 +141,6 @@ target_link_libraries(
 
         ${MAGIC_LIB}
         unofficial::sqlite3::sqlite3
-        OpenBLAS::OpenBLAS
 )
 
 add_custom_target(

Dockerfile

@@ -1,6 +1,11 @@
 FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
+RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /build/
 
 COPY scripts scripts
@@ -14,12 +19,14 @@ COPY sist2-admin sist2-admin
 RUN cd sist2-vue/ && npm install && npm run build
 RUN cd sist2-admin/frontend/ && npm install && npm run build
 
-RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
 RUN cd build && make -j$(nproc)
 RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
 
+WORKDIR /root
+
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 
@@ -30,24 +37,21 @@ RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 li
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
 COPY --from=build /build/build/sist2 /root/sist2
 
 # sist2-admin
-WORKDIR /root/sist2-admin
+COPY sist2-admin/requirements.txt sist2-admin/
-COPY sist2-admin/requirements.txt /root/sist2-admin/
+RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
-RUN ln /usr/bin/python3 /usr/bin/python
+COPY --from=build /build/sist2-admin/ sist2-admin/
-RUN python -m pip install --no-cache -r /root/sist2-admin/requirements.txt
-COPY --from=build /build/sist2-admin/ /root/sist2-admin/

Dockerfile.arm64

@@ -1,22 +1,9 @@
 FROM simon987/sist2-build-arm64 as build
 MAINTAINER simon987 <me@simon987.net>
 
-WORKDIR /build/
-
-COPY scripts scripts
-COPY schema schema
-COPY CMakeLists.txt .
-COPY third-party third-party
-COPY src src
-COPY sist2-vue sist2-vue
-COPY sist2-admin sist2-admin
-
-RUN cd sist2-vue/ && npm install && npm run build
-RUN cd sist2-admin/frontend/ && npm install && npm run build
-
 WORKDIR /build/
 ADD . /build/
-RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
 RUN cd build && make -j$(nproc)
 RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
@@ -33,17 +20,16 @@ RUN apt update && apt install -y curl libasan5 libmagic1 tesseract-ocr python3-p
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
 COPY --from=build /build/build/sist2 /root/sist2

README.md

@@ -10,13 +10,13 @@ sist2 (Simple incremental search tool)
 
 *Warning: sist2 is in early development*
 
-![search panel](docs/sist2.gif)
+![search panel](docs/sist2.png)
 
 ## Features
 
 * Fast, low memory usage, multi-threaded
-* Manage & schedule scan jobs with simple web interface (Docker only)
 * Mobile-friendly Web interface
+* Portable (all its features are packaged in a single executable)
 * Extracts text and metadata from common file types \*
 * Generates thumbnails \*
 * Incremental scanning
@@ -24,64 +24,47 @@ sist2 (Simple incremental search tool)
 * Recursive scan inside archive files \*\*
 * OCR support with tesseract \*\*\*
 * Stats page & disk utilisation visualization
-* Named-entity recognition (client-side) \*\*\*\*
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
 \*\*\* See [OCR](#ocr)
-\*\*\*\* See [Named-Entity Recognition](#NER)
+
+![stats](docs/stats.png)
 
 ## Getting Started
 
-### Using Docker Compose *(Windows/Linux/Mac)*
+1. Have an Elasticsearch (>= 6.8.X, ideally >=7.14.0) instance running
-
-```yaml
-version: "3"
-
-services:
-  elasticsearch:
-    image: elasticsearch:7.17.9
-    restart: unless-stopped
-    environment:
-      - "discovery.type=single-node"
-      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
-  sist2-admin:
-    image: simon987/sist2:3.3.4-x64-linux
-    restart: unless-stopped
-    volumes:
-      - ./sist2-admin-data/:/sist2-admin/
-      - /:/host
-    ports:
-      - 4090:4090 # sist2
-      - 8080:8080 # sist2-admin
-    working_dir: /root/sist2-admin/
-    entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
-```
-
-Navigate to http://localhost:8080/ to configure sist2-admin.
-
-### Using the executable file *(Linux/WSL only)*
-
-1. Choose search backend (See [comparison](#search-backends)):
-    * **Elasticsearch**: have an Elasticsearch (version >= 6.8.X, ideally >=7.14.0) instance running
     1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
-        2. *(or)* Run using docker:
+    1. *(or)* Run using docker:
         ```bash
         docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
         ```
-    * **SQLite**: No installation required
+    1. *(or)* Run using docker-compose:
+        ```yaml
+          elasticsearch:
+            image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9
+            environment:
+              - discovery.type=single-node
+              - "ES_JAVA_OPTS=-Xms1G -Xmx2G"
+        ```
+1. Download sist2 executable
+    1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases). 
+Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x` *
+    2. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
+       recommended!)*
+    3. *(or)* `docker pull simon987/sist2:2.12.1-x64-linux`
 
-2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
+1. See [Usage guide](docs/USAGE.md)
-   Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
-3. See [usage guide](docs/USAGE.md) for command line usage.
 
-Example usage:
+\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
 
-1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
+## Example usage
-2. Prepare search index:
+
-    * **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
+See [Usage guide](docs/USAGE.md) for more details
-    * **SQLite**: `sist2 index --search-index ./search.sist2 ./documents.sist2`
+
-3. Start web interface: `sist2 web ./documents.sist2`
+1. Scan a directory: `sist2 scan ~/Documents -o ./docs_idx`
+1. Push index to Elasticsearch: `sist2 index ./docs_idx`
+1. Start web interface: `sist2 web ./docs_idx`
 
 ## Format support
 
@@ -99,7 +82,7 @@ Example usage:
 | tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
 | docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
 | doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
-| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | yes         | author, title                                                                                                                          |
+| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | no          | author, title                                                                                                                          |
 | wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
 | json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
@@ -126,7 +109,7 @@ Download the language data files with your package manager (`apt install tessera
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
 The `simon987/sist2` image comes with common languages
-(hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
+(hin, jpn, eng, fra, rus, spa, chi_sim, deu) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
 name must be identical to the `*.traineddata` file installed on your system 
@@ -140,62 +123,20 @@ sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
 sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
 ```
 
-### Search backends
-
-sist2 v3.0.7+ supports SQLite search backend. The SQLite search backend has
-fewer features and generally comparable query performance for medium-size
-indices, but it uses much less memory and is easier to set up.
-
-|                                              |                       SQLite                        |                                                             Elasticsearch                                                             |
-|----------------------------------------------|:---------------------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------:|
-| Requires separate search engine installation |                                                     |                                                                   ✓                                                                   |
-| Memory footprint                             |                        ~20MB                        |                                                                >500MB                                                                 |
-| Query syntax                                 |      [fts5](https://www.sqlite.org/fts5.html)       | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
-| Fuzzy search                                 |                                                     |                                                                   ✓                                                                   |
-| Media Types tree real-time updating          |                                                     |                                                                   ✓                                                                   |
-| Search in file `path`                        | [WIP](https://github.com/simon987/sist2/issues/402) |                                                                   ✓                                                                   |
-| Manual tagging                               |                          ✓                          |                                                                   ✓                                                                   |
-| User scripts                                 |                          ✓                          |                                                                   ✓                                                                   |
-| Media Type breakdown for search results      |                                                     |                                                                   ✓                                                                   |
-| Embeddings search                            |                      ✓ *O(n)*                       |                                                              ✓ *O(logn)*                                                              |
-
-### NER
-
-sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to
-**Configuration** > **Machine learning options** > **Model repositories**
-to enable it.
-
-The text processing is done in your browser, no data is sent to any third-party services.
-See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) for more details.
-
-#### List of available repositories:
-
-| URL                                                                                                     | Maintainer                              | Purpose |
-|---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
-| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
-
-<details>
-  <summary>Screenshot</summary>
-
-![ner](docs/ner.png)
-
-</details>
-
 ## Build from source
 
 You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries
 
-### Using docker
+### With docker (recommended)
 
 ```bash
 git clone --recursive https://github.com/simon987/sist2/
 cd sist2
-docker build . -t my-sist2-image
+docker build . -f ./Dockerfile -t my-sist2-image
-# Copy sist2 executable from docker image
 docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 ```
 
-### Using a linux computer
+### On a linux computer
 
 1. Install compile-time dependencies
 
@@ -203,14 +144,15 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
    apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
    ```
 
-2. Install vcpkg using my fork: https://github.com/simon987/vcpkg
+1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
-3. Install vcpkg dependencies
+
+1. Install vcpkg dependencies
 
     ```bash
-    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib]
+    vcpkg install curl[core,openssl] sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
     ```
 
-4. Build
+1. Build
     ```bash
     git clone --recursive https://github.com/simon987/sist2/
     (cd sist2-vue; npm install; npm run build)

docs/USAGE.md

@@ -1,68 +1,78 @@
 # Usage
 
+*More examples (specifically with docker/compose) are in progress*
+
+* [scan](#scan)
+    * [options](#scan-options)
+    * [examples](#scan-examples)
+    * [index format](#index-format)
+* [index](#index)
+    * [options](#index-options)
+    * [examples](#index-examples)
+* [web](#web)
+    * [options](#web-options)
+    * [examples](#web-examples)
+    * [rewrite_url](#rewrite_url)
+* [elasticsearch](#elasticsearch)
+* [exec-script](#exec-script)
+* [tagging](#tagging)
+* [sidecar files](#sidecar-files)
+
 ```
 Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
-   or: sist2 sqlite-index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
-
+   or: sist2 exec-script [OPTION]... INDEX
 Lightning-fast file system indexer and search tool.
 
     -h, --help                        show this help message and exit
-    -v, --version                     Print version and exit.
+    -v, --version                     Show version and exit
-    --verbose                         Turn on logging.
+    --verbose                         Turn on logging
-    --very-verbose                    Turn on debug messages.
+    --very-verbose                    Turn on debug messages
-    --json-logs                       Output logs in JSON format.
 
 Scan options
-    -t, --threads=<int>               Number of threads. DEFAULT: 1
+    -t, --threads=<int>               Number of threads. DEFAULT=1
-    -q, --thumbnail_count-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
+    --mem-throttle=<int>              Total memory threshold in MiB for scan throttling. DEFAULT=0
-    --thumbnail_count-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
+    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT=2
-    --thumbnail_count-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
+    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT=500
-    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
+    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT=1
-    -o, --output=<str>                Output index file path. DEFAULT: index.sist2
+    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT=32768
-    --incremental                     If the output file path exists, only scan new or modified files.
+    --incremental=<str>               Reuse an existing index and only scan modified files.
-    --optimize-index                  Defragment index file after scan to reduce its file size.
+    -o, --output=<str>                Output directory. DEFAULT=index.sist2/
     --rewrite-url=<str>               Serve files from this url instead of from disk.
-    --name=<str>                      Index display name. DEFAULT: index
+    --name=<str>                      Index display name. DEFAULT: (name of the directory)
     --depth=<int>                     Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: don't scan, list: only save file names as text, shallow: don't scan archives inside archives. DEFAULT: recurse
+    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
     --archive-passphrase=<str>        Passphrase for encrypted archive files
     --ocr-lang=<str>                  Tesseract language (use 'tesseract --list-langs' to see which are installed on your machine)
     --ocr-images                      Enable OCR'ing of image files.
     --ocr-ebooks                      Enable OCR'ing of ebook files.
-    -e, --exclude=<str>               Files that match this regex will not be scanned.
+    -e, --exclude=<str>               Files that match this regex will not be scanned
-    --fast                            Only index file names & mime type.
+    --fast                            Only index file names & mime type
     --treemap-threshold=<str>         Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
     --mem-buffer=<int>                Maximum memory buffer size per thread in MiB for files inside archives (see USAGE.md). DEFAULT: 2000
     --read-subtitles                  Read subtitles from media files.
-    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata).
+    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata)
     --checksums                       Calculate file checksums when scanning.
     --list-file=<str>                 Specify a list of newline-delimited paths to be scanned instead of normal directory traversal. Use '-' to read from stdin.
 
 Index options
-    -t, --threads=<int>               Number of threads. DEFAULT: 1
+    -t, --threads=<int>               Number of threads. DEFAULT=1
-    --es-url=<str>                    Elasticsearch url with port. DEFAULT: http://localhost:9200
+    --es-url=<str>                    Elasticsearch url with port. DEFAULT=http://localhost:9200
-    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
-    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
+    -p, --print                       Just print JSON documents to stdout.
-    -p, --print                       Print JSON documents to stdout instead of indexing to elasticsearch.
+    --incremental-index               Conduct incremental indexing, assumes that the old index is already digested by Elasticsearch.
-    --incremental-index               Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
     --script-file=<str>               Path to user script.
     --mappings-file=<str>             Path to Elasticsearch mappings.
     --settings-file=<str>             Path to Elasticsearch settings.
     --async-script                    Execute user script asynchronously.
-    --batch-size=<int>                Index batch size. DEFAULT: 70
+    --batch-size=<int>                Index batch size. DEFAULT: 100
-    -f, --force-reset                 Reset Elasticsearch mappings and settings.
+    -f, --force-reset                 Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
-
-sqlite-index options
-    --search-index=<str>              Path to search index. Will be created if it does not exist yet.
 
 Web options
-    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
+    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
-    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
-    --search-index=<str>              Path to SQLite search index.
+    --bind=<str>                      Listen on this address. DEFAULT=localhost:4090
-    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
-    --bind=<str>                      Listen for connections on this address. DEFAULT: localhost:4090
     --auth=<str>                      Basic auth in user:password format
     --auth0-audience=<str>            API audience/identifier
     --auth0-domain=<str>              Application domain
@@ -73,18 +83,89 @@ Web options
     --dev                             Serve html & js files from disk (for development)
     --lang=<str>                      Default UI language. Can be changed by the user
 
+Exec-script options
+    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --script-file=<str>               Path to user script.
+    --async-script                    Execute user script asynchronously.
 Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
+## Scan
+
+### Scan options
+
+* `-t, --threads` 
+      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
+* `--mem-throttle`
+    Total memory threshold in MiB for scan throttling. Worker threads will not start a new parse job
+    until the total memory usage of sist2 is below this threshold. Set to 0 to disable. DEFAULT=0
+* `-q, --thumbnail-quality` 
+    Thumbnail quality, on a scale of 2 to 32, 2 being the best. See section below for a rough estimate of thumbnail database size
+* `--thumbnail-size` 
+    Thumbnail size in pixels.
+* `--thumbnail-count`
+    Maximum number of thumbnails to generate. When set to a value >= 2, thumbnails for video previews
+    will be generated. The actual number of thumbnails generated depends on the length of the video (maximum 1 image 
+    every ~7s). Set to 0 to completely disable thumbnails.
+* `--content-size` 
+    Number of bytes of text to be extracted from the content of files (plain text, PDFs etc.).
+    Repeated whitespace and special characters do not count toward this limit.
+    Set to 0 to completely disable content parsing.
+* `--incremental`
+    Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
+    will be copied to the new index and will not be parsed again.
+* `-o, --output` Output directory. 
+* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
+* `--name` Set the `name` option for the web module
+* `--depth` Maximum scan dept. Set to 0 only scan files directly in the root directory, set to -1 for infinite depth
+* `--archive` Archive file mode.
+    * skip: Don't parse
+    * list: Only get file names as text
+    * shallow: Don't parse archives inside archives.
+    * recurse: Scan archives recursively (default)
+* `--ocr-lang`, `--ocr-ebooks`, `--ocr-images` See [OCR](../README.md#OCR)
+* `-e, --exclude` Regex pattern to exclude files. A file is excluded if the pattern matches any 
+    part of the full absolute path.
+    
+    Examples: 
+    * `-e ".*\.ttf"`: Ignore ttf files
+    * `-e ".*\.(ttf|rar)"`: Ignore ttf and rar files
+    * `-e "^/mnt/backups/"`: Ignore all files in the `/mnt/backups/` directory
+    * `-e "^/mnt/Data[12]/"`: Ignore all files in the `/mnt/Data1/` and `/mnt/Data2/` directory
+    * `-e "(^/usr/)|(^/var/)|(^/media/DRIVE-A/tmp/)|(^/media/DRIVE-B/Trash/)"` Exclude the
+     `/usr`, `/var`, `/media/DRIVE-A/tmp`, `/media/DRIVE-B/Trash` directories
+* `--fast` Only index file names and mime type
+* `--treemap-threshold` Directories smaller than (`treemap-threshold` * `<total size of the index>`)
+    will not be considered for the disk utilisation visualization; their size will be added to
+    the parent directory. If the parent directory is still smaller than the threshold, it will also be "merged upwards"
+    and so on.
+    
+    In effect, smaller `treemap-threshold` values will yield a more detailed 
+    (but also a more cluttered and harder to read) visualization. 
+    
+* `--mem-buffer` Maximum memory buffer size in MiB (per thread) for files inside archives. Media files 
+    larger than this number will be read sequentially and no *seek* operations will be supported.
+
+    To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
+* `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
+* `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
+* `--checksums` Calculate file checksums (SHA1) when scanning files. This option does not cause any additional read 
+  operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
+  files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
+
+
 #### Thumbnail database size estimation
 
-See chart below for rough estimate of thumbnail_count size vs. thumbnail_count size & quality arguments:
+See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
 
-For example, `--thumbnail_count-size=500`, `--thumbnail_count-quality=50` for a directory with 8 million images will create a thumbnail_count database 
+For example, `--thumbnail-size=500`, `--thumbnail-quality=2` for a directory with 8 million images will create a thumbnail database 
-that is about `8000000 * 11.8kB = 94.4GB`.
+that is about `8000000 * 36kB = 288GB`.
 
 ![thumbnail_size](thumbnail_size.png)
 
+// TODO: add note about LMDB page size 4096
+
 ### Scan examples
 
 Simple scan
@@ -92,70 +173,132 @@ Simple scan
 sist2 scan ~/Documents
 
 sist2 scan \
-    --threads 4 --content-size 16000000 --thumbnail_count-quality 2 --archive shallow \
+    --threads 4 --content-size 16000000 --thumbnail-quality 2 --archive shallow \
     --name "My Documents" --rewrite-url "http://nas.domain.local/My Documents/" \
-    ~/Documents -o ./documents.sist2
+    ~/Documents -o ./documents.idx/
 ```
 
 Incremental scan
-
+```
-If the index file does not exist, `--incremental` has no effect.
+sist2 scan --incremental ./orig_idx/ -o ./updated_idx/ ~/Documents
-```bash
-sist scan ~/Documents -o ./documents.sist2
-sist scan ~/Documents -o ./documents.sist2 --incremental
-# or
-sist scan ~/Documents -o ./documents.sist2 --incremental
-sist scan ~/Documents -o ./documents.sist2 --incremental
 ```
 
-### Index documents to Elasticsearch search backend
+### Index format
 
-```bash
+A typical `ndjson` type index structure looks like this:
-sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index.sist2
+```
-sist2 index ./my_index.sist2
+documents.idx/
+├── descriptor.json
+├── _index_main.ndjson.zst
+├── treemap.csv
+├── agg_mime.csv
+├── agg_date.csv
+├── add_size.csv
+├── thumbs/
+|   ├── data.mdb
+|   └── lock.mdb
+├── tags/
+|   ├── data.mdb
+|   └── lock.mdb
+└── meta/
+    ├── data.mdb
+    └── lock.mdb
 ```
 
-#### Index documents to SQLite search backend
+The `_index_*.ndjson.zst` files contain the document data in JSON format, in a compressed newline-delemited file.
+
+The `thumbs/` folder is a [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database)
+database containing the thumbnails.
+
+The `descriptor.json` file contains general information about the index. The 
+following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rewrite_url) and `timestamp`.
+
+The `.csv` are pre-computed aggregations necessary for the stats page.
+
+*thumbs/*:
+
+LMDB key-value store. Keys are **binary** 16-byte md5 hash* (`_id` field)
+and values are raw image bytes.
+
+*\* Hash is calculated from the full path of the file, including the extension, relative to the index root*
+
+
+## Index
+### Index options
+ * `--es-url` 
+ Elasticsearch url and port. If you are using docker, make sure that both containers are on the
+ same network.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
+ * `-p, --print` 
+    Print index in JSON format to stdout.
+ * `--incremental-index`
+   Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
+   Only the new changes since the last scan will be sent.
+ * `--script-file` 
+    Path to user script. See [Scripting](scripting.md).
+ * `--mappings-file`
+    Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used.
+ * `--settings-file`
+    Path to custom Elasticsearch settings. *(See above)*
+ * `--async-script` 
+    Use `wait_for_completion=false` elasticsearch option while executing user script.
+     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
+ * `--batch-size=<int>` 
+    Index batch size. Indexing is generally faster with larger batches, but payloads that
+    are too large will fail and additional overhead for retrying with smaller sizes may slow
+    down the process.
+ * `-f, --force-reset` 
+    Reset Elasticsearch mappings and settings.
+ * `-t, --threads` Number of threads to use. Ideally, choose a number equal to the number of logical cores of the machine hosting Elasticsearch.
+    
+### Index examples
+
+**Push to elasticsearch**
 ```bash
-# The search index will be created if it does not exist already
+sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index/
-sist2 sqlite-index ./index1.sist2 --search-index search.sist2
+sist2 index ./my_index/
-sist2 sqlite-index ./index2.sist2 --search-index search.sist2
 ```
 
 **Save index in JSON format**
 ```bash
-sist2 index --print ./my_index.sist2 > my_index.ndjson
+sist2 index --print ./my_index/ > my_index.ndjson
 ```
 
 **Inspect contents of an index**
 ```bash
-sist2 index --print ./my_index.sist2 | jq | less
+sist2 index --print ./my_index/ | jq | less
 ```
 
 ## Web
 
+### Web options
+ * `--es-url=<str>` Elasticsearch url.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
+ * `--bind=<str>` Listen on this address.
+ * `--auth=<str>` Basic auth in user:password format
+ * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
+    `--auth` argument, but authentication is only applied the `/tag/` endpoint.
+ * `--tagline=<str>` When specified, will replace the default tagline in the navbar.
+ * `--dev` Serve html & js files from disk (for development, used to modify frontend files without having to recompile)
+ * `--lang=<str>` Set the default web UI language (See #180 for a list of supported languages, default
+   is `en`). The user can change the language in the configuration page
+ * `--auth0-audience`, `--auth0-domain`, `--auth0-client-id`, `--auth0-public-key-file` See [Authentication with Auth0](auth0.md)
+
 ### Web examples
 
-**Single index (Elasticsearch backend)**
+**Single index**
 ```bash
-sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index.sist2
+sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index
 ```
 
-**Multiple indices (Elasticsearch backend)**
+**Multiple indices**
 ```bash
 # Indices will be displayed in this order in the web interface
-sist2 web index1.sist2 index2.sist2 index3.sist2 index4.sist2
+sist2 web index1 index2 index3 index4
 ```
 
-**SQLite search backend**
-```bash
-sist2 web --search-index search.sist2 index1.sist2
-```
-
-#### Auth0 authentication
-
-See [auth0.md](auth0.md)
-
 ### rewrite_url
 
 When the `rewrite_url` field is not empty, the web module ignores the `root`
@@ -175,30 +318,9 @@ Using a version >=7.14.0 is recommended to enable the following features:
 When using a legacy version of ES, a notice will be displayed next to the sist2 version in the web UI.
 If you don't care about the features above, you can ignore it or disable it in the configuration page.
 
-# Embeddings search
+## exec-script
 
-Since v3.2.0, User scripts can be used to generate _embeddings_ (vector of float32 numbers) which are stored in the .sist2 index file 
+The `exec-script` command is used to execute a user script for an index that has already been imported to Elasticsearch with the `index` command. Note that the documents will not be reset to their default state before each execution as the `index` command does: if you make undesired changes to the documents by accident, you will need to run `index` again to revert to the original state.
-(see [scripting](scripting.md)). Embeddings can be used for:
-
-* Nearest-neighbor queries (e.g. "return the documents most similar to this one")
-* Semantic searches (e.g. "return the documents that are most closely related to the given topic")
-
-In theory, embeddings can be created for any type of documents (image, text, audio etc.).
-
-For example, the [clip](https://github.com/simon987/sist2-script-clip) User Script, generates 512-d embeddings of images
-(videos are also supported using the thumbnails generated by sist2). When the user enters a query in the "Embeddings Search"
-textbox, the query's embedding is generated in their browser, leveraging the ONNX web runtime.
-
-<details>
-  <summary>Screenshots</summary>
- 
- ![embeddings-1](embeddings-1.png)
- ![embeddings-2](embeddings-2.png)
-
-1. Embeddings search bar. You can select the model using the dropdown on the left.
-2. This icon appears for indices with embeddings search enabled.
-3. Documents with this icon have embeddings. Click on the icon to perform KNN search.
-</details>
 
 
 # Tagging
@@ -226,3 +348,42 @@ See [Automatic tagging](#automatic-tagging) for information about tag
 ### Automatic tagging
 
 See [scripting](scripting.md) documentation.
+
+# Sidecar files
+
+When scanning, sist2 will read metadata from `.s2meta` JSON files and overwrite the 
+original document's indexed metadata (does not modify the actual file). Sidecar metadata files will also work inside archives.
+Sidecar files themselves are not saved in the index.
+
+This feature is useful to leverage third-party applications such as speech-to-text or
+OCR to add additional metadata to a file.
+
+**Example**
+
+```
+~/Documents/
+├── Video.mp4
+└── Video.mp4.s2meta
+```
+
+The sidecar file must have exactly the same file path and the `.s2meta` suffix.
+
+`Video.mp4.s2meta`:
+```json
+{
+  "content": "This sidecar file will overwrite some metadata fields of Video.mp4",
+  "author": "Some author",
+  "duration": 12345,
+  "bitrate": 67890,
+  "some_arbitrary_field": [1,2,3]
+}
+```
+
+```
+sist2 scan ~/Documents -o ./docs.idx
+sist2 index ./docs.idx
+```
+
+*NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,
+it is not currently possible to restore both manual tags and sidecar tags without user scripts
+while reindexing.

docs/scripting.md

@@ -1,34 +1,6 @@
 ## User scripts
 
-User scripts are used to augment your sist2 index with additional metadata, neural network embeddings, tags etc.
+*This document is under construction, more in-depth guide coming soon*
-
-
-Since version 3.2.0, user scripts are written in Python, and are ran against the sist2 index file. User scripts do not 
-need a connection to the search backend.
-
-You can create a user script based on a template from the sist2-admin interface:
-
-![sist2-admin-scripts](sist2-admin-scripts.png)
-
-User scripts leverage the [sist2-python](https://github.com/simon987/sist2-python) library to interface with the
-index file*. You can find sist2-python documentation and examples
-here: [sist2-python.readthedocs.io](https://sist2-python.readthedocs.io/).
-
-If you are not using the sist2-admin interface, you can run user scripts manually from the command line:
-
-```
-pip install git+https://github.com/simon987/sist2-python.git
-
-python my_script.py /path/to/my_index.sist2
-```
-
-\* It is possible to manually update the index using raw SQL queries, but the database schema is not stable and
-can change at any time; it is recommended to use the more stable sist2-python wrapper instead.
-
-<hr>
-
-<details>
-  <summary>Legacy user scripts (sist2 version < 3.2.0)</summary>
 
 During the `index` step, you can use the `--script-file <script>` option to
 modify documents or add user tags. This option is mainly used to
@@ -41,7 +13,6 @@ without programming experience at all if you're somewhat familiar with
 regex.
 
 This is the base structure of the documents we're working with:
-
 ```json
 {
   "_id": "e171405c-fdb5-4feb-bb32-82637bc32084",
@@ -64,7 +35,6 @@ This is the base structure of the documents we're working with:
 
 This script checks if the `genre` attribute exists, if it does
 it adds the `genre.<genre>` tag. 
-
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -77,12 +47,11 @@ You can use `.` to create a hierarchical tag tree:
 
 ![scripting/genre_example](genre_example.png)
 
-To use regular expressions, you need to add this line in `/etc/elasticsearch/elasticsearch.yml`
 
+To use regular expressions, you need to add this line in `/etc/elasticsearch/elasticsearch.yml`
 ```yaml
 script.painless.regex.enabled: true
 ```
-
 Or, if you're using docker add `-e "script.painless.regex.enabled=true"`
 
 **Tag color**
@@ -93,7 +62,6 @@ hexadecimal color code (`#RRGGBBAA`) to the tag name.
 ### Examples
 
 If `(20XX)` is in the file name, add the `year.<year>` tag:
-
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -104,7 +72,6 @@ if (m.find()) {
 ```
 
 Use default *Calibre* folder structure to infer author.
-
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -119,7 +86,6 @@ if (ctx._source.name.contains("-") && ctx._source.extension == "pdf") {
 
 If the file matches a specific pattern `AAAA-000 fName1 lName1, <fName2 lName2>...`, add the `actress.<actress>` and 
 `studio.<studio>` tag:
-
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -136,18 +102,16 @@ if (m.find()) {
 ```
 
 Set the name of the last folder (`/path/to/<studio>/file.mp4`) to `studio.<studio>` tag
-
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
 if (ctx._source.path != "") {
-    String[] names = ctx._source.path.splitOnToken('/');
+    String[] names = ctx._source.path.splitOnToken('/');
     tags.add("studio." + names[names.length-1]);
 }
 ```
 
 Parse `EXIF:F Number` tag
-
 ```Java
 if (ctx._source?.exif_fnumber != null) {
     String[] values = ctx._source.exif_fnumber.splitOnToken(' ');
@@ -160,7 +124,6 @@ if (ctx._source?.exif_fnumber != null) {
 ```
 
 Display year and months from `EXIF:DateTime` tag
-
 ```Java
 if (ctx._source?.exif_datetime != null) {
     SimpleDateFormat parser = new SimpleDateFormat("yyyy:MM:dd HH:mm:ss");
@@ -177,6 +140,3 @@ if (ctx._source?.exif_datetime != null) {
 }
 
 ```
-
-</details>
-

schema/mappings.json

@@ -68,7 +68,7 @@
     },
     "mtime": {
       "type": "date",
-      "format": "epoch_second"
+      "format": "epoch_millis"
     },
     "size": {
       "type": "long"
@@ -202,46 +202,6 @@
     },
     "modified_by": {
       "type": "text"
-    },
-    "emb.384.*": {
-      "type": "dense_vector",
-      "dims": 384
-    },
-    "emb.idx_384.*": {
-      "type": "dense_vector",
-      "dims": 384,
-      "index": true,
-      "similarity": "cosine"
-    },
-    "emb.idx_512.clip": {
-      "type": "dense_vector",
-      "dims": 512,
-      "index": true,
-      "similarity": "cosine"
-    },
-    "emb.512.*": {
-      "type": "dense_vector",
-      "dims": 512
-    },
-    "emb.idx_768.*": {
-      "type": "dense_vector",
-      "dims": 768,
-      "index": true,
-      "similarity": "cosine"
-    },
-    "emb.768.*": {
-      "type": "dense_vector",
-      "dims": 768
-    },
-    "emb.idx_1024.*": {
-      "type": "dense_vector",
-      "dims": 1024,
-      "index": true,
-      "similarity": "cosine"
-    },
-    "emb.1024.*": {
-      "type": "dense_vector",
-      "dims": 1024
     }
   }
 }

scripts/build.sh

@@ -4,24 +4,10 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
-(
-  cd sist2-vue/
-  npm install
-  npm run build
-) &
-
-(
-  cd sist2-admin/frontend/
-  npm install
-  npm run build
-) &
-
-wait
-
 mkdir build
 (
   cd build
-  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
   make -j $(nproc)
   strip sist2
   ./sist2 -v > VERSION
@@ -31,7 +17,7 @@ mv build/sist2 sist2-x64-linux
 (
   cd build
   rm -rf CMakeFiles CMakeCache.txt
-  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
   make -j  $(nproc)
 )
 mv build/sist2_debug sist2-x64-linux-debug

scripts/build_arm64.sh

@@ -4,24 +4,10 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
-(
-  cd sist2-vue/
-  npm install
-  npm run build
-) &
-
-(
-  cd sist2-admin/frontend/
-  npm install
-  npm run build
-) &
-
-wait
-
 mkdir build
 (
   cd build
-  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
   make -j $(nproc)
   strip sist2
 )
@@ -30,7 +16,7 @@ mv build/sist2 sist2-arm64-linux
 rm -rf CMakeFiles CMakeCache.txt
 (
   cd build
-  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
   make -j $(nproc)
 )
 mv build/sist2_debug sist2-arm64-linux-debug

scripts/mime.csv

@@ -450,3 +450,4 @@ image/x-sony-arw, arw
 image/x-sony-sr2, sr2
 image/x-sony-srf, srf
 image/x-epson-erf, erf
+sist2/sidecar, s2meta

scripts/mime.py

@@ -3,7 +3,6 @@ import zlib
 mimes = {}
 noparse = set()
 ext_in_hash = set()
-mime_ids = {}
 
 major_mime = {
     "sist2": 0,
@@ -103,9 +102,6 @@ cnt = 1
 
 
 def mime_id(mime):
-    if mime in mime_ids:
-        return mime_ids[mime]
-
     global cnt
     major = mime.split("/")[0]
     mime_id = str((major_mime[major] << 16) + cnt)
@@ -131,7 +127,9 @@ def mime_id(mime):
     elif mime == "application/x-empty":
         cnt -= 1
         return "1"
-    mime_ids[mime] = mime_id
+    elif mime == "sist2/sidecar":
+        cnt -= 1
+        return "2"
     return mime_id
 
 
@@ -199,12 +197,4 @@ with open("scripts/mime.csv") as f:
         print(f"case {crc(mime)}: return {clean(mime)};")
 
     print("default: return 0;}}")
-
-    # mime list
-
-    mime_list = ",".join(mime_id(x) for x in mimes.keys()) + ",0"
-
-    print(f"unsigned int mime_ids[] = {{{mime_list}}};")
-    print("unsigned int* get_mime_ids() { return mime_ids; }")
-
     print("#endif")

scripts/sqlite_extension.c

@@ -1,84 +0,0 @@
-#include <sqlite3ext.h>
-#include <string.h>
-#include <stdlib.h>
-
-SQLITE_EXTENSION_INIT1
-
-static int sep_rfind(const char *str) {
-    for (int i = (int) strlen(str); i >= 0; i--) {
-        if (str[i] == '/') {
-            return i;
-        }
-    }
-    return -1;
-}
-
-void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
-    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
-        sqlite3_result_error(ctx, "Invalid parameters", -1);
-    }
-
-    const char *value = (const char *) sqlite3_value_text(argv[0]);
-
-    int stop = sep_rfind(value);
-    if (stop == -1) {
-        sqlite3_result_null(ctx);
-        return;
-    }
-    char parent[4096 * 3];
-    strncpy(parent, value, stop);
-
-    sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
-}
-
-void random_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
-    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
-        sqlite3_result_error(ctx, "Invalid parameters", -1);
-    }
-
-    char state_buf[32] = {0,};
-    struct random_data buf;
-    int result;
-
-    long seed = sqlite3_value_int64(argv[0]);
-
-    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
-
-    random_r(&buf, &result);
-
-    sqlite3_result_int(ctx, result);
-}
-
-
-int sqlite3_extension_init(
-        sqlite3 *db,
-        char **pzErrMsg,
-        const sqlite3_api_routines *pApi
-) {
-    SQLITE_EXTENSION_INIT2(pApi);
-
-
-    sqlite3_create_function(
-            db,
-            "path_parent",
-            1,
-            SQLITE_UTF8,
-            NULL,
-            path_parent_func,
-            NULL,
-            NULL
-    );
-
-    sqlite3_create_function(
-            db,
-            "random_seeded",
-            1,
-            SQLITE_UTF8,
-            NULL,
-            random_func,
-            NULL,
-            NULL
-    );
-
-    return SQLITE_OK;
-}

scripts/sqlite_extension_compile.sh

@@ -1 +0,0 @@
-gcc -I/mnt/work/vcpkg/installed/x64-linux/include -g -fPIC -shared sqlite_extension.c -o sist2funcs.so

scripts/start_dev_es.sh

@@ -1,3 +1,3 @@
-docker run --rm -it --name "sist2-dev-es3"\
+docker run --rm -it --name "sist2-dev-es"\
        	-p 9200:9200 -e "discovery.type=single-node" \
 	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.17.9

scripts/start_dev_es_8.sh

@@ -1,3 +1,3 @@
-docker run --rm -it --name "sist2-dev-es3"\
+docker run --rm -it --name "sist2-dev-es"\
        	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
-	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.7.0
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.1.2

scripts/test_in_docker.sh

@@ -1,7 +0,0 @@
-docker build . -t tmp
-
-docker run --rm -it\
-  -v $(pwd):/host \
-  tmp \
-  scan --ocr-lang eng --ocr-ebooks -t6 --incremental --very-verbose \
-  -o /host/docker.sist2 /host/third-party/libscan/libscan-test-files/test_files/

sist2-admin/frontend/src/App.vue

@@ -4,10 +4,9 @@
     <b-container class="pt-4">
       <b-alert show dismissible variant="info">
         This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
-                to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
+        to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on Github</a>. Thank you!
-                Github</a>. Thank you!
       </b-alert>
-            <router-view v-if="$store.state.sist2AdminInfo"/>
+      <router-view/>
     </b-container>
   </div>
 </template>
@@ -32,11 +31,7 @@ export default {
   },
   methods: {
     connectNotifications() {
-            if (window.location.protocol === "https:") {
-                this.socket = new WebSocket(`wss://${window.location.host}/notifications`);
-            } else {
       this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
-            }
       this.socket.onopen = () => {
         this.socket.send("Hello from client");
       }
@@ -71,12 +66,10 @@ html, body {
 
 .info-icon {
   width: 1rem;
-    min-width: 1rem;
   margin-right: 0.2rem;
   cursor: pointer;
   line-height: 1rem;
   height: 1rem;
-    min-height: 1rem;
   background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
   filter: brightness(45%);
   display: block;

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -7,15 +7,15 @@ class Sist2AdminApi {
     }
 
     getJobs() {
-        return axios.get(`${this.baseUrl}/api/job`);
+        return axios.get(`${this.baseUrl}/api/job/`);
     }
 
     getFrontends() {
-        return axios.get(`${this.baseUrl}/api/frontend`);
+        return axios.get(`${this.baseUrl}/api/frontend/`);
     }
 
     getTasks() {
-        return axios.get(`${this.baseUrl}/api/task`);
+        return axios.get(`${this.baseUrl}/api/task/`);
     }
 
     killTask(taskId) {
@@ -33,26 +33,9 @@ class Sist2AdminApi {
         return axios.get(`${this.baseUrl}/api/job/${name}`);
     }
 
-    getSearchBackend(name) {
+    /**
-        return axios.get(`${this.baseUrl}/api/search_backend/${name}`);
+     * @param {string} name
-    }
+     */
-
-    updateSearchBackend(name, data) {
-        return axios.put(`${this.baseUrl}/api/search_backend/${name}`, data);
-    }
-
-    getSearchBackends() {
-        return axios.get(`${this.baseUrl}/api/search_backend`);
-    }
-
-    deleteBackend(name) {
-        return axios.delete(`${this.baseUrl}/api/search_backend/${name}`)
-    }
-
-    createBackend(name) {
-        return axios.post(`${this.baseUrl}/api/search_backend/${name}`);
-    }
-
     getFrontend(name) {
         return axios.get(`${this.baseUrl}/api/frontend/${name}`);
     }
@@ -127,49 +110,7 @@ class Sist2AdminApi {
     }
 
     getSist2AdminInfo() {
-        return axios.get(`${this.baseUrl}/api`);
+        return axios.get(`${this.baseUrl}/api/`);
-    }
-
-    getLogsToDelete(jobName, n) {
-        return axios.get(`${this.baseUrl}/api/job/${jobName}/logs_to_delete`, {
-            params: {n: n}
-        });
-    }
-
-    deleteTaskLogs(taskId) {
-        return axios.post(`${this.baseUrl}/api/task/${taskId}/delete_logs`);
-    }
-
-    getUserScripts() {
-        return axios.get(`${this.baseUrl}/api/user_script`);
-    }
-
-    getUserScript(name) {
-        return axios.get(`${this.baseUrl}/api/user_script/${name}`);
-    }
-
-    createUserScript(name, template) {
-        return axios.post(`${this.baseUrl}/api/user_script/${name}`, null, {
-            params: {
-                template: template
-            }
-        });
-    }
-
-    updateUserScript(name, data) {
-        return axios.put(`${this.baseUrl}/api/user_script/${name}`, data);
-    }
-
-    deleteUserScript(name) {
-        return axios.delete(`${this.baseUrl}/api/user_script/${name}`);
-    }
-
-    testUserScript(name, job) {
-        return axios.get(`${this.baseUrl}/api/user_script/${name}/run`, {
-            params: {
-                job: job
-            }
-        });
     }
 }
 

sist2-admin/frontend/src/components/IndexOptions.vue

@@ -0,0 +1,64 @@
+<template>
+  <div>
+    <label>{{ $t("indexOptions.threads") }}</label>
+    <b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("webOptions.esUrl") }}</label>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <label>{{ $t("indexOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
+    <br>
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("indexOptions.batchSize") }}</label>
+    <b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
+
+    <label>{{ $t("indexOptions.script") }}</label>
+    <b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
+  </div>
+
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+  name: "IndexOptions",
+  props: ["options"],
+  data() {
+    return {
+      showEsTestAlert: false,
+      esTestOk: false,
+      esTestMessage: "",
+    }
+  },
+  methods: {
+    update() {
+      this.$emit("change", this.options);
+    },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
+  },
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/JobCheckboxGroup.vue

@@ -9,18 +9,13 @@
           @input="frontend.jobs = $event; $emit('input')"
       >
         <div v-for="job in jobs" :key="job.name">
-                    <b-form-checkbox :disabled="job.status !== 'indexed'"
+          <b-form-checkbox :disabled="job.status !== 'indexed'" :value="job.name">[{{ job.name }}]</b-form-checkbox>
-                                     :value="job.name">
-                        <template #default><span
-                                :title="job.status !== 'indexed' ? $t('jobOptions.notIndexed') : ''"
-                        >[{{ job.name }}]</span></template>
-                    </b-form-checkbox>
           <br/>
         </div>
       </b-form-checkbox-group>
       <div v-else>
         <span class="text-muted">{{ $t('jobOptions.noJobAvailable') }}</span>
-                <router-link to="/">{{ $t("create") }}</router-link>
+        &nbsp;<router-link to="/">{{$t("create")}}</router-link>
       </div>
     </b-form-group>
   </div>
@@ -34,20 +29,13 @@ export default {
   props: ["frontend"],
   mounted() {
     Sist2AdminApi.getJobs().then(resp => {
-            this._jobs = resp.data;
+      this.jobs = resp.data;
       this.loading = false;
     });
   },
-    computed: {
-        jobs() {
-            return this._jobs
-                .filter(job => job.index_options.search_backend === this.frontend.web_options.search_backend)
-        }
-    },
   data() {
     return {
       loading: true,
-            _jobs: null
     }
   }
 }

sist2-admin/frontend/src/components/JobListItem.vue

@@ -44,7 +44,8 @@ export default {
         return "";
       }
 
-      return moment.utc(dateString).local().fromNow();
+      const date = Date.parse(dateString);
+      return moment(date).fromNow();
     }
   }
 }

sist2-admin/frontend/src/components/JobOptions.vue

@@ -9,31 +9,18 @@
     </b-form-checkbox>
 
     <label>{{ $t("jobOptions.cron") }}</label>
-        <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression"
+    <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression" :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
-                      :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
-
-        <label>{{ $t("jobOptions.keepNLogs") }}</label>
-        <b-input-group>
-            <b-form-input type="number" v-model="job.keep_last_n_logs" @change="update()"></b-form-input>
-            <b-input-group-append>
-                <b-button variant="danger" @click="onDeleteNowClick()">{{ $t("jobOptions.deleteNow") }}</b-button>
-            </b-input-group-append>
-        </b-input-group>
-
   </div>
 </template>
 
 <script>
 
-import Sist2AdminApi from "@/Sist2AdminApi";
-
 export default {
   name: "JobOptions",
   props: ["job"],
   data() {
     return {
-            cronValid: undefined,
+      cronValid: undefined
-            logsToDelete: null
     }
   },
   computed: {
@@ -65,30 +52,6 @@ export default {
         this.$emit("change", this.job);
       }
     },
-        onDeleteNowClick() {
-            Sist2AdminApi.getLogsToDelete(this.job.name, this.job.keep_last_n_logs).then(resp => {
-                const toDelete = resp.data;
-                const message = `Delete ${toDelete.length} log files?`;
-
-                this.$bvModal.msgBoxConfirm(message, {
-                    title: this.$t("confirmation"),
-                    size: "sm",
-                    buttonSize: "sm",
-                    okVariant: "danger",
-                    okTitle: this.$t("delete"),
-                    cancelTitle: this.$t("cancel"),
-                    footerClass: "p-2",
-                    hideHeaderClose: false,
-                    centered: true
-                }).then(value => {
-                    if (value) {
-                        toDelete.forEach(row => {
-                            Sist2AdminApi.deleteTaskLogs(row["id"]);
-                        });
-                    }
-                });
-            })
-        }
   },
 }
 </script>

sist2-admin/frontend/src/components/JobSelect.vue

@@ -1,34 +0,0 @@
-<template>
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <span v-else-if="jobs.length === 0"></span>
-    <b-form-select v-else :options="jobs" text-field="name" value-field="name"
-                   @change="$emit('change', $event)" :value="$t('selectJob')"></b-form-select>
-</template>
-
-<script>
-import Sist2AdminApi from "@/Sist2AdminApi";
-
-export default {
-    name: "JobSelect",
-    mounted() {
-        Sist2AdminApi.getJobs().then(resp => {
-            this._jobs = resp.data;
-            this.loading = false;
-        });
-    },
-    computed: {
-        jobs() {
-            return [
-                {name: this.$t("selectJob"), disabled: true},
-                ...this._jobs.filter(job => job.index_path)
-            ]
-        }
-    },
-    data() {
-        return {
-            loading: true,
-            _jobs: null
-        }
-    }
-}
-</script>

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -7,7 +7,7 @@
     <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
-    <b-form-input type="number" min="0" max="100" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailCount") }}</label>
     <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>

sist2-admin/frontend/src/components/SearchBackendListItem.vue

@@ -1,24 +0,0 @@
-<template>
-    <b-list-group-item action :to="`/searchBackend/${backend.name}`">
-
-        <div class="d-flex w-100 justify-content-between">
-            <h5 class="mb-1">
-                {{ backend.name }}
-            </h5>
-
-            <div>
-                <b-badge v-if="backend.backend_type === 'sqlite'" variant="info">SQLite</b-badge>
-                <b-badge v-else variant="info">Elasticsearch</b-badge>
-            </div>
-        </div>
-
-    </b-list-group-item>
-</template>
-
-<script>
-
-export default {
-    name: "SearchBackendListItem",
-    props: ["backend"],
-}
-</script>

sist2-admin/frontend/src/components/SearchBackendSelect.vue

@@ -1,37 +0,0 @@
-<template>
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <div v-else>
-        <label>{{$t("backendOptions.searchBackend")}}</label>
-        <b-select :options="options" :value="value" @change="$emit('change', $event)"></b-select>
-    </div>
-</template>
-
-<script>
-import Sist2AdminApi from "@/Sist2AdminApi";
-
-export default {
-    name: "SearchBackendSelect",
-    props: ["value"],
-    data() {
-        return {
-            loading: true,
-            backends: null,
-        }
-    },
-    computed: {
-        options() {
-            return this.backends.map(backend => backend.name)
-        }
-    },
-    mounted() {
-        Sist2AdminApi.getSearchBackends().then(resp => {
-            this.loading = false;
-            this.backends = resp.data
-        })
-    }
-}
-</script>
-
-<style scoped>
-
-</style>

sist2-admin/frontend/src/components/UserScriptListItem.vue

@@ -1,18 +0,0 @@
-<template>
-    <b-list-group-item action :to="`/userScript/${script.name}`">
-
-        <div class="d-flex w-100 justify-content-between">
-            <h5 class="mb-1">
-                {{ script.name }}
-            </h5>
-        </div>
-    </b-list-group-item>
-</template>
-
-<script>
-
-export default {
-    name: "UserScriptListItem",
-    props: ["script"],
-}
-</script>

sist2-admin/frontend/src/components/UserScriptPicker.vue

@@ -1,88 +0,0 @@
-<template>
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
-
-    <b-row v-else>
-        <b-col cols="6">
-            <h5>Selected scripts</h5>
-            <b-list-group>
-                <b-list-group-item v-for="script in selectedScripts" :key="script"
-                                   button
-                                   @click="onRemoveScript(script)"
-                                   class="d-flex justify-content-between align-items-center">
-                    {{ script }}
-                    <b-button-group>
-                        <b-button variant="light" @click.stop="moveUpScript(script)">↑</b-button>
-                        <b-button variant="light" @click.stop="moveDownScript(script)">↓</b-button>
-                    </b-button-group>
-                </b-list-group-item>
-            </b-list-group>
-        </b-col>
-        <b-col cols="6">
-            <h5>Available scripts</h5>
-            <b-list-group>
-                <b-list-group-item v-for="script in availableScripts" :key="script" button
-                                   @click="onSelectScript(script)">
-                    {{ script }}
-                </b-list-group-item>
-            </b-list-group>
-        </b-col>
-    </b-row>
-
-  <!--    <b-checkbox-group v-else :options="scripts" stacked :checked="selectedScripts"-->
-  <!--                      @input="$emit('change', $event)"></b-checkbox-group>-->
-</template>
-
-<script>
-import Sist2AdminApi from "@/Sist2AdminApi";
-
-export default {
-    name: "UserScriptPicker",
-    props: ["selectedScripts"],
-    data() {
-        return {
-            loading: true,
-            scripts: []
-        }
-    },
-    computed: {
-        availableScripts() {
-            return this.scripts.filter(script => !this.selectedScripts.includes(script))
-        }
-    },
-    mounted() {
-        Sist2AdminApi.getUserScripts().then(resp => {
-            this.scripts = resp.data.map(script => script.name);
-            this.loading = false;
-        });
-    },
-    methods: {
-        onSelectScript(name) {
-            this.selectedScripts.push(name);
-            this.$emit("change", this.selectedScripts)
-        },
-        onRemoveScript(name) {
-            this.selectedScripts.splice(this.selectedScripts.indexOf(name), 1);
-            this.$emit("change", this.selectedScripts);
-        },
-        moveUpScript(name) {
-            const index = this.selectedScripts.indexOf(name);
-            if (index > 0) {
-                this.selectedScripts.splice(index, 1);
-                this.selectedScripts.splice(index - 1, 0, name);
-            }
-            this.$emit("change", this.selectedScripts);
-        },
-        moveDownScript(name) {
-            const index = this.selectedScripts.indexOf(name);
-            if (index < this.selectedScripts.length - 1) {
-                this.selectedScripts.splice(index, 1);
-                this.selectedScripts.splice(index + 1, 0, name);
-            }
-            this.$emit("change", this.selectedScripts);
-        }
-    }
-}
-</script>
-
-<style scoped>
-</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -1,10 +1,26 @@
 <template>
   <div>
-        <h4>{{ $t("webOptions.title") }}</h4>
+    <label>{{ $t("webOptions.esUrl") }}</label>
-        <b-card>
+    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+      {{ esTestMessage }}
+    </b-alert>
+
+    <b-input-group>
+      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+      </b-input-group-append>
+    </b-input-group>
+
+    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
+      {{ $t("webOptions.esInsecure") }}
+    </b-form-checkbox>
+
+    <label>{{ $t("webOptions.esIndex") }}</label>
+    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+
     <label>{{ $t("webOptions.lang") }}</label>
-            <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
+    <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
-                           @change="update()"></b-form-select>
 
     <label>{{ $t("webOptions.bind") }}</label>
     <b-form-input v-model="options.bind" @change="update()"></b-form-input>
@@ -17,11 +33,9 @@
 
     <label>{{ $t("webOptions.tagAuth") }}</label>
     <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
-        </b-card>
 
     <br>
-        <h4>Auth0 options</h4>
+    <h5>Auth0 options</h5>
-        <b-card>
     <label>{{ $t("webOptions.auth0Audience") }}</label>
     <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
@@ -33,12 +47,16 @@
 
     <label>{{ $t("webOptions.auth0PublicKey") }}</label>
     <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
-        </b-card>
+
+
+
   </div>
 </template>
 
 <script>
 
+import sist2AdminApi from "@/Sist2AdminApi";
+
 export default {
   name: "WebOptions",
   props: ["options", "frontendName"],
@@ -51,8 +69,19 @@ export default {
   },
   methods: {
     update() {
+      if (!this.options.es_url.startsWith("https")) {
+        this.options.es_insecure_ssl = false;
+      }
+
       this.$emit("change", this.options);
     },
+    testEs() {
+      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
+        this.showEsTestAlert = true;
+        this.esTestOk = resp.data.ok;
+        this.esTestMessage = resp.data.message;
+      });
+    }
   }
 }
 </script>

sist2-admin/frontend/src/i18n/messages.js

@@ -5,13 +5,10 @@ export default {
         go: "Go",
         online: "online",
         offline: "offline",
-        view: "View",
         delete: "Delete",
         runNow: "Index now",
         create: "Create",
-        cancel: "Cancel",
         test: "Test",
-        confirmation: "Confirmation",
 
         jobTitle: "job configuration",
         tasks: "Tasks",
@@ -48,26 +45,12 @@ export default {
         extraQueryArgs: "Extra query arguments when launching from sist2-admin",
         customUrl: "Custom URL when launching from sist2-admin",
 
-        searchBackends: "Search backends",
+        selectJobs: "Select jobs",
-        searchBackendTitle: "search backend configuration",
-        newBackendName: "New search backend name",
-
-        frontendTab: "Frontend",
-        backendTab: "Backend",
-        scripts: "User Scripts",
-        script: "User Script",
-        testScript: "Test/debug User Script",
-        newScriptName: "New script name",
-        scriptType: "Script type",
-        scriptCode: "Script code (Python)",
-        scriptOptions: "User scripts",
-        gitRepository: "Git repository URL",
-        extraArgs: "Extra command line arguments",
-
-        selectJobs: "Available jobs",
-        selectJob: "Select a job",
         webOptions: {
             title: "Web options",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
             lang: "UI Language",
             bind: "Listen address",
             tagline: "Tagline in navbar",
@@ -78,24 +61,12 @@ export default {
             auth0ClientId: "Auth0 client ID",
             auth0PublicKey: "Auth0 public key",
         },
-        backendOptions: {
-            title: "Search backend options",
-            searchBackend: "Search backend",
-            type: "Search backend type",
-            esUrl: "Elasticsearch URL",
-            esIndex: "Elasticsearch index name",
-            esInsecure: "Do not verify SSL connections to Elasticsearch.",
-            threads: "Number of threads",
-            batchSize: "Index batch size",
-            script: "User script",
-            searchIndex: "Search index file location"
-        },
         scanOptions: {
             title: "Scanning options",
             path: "Path",
             threads: "Number of threads",
             memThrottle: "Total memory threshold in MiB for scan throttling",
-            thumbnailQuality: "Thumbnail quality, on a scale of 0 to 100, 100 being the best",
+            thumbnailQuality: "Thumbnail quality, on a scale of 2 to 32, 2 being the best",
             thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
             thumbnailSize: "Thumbnail size, in pixels",
             contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",
@@ -116,19 +87,24 @@ export default {
             treemapThreshold: "Relative size threshold for treemap",
             optimizeIndex: "Defragment index file after scan to reduce its file size."
         },
+        indexOptions: {
+            title: "Indexing options",
+            threads: "Number of threads",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            batchSize: "Index batch size",
+            script: "User script"
+        },
         jobOptions: {
             title: "Job options",
             cron: "Job schedule",
-            keepNLogs: "Keep last N log files. Set to -1 to keep all logs.",
-            deleteNow: "Delete now",
             scheduleEnabled: "Enable scheduled re-scan",
-            noJobAvailable: "No jobs available for this search backend.",
+            noJobAvailable: "No jobs available.",
-            notIndexed: "Has not been indexed yet",
-            noBackendError: "You must select a search backend to run this job",
             desktopNotifications: "Desktop notifications"
         },
         frontendOptions: {
-            title: "Advanced options",
+            title: "Frontend options",
             noJobSelectedWarning: "You must select at least one job to start this frontend"
         },
         notifications: {

sist2-admin/frontend/src/router/index.js

@@ -5,19 +5,12 @@ import Job from "@/views/Job";
 import Tasks from "@/views/Tasks";
 import Frontend from "@/views/Frontend";
 import Tail from "@/views/Tail";
-import SearchBackend from "@/views/SearchBackend.vue";
-import UserScript from "@/views/UserScript.vue";
 
 Vue.use(VueRouter);
 
 const routes = [
   {
-    path: "/task",
+    path: "/",
-    name: "Tasks",
-    component: Tasks
-  },
-  {
-    path: "/:tab?",
     name: "Home",
     component: Home
   },
@@ -26,21 +19,16 @@ const routes = [
     name: "Job",
     component: Job
   },
+  {
+    path: "/task/",
+    name: "Tasks",
+    component: Tasks
+  },
   {
     path: "/frontend/:name",
     name: "Frontend",
     component: Frontend
   },
-  {
-    path: "/searchBackend/:name",
-    name: "SearchBackend",
-    component: SearchBackend
-  },
-  {
-    path: "/userScript/:name",
-    name: "UserScript",
-    component: UserScript
-  },
   {
     path: "/log/:taskId",
     name: "Tail",

sist2-admin/frontend/src/views/Frontend.vue

@@ -8,7 +8,6 @@
       </small>
     </b-card-title>
 
-        <!-- Action buttons-->
     <div class="mb-3" v-if="!loading">
       <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
           $t("start")
@@ -24,26 +23,10 @@
       <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
     </div>
 
+
     <b-progress v-if="loading" striped animated value="100"></b-progress>
     <b-card-body v-else>
 
-            <h4>{{ $t("backendOptions.title") }}</h4>
-            <b-card>
-                <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
-
-                <SearchBackendSelect :value="frontend.web_options.search_backend"
-                                     @change="onBackendSelect($event)"></SearchBackendSelect>
-
-                <br>
-                <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
-            </b-card>
-
-            <br/>
-
-            <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
-                        @change="update()"></WebOptions>
-            <br/>
-
       <h4>{{ $t("frontendOptions.title") }}</h4>
       <b-card>
         <b-form-checkbox v-model="frontend.auto_start" @change="update()">
@@ -55,8 +38,22 @@
 
         <label>{{ $t("customUrl") }}</label>
         <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+
+        <br/>
+
+        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+
+        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+      </b-card>
+
+      <br/>
+
+      <h4>{{ $t("webOptions.title") }}</h4>
+      <b-card>
+        <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
       </b-card>
     </b-card-body>
+
   </b-card>
 </template>
 
@@ -65,11 +62,10 @@
 import Sist2AdminApi from "@/Sist2AdminApi";
 import JobCheckboxGroup from "@/components/JobCheckboxGroup";
 import WebOptions from "@/components/WebOptions";
-import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
   name: 'Frontend',
-    components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
+  components: {JobCheckboxGroup, WebOptions},
   data() {
     return {
       loading: true,
@@ -122,17 +118,12 @@ export default {
     },
     deleteFrontend() {
       Sist2AdminApi.deleteFrontend(this.name).then(() => {
-                this.$router.push("/");
+        this.$router.push("/frontends");
       });
     },
     update() {
       Sist2AdminApi.updateFrontend(this.name, this.frontend);
     },
-        onBackendSelect(backend) {
-            this.frontend.web_options.search_backend = backend;
-            this.frontend.jobs = [];
-            this.update();
-        }
   }
 }
 </script>

sist2-admin/frontend/src/views/Home.vue

@@ -1,34 +1,5 @@
 <template>
   <div>
-        <b-tabs content-class="mt-3" v-model="tab" @input="onTabChange($event)">
-            <b-tab :title="$t('backendTab')">
-
-                <b-card>
-                    <b-card-title>{{ $t("searchBackends") }}</b-card-title>
-
-                    <b-row>
-                        <b-col>
-                            <b-input v-model="newBackendName" :placeholder="$t('newBackendName')"></b-input>
-                        </b-col>
-                        <b-col>
-                            <b-button variant="primary" @click="createBackend()"
-                                      :disabled="!backendNameValid(newBackendName)">
-                                {{ $t("create") }}
-                            </b-button>
-                        </b-col>
-                    </b-row>
-
-                    <hr/>
-
-                    <b-progress v-if="backendsLoading" striped animated value="100"></b-progress>
-                    <b-list-group v-else>
-                        <SearchBackendListItem v-for="backend in backends"
-                                               :key="backend.name" :backend="backend"></SearchBackendListItem>
-                    </b-list-group>
-                </b-card>
-
-                <br/>
-
     <b-card>
       <b-card-title>{{ $t("jobs") }}</b-card-title>
       <b-row>
@@ -44,8 +15,7 @@
           ></b-popover>
         </b-col>
         <b-col>
-                            <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">
+          <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
-                                {{ $t("create") }}
           </b-button>
         </b-col>
       </b-row>
@@ -57,39 +27,9 @@
         <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
       </b-list-group>
     </b-card>
-            </b-tab>
-            <b-tab :title="$t('scripts')">
 
-                <b-progress v-if="scriptsLoading" striped animated value="100"></b-progress>
+    <br/>
-                <b-card v-else>
-                    <b-card-title>{{ $t("scripts") }}</b-card-title>
 
-                    <label>Select template</label>
-                    <b-form-radio-group stacked :options="scriptTemplates" v-model="scriptTemplate"></b-form-radio-group>
-                    <br>
-
-                    <b-row>
-                        <b-col>
-                            <b-form-input v-model="newScriptName" :disabled="!scriptTemplate" :placeholder="$t('newScriptName')"></b-form-input>
-                        </b-col>
-                        <b-col>
-                            <b-button variant="primary" @click="createScript()"
-                                      :disabled="!scriptNameValid(newScriptName)">
-                                {{ $t("create") }}
-                            </b-button>
-                        </b-col>
-                    </b-row>
-
-                    <hr/>
-
-                    <b-list-group>
-                        <UserScriptListItem v-for="script in scripts"
-                                            :key="script.name" :script="script"></UserScriptListItem>
-                    </b-list-group>
-
-                </b-card>
-            </b-tab>
-            <b-tab :title="$t('frontendTab')">
     <b-card>
 
       <b-card-title>{{ $t("frontends") }}</b-card-title>
@@ -99,8 +39,7 @@
           <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
         </b-col>
         <b-col>
-                            <b-button variant="primary" @click="createFrontend()"
+          <b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
-                                      :disabled="!frontendNameValid(newFrontendName)">
             {{ $t("create") }}
           </b-button>
         </b-col>
@@ -115,8 +54,6 @@
       </b-list-group>
 
     </b-card>
-            </b-tab>
-        </b-tabs>
   </div>
 </template>
 
@@ -125,12 +62,10 @@ import JobListItem from "@/components/JobListItem";
 import {formatBindAddress} from "@/util";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import FrontendListItem from "@/components/FrontendListItem";
-import SearchBackendListItem from "@/components/SearchBackendListItem.vue";
-import UserScriptListItem from "@/components/UserScriptListItem.vue";
 
 export default {
   name: "Jobs",
-    components: {UserScriptListItem, SearchBackendListItem, JobListItem, FrontendListItem},
+  components: {JobListItem, FrontendListItem},
   data() {
     return {
       jobsLoading: true,
@@ -142,28 +77,11 @@ export default {
       formatBindAddress,
       newFrontendName: "",
 
-            backends: [],
+      showHelp: false
-            backendsLoading: true,
-            newBackendName: "",
-
-            scripts: [],
-            scriptTemplates: [],
-            newScriptName: "",
-            scriptTemplate: null,
-            scriptsLoading: true,
-
-            showHelp: false,
-            tab: 0
     }
   },
   mounted() {
     this.loading = true;
-        if (this.$route.params.tab) {
-            console.log("mounted " + this.$route.params.tab)
-            window.setTimeout(() => {
-                this.tab = Math.round(Number(this.$route.params.tab));
-            }, 1)
-        }
     this.reload();
   },
   methods: {
@@ -175,24 +93,7 @@ export default {
       return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
     },
     frontendNameValid(name) {
-            if (this.frontends.some(frontend => frontend.name === name)) {
+      if (this.frontends.some(job => job.name === name)) {
-                return false;
-            }
-
-            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
-        },
-        backendNameValid(name) {
-            if (this.backends.some(backend => backend.name === name)) {
-                return false;
-            }
-
-            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
-        },
-        scriptNameValid(name) {
-            if (this.scripts.some(script => script.name === name)) {
-                return false;
-            }
-            if (name.length > 16) {
         return false;
       }
 
@@ -202,38 +103,19 @@ export default {
       Sist2AdminApi.getJobs().then(resp => {
         this.jobs = resp.data;
         this.jobsLoading = false;
+
         this.showHelp = this.jobs.length === 0;
       });
       Sist2AdminApi.getFrontends().then(resp => {
         this.frontends = resp.data;
         this.frontendsLoading = false;
       });
-            Sist2AdminApi.getSearchBackends().then(resp => {
-                this.backends = resp.data;
-                this.backendsLoading = false;
-            })
-            Sist2AdminApi.getUserScripts().then(resp => {
-                this.scripts = resp.data;
-                this.scriptTemplates = this.$store.state.sist2AdminInfo.user_script_templates;
-                this.scriptsLoading = false;
-            })
     },
     createJob() {
       Sist2AdminApi.createJob(this.newJobName).then(this.reload);
     },
     createFrontend() {
       Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
-        },
-        createBackend() {
-            Sist2AdminApi.createBackend(this.newBackendName).then(this.reload);
-        },
-        createScript() {
-            Sist2AdminApi.createUserScript(this.newScriptName, this.scriptTemplate).then(this.reload)
-        },
-        onTabChange(tab) {
-            if (this.$route.params.tab != tab) {
-                this.$router.push({params: {tab: tab}})
-            }
     }
   }
 }

sist2-admin/frontend/src/views/Job.vue

@@ -6,7 +6,7 @@
     </b-card-title>
 
     <div class="mb-3">
-            <b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
+      <b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
       <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
     </div>
 
@@ -24,27 +24,18 @@
 
       <br/>
 
-            <h4>{{ $t("backendOptions.title") }}</h4>
-            <b-card>
-                <b-alert v-if="!valid" variant="warning" show>{{ $t("jobOptions.noBackendError") }}</b-alert>
-                <SearchBackendSelect :value="job.index_options.search_backend"
-                                     @change="onBackendSelect($event)"></SearchBackendSelect>
-            </b-card>
-            <br/>
-
-            <h4>{{ $t("scriptOptions") }}</h4>
-            <b-card>
-                <UserScriptPicker :selected-scripts="job.user_scripts"
-                                  @change="onScriptChange($event)"></UserScriptPicker>
-            </b-card>
-
-            <br/>
-
       <h4>{{ $t("scanOptions.title") }}</h4>
       <b-card>
         <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
       </b-card>
 
+      <br/>
+
+      <h4>{{ $t("indexOptions.title") }}</h4>
+      <b-card>
+        <IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
+      </b-card>
+
     </b-card-body>
 
   </b-card>
@@ -53,22 +44,20 @@
 <script>
 import ScanOptions from "@/components/ScanOptions";
 import Sist2AdminApi from "@/Sist2AdminApi";
+import IndexOptions from "@/components/IndexOptions";
 import JobOptions from "@/components/JobOptions";
-import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
-import UserScriptPicker from "@/components/UserScriptPicker.vue";
 
 export default {
   name: "Job",
   components: {
-        UserScriptPicker,
+    IndexOptions,
-        SearchBackendSelect,
     ScanOptions,
     JobOptions
   },
   data() {
     return {
       loading: true,
-            job: null,
+      job: null
     }
   },
   methods: {
@@ -88,26 +77,9 @@ export default {
       });
     },
     deleteJob() {
-            Sist2AdminApi.deleteJob(this.getName())
+      Sist2AdminApi.deleteJob(this.getName()).then(() => {
-                .then(() => {
         this.$router.push("/");
       })
-                .catch(err => {
-                    this.$bvToast.toast("Cannot delete job " +
-                        "because it is referenced by a frontend", {
-                        title: "Error",
-                        variant: "danger",
-                        toaster: "b-toaster-bottom-right"
-                    });
-                })
-        },
-        onBackendSelect(backend) {
-            this.job.index_options.search_backend = backend;
-            this.update();
-        },
-        onScriptChange(scripts) {
-            this.job.user_scripts = scripts;
-            this.update();
     }
   },
   mounted() {
@@ -115,11 +87,6 @@ export default {
       this.loading = false;
       this.job = resp.data;
     })
-    },
-    computed: {
-        valid() {
-            return this.job?.index_options.search_backend != null;
-        }
   }
 }
 </script>

sist2-admin/frontend/src/views/SearchBackend.vue

@@ -1,123 +0,0 @@
-<template>
-
-    <b-card>
-        <b-card-title>
-            <span class="text-monospace">{{ getName() }}</span>
-            {{ $t("searchBackendTitle") }}
-        </b-card-title>
-
-        <div class="mb-3">
-            <b-button variant="danger" @click="deleteBackend()">{{ $t("delete") }}</b-button>
-        </div>
-
-        <b-progress v-if="loading" striped animated value="100"></b-progress>
-        <b-card-body v-else>
-
-            <label>{{ $t("backendOptions.type") }}</label>
-            <b-select :options="backendTypeOptions" v-model="backend.backend_type" @change="update()"></b-select>
-
-            <hr/>
-
-            <template v-if="backend.backend_type === 'elasticsearch'">
-                <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
-                    {{ esTestMessage }}
-                </b-alert>
-
-                <label>{{ $t("backendOptions.esUrl") }}</label>
-                <b-input-group>
-                    <b-form-input v-model="backend.es_url" @change="update()"></b-form-input>
-                    <b-input-group-append>
-                        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
-                    </b-input-group-append>
-                </b-input-group>
-
-                <b-form-checkbox v-model="backend.es_insecure_ssl" :disabled="!this.backend.es_url.startsWith('https')"
-                                 @change="update()">
-                    {{ $t("backendOptions.esInsecure") }}
-                </b-form-checkbox>
-
-                <label>{{ $t("backendOptions.esIndex") }}</label>
-                <b-form-input v-model="backend.es_index" @change="update()"></b-form-input>
-
-                <label>{{ $t("backendOptions.threads") }}</label>
-                <b-form-input v-model="backend.threads" type="number" min="1" @change="update()"></b-form-input>
-
-                <label>{{ $t("backendOptions.batchSize") }}</label>
-                <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
-            </template>
-            <template v-else>
-                <label>{{ $t("backendOptions.searchIndex") }}</label>
-                <b-form-input v-model="backend.search_index" disabled></b-form-input>
-            </template>
-        </b-card-body>
-
-    </b-card>
-</template>
-
-<script>
-import sist2AdminApi from "@/Sist2AdminApi";
-import Sist2AdminApi from "@/Sist2AdminApi";
-
-export default {
-    name: "SearchBackend",
-    data() {
-        return {
-            showEsTestAlert: false,
-            esTestOk: false,
-            esTestMessage: "",
-            loading: true,
-            backend: null,
-            backendTypeOptions: [
-                {
-                    text: "Elasticsearch",
-                    value: "elasticsearch"
-                },
-                {
-                    text: "SQLite",
-                    value: "sqlite"
-                }
-            ]
-        }
-    },
-    mounted() {
-        Sist2AdminApi.getSearchBackend(this.getName()).then(resp => {
-            this.backend = resp.data;
-            this.loading = false;
-        });
-    },
-    methods: {
-        getName() {
-            return this.$route.params.name;
-        },
-        testEs() {
-            sist2AdminApi.pingEs(this.backend.es_url, this.backend.es_insecure_ssl)
-                .then((resp) => {
-                    this.showEsTestAlert = true;
-                    this.esTestOk = resp.data.ok;
-                    this.esTestMessage = resp.data.message;
-                });
-        },
-        update() {
-            Sist2AdminApi.updateSearchBackend(this.getName(), this.backend);
-        },
-        deleteBackend() {
-            Sist2AdminApi.deleteBackend(this.getName())
-                .then(() => {
-                    this.$router.push("/");
-                })
-                .catch(err => {
-                    this.$bvToast.toast("Cannot delete search backend " +
-                        "because it is referenced by a job or frontend", {
-                        title: "Error",
-                        variant: "danger",
-                        toaster: "b-toaster-bottom-right"
-                    });
-                })
-        }
-    }
-}
-</script>
-
-<style scoped>
-
-</style>

sist2-admin/frontend/src/views/Tail.vue

@@ -65,11 +65,7 @@ export default {
       }
 
       const n = this.mode === "follow" ? 32 : 9999999999;
-            if (window.location.protocol === "https:") {
-                this.socket = new WebSocket(`wss://${window.location.host}/log/${this.taskId}?n=${n}`);
-            } else {
       this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
-            }
       this.socket.onopen = () => {
         this.socket.send("Hello from client");
       }
@@ -92,9 +88,6 @@ export default {
           if ("stderr" in message) {
             message.level = "ERROR";
             message.message = message["stderr"];
-                    } else if ("stdout" in message) {
-                        message.level = "INFO";
-                        message.message = message["stdout"];
           } else {
             message.level = "ADMIN";
             message.message = message["sist2-admin"];

sist2-admin/frontend/src/views/Tasks.vue

@@ -23,18 +23,7 @@
           :per-page="10"
       >
         <template #cell(logs)="data">
-                    <template v-if="data.item._row.has_logs">
+          <router-link :to="`/log/${data.item.logs}`">{{ $t("logs") }}</router-link>
-                        <b-button variant="link" size="sm" :to="`/log/${data.item.id}`">
-                            {{ $t("view") }}
-                        </b-button>
-                        /
-                        <b-button variant="link" size="sm" @click="deleteLogs(data.item.id)">
-                            {{ $t("delete") }}
-                        </b-button>
-                    </template>
-                </template>
-
-                <template #cell(delete)="data">
         </template>
 
       </b-table>
@@ -81,7 +70,7 @@ function humanDuration(sec_num) {
     return `${seconds}s`;
   }
 
-    return "<1s";
+  return "<0s";
 }
 
 export default {
@@ -132,10 +121,9 @@ export default {
           id: row.id,
           name: row.name,
           duration: this.taskDuration(row),
-                    time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"),
+          time: moment(row.started).format("dd, MMM Do YYYY, HH:mm:ss"),
-                    logs: null,
+          logs: row.id,
-                    status: row.return_code === 0 ? "ok" : "failed",
+          status: row.return_code === 0 ? "ok" : "failed"
-                    _row: row
         }));
       });
     },
@@ -149,11 +137,6 @@ export default {
       const end = moment.utc(task.ended);
 
       return humanDuration(end.diff(start))
-        },
-        deleteLogs(taskId) {
-            Sist2AdminApi.deleteTaskLogs(taskId).then(() => {
-                this.updateHistory();
-            })
     }
   }
 }
@@ -164,8 +147,4 @@ export default {
   font-family: monospace;
   font-size: 12px;
 }
-
-.btn-link {
-    padding: 0;
-}
 </style>

sist2-admin/frontend/src/views/UserScript.vue

@@ -1,117 +0,0 @@
-<template>
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <b-card v-else>
-        <b-card-title>
-            {{ $route.params.name }}
-            {{ $t("script") }}
-        </b-card-title>
-
-        <div class="mb-3">
-            <b-button variant="danger" @click="deleteScript()">{{ $t("delete") }}</b-button>
-        </div>
-
-        <b-card>
-            <h5>{{ $t("testScript") }}</h5>
-
-            <b-row>
-                <b-col cols="11">
-                    <JobSelect @change="onJobSelect($event)"></JobSelect>
-                </b-col>
-                <b-col cols="1">
-                    <b-button :disabled="!selectedTestJob" variant="primary" @click="testScript()">{{ $t("test") }}
-                    </b-button>
-                </b-col>
-            </b-row>
-
-        </b-card>
-        <br/>
-
-        <label>{{ $t("scriptType") }}</label>
-        <b-form-select :options="['git', 'simple']" v-model="script.type" @change="update()"></b-form-select>
-
-        <template v-if="script.type === 'git'">
-            <label>{{ $t("gitRepository") }}</label>
-            <b-form-input v-model="script.git_repository" placeholder="https://github.com/example/example.git"
-                          @change="update()"></b-form-input>
-
-            <label>{{ $t("extraArgs") }}</label>
-            <b-form-input v-model="script.extra_args" @change="update()" class="text-monospace"></b-form-input>
-        </template>
-
-        <template v-if="script.type === 'simple'">
-
-            <label>{{ $t("scriptCode") }}</label>
-            <p>Find sist2-python documentation <a href="https://sist2-python.readthedocs.io/" target="_blank">here</a></p>
-            <b-textarea rows="15" class="text-monospace" v-model="script.script" @change="update()" spellcheck="false"></b-textarea>
-        </template>
-
-        <template v-if="script.type === 'local'">
-            <!-- TODO-->
-        </template>
-
-
-    </b-card>
-</template>
-
-<script>
-
-import Sist2AdminApi from "@/Sist2AdminApi";
-import JobOptions from "@/components/JobOptions.vue";
-import JobCheckboxGroup from "@/components/JobCheckboxGroup.vue";
-import JobSelect from "@/components/JobSelect.vue";
-
-export default {
-    name: "UserScript",
-    components: {JobSelect, JobCheckboxGroup, JobOptions},
-    data() {
-        return {
-            loading: true,
-            script: null,
-            selectedTestJob: null
-        }
-    },
-    methods: {
-        update() {
-            Sist2AdminApi.updateUserScript(this.name, this.script);
-        },
-        onJobSelect(job) {
-            this.selectedTestJob = job;
-        },
-        deleteScript() {
-            Sist2AdminApi.deleteUserScript(this.name)
-                .then(() => {
-                    this.$router.push("/");
-                })
-                .catch(err => {
-                    this.$bvToast.toast("Cannot delete user script " +
-                        "because it is referenced by a job", {
-                        title: "Error",
-                        variant: "danger",
-                        toaster: "b-toaster-bottom-right"
-                    });
-                })
-        },
-        testScript() {
-            Sist2AdminApi.testUserScript(this.name, this.selectedTestJob)
-                .then(() => {
-                    this.$bvToast.toast(this.$t("runJobConfirmation"), {
-                        title: this.$t("runJobConfirmationTitle"),
-                        variant: "success",
-                        toaster: "b-toaster-bottom-right"
-                    });
-                })
-        }
-    },
-    mounted() {
-        Sist2AdminApi.getUserScript(this.name).then(resp => {
-            this.script = resp.data;
-            this.loading = false;
-        });
-    },
-    computed: {
-        name() {
-            return this.$route.params.name;
-        },
-    },
-}
-</script>

sist2-admin/requirements.txt

@@ -3,5 +3,3 @@ git+https://github.com/simon987/hexlib.git
 uvicorn
 websockets
 pycron
-GitPython
-git+https://github.com/simon987/sist2-python.git

sist2-admin/sist2_admin/app.py

@@ -18,13 +18,11 @@ from websockets.exceptions import ConnectionClosed
 
 import cron
 from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
-from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus, Sist2UserScriptTask
+from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
 from notifications import Subscribe, Notifications
-from sist2 import Sist2, Sist2SearchBackend
+from sist2 import Sist2
-from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION, migrate_v3_to_v4, \
+from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
-    get_log_files_to_remove, delete_log_file, create_default_search_backends
 from web import Sist2Frontend
-from script import UserScript, SCRIPT_TEMPLATES
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
 db = PersistentState(dbfile=os.path.join(DATA_FOLDER, "state.db"))
@@ -53,8 +51,7 @@ async def home():
 async def api():
     return {
         "tesseract_langs": TESSERACT_LANGS,
-        "logs_folder": LOG_FOLDER,
+        "logs_folder": LOG_FOLDER
-        "user_script_templates": list(SCRIPT_TEMPLATES.keys())
     }
 
 
@@ -76,14 +73,16 @@ async def get_frontend(name: str):
     raise HTTPException(status_code=404)
 
 
-@app.get("/api/job")
+@app.get("/api/job/")
 async def get_jobs():
     return list(db["jobs"])
 
 
 @app.put("/api/job/{name:str}")
 async def update_job(name: str, new_job: Sist2Job):
-    new_job.last_modified = datetime.utcnow()
+    # TODO: Check etag
+
+    new_job.last_modified = datetime.now()
     job = db["jobs"][name]
     if not job:
         raise HTTPException(status_code=404)
@@ -115,10 +114,12 @@ async def update_job(name: str, new_job: Sist2Job):
 async def update_frontend(name: str, frontend: Sist2Frontend):
     db["frontends"][name] = frontend
 
+    # TODO: Check etag
+
     return "ok"
 
 
-@app.get("/api/task")
+@app.get("/api/task/")
 async def get_tasks():
     return list(map(lambda t: t.json(), task_queue.tasks()))
 
@@ -133,38 +134,16 @@ async def kill_job(task_id: str):
     return task_queue.kill_task(task_id)
 
 
-@app.post("/api/task/{task_id:str}/delete_logs")
-async def delete_task_logs(task_id: str):
-    if not db["task_done"][task_id]:
-        raise HTTPException(status_code=404)
-
-    delete_log_file(db, task_id)
-
-    return "ok"
-
-
 def _run_job(job: Sist2Job):
-    job.last_modified = datetime.utcnow()
+    job.last_modified = datetime.now()
     if job.status == JobStatus("created"):
         job.status = JobStatus("started")
     db["jobs"][job.name] = job
 
     scan_task = Sist2ScanTask(job, f"Scan [{job.name}]")
-
+    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=scan_task)
-    index_depends_on = scan_task
-    script_tasks = []
-    for script_name in job.user_scripts:
-        script = db["user_scripts"][script_name]
-
-        task = Sist2UserScriptTask(script, job, f"Script <{script_name}> [{job.name}]", depends_on=scan_task)
-        script_tasks.append(task)
-        index_depends_on = task
-
-    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=index_depends_on)
 
     task_queue.submit(scan_task)
-    for task in script_tasks:
-        task_queue.submit(task)
     task_queue.submit(index_task)
 
 
@@ -179,53 +158,19 @@ async def run_job(name: str):
     return "ok"
 
 
-@app.get("/api/user_script/{name:str}/run")
-def run_user_script(name: str, job: str):
-    script = db["user_scripts"][name]
-    if not script:
-        raise HTTPException(status_code=404)
-    job = db["jobs"][job]
-    if not job:
-        raise HTTPException(status_code=404)
-
-    script_task = Sist2UserScriptTask(script, job, f"Script <{name}> [{job.name}]")
-
-    task_queue.submit(script_task)
-
-    return "ok"
-
-
-@app.get("/api/job/{name:str}/logs_to_delete")
-async def task_history(n: int, name: str):
-    return get_log_files_to_remove(db, name, n)
-
-
 @app.delete("/api/job/{name:str}")
 async def delete_job(name: str):
-    job: Sist2Job = db["jobs"][name]
+    job = db["jobs"][name]
-    if not job:
+    if job:
-        raise HTTPException(status_code=404)
-
-    if any(name in frontend.jobs for frontend in db["frontends"]):
-        raise HTTPException(status_code=400, detail="in use (frontend)")
-
-    try:
-        os.remove(job.previous_index)
-    except:
-        pass
-
         del db["jobs"][name]
-
+    else:
-    return "ok"
+        raise HTTPException(status_code=404)
 
 
 @app.delete("/api/frontend/{name:str}")
 async def delete_frontend(name: str):
     if name in RUNNING_FRONTENDS:
-        try:
         os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
-        except ProcessLookupError:
-            pass
         del RUNNING_FRONTENDS[name]
 
     frontend = db["frontends"][name]
@@ -270,7 +215,7 @@ def check_es_version(es_url: str, insecure: bool):
             es_url = f"{url.scheme}://{url.hostname}:{url.port}"
         else:
             auth = None
-        r = requests.get(es_url, verify=not insecure, auth=auth)
+        r = requests.get(es_url, verify=insecure, auth=auth)
     except SSLError:
         return {
             "ok": False,
@@ -306,21 +251,9 @@ def check_es_version(es_url: str, insecure: bool):
 
 
 def start_frontend_(frontend: Sist2Frontend):
-    frontend.web_options.indices = [
+    frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
-        os.path.join(DATA_FOLDER, db["jobs"][j].index_path)
-        for j in frontend.jobs
-    ]
 
-    backend_name = frontend.web_options.search_backend
+    pid = sist2.web(frontend.web_options, frontend.name)
-    search_backend = db["search_backends"][backend_name]
-    if search_backend is None:
-        logger.error(
-            f"Error while running task: search backend not found: {backend_name}")
-        return -1
-
-    logger.debug(f"Fetched search backend options for {backend_name}")
-
-    pid = sist2.web(frontend.web_options, search_backend, frontend.name)
     RUNNING_FRONTENDS[frontend.name] = pid
 
 
@@ -340,7 +273,7 @@ async def stop_frontend(name: str):
         del RUNNING_FRONTENDS[name]
 
 
-@app.get("/api/frontend")
+@app.get("/api/frontend/")
 async def get_frontends():
     res = []
     for frontend in db["frontends"]:
@@ -350,115 +283,6 @@ async def get_frontends():
     return res
 
 
-@app.get("/api/search_backend")
-async def get_search_backends():
-    return list(db["search_backends"])
-
-
-@app.put("/api/search_backend/{name:str}")
-async def update_search_backend(name: str, backend: Sist2SearchBackend):
-    if not db["search_backends"][name]:
-        raise HTTPException(status_code=404)
-
-    db["search_backends"][name] = backend
-    return "ok"
-
-
-@app.get("/api/search_backend/{name:str}")
-def get_search_backend(name: str):
-    backend = db["search_backends"][name]
-    if not backend:
-        raise HTTPException(status_code=404)
-
-    return backend
-
-
-@app.delete("/api/search_backend/{name:str}")
-def delete_search_backend(name: str):
-    backend: Sist2SearchBackend = db["search_backends"][name]
-    if not backend:
-        raise HTTPException(status_code=404)
-
-    if any(frontend.web_options.search_backend == name for frontend in db["frontends"]):
-        raise HTTPException(status_code=400, detail="in use (frontend)")
-
-    if any(job.index_options.search_backend == name for job in db["jobs"]):
-        raise HTTPException(status_code=400, detail="in use (job)")
-
-    del db["search_backends"][name]
-
-    try:
-        os.remove(os.path.join(DATA_FOLDER, backend.search_index))
-    except:
-        pass
-
-    return "ok"
-
-
-@app.post("/api/search_backend/{name:str}")
-def create_search_backend(name: str):
-    if db["search_backends"][name] is not None:
-        return HTTPException(status_code=400, detail="already exists")
-
-    backend = Sist2SearchBackend.create_default(name)
-    db["search_backends"][name] = backend
-
-    return backend
-
-
-@app.delete("/api/user_script/{name:str}")
-def delete_user_script(name: str):
-    if db["user_scripts"][name] is None:
-        return HTTPException(status_code=404)
-
-    if any(name in job.user_scripts for job in db["jobs"]):
-        raise HTTPException(status_code=400, detail="in use (job)")
-
-    script: UserScript = db["user_scripts"][name]
-    script.delete_dir()
-
-    del db["user_scripts"][name]
-
-    return "ok"
-
-
-@app.post("/api/user_script/{name:str}")
-def create_user_script(name: str, template: str):
-    if db["user_scripts"][name] is not None:
-        return HTTPException(status_code=400, detail="already exists")
-
-    script = SCRIPT_TEMPLATES[template](name)
-    db["user_scripts"][name] = script
-
-    return script
-
-
-@app.get("/api/user_script")
-async def get_user_scripts():
-    return list(db["user_scripts"])
-
-
-@app.get("/api/user_script/{name:str}")
-async def get_user_script(name: str):
-    backend = db["user_scripts"][name]
-    if not backend:
-        raise HTTPException(status_code=404)
-
-    return backend
-
-
-@app.put("/api/user_script/{name:str}")
-async def update_user_script(name: str, script: UserScript):
-    previous_version: UserScript = db["user_scripts"][name]
-
-    if previous_version and previous_version.git_repository != script.git_repository:
-        script.force_clone = True
-
-    db["user_scripts"][name] = script
-
-    return "ok"
-
-
 def tail(filepath: str, n: int):
     with open(filepath) as file:
 
@@ -497,6 +321,7 @@ async def ws_tail_log(websocket: WebSocket):
         async with Subscribe(notifications) as ob:
             async for notification in ob.notifications():
                 await websocket.send_json(notification)
+                print(notification)
 
     except ConnectionClosed:
         return
@@ -527,7 +352,7 @@ async def ws_tail_log(websocket: WebSocket, task_id: str, n: int):
 
 
 def main():
-    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0", timeout_graceful_shutdown=0)
+    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0")
 
 
 def initialize_db():
@@ -536,8 +361,6 @@ def initialize_db():
     frontend = Sist2Frontend.create_default("default")
     db["frontends"]["default"] = frontend
 
-    create_default_search_backends(db)
-
     logger.info("Initialized database.")
 
 
@@ -558,13 +381,6 @@ if __name__ == '__main__':
     if db["sist2_admin"]["info"]["version"] == "2":
         logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.")
         exit(-1)
-    if db["sist2_admin"]["info"]["version"] == "3":
-        logger.info("Migrating to v4 database schema")
-        migrate_v3_to_v4(db)
-
-    if db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
-        raise Exception(f"Incompatible database {db.dbfile}. "
-                        f"Automatic migration is not available, please delete the database file to continue.")
 
     start_frontends()
     cron.initialize(db, _run_job)

sist2-admin/sist2_admin/config.py

@@ -9,11 +9,9 @@ MAX_LOG_SIZE = 1 * 1024 * 1024
 SIST2_BINARY = os.environ.get("SIST2_BINARY", "/root/sist2")
 DATA_FOLDER = os.environ.get("DATA_FOLDER", "/sist2-admin/")
 LOG_FOLDER = os.path.join(DATA_FOLDER, "logs")
-SCRIPT_FOLDER = os.path.join(DATA_FOLDER, "scripts")
 WEBSERVER_PORT = 8080
 
 os.makedirs(LOG_FOLDER, exist_ok=True)
-os.makedirs(SCRIPT_FOLDER, exist_ok=True)
 os.makedirs(DATA_FOLDER, exist_ok=True)
 
 logger = logging.Logger("sist2-admin")

sist2-admin/sist2_admin/cron.py

@@ -10,9 +10,7 @@ from jobs import Sist2Job
 
 
 def _check_schedule(db: PersistentState, run_job):
-    jobs = list(db["jobs"])
+    for job in db["jobs"]:
-
-    for job in jobs:
         job: Sist2Job
 
         if job.schedule_enabled:

sist2-admin/sist2_admin/jobs.py

@@ -1,29 +1,23 @@
 import json
 import logging
 import os.path
-import shlex
 import signal
 import uuid
 from datetime import datetime
 from enum import Enum
-from io import TextIOWrapper
 from logging import FileHandler
-from subprocess import Popen
-import subprocess
 from threading import Lock, Thread
 from time import sleep
-from typing import List
 from uuid import uuid4, UUID
 
 from hexlib.db import PersistentState
 from pydantic import BaseModel
 
-from config import logger, LOG_FOLDER, DATA_FOLDER
+from config import logger, LOG_FOLDER
 from notifications import Notifications
 from sist2 import ScanOptions, IndexOptions, Sist2
-from state import RUNNING_FRONTENDS, get_log_files_to_remove, delete_log_file
+from state import RUNNING_FRONTENDS
 from web import Sist2Frontend
-from script import UserScript
 
 
 class JobStatus(Enum):
@@ -38,13 +32,9 @@ class Sist2Job(BaseModel):
     scan_options: ScanOptions
     index_options: IndexOptions
 
-    user_scripts: List[str] = []
-
     cron_expression: str
     schedule_enabled: bool = False
 
-    keep_last_n_logs: int = -1
-
     previous_index: str = None
     index_path: str = None
     previous_index_path: str = None
@@ -63,10 +53,15 @@ class Sist2Job(BaseModel):
             name=name,
             scan_options=ScanOptions(path="/"),
             index_options=IndexOptions(),
-            last_modified=datetime.utcnow(),
+            last_modified=datetime.now(),
             cron_expression="0 0 * * *"
         )
 
+    # @validator("etag", always=True)
+    # def validate_etag(cls, value, values):
+    #     s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
+    #     return md5(s.encode()).hexdigest()
+
 
 class Sist2TaskProgress:
 
@@ -116,14 +111,10 @@ class Sist2Task:
             self._logger.info(json.dumps(log_json))
 
     def run(self, sist2: Sist2, db: PersistentState):
-        self.started = datetime.utcnow()
+        self.started = datetime.now()
 
         logger.info(f"Started task {self.display_name}")
 
-    def set_pid(self, pid):
-        self.pid = pid
-
-
 
 class Sist2ScanTask(Sist2Task):
 
@@ -137,17 +128,18 @@ class Sist2ScanTask(Sist2Task):
         else:
             self.job.scan_options.output = None
 
-        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
+        def set_pid(pid):
-        self.ended = datetime.utcnow()
+            self.pid = pid
 
-        is_ok = (return_code in (0, 1)) if "debug" in sist2.bin_path else (return_code == 0)
+        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
+        self.ended = datetime.now()
 
-        if not is_ok:
+        if return_code != 0:
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
             logger.info(f"Task {self.display_name} failed ({return_code})")
         else:
             self.job.index_path = self.job.scan_options.output
-            self.job.last_index_date = datetime.utcnow()
+            self.job.last_index_date = datetime.now()
             self.job.do_full_scan = False
             db["jobs"][self.job.name] = self.job
             self._logger.info(json.dumps({"sist2-admin": f"Save last_index_date={self.job.last_index_date}"}))
@@ -155,7 +147,7 @@ class Sist2ScanTask(Sist2Task):
         logger.info(f"Completed {self.display_name} ({return_code=})")
 
         # Remove old index
-        if is_ok:
+        if return_code == 0:
             if self.job.previous_index_path is not None and self.job.previous_index_path != self.job.index_path:
                 self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index_path=}"}))
                 try:
@@ -166,9 +158,6 @@ class Sist2ScanTask(Sist2Task):
             self.job.previous_index_path = self.job.index_path
             db["jobs"][self.job.name] = self.job
 
-        if is_ok:
-            return 0
-
         return return_code
 
 
@@ -182,19 +171,12 @@ class Sist2IndexTask(Sist2Task):
 
         self.job.index_options.path = self.job.scan_options.output
 
-        search_backend = db["search_backends"][self.job.index_options.search_backend]
+        return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
-        if search_backend is None:
+        self.ended = datetime.now()
-            logger.error(f"Error while running task: search backend not found: {self.job.index_options.search_backend}")
-            return -1
-
-        logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
-
-        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
-        self.ended = datetime.utcnow()
 
         duration = self.ended - self.started
 
-        ok = return_code in (0, 1)
+        ok = return_code == 0
 
         if ok:
             self.restart_running_frontends(db, sist2)
@@ -224,84 +206,14 @@ class Sist2IndexTask(Sist2Task):
             except ChildProcessError:
                 pass
 
-            backend_name = frontend.web_options.search_backend
+            frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
-            search_backend = db["search_backends"][backend_name]
-            if search_backend is None:
-                logger.error(f"Error while running task: search backend not found: {backend_name}")
-                return -1
 
-            logger.debug(f"Fetched search backend options for {backend_name}")
+            pid = sist2.web(frontend.web_options, frontend.name)
-
-            frontend.web_options.indices = [
-                os.path.join(DATA_FOLDER, db["jobs"][j].index_path)
-                for j in frontend.jobs
-            ]
-
-            pid = sist2.web(frontend.web_options, search_backend, frontend.name)
             RUNNING_FRONTENDS[frontend_name] = pid
 
             self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))
 
 
-class Sist2UserScriptTask(Sist2Task):
-
-    def __init__(self, user_script: UserScript, job: Sist2Job, display_name: str, depends_on: Sist2Task = None):
-        super().__init__(job, display_name, depends_on=depends_on.id if depends_on else None)
-        self.user_script = user_script
-
-    def run(self, sist2: Sist2, db: PersistentState):
-        super().run(sist2, db)
-
-        try:
-            self.user_script.setup(self.log_callback, self.set_pid)
-        except Exception as e:
-            logger.error(f"Setup for {self.user_script.name} failed: ")
-            logger.exception(e)
-            self.log_callback({"sist2-admin": f"Setup for {self.user_script.name} failed: {e}"})
-            return -1
-
-        executable = self.user_script.get_executable()
-        index_path = os.path.join(DATA_FOLDER, self.job.index_path)
-        extra_args = self.user_script.extra_args
-
-        args = [
-            executable,
-            index_path,
-            *shlex.split(extra_args)
-        ]
-
-        self.log_callback({"sist2-admin": f"Starting user script with {executable=}, {index_path=}, {extra_args=}"})
-
-        proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.user_script.script_dir())
-        self.set_pid(proc.pid)
-
-        t_stderr = Thread(target=self._consume_logs, args=(self.log_callback, proc, "stderr", False))
-        t_stderr.start()
-
-        self._consume_logs(self.log_callback, proc, "stdout", True)
-
-        self.ended = datetime.utcnow()
-
-        return 0
-
-    @staticmethod
-    def _consume_logs(logs_cb, proc, stream, wait):
-        pipe_wrapper = TextIOWrapper(getattr(proc, stream), encoding="utf8", errors="ignore")
-        try:
-            for line in pipe_wrapper:
-                if line.strip() == "":
-                    continue
-                if line.startswith("$PROGRESS"):
-                    progress = json.loads(line[len("$PROGRESS "):])
-                    logs_cb({"progress": progress})
-                    continue
-                logs_cb({stream: line})
-        finally:
-            if wait:
-                proc.wait()
-            pipe_wrapper.close()
-
-
 class TaskQueue:
     def __init__(self, sist2: Sist2, db: PersistentState, notifications: Notifications):
         self._lock = Lock()
@@ -389,14 +301,8 @@ class TaskQueue:
                 "ended": task.ended,
                 "started": task.started,
                 "name": task.display_name,
-                "return_code": task_result,
+                "return_code": task_result
-                "has_logs": 1
             }
-
-            logs_to_delete = get_log_files_to_remove(self._db, task.job.name, task.job.keep_last_n_logs)
-            for row in logs_to_delete:
-                delete_log_file(self._db, row["id"])
-
         if isinstance(task, Sist2IndexTask):
             self._notifications.notify({
                 "message": "notifications.indexCompleted",

sist2-admin/sist2_admin/script.py

@@ -1,130 +0,0 @@
-import os
-import shutil
-import stat
-import subprocess
-from enum import Enum
-
-from git import Repo
-from pydantic import BaseModel
-
-from config import SCRIPT_FOLDER
-
-
-class ScriptType(Enum):
-    LOCAL = "local"
-    SIMPLE = "simple"
-    GIT = "git"
-
-
-def set_executable(file):
-    os.chmod(file, os.stat(file).st_mode | stat.S_IEXEC)
-
-
-def _initialize_git_repository(url, path, log_cb, force_clone, set_pid_cb):
-    log_cb({"sist2-admin": f"Cloning {url}"})
-
-    if force_clone or not os.path.exists(os.path.join(path, ".git")):
-        if force_clone:
-            shutil.rmtree(path, ignore_errors=True)
-        Repo.clone_from(url, path)
-    else:
-        repo = Repo(path)
-        repo.remote("origin").pull()
-
-    setup_script = os.path.join(path, "setup.sh")
-    if setup_script:
-        log_cb({"sist2-admin": f"Executing setup script {setup_script}"})
-
-        set_executable(setup_script)
-        proc = subprocess.Popen([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        set_pid_cb(proc.pid)
-        proc.wait()
-        stdout = proc.stdout.read()
-
-        for line in stdout.split(b"\n"):
-            if line:
-                log_cb({"stdout": line.decode()})
-
-        log_cb({"stdout": f"Executed setup script {setup_script}, return code = {proc.returncode}"})
-
-        if proc.returncode != 0:
-            raise Exception("Error when running setup script!")
-
-    log_cb({"sist2-admin": f"Initialized git repository in {path}"})
-
-
-class UserScript(BaseModel):
-    name: str
-    type: ScriptType
-    git_repository: str = None
-    force_clone: bool = False
-    script: str = None
-    extra_args: str = ""
-
-    def script_dir(self):
-        return os.path.join(SCRIPT_FOLDER, self.name)
-
-    def setup(self, log_cb, set_pid_cb):
-        os.makedirs(self.script_dir(), exist_ok=True)
-
-        if self.type == ScriptType.GIT:
-            _initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone, set_pid_cb)
-            self.force_clone = False
-        elif self.type == ScriptType.SIMPLE:
-            self._setup_simple()
-
-        set_executable(self.get_executable())
-
-    def _setup_simple(self):
-        with open(self.get_executable(), "w") as f:
-            f.write(
-                "#!/bin/bash\n"
-                "python run.py \"$@\""
-            )
-
-        with open(os.path.join(self.script_dir(), "run.py"), "w") as f:
-            f.write(self.script)
-
-    def get_executable(self):
-        return os.path.join(self.script_dir(), "run.sh")
-
-    def delete_dir(self):
-        shutil.rmtree(self.script_dir(), ignore_errors=True)
-
-
-SCRIPT_TEMPLATES = {
-    "CLIP - Generate embeddings to predict the most relevant image based on the text prompt": lambda name: UserScript(
-        name=name,
-        type=ScriptType.GIT,
-        git_repository="https://github.com/simon987/sist2-script-clip",
-        extra_args="--num-tags=1 --tags-file=general.txt --color=#dcd7ff"
-    ),
-    "Whisper - Speech to text with OpenAI Whisper": lambda name: UserScript(
-        name=name,
-        type=ScriptType.GIT,
-        git_repository="https://github.com/simon987/sist2-script-whisper",
-        extra_args="--model=base --num-threads=4 --color=#51da4c --tag"
-    ),
-    "Hamburger - Simple script example": lambda name: UserScript(
-        name=name,
-        type=ScriptType.SIMPLE,
-        script=
-        'from sist2 import Sist2Index\n'
-        'import sys\n'
-        '\n'
-        'index = Sist2Index(sys.argv[1])\n'
-        'for doc in index.document_iter():\n'
-        '    doc.json_data["tag"] = ["hamburger.#00FF00"]\n'
-        '    index.update_document(doc)\n'
-        '\n'
-        'index.sync_tag_table()\n'
-        'index.commit()\n'
-        '\n'
-        'print("Done!")\n'
-    ),
-    "(Blank)": lambda name: UserScript(
-        name=name,
-        type=ScriptType.SIMPLE,
-        script=""
-    )
-}

sist2-admin/sist2_admin/sist2.py

@@ -3,7 +3,6 @@ import json
 import logging
 import os.path
 from datetime import datetime
-from enum import Enum
 from io import TextIOWrapper
 from logging import FileHandler
 from subprocess import Popen, PIPE
@@ -13,7 +12,7 @@ from typing import List
 
 from pydantic import BaseModel
 
-from config import logger, LOG_FOLDER, DATA_FOLDER
+from config import logger, LOG_FOLDER
 
 
 class Sist2Version:
@@ -26,53 +25,74 @@ class Sist2Version:
         return f"{self.major}.{self.minor}.{self.patch}"
 
 
-class SearchBackendType(Enum):
+class WebOptions(BaseModel):
-    SQLITE = "sqlite"
+    indices: List[str] = []
-    ELASTICSEARCH = "elasticsearch"
-
-
-class Sist2SearchBackend(BaseModel):
-    backend_type: SearchBackendType = SearchBackendType("elasticsearch")
-    name: str
-
-    search_index: str = ""
-
     es_url: str = "http://elasticsearch:9200"
     es_insecure_ssl: bool = False
     es_index: str = "sist2"
-    threads: int = 1
+    bind: str = "0.0.0.0:4090"
-    batch_size: int = 70
+    auth: str = None
-
+    tag_auth: str = None
-    @staticmethod
+    tagline: str = "Lightning-fast file system indexer and search tool"
-    def create_default(name: str, backend_type: SearchBackendType = SearchBackendType("elasticsearch")):
+    dev: bool = False
-        return Sist2SearchBackend(
+    lang: str = "en"
-            name=name,
+    auth0_audience: str = None
-            search_index=f"search-index-{name.replace('/', '_')}.sist2",
+    auth0_domain: str = None
-            backend_type=backend_type
+    auth0_client_id: str = None
-        )
+    auth0_public_key: str = None
-
+    auth0_public_key_file: str = None
-
-class IndexOptions(BaseModel):
-    path: str = None
-    incremental_index: bool = True
-    search_backend: str = None
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def args(self, search_backend):
+    def args(self):
-        absolute_path = os.path.join(DATA_FOLDER, self.path)
+        args = ["web", f"--es-url={self.es_url}", f"--bind={self.bind}",
+                f"--tagline={self.tagline}", f"--lang={self.lang}"]
 
-        if search_backend.backend_type == SearchBackendType("sqlite"):
+        if self.auth0_audience:
-            search_index_absolute = os.path.join(DATA_FOLDER, search_backend.search_index)
+            args.append(f"--auth0-audience={self.auth0_audience}")
-            args = ["sqlite-index", absolute_path, "--search-index", search_index_absolute]
+        if self.auth0_domain:
-        else:
+            args.append(f"--auth0-domain={self.auth0_domain}")
-            args = ["index", absolute_path, f"--threads={search_backend.threads}",
+        if self.auth0_client_id:
-                    f"--es-url={search_backend.es_url}",
+            args.append(f"--auth0-client-id={self.auth0_client_id}")
-                    f"--es-index={search_backend.es_index}",
+        if self.auth0_public_key_file:
-                    f"--batch-size={search_backend.batch_size}"]
+            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
+        if self.es_insecure_ssl:
+            args.append(f"--es-insecure-ssl")
+        if self.auth:
+            args.append(f"--auth={self.auth}")
+        if self.tag_auth:
+            args.append(f"--tag-auth={self.tag_auth}")
+        if self.dev:
+            args.append(f"--dev")
 
-            if search_backend.es_insecure_ssl:
+        args.extend(self.indices)
+
+        return args
+
+
+class IndexOptions(BaseModel):
+    path: str = None
+    threads: int = 1
+    es_url: str = "http://elasticsearch:9200"
+    es_insecure_ssl: bool = False
+    es_index: str = "sist2"
+    incremental_index: bool = True
+    script: str = ""
+    script_file: str = None
+    batch_size: int = 70
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self):
+
+        args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
+                f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
+
+        if self.script_file:
+            args.append(f"--script-file={self.script_file}")
+        if self.es_insecure_ssl:
             args.append(f"--es-insecure-ssl")
         if self.incremental_index:
             args.append(f"--incremental-index")
@@ -89,7 +109,7 @@ ARCHIVE_RECURSE = "recurse"
 class ScanOptions(BaseModel):
     path: str
     threads: int = 1
-    thumbnail_quality: int = 50
+    thumbnail_quality: int = 2
     thumbnail_size: int = 552
     thumbnail_count: int = 1
     content_size: int = 32768
@@ -117,12 +137,9 @@ class ScanOptions(BaseModel):
         super().__init__(**kwargs)
 
     def args(self):
-
-        output_path = os.path.join(DATA_FOLDER, self.output)
-
         args = ["scan", self.path, f"--threads={self.threads}", f"--thumbnail-quality={self.thumbnail_quality}",
                 f"--thumbnail-count={self.thumbnail_count}", f"--thumbnail-size={self.thumbnail_size}",
-                f"--content-size={self.content_size}", f"--output={output_path}", f"--depth={self.depth}",
+                f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
                 f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
 
         if self.incremental:
@@ -184,79 +201,29 @@ class Sist2Index:
         return self._descriptor["name"]
 
 
-class WebOptions(BaseModel):
-    indices: List[str] = []
-
-    search_backend: str = "elasticsearch"
-
-    bind: str = "0.0.0.0:4090"
-    auth: str = None
-    tag_auth: str = None
-    tagline: str = "Lightning-fast file system indexer and search tool"
-    dev: bool = False
-    lang: str = "en"
-    auth0_audience: str = None
-    auth0_domain: str = None
-    auth0_client_id: str = None
-    auth0_public_key: str = None
-    auth0_public_key_file: str = None
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def args(self, search_backend: Sist2SearchBackend):
-        args = ["web", f"--bind={self.bind}", f"--tagline={self.tagline}",
-                f"--lang={self.lang}"]
-
-        if search_backend.backend_type == SearchBackendType("sqlite"):
-            search_index_absolute = os.path.join(DATA_FOLDER, search_backend.search_index)
-            args.append(f"--search-index={search_index_absolute}")
-        else:
-            args.append(f"--es-url={search_backend.es_url}")
-            args.append(f"--es-index={search_backend.es_index}")
-            if search_backend.es_insecure_ssl:
-                args.append(f"--es-insecure-ssl")
-
-        if self.auth0_audience:
-            args.append(f"--auth0-audience={self.auth0_audience}")
-        if self.auth0_domain:
-            args.append(f"--auth0-domain={self.auth0_domain}")
-        if self.auth0_client_id:
-            args.append(f"--auth0-client-id={self.auth0_client_id}")
-        if self.auth0_public_key_file:
-            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
-        if self.auth:
-            args.append(f"--auth={self.auth}")
-        if self.tag_auth:
-            args.append(f"--tag-auth={self.tag_auth}")
-        if self.dev:
-            args.append(f"--dev")
-
-        args.extend(self.indices)
-
-        return args
-
-
 class Sist2:
 
     def __init__(self, bin_path: str, data_directory: str):
-        self.bin_path = bin_path
+        self._bin_path = bin_path
         self._data_dir = data_directory
 
-    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb, set_pid_cb):
+    def index(self, options: IndexOptions, logs_cb):
+
+        if options.script:
+            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
+                f.write(options.script)
+            options.script_file = f.name
+        else:
+            options.script_file = None
 
         args = [
-            self.bin_path,
+            self._bin_path,
-            *options.args(search_backend),
+            *options.args(),
             "--json-logs",
             "--very-verbose"
         ]
-
-        logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
-        set_pid_cb(proc.pid)
-
         t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
         t_stderr.start()
 
@@ -269,10 +236,13 @@ class Sist2:
     def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
 
         if options.output is None:
-            options.output = f"scan-{options.name.replace('/', '_')}-{datetime.utcnow()}.sist2"
+            options.output = os.path.join(
+                self._data_dir,
+                f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2"
+            )
 
         args = [
-            self.bin_path,
+            self._bin_path,
             *options.args(),
             "--json-logs",
             "--very-verbose"
@@ -320,7 +290,7 @@ class Sist2:
                 except NameError:
                     pass
 
-    def web(self, options: WebOptions, search_backend: Sist2SearchBackend, name: str):
+    def web(self, options: WebOptions, name: str):
 
         if options.auth0_public_key:
             with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
@@ -330,8 +300,8 @@ class Sist2:
             options.auth0_public_key_file = None
 
         args = [
-            self.bin_path,
+            self._bin_path,
-            *options.args(search_backend)
+            *options.args()
         ]
 
         web_logger = logging.Logger(name=f"sist2-frontend-{name}")

sist2-admin/sist2_admin/state.py

@@ -1,20 +1,16 @@
 from typing import Dict
-import os
 import shutil
 
 from hexlib.db import Table, PersistentState
 import pickle
 
 from tesseract import get_tesseract_langs
-import sqlite3
-from config import LOG_FOLDER, logger
-from sist2 import SearchBackendType, Sist2SearchBackend
 
 RUNNING_FRONTENDS: Dict[str, int] = {}
 
 TESSERACT_LANGS = get_tesseract_langs()
 
-DB_SCHEMA_VERSION = "5"
+DB_SCHEMA_VERSION = "3"
 
 from pydantic import BaseModel
 
@@ -54,35 +50,8 @@ class PickleTable(Table):
             yield dict((k, _deserialize(v)) for k, v in row.items())
 
 
-def get_log_files_to_remove(db: PersistentState, job_name: str, n: int):
-    if n < 0:
-        return []
-
-    counter = 0
-    to_remove = []
-
-    for row in db["task_done"].sql("WHERE has_logs=1 ORDER BY started DESC"):
-        if row["name"].endswith(f"[{job_name}]"):
-            counter += 1
-
-            if counter > n:
-                to_remove.append(row)
-
-    return to_remove
-
-
-def delete_log_file(db: PersistentState, task_id: str):
-    db["task_done"][task_id] = {
-        "has_logs": 0
-    }
-
-    try:
-        os.remove(os.path.join(LOG_FOLDER, f"sist2-{task_id}.log"))
-    except:
-        pass
-
-
 def migrate_v1_to_v2(db: PersistentState):
+
     shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v2.bak")
 
     # Frontends
@@ -108,29 +77,3 @@ def migrate_v1_to_v2(db: PersistentState):
     db["sist2_admin"]["info"] = {
         "version": "2"
     }
-
-
-def create_default_search_backends(db: PersistentState):
-    es_backend = Sist2SearchBackend.create_default(name="elasticsearch",
-                                                   backend_type=SearchBackendType("elasticsearch"))
-    db["search_backends"]["elasticsearch"] = es_backend
-    sqlite_backend = Sist2SearchBackend.create_default(name="sqlite", backend_type=SearchBackendType("sqlite"))
-    db["search_backends"]["sqlite"] = sqlite_backend
-
-
-def migrate_v3_to_v4(db: PersistentState):
-    shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v4.bak")
-
-    create_default_search_backends(db)
-
-    try:
-        conn = sqlite3.connect(db.dbfile)
-        conn.execute("ALTER TABLE task_done ADD COLUMN has_logs INTEGER DEFAULT 1")
-        conn.commit()
-        conn.close()
-    except Exception as e:
-        logger.exception(e)
-
-    db["sist2_admin"]["info"] = {
-        "version": "4"
-    }

sist2-vue/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sist2",
-  "version": "1.0.0",
+  "version": "2.11.0",
   "private": true,
   "scripts": {
     "serve": "vue-cli-service serve",
@@ -17,7 +17,6 @@
     "dom-to-image": "^2.6.0",
     "fslightbox-vue": "fslightbox-vue.tgz",
     "nouislider": "^15.2.0",
-    "onnxruntime-web": "^1.15.1",
     "underscore": "^1.13.1",
     "vue": "^2.6.12",
     "vue-color": "^2.8.1",
@@ -29,9 +28,9 @@
   },
   "devDependencies": {
     "@babel/polyfill": "^7.12.1",
-    "@types/underscore": "^1.11.6",
     "@vue/cli-plugin-babel": "~5.0.8",
     "@vue/cli-plugin-router": "~5.0.8",
+    "@vue/cli-plugin-typescript": "^5.0.8",
     "@vue/cli-plugin-vuex": "~5.0.8",
     "@vue/cli-service": "^5.0.8",
     "@vue/test-utils": "^1.0.3",
@@ -43,7 +42,8 @@
     "portal-vue": "^2.1.7",
     "sass": "^1.26.11",
     "sass-loader": "^10.0.2",
-    "vue-cli-plugin-bootstrap-vue": "~0.8.2",
+    "typescript": "~4.1.5",
+    "vue-cli-plugin-bootstrap-vue": "~0.7.0",
     "vue-template-compiler": "^2.6.11"
   },
   "browserslist": [

sist2-vue/public/index.html

@@ -19,6 +19,12 @@
             We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled.
             Please enable it to continue.
         </strong>
+        <br/>
+        <strong>
+            Nous sommes désolés mais <%= htmlWebpackPlugin.options.title %> ne fonctionne pas correctement
+            si JavaScript est activé.
+            Veuillez l'activer pour continuer.
+        </strong>
     </div>
 </noscript>
 <div id="app"></div>

sist2-vue/src/App.vue

@@ -10,7 +10,7 @@
       <b-spinner type="grow" variant="primary"></b-spinner>
     </div>
     <div class="loading-text">
-            Loading • Chargement • 装载 • Wird geladen • Ładowanie
+      Loading • Chargement • 装载 • Wird geladen
     </div>
   </div>
 </template>
@@ -19,10 +19,7 @@
 import NavBar from "@/components/NavBar";
 import {mapActions, mapGetters, mapMutations} from "vuex";
 import Sist2Api from "@/Sist2Api";
-import ModelsRepo from "@/ml/modelsRepo";
 import {setupAuth0} from "@/main";
-import Sist2ElasticsearchQuery from "@/Sist2ElasticsearchQuery";
-import Sist2SqliteQuery from "@/Sist2SqliteQuery";
 
 export default {
   components: {NavBar},
@@ -39,17 +36,6 @@ export default {
   mounted() {
     this.$store.dispatch("loadConfiguration").then(() => {
       this.$root.$i18n.locale = this.$store.state.optLang;
-            ModelsRepo.init(this.$store.getters.mlRepositoryList).catch(err => {
-                this.$bvToast.toast(
-                    this.$t("ml.repoFetchError"),
-                    {
-                        title: this.$t("ml.repoFetchErrorTitle"),
-                        noAutoHide: true,
-                        toaster: "b-toaster-bottom-right",
-                        headerClass: "toast-header-warning",
-                        bodyClass: "toast-body-warning",
-                    });
-            });
     });
 
     this.$store.subscribe((mutation) => {
@@ -90,13 +76,6 @@ export default {
 
       this.setSist2Info(data);
       this.setIndices(data.indices)
-
-            if (Sist2Api.backend() === "sqlite") {
-                Sist2Api.init(Sist2SqliteQuery.searchQuery);
-                this.$store.commit("setUiSqliteMode", true);
-            } else {
-                Sist2Api.init(Sist2ElasticsearchQuery.searchQuery);
-            }
     });
   },
   methods: {
@@ -308,21 +287,15 @@ html, body {
 
 .info-icon {
   width: 1rem;
-    min-width: 1rem;
   margin-right: 0.2rem;
   cursor: pointer;
   line-height: 1rem;
   height: 1rem;
-    min-height: 1rem;
   background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
   filter: brightness(45%);
   display: block;
 }
 
-.theme-black .info-icon {
-    filter: brightness(80%);
-}
-
 .tabs {
   margin-top: 10px;
 }

sist2-vue/src/Sist2Api.js

@@ -1,635 +0,0 @@
-import axios from "axios";
-import {strUnescape, lum, sid} from "./util";
-import Sist2Query from "@/Sist2ElasticsearchQuery";
-import store from "@/store";
-
-
-class Sist2Api {
-
-    baseUrl;
-    sist2Info;
-    queryfunc;
-
-    constructor(baseUrl) {
-        this.baseUrl = baseUrl;
-    }
-
-    init(queryFunc) {
-        this.queryfunc = queryFunc;
-    }
-
-    backend() {
-        return this.sist2Info.searchBackend;
-    }
-
-    models() {
-        const allModels = this.sist2Info.indices
-            .map(idx => idx.models)
-            .flat();
-
-        return allModels
-            .filter((v, i, a) => a.findIndex(v2 => (v2.id === v.id)) === i)
-    }
-
-    getSist2Info() {
-        return axios.get(`${this.baseUrl}i`).then(resp => {
-            const indices = resp.data.indices;
-
-            resp.data.indices = indices.map(idx => {
-                return {
-                    id: idx.id,
-                    name: idx.name,
-                    timestamp: idx.timestamp,
-                    version: idx.version,
-                    models: idx.models,
-                };
-            });
-
-            this.sist2Info = resp.data;
-
-            return resp.data;
-        })
-    }
-
-    setHitProps(hit) {
-        hit["_props"] = {};
-
-        const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
-
-        if ("parent" in hit._source) {
-            hit._props.isSubDocument = true;
-        }
-
-        if ("thumbnail" in hit._source && hit._source.thumbnail > 0) {
-            hit._props.hasThumbnail = true;
-
-            if (Number.isNaN(Number(hit._source.thumbnail))) {
-                // Backwards compatibility
-                hit._props.tnNum = 1;
-                hit._props.hasVidPreview = false;
-            } else {
-                hit._props.tnNum = Number(hit._source.thumbnail);
-                hit._props.hasVidPreview = hit._props.tnNum > 1;
-            }
-        }
-
-        switch (mimeCategory) {
-            case "image":
-                if (hit._source.videoc === "gif") {
-                    hit._props.isGif = true;
-                } else {
-                    hit._props.isImage = true;
-                }
-                if ("width" in hit._source && !hit._props.isSubDocument && hit._source.videoc !== "tiff"
-                    && hit._source.videoc !== "raw" && hit._source.videoc !== "ppm") {
-                    hit._props.isPlayableImage = true;
-                }
-                if ("width" in hit._source && "height" in hit._source) {
-                    hit._props.imageAspectRatio = hit._source.width / hit._source.height;
-                }
-                break;
-            case "video":
-                if ("videoc" in hit._source) {
-                    hit._props.isVideo = true;
-                }
-                if (hit._props.isVideo) {
-                    const videoc = hit._source.videoc;
-                    const mime = hit._source.mime;
-
-                    hit._props.isPlayableVideo = mime != null &&
-                        mime.startsWith("video/") &&
-                        !hit._props.isSubDocument &&
-                        hit._source.extension !== "mkv" &&
-                        hit._source.extension !== "avi" &&
-                        hit._source.extension !== "mov" &&
-                        videoc !== "hevc" &&
-                        videoc !== "mpeg1video" &&
-                        videoc !== "mpeg2video" &&
-                        videoc !== "wmv3";
-                }
-                break;
-            case "audio":
-                if ("audioc" in hit._source && !hit._props.isSubDocument) {
-                    hit._props.isAudio = true;
-                }
-                break;
-        }
-    }
-
-    setHitTags(hit) {
-        const tags = [];
-
-        // User tags
-        if ("tag" in hit._source) {
-            hit._source.tag.forEach(tag => {
-                tags.push(this.createUserTag(tag));
-            })
-        }
-
-        hit._tags = tags;
-    }
-
-    createUserTag(tag) {
-        const tokens = tag.split(".");
-
-        const colorToken = tokens.pop();
-
-        const bg = colorToken;
-        const fg = lum(colorToken) > 50 ? "#000" : "#fff";
-
-        return {
-            style: "user",
-            fg: fg,
-            bg: bg,
-            text: tokens.join("."),
-            rawText: tag,
-            userTag: true,
-        };
-    }
-
-    search() {
-        if (this.backend() === "sqlite") {
-            return this.ftsQuery(this.queryfunc())
-        } else {
-            return this.esQuery(this.queryfunc());
-        }
-    }
-
-    esQuery(query) {
-        return axios.post(`${this.baseUrl}es`, query).then(resp => {
-            const res = resp.data;
-
-            if (res.hits?.hits) {
-                res.hits.hits.forEach((hit) => {
-                    hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
-                    hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
-
-                    this.setHitProps(hit);
-                    this.setHitTags(hit);
-                });
-            }
-
-            return res;
-        });
-    }
-
-    ftsQuery(query) {
-        return axios.post(`${this.baseUrl}fts/search`, query).then(resp => {
-            const res = resp.data;
-
-            if (res.hits.hits) {
-                res.hits.hits.forEach(hit => {
-                    hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
-                    hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
-
-                    this.setHitProps(hit);
-                    this.setHitTags(hit);
-
-                    if ("highlight" in hit) {
-                        hit["highlight"]["name"] = [hit["highlight"]["name"]];
-                        hit["highlight"]["content"] = [hit["highlight"]["content"]];
-                    }
-                });
-            }
-
-            return res;
-        });
-    }
-
-    getMimeTypesEs(query) {
-        const AGGS = {
-            mimeTypes: {
-                terms: {
-                    field: "mime",
-                    size: 10000
-                }
-            }
-        };
-
-        if (!query) {
-            query = {
-                aggs: AGGS,
-                size: 0,
-            };
-        } else {
-            query.size = 0;
-            query.aggs = AGGS;
-        }
-
-        return this.esQuery(query).then(resp => {
-            return resp["aggregations"]["mimeTypes"]["buckets"].map(bucket => ({
-                mime: bucket.key,
-                count: bucket.doc_count
-            }));
-
-        });
-    }
-
-    getMimeTypesSqlite() {
-        return axios.get(`${this.baseUrl}fts/mimetypes`)
-            .then(resp => {
-                return resp.data;
-            });
-    }
-
-    async getMimeTypes(query = undefined) {
-        let buckets;
-
-        if (this.backend() === "sqlite") {
-            buckets = await this.getMimeTypesSqlite();
-        } else {
-            buckets = await this.getMimeTypesEs(query);
-        }
-
-        const mimeMap = [];
-
-        buckets.sort((a, b) => a.mime > b.mime).forEach((bucket) => {
-            const tmp = bucket.mime.split("/");
-            const category = tmp[0];
-            const mime = tmp[1];
-
-            let category_exists = false;
-
-            const child = {
-                "id": bucket.mime,
-                "text": `${mime} (${bucket.count})`
-            };
-
-            mimeMap.forEach(node => {
-                if (node.text === category) {
-                    node.children.push(child);
-                    category_exists = true;
-                }
-            });
-
-            if (!category_exists) {
-                mimeMap.push({text: category, children: [child], id: category});
-            }
-        })
-
-        mimeMap.forEach(node => {
-            if (node.children) {
-                node.children.sort((a, b) => a.id.localeCompare(b.id));
-            }
-        })
-        mimeMap.sort((a, b) => a.id.localeCompare(b.id))
-
-        return {buckets, mimeMap};
-    }
-
-    _createEsTag(tag, count) {
-        const tokens = tag.split(".");
-
-        if (/.*\.#[0-9a-fA-F]{6}/.test(tag)) {
-            return {
-                id: tokens.slice(0, -1).join("."),
-                color: tokens.pop(),
-                isLeaf: true,
-                count: count
-            };
-        }
-
-        return {
-            id: tag,
-            count: count,
-            isLeaf: false,
-            color: undefined
-        };
-    }
-
-    getTagsEs() {
-        return this.esQuery({
-            aggs: {
-                tags: {
-                    terms: {
-                        field: "tag",
-                        size: 65535
-                    }
-                }
-            },
-            size: 0,
-        }).then(resp => {
-            return resp["aggregations"]["tags"]["buckets"]
-                .sort((a, b) => a["key"].localeCompare(b["key"]))
-                .map((bucket) => this._createEsTag(bucket["key"], bucket["doc_count"]));
-        });
-    }
-
-    getTagsSqlite() {
-        return axios.get(`${this.baseUrl}/fts/tags`)
-            .then(resp => {
-                return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
-            });
-    }
-
-    async getTags() {
-        let tags;
-        if (this.backend() === "sqlite") {
-            tags = await this.getTagsSqlite();
-        } else {
-            tags = await this.getTagsEs();
-        }
-
-        // Remove duplicates (same tag with different color)
-        const seen = new Set();
-
-        return tags.filter((t) => {
-            if (seen.has(t.id)) {
-                return false;
-            }
-            seen.add(t.id);
-            return true;
-        });
-    }
-
-    saveTag(tag, hit) {
-        return axios.post(`${this.baseUrl}tag/${sid(hit)}`, {
-            delete: false,
-            name: tag,
-        });
-    }
-
-    deleteTag(tag, hit) {
-        return axios.post(`${this.baseUrl}tag/${sid(hit)}`, {
-            delete: true,
-            name: tag,
-        });
-    }
-
-    searchPaths(indexId, minDepth, maxDepth, prefix = null) {
-        if (this.backend() === "sqlite") {
-            return this.searchPathsSqlite(indexId, minDepth, minDepth, prefix);
-        } else {
-            return this.searchPathsEs(indexId, minDepth, maxDepth, prefix);
-        }
-    }
-
-    searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
-        return axios.post(`${this.baseUrl}fts/paths`, {
-            indexId, minDepth, maxDepth, prefix
-        }).then(resp => {
-            return resp.data;
-        });
-    }
-
-    searchPathsEs(indexId, minDepth, maxDepth, prefix) {
-
-        const query = {
-            query: {
-                bool: {
-                    filter: [
-                        {term: {index: indexId}},
-                        {range: {_depth: {gte: minDepth, lte: maxDepth}}},
-                    ]
-                }
-            },
-            aggs: {
-                paths: {
-                    terms: {
-                        field: "path",
-                        size: 10000
-                    }
-                }
-            },
-            size: 0
-        };
-
-        if (prefix != null) {
-            query["query"]["bool"]["must"] = {
-                prefix: {
-                    path: prefix,
-                }
-            };
-        }
-
-        return this.esQuery(query).then(resp => {
-            const buckets = resp["aggregations"]["paths"]["buckets"];
-
-            if (!buckets) {
-                return [];
-            }
-
-            return buckets
-                .map(bucket => ({
-                    path: bucket.key,
-                    count: bucket.doc_count
-                }));
-        });
-    }
-
-    getDateRangeSqlite() {
-        return axios.get(`${this.baseUrl}fts/dateRange`)
-            .then(resp => ({
-                min: resp.data.dateMin,
-                max: (resp.data.dateMax === resp.data.dateMin)
-                    ? resp.data.dateMax + 1
-                    : resp.data.dateMax,
-            }));
-    }
-
-    getDateRange() {
-        if (this.backend() === "sqlite") {
-            return this.getDateRangeSqlite();
-        } else {
-            return this.getDateRangeEs();
-        }
-    }
-
-    getDateRangeEs() {
-        return this.esQuery({
-            // TODO: filter current selected indices
-            aggs: {
-                dateMin: {min: {field: "mtime"}},
-                dateMax: {max: {field: "mtime"}},
-            },
-            size: 0
-        }).then(res => {
-            const range = {
-                min: res.aggregations.dateMin.value / 1000,
-                max: res.aggregations.dateMax.value / 1000,
-            }
-
-            if (range.min == null) {
-                range.min = 0;
-                range.max = 1;
-            } else if (range.min === range.max) {
-                range.max += 1;
-            }
-
-            return range;
-        });
-    }
-
-    getPathSuggestionsSqlite(text) {
-        return axios.post(`${this.baseUrl}fts/paths`, {
-            prefix: text,
-            minDepth: 1,
-            maxDepth: 10000
-        }).then(resp => {
-            return resp.data.map(bucket => bucket.path);
-        })
-    }
-
-    getPathSuggestionsEs(text) {
-        return this.esQuery({
-            suggest: {
-                path: {
-                    prefix: text,
-                    completion: {
-                        field: "suggest-path",
-                        skip_duplicates: true,
-                        size: 10000
-                    }
-                }
-            }
-        }).then(resp => {
-            return resp["suggest"]["path"][0]["options"]
-                .map(opt => opt["_source"]["path"]);
-        });
-    }
-
-    getPathSuggestions(text) {
-        if (this.backend() === "sqlite") {
-            return this.getPathSuggestionsSqlite(text);
-        } else {
-            return this.getPathSuggestionsEs(text)
-        }
-    }
-
-    getTreemapStat(indexId) {
-        return `${this.baseUrl}s/${indexId}/TMAP`;
-    }
-
-    getMimeStat(indexId) {
-        return `${this.baseUrl}s/${indexId}/MAGG`;
-    }
-
-    getSizeStat(indexId) {
-        return `${this.baseUrl}s/${indexId}/SAGG`;
-    }
-
-    getDateStat(indexId) {
-        return `${this.baseUrl}s/${indexId}/DAGG`;
-    }
-
-    getDocumentEs(sid, highlight, fuzzy) {
-        const query = Sist2Query.searchQuery();
-
-        if (highlight) {
-            const fields = fuzzy
-                ? {"content.nGram": {}}
-                : {content: {}};
-
-            query.highlight = {
-                pre_tags: ["<mark>"],
-                post_tags: ["</mark>"],
-                number_of_fragments: 0,
-                fields,
-            };
-
-            if (!store.state.sist2Info.esVersionLegacy) {
-                query.highlight.max_analyzed_offset = 999_999;
-            }
-        }
-
-        if ("knn" in query) {
-            query.query = {
-                bool: {
-                    must: []
-                }
-            };
-            delete query.knn;
-        }
-
-        if ("function_score" in query.query) {
-            query.query = query.query.function_score.query;
-        }
-
-        if (!("must" in query.query.bool)) {
-            query.query.bool.must = [];
-        } else if (!Array.isArray(query.query.bool.must)) {
-            query.query.bool.must = [query.query.bool.must];
-        }
-
-        query.query.bool.must.push({match: {_id: sid}});
-
-        delete query["sort"];
-        delete query["aggs"];
-        delete query["search_after"];
-        delete query.query["function_score"];
-
-        query._source = {
-            includes: ["content", "name", "path", "extension"]
-        }
-
-        query.size = 1;
-
-        return this.esQuery(query).then(resp => {
-            if (resp.hits.hits.length === 1) {
-                return resp.hits.hits[0];
-            }
-            return null;
-        });
-    }
-
-    getDocumentSqlite(sid) {
-        return axios.get(`${this.baseUrl}/fts/d/${sid}`)
-            .then(resp => ({
-                _source: resp.data
-            }));
-    }
-
-    getDocument(sid, highlight, fuzzy) {
-        if (this.backend() === "sqlite") {
-            return this.getDocumentSqlite(sid);
-        } else {
-            return this.getDocumentEs(sid, highlight, fuzzy);
-        }
-    }
-
-    getTagSuggestions(prefix) {
-        if (this.backend() === "sqlite") {
-            return this.getTagSuggestionsSqlite(prefix);
-        } else {
-            return this.getTagSuggestionsEs(prefix);
-        }
-    }
-
-    getTagSuggestionsSqlite(prefix) {
-        return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
-            .then(resp => (resp.data));
-    }
-
-    getTagSuggestionsEs(prefix) {
-        return this.esQuery({
-            suggest: {
-                tag: {
-                    prefix: prefix,
-                    completion: {
-                        field: "suggest-tag",
-                        skip_duplicates: true,
-                        size: 10000
-                    }
-                }
-            }
-        }).then(resp => {
-            const result = [];
-            resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
-                tags.forEach(tag => {
-                    const t = tag.slice(0, -8);
-                    if (!result.find(x => x.slice(0, -8) === t)) {
-                        result.push(tag);
-                    }
-                });
-            });
-            return result;
-        });
-    }
-
-    getEmbeddings(sid, modelId) {
-        return axios.post(`${this.baseUrl}/e/${sid}/${modelId.toString().padStart(3, '0')}`)
-            .then(resp => (resp.data));
-    }
-}
-
-export default new Sist2Api("");

sist2-vue/src/Sist2Api.ts

@@ -0,0 +1,381 @@
+import axios from "axios";
+import {ext, strUnescape, lum} from "./util";
+
+export interface EsTag {
+    id: string
+    count: number
+    color: string | undefined
+    isLeaf: boolean
+}
+
+export interface Tag {
+    style: string
+    text: string
+    rawText: string
+    fg: string
+    bg: string
+    userTag: boolean
+}
+
+export interface Index {
+    name: string
+    version: string
+    id: string
+    idPrefix: string
+    timestamp: number
+}
+
+export interface EsHit {
+    _index: string
+    _id: string
+    _score: number
+    _type: string
+    _tags: Tag[]
+    _seq: number
+    _source: {
+        path: string
+        size: number
+        mime: string
+        name: string
+        extension: string
+        index: string
+        _depth: number
+        mtime: number
+        videoc: string
+        audioc: string
+        parent: string
+        width: number
+        height: number
+        duration: number
+        tag: string[]
+        checksum: string
+        thumbnail: string
+    }
+    _props: {
+        isSubDocument: boolean
+        isImage: boolean
+        isGif: boolean
+        isVideo: boolean
+        isPlayableVideo: boolean
+        isPlayableImage: boolean
+        isAudio: boolean
+        hasThumbnail: boolean
+        hasVidPreview: boolean
+        imageAspectRatio: number
+        /** Number of thumbnails available */
+        tnNum: number
+    }
+    highlight: {
+        name: string[] | undefined,
+        content: string[] | undefined,
+    }
+}
+
+function getIdPrefix(indices: Index[], id: string): string {
+    for (let i = 4; i < 32; i++) {
+        const prefix = id.slice(0, i);
+
+        if (indices.filter(idx => idx.id.slice(0, i) == prefix).length == 1) {
+            return prefix;
+        }
+    }
+
+    return id;
+}
+
+export interface EsResult {
+    took: number
+
+    hits: {
+        // TODO: ES 6.X ?
+        total: {
+            value: number
+        }
+        hits: EsHit[]
+    }
+
+    aggregations: any
+}
+
+class Sist2Api {
+
+    private baseUrl: string
+
+    constructor(baseUrl: string) {
+        this.baseUrl = baseUrl;
+    }
+
+    getSist2Info(): Promise<any> {
+        return axios.get(`${this.baseUrl}i`).then(resp => {
+            const indices = resp.data.indices as Index[];
+
+            resp.data.indices = indices.map(idx => {
+                return {
+                    id: idx.id,
+                    name: idx.name,
+                    timestamp: idx.timestamp,
+                    version: idx.version,
+                    idPrefix: getIdPrefix(indices, idx.id)
+                } as Index;
+            });
+
+            return resp.data;
+        })
+    }
+
+    setHitProps(hit: EsHit): void {
+        hit["_props"] = {} as any;
+
+        const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
+
+        if ("parent" in hit._source) {
+            hit._props.isSubDocument = true;
+        }
+
+        if ("thumbnail" in hit._source) {
+            hit._props.hasThumbnail = true;
+
+            if (Number.isNaN(Number(hit._source.thumbnail))) {
+                // Backwards compatibility
+                hit._props.tnNum = 1;
+                hit._props.hasVidPreview = false;
+            } else {
+                hit._props.tnNum = Number(hit._source.thumbnail);
+                hit._props.hasVidPreview = hit._props.tnNum > 1;
+            }
+        }
+
+        switch (mimeCategory) {
+            case "image":
+                if (hit._source.videoc === "gif") {
+                    hit._props.isGif = true;
+                } else {
+                    hit._props.isImage = true;
+                }
+                if ("width" in hit._source && !hit._props.isSubDocument && hit._source.videoc !== "tiff"
+                    && hit._source.videoc !== "raw" && hit._source.videoc !== "ppm") {
+                    hit._props.isPlayableImage = true;
+                }
+                if ("width" in hit._source && "height" in hit._source) {
+                    hit._props.imageAspectRatio = hit._source.width / hit._source.height;
+                }
+                break;
+            case "video":
+                if ("videoc" in hit._source) {
+                    hit._props.isVideo = true;
+                }
+                if (hit._props.isVideo) {
+                    const videoc = hit._source.videoc;
+                    const mime = hit._source.mime;
+
+                    hit._props.isPlayableVideo = mime != null &&
+                        mime.startsWith("video/") &&
+                        !hit._props.isSubDocument &&
+                        hit._source.extension !== "mkv" &&
+                        hit._source.extension !== "avi" &&
+                        hit._source.extension !== "mov" &&
+                        videoc !== "hevc" &&
+                        videoc !== "mpeg1video" &&
+                        videoc !== "mpeg2video" &&
+                        videoc !== "wmv3";
+                }
+                break;
+            case "audio":
+                if ("audioc" in hit._source && !hit._props.isSubDocument) {
+                    hit._props.isAudio = true;
+                }
+                break;
+        }
+    }
+
+    setHitTags(hit: EsHit): void {
+        const tags = [] as Tag[];
+
+        // User tags
+        if ("tag" in hit._source) {
+            hit._source.tag.forEach(tag => {
+                tags.push(this.createUserTag(tag));
+            })
+        }
+
+        hit._tags = tags;
+    }
+
+    createUserTag(tag: string): Tag {
+        const tokens = tag.split(".");
+
+        const colorToken = tokens.pop() as string;
+
+        const bg = colorToken;
+        const fg = lum(colorToken) > 50 ? "#000" : "#fff";
+
+        return {
+            style: "user",
+            fg: fg,
+            bg: bg,
+            text: tokens.join("."),
+            rawText: tag,
+            userTag: true,
+        } as Tag;
+    }
+
+    esQuery(query: any): Promise<EsResult> {
+        return axios.post(`${this.baseUrl}es`, query).then(resp => {
+            const res = resp.data as EsResult;
+
+            if (res.hits?.hits) {
+                res.hits.hits.forEach((hit: EsHit) => {
+                    hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
+                    hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+
+                    this.setHitProps(hit);
+                    this.setHitTags(hit);
+                });
+            }
+
+            return res;
+        });
+    }
+
+    getMimeTypes(query = undefined) {
+        const AGGS = {
+            mimeTypes: {
+                terms: {
+                    field: "mime",
+                    size: 10000
+                }
+            }
+        };
+
+        if (!query) {
+            query = {
+                aggs: AGGS,
+                size: 0,
+            };
+        } else {
+            query.size = 0;
+            query.aggs = AGGS;
+        }
+
+        return this.esQuery(query).then(resp => {
+            const mimeMap: any[] = [];
+            const buckets = resp["aggregations"]["mimeTypes"]["buckets"];
+
+            buckets.sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
+                const tmp = bucket["key"].split("/");
+                const category = tmp[0];
+                const mime = tmp[1];
+
+                let category_exists = false;
+
+                const child = {
+                    "id": bucket["key"],
+                    "text": `${mime} (${bucket["doc_count"]})`
+                };
+
+                mimeMap.forEach(node => {
+                    if (node.text === category) {
+                        node.children.push(child);
+                        category_exists = true;
+                    }
+                });
+
+                if (!category_exists) {
+                    mimeMap.push({text: category, children: [child], id: category});
+                }
+            })
+
+            mimeMap.forEach(node => {
+                if (node.children) {
+                    node.children.sort((a, b) => a.id.localeCompare(b.id));
+                }
+            })
+            mimeMap.sort((a, b) => a.id.localeCompare(b.id))
+
+            return {buckets, mimeMap};
+        });
+    }
+
+    _createEsTag(tag: string, count: number): EsTag {
+        const tokens = tag.split(".");
+
+        if (/.*\.#[0-9a-f]{6}/.test(tag)) {
+            return {
+                id: tokens.slice(0, -1).join("."),
+                color: tokens.pop(),
+                isLeaf: true,
+                count: count
+            };
+        }
+
+        return {
+            id: tag,
+            count: count,
+            isLeaf: false,
+            color: undefined
+        };
+    }
+
+    getTags() {
+        return this.esQuery({
+            aggs: {
+                tags: {
+                    terms: {
+                        field: "tag",
+                        size: 10000
+                    }
+                }
+            },
+            size: 0,
+        }).then(resp => {
+            const seen = new Set();
+
+            const tags = resp["aggregations"]["tags"]["buckets"]
+                .sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
+                .map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
+
+            // Remove duplicates (same tag with different color)
+            return tags.filter((t: EsTag) => {
+                if (seen.has(t.id)) {
+                    return false;
+                }
+                seen.add(t.id);
+                return true;
+            });
+        });
+    }
+
+    saveTag(tag: string, hit: EsHit) {
+        return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
+            delete: false,
+            name: tag,
+            doc_id: hit["_id"]
+        });
+    }
+
+    deleteTag(tag: string, hit: EsHit) {
+        return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
+            delete: true,
+            name: tag,
+            doc_id: hit["_id"]
+        });
+    }
+
+    getTreemapCsvUrl(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/1`;
+    }
+
+    getMimeCsvUrl(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/2`;
+    }
+
+    getSizeCsv(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/3`;
+    }
+
+    getDateCsv(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/4`;
+    }
+}
+
+export default new Sist2Api("");

sist2-vue/src/Sist2Query.ts

@@ -1,5 +1,5 @@
-import store from "@/store";
+import store from "./store";
-import Sist2Api from "@/Sist2Api";
+import {EsHit, Index} from "@/Sist2Api";
 
 const SORT_MODES = {
     score: {
@@ -7,62 +7,69 @@ const SORT_MODES = {
             {_score: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit) => hit._score
+        key: (hit: EsHit) => hit._score
     },
     random: {
         mode: [
             {_score: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit) => hit._score
+        key: (hit: EsHit) => hit._score
     },
     dateAsc: {
         mode: [
             {mtime: {order: "asc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit) => hit._source.mtime
+        key: (hit: EsHit) => hit._source.mtime
     },
     dateDesc: {
         mode: [
             {mtime: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit) => hit._source.mtime
+        key: (hit: EsHit) => hit._source.mtime
     },
     sizeAsc: {
         mode: [
             {size: {order: "asc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit) => hit._source.size
+        key: (hit: EsHit) => hit._source.size
     },
     sizeDesc: {
         mode: [
             {size: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit) => hit._source.size
+        key: (hit: EsHit) => hit._source.size
     },
     nameAsc: {
         mode: [
             {name: {order: "asc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit) => hit._source.name
+        key: (hit: EsHit) => hit._source.name
     },
     nameDesc: {
         mode: [
             {name: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit) => hit._source.name
+        key: (hit: EsHit) => hit._source.name
     }
-};
+} as any;
 
-class Sist2ElasticsearchQuery {
+interface SortMode {
+    text: string
+    mode: any[]
+    key: (hit: EsHit) => any
+}
 
-    searchQuery(blankSearch = false) {
+
+class Sist2Query {
+
+    searchQuery(blankSearch: boolean = false): any {
 
         const getters = store.getters;
 
@@ -76,17 +83,15 @@ class Sist2ElasticsearchQuery {
         const fuzzy = getters.fuzzy;
         const size = getters.size;
         const after = getters.lastDoc;
-        const selectedIndexIds = getters.selectedIndices.map((idx) => idx.id)
+        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
         const selectedMimeTypes = getters.selectedMimeTypes;
         const selectedTags = getters.selectedTags;
-        const sortMode = getters.embedding ? "score" : getters.sortMode;
 
         const legacyES = store.state.sist2Info.esVersionLegacy;
-        const hasKnn = store.state.sist2Info.esVersionHasKnn;
 
         const filters = [
             {terms: {index: selectedIndexIds}}
-        ];
+        ] as any[];
 
         const fields = [
             "name^8",
@@ -138,7 +143,7 @@ class Sist2ElasticsearchQuery {
                 if (getters.optTagOrOperator) {
                     filters.push({terms: {"tag": selectedTags}});
                 } else {
-                    selectedTags.forEach((tag) => filters.push({term: {"tag": tag}}));
+                    selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
                 }
             }
         }
@@ -164,76 +169,31 @@ class Sist2ElasticsearchQuery {
 
         const q = {
             _source: {
-                excludes: ["content", "_tie", "emb.*"]
+                excludes: ["content", "_tie"]
             },
             query: {
                 bool: {
                     filter: filters,
                 }
             },
-            sort: SORT_MODES[sortMode].mode,
+            sort: SORT_MODES[getters.sortMode].mode,
-            size: size,
+            aggs:
-        };
+                {
-
-        if (!after) {
-            q.aggs = {
                     total_size: {"sum": {"field": "size"}},
                     total_count: {"value_count": {"field": "size"}}
-            };
+                },
-        }
+            size: size,
+        } as any;
 
         if (!empty && !blankSearch) {
-            if (getters.embedding) {
-                filters.push(query)
-            } else {
             q.query.bool.must = query;
         }
-        }
-
-        if (getters.embedding) {
-            delete q.query;
-
-            const field = "emb." + Sist2Api.models().find(m => m.id === getters.embeddingsModel).path;
-
-            if (hasKnn) {
-                // Use knn (8.8+)
-                q.knn = {
-                    field: field,
-                    query_vector: getters.embedding,
-
-                    k: 600,
-                    num_candidates: 600,
-
-                    filter: filters
-                }
-            } else {
-                // Use brute-force as a fallback
-
-                filters.push({exists: {field: field}});
-
-                q.query = {
-                    function_score: {
-                        query: {
-                            bool: {
-                                must: filters,
-                            }
-                        },
-                        script_score: {
-                            script: {
-                                source: `cosineSimilarity(params.query_vector, "${field}") + 1.0`,
-                                params: {query_vector: getters.embedding}
-                            }
-                        }
-                    }
-                }
-            }
-        }
 
         if (after) {
-            q.search_after = [SORT_MODES[sortMode].key(after), after["_id"]];
+            q.search_after = [SORT_MODES[getters.sortMode].key(after), after["_id"]];
         }
 
-        if (getters.optHighlight && !getters.embedding) {
+        if (getters.optHighlight) {
             q.highlight = {
                 pre_tags: ["<mark>"],
                 post_tags: ["</mark>"],
@@ -259,7 +219,7 @@ class Sist2ElasticsearchQuery {
             }
         }
 
-        if (sortMode === "random") {
+        if (getters.sortMode === "random") {
             q.query = {
                 function_score: {
                     query: {
@@ -289,5 +249,4 @@ class Sist2ElasticsearchQuery {
     }
 }
 
-
+export default new Sist2Query();
-export default new Sist2ElasticsearchQuery();

sist2-vue/src/Sist2SqliteQuery.js

@@ -1,114 +0,0 @@
-import store from "@/store";
-
-const SORT_MODES = {
-    score: {
-        "sort": "score",
-    },
-    random: {
-        "sort": "random"
-    },
-    dateAsc: {
-        "sort": "mtime"
-    },
-    dateDesc: {
-        "sort": "mtime",
-        "sortAsc": false
-    },
-    sizeAsc: {
-        "sort": "size",
-    },
-    sizeDesc: {
-        "sort": "size",
-        "sortAsc": false
-    },
-    nameAsc: {
-        "sort": "name",
-    },
-    nameDesc: {
-        "sort": "name",
-        "sortAsc": false
-    }
-};
-
-
-class Sist2ElasticsearchQuery {
-
-    searchQuery() {
-
-        const getters = store.getters;
-
-        const searchText = getters.searchText;
-        const pathText = getters.pathText;
-        const sizeMin = getters.sizeMin;
-        const sizeMax = getters.sizeMax;
-        const dateMin = getters.dateMin;
-        const dateMax = getters.dateMax;
-        const size = getters.size;
-        const after = getters.lastDoc;
-        const selectedIndexIds = getters.selectedIndices.map((idx) => idx.id)
-        const selectedMimeTypes = getters.selectedMimeTypes;
-        const selectedTags = getters.selectedTags;
-
-        const q = {
-            "pageSize": size
-        }
-
-        Object.assign(q, SORT_MODES[getters.sortMode]);
-
-        if (!after) {
-            q["fetchAggregations"] = true;
-        }
-        if (searchText) {
-            q["query"] = searchText;
-        }
-        if (pathText) {
-            q["path"] = pathText.endsWith("/") ? pathText.slice(0, -1) : pathText;
-        }
-        if (sizeMin) {
-            q["sizeMin"] = sizeMin;
-        }
-        if (sizeMax) {
-            q["sizeMax"] = sizeMax;
-        }
-        if (dateMin) {
-            q["dateMin"] = dateMin;
-        }
-        if (dateMax) {
-            q["dateMax"] = dateMax;
-        }
-        if (after) {
-            q["after"] = after.sort;
-        }
-        if (selectedIndexIds.length > 0) {
-            q["indexIds"] = selectedIndexIds;
-        }
-        if (selectedMimeTypes.length > 0) {
-            q["mimeTypes"] = selectedMimeTypes;
-        }
-        if (selectedTags.length > 0) {
-            q["tags"] = selectedTags
-        }
-        if (getters.sortMode === "random") {
-            q["seed"] = getters.seed;
-        }
-        if (getters.optHighlight) {
-            q["highlight"] = true;
-            q["highlightContextSize"] = Number(getters.optFragmentSize);
-        }
-
-        if (getters.embedding) {
-            q["model"] = getters.embeddingsModel;
-            q["embedding"] = getters.embedding;
-            q["sort"] = "embedding";
-            q["sortAsc"] = false;
-        } else if (getters.sortMode === "embedding") {
-            q["sort"] = "sort"
-            q["sortAsc"] = true;
-        }
-
-        return q;
-    }
-}
-
-
-export default new Sist2ElasticsearchQuery();

sist2-vue/src/components/AnalyzedContentSpan.vue

@@ -1,21 +0,0 @@
-<template>
-    <span :style="getStyle()">{{span.text}}</span>
-</template>
-
-<script>
-
-
-import ModelsRepo from "@/ml/modelsRepo";
-
-export default {
-    name: "AnalyzedContentSpan",
-    props: ["span", "text"],
-    methods: {
-        getStyle() {
-            return ModelsRepo.data[this.$store.getters.nerModel.name].labelStyles[this.span.label];
-        }
-    }
-}
-</script>
-
-<style scoped></style>

sist2-vue/src/components/AnalyzedContentSpanContainer.vue

@@ -1,75 +0,0 @@
-<template>
-    <div>
-        <b-card class="mb-2">
-            <AnalyzedContentSpan v-for="span of legend" :key="span.id" :span="span"
-                                 class="mr-2"></AnalyzedContentSpan>
-        </b-card>
-        <div class="content-div">
-            <AnalyzedContentSpan v-for="span of mergedSpans" :key="span.id" :span="span"></AnalyzedContentSpan>
-        </div>
-    </div>
-</template>
-
-<script>
-
-
-import AnalyzedContentSpan from "@/components/AnalyzedContentSpan.vue";
-import ModelsRepo from "@/ml/modelsRepo";
-
-export default {
-    name: "AnalyzedContentSpanContainer",
-    components: {AnalyzedContentSpan},
-    props: ["spans", "text"],
-    computed: {
-        legend() {
-            return Object.entries(ModelsRepo.data[this.$store.state.nerModel.name].legend)
-                .map(([label, name]) => ({
-                    text: name,
-                    id: label,
-                    label: label
-                }));
-        },
-        mergedSpans() {
-            const spans = this.spans;
-
-            const merged = [];
-
-            let lastLabel = null;
-            let fixSpace = false;
-            for (let i = 0; i < spans.length; i++) {
-
-                if (spans[i].label !== lastLabel) {
-                    let start = spans[i].wordIndex;
-                    const nextSpan = spans.slice(i + 1).find(s => s.label !== spans[i].label)
-                    let end = nextSpan ? nextSpan.wordIndex : undefined;
-
-                    if (end !== undefined && this.text[end - 1] === " ") {
-                        end -= 1;
-                        fixSpace = true;
-                    }
-
-                    merged.push({
-                        text: this.text.slice(start, end),
-                        label: spans[i].label,
-                        id: spans[i].wordIndex
-                    });
-
-                    if (fixSpace) {
-                        merged.push({
-                            text: " ",
-                            label: "O",
-                            id: end
-                        });
-                        fixSpace = false;
-                    }
-                    lastLabel = spans[i].label;
-                }
-            }
-
-            return merged;
-        },
-    },
-}
-</script>
-
-<style scoped></style>

sist2-vue/src/components/D3DateHistogram.vue

@@ -120,7 +120,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#date-histogram");
 
-      d3.json(Sist2Api.getDateStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getDateCsv(indexId)).then(tabularData => {
         dateHistogram(tabularData.slice(), svg, this.$t("d3.dateHistogram"));
       });
     }

sist2-vue/src/components/D3MimeBarCount.vue

@@ -91,7 +91,7 @@ export default {
       const mimeSvgCount = d3.select("#agg-mime-count");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
         mimeBarCount(tabularData.slice(), mimeSvgCount, fillOpacity, this.$t("d3.mimeCount"));
       });
     }

sist2-vue/src/components/D3MimeBarSize.vue

@@ -90,7 +90,7 @@ export default {
       const mimeSvgSize = d3.select("#agg-mime-size");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
         mimeBarSize(tabularData.slice(), mimeSvgSize, fillOpacity, this.$t("d3.mimeSize"));
       });
     }

sist2-vue/src/components/D3SizeHistogram.vue

@@ -117,7 +117,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#size-histogram");
 
-      d3.json(Sist2Api.getSizeStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getSizeCsv(indexId)).then(tabularData => {
         sizeHistogram(tabularData.slice(), svg, this.$t("d3.sizeHistogram"));
       });
     }

sist2-vue/src/components/D3Treemap.vue

@@ -9,7 +9,8 @@
 
 <script>
 import * as d3 from "d3";
-import {humanFileSize, burrow} from "@/util";
+import {burrow} from "@/util-js"
+import {humanFileSize} from "@/util";
 import Sist2Api from "@/Sist2Api";
 import domtoimage from "dom-to-image";
 
@@ -239,7 +240,7 @@ export default {
           .style("overflow", "visible")
           .style("font", "10px sans-serif");
 
-            d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getTreemapCsvUrl(indexId)).then(tabularData => {
         tabularData.forEach(row => {
           row.taxonomy = row.path.split("/");
           row.size = Number(row.size);

sist2-vue/src/components/DateSlider.vue

@@ -31,7 +31,8 @@
 <script>
 import noUiSlider from 'nouislider';
 import 'nouislider/dist/nouislider.css';
-import {humanDate, mergeTooltips} from "@/util";
+import {humanDate} from "@/util";
+import {mergeTooltips} from "@/util-js";
 
 export default {
   name: "DateSlider",

sist2-vue/src/components/DebugInfo.vue

@@ -1,18 +1,14 @@
 <template>
-    <b-card v-if="$store.state.sist2Info.showDebugInfo" class="mb-4 mt-4">
+  <b-card class="mb-4 mt-4">
-        <b-card-title>
+    <b-card-title><DebugIcon class="mr-1"></DebugIcon>{{ $t("debug") }}</b-card-title>
-            <DebugIcon class="mr-1"></DebugIcon>
-            {{ $t("debug") }}
-        </b-card-title>
     <p v-html="$t('debugDescription')"></p>
 
     <b-card-body>
 
       <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
 
-            <hr/>
+      <hr />
-            <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx"
+      <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx" class="mt-2"></IndexDebugInfo>
-                            class="mt-2"></IndexDebugInfo>
     </b-card-body>
   </b-card>
 </template>
@@ -20,17 +16,13 @@
 <script>
 import IndexDebugInfo from "@/components/IndexDebugInfo";
 import DebugIcon from "@/components/icons/DebugIcon";
-import {mapGetters} from "vuex";
 
 export default {
   name: "DebugInfo.vue",
   components: {DebugIcon, IndexDebugInfo},
   computed: {
-        ...mapGetters([
-            "uiSqliteMode",
-        ]),
     tableItems() {
-            const items = [
+      return [
         {key: "version", value: this.$store.state.sist2Info.version},
         {key: "platform", value: this.$store.state.sist2Info.platform},
         {key: "debugBinary", value: this.$store.state.sist2Info.debug},
@@ -39,18 +31,10 @@ export default {
         {key: "tagline", value: this.$store.state.sist2Info.tagline},
         {key: "dev", value: this.$store.state.sist2Info.dev},
         {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
-            ];
-
-            if (!this.uiSqliteMode) {
-                items.push(
         {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
         {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
         {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
-                    {key: "esVersionHasKnn", value: this.$store.state.sist2Info.esVersionHasKnn},
+      ]
-                );
-            }
-
-            return items;
     }
   }
 }

sist2-vue/src/components/DocCard.vue

@@ -16,7 +16,7 @@
       <!-- Audio player-->
       <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
              :type="doc._source.mime"
-                   :src="`f/${sid(doc)}`"
+             :src="`f/${doc._id}`"
              @play="onAudioPlay()"></audio>
 
       <b-card-body class="padding-03">
@@ -24,7 +24,6 @@
         <!-- Title line -->
         <div style="display: flex">
           <span class="info-icon" @click="onInfoClick()"></span>
-                    <MLIcon v-if="doc._source.embedding" clickable @click="onEmbeddingClick()"></MLIcon>
           <DocFileTitle :doc="doc"></DocFileTitle>
         </div>
 
@@ -43,19 +42,17 @@
 </template>
 
 <script>
-import {ext, humanFileSize, humanTime, sid} from "@/util";
+import {ext, humanFileSize, humanTime} from "@/util";
 import TagContainer from "@/components/TagContainer.vue";
 import DocFileTitle from "@/components/DocFileTitle.vue";
 import DocInfoModal from "@/components/DocInfoModal.vue";
 import ContentDiv from "@/components/ContentDiv.vue";
 import FullThumbnail from "@/components/FullThumbnail";
 import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
-import MLIcon from "@/components/icons/MlIcon.vue";
-import Sist2Api from "@/Sist2Api";
 
 
 export default {
-    components: {MLIcon, FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+  components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
   props: ["doc", "width"],
   data() {
     return {
@@ -69,25 +66,17 @@ export default {
     }
   },
   methods: {
-        sid: sid,
     humanFileSize: humanFileSize,
     humanTime: humanTime,
     onInfoClick() {
       this.showInfo = true;
     },
-        onEmbeddingClick() {
-            Sist2Api.getEmbeddings(sid(this.doc), this.$store.state.embeddingsModel).then(embeddings => {
-                this.$store.commit("setEmbeddingText", "");
-                this.$store.commit("setEmbedding", embeddings);
-                this.$store.commit("setEmbeddingDoc", this.doc);
-            })
-        },
     async onThumbnailClick() {
       this.$store.commit("setUiLightboxSlide", this.doc._seq);
       await this.$store.dispatch("showLightbox");
     },
     onAudioPlay() {
-            Array.prototype.slice.call(document.getElementsByTagName("audio")).forEach((el) => {
+      document.getElementsByTagName("audio").forEach((el) => {
         if (el !== this.$refs["audio"]) {
           el.pause();
         }

sist2-vue/src/components/DocCardWall.vue

@@ -5,12 +5,11 @@
       @append="append"
       @layout-complete="$emit('layout-complete')"
   >
-        <DocCard v-for="doc in docs" :key="sid(doc)" :doc="doc" :width="width"></DocCard>
+    <DocCard v-for="doc in docs" :key="doc._id" :doc="doc" :width="width"></DocCard>
   </GridLayout>
 </template>
 
 <script>
-import {sid} from "@/util";
 import Vue from "vue";
 import DocCard from "@/components/DocCard";
 
@@ -40,9 +39,6 @@ export default Vue.extend({
       }
     }
   },
-    methods: {
-        sid,
-    },
   computed: {
     colCount() {
       const columns = this.$store.getters["optColumns"];

sist2-vue/src/components/DocFileTitle.vue

@@ -1,19 +1,17 @@
 <template>
-    <a :href="`f/${sid(doc)}`"
+  <a :href="`f/${doc._id}`" class="file-title-anchor" target="_blank">
-       :class="doc._source.embedding ? 'file-title-anchor-with-embedding' : 'file-title-anchor'" target="_blank">
     <div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
          v-html="fileName() + ext(doc)"></div>
   </a>
 </template>
 
 <script>
-import {ext, sid} from "@/util";
+import {ext} from "@/util";
 
 export default {
   name: "DocFileTitle",
   props: ["doc"],
   methods: {
-        sid: sid,
     ext: ext,
     fileName() {
       if (!this.doc.highlight) {
@@ -36,13 +34,8 @@ export default {
   max-width: calc(100% - 1.2rem);
 }
 
-.file-title-anchor-with-embedding {
-    max-width: calc(100% - 2.2rem);
-}
-
 .file-title {
   width: 100%;
-    max-width: 100%;
   line-height: 1rem;
   height: 1.1rem;
   white-space: nowrap;
@@ -56,7 +49,6 @@ export default {
 .theme-black .file-title {
   color: #ddd;
 }
-
 .theme-black .file-title:hover {
   color: #fff;
 }

sist2-vue/src/components/DocInfoModal.vue

@@ -8,16 +8,16 @@
       </h5>
     </template>
 
-        <img v-if="doc._props.hasThumbnail" :src="`t/${sid(doc)}`" alt="" class="fit card-img-top">
+    <img v-if="doc._props.hasThumbnail" :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">
 
     <InfoTable :doc="doc"></InfoTable>
 
-        <LazyContentDiv :sid="sid(doc)"></LazyContentDiv>
+    <LazyContentDiv :doc-id="doc._id"></LazyContentDiv>
   </b-modal>
 </template>
 
 <script>
-import {ext, sid} from "@/util";
+import {ext} from "@/util";
 import InfoTable from "@/components/InfoTable";
 import LazyContentDiv from "@/components/LazyContentDiv";
 
@@ -27,7 +27,6 @@ export default {
   props: ["doc", "show"],
   methods: {
     ext: ext,
-        sid: sid
   }
 }
 </script>

sist2-vue/src/components/DocList.vue

@@ -1,11 +1,10 @@
 <template>
   <b-list-group class="mt-3">
-        <DocListItem v-for="doc in docs" :key="sid(doc)" :doc="doc"></DocListItem>
+    <DocListItem v-for="doc in docs" :key="doc._id" :doc="doc"></DocListItem>
   </b-list-group>
 </template>
 
-<script>
+<script lang="ts">
-import {sid} from "@/util";
 import DocListItem from "@/components/DocListItem.vue";
 import Vue from "vue";
 
@@ -22,9 +21,6 @@ export default Vue.extend({
         this.append();
       }
     });
-    },
-    methods: {
-        sid: sid
   }
 });
 </script>

sist2-vue/src/components/DocListItem.vue

@@ -17,10 +17,10 @@
           </div>
 
           <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-                         :src="(doc._props.isGif && hover) ? `f/${sid(doc)}` : `t/${sid(doc)}`"
+               :src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
                alt=""
                class="pointer fit-sm" @click="onThumbnailClick()">
-                    <img v-else :src="`t/${sid(doc)}`" alt=""
+          <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
                class="fit-sm">
         </div>
       </div>
@@ -32,7 +32,6 @@
       <div class="doc-line ml-3">
         <div style="display: flex">
           <span class="info-icon" @click="showInfo = true"></span>
-                    <MLIcon v-if="doc._source.embedding" clickable @click="onEmbeddingClick()"></MLIcon>
           <DocFileTitle :doc="doc"></DocFileTitle>
         </div>
 
@@ -68,13 +67,10 @@ import DocInfoModal from "@/components/DocInfoModal";
 import ContentDiv from "@/components/ContentDiv";
 import FileIcon from "@/components/icons/FileIcon";
 import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
-import MLIcon from "@/components/icons/MlIcon.vue";
-import Sist2Api from "@/Sist2Api";
-import {sid} from "@/util";
 
 export default {
   name: "DocListItem",
-    components: {MLIcon, FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
+  components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
   props: ["doc"],
   data() {
     return {
@@ -83,18 +79,10 @@ export default {
     }
   },
   methods: {
-        sid: sid,
     async onThumbnailClick() {
       this.$store.commit("setUiLightboxSlide", this.doc._seq);
       await this.$store.dispatch("showLightbox");
     },
-        onEmbeddingClick() {
-            Sist2Api.getEmbeddings(sid(this.doc), this.$store.state.embeddingsModel).then(embeddings => {
-                this.$store.commit("setEmbeddingText", "");
-                this.$store.commit("setEmbedding", embeddings);
-                this.$store.commit("setEmbeddingDoc", this.doc);
-            })
-        },
     path() {
       if (!this.doc.highlight) {
         return this.doc._source.path + "/"

sist2-vue/src/components/EmbeddingsSearchBar.vue

@@ -1,155 +0,0 @@
-<template>
-    <div>
-        <b-progress v-if="modelLoading && [0, 1].includes(modelLoadingProgress)" max="1" class="mb-1" variant="primary"
-                    striped animated :value="1">
-        </b-progress>
-        <b-progress v-else-if="modelLoading" :value="modelLoadingProgress" max="1" class="mb-1" variant="warning"
-                    show-progress>
-        </b-progress>
-        <div style="display: flex">
-            <b-select :options="modelOptions()" class="mr-2 input-prepend" :value="modelName"
-                      @change="onModelChange($event)"></b-select>
-
-            <b-input-group>
-                <b-form-input :value="embeddingText"
-                              :placeholder="$store.state.embeddingDoc ? ' ' : $t('embeddingsSearchPlaceholder')"
-                              @input="onInput($event)"
-                              :disabled="modelLoading"
-                              :style="{'pointer-events': $store.state.embeddingDoc ? 'none' : undefined}"
-                ></b-form-input>
-                <b-badge v-if="$store.state.embeddingDoc" pill variant="primary" class="overlay-badge" href="#"
-                         @click="onBadgeClick()">{{ docName }}
-                </b-badge>
-
-                <template #prepend>
-                </template>
-
-                <template #append>
-                    <b-input-group-text>
-                        <MLIcon class="ml-append" big></MLIcon>
-                    </b-input-group-text>
-                </template>
-
-            </b-input-group>
-        </div>
-
-    </div>
-</template>
-
-<script>
-import {mapGetters, mapMutations} from "vuex";
-import {CLIPTransformerModel} from "@/ml/CLIPTransformerModel"
-import _debounce from "lodash/debounce";
-import MLIcon from "@/components/icons/MlIcon.vue";
-import Sist2AdminApi from "@/Sist2Api";
-
-export default {
-    components: {MLIcon},
-    data() {
-        return {
-            modelLoading: false,
-            modelLoadingProgress: 0,
-            modelLoaded: false,
-            model: null,
-            modelName: null
-        }
-    },
-    computed: {
-        ...mapGetters({
-            optQueryMode: "optQueryMode",
-            embeddingText: "embeddingText",
-            fuzzy: "fuzzy",
-        }),
-        docName() {
-            const ext = this.$store.state.embeddingDoc._source.extension;
-            return this.$store.state.embeddingDoc._source.name +
-                (ext ? "." + ext : "")
-        }
-    },
-    mounted() {
-        // Set default model
-        this.modelName = Sist2AdminApi.models()[0].name;
-        this.onModelChange(this.modelName);
-
-        this.onInput = _debounce(this._onInput, 450, {leading: false});
-    },
-    methods: {
-        ...mapMutations({
-            setEmbeddingText: "setEmbeddingText",
-            setEmbedding: "setEmbedding",
-            setEmbeddingModel: "setEmbeddingsModel",
-        }),
-        async loadModel() {
-            this.modelLoading = true;
-
-            await this.model.init(async progress => {
-                this.modelLoadingProgress = progress;
-            });
-            this.modelLoading = false;
-            this.modelLoaded = true;
-        },
-        async _onInput(text) {
-            try {
-
-                if (!this.modelLoaded) {
-                    await this.loadModel();
-                }
-
-                if (text.length === 0) {
-                    this.setEmbeddingText("");
-                    this.setEmbedding(null);
-                    return;
-                }
-
-                const embeddings = await this.model.predict(text);
-
-                this.setEmbeddingText(text);
-                this.setEmbedding(embeddings);
-            } catch (e) {
-                alert(e)
-            }
-        },
-        modelOptions() {
-            return Sist2AdminApi.models().map(model => model.name);
-        },
-        onModelChange(name) {
-            this.modelLoaded = false;
-            this.modelLoadingProgress = 0;
-
-            const modelInfo = Sist2AdminApi.models().find(m => m.name === name);
-
-            if (modelInfo.name === "CLIP") {
-                const tokenizerUrl = new URL("./tokenizer.json", modelInfo.url).href;
-                this.model = new CLIPTransformerModel(modelInfo.url, tokenizerUrl)
-                this.setEmbeddingModel(modelInfo.id);
-            } else {
-                throw new Error("Unknown model: " + name);
-            }
-        },
-        onBadgeClick() {
-            this.$store.commit("setEmbedding", null);
-            this.$store.commit("setEmbeddingDoc", null);
-        }
-    }
-}
-</script>
-<style>
-.overlay-badge {
-    position: absolute;
-    z-index: 1;
-    left: 0.375rem;
-    top: 8px;
-    line-height: 1.1rem;
-    overflow: hidden;
-    max-width: 200px;
-    text-overflow: ellipsis;
-}
-
-.input-prepend {
-    max-width: 100px;
-}
-
-.theme-black .ml-append {
-    filter: brightness(0.95) !important;
-}
-</style>

sist2-vue/src/components/FeaturedFieldsLine.vue

@@ -16,10 +16,6 @@ export default {
   props: ["doc"],
   computed: {
     featuredLineHtml() {
-            if (this.$store.getters.optFeaturedFields === undefined) {
-                return "";
-            }
-
       const scope = {doc: this.doc._source, humanDate: humanDate, humanFileSize: humanFileSize};
 
       return this.$store.getters.optFeaturedFields

sist2-vue/src/components/FullThumbnail.vue

@@ -40,7 +40,7 @@
 </template>
 
 <script>
-import {humanTime, sid} from "@/util";
+import {humanTime} from "@/util";
 import ThumbnailProgressBar from "@/components/ThumbnailProgressBar";
 
 export default {
@@ -67,16 +67,15 @@ export default {
     },
   },
   methods: {
-        sid: sid,
     getThumbnailSrc(thumbnailNum) {
       const doc = this.doc;
       const props = doc._props;
       if (props.isGif && this.hover) {
-                return `f/${sid(doc)}`;
+        return `f/${doc._id}`;
       }
       return (this.currentThumbnailNum === 0)
-                ? `t/${sid(doc)}`
+          ? `t/${doc._source.index}/${doc._id}`
-                : `t/${sid(doc)}/${String(thumbnailNum).padStart(4, "0")}`;
+          : `t/${doc._source.index}/${doc._id}/${String(thumbnailNum).padStart(4, "0")}`;
     },
     humanTime: humanTime,
     onThumbnailClick() {

sist2-vue/src/components/IndexPicker.vue

@@ -27,16 +27,11 @@
           @click.shift="shiftClick(idx, $event)"
           class="d-flex justify-content-between align-items-center list-group-item-action pointer"
           :class="{active: lastClickIndex === idx}"
-                    :key="idx.id"
       >
         <div class="d-flex">
           <b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
           {{ idx.name }}
-                    <div v-if="hasEmbeddings(idx)" style="vertical-align: center; margin-left: 5px">
+          <span class="text-muted timestamp-text ml-2">{{ formatIdxDate(idx.timestamp) }}</span>
-                        <MLIcon small style="top: -1px; position: relative"></MLIcon>
-                    </div>
-                    <span class="text-muted timestamp-text ml-2"
-                          style="top: 1px; position: relative">{{ formatIdxDate(idx.timestamp) }}</span>
         </div>
         <b-badge class="version-badge">v{{ idx.version }}</b-badge>
       </b-list-group-item>
@@ -44,16 +39,14 @@
   </div>
 </template>
 
-<script>
+<script lang="ts">
 import SmallBadge from "./SmallBadge.vue"
 import {mapActions, mapGetters} from "vuex";
 import Vue from "vue";
 import {format} from "date-fns";
-import MLIcon from "@/components/icons/MlIcon.vue";
 
 export default Vue.extend({
   components: {
-        MLIcon,
     SmallBadge
   },
   data() {
@@ -112,7 +105,7 @@ export default Vue.extend({
     onSelect(value) {
       this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
     },
-        formatIdxDate(timestamp) {
+    formatIdxDate(timestamp: number): string {
       return format(new Date(timestamp * 1000), "yyyy-MM-dd");
     },
     toggleIndex(index, e) {
@@ -122,17 +115,14 @@ export default Vue.extend({
 
       this.lastClickIndex = index;
       if (this.isSelected(index)) {
-                this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id !== index.id));
+        this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
       } else {
         this.setSelectedIndices([index, ...this.selectedIndices]);
       }
     },
     isSelected(index) {
-            return this.selectedIndices.find(idx => idx.id === index.id) != null;
+      return this.selectedIndices.find(idx => idx.id == index.id) != null;
-        },
+    }
-        hasEmbeddings(index) {
-            return index.models.length > 0;
-        },
   },
 })
 </script>
@@ -180,18 +170,18 @@ export default Vue.extend({
 }
 
 .theme-black .list-group-item {
-    border: 1px solid rgba(255, 255, 255, 0.1);
+  border: 1px solid rgba(255,255,255, 0.1);
 }
 
 .theme-black .list-group-item:first-child {
-    border: 1px solid rgba(255, 255, 255, 0.05);
+  border: 1px solid rgba(255,255,255, 0.05);
 }
 
 .theme-black .list-group-item.active {
   z-index: 2;
   background-color: inherit;
   color: inherit;
-    border: 1px solid rgba(255, 255, 255, 0.3);
+  border: 1px solid rgba(255,255,255, 0.3);
   border-radius: 0;
 }
 

sist2-vue/src/components/LazyContentDiv.vue

@@ -1,103 +1,80 @@
 <template>
   <Preloader v-if="loading"></Preloader>
-    <div v-else-if="content">
+  <div v-else-if="content" class="content-div" v-html="content"></div>
-        <b-form inline class="my-2" v-if="ModelsRepo.getOptions().length > 0">
-            <b-checkbox class="ml-auto mr-2" :checked="optAutoAnalyze"
-                        @input="setOptAutoAnalyze($event); $store.dispatch('updateConfiguration')">
-                {{ $t("ml.auto") }}
-            </b-checkbox>
-            <b-button :disabled="mlPredictionsLoading || mlLoading" @click="mlAnalyze" variant="primary"
-            >{{ $t("ml.analyzeText") }}
-            </b-button>
-            <b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="nerModel">
-                <b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()" :key="opt.value">{{ opt.text }}
-                </b-select-option>
-            </b-select>
-        </b-form>
-
-        <b-progress v-if="mlLoading" variant="warning" show-progress :max="1" class="mb-3"
-        >
-            <b-progress-bar :value="modelLoadingProgress">
-                <strong>{{ ((modelLoadingProgress * modelSize) / (1024 * 1024)).toFixed(1) }}MB / {{
-                    (modelSize / (1024 * 1024)).toFixed(1)
-                    }}MB</strong>
-            </b-progress-bar>
-        </b-progress>
-
-        <b-progress v-if="mlPredictionsLoading" variant="primary" :value="modelPredictionProgress"
-                    :max="content.length" class="mb-3"></b-progress>
-
-        <AnalyzedContentSpansContainer v-if="analyzedContentSpans.length > 0"
-                                       :spans="analyzedContentSpans" :text="rawContent"></AnalyzedContentSpansContainer>
-        <div v-else class="content-div" v-html="content"></div>
-    </div>
 </template>
 
 <script>
 import Sist2Api from "@/Sist2Api";
 import Preloader from "@/components/Preloader";
-import Sist2Query from "@/Sist2ElasticsearchQuery";
+import Sist2Query from "@/Sist2Query";
 import store from "@/store";
-import BertNerModel from "@/ml/BertNerModel";
-import AnalyzedContentSpansContainer from "@/components/AnalyzedContentSpanContainer.vue";
-import ModelsRepo from "@/ml/modelsRepo";
-import {mapGetters, mapMutations} from "vuex";
 
 export default {
   name: "LazyContentDiv",
-    components: {AnalyzedContentSpansContainer, Preloader},
+  components: {Preloader},
-    props: ["sid"],
+  props: ["docId"],
   data() {
     return {
-            ModelsRepo,
       content: "",
-            rawContent: "",
+      loading: true
-            loading: true,
-            modelLoadingProgress: 0,
-            modelPredictionProgress: 0,
-            mlPredictionsLoading: false,
-            mlLoading: false,
-            nerModel: null,
-            analyzedContentSpans: []
     }
   },
   mounted() {
+    const query = Sist2Query.searchQuery();
 
-        if (this.$store.getters.optMlDefaultModel) {
+    if (this.$store.state.optHighlight) {
-            this.nerModel = this.$store.getters.optMlDefaultModel
+
-        } else {
+      const fields = this.$store.state.fuzzy
-            this.nerModel = ModelsRepo.getDefaultModel();
+          ? {"content.nGram": {}}
+          : {content: {}};
+
+      query.highlight = {
+        pre_tags: ["<mark>"],
+        post_tags: ["</mark>"],
+        number_of_fragments: 0,
+        fields,
+      };
+
+      if (!store.state.sist2Info.esVersionLegacy) {
+        query.highlight.max_analyzed_offset = 999_999;
+      }
     }
 
-        Sist2Api
+    if ("function_score" in query.query) {
-            .getDocument(this.sid, this.$store.state.optHighlight, this.$store.state.fuzzy)
+      query.query = query.query.function_score.query;
-            .then(doc => {
+    }
+
+    if (!("must" in query.query.bool)) {
+      query.query.bool.must = [];
+    } else if (!Array.isArray(query.query.bool.must)) {
+      query.query.bool.must = [query.query.bool.must];
+    }
+
+    query.query.bool.must.push({match: {_id: this.docId}});
+
+    delete query["sort"];
+    delete query["aggs"];
+    delete query["search_after"];
+    delete query.query["function_score"];
+
+    query._source = {
+      includes: ["content", "name", "path", "extension"]
+    }
+
+    query.size = 1;
+
+    Sist2Api.esQuery(query).then(resp => {
       this.loading = false;
-
+      if (resp.hits.hits.length === 1) {
-                if (doc) {
+        this.content = this.getContent(resp.hits.hits[0]);
-                    this.content = this.getContent(doc)
+      } else {
-                }
+        console.log("FIXME: could not get content")
-
+        console.log(resp)
-                if (this.optAutoAnalyze) {
-                    this.mlAnalyze();
       }
     });
   },
-    computed: {
-        ...mapGetters(["optAutoAnalyze"]),
-        modelSize() {
-            const modelData = ModelsRepo.data[this.nerModel];
-            if (!modelData) {
-                return 0;
-            }
-            return modelData.size;
-        }
-    },
   methods: {
-        ...mapMutations(["setOptAutoAnalyze"]),
     getContent(doc) {
-            this.rawContent = doc._source.content;
-
       if (!doc.highlight) {
         return doc._source.content;
       }
@@ -108,60 +85,10 @@ export default {
       if (doc.highlight.content) {
         return doc.highlight.content[0];
       }
-        },
-        async getMlModel() {
-            if (this.$store.getters.nerModel.name !== this.nerModel) {
-                this.mlLoading = true;
-                this.modelLoadingProgress = 0;
-                const modelInfo = ModelsRepo.data[this.nerModel];
-
-                const model = new BertNerModel(
-                    modelInfo.vocabUrl,
-                    modelInfo.modelUrl,
-                    modelInfo.id2label,
-                )
-
-                await model.init(progress => this.modelLoadingProgress = progress);
-                this.$store.commit("setNerModel", {model, name: this.nerModel});
-
-                this.mlLoading = false;
-                return model
-            }
-
-            return this.$store.getters.nerModel.model;
-        },
-        async mlAnalyze() {
-            if (!this.content) {
-                return;
-            }
-
-            const modelInfo = ModelsRepo.data[this.nerModel];
-            if (modelInfo === undefined) {
-                return;
-            }
-
-            this.$store.commit("setOptMlDefaultModel", this.nerModel);
-            await this.$store.dispatch("updateConfiguration");
-
-            const model = await this.getMlModel();
-
-            this.analyzedContentSpans = [];
-
-            this.mlPredictionsLoading = true;
-
-            await model.predict(this.rawContent, results => {
-                results.forEach(result => result.label = modelInfo.humanLabels[result.label]);
-                this.analyzedContentSpans.push(...results);
-                this.modelPredictionProgress = results[results.length - 1].wordIndex;
-            });
-            this.mlPredictionsLoading = false;
     }
   }
 }
 </script>
 
-<style>
+<style scoped>
-.progress-bar {
-    transition: none;
-}
 </style>

sist2-vue/src/components/MimePicker.vue

@@ -9,7 +9,7 @@ import InspireTreeDOM from "inspire-tree-dom";
 import "inspire-tree-dom/dist/inspire-tree-light.min.css";
 import {getSelectedTreeNodes, getTreeNodeAttributes} from "@/util";
 import Sist2Api from "@/Sist2Api";
-import Sist2Query from "@/Sist2ElasticsearchQuery";
+import Sist2Query from "@/Sist2Query";
 
 export default {
   name: "MimePicker",

sist2-vue/src/components/NavBar.vue

@@ -10,7 +10,7 @@
     <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
       v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
         href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
-        target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
+        target="_blank">legacyES</a></span>
     </span>
 
     <span v-if="$store && $store.state.sist2Info" class="tagline" v-html="tagline()"></span>

sist2-vue/src/components/PathTree.vue

@@ -20,7 +20,6 @@
           :list="suggestPath"
           :max-suggestions="0"
           :placeholder="$t('pathBar.placeholder')"
-                    :debounce="200"
       >
         <!-- Suggestion item template-->
         <div slot="suggestion-item" slot-scope="{ suggestion, query }">
@@ -102,8 +101,22 @@ export default {
     },
     async getPathChoices() {
       return new Promise(getPaths => {
-                Sist2Api.getPathSuggestions(this.getPathText).then(getPaths);
+        const q = {
-            });
+          suggest: {
+            path: {
+              prefix: this.getPathText,
+              completion: {
+                field: "suggest-path",
+                skip_duplicates: true,
+                size: 10000
+              }
+            }
+          }
+        };
+
+        Sist2Api.esQuery(q)
+            .then(resp => getPaths(resp["suggest"]["path"][0]["options"].map(opt => opt["_source"]["path"])));
+      })
     },
     async suggestPath(term) {
       if (!this.$store.state.optSuggestPath) {
@@ -123,33 +136,64 @@ export default {
       return matches.sort((a, b) => a.length - b.length);
     },
     getNextDepth(node) {
-            return Sist2Api
+      const q = {
-                .searchPaths(node.index, node.depth + 1, node.depth + 3, node.depth > 0 ? node.id : null)
+        query: {
-                .then(buckets => {
+          bool: {
+            filter: [
+              {term: {index: node.index}},
+              {range: {_depth: {gte: node.depth + 1, lte: node.depth + 3}}},
+            ]
+          }
+        },
+        aggs: {
+          paths: {
+            terms: {
+              field: "path",
+              size: 10000
+            }
+          }
+        },
+        size: 0
+      };
+
+      if (node.depth > 0) {
+        q.query.bool.must = {
+          prefix: {
+            path: node.id,
+          }
+        };
+      }
+
+      return Sist2Api.esQuery(q).then(resp => {
+        const buckets = resp["aggregations"]["paths"]["buckets"];
+        if (!buckets) {
+          return false;
+        }
+
         const paths = [];
 
         return buckets
-                        .filter(bucket => bucket.path.length > node.id.length || node.id.startsWith("/"))
+            .filter(bucket => bucket.key.length > node.id.length || node.id.startsWith("/"))
-                        .sort((a, b) => a.path > b.path ? 1 : -1)
+            .sort((a, b) => a.key > b.key)
             .map(bucket => {
-                            if (paths.some(n => bucket.path.startsWith(n))) {
+
+              if (paths.some(n => bucket.key.startsWith(n))) {
                 return null;
               }
 
-                            const name = node.id.startsWith("/") ? bucket.path : bucket.path.slice(node.id.length + 1);
+              const name = node.id.startsWith("/") ? bucket.key : bucket.key.slice(node.id.length + 1);
 
-                            paths.push(bucket.path);
+              paths.push(bucket.key);
 
               return {
-                                id: bucket.path,
+                id: bucket.key,
-                                text: `${name}/ (${bucket.count})`,
+                text: `${name}/ (${bucket.doc_count})`,
                 depth: node.depth + 1,
                 index: node.index,
-                                values: [bucket.path],
+                values: [bucket.key],
                 children: true,
               }
-                        })
+            }).filter(x => x !== null)
-                        .filter(bucket => bucket !== null);
       });
     },
     handleTreeClick(e, node, handler) {

sist2-vue/src/components/ResultsCard.vue

@@ -17,16 +17,12 @@
 
     <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
       <b-card>
-                <b-table :items="tableItems" small borderless thead-class="hidden" class="mb-0"></b-table>
+        <b-table :items="tableItems" small borderless bordered thead-class="hidden" class="mb-0"></b-table>
-
-                <template v-if="!$store.state.uiSqliteMode">
 
         <br/>
         <h4>
-                        {{ $t("mimeTypes") }}
+          {{$t("mimeTypes")}}
-                        <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick">
+          <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick"><ClipboardIcon/></b-button>
-                            <ClipboardIcon/>
-                        </b-button>
         </h4>
         <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
         <b-table
@@ -34,24 +30,22 @@
             sort-by="doc_count"
             :sort-desc="true"
             thead-class="hidden"
-                            bordered
+            :items="$store.state.uiDetailsMimeAgg" small bordered class="mb-0"
-                            :items="$store.state.uiDetailsMimeAgg" small class="mb-0"
         ></b-table>
-                </template>
       </b-card>
     </b-collapse>
   </b-card>
 </template>
 
-<script>
+<script lang="ts">
+import Sist2Api, {EsResult} from "@/Sist2Api";
 import Vue from "vue";
 import {humanFileSize} from "@/util";
 import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
 import SortSelect from "@/components/SortSelect.vue";
 import Preloader from "@/components/Preloader.vue";
-import Sist2Query from "@/Sist2ElasticsearchQuery";
+import Sist2Query from "@/Sist2Query";
 import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
-import Sist2Api from "@/Sist2Api";
 
 export default Vue.extend({
   name: "ResultsCard",
@@ -64,14 +58,13 @@ export default Vue.extend({
       return this.$store.state.lastQueryResults != null;
     },
     hitCount() {
-            return (this.$store.state.firstQueryResults).aggregations.total_count.value;
+      return (this.$store.state.lastQueryResults as EsResult).aggregations.total_count.value;
     },
     tableItems() {
       const items = [];
 
-            if (!this.$store.state.uiSqliteMode) {
+
       items.push({key: this.$t("queryTime"), value: this.took()});
-            }
       items.push({key: this.$t("totalSize"), value: this.totalSize()});
 
       return items;
@@ -79,19 +72,15 @@ export default Vue.extend({
   },
   methods: {
     took() {
-            return (this.$store.state.lastQueryResults).took + "ms";
+      return (this.$store.state.lastQueryResults as EsResult).took + "ms";
     },
     totalSize() {
-            return humanFileSize((this.$store.state.firstQueryResults).aggregations.total_size.value);
+      return humanFileSize((this.$store.state.lastQueryResults as EsResult).aggregations.total_size.value);
     },
     onToggle() {
       const show = !document.getElementById("collapse-1").classList.contains("show");
       this.$store.commit("setUiShowDetails", show);
 
-            if (this.$store.state.uiSqliteMode) {
-                return;
-            }
-
       if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
         // Mime aggs are not updated automatically, update now
         this.forceUpdateMimeAgg();
@@ -99,7 +88,7 @@ export default Vue.extend({
     },
     onCopyClick() {
       let tsvString = "";
-            this.$store.state.uiDetailsMimeAgg.slice().sort((a, b) => b["doc_count"] - a["doc_count"]).forEach(row => {
+      this.$store.state.uiDetailsMimeAgg.slice().sort((a,b) => b["doc_count"] - a["doc_count"]).forEach(row => {
         tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
       });
 

sist2-vue/src/components/SearchBar.vue

@@ -6,7 +6,7 @@
                     @input="setSearchText($event)"></b-form-input>
 
       <template #prepend>
-        <b-input-group-text v-if="!$store.state.uiSqliteMode">
+        <b-input-group-text>
           <b-form-checkbox :checked="fuzzy" title="Toggle fuzzy searching" @change="setFuzzy($event)">
             {{ $t("searchBar.fuzzy") }}
           </b-form-checkbox>

sist2-vue/src/components/SizeSlider.vue

@@ -5,7 +5,8 @@
 <script>
 import noUiSlider from 'nouislider';
 import 'nouislider/dist/nouislider.css';
-import {humanFileSize, mergeTooltips} from "@/util";
+import {humanFileSize} from "@/util";
+import {mergeTooltips} from "@/util-js";
 
 export default {
   name: "SizeSlider",

sist2-vue/src/components/SmallBadge.vue

@@ -2,7 +2,7 @@
   <b-badge variant="secondary" :pill="pill">{{ text }}</b-badge>
 </template>
 
-<script>
+<script lang="ts">
 import Vue from "vue";
 
 export default Vue.extend({

sist2-vue/src/components/SortSelect.vue

@@ -1,5 +1,5 @@
 <template>
-  <b-dropdown variant="primary" :disabled="$store.getters.embedding !== null">
+  <b-dropdown variant="primary">
     <b-dropdown-item :class="{'dropdown-active': sort === 'score'}" @click="onSelect('score')">{{
         $t("sort.relevance")
       }}

sist2-vue/src/components/TagContainer.vue

@@ -1,8 +1,7 @@
 <template>
   <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
 
-        <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static
+    <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static lazy>
-                 lazy>
       <b-row>
         <b-col style="flex-grow: 2" sm>
           <VueSimpleSuggest
@@ -31,8 +30,7 @@
           </VueSimpleSuggest>
         </b-col>
         <b-col class="mt-4">
-                    <TwitterColorPicker v-model="color" triangle="hide" :width="252"
+          <TwitterColorPicker v-model="color" triangle="hide" :width="252" class="mr-auto ml-auto"></TwitterColorPicker>
-                                        class="mr-auto ml-auto"></TwitterColorPicker>
         </b-col>
       </b-row>
 
@@ -66,7 +64,7 @@
     </template>
 
     <!-- Add button -->
-        <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{ $t("addTag") }}</small>
+    <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{$t("addTag")}}</small>
 
     <!-- Size tag-->
     <small v-else class="text-muted badge-size" style="padding-left: 2px">{{
@@ -98,6 +96,11 @@ export default Vue.extend({
       },
     }
   },
+  computed: {
+    tagHover() {
+      return this.$store.getters["uiTagHover"];
+    }
+  },
   methods: {
     humanFileSize: humanFileSize,
     getSuggestionWithoutQueryPrefix(suggestion, query) {
@@ -135,6 +138,14 @@ export default Vue.extend({
         color: tag.fg,
       };
     },
+    onTagHover(tag) {
+      if (tag.userTag) {
+        this.$store.commit("setUiTagHover", tag);
+      }
+    },
+    onTagLeave() {
+      this.$store.commit("setUiTagHover", null);
+    },
     onTagDeleteClick(tag, e) {
       this.hit._tags = this.hit._tags.filter(t => t !== tag);
 
@@ -208,8 +219,29 @@ export default Vue.extend({
     },
     getTagChoices(prefix) {
       return new Promise(getPaths => {
-                Sist2Api.getTagSuggestions(prefix)
+        Sist2Api.esQuery({
-                    .then(paths => getPaths(paths))
+          suggest: {
+            tag: {
+              prefix: prefix,
+              completion: {
+                field: "suggest-tag",
+                skip_duplicates: true,
+                size: 10000
+              }
+            }
+          }
+        }).then(resp => {
+          const result = [];
+          resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
+            tags.forEach(tag => {
+              const t = tag.slice(0, -8);
+              if (!result.find(x => x.slice(0, -8) === t)) {
+                result.push(tag);
+              }
+            });
+          });
+          getPaths(result);
+        });
       });
     }
   },

sist2-vue/src/components/TagPicker.vue

@@ -134,12 +134,8 @@ export default {
         this.initializeTree();
         this.updateTree();
       } else if (mutation.type === "busUpdateTags") {
-                if (this.$store.state.uiSqliteMode) {
-                    this.updateTree();
-                } else {
         window.setTimeout(this.updateTree, 2000);
       }
-            }
     });
   },
   methods: {
@@ -210,8 +206,4 @@ export default {
 .theme-black .inspire-tree .matched > .wholerow {
   background: rgba(251, 191, 41, 0.25);
 }
-#tagTree {
-    max-height: 350px;
-    overflow: auto;
-}
 </style>

sist2-vue/src/components/ThumbnailProgressBar.vue

@@ -1,5 +1,5 @@
 <template>
-  <div class="thumbnail_count-progress-bar" :style="{width: `${percentProgress}%`}"></div>
+  <div class="thumbnail-progress-bar" :style="{width: `${percentProgress}%`}"></div>
 </template>
 
 <script>
@@ -16,7 +16,7 @@ export default {
 
 <style scoped>
 
-.thumbnail_count-progress-bar {
+.thumbnail-progress-bar {
   position: absolute;
   left: 0;
   bottom: 0;
@@ -27,11 +27,11 @@ export default {
   z-index: 9;
 }
 
-.theme-black .thumbnail_count-progress-bar {
+.theme-black .thumbnail-progress-bar {
   background: rgba(0, 188, 212, 0.95);
 }
 
-.sub-document .thumbnail_count-progress-bar {
+.sub-document .thumbnail-progress-bar {
   max-width: calc(100% - 8px);
   left: 4px;
 }

sist2-vue/src/components/icons/MlIcon.vue

@@ -1,76 +0,0 @@
-<template>
-    <svg class="ml-icon" :class="{'m-icon': 1, 'ml-icon-big': big, 'ml-icon-clickable': clickable}" xmlns="http://www.w3.org/2000/svg"
-         viewBox="0 0 512 512" xml:space="preserve" fill="currentColor" stroke="currentColor" @click="$emit('click')">
-                <g>
-	<path class="st0" d="M167.314,14.993C167.314,6.712,160.602,0,152.332,0h-5.514c-8.27,0-14.982,6.712-14.982,14.993v41.466h35.478
-		V14.993z"/>
-                    <path class="st0"
-                          d="M238.26,14.993C238.26,6.712,231.549,0,223.278,0h-5.504c-8.271,0-14.982,6.712-14.982,14.993v41.466h35.468 V14.993z"/>
-                    <path class="st0"
-                          d="M309.207,14.993C309.207,6.712,302.496,0,294.225,0h-5.504c-8.271,0-14.982,6.712-14.982,14.993v41.466h35.468 V14.993z"/>
-                    <path class="st0"
-                          d="M380.164,14.993C380.164,6.712,373.453,0,365.182,0h-5.514c-8.27,0-14.982,6.712-14.982,14.993v41.466h35.478 V14.993z"/>
-                    <path class="st0"
-                          d="M131.836,497.007c0,8.282,6.712,14.993,14.982,14.993h5.514c8.27,0,14.982-6.711,14.982-14.993V455.55h-35.478 V497.007z"/>
-                    <path class="st0"
-                          d="M202.792,497.007c0,8.282,6.712,14.993,14.982,14.993h5.504c8.27,0,14.982-6.711,14.982-14.993V455.55h-35.468 V497.007z"/>
-                    <path class="st0"
-                          d="M273.739,497.007c0,8.282,6.712,14.993,14.982,14.993h5.504c8.271,0,14.982-6.711,14.982-14.993V455.55 h-35.468V497.007z"/>
-                    <path class="st0"
-                          d="M344.686,497.007c0,8.282,6.712,14.993,14.982,14.993h5.514c8.271,0,14.982-6.711,14.982-14.993V455.55 h-35.478V497.007z"/>
-                    <path class="st0"
-                          d="M497.018,131.836H455.55v35.479h41.468c8.27,0,14.982-6.712,14.982-14.993v-5.493 C512,138.548,505.288,131.836,497.018,131.836z"/>
-                    <path class="st0"
-                          d="M497.018,202.793H455.55v35.468h41.468c8.27,0,14.982-6.712,14.982-14.982v-5.494 C512,209.504,505.288,202.793,497.018,202.793z"/>
-                    <path class="st0"
-                          d="M497.018,273.739H455.55v35.468h41.468c8.27,0,14.982-6.711,14.982-14.992v-5.494 C512,280.451,505.288,273.739,497.018,273.739z"/>
-                    <path class="st0"
-                          d="M497.018,344.686H455.55v35.479h41.468c8.27,0,14.982-6.712,14.982-14.993v-5.493 C512,351.398,505.288,344.686,497.018,344.686z"/>
-                    <path class="st0"
-                          d="M0,146.828v5.493c0,8.281,6.711,14.993,14.982,14.993H56.46v-35.479H14.982C6.711,131.836,0,138.548,0,146.828 z"/>
-                    <path class="st0"
-                          d="M0,217.785v5.494c0,8.27,6.711,14.982,14.982,14.982H56.46v-35.468H14.982C6.711,202.793,0,209.504,0,217.785z "/>
-                    <path class="st0"
-                          d="M0,288.721v5.494c0,8.281,6.711,14.992,14.982,14.992H56.46v-35.468H14.982C6.711,273.739,0,280.451,0,288.721 z"/>
-                    <path class="st0"
-                          d="M0,359.679v5.493c0,8.281,6.711,14.993,14.982,14.993H56.46v-35.479H14.982C6.711,344.686,0,351.398,0,359.679 z"/>
-                    <path class="st0"
-                          d="M78.628,433.382h354.753V78.628H78.628V433.382z M376.56,120.2c9.18,0,16.635,7.445,16.635,16.634 c0,9.18-7.455,16.624-16.635,16.624c-9.179,0-16.624-7.445-16.624-16.624C359.936,127.644,367.381,120.2,376.56,120.2z M376.56,361.32c9.18,0,16.635,7.445,16.635,16.635c0,9.179-7.455,16.623-16.635,16.623c-9.179,0-16.624-7.444-16.624-16.623 C359.936,368.764,367.381,361.32,376.56,361.32z M184.362,184.362h143.287v143.287H184.362V184.362z M135.439,120.2 c9.19,0,16.635,7.445,16.635,16.634c0,9.169-7.445,16.624-16.635,16.624c-9.178,0-16.623-7.455-16.623-16.624 C118.816,127.644,126.26,120.2,135.439,120.2z M135.439,361.32c9.19,0,16.635,7.445,16.635,16.635 c0,9.169-7.445,16.623-16.635,16.623c-9.178,0-16.623-7.454-16.623-16.623C118.816,368.764,126.26,361.32,135.439,361.32z"/>
-</g>
-</svg>
-</template>
-
-<script>
-export default {
-    name: "MLIcon",
-    props: {
-        "big": Boolean,
-        "clickable": Boolean
-    }
-}
-</script>
-
-<style scoped>
-.ml-icon-clickable {
-    cursor: pointer;
-}
-
-.ml-icon-big {
-    width: 24px !important;
-    height: 24px !important;
-}
-
-.ml-icon {
-    width: 1rem;
-    min-width: 1rem;
-    margin-right: 0.2rem;
-    line-height: 1rem;
-    height: 1rem;
-    min-height: 1rem;
-    filter: brightness(45%);
-}
-
-.theme-black .ml-icon {
-    filter: brightness(80%);
-}
-</style>

sist2-vue/src/i18n/messages.ts

@@ -18,7 +18,6 @@ export default {
         tags: "Tags",
         tagFilter: "Filter tags",
         forExample: "For example:",
-        embeddingsSearchPlaceholder: "Embeddings search",
         help: {
             simpleSearch: "Simple search",
             advancedSearch: "Advanced search",
@@ -50,7 +49,6 @@ export default {
         configReset: "Reset configuration",
         searchOptions: "Search options",
         treemapOptions: "Treemap options",
-        mlOptions: "Machine learning options",
         displayOptions: "Display options",
         opt: {
             lang: "Language",
@@ -58,7 +56,7 @@ export default {
             fuzzy: "Set fuzzy search by default",
             searchInPath: "Enable matching query against document path",
             suggestPath: "Enable auto-complete in path filter bar",
-            fragmentSize: "Highlight context size",
+            fragmentSize: "Highlight context size in characters",
             queryMode: "Search mode",
             displayMode: "Display",
             columns: "Column count",
@@ -80,10 +78,7 @@ export default {
             simpleLightbox: "Disable animations in image viewer",
             showTagPickerFilter: "Display the tag filter bar",
             featuredFields: "Featured fields Javascript template string. Will appear in the search results.",
-            featuredFieldsList: "Available variables",
+            featuredFieldsList: "Available variables"
-            autoAnalyze: "Automatically analyze text",
-            defaultModel: "Default model",
-            mlRepositories: "Model repositories (one per line)"
         },
         queryMode: {
             simple: "Simple",
@@ -93,7 +88,6 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
-            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -177,12 +171,6 @@ export default {
             selectedIndex: "selected index",
             selectedIndices: "selected indices",
         },
-        ml: {
-            analyzeText: "Analyze",
-            auto: "Auto",
-            repoFetchError: "Failed to get list of models. Check browser console for more details.",
-            repoFetchErrorTitle: "Could not fetch model repositories",
-        }
     },
     de: {
         filePage: {
@@ -241,7 +229,7 @@ export default {
             fuzzy: "Aktiviere Fuzzy-Suche standardmäßig",
             searchInPath: "Abgleich der Abfrage mit dem Dokumentpfad aktivieren",
             suggestPath: "Aktiviere Auto-Vervollständigung in Pfadfilter-Leiste",
-            fragmentSize: "Kontextgröße",
+            fragmentSize: "Kontextgröße in Zeichen hervorheben",
             queryMode: "Such-Modus",
             displayMode: "Ansicht",
             columns: "Anzahl Spalten",
@@ -262,8 +250,8 @@ export default {
             vidPreviewInterval: "Videovorschau Framedauer in ms",
             simpleLightbox: "Schalte Animationen im Image-Viewer ab",
             showTagPickerFilter: "Zeige die Tag-Filter-Leiste",
-            featuredFields: "Variablen, welche zusätzlich in den Suchergebnissen angezeigt werden können.",
+            featuredFields: "Ausgewählte Felder Javascript Vorlage String. Wird in den Suchergebnissen angezeigt.",
-            featuredFieldsList: "verfügbare Variablen"
+            featuredFieldsList: "Verfügbare Variablen"
         },
         queryMode: {
             simple: "Einfach",
@@ -273,7 +261,6 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
-            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -346,10 +333,10 @@ export default {
             random: "zufällig",
         },
         d3: {
-            mimeCount: "Anzahl nach Medientyp",
+            mimeCount: "Anzahlverteilung nach Medientyp",
-            mimeSize: "Größen nach Medientyp",
+            mimeSize: "Größenverteilung nach Medientyp",
-            dateHistogram: "Änderungszeiten",
+            dateHistogram: "Verteilung der Änderungszeiten",
-            sizeHistogram: "Dateigrößen",
+            sizeHistogram: "Verteilung der Dateigrößen",
         },
         indexPicker: {
             selectNone: "keinen auswählen",
@@ -416,7 +403,7 @@ export default {
             fuzzy: "Activer la recherche approximative par défaut",
             searchInPath: "Activer la recherche dans le chemin des documents",
             suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
-            fragmentSize: "Longueur du contexte de surlignage",
+            fragmentSize: "Longueur du contexte de surlignage, en nombre de caractères",
             queryMode: "Mode de recherche",
             displayMode: "Affichage",
             columns: "Nombre de colonnes",
@@ -448,7 +435,6 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
-            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -623,7 +609,6 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
-            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -708,188 +693,4 @@ export default {
             selectedIndices: "选中索引",
         },
     },
-    pl: {
-        filePage: {
-            notFound: "Nie znaleziono"
-        },
-        searchBar: {
-            simple: "Szukaj",
-            advanced: "Zaawansowane szukanie",
-            fuzzy: "Również podobne"
-        },
-        addTag: "Tag",
-        deleteTag: "Usuń",
-        download: "Pobierz",
-        and: "i",
-        page: "strona",
-        pages: "stron",
-        mimeTypes: "Typy danych",
-        tags: "Tagi",
-        tagFilter: "Filtruj tagi",
-        forExample: "Na przykład:",
-        help: {
-            simpleSearch: "Proste szukanie",
-            advancedSearch: "Zaawansowane szukanie",
-            help: "Pomoc",
-            term: "<WYRAZ>",
-            and: "operator I",
-            or: "operator LUB",
-            not: "zabrania danego wyrazu",
-            quotes: "znajdzie objętą sekwencję wyrazów w podanej kolejności",
-            prefix: "znajdzie dowolny wyraz rozpoczynający się na takie litery, jeśli zastosowane na końcu wyrazu",
-            parens: "używane do grupowania wyrażeń",
-            tildeTerm: "znajdzie wyraz w podanej odległości",
-            tildePhrase: "znajdzie frazę przeplecioną podaną liczbą niepasujących wyrazów",
-            example1:
-                "Na przykład: <code>\"pieczone jajko\" +(kiełbasa | ziemniak) -frytki</code> znajdzie frazę " +
-                "<i>pieczone jajko</i> gdzie występuje też: <i>kiełbasa</i> albo <i>ziemniak</i>, ale zignoruje rezultat " +
-                "zawierający <i>frytki</i>.",
-            defaultOperator:
-                "Kiedy nie podano ani <code>+</code>, ani <code>|</code>, to domyślnym operatorem jest " +
-                "<code>+</code> (i).",
-            fuzzy:
-                "Kiedy opcja <b>Również podobne</b> jest zaznaczona, częściowo zgodne wyrazy są również znajdywane.",
-            moreInfoSimple: "Po więcej informacji sięgnij do <a target=\"_blank\" " +
-                "rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">dokumentacji Elasticsearch</a>",
-            moreInfoAdvanced: "Aby uzyskać więcej informacji o zaawansowanym szukaniu, przeczytaj <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">dokumentację Elasticsearch</a>"
-        },
-        config: "Ustawienia",
-        configDescription: "Ustawienia są zapisywane na żywo w tej przeglądarce.",
-        configReset: "Zresetuj ustawienia",
-        searchOptions: "Opcje szukania",
-        treemapOptions: "Opcje mapy",
-        mlOptions: "Opcje uczenia maszynowego",
-        displayOptions: "Opcje wyświetlania",
-        opt: {
-            lang: "Język",
-            highlight: "Zaznaczaj znalezione fragmenty",
-            fuzzy: "Ustaw szukanie również podobnych jako domyślne",
-            searchInPath: "Włącz szukanie również w ścieżce dokumentu",
-            suggestPath: "Włącz auto-uzupełnianie w filtrze ścieżek",
-            fragmentSize: "Podświetl wielkość kontekstu w znakach",
-            queryMode: "Tryb szukania",
-            displayMode: "Wyświetlanie",
-            columns: "Liczba kolumn",
-            treemapType: "Typ mapy",
-            treemapTiling: "Układanie mapy",
-            treemapColorGroupingDepth: "Jak głęboko grupować kolory mapy (na płasko)",
-            treemapColor: "Kolor mapy (kaskadowo)",
-            treemapSize: "Wielkość mapy",
-            theme: "Styl graficzny",
-            lightboxLoadOnlyCurrent: "Nie pobieraj od razu obrazów w pełnej wielkości dla sąsiednich obrazów podglądu.",
-            slideDuration: "Czas trwania jednego slajdu w pokazie slajdów",
-            resultSize: "Liczba wyników na stronę",
-            tagOrOperator: "Użyj operatora LUB przy wyborze kilku tagów",
-            hideDuplicates: "Ukryj zduplikowane wyniki (według sumy kontrolnej)",
-            hideLegacy: "Ukryj powiadomienie Elasticsearch 'legacyES'",
-            updateMimeMap: "Uaktualniaj drzewo typów mediów na żywo",
-            useDatePicker: "Używaj kalendarza do wyboru dat, zamiast suwaka",
-            vidPreviewInterval: "Czas trwania jednej klatki w podglądzie wideo (w ms)",
-            simpleLightbox: "Wyłącz animacje w podglądzie obrazów",
-            showTagPickerFilter: "Pokazuj pole filtrowania tagów",
-            featuredFields: "Wybrane pola szablonu Javascript. Będą pojawiać się przy wynikach wyszukiwania.",
-            featuredFieldsList: "Dostępne zmienne",
-            autoAnalyze: "Automatycznie analizuj tekst",
-            defaultModel: "Domyślny model",
-            mlRepositories: "Repozytoria modeli (każde w osobnej linii)"
-        },
-        queryMode: {
-            simple: "Proste",
-            advanced: "Zaawansowane",
-        },
-        lang: {
-            en: "English",
-            de: "Deutsch",
-            fr: "Français",
-            pl: "Polski",
-            "zh-CN": "简体中文",
-        },
-        displayMode: {
-            grid: "Siatka",
-            list: "Lista",
-        },
-        columns: {
-            auto: "Automatyczna"
-        },
-        treemapType: {
-            cascaded: "Kaskadowa",
-            flat: "Płaska (kompaktowa)"
-        },
-        treemapSize: {
-            small: "Mała",
-            medium: "Średnia",
-            large: "Duża",
-            xLarge: "Bardzo duża",
-            xxLarge: "Ogromna",
-            custom: "Inna",
-        },
-        treemapTiling: {
-            binary: "Binarnie",
-            squarify: "Kwadratowo",
-            slice: "Wycinek",
-            dice: "Kostka",
-            sliceDice: "Wycinek i kostka",
-        },
-        theme: {
-            light: "Jasny",
-            black: "Czarny"
-        },
-        hit: "traf",
-        hits: "trafień",
-        details: "Szczegóły",
-        stats: "Statystyki",
-        queryTime: "Czas szukania",
-        totalSize: "Całkowita wielkość",
-        pathBar: {
-            placeholder: "Filtruj ścieżki",
-            modalTitle: "Wybierz ścieżkę"
-        },
-        debug: "Informacje dla programistów",
-        debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
-            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
-        tagline: "Slogan",
-        toast: {
-            esConnErrTitle: "Problem z połączeniem z Elasticsearch",
-            esConnErr: "Moduł strony internetowej sist2 napotkał problem przy połączeniu z Elasticsearch." +
-                " Zobacz logi serwera, aby uzyskać więcej informacji.",
-            esQueryErrTitle: "Problem z kwerendą",
-            esQueryErr: "Kwerenda szukania jest niezrozumiała albo nie udało się jej przesłać. Sprawdź dokumentację zaawansowanego szukania. " +
-                "Zobacz logi serwera, aby uzyskać więcej informacji.",
-            dupeTagTitle: "Zduplikowany tag",
-            dupeTag: "Ten dokument już ma taki tag.",
-            copiedToClipboard: "Skopiowano do schowka"
-        },
-        saveTagModalTitle: "Dodaj tag",
-        saveTagPlaceholder: "Nazwa",
-        confirm: "Zatwierdź",
-        indexPickerPlaceholder: "Wybierz indeks",
-        sort: {
-            relevance: "Zgodność z szukanym",
-            dateAsc: "Data (najpierw starsze)",
-            dateDesc: "Data (najpierw nowsze)",
-            sizeAsc: "Wielkość (najpierw mniejsze)",
-            sizeDesc: "Wielkość (najpierw większe)",
-            nameAsc: "Nazwa (A-z)",
-            nameDesc: "Nazwa (Z-a)",
-            random: "Losowo",
-        },
-        d3: {
-            mimeCount: "Dystrybucja liczby plików według typów mediów",
-            mimeSize: "Dystrybucja wielkości plików według typów mediów",
-            dateHistogram: "Dystrybucja dat modyfikacji plików",
-            sizeHistogram: "Dystrybucja wielkości plików",
-        },
-        indexPicker: {
-            selectNone: "Zaznacz nic",
-            selectAll: "Zaznacz wszystko",
-            selectedIndex: "wybrany indeks",
-            selectedIndices: "wybrane indeksy",
-        },
-        ml: {
-            analyzeText: "Analizuj",
-            auto: "Automatycznie",
-            repoFetchError: "Nie udało się uzyskać listy modeli. Zobacz konsolę przeglądarki, aby uzyskać więcej informacji.",
-            repoFetchErrorTitle: "Nie udało się pobrać repozytoriów modeli",
-        }
-    }
 }

sist2-vue/src/main.js

@@ -1,3 +1,4 @@
+import '@babel/polyfill'
 import 'mutationobserver-shim'
 import Vue from 'vue'
 import './plugins/bootstrap-vue'