Merge branch 'master' into embeddings

Bump semver from 5.7.1 to 5.7.2 in /sist2-admin/frontend

.devcontainer/Dockerfile

@@ -0,0 +1,9 @@
+FROM simon987/sist2-build
+
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
+RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+

.devcontainer/devcontainer.json

@@ -0,0 +1,16 @@
+{
+  "name": "sist2-dev",
+  "dockerComposeFile": [
+    "docker-compose.yml"
+  ],
+  "service": "sist2-dev",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-vscode.cpptools-extension-pack"
+      ]
+    }
+  },
+  "remoteUser": "root",
+  "workspaceFolder": "/app/"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,8 @@
+version: "3"
+
+services:
+  sist2-dev:
+    build: .
+    command: sleep infinity
+    volumes:
+      - ../:/app

.dockerignore

@@ -37,4 +37,5 @@ state.db
 build/
 __pycache__/
 sist2-vue/dist
-sist2-admin/frontend/dist
+sist2-admin/frontend/dist
+*.fts

.gitignore

@@ -43,4 +43,6 @@ src/magic_generated.c
 src/index/static_generated.c
 *.sist2
 *-shm
-*-journal
+*-journal
+.vscode
+*.fts

CMakeLists.txt

@@ -31,7 +31,8 @@ add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
-add_executable(sist2
+add_executable(
+        sist2
         # argparse
         third-party/argparse/argparse.h third-party/argparse/argparse.c
 
@@ -58,7 +59,11 @@ add_executable(sist2
 
         src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp
 
-        src/database/database_stats.c src/database/database_schema.c)
+        src/database/database_stats.c
+        src/database/database_schema.c
+        src/database/database_fts.c
+        src/web/web_fts.c
+        src/database/database_embeddings.c)
 set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
@@ -71,6 +76,7 @@ find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
 find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
 find_package(unofficial-sqlite3 CONFIG REQUIRED)
+find_package(OpenBLAS CONFIG REQUIRED)
 
 
 target_include_directories(
@@ -126,6 +132,7 @@ else ()
             PRIVATE
 
             -Ofast
+#            -g
             -fno-stack-protector
             -fomit-frame-pointer
             -w
@@ -152,6 +159,7 @@ target_link_libraries(
 
         ${MAGIC_LIB}
         unofficial::sqlite3::sqlite3
+        OpenBLAS::OpenBLAS
 )
 
 add_custom_target(

Dockerfile

@@ -1,11 +1,6 @@
 FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
-RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
-
 WORKDIR /build/
 
 COPY scripts scripts
@@ -25,7 +20,6 @@ RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
 
-
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 
@@ -45,6 +39,7 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
     curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2

Dockerfile.arm64

@@ -1,6 +1,19 @@
 FROM simon987/sist2-build-arm64 as build
 MAINTAINER simon987 <me@simon987.net>
 
+WORKDIR /build/
+
+COPY scripts scripts
+COPY schema schema
+COPY CMakeLists.txt .
+COPY third-party third-party
+COPY src src
+COPY sist2-vue sist2-vue
+COPY sist2-admin sist2-admin
+
+RUN cd sist2-vue/ && npm install && npm run build
+RUN cd sist2-admin/frontend/ && npm install && npm run build
+
 WORKDIR /build/
 ADD . /build/
 RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
@@ -20,16 +33,17 @@ RUN apt update && apt install -y curl libasan5 libmagic1 tesseract-ocr python3-p
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
-    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
-    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
-    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
 COPY --from=build /build/build/sist2 /root/sist2

README.md

@@ -28,7 +28,7 @@ sist2 (Simple incremental search tool)
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
-\*\*\* See [OCR](#ocr)
+\*\*\* See [OCR](#ocr)    
 \*\*\*\* See [Named-Entity Recognition](#NER)
 
 ## Getting Started
@@ -46,7 +46,7 @@ services:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
   sist2-admin:
-    image: simon987/sist2:3.0.4-x64-linux
+    image: simon987/sist2:3.1.4-x64-linux
     restart: unless-stopped
     volumes:
       - ./sist2-admin-data/:/sist2-admin/
@@ -62,12 +62,14 @@ Navigate to http://localhost:8080/ to configure sist2-admin.
 
 ### Using the executable file *(Linux/WSL only)*
 
-1. Have an Elasticsearch (>= 6.8.X, ideally >=7.14.0) instance running
-    1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
-    2. *(or)* Run using docker:
-        ```bash
-        docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
-        ```
+1. Choose search backend (See [comparison](#search-backends)):
+    * **Elasticsearch**: have an Elasticsearch (version >= 6.8.X, ideally >=7.14.0) instance running
+        1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
+        2. *(or)* Run using docker:
+            ```bash
+            docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
+            ```
+    * **SQLite**: No installation required
 
 2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
    Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
@@ -76,7 +78,9 @@ Navigate to http://localhost:8080/ to configure sist2-admin.
 Example usage:
 
 1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
-2. Push index to Elasticsearch: `sist2 index ./documents.sist2`
+2. Prepare search index:
+    * **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
+    * **SQLite**: `sist2 index --search-index ./search.sist2 ./documents.sist2`
 3. Start web interface: `sist2 web ./documents.sist2`
 
 ## Format support
@@ -122,7 +126,7 @@ Download the language data files with your package manager (`apt install tessera
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
 The `simon987/sist2` image comes with common languages
-(hin, jpn, eng, fra, rus, spa, chi_sim, deu) pre-installed.
+(hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
 name must be identical to the `*.traineddata` file installed on your system
@@ -136,9 +140,27 @@ sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
 sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
 ```
 
+### Search backends
+
+sist2 v3.0.7+ supports SQLite search backend. The SQLite search backend has
+fewer features and generally comparable query performance for medium-size
+indices, but it uses much less memory and is easier to set up.
+
+|                                              |                       SQLite                        |                                                             Elasticsearch                                                             |
+|----------------------------------------------|:---------------------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------:|
+| Requires separate search engine installation |                                                     |                                                                   ✓                                                                   |
+| Memory footprint                             |                        ~20MB                        |                                                                >500MB                                                                 |
+| Query syntax                                 |      [fts5](https://www.sqlite.org/fts5.html)       | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
+| Fuzzy search                                 |                                                     |                                                                   ✓                                                                   |
+| Media Types tree real-time updating          |                                                     |                                                                   ✓                                                                   |
+| Search in file `path`                        | [WIP](https://github.com/simon987/sist2/issues/402) |                                                                   ✓                                                                   |
+| Manual tagging                               |                          ✓                          |                                                                   ✓                                                                   |
+| User scripts                                 |                          ✓                          |                                                                   ✓                                                                   |
+| Media Type breakdown for search results      |                                                     |                                                                   ✓                                                                   |
+
 ### NER
 
-sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to 
+sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to
 **Configuration** > **Machine learning options** > **Model repositories**
 to enable it.
 
@@ -151,7 +173,6 @@ See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) fo
 |---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
 | [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
 
-
 <details>
   <summary>Screenshot</summary>
 
@@ -185,7 +206,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl] sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
+    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib]
     ```
 
 4. Build

docs/USAGE.md

@@ -3,8 +3,8 @@
 ```
 Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
+   or: sist2 sqlite-index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
-   or: sist2 exec-script [OPTION]... INDEX
 
 Lightning-fast file system indexer and search tool.
 
@@ -16,7 +16,7 @@ Lightning-fast file system indexer and search tool.
 
 Scan options
     -t, --threads=<int>               Number of threads. DEFAULT: 1
-    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2
+    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
     --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
     --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
     --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
@@ -54,9 +54,13 @@ Index options
     --batch-size=<int>                Index batch size. DEFAULT: 70
     -f, --force-reset                 Reset Elasticsearch mappings and settings.
 
+sqlite-index options
+    --search-index=<str>              Path to search index. Will be created if it does not exist yet.
+
 Web options
     --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
     --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --search-index=<str>              Path to SQLite search index.
     --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
     --bind=<str>                      Listen for connections on this address. DEFAULT: localhost:4090
     --auth=<str>                      Basic auth in user:password format
@@ -69,13 +73,6 @@ Web options
     --dev                             Serve html & js files from disk (for development)
     --lang=<str>                      Default UI language. Can be changed by the user
 
-Exec-script options
-    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
-    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
-    --script-file=<str>               Path to user script.
-    --async-script                    Execute user script asynchronously.
-
 Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
@@ -83,8 +80,8 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 
 See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
 
-For example, `--thumbnail-size=500`, `--thumbnail-quality=2` for a directory with 8 million images will create a thumbnail database 
-that is about `8000000 * 36kB = 288GB`.
+For example, `--thumbnail-size=500`, `--thumbnail-quality=50` for a directory with 8 million images will create a thumbnail database 
+that is about `8000000 * 11.8kB = 94.4GB`.
 
 ![thumbnail_size](thumbnail_size.png)
 
@@ -111,53 +108,54 @@ sist scan ~/Documents -o ./documents.sist2 --incremental
 sist scan ~/Documents -o ./documents.sist2 --incremental
 ```
 
-### Index examples
+### Index documents to Elasticsearch search backend
 
-**Push to elasticsearch**
 ```bash
-sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index/
-sist2 index ./my_index/
+sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index.sist2
+sist2 index ./my_index.sist2
+```
+
+#### Index documents to SQLite search backend
+```bash
+# The search index will be created if it does not exist already
+sist2 sqlite-index ./index1.sist2 --search-index search.sist2
+sist2 sqlite-index ./index2.sist2 --search-index search.sist2
 ```
 
 **Save index in JSON format**
 ```bash
-sist2 index --print ./my_index/ > my_index.ndjson
+sist2 index --print ./my_index.sist2 > my_index.ndjson
 ```
 
 **Inspect contents of an index**
 ```bash
-sist2 index --print ./my_index/ | jq | less
+sist2 index --print ./my_index.sist2 | jq | less
 ```
 
 ## Web
 
-### Web options
- * `--es-url=<str>` Elasticsearch url.
- * `--es-index` 
-    Elasticsearch index name. DEFAULT=sist2
- * `--bind=<str>` Listen on this address.
- * `--auth=<str>` Basic auth in user:password format
- * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
-    `--auth` argument, but authentication is only applied the `/tag/` endpoint.
- * `--tagline=<str>` When specified, will replace the default tagline in the navbar.
- * `--dev` Serve html & js files from disk (for development, used to modify frontend files without having to recompile)
- * `--lang=<str>` Set the default web UI language (See #180 for a list of supported languages, default
-   is `en`). The user can change the language in the configuration page
- * `--auth0-audience`, `--auth0-domain`, `--auth0-client-id`, `--auth0-public-key-file` See [Authentication with Auth0](auth0.md)
-
 ### Web examples
 
-**Single index**
+**Single index (Elasticsearch backend)**
 ```bash
-sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index
+sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index.sist2
 ```
 
-**Multiple indices**
+**Multiple indices (Elasticsearch backend)**
 ```bash
 # Indices will be displayed in this order in the web interface
-sist2 web index1 index2 index3 index4
+sist2 web index1.sist2 index2.sist2 index3.sist2 index4.sist2
 ```
 
+**SQLite search backend**
+```bash
+sist2 web --search-index search.sist2 index1.sist2
+```
+
+#### Auth0 authentication
+
+See [auth0.md](auth0.md)
+
 ### rewrite_url
 
 When the `rewrite_url` field is not empty, the web module ignores the `root`
@@ -177,11 +175,6 @@ Using a version >=7.14.0 is recommended to enable the following features:
 When using a legacy version of ES, a notice will be displayed next to the sist2 version in the web UI.
 If you don't care about the features above, you can ignore it or disable it in the configuration page.
 
-## exec-script
-
-The `exec-script` command is used to execute a user script for an index that has already been imported to Elasticsearch with the `index` command. Note that the documents will not be reset to their default state before each execution as the `index` command does: if you make undesired changes to the documents by accident, you will need to run `index` again to revert to the original state.
-
-
 # Tagging
 
 ### Manual tagging

docs/scripting.md

@@ -1,18 +1,47 @@
 ## User scripts
 
-*This document is under construction, more in-depth guide coming soon*
+User scripts are used to augment your sist2 index with additional metadata, neural network embeddings, tags etc.
+
+
+Since version 3.2.0, user scripts are written in Python, and are ran against the sist2 index file. User scripts do not 
+need a connection to the search backend.
+
+You can create a user script based on a template from the sist2-admin interface:
+
+![sist2-admin-scripts](sist2-admin-scripts.png)
+
+User scripts leverage the [sist2-python](https://github.com/simon987/sist2-python) library to interface with the
+index file*. You can find sist2-python documentation and examples
+here: [sist2-python.readthedocs.io](https://sist2-python.readthedocs.io/).
+
+If you are not using the sist2-admin interface, you can run user scripts manually from the command line:
+
+```
+pip install git+https://github.com/simon987/sist2-python.git
+
+python my_script.py /path/to/my_index.sist2
+```
+
+\* It is possible to manually update the index using raw SQL queries, but the database schema is not stable and
+can change at any time; it is recommended to use the more stable sist2-python wrapper instead.
+
+<hr>
+
+<details>
+  <summary>Legacy user scripts (sist2 version < 3.2.0)</summary>
 
 During the `index` step, you can use the `--script-file <script>` option to
 modify documents or add user tags. This option is mainly used to
 implement automatic tagging based on file attributes.
 
-The scripting language used 
-([Painless Scripting Language](https://www.elastic.co/guide/en/elasticsearch/painless/7.4/index.html)) 
+The scripting language used
+([Painless Scripting Language](https://www.elastic.co/guide/en/elasticsearch/painless/7.4/index.html))
 is very similar to Java, but you should be able to create user scripts
 without programming experience at all if you're somewhat familiar with
 regex.
 
 This is the base structure of the documents we're working with:
+
 ```json
 {
   "_id": "e171405c-fdb5-4feb-bb32-82637bc32084",
@@ -34,7 +63,8 @@ This is the base structure of the documents we're working with:
 **Example script**
 
 This script checks if the `genre` attribute exists, if it does
-it adds the `genre.<genre>` tag. 
+it adds the `genre.<genre>` tag.
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -47,21 +77,23 @@ You can use `.` to create a hierarchical tag tree:
 
 ![scripting/genre_example](genre_example.png)
 
-
 To use regular expressions, you need to add this line in `/etc/elasticsearch/elasticsearch.yml`
+
 ```yaml
 script.painless.regex.enabled: true
 ```
+
 Or, if you're using docker add `-e "script.painless.regex.enabled=true"`
 
 **Tag color**
 
-You can specify the color for an individual tag by appending an 
+You can specify the color for an individual tag by appending an
 hexadecimal color code (`#RRGGBBAA`) to the tag name.
 
 ### Examples
 
 If `(20XX)` is in the file name, add the `year.<year>` tag:
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -72,6 +104,7 @@ if (m.find()) {
 ```
 
 Use default *Calibre* folder structure to infer author.
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -84,8 +117,9 @@ if (ctx._source.name.contains("-") && ctx._source.extension == "pdf") {
 }
 ```
 
-If the file matches a specific pattern `AAAA-000 fName1 lName1, <fName2 lName2>...`, add the `actress.<actress>` and 
+If the file matches a specific pattern `AAAA-000 fName1 lName1, <fName2 lName2>...`, add the `actress.<actress>` and
 `studio.<studio>` tag:
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -102,16 +136,18 @@ if (m.find()) {
 ```
 
 Set the name of the last folder (`/path/to/<studio>/file.mp4`) to `studio.<studio>` tag
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
 if (ctx._source.path != "") {
-    String[] names = ctx._source.path.splitOnToken('/');
+    String[] names = ctx._source.path.splitOnToken('/');
     tags.add("studio." + names[names.length-1]);
 }
 ```
 
 Parse `EXIF:F Number` tag
+
 ```Java
 if (ctx._source?.exif_fnumber != null) {
     String[] values = ctx._source.exif_fnumber.splitOnToken(' ');
@@ -124,6 +160,7 @@ if (ctx._source?.exif_fnumber != null) {
 ```
 
 Display year and months from `EXIF:DateTime` tag
+
 ```Java
 if (ctx._source?.exif_datetime != null) {
     SimpleDateFormat parser = new SimpleDateFormat("yyyy:MM:dd HH:mm:ss");
@@ -140,3 +177,6 @@ if (ctx._source?.exif_datetime != null) {
 }
 
 ```
+
+</details>
+

schema/mappings.json

@@ -68,7 +68,7 @@
     },
     "mtime": {
       "type": "date",
-      "format": "epoch_millis"
+      "format": "epoch_second"
     },
     "size": {
       "type": "long"
@@ -202,6 +202,46 @@
     },
     "modified_by": {
       "type": "text"
+    },
+    "emb.384.*": {
+      "type": "dense_vector",
+      "dims": 384
+    },
+    "emb.idx_384.*": {
+      "type": "dense_vector",
+      "dims": 384,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.idx_512.clip": {
+      "type": "dense_vector",
+      "dims": 512,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.512.*": {
+      "type": "dense_vector",
+      "dims": 512
+    },
+    "emb.idx_768.*": {
+      "type": "dense_vector",
+      "dims": 768,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.768.*": {
+      "type": "dense_vector",
+      "dims": 768
+    },
+    "emb.idx_1024.*": {
+      "type": "dense_vector",
+      "dims": 1024,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.1024.*": {
+      "type": "dense_vector",
+      "dims": 1024
     }
   }
 }

scripts/build.sh

@@ -4,6 +4,20 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
+(
+  cd sist2-vue/
+  npm install
+  npm run build
+) &
+
+(
+  cd sist2-admin/frontend/
+  npm install
+  npm run build
+) &
+
+wait
+
 mkdir build
 (
   cd build

scripts/build_arm64.sh

@@ -4,6 +4,20 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
+(
+  cd sist2-vue/
+  npm install
+  npm run build
+) &
+
+(
+  cd sist2-admin/frontend/
+  npm install
+  npm run build
+) &
+
+wait
+
 mkdir build
 (
   cd build

scripts/generate_big_index.py

@@ -0,0 +1,131 @@
+import sqlite3
+import orjson as json
+import os
+import string
+from hashlib import md5
+import random
+from tqdm import tqdm
+
+schema = """
+CREATE TABLE thumbnail (
+  id TEXT NOT NULL CHECK (
+    length(id) = 32
+  ), 
+  num INTEGER NOT NULL, 
+  data BLOB NOT NULL, 
+  PRIMARY KEY(id, num)
+) WITHOUT ROWID;
+CREATE TABLE version (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, 
+  date TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP)
+);
+CREATE TABLE document (
+  id TEXT PRIMARY KEY NOT NULL CHECK (
+    length(id) = 32
+  ), 
+  marked INTEGER NOT NULL DEFAULT (1), 
+  version INTEGER NOT NULL REFERENCES version(id), 
+  mtime INTEGER NOT NULL, 
+  size INTEGER NOT NULL, 
+  json_data TEXT NOT NULL CHECK (
+    json_valid(json_data)
+  )
+);
+CREATE TABLE delete_list (
+  id TEXT PRIMARY KEY CHECK (
+    length(id) = 32
+  )
+) WITHOUT ROWID;
+CREATE TABLE tag (
+  id TEXT NOT NULL, 
+  tag TEXT NOT NULL, 
+  PRIMARY KEY (id, tag)
+);
+CREATE TABLE document_sidecar (
+  id TEXT PRIMARY KEY NOT NULL, json_data TEXT NOT NULL
+) WITHOUT ROWID;
+CREATE TABLE descriptor (
+  id TEXT NOT NULL, version_major INTEGER NOT NULL, 
+  version_minor INTEGER NOT NULL, version_patch INTEGER NOT NULL, 
+  root TEXT NOT NULL, name TEXT NOT NULL, 
+  rewrite_url TEXT, timestamp INTEGER NOT NULL
+);
+CREATE TABLE stats_treemap (
+  path TEXT NOT NULL, size INTEGER NOT NULL
+);
+CREATE TABLE stats_size_agg (
+  bucket INTEGER NOT NULL, count INTEGER NOT NULL
+);
+CREATE TABLE stats_date_agg (
+  bucket INTEGER NOT NULL, count INTEGER NOT NULL
+);
+CREATE TABLE stats_mime_agg (
+  mime TEXT NOT NULL, size INTEGER NOT NULL, 
+  count INTEGER NOT NULL
+);
+CREATE TABLE embedding (
+  id TEXT REFERENCES document(id), 
+  model_id INTEGER NOT NULL references model(id), 
+  start INTEGER NOT NULL, 
+  end INTEGER, 
+  embedding BLOB NOT NULL, 
+  PRIMARY KEY (id, model_id, start)
+);
+CREATE TABLE model (
+  id INTEGER PRIMARY KEY, 
+  name TEXT NOT NULL UNIQUE CHECK (
+    length(name) < 16
+  ), 
+  url TEXT, 
+  path TEXT NOT NULL UNIQUE, 
+  size INTEGER NOT NULL, 
+  type TEXT NOT NULL CHECK (
+    type IN ('flat', 'nested')
+  )
+);
+"""
+
+content = "".join(random.choices(string.ascii_letters, k=500))
+
+
+def gen_document():
+    return [
+        md5(random.randbytes(8)).hexdigest(),
+        json.dumps({
+            "content": content,
+            "mime": "image/jpeg",
+            "extension": "jpeg",
+            "name": "test",
+            "path": "",
+        })
+    ]
+
+
+if __name__ == "__main__":
+    DB_NAME = "big_index.sist2"
+    SIZE = 30_000_000
+
+    os.remove(DB_NAME)
+    db = sqlite3.connect(DB_NAME)
+    db.executescript(schema)
+
+    db.executescript("""
+    PRAGMA journal_mode = OFF;
+    PRAGMA synchronous = 0;
+    """)
+
+    for _ in tqdm(range(SIZE), total=SIZE):
+        db.execute(
+            "INSERT INTO document (id, version, mtime, size, json_data) VALUES (?, 1, 1000000, 10000, ?)",
+            gen_document()
+        )
+
+    # 1. Enable rowid from document
+    # 2. CREATE TABLE marked (
+    #     id INTEGER PRIMARY KEY,
+    #     marked int
+    #    );
+    # 3. Set FK for document_sidecar, embedding, tag, thumbnail
+    # 4. Toggle FK if debug
+
+    db.commit()

scripts/sqlite_extension.c

@@ -0,0 +1,84 @@
+#include <sqlite3ext.h>
+#include <string.h>
+#include <stdlib.h>
+
+SQLITE_EXTENSION_INIT1
+
+static int sep_rfind(const char *str) {
+    for (int i = (int) strlen(str); i >= 0; i--) {
+        if (str[i] == '/') {
+            return i;
+        }
+    }
+    return -1;
+}
+
+void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    const char *value = (const char *) sqlite3_value_text(argv[0]);
+
+    int stop = sep_rfind(value);
+    if (stop == -1) {
+        sqlite3_result_null(ctx);
+        return;
+    }
+    char parent[4096 * 3];
+    strncpy(parent, value, stop);
+
+    sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
+}
+
+void random_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    char state_buf[32] = {0,};
+    struct random_data buf;
+    int result;
+
+    long seed = sqlite3_value_int64(argv[0]);
+
+    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
+
+    random_r(&buf, &result);
+
+    sqlite3_result_int(ctx, result);
+}
+
+
+int sqlite3_extension_init(
+        sqlite3 *db,
+        char **pzErrMsg,
+        const sqlite3_api_routines *pApi
+) {
+    SQLITE_EXTENSION_INIT2(pApi);
+
+
+    sqlite3_create_function(
+            db,
+            "path_parent",
+            1,
+            SQLITE_UTF8,
+            NULL,
+            path_parent_func,
+            NULL,
+            NULL
+    );
+
+    sqlite3_create_function(
+            db,
+            "random_seeded",
+            1,
+            SQLITE_UTF8,
+            NULL,
+            random_func,
+            NULL,
+            NULL
+    );
+
+    return SQLITE_OK;
+}

scripts/sqlite_extension_compile.sh

@@ -0,0 +1 @@
+gcc -I/mnt/work/vcpkg/installed/x64-linux/include -g -fPIC -shared sqlite_extension.c -o sist2funcs.so

scripts/start_dev_es.sh

@@ -1,3 +1,3 @@
-docker run --rm -it --name "sist2-dev-es"\
+docker run --rm -it --name "sist2-dev-es3"\
        	-p 9200:9200 -e "discovery.type=single-node" \
 	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.17.9

scripts/start_dev_es_8.sh

@@ -1,3 +1,3 @@
-docker run --rm -it --name "sist2-dev-es"\
+docker run --rm -it --name "sist2-dev-es3"\
        	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
 	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.7.0

scripts/test_in_docker.sh

@@ -0,0 +1,7 @@
+docker build . -t tmp
+
+docker run --rm -it\
+  -v $(pwd):/host \
+  tmp \
+  scan --ocr-lang eng --ocr-ebooks -t6 --incremental --very-verbose \
+  -o /host/docker.sist2 /host/third-party/libscan/libscan-test-files/test_files/

sist2-admin/frontend/src/App.vue

@@ -1,14 +1,15 @@
 <template>
-  <div id="app">
-    <NavBar></NavBar>
-    <b-container class="pt-4">
-      <b-alert show dismissible variant="info">
-        This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
-        to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on Github</a>. Thank you!
-      </b-alert>
-      <router-view/>
-    </b-container>
-  </div>
+    <div id="app">
+        <NavBar></NavBar>
+        <b-container class="pt-4">
+            <b-alert show dismissible variant="info">
+                This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
+                to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
+                Github</a>. Thank you!
+            </b-alert>
+            <router-view v-if="$store.state.sist2AdminInfo"/>
+        </b-container>
+    </div>
 </template>
 
 <script>
@@ -16,83 +17,89 @@ import NavBar from "@/components/NavBar";
 import Sist2AdminApi from "@/Sist2AdminApi";
 
 export default {
-  components: {NavBar},
-  data() {
-    return {
-      socket: null
-    }
-  },
-  mounted() {
-    Sist2AdminApi.getSist2AdminInfo()
-        .then(resp => this.$store.commit("setSist2AdminInfo", resp.data));
-    this.$store.dispatch("loadBrowserSettings");
-    this.connectNotifications();
-    // this.socket.onclose = this.connectNotifications;
-  },
-  methods: {
-    connectNotifications() {
-      this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
-      this.socket.onopen = () => {
-        this.socket.send("Hello from client");
-      }
-
-      this.socket.onmessage = e => {
-        const notification = JSON.parse(e.data);
-        if (notification.message) {
-          notification.messageString = this.$t(notification.message).toString();
+    components: {NavBar},
+    data() {
+        return {
+            socket: null
         }
+    },
+    mounted() {
+        Sist2AdminApi.getSist2AdminInfo()
+            .then(resp => this.$store.commit("setSist2AdminInfo", resp.data));
+        this.$store.dispatch("loadBrowserSettings");
+        this.connectNotifications();
+        // this.socket.onclose = this.connectNotifications;
+    },
+    methods: {
+        connectNotifications() {
+            if (window.location.protocol === "https:") {
+                this.socket = new WebSocket(`wss://${window.location.host}/notifications`);
+            } else {
+                this.socket = new WebSocket(`ws://${window.location.host}/notifications`);
+            }
+            this.socket.onopen = () => {
+                this.socket.send("Hello from client");
+            }
 
-        this.$store.dispatch("notify", notification)
-      }
+            this.socket.onmessage = e => {
+                const notification = JSON.parse(e.data);
+                if (notification.message) {
+                    notification.messageString = this.$t(notification.message).toString();
+                }
+
+                this.$store.dispatch("notify", notification)
+            }
+        }
     }
-  }
 }
 </script>
 
 <style>
 html, body {
-  height: 100%;
+    height: 100%;
 }
 
 #app {
-  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-  /*text-align: center;*/
-  color: #2c3e50;
-  padding-bottom: 1em;
-  min-height: 100%;
+    /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+    -webkit-font-smoothing: antialiased;
+    -moz-osx-font-smoothing: grayscale;
+    /*text-align: center;*/
+    color: #2c3e50;
+    padding-bottom: 1em;
+    min-height: 100%;
 }
 
 .info-icon {
-  width: 1rem;
-  margin-right: 0.2rem;
-  cursor: pointer;
-  line-height: 1rem;
-  height: 1rem;
-  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
-  filter: brightness(45%);
-  display: block;
+    width: 1rem;
+    min-width: 1rem;
+    margin-right: 0.2rem;
+    cursor: pointer;
+    line-height: 1rem;
+    height: 1rem;
+    min-height: 1rem;
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+    filter: brightness(45%);
+    display: block;
 }
 
 .tabs {
-  margin-top: 10px;
+    margin-top: 10px;
 }
 
 .modal-title {
-  text-overflow: ellipsis;
-  overflow: hidden;
-  white-space: nowrap;
+    text-overflow: ellipsis;
+    overflow: hidden;
+    white-space: nowrap;
 }
 
 @media screen and (min-width: 1500px) {
-  .container {
-    max-width: 1440px;
-  }
+    .container {
+        max-width: 1440px;
+    }
 }
 
 label {
-  margin-top: 0.5rem;
-  margin-bottom: 0;
+    margin-top: 0.5rem;
+    margin-bottom: 0;
 }
 </style>

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -7,15 +7,15 @@ class Sist2AdminApi {
     }
 
     getJobs() {
-        return axios.get(`${this.baseUrl}/api/job/`);
+        return axios.get(`${this.baseUrl}/api/job`);
     }
 
     getFrontends() {
-        return axios.get(`${this.baseUrl}/api/frontend/`);
+        return axios.get(`${this.baseUrl}/api/frontend`);
     }
 
     getTasks() {
-        return axios.get(`${this.baseUrl}/api/task/`);
+        return axios.get(`${this.baseUrl}/api/task`);
     }
 
     killTask(taskId) {
@@ -33,9 +33,26 @@ class Sist2AdminApi {
         return axios.get(`${this.baseUrl}/api/job/${name}`);
     }
 
-    /**
-     * @param {string} name
-     */
+    getSearchBackend(name) {
+        return axios.get(`${this.baseUrl}/api/search_backend/${name}`);
+    }
+
+    updateSearchBackend(name, data) {
+        return axios.put(`${this.baseUrl}/api/search_backend/${name}`, data);
+    }
+
+    getSearchBackends() {
+        return axios.get(`${this.baseUrl}/api/search_backend`);
+    }
+
+    deleteBackend(name) {
+        return axios.delete(`${this.baseUrl}/api/search_backend/${name}`)
+    }
+
+    createBackend(name) {
+        return axios.post(`${this.baseUrl}/api/search_backend/${name}`);
+    }
+
     getFrontend(name) {
         return axios.get(`${this.baseUrl}/api/frontend/${name}`);
     }
@@ -110,7 +127,49 @@ class Sist2AdminApi {
     }
 
     getSist2AdminInfo() {
-        return axios.get(`${this.baseUrl}/api/`);
+        return axios.get(`${this.baseUrl}/api`);
+    }
+
+    getLogsToDelete(jobName, n) {
+        return axios.get(`${this.baseUrl}/api/job/${jobName}/logs_to_delete`, {
+            params: {n: n}
+        });
+    }
+
+    deleteTaskLogs(taskId) {
+        return axios.post(`${this.baseUrl}/api/task/${taskId}/delete_logs`);
+    }
+
+    getUserScripts() {
+        return axios.get(`${this.baseUrl}/api/user_script`);
+    }
+
+    getUserScript(name) {
+        return axios.get(`${this.baseUrl}/api/user_script/${name}`);
+    }
+
+    createUserScript(name, template) {
+        return axios.post(`${this.baseUrl}/api/user_script/${name}`, null, {
+            params: {
+                template: template
+            }
+        });
+    }
+
+    updateUserScript(name, data) {
+        return axios.put(`${this.baseUrl}/api/user_script/${name}`, data);
+    }
+
+    deleteUserScript(name) {
+        return axios.delete(`${this.baseUrl}/api/user_script/${name}`);
+    }
+
+    testUserScript(name, job) {
+        return axios.get(`${this.baseUrl}/api/user_script/${name}/run`, {
+            params: {
+                job: job
+            }
+        });
     }
 }
 

sist2-admin/frontend/src/components/IndexOptions.vue

@@ -1,64 +0,0 @@
-<template>
-  <div>
-    <label>{{ $t("indexOptions.threads") }}</label>
-    <b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.esUrl") }}</label>
-    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
-      {{ esTestMessage }}
-    </b-alert>
-    <b-input-group>
-      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
-      <b-input-group-append>
-        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
-      </b-input-group-append>
-    </b-input-group>
-
-    <label>{{ $t("indexOptions.esIndex") }}</label>
-    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
-
-    <br>
-    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
-      {{ $t("webOptions.esInsecure") }}
-    </b-form-checkbox>
-
-    <label>{{ $t("indexOptions.batchSize") }}</label>
-    <b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
-
-    <label>{{ $t("indexOptions.script") }}</label>
-    <b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
-  </div>
-
-</template>
-
-<script>
-import sist2AdminApi from "@/Sist2AdminApi";
-
-export default {
-  name: "IndexOptions",
-  props: ["options"],
-  data() {
-    return {
-      showEsTestAlert: false,
-      esTestOk: false,
-      esTestMessage: "",
-    }
-  },
-  methods: {
-    update() {
-      this.$emit("change", this.options);
-    },
-    testEs() {
-      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
-        this.showEsTestAlert = true;
-        this.esTestOk = resp.data.ok;
-        this.esTestMessage = resp.data.message;
-      });
-    }
-  },
-}
-</script>
-
-<style scoped>
-
-</style>

sist2-admin/frontend/src/components/JobCheckboxGroup.vue

@@ -1,42 +1,54 @@
 <template>
-  <div>
-    <h5>{{ $t("selectJobs") }}</h5>
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <b-form-group v-else>
-      <b-form-checkbox-group
-          v-if="jobs.length > 0"
-          :checked="frontend.jobs"
-          @input="frontend.jobs = $event; $emit('input')"
-      >
-        <div v-for="job in jobs" :key="job.name">
-          <b-form-checkbox :disabled="job.status !== 'indexed'" :value="job.name">[{{ job.name }}]</b-form-checkbox>
-          <br/>
-        </div>
-      </b-form-checkbox-group>
-      <div v-else>
-        <span class="text-muted">{{ $t('jobOptions.noJobAvailable') }}</span>
-        &nbsp;<router-link to="/">{{$t("create")}}</router-link>
-      </div>
-    </b-form-group>
-  </div>
+    <div>
+        <h5>{{ $t("selectJobs") }}</h5>
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-form-group v-else>
+            <b-form-checkbox-group
+                    v-if="jobs.length > 0"
+                    :checked="frontend.jobs"
+                    @input="frontend.jobs = $event; $emit('input')"
+            >
+                <div v-for="job in jobs" :key="job.name">
+                    <b-form-checkbox :disabled="job.status !== 'indexed'"
+                                     :value="job.name">
+                        <template #default><span
+                                :title="job.status !== 'indexed' ? $t('jobOptions.notIndexed') : ''"
+                        >[{{ job.name }}]</span></template>
+                    </b-form-checkbox>
+                    <br/>
+                </div>
+            </b-form-checkbox-group>
+            <div v-else>
+                <span class="text-muted">{{ $t('jobOptions.noJobAvailable') }}</span>
+                <router-link to="/">{{ $t("create") }}</router-link>
+            </div>
+        </b-form-group>
+    </div>
 </template>
 
 <script>
 import Sist2AdminApi from "@/Sist2AdminApi";
 
 export default {
-  name: "JobCheckboxGroup",
-  props: ["frontend"],
-  mounted() {
-    Sist2AdminApi.getJobs().then(resp => {
-      this.jobs = resp.data;
-      this.loading = false;
-    });
-  },
-  data() {
-    return {
-      loading: true,
+    name: "JobCheckboxGroup",
+    props: ["frontend"],
+    mounted() {
+        Sist2AdminApi.getJobs().then(resp => {
+            this._jobs = resp.data;
+            this.loading = false;
+        });
+    },
+    computed: {
+        jobs() {
+            return this._jobs
+                .filter(job => job.index_options.search_backend === this.frontend.web_options.search_backend)
+        }
+    },
+    data() {
+        return {
+            loading: true,
+            _jobs: null
+        }
     }
-  }
 }
 </script>

sist2-admin/frontend/src/components/JobListItem.vue

@@ -44,8 +44,7 @@ export default {
         return "";
       }
 
-      const date = Date.parse(dateString);
-      return moment(date).fromNow();
+      return moment.utc(dateString).local().fromNow();
     }
   }
 }

sist2-admin/frontend/src/components/JobOptions.vue

@@ -1,57 +1,94 @@
 <template>
-  <div>
-    <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
-      {{ $t("jobOptions.desktopNotifications") }}
-    </b-form-checkbox>
+    <div>
+        <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
+            {{ $t("jobOptions.desktopNotifications") }}
+        </b-form-checkbox>
 
-    <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
-      {{ $t("jobOptions.scheduleEnabled") }}
-    </b-form-checkbox>
+        <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
+            {{ $t("jobOptions.scheduleEnabled") }}
+        </b-form-checkbox>
 
-    <label>{{ $t("jobOptions.cron") }}</label>
-    <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression" :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
-  </div>
+        <label>{{ $t("jobOptions.cron") }}</label>
+        <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression"
+                      :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
+
+        <label>{{ $t("jobOptions.keepNLogs") }}</label>
+        <b-input-group>
+            <b-form-input type="number" v-model="job.keep_last_n_logs" @change="update()"></b-form-input>
+            <b-input-group-append>
+                <b-button variant="danger" @click="onDeleteNowClick()">{{ $t("jobOptions.deleteNow") }}</b-button>
+            </b-input-group-append>
+        </b-input-group>
+
+    </div>
 </template>
 
 <script>
 
+import Sist2AdminApi from "@/Sist2AdminApi";
+
 export default {
-  name: "JobOptions",
-  props: ["job"],
-  data() {
-    return {
-      cronValid: undefined
-    }
-  },
-  computed: {
-    desktopNotificationsEnabled() {
-      return this.$store.state.jobDesktopNotificationMap[this.job.name];
-    }
-  },
+    name: "JobOptions",
+    props: ["job"],
+    data() {
+        return {
+            cronValid: undefined,
+            logsToDelete: null
+        }
+    },
+    computed: {
+        desktopNotificationsEnabled() {
+            return this.$store.state.jobDesktopNotificationMap[this.job.name];
+        }
+    },
     mounted() {
-      this.cronValid = this.checkCron(this.job.cron_expression)
+        this.cronValid = this.checkCron(this.job.cron_expression)
     },
     methods: {
-      checkCron(expression) {
-          return /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(expression);
-      },
-    updateNotifications(value) {
-      this.$store.dispatch("setJobDesktopNotification", {
-        job: this.job.name,
-        enabled: value
-      });
-    },
-    update() {
-      if (this.job.schedule_enabled) {
-        this.cronValid = this.checkCron(this.job.cron_expression);
-      } else {
-        this.cronValid = undefined;
-      }
+        checkCron(expression) {
+            return /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(expression);
+        },
+        updateNotifications(value) {
+            this.$store.dispatch("setJobDesktopNotification", {
+                job: this.job.name,
+                enabled: value
+            });
+        },
+        update() {
+            if (this.job.schedule_enabled) {
+                this.cronValid = this.checkCron(this.job.cron_expression);
+            } else {
+                this.cronValid = undefined;
+            }
 
-      if (this.cronValid !== false) {
-        this.$emit("change", this.job);
-      }
+            if (this.cronValid !== false) {
+                this.$emit("change", this.job);
+            }
+        },
+        onDeleteNowClick() {
+            Sist2AdminApi.getLogsToDelete(this.job.name, this.job.keep_last_n_logs).then(resp => {
+                const toDelete = resp.data;
+                const message = `Delete ${toDelete.length} log files?`;
+
+                this.$bvModal.msgBoxConfirm(message, {
+                    title: this.$t("confirmation"),
+                    size: "sm",
+                    buttonSize: "sm",
+                    okVariant: "danger",
+                    okTitle: this.$t("delete"),
+                    cancelTitle: this.$t("cancel"),
+                    footerClass: "p-2",
+                    hideHeaderClose: false,
+                    centered: true
+                }).then(value => {
+                    if (value) {
+                        toDelete.forEach(row => {
+                            Sist2AdminApi.deleteTaskLogs(row["id"]);
+                        });
+                    }
+                });
+            })
+        }
     },
-  },
 }
 </script>

sist2-admin/frontend/src/components/JobSelect.vue

@@ -0,0 +1,34 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <span v-else-if="jobs.length === 0"></span>
+    <b-form-select v-else :options="jobs" text-field="name" value-field="name"
+                   @change="$emit('change', $event)" :value="$t('selectJob')"></b-form-select>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "JobSelect",
+    mounted() {
+        Sist2AdminApi.getJobs().then(resp => {
+            this._jobs = resp.data;
+            this.loading = false;
+        });
+    },
+    computed: {
+        jobs() {
+            return [
+                {name: this.$t("selectJob"), disabled: true},
+                ...this._jobs.filter(job => job.index_path)
+            ]
+        }
+    },
+    data() {
+        return {
+            loading: true,
+            _jobs: null
+        }
+    }
+}
+</script>

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -7,7 +7,7 @@
     <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
-    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+    <b-form-input type="number" min="0" max="100" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailCount") }}</label>
     <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>

sist2-admin/frontend/src/components/SearchBackendListItem.vue

@@ -0,0 +1,24 @@
+<template>
+    <b-list-group-item action :to="`/searchBackend/${backend.name}`">
+
+        <div class="d-flex w-100 justify-content-between">
+            <h5 class="mb-1">
+                {{ backend.name }}
+            </h5>
+
+            <div>
+                <b-badge v-if="backend.backend_type === 'sqlite'" variant="info">SQLite</b-badge>
+                <b-badge v-else variant="info">Elasticsearch</b-badge>
+            </div>
+        </div>
+
+    </b-list-group-item>
+</template>
+
+<script>
+
+export default {
+    name: "SearchBackendListItem",
+    props: ["backend"],
+}
+</script>

sist2-admin/frontend/src/components/SearchBackendSelect.vue

@@ -0,0 +1,37 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <div v-else>
+        <label>{{$t("backendOptions.searchBackend")}}</label>
+        <b-select :options="options" :value="value" @change="$emit('change', $event)"></b-select>
+    </div>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "SearchBackendSelect",
+    props: ["value"],
+    data() {
+        return {
+            loading: true,
+            backends: null,
+        }
+    },
+    computed: {
+        options() {
+            return this.backends.map(backend => backend.name)
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getSearchBackends().then(resp => {
+            this.loading = false;
+            this.backends = resp.data
+        })
+    }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/UserScriptListItem.vue

@@ -0,0 +1,18 @@
+<template>
+    <b-list-group-item action :to="`/userScript/${script.name}`">
+
+        <div class="d-flex w-100 justify-content-between">
+            <h5 class="mb-1">
+                {{ script.name }}
+            </h5>
+        </div>
+    </b-list-group-item>
+</template>
+
+<script>
+
+export default {
+    name: "UserScriptListItem",
+    props: ["script"],
+}
+</script>

sist2-admin/frontend/src/components/UserScriptPicker.vue

@@ -0,0 +1,88 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+
+    <b-row v-else>
+        <b-col cols="6">
+            <h5>Selected scripts</h5>
+            <b-list-group>
+                <b-list-group-item v-for="script in selectedScripts" :key="script"
+                                   button
+                                   @click="onRemoveScript(script)"
+                                   class="d-flex justify-content-between align-items-center">
+                    {{ script }}
+                    <b-button-group>
+                        <b-button variant="light" @click.stop="moveUpScript(script)">↑</b-button>
+                        <b-button variant="light" @click.stop="moveDownScript(script)">↓</b-button>
+                    </b-button-group>
+                </b-list-group-item>
+            </b-list-group>
+        </b-col>
+        <b-col cols="6">
+            <h5>Available scripts</h5>
+            <b-list-group>
+                <b-list-group-item v-for="script in availableScripts" :key="script" button
+                                   @click="onSelectScript(script)">
+                    {{ script }}
+                </b-list-group-item>
+            </b-list-group>
+        </b-col>
+    </b-row>
+
+  <!--    <b-checkbox-group v-else :options="scripts" stacked :checked="selectedScripts"-->
+  <!--                      @input="$emit('change', $event)"></b-checkbox-group>-->
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "UserScriptPicker",
+    props: ["selectedScripts"],
+    data() {
+        return {
+            loading: true,
+            scripts: []
+        }
+    },
+    computed: {
+        availableScripts() {
+            return this.scripts.filter(script => !this.selectedScripts.includes(script))
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getUserScripts().then(resp => {
+            this.scripts = resp.data.map(script => script.name);
+            this.loading = false;
+        });
+    },
+    methods: {
+        onSelectScript(name) {
+            this.selectedScripts.push(name);
+            this.$emit("change", this.selectedScripts)
+        },
+        onRemoveScript(name) {
+            this.selectedScripts.splice(this.selectedScripts.indexOf(name), 1);
+            this.$emit("change", this.selectedScripts);
+        },
+        moveUpScript(name) {
+            const index = this.selectedScripts.indexOf(name);
+            if (index > 0) {
+                this.selectedScripts.splice(index, 1);
+                this.selectedScripts.splice(index - 1, 0, name);
+            }
+            this.$emit("change", this.selectedScripts);
+        },
+        moveDownScript(name) {
+            const index = this.selectedScripts.indexOf(name);
+            if (index < this.selectedScripts.length - 1) {
+                this.selectedScripts.splice(index, 1);
+                this.selectedScripts.splice(index + 1, 0, name);
+            }
+            this.$emit("change", this.selectedScripts);
+        }
+    }
+}
+</script>
+
+<style scoped>
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -1,88 +1,59 @@
 <template>
-  <div>
-    <label>{{ $t("webOptions.esUrl") }}</label>
-    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
-      {{ esTestMessage }}
-    </b-alert>
+    <div>
+        <h4>{{ $t("webOptions.title") }}</h4>
+        <b-card>
+            <label>{{ $t("webOptions.lang") }}</label>
+            <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
+                           @change="update()"></b-form-select>
 
-    <b-input-group>
-      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
-      <b-input-group-append>
-        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
-      </b-input-group-append>
-    </b-input-group>
+            <label>{{ $t("webOptions.bind") }}</label>
+            <b-form-input v-model="options.bind" @change="update()"></b-form-input>
 
-    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
-      {{ $t("webOptions.esInsecure") }}
-    </b-form-checkbox>
+            <label>{{ $t("webOptions.tagline") }}</label>
+            <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
 
-    <label>{{ $t("webOptions.esIndex") }}</label>
-    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+            <label>{{ $t("webOptions.auth") }}</label>
+            <b-form-input v-model="options.auth" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.lang") }}</label>
-    <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
+            <label>{{ $t("webOptions.tagAuth") }}</label>
+            <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+        </b-card>
 
-    <label>{{ $t("webOptions.bind") }}</label>
-    <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+        <br>
+        <h4>Auth0 options</h4>
+        <b-card>
+            <label>{{ $t("webOptions.auth0Audience") }}</label>
+            <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.tagline") }}</label>
-    <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+            <label>{{ $t("webOptions.auth0Domain") }}</label>
+            <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.auth") }}</label>
-    <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+            <label>{{ $t("webOptions.auth0ClientId") }}</label>
+            <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.tagAuth") }}</label>
-    <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
-
-    <br>
-    <h5>Auth0 options</h5>
-    <label>{{ $t("webOptions.auth0Audience") }}</label>
-    <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.auth0Domain") }}</label>
-    <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.auth0ClientId") }}</label>
-    <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.auth0PublicKey") }}</label>
-    <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
-
-
-
-  </div>
+            <label>{{ $t("webOptions.auth0PublicKey") }}</label>
+            <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
+        </b-card>
+    </div>
 </template>
 
 <script>
 
-import sist2AdminApi from "@/Sist2AdminApi";
-
 export default {
-  name: "WebOptions",
-  props: ["options", "frontendName"],
-  data() {
-    return {
-      showEsTestAlert: false,
-      esTestOk: false,
-      esTestMessage: "",
-    }
-  },
-  methods: {
-    update() {
-      if (!this.options.es_url.startsWith("https")) {
-        this.options.es_insecure_ssl = false;
-      }
-
-      this.$emit("change", this.options);
+    name: "WebOptions",
+    props: ["options", "frontendName"],
+    data() {
+        return {
+            showEsTestAlert: false,
+            esTestOk: false,
+            esTestMessage: "",
+        }
     },
-    testEs() {
-      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
-        this.showEsTestAlert = true;
-        this.esTestOk = resp.data.ok;
-        this.esTestMessage = resp.data.message;
-      });
+    methods: {
+        update() {
+            this.$emit("change", this.options);
+        },
     }
-  }
 }
 </script>
 

sist2-admin/frontend/src/i18n/messages.js

@@ -5,10 +5,13 @@ export default {
         go: "Go",
         online: "online",
         offline: "offline",
+        view: "View",
         delete: "Delete",
         runNow: "Index now",
         create: "Create",
+        cancel: "Cancel",
         test: "Test",
+        confirmation: "Confirmation",
 
         jobTitle: "job configuration",
         tasks: "Tasks",
@@ -45,12 +48,26 @@ export default {
         extraQueryArgs: "Extra query arguments when launching from sist2-admin",
         customUrl: "Custom URL when launching from sist2-admin",
 
-        selectJobs: "Select jobs",
+        searchBackends: "Search backends",
+        searchBackendTitle: "search backend configuration",
+        newBackendName: "New search backend name",
+
+        frontendTab: "Frontend",
+        backendTab: "Backend",
+        scripts: "User Scripts",
+        script: "User Script",
+        testScript: "Test/debug User Script",
+        newScriptName: "New script name",
+        scriptType: "Script type",
+        scriptCode: "Script code (Python)",
+        scriptOptions: "User scripts",
+        gitRepository: "Git repository URL",
+        extraArgs: "Extra command line arguments",
+
+        selectJobs: "Available jobs",
+        selectJob: "Select a job",
         webOptions: {
             title: "Web options",
-            esUrl: "Elasticsearch URL",
-            esIndex: "Elasticsearch index name",
-            esInsecure: "Do not verify SSL connections to Elasticsearch.",
             lang: "UI Language",
             bind: "Listen address",
             tagline: "Tagline in navbar",
@@ -61,12 +78,24 @@ export default {
             auth0ClientId: "Auth0 client ID",
             auth0PublicKey: "Auth0 public key",
         },
+        backendOptions: {
+            title: "Search backend options",
+            searchBackend: "Search backend",
+            type: "Search backend type",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            threads: "Number of threads",
+            batchSize: "Index batch size",
+            script: "User script",
+            searchIndex: "Search index file location"
+        },
         scanOptions: {
             title: "Scanning options",
             path: "Path",
             threads: "Number of threads",
             memThrottle: "Total memory threshold in MiB for scan throttling",
-            thumbnailQuality: "Thumbnail quality, on a scale of 2 to 32, 2 being the best",
+            thumbnailQuality: "Thumbnail quality, on a scale of 0 to 100, 100 being the best",
             thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
             thumbnailSize: "Thumbnail size, in pixels",
             contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",
@@ -87,24 +116,19 @@ export default {
             treemapThreshold: "Relative size threshold for treemap",
             optimizeIndex: "Defragment index file after scan to reduce its file size."
         },
-        indexOptions: {
-            title: "Indexing options",
-            threads: "Number of threads",
-            esUrl: "Elasticsearch URL",
-            esIndex: "Elasticsearch index name",
-            esInsecure: "Do not verify SSL connections to Elasticsearch.",
-            batchSize: "Index batch size",
-            script: "User script"
-        },
         jobOptions: {
             title: "Job options",
             cron: "Job schedule",
+            keepNLogs: "Keep last N log files. Set to -1 to keep all logs.",
+            deleteNow: "Delete now",
             scheduleEnabled: "Enable scheduled re-scan",
-            noJobAvailable: "No jobs available.",
+            noJobAvailable: "No jobs available for this search backend.",
+            notIndexed: "Has not been indexed yet",
+            noBackendError: "You must select a search backend to run this job",
             desktopNotifications: "Desktop notifications"
         },
         frontendOptions: {
-            title: "Frontend options",
+            title: "Advanced options",
             noJobSelectedWarning: "You must select at least one job to start this frontend"
         },
         notifications: {

sist2-admin/frontend/src/router/index.js

@@ -5,12 +5,19 @@ import Job from "@/views/Job";
 import Tasks from "@/views/Tasks";
 import Frontend from "@/views/Frontend";
 import Tail from "@/views/Tail";
+import SearchBackend from "@/views/SearchBackend.vue";
+import UserScript from "@/views/UserScript.vue";
 
 Vue.use(VueRouter);
 
 const routes = [
   {
-    path: "/",
+    path: "/task",
+    name: "Tasks",
+    component: Tasks
+  },
+  {
+    path: "/:tab?",
     name: "Home",
     component: Home
   },
@@ -19,16 +26,21 @@ const routes = [
     name: "Job",
     component: Job
   },
-  {
-    path: "/task/",
-    name: "Tasks",
-    component: Tasks
-  },
   {
     path: "/frontend/:name",
     name: "Frontend",
     component: Frontend
   },
+  {
+    path: "/searchBackend/:name",
+    name: "SearchBackend",
+    component: SearchBackend
+  },
+  {
+    path: "/userScript/:name",
+    name: "UserScript",
+    component: UserScript
+  },
   {
     path: "/log/:taskId",
     name: "Tail",

sist2-admin/frontend/src/views/Frontend.vue

@@ -1,60 +1,63 @@
 <template>
-  <b-card>
-    <b-card-title>
-      {{ name }}
-      <small style="vertical-align: top">
-        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
-        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
-      </small>
-    </b-card-title>
+    <b-card>
+        <b-card-title>
+            {{ name }}
+            <small style="vertical-align: top">
+                <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+                <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+            </small>
+        </b-card-title>
 
-    <div class="mb-3" v-if="!loading">
-      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
-          $t("start")
-        }}
-      </b-button>
-      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
-          $t("stop")
-        }}
-      </b-button>
-      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
-        {{ $t("go") }}
-      </b-button>
-      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
-    </div>
+        <!-- Action buttons-->
+        <div class="mb-3" v-if="!loading">
+            <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+                $t("start")
+                }}
+            </b-button>
+            <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+                $t("stop")
+                }}
+            </b-button>
+            <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+                {{ $t("go") }}
+            </b-button>
+            <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+        </div>
 
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-card-body v-else>
 
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <b-card-body v-else>
+            <h4>{{ $t("backendOptions.title") }}</h4>
+            <b-card>
+                <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
 
-      <h4>{{ $t("frontendOptions.title") }}</h4>
-      <b-card>
-        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
-          {{ $t("autoStart") }}
-        </b-form-checkbox>
+                <SearchBackendSelect :value="frontend.web_options.search_backend"
+                                     @change="onBackendSelect($event)"></SearchBackendSelect>
 
-        <label>{{ $t("extraQueryArgs") }}</label>
-        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+                <br>
+                <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+            </b-card>
 
-        <label>{{ $t("customUrl") }}</label>
-        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+            <br/>
 
-        <br/>
+            <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
+                        @change="update()"></WebOptions>
+            <br/>
 
-        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+            <h4>{{ $t("frontendOptions.title") }}</h4>
+            <b-card>
+                <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+                    {{ $t("autoStart") }}
+                </b-form-checkbox>
 
-        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
-      </b-card>
+                <label>{{ $t("extraQueryArgs") }}</label>
+                <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
 
-      <br/>
-
-      <h4>{{ $t("webOptions.title") }}</h4>
-      <b-card>
-        <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
-      </b-card>
-    </b-card-body>
-
-  </b-card>
+                <label>{{ $t("customUrl") }}</label>
+                <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+            </b-card>
+        </b-card-body>
+    </b-card>
 </template>
 
 <script>
@@ -62,68 +65,74 @@
 import Sist2AdminApi from "@/Sist2AdminApi";
 import JobCheckboxGroup from "@/components/JobCheckboxGroup";
 import WebOptions from "@/components/WebOptions";
+import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
-  name: 'Frontend',
-  components: {JobCheckboxGroup, WebOptions},
-  data() {
-    return {
-      loading: true,
-      frontend: null,
-    }
-  },
-  computed: {
-    valid() {
-      return !this.loading && this.frontend.jobs.length > 0;
+    name: 'Frontend',
+    components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
+    data() {
+        return {
+            loading: true,
+            frontend: null,
+        }
     },
-    frontendUrl() {
-      if (this.frontend.custom_url) {
-        return this.frontend.custom_url + this.args;
-      }
+    computed: {
+        valid() {
+            return !this.loading && this.frontend.jobs.length > 0;
+        },
+        frontendUrl() {
+            if (this.frontend.custom_url) {
+                return this.frontend.custom_url + this.args;
+            }
 
-      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
-        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
-      }
+            if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+                return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+            }
 
-      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+            return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+        },
+        name() {
+            return this.$route.params.name;
+        },
+        port() {
+            return this.frontend.web_options.bind.split(":")[1]
+        },
+        args() {
+            const args = this.frontend.extra_query_args;
+            if (args !== "") {
+                return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+            }
+            return "";
+        }
     },
-    name() {
-      return this.$route.params.name;
+    mounted() {
+        Sist2AdminApi.getFrontend(this.name).then(resp => {
+            this.frontend = resp.data;
+            this.loading = false;
+        });
     },
-    port() {
-      return this.frontend.web_options.bind.split(":")[1]
-    },
-    args() {
-      const args = this.frontend.extra_query_args;
-      if (args !== "") {
-        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
-      }
-      return "";
+    methods: {
+        start() {
+            this.frontend.running = true;
+            Sist2AdminApi.startFrontend(this.name)
+        },
+        stop() {
+            this.frontend.running = false;
+            Sist2AdminApi.stopFrontend(this.name)
+        },
+        deleteFrontend() {
+            Sist2AdminApi.deleteFrontend(this.name).then(() => {
+                this.$router.push("/");
+            });
+        },
+        update() {
+            Sist2AdminApi.updateFrontend(this.name, this.frontend);
+        },
+        onBackendSelect(backend) {
+            this.frontend.web_options.search_backend = backend;
+            this.frontend.jobs = [];
+            this.update();
+        }
     }
-  },
-  mounted() {
-    Sist2AdminApi.getFrontend(this.name).then(resp => {
-      this.frontend = resp.data;
-      this.loading = false;
-    });
-  },
-  methods: {
-    start() {
-      this.frontend.running = true;
-      Sist2AdminApi.startFrontend(this.name)
-    },
-    stop() {
-      this.frontend.running = false;
-      Sist2AdminApi.stopFrontend(this.name)
-    },
-    deleteFrontend() {
-      Sist2AdminApi.deleteFrontend(this.name).then(() => {
-        this.$router.push("/frontends");
-      });
-    },
-    update() {
-      Sist2AdminApi.updateFrontend(this.name, this.frontend);
-    },
-  }
 }
 </script>

sist2-admin/frontend/src/views/Home.vue

@@ -1,60 +1,123 @@
 <template>
-  <div>
-    <b-card>
-      <b-card-title>{{ $t("jobs") }}</b-card-title>
-      <b-row>
-        <b-col>
-          <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
-          <b-popover
-              :show.sync="showHelp"
-              target="new-job"
-              placement="top"
-              triggers="manual"
-              variant="primary"
-              :content="$t('newJobHelp')"
-          ></b-popover>
-        </b-col>
-        <b-col>
-          <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
-          </b-button>
-        </b-col>
-      </b-row>
+    <div>
+        <b-tabs content-class="mt-3" v-model="tab" @input="onTabChange($event)">
+            <b-tab :title="$t('backendTab')">
 
-      <hr/>
+                <b-card>
+                    <b-card-title>{{ $t("searchBackends") }}</b-card-title>
 
-      <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
-      <b-list-group v-else>
-        <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
-      </b-list-group>
-    </b-card>
+                    <b-row>
+                        <b-col>
+                            <b-input v-model="newBackendName" :placeholder="$t('newBackendName')"></b-input>
+                        </b-col>
+                        <b-col>
+                            <b-button variant="primary" @click="createBackend()"
+                                      :disabled="!backendNameValid(newBackendName)">
+                                {{ $t("create") }}
+                            </b-button>
+                        </b-col>
+                    </b-row>
 
-    <br/>
+                    <hr/>
 
-    <b-card>
+                    <b-progress v-if="backendsLoading" striped animated value="100"></b-progress>
+                    <b-list-group v-else>
+                        <SearchBackendListItem v-for="backend in backends"
+                                               :key="backend.name" :backend="backend"></SearchBackendListItem>
+                    </b-list-group>
+                </b-card>
 
-      <b-card-title>{{ $t("frontends") }}</b-card-title>
+                <br/>
 
-      <b-row>
-        <b-col>
-          <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
-        </b-col>
-        <b-col>
-          <b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
-            {{ $t("create") }}
-          </b-button>
-        </b-col>
-      </b-row>
+                <b-card>
+                    <b-card-title>{{ $t("jobs") }}</b-card-title>
+                    <b-row>
+                        <b-col>
+                            <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
+                            <b-popover
+                                    :show.sync="showHelp"
+                                    target="new-job"
+                                    placement="top"
+                                    triggers="manual"
+                                    variant="primary"
+                                    :content="$t('newJobHelp')"
+                            ></b-popover>
+                        </b-col>
+                        <b-col>
+                            <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">
+                                {{ $t("create") }}
+                            </b-button>
+                        </b-col>
+                    </b-row>
 
-      <hr/>
+                    <hr/>
 
-      <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
-      <b-list-group v-else>
-        <FrontendListItem v-for="frontend in frontends"
-                          :key="frontend.name" :frontend="frontend"></FrontendListItem>
-      </b-list-group>
+                    <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
+                    <b-list-group v-else>
+                        <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
+                    </b-list-group>
+                </b-card>
+            </b-tab>
+            <b-tab :title="$t('scripts')">
 
-    </b-card>
-  </div>
+                <b-progress v-if="scriptsLoading" striped animated value="100"></b-progress>
+                <b-card v-else>
+                    <b-card-title>{{ $t("scripts") }}</b-card-title>
+
+                    <label>Select template</label>
+                    <b-form-radio-group stacked :options="scriptTemplates" v-model="scriptTemplate"></b-form-radio-group>
+                    <br>
+
+                    <b-row>
+                        <b-col>
+                            <b-form-input v-model="newScriptName" :disabled="!scriptTemplate" :placeholder="$t('newScriptName')"></b-form-input>
+                        </b-col>
+                        <b-col>
+                            <b-button variant="primary" @click="createScript()"
+                                      :disabled="!scriptNameValid(newScriptName)">
+                                {{ $t("create") }}
+                            </b-button>
+                        </b-col>
+                    </b-row>
+
+                    <hr/>
+
+                    <b-list-group>
+                        <UserScriptListItem v-for="script in scripts"
+                                            :key="script.name" :script="script"></UserScriptListItem>
+                    </b-list-group>
+
+                </b-card>
+            </b-tab>
+            <b-tab :title="$t('frontendTab')">
+                <b-card>
+
+                    <b-card-title>{{ $t("frontends") }}</b-card-title>
+
+                    <b-row>
+                        <b-col>
+                            <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
+                        </b-col>
+                        <b-col>
+                            <b-button variant="primary" @click="createFrontend()"
+                                      :disabled="!frontendNameValid(newFrontendName)">
+                                {{ $t("create") }}
+                            </b-button>
+                        </b-col>
+                    </b-row>
+
+                    <hr/>
+
+                    <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
+                    <b-list-group v-else>
+                        <FrontendListItem v-for="frontend in frontends"
+                                          :key="frontend.name" :frontend="frontend"></FrontendListItem>
+                    </b-list-group>
+
+                </b-card>
+            </b-tab>
+        </b-tabs>
+    </div>
 </template>
 
 <script>
@@ -62,61 +125,116 @@ import JobListItem from "@/components/JobListItem";
 import {formatBindAddress} from "@/util";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import FrontendListItem from "@/components/FrontendListItem";
+import SearchBackendListItem from "@/components/SearchBackendListItem.vue";
+import UserScriptListItem from "@/components/UserScriptListItem.vue";
 
 export default {
-  name: "Jobs",
-  components: {JobListItem, FrontendListItem},
-  data() {
-    return {
-      jobsLoading: true,
-      newJobName: "",
-      jobs: [],
+    name: "Jobs",
+    components: {UserScriptListItem, SearchBackendListItem, JobListItem, FrontendListItem},
+    data() {
+        return {
+            jobsLoading: true,
+            newJobName: "",
+            jobs: [],
 
-      frontendsLoading: true,
-      frontends: [],
-      formatBindAddress,
-      newFrontendName: "",
+            frontendsLoading: true,
+            frontends: [],
+            formatBindAddress,
+            newFrontendName: "",
 
-      showHelp: false
+            backends: [],
+            backendsLoading: true,
+            newBackendName: "",
+
+            scripts: [],
+            scriptTemplates: [],
+            newScriptName: "",
+            scriptTemplate: null,
+            scriptsLoading: true,
+
+            showHelp: false,
+            tab: 0
+        }
+    },
+    mounted() {
+        this.loading = true;
+        if (this.$route.params.tab) {
+            console.log("mounted " + this.$route.params.tab)
+            window.setTimeout(() => {
+                this.tab = Math.round(Number(this.$route.params.tab));
+            }, 1)
+        }
+        this.reload();
+    },
+    methods: {
+        jobNameValid(name) {
+            if (this.jobs.some(job => job.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        frontendNameValid(name) {
+            if (this.frontends.some(frontend => frontend.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        backendNameValid(name) {
+            if (this.backends.some(backend => backend.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        scriptNameValid(name) {
+            if (this.scripts.some(script => script.name === name)) {
+                return false;
+            }
+            if (name.length > 16) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        reload() {
+            Sist2AdminApi.getJobs().then(resp => {
+                this.jobs = resp.data;
+                this.jobsLoading = false;
+                this.showHelp = this.jobs.length === 0;
+            });
+            Sist2AdminApi.getFrontends().then(resp => {
+                this.frontends = resp.data;
+                this.frontendsLoading = false;
+            });
+            Sist2AdminApi.getSearchBackends().then(resp => {
+                this.backends = resp.data;
+                this.backendsLoading = false;
+            })
+            Sist2AdminApi.getUserScripts().then(resp => {
+                this.scripts = resp.data;
+                this.scriptTemplates = this.$store.state.sist2AdminInfo.user_script_templates;
+                this.scriptsLoading = false;
+            })
+        },
+        createJob() {
+            Sist2AdminApi.createJob(this.newJobName).then(this.reload);
+        },
+        createFrontend() {
+            Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
+        },
+        createBackend() {
+            Sist2AdminApi.createBackend(this.newBackendName).then(this.reload);
+        },
+        createScript() {
+            Sist2AdminApi.createUserScript(this.newScriptName, this.scriptTemplate).then(this.reload)
+        },
+        onTabChange(tab) {
+            if (this.$route.params.tab != tab) {
+                this.$router.push({params: {tab: tab}})
+            }
+        }
     }
-  },
-  mounted() {
-    this.loading = true;
-    this.reload();
-  },
-  methods: {
-    jobNameValid(name) {
-      if (this.jobs.some(job => job.name === name)) {
-        return false;
-      }
-
-      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
-    },
-    frontendNameValid(name) {
-      if (this.frontends.some(job => job.name === name)) {
-        return false;
-      }
-
-      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
-    },
-    reload() {
-      Sist2AdminApi.getJobs().then(resp => {
-        this.jobs = resp.data;
-        this.jobsLoading = false;
-
-        this.showHelp = this.jobs.length === 0;
-      });
-      Sist2AdminApi.getFrontends().then(resp => {
-        this.frontends = resp.data;
-        this.frontendsLoading = false;
-      });
-    },
-    createJob() {
-      Sist2AdminApi.createJob(this.newJobName).then(this.reload);
-    },
-    createFrontend() {
-      Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
-    }
-  }
 }
 </script>

sist2-admin/frontend/src/views/Job.vue

@@ -1,92 +1,125 @@
 <template>
-  <b-card>
-    <b-card-title>
-      [{{ getName() }}]
-      {{ $t("jobTitle") }}
-    </b-card-title>
+    <b-card>
+        <b-card-title>
+            [{{ getName() }}]
+            {{ $t("jobTitle") }}
+        </b-card-title>
 
-    <div class="mb-3">
-      <b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
-      <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
-    </div>
+        <div class="mb-3">
+            <b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
+            <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
+        </div>
 
-    <div v-if="job">
-      {{ $t("status") }}: <code>{{ job.status }}</code>
-    </div>
+        <div v-if="job">
+            {{ $t("status") }}: <code>{{ job.status }}</code>
+        </div>
 
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <b-card-body v-else>
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-card-body v-else>
 
-      <h4>{{ $t("jobOptions.title") }}</h4>
-      <b-card>
-        <JobOptions :job="job" @change="update"></JobOptions>
-      </b-card>
+            <h4>{{ $t("jobOptions.title") }}</h4>
+            <b-card>
+                <JobOptions :job="job" @change="update"></JobOptions>
+            </b-card>
 
-      <br/>
+            <br/>
 
-      <h4>{{ $t("scanOptions.title") }}</h4>
-      <b-card>
-        <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
-      </b-card>
+            <h4>{{ $t("backendOptions.title") }}</h4>
+            <b-card>
+                <b-alert v-if="!valid" variant="warning" show>{{ $t("jobOptions.noBackendError") }}</b-alert>
+                <SearchBackendSelect :value="job.index_options.search_backend"
+                                     @change="onBackendSelect($event)"></SearchBackendSelect>
+            </b-card>
+            <br/>
 
-      <br/>
+            <h4>{{ $t("scriptOptions") }}</h4>
+            <b-card>
+                <UserScriptPicker :selected-scripts="job.user_scripts"
+                                  @change="onScriptChange($event)"></UserScriptPicker>
+            </b-card>
 
-      <h4>{{ $t("indexOptions.title") }}</h4>
-      <b-card>
-        <IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
-      </b-card>
+            <br/>
 
-    </b-card-body>
+            <h4>{{ $t("scanOptions.title") }}</h4>
+            <b-card>
+                <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
+            </b-card>
 
-  </b-card>
+        </b-card-body>
+
+    </b-card>
 </template>
 
 <script>
 import ScanOptions from "@/components/ScanOptions";
 import Sist2AdminApi from "@/Sist2AdminApi";
-import IndexOptions from "@/components/IndexOptions";
 import JobOptions from "@/components/JobOptions";
+import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
+import UserScriptPicker from "@/components/UserScriptPicker.vue";
 
 export default {
-  name: "Job",
-  components: {
-    IndexOptions,
-    ScanOptions,
-    JobOptions
-  },
-  data() {
-    return {
-      loading: true,
-      job: null
+    name: "Job",
+    components: {
+        UserScriptPicker,
+        SearchBackendSelect,
+        ScanOptions,
+        JobOptions
+    },
+    data() {
+        return {
+            loading: true,
+            job: null,
+        }
+    },
+    methods: {
+        getName() {
+            return this.$route.params.name;
+        },
+        update() {
+            Sist2AdminApi.updateJob(this.getName(), this.job);
+        },
+        runJob() {
+            Sist2AdminApi.runJob(this.getName()).then(() => {
+                this.$bvToast.toast(this.$t("runJobConfirmation"), {
+                    title: this.$t("runJobConfirmationTitle"),
+                    variant: "success",
+                    toaster: "b-toaster-bottom-right"
+                });
+            });
+        },
+        deleteJob() {
+            Sist2AdminApi.deleteJob(this.getName())
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete job " +
+                        "because it is referenced by a frontend", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        },
+        onBackendSelect(backend) {
+            this.job.index_options.search_backend = backend;
+            this.update();
+        },
+        onScriptChange(scripts) {
+            this.job.user_scripts = scripts;
+            this.update();
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getJob(this.getName()).then(resp => {
+            this.loading = false;
+            this.job = resp.data;
+        })
+    },
+    computed: {
+        valid() {
+            return this.job?.index_options.search_backend != null;
+        }
     }
-  },
-  methods: {
-    getName() {
-      return this.$route.params.name;
-    },
-    update() {
-      Sist2AdminApi.updateJob(this.getName(), this.job);
-    },
-    runJob() {
-      Sist2AdminApi.runJob(this.getName()).then(() => {
-        this.$bvToast.toast(this.$t("runJobConfirmation"), {
-          title: this.$t("runJobConfirmationTitle"),
-          variant: "success",
-          toaster: "b-toaster-bottom-right"
-        });
-      });
-    },
-    deleteJob() {
-      Sist2AdminApi.deleteJob(this.getName()).then(() => {
-        this.$router.push("/");
-      })
-    }
-  },
-  mounted() {
-    Sist2AdminApi.getJob(this.getName()).then(resp => {
-      this.loading = false;
-      this.job = resp.data;
-    })
-  }
 }
 </script>

sist2-admin/frontend/src/views/SearchBackend.vue

@@ -0,0 +1,123 @@
+<template>
+
+    <b-card>
+        <b-card-title>
+            <span class="text-monospace">{{ getName() }}</span>
+            {{ $t("searchBackendTitle") }}
+        </b-card-title>
+
+        <div class="mb-3">
+            <b-button variant="danger" @click="deleteBackend()">{{ $t("delete") }}</b-button>
+        </div>
+
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-card-body v-else>
+
+            <label>{{ $t("backendOptions.type") }}</label>
+            <b-select :options="backendTypeOptions" v-model="backend.backend_type" @change="update()"></b-select>
+
+            <hr/>
+
+            <template v-if="backend.backend_type === 'elasticsearch'">
+                <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+                    {{ esTestMessage }}
+                </b-alert>
+
+                <label>{{ $t("backendOptions.esUrl") }}</label>
+                <b-input-group>
+                    <b-form-input v-model="backend.es_url" @change="update()"></b-form-input>
+                    <b-input-group-append>
+                        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+                    </b-input-group-append>
+                </b-input-group>
+
+                <b-form-checkbox v-model="backend.es_insecure_ssl" :disabled="!this.backend.es_url.startsWith('https')"
+                                 @change="update()">
+                    {{ $t("backendOptions.esInsecure") }}
+                </b-form-checkbox>
+
+                <label>{{ $t("backendOptions.esIndex") }}</label>
+                <b-form-input v-model="backend.es_index" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.threads") }}</label>
+                <b-form-input v-model="backend.threads" type="number" min="1" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.batchSize") }}</label>
+                <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
+            </template>
+            <template v-else>
+                <label>{{ $t("backendOptions.searchIndex") }}</label>
+                <b-form-input v-model="backend.search_index" disabled></b-form-input>
+            </template>
+        </b-card-body>
+
+    </b-card>
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "SearchBackend",
+    data() {
+        return {
+            showEsTestAlert: false,
+            esTestOk: false,
+            esTestMessage: "",
+            loading: true,
+            backend: null,
+            backendTypeOptions: [
+                {
+                    text: "Elasticsearch",
+                    value: "elasticsearch"
+                },
+                {
+                    text: "SQLite",
+                    value: "sqlite"
+                }
+            ]
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getSearchBackend(this.getName()).then(resp => {
+            this.backend = resp.data;
+            this.loading = false;
+        });
+    },
+    methods: {
+        getName() {
+            return this.$route.params.name;
+        },
+        testEs() {
+            sist2AdminApi.pingEs(this.backend.es_url, this.backend.es_insecure_ssl)
+                .then((resp) => {
+                    this.showEsTestAlert = true;
+                    this.esTestOk = resp.data.ok;
+                    this.esTestMessage = resp.data.message;
+                });
+        },
+        update() {
+            Sist2AdminApi.updateSearchBackend(this.getName(), this.backend);
+        },
+        deleteBackend() {
+            Sist2AdminApi.deleteBackend(this.getName())
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete search backend " +
+                        "because it is referenced by a job or frontend", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        }
+    }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/views/Tail.vue

@@ -1,168 +1,175 @@
 <template>
-  <b-card>
-    <b-card-body>
+    <b-card>
+        <b-card-body>
 
-      <h4 class="mb-3">{{ taskId }} {{ $t("logs") }}</h4>
+            <h4 class="mb-3">{{ taskId }} {{ $t("logs") }}</h4>
 
-      <div v-if="$store.state.sist2AdminInfo">
-        {{ $t("logFile") }}
-        <code>{{ $store.state.sist2AdminInfo.logs_folder }}/sist2-{{ taskId }}.log</code>
-        <br/>
-        <br/>
-      </div>
+            <div v-if="$store.state.sist2AdminInfo">
+                {{ $t("logFile") }}
+                <code>{{ $store.state.sist2AdminInfo.logs_folder }}/sist2-{{ taskId }}.log</code>
+                <br/>
+                <br/>
+            </div>
 
-      <b-row>
-        <b-col>
-          <span>{{ $t("logLevel") }}</span>
-          <b-select :options="levels.slice(0, -1)" v-model="logLevel" @input="connect()"></b-select>
-        </b-col>
-        <b-col>
-          <span>{{ $t("logMode") }}</span>
-          <b-select :options="modeOptions" v-model="mode" @input="connect()"></b-select>
-        </b-col>
-      </b-row>
+            <b-row>
+                <b-col>
+                    <span>{{ $t("logLevel") }}</span>
+                    <b-select :options="levels.slice(0, -1)" v-model="logLevel" @input="connect()"></b-select>
+                </b-col>
+                <b-col>
+                    <span>{{ $t("logMode") }}</span>
+                    <b-select :options="modeOptions" v-model="mode" @input="connect()"></b-select>
+                </b-col>
+            </b-row>
 
-      <div id="log-tail-output" class="mt-3 ml-1"></div>
+            <div id="log-tail-output" class="mt-3 ml-1"></div>
 
-    </b-card-body>
-  </b-card>
+        </b-card-body>
+    </b-card>
 </template>
 
 <script>
 
 export default {
-  name: "Tail",
-  data() {
-    return {
-      logLevel: "DEBUG",
-      levels: ["DEBUG", "INFO", "WARNING", "ERROR", "ADMIN", "FATAL"],
-      socket: null,
-      mode: "follow",
-      modeOptions: [
-        {
-          "text": this.$t('follow'),
-          "value": "follow"
-        },
-        {
-          "text": this.$t('wholeFile'),
-          "value": "wholeFile"
+    name: "Tail",
+    data() {
+        return {
+            logLevel: "DEBUG",
+            levels: ["DEBUG", "INFO", "WARNING", "ERROR", "ADMIN", "FATAL"],
+            socket: null,
+            mode: "follow",
+            modeOptions: [
+                {
+                    "text": this.$t('follow'),
+                    "value": "follow"
+                },
+                {
+                    "text": this.$t('wholeFile'),
+                    "value": "wholeFile"
+                }
+            ]
         }
-      ]
+    },
+    computed: {
+        taskId: function () {
+            return this.$route.params.taskId;
+        }
+    },
+    methods: {
+        connect() {
+            let lineCount = 0;
+            const outputElem = document.getElementById("log-tail-output")
+            outputElem.replaceChildren();
+            if (this.socket !== null) {
+                this.socket.close();
+            }
+
+            const n = this.mode === "follow" ? 32 : 9999999999;
+            if (window.location.protocol === "https:") {
+                this.socket = new WebSocket(`wss://${window.location.host}/log/${this.taskId}?n=${n}`);
+            } else {
+                this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
+            }
+            this.socket.onopen = () => {
+                this.socket.send("Hello from client");
+            }
+
+            this.socket.onmessage = e => {
+                let message;
+                try {
+                    message = JSON.parse(e.data);
+                } catch {
+                    console.error(e.data)
+                    return;
+                }
+
+                if ("ping" in message) {
+                    return;
+                }
+
+                if (message.level === undefined) {
+
+                    if ("stderr" in message) {
+                        message.level = "ERROR";
+                        message.message = message["stderr"];
+                    } else if ("stdout" in message) {
+                        message.level = "INFO";
+                        message.message = message["stdout"];
+                    } else {
+                        message.level = "ADMIN";
+                        message.message = message["sist2-admin"];
+                    }
+                    message.datetime = ""
+                    message.filepath = ""
+                }
+
+                if (this.levels.indexOf(message.level) < this.levels.indexOf(this.logLevel)) {
+                    return;
+                }
+
+                const logLine = `${message.datetime} [${message.level} ${message.filepath}] ${message.message}`;
+
+                const span = document.createElement("span");
+                span.setAttribute("class", message.level);
+                span.appendChild(document.createTextNode(logLine));
+
+                outputElem.appendChild(span);
+                lineCount += 1;
+
+                if (this.mode === "follow" && lineCount >= n) {
+                    outputElem.firstChild.remove();
+                }
+            }
+        }
+    },
+    mounted() {
+        this.connect()
     }
-  },
-  computed: {
-    taskId: function () {
-      return this.$route.params.taskId;
-    }
-  },
-  methods: {
-    connect() {
-      let lineCount = 0;
-      const outputElem = document.getElementById("log-tail-output")
-      outputElem.replaceChildren();
-      if (this.socket !== null) {
-        this.socket.close();
-      }
-
-      const n = this.mode === "follow" ? 32 : 9999999999;
-      this.socket = new WebSocket(`ws://${window.location.host}/log/${this.taskId}?n=${n}`);
-      this.socket.onopen = () => {
-        this.socket.send("Hello from client");
-      }
-
-      this.socket.onmessage = e => {
-        let message;
-        try {
-          message = JSON.parse(e.data);
-        } catch {
-          console.error(e.data)
-          return;
-        }
-
-        if ("ping" in message) {
-          return;
-        }
-
-        if (message.level === undefined) {
-
-          if ("stderr" in message) {
-            message.level = "ERROR";
-            message.message = message["stderr"];
-          } else {
-            message.level = "ADMIN";
-            message.message = message["sist2-admin"];
-          }
-          message.datetime = ""
-          message.filepath = ""
-        }
-
-        if (this.levels.indexOf(message.level) < this.levels.indexOf(this.logLevel)) {
-          return;
-        }
-
-        const logLine = `${message.datetime} [${message.level} ${message.filepath}] ${message.message}`;
-
-        const span = document.createElement("span");
-        span.setAttribute("class", message.level);
-        span.appendChild(document.createTextNode(logLine));
-
-        outputElem.appendChild(span);
-        lineCount += 1;
-
-        if (this.mode === "follow" && lineCount >= n) {
-          outputElem.firstChild.remove();
-        }
-      }
-    }
-  },
-  mounted() {
-    this.connect()
-  }
 }
 
 </script>
 
 <style>
 #log-tail-output span {
-  display: block;
+    display: block;
 }
 
 span.DEBUG {
-  color: #9E9E9E;
+    color: #9E9E9E;
 }
 
 span.WARNING {
-  color: #FFB300;
+    color: #FFB300;
 }
 
 span.INFO {
-  color: #039BE5;
+    color: #039BE5;
 }
 
 span.ERROR {
-  color: #F4511E;
+    color: #F4511E;
 }
 
 span.FATAL {
-  color: #F4511E;
+    color: #F4511E;
 }
 
 span.ADMIN {
-  color: #ee05ff;
+    color: #ee05ff;
 }
 
 
 #log-tail-output {
-  font-size: 13px;
-  font-family: monospace;
+    font-size: 13px;
+    font-family: monospace;
 
-  padding: 6px;
-  background-color: #f5f5f5;
-  border: 1px solid #ccc;
-  border-radius: 4px;
-  margin: 3px;
-  white-space: pre;
-  color: #000;
-  overflow: hidden;
+    padding: 6px;
+    background-color: #f5f5f5;
+    border: 1px solid #ccc;
+    border-radius: 4px;
+    margin: 3px;
+    white-space: pre;
+    color: #000;
+    overflow: hidden;
 }
 </style>

sist2-admin/frontend/src/views/Tasks.vue

@@ -1,38 +1,49 @@
 <template>
-  <div>
+    <div>
 
-    <b-card v-if="tasks.length > 0">
-      <h2>{{ $t("runningTasks") }}</h2>
-      <b-list-group>
-        <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
-      </b-list-group>
-    </b-card>
+        <b-card v-if="tasks.length > 0">
+            <h2>{{ $t("runningTasks") }}</h2>
+            <b-list-group>
+                <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
+            </b-list-group>
+        </b-card>
 
-    <b-card class="mt-4">
+        <b-card class="mt-4">
 
-      <b-card-title>{{ $t("taskHistory") }}</b-card-title>
+            <b-card-title>{{ $t("taskHistory") }}</b-card-title>
 
-      <br/>
+            <br/>
 
-      <b-table
-          id="task-history"
-          :items="historyItems"
-          :fields="historyFields"
-          :current-page="historyCurrentPage"
-          :tbody-tr-class="rowClass"
-          :per-page="10"
-      >
-        <template #cell(logs)="data">
-          <router-link :to="`/log/${data.item.logs}`">{{ $t("logs") }}</router-link>
-        </template>
+            <b-table
+                    id="task-history"
+                    :items="historyItems"
+                    :fields="historyFields"
+                    :current-page="historyCurrentPage"
+                    :tbody-tr-class="rowClass"
+                    :per-page="10"
+            >
+                <template #cell(logs)="data">
+                    <template v-if="data.item._row.has_logs">
+                        <b-button variant="link" size="sm" :to="`/log/${data.item.id}`">
+                            {{ $t("view") }}
+                        </b-button>
+                        /
+                        <b-button variant="link" size="sm" @click="deleteLogs(data.item.id)">
+                            {{ $t("delete") }}
+                        </b-button>
+                    </template>
+                </template>
 
-      </b-table>
+                <template #cell(delete)="data">
+                </template>
 
-      <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
-                    :per-page="10"></b-pagination>
+            </b-table>
 
-    </b-card>
-  </div>
+            <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
+                          :per-page="10"></b-pagination>
+
+        </b-card>
+    </div>
 </template>
 
 <script>
@@ -45,106 +56,116 @@ const HOUR = 3600;
 const MINUTE = 60;
 
 function humanDuration(sec_num) {
-  sec_num = sec_num / 1000;
-  const days = Math.floor(sec_num / DAY);
-  sec_num -= days * DAY;
-  const hours = Math.floor(sec_num / HOUR);
-  sec_num -= hours * HOUR;
-  const minutes = Math.floor(sec_num / MINUTE);
-  sec_num -= minutes * MINUTE;
-  const seconds = Math.floor(sec_num);
+    sec_num = sec_num / 1000;
+    const days = Math.floor(sec_num / DAY);
+    sec_num -= days * DAY;
+    const hours = Math.floor(sec_num / HOUR);
+    sec_num -= hours * HOUR;
+    const minutes = Math.floor(sec_num / MINUTE);
+    sec_num -= minutes * MINUTE;
+    const seconds = Math.floor(sec_num);
 
-  if (days > 0) {
-    return `${days} days ${hours}h ${minutes}m ${seconds}s`;
-  }
+    if (days > 0) {
+        return `${days} days ${hours}h ${minutes}m ${seconds}s`;
+    }
 
-  if (hours > 0) {
-    return `${hours}h ${minutes}m ${seconds}s`;
-  }
+    if (hours > 0) {
+        return `${hours}h ${minutes}m ${seconds}s`;
+    }
 
-  if (minutes > 0) {
-    return `${minutes}m ${seconds}s`;
-  }
+    if (minutes > 0) {
+        return `${minutes}m ${seconds}s`;
+    }
 
-  if (seconds > 0) {
-    return `${seconds}s`;
-  }
+    if (seconds > 0) {
+        return `${seconds}s`;
+    }
 
-  return "<0s";
+    return "<0s";
 }
 
 export default {
-  name: 'Tasks',
-  components: {TaskListItem},
-  data() {
-    return {
-      loading: true,
-      tasks: [],
-      taskHistory: [],
-      timerId: null,
-      historyFields: [
-        {key: "name", label: this.$t("taskName")},
-        {key: "time", label: this.$t("taskStarted")},
-        {key: "duration", label: this.$t("taskDuration")},
-        {key: "status", label: this.$t("taskStatus")},
-        {key: "logs", label: this.$t("logs")},
-      ],
-      historyCurrentPage: 1,
-      historyItems: []
-    }
-  },
-  props: {
-    msg: String
-  },
-  mounted() {
-    this.loading = true;
-    this.update().then(() => this.loading = false);
+    name: 'Tasks',
+    components: {TaskListItem},
+    data() {
+        return {
+            loading: true,
+            tasks: [],
+            taskHistory: [],
+            timerId: null,
+            historyFields: [
+                {key: "name", label: this.$t("taskName")},
+                {key: "time", label: this.$t("taskStarted")},
+                {key: "duration", label: this.$t("taskDuration")},
+                {key: "status", label: this.$t("taskStatus")},
+                {key: "logs", label: this.$t("logs")},
+            ],
+            historyCurrentPage: 1,
+            historyItems: []
+        }
+    },
+    props: {
+        msg: String
+    },
+    mounted() {
+        this.loading = true;
+        this.update().then(() => this.loading = false);
 
-    this.timerId = window.setInterval(this.update, 1000);
-    this.updateHistory();
-  },
-  destroyed() {
-    if (this.timerId) {
-      window.clearInterval(this.timerId);
-    }
-  },
-  methods: {
-    rowClass(row) {
-      if (row.status === "failed") {
-        return "table-danger";
-      }
-      return null;
+        this.timerId = window.setInterval(this.update, 1000);
+        this.updateHistory();
     },
-    updateHistory() {
-      Sist2AdminApi.getTaskHistory().then(resp => {
-        this.historyItems = resp.data.map(row => ({
-          id: row.id,
-          name: row.name,
-          duration: this.taskDuration(row),
-          time: moment(row.started).format("dd, MMM Do YYYY, HH:mm:ss"),
-          logs: row.id,
-          status: row.return_code === 0 ? "ok" : "failed"
-        }));
-      });
+    destroyed() {
+        if (this.timerId) {
+            window.clearInterval(this.timerId);
+        }
     },
-    update() {
-      return Sist2AdminApi.getTasks().then(resp => {
-        this.tasks = resp.data;
-      })
-    },
-    taskDuration(task) {
-      const start = moment.utc(task.started);
-      const end = moment.utc(task.ended);
+    methods: {
+        rowClass(row) {
+            if (row.status === "failed") {
+                return "table-danger";
+            }
+            return null;
+        },
+        updateHistory() {
+            Sist2AdminApi.getTaskHistory().then(resp => {
+                this.historyItems = resp.data.map(row => ({
+                    id: row.id,
+                    name: row.name,
+                    duration: this.taskDuration(row),
+                    time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"),
+                    logs: null,
+                    status: [0,1].includes(row.return_code) ? "ok" : "failed",
+                    _row: row
+                }));
+            });
+        },
+        update() {
+            return Sist2AdminApi.getTasks().then(resp => {
+                this.tasks = resp.data;
+            })
+        },
+        taskDuration(task) {
+            const start = moment.utc(task.started);
+            const end = moment.utc(task.ended);
 
-      return humanDuration(end.diff(start))
+            return humanDuration(end.diff(start))
+        },
+        deleteLogs(taskId) {
+            Sist2AdminApi.deleteTaskLogs(taskId).then(() => {
+                this.updateHistory();
+            })
+        }
     }
-  }
 }
 </script>
 
 <style scoped>
 #task-history {
-  font-family: monospace;
-  font-size: 12px;
+    font-family: monospace;
+    font-size: 12px;
+}
+
+.btn-link {
+    padding: 0;
 }
 </style>

sist2-admin/frontend/src/views/UserScript.vue

@@ -0,0 +1,117 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card v-else>
+        <b-card-title>
+            {{ $route.params.name }}
+            {{ $t("script") }}
+        </b-card-title>
+
+        <div class="mb-3">
+            <b-button variant="danger" @click="deleteScript()">{{ $t("delete") }}</b-button>
+        </div>
+
+        <b-card>
+            <h5>{{ $t("testScript") }}</h5>
+
+            <b-row>
+                <b-col cols="11">
+                    <JobSelect @change="onJobSelect($event)"></JobSelect>
+                </b-col>
+                <b-col cols="1">
+                    <b-button :disabled="!selectedTestJob" variant="primary" @click="testScript()">{{ $t("test") }}
+                    </b-button>
+                </b-col>
+            </b-row>
+
+        </b-card>
+        <br/>
+
+        <label>{{ $t("scriptType") }}</label>
+        <b-form-select :options="['git', 'simple']" v-model="script.type" @change="update()"></b-form-select>
+
+        <template v-if="script.type === 'git'">
+            <label>{{ $t("gitRepository") }}</label>
+            <b-form-input v-model="script.git_repository" placeholder="https://github.com/example/example.git"
+                          @change="update()"></b-form-input>
+
+            <label>{{ $t("extraArgs") }}</label>
+            <b-form-input v-model="script.extra_args" @change="update()" class="text-monospace"></b-form-input>
+        </template>
+
+        <template v-if="script.type === 'simple'">
+
+            <label>{{ $t("scriptCode") }}</label>
+            <p>Find sist2-python documentation <a href="https://sist2-python.readthedocs.io/" target="_blank">here</a></p>
+            <b-textarea rows="15" class="text-monospace" v-model="script.script" @change="update()" spellcheck="false"></b-textarea>
+        </template>
+
+        <template v-if="script.type === 'local'">
+            <!-- TODO-->
+        </template>
+
+
+    </b-card>
+</template>
+
+<script>
+
+import Sist2AdminApi from "@/Sist2AdminApi";
+import JobOptions from "@/components/JobOptions.vue";
+import JobCheckboxGroup from "@/components/JobCheckboxGroup.vue";
+import JobSelect from "@/components/JobSelect.vue";
+
+export default {
+    name: "UserScript",
+    components: {JobSelect, JobCheckboxGroup, JobOptions},
+    data() {
+        return {
+            loading: true,
+            script: null,
+            selectedTestJob: null
+        }
+    },
+    methods: {
+        update() {
+            Sist2AdminApi.updateUserScript(this.name, this.script);
+        },
+        onJobSelect(job) {
+            this.selectedTestJob = job;
+        },
+        deleteScript() {
+            Sist2AdminApi.deleteUserScript(this.name)
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete user script " +
+                        "because it is referenced by a job", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        },
+        testScript() {
+            Sist2AdminApi.testUserScript(this.name, this.selectedTestJob)
+                .then(() => {
+                    this.$bvToast.toast(this.$t("runJobConfirmation"), {
+                        title: this.$t("runJobConfirmationTitle"),
+                        variant: "success",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getUserScript(this.name).then(resp => {
+            this.script = resp.data;
+            this.loading = false;
+        });
+    },
+    computed: {
+        name() {
+            return this.$route.params.name;
+        },
+    },
+}
+</script>

sist2-admin/requirements.txt

@@ -2,4 +2,6 @@ fastapi
 git+https://github.com/simon987/hexlib.git
 uvicorn
 websockets
-pycron
+pycron
+GitPython
+git+https://github.com/simon987/sist2-python.git

sist2-admin/sist2_admin/app.py

@@ -18,11 +18,13 @@ from websockets.exceptions import ConnectionClosed
 
 import cron
 from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
-from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
+from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus, Sist2UserScriptTask
 from notifications import Subscribe, Notifications
-from sist2 import Sist2
-from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
+from sist2 import Sist2, Sist2SearchBackend
+from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION, migrate_v3_to_v4, \
+    get_log_files_to_remove, delete_log_file, create_default_search_backends
 from web import Sist2Frontend
+from script import UserScript, SCRIPT_TEMPLATES
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
 db = PersistentState(dbfile=os.path.join(DATA_FOLDER, "state.db"))
@@ -51,7 +53,8 @@ async def home():
 async def api():
     return {
         "tesseract_langs": TESSERACT_LANGS,
-        "logs_folder": LOG_FOLDER
+        "logs_folder": LOG_FOLDER,
+        "user_script_templates": list(SCRIPT_TEMPLATES.keys())
     }
 
 
@@ -73,16 +76,14 @@ async def get_frontend(name: str):
     raise HTTPException(status_code=404)
 
 
-@app.get("/api/job/")
+@app.get("/api/job")
 async def get_jobs():
     return list(db["jobs"])
 
 
 @app.put("/api/job/{name:str}")
 async def update_job(name: str, new_job: Sist2Job):
-    # TODO: Check etag
-
-    new_job.last_modified = datetime.now()
+    new_job.last_modified = datetime.utcnow()
     job = db["jobs"][name]
     if not job:
         raise HTTPException(status_code=404)
@@ -114,12 +115,10 @@ async def update_job(name: str, new_job: Sist2Job):
 async def update_frontend(name: str, frontend: Sist2Frontend):
     db["frontends"][name] = frontend
 
-    # TODO: Check etag
-
     return "ok"
 
 
-@app.get("/api/task/")
+@app.get("/api/task")
 async def get_tasks():
     return list(map(lambda t: t.json(), task_queue.tasks()))
 
@@ -134,16 +133,38 @@ async def kill_job(task_id: str):
     return task_queue.kill_task(task_id)
 
 
+@app.post("/api/task/{task_id:str}/delete_logs")
+async def delete_task_logs(task_id: str):
+    if not db["task_done"][task_id]:
+        raise HTTPException(status_code=404)
+
+    delete_log_file(db, task_id)
+
+    return "ok"
+
+
 def _run_job(job: Sist2Job):
-    job.last_modified = datetime.now()
+    job.last_modified = datetime.utcnow()
     if job.status == JobStatus("created"):
         job.status = JobStatus("started")
     db["jobs"][job.name] = job
 
     scan_task = Sist2ScanTask(job, f"Scan [{job.name}]")
-    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=scan_task)
+
+    index_depends_on = scan_task
+    script_tasks = []
+    for script_name in job.user_scripts:
+        script = db["user_scripts"][script_name]
+
+        task = Sist2UserScriptTask(script, job, f"Script <{script_name}> [{job.name}]", depends_on=scan_task)
+        script_tasks.append(task)
+        index_depends_on = task
+
+    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=index_depends_on)
 
     task_queue.submit(scan_task)
+    for task in script_tasks:
+        task_queue.submit(task)
     task_queue.submit(index_task)
 
 
@@ -158,14 +179,45 @@ async def run_job(name: str):
     return "ok"
 
 
+@app.get("/api/user_script/{name:str}/run")
+def run_user_script(name: str, job: str):
+    script = db["user_scripts"][name]
+    if not script:
+        raise HTTPException(status_code=404)
+    job = db["jobs"][job]
+    if not job:
+        raise HTTPException(status_code=404)
+
+    script_task = Sist2UserScriptTask(script, job, f"Script <{name}> [{job.name}]")
+
+    task_queue.submit(script_task)
+
+    return "ok"
+
+
+@app.get("/api/job/{name:str}/logs_to_delete")
+async def task_history(n: int, name: str):
+    return get_log_files_to_remove(db, name, n)
+
+
 @app.delete("/api/job/{name:str}")
 async def delete_job(name: str):
-    job = db["jobs"][name]
-    if job:
-        del db["jobs"][name]
-    else:
+    job: Sist2Job = db["jobs"][name]
+    if not job:
         raise HTTPException(status_code=404)
 
+    if any(name in frontend.jobs for frontend in db["frontends"]):
+        raise HTTPException(status_code=400, detail="in use (frontend)")
+
+    try:
+        os.remove(job.previous_index)
+    except:
+        pass
+
+    del db["jobs"][name]
+
+    return "ok"
+
 
 @app.delete("/api/frontend/{name:str}")
 async def delete_frontend(name: str):
@@ -215,7 +267,7 @@ def check_es_version(es_url: str, insecure: bool):
             es_url = f"{url.scheme}://{url.hostname}:{url.port}"
         else:
             auth = None
-        r = requests.get(es_url, verify=insecure, auth=auth)
+        r = requests.get(es_url, verify=not insecure, auth=auth)
     except SSLError:
         return {
             "ok": False,
@@ -251,9 +303,21 @@ def check_es_version(es_url: str, insecure: bool):
 
 
 def start_frontend_(frontend: Sist2Frontend):
-    frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
+    frontend.web_options.indices = [
+        os.path.join(DATA_FOLDER, db["jobs"][j].index_path)
+        for j in frontend.jobs
+    ]
 
-    pid = sist2.web(frontend.web_options, frontend.name)
+    backend_name = frontend.web_options.search_backend
+    search_backend = db["search_backends"][backend_name]
+    if search_backend is None:
+        logger.error(
+            f"Error while running task: search backend not found: {backend_name}")
+        return -1
+
+    logger.debug(f"Fetched search backend options for {backend_name}")
+
+    pid = sist2.web(frontend.web_options, search_backend, frontend.name)
     RUNNING_FRONTENDS[frontend.name] = pid
 
 
@@ -273,7 +337,7 @@ async def stop_frontend(name: str):
         del RUNNING_FRONTENDS[name]
 
 
-@app.get("/api/frontend/")
+@app.get("/api/frontend")
 async def get_frontends():
     res = []
     for frontend in db["frontends"]:
@@ -283,6 +347,115 @@ async def get_frontends():
     return res
 
 
+@app.get("/api/search_backend")
+async def get_search_backends():
+    return list(db["search_backends"])
+
+
+@app.put("/api/search_backend/{name:str}")
+async def update_search_backend(name: str, backend: Sist2SearchBackend):
+    if not db["search_backends"][name]:
+        raise HTTPException(status_code=404)
+
+    db["search_backends"][name] = backend
+    return "ok"
+
+
+@app.get("/api/search_backend/{name:str}")
+def get_search_backend(name: str):
+    backend = db["search_backends"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    return backend
+
+
+@app.delete("/api/search_backend/{name:str}")
+def delete_search_backend(name: str):
+    backend: Sist2SearchBackend = db["search_backends"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    if any(frontend.web_options.search_backend == name for frontend in db["frontends"]):
+        raise HTTPException(status_code=400, detail="in use (frontend)")
+
+    if any(job.index_options.search_backend == name for job in db["jobs"]):
+        raise HTTPException(status_code=400, detail="in use (job)")
+
+    del db["search_backends"][name]
+
+    try:
+        os.remove(os.path.join(DATA_FOLDER, backend.search_index))
+    except:
+        pass
+
+    return "ok"
+
+
+@app.post("/api/search_backend/{name:str}")
+def create_search_backend(name: str):
+    if db["search_backends"][name] is not None:
+        return HTTPException(status_code=400, detail="already exists")
+
+    backend = Sist2SearchBackend.create_default(name)
+    db["search_backends"][name] = backend
+
+    return backend
+
+
+@app.delete("/api/user_script/{name:str}")
+def delete_user_script(name: str):
+    if db["user_scripts"][name] is None:
+        return HTTPException(status_code=404)
+
+    if any(name in job.user_scripts for job in db["jobs"]):
+        raise HTTPException(status_code=400, detail="in use (job)")
+
+    script: UserScript = db["user_scripts"][name]
+    script.delete_dir()
+
+    del db["user_scripts"][name]
+
+    return "ok"
+
+
+@app.post("/api/user_script/{name:str}")
+def create_user_script(name: str, template: str):
+    if db["user_scripts"][name] is not None:
+        return HTTPException(status_code=400, detail="already exists")
+
+    script = SCRIPT_TEMPLATES[template](name)
+    db["user_scripts"][name] = script
+
+    return script
+
+
+@app.get("/api/user_script")
+async def get_user_scripts():
+    return list(db["user_scripts"])
+
+
+@app.get("/api/user_script/{name:str}")
+async def get_user_script(name: str):
+    backend = db["user_scripts"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    return backend
+
+
+@app.put("/api/user_script/{name:str}")
+async def update_user_script(name: str, script: UserScript):
+    previous_version: UserScript = db["user_scripts"][name]
+
+    if previous_version and previous_version.git_repository != script.git_repository:
+        script.force_clone = True
+
+    db["user_scripts"][name] = script
+
+    return "ok"
+
+
 def tail(filepath: str, n: int):
     with open(filepath) as file:
 
@@ -321,7 +494,6 @@ async def ws_tail_log(websocket: WebSocket):
         async with Subscribe(notifications) as ob:
             async for notification in ob.notifications():
                 await websocket.send_json(notification)
-                print(notification)
 
     except ConnectionClosed:
         return
@@ -352,7 +524,7 @@ async def ws_tail_log(websocket: WebSocket, task_id: str, n: int):
 
 
 def main():
-    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0")
+    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0", timeout_graceful_shutdown=0)
 
 
 def initialize_db():
@@ -361,6 +533,8 @@ def initialize_db():
     frontend = Sist2Frontend.create_default("default")
     db["frontends"]["default"] = frontend
 
+    create_default_search_backends(db)
+
     logger.info("Initialized database.")
 
 
@@ -381,6 +555,13 @@ if __name__ == '__main__':
     if db["sist2_admin"]["info"]["version"] == "2":
         logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.")
         exit(-1)
+    if db["sist2_admin"]["info"]["version"] == "3":
+        logger.info("Migrating to v4 database schema")
+        migrate_v3_to_v4(db)
+
+    if db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
+        raise Exception(f"Incompatible database {db.dbfile}. "
+                        f"Automatic migration is not available, please delete the database file to continue.")
 
     start_frontends()
     cron.initialize(db, _run_job)

sist2-admin/sist2_admin/config.py

@@ -9,9 +9,11 @@ MAX_LOG_SIZE = 1 * 1024 * 1024
 SIST2_BINARY = os.environ.get("SIST2_BINARY", "/root/sist2")
 DATA_FOLDER = os.environ.get("DATA_FOLDER", "/sist2-admin/")
 LOG_FOLDER = os.path.join(DATA_FOLDER, "logs")
+SCRIPT_FOLDER = os.path.join(DATA_FOLDER, "scripts")
 WEBSERVER_PORT = 8080
 
 os.makedirs(LOG_FOLDER, exist_ok=True)
+os.makedirs(SCRIPT_FOLDER, exist_ok=True)
 os.makedirs(DATA_FOLDER, exist_ok=True)
 
 logger = logging.Logger("sist2-admin")

sist2-admin/sist2_admin/cron.py

@@ -10,7 +10,9 @@ from jobs import Sist2Job
 
 
 def _check_schedule(db: PersistentState, run_job):
-    for job in db["jobs"]:
+    jobs = list(db["jobs"])
+
+    for job in jobs:
         job: Sist2Job
 
         if job.schedule_enabled:

sist2-admin/sist2_admin/jobs.py

@@ -1,23 +1,29 @@
 import json
 import logging
 import os.path
+import shlex
 import signal
 import uuid
 from datetime import datetime
 from enum import Enum
+from io import TextIOWrapper
 from logging import FileHandler
+from subprocess import Popen
+import subprocess
 from threading import Lock, Thread
 from time import sleep
+from typing import List
 from uuid import uuid4, UUID
 
 from hexlib.db import PersistentState
 from pydantic import BaseModel
 
-from config import logger, LOG_FOLDER
+from config import logger, LOG_FOLDER, DATA_FOLDER
 from notifications import Notifications
 from sist2 import ScanOptions, IndexOptions, Sist2
-from state import RUNNING_FRONTENDS
+from state import RUNNING_FRONTENDS, get_log_files_to_remove, delete_log_file
 from web import Sist2Frontend
+from script import UserScript
 
 
 class JobStatus(Enum):
@@ -32,9 +38,13 @@ class Sist2Job(BaseModel):
     scan_options: ScanOptions
     index_options: IndexOptions
 
+    user_scripts: List[str] = []
+
     cron_expression: str
     schedule_enabled: bool = False
 
+    keep_last_n_logs: int = -1
+
     previous_index: str = None
     index_path: str = None
     previous_index_path: str = None
@@ -53,15 +63,10 @@ class Sist2Job(BaseModel):
             name=name,
             scan_options=ScanOptions(path="/"),
             index_options=IndexOptions(),
-            last_modified=datetime.now(),
+            last_modified=datetime.utcnow(),
             cron_expression="0 0 * * *"
         )
 
-    # @validator("etag", always=True)
-    # def validate_etag(cls, value, values):
-    #     s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
-    #     return md5(s.encode()).hexdigest()
-
 
 class Sist2TaskProgress:
 
@@ -111,7 +116,7 @@ class Sist2Task:
             self._logger.info(json.dumps(log_json))
 
     def run(self, sist2: Sist2, db: PersistentState):
-        self.started = datetime.now()
+        self.started = datetime.utcnow()
 
         logger.info(f"Started task {self.display_name}")
 
@@ -132,14 +137,16 @@ class Sist2ScanTask(Sist2Task):
             self.pid = pid
 
         return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
-        self.ended = datetime.now()
+        self.ended = datetime.utcnow()
 
-        if return_code != 0:
+        is_ok = return_code in (0, 1)
+
+        if not is_ok:
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
             logger.info(f"Task {self.display_name} failed ({return_code})")
         else:
             self.job.index_path = self.job.scan_options.output
-            self.job.last_index_date = datetime.now()
+            self.job.last_index_date = datetime.utcnow()
             self.job.do_full_scan = False
             db["jobs"][self.job.name] = self.job
             self._logger.info(json.dumps({"sist2-admin": f"Save last_index_date={self.job.last_index_date}"}))
@@ -147,7 +154,7 @@ class Sist2ScanTask(Sist2Task):
         logger.info(f"Completed {self.display_name} ({return_code=})")
 
         # Remove old index
-        if return_code == 0:
+        if is_ok:
             if self.job.previous_index_path is not None and self.job.previous_index_path != self.job.index_path:
                 self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index_path=}"}))
                 try:
@@ -171,12 +178,19 @@ class Sist2IndexTask(Sist2Task):
 
         self.job.index_options.path = self.job.scan_options.output
 
-        return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
-        self.ended = datetime.now()
+        search_backend = db["search_backends"][self.job.index_options.search_backend]
+        if search_backend is None:
+            logger.error(f"Error while running task: search backend not found: {self.job.index_options.search_backend}")
+            return -1
+
+        logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
+
+        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback)
+        self.ended = datetime.utcnow()
 
         duration = self.ended - self.started
 
-        ok = return_code == 0
+        ok = return_code in (0, 1)
 
         if ok:
             self.restart_running_frontends(db, sist2)
@@ -206,14 +220,84 @@ class Sist2IndexTask(Sist2Task):
             except ChildProcessError:
                 pass
 
-            frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
+            backend_name = frontend.web_options.search_backend
+            search_backend = db["search_backends"][backend_name]
+            if search_backend is None:
+                logger.error(f"Error while running task: search backend not found: {backend_name}")
+                return -1
 
-            pid = sist2.web(frontend.web_options, frontend.name)
+            logger.debug(f"Fetched search backend options for {backend_name}")
+
+            frontend.web_options.indices = [
+                os.path.join(DATA_FOLDER, db["jobs"][j].index_path)
+                for j in frontend.jobs
+            ]
+
+            pid = sist2.web(frontend.web_options, search_backend, frontend.name)
             RUNNING_FRONTENDS[frontend_name] = pid
 
             self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))
 
 
+class Sist2UserScriptTask(Sist2Task):
+
+    def __init__(self, user_script: UserScript, job: Sist2Job, display_name: str, depends_on: Sist2Task = None):
+        super().__init__(job, display_name, depends_on=depends_on.id if depends_on else None)
+        self.user_script = user_script
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        try:
+            self.user_script.setup(self.log_callback)
+        except Exception as e:
+            logger.error(f"Setup for {self.user_script.name} failed: ")
+            logger.exception(e)
+            self.log_callback({"sist2-admin": f"Setup for {self.user_script.name} failed: {e}"})
+            return -1
+
+        executable = self.user_script.get_executable()
+        index_path = os.path.join(DATA_FOLDER, self.job.index_path)
+        extra_args = self.user_script.extra_args
+
+        args = [
+            executable,
+            index_path,
+            *shlex.split(extra_args)
+        ]
+
+        self.log_callback({"sist2-admin": f"Starting user script with {executable=}, {index_path=}, {extra_args=}"})
+
+        proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.user_script.script_dir())
+        self.pid = proc.pid
+
+        t_stderr = Thread(target=self._consume_logs, args=(self.log_callback, proc, "stderr", False))
+        t_stderr.start()
+
+        self._consume_logs(self.log_callback, proc, "stdout", True)
+
+        self.ended = datetime.utcnow()
+
+        return 0
+
+    @staticmethod
+    def _consume_logs(logs_cb, proc, stream, wait):
+        pipe_wrapper = TextIOWrapper(getattr(proc, stream), encoding="utf8", errors="ignore")
+        try:
+            for line in pipe_wrapper:
+                if line.strip() == "":
+                    continue
+                if line.startswith("$PROGRESS"):
+                    progress = json.loads(line[len("$PROGRESS "):])
+                    logs_cb({"progress": progress})
+                    continue
+                logs_cb({stream: line})
+        finally:
+            if wait:
+                proc.wait()
+            pipe_wrapper.close()
+
+
 class TaskQueue:
     def __init__(self, sist2: Sist2, db: PersistentState, notifications: Notifications):
         self._lock = Lock()
@@ -232,7 +316,7 @@ class TaskQueue:
     def _tasks_failed(self):
         done = set()
 
-        for row in self._db["task_done"].sql("WHERE return_code != 0"):
+        for row in self._db["task_done"].sql("WHERE return_code NOT IN (0,1)"):
             done.add(uuid.UUID(row["id"]))
 
         return done
@@ -301,8 +385,14 @@ class TaskQueue:
                 "ended": task.ended,
                 "started": task.started,
                 "name": task.display_name,
-                "return_code": task_result
+                "return_code": task_result,
+                "has_logs": 1
             }
+
+            logs_to_delete = get_log_files_to_remove(self._db, task.job.name, task.job.keep_last_n_logs)
+            for row in logs_to_delete:
+                delete_log_file(self._db, row["id"])
+
         if isinstance(task, Sist2IndexTask):
             self._notifications.notify({
                 "message": "notifications.indexCompleted",

sist2-admin/sist2_admin/script.py

@@ -0,0 +1,126 @@
+import os
+import shutil
+import stat
+import subprocess
+from enum import Enum
+
+from git import Repo
+from pydantic import BaseModel
+
+from config import SCRIPT_FOLDER
+
+
+class ScriptType(Enum):
+    LOCAL = "local"
+    SIMPLE = "simple"
+    GIT = "git"
+
+
+def set_executable(file):
+    os.chmod(file, os.stat(file).st_mode | stat.S_IEXEC)
+
+
+def _initialize_git_repository(url, path, log_cb, force_clone):
+    log_cb({"sist2-admin": f"Cloning {url}"})
+
+    if force_clone or not os.path.exists(os.path.join(path, ".git")):
+        if force_clone:
+            shutil.rmtree(path, ignore_errors=True)
+        Repo.clone_from(url, path)
+    else:
+        repo = Repo(path)
+        repo.remote("origin").pull()
+
+    setup_script = os.path.join(path, "setup.sh")
+    if setup_script:
+        log_cb({"sist2-admin": f"Executing setup script {setup_script}"})
+
+        set_executable(setup_script)
+        result = subprocess.run([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        for line in result.stdout.split(b"\n"):
+            if line:
+                log_cb({"stdout": line.decode()})
+
+        log_cb({"stdout": f"Executed setup script {setup_script}, return code = {result.returncode}"})
+
+        if result.returncode != 0:
+            raise Exception("Error when running setup script!")
+
+    log_cb({"sist2-admin": f"Initialized git repository in {path}"})
+
+
+class UserScript(BaseModel):
+    name: str
+    type: ScriptType
+    git_repository: str = None
+    force_clone: bool = False
+    script: str = None
+    extra_args: str = ""
+
+    def script_dir(self):
+        return os.path.join(SCRIPT_FOLDER, self.name)
+
+    def setup(self, log_cb):
+        os.makedirs(self.script_dir(), exist_ok=True)
+
+        if self.type == ScriptType.GIT:
+            _initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone)
+            self.force_clone = False
+        elif self.type == ScriptType.SIMPLE:
+            self._setup_simple()
+
+        set_executable(self.get_executable())
+
+    def _setup_simple(self):
+        with open(self.get_executable(), "w") as f:
+            f.write(
+                "#!/bin/bash\n"
+                "python run.py \"$@\""
+            )
+
+        with open(os.path.join(self.script_dir(), "run.py"), "w") as f:
+            f.write(self.script)
+
+    def get_executable(self):
+        return os.path.join(self.script_dir(), "run.sh")
+
+    def delete_dir(self):
+        shutil.rmtree(self.script_dir(), ignore_errors=True)
+
+
+SCRIPT_TEMPLATES = {
+    "CLIP - Generate embeddings to predict the most relevant image based on the text prompt": lambda name: UserScript(
+        name=name,
+        type=ScriptType.GIT,
+        git_repository="https://github.com/simon987/sist2-script-clip",
+        extra_args="--num-tags=1 --tags-file=general.txt --color=#dcd7ff"
+    ),
+    "Whisper - Speech to text with OpenAI Whisper": lambda name: UserScript(
+        name=name,
+        type=ScriptType.GIT,
+        git_repository="https://github.com/simon987/sist2-script-whisper",
+        extra_args="--model=base --num-threads=4 --color=#51da4c --tag"
+    ),
+    "Hamburger - Simple script example": lambda name: UserScript(
+        name=name,
+        type=ScriptType.SIMPLE,
+        script=
+        'from sist2 import Sist2Index\n'
+        'import sys\n'
+        '\n'
+        'index = Sist2Index(sys.argv[1])\n'
+        'for doc in index.document_iter():\n'
+        '    doc.json_data["tag"] = ["hamburger.#00FF00"]\n'
+        '    index.update_document(doc)\n'
+        '\n'
+        'index.sync_tag_table()\n'
+        'index.commit()\n'
+        '\n'
+        'print("Done!")\n'
+    ),
+    "(Blank)": lambda name: UserScript(
+        name=name,
+        type=ScriptType.SIMPLE,
+        script=""
+    )
+}

sist2-admin/sist2_admin/sist2.py

@@ -3,6 +3,7 @@ import json
 import logging
 import os.path
 from datetime import datetime
+from enum import Enum
 from io import TextIOWrapper
 from logging import FileHandler
 from subprocess import Popen, PIPE
@@ -12,7 +13,7 @@ from typing import List
 
 from pydantic import BaseModel
 
-from config import logger, LOG_FOLDER
+from config import logger, LOG_FOLDER, DATA_FOLDER
 
 
 class Sist2Version:
@@ -25,77 +26,56 @@ class Sist2Version:
         return f"{self.major}.{self.minor}.{self.patch}"
 
 
-class WebOptions(BaseModel):
-    indices: List[str] = []
+class SearchBackendType(Enum):
+    SQLITE = "sqlite"
+    ELASTICSEARCH = "elasticsearch"
+
+
+class Sist2SearchBackend(BaseModel):
+    backend_type: SearchBackendType = SearchBackendType("elasticsearch")
+    name: str
+
+    search_index: str = ""
+
     es_url: str = "http://elasticsearch:9200"
     es_insecure_ssl: bool = False
     es_index: str = "sist2"
-    bind: str = "0.0.0.0:4090"
-    auth: str = None
-    tag_auth: str = None
-    tagline: str = "Lightning-fast file system indexer and search tool"
-    dev: bool = False
-    lang: str = "en"
-    auth0_audience: str = None
-    auth0_domain: str = None
-    auth0_client_id: str = None
-    auth0_public_key: str = None
-    auth0_public_key_file: str = None
+    threads: int = 1
+    batch_size: int = 70
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def args(self):
-        args = ["web", f"--es-url={self.es_url}", f"--bind={self.bind}",
-                f"--tagline={self.tagline}", f"--lang={self.lang}"]
-
-        if self.auth0_audience:
-            args.append(f"--auth0-audience={self.auth0_audience}")
-        if self.auth0_domain:
-            args.append(f"--auth0-domain={self.auth0_domain}")
-        if self.auth0_client_id:
-            args.append(f"--auth0-client-id={self.auth0_client_id}")
-        if self.auth0_public_key_file:
-            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
-        if self.es_insecure_ssl:
-            args.append(f"--es-insecure-ssl")
-        if self.auth:
-            args.append(f"--auth={self.auth}")
-        if self.tag_auth:
-            args.append(f"--tag-auth={self.tag_auth}")
-        if self.dev:
-            args.append(f"--dev")
-
-        args.extend(self.indices)
-
-        return args
+    @staticmethod
+    def create_default(name: str, backend_type: SearchBackendType = SearchBackendType("elasticsearch")):
+        return Sist2SearchBackend(
+            name=name,
+            search_index=f"search-index-{name.replace('/', '_')}.sist2",
+            backend_type=backend_type
+        )
 
 
 class IndexOptions(BaseModel):
     path: str = None
-    threads: int = 1
-    es_url: str = "http://elasticsearch:9200"
-    es_insecure_ssl: bool = False
-    es_index: str = "sist2"
     incremental_index: bool = True
-    script: str = ""
-    script_file: str = None
-    batch_size: int = 70
+    search_backend: str = None
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def args(self):
+    def args(self, search_backend):
+        absolute_path = os.path.join(DATA_FOLDER, self.path)
 
-        args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
-                f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
+        if search_backend.backend_type == SearchBackendType("sqlite"):
+            search_index_absolute = os.path.join(DATA_FOLDER, search_backend.search_index)
+            args = ["sqlite-index", absolute_path, "--search-index", search_index_absolute]
+        else:
+            args = ["index", absolute_path, f"--threads={search_backend.threads}",
+                    f"--es-url={search_backend.es_url}",
+                    f"--es-index={search_backend.es_index}",
+                    f"--batch-size={search_backend.batch_size}"]
 
-        if self.script_file:
-            args.append(f"--script-file={self.script_file}")
-        if self.es_insecure_ssl:
-            args.append(f"--es-insecure-ssl")
-        if self.incremental_index:
-            args.append(f"--incremental-index")
+            if search_backend.es_insecure_ssl:
+                args.append(f"--es-insecure-ssl")
+            if self.incremental_index:
+                args.append(f"--incremental-index")
 
         return args
 
@@ -109,7 +89,7 @@ ARCHIVE_RECURSE = "recurse"
 class ScanOptions(BaseModel):
     path: str
     threads: int = 1
-    thumbnail_quality: int = 2
+    thumbnail_quality: int = 50
     thumbnail_size: int = 552
     thumbnail_count: int = 1
     content_size: int = 32768
@@ -137,9 +117,12 @@ class ScanOptions(BaseModel):
         super().__init__(**kwargs)
 
     def args(self):
+
+        output_path = os.path.join(DATA_FOLDER, self.output)
+
         args = ["scan", self.path, f"--threads={self.threads}", f"--thumbnail-quality={self.thumbnail_quality}",
                 f"--thumbnail-count={self.thumbnail_count}", f"--thumbnail-size={self.thumbnail_size}",
-                f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
+                f"--content-size={self.content_size}", f"--output={output_path}", f"--depth={self.depth}",
                 f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
 
         if self.incremental:
@@ -201,27 +184,75 @@ class Sist2Index:
         return self._descriptor["name"]
 
 
+class WebOptions(BaseModel):
+    indices: List[str] = []
+
+    search_backend: str = "elasticsearch"
+
+    bind: str = "0.0.0.0:4090"
+    auth: str = None
+    tag_auth: str = None
+    tagline: str = "Lightning-fast file system indexer and search tool"
+    dev: bool = False
+    lang: str = "en"
+    auth0_audience: str = None
+    auth0_domain: str = None
+    auth0_client_id: str = None
+    auth0_public_key: str = None
+    auth0_public_key_file: str = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self, search_backend: Sist2SearchBackend):
+        args = ["web", f"--bind={self.bind}", f"--tagline={self.tagline}",
+                f"--lang={self.lang}"]
+
+        if search_backend.backend_type == SearchBackendType("sqlite"):
+            search_index_absolute = os.path.join(DATA_FOLDER, search_backend.search_index)
+            args.append(f"--search-index={search_index_absolute}")
+        else:
+            args.append(f"--es-url={search_backend.es_url}")
+            args.append(f"--es-index={search_backend.es_index}")
+            if search_backend.es_insecure_ssl:
+                args.append(f"--es-insecure-ssl")
+
+        if self.auth0_audience:
+            args.append(f"--auth0-audience={self.auth0_audience}")
+        if self.auth0_domain:
+            args.append(f"--auth0-domain={self.auth0_domain}")
+        if self.auth0_client_id:
+            args.append(f"--auth0-client-id={self.auth0_client_id}")
+        if self.auth0_public_key_file:
+            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
+        if self.auth:
+            args.append(f"--auth={self.auth}")
+        if self.tag_auth:
+            args.append(f"--tag-auth={self.tag_auth}")
+        if self.dev:
+            args.append(f"--dev")
+
+        args.extend(self.indices)
+
+        return args
+
+
 class Sist2:
 
     def __init__(self, bin_path: str, data_directory: str):
-        self._bin_path = bin_path
+        self.bin_path = bin_path
         self._data_dir = data_directory
 
-    def index(self, options: IndexOptions, logs_cb):
-
-        if options.script:
-            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
-                f.write(options.script)
-            options.script_file = f.name
-        else:
-            options.script_file = None
+    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
 
         args = [
-            self._bin_path,
-            *options.args(),
+            self.bin_path,
+            *options.args(search_backend),
             "--json-logs",
             "--very-verbose"
         ]
+
+        logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
         t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
@@ -236,13 +267,10 @@ class Sist2:
     def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
 
         if options.output is None:
-            options.output = os.path.join(
-                self._data_dir,
-                f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2"
-            )
+            options.output = f"scan-{options.name.replace('/', '_')}-{datetime.utcnow()}.sist2"
 
         args = [
-            self._bin_path,
+            self.bin_path,
             *options.args(),
             "--json-logs",
             "--very-verbose"
@@ -290,7 +318,7 @@ class Sist2:
                 except NameError:
                     pass
 
-    def web(self, options: WebOptions, name: str):
+    def web(self, options: WebOptions, search_backend: Sist2SearchBackend, name: str):
 
         if options.auth0_public_key:
             with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
@@ -300,8 +328,8 @@ class Sist2:
             options.auth0_public_key_file = None
 
         args = [
-            self._bin_path,
-            *options.args()
+            self.bin_path,
+            *options.args(search_backend)
         ]
 
         web_logger = logging.Logger(name=f"sist2-frontend-{name}")

sist2-admin/sist2_admin/state.py

@@ -1,16 +1,20 @@
 from typing import Dict
+import os
 import shutil
 
 from hexlib.db import Table, PersistentState
 import pickle
 
 from tesseract import get_tesseract_langs
+import sqlite3
+from config import LOG_FOLDER, logger
+from sist2 import SearchBackendType, Sist2SearchBackend
 
 RUNNING_FRONTENDS: Dict[str, int] = {}
 
 TESSERACT_LANGS = get_tesseract_langs()
 
-DB_SCHEMA_VERSION = "3"
+DB_SCHEMA_VERSION = "5"
 
 from pydantic import BaseModel
 
@@ -50,8 +54,35 @@ class PickleTable(Table):
             yield dict((k, _deserialize(v)) for k, v in row.items())
 
 
-def migrate_v1_to_v2(db: PersistentState):
+def get_log_files_to_remove(db: PersistentState, job_name: str, n: int):
+    if n < 0:
+        return []
 
+    counter = 0
+    to_remove = []
+
+    for row in db["task_done"].sql("WHERE has_logs=1 ORDER BY started DESC"):
+        if row["name"].endswith(f"[{job_name}]"):
+            counter += 1
+
+            if counter > n:
+                to_remove.append(row)
+
+    return to_remove
+
+
+def delete_log_file(db: PersistentState, task_id: str):
+    db["task_done"][task_id] = {
+        "has_logs": 0
+    }
+
+    try:
+        os.remove(os.path.join(LOG_FOLDER, f"sist2-{task_id}.log"))
+    except:
+        pass
+
+
+def migrate_v1_to_v2(db: PersistentState):
     shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v2.bak")
 
     # Frontends
@@ -77,3 +108,29 @@ def migrate_v1_to_v2(db: PersistentState):
     db["sist2_admin"]["info"] = {
         "version": "2"
     }
+
+
+def create_default_search_backends(db: PersistentState):
+    es_backend = Sist2SearchBackend.create_default(name="elasticsearch",
+                                                   backend_type=SearchBackendType("elasticsearch"))
+    db["search_backends"]["elasticsearch"] = es_backend
+    sqlite_backend = Sist2SearchBackend.create_default(name="sqlite", backend_type=SearchBackendType("sqlite"))
+    db["search_backends"]["sqlite"] = sqlite_backend
+
+
+def migrate_v3_to_v4(db: PersistentState):
+    shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v4.bak")
+
+    create_default_search_backends(db)
+
+    try:
+        conn = sqlite3.connect(db.dbfile)
+        conn.execute("ALTER TABLE task_done ADD COLUMN has_logs INTEGER DEFAULT 1")
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        logger.exception(e)
+
+    db["sist2_admin"]["info"] = {
+        "version": "4"
+    }

sist2-vue/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sist2",
-  "version": "2.11.0",
+  "version": "1.0.0",
   "private": true,
   "scripts": {
     "serve": "vue-cli-service serve",
@@ -9,15 +9,15 @@
   "dependencies": {
     "@auth0/auth0-spa-js": "^2.0.2",
     "@egjs/vue-infinitegrid": "3.3.0",
-    "@tensorflow/tfjs": "^4.4.0",
     "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "d3": "^7.8.4",
+    "d3": "^5.6.1",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",
     "fslightbox-vue": "fslightbox-vue.tgz",
     "nouislider": "^15.2.0",
+    "onnxruntime-web": "^1.15.1",
     "underscore": "^1.13.1",
     "vue": "^2.6.12",
     "vue-color": "^2.8.1",
@@ -29,6 +29,7 @@
   },
   "devDependencies": {
     "@babel/polyfill": "^7.12.1",
+    "@types/underscore": "^1.11.6",
     "@vue/cli-plugin-babel": "~5.0.8",
     "@vue/cli-plugin-router": "~5.0.8",
     "@vue/cli-plugin-typescript": "^5.0.8",
@@ -43,8 +44,8 @@
     "portal-vue": "^2.1.7",
     "sass": "^1.26.11",
     "sass-loader": "^10.0.2",
-    "typescript": "~4.1.5",
-    "vue-cli-plugin-bootstrap-vue": "~0.7.0",
+    "typescript": "^4.9.5",
+    "vue-cli-plugin-bootstrap-vue": "~0.8.2",
     "vue-template-compiler": "^2.6.11"
   },
   "browserslist": [

sist2-vue/public/index.html

@@ -19,12 +19,6 @@
             We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled.
             Please enable it to continue.
         </strong>
-        <br/>
-        <strong>
-            Nous sommes désolés mais <%= htmlWebpackPlugin.options.title %> ne fonctionne pas correctement
-            si JavaScript est activé.
-            Veuillez l'activer pour continuer.
-        </strong>
     </div>
 </noscript>
 <div id="app"></div>

sist2-vue/src/App.vue

@@ -10,7 +10,7 @@
             <b-spinner type="grow" variant="primary"></b-spinner>
         </div>
         <div class="loading-text">
-            Loading • Chargement • 装载 • Wird geladen
+            Loading • Chargement • 装载 • Wird geladen • Ładowanie
         </div>
     </div>
 </template>
@@ -21,6 +21,8 @@ import {mapActions, mapGetters, mapMutations} from "vuex";
 import Sist2Api from "@/Sist2Api";
 import ModelsRepo from "@/ml/modelsRepo";
 import {setupAuth0} from "@/main";
+import Sist2ElasticsearchQuery from "@/Sist2ElasticsearchQuery";
+import Sist2SqliteQuery from "@/Sist2SqliteQuery";
 
 export default {
     components: {NavBar},
@@ -88,6 +90,13 @@ export default {
 
             this.setSist2Info(data);
             this.setIndices(data.indices)
+
+            if (Sist2Api.backend() === "sqlite") {
+                Sist2Api.init(Sist2SqliteQuery.searchQuery);
+                this.$store.commit("setUiSqliteMode", true);
+            } else {
+                Sist2Api.init(Sist2ElasticsearchQuery.searchQuery);
+            }
         });
     },
     methods: {
@@ -299,15 +308,21 @@ html, body {
 
 .info-icon {
     width: 1rem;
+    min-width: 1rem;
     margin-right: 0.2rem;
     cursor: pointer;
     line-height: 1rem;
     height: 1rem;
+    min-height: 1rem;
     background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
     filter: brightness(45%);
     display: block;
 }
 
+.theme-black .info-icon {
+    filter: brightness(80%);
+}
+
 .tabs {
     margin-top: 10px;
 }

sist2-vue/src/Sist2Api.ts

@@ -1,5 +1,7 @@
 import axios from "axios";
 import {ext, strUnescape, lum} from "./util";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
+import store from "@/store";
 
 export interface EsTag {
     id: string
@@ -23,6 +25,7 @@ export interface Index {
     id: string
     idPrefix: string
     timestamp: number
+    models: []
 }
 
 export interface EsHit {
@@ -99,12 +102,31 @@ export interface EsResult {
 
 class Sist2Api {
 
-    private baseUrl: string
+    private readonly baseUrl: string
+    private sist2Info: any
+    private queryfunc: () => EsResult;
 
     constructor(baseUrl: string) {
         this.baseUrl = baseUrl;
     }
 
+    init(queryFunc: () => EsResult) {
+        this.queryfunc = queryFunc;
+    }
+
+    backend() {
+        return this.sist2Info.searchBackend;
+    }
+
+    models() {
+        const allModels = this.sist2Info.indices
+            .map(idx => idx.models)
+            .flat();
+
+        return allModels
+            .filter((v, i, a) => a.findIndex(v2 => (v2.id === v.id)) === i)
+    }
+
     getSist2Info(): Promise<any> {
         return axios.get(`${this.baseUrl}i`).then(resp => {
             const indices = resp.data.indices as Index[];
@@ -115,10 +137,13 @@ class Sist2Api {
                     name: idx.name,
                     timestamp: idx.timestamp,
                     version: idx.version,
-                    idPrefix: getIdPrefix(indices, idx.id)
+                    models: idx.models,
+                    idPrefix: getIdPrefix(indices, idx.id),
                 } as Index;
             });
 
+            this.sist2Info = resp.data;
+
             return resp.data;
         })
     }
@@ -219,6 +244,14 @@ class Sist2Api {
         } as Tag;
     }
 
+    search(): Promise<EsResult> {
+        if (this.backend() == "sqlite") {
+            return this.ftsQuery(this.queryfunc())
+        } else {
+            return this.esQuery(this.queryfunc());
+        }
+    }
+
     esQuery(query: any): Promise<EsResult> {
         return axios.post(`${this.baseUrl}es`, query).then(resp => {
             const res = resp.data as EsResult;
@@ -237,7 +270,30 @@ class Sist2Api {
         });
     }
 
-    getMimeTypes(query = undefined) {
+    ftsQuery(query: any): Promise<EsResult> {
+        return axios.post(`${this.baseUrl}fts/search`, query).then(resp => {
+            const res = resp.data as any;
+
+            if (res.hits.hits) {
+                res.hits.hits.forEach(hit => {
+                    hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
+                    hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+
+                    this.setHitProps(hit);
+                    this.setHitTags(hit);
+
+                    if ("highlight" in hit) {
+                        hit["highlight"]["name"] = [hit["highlight"]["name"]];
+                        hit["highlight"]["content"] = [hit["highlight"]["content"]];
+                    }
+                });
+            }
+
+            return res;
+        });
+    }
+
+    private getMimeTypesEs(query) {
         const AGGS = {
             mimeTypes: {
                 terms: {
@@ -258,48 +314,70 @@ class Sist2Api {
         }
 
         return this.esQuery(query).then(resp => {
-            const mimeMap: any[] = [];
-            const buckets = resp["aggregations"]["mimeTypes"]["buckets"];
+            return resp["aggregations"]["mimeTypes"]["buckets"].map(bucket => ({
+                mime: bucket.key,
+                count: bucket.doc_count
+            }));
 
-            buckets.sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
-                const tmp = bucket["key"].split("/");
-                const category = tmp[0];
-                const mime = tmp[1];
+        });
+    }
 
-                let category_exists = false;
+    private getMimeTypesSqlite(): Promise<[{ mime: string, count: number }]> {
+        return axios.get(`${this.baseUrl}fts/mimetypes`)
+            .then(resp => {
+                return resp.data;
+            });
+    }
 
-                const child = {
-                    "id": bucket["key"],
-                    "text": `${mime} (${bucket["doc_count"]})`
-                };
+    async getMimeTypes(query = undefined) {
+        let buckets;
 
-                mimeMap.forEach(node => {
-                    if (node.text === category) {
-                        node.children.push(child);
-                        category_exists = true;
-                    }
-                });
+        if (this.backend() == "sqlite") {
+            buckets = await this.getMimeTypesSqlite();
+        } else {
+            buckets = await this.getMimeTypesEs(query);
+        }
 
-                if (!category_exists) {
-                    mimeMap.push({text: category, children: [child], id: category});
-                }
-            })
+        const mimeMap: any[] = [];
+
+        buckets.sort((a: any, b: any) => a.mime > b.mime).forEach((bucket: any) => {
+            const tmp = bucket.mime.split("/");
+            const category = tmp[0];
+            const mime = tmp[1];
+
+            let category_exists = false;
+
+            const child = {
+                "id": bucket.mime,
+                "text": `${mime} (${bucket.count})`
+            };
 
             mimeMap.forEach(node => {
-                if (node.children) {
-                    node.children.sort((a, b) => a.id.localeCompare(b.id));
+                if (node.text === category) {
+                    node.children.push(child);
+                    category_exists = true;
                 }
-            })
-            mimeMap.sort((a, b) => a.id.localeCompare(b.id))
+            });
 
-            return {buckets, mimeMap};
-        });
+            if (!category_exists) {
+                mimeMap.push({text: category, children: [child], id: category});
+            }
+        })
+
+        mimeMap.forEach(node => {
+            if (node.children) {
+                node.children.sort((a, b) => a.id.localeCompare(b.id));
+            }
+        })
+        mimeMap.sort((a, b) => a.id.localeCompare(b.id))
+
+        return {buckets, mimeMap};
     }
 
     _createEsTag(tag: string, count: number): EsTag {
         const tokens = tag.split(".");
 
-        if (/.*\.#[0-9a-f]{6}/.test(tag)) {
+        if (/.*\.#[0-9a-fA-F]{6}/.test(tag)) {
             return {
                 id: tokens.slice(0, -1).join("."),
                 color: tokens.pop(),
@@ -316,32 +394,48 @@ class Sist2Api {
         };
     }
 
-    getTags() {
+    private getTagsEs() {
         return this.esQuery({
             aggs: {
                 tags: {
                     terms: {
                         field: "tag",
-                        size: 10000
+                        size: 65535
                     }
                 }
             },
             size: 0,
         }).then(resp => {
-            const seen = new Set();
-
-            const tags = resp["aggregations"]["tags"]["buckets"]
+            return resp["aggregations"]["tags"]["buckets"]
                 .sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
                 .map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
+        });
+    }
 
-            // Remove duplicates (same tag with different color)
-            return tags.filter((t: EsTag) => {
-                if (seen.has(t.id)) {
-                    return false;
-                }
-                seen.add(t.id);
-                return true;
+    private getTagsSqlite() {
+        return axios.get(`${this.baseUrl}/fts/tags`)
+            .then(resp => {
+                return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
             });
+    }
+
+    async getTags(): Promise<EsTag[]> {
+        let tags;
+        if (this.backend() == "sqlite") {
+            tags = await this.getTagsSqlite();
+        } else {
+            tags = await this.getTagsEs();
+        }
+
+        // Remove duplicates (same tag with different color)
+        const seen = new Set();
+
+        return tags.filter((t: EsTag) => {
+            if (seen.has(t.id)) {
+                return false;
+            }
+            seen.add(t.id);
+            return true;
         });
     }
 
@@ -361,6 +455,144 @@ class Sist2Api {
         });
     }
 
+    searchPaths(indexId, minDepth, maxDepth, prefix = null) {
+        if (this.backend() == "sqlite") {
+            return this.searchPathsSqlite(indexId, minDepth, minDepth, prefix);
+        } else {
+            return this.searchPathsEs(indexId, minDepth, maxDepth, prefix);
+        }
+    }
+
+    private searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
+        return axios.post(`${this.baseUrl}fts/paths`, {
+            indexId, minDepth, maxDepth, prefix
+        }).then(resp => {
+            return resp.data;
+        });
+    }
+
+    private searchPathsEs(indexId, minDepth, maxDepth, prefix): Promise<[{ path: string, count: number }]> {
+
+        const query = {
+            query: {
+                bool: {
+                    filter: [
+                        {term: {index: indexId}},
+                        {range: {_depth: {gte: minDepth, lte: maxDepth}}},
+                    ]
+                }
+            },
+            aggs: {
+                paths: {
+                    terms: {
+                        field: "path",
+                        size: 10000
+                    }
+                }
+            },
+            size: 0
+        };
+
+        if (prefix != null) {
+            query["query"]["bool"]["must"] = {
+                prefix: {
+                    path: prefix,
+                }
+            };
+        }
+
+        return this.esQuery(query).then(resp => {
+            const buckets = resp["aggregations"]["paths"]["buckets"];
+
+            if (!buckets) {
+                return [];
+            }
+
+            return buckets
+                .map(bucket => ({
+                    path: bucket.key,
+                    count: bucket.doc_count
+                }));
+        });
+    }
+
+    private getDateRangeSqlite() {
+        return axios.get(`${this.baseUrl}fts/dateRange`)
+            .then(resp => ({
+                min: resp.data.dateMin,
+                max: resp.data.dateMax,
+            }));
+    }
+
+    getDateRange(): Promise<{ min: number, max: number }> {
+        if (this.backend() == "sqlite") {
+            return this.getDateRangeSqlite();
+        } else {
+            return this.getDateRangeEs();
+        }
+    }
+
+    private getDateRangeEs() {
+        return this.esQuery({
+            // TODO: filter current selected indices
+            aggs: {
+                dateMin: {min: {field: "mtime"}},
+                dateMax: {max: {field: "mtime"}},
+            },
+            size: 0
+        }).then(res => {
+            const range = {
+                min: res.aggregations.dateMin.value / 1000,
+                max: res.aggregations.dateMax.value / 1000,
+            }
+
+            if (range.min == null) {
+                range.min = 0;
+                range.max = 1;
+            } else if (range.min == range.max) {
+                range.max += 1;
+            }
+
+            return range;
+        });
+    }
+
+    private getPathSuggestionsSqlite(text: string) {
+        return axios.post(`${this.baseUrl}fts/paths`, {
+            prefix: text,
+            minDepth: 1,
+            maxDepth: 10000
+        }).then(resp => {
+            return resp.data.map(bucket => bucket.path);
+        })
+    }
+
+    private getPathSuggestionsEs(text) {
+        return this.esQuery({
+            suggest: {
+                path: {
+                    prefix: text,
+                    completion: {
+                        field: "suggest-path",
+                        skip_duplicates: true,
+                        size: 10000
+                    }
+                }
+            }
+        }).then(resp => {
+            return resp["suggest"]["path"][0]["options"]
+                .map(opt => opt["_source"]["path"]);
+        });
+    }
+
+    getPathSuggestions(text: string): Promise<string[]> {
+        if (this.backend() == "sqlite") {
+            return this.getPathSuggestionsSqlite(text);
+        } else {
+            return this.getPathSuggestionsEs(text)
+        }
+    }
+
     getTreemapStat(indexId: string) {
         return `${this.baseUrl}s/${indexId}/TMAP`;
     }
@@ -376,6 +608,125 @@ class Sist2Api {
     getDateStat(indexId: string) {
         return `${this.baseUrl}s/${indexId}/DAGG`;
     }
+
+    private getDocumentEs(docId: string, highlight: boolean, fuzzy: boolean) {
+        const query = Sist2Query.searchQuery();
+
+        if (highlight) {
+            const fields = fuzzy
+                ? {"content.nGram": {}}
+                : {content: {}};
+
+            query.highlight = {
+                pre_tags: ["<mark>"],
+                post_tags: ["</mark>"],
+                number_of_fragments: 0,
+                fields,
+            };
+
+            if (!store.state.sist2Info.esVersionLegacy) {
+                query.highlight.max_analyzed_offset = 999_999;
+            }
+        }
+
+        if ("knn" in query) {
+            query.query = {
+                bool: {
+                    must: []
+                }
+            };
+            delete query.knn;
+        }
+
+        if ("function_score" in query.query) {
+            query.query = query.query.function_score.query;
+        }
+
+        if (!("must" in query.query.bool)) {
+            query.query.bool.must = [];
+        } else if (!Array.isArray(query.query.bool.must)) {
+            query.query.bool.must = [query.query.bool.must];
+        }
+
+        query.query.bool.must.push({match: {_id: docId}});
+
+        delete query["sort"];
+        delete query["aggs"];
+        delete query["search_after"];
+        delete query.query["function_score"];
+
+        query._source = {
+            includes: ["content", "name", "path", "extension"]
+        }
+
+        query.size = 1;
+
+        return this.esQuery(query).then(resp => {
+            if (resp.hits.hits.length === 1) {
+                return resp.hits.hits[0];
+            }
+            return null;
+        });
+    }
+
+    private getDocumentSqlite(docId: string): Promise<EsHit> {
+        return axios.get(`${this.baseUrl}/fts/d/${docId}`)
+            .then(resp => ({
+                _source: resp.data
+            } as EsHit));
+    }
+
+    getDocument(docId: string, highlight: boolean, fuzzy: boolean): Promise<EsHit | null> {
+        if (this.backend() == "sqlite") {
+            return this.getDocumentSqlite(docId);
+        } else {
+            return this.getDocumentEs(docId, highlight, fuzzy);
+        }
+    }
+
+    getTagSuggestions(prefix: string): Promise<string[]> {
+        if (this.backend() == "sqlite") {
+            return this.getTagSuggestionsSqlite(prefix);
+        } else {
+            return this.getTagSuggestionsEs(prefix);
+        }
+    }
+
+    private getTagSuggestionsSqlite(prefix): Promise<string[]> {
+        return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
+            .then(resp => (resp.data));
+    }
+
+    private getTagSuggestionsEs(prefix): Promise<string[]> {
+        return this.esQuery({
+            suggest: {
+                tag: {
+                    prefix: prefix,
+                    completion: {
+                        field: "suggest-tag",
+                        skip_duplicates: true,
+                        size: 10000
+                    }
+                }
+            }
+        }).then(resp => {
+            const result = [];
+            resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
+                tags.forEach(tag => {
+                    const t = tag.slice(0, -8);
+                    if (!result.find(x => x.slice(0, -8) === t)) {
+                        result.push(tag);
+                    }
+                });
+            });
+            return result;
+        });
+    }
+
+    getEmbeddings(indexId, docId, modelId) {
+        return axios.post(`${this.baseUrl}/e/${indexId}/${docId}/${modelId.toString().padStart(3, '0')}`)
+            .then(resp => (resp.data));
+    }
 }
 
 export default new Sist2Api("");

sist2-vue/src/Sist2ElasticsearchQuery.ts

@@ -1,5 +1,5 @@
 import store from "./store";
-import {EsHit, Index} from "@/Sist2Api";
+import sist2Api, {EsHit, Index} from "@/Sist2Api";
 
 const SORT_MODES = {
     score: {
@@ -60,14 +60,7 @@ const SORT_MODES = {
     }
 } as any;
 
-interface SortMode {
-    text: string
-    mode: any[]
-    key: (hit: EsHit) => any
-}
-
-
-class Sist2Query {
+class Sist2ElasticsearchQuery {
 
     searchQuery(blankSearch: boolean = false): any {
 
@@ -86,8 +79,10 @@ class Sist2Query {
         const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
         const selectedMimeTypes = getters.selectedMimeTypes;
         const selectedTags = getters.selectedTags;
+        const sortMode = getters.embedding ? "score" : getters.sortMode;
 
         const legacyES = store.state.sist2Info.esVersionLegacy;
+        const hasKnn = store.state.sist2Info.esVersionHasKnn;
 
         const filters = [
             {terms: {index: selectedIndexIds}}
@@ -169,31 +164,76 @@ class Sist2Query {
 
         const q = {
             _source: {
-                excludes: ["content", "_tie"]
+                excludes: ["content", "_tie", "emb.*"]
             },
             query: {
                 bool: {
                     filter: filters,
                 }
             },
-            sort: SORT_MODES[getters.sortMode].mode,
-            aggs:
-                {
-                    total_size: {"sum": {"field": "size"}},
-                    total_count: {"value_count": {"field": "size"}}
-                },
+            sort: SORT_MODES[sortMode].mode,
             size: size,
         } as any;
 
+        if (!after) {
+            q.aggs = {
+                total_size: {"sum": {"field": "size"}},
+                total_count: {"value_count": {"field": "size"}}
+            };
+        }
+
         if (!empty && !blankSearch) {
-            q.query.bool.must = query;
+            if (getters.embedding) {
+                filters.push(query)
+            } else {
+                q.query.bool.must = query;
+            }
+        }
+
+        if (getters.embedding) {
+            delete q.query;
+
+            const field = "emb." + sist2Api.models().find(m => m.id == getters.embeddingsModel).path;
+
+            if (hasKnn) {
+                // Use knn (8.8+)
+                q.knn = {
+                    field: field,
+                    query_vector: getters.embedding,
+
+                    k: 600,
+                    num_candidates: 600,
+
+                    filter: filters
+                }
+            } else {
+                // Use brute-force as a fallback
+
+                filters.push({exists: {field: field}});
+
+                q.query = {
+                    function_score: {
+                        query: {
+                            bool: {
+                                must: filters,
+                            }
+                        },
+                        script_score: {
+                            script: {
+                                source: `cosineSimilarity(params.query_vector, "${field}") + 1.0`,
+                                params: {query_vector: getters.embedding}
+                            }
+                        }
+                    }
+                }
+            }
         }
 
         if (after) {
-            q.search_after = [SORT_MODES[getters.sortMode].key(after), after["_id"]];
+            q.search_after = [SORT_MODES[sortMode].key(after), after["_id"]];
         }
 
-        if (getters.optHighlight) {
+        if (getters.optHighlight && !getters.embedding) {
             q.highlight = {
                 pre_tags: ["<mark>"],
                 post_tags: ["</mark>"],
@@ -219,7 +259,7 @@ class Sist2Query {
             }
         }
 
-        if (getters.sortMode === "random") {
+        if (sortMode === "random") {
             q.query = {
                 function_score: {
                     query: {
@@ -249,4 +289,5 @@ class Sist2Query {
     }
 }
 
-export default new Sist2Query();
+
+export default new Sist2ElasticsearchQuery();

sist2-vue/src/Sist2SqliteQuery.ts

@@ -0,0 +1,121 @@
+import store from "./store";
+import {EsHit, Index} from "@/Sist2Api";
+
+const SORT_MODES = {
+    score: {
+        "sort": "score",
+    },
+    random: {
+        "sort": "random"
+    },
+    dateAsc: {
+        "sort": "mtime"
+    },
+    dateDesc: {
+        "sort": "mtime",
+        "sortAsc": false
+    },
+    sizeAsc: {
+        "sort": "size",
+    },
+    sizeDesc: {
+        "sort": "size",
+        "sortAsc": false
+    },
+    nameAsc: {
+        "sort": "name",
+    },
+    nameDesc: {
+        "sort": "name",
+        "sortAsc": false
+    }
+} as any;
+
+interface SortMode {
+    text: string
+    mode: any[]
+    key: (hit: EsHit) => any
+}
+
+
+class Sist2ElasticsearchQuery {
+
+    searchQuery(): any {
+
+        const getters = store.getters;
+
+        const searchText = getters.searchText;
+        const pathText = getters.pathText;
+        const sizeMin = getters.sizeMin;
+        const sizeMax = getters.sizeMax;
+        const dateMin = getters.dateMin;
+        const dateMax = getters.dateMax;
+        const size = getters.size;
+        const after = getters.lastDoc;
+        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
+        const selectedMimeTypes = getters.selectedMimeTypes;
+        const selectedTags = getters.selectedTags;
+
+        const q = {
+            "pageSize": size
+        }
+
+        Object.assign(q, SORT_MODES[getters.sortMode]);
+
+        if (!after) {
+            q["fetchAggregations"] = true;
+        }
+        if (searchText) {
+            q["query"] = searchText;
+        }
+        if (pathText) {
+            q["path"] = pathText.endsWith("/") ? pathText.slice(0, -1) : pathText;
+        }
+        if (sizeMin) {
+            q["sizeMin"] = sizeMin;
+        }
+        if (sizeMax) {
+            q["sizeMax"] = sizeMax;
+        }
+        if (dateMin) {
+            q["dateMin"] = dateMin;
+        }
+        if (dateMax) {
+            q["dateMax"] = dateMax;
+        }
+        if (after) {
+            q["after"] = after.sort;
+        }
+        if (selectedIndexIds.length > 0) {
+            q["indexIds"] = selectedIndexIds;
+        }
+        if (selectedMimeTypes.length > 0) {
+            q["mimeTypes"] = selectedMimeTypes;
+        }
+        if (selectedTags.length > 0) {
+            q["tags"] = selectedTags
+        }
+        if (getters.sortMode == "random") {
+            q["seed"] = getters.seed;
+        }
+        if (getters.optHighlight) {
+            q["highlight"] = true;
+            q["highlightContextSize"] = Number(getters.optFragmentSize);
+        }
+
+        if (getters.embedding) {
+            q["model"] = getters.embeddingsModel;
+            q["embedding"] = getters.embedding;
+            q["sort"] = "embedding";
+            q["sortAsc"] = false;
+        } else if (getters.sortMode == "embedding") {
+            q["sort"] = "sort"
+            q["sortAsc"] = true;
+        }
+
+        return q;
+    }
+}
+
+
+export default new Sist2ElasticsearchQuery();

sist2-vue/src/components/AnalyzedContentSpan.vue

@@ -12,7 +12,7 @@ export default {
     props: ["span", "text"],
     methods: {
         getStyle() {
-            return ModelsRepo.data[this.$store.getters.mlModel.name].labelStyles[this.span.label];
+            return ModelsRepo.data[this.$store.getters.nerModel.name].labelStyles[this.span.label];
         }
     }
 }

sist2-vue/src/components/AnalyzedContentSpanContainer.vue

@@ -22,7 +22,7 @@ export default {
     props: ["spans", "text"],
     computed: {
         legend() {
-            return Object.entries(ModelsRepo.data[this.$store.state.mlModel.name].legend)
+            return Object.entries(ModelsRepo.data[this.$store.state.nerModel.name].legend)
                 .map(([label, name]) => ({
                     text: name,
                     id: label,

sist2-vue/src/components/D3Treemap.vue

@@ -1,10 +1,10 @@
 <template>
-  <div>
-    <b-btn style="float:right;margin-bottom: 10px" @click="downloadTreemap()" variant="primary">
-      {{ $t("download") }}
-    </b-btn>
-    <svg id="treemap"></svg>
-  </div>
+    <div>
+        <b-btn style="float:right;margin-bottom: 10px" @click="downloadTreemap()" variant="primary">
+            {{ $t("download") }}
+        </b-btn>
+        <svg id="treemap"></svg>
+    </div>
 </template>
 
 <script>
@@ -16,252 +16,252 @@ import domtoimage from "dom-to-image";
 
 
 const TILING_MODES = {
-  "squarify": d3.treemapSquarify,
-  "binary": d3.treemapBinary,
-  "sliceDice": d3.treemapSliceDice,
-  "slice": d3.treemapSlice,
-  "dice": d3.treemapDice,
+    "squarify": d3.treemapSquarify,
+    "binary": d3.treemapBinary,
+    "sliceDice": d3.treemapSliceDice,
+    "slice": d3.treemapSlice,
+    "dice": d3.treemapDice,
 };
 
 const COLORS = {
-  "PuBuGn": d3.interpolatePuBuGn,
-  "PuRd": d3.interpolatePuRd,
-  "PuBu": d3.interpolatePuBu,
-  "YlOrBr": d3.interpolateYlOrBr,
-  "YlOrRd": d3.interpolateYlOrRd,
-  "YlGn": d3.interpolateYlGn,
-  "YlGnBu": d3.interpolateYlGnBu,
-  "Plasma": d3.interpolatePlasma,
-  "Magma": d3.interpolateMagma,
-  "Inferno": d3.interpolateInferno,
-  "Viridis": d3.interpolateViridis,
-  "Turbo": d3.interpolateTurbo,
+    "PuBuGn": d3.interpolatePuBuGn,
+    "PuRd": d3.interpolatePuRd,
+    "PuBu": d3.interpolatePuBu,
+    "YlOrBr": d3.interpolateYlOrBr,
+    "YlOrRd": d3.interpolateYlOrRd,
+    "YlGn": d3.interpolateYlGn,
+    "YlGnBu": d3.interpolateYlGnBu,
+    "Plasma": d3.interpolatePlasma,
+    "Magma": d3.interpolateMagma,
+    "Inferno": d3.interpolateInferno,
+    "Viridis": d3.interpolateViridis,
+    "Turbo": d3.interpolateTurbo,
 };
 
 const SIZES = {
-  "small": [800, 600],
-  "medium": [1300, 750],
-  "large": [1900, 900],
-  "x-large": [2800, 1700],
-  "xx-large": [3600, 2000],
+    "small": [800, 600],
+    "medium": [1300, 750],
+    "large": [1900, 900],
+    "x-large": [2800, 1700],
+    "xx-large": [3600, 2000],
 };
 
 
 const uids = {};
 
 function uid(name) {
-  let id = uids[name] || 0;
-  uids[name] = id + 1;
-  return name + id;
+    let id = uids[name] || 0;
+    uids[name] = id + 1;
+    return name + id;
 }
 
 function cascade(root, offset) {
-  const x = new Map;
-  const y = new Map;
-  return root.eachAfter(d => {
-    if (d.children && d.children.length !== 0) {
-      x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
-      y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
-    } else {
-      x.set(d, 0);
-      y.set(d, 0);
-    }
-  }).eachBefore(d => {
-    d.x1 -= 2 * offset * x.get(d);
-    d.y1 -= 2 * offset * y.get(d);
-  });
+    const x = new Map;
+    const y = new Map;
+    return root.eachAfter(d => {
+        if (d.children && d.children.length !== 0) {
+            x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
+            y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
+        } else {
+            x.set(d, 0);
+            y.set(d, 0);
+        }
+    }).eachBefore(d => {
+        d.x1 -= 2 * offset * x.get(d);
+        d.y1 -= 2 * offset * y.get(d);
+    });
 }
 
 function cascadeTreemap(data, svg, width, height, tilingMode, treemapColor) {
-  const root = cascade(
-      d3.treemap()
-          .size([width, height])
-          .tile(TILING_MODES[tilingMode])
-          .paddingOuter(3)
-          .paddingTop(16)
-          .paddingInner(1)
-          .round(true)(
-              d3.hierarchy(data)
-                  .sum(d => d.value)
-                  .sort((a, b) => b.value - a.value)
-          ),
-      3 // treemap.paddingOuter
-  );
+    const root = cascade(
+        d3.treemap()
+            .size([width, height])
+            .tile(TILING_MODES[tilingMode])
+            .paddingOuter(3)
+            .paddingTop(16)
+            .paddingInner(1)
+            .round(true)(
+                d3.hierarchy(data)
+                    .sum(d => d.value)
+                    .sort((a, b) => b.value - a.value)
+            ),
+        3 // treemap.paddingOuter
+    );
 
-  const maxDepth = Math.max(...root.descendants().map(d => d.depth));
-  const color = d3.scaleSequential([maxDepth, -1], COLORS[treemapColor]);
+    const maxDepth = Math.max(...root.descendants().map(d => d.depth));
+    const color = d3.scaleSequential([maxDepth, -1], COLORS[treemapColor]);
 
-  svg.append("filter")
-      .attr("id", "shadow")
-      .append("feDropShadow")
-      .attr("flood-opacity", 0.3)
-      .attr("dx", 0)
-      .attr("stdDeviation", 3);
+    svg.append("filter")
+        .attr("id", "shadow")
+        .append("feDropShadow")
+        .attr("flood-opacity", 0.3)
+        .attr("dx", 0)
+        .attr("stdDeviation", 3);
 
-  const node = svg.selectAll("g")
-      .data(
-          d3.nest()
-              .key(d => d.depth).sortKeys(d3.ascending)
-              .entries(root.descendants())
-      )
-      .join("g")
-      .attr("filter", "url(#shadow)")
-      .selectAll("g")
-      .data(d => d.values)
-      .join("g")
-      .attr("transform", d => `translate(${d.x0},${d.y0})`);
+    const node = svg.selectAll("g")
+        .data(
+            d3.nest()
+                .key(d => d.depth).sortKeys(d3.ascending)
+                .entries(root.descendants())
+        )
+        .join("g")
+        .attr("filter", "url(#shadow)")
+        .selectAll("g")
+        .data(d => d.values)
+        .join("g")
+        .attr("transform", d => `translate(${d.x0},${d.y0})`);
 
-  node.append("title")
-      .text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+    node.append("title")
+        .text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
 
-  node.append("rect")
-      .attr("id", d => (d.nodeUid = uid("node")))
-      .attr("fill", d => color(d.depth))
-      .attr("width", d => d.x1 - d.x0)
-      .attr("height", d => d.y1 - d.y0);
+    node.append("rect")
+        .attr("id", d => (d.nodeUid = uid("node")))
+        .attr("fill", d => color(d.depth))
+        .attr("width", d => d.x1 - d.x0)
+        .attr("height", d => d.y1 - d.y0);
 
-  node.append("clipPath")
-      .attr("id", d => (d.clipUid = uid("clip")))
-      .append("use")
-      .attr("href", d => `#${d.nodeUid}`);
+    node.append("clipPath")
+        .attr("id", d => (d.clipUid = uid("clip")))
+        .append("use")
+        .attr("href", d => `#${d.nodeUid}`);
 
-  node.append("text")
-      .attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
-      .attr("clip-path", d => `url(#${d.clipUid})`)
-      .selectAll("tspan")
-      .data(d => [d.data.name, humanFileSize(d.value)])
-      .join("tspan")
-      .text(d => d);
+    node.append("text")
+        .attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
+        .attr("clip-path", d => `url(#${d.clipUid})`)
+        .selectAll("tspan")
+        .data(d => [d.data.name, humanFileSize(d.value)])
+        .join("tspan")
+        .text(d => d);
 
-  node.filter(d => d.children).selectAll("tspan")
-      .attr("dx", 3)
-      .attr("y", 13);
+    node.filter(d => d.children).selectAll("tspan")
+        .attr("dx", 3)
+        .attr("y", 13);
 
-  node.filter(d => !d.children).selectAll("tspan")
-      .attr("x", 3)
-      .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`);
+    node.filter(d => !d.children).selectAll("tspan")
+        .attr("x", 3)
+        .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`);
 }
 
 function flatTreemap(data, svg, width, height, groupingDepth, tilingMode, fillOpacity) {
-  const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
+    const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
 
-  const root = d3.treemap()
-      .tile(TILING_MODES[tilingMode])
-      .size([width, height])
-      .padding(1)
-      .round(true)(
-          d3.hierarchy(data)
-              .sum(d => d.value)
-              .sort((a, b) => b.value - a.value)
-      );
+    const root = d3.treemap()
+        .tile(TILING_MODES[tilingMode])
+        .size([width, height])
+        .padding(1)
+        .round(true)(
+            d3.hierarchy(data)
+                .sum(d => d.value)
+                .sort((a, b) => b.value - a.value)
+        );
 
-  const leaf = svg.selectAll("g")
-      .data(root.leaves())
-      .join("g")
-      .attr("transform", d => `translate(${d.x0},${d.y0})`);
+    const leaf = svg.selectAll("g")
+        .data(root.leaves())
+        .join("g")
+        .attr("transform", d => `translate(${d.x0},${d.y0})`);
 
-  leaf.append("title")
-      .text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+    leaf.append("title")
+        .text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
 
-  leaf.append("rect")
-      .attr("id", d => (d.leafUid = uid("leaf")))
-      .attr("fill", d => {
-        while (d.depth > groupingDepth) d = d.parent;
-        return ordinalColor(d.data.name);
-      })
-      .attr("fill-opacity", fillOpacity)
-      .attr("width", d => d.x1 - d.x0)
-      .attr("height", d => d.y1 - d.y0);
+    leaf.append("rect")
+        .attr("id", d => (d.leafUid = uid("leaf")))
+        .attr("fill", d => {
+            while (d.depth > groupingDepth) d = d.parent;
+            return ordinalColor(d.data.name);
+        })
+        .attr("fill-opacity", fillOpacity)
+        .attr("width", d => d.x1 - d.x0)
+        .attr("height", d => d.y1 - d.y0);
 
-  leaf.append("clipPath")
-      .attr("id", d => (d.clipUid = uid("clip")))
-      .append("use")
-      .attr("href", d => `#${d.leafUid}`);
+    leaf.append("clipPath")
+        .attr("id", d => (d.clipUid = uid("clip")))
+        .append("use")
+        .attr("href", d => `#${d.leafUid}`);
 
-  leaf.append("text")
-      .attr("clip-path", d => `url(#${d.clipUid})`)
-      .selectAll("tspan")
-      .data(d => {
-        if (d.data.name === ".") {
-          d = d.parent;
-        }
-        return [d.data.name, humanFileSize(d.value)]
-      })
-      .join("tspan")
-      .attr("x", 2)
-      .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`)
-      .text(d => d);
+    leaf.append("text")
+        .attr("clip-path", d => `url(#${d.clipUid})`)
+        .selectAll("tspan")
+        .data(d => {
+            if (d.data.name === ".") {
+                d = d.parent;
+            }
+            return [d.data.name, humanFileSize(d.value)]
+        })
+        .join("tspan")
+        .attr("x", 2)
+        .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`)
+        .text(d => d);
 }
 
 function exportTreemap(indexName, width, height) {
-  domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
-      .then(function (blob) {
-        let a = document.createElement("a");
-        let url = URL.createObjectURL(blob);
+    domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
+        .then(function (blob) {
+            let a = document.createElement("a");
+            let url = URL.createObjectURL(blob);
 
-        a.href = url;
-        a.download = `${indexName}_treemap.png`;
-        document.body.appendChild(a);
-        a.click();
-        setTimeout(function () {
-          document.body.removeChild(a);
-          window.URL.revokeObjectURL(url);
-        }, 0);
-      });
+            a.href = url;
+            a.download = `${indexName}_treemap.png`;
+            document.body.appendChild(a);
+            a.click();
+            setTimeout(function () {
+                document.body.removeChild(a);
+                window.URL.revokeObjectURL(url);
+            }, 0);
+        });
 }
 
 export default {
-  name: "D3Treemap",
-  props: ["indexId"],
-  watch: {
-    indexId: function () {
-      this.update(this.indexId);
-    }
-  },
-  mounted() {
-    this.update(this.indexId);
-  },
-  methods: {
-    update(indexId) {
-      const width = SIZES[this.$store.state.optTreemapSize][0];
-      const height = SIZES[this.$store.state.optTreemapSize][1];
-      const tilingMode = this.$store.state.optTreemapTiling;
-      const groupingDepth = this.$store.state.optTreemapColorGroupingDepth;
-      const treemapColor = this.$store.state.optTreemapColor;
-      const treemapType = this.$store.state.optTreemapType;
-
-      const treemapSvg = d3.select("#treemap");
-
-      treemapSvg.selectAll("*").remove();
-      treemapSvg.attr("viewBox", [0, 0, width, height])
-          .attr("xmlns", "http://www.w3.org/2000/svg")
-          .attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
-          .attr("version", "1.1")
-          .style("overflow", "visible")
-          .style("font", "10px sans-serif");
-
-      d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
-        tabularData.forEach(row => {
-          row.taxonomy = row.path.split("/");
-          row.size = Number(row.size);
-        });
-
-        if (treemapType === "cascaded") {
-          const data = burrow(tabularData, false);
-          cascadeTreemap(data, treemapSvg, width, height, tilingMode, treemapColor);
-        } else {
-          const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
-          const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
-          flatTreemap(data, treemapSvg, width, height, groupingDepth, tilingMode, fillOpacity);
+    name: "D3Treemap",
+    props: ["indexId"],
+    watch: {
+        indexId: function () {
+            this.update(this.indexId);
         }
-      });
     },
-    downloadTreemap() {
-      const width = SIZES[this.$store.state.optTreemapSize][0];
-      const height = SIZES[this.$store.state.optTreemapSize][1];
+    mounted() {
+        this.update(this.indexId);
+    },
+    methods: {
+        update(indexId) {
+            const width = SIZES[this.$store.state.optTreemapSize][0];
+            const height = SIZES[this.$store.state.optTreemapSize][1];
+            const tilingMode = this.$store.state.optTreemapTiling;
+            const groupingDepth = this.$store.state.optTreemapColorGroupingDepth;
+            const treemapColor = this.$store.state.optTreemapColor;
+            const treemapType = this.$store.state.optTreemapType;
 
-      exportTreemap(this.indexId, width, height);
+            const treemapSvg = d3.select("#treemap");
+
+            treemapSvg.selectAll("*").remove();
+            treemapSvg.attr("viewBox", [0, 0, width, height])
+                .attr("xmlns", "http://www.w3.org/2000/svg")
+                .attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
+                .attr("version", "1.1")
+                .style("overflow", "visible")
+                .style("font", "10px sans-serif");
+
+            d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
+                tabularData.forEach(row => {
+                    row.taxonomy = row.path.split("/");
+                    row.size = Number(row.size);
+                });
+
+                if (treemapType === "cascaded") {
+                    const data = burrow(tabularData, false);
+                    cascadeTreemap(data, treemapSvg, width, height, tilingMode, treemapColor);
+                } else {
+                    const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
+                    const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
+                    flatTreemap(data, treemapSvg, width, height, groupingDepth, tilingMode, fillOpacity);
+                }
+            });
+        },
+        downloadTreemap() {
+            const width = SIZES[this.$store.state.optTreemapSize][0];
+            const height = SIZES[this.$store.state.optTreemapSize][1];
+
+            exportTreemap(this.indexId, width, height);
+        }
     }
-  }
 }
 </script>

sist2-vue/src/components/DebugInfo.vue

@@ -1,41 +1,57 @@
 <template>
-  <b-card v-if="$store.state.sist2Info.showDebugInfo" class="mb-4 mt-4">
-    <b-card-title><DebugIcon class="mr-1"></DebugIcon>{{ $t("debug") }}</b-card-title>
-    <p v-html="$t('debugDescription')"></p>
+    <b-card v-if="$store.state.sist2Info.showDebugInfo" class="mb-4 mt-4">
+        <b-card-title>
+            <DebugIcon class="mr-1"></DebugIcon>
+            {{ $t("debug") }}
+        </b-card-title>
+        <p v-html="$t('debugDescription')"></p>
 
-    <b-card-body>
+        <b-card-body>
 
-      <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
+            <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
 
-      <hr />
-      <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx" class="mt-2"></IndexDebugInfo>
-    </b-card-body>
-  </b-card>
+            <hr/>
+            <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx"
+                            class="mt-2"></IndexDebugInfo>
+        </b-card-body>
+    </b-card>
 </template>
 
 <script>
 import IndexDebugInfo from "@/components/IndexDebugInfo";
 import DebugIcon from "@/components/icons/DebugIcon";
+import {mapGetters} from "vuex";
 
 export default {
-  name: "DebugInfo.vue",
-  components: {DebugIcon, IndexDebugInfo},
-  computed: {
-    tableItems() {
-      return [
-        {key: "version", value: this.$store.state.sist2Info.version},
-        {key: "platform", value: this.$store.state.sist2Info.platform},
-        {key: "debugBinary", value: this.$store.state.sist2Info.debug},
-        {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
-        {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
-        {key: "tagline", value: this.$store.state.sist2Info.tagline},
-        {key: "dev", value: this.$store.state.sist2Info.dev},
-        {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
-        {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
-        {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
-        {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
-      ]
+    name: "DebugInfo.vue",
+    components: {DebugIcon, IndexDebugInfo},
+    computed: {
+        ...mapGetters([
+            "uiSqliteMode",
+        ]),
+        tableItems() {
+            const items = [
+                {key: "version", value: this.$store.state.sist2Info.version},
+                {key: "platform", value: this.$store.state.sist2Info.platform},
+                {key: "debugBinary", value: this.$store.state.sist2Info.debug},
+                {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
+                {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
+                {key: "tagline", value: this.$store.state.sist2Info.tagline},
+                {key: "dev", value: this.$store.state.sist2Info.dev},
+                {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
+            ];
+
+            if (!this.uiSqliteMode) {
+                items.push(
+                    {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
+                    {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
+                    {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
+                    {key: "esVersionHasKnn", value: this.$store.state.sist2Info.esVersionHasKnn},
+                );
+            }
+
+            return items;
+        }
     }
-  }
 }
 </script>

sist2-vue/src/components/DocCard.vue

@@ -1,44 +1,45 @@
 <template>
-  <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
-       @click="$store.commit('busTnTouchStart', null)">
-    <b-card
-        no-body
-        img-top
-    >
-      <!-- Info modal-->
-      <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
+    <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
+         @click="$store.commit('busTnTouchStart', null)">
+        <b-card
+                no-body
+                img-top
+        >
+            <!-- Info modal-->
+            <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
 
-      <ContentDiv :doc="doc"></ContentDiv>
+            <ContentDiv :doc="doc"></ContentDiv>
 
-      <!-- Thumbnail-->
-      <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
+            <!-- Thumbnail-->
+            <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
 
-      <!-- Audio player-->
-      <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
-             :type="doc._source.mime"
-             :src="`f/${doc._id}`"
-             @play="onAudioPlay()"></audio>
+            <!-- Audio player-->
+            <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
+                   :type="doc._source.mime"
+                   :src="`f/${doc._source.index}/${doc._id}`"
+                   @play="onAudioPlay()"></audio>
 
-      <b-card-body class="padding-03">
+            <b-card-body class="padding-03">
 
-        <!-- Title line -->
-        <div style="display: flex">
-          <span class="info-icon" @click="onInfoClick()"></span>
-          <DocFileTitle :doc="doc"></DocFileTitle>
-        </div>
+                <!-- Title line -->
+                <div style="display: flex">
+                    <span class="info-icon" @click="onInfoClick()"></span>
+                    <MLIcon v-if="doc._source.embedding" clickable @click="onEmbeddingClick()"></MLIcon>
+                    <DocFileTitle :doc="doc"></DocFileTitle>
+                </div>
 
-        <!-- Featured line -->
-        <div style="display: flex">
-          <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
-        </div>
+                <!-- Featured line -->
+                <div style="display: flex">
+                    <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
+                </div>
 
-        <!-- Tags -->
-        <div class="card-text">
-          <TagContainer :hit="doc"></TagContainer>
-        </div>
-      </b-card-body>
-    </b-card>
-  </div>
+                <!-- Tags -->
+                <div class="card-text">
+                    <TagContainer :hit="doc"></TagContainer>
+                </div>
+            </b-card-body>
+        </b-card>
+    </div>
 </template>
 
 <script>
@@ -49,94 +50,103 @@ import DocInfoModal from "@/components/DocInfoModal.vue";
 import ContentDiv from "@/components/ContentDiv.vue";
 import FullThumbnail from "@/components/FullThumbnail";
 import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
+import MLIcon from "@/components/icons/MlIcon.vue";
+import Sist2Api from "@/Sist2Api";
 
 
 export default {
-  components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
-  props: ["doc", "width"],
-  data() {
-    return {
-      ext: ext,
-      showInfo: false,
-    }
-  },
-  computed: {
-    smallBadge() {
-      return this.width < 150;
-    }
-  },
-  methods: {
-    humanFileSize: humanFileSize,
-    humanTime: humanTime,
-    onInfoClick() {
-      this.showInfo = true;
-    },
-    async onThumbnailClick() {
-      this.$store.commit("setUiLightboxSlide", this.doc._seq);
-      await this.$store.dispatch("showLightbox");
-    },
-    onAudioPlay() {
-      document.getElementsByTagName("audio").forEach((el) => {
-        if (el !== this.$refs["audio"]) {
-          el.pause();
+    components: {MLIcon, FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+    props: ["doc", "width"],
+    data() {
+        return {
+            ext: ext,
+            showInfo: false,
         }
-      });
     },
-  },
+    computed: {
+        smallBadge() {
+            return this.width < 150;
+        }
+    },
+    methods: {
+        humanFileSize: humanFileSize,
+        humanTime: humanTime,
+        onInfoClick() {
+            this.showInfo = true;
+        },
+        onEmbeddingClick() {
+            Sist2Api.getEmbeddings(this.doc._source.index, this.doc._id, this.$store.state.embeddingsModel).then(embeddings => {
+                this.$store.commit("setEmbeddingText", "");
+                this.$store.commit("setEmbedding", embeddings);
+                this.$store.commit("setEmbeddingDoc", this.doc);
+            })
+        },
+        async onThumbnailClick() {
+            this.$store.commit("setUiLightboxSlide", this.doc._seq);
+            await this.$store.dispatch("showLightbox");
+        },
+        onAudioPlay() {
+            Array.prototype.slice.call(document.getElementsByTagName("audio")).forEach((el) => {
+                if (el !== this.$refs["audio"]) {
+                    el.pause();
+                }
+            });
+        },
+    },
 }
 </script>
 <style>
 .fit {
-  display: block;
-  min-width: 64px;
-  max-width: 100%;
-  /*max-height: 400px;*/
-  margin: 0 auto 0;
-  width: auto;
-  height: auto;
+    display: block;
+    min-width: 64px;
+    max-width: 100%;
+    /*max-height: 400px;*/
+    margin: 0 auto 0;
+    width: auto;
+    height: auto;
 }
 
 .audio-fit {
-  height: 39px;
-  vertical-align: bottom;
-  display: inline;
-  width: 100%;
+    height: 39px;
+    vertical-align: bottom;
+    display: inline;
+    width: 100%;
 }
 </style>
 
 <style scoped>
 
 .padding-03 {
-  padding: 0.3rem;
+    padding: 0.3rem;
 }
 
 .card {
-  margin-top: 1em;
-  margin-left: 0;
-  margin-right: 0;
-  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-  border-radius: 0;
-  border: none;
+    margin-top: 1em;
+    margin-left: 0;
+    margin-right: 0;
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
 }
 
 .card-body {
-  padding: 0.3rem;
+    padding: 0.3rem;
 }
 
 .doc-card {
-  padding-left: 3px;
-  padding-right: 3px;
+    padding-left: 3px;
+    padding-right: 3px;
 }
 
 .sub-document .card {
-  background: #AB47BC1F !important;
+    background: #AB47BC1F !important;
 }
 
 .theme-black .sub-document .card {
-  background: #37474F !important;
+    background: #37474F !important;
 }
 
 .sub-document .fit {
-  padding: 4px 4px 0 4px;
+    padding: 4px 4px 0 4px;
 }
 </style>

sist2-vue/src/components/DocFileTitle.vue

@@ -1,63 +1,70 @@
 <template>
-  <a :href="`f/${doc._id}`" class="file-title-anchor" target="_blank">
-    <div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
-         v-html="fileName() + ext(doc)"></div>
-  </a>
+    <a :href="`f/${doc._source.index}/${doc._id}`"
+       :class="doc._source.embedding ? 'file-title-anchor-with-embedding' : 'file-title-anchor'" target="_blank">
+        <div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
+             v-html="fileName() + ext(doc)"></div>
+    </a>
 </template>
 
 <script>
 import {ext} from "@/util";
 
 export default {
-  name: "DocFileTitle",
-  props: ["doc"],
-  methods: {
-    ext: ext,
-    fileName() {
-      if (!this.doc.highlight) {
-        return this.doc._source.name;
-      }
-      if (this.doc.highlight["name.nGram"]) {
-        return this.doc.highlight["name.nGram"];
-      }
-      if (this.doc.highlight.name) {
-        return this.doc.highlight.name;
-      }
-      return this.doc._source.name;
+    name: "DocFileTitle",
+    props: ["doc"],
+    methods: {
+        ext: ext,
+        fileName() {
+            if (!this.doc.highlight) {
+                return this.doc._source.name;
+            }
+            if (this.doc.highlight["name.nGram"]) {
+                return this.doc.highlight["name.nGram"];
+            }
+            if (this.doc.highlight.name) {
+                return this.doc.highlight.name;
+            }
+            return this.doc._source.name;
+        }
     }
-  }
 }
 </script>
 
 <style scoped>
 .file-title-anchor {
-  max-width: calc(100% - 1.2rem);
+    max-width: calc(100% - 1.2rem);
+}
+
+.file-title-anchor-with-embedding {
+    max-width: calc(100% - 2.2rem);
 }
 
 .file-title {
-  width: 100%;
-  line-height: 1rem;
-  height: 1.1rem;
-  white-space: nowrap;
-  text-overflow: ellipsis;
-  overflow: hidden;
-  font-size: 16px;
-  font-family: "Source Sans Pro", sans-serif;
-  font-weight: bold;
+    width: 100%;
+    max-width: 100%;
+    line-height: 1rem;
+    height: 1.1rem;
+    white-space: nowrap;
+    text-overflow: ellipsis;
+    overflow: hidden;
+    font-size: 16px;
+    font-family: "Source Sans Pro", sans-serif;
+    font-weight: bold;
 }
 
 .theme-black .file-title {
-  color: #ddd;
+    color: #ddd;
 }
+
 .theme-black .file-title:hover {
-  color: #fff;
+    color: #fff;
 }
 
 .theme-light .file-title {
-  color: black;
+    color: black;
 }
 
 .doc-card .file-title {
-  font-size: 12px;
+    font-size: 12px;
 }
 </style>

sist2-vue/src/components/DocListItem.vue

@@ -1,63 +1,64 @@
 <template>
-  <b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}"
-                     @mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
+    <b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}"
+                       @mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
 
-    <!-- Info modal-->
-    <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
+        <!-- Info modal-->
+        <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
 
-    <div class="media ml-2">
+        <div class="media ml-2">
 
-      <!-- Thumbnail-->
-      <div v-if="doc._props.hasThumbnail" class="align-self-start mr-2 wrapper-sm">
-        <div class="img-wrapper">
-          <div v-if="doc._props.isPlayableVideo" class="play">
-            <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
-              <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
-            </svg>
-          </div>
+            <!-- Thumbnail-->
+            <div v-if="doc._props.hasThumbnail" class="align-self-start mr-2 wrapper-sm">
+                <div class="img-wrapper">
+                    <div v-if="doc._props.isPlayableVideo" class="play">
+                        <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
+                            <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
+                        </svg>
+                    </div>
 
-          <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-               :src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
-               alt=""
-               class="pointer fit-sm" @click="onThumbnailClick()">
-          <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
-               class="fit-sm">
-        </div>
-      </div>
-      <div v-else class="file-icon-wrapper" style="">
-        <FileIcon></FileIcon>
-      </div>
+                    <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
+                         :src="(doc._props.isGif && hover) ? `f/${doc._source.index}/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
+                         alt=""
+                         class="pointer fit-sm" @click="onThumbnailClick()">
+                    <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
+                         class="fit-sm">
+                </div>
+            </div>
+            <div v-else class="file-icon-wrapper" style="">
+                <FileIcon></FileIcon>
+            </div>
 
-      <!-- Doc line-->
-      <div class="doc-line ml-3">
-        <div style="display: flex">
-          <span class="info-icon" @click="showInfo = true"></span>
-          <DocFileTitle :doc="doc"></DocFileTitle>
-        </div>
+            <!-- Doc line-->
+            <div class="doc-line ml-3">
+                <div style="display: flex">
+                    <span class="info-icon" @click="showInfo = true"></span>
+                    <MLIcon v-if="doc._source.embedding" clickable @click="onEmbeddingClick()"></MLIcon>
+                    <DocFileTitle :doc="doc"></DocFileTitle>
+                </div>
 
-        <!-- Content highlight -->
-        <ContentDiv :doc="doc"></ContentDiv>
+                <!-- Content highlight -->
+                <ContentDiv :doc="doc"></ContentDiv>
 
-        <div class="path-row">
-          <div class="path-line" v-html="path()"></div>
-          <TagContainer :hit="doc"></TagContainer>
-        </div>
+                <div class="path-row">
+                    <div class="path-line" v-html="path()"></div>
+                    <TagContainer :hit="doc"></TagContainer>
+                </div>
 
-        <div v-if="doc._source.pages || doc._source.author" class="path-row text-muted">
+                <div v-if="doc._source.pages || doc._source.author" class="path-row text-muted">
           <span v-if="doc._source.pages">{{ doc._source.pages }} {{
               doc._source.pages > 1 ? $t("pages") : $t("page")
-            }}</span>
-          <span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
-          <span v-if="doc._source.author">{{ doc._source.author }}</span>
-        </div>
+              }}</span>
+                    <span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
+                    <span v-if="doc._source.author">{{ doc._source.author }}</span>
+                </div>
 
-        <!-- Featured line -->
-        <div style="display: flex">
-          <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
+                <!-- Featured line -->
+                <div style="display: flex">
+                    <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
+                </div>
+            </div>
         </div>
-      </div>
-    </div>
-  </b-list-group-item>
+    </b-list-group-item>
 </template>
 
 <script>
@@ -67,131 +68,140 @@ import DocInfoModal from "@/components/DocInfoModal";
 import ContentDiv from "@/components/ContentDiv";
 import FileIcon from "@/components/icons/FileIcon";
 import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
+import MLIcon from "@/components/icons/MlIcon.vue";
+import Sist2Api from "@/Sist2Api";
 
 export default {
-  name: "DocListItem",
-  components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
-  props: ["doc"],
-  data() {
-    return {
-      hover: false,
-      showInfo: false
-    }
-  },
-  methods: {
-    async onThumbnailClick() {
-      this.$store.commit("setUiLightboxSlide", this.doc._seq);
-      await this.$store.dispatch("showLightbox");
+    name: "DocListItem",
+    components: {MLIcon, FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
+    props: ["doc"],
+    data() {
+        return {
+            hover: false,
+            showInfo: false
+        }
     },
-    path() {
-      if (!this.doc.highlight) {
-        return this.doc._source.path + "/"
-      }
-      if (this.doc.highlight["path.text"]) {
-        return this.doc.highlight["path.text"] + "/"
-      }
+    methods: {
+        async onThumbnailClick() {
+            this.$store.commit("setUiLightboxSlide", this.doc._seq);
+            await this.$store.dispatch("showLightbox");
+        },
+        onEmbeddingClick() {
+            Sist2Api.getEmbeddings(this.doc._source.index, this.doc._id, this.$store.state.embeddingsModel).then(embeddings => {
+                this.$store.commit("setEmbeddingText", "");
+                this.$store.commit("setEmbedding", embeddings);
+                this.$store.commit("setEmbeddingDoc", this.doc);
+            })
+        },
+        path() {
+            if (!this.doc.highlight) {
+                return this.doc._source.path + "/"
+            }
+            if (this.doc.highlight["path.text"]) {
+                return this.doc.highlight["path.text"] + "/"
+            }
 
-      if (this.doc.highlight["path.nGram"]) {
-        return this.doc.highlight["path.nGram"] + "/"
-      }
-      return this.doc._source.path + "/"
-    },
-    onTnEnter() {
-      this.hover = true;
-    },
-    onTnLeave() {
-      this.hover = false;
-    },
-  }
+            if (this.doc.highlight["path.nGram"]) {
+                return this.doc.highlight["path.nGram"] + "/"
+            }
+            return this.doc._source.path + "/"
+        },
+        onTnEnter() {
+            this.hover = true;
+        },
+        onTnLeave() {
+            this.hover = false;
+        },
+    }
 }
 </script>
 
 <style scoped>
 .sub-document {
-  background: #AB47BC1F !important;
+    background: #AB47BC1F !important;
 }
 
 .theme-black .sub-document {
-  background: #37474F !important;
+    background: #37474F !important;
 }
 
 .list-group {
-  margin-top: 1em;
+    margin-top: 1em;
 }
 
 .list-group-item {
-  padding: .25rem 0.5rem;
+    padding: .25rem 0.5rem;
 
-  box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
-  border-radius: 0;
-  border: none;
+    box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
+    border-radius: 0;
+    border: none;
 }
 
 .path-row {
-  display: -ms-flexbox;
-  display: flex;
-  -ms-flex-align: start;
-  align-items: flex-start;
+    display: -ms-flexbox;
+    display: flex;
+    -ms-flex-align: start;
+    align-items: flex-start;
 }
 
 .path-line {
-  color: #808080;
-  text-overflow: ellipsis;
-  overflow: hidden;
-  white-space: nowrap;
-  margin-right: 0.3em;
+    color: #808080;
+    text-overflow: ellipsis;
+    overflow: hidden;
+    white-space: nowrap;
+    margin-right: 0.3em;
 }
 
 .theme-black .path-line {
-  color: #bbb;
+    color: #bbb;
 }
 
 .play {
-  position: absolute;
-  width: 18px;
-  height: 18px;
-  left: 50%;
-  top: 50%;
-  transform: translate(-50%, -50%);
-  pointer-events: none;
+    position: absolute;
+    width: 18px;
+    height: 18px;
+    left: 50%;
+    top: 50%;
+    transform: translate(-50%, -50%);
+    pointer-events: none;
 }
 
 .play svg {
-  fill: rgba(0, 0, 0, 0.7);
+    fill: rgba(0, 0, 0, 0.7);
 }
 
 .list-group-item .img-wrapper {
-  width: 88px;
-  height: 88px;
-  position: relative;
+    width: 88px;
+    height: 88px;
+    position: relative;
 }
 
 .fit-sm {
-  max-height: 100%;
-  max-width: 100%;
-  width: auto;
-  height: auto;
-  position: absolute;
-  top: 0;
-  bottom: 0;
-  left: 0;
-  right: 0;
-  margin: auto;
+    max-height: 100%;
+    max-width: 100%;
+    width: auto;
+    height: auto;
+    position: absolute;
+    top: 0;
+    bottom: 0;
+    left: 0;
+    right: 0;
+    margin: auto;
 
-  /*box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.12);*/
+    /*box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.12);*/
 }
 
 .doc-line {
-  max-width: calc(100% - 88px - 1.5rem);
-  flex: 1;
-  vertical-align: middle;
-  margin-top: auto;
-  margin-bottom: auto;
+    max-width: calc(100% - 88px - 1.5rem);
+    flex: 1;
+    vertical-align: middle;
+    margin-top: auto;
+    margin-bottom: auto;
 }
 
 .file-icon-wrapper {
-  width: calc(88px + .5rem);
-  height: 88px;
-  position: relative;
+    width: calc(88px + .5rem);
+    height: 88px;
+    position: relative;
 }
 </style>

sist2-vue/src/components/EmbeddingsSearchBar.vue

@@ -0,0 +1,155 @@
+<template>
+    <div>
+        <b-progress v-if="modelLoading && [0, 1].includes(modelLoadingProgress)" max="1" class="mb-1" variant="primary"
+                    striped animated :value="1">
+        </b-progress>
+        <b-progress v-else-if="modelLoading" :value="modelLoadingProgress" max="1" class="mb-1" variant="warning"
+                    show-progress>
+        </b-progress>
+        <div style="display: flex">
+            <b-select :options="modelOptions()" class="mr-2 input-prepend" :value="modelName"
+                      @change="onModelChange($event)"></b-select>
+
+            <b-input-group>
+                <b-form-input :value="embeddingText"
+                              :placeholder="$store.state.embeddingDoc ? ' ' : $t('embeddingsSearchPlaceholder')"
+                              @input="onInput($event)"
+                              :disabled="modelLoading"
+                              :style="{'pointer-events': $store.state.embeddingDoc ? 'none' : undefined}"
+                ></b-form-input>
+                <b-badge v-if="$store.state.embeddingDoc" pill variant="primary" class="overlay-badge" href="#"
+                         @click="onBadgeClick()">{{ docName }}
+                </b-badge>
+
+                <template #prepend>
+                </template>
+
+                <template #append>
+                    <b-input-group-text>
+                        <MLIcon class="ml-append" big></MLIcon>
+                    </b-input-group-text>
+                </template>
+
+            </b-input-group>
+        </div>
+
+    </div>
+</template>
+
+<script>
+import {mapGetters, mapMutations} from "vuex";
+import {CLIPTransformerModel} from "@/ml/CLIPTransformerModel"
+import _debounce from "lodash/debounce";
+import MLIcon from "@/components/icons/MlIcon.vue";
+import Sist2AdminApi from "@/Sist2Api";
+
+export default {
+    components: {MLIcon},
+    data() {
+        return {
+            modelLoading: false,
+            modelLoadingProgress: 0,
+            modelLoaded: false,
+            model: null,
+            modelName: null
+        }
+    },
+    computed: {
+        ...mapGetters({
+            optQueryMode: "optQueryMode",
+            embeddingText: "embeddingText",
+            fuzzy: "fuzzy",
+        }),
+        docName() {
+            const ext = this.$store.state.embeddingDoc._source.extension;
+            return this.$store.state.embeddingDoc._source.name +
+                (ext ? "." + ext : "")
+        }
+    },
+    mounted() {
+        // Set default model
+        this.modelName = Sist2AdminApi.models()[0].name;
+        this.onModelChange(this.modelName);
+
+        this.onInput = _debounce(this._onInput, 450, {leading: false});
+    },
+    methods: {
+        ...mapMutations({
+            setEmbeddingText: "setEmbeddingText",
+            setEmbedding: "setEmbedding",
+            setEmbeddingModel: "setEmbeddingsModel",
+        }),
+        async loadModel() {
+            this.modelLoading = true;
+
+            await this.model.init(async progress => {
+                this.modelLoadingProgress = progress;
+            });
+            this.modelLoading = false;
+            this.modelLoaded = true;
+        },
+        async _onInput(text) {
+            try {
+
+                if (!this.modelLoaded) {
+                    await this.loadModel();
+                }
+
+                if (text.length === 0) {
+                    this.setEmbeddingText("");
+                    this.setEmbedding(null);
+                    return;
+                }
+
+                const embeddings = await this.model.predict(text);
+
+                this.setEmbeddingText(text);
+                this.setEmbedding(embeddings);
+            } catch (e) {
+                alert(e)
+            }
+        },
+        modelOptions() {
+            return Sist2AdminApi.models().map(model => model.name);
+        },
+        onModelChange(name) {
+            this.modelLoaded = false;
+            this.modelLoadingProgress = 0;
+
+            const modelInfo = Sist2AdminApi.models().find(m => m.name === name);
+
+            if (modelInfo.name === "CLIP") {
+                const tokenizerUrl = new URL("./tokenizer.json", modelInfo.url).href;
+                this.model = new CLIPTransformerModel(modelInfo.url, tokenizerUrl)
+                this.setEmbeddingModel(modelInfo.id);
+            } else {
+                throw new Error("Unknown model: " + name);
+            }
+        },
+        onBadgeClick() {
+            this.$store.commit("setEmbedding", null);
+            this.$store.commit("setEmbeddingDoc", null);
+        }
+    }
+}
+</script>
+<style>
+.overlay-badge {
+    position: absolute;
+    z-index: 1;
+    left: 0.375rem;
+    top: 8px;
+    line-height: 1.1rem;
+    overflow: hidden;
+    max-width: 200px;
+    text-overflow: ellipsis;
+}
+
+.input-prepend {
+    max-width: 100px;
+}
+
+.theme-black .ml-append {
+    filter: brightness(0.95) !important;
+}
+</style>

sist2-vue/src/components/FullThumbnail.vue

@@ -71,7 +71,7 @@ export default {
       const doc = this.doc;
       const props = doc._props;
       if (props.isGif && this.hover) {
-        return `f/${doc._id}`;
+        return `f/${doc._source.index}/${doc._id}`;
       }
       return (this.currentThumbnailNum === 0)
           ? `t/${doc._source.index}/${doc._id}`

sist2-vue/src/components/IndexPicker.vue

@@ -1,42 +1,46 @@
 <template>
-  <div v-if="isMobile">
-    <b-form-select
-        :value="selectedIndicesIds"
-        @change="onSelect($event)"
-        :options="indices" multiple :select-size="6" text-field="name"
-        value-field="id"></b-form-select>
-  </div>
-  <div v-else>
+    <div v-if="isMobile">
+        <b-form-select
+                :value="selectedIndicesIds"
+                @change="onSelect($event)"
+                :options="indices" multiple :select-size="6" text-field="name"
+                value-field="id"></b-form-select>
+    </div>
+    <div v-else>
 
-    <div class="d-flex justify-content-between align-content-center">
+        <div class="d-flex justify-content-between align-content-center">
       <span>
         {{ selectedIndices.length }}
         {{ selectedIndices.length === 1 ? $t("indexPicker.selectedIndex") : $t("indexPicker.selectedIndices") }}
       </span>
 
-      <div>
-        <b-button variant="link" @click="selectAll()"> {{ $t("indexPicker.selectAll") }}</b-button>
-        <b-button variant="link" @click="selectNone()"> {{ $t("indexPicker.selectNone") }}</b-button>
-      </div>
-    </div>
-
-    <b-list-group id="index-picker-desktop" class="unselectable">
-      <b-list-group-item
-          v-for="idx in indices"
-          @click="toggleIndex(idx, $event)"
-          @click.shift="shiftClick(idx, $event)"
-          class="d-flex justify-content-between align-items-center list-group-item-action pointer"
-          :class="{active: lastClickIndex === idx}"
-      >
-        <div class="d-flex">
-          <b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
-          {{ idx.name }}
-          <span class="text-muted timestamp-text ml-2">{{ formatIdxDate(idx.timestamp) }}</span>
+            <div>
+                <b-button variant="link" @click="selectAll()"> {{ $t("indexPicker.selectAll") }}</b-button>
+                <b-button variant="link" @click="selectNone()"> {{ $t("indexPicker.selectNone") }}</b-button>
+            </div>
         </div>
-        <b-badge class="version-badge">v{{ idx.version }}</b-badge>
-      </b-list-group-item>
-    </b-list-group>
-  </div>
+
+        <b-list-group id="index-picker-desktop" class="unselectable">
+            <b-list-group-item
+                    v-for="idx in indices"
+                    @click="toggleIndex(idx, $event)"
+                    @click.shift="shiftClick(idx, $event)"
+                    class="d-flex justify-content-between align-items-center list-group-item-action pointer"
+                    :class="{active: lastClickIndex === idx}"
+            >
+                <div class="d-flex">
+                    <b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
+                    {{ idx.name }}
+                    <div style="vertical-align: center; margin-left: 5px">
+                        <MLIcon small style="top: -1px; position: relative"></MLIcon>
+                    </div>
+                    <span class="text-muted timestamp-text ml-2"
+                          style="top: 1px; position: relative">{{ formatIdxDate(idx.timestamp) }}</span>
+                </div>
+                <b-badge class="version-badge">v{{ idx.version }}</b-badge>
+            </b-list-group-item>
+        </b-list-group>
+    </div>
 </template>
 
 <script lang="ts">
@@ -44,148 +48,150 @@ import SmallBadge from "./SmallBadge.vue"
 import {mapActions, mapGetters} from "vuex";
 import Vue from "vue";
 import {format} from "date-fns";
+import MLIcon from "@/components/icons/MlIcon.vue";
 
 export default Vue.extend({
-  components: {
-    SmallBadge
-  },
-  data() {
-    return {
-      loading: true,
-      lastClickIndex: null
-    }
-  },
-  computed: {
-    ...mapGetters([
-      "indices", "selectedIndices"
-    ]),
-    selectedIndicesIds() {
-      return this.selectedIndices.map(idx => idx.id)
+    components: {
+        MLIcon,
+        SmallBadge
     },
-    isMobile() {
-      return window.innerWidth <= 650;
-    }
-  },
-  methods: {
-    ...mapActions({
-      setSelectedIndices: "setSelectedIndices"
-    }),
-    shiftClick(index, e) {
-      if (this.lastClickIndex === null) {
-        return;
-      }
-
-      const select = this.isSelected(this.lastClickIndex);
-
-      let leftBoundary = this.indices.indexOf(this.lastClickIndex);
-      let rightBoundary = this.indices.indexOf(index);
-
-      if (rightBoundary < leftBoundary) {
-        let tmp = leftBoundary;
-        leftBoundary = rightBoundary;
-        rightBoundary = tmp;
-      }
-
-      for (let i = leftBoundary; i <= rightBoundary; i++) {
-        if (select) {
-          if (!this.isSelected(this.indices[i])) {
-            this.setSelectedIndices([this.indices[i], ...this.selectedIndices]);
-          }
-        } else {
-          this.setSelectedIndices(this.selectedIndices.filter(idx => idx !== this.indices[i]));
+    data() {
+        return {
+            loading: true,
+            lastClickIndex: null
         }
-      }
     },
-    selectAll() {
-      this.setSelectedIndices(this.indices);
+    computed: {
+        ...mapGetters([
+            "indices", "selectedIndices"
+        ]),
+        selectedIndicesIds() {
+            return this.selectedIndices.map(idx => idx.id)
+        },
+        isMobile() {
+            return window.innerWidth <= 650;
+        }
     },
-    selectNone() {
-      this.setSelectedIndices([]);
-    },
-    onSelect(value) {
-      this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
-    },
-    formatIdxDate(timestamp: number): string {
-      return format(new Date(timestamp * 1000), "yyyy-MM-dd");
-    },
-    toggleIndex(index, e) {
-      if (e.shiftKey) {
-        return;
-      }
+    methods: {
+        ...mapActions({
+            setSelectedIndices: "setSelectedIndices"
+        }),
+        shiftClick(index, e) {
+            if (this.lastClickIndex === null) {
+                return;
+            }
 
-      this.lastClickIndex = index;
-      if (this.isSelected(index)) {
-        this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
-      } else {
-        this.setSelectedIndices([index, ...this.selectedIndices]);
-      }
+            const select = this.isSelected(this.lastClickIndex);
+
+            let leftBoundary = this.indices.indexOf(this.lastClickIndex);
+            let rightBoundary = this.indices.indexOf(index);
+
+            if (rightBoundary < leftBoundary) {
+                let tmp = leftBoundary;
+                leftBoundary = rightBoundary;
+                rightBoundary = tmp;
+            }
+
+            for (let i = leftBoundary; i <= rightBoundary; i++) {
+                if (select) {
+                    if (!this.isSelected(this.indices[i])) {
+                        this.setSelectedIndices([this.indices[i], ...this.selectedIndices]);
+                    }
+                } else {
+                    this.setSelectedIndices(this.selectedIndices.filter(idx => idx !== this.indices[i]));
+                }
+            }
+        },
+        selectAll() {
+            this.setSelectedIndices(this.indices);
+        },
+        selectNone() {
+            this.setSelectedIndices([]);
+        },
+        onSelect(value) {
+            this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
+        },
+        formatIdxDate(timestamp: number): string {
+            return format(new Date(timestamp * 1000), "yyyy-MM-dd");
+        },
+        toggleIndex(index, e) {
+            if (e.shiftKey) {
+                return;
+            }
+
+            this.lastClickIndex = index;
+            if (this.isSelected(index)) {
+                this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
+            } else {
+                this.setSelectedIndices([index, ...this.selectedIndices]);
+            }
+        },
+        isSelected(index) {
+            return this.selectedIndices.find(idx => idx.id == index.id) != null;
+        }
     },
-    isSelected(index) {
-      return this.selectedIndices.find(idx => idx.id == index.id) != null;
-    }
-  },
 })
 </script>
 
 <style scoped>
 .timestamp-text {
-  line-height: 24px;
-  font-size: 80%;
+    line-height: 24px;
+    font-size: 80%;
 }
 
 .theme-black .version-badge {
-  color: #eee !important;
-  background: none;
+    color: #eee !important;
+    background: none;
 }
 
 .version-badge {
-  color: #222 !important;
-  background: none;
+    color: #222 !important;
+    background: none;
 }
 
 .list-group-item {
-  padding: 0.2em 0.4em;
+    padding: 0.2em 0.4em;
 }
 
 #index-picker-desktop {
-  overflow-y: auto;
-  max-height: 132px;
+    overflow-y: auto;
+    max-height: 132px;
 }
 
 .btn-link:focus {
-  box-shadow: none;
+    box-shadow: none;
 }
 
 .unselectable {
-  user-select: none;
-  -ms-user-select: none;
-  -moz-user-select: none;
-  -webkit-user-select: none;
+    user-select: none;
+    -ms-user-select: none;
+    -moz-user-select: none;
+    -webkit-user-select: none;
 }
 
 .list-group-item.active {
-  z-index: 2;
-  background-color: inherit;
-  color: inherit;
+    z-index: 2;
+    background-color: inherit;
+    color: inherit;
 }
 
 .theme-black .list-group-item {
-  border: 1px solid rgba(255,255,255, 0.1);
+    border: 1px solid rgba(255, 255, 255, 0.1);
 }
 
 .theme-black .list-group-item:first-child {
-  border: 1px solid rgba(255,255,255, 0.05);
+    border: 1px solid rgba(255, 255, 255, 0.05);
 }
 
 .theme-black .list-group-item.active {
-  z-index: 2;
-  background-color: inherit;
-  color: inherit;
-  border: 1px solid rgba(255,255,255, 0.3);
-  border-radius: 0;
+    z-index: 2;
+    background-color: inherit;
+    color: inherit;
+    border: 1px solid rgba(255, 255, 255, 0.3);
+    border-radius: 0;
 }
 
 .theme-black .list-group {
-  border-radius: 0;
+    border-radius: 0;
 }
 </style>

sist2-vue/src/components/LazyContentDiv.vue

@@ -9,7 +9,7 @@
             <b-button :disabled="mlPredictionsLoading || mlLoading" @click="mlAnalyze" variant="primary"
             >{{ $t("ml.analyzeText") }}
             </b-button>
-            <b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="mlModel">
+            <b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="nerModel">
                 <b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()">{{ opt.text }}
                 </b-select-option>
             </b-select>
@@ -18,7 +18,7 @@
         <b-progress v-if="mlLoading" variant="warning" show-progress :max="1" class="mb-3"
         >
             <b-progress-bar :value="modelLoadingProgress">
-                <strong>{{ ((modelLoadingProgress * modelSize) / (1024*1024)).toFixed(1) }}MB / {{
+                <strong>{{ ((modelLoadingProgress * modelSize) / (1024 * 1024)).toFixed(1) }}MB / {{
                     (modelSize / (1024 * 1024)).toFixed(1)
                     }}MB</strong>
             </b-progress-bar>
@@ -36,7 +36,7 @@
 <script>
 import Sist2Api from "@/Sist2Api";
 import Preloader from "@/components/Preloader";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 import store from "@/store";
 import BertNerModel from "@/ml/BertNerModel";
 import AnalyzedContentSpansContainer from "@/components/AnalyzedContentSpanContainer.vue";
@@ -57,75 +57,36 @@ export default {
             modelPredictionProgress: 0,
             mlPredictionsLoading: false,
             mlLoading: false,
-            mlModel: null,
+            nerModel: null,
             analyzedContentSpans: []
         }
     },
     mounted() {
 
         if (this.$store.getters.optMlDefaultModel) {
-            this.mlModel = this.$store.getters.optMlDefaultModel
+            this.nerModel = this.$store.getters.optMlDefaultModel
         } else {
-            this.mlModel = ModelsRepo.getDefaultModel();
+            this.nerModel = ModelsRepo.getDefaultModel();
         }
 
-        const query = Sist2Query.searchQuery();
+        Sist2Api
+            .getDocument(this.docId, this.$store.state.optHighlight, this.$store.state.fuzzy)
+            .then(doc => {
+                this.loading = false;
 
-        if (this.$store.state.optHighlight) {
-            const fields = this.$store.state.fuzzy
-                ? {"content.nGram": {}}
-                : {content: {}};
+                if (doc) {
+                    this.content = this.getContent(doc)
+                }
 
-            query.highlight = {
-                pre_tags: ["<mark>"],
-                post_tags: ["</mark>"],
-                number_of_fragments: 0,
-                fields,
-            };
-
-            if (!store.state.sist2Info.esVersionLegacy) {
-                query.highlight.max_analyzed_offset = 999_999;
-            }
-        }
-
-        if ("function_score" in query.query) {
-            query.query = query.query.function_score.query;
-        }
-
-        if (!("must" in query.query.bool)) {
-            query.query.bool.must = [];
-        } else if (!Array.isArray(query.query.bool.must)) {
-            query.query.bool.must = [query.query.bool.must];
-        }
-
-        query.query.bool.must.push({match: {_id: this.docId}});
-
-        delete query["sort"];
-        delete query["aggs"];
-        delete query["search_after"];
-        delete query.query["function_score"];
-
-        query._source = {
-            includes: ["content", "name", "path", "extension"]
-        }
-
-        query.size = 1;
-
-        Sist2Api.esQuery(query).then(resp => {
-            this.loading = false;
-            if (resp.hits.hits.length === 1) {
-                this.content = this.getContent(resp.hits.hits[0]);
-            }
-
-            if (this.optAutoAnalyze) {
-                this.mlAnalyze();
-            }
-        });
+                if (this.optAutoAnalyze) {
+                    this.mlAnalyze();
+                }
+            });
     },
     computed: {
         ...mapGetters(["optAutoAnalyze"]),
         modelSize() {
-            const modelData = ModelsRepo.data[this.mlModel];
+            const modelData = ModelsRepo.data[this.nerModel];
             if (!modelData) {
                 return 0;
             }
@@ -149,10 +110,10 @@ export default {
             }
         },
         async getMlModel() {
-            if (this.$store.getters.mlModel.name !== this.mlModel) {
+            if (this.$store.getters.nerModel.name !== this.nerModel) {
                 this.mlLoading = true;
                 this.modelLoadingProgress = 0;
-                const modelInfo = ModelsRepo.data[this.mlModel];
+                const modelInfo = ModelsRepo.data[this.nerModel];
 
                 const model = new BertNerModel(
                     modelInfo.vocabUrl,
@@ -161,25 +122,25 @@ export default {
                 )
 
                 await model.init(progress => this.modelLoadingProgress = progress);
-                this.$store.commit("setMlModel", {model, name: this.mlModel});
+                this.$store.commit("setNerModel", {model, name: this.nerModel});
 
                 this.mlLoading = false;
                 return model
             }
 
-            return this.$store.getters.mlModel.model;
+            return this.$store.getters.nerModel.model;
         },
         async mlAnalyze() {
             if (!this.content) {
                 return;
             }
 
-            const modelInfo = ModelsRepo.data[this.mlModel];
+            const modelInfo = ModelsRepo.data[this.nerModel];
             if (modelInfo === undefined) {
                 return;
             }
 
-            this.$store.commit("setOptMlDefaultModel", this.mlModel);
+            this.$store.commit("setOptMlDefaultModel", this.nerModel);
             await this.$store.dispatch("updateConfiguration");
 
             const model = await this.getMlModel();

sist2-vue/src/components/MimePicker.vue

@@ -9,7 +9,7 @@ import InspireTreeDOM from "inspire-tree-dom";
 import "inspire-tree-dom/dist/inspire-tree-light.min.css";
 import {getSelectedTreeNodes, getTreeNodeAttributes} from "@/util";
 import Sist2Api from "@/Sist2Api";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 
 export default {
   name: "MimePicker",

sist2-vue/src/components/NavBar.vue

@@ -10,7 +10,7 @@
     <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
       v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
         href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
-        target="_blank">legacyES</a></span>
+        target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
     </span>
 
     <span v-if="$store && $store.state.sist2Info" class="tagline" v-html="tagline()"></span>

sist2-vue/src/components/PathTree.vue

@@ -1,40 +1,41 @@
 <template>
-  <div>
-    <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
-      <div class="input-group-prepend">
+    <div>
+        <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
+            <div class="input-group-prepend">
 
-        <b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
-          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
-            <path
-                fill="currentColor"
-                d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
-            />
-          </svg>
-        </b-button>
-      </div>
+                <b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
+                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
+                        <path
+                                fill="currentColor"
+                                d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
+                        />
+                    </svg>
+                </b-button>
+            </div>
+
+            <VueSimpleSuggest
+                    class="form-control-fix-flex"
+                    @input="setPathText"
+                    :value="getPathText"
+                    :list="suggestPath"
+                    :max-suggestions="0"
+                    :placeholder="$t('pathBar.placeholder')"
+                    :debounce="200"
+            >
+                <!-- Suggestion item template-->
+                <div slot="suggestion-item" slot-scope="{ suggestion, query }">
+                    <div class="suggestion-line" :title="suggestion">
+                        <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
+                    </div>
+                </div>
+            </VueSimpleSuggest>
 
-      <VueSimpleSuggest
-          class="form-control-fix-flex"
-          @input="setPathText"
-          :value="getPathText"
-          :list="suggestPath"
-          :max-suggestions="0"
-          :placeholder="$t('pathBar.placeholder')"
-      >
-        <!-- Suggestion item template-->
-        <div slot="suggestion-item" slot-scope="{ suggestion, query }">
-          <div class="suggestion-line" :title="suggestion">
-            <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
-          </div>
         </div>
-      </VueSimpleSuggest>
 
+        <b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
+            <div id="pathTree"></div>
+        </b-modal>
     </div>
-
-    <b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
-      <div id="pathTree"></div>
-    </b-modal>
-  </div>
 </template>
 
 <script>
@@ -48,198 +49,153 @@ import VueSimpleSuggest from 'vue-simple-suggest'
 import 'vue-simple-suggest/dist/styles.css' // Optional CSS
 
 export default {
-  name: "PathTree",
-  components: {
-    VueSimpleSuggest
-  },
-  data() {
-    return {
-      mimeTree: null,
-      pathItems: [],
-      tmpPath: ""
-
-    }
-  },
-  computed: {
-    ...mapGetters(["getPathText"])
-  },
-  mounted() {
-    this.$store.subscribe((mutation) => {
-      // Wait until indices are loaded to get the root paths
-      if (mutation.type === "setIndices") {
-        let pathTree = new InspireTree({
-          data: (node, resolve, reject) => {
-            return this.getNextDepth(node);
-          },
-          sort: "text"
-        });
-
-        this.$store.state.indices.forEach(idx => {
-          pathTree.addNode({
-            id: "/" + idx.id,
-            values: ["/" + idx.id],
-            text: `/[${idx.name}]`,
-            index: idx.id,
-            depth: 0,
-            children: true
-          })
-        });
-
-        new InspireTreeDOM(pathTree, {
-          target: "#pathTree"
-        });
-
-        pathTree.on("node.click", this.handleTreeClick);
-        pathTree.expand();
-      }
-    });
-  },
-  methods: {
-    ...mapMutations(["setPathText"]),
-    getSuggestionWithoutQueryPrefix(suggestion, query) {
-      return suggestion.slice(query.length)
+    name: "PathTree",
+    components: {
+        VueSimpleSuggest
     },
-    async getPathChoices() {
-      return new Promise(getPaths => {
-        const q = {
-          suggest: {
-            path: {
-              prefix: this.getPathText,
-              completion: {
-                field: "suggest-path",
-                skip_duplicates: true,
-                size: 10000
-              }
-            }
-          }
-        };
+    data() {
+        return {
+            mimeTree: null,
+            pathItems: [],
+            tmpPath: ""
 
-        Sist2Api.esQuery(q)
-            .then(resp => getPaths(resp["suggest"]["path"][0]["options"].map(opt => opt["_source"]["path"])));
-      })
-    },
-    async suggestPath(term) {
-      if (!this.$store.state.optSuggestPath) {
-        return []
-      }
-
-      term = term.toLowerCase();
-
-      const choices = await this.getPathChoices();
-
-      let matches = [];
-      for (let i = 0; i < choices.length; i++) {
-        if (~choices[i].toLowerCase().indexOf(term)) {
-          matches.push(choices[i]);
         }
-      }
-      return matches.sort((a, b) => a.length - b.length);
     },
-    getNextDepth(node) {
-      const q = {
-        query: {
-          bool: {
-            filter: [
-              {term: {index: node.index}},
-              {range: {_depth: {gte: node.depth + 1, lte: node.depth + 3}}},
-            ]
-          }
-        },
-        aggs: {
-          paths: {
-            terms: {
-              field: "path",
-              size: 10000
+    computed: {
+        ...mapGetters(["getPathText"])
+    },
+    mounted() {
+        this.$store.subscribe((mutation) => {
+            // Wait until indices are loaded to get the root paths
+            if (mutation.type === "setIndices") {
+                let pathTree = new InspireTree({
+                    data: (node, resolve, reject) => {
+                        return this.getNextDepth(node);
+                    },
+                    sort: "text"
+                });
+
+                this.$store.state.indices.forEach(idx => {
+                    pathTree.addNode({
+                        id: "/" + idx.id,
+                        values: ["/" + idx.id],
+                        text: `/[${idx.name}]`,
+                        index: idx.id,
+                        depth: 0,
+                        children: true
+                    })
+                });
+
+                new InspireTreeDOM(pathTree, {
+                    target: "#pathTree"
+                });
+
+                pathTree.on("node.click", this.handleTreeClick);
+                pathTree.expand();
             }
-          }
+        });
+    },
+    methods: {
+        ...mapMutations(["setPathText"]),
+        getSuggestionWithoutQueryPrefix(suggestion, query) {
+            return suggestion.slice(query.length)
         },
-        size: 0
-      };
+        async getPathChoices() {
+            return new Promise(getPaths => {
+                Sist2Api.getPathSuggestions(this.getPathText).then(getPaths);
+            });
+        },
+        async suggestPath(term) {
+            if (!this.$store.state.optSuggestPath) {
+                return []
+            }
 
-      if (node.depth > 0) {
-        q.query.bool.must = {
-          prefix: {
-            path: node.id,
-          }
-        };
-      }
+            term = term.toLowerCase();
 
-      return Sist2Api.esQuery(q).then(resp => {
-        const buckets = resp["aggregations"]["paths"]["buckets"];
-        if (!buckets) {
-          return false;
-        }
+            const choices = await this.getPathChoices();
 
-        const paths = [];
+            let matches = [];
+            for (let i = 0; i < choices.length; i++) {
+                if (~choices[i].toLowerCase().indexOf(term)) {
+                    matches.push(choices[i]);
+                }
+            }
+            return matches.sort((a, b) => a.length - b.length);
+        },
+        getNextDepth(node) {
+            return Sist2Api
+                .searchPaths(node.index, node.depth + 1, node.depth + 3, node.depth > 0 ? node.id : null)
+                .then(buckets => {
+                    const paths = [];
 
-        return buckets
-            .filter(bucket => bucket.key.length > node.id.length || node.id.startsWith("/"))
-            .sort((a, b) => a.key > b.key)
-            .map(bucket => {
+                    return buckets
+                        .filter(bucket => bucket.path.length > node.id.length || node.id.startsWith("/"))
+                        .sort((a, b) => a.path > b.path ? 1 : -1)
+                        .map(bucket => {
+                            if (paths.some(n => bucket.path.startsWith(n))) {
+                                return null;
+                            }
 
-              if (paths.some(n => bucket.key.startsWith(n))) {
-                return null;
-              }
+                            const name = node.id.startsWith("/") ? bucket.path : bucket.path.slice(node.id.length + 1);
 
-              const name = node.id.startsWith("/") ? bucket.key : bucket.key.slice(node.id.length + 1);
+                            paths.push(bucket.path);
 
-              paths.push(bucket.key);
+                            return {
+                                id: bucket.path,
+                                text: `${name}/ (${bucket.count})`,
+                                depth: node.depth + 1,
+                                index: node.index,
+                                values: [bucket.path],
+                                children: true,
+                            }
+                        })
+                        .filter(bucket => bucket !== null);
+                });
+        },
+        handleTreeClick(e, node, handler) {
+            if (node.depth !== 0) {
+                this.setPathText(node.id);
+                this.$refs['path-modal'].hide()
 
-              return {
-                id: bucket.key,
-                text: `${name}/ (${bucket.doc_count})`,
-                depth: node.depth + 1,
-                index: node.index,
-                values: [bucket.key],
-                children: true,
-              }
-            }).filter(x => x !== null)
-      });
+                this.$emit("search");
+            }
+
+            handler();
+        },
     },
-    handleTreeClick(e, node, handler) {
-      if (node.depth !== 0) {
-        this.setPathText(node.id);
-        this.$refs['path-modal'].hide()
-
-        this.$emit("search");
-      }
-
-      handler();
-    },
-  },
 }
 </script>
 
 <style scoped>
 #mimeTree {
-  max-height: 350px;
-  overflow: auto;
+    max-height: 350px;
+    overflow: auto;
 }
 
 .form-control-fix-flex {
-  flex: 1 1 auto;
-  width: 1%;
-  min-width: 0;
-  margin-bottom: 0;
+    flex: 1 1 auto;
+    width: 1%;
+    min-width: 0;
+    margin-bottom: 0;
 }
 
 .suggestion-line {
-  max-width: 100%;
-  white-space: nowrap;
-  overflow: hidden;
-  text-overflow: ellipsis;
-  line-height: 1.1;
+    max-width: 100%;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    line-height: 1.1;
 }
 
 </style>
 
 <style>
 .suggestions {
-  max-height: 250px;
-  overflow-y: auto;
+    max-height: 250px;
+    overflow-y: auto;
 }
 
 .theme-black .suggestions {
-  color: black
+    color: black
 }
 </style>

sist2-vue/src/components/ResultsCard.vue

@@ -1,40 +1,46 @@
 <template>
-  <b-card v-if="lastResultsLoaded" id="results">
-    <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
+    <b-card v-if="lastResultsLoaded" id="results">
+        <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
 
-    <div style="float: right">
-      <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
-          $t("details")
-        }}
-      </b-button>
+        <div style="float: right">
+            <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
+                $t("details")
+                }}
+            </b-button>
 
-      <template v-if="hitCount !== 0">
-        <SortSelect class="ml-2"></SortSelect>
+            <template v-if="hitCount !== 0">
+                <SortSelect class="ml-2"></SortSelect>
 
-        <DisplayModeToggle class="ml-2"></DisplayModeToggle>
-      </template>
-    </div>
+                <DisplayModeToggle class="ml-2"></DisplayModeToggle>
+            </template>
+        </div>
 
-    <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
-      <b-card>
-        <b-table :items="tableItems" small borderless bordered thead-class="hidden" class="mb-0"></b-table>
+        <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
+            <b-card>
+                <b-table :items="tableItems" small borderless thead-class="hidden" class="mb-0"></b-table>
 
-        <br/>
-        <h4>
-          {{$t("mimeTypes")}}
-          <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick"><ClipboardIcon/></b-button>
-        </h4>
-        <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
-        <b-table
-            v-else
-            sort-by="doc_count"
-            :sort-desc="true"
-            thead-class="hidden"
-            :items="$store.state.uiDetailsMimeAgg" small bordered class="mb-0"
-        ></b-table>
-      </b-card>
-    </b-collapse>
-  </b-card>
+                <template v-if="!$store.state.uiSqliteMode">
+
+                    <br/>
+                    <h4>
+                        {{ $t("mimeTypes") }}
+                        <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick">
+                            <ClipboardIcon/>
+                        </b-button>
+                    </h4>
+                    <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
+                    <b-table
+                            v-else
+                            sort-by="doc_count"
+                            :sort-desc="true"
+                            thead-class="hidden"
+                            bordered
+                            :items="$store.state.uiDetailsMimeAgg" small class="mb-0"
+                    ></b-table>
+                </template>
+            </b-card>
+        </b-collapse>
+    </b-card>
 </template>
 
 <script lang="ts">
@@ -44,91 +50,96 @@ import {humanFileSize} from "@/util";
 import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
 import SortSelect from "@/components/SortSelect.vue";
 import Preloader from "@/components/Preloader.vue";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
 
 export default Vue.extend({
-  name: "ResultsCard",
-  components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
-  created() {
+    name: "ResultsCard",
+    components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
+    created() {
 
-  },
-  computed: {
-    lastResultsLoaded() {
-      return this.$store.state.lastQueryResults != null;
     },
-    hitCount() {
-      return (this.$store.state.lastQueryResults as EsResult).aggregations.total_count.value;
+    computed: {
+        lastResultsLoaded() {
+            return this.$store.state.lastQueryResults != null;
+        },
+        hitCount() {
+            return (this.$store.state.firstQueryResults as EsResult).aggregations.total_count.value;
+        },
+        tableItems() {
+            const items = [];
+
+            if (!this.$store.state.uiSqliteMode) {
+                items.push({key: this.$t("queryTime"), value: this.took()});
+            }
+            items.push({key: this.$t("totalSize"), value: this.totalSize()});
+
+            return items;
+        }
     },
-    tableItems() {
-      const items = [];
+    methods: {
+        took() {
+            return (this.$store.state.lastQueryResults as EsResult).took + "ms";
+        },
+        totalSize() {
+            return humanFileSize((this.$store.state.firstQueryResults as EsResult).aggregations.total_size.value);
+        },
+        onToggle() {
+            const show = !document.getElementById("collapse-1").classList.contains("show");
+            this.$store.commit("setUiShowDetails", show);
 
+            if (this.$store.state.uiSqliteMode) {
+                return;
+            }
 
-      items.push({key: this.$t("queryTime"), value: this.took()});
-      items.push({key: this.$t("totalSize"), value: this.totalSize()});
+            if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
+                // Mime aggs are not updated automatically, update now
+                this.forceUpdateMimeAgg();
+            }
+        },
+        onCopyClick() {
+            let tsvString = "";
+            this.$store.state.uiDetailsMimeAgg.slice().sort((a, b) => b["doc_count"] - a["doc_count"]).forEach(row => {
+                tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
+            });
 
-      return items;
-    }
-  },
-  methods: {
-    took() {
-      return (this.$store.state.lastQueryResults as EsResult).took + "ms";
+            navigator.clipboard.writeText(tsvString);
+
+            this.$bvToast.toast(
+                this.$t("toast.copiedToClipboard"),
+                {
+                    title: null,
+                    noAutoHide: false,
+                    toaster: "b-toaster-bottom-right",
+                    headerClass: "hidden",
+                    bodyClass: "toast-body-info",
+                });
+        },
+        forceUpdateMimeAgg() {
+            const query = Sist2Query.searchQuery();
+            Sist2Api.getMimeTypes(query).then(({buckets}) => {
+                this.$store.commit("setUiDetailsMimeAgg", buckets);
+            });
+        }
     },
-    totalSize() {
-      return humanFileSize((this.$store.state.lastQueryResults as EsResult).aggregations.total_size.value);
-    },
-    onToggle() {
-      const show = !document.getElementById("collapse-1").classList.contains("show");
-      this.$store.commit("setUiShowDetails", show);
-
-      if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
-        // Mime aggs are not updated automatically, update now
-        this.forceUpdateMimeAgg();
-      }
-    },
-    onCopyClick() {
-      let tsvString = "";
-      this.$store.state.uiDetailsMimeAgg.slice().sort((a,b) => b["doc_count"] - a["doc_count"]).forEach(row => {
-        tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
-      });
-
-      navigator.clipboard.writeText(tsvString);
-
-      this.$bvToast.toast(
-          this.$t("toast.copiedToClipboard"),
-          {
-            title: null,
-            noAutoHide: false,
-            toaster: "b-toaster-bottom-right",
-            headerClass: "hidden",
-            bodyClass: "toast-body-info",
-          });
-    },
-    forceUpdateMimeAgg() {
-      const query = Sist2Query.searchQuery();
-      Sist2Api.getMimeTypes(query).then(({buckets}) => {
-        this.$store.commit("setUiDetailsMimeAgg", buckets);
-      });
-    }
-  },
 });
 
 </script>
 
 <style>
 #results {
-  margin-top: 1em;
+    margin-top: 1em;
 
-  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-  border-radius: 0;
-  border: none;
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
 }
 
 #results .card-body {
-  padding: 0.7em 1.25em;
+    padding: 0.7em 1.25em;
 }
 
 .hidden {
-  display: none;
+    display: none;
 }
 </style>

sist2-vue/src/components/SearchBar.vue

@@ -6,7 +6,7 @@
                     @input="setSearchText($event)"></b-form-input>
 
       <template #prepend>
-        <b-input-group-text>
+        <b-input-group-text v-if="!$store.state.uiSqliteMode">
           <b-form-checkbox :checked="fuzzy" title="Toggle fuzzy searching" @change="setFuzzy($event)">
             {{ $t("searchBar.fuzzy") }}
           </b-form-checkbox>

sist2-vue/src/components/SortSelect.vue

@@ -1,5 +1,5 @@
 <template>
-  <b-dropdown variant="primary">
+  <b-dropdown variant="primary" :disabled="$store.getters.embedding !== null">
     <b-dropdown-item :class="{'dropdown-active': sort === 'score'}" @click="onSelect('score')">{{
         $t("sort.relevance")
       }}

sist2-vue/src/components/TagContainer.vue

@@ -1,76 +1,78 @@
 <template>
-  <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
+    <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
 
-    <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static lazy>
-      <b-row>
-        <b-col style="flex-grow: 2" sm>
-          <VueSimpleSuggest
-              ref="suggest"
-              :value="tagText"
-              @select="setTagText($event)"
-              @input="setTagText($event)"
-              class="form-control-fix-flex"
-              style="margin-top: 17px"
-              :list="suggestTag"
-              :max-suggestions="0"
-              :placeholder="$t('saveTagPlaceholder')"
-          >
-            <!-- Suggestion item template-->
-            <div slot="suggestion-item" slot-scope="{ suggestion, query}"
-            >
-              <div class="suggestion-line">
+        <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static
+                 lazy>
+            <b-row>
+                <b-col style="flex-grow: 2" sm>
+                    <VueSimpleSuggest
+                            ref="suggest"
+                            :value="tagText"
+                            @select="setTagText($event)"
+                            @input="setTagText($event)"
+                            class="form-control-fix-flex"
+                            style="margin-top: 17px"
+                            :list="suggestTag"
+                            :max-suggestions="0"
+                            :placeholder="$t('saveTagPlaceholder')"
+                    >
+                        <!-- Suggestion item template-->
+                        <div slot="suggestion-item" slot-scope="{ suggestion, query}"
+                        >
+                            <div class="suggestion-line">
                 <span
-                    class="badge badge-suggestion"
-                    :style="{background: getBg(suggestion), color: getFg(suggestion)}"
+                        class="badge badge-suggestion"
+                        :style="{background: getBg(suggestion), color: getFg(suggestion)}"
                 >
                 <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
                 </span>
-              </div>
-            </div>
-          </VueSimpleSuggest>
-        </b-col>
-        <b-col class="mt-4">
-          <TwitterColorPicker v-model="color" triangle="hide" :width="252" class="mr-auto ml-auto"></TwitterColorPicker>
-        </b-col>
-      </b-row>
+                            </div>
+                        </div>
+                    </VueSimpleSuggest>
+                </b-col>
+                <b-col class="mt-4">
+                    <TwitterColorPicker v-model="color" triangle="hide" :width="252"
+                                        class="mr-auto ml-auto"></TwitterColorPicker>
+                </b-col>
+            </b-row>
 
-      <b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
-      </b-button>
-    </b-modal>
+            <b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
+            </b-button>
+        </b-modal>
 
 
-    <template v-for="tag in hit._tags">
-      <!-- User tag-->
-      <div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
+        <template v-for="tag in hit._tags">
+            <!-- User tag-->
+            <div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
         <span
-            :id="hit._id+tag.rawText"
-            :title="tag.text"
-            tabindex="-1"
-            class="badge pointer"
-            :style="badgeStyle(tag)" :class="badgeClass(tag)"
-            @click.right="onTagRightClick(tag, $event)"
+                :id="hit._id+tag.rawText"
+                :title="tag.text"
+                tabindex="-1"
+                class="badge pointer"
+                :style="badgeStyle(tag)" :class="badgeClass(tag)"
+                @click.right="onTagRightClick(tag, $event)"
         >{{ tag.text.split(".").pop() }}</span>
 
-        <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
-          <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{ $t("deleteTag") }}</b-button>
-        </b-popover>
-      </div>
+                <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
+                    <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{ $t("deleteTag") }}</b-button>
+                </b-popover>
+            </div>
 
-      <span
-          v-else :key="tag.text"
-          class="badge"
-          :style="badgeStyle(tag)" :class="badgeClass(tag)"
-      >{{ tag.text.split(".").pop() }}</span>
-    </template>
+            <span
+                    v-else :key="tag.text"
+                    class="badge"
+                    :style="badgeStyle(tag)" :class="badgeClass(tag)"
+            >{{ tag.text.split(".").pop() }}</span>
+        </template>
 
-    <!-- Add button -->
-    <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{$t("addTag")}}</small>
+        <!-- Add button -->
+        <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{ $t("addTag") }}</small>
 
-    <!-- Size tag-->
-    <small v-else class="text-muted badge-size" style="padding-left: 2px">{{
-        humanFileSize(hit._source.size)
-      }}</small>
-  </div>
+        <!-- Size tag-->
+        <small v-else class="text-muted badge-size" style="padding-left: 2px">{{
+            humanFileSize(hit._source.size)
+            }}</small>
+    </div>
 </template>
 
 <script>
@@ -81,170 +83,136 @@ import Sist2Api from "@/Sist2Api";
 import VueSimpleSuggest from 'vue-simple-suggest'
 
 export default Vue.extend({
-  components: {
-    "TwitterColorPicker": Twitter,
-    VueSimpleSuggest
-  },
-  props: ["hit"],
-  data() {
-    return {
-      showAddButton: false,
-      showModal: false,
-      tagText: null,
-      color: {
-        hex: "#e0e0e0",
-      },
-    }
-  },
-  computed: {
-    tagHover() {
-      return this.$store.getters["uiTagHover"];
-    }
-  },
-  methods: {
-    humanFileSize: humanFileSize,
-    getSuggestionWithoutQueryPrefix(suggestion, query) {
-      return suggestion.id.slice(query.length, -8)
+    components: {
+        "TwitterColorPicker": Twitter,
+        VueSimpleSuggest
     },
-    getBg(suggestion) {
-      return suggestion.id.slice(-7);
-    },
-    getFg(suggestion) {
-      return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
-    },
-    setTagText(value) {
-      this.$refs.suggest.clearSuggestions();
-
-      if (typeof value === "string") {
-        this.tagText = {
-          id: value,
-          title: value
-        };
-        return;
-      }
-
-      this.color = {
-        hex: "#" + value.id.split("#")[1]
-      }
-
-      this.tagText = value;
-    },
-    badgeClass(tag) {
-      return `badge-${tag.style}`;
-    },
-    badgeStyle(tag) {
-      return {
-        background: tag.bg,
-        color: tag.fg,
-      };
-    },
-    onTagHover(tag) {
-      if (tag.userTag) {
-        this.$store.commit("setUiTagHover", tag);
-      }
-    },
-    onTagLeave() {
-      this.$store.commit("setUiTagHover", null);
-    },
-    onTagDeleteClick(tag, e) {
-      this.hit._tags = this.hit._tags.filter(t => t !== tag);
-
-      Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
-        //toast
-        this.$store.commit("busUpdateWallItems");
-        this.$store.commit("busUpdateTags");
-      });
-    },
-    tagAdd() {
-      this.showModal = true;
-    },
-    saveTag() {
-      if (this.tagText.id.includes("#")) {
-        this.$bvToast.toast(
-            this.$t("toast.invalidTag"),
-            {
-              title: this.$t("toast.invalidTagTitle"),
-              noAutoHide: true,
-              toaster: "b-toaster-bottom-right",
-              headerClass: "toast-header-error",
-              bodyClass: "toast-body-error",
-            });
-        return;
-      }
-
-      let tag = this.tagText.id + this.color.hex.replace("#", ".#");
-      const userTags = this.hit._tags.filter(t => t.userTag);
-
-      if (userTags.find(t => t.rawText === tag) != null) {
-        this.$bvToast.toast(
-            this.$t("toast.dupeTag"),
-            {
-              title: this.$t("toast.dupeTagTitle"),
-              noAutoHide: true,
-              toaster: "b-toaster-bottom-right",
-              headerClass: "toast-header-error",
-              bodyClass: "toast-body-error",
-            });
-        return;
-      }
-
-      this.hit._tags.push(Sist2Api.createUserTag(tag));
-
-      Sist2Api.saveTag(tag, this.hit).then(() => {
-        this.tagText = null;
-        this.showModal = false;
-        this.$store.commit("busUpdateWallItems");
-        this.$store.commit("busUpdateTags");
-        // TODO: toast
-      });
-    },
-    async suggestTag(term) {
-      term = term.toLowerCase();
-
-      const choices = await this.getTagChoices(term);
-
-      let matches = [];
-      for (let i = 0; i < choices.length; i++) {
-        if (~choices[i].toLowerCase().indexOf(term)) {
-          matches.push(choices[i]);
-        }
-      }
-
-      return matches.sort().map(match => {
+    props: ["hit"],
+    data() {
         return {
-          title: match.split(".").slice(0, -1).join("."),
-          id: match
+            showAddButton: false,
+            showModal: false,
+            tagText: null,
+            color: {
+                hex: "#e0e0e0",
+            },
         }
-      });
     },
-    getTagChoices(prefix) {
-      return new Promise(getPaths => {
-        Sist2Api.esQuery({
-          suggest: {
-            tag: {
-              prefix: prefix,
-              completion: {
-                field: "suggest-tag",
-                skip_duplicates: true,
-                size: 10000
-              }
+    methods: {
+        humanFileSize: humanFileSize,
+        getSuggestionWithoutQueryPrefix(suggestion, query) {
+            return suggestion.id.slice(query.length, -8)
+        },
+        getBg(suggestion) {
+            return suggestion.id.slice(-7);
+        },
+        getFg(suggestion) {
+            return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
+        },
+        setTagText(value) {
+            this.$refs.suggest.clearSuggestions();
+
+            if (typeof value === "string") {
+                this.tagText = {
+                    id: value,
+                    title: value
+                };
+                return;
             }
-          }
-        }).then(resp => {
-          const result = [];
-          resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
-            tags.forEach(tag => {
-              const t = tag.slice(0, -8);
-              if (!result.find(x => x.slice(0, -8) === t)) {
-                result.push(tag);
-              }
+
+            this.color = {
+                hex: "#" + value.id.split("#")[1]
+            }
+
+            this.tagText = value;
+        },
+        badgeClass(tag) {
+            return `badge-${tag.style}`;
+        },
+        badgeStyle(tag) {
+            return {
+                background: tag.bg,
+                color: tag.fg,
+            };
+        },
+        onTagDeleteClick(tag, e) {
+            this.hit._tags = this.hit._tags.filter(t => t !== tag);
+
+            Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
+                //toast
+                this.$store.commit("busUpdateWallItems");
+                this.$store.commit("busUpdateTags");
             });
-          });
-          getPaths(result);
-        });
-      });
-    }
-  },
+        },
+        tagAdd() {
+            this.showModal = true;
+        },
+        saveTag() {
+            if (this.tagText.id.includes("#")) {
+                this.$bvToast.toast(
+                    this.$t("toast.invalidTag"),
+                    {
+                        title: this.$t("toast.invalidTagTitle"),
+                        noAutoHide: true,
+                        toaster: "b-toaster-bottom-right",
+                        headerClass: "toast-header-error",
+                        bodyClass: "toast-body-error",
+                    });
+                return;
+            }
+
+            let tag = this.tagText.id + this.color.hex.replace("#", ".#");
+            const userTags = this.hit._tags.filter(t => t.userTag);
+
+            if (userTags.find(t => t.rawText === tag) != null) {
+                this.$bvToast.toast(
+                    this.$t("toast.dupeTag"),
+                    {
+                        title: this.$t("toast.dupeTagTitle"),
+                        noAutoHide: true,
+                        toaster: "b-toaster-bottom-right",
+                        headerClass: "toast-header-error",
+                        bodyClass: "toast-body-error",
+                    });
+                return;
+            }
+
+            this.hit._tags.push(Sist2Api.createUserTag(tag));
+
+            Sist2Api.saveTag(tag, this.hit).then(() => {
+                this.tagText = null;
+                this.showModal = false;
+                this.$store.commit("busUpdateWallItems");
+                this.$store.commit("busUpdateTags");
+                // TODO: toast
+            });
+        },
+        async suggestTag(term) {
+            term = term.toLowerCase();
+
+            const choices = await this.getTagChoices(term);
+
+            let matches = [];
+            for (let i = 0; i < choices.length; i++) {
+                if (~choices[i].toLowerCase().indexOf(term)) {
+                    matches.push(choices[i]);
+                }
+            }
+
+            return matches.sort().map(match => {
+                return {
+                    title: match.split(".").slice(0, -1).join("."),
+                    id: match
+                }
+            });
+        },
+        getTagChoices(prefix) {
+            return new Promise(getPaths => {
+                Sist2Api.getTagSuggestions(prefix)
+                    .then(paths => getPaths(paths))
+            });
+        }
+    },
 });
 </script>
 
@@ -252,87 +220,87 @@ export default Vue.extend({
 
 
 .badge-video {
-  color: #FFFFFF;
-  background-color: #F27761;
+    color: #FFFFFF;
+    background-color: #F27761;
 }
 
 .badge-image {
-  color: #FFFFFF;
-  background-color: #AA99C9;
+    color: #FFFFFF;
+    background-color: #AA99C9;
 }
 
 .badge-audio {
-  color: #FFFFFF;
-  background-color: #00ADEF;
+    color: #FFFFFF;
+    background-color: #00ADEF;
 }
 
 .badge-user {
-  color: #212529;
-  background-color: #e0e0e0;
+    color: #212529;
+    background-color: #e0e0e0;
 }
 
 .badge-user:hover, .add-tag-button:hover {
-  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
+    box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
 }
 
 .badge-text {
-  color: #FFFFFF;
-  background-color: #FAAB3C;
+    color: #FFFFFF;
+    background-color: #FAAB3C;
 }
 
 .badge {
-  margin-right: 3px;
+    margin-right: 3px;
 }
 
 .badge-delete {
-  margin-right: -2px;
-  margin-left: 2px;
-  margin-top: -1px;
-  font-family: monospace;
-  font-size: 90%;
-  background: rgba(0, 0, 0, 0.2);
-  padding: 0.1em 0.4em;
-  color: white;
-  cursor: pointer;
+    margin-right: -2px;
+    margin-left: 2px;
+    margin-top: -1px;
+    font-family: monospace;
+    font-size: 90%;
+    background: rgba(0, 0, 0, 0.2);
+    padding: 0.1em 0.4em;
+    color: white;
+    cursor: pointer;
 }
 
 .badge-size {
-  width: 50px;
-  display: inline-block;
+    width: 50px;
+    display: inline-block;
 }
 
 .add-tag-button {
-  cursor: pointer;
-  color: #212529;
-  background-color: #e0e0e0;
-  width: 50px;
+    cursor: pointer;
+    color: #212529;
+    background-color: #e0e0e0;
+    width: 50px;
 }
 
 .badge {
-  user-select: none;
+    user-select: none;
 }
 
 .badge-suggestion {
-  font-size: 90%;
-  font-weight: normal;
+    font-size: 90%;
+    font-weight: normal;
 }
 </style>
 
 <style>
 .vc-twitter-body {
-  padding: 0 !important;
+    padding: 0 !important;
 }
 
 .vc-twitter {
-  box-shadow: none !important;
-  background: none !important;
+    box-shadow: none !important;
+    background: none !important;
 }
 
 .tooltip {
-  user-select: none;
+    user-select: none;
 }
 
 .toast {
-  border: none;
+    border: none;
 }
 </style>

sist2-vue/src/components/TagPicker.vue

@@ -1,13 +1,13 @@
 <template>
-  <div>
-    <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
-      <b-form-input :value="filter"
-                    :placeholder="$t('tagFilter')"
-                    @input="onFilter($event)"></b-form-input>
-    </b-input-group>
+    <div>
+        <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
+            <b-form-input :value="filter"
+                          :placeholder="$t('tagFilter')"
+                          @input="onFilter($event)"></b-form-input>
+        </b-input-group>
 
-    <div id="tagTree"></div>
-  </div>
+        <div id="tagTree"></div>
+    </div>
 </template>
 
 <script>
@@ -19,191 +19,199 @@ import {getSelectedTreeNodes} from "@/util";
 import Sist2Api from "@/Sist2Api";
 
 function resetState(node) {
-  node._tree.defaultState.forEach(function (val, prop) {
-    node.state(prop, val);
-  });
+    node._tree.defaultState.forEach(function (val, prop) {
+        node.state(prop, val);
+    });
 
-  return node;
+    return node;
 }
 
 function baseStateChange(prop, value, verb, node, deep) {
-  if (node.state(prop) !== value) {
-    node._tree.batch();
+    if (node.state(prop) !== value) {
+        node._tree.batch();
 
-    if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
-      resetState(node);
+        if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
+            resetState(node);
+        }
+
+        node.state(prop, value);
+
+        node._tree.emit('node.' + verb, node, false);
+
+        if (deep && node.hasChildren()) {
+            node.children.recurseDown(function (child) {
+                baseStateChange(prop, value, verb, child);
+            });
+        }
+
+        node.markDirty();
+        node._tree.end();
     }
 
-    node.state(prop, value);
-
-    node._tree.emit('node.' + verb, node, false);
-
-    if (deep && node.hasChildren()) {
-      node.children.recurseDown(function (child) {
-        baseStateChange(prop, value, verb, child);
-      });
-    }
-
-    node.markDirty();
-    node._tree.end();
-  }
-
-  return node;
+    return node;
 }
 
 function addTag(map, tag, id, count) {
-  const tags = tag.split(".");
+    const tags = tag.split(".");
 
-  const child = {
-    id: id,
-    count: count,
-    text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
-    name: tags[0],
-    children: [],
-    // Overwrite base functions
-    blur: function () {
-      // noop
-    },
-    select: function () {
-      this.state("selected", true);
-      return this.check()
-    },
-    deselect: function () {
-      this.state("selected", false);
-      return this.uncheck()
-    },
-    uncheck: function () {
-      baseStateChange('checked', false, 'unchecked', this, false);
-      this.state('indeterminate', false);
+    const child = {
+        id: id,
+        count: count,
+        text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
+        name: tags[0],
+        children: [],
+        // Overwrite base functions
+        blur: function () {
+            // noop
+        },
+        select: function () {
+            this.state("selected", true);
+            return this.check()
+        },
+        deselect: function () {
+            this.state("selected", false);
+            return this.uncheck()
+        },
+        uncheck: function () {
+            baseStateChange('checked', false, 'unchecked', this, false);
+            this.state('indeterminate', false);
 
-      if (this.hasParent()) {
-        this.getParent().refreshIndeterminateState();
-      }
+            if (this.hasParent()) {
+                this.getParent().refreshIndeterminateState();
+            }
 
-      this._tree.end();
-      return this;
-    },
-    check: function () {
-      baseStateChange('checked', true, 'checked', this, false);
+            this._tree.end();
+            return this;
+        },
+        check: function () {
+            baseStateChange('checked', true, 'checked', this, false);
 
-      if (this.hasParent()) {
-        this.getParent().refreshIndeterminateState();
-      }
+            if (this.hasParent()) {
+                this.getParent().refreshIndeterminateState();
+            }
 
-      this._tree.end();
-      return this;
+            this._tree.end();
+            return this;
+        }
+    };
+
+    let found = false;
+    map.forEach(node => {
+        if (node.name === child.name) {
+            found = true;
+            if (tags.length !== 1) {
+                addTag(node.children, tags.slice(1).join("."), id, count);
+            } else {
+                // Same name, different color
+                console.error("FIXME: Duplicate tag?")
+                console.trace(node)
+            }
+        }
+    });
+    if (!found) {
+        if (tags.length !== 1) {
+            addTag(child.children, tags.slice(1).join("."), id, count);
+            map.push(child);
+        } else {
+            map.push(child);
+        }
     }
-  };
-
-  let found = false;
-  map.forEach(node => {
-    if (node.name === child.name) {
-      found = true;
-      if (tags.length !== 1) {
-        addTag(node.children, tags.slice(1).join("."), id, count);
-      } else {
-        // Same name, different color
-        console.error("FIXME: Duplicate tag?")
-        console.trace(node)
-      }
-    }
-  });
-  if (!found) {
-    if (tags.length !== 1) {
-      addTag(child.children, tags.slice(1).join("."), id, count);
-      map.push(child);
-    } else {
-      map.push(child);
-    }
-  }
 }
 
 export default {
-  name: "TagPicker",
-  props: ["showSearchBar"],
-  data() {
-    return {
-      tagTree: null,
-      loadedFromArgs: false,
-      filter: ""
-    }
-  },
-  mounted() {
-    this.$store.subscribe((mutation) => {
-      if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
-        this.initializeTree();
-        this.updateTree();
-      } else if (mutation.type === "busUpdateTags") {
-        window.setTimeout(this.updateTree, 2000);
-      }
-    });
-  },
-  methods: {
-    onFilter(value) {
-      this.filter = value;
-      this.tagTree.search(value);
-    },
-    initializeTree() {
-      const tagMap = [];
-      this.tagTree = new InspireTree({
-        selection: {
-          mode: "checkbox",
-          autoDeselect: false,
-        },
-        checkbox: {
-          autoCheckChildren: false,
-        },
-        data: tagMap
-      });
-      new InspireTreeDOM(this.tagTree, {
-        target: '#tagTree'
-      });
-      this.tagTree.on("node.state.changed", this.handleTreeClick);
-    },
-    updateTree() {
-      // TODO: remember which tags are selected and restore?
-      const tagMap = [];
-      Sist2Api.getTags().then(tags => {
-        tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
-        this.tagTree.removeAll();
-        this.tagTree.addNodes(tagMap);
-
-        if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
-          this.$store.state._onLoadSelectedTags.forEach(mime => {
-            this.tagTree.node(mime).select();
-            this.loadedFromArgs = true;
-          });
+    name: "TagPicker",
+    props: ["showSearchBar"],
+    data() {
+        return {
+            tagTree: null,
+            loadedFromArgs: false,
+            filter: ""
         }
-      });
     },
-    handleTreeClick(node, e) {
-      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
-          || e === "matched" || e === "hidden") {
-        return;
-      }
+    mounted() {
+        this.$store.subscribe((mutation) => {
+            if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
+                this.initializeTree();
+                this.updateTree();
+            } else if (mutation.type === "busUpdateTags") {
+                if (this.$store.state.uiSqliteMode) {
+                    this.updateTree();
+                } else {
+                    window.setTimeout(this.updateTree, 2000);
+                }
+            }
+        });
+    },
+    methods: {
+        onFilter(value) {
+            this.filter = value;
+            this.tagTree.search(value);
+        },
+        initializeTree() {
+            const tagMap = [];
+            this.tagTree = new InspireTree({
+                selection: {
+                    mode: "checkbox",
+                    autoDeselect: false,
+                },
+                checkbox: {
+                    autoCheckChildren: false,
+                },
+                data: tagMap
+            });
+            new InspireTreeDOM(this.tagTree, {
+                target: '#tagTree'
+            });
+            this.tagTree.on("node.state.changed", this.handleTreeClick);
+        },
+        updateTree() {
+            // TODO: remember which tags are selected and restore?
+            const tagMap = [];
+            Sist2Api.getTags().then(tags => {
+                tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
+                this.tagTree.removeAll();
+                this.tagTree.addNodes(tagMap);
 
-      this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
-    },
-  }
+                if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
+                    this.$store.state._onLoadSelectedTags.forEach(mime => {
+                        this.tagTree.node(mime).select();
+                        this.loadedFromArgs = true;
+                    });
+                }
+            });
+        },
+        handleTreeClick(node, e) {
+            if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
+                || e === "matched" || e === "hidden") {
+                return;
+            }
+
+            this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
+        },
+    }
 }
 </script>
 
 <style scoped>
 #mimeTree {
-  max-height: 350px;
-  overflow: auto;
+    max-height: 350px;
+    overflow: auto;
 }
 </style>
 <style>
 .inspire-tree .focused > .wholerow {
-  border: none;
+    border: none;
 }
 
 #tag-picker-filter-bar {
-  padding: 10px 4px 4px;
+    padding: 10px 4px 4px;
 }
 
 .theme-black .inspire-tree .matched > .wholerow {
-  background: rgba(251, 191, 41, 0.25);
+    background: rgba(251, 191, 41, 0.25);
+}
+#tagTree {
+    max-height: 350px;
+    overflow: auto;
 }
 </style>

sist2-vue/src/components/icons/MlIcon.vue

@@ -0,0 +1,76 @@
+<template>
+    <svg class="ml-icon" :class="{'m-icon': 1, 'ml-icon-big': big, 'ml-icon-clickable': clickable}" xmlns="http://www.w3.org/2000/svg"
+         viewBox="0 0 512 512" xml:space="preserve" fill="currentColor" stroke="currentColor" @click="$emit('click')">
+                <g>
+	<path class="st0" d="M167.314,14.993C167.314,6.712,160.602,0,152.332,0h-5.514c-8.27,0-14.982,6.712-14.982,14.993v41.466h35.478
+		V14.993z"/>
+                    <path class="st0"
+                          d="M238.26,14.993C238.26,6.712,231.549,0,223.278,0h-5.504c-8.271,0-14.982,6.712-14.982,14.993v41.466h35.468 V14.993z"/>
+                    <path class="st0"
+                          d="M309.207,14.993C309.207,6.712,302.496,0,294.225,0h-5.504c-8.271,0-14.982,6.712-14.982,14.993v41.466h35.468 V14.993z"/>
+                    <path class="st0"
+                          d="M380.164,14.993C380.164,6.712,373.453,0,365.182,0h-5.514c-8.27,0-14.982,6.712-14.982,14.993v41.466h35.478 V14.993z"/>
+                    <path class="st0"
+                          d="M131.836,497.007c0,8.282,6.712,14.993,14.982,14.993h5.514c8.27,0,14.982-6.711,14.982-14.993V455.55h-35.478 V497.007z"/>
+                    <path class="st0"
+                          d="M202.792,497.007c0,8.282,6.712,14.993,14.982,14.993h5.504c8.27,0,14.982-6.711,14.982-14.993V455.55h-35.468 V497.007z"/>
+                    <path class="st0"
+                          d="M273.739,497.007c0,8.282,6.712,14.993,14.982,14.993h5.504c8.271,0,14.982-6.711,14.982-14.993V455.55 h-35.468V497.007z"/>
+                    <path class="st0"
+                          d="M344.686,497.007c0,8.282,6.712,14.993,14.982,14.993h5.514c8.271,0,14.982-6.711,14.982-14.993V455.55 h-35.478V497.007z"/>
+                    <path class="st0"
+                          d="M497.018,131.836H455.55v35.479h41.468c8.27,0,14.982-6.712,14.982-14.993v-5.493 C512,138.548,505.288,131.836,497.018,131.836z"/>
+                    <path class="st0"
+                          d="M497.018,202.793H455.55v35.468h41.468c8.27,0,14.982-6.712,14.982-14.982v-5.494 C512,209.504,505.288,202.793,497.018,202.793z"/>
+                    <path class="st0"
+                          d="M497.018,273.739H455.55v35.468h41.468c8.27,0,14.982-6.711,14.982-14.992v-5.494 C512,280.451,505.288,273.739,497.018,273.739z"/>
+                    <path class="st0"
+                          d="M497.018,344.686H455.55v35.479h41.468c8.27,0,14.982-6.712,14.982-14.993v-5.493 C512,351.398,505.288,344.686,497.018,344.686z"/>
+                    <path class="st0"
+                          d="M0,146.828v5.493c0,8.281,6.711,14.993,14.982,14.993H56.46v-35.479H14.982C6.711,131.836,0,138.548,0,146.828 z"/>
+                    <path class="st0"
+                          d="M0,217.785v5.494c0,8.27,6.711,14.982,14.982,14.982H56.46v-35.468H14.982C6.711,202.793,0,209.504,0,217.785z "/>
+                    <path class="st0"
+                          d="M0,288.721v5.494c0,8.281,6.711,14.992,14.982,14.992H56.46v-35.468H14.982C6.711,273.739,0,280.451,0,288.721 z"/>
+                    <path class="st0"
+                          d="M0,359.679v5.493c0,8.281,6.711,14.993,14.982,14.993H56.46v-35.479H14.982C6.711,344.686,0,351.398,0,359.679 z"/>
+                    <path class="st0"
+                          d="M78.628,433.382h354.753V78.628H78.628V433.382z M376.56,120.2c9.18,0,16.635,7.445,16.635,16.634 c0,9.18-7.455,16.624-16.635,16.624c-9.179,0-16.624-7.445-16.624-16.624C359.936,127.644,367.381,120.2,376.56,120.2z M376.56,361.32c9.18,0,16.635,7.445,16.635,16.635c0,9.179-7.455,16.623-16.635,16.623c-9.179,0-16.624-7.444-16.624-16.623 C359.936,368.764,367.381,361.32,376.56,361.32z M184.362,184.362h143.287v143.287H184.362V184.362z M135.439,120.2 c9.19,0,16.635,7.445,16.635,16.634c0,9.169-7.445,16.624-16.635,16.624c-9.178,0-16.623-7.455-16.623-16.624 C118.816,127.644,126.26,120.2,135.439,120.2z M135.439,361.32c9.19,0,16.635,7.445,16.635,16.635 c0,9.169-7.445,16.623-16.635,16.623c-9.178,0-16.623-7.454-16.623-16.623C118.816,368.764,126.26,361.32,135.439,361.32z"/>
+</g>
+</svg>
+</template>
+
+<script>
+export default {
+    name: "MLIcon",
+    props: {
+        "big": Boolean,
+        "clickable": Boolean
+    }
+}
+</script>
+
+<style scoped>
+.ml-icon-clickable {
+    cursor: pointer;
+}
+
+.ml-icon-big {
+    width: 24px !important;
+    height: 24px !important;
+}
+
+.ml-icon {
+    width: 1rem;
+    min-width: 1rem;
+    margin-right: 0.2rem;
+    line-height: 1rem;
+    height: 1rem;
+    min-height: 1rem;
+    filter: brightness(45%);
+}
+
+.theme-black .ml-icon {
+    filter: brightness(80%);
+}
+</style>

sist2-vue/src/i18n/messages.ts

@@ -1,7 +1,7 @@
 export default {
     en: {
         filePage: {
-          notFound: "Not found"
+            notFound: "Not found"
         },
         searchBar: {
             simple: "Search",
@@ -18,6 +18,7 @@ export default {
         tags: "Tags",
         tagFilter: "Filter tags",
         forExample: "For example:",
+        embeddingsSearchPlaceholder: "Embeddings search",
         help: {
             simpleSearch: "Simple search",
             advancedSearch: "Advanced search",
@@ -57,7 +58,7 @@ export default {
             fuzzy: "Set fuzzy search by default",
             searchInPath: "Enable matching query against document path",
             suggestPath: "Enable auto-complete in path filter bar",
-            fragmentSize: "Highlight context size in characters",
+            fragmentSize: "Highlight context size",
             queryMode: "Search mode",
             displayMode: "Display",
             columns: "Column count",
@@ -92,6 +93,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -184,7 +186,7 @@ export default {
     },
     de: {
         filePage: {
-          notFound: "Nicht gefunden"
+            notFound: "Nicht gefunden"
         },
         searchBar: {
             simple: "Suche",
@@ -239,7 +241,7 @@ export default {
             fuzzy: "Aktiviere Fuzzy-Suche standardmäßig",
             searchInPath: "Abgleich der Abfrage mit dem Dokumentpfad aktivieren",
             suggestPath: "Aktiviere Auto-Vervollständigung in Pfadfilter-Leiste",
-            fragmentSize: "Kontextgröße in Zeichen hervorheben",
+            fragmentSize: "Kontextgröße",
             queryMode: "Such-Modus",
             displayMode: "Ansicht",
             columns: "Anzahl Spalten",
@@ -271,6 +273,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -413,7 +416,7 @@ export default {
             fuzzy: "Activer la recherche approximative par défaut",
             searchInPath: "Activer la recherche dans le chemin des documents",
             suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
-            fragmentSize: "Longueur du contexte de surlignage, en nombre de caractères",
+            fragmentSize: "Longueur du contexte de surlignage",
             queryMode: "Mode de recherche",
             displayMode: "Affichage",
             columns: "Nombre de colonnes",
@@ -445,6 +448,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -619,6 +623,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -703,4 +708,188 @@ export default {
             selectedIndices: "选中索引",
         },
     },
+    pl: {
+        filePage: {
+            notFound: "Nie znaleziono"
+        },
+        searchBar: {
+            simple: "Szukaj",
+            advanced: "Zaawansowane szukanie",
+            fuzzy: "Również podobne"
+        },
+        addTag: "Tag",
+        deleteTag: "Usuń",
+        download: "Pobierz",
+        and: "i",
+        page: "strona",
+        pages: "stron",
+        mimeTypes: "Typy danych",
+        tags: "Tagi",
+        tagFilter: "Filtruj tagi",
+        forExample: "Na przykład:",
+        help: {
+            simpleSearch: "Proste szukanie",
+            advancedSearch: "Zaawansowane szukanie",
+            help: "Pomoc",
+            term: "<WYRAZ>",
+            and: "operator I",
+            or: "operator LUB",
+            not: "zabrania danego wyrazu",
+            quotes: "znajdzie objętą sekwencję wyrazów w podanej kolejności",
+            prefix: "znajdzie dowolny wyraz rozpoczynający się na takie litery, jeśli zastosowane na końcu wyrazu",
+            parens: "używane do grupowania wyrażeń",
+            tildeTerm: "znajdzie wyraz w podanej odległości",
+            tildePhrase: "znajdzie frazę przeplecioną podaną liczbą niepasujących wyrazów",
+            example1:
+                "Na przykład: <code>\"pieczone jajko\" +(kiełbasa | ziemniak) -frytki</code> znajdzie frazę " +
+                "<i>pieczone jajko</i> gdzie występuje też: <i>kiełbasa</i> albo <i>ziemniak</i>, ale zignoruje rezultat " +
+                "zawierający <i>frytki</i>.",
+            defaultOperator:
+                "Kiedy nie podano ani <code>+</code>, ani <code>|</code>, to domyślnym operatorem jest " +
+                "<code>+</code> (i).",
+            fuzzy:
+                "Kiedy opcja <b>Również podobne</b> jest zaznaczona, częściowo zgodne wyrazy są również znajdywane.",
+            moreInfoSimple: "Po więcej informacji sięgnij do <a target=\"_blank\" " +
+                "rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">dokumentacji Elasticsearch</a>",
+            moreInfoAdvanced: "Aby uzyskać więcej informacji o zaawansowanym szukaniu, przeczytaj <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">dokumentację Elasticsearch</a>"
+        },
+        config: "Ustawienia",
+        configDescription: "Ustawienia są zapisywane na żywo w tej przeglądarce.",
+        configReset: "Zresetuj ustawienia",
+        searchOptions: "Opcje szukania",
+        treemapOptions: "Opcje mapy",
+        mlOptions: "Opcje uczenia maszynowego",
+        displayOptions: "Opcje wyświetlania",
+        opt: {
+            lang: "Język",
+            highlight: "Zaznaczaj znalezione fragmenty",
+            fuzzy: "Ustaw szukanie również podobnych jako domyślne",
+            searchInPath: "Włącz szukanie również w ścieżce dokumentu",
+            suggestPath: "Włącz auto-uzupełnianie w filtrze ścieżek",
+            fragmentSize: "Podświetl wielkość kontekstu w znakach",
+            queryMode: "Tryb szukania",
+            displayMode: "Wyświetlanie",
+            columns: "Liczba kolumn",
+            treemapType: "Typ mapy",
+            treemapTiling: "Układanie mapy",
+            treemapColorGroupingDepth: "Jak głęboko grupować kolory mapy (na płasko)",
+            treemapColor: "Kolor mapy (kaskadowo)",
+            treemapSize: "Wielkość mapy",
+            theme: "Styl graficzny",
+            lightboxLoadOnlyCurrent: "Nie pobieraj od razu obrazów w pełnej wielkości dla sąsiednich obrazów podglądu.",
+            slideDuration: "Czas trwania jednego slajdu w pokazie slajdów",
+            resultSize: "Liczba wyników na stronę",
+            tagOrOperator: "Użyj operatora LUB przy wyborze kilku tagów",
+            hideDuplicates: "Ukryj zduplikowane wyniki (według sumy kontrolnej)",
+            hideLegacy: "Ukryj powiadomienie Elasticsearch 'legacyES'",
+            updateMimeMap: "Uaktualniaj drzewo typów mediów na żywo",
+            useDatePicker: "Używaj kalendarza do wyboru dat, zamiast suwaka",
+            vidPreviewInterval: "Czas trwania jednej klatki w podglądzie wideo (w ms)",
+            simpleLightbox: "Wyłącz animacje w podglądzie obrazów",
+            showTagPickerFilter: "Pokazuj pole filtrowania tagów",
+            featuredFields: "Wybrane pola szablonu Javascript. Będą pojawiać się przy wynikach wyszukiwania.",
+            featuredFieldsList: "Dostępne zmienne",
+            autoAnalyze: "Automatycznie analizuj tekst",
+            defaultModel: "Domyślny model",
+            mlRepositories: "Repozytoria modeli (każde w osobnej linii)"
+        },
+        queryMode: {
+            simple: "Proste",
+            advanced: "Zaawansowane",
+        },
+        lang: {
+            en: "English",
+            de: "Deutsch",
+            fr: "Français",
+            pl: "Polski",
+            "zh-CN": "简体中文",
+        },
+        displayMode: {
+            grid: "Siatka",
+            list: "Lista",
+        },
+        columns: {
+            auto: "Automatyczna"
+        },
+        treemapType: {
+            cascaded: "Kaskadowa",
+            flat: "Płaska (kompaktowa)"
+        },
+        treemapSize: {
+            small: "Mała",
+            medium: "Średnia",
+            large: "Duża",
+            xLarge: "Bardzo duża",
+            xxLarge: "Ogromna",
+            custom: "Inna",
+        },
+        treemapTiling: {
+            binary: "Binarnie",
+            squarify: "Kwadratowo",
+            slice: "Wycinek",
+            dice: "Kostka",
+            sliceDice: "Wycinek i kostka",
+        },
+        theme: {
+            light: "Jasny",
+            black: "Czarny"
+        },
+        hit: "traf",
+        hits: "trafień",
+        details: "Szczegóły",
+        stats: "Statystyki",
+        queryTime: "Czas szukania",
+        totalSize: "Całkowita wielkość",
+        pathBar: {
+            placeholder: "Filtruj ścieżki",
+            modalTitle: "Wybierz ścieżkę"
+        },
+        debug: "Informacje dla programistów",
+        debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
+            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
+        tagline: "Slogan",
+        toast: {
+            esConnErrTitle: "Problem z połączeniem z Elasticsearch",
+            esConnErr: "Moduł strony internetowej sist2 napotkał problem przy połączeniu z Elasticsearch." +
+                " Zobacz logi serwera, aby uzyskać więcej informacji.",
+            esQueryErrTitle: "Problem z kwerendą",
+            esQueryErr: "Kwerenda szukania jest niezrozumiała albo nie udało się jej przesłać. Sprawdź dokumentację zaawansowanego szukania. " +
+                "Zobacz logi serwera, aby uzyskać więcej informacji.",
+            dupeTagTitle: "Zduplikowany tag",
+            dupeTag: "Ten dokument już ma taki tag.",
+            copiedToClipboard: "Skopiowano do schowka"
+        },
+        saveTagModalTitle: "Dodaj tag",
+        saveTagPlaceholder: "Nazwa",
+        confirm: "Zatwierdź",
+        indexPickerPlaceholder: "Wybierz indeks",
+        sort: {
+            relevance: "Zgodność z szukanym",
+            dateAsc: "Data (najpierw starsze)",
+            dateDesc: "Data (najpierw nowsze)",
+            sizeAsc: "Wielkość (najpierw mniejsze)",
+            sizeDesc: "Wielkość (najpierw większe)",
+            nameAsc: "Nazwa (A-z)",
+            nameDesc: "Nazwa (Z-a)",
+            random: "Losowo",
+        },
+        d3: {
+            mimeCount: "Dystrybucja liczby plików według typów mediów",
+            mimeSize: "Dystrybucja wielkości plików według typów mediów",
+            dateHistogram: "Dystrybucja dat modyfikacji plików",
+            sizeHistogram: "Dystrybucja wielkości plików",
+        },
+        indexPicker: {
+            selectNone: "Zaznacz nic",
+            selectAll: "Zaznacz wszystko",
+            selectedIndex: "wybrany indeks",
+            selectedIndices: "wybrane indeksy",
+        },
+        ml: {
+            analyzeText: "Analizuj",
+            auto: "Automatycznie",
+            repoFetchError: "Nie udało się uzyskać listy modeli. Zobacz konsolę przeglądarki, aby uzyskać więcej informacji.",
+            repoFetchErrorTitle: "Nie udało się pobrać repozytoriów modeli",
+        }
+    }
 }

sist2-vue/src/main.js

@@ -1,4 +1,3 @@
-import '@babel/polyfill'
 import 'mutationobserver-shim'
 import Vue from 'vue'
 import './plugins/bootstrap-vue'

sist2-vue/src/ml/BPETokenizer.js

@@ -0,0 +1,118 @@
+const inf = Number.POSITIVE_INFINITY;
+const START_TOK = 49406;
+const END_TOK = 49407;
+
+function min(array, key) {
+    return array
+        .reduce((a, b) => (key(a, b) ? b : a))
+}
+
+class TupleSet extends Set {
+    add(elem) {
+        return super.add(elem.join("`"));
+    }
+
+    has(elem) {
+        return super.has(elem.join("`"));
+    }
+
+    toList() {
+        return [...this].map(x => x.split("`"))
+    }
+}
+
+export class BPETokenizer {
+
+    _encoder = null;
+    _bpeRanks = null;
+
+    constructor(encoder, bpeRanks) {
+        this._encoder = encoder;
+        this._bpeRanks = bpeRanks;
+    }
+
+    getPairs(word) {
+        const pairs = new TupleSet();
+
+        let prevChar = word[0];
+        for (let i = 1; i < word.length; i++) {
+            pairs.add([prevChar, word[i]])
+            prevChar = word[i];
+        }
+
+        return pairs.toList();
+    }
+
+    bpe(token) {
+        let word = [...token];
+        word[word.length - 1] += "</w>";
+        let pairs = this.getPairs(word)
+
+        if (pairs.length === 0) {
+            return token + "</w>"
+        }
+
+        while (true) {
+            const bigram = min(pairs, (a, b) => {
+                return (this._bpeRanks[a.join("`")] ?? inf) > (this._bpeRanks[b.join("`") ?? inf])
+            });
+
+            if (this._bpeRanks[bigram.join("`")] === undefined) {
+                break;
+            }
+
+            const [first, second] = bigram;
+            let newWord = [];
+            let i = 0;
+
+            while (i < word.length) {
+                const j = word.indexOf(first, i);
+                if (j === -1) {
+                    newWord.push(...word.slice(i));
+                    break;
+                } else {
+                    newWord.push(...word.slice(i, j));
+                    i = j;
+                }
+
+                if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
+                    newWord.push(first + second);
+                    i += 2;
+                } else {
+                    newWord.push(word[i]);
+                    i += 1;
+                }
+            }
+
+            word = [...newWord]
+            if (word.length === 1) {
+                break;
+            } else {
+                pairs = this.getPairs(word);
+            }
+        }
+
+        return word.join(" ");
+    }
+
+    encode(text) {
+        let bpeTokens = [];
+        text = text.trim();
+        text = text.replaceAll(/\s+/g, " ");
+
+        text
+            .match(/<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[a-zA-Z0-9]+/ig)
+            .forEach(token => {
+                bpeTokens.push(...this.bpe(token).split(" ").map(t => this._encoder[t]));
+            });
+
+        bpeTokens.unshift(START_TOK);
+        bpeTokens = bpeTokens.slice(0, 76);
+        bpeTokens.push(END_TOK);
+        while (bpeTokens.length < 77) {
+            bpeTokens.push(0);
+        }
+
+        return bpeTokens;
+    }
+}

sist2-vue/src/ml/BertNerModel.js

@@ -1,6 +1,8 @@
 import BertTokenizer from "@/ml/BertTokenizer";
-import * as tf from "@tensorflow/tfjs";
 import axios from "axios";
+import {chunk as _chunk} from "underscore";
+import * as ort from "onnxruntime-web";
+import {argMax, downloadToBuffer, ORT_WASM_PATHS} from "@/ml/mlUtils";
 
 export default class BertNerModel {
     vocabUrl;
@@ -29,7 +31,10 @@ export default class BertNerModel {
     }
 
     async loadModel(onProgress) {
-        this._model = await tf.loadGraphModel(this.modelUrl, {onProgress});
+        ort.env.wasm.wasmPaths = ORT_WASM_PATHS;
+        const buf = await downloadToBuffer(this.modelUrl, onProgress);
+
+        this._model = await ort.InferenceSession.create(buf.buffer, {executionProviders: ["wasm"]});
     }
 
     alignLabels(labels, wordIds, words) {
@@ -57,21 +62,28 @@ export default class BertNerModel {
 
     async predict(text, callback) {
         this._previousWordId = null;
-        const encoded = this._tokenizer.encodeText(text, this.inputSize)
+        const encoded = this._tokenizer.encodeText(text, this.inputSize);
 
+        let i = 0;
         for (let chunk of encoded.inputChunks) {
-            const rawResult = tf.tidy(() => this._model.execute({
-                input_ids: tf.tensor2d(chunk.inputIds, [1, this.inputSize], "int32"),
-                token_type_ids: tf.tensor2d(chunk.segmentIds, [1, this.inputSize], "int32"),
-                attention_mask: tf.tensor2d(chunk.inputMask, [1, this.inputSize], "int32"),
-            }));
 
-            const labelIds = await tf.argMax(rawResult, -1);
-            const labelIdsArray = await labelIds.array();
-            const labels = labelIdsArray[0].map(id => this.id2label[id]);
-            rawResult.dispose()
+            const results = await this._model.run({
+                input_ids: new ort.Tensor("int32", chunk.inputIds, [1, this.inputSize]),
+                token_type_ids: new ort.Tensor("int32", chunk.segmentIds, [1, this.inputSize]),
+                attention_mask: new ort.Tensor("int32", chunk.inputMask, [1, this.inputSize]),
+            });
 
-            callback(this.alignLabels(labels, chunk.wordIds, encoded.words))
+            const labelIds = _chunk(results["output"].data, this.id2label.length).map(argMax);
+            const labels = labelIds.map(id => this.id2label[id]);
+
+            callback(this.alignLabels(labels, chunk.wordIds, encoded.words));
+
+            i += 1;
+
+            // give browser some time to repaint
+            if (i % 2 === 0) {
+                await new Promise(resolve => setTimeout(resolve, 0));
+            }
         }
     }
 }

sist2-vue/src/ml/BertTokenizer.js

@@ -1,4 +1,5 @@
 import {zip, chunk} from "underscore";
+import {toInt64} from "@/ml/mlUtils";
 
 const UNK_INDEX = 100;
 const CLS_INDEX = 101;

sist2-vue/src/ml/CLIPTransformerModel.js

@@ -0,0 +1,57 @@
+import * as ort from "onnxruntime-web";
+import {BPETokenizer} from "@/ml/BPETokenizer";
+import axios from "axios";
+import {downloadToBuffer, ORT_WASM_PATHS} from "@/ml/mlUtils";
+import ModelStore from "@/ml/ModelStore";
+
+export class CLIPTransformerModel {
+
+    _modelUrl = null;
+    _tokenizerUrl = null;
+    _model = null;
+    _tokenizer = null;
+
+    constructor(modelUrl, tokenizerUrl) {
+        this._modelUrl = modelUrl;
+        this._tokenizerUrl = tokenizerUrl;
+    }
+
+    async init(onProgress) {
+        await Promise.all([this.loadTokenizer(), this.loadModel(onProgress)]);
+    }
+
+    async loadModel(onProgress) {
+        ort.env.wasm.wasmPaths = ORT_WASM_PATHS;
+        ort.env.wasm.numThreads = 2;
+
+        let buf = await ModelStore.get(this._modelUrl);
+        if (!buf) {
+            buf = await downloadToBuffer(this._modelUrl, onProgress);
+            await ModelStore.set(this._modelUrl, buf);
+        }
+
+        this._model = await ort.InferenceSession.create(buf.buffer, {
+            executionProviders: ["wasm"],
+        });
+    }
+
+    async loadTokenizer() {
+        const resp = await axios.get(this._tokenizerUrl);
+        this._tokenizer = new BPETokenizer(resp.data.encoder, resp.data.bpe_ranks)
+    }
+
+    async predict(text) {
+        const tokenized = this._tokenizer.encode(text);
+
+        const inputs = {
+            input_ids: new ort.Tensor("int32", tokenized, [1, 77])
+        };
+
+        const results = await this._model.run(inputs);
+
+        return Array.from(
+            Object.values(results)
+                .find(result => result.size === 512).data
+        );
+    }
+}

sist2-vue/src/ml/ModelStore.js

@@ -0,0 +1,67 @@
+class ModelStore {
+
+    _ok;
+    _db;
+    _resolve;
+    _loadingPromise;
+
+    constructor() {
+        const request = window.indexedDB.open("ModelStore", 1);
+
+        request.onerror = () => {
+            this._ok = false;
+        }
+
+        request.onupgradeneeded = event => {
+            const db = event.target.result;
+            db.createObjectStore("models");
+        }
+
+        request.onsuccess = () => {
+            this._ok = true;
+            this._db = request.result;
+
+            this._resolve();
+        }
+
+        this._loadingPromise = new Promise(resolve => this._resolve = resolve);
+    }
+
+    async get(key) {
+        await this._loadingPromise;
+
+        const req = this._db.transaction(["models"], "readwrite")
+            .objectStore("models")
+            .get(key);
+
+        return new Promise(resolve => {
+            req.onsuccess = event => {
+                resolve(event.target.result);
+            };
+            req.onerror = event => {
+                console.log("ERROR:");
+                console.log(event);
+                resolve(null);
+            };
+        });
+    }
+
+    async set(key, val) {
+        await this._loadingPromise;
+
+        const req = this._db.transaction(["models"], "readwrite")
+            .objectStore("models")
+            .put(val, key);
+
+        return new Promise(resolve => {
+            req.onsuccess = () => {
+                resolve(true);
+            };
+            req.onerror = () => {
+                resolve(false);
+            };
+        });
+    }
+}
+
+export default new ModelStore();

sist2-vue/src/ml/mlUtils.js

@@ -0,0 +1,46 @@
+export async function downloadToBuffer(url, onProgress) {
+    const resp = await fetch(url);
+
+    const contentLength = +resp.headers.get("Content-Length");
+    const buf = new Uint8ClampedArray(contentLength);
+    const reader = resp.body.getReader();
+    let cursor = 0;
+
+    if (onProgress) {
+        onProgress(0);
+    }
+
+    while (true) {
+        const {done, value} = await reader.read();
+
+        if (done) {
+            break;
+        }
+
+        buf.set(value, cursor);
+        cursor += value.length;
+
+        if (onProgress) {
+            onProgress(cursor / contentLength);
+        }
+    }
+
+    return buf;
+}
+
+export function argMax(array) {
+    return array
+        .map((x, i) => [x, i])
+        .reduce((r, a) => (a[0] > r[0] ? a : r))[1];
+}
+
+export function toInt64(array) {
+    return new BigInt64Array(array.map(BigInt));
+}
+
+export const ORT_WASM_PATHS = {
+    "ort-wasm-simd.wasm": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.15.1/dist/ort-wasm-simd.wasm",
+    "ort-wasm.wasm": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.15.1/dist/ort-wasm.wasm",
+    "ort-wasm-simd-threaded.wasm": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.15.1/dist/ort-wasm-simd-threaded.wasm",
+    "ort-wasm-threaded.wasm": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.15.1/dist/ort-wasm-threaded.wasm",
+}

sist2-vue/src/store/index.ts

@@ -1,7 +1,7 @@
 import Vue from "vue"
 import Vuex from "vuex"
 import VueRouter, {Route} from "vue-router";
-import {EsHit, EsResult, EsTag, Index, Tag} from "@/Sist2Api";
+import {EsHit, EsResult, EsTag, Index} from "@/Sist2Api";
 import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
 import {getInstance} from "@/plugins/auth0.js";
 
@@ -23,6 +23,9 @@ export default new Vuex.Store({
         dateMin: undefined,
         dateMax: undefined,
         searchText: "",
+        embeddingText: "",
+        embedding: null,
+        embeddingDoc: null,
         pathText: "",
         sortMode: "score",
 
@@ -69,11 +72,12 @@ export default new Vuex.Store({
         selectedTags: [] as string[],
 
         lastQueryResults: null,
+        firstQueryResults: null,
 
         keySequence: 0,
         querySequence: 0,
 
-        uiTagHover: null as Tag | null,
+        uiSqliteMode: false,
         uiLightboxIsOpen: false,
         uiShowLightbox: false,
         uiLightboxSources: [] as string[],
@@ -90,10 +94,11 @@ export default new Vuex.Store({
         uiMimeMap: [] as any[],
 
         auth0Token: null,
-        mlModel: {
+        nerModel: {
             model: null,
             name: null
         },
+        embeddingsModel: null
     },
     mutations: {
         setUiShowDetails: (state, val) => state.uiShowDetails = val,
@@ -128,15 +133,18 @@ export default new Vuex.Store({
         setDateBoundsMin: (state, val) => state.dateBoundsMin = val,
         setDateBoundsMax: (state, val) => state.dateBoundsMax = val,
         setSearchText: (state, val) => state.searchText = val,
+        setEmbeddingText: (state, val) => state.embeddingText = val,
+        setEmbedding: (state, val) => state.embedding = val,
+        setEmbeddingDoc: (state, val) => state.embeddingDoc = val,
         setFuzzy: (state, val) => state.fuzzy = val,
         setLastQueryResult: (state, val) => state.lastQueryResults = val,
+        setFirstQueryResult: (state, val) => state.firstQueryResults = val,
         _setOnLoadSelectedIndices: (state, val) => state._onLoadSelectedIndices = val,
         _setOnLoadSelectedMimeTypes: (state, val) => state._onLoadSelectedMimeTypes = val,
         _setOnLoadSelectedTags: (state, val) => state._onLoadSelectedTags = val,
         setSelectedIndices: (state, val) => state.selectedIndices = val,
         setSelectedMimeTypes: (state, val) => state.selectedMimeTypes = val,
         setSelectedTags: (state, val) => state.selectedTags = val,
-        setUiTagHover: (state, val: Tag | null) => state.uiTagHover = val,
         setUiLightboxIsOpen: (state, val: boolean) => state.uiLightboxIsOpen = val,
         _setUiShowLightbox: (state, val: boolean) => state.uiShowLightbox = val,
         setUiLightboxKey: (state, val: number) => state.uiLightboxKey = val,
@@ -154,6 +162,7 @@ export default new Vuex.Store({
         setUiLightboxThumbs: (state, val) => state.uiLightboxThumbs = val,
         setUiLightboxTypes: (state, val) => state.uiLightboxTypes = val,
         setUiLightboxCaptions: (state, val) => state.uiLightboxCaptions = val,
+        setUiSqliteMode: (state, val) => state.uiSqliteMode = val,
 
         setOptTheme: (state, val) => state.optTheme = val,
         setOptDisplay: (state, val) => state.optDisplay = val,
@@ -179,9 +188,15 @@ export default new Vuex.Store({
         setOptVidPreviewInterval: (state, val) => state.optVidPreviewInterval = val,
         setOptSimpleLightbox: (state, val) => state.optSimpleLightbox = val,
         setOptShowTagPickerFilter: (state, val) => state.optShowTagPickerFilter = val,
-        setOptAutoAnalyze: (state, val) => {state.optAutoAnalyze = val},
-        setOptMlRepositories: (state, val) => {state.optMlRepositories = val},
-        setOptMlDefaultModel: (state, val) => {state.optMlDefaultModel = val},
+        setOptAutoAnalyze: (state, val) => {
+            state.optAutoAnalyze = val
+        },
+        setOptMlRepositories: (state, val) => {
+            state.optMlRepositories = val
+        },
+        setOptMlDefaultModel: (state, val) => {
+            state.optMlDefaultModel = val
+        },
 
         setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
         setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val,
@@ -204,7 +219,8 @@ export default new Vuex.Store({
             // noop
         },
         setAuth0Token: (state, val) => state.auth0Token = val,
-        setMlModel: (state, val) => state.mlModel = val,
+        setNerModel: (state, val) => state.nerModel = val,
+        setEmbeddingsModel: (state, val) => state.embeddingsModel = val,
     },
     actions: {
         setSist2Info: (store, val) => {
@@ -340,6 +356,7 @@ export default new Vuex.Store({
             commit("_setUiShowLightbox", !state.uiShowLightbox);
         },
         clearResults({commit}) {
+            commit("setFirstQueryResult", null);
             commit("setLastQueryResult", null);
             commit("_setKeySequence", 0);
             commit("_setUiShowLightbox", false);
@@ -361,7 +378,9 @@ export default new Vuex.Store({
     },
     modules: {},
     getters: {
-        mlModel: (state) => state.mlModel,
+        nerModel: (state) => state.nerModel,
+        embeddingsModel: (state) => state.embeddingsModel,
+        embedding: (state) => state.embedding,
         seed: (state) => state.seed,
         getPathText: (state) => state.pathText,
         indices: state => state.indices,
@@ -380,6 +399,7 @@ export default new Vuex.Store({
         sizeMin: state => state.sizeMin,
         sizeMax: state => state.sizeMax,
         searchText: state => state.searchText,
+        embeddingText: state => state.embeddingText,
         pathText: state => state.pathText,
         fuzzy: state => state.fuzzy,
         size: state => state.optSize,
@@ -392,7 +412,6 @@ export default new Vuex.Store({
 
             return (state.lastQueryResults as unknown as EsResult).hits.hits.slice(-1)[0];
         },
-        uiTagHover: state => state.uiTagHover,
         uiShowLightbox: state => state.uiShowLightbox,
         uiLightboxSources: state => state.uiLightboxSources,
         uiLightboxThumbs: state => state.uiLightboxThumbs,
@@ -400,6 +419,7 @@ export default new Vuex.Store({
         uiLightboxTypes: state => state.uiLightboxTypes,
         uiLightboxKey: state => state.uiLightboxKey,
         uiLightboxSlide: state => state.uiLightboxSlide,
+        uiSqliteMode: state => state.uiSqliteMode,
 
         optHideDuplicates: state => state.optHideDuplicates,
         optLang: state => state.optLang,

sist2-vue/src/views/Configuration.vue

@@ -37,11 +37,11 @@
                         {{ $t("opt.lightboxLoadOnlyCurrent") }}
                     </b-form-checkbox>
 
-                    <b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy">
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optHideLegacy" @input="setOptHideLegacy">
                         {{ $t("opt.hideLegacy") }}
                     </b-form-checkbox>
 
-                    <b-form-checkbox :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
                         {{ $t("opt.updateMimeMap") }}
                     </b-form-checkbox>
 
@@ -132,8 +132,11 @@
                         $t("opt.tagOrOperator")
                         }}
                     </b-form-checkbox>
-                    <b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
-                    <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optFuzzy" @input="setOptFuzzy">
+                        {{ $t("opt.fuzzy") }}
+                    </b-form-checkbox>
+
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optSearchInPath" @input="setOptSearchInPath">{{
                         $t("opt.searchInPath")
                         }}
                     </b-form-checkbox>
@@ -151,8 +154,8 @@
                     <b-form-input :value="optResultSize" type="number" min="10"
                                   @input="setOptResultSize"></b-form-input>
 
-                    <label>{{ $t("opt.queryMode") }}</label>
-                    <b-form-select :options="queryModeOptions" :value="optQueryMode"
+                    <label :class="{'text-muted': uiSqliteMode}">{{ $t("opt.queryMode") }}</label>
+                    <b-form-select :disabled="uiSqliteMode" :options="queryModeOptions" :value="optQueryMode"
                                    @input="setOptQueryMode"></b-form-select>
 
                     <label>{{ $t("opt.slideDuration") }}</label>
@@ -170,7 +173,7 @@
                     <b-textarea rows="3" :value="optMlRepositories" @input="setOptMlRepositories"></b-textarea>
                     <br>
                     <b-form-checkbox :checked="optAutoAnalyze" @input="setOptAutoAnalyze">{{
-                            $t("opt.autoAnalyze")
+                        $t("opt.autoAnalyze")
                         }}
                     </b-form-checkbox>
                 </b-card>
@@ -234,6 +237,7 @@ export default {
                 {value: "fr", text: this.$t("lang.fr")},
                 {value: "zh-CN", text: this.$t("lang.zh-CN")},
                 {value: "de", text: this.$t("lang.de")},
+                {value: "pl", text: this.$t("lang.pl")},
             ],
             queryModeOptions: [
                 {value: "simple", text: this.$t("queryMode.simple")},
@@ -300,6 +304,7 @@ export default {
     },
     computed: {
         ...mapGetters([
+            "uiSqliteMode",
             "optTheme",
             "optDisplay",
             "optColumns",

sist2-vue/src/views/FilePage.vue

@@ -14,7 +14,7 @@
       <!-- Audio player-->
       <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
              :type="doc._source.mime"
-             :src="`f/${doc._id}`"></audio>
+             :src="`f/${doc._source.index}/${doc._id}`"></audio>
 
       <InfoTable :doc="doc" v-if="doc"></InfoTable>
 
@@ -54,7 +54,7 @@ export default Vue.extend({
   methods: {
     ext: ext,
     onThumbnailClick() {
-      window.open(`/f/${this.doc._id}`, "_blank");
+      window.open(`/f/${this.doc.index}/${this.doc._id}`, "_blank");
     },
     findByCustomField(field, id) {
       return {

sist2-vue/src/views/SearchPage.vue

@@ -13,6 +13,7 @@
 
         <b-card v-show="!uiLoading && !showEsConnectionError" id="search-panel">
             <SearchBar @show-help="showHelp=true"></SearchBar>
+            <EmbeddingsSearchBar v-if="hasEmbeddings" class="mt-3"></EmbeddingsSearchBar>
             <b-row>
                 <b-col style="height: 70px;" sm="6">
                     <SizeSlider></SizeSlider>
@@ -58,16 +59,14 @@
     </div>
 </template>
 
-<script lang="ts">
+<script>
 import Preloader from "@/components/Preloader.vue";
 import {mapActions, mapGetters, mapMutations} from "vuex";
-import sist2 from "../Sist2Api";
-import Sist2Api, {EsHit, EsResult} from "../Sist2Api";
 import SearchBar from "@/components/SearchBar.vue";
 import IndexPicker from "@/components/IndexPicker.vue";
 import Vue from "vue";
-import Sist2Query from "@/Sist2Query";
-import _debounce from "lodash/debounce";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
+import {debounce as _debounce} from "underscore";
 import DocCardWall from "@/components/DocCardWall.vue";
 import Lightbox from "@/components/Lightbox.vue";
 import LightboxCaption from "@/components/LightboxCaption.vue";
@@ -79,10 +78,13 @@ import DateSlider from "@/components/DateSlider.vue";
 import TagPicker from "@/components/TagPicker.vue";
 import DocList from "@/components/DocList.vue";
 import HelpDialog from "@/components/HelpDialog.vue";
+import EmbeddingsSearchBar from "@/components/EmbeddingsSearchBar.vue";
+import Sist2Api from "@/Sist2Api";
 
 
 export default Vue.extend({
     components: {
+        EmbeddingsSearchBar,
         HelpDialog,
         DocList,
         TagPicker,
@@ -92,8 +94,8 @@ export default Vue.extend({
     data: () => ({
         loading: false,
         uiLoading: true,
-        search: undefined as any,
-        docs: [] as EsHit[],
+        search: undefined,
+        docs: [],
         docIds: new Set(),
         docChecksums: new Set(),
         searchBusy: false,
@@ -107,16 +109,16 @@ export default Vue.extend({
     mounted() {
         // Handle touch events
         window.ontouchend = () => this.$store.commit("busTouchEnd");
-        window.ontouchcancel = this.$store.commit("busTouchEnd");
+        window.ontouchcancel = () => this.$store.commit("busTouchEnd");
 
-        this.search = _debounce(async (clear: boolean) => {
+        this.search = _debounce(async (clear) => {
             if (clear) {
                 await this.clearResults();
             }
 
-            await this.searchNow(Sist2Query.searchQuery());
+            await this.searchNow();
 
-        }, 350, {leading: false});
+        }, 350, false);
 
         this.$store.dispatch("loadFromArgs", this.$route).then(() => {
             this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
@@ -125,6 +127,7 @@ export default Vue.extend({
                     "setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
                     "setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
                     "setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
+                    "setEmbedding"
                 ].includes(mutation.type)) {
                     if (this.searchBusy) {
                         return;
@@ -137,7 +140,7 @@ export default Vue.extend({
 
         this.setIndices(this.$store.getters["sist2Info"].indices)
 
-        this.getDateRange().then((range: { min: number, max: number }) => {
+        Sist2Api.getDateRange().then((range) => {
             this.setDateBoundsMin(range.min);
             this.setDateBoundsMax(range.max);
 
@@ -151,7 +154,7 @@ export default Vue.extend({
         }).catch(error => {
             console.log(error);
 
-            if (error.response.status == 503 || error.response.status == 500) {
+            if (error.response.status === 503 || error.response.status === 500) {
                 this.showEsConnectionError = true;
                 this.uiLoading = false;
             } else {
@@ -169,6 +172,12 @@ export default Vue.extend({
             setDateBoundsMax: "setDateBoundsMax",
             setTags: "setTags",
         }),
+        hasEmbeddings() {
+            if (!this.loading) {
+                return false;
+            }
+            return Sist2Api.models().some();
+        },
         showErrorToast() {
             this.$bvToast.toast(
                 this.$t("toast.esConnErr"),
@@ -180,7 +189,7 @@ export default Vue.extend({
                     bodyClass: "toast-body-error",
                 });
         },
-        showSyntaxErrorToast: function (): void {
+        showSyntaxErrorToast: function () {
             this.$bvToast.toast(
                 this.$t("toast.esQueryErr"),
                 {
@@ -191,15 +200,16 @@ export default Vue.extend({
                     bodyClass: "toast-body-warning",
                 });
         },
-        async searchNow(q: any) {
+        async searchNow() {
             this.searchBusy = true;
             await this.$store.dispatch("incrementQuerySequence");
             this.$store.commit("busSearch");
 
-            Sist2Api.esQuery(q).then(async (resp: EsResult) => {
+            Sist2Api.search().then(async (resp) => {
                 await this.handleSearch(resp);
                 this.searchBusy = false;
             }).catch(err => {
+                console.log(err)
                 if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
                     this.showSyntaxErrorToast();
                 } else {
@@ -214,8 +224,8 @@ export default Vue.extend({
             await this.$store.dispatch("clearResults");
             this.$store.commit("setUiReachedScrollEnd", false);
         },
-        async handleSearch(resp: EsResult) {
-            if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
+        async handleSearch(resp) {
+            if (resp.hits.hits.length === 0 || resp.hits.hits.length < this.$store.state.optSize) {
                 this.$store.commit("setUiReachedScrollEnd", true);
             }
 
@@ -238,7 +248,7 @@ export default Vue.extend({
                 if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
                     hit._seq = await this.$store.dispatch("getKeySequence");
                     this.$store.commit("addLightboxSource", {
-                        source: `f/${hit._id}`,
+                        source: `f/${hit._source.index}/${hit._id}`,
                         thumbnail: hit._props.hasThumbnail
                             ? `t/${hit._source.index}/${hit._id}`
                             : null,
@@ -253,38 +263,17 @@ export default Vue.extend({
 
             await this.$store.dispatch("remountLightbox");
             this.$store.commit("setLastQueryResult", resp);
+            if (this.$store.state.firstQueryResults == null) {
+                this.$store.commit("setFirstQueryResult", resp);
+            }
 
             this.docs.push(...resp.hits.hits);
 
             resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
         },
-        getDateRange(): Promise<{ min: number, max: number }> {
-            return sist2.esQuery({
-                // TODO: filter current selected indices
-                aggs: {
-                    dateMin: {min: {field: "mtime"}},
-                    dateMax: {max: {field: "mtime"}},
-                },
-                size: 0
-            }).then(res => {
-                const range = {
-                    min: res.aggregations.dateMin.value,
-                    max: res.aggregations.dateMax.value,
-                }
-
-                if (range.min == null) {
-                    range.min = 0;
-                    range.max = 1;
-                } else if (range.min == range.max) {
-                    range.max += 1;
-                }
-
-                return range;
-            });
-        },
         appendFunc() {
             if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
-                this.searchNow(Sist2Query.searchQuery());
+                this.searchNow();
             }
         }
     },

sist2-vue/src/views/StatsPage.vue

@@ -1,37 +1,36 @@
 <template>
-  <b-container>
+    <b-container>
 
-    <b-card v-if="loading">
-      <Preloader></Preloader>
-    </b-card>
+        <template>
+            <b-card>
+                <b-card-body>
+                    <b-select v-model="selectedIndex" :options="indexOptions">
+                        <template #first>
+                            <b-form-select-option :value="null" disabled>{{
+                                $t("indexPickerPlaceholder")
+                                }}
+                            </b-form-select-option>
+                        </template>
+                    </b-select>
+                </b-card-body>
+            </b-card>
 
-    <template v-else>
-      <b-card>
-        <b-card-body>
-          <b-select v-model="selectedIndex" :options="indexOptions">
-            <template #first>
-              <b-form-select-option :value="null" disabled>{{ $t("indexPickerPlaceholder") }}</b-form-select-option>
-            </template>
-          </b-select>
-        </b-card-body>
-      </b-card>
-
-      <b-card v-if="selectedIndex !== null" class="mt-3">
-        <b-card-body>
-          <D3Treemap :index-id="selectedIndex"></D3Treemap>
+            <b-card v-if="selectedIndex !== null" class="mt-3">
+                <b-card-body>
+                    <D3Treemap :index-id="selectedIndex"></D3Treemap>
 
 
-        </b-card-body>
-      </b-card>
+                </b-card-body>
+            </b-card>
 
-      <b-card v-if="selectedIndex !== null" class="stats-card mt-3">
-        <D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
-        <D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
-        <D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
-        <D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
-      </b-card>
-    </template>
-  </b-container>
+            <b-card v-if="selectedIndex !== null" class="stats-card mt-3">
+                <D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
+                <D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
+                <D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
+                <D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
+            </b-card>
+        </template>
+    </b-container>
 </template>
 <script>
 import D3Treemap from "@/components/D3Treemap";
@@ -43,37 +42,32 @@ import D3DateHistogram from "@/components/D3DateHistogram";
 import D3SizeHistogram from "@/components/D3SizeHistogram";
 
 export default {
-  components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
-  data() {
-    return {
-      loading: true,
-      selectedIndex: null,
-      indices: []
-    }
-  },
-  computed: {
-    indexOptions() {
-      return this.indices.map(idx => {
+    components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
+    data() {
         return {
-          text: idx.name,
-          value: idx.id
-        };
-      })
+            selectedIndex: null,
+        }
+    },
+    computed: {
+        indexOptions() {
+            return this.indices.map(idx => {
+                return {
+                    text: idx.name,
+                    value: idx.id
+                };
+            })
+        },
+        indices() {
+            return this.$store.state.indices;
+        }
     }
-  },
-  mounted() {
-    Sist2Api.getSist2Info().then(data => {
-      this.indices = data.indices;
-      this.loading = false;
-    })
-  }
 }
 </script>
 
 <style>
 
 .stats-card {
-  text-align: center;
-  padding: 1em;
+    text-align: center;
+    padding: 1em;
 }
 </style>

sist2-vue/vue.config.js

@@ -1,3 +1,5 @@
+const TerserPlugin = require("terser-webpack-plugin");
+
 module.exports = {
     filenameHashing: false,
     productionSourceMap: false,
@@ -6,5 +8,19 @@ module.exports = {
         index: {
             entry: "src/main.js"
         }
+    },
+    configureWebpack: config => {
+        config.optimization.minimizer = [new TerserPlugin({
+            terserOptions: {
+                compress: {
+                    passes: 2,
+                    module: true,
+                    hoist_funs: true,
+                    // https://github.com/microsoft/onnxruntime/issues/16984
+                    unused: false,
+                },
+                mangle: true,
+            }
+        })]
     }
 }

src/cli.c

@@ -5,7 +5,7 @@
 #define DEFAULT_OUTPUT "index.sist2"
 #define DEFAULT_NAME "index"
 #define DEFAULT_CONTENT_SIZE 32768
-#define DEFAULT_QUALITY 2
+#define DEFAULT_QUALITY 50
 #define DEFAULT_THUMBNAIL_SIZE 552
 #define DEFAULT_THUMBNAIL_COUNT 1
 #define DEFAULT_REWRITE_URL ""
@@ -38,11 +38,6 @@ scan_args_t *scan_args_create() {
     return args;
 }
 
-exec_args_t *exec_args_create() {
-    exec_args_t *args = calloc(sizeof(exec_args_t), 1);
-    return args;
-}
-
 void scan_args_destroy(scan_args_t *args) {
     if (args->name != NULL) {
         free(args->name);
@@ -74,12 +69,8 @@ void web_args_destroy(web_args_t *args) {
     free(args);
 }
 
-void exec_args_destroy(exec_args_t *args) {
-
-    if (args->index_path != NULL) {
-        free(args->index_path);
-    }
-
+void sqlite_index_args_destroy(sqlite_index_args_t *args) {
+    free(args->index_path);
     free(args);
 }
 
@@ -93,15 +84,18 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (abs_path == NULL) {
         LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
     } else {
-        abs_path = realloc(abs_path, strlen(abs_path) + 2);
-        strcat(abs_path, "/");
-        args->path = abs_path;
+        char *new_abs_path = realloc(abs_path, strlen(abs_path) + 2);
+        if (new_abs_path == NULL) {
+            LOG_FATALF("cli.c", "FIXME: realloc() failed for argv[1]=%s, abs_path=%s", argv[1], abs_path);
+        }
+        strcat(new_abs_path, "/");
+        args->path = new_abs_path;
     }
 
     if (args->tn_quality == OPTION_VALUE_UNSPECIFIED) {
         args->tn_quality = DEFAULT_QUALITY;
-    } else if (args->tn_quality < 2 || args->tn_quality > 31) {
-        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [2, 31].\n",
+    } else if (args->tn_quality < 0 || args->tn_quality > 100) {
+        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [0, 100].\n",
                 args->tn_quality);
         return 1;
     }
@@ -109,7 +103,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (args->tn_size == OPTION_VALUE_UNSPECIFIED) {
         args->tn_size = DEFAULT_THUMBNAIL_SIZE;
     } else if (args->tn_size < 32) {
-        printf("Invalid value --thumbnail-size argument: %d. Must be greater than 32 pixels.\n", args->tn_size);
+        printf("Invalid value --thumbnail-size argument: %d. Must be >= 32 pixels.\n", args->tn_size);
         return 1;
     }
 
@@ -142,10 +136,14 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     char *abs_output = abspath(args->output);
     if (args->incremental && abs_output == NULL) {
-        LOG_WARNINGF("main.c", "Could not open original index for incremental scan: %s. Will not perform incremental scan.", args->output);
+        LOG_WARNINGF("main.c",
+                     "Could not open original index for incremental scan: %s. Will not perform incremental scan.",
+                     args->output);
         args->incremental = FALSE;
     } else if (!args->incremental && abs_output != NULL) {
-        LOG_FATALF("main.c", "Index already exists: %s. If you wish to perform incremental scan, you must specify --incremental", abs_output);
+        LOG_FATALF("main.c",
+                   "Index already exists: %s. If you wish to perform incremental scan, you must specify --incremental",
+                   abs_output);
     }
     free(abs_output);
 
@@ -410,6 +408,32 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
     return 0;
 }
 
+int sqlite_index_args_validate(sqlite_index_args_t *args, int argc, const char **argv) {
+
+    LogCtx.verbose = 1;
+
+    if (argc < 2) {
+        fprintf(stderr, "Required positional argument: PATH.\n");
+        return 1;
+    }
+
+    char *index_path = abspath(argv[1]);
+    if (index_path == NULL) {
+        LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
+    } else {
+        args->index_path = index_path;
+    }
+
+    if (args->search_index_path == NULL) {
+        LOG_FATAL("cli.c", "Missing required argument --search-index");
+    }
+
+    LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path);
+    LOG_DEBUGF("cli.c", "arg search_index_path=%s", args->search_index_path);
+
+    return 0;
+}
+
 int web_args_validate(web_args_t *args, int argc, const char **argv) {
 
     LogCtx.verbose = 1;
@@ -419,6 +443,16 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
+    if (args->search_index_path != NULL && args->es_url != NULL) {
+        LOG_FATAL("cli.c", "--search-index and --es-url arguments are mutually exclusive.");
+    }
+    if (args->search_index_path != NULL && args->es_index != NULL) {
+        LOG_FATAL("cli.c", "--search-index and --es-index arguments are mutually exclusive.");
+    }
+    if (args->search_index_path != NULL && args->es_insecure_ssl == TRUE) {
+        LOG_FATAL("cli.c", "--search-index and --es-insecure_ssl arguments are mutually exclusive.");
+    }
+
     if (args->es_url == NULL) {
         args->es_url = DEFAULT_ES_URL;
     }
@@ -531,9 +565,25 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         free(abs_path);
     }
 
+    if (args->search_index_path != NULL) {
+        char *abs_path = abspath(args->search_index_path);
+        if (abs_path == NULL) {
+            LOG_FATALF("cli.c", "Search index not found: %s", args->search_index_path);
+        }
+
+        args->es_index = NULL;
+        args->es_url = NULL;
+        args->es_insecure_ssl = FALSE;
+        args->search_backend = SQLITE_SEARCH_BACKEND;
+    } else {
+        args->search_backend = ES_SEARCH_BACKEND;
+    }
+
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url);
     LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index);
     LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl);
+    LOG_DEBUGF("cli.c", "arg search_index_path=%s", args->search_index_path);
+    LOG_DEBUGF("cli.c", "arg search_backend=%d", args->search_backend);
     LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline);
     LOG_DEBUGF("cli.c", "arg dev=%d", args->dev);
     LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address);
@@ -554,47 +604,12 @@ index_args_t *index_args_create() {
     return args;
 }
 
+sqlite_index_args_t *sqlite_index_args_create() {
+    sqlite_index_args_t *args = calloc(sizeof(sqlite_index_args_t), 1);
+    return args;
+}
+
 web_args_t *web_args_create() {
     web_args_t *args = calloc(sizeof(web_args_t), 1);
     return args;
 }
-
-int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
-
-    if (argc < 2) {
-        fprintf(stderr, "Required positional argument: PATH.\n");
-        return 1;
-    }
-
-    char *index_path = abspath(argv[1]);
-    if (index_path == NULL) {
-        LOG_FATALF("cli.c", "Invalid index PATH argument. File not found: %s", argv[1]);
-    } else {
-        args->index_path = index_path;
-    }
-
-    if (args->es_url == NULL) {
-        args->es_url = DEFAULT_ES_URL;
-    }
-
-    if (args->es_index == NULL) {
-        args->es_index = DEFAULT_ES_INDEX;
-    }
-
-    if (args->script_path == NULL) {
-        LOG_FATAL("cli.c", "--script-file argument is required");
-    }
-
-    if (load_external_file(args->script_path, &args->script) != 0) {
-        return 1;
-    }
-
-    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path);
-
-    char log_buf[5000];
-    strncpy(log_buf, args->script, sizeof(log_buf));
-    *(log_buf + sizeof(log_buf) - 1) = '\0';
-    LOG_DEBUGF("cli.c", "arg script=%s", log_buf);
-
-    return 0;
-}

src/cli.h

@@ -66,10 +66,21 @@ typedef struct index_args {
     int incremental;
 } index_args_t;
 
+typedef struct {
+    char *index_path;
+    char *search_index_path;
+} sqlite_index_args_t;
+
+typedef enum {
+    ES_SEARCH_BACKEND,
+    SQLITE_SEARCH_BACKEND,
+} search_backend_t;
+
 typedef struct web_args {
     char *es_url;
     char *es_index;
     int es_insecure_ssl;
+    char *search_index_path;
     char *listen_address;
     char *credentials;
     char *tag_credentials;
@@ -88,20 +99,13 @@ typedef struct web_args {
     int index_count;
     int dev;
     const char **indices;
+    search_backend_t search_backend;
 } web_args_t;
 
-typedef struct exec_args {
-    char *es_url;
-    char *es_index;
-    int es_insecure_ssl;
-    char *index_path;
-    const char *script_path;
-    int async_script;
-    char *script;
-} exec_args_t;
-
 index_args_t *index_args_create();
 
+sqlite_index_args_t *sqlite_index_args_create();
+
 void index_args_destroy(index_args_t *args);
 
 web_args_t *web_args_create();
@@ -110,12 +114,11 @@ void web_args_destroy(web_args_t *args);
 
 int index_args_validate(index_args_t *args, int argc, const char **argv);
 
+int sqlite_index_args_validate(sqlite_index_args_t *args, int argc, const char **argv);
+
 int web_args_validate(web_args_t *args, int argc, const char **argv);
 
-exec_args_t *exec_args_create();
+void sqlite_index_args_destroy(sqlite_index_args_t *args);
 
-void exec_args_destroy(exec_args_t *args);
-
-int exec_args_validate(exec_args_t *args, int argc, const char **argv);
 
 #endif

src/ctx.h

@@ -76,6 +76,7 @@ typedef struct {
     char *es_url;
     es_version_t *es_version;
     char *es_index;
+    database_t *search_db;
     int es_insecure_ssl;
     int index_count;
     char *auth_user;
@@ -93,6 +94,7 @@ typedef struct {
     struct index_t indices[256];
     char lang[10];
     int dev;
+    int search_backend;
 } WebCtx_t;
 
 

src/database/database.c

@@ -14,12 +14,45 @@ database_t *database_create(const char *filename, database_type_t type) {
     strcpy(db->filename, filename);
     db->type = type;
     db->select_thumbnail_stmt = NULL;
+    db->db = NULL;
+    db->tag_array = NULL;
 
     db->ipc_ctx = NULL;
 
     return db;
 }
 
+int tag_matches(const char *query, const char *tag) {
+    size_t query_len = strlen(query);
+    size_t tag_len = strlen(tag);
+
+    if (query_len >= tag_len) {
+        return FALSE;
+    }
+
+    return strncmp(tag, query, query_len) == 0 && *(tag + query_len) == '.';
+}
+
+void tag_matches_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    const char *tag = (const char *) sqlite3_value_text(argv[0]);
+
+    char **tags = *(char ***) sqlite3_user_data(ctx);
+
+    array_foreach(tags) {
+        if (tag_matches(tags[i], tag)) {
+            sqlite3_result_int(ctx, TRUE);
+            return;
+        }
+    }
+
+    sqlite3_result_int(ctx, FALSE);
+}
+
 __always_inline
 static int sep_rfind(const char *str) {
     for (int i = (int) strlen(str); i >= 0; i--) {
@@ -48,6 +81,24 @@ void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
     sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
 }
 
+void random_func(sqlite3_context *ctx, int argc, UNUSED(sqlite3_value **argv)) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    char state_buf[128] = {0,};
+    struct random_data buf;
+    int result;
+
+    long seed = sqlite3_value_int64(argv[0]);
+
+    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
+
+    random_r(&buf, &result);
+
+    sqlite3_result_int(ctx, result);
+}
+
 
 void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
     if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
@@ -74,6 +125,8 @@ void database_initialize(database_t *db) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IndexDatabaseSchema, NULL, NULL, NULL));
     } else if (db->type == IPC_CONSUMER_DATABASE || db->type == IPC_PRODUCER_DATABASE) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IpcDatabaseSchema, NULL, NULL, NULL));
+    } else if (db->type == FTS_DATABASE) {
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, FtsDatabaseSchema, NULL, NULL, NULL));
     }
 
     sqlite3_close(db->db);
@@ -85,7 +138,8 @@ void database_open(database_t *db) {
     CRASH_IF_NOT_SQLITE_OK(sqlite3_open(db->filename, &db->db));
     sqlite3_busy_timeout(db->db, 1000);
 
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA cache_size = -200000;", NULL, NULL, NULL));
+    // TODO: Optional argument?
+//    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA cache_size = -200000;", NULL, NULL, NULL));
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL));
 
     if (db->type == INDEX_DATABASE) {
@@ -109,7 +163,8 @@ void database_open(database_t *db) {
                 &db->write_document_sidecar_stmt, NULL));
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
                 db->db,
-                "REPLACE INTO document (id, mtime, size, json_data) VALUES (?, ?, ?, ?);", -1,
+                "REPLACE INTO document (id, mtime, size, json_data, version) VALUES (?, ?, ?, ?, (SELECT max(id) FROM version));",
+                -1,
                 &db->write_document_stmt, NULL));
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
                 db->db,
@@ -117,6 +172,18 @@ void database_open(database_t *db) {
                 -1,
                 &db->write_thumbnail_stmt, NULL));
 
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT json_data FROM document WHERE id=?", -1,
+                &db->get_document, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT * FROM model", -1,
+                &db->get_models, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT embedding FROM embedding WHERE id=? AND model_id=? AND start=0", -1,
+                &db->get_embedding, NULL));
+
         // Create functions
         sqlite3_create_function(
                 db->db,
@@ -128,6 +195,17 @@ void database_open(database_t *db) {
                 NULL,
                 NULL
         );
+
+        sqlite3_create_function(
+                db->db,
+                "emb_to_json",
+                1,
+                SQLITE_UTF8,
+                NULL,
+                emb_to_json_func,
+                NULL,
+                NULL
+        );
     } else if (db->type == IPC_CONSUMER_DATABASE) {
 
         sqlite3_create_function(
@@ -168,6 +246,65 @@ void database_open(database_t *db) {
                 db->db, "INSERT INTO index_job (doc_id,type,line) VALUES (?,?,?);", -1,
                 &db->insert_index_job_stmt, NULL));
 
+    } else if (db->type == FTS_DATABASE) {
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT path, count FROM path_index"
+                        " WHERE index_id=? AND depth BETWEEN ? AND ?"
+                        " LIMIT 65536", -1,
+                &db->fts_search_paths, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT json_data FROM document_index"
+                        " WHERE id=?", -1,
+                &db->fts_get_document, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT DISTINCT tag FROM tag"
+                        " WHERE tag GLOB (? || '*') ORDER BY tag LIMIT 100", -1,
+                &db->fts_suggest_tag, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT tag, count(*) FROM tag GROUP BY tag", -1,
+                &db->fts_get_tags, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT size FROM model WHERE id=?", -1,
+                &db->fts_model_size, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT path, count FROM path_index"
+                        " WHERE (index_id=?1 OR ?1 IS NULL) AND depth BETWEEN ? AND ?"
+                        " AND (path = ?4 or path GLOB ?5)"
+                        " LIMIT 65536", -1,
+                &db->fts_search_paths_w_prefix, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT path, count FROM path_index"
+                        " WHERE depth BETWEEN ? AND ?"
+                        " AND path GLOB ?"
+                        " LIMIT 65536", -1,
+                &db->fts_suggest_paths, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT * FROM stats", -1,
+                &db->fts_date_range, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT mime, sum(count) FROM mime_index WHERE mime is not NULL GROUP BY mime", -1,
+                &db->fts_get_mimetypes, NULL));
+
+        sqlite3_create_function(
+                db->db,
+                "random_seeded",
+                1,
+                SQLITE_UTF8,
+                NULL,
+                random_func,
+                NULL,
+                NULL
+        );
+
         sqlite3_create_function(
                 db->db,
                 "path_parent",
@@ -178,8 +315,44 @@ void database_open(database_t *db) {
                 NULL,
                 NULL
         );
+
+        sqlite3_create_function(
+                db->db,
+                "tag_matches",
+                1,
+                SQLITE_UTF8,
+                &db->tag_array,
+                tag_matches_func,
+                NULL,
+                NULL
+        );
+
+        sqlite3_create_function(
+                db->db,
+                "cosine_sim",
+                3,
+                SQLITE_UTF8,
+                NULL,
+                cosine_sim_func,
+                NULL,
+                NULL
+        );
     }
 
+    if (db->type == FTS_DATABASE || db->type == INDEX_DATABASE) {
+        // Tag table is the same schema for FTS database & index database
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "INSERT INTO tag (id, tag) VALUES (?,?) ON CONFLICT DO NOTHING;",
+                -1,
+                &db->write_tag_stmt, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "DELETE FROM tag WHERE id=? AND tag=?;",
+                -1,
+                &db->delete_tag_stmt, NULL));
+    }
 }
 
 void database_close(database_t *db, int optimize) {
@@ -191,7 +364,9 @@ void database_close(database_t *db, int optimize) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA optimize;", NULL, NULL, NULL));
     }
 
-    sqlite3_close(db->db);
+    if (db->db) {
+        sqlite3_close(db->db);
+    }
 
     if (db->type == IPC_PRODUCER_DATABASE) {
         remove(db->filename);
@@ -323,31 +498,31 @@ database_iterator_t *database_create_document_iterator(database_t *db) {
 
     sqlite3_stmt *stmt;
 
-    // TODO optimization: remove mtime, size, _id from json_data
-
-    sqlite3_prepare_v2(db->db, "WITH doc (j) AS (SELECT CASE"
-                               " WHEN sc.json_data IS NULL THEN"
-                               "  CASE"
-                               "   WHEN t.tag IS NULL THEN"
-                               "    json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime)"
-                               "   ELSE"
-                               "    json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime, '$.tag', json_group_array(t.tag))"
-                               "   END"
-                               " ELSE"
-                               "  CASE"
-                               "   WHEN t.tag IS NULL THEN"
-                               "    json_patch(json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime), sc.json_data)"
-                               "   ELSE"
-                               //   This will overwrite any tags specified in the sidecar file!
-                               //   TODO: concatenate the two arrays?
-                               "    json_set(json_patch(document.json_data, sc.json_data), '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime, '$.tag', json_group_array(t.tag))"
-                               "   END"
-                               " END"
-                               " FROM document"
-                               " LEFT JOIN document_sidecar sc ON document.id = sc.id"
-                               " LEFT JOIN tag t ON document.id = t.id"
-                               " GROUP BY document.id)"
-                               " SELECT json_set(j, '$.index', (SELECT id FROM descriptor)) FROM doc", -1, &stmt, NULL);
+    CRASH_IF_NOT_SQLITE_OK(
+            sqlite3_prepare_v2(
+                    db->db,
+                    "WITH doc (j) AS (SELECT CASE"
+                    " WHEN emb.embedding IS NULL THEN"
+                    "  json_set(document.json_data, "
+                    "      '$._id', document.id, "
+                    "      '$.size', document.size, "
+                    "      '$.mtime', document.mtime, "
+                    "      '$.tag', json_group_array((SELECT tag FROM tag WHERE document.id = tag.id)))"
+                    " ELSE"
+                    "  json_set(document.json_data,"
+                    "      '$._id', document.id,"
+                    "      '$.size', document.size,"
+                    "      '$.mtime', document.mtime,"
+                    "      '$.tag', json_group_array((SELECT tag FROM tag WHERE document.id = tag.id)),"
+                    "      '$.emb', json_group_object(m.path, json(emb_to_json(emb.embedding))),"
+                    "      '$.embedding', 1)"
+                    " END"
+                    " FROM document"
+                    " LEFT JOIN embedding emb ON document.id = emb.id"
+                    " LEFT JOIN model m ON emb.model_id = m.id"
+                    " GROUP BY document.id)"
+                    " SELECT json_set(j, '$.index', (SELECT id FROM descriptor)) FROM doc",
+                    -1, &stmt, NULL));
 
     database_iterator_t *iter = malloc(sizeof(database_iterator_t));
 
@@ -357,6 +532,13 @@ database_iterator_t *database_create_document_iterator(database_t *db) {
     return iter;
 }
 
+void remove_tag_if_null(cJSON *doc) {
+    cJSON *tags = cJSON_GetObjectItem(doc, "tag");
+    if (tags != NULL && cJSON_IsNull(cJSON_GetArrayItem(tags, 0))) {
+        cJSON_DeleteItemFromObject(doc, "tag");
+    }
+}
+
 cJSON *database_document_iter(database_iterator_t *iter) {
 
     if (iter->stmt == NULL) {
@@ -368,7 +550,12 @@ cJSON *database_document_iter(database_iterator_t *iter) {
 
     if (ret == SQLITE_ROW) {
         const char *json_string = (const char *) sqlite3_column_text(iter->stmt, 0);
-        return cJSON_Parse(json_string);
+
+        cJSON *doc = cJSON_Parse(json_string);
+
+        remove_tag_if_null(doc);
+
+        return doc;
     }
 
     if (ret != SQLITE_DONE) {
@@ -479,28 +666,6 @@ void database_write_thumbnail(database_t *db, const char *id, int num, void *dat
 }
 
 
-//void database_create_fts_index(database_t *db, database_t *fts_db) {
-//    // In a separate file,
-//
-//    // use database_initialize() to create FTS schema
-//    // if --force-reset, then truncate the tables first
-//
-//    /*
-//     * create/append fts table
-//     *
-//     * create/append scalar index table with
-//     *  id,index,size,mtime,mime
-//     *
-//     * create/append path index table with
-//     *  index,path,depth
-//     *
-//     * content table is a view with SELECT UNION for all attached tables
-//     *  random_seed column
-//     */
-//
-//    // INSERT INTO ft(ft) VALUES('optimize');
-//}
-
 job_t *database_get_work(database_t *db, job_type_t job_type) {
     job_t *job;
 
@@ -642,3 +807,44 @@ void database_add_work(database_t *db, job_t *job) {
     pthread_cond_signal(&db->ipc_ctx->has_work_cond);
     pthread_mutex_unlock(&db->ipc_ctx->mutex);
 }
+
+void database_write_tag(database_t *db, char *doc_id, char *tag) {
+    sqlite3_bind_text(db->write_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_text(db->write_tag_stmt, 2, tag, -1, SQLITE_STATIC);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->write_tag_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_tag_stmt));
+}
+
+void database_delete_tag(database_t *db, char *doc_id, char *tag) {
+    sqlite3_bind_text(db->delete_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_text(db->delete_tag_stmt, 2, tag, -1, SQLITE_STATIC);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->delete_tag_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->delete_tag_stmt));
+}
+
+cJSON *database_get_document(database_t *db, char *doc_id) {
+    sqlite3_bind_text(db->get_document, 1, doc_id, -1, SQLITE_STATIC);
+
+    int ret = sqlite3_step(db->get_document);
+    CRASH_IF_STMT_FAIL(ret);
+
+    cJSON *json;
+
+    if (ret == SQLITE_ROW) {
+        const char *json_str = sqlite3_column_text(db->get_document, 0);
+        json = cJSON_Parse(json_str);
+    } else {
+        json = NULL;
+    }
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->get_document));
+
+    return json;
+}
+
+void database_increment_version(database_t *db) {
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "INSERT INTO version DEFAULT VALUES", NULL, NULL, NULL));
+}

src/database/database.h

@@ -10,6 +10,7 @@ typedef struct index_descriptor index_descriptor_t;
 
 extern const char *IpcDatabaseSchema;
 extern const char *IndexDatabaseSchema;
+extern const char *FtsDatabaseSchema;
 
 typedef enum {
     INDEX_DATABASE,
@@ -32,6 +33,17 @@ typedef enum {
     JOB_PARSE_JOB
 } job_type_t;
 
+typedef enum {
+    FTS_SORT_INVALID,
+    FTS_SORT_SCORE,
+    FTS_SORT_SIZE,
+    FTS_SORT_MTIME,
+    FTS_SORT_RANDOM,
+    FTS_SORT_NAME,
+    FTS_SORT_ID,
+    FTS_SORT_EMBEDDING
+} fts_sort_t;
+
 typedef struct {
     job_type_t type;
     union {
@@ -52,6 +64,11 @@ typedef struct {
     char current_job[MAX_THREADS][PATH_MAX * 2];
 } database_ipc_ctx_t;
 
+typedef struct {
+    double date_min;
+    double date_max;
+} database_summary_stats_t;
+
 typedef struct database {
     char filename[PATH_MAX];
     database_type_t type;
@@ -66,12 +83,31 @@ typedef struct database {
     sqlite3_stmt *write_document_stmt;
     sqlite3_stmt *write_document_sidecar_stmt;
     sqlite3_stmt *write_thumbnail_stmt;
+    sqlite3_stmt *get_document;
+    sqlite3_stmt *get_models;
+    sqlite3_stmt *get_embedding;
+
+    sqlite3_stmt *delete_tag_stmt;
+    sqlite3_stmt *write_tag_stmt;
 
     sqlite3_stmt *insert_parse_job_stmt;
     sqlite3_stmt *insert_index_job_stmt;
     sqlite3_stmt *pop_parse_job_stmt;
     sqlite3_stmt *pop_index_job_stmt;
 
+    sqlite3_stmt *fts_search_paths;
+    sqlite3_stmt *fts_search_paths_w_prefix;
+    sqlite3_stmt *fts_suggest_paths;
+    sqlite3_stmt *fts_date_range;
+    sqlite3_stmt *fts_get_mimetypes;
+    sqlite3_stmt *fts_get_document;
+    sqlite3_stmt *fts_suggest_tag;
+    sqlite3_stmt *fts_get_tags;
+    sqlite3_stmt *fts_model_size;
+
+
+    char **tag_array;
+
     database_ipc_ctx_t *ipc_ctx;
 } database_t;
 
@@ -86,8 +122,6 @@ typedef struct {
     long size;
 } treemap_row_t;
 
-static treemap_row_t null_treemap_row = {0, 0, 0};
-
 
 database_t *database_create(const char *filename, database_type_t type);
 
@@ -97,6 +131,8 @@ void database_open(database_t *db);
 
 void database_close(database_t *, int optimize);
 
+void database_increment_version(database_t *db);
+
 void database_write_thumbnail(database_t *db, const char *id, int num, void *data, size_t data_size);
 
 void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *return_value_len);
@@ -109,6 +145,8 @@ void database_write_document(database_t *db, document_t *doc, const char *json_d
 
 database_iterator_t *database_create_document_iterator(database_t *db);
 
+void emb_to_json_func(sqlite3_context *ctx, int argc, sqlite3_value **argv);
+
 cJSON *database_document_iter(database_iterator_t *);
 
 #define database_document_iter_foreach(element, iter) \
@@ -116,7 +154,7 @@ cJSON *database_document_iter(database_iterator_t *);
 
 database_iterator_t *database_create_delete_list_iterator(database_t *db);
 
-char * database_delete_list_iter(database_iterator_t *iter);
+char *database_delete_list_iter(database_iterator_t *iter);
 
 #define database_delete_list_iter_foreach(element, iter) \
     for (char *(element) = database_delete_list_iter(iter); (element) != NULL; (element) = database_delete_list_iter(iter))
@@ -135,7 +173,7 @@ database_iterator_t *database_create_treemap_iterator(database_t *db, long thres
 treemap_row_t database_treemap_iter(database_iterator_t *iter);
 
 #define database_treemap_iter_foreach(element, iter) \
-    for (treemap_row_t element = database_treemap_iter(iter); element.path != NULL; element = database_treemap_iter(iter))
+    for (treemap_row_t element = database_treemap_iter(iter); (element).path != NULL; (element) = database_treemap_iter(iter))
 
 
 void database_generate_stats(database_t *db, double treemap_threshold);
@@ -146,22 +184,64 @@ job_t *database_get_work(database_t *db, job_type_t job_type);
 
 void database_add_work(database_t *db, job_t *job);
 
-//void database_index(database_t *db);
-
 cJSON *database_get_stats(database_t *db, database_stat_type_d type);
 
 #define CRASH_IF_STMT_FAIL(x) do { \
         int return_value = x;                \
         if (return_value != SQLITE_DONE && return_value != SQLITE_ROW) {     \
-            LOG_FATALF("database.c", "Sqlite error @ database.c:%d : (%d) %s", __LINE__, return_value, sqlite3_errmsg(db->db)); \
+            LOG_FATALF("database.c", "Sqlite error @ %s:%d : (%d) %s", __BASE_FILE__, __LINE__, return_value, sqlite3_errmsg(db->db)); \
         }                           \
     } while (0)
 
 #define CRASH_IF_NOT_SQLITE_OK(x) do { \
         int return_value = x;                \
         if (return_value != SQLITE_OK) {     \
-            LOG_FATALF("database.c", "Sqlite error @ database.c:%d : (%d) %s", __LINE__, return_value, sqlite3_errmsg(db->db)); \
+            LOG_FATALF("database.c", "Sqlite error @ %s:%d : (%d) %s", __BASE_FILE__, __LINE__, return_value, sqlite3_errmsg(db->db)); \
         }                           \
     } while (0)
 
-#endif //SIST2_DATABASE_H
+void database_fts_attach(database_t *db, const char *fts_database_path);
+
+void database_fts_index(database_t *db);
+
+void database_fts_optimize(database_t *db);
+
+cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
+                              int suggest);
+
+cJSON *database_fts_get_mimetypes(database_t *db);
+
+database_summary_stats_t database_fts_get_date_range(database_t *db);
+
+cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
+                           long size_max, long date_min, long date_max, int page_size,
+                           char **index_ids, char **mime_types, char **tags, int sort_asc,
+                           fts_sort_t sort, int seed, char **after, int fetch_aggregations,
+                           int highlight, int highlight_context_size, int model,
+                           const float *embedding, int embedding_size);
+
+void database_write_tag(database_t *db, char *doc_id, char *tag);
+
+void database_delete_tag(database_t *db, char *doc_id, char *tag);
+
+void database_fts_detach(database_t *db);
+
+cJSON *database_fts_get_document(database_t *db, char *doc_id);
+
+database_summary_stats_t database_fts_sync_tags(database_t *db);
+
+cJSON *database_fts_suggest_tag(database_t *db, char *prefix);
+
+cJSON *database_fts_get_tags(database_t *db);
+
+cJSON *database_get_document(database_t *db, char *doc_id);
+
+void cosine_sim_func(sqlite3_context *ctx, int argc, sqlite3_value **argv);
+
+cJSON *database_get_models(database_t *db);
+
+int database_fts_get_model_size(database_t *db, int model_id);
+
+cJSON *database_get_embedding(database_t *db, char *doc_id, int model_id);
+
+#endif

src/database/database_embeddings.c

@@ -0,0 +1,107 @@
+#include <openblas/cblas.h>
+#include "database.h"
+#include "src/ctx.h"
+
+
+static float cosine_sim(int n, const float *a, const float *b) {
+    float dot_product = cblas_sdot(n, a, 1, b, 1);
+    float norm_a = cblas_snrm2(n, a, 1);
+    float norm_b = cblas_snrm2(n, b, 1);
+
+    return dot_product / (norm_a * norm_b);
+}
+
+
+void cosine_sim_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 3) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    int n = sqlite3_value_int(argv[0]);
+    const float *a = sqlite3_value_blob(argv[1]);
+    const float *b = sqlite3_value_blob(argv[2]);
+
+    if (a == NULL || b == NULL) {
+        sqlite3_result_double(ctx, -1);
+        return;
+    }
+
+    float result = cosine_sim(n, a, b);
+    if (result != result) {
+        result = -1;
+    }
+
+    sqlite3_result_double(ctx, result);
+}
+
+cJSON *database_get_models(database_t *db) {
+    cJSON *json = cJSON_CreateArray();
+    sqlite3_stmt *stmt = db->get_models;
+
+    int ret;
+    do {
+        ret = sqlite3_step(stmt);
+        if (ret == SQLITE_BUSY) {
+            // Database is busy (probably scanning)
+            LOG_WARNING("database_embeddings.c",
+                         "Database is busy, could not fetch list of models");
+            break;
+        }
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON *row = cJSON_CreateObject();
+
+        cJSON_AddNumberToObject(row, "id", sqlite3_column_int(stmt, 0));
+        cJSON_AddStringToObject(row, "name", (const char *) sqlite3_column_text(stmt, 1));
+        cJSON_AddStringToObject(row, "url", (const char *) sqlite3_column_text(stmt, 2));
+        cJSON_AddStringToObject(row, "path", (const char *) sqlite3_column_text(stmt, 3));
+        cJSON_AddNumberToObject(row, "size", sqlite3_column_int(stmt, 4));
+        cJSON_AddStringToObject(row, "type", (const char *) sqlite3_column_text(stmt, 5));
+
+        cJSON_AddItemToArray(json, row);
+    } while (TRUE);
+
+    sqlite3_reset(stmt);
+
+    return json;
+}
+
+cJSON *database_get_embedding(database_t *db, char *doc_id, int model_id) {
+
+    sqlite3_bind_text(db->get_embedding, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->get_embedding, 2, model_id);
+    int ret = sqlite3_step(db->get_embedding);
+    CRASH_IF_STMT_FAIL(ret);
+
+    if (ret == SQLITE_DONE) {
+        sqlite3_reset(db->get_embedding);
+        return NULL;
+    }
+
+    float *embedding = (float *) sqlite3_column_blob(db->get_embedding, 0);
+    size_t size = sqlite3_column_bytes(db->get_embedding, 0) / sizeof(float);
+
+    cJSON *json = cJSON_CreateFloatArray(embedding, (int) size);
+    sqlite3_reset(db->get_embedding);
+
+    return json;
+}
+
+void emb_to_json_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    float *embedding = (float *) sqlite3_value_blob(argv[0]);
+    int size = sqlite3_value_bytes(argv[0]) / 4;
+
+    cJSON *json = cJSON_CreateFloatArray(embedding, size);
+    char *json_str = cJSON_PrintUnformatted(json);
+
+    sqlite3_result_text(ctx, json_str, -1, SQLITE_TRANSIENT);
+    free(json_str);
+    cJSON_Delete(json);
+}