Fix #521

Update README.md

.devcontainer/Dockerfile

@@ -1,9 +0,0 @@
-FROM simon987/sist2-build
-
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
-RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-

.devcontainer/devcontainer.json

@@ -1,16 +0,0 @@
-{
-  "name": "sist2-dev",
-  "dockerComposeFile": [
-    "docker-compose.yml"
-  ],
-  "service": "sist2-dev",
-  "customizations": {
-    "vscode": {
-      "extensions": [
-        "ms-vscode.cpptools-extension-pack"
-      ]
-    }
-  },
-  "remoteUser": "root",
-  "workspaceFolder": "/app/"
-}

.devcontainer/docker-compose.yml

@@ -1,8 +0,0 @@
-version: "3"
-
-services:
-  sist2-dev:
-    build: .
-    command: sleep infinity
-    volumes:
-      - ../:/app

.dockerignore

@@ -39,3 +39,5 @@ __pycache__/
 sist2-vue/dist
 sist2-admin/frontend/dist
 *.fts
+.git
+third-party/libscan/third-party/ext_*/*

.drone.yml

@@ -7,11 +7,36 @@ platform:
   arch: amd64
 
 steps:
+  - name: submodules
+    image: alpine/git
+    commands:
+      - git submodule update --init --recursive
+  - name: docker
+    image: plugins/docker
+    depends_on:
+      - submodules
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: sist2app/sist2
+      context: ./
+      dockerfile: ./Dockerfile
+      auto_tag: true
+      auto_tag_suffix: x64-linux
+      when:
+        event:
+          - tag
   - name: build
-    image: simon987/sist2-build
+    image: sist2app/sist2-build
+    depends_on:
+    - submodules
     commands:
       - ./scripts/build.sh
   - name: scp files
+    depends_on:
+      - build
     image: appleboy/drone-scp
     settings:
       host:
@@ -22,26 +47,11 @@ steps:
         from_secret: SSH_USER
       key:
         from_secret: SSH_KEY
-      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
       source:
         - ./VERSION
         - ./sist2-x64-linux
         - ./sist2-x64-linux-debug
-  - name: docker
-    image: plugins/docker
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: simon987/sist2
-      context: ./
-      dockerfile: ./Dockerfile
-      auto_tag: true
-      auto_tag_suffix: x64-linux
-      when:
-        event:
-          - tag
 
 ---
 kind: pipeline
@@ -52,11 +62,36 @@ platform:
   arch: arm64
 
 steps:
+  - name: submodules
+    image: alpine/git
+    commands:
+      - git submodule update --init --recursive
+  - name: docker
+    image: plugins/docker
+    depends_on:
+      - submodules
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: sist2app/sist2
+      context: ./
+      dockerfile: ./Dockerfile.arm64
+      auto_tag: true
+      auto_tag_suffix: arm64-linux
+      when:
+        event:
+          - tag
   - name: build
-    image: simon987/sist2-build-arm64
+    image: sist2app/sist2-build-arm64
+    depends_on:
+      - submodules
     commands:
       - ./scripts/build_arm64.sh
   - name: scp files
+    depends_on:
+      - build
     image: appleboy/drone-scp
     settings:
       host:
@@ -67,22 +102,7 @@ steps:
         from_secret: SSH_USER
       key:
         from_secret: SSH_KEY
-      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
       source:
         - ./sist2-arm64-linux
         - ./sist2-arm64-linux-debug
-  - name: docker
-    image: plugins/docker
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: simon987/sist2
-      context: ./
-      dockerfile: ./Dockerfile.arm64
-      auto_tag: true
-      auto_tag_suffix: arm64-linux
-      when:
-        event:
-          - tag

.gitignore

@@ -3,6 +3,7 @@ thumbs
 *.cbp
 CMakeCache.txt
 CMakeFiles
+cmake-build-default-event-trace
 cmake-build-debug
 cmake_install.cmake
 Makefile

CMakeLists.txt

@@ -53,7 +53,6 @@ add_executable(
         src/types.h
         src/log.c src/log.h
         src/cli.c src/cli.h
-        src/parsing/sidecar.c src/parsing/sidecar.h
         src/database/database.c src/database/database.h
         src/parsing/fs_util.h
 
@@ -63,7 +62,7 @@ add_executable(
         src/database/database_schema.c
         src/database/database_fts.c
         src/web/web_fts.c
-)
+        src/database/database_embeddings.c)
 set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
@@ -76,6 +75,7 @@ find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
 find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
 find_package(unofficial-sqlite3 CONFIG REQUIRED)
+find_package(OpenBLAS CONFIG REQUIRED)
 
 
 target_include_directories(
@@ -147,6 +147,7 @@ add_dependencies(
 target_link_libraries(
         sist2
 
+        m
         z
         argparse
         unofficial::mongoose::mongoose
@@ -158,6 +159,7 @@ target_link_libraries(
 
         ${MAGIC_LIB}
         unofficial::sqlite3::sqlite3
+        OpenBLAS::OpenBLAS
 )
 
 add_custom_target(

Dockerfile

@@ -1,5 +1,4 @@
-FROM simon987/sist2-build as build
+FROM sist2app/sist2-build as build
-MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 
@@ -48,5 +47,6 @@ COPY --from=build /build/build/sist2 /root/sist2
 # sist2-admin
 WORKDIR /root/sist2-admin
 COPY sist2-admin/requirements.txt /root/sist2-admin/
-RUN python3 -m pip install --no-cache -r /root/sist2-admin/requirements.txt
+RUN ln /usr/bin/python3 /usr/bin/python
+RUN python -m pip install --no-cache -r /root/sist2-admin/requirements.txt
 COPY --from=build /build/sist2-admin/ /root/sist2-admin/

Dockerfile.arm64

@@ -1,5 +1,4 @@
-FROM simon987/sist2-build-arm64 as build
+FROM sist2app/sist2-build-arm64 as build
-MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 

README.md

@@ -1,9 +1,11 @@
-![GitHub](https://img.shields.io/github/license/simon987/sist2.svg)
+![GitHub](https://img.shields.io/github/license/sist2app/sist2.svg)
-[![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
+[![CodeFactor](https://www.codefactor.io/repository/github/sist2app/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/sist2app/sist2)
 [![Development snapshots](https://ci.simon987.net/api/badges/simon987/sist2/status.svg)](https://files.simon987.net/.gate/sist2/simon987_sist2/)
 
 **Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
 
+**Community URL:** [Discord](https://discord.gg/2PEjDy3Rfs)
+
 # sist2
 
 sist2 (Simple incremental search tool)
@@ -36,26 +38,32 @@ sist2 (Simple incremental search tool)
 ### Using Docker Compose *(Windows/Linux/Mac)*
 
 ```yaml
-version: "3"
-
 services:
   elasticsearch:
     image: elasticsearch:7.17.9
     restart: unless-stopped
+    volumes:
+      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
+      - /data/sist2-es-data/:/usr/share/elasticsearch/data
     environment:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "PUID=1000"
+      - "PGID=1000"
   sist2-admin:
-    image: simon987/sist2:3.1.0-x64-linux
+    image: sist2app/sist2:x64-linux
     restart: unless-stopped
     volumes:
-      - ./sist2-admin-data/:/sist2-admin/
+      - /data/sist2-admin-data/:/sist2-admin/
-      - /:/host
+      - /<path to index>/:/host
     ports:
-      - 4090:4090 # sist2
+      - 4090:4090
-      - 8080:8080 # sist2-admin
+      # NOTE: Don't expose this port publicly!
+      - 8080:8080
     working_dir: /root/sist2-admin/
-    entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
+    entrypoint: python3
+    command:
+      - /root/sist2-admin/sist2_admin/app.py
 ```
 
 Navigate to http://localhost:8080/ to configure sist2-admin.
@@ -71,7 +79,7 @@ Navigate to http://localhost:8080/ to configure sist2-admin.
             ```
     * **SQLite**: No installation required
 
-2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
+2. Download the [latest sist2 release](https://github.com/sist2app/sist2/releases).
    Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
 3. See [usage guide](docs/USAGE.md) for command line usage.
 
@@ -80,28 +88,30 @@ Example usage:
 1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
 2. Prepare search index:
     * **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
-    * **SQLite**: `sist2 index --search-index ./search.sist2 ./documents.sist2`
+    * **SQLite**: `sist2 sqlite-index --search-index ./search.sist2 ./documents.sist2`
-3. Start web interface: `sist2 web ./documents.sist2`
+3. Start web interface: 
+   * **Elasticsearch**: `sist2 web ./documents.sist2`
+   * **SQLite**: `sist2 web --search-index ./search.sist2 ./documents.sist2`
 
 ## Format support
 
 | File type                                                                 | Library                                                                      | Content  | Thumbnail   | Metadata                                                                                                                               |
 |:--------------------------------------------------------------------------|:-----------------------------------------------------------------------------|:---------|:------------|:---------------------------------------------------------------------------------------------------------------------------------------|
 | pdf,xps,fb2,epub                                                          | MuPDF                                                                        | text+ocr | yes         | author, title                                                                                                                          |
-| cbz,cbr                                                                   | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | -        | yes         | -                                                                                                                                      |
+| cbz,cbr                                                                   | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | -        | yes         | -                                                                                                                                      |
 | `audio/*`                                                                 | ffmpeg                                                                       | -        | yes         | ID3 tags                                                                                                                               |
 | `video/*`                                                                 | ffmpeg                                                                       | -        | yes         | title, comment, artist                                                                                                                 |
-| `image/*`                                                                 | ffmpeg                                                                       | ocr      | yes         | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
+| `image/*`                                                                 | ffmpeg                                                                       | ocr      | yes         | [Common EXIF tags](https://github.com/sist2app/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
 | raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf | LibRaw                                                                       | no       | yes         | Common EXIF tags, GPS tags                                                                                                             |
 | ttf,ttc,cff,woff,fnt,otf                                                  | Freetype2                                                                    | -        | yes, `bmp`  | Name & style                                                                                                                           |
-| `text/plain`                                                              | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| `text/plain`                                                              | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
-| html, xml                                                                 | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| html, xml                                                                 | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
 | tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
-| docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
+| docx, xlsx, pptx                                                          | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
 | doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
 | mobi, azw, azw3                                                           | libmobi                                                                      | yes      | yes         | author, title                                                                                                                          |
 | wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
-| json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
+| json, jsonl, ndjson                                                       | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
 \* *See [Archive files](#archive-files)*
 
@@ -125,7 +135,7 @@ You can enable OCR support for ebook (pdf,xps,fb2,epub) or image file types with
 Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
-The `simon987/sist2` image comes with common languages
+The `sist2app/sist2` image comes with common languages
 (hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
@@ -147,16 +157,16 @@ fewer features and generally comparable query performance for medium-size
 indices, but it uses much less memory and is easier to set up.
 
 |                                              |                       SQLite                        |                                                             Elasticsearch                                                             |
-|----------------------------------------------|:----------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------:|
+|----------------------------------------------|:---------------------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------:|
 | Requires separate search engine installation |                                                     |                                                                   ✓                                                                   |
 | Memory footprint                             |                        ~20MB                        |                                                                >500MB                                                                 |
 | Query syntax                                 |      [fts5](https://www.sqlite.org/fts5.html)       | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
 | Fuzzy search                                 |                                                     |                                                                   ✓                                                                   |
 | Media Types tree real-time updating          |                                                     |                                                                   ✓                                                                   |
-| Search in file `path`                        |                                          |                                                                   ✓                                                                   |
 | Manual tagging                               |                          ✓                          |                                                                   ✓                                                                   |
-| User scripts                                 |                                          |                                                                   ✓                                                                   |
+| User scripts                                 |                          ✓                          |                                                                   ✓                                                                   |
 | Media Type breakdown for search results      |                                                     |                                                                   ✓                                                                   |
+| Embeddings search                            |                      ✓ *O(n)*                       |                                                              ✓ *O(logn)*                                                              |
 
 ### NER
 
@@ -165,13 +175,13 @@ sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported re
 to enable it.
 
 The text processing is done in your browser, no data is sent to any third-party services.
-See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) for more details.
+See [sist2app/sist2-ner-models](https://github.com/sist2app/sist2-ner-models) for more details.
 
 #### List of available repositories:
 
 | URL                                                                                                     | Maintainer                              | Purpose |
 |---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
-| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
+| [sist2app/sist2-ner-models](https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json) | [sist2app](https://github.com/sist2app) | General |
 
 <details>
   <summary>Screenshot</summary>
@@ -187,7 +197,7 @@ You can compile **sist2** by yourself if you don't want to use the pre-compiled
 ### Using docker
 
 ```bash
-git clone --recursive https://github.com/simon987/sist2/
+git clone --recursive https://github.com/sist2app/sist2/
 cd sist2
 docker build . -t my-sist2-image
 # Copy sist2 executable from docker image
@@ -202,16 +212,16 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
    apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
    ```
 
-2. Install vcpkg using my fork: https://github.com/simon987/vcpkg
+2. Install vcpkg using my fork: https://github.com/sist2app/vcpkg
 3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,ffprobe,zlib]
+    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5,json1] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib]
     ```
 
 4. Build
     ```bash
-    git clone --recursive https://github.com/simon987/sist2/
+    git clone --recursive https://github.com/sist2app/sist2/
     (cd sist2-vue; npm install; npm run build)
     (cd sist2-admin/frontend; npm install; npm run build)
     cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .

docker-compose.yml

@@ -4,16 +4,21 @@ services:
   elasticsearch:
     image: elasticsearch:7.17.9
     container_name: sist2-es
+    volumes:
+      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
+      - /data/sist2-es-data/:/usr/share/elasticsearch/data
     environment:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "PUID=1000"
+      - "PGID=1000"
   sist2-admin:
     build:
       context: .
     container_name: sist2-admin
     volumes:
-      - /mnt/array/sist2-admin-data/:/sist2-admin/
+      - /data/sist2-admin-data/:/sist2-admin/
-      - /:/host
+      - /<path to index>/:/host
     ports:
       - 4090:4090
       # NOTE: Don't export this port publicly!

docs/USAGE.md

@@ -5,7 +5,6 @@ Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
    or: sist2 sqlite-index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
-   or: sist2 exec-script [OPTION]... INDEX
 
 Lightning-fast file system indexer and search tool.
 
@@ -17,9 +16,9 @@ Lightning-fast file system indexer and search tool.
 
 Scan options
     -t, --threads=<int>               Number of threads. DEFAULT: 1
-    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
+    -q, --thumbnail_count-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
-    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
+    --thumbnail_count-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
-    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
+    --thumbnail_count-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
     --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
     -o, --output=<str>                Output index file path. DEFAULT: index.sist2
     --incremental                     If the output file path exists, only scan new or modified files.
@@ -74,21 +73,14 @@ Web options
     --dev                             Serve html & js files from disk (for development)
     --lang=<str>                      Default UI language. Can be changed by the user
 
-Exec-script options
-    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
-    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
-    --script-file=<str>               Path to user script.
-    --async-script                    Execute user script asynchronously.
-
 Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
 #### Thumbnail database size estimation
 
-See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
+See chart below for rough estimate of thumbnail_count size vs. thumbnail_count size & quality arguments:
 
-For example, `--thumbnail-size=500`, `--thumbnail-quality=50` for a directory with 8 million images will create a thumbnail database 
+For example, `--thumbnail_count-size=500`, `--thumbnail_count-quality=50` for a directory with 8 million images will create a thumbnail_count database 
 that is about `8000000 * 11.8kB = 94.4GB`.
 
 ![thumbnail_size](thumbnail_size.png)
@@ -100,7 +92,7 @@ Simple scan
 sist2 scan ~/Documents
 
 sist2 scan \
-    --threads 4 --content-size 16000000 --thumbnail-quality 2 --archive shallow \
+    --threads 4 --content-size 16000000 --thumbnail_count-quality 2 --archive shallow \
     --name "My Documents" --rewrite-url "http://nas.domain.local/My Documents/" \
     ~/Documents -o ./documents.sist2
 ```
@@ -180,12 +172,37 @@ Using a version >=7.14.0 is recommended to enable the following features:
 
 - Bug fix for large documents (See #198)
 
+Using a version >=8.0.0 is recommended to enable the following features:
+
+- Approximate KNN search for Embeddings search (faster queries).
+
 When using a legacy version of ES, a notice will be displayed next to the sist2 version in the web UI.
 If you don't care about the features above, you can ignore it or disable it in the configuration page.
 
-## exec-script
+# Embeddings search
 
-The `exec-script` command is used to execute a user script for an index that has already been imported to Elasticsearch with the `index` command. Note that the documents will not be reset to their default state before each execution as the `index` command does: if you make undesired changes to the documents by accident, you will need to run `index` again to revert to the original state.
+Since v3.2.0, User scripts can be used to generate _embeddings_ (vector of float32 numbers) which are stored in the .sist2 index file 
+(see [scripting](scripting.md)). Embeddings can be used for:
+
+* Nearest-neighbor queries (e.g. "return the documents most similar to this one")
+* Semantic searches (e.g. "return the documents that are most closely related to the given topic")
+
+In theory, embeddings can be created for any type of documents (image, text, audio etc.).
+
+For example, the [clip](https://github.com/sist2app/sist2-script-clip) User Script, generates 512-d embeddings of images
+(videos are also supported using the thumbnails generated by sist2). When the user enters a query in the "Embeddings Search"
+textbox, the query's embedding is generated in their browser, leveraging the ONNX web runtime.
+
+<details>
+  <summary>Screenshots</summary>
+ 
+ ![embeddings-1](embeddings-1.png)
+ ![embeddings-2](embeddings-2.png)
+
+1. Embeddings search bar. You can select the model using the dropdown on the left.
+2. This icon appears for indices with embeddings search enabled.
+3. Documents with this icon have embeddings. Click on the icon to perform KNN search.
+</details>
 
 
 # Tagging
@@ -213,42 +230,3 @@ See [Automatic tagging](#automatic-tagging) for information about tag
 ### Automatic tagging
 
 See [scripting](scripting.md) documentation.
-
-# Sidecar files
-
-When scanning, sist2 will read metadata from `.s2meta` JSON files and overwrite the 
-original document's indexed metadata (does not modify the actual file). Sidecar metadata files will also work inside archives.
-Sidecar files themselves are not saved in the index.
-
-This feature is useful to leverage third-party applications such as speech-to-text or
-OCR to add additional metadata to a file.
-
-**Example**
-
-```
-~/Documents/
-├── Video.mp4
-└── Video.mp4.s2meta
-```
-
-The sidecar file must have exactly the same file path and the `.s2meta` suffix.
-
-`Video.mp4.s2meta`:
-```json
-{
-  "content": "This sidecar file will overwrite some metadata fields of Video.mp4",
-  "author": "Some author",
-  "duration": 12345,
-  "bitrate": 67890,
-  "some_arbitrary_field": [1,2,3]
-}
-```
-
-```
-sist2 scan ~/Documents -o ./docs.sist2
-sist2 index ./docs.sist2
-```
-
-*NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,
-it is not currently possible to restore both manual tags and sidecar tags without user scripts
-while reindexing.

docs/scripting.md

@@ -1,6 +1,34 @@
 ## User scripts
 
-*This document is under construction, more in-depth guide coming soon*
+User scripts are used to augment your sist2 index with additional metadata, neural network embeddings, tags etc.
+
+
+Since version 3.2.0, user scripts are written in Python, and are ran against the sist2 index file. User scripts do not 
+need a connection to the search backend.
+
+You can create a user script based on a template from the sist2-admin interface:
+
+![sist2-admin-scripts](sist2-admin-scripts.png)
+
+User scripts leverage the [sist2-python](https://github.com/simon987/sist2-python) library to interface with the
+index file*. You can find sist2-python documentation and examples
+here: [sist2-python.readthedocs.io](https://sist2-python.readthedocs.io/).
+
+If you are not using the sist2-admin interface, you can run user scripts manually from the command line:
+
+```
+pip install git+https://github.com/simon987/sist2-python.git
+
+python my_script.py /path/to/my_index.sist2
+```
+
+\* It is possible to manually update the index using raw SQL queries, but the database schema is not stable and
+can change at any time; it is recommended to use the more stable sist2-python wrapper instead.
+
+<hr>
+
+<details>
+  <summary>Legacy user scripts (sist2 version < 3.2.0)</summary>
 
 During the `index` step, you can use the `--script-file <script>` option to
 modify documents or add user tags. This option is mainly used to
@@ -13,6 +41,7 @@ without programming experience at all if you're somewhat familiar with
 regex.
 
 This is the base structure of the documents we're working with:
+
 ```json
 {
   "_id": "e171405c-fdb5-4feb-bb32-82637bc32084",
@@ -35,6 +64,7 @@ This is the base structure of the documents we're working with:
 
 This script checks if the `genre` attribute exists, if it does
 it adds the `genre.<genre>` tag.
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -47,11 +77,12 @@ You can use `.` to create a hierarchical tag tree:
 
 ![scripting/genre_example](genre_example.png)
 
-
 To use regular expressions, you need to add this line in `/etc/elasticsearch/elasticsearch.yml`
+
 ```yaml
 script.painless.regex.enabled: true
 ```
+
 Or, if you're using docker add `-e "script.painless.regex.enabled=true"`
 
 **Tag color**
@@ -62,6 +93,7 @@ hexadecimal color code (`#RRGGBBAA`) to the tag name.
 ### Examples
 
 If `(20XX)` is in the file name, add the `year.<year>` tag:
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -72,6 +104,7 @@ if (m.find()) {
 ```
 
 Use default *Calibre* folder structure to infer author.
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -86,6 +119,7 @@ if (ctx._source.name.contains("-") && ctx._source.extension == "pdf") {
 
 If the file matches a specific pattern `AAAA-000 fName1 lName1, <fName2 lName2>...`, add the `actress.<actress>` and
 `studio.<studio>` tag:
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
@@ -102,16 +136,18 @@ if (m.find()) {
 ```
 
 Set the name of the last folder (`/path/to/<studio>/file.mp4`) to `studio.<studio>` tag
+
 ```Java
 ArrayList tags = ctx._source.tag = new ArrayList();
 
 if (ctx._source.path != "") {
-    String[] names = ctx._source.path.splitOnToken('/');
+    String[] names = ctx._source.path.splitOnToken('/');
     tags.add("studio." + names[names.length-1]);
 }
 ```
 
 Parse `EXIF:F Number` tag
+
 ```Java
 if (ctx._source?.exif_fnumber != null) {
     String[] values = ctx._source.exif_fnumber.splitOnToken(' ');
@@ -124,6 +160,7 @@ if (ctx._source?.exif_fnumber != null) {
 ```
 
 Display year and months from `EXIF:DateTime` tag
+
 ```Java
 if (ctx._source?.exif_datetime != null) {
     SimpleDateFormat parser = new SimpleDateFormat("yyyy:MM:dd HH:mm:ss");
@@ -140,3 +177,6 @@ if (ctx._source?.exif_datetime != null) {
 }
 
 ```
+
+</details>
+

schema/mappings.json

@@ -202,6 +202,46 @@
     },
     "modified_by": {
       "type": "text"
+    },
+    "emb.384.*": {
+      "type": "dense_vector",
+      "dims": 384
+    },
+    "emb.idx_384.*": {
+      "type": "dense_vector",
+      "dims": 384,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.idx_512.clip": {
+      "type": "dense_vector",
+      "dims": 512,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.512.*": {
+      "type": "dense_vector",
+      "dims": 512
+    },
+    "emb.idx_768.*": {
+      "type": "dense_vector",
+      "dims": 768,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.768.*": {
+      "type": "dense_vector",
+      "dims": 768
+    },
+    "emb.idx_1024.*": {
+      "type": "dense_vector",
+      "dims": 1024,
+      "index": true,
+      "similarity": "cosine"
+    },
+    "emb.1024.*": {
+      "type": "dense_vector",
+      "dims": 1024
     }
   }
 }

scripts/build.sh

@@ -2,8 +2,6 @@
 
 VCPKG_ROOT="/vcpkg"
 
-git submodule update --init --recursive
-
 (
   cd sist2-vue/
   npm install

scripts/magic_static.py

@@ -1,8 +1,16 @@
 
-try:
+MAGIC_PATHS = [
-    with open("/usr/lib/file/magic.mgc", "rb") as f:
+    "/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
+    "/work/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
+    "/usr/lib/file/magic.mgc"
+]
+
+for path in MAGIC_PATHS:
+    try:
+        with open(path, "rb") as f:
             data = f.read()
-except:
+        break
-    data = bytes([])
+    except:
+        continue
 
 print("char magic_database_buffer[%d] = {%s};" % (len(data), ",".join(str(int(b)) for b in data)))

scripts/mime.csv

@@ -450,4 +450,3 @@ image/x-sony-arw, arw
 image/x-sony-sr2, sr2
 image/x-sony-srf, srf
 image/x-epson-erf, erf
-sist2/sidecar, s2meta

scripts/mime.py

@@ -3,6 +3,7 @@ import zlib
 mimes = {}
 noparse = set()
 ext_in_hash = set()
+mime_ids = {}
 
 major_mime = {
     "sist2": 0,
@@ -102,6 +103,9 @@ cnt = 1
 
 
 def mime_id(mime):
+    if mime in mime_ids:
+        return mime_ids[mime]
+
     global cnt
     major = mime.split("/")[0]
     mime_id = str((major_mime[major] << 16) + cnt)
@@ -127,9 +131,7 @@ def mime_id(mime):
     elif mime == "application/x-empty":
         cnt -= 1
         return "1"
-    elif mime == "sist2/sidecar":
+    mime_ids[mime] = mime_id
-        cnt -= 1
-        return "2"
     return mime_id
 
 
@@ -197,4 +199,12 @@ with open("scripts/mime.csv") as f:
         print(f"case {crc(mime)}: return {clean(mime)};")
 
     print("default: return 0;}}")
+
+    # mime list
+
+    mime_list = ",".join(mime_id(x) for x in mimes.keys()) + ",0"
+
+    print(f"unsigned int mime_ids[] = {{{mime_list}}};")
+    print("unsigned int* get_mime_ids() { return mime_ids; }")
+
     print("#endif")

scripts/start_dev_es.sh

@@ -1,3 +1,3 @@
-docker run --rm -it --name "sist2-dev-es"\
+docker run --rm -it --name "sist2-dev-es3"\
        	-p 9200:9200 -e "discovery.type=single-node" \
 	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.17.9

scripts/start_dev_es_8.sh

@@ -1,3 +1,3 @@
-docker run --rm -it --name "sist2-dev-es"\
+docker run --rm -it --name "sist2-dev-es3"\
        	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
 	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.7.0

sist2-admin/frontend/package.json

@@ -8,7 +8,7 @@
     "watch": "vue-cli-service build --watch"
   },
   "dependencies": {
-    "axios": "^0.27.2",
+    "axios": "^1.6.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
     "moment": "^2.29.3",

sist2-admin/frontend/src/App.vue

@@ -4,10 +4,10 @@
         <b-container class="pt-4">
             <b-alert show dismissible variant="info">
                 This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
-                to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
+                to the <a href="https://github.com/sist2app/sist2/issues/new/choose" target="_blank">issue tracker on
                 Github</a>. Thank you!
             </b-alert>
-            <router-view/>
+            <router-view v-if="$store.state.sist2AdminInfo"/>
         </b-container>
     </div>
 </template>
@@ -71,10 +71,12 @@ html, body {
 
 .info-icon {
     width: 1rem;
+    min-width: 1rem;
     margin-right: 0.2rem;
     cursor: pointer;
     line-height: 1rem;
     height: 1rem;
+    min-height: 1rem;
     background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
     filter: brightness(45%);
     display: block;

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -89,9 +89,12 @@ class Sist2AdminApi {
 
     /**
      * @param {string} name
+     * @param {bool} full
      */
-    runJob(name) {
+    runJob(name, full) {
-        return axios.get(`${this.baseUrl}/api/job/${name}/run`);
+        return axios.get(`${this.baseUrl}/api/job/${name}/run`, {
+            params: {full}
+        });
     }
 
     /**
@@ -139,6 +142,38 @@ class Sist2AdminApi {
     deleteTaskLogs(taskId) {
         return axios.post(`${this.baseUrl}/api/task/${taskId}/delete_logs`);
     }
+
+    getUserScripts() {
+        return axios.get(`${this.baseUrl}/api/user_script`);
+    }
+
+    getUserScript(name) {
+        return axios.get(`${this.baseUrl}/api/user_script/${name}`);
+    }
+
+    createUserScript(name, template) {
+        return axios.post(`${this.baseUrl}/api/user_script/${name}`, null, {
+            params: {
+                template: template
+            }
+        });
+    }
+
+    updateUserScript(name, data) {
+        return axios.put(`${this.baseUrl}/api/user_script/${name}`, data);
+    }
+
+    deleteUserScript(name) {
+        return axios.delete(`${this.baseUrl}/api/user_script/${name}`);
+    }
+
+    testUserScript(name, job) {
+        return axios.get(`${this.baseUrl}/api/user_script/${name}/run`, {
+            params: {
+                job: job
+            }
+        });
+    }
 }
 
 export default new Sist2AdminApi()

sist2-admin/frontend/src/components/JobCheckboxGroup.vue

@@ -9,13 +9,18 @@
                     @input="frontend.jobs = $event; $emit('input')"
             >
                 <div v-for="job in jobs" :key="job.name">
-          <b-form-checkbox :disabled="job.status !== 'indexed'" :value="job.name">[{{ job.name }}]</b-form-checkbox>
+                    <b-form-checkbox :disabled="job.status !== 'indexed'"
+                                     :value="job.name">
+                        <template #default><span
+                                :title="job.status !== 'indexed' ? $t('jobOptions.notIndexed') : ''"
+                        >[{{ job.name }}]</span></template>
+                    </b-form-checkbox>
                     <br/>
                 </div>
             </b-form-checkbox-group>
             <div v-else>
                 <span class="text-muted">{{ $t('jobOptions.noJobAvailable') }}</span>
-        &nbsp;<router-link to="/">{{$t("create")}}</router-link>
+                <router-link to="/">{{ $t("create") }}</router-link>
             </div>
         </b-form-group>
     </div>
@@ -29,13 +34,20 @@ export default {
     props: ["frontend"],
     mounted() {
         Sist2AdminApi.getJobs().then(resp => {
-      this.jobs = resp.data;
+            this._jobs = resp.data;
             this.loading = false;
         });
     },
+    computed: {
+        jobs() {
+            return this._jobs
+                .filter(job => job.index_options.search_backend === this.frontend.web_options.search_backend)
+        }
+    },
     data() {
         return {
             loading: true,
+            _jobs: null
         }
     }
 }

sist2-admin/frontend/src/components/JobSelect.vue

@@ -0,0 +1,34 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <span v-else-if="jobs.length === 0"></span>
+    <b-form-select v-else :options="jobs" text-field="name" value-field="name"
+                   @change="$emit('change', $event)" :value="$t('selectJob')"></b-form-select>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "JobSelect",
+    mounted() {
+        Sist2AdminApi.getJobs().then(resp => {
+            this._jobs = resp.data;
+            this.loading = false;
+        });
+    },
+    computed: {
+        jobs() {
+            return [
+                {name: this.$t("selectJob"), disabled: true},
+                ...this._jobs.filter(job => job.index_path)
+            ]
+        }
+    },
+    data() {
+        return {
+            loading: true,
+            _jobs: null
+        }
+    }
+}
+</script>

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -95,6 +95,7 @@ export default {
   methods: {
     onOcrLangChange() {
       this.options.ocr_lang = this.selectedOcrLangs.join("+");
+      this.update();
     },
     update() {
       this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;

sist2-admin/frontend/src/components/UserScriptListItem.vue

@@ -0,0 +1,18 @@
+<template>
+    <b-list-group-item action :to="`/userScript/${script.name}`">
+
+        <div class="d-flex w-100 justify-content-between">
+            <h5 class="mb-1">
+                {{ script.name }}
+            </h5>
+        </div>
+    </b-list-group-item>
+</template>
+
+<script>
+
+export default {
+    name: "UserScriptListItem",
+    props: ["script"],
+}
+</script>

sist2-admin/frontend/src/components/UserScriptPicker.vue

@@ -0,0 +1,88 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+
+    <b-row v-else>
+        <b-col cols="6">
+            <h5>Selected scripts</h5>
+            <b-list-group>
+                <b-list-group-item v-for="script in selectedScripts" :key="script"
+                                   button
+                                   @click="onRemoveScript(script)"
+                                   class="d-flex justify-content-between align-items-center">
+                    {{ script }}
+                    <b-button-group>
+                        <b-button variant="light" @click.stop="moveUpScript(script)">↑</b-button>
+                        <b-button variant="light" @click.stop="moveDownScript(script)">↓</b-button>
+                    </b-button-group>
+                </b-list-group-item>
+            </b-list-group>
+        </b-col>
+        <b-col cols="6">
+            <h5>Available scripts</h5>
+            <b-list-group>
+                <b-list-group-item v-for="script in availableScripts" :key="script" button
+                                   @click="onSelectScript(script)">
+                    {{ script }}
+                </b-list-group-item>
+            </b-list-group>
+        </b-col>
+    </b-row>
+
+  <!--    <b-checkbox-group v-else :options="scripts" stacked :checked="selectedScripts"-->
+  <!--                      @input="$emit('change', $event)"></b-checkbox-group>-->
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "UserScriptPicker",
+    props: ["selectedScripts"],
+    data() {
+        return {
+            loading: true,
+            scripts: []
+        }
+    },
+    computed: {
+        availableScripts() {
+            return this.scripts.filter(script => !this.selectedScripts.includes(script))
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getUserScripts().then(resp => {
+            this.scripts = resp.data.map(script => script.name);
+            this.loading = false;
+        });
+    },
+    methods: {
+        onSelectScript(name) {
+            this.selectedScripts.push(name);
+            this.$emit("change", this.selectedScripts)
+        },
+        onRemoveScript(name) {
+            this.selectedScripts.splice(this.selectedScripts.indexOf(name), 1);
+            this.$emit("change", this.selectedScripts);
+        },
+        moveUpScript(name) {
+            const index = this.selectedScripts.indexOf(name);
+            if (index > 0) {
+                this.selectedScripts.splice(index, 1);
+                this.selectedScripts.splice(index - 1, 0, name);
+            }
+            this.$emit("change", this.selectedScripts);
+        },
+        moveDownScript(name) {
+            const index = this.selectedScripts.indexOf(name);
+            if (index < this.selectedScripts.length - 1) {
+                this.selectedScripts.splice(index, 1);
+                this.selectedScripts.splice(index + 1, 0, name);
+            }
+            this.$emit("change", this.selectedScripts);
+        }
+    }
+}
+</script>
+
+<style scoped>
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -1,8 +1,10 @@
 <template>
   <div>
-
+    <h4>{{ $t("webOptions.title") }}</h4>
+    <b-card>
       <label>{{ $t("webOptions.lang") }}</label>
-        <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
+      <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
+                     @change="update()"></b-form-select>
 
       <label>{{ $t("webOptions.bind") }}</label>
       <b-form-input v-model="options.bind" @change="update()"></b-form-input>
@@ -14,10 +16,16 @@
       <b-form-input v-model="options.auth" @change="update()"></b-form-input>
 
       <label>{{ $t("webOptions.tagAuth") }}</label>
-        <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+      <b-form-input v-model="options.tag_auth" @change="update()" :disabled="Boolean(options.auth)"></b-form-input>
+
+      <b-form-checkbox v-model="options.verbose" @change="update()">
+        {{$t("webOptions.verbose")}}
+      </b-form-checkbox>
+    </b-card>
 
     <br>
-        <h5>Auth0 options</h5>
+    <h4>Auth0 options</h4>
+    <b-card>
       <label>{{ $t("webOptions.auth0Audience") }}</label>
       <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
@@ -29,13 +37,12 @@
 
       <label>{{ $t("webOptions.auth0PublicKey") }}</label>
       <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
+    </b-card>
   </div>
 </template>
 
 <script>
 
-import sist2AdminApi from "@/Sist2AdminApi";
-
 export default {
   name: "WebOptions",
   props: ["options", "frontendName"],
@@ -43,13 +50,16 @@ export default {
     return {
       showEsTestAlert: false,
       esTestOk: false,
-            esTestMessage: "",
+      esTestMessage: ""
     }
   },
   methods: {
     update() {
-            if (!this.options.es_url.startsWith("https")) {
+
-                this.options.es_insecure_ssl = false;
+      console.log(this.options)
+      if (this.options.auth && this.options.tag_auth) {
+        // If both are set, remove tagAuth
+        this.options.tag_auth = "";
       }
 
       this.$emit("change", this.options);

sist2-admin/frontend/src/i18n/messages.js

@@ -8,6 +8,7 @@ export default {
         view: "View",
         delete: "Delete",
         runNow: "Index now",
+        runNowFull: "Full re-index",
         create: "Create",
         cancel: "Cancel",
         test: "Test",
@@ -52,7 +53,23 @@ export default {
         searchBackendTitle: "search backend configuration",
         newBackendName: "New search backend name",
 
-        selectJobs: "Select jobs",
+        frontendTab: "Frontend",
+        backendTab: "Backend",
+        scripts: "User Scripts",
+        script: "User Script",
+        testScript: "Test/debug User Script",
+        newScriptName: "New script name",
+        scriptType: "Script type",
+        scriptCode: "Script code (Python)",
+        scriptOptions: "User scripts",
+        gitRepository: "Git repository URL",
+        extraArgs: "Extra command line arguments",
+
+        couldNotStartFrontend: "Could not start frontend",
+        couldNotStartFrontendBody: "Unable to start the frontend, check server logs for more details.",
+
+        selectJobs: "Available jobs",
+        selectJob: "Select a job",
         webOptions: {
             title: "Web options",
             lang: "UI Language",
@@ -64,6 +81,7 @@ export default {
             auth0Domain: "Auth0 domain",
             auth0ClientId: "Auth0 client ID",
             auth0PublicKey: "Auth0 public key",
+            verbose: "Verbose logs"
         },
         backendOptions: {
             title: "Search backend options",
@@ -109,12 +127,13 @@ export default {
             keepNLogs: "Keep last N log files. Set to -1 to keep all logs.",
             deleteNow: "Delete now",
             scheduleEnabled: "Enable scheduled re-scan",
-            noJobAvailable: "No jobs available.",
+            noJobAvailable: "No jobs available for this search backend.",
+            notIndexed: "Has not been indexed yet",
             noBackendError: "You must select a search backend to run this job",
             desktopNotifications: "Desktop notifications"
         },
         frontendOptions: {
-            title: "Frontend options",
+            title: "Advanced options",
             noJobSelectedWarning: "You must select at least one job to start this frontend"
         },
         notifications: {

sist2-admin/frontend/src/router/index.js

@@ -6,12 +6,18 @@ import Tasks from "@/views/Tasks";
 import Frontend from "@/views/Frontend";
 import Tail from "@/views/Tail";
 import SearchBackend from "@/views/SearchBackend.vue";
+import UserScript from "@/views/UserScript.vue";
 
 Vue.use(VueRouter);
 
 const routes = [
   {
-    path: "/",
+    path: "/task",
+    name: "Tasks",
+    component: Tasks
+  },
+  {
+    path: "/:tab?",
     name: "Home",
     component: Home
   },
@@ -20,11 +26,6 @@ const routes = [
     name: "Job",
     component: Job
   },
-  {
-    path: "/task/",
-    name: "Tasks",
-    component: Tasks
-  },
   {
     path: "/frontend/:name",
     name: "Frontend",
@@ -35,6 +36,11 @@ const routes = [
     name: "SearchBackend",
     component: SearchBackend
   },
+  {
+    path: "/userScript/:name",
+    name: "UserScript",
+    component: UserScript
+  },
   {
     path: "/log/:taskId",
     name: "Tail",

sist2-admin/frontend/src/views/Frontend.vue

@@ -8,6 +8,7 @@
       </small>
     </b-card-title>
 
+    <!-- Action buttons-->
     <div class="mb-3" v-if="!loading">
       <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
           $t("start")
@@ -23,10 +24,26 @@
       <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
     </div>
 
-
     <b-progress v-if="loading" striped animated value="100"></b-progress>
     <b-card-body v-else>
 
+      <h4>{{ $t("backendOptions.title") }}</h4>
+      <b-card>
+        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+
+        <SearchBackendSelect :value="frontend.web_options.search_backend"
+                             @change="onBackendSelect($event)"></SearchBackendSelect>
+
+        <br>
+        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+      </b-card>
+
+      <br/>
+
+      <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
+                  @change="update()"></WebOptions>
+      <br/>
+
       <h4>{{ $t("frontendOptions.title") }}</h4>
       <b-card>
         <b-form-checkbox v-model="frontend.auto_start" @change="update()">
@@ -38,32 +55,8 @@
 
         <label>{{ $t("customUrl") }}</label>
         <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
-
-                <br/>
-
-                <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
-
-                <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
-            </b-card>
-
-            <br/>
-
-            <h4>{{ $t("webOptions.title") }}</h4>
-            <b-card>
-                <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
-                            @change="update()"></WebOptions>
-            </b-card>
-
-            <br>
-
-            <h4>{{ $t("backendOptions.title") }}</h4>
-            <b-card>
-                <SearchBackendSelect :value="frontend.web_options.search_backend"
-                                     @change="onBackendSelect($event)"></SearchBackendSelect>
       </b-card>
     </b-card-body>
-
-
   </b-card>
 </template>
 
@@ -120,8 +113,15 @@ export default {
   },
   methods: {
     start() {
+      Sist2AdminApi.startFrontend(this.name).then(() => {
         this.frontend.running = true;
-            Sist2AdminApi.startFrontend(this.name)
+      }).catch(() => {
+        this.$bvToast.toast(this.$t("couldNotStartFrontendBody"), {
+          title: this.$t("couldNotStartFrontend"),
+          variant: "danger",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
     },
     stop() {
       this.frontend.running = false;
@@ -137,6 +137,7 @@ export default {
     },
     onBackendSelect(backend) {
       this.frontend.web_options.search_backend = backend;
+      this.frontend.jobs = [];
       this.update();
     }
   }

sist2-admin/frontend/src/views/Home.vue

@@ -1,5 +1,34 @@
 <template>
     <div>
+        <b-tabs content-class="mt-3" v-model="tab" @input="onTabChange($event)">
+            <b-tab :title="$t('backendTab')">
+
+                <b-card>
+                    <b-card-title>{{ $t("searchBackends") }}</b-card-title>
+
+                    <b-row>
+                        <b-col>
+                            <b-input v-model="newBackendName" :placeholder="$t('newBackendName')"></b-input>
+                        </b-col>
+                        <b-col>
+                            <b-button variant="primary" @click="createBackend()"
+                                      :disabled="!backendNameValid(newBackendName)">
+                                {{ $t("create") }}
+                            </b-button>
+                        </b-col>
+                    </b-row>
+
+                    <hr/>
+
+                    <b-progress v-if="backendsLoading" striped animated value="100"></b-progress>
+                    <b-list-group v-else>
+                        <SearchBackendListItem v-for="backend in backends"
+                                               :key="backend.name" :backend="backend"></SearchBackendListItem>
+                    </b-list-group>
+                </b-card>
+
+                <br/>
+
                 <b-card>
                     <b-card-title>{{ $t("jobs") }}</b-card-title>
                     <b-row>
@@ -28,9 +57,39 @@
                         <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
                     </b-list-group>
                 </b-card>
+            </b-tab>
+            <b-tab :title="$t('scripts')">
 
-        <br/>
+                <b-progress v-if="scriptsLoading" striped animated value="100"></b-progress>
+                <b-card v-else>
+                    <b-card-title>{{ $t("scripts") }}</b-card-title>
 
+                    <label>Select template</label>
+                    <b-form-radio-group stacked :options="scriptTemplates" v-model="scriptTemplate"></b-form-radio-group>
+                    <br>
+
+                    <b-row>
+                        <b-col>
+                            <b-form-input v-model="newScriptName" :disabled="!scriptTemplate" :placeholder="$t('newScriptName')"></b-form-input>
+                        </b-col>
+                        <b-col>
+                            <b-button variant="primary" @click="createScript()"
+                                      :disabled="!scriptNameValid(newScriptName)">
+                                {{ $t("create") }}
+                            </b-button>
+                        </b-col>
+                    </b-row>
+
+                    <hr/>
+
+                    <b-list-group>
+                        <UserScriptListItem v-for="script in scripts"
+                                            :key="script.name" :script="script"></UserScriptListItem>
+                    </b-list-group>
+
+                </b-card>
+            </b-tab>
+            <b-tab :title="$t('frontendTab')">
                 <b-card>
 
                     <b-card-title>{{ $t("frontends") }}</b-card-title>
@@ -56,33 +115,8 @@
                     </b-list-group>
 
                 </b-card>
-
+            </b-tab>
-        <br/>
+        </b-tabs>
-
-        <b-card>
-            <b-card-title>{{ $t("searchBackends") }}</b-card-title>
-
-            <b-row>
-                <b-col>
-                    <b-input v-model="newBackendName" :placeholder="$t('newBackendName')"></b-input>
-                </b-col>
-                <b-col>
-                    <b-button variant="primary" @click="createBackend()"
-                              :disabled="!backendNameValid(newBackendName)">
-                        {{ $t("create") }}
-                    </b-button>
-                </b-col>
-            </b-row>
-
-            <hr/>
-
-            <b-progress v-if="backendsLoading" striped animated value="100"></b-progress>
-            <b-list-group v-else>
-                <SearchBackendListItem v-for="backend in backends"
-                                  :key="backend.name" :backend="backend"></SearchBackendListItem>
-            </b-list-group>
-
-        </b-card>
     </div>
 </template>
 
@@ -92,10 +126,11 @@ import {formatBindAddress} from "@/util";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import FrontendListItem from "@/components/FrontendListItem";
 import SearchBackendListItem from "@/components/SearchBackendListItem.vue";
+import UserScriptListItem from "@/components/UserScriptListItem.vue";
 
 export default {
     name: "Jobs",
-    components: {SearchBackendListItem, JobListItem, FrontendListItem},
+    components: {UserScriptListItem, SearchBackendListItem, JobListItem, FrontendListItem},
     data() {
         return {
             jobsLoading: true,
@@ -111,11 +146,24 @@ export default {
             backendsLoading: true,
             newBackendName: "",
 
-            showHelp: false
+            scripts: [],
+            scriptTemplates: [],
+            newScriptName: "",
+            scriptTemplate: null,
+            scriptsLoading: true,
+
+            showHelp: false,
+            tab: 0
         }
     },
     mounted() {
         this.loading = true;
+        if (this.$route.params.tab) {
+            console.log("mounted " + this.$route.params.tab)
+            window.setTimeout(() => {
+                this.tab = Math.round(Number(this.$route.params.tab));
+            }, 1)
+        }
         this.reload();
     },
     methods: {
@@ -140,11 +188,20 @@ export default {
 
             return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
         },
+        scriptNameValid(name) {
+            if (this.scripts.some(script => script.name === name)) {
+                return false;
+            }
+            if (name.length > 16) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
         reload() {
             Sist2AdminApi.getJobs().then(resp => {
                 this.jobs = resp.data;
                 this.jobsLoading = false;
-
                 this.showHelp = this.jobs.length === 0;
             });
             Sist2AdminApi.getFrontends().then(resp => {
@@ -155,6 +212,11 @@ export default {
                 this.backends = resp.data;
                 this.backendsLoading = false;
             })
+            Sist2AdminApi.getUserScripts().then(resp => {
+                this.scripts = resp.data;
+                this.scriptTemplates = this.$store.state.sist2AdminInfo.user_script_templates;
+                this.scriptsLoading = false;
+            })
         },
         createJob() {
             Sist2AdminApi.createJob(this.newJobName).then(this.reload);
@@ -164,6 +226,14 @@ export default {
         },
         createBackend() {
             Sist2AdminApi.createBackend(this.newBackendName).then(this.reload);
+        },
+        createScript() {
+            Sist2AdminApi.createUserScript(this.newScriptName, this.scriptTemplate).then(this.reload)
+        },
+        onTabChange(tab) {
+            if (this.$route.params.tab != tab) {
+                this.$router.push({params: {tab: tab}})
+            }
         }
     }
 }

sist2-admin/frontend/src/views/Job.vue

@@ -6,7 +6,19 @@
         </b-card-title>
 
         <div class="mb-3">
-            <b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
+
+            <b-dropdown
+                    split
+                    split-variant="primary"
+                    variant="primary"
+                    :text="$t('runNow')"
+                    class="mr-1"
+                    :disabled="!valid"
+                    @click="runJob()"
+            >
+                <b-dropdown-item href="#" @click="runJob(true)">{{ $t("runNowFull") }}</b-dropdown-item>
+            </b-dropdown>
+
             <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
         </div>
 
@@ -24,19 +36,26 @@
 
             <br/>
 
-            <h4>{{ $t("scanOptions.title") }}</h4>
-            <b-card>
-                <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
-            </b-card>
-
-            <br/>
-
             <h4>{{ $t("backendOptions.title") }}</h4>
             <b-card>
                 <b-alert v-if="!valid" variant="warning" show>{{ $t("jobOptions.noBackendError") }}</b-alert>
                 <SearchBackendSelect :value="job.index_options.search_backend"
                                      @change="onBackendSelect($event)"></SearchBackendSelect>
             </b-card>
+            <br/>
+
+            <h4>{{ $t("scriptOptions") }}</h4>
+            <b-card>
+                <UserScriptPicker :selected-scripts="job.user_scripts"
+                                  @change="onScriptChange($event)"></UserScriptPicker>
+            </b-card>
+
+            <br/>
+
+            <h4>{{ $t("scanOptions.title") }}</h4>
+            <b-card>
+                <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
+            </b-card>
 
         </b-card-body>
 
@@ -48,10 +67,12 @@ import ScanOptions from "@/components/ScanOptions";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import JobOptions from "@/components/JobOptions";
 import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
+import UserScriptPicker from "@/components/UserScriptPicker.vue";
 
 export default {
     name: "Job",
     components: {
+        UserScriptPicker,
         SearchBackendSelect,
         ScanOptions,
         JobOptions
@@ -60,6 +81,7 @@ export default {
         return {
             loading: true,
             job: null,
+            console: console
         }
     },
     methods: {
@@ -69,8 +91,8 @@ export default {
         update() {
             Sist2AdminApi.updateJob(this.getName(), this.job);
         },
-        runJob() {
+        runJob(full = false) {
-            Sist2AdminApi.runJob(this.getName()).then(() => {
+            Sist2AdminApi.runJob(this.getName(), full).then(() => {
                 this.$bvToast.toast(this.$t("runJobConfirmation"), {
                     title: this.$t("runJobConfirmationTitle"),
                     variant: "success",
@@ -95,6 +117,10 @@ export default {
         onBackendSelect(backend) {
             this.job.index_options.search_backend = backend;
             this.update();
+        },
+        onScriptChange(scripts) {
+            this.job.user_scripts = scripts;
+            this.update();
         }
     },
     mounted() {

sist2-admin/frontend/src/views/SearchBackend.vue

@@ -44,9 +44,6 @@
 
                 <label>{{ $t("backendOptions.batchSize") }}</label>
                 <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
-
-                <label>{{ $t("backendOptions.script") }}</label>
-                <b-form-textarea v-model="backend.script" rows="6" @change="update()"></b-form-textarea>
             </template>
             <template v-else>
                 <label>{{ $t("backendOptions.searchIndex") }}</label>

sist2-admin/frontend/src/views/Tail.vue

@@ -92,6 +92,9 @@ export default {
                     if ("stderr" in message) {
                         message.level = "ERROR";
                         message.message = message["stderr"];
+                    } else if ("stdout" in message) {
+                        message.level = "INFO";
+                        message.message = message["stdout"];
                     } else {
                         message.level = "ADMIN";
                         message.message = message["sist2-admin"];
@@ -167,6 +170,6 @@ span.ADMIN {
     margin: 3px;
     white-space: pre;
     color: #000;
-    overflow: hidden;
+    overflow-y: hidden;
 }
 </style>

sist2-admin/frontend/src/views/Tasks.vue

@@ -81,7 +81,7 @@ function humanDuration(sec_num) {
         return `${seconds}s`;
     }
 
-    return "<0s";
+    return "<1s";
 }
 
 export default {
@@ -134,7 +134,7 @@ export default {
                     duration: this.taskDuration(row),
                     time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"),
                     logs: null,
-                    status: [0,1].includes(row.return_code) ? "ok" : "failed",
+                    status: row.return_code === 0 ? "ok" : "failed",
                     _row: row
                 }));
             });

sist2-admin/frontend/src/views/UserScript.vue

@@ -0,0 +1,117 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-card v-else>
+        <b-card-title>
+            {{ $route.params.name }}
+            {{ $t("script") }}
+        </b-card-title>
+
+        <div class="mb-3">
+            <b-button variant="danger" @click="deleteScript()">{{ $t("delete") }}</b-button>
+        </div>
+
+        <b-card>
+            <h5>{{ $t("testScript") }}</h5>
+
+            <b-row>
+                <b-col cols="11">
+                    <JobSelect @change="onJobSelect($event)"></JobSelect>
+                </b-col>
+                <b-col cols="1">
+                    <b-button :disabled="!selectedTestJob" variant="primary" @click="testScript()">{{ $t("test") }}
+                    </b-button>
+                </b-col>
+            </b-row>
+
+        </b-card>
+        <br/>
+
+        <label>{{ $t("scriptType") }}</label>
+        <b-form-select :options="['git', 'simple']" v-model="script.type" @change="update()"></b-form-select>
+
+        <template v-if="script.type === 'git'">
+            <label>{{ $t("gitRepository") }}</label>
+            <b-form-input v-model="script.git_repository" placeholder="https://github.com/example/example.git"
+                          @change="update()"></b-form-input>
+
+            <label>{{ $t("extraArgs") }}</label>
+            <b-form-input v-model="script.extra_args" @change="update()" class="text-monospace"></b-form-input>
+        </template>
+
+        <template v-if="script.type === 'simple'">
+
+            <label>{{ $t("scriptCode") }}</label>
+            <p>Find sist2-python documentation <a href="https://sist2-python.readthedocs.io/" target="_blank">here</a></p>
+            <b-textarea rows="15" class="text-monospace" v-model="script.script" @change="update()" spellcheck="false"></b-textarea>
+        </template>
+
+        <template v-if="script.type === 'local'">
+            <!-- TODO-->
+        </template>
+
+
+    </b-card>
+</template>
+
+<script>
+
+import Sist2AdminApi from "@/Sist2AdminApi";
+import JobOptions from "@/components/JobOptions.vue";
+import JobCheckboxGroup from "@/components/JobCheckboxGroup.vue";
+import JobSelect from "@/components/JobSelect.vue";
+
+export default {
+    name: "UserScript",
+    components: {JobSelect, JobCheckboxGroup, JobOptions},
+    data() {
+        return {
+            loading: true,
+            script: null,
+            selectedTestJob: null
+        }
+    },
+    methods: {
+        update() {
+            Sist2AdminApi.updateUserScript(this.name, this.script);
+        },
+        onJobSelect(job) {
+            this.selectedTestJob = job;
+        },
+        deleteScript() {
+            Sist2AdminApi.deleteUserScript(this.name)
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete user script " +
+                        "because it is referenced by a job", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        },
+        testScript() {
+            Sist2AdminApi.testUserScript(this.name, this.selectedTestJob)
+                .then(() => {
+                    this.$bvToast.toast(this.$t("runJobConfirmation"), {
+                        title: this.$t("runJobConfirmationTitle"),
+                        variant: "success",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getUserScript(this.name).then(resp => {
+            this.script = resp.data;
+            this.loading = false;
+        });
+    },
+    computed: {
+        name() {
+            return this.$route.params.name;
+        },
+    },
+}
+</script>

sist2-admin/requirements.txt

@@ -3,3 +3,5 @@ git+https://github.com/simon987/hexlib.git
 uvicorn
 websockets
 pycron
+GitPython
+git+https://github.com/sist2app/sist2-python.git@2.1

sist2-admin/sist2_admin/app.py

@@ -2,6 +2,7 @@ import asyncio
 import os
 import signal
 from datetime import datetime
+from time import sleep
 from urllib.parse import urlparse
 
 import requests
@@ -18,12 +19,14 @@ from websockets.exceptions import ConnectionClosed
 
 import cron
 from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
-from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
+from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus, Sist2UserScriptTask
 from notifications import Subscribe, Notifications
 from sist2 import Sist2, Sist2SearchBackend
 from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION, migrate_v3_to_v4, \
     get_log_files_to_remove, delete_log_file, create_default_search_backends
 from web import Sist2Frontend
+from script import UserScript, SCRIPT_TEMPLATES
+from util import tail_sync, pid_is_running
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
 db = PersistentState(dbfile=os.path.join(DATA_FOLDER, "state.db"))
@@ -52,7 +55,8 @@ async def home():
 async def api():
     return {
         "tesseract_langs": TESSERACT_LANGS,
-        "logs_folder": LOG_FOLDER
+        "logs_folder": LOG_FOLDER,
+        "user_script_templates": list(SCRIPT_TEMPLATES.keys())
     }
 
 
@@ -74,7 +78,7 @@ async def get_frontend(name: str):
     raise HTTPException(status_code=404)
 
 
-@app.get("/api/job/")
+@app.get("/api/job")
 async def get_jobs():
     return list(db["jobs"])
 
@@ -113,12 +117,10 @@ async def update_job(name: str, new_job: Sist2Job):
 async def update_frontend(name: str, frontend: Sist2Frontend):
     db["frontends"][name] = frontend
 
-    # TODO: Check etag
-
     return "ok"
 
 
-@app.get("/api/task/")
+@app.get("/api/task")
 async def get_tasks():
     return list(map(lambda t: t.json(), task_queue.tasks()))
 
@@ -150,23 +152,54 @@ def _run_job(job: Sist2Job):
     db["jobs"][job.name] = job
 
     scan_task = Sist2ScanTask(job, f"Scan [{job.name}]")
-    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=scan_task)
+
+    index_depends_on = scan_task
+    script_tasks = []
+    for script_name in job.user_scripts:
+        script = db["user_scripts"][script_name]
+
+        task = Sist2UserScriptTask(script, job, f"Script <{script_name}> [{job.name}]", depends_on=scan_task)
+        script_tasks.append(task)
+        index_depends_on = task
+
+    index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=index_depends_on)
 
     task_queue.submit(scan_task)
+    for task in script_tasks:
+        task_queue.submit(task)
     task_queue.submit(index_task)
 
 
 @app.get("/api/job/{name:str}/run")
-async def run_job(name: str):
+async def run_job(name: str, full: bool = False):
-    job = db["jobs"][name]
+    job: Sist2Job = db["jobs"][name]
     if not job:
         raise HTTPException(status_code=404)
 
+    if full:
+        job.do_full_scan = True
+
     _run_job(job)
 
     return "ok"
 
 
+@app.get("/api/user_script/{name:str}/run")
+def run_user_script(name: str, job: str):
+    script = db["user_scripts"][name]
+    if not script:
+        raise HTTPException(status_code=404)
+    job = db["jobs"][job]
+    if not job:
+        raise HTTPException(status_code=404)
+
+    script_task = Sist2UserScriptTask(script, job, f"Script <{name}> [{job.name}]")
+
+    task_queue.submit(script_task)
+
+    return "ok"
+
+
 @app.get("/api/job/{name:str}/logs_to_delete")
 async def task_history(n: int, name: str):
     return get_log_files_to_remove(db, name, n)
@@ -194,7 +227,10 @@ async def delete_job(name: str):
 @app.delete("/api/frontend/{name:str}")
 async def delete_frontend(name: str):
     if name in RUNNING_FRONTENDS:
+        try:
             os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        except ProcessLookupError:
+            pass
         del RUNNING_FRONTENDS[name]
 
     frontend = db["frontends"][name]
@@ -239,7 +275,7 @@ def check_es_version(es_url: str, insecure: bool):
             es_url = f"{url.scheme}://{url.hostname}:{url.port}"
         else:
             auth = None
-        r = requests.get(es_url, verify=insecure, auth=auth)
+        r = requests.get(es_url, verify=not insecure, auth=auth)
     except SSLError:
         return {
             "ok": False,
@@ -290,7 +326,18 @@ def start_frontend_(frontend: Sist2Frontend):
     logger.debug(f"Fetched search backend options for {backend_name}")
 
     pid = sist2.web(frontend.web_options, search_backend, frontend.name)
+
+    sleep(0.2)
+    if not pid_is_running(pid):
+        frontend_log = frontend.get_log_path(LOG_FOLDER)
+        logger.error(f"Frontend exited too quickly, check {frontend_log} for more details:")
+        for line in tail_sync(frontend.get_log_path(LOG_FOLDER), 3):
+            logger.error(line.strip())
+
+        return False
+
     RUNNING_FRONTENDS[frontend.name] = pid
+    return True
 
 
 @app.post("/api/frontend/{name:str}/start")
@@ -299,7 +346,12 @@ async def start_frontend(name: str):
     if not frontend:
         raise HTTPException(status_code=404)
 
-    start_frontend_(frontend)
+    ok = start_frontend_(frontend)
+
+    if not ok:
+        raise HTTPException(status_code=500)
+
+    return "ok"
 
 
 @app.post("/api/frontend/{name:str}/stop")
@@ -309,7 +361,7 @@ async def stop_frontend(name: str):
         del RUNNING_FRONTENDS[name]
 
 
-@app.get("/api/frontend/")
+@app.get("/api/frontend")
 async def get_frontends():
     res = []
     for frontend in db["frontends"]:
@@ -319,7 +371,7 @@ async def get_frontends():
     return res
 
 
-@app.get("/api/search_backend/")
+@app.get("/api/search_backend")
 async def get_search_backends():
     return list(db["search_backends"])
 
@@ -375,6 +427,59 @@ def create_search_backend(name: str):
     return backend
 
 
+@app.delete("/api/user_script/{name:str}")
+def delete_user_script(name: str):
+    if db["user_scripts"][name] is None:
+        return HTTPException(status_code=404)
+
+    if any(name in job.user_scripts for job in db["jobs"]):
+        raise HTTPException(status_code=400, detail="in use (job)")
+
+    script: UserScript = db["user_scripts"][name]
+    script.delete_dir()
+
+    del db["user_scripts"][name]
+
+    return "ok"
+
+
+@app.post("/api/user_script/{name:str}")
+def create_user_script(name: str, template: str):
+    if db["user_scripts"][name] is not None:
+        return HTTPException(status_code=400, detail="already exists")
+
+    script = SCRIPT_TEMPLATES[template](name)
+    db["user_scripts"][name] = script
+
+    return script
+
+
+@app.get("/api/user_script")
+async def get_user_scripts():
+    return list(db["user_scripts"])
+
+
+@app.get("/api/user_script/{name:str}")
+async def get_user_script(name: str):
+    backend = db["user_scripts"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    return backend
+
+
+@app.put("/api/user_script/{name:str}")
+async def update_user_script(name: str, script: UserScript):
+    previous_version: UserScript = db["user_scripts"][name]
+
+    if previous_version and previous_version.git_repository != script.git_repository:
+        script.force_clone = True
+
+    db["user_scripts"][name] = script
+
+    return "ok"
+
+
 def tail(filepath: str, n: int):
     with open(filepath) as file:
 
@@ -479,7 +584,8 @@ if __name__ == '__main__':
         migrate_v3_to_v4(db)
 
     if db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
-        raise Exception(f"Incompatible database version for {db.dbfile}")
+        raise Exception(f"Incompatible database {db.dbfile}. "
+                        f"Automatic migration is not available, please delete the database file to continue.")
 
     start_frontends()
     cron.initialize(db, _run_job)

sist2-admin/sist2_admin/config.py

@@ -9,9 +9,11 @@ MAX_LOG_SIZE = 1 * 1024 * 1024
 SIST2_BINARY = os.environ.get("SIST2_BINARY", "/root/sist2")
 DATA_FOLDER = os.environ.get("DATA_FOLDER", "/sist2-admin/")
 LOG_FOLDER = os.path.join(DATA_FOLDER, "logs")
+SCRIPT_FOLDER = os.path.join(DATA_FOLDER, "scripts")
 WEBSERVER_PORT = 8080
 
 os.makedirs(LOG_FOLDER, exist_ok=True)
+os.makedirs(SCRIPT_FOLDER, exist_ok=True)
 os.makedirs(DATA_FOLDER, exist_ok=True)
 
 logger = logging.Logger("sist2-admin")

sist2-admin/sist2_admin/jobs.py

@@ -1,13 +1,18 @@
 import json
 import logging
 import os.path
+import shlex
 import signal
 import uuid
 from datetime import datetime
 from enum import Enum
+from io import TextIOWrapper
 from logging import FileHandler
+from subprocess import Popen
+import subprocess
 from threading import Lock, Thread
 from time import sleep
+from typing import List
 from uuid import uuid4, UUID
 
 from hexlib.db import PersistentState
@@ -18,6 +23,7 @@ from notifications import Notifications
 from sist2 import ScanOptions, IndexOptions, Sist2
 from state import RUNNING_FRONTENDS, get_log_files_to_remove, delete_log_file
 from web import Sist2Frontend
+from script import UserScript
 
 
 class JobStatus(Enum):
@@ -32,6 +38,8 @@ class Sist2Job(BaseModel):
     scan_options: ScanOptions
     index_options: IndexOptions
 
+    user_scripts: List[str] = []
+
     cron_expression: str
     schedule_enabled: bool = False
 
@@ -112,6 +120,10 @@ class Sist2Task:
 
         logger.info(f"Started task {self.display_name}")
 
+    def set_pid(self, pid):
+        self.pid = pid
+
+
 
 class Sist2ScanTask(Sist2Task):
 
@@ -125,13 +137,10 @@ class Sist2ScanTask(Sist2Task):
         else:
             self.job.scan_options.output = None
 
-        def set_pid(pid):
+        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
-            self.pid = pid
-
-        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
         self.ended = datetime.utcnow()
 
-        is_ok = return_code in (0, 1)
+        is_ok = (return_code in (0, 1)) if "debug" in sist2.bin_path else (return_code == 0)
 
         if not is_ok:
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
@@ -157,6 +166,9 @@ class Sist2ScanTask(Sist2Task):
             self.job.previous_index_path = self.job.index_path
             db["jobs"][self.job.name] = self.job
 
+        if is_ok:
+            return 0
+
         return return_code
 
 
@@ -177,12 +189,12 @@ class Sist2IndexTask(Sist2Task):
 
         logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
 
-        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback)
+        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
         self.ended = datetime.utcnow()
 
         duration = self.ended - self.started
 
-        ok = return_code == 0
+        ok = return_code in (0, 1)
 
         if ok:
             self.restart_running_frontends(db, sist2)
@@ -192,7 +204,7 @@ class Sist2IndexTask(Sist2Task):
         self.job.previous_index_path = self.job.index_path
         db["jobs"][self.job.name] = self.job
 
-        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
+        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}, {ok=}"}))
 
         logger.info(f"Completed {self.display_name} ({return_code=})")
 
@@ -231,6 +243,65 @@ class Sist2IndexTask(Sist2Task):
             self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))
 
 
+class Sist2UserScriptTask(Sist2Task):
+
+    def __init__(self, user_script: UserScript, job: Sist2Job, display_name: str, depends_on: Sist2Task = None):
+        super().__init__(job, display_name, depends_on=depends_on.id if depends_on else None)
+        self.user_script = user_script
+
+    def run(self, sist2: Sist2, db: PersistentState):
+        super().run(sist2, db)
+
+        try:
+            self.user_script.setup(self.log_callback, self.set_pid)
+        except Exception as e:
+            logger.error(f"Setup for {self.user_script.name} failed: ")
+            logger.exception(e)
+            self.log_callback({"sist2-admin": f"Setup for {self.user_script.name} failed: {e}"})
+            return -1
+
+        executable = self.user_script.get_executable()
+        index_path = os.path.join(DATA_FOLDER, self.job.index_path)
+        extra_args = self.user_script.extra_args
+
+        args = [
+            executable,
+            index_path,
+            *shlex.split(extra_args)
+        ]
+
+        self.log_callback({"sist2-admin": f"Starting user script with {executable=}, {index_path=}, {extra_args=}"})
+
+        proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.user_script.script_dir())
+        self.set_pid(proc.pid)
+
+        t_stderr = Thread(target=self._consume_logs, args=(self.log_callback, proc, "stderr", False))
+        t_stderr.start()
+
+        self._consume_logs(self.log_callback, proc, "stdout", True)
+
+        self.ended = datetime.utcnow()
+
+        return 0
+
+    @staticmethod
+    def _consume_logs(logs_cb, proc, stream, wait):
+        pipe_wrapper = TextIOWrapper(getattr(proc, stream), encoding="utf8", errors="ignore")
+        try:
+            for line in pipe_wrapper:
+                if line.strip() == "":
+                    continue
+                if line.startswith("$PROGRESS"):
+                    progress = json.loads(line[len("$PROGRESS "):])
+                    logs_cb({"progress": progress})
+                    continue
+                logs_cb({stream: line})
+        finally:
+            if wait:
+                proc.wait()
+            pipe_wrapper.close()
+
+
 class TaskQueue:
     def __init__(self, sist2: Sist2, db: PersistentState, notifications: Notifications):
         self._lock = Lock()
@@ -249,7 +320,7 @@ class TaskQueue:
     def _tasks_failed(self):
         done = set()
 
-        for row in self._db["task_done"].sql("WHERE return_code NOT IN (0,1)"):
+        for row in self._db["task_done"].sql("WHERE return_code != 0"):
             done.add(uuid.UUID(row["id"]))
 
         return done

sist2-admin/sist2_admin/script.py

@@ -0,0 +1,130 @@
+import os
+import shutil
+import stat
+import subprocess
+from enum import Enum
+
+from git import Repo
+from pydantic import BaseModel
+
+from config import SCRIPT_FOLDER
+
+
+class ScriptType(Enum):
+    LOCAL = "local"
+    SIMPLE = "simple"
+    GIT = "git"
+
+
+def set_executable(file):
+    os.chmod(file, os.stat(file).st_mode | stat.S_IEXEC)
+
+
+def _initialize_git_repository(url, path, log_cb, force_clone, set_pid_cb):
+    log_cb({"sist2-admin": f"Cloning {url}"})
+
+    if force_clone or not os.path.exists(os.path.join(path, ".git")):
+        if force_clone:
+            shutil.rmtree(path, ignore_errors=True)
+        Repo.clone_from(url, path)
+    else:
+        repo = Repo(path)
+        repo.remote("origin").pull()
+
+    setup_script = os.path.join(path, "setup.sh")
+    if setup_script:
+        log_cb({"sist2-admin": f"Executing setup script {setup_script}"})
+
+        set_executable(setup_script)
+        proc = subprocess.Popen([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        set_pid_cb(proc.pid)
+        proc.wait()
+        stdout = proc.stdout.read()
+
+        for line in stdout.split(b"\n"):
+            if line:
+                log_cb({"stdout": line.decode()})
+
+        log_cb({"stdout": f"Executed setup script {setup_script}, return code = {proc.returncode}"})
+
+        if proc.returncode != 0:
+            raise Exception("Error when running setup script!")
+
+    log_cb({"sist2-admin": f"Initialized git repository in {path}"})
+
+
+class UserScript(BaseModel):
+    name: str
+    type: ScriptType
+    git_repository: str = None
+    force_clone: bool = False
+    script: str = None
+    extra_args: str = ""
+
+    def script_dir(self):
+        return os.path.join(SCRIPT_FOLDER, self.name)
+
+    def setup(self, log_cb, set_pid_cb):
+        os.makedirs(self.script_dir(), exist_ok=True)
+
+        if self.type == ScriptType.GIT:
+            _initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone, set_pid_cb)
+            self.force_clone = False
+        elif self.type == ScriptType.SIMPLE:
+            self._setup_simple()
+
+        set_executable(self.get_executable())
+
+    def _setup_simple(self):
+        with open(self.get_executable(), "w") as f:
+            f.write(
+                "#!/bin/bash\n"
+                "python run.py \"$@\""
+            )
+
+        with open(os.path.join(self.script_dir(), "run.py"), "w") as f:
+            f.write(self.script)
+
+    def get_executable(self):
+        return os.path.join(self.script_dir(), "run.sh")
+
+    def delete_dir(self):
+        shutil.rmtree(self.script_dir(), ignore_errors=True)
+
+
+SCRIPT_TEMPLATES = {
+    "CLIP - Generate embeddings to predict the most relevant image based on the text prompt": lambda name: UserScript(
+        name=name,
+        type=ScriptType.GIT,
+        git_repository="https://github.com/sist2app/sist2-script-clip",
+        extra_args="--num-tags=1 --tags-file=general.txt --color=#dcd7ff"
+    ),
+    "Whisper - Speech to text with OpenAI Whisper": lambda name: UserScript(
+        name=name,
+        type=ScriptType.GIT,
+        git_repository="https://github.com/simon987/sist2-script-whisper",
+        extra_args="--model=base --num-threads=4 --color=#51da4c --tag"
+    ),
+    "Hamburger - Simple script example": lambda name: UserScript(
+        name=name,
+        type=ScriptType.SIMPLE,
+        script=
+        'from sist2 import Sist2Index\n'
+        'import sys\n'
+        '\n'
+        'index = Sist2Index(sys.argv[1])\n'
+        'for doc in index.document_iter():\n'
+        '    doc.json_data["tag"] = ["hamburger.#00FF00"]\n'
+        '    index.update_document(doc)\n'
+        '\n'
+        'index.sync_tag_table()\n'
+        'index.commit()\n'
+        '\n'
+        'print("Done!")\n'
+    ),
+    "(Blank)": lambda name: UserScript(
+        name=name,
+        type=ScriptType.SIMPLE,
+        script=""
+    )
+}

sist2-admin/sist2_admin/sist2.py

@@ -2,10 +2,11 @@ import datetime
 import json
 import logging
 import os.path
+import sys
 from datetime import datetime
 from enum import Enum
 from io import TextIOWrapper
-from logging import FileHandler
+from logging import FileHandler, StreamHandler
 from subprocess import Popen, PIPE
 from tempfile import NamedTemporaryFile
 from threading import Thread
@@ -41,8 +42,6 @@ class Sist2SearchBackend(BaseModel):
     es_insecure_ssl: bool = False
     es_index: str = "sist2"
     threads: int = 1
-    script: str = ""
-    script_file: str = None
     batch_size: int = 70
 
     @staticmethod
@@ -74,8 +73,6 @@ class IndexOptions(BaseModel):
                     f"--es-index={search_backend.es_index}",
                     f"--batch-size={search_backend.batch_size}"]
 
-            if search_backend.script_file:
-                args.append(f"--script-file={search_backend.script_file}")
             if search_backend.es_insecure_ssl:
                 args.append(f"--es-insecure-ssl")
             if self.incremental_index:
@@ -204,6 +201,7 @@ class WebOptions(BaseModel):
     auth0_client_id: str = None
     auth0_public_key: str = None
     auth0_public_key_file: str = None
+    verbose: bool = False
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -235,6 +233,8 @@ class WebOptions(BaseModel):
             args.append(f"--tag-auth={self.tag_auth}")
         if self.dev:
             args.append(f"--dev")
+        if self.verbose:
+            args.append(f"--very-verbose")
 
         args.extend(self.indices)
 
@@ -247,14 +247,7 @@ class Sist2:
         self.bin_path = bin_path
         self._data_dir = data_directory
 
-    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
+    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb, set_pid_cb):
-
-        if search_backend.script and search_backend.backend_type == SearchBackendType("elasticsearch"):
-            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
-                f.write(search_backend.script)
-            search_backend.script_file = f.name
-        else:
-            search_backend.script_file = None
 
         args = [
             self.bin_path,
@@ -266,7 +259,9 @@ class Sist2:
         logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        set_pid_cb(proc.pid)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
         t_stderr.start()
 
         self._consume_logs_stdout(logs_cb, proc)
@@ -293,7 +288,7 @@ class Sist2:
 
         set_pid_cb(proc.pid)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
         t_stderr.start()
 
         self._consume_logs_stdout(logs_cb, proc)
@@ -303,7 +298,7 @@ class Sist2:
         return proc.returncode
 
     @staticmethod
-    def _consume_logs_stderr(logs_cb, proc):
+    def _consume_logs_stderr(logs_cb, exit_cb, proc):
         pipe_wrapper = TextIOWrapper(proc.stderr, encoding="utf8", errors="ignore")
         try:
             for line in pipe_wrapper:
@@ -311,7 +306,9 @@ class Sist2:
                     continue
                 logs_cb({"stderr": line})
         finally:
-            proc.wait()
+            return_code = proc.wait()
+            if exit_cb:
+                exit_cb(return_code)
             pipe_wrapper.close()
 
     @staticmethod
@@ -345,15 +342,19 @@ class Sist2:
 
         web_logger = logging.Logger(name=f"sist2-frontend-{name}")
         web_logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"frontend-{name}.log")))
+        web_logger.addHandler(StreamHandler())
 
         def logs_cb(message):
             web_logger.info(json.dumps(message))
 
+        def exit_cb(return_code):
+            logger.info(f"Web frontend exited with return code {return_code}")
+
         logger.info(f"Starting frontend {' '.join(args)}")
 
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, exit_cb, proc))
         t_stderr.start()
 
         t_stdout = Thread(target=self._consume_logs_stdout, args=(logs_cb, proc))

sist2-admin/sist2_admin/state.py

@@ -14,7 +14,7 @@ RUNNING_FRONTENDS: Dict[str, int] = {}
 
 TESSERACT_LANGS = get_tesseract_langs()
 
-DB_SCHEMA_VERSION = "4"
+DB_SCHEMA_VERSION = "5"
 
 from pydantic import BaseModel
 

sist2-admin/sist2_admin/util.py

@@ -0,0 +1,41 @@
+from glob import glob
+import os
+from config import DATA_FOLDER
+
+
+def get_old_index_files(name):
+    files = glob(os.path.join(DATA_FOLDER, f"scan-{name.replace('/', '_')}-*.sist2"))
+    files = list(sorted(files, key=lambda f: os.stat(f).st_mtime))
+    files = files[-1:]
+
+    return files
+
+
+def tail_sync(filename, lines=1, _buffer=4098):
+    with open(filename) as f:
+        lines_found = []
+
+        block_counter = -1
+
+        while len(lines_found) < lines:
+            try:
+                f.seek(block_counter * _buffer, os.SEEK_END)
+            except IOError:
+                f.seek(0)
+                lines_found = f.readlines()
+                break
+
+            lines_found = f.readlines()
+
+            block_counter -= 1
+
+        return lines_found[-lines:]
+
+
+def pid_is_running(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+
+    return True

sist2-vue/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sist2",
-  "version": "2.11.0",
+  "version": "1.0.0",
   "private": true,
   "scripts": {
     "serve": "vue-cli-service serve",
@@ -9,15 +9,15 @@
   "dependencies": {
     "@auth0/auth0-spa-js": "^2.0.2",
     "@egjs/vue-infinitegrid": "3.3.0",
-    "@tensorflow/tfjs": "^4.4.0",
+    "axios": "^1.6.0",
-    "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "d3": "^7.8.4",
+    "d3": "^5.6.1",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",
     "fslightbox-vue": "fslightbox-vue.tgz",
     "nouislider": "^15.2.0",
+    "onnxruntime-web": "1.15.1",
     "underscore": "^1.13.1",
     "vue": "^2.6.12",
     "vue-color": "^2.8.1",
@@ -29,9 +29,9 @@
   },
   "devDependencies": {
     "@babel/polyfill": "^7.12.1",
+    "@types/underscore": "^1.11.6",
     "@vue/cli-plugin-babel": "~5.0.8",
     "@vue/cli-plugin-router": "~5.0.8",
-    "@vue/cli-plugin-typescript": "^5.0.8",
     "@vue/cli-plugin-vuex": "~5.0.8",
     "@vue/cli-service": "^5.0.8",
     "@vue/test-utils": "^1.0.3",
@@ -43,8 +43,7 @@
     "portal-vue": "^2.1.7",
     "sass": "^1.26.11",
     "sass-loader": "^10.0.2",
-    "typescript": "~4.1.5",
+    "vue-cli-plugin-bootstrap-vue": "~0.8.2",
-    "vue-cli-plugin-bootstrap-vue": "~0.7.0",
     "vue-template-compiler": "^2.6.11"
   },
   "browserslist": [

sist2-vue/src/App.vue

@@ -308,15 +308,21 @@ html, body {
 
 .info-icon {
     width: 1rem;
+    min-width: 1rem;
     margin-right: 0.2rem;
     cursor: pointer;
     line-height: 1rem;
     height: 1rem;
+    min-height: 1rem;
     background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
     filter: brightness(45%);
     display: block;
 }
 
+.theme-black .info-icon {
+    filter: brightness(80%);
+}
+
 .tabs {
     margin-top: 10px;
 }

sist2-vue/src/Sist2Api.js

@@ -1,115 +1,20 @@
 import axios from "axios";
-import {ext, strUnescape, lum} from "./util";
+import {strUnescape, lum, sid} from "./util";
 import Sist2Query from "@/Sist2ElasticsearchQuery";
 import store from "@/store";
 
-export interface EsTag {
-    id: string
-    count: number
-    color: string | undefined
-    isLeaf: boolean
-}
-
-export interface Tag {
-    style: string
-    text: string
-    rawText: string
-    fg: string
-    bg: string
-    userTag: boolean
-}
-
-export interface Index {
-    name: string
-    version: string
-    id: string
-    idPrefix: string
-    timestamp: number
-}
-
-export interface EsHit {
-    _index: string
-    _id: string
-    _score: number
-    _type: string
-    _tags: Tag[]
-    _seq: number
-    _source: {
-        path: string
-        size: number
-        mime: string
-        name: string
-        extension: string
-        index: string
-        _depth: number
-        mtime: number
-        videoc: string
-        audioc: string
-        parent: string
-        width: number
-        height: number
-        duration: number
-        tag: string[]
-        checksum: string
-        thumbnail: string
-    }
-    _props: {
-        isSubDocument: boolean
-        isImage: boolean
-        isGif: boolean
-        isVideo: boolean
-        isPlayableVideo: boolean
-        isPlayableImage: boolean
-        isAudio: boolean
-        hasThumbnail: boolean
-        hasVidPreview: boolean
-        imageAspectRatio: number
-        /** Number of thumbnails available */
-        tnNum: number
-    }
-    highlight: {
-        name: string[] | undefined,
-        content: string[] | undefined,
-    }
-}
-
-function getIdPrefix(indices: Index[], id: string): string {
-    for (let i = 4; i < 32; i++) {
-        const prefix = id.slice(0, i);
-
-        if (indices.filter(idx => idx.id.slice(0, i) == prefix).length == 1) {
-            return prefix;
-        }
-    }
-
-    return id;
-}
-
-export interface EsResult {
-    took: number
-
-    hits: {
-        // TODO: ES 6.X ?
-        total: {
-            value: number
-        }
-        hits: EsHit[]
-    }
-
-    aggregations: any
-}
 
 class Sist2Api {
 
-    private readonly baseUrl: string
+    baseUrl;
-    private sist2Info: any
+    sist2Info;
-    private queryfunc: () => EsResult;
+    queryfunc;
 
-    constructor(baseUrl: string) {
+    constructor(baseUrl) {
         this.baseUrl = baseUrl;
     }
 
-    init(queryFunc: () => EsResult) {
+    init(queryFunc) {
         this.queryfunc = queryFunc;
     }
 
@@ -117,28 +22,25 @@ class Sist2Api {
         return this.sist2Info.searchBackend;
     }
 
-    getSist2Info(): Promise<any> {
+    models() {
+        const allModels = this.sist2Info.indices
+            .map(idx => idx.models)
+            .flat();
+
+        return allModels
+            .filter((v, i, a) => a.findIndex(v2 => (v2.id === v.id)) === i)
+    }
+
+    getSist2Info() {
         return axios.get(`${this.baseUrl}i`).then(resp => {
-            const indices = resp.data.indices as Index[];
-
-            resp.data.indices = indices.map(idx => {
-                return {
-                    id: idx.id,
-                    name: idx.name,
-                    timestamp: idx.timestamp,
-                    version: idx.version,
-                    idPrefix: getIdPrefix(indices, idx.id)
-                } as Index;
-            });
-
             this.sist2Info = resp.data;
 
             return resp.data;
         })
     }
 
-    setHitProps(hit: EsHit): void {
+    setHitProps(hit) {
-        hit["_props"] = {} as any;
+        hit["_props"] = {};
 
         const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
 
@@ -146,7 +48,7 @@ class Sist2Api {
             hit._props.isSubDocument = true;
         }
 
-        if ("thumbnail" in hit._source) {
+        if ("thumbnail" in hit._source && hit._source.thumbnail > 0) {
             hit._props.hasThumbnail = true;
 
             if (Number.isNaN(Number(hit._source.thumbnail))) {
@@ -202,8 +104,8 @@ class Sist2Api {
         }
     }
 
-    setHitTags(hit: EsHit): void {
+    setHitTags(hit) {
-        const tags = [] as Tag[];
+        const tags = [];
 
         // User tags
         if ("tag" in hit._source) {
@@ -215,10 +117,10 @@ class Sist2Api {
         hit._tags = tags;
     }
 
-    createUserTag(tag: string): Tag {
+    createUserTag(tag) {
         const tokens = tag.split(".");
 
-        const colorToken = tokens.pop() as string;
+        const colorToken = tokens.pop();
 
         const bg = colorToken;
         const fg = lum(colorToken) > 50 ? "#000" : "#fff";
@@ -230,25 +132,30 @@ class Sist2Api {
             text: tokens.join("."),
             rawText: tag,
             userTag: true,
-        } as Tag;
+        };
     }
 
-    search(): Promise<EsResult> {
+    search() {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.ftsQuery(this.queryfunc())
         } else {
             return this.esQuery(this.queryfunc());
         }
     }
 
-    esQuery(query: any): Promise<EsResult> {
+    _getIndexRoot(indexId) {
+        return this.sist2Info.indices.find(idx => idx.id === indexId).root;
+    }
+
+    esQuery(query) {
         return axios.post(`${this.baseUrl}es`, query).then(resp => {
-            const res = resp.data as EsResult;
+            const res = resp.data;
 
             if (res.hits?.hits) {
-                res.hits.hits.forEach((hit: EsHit) => {
+                res.hits.hits.forEach((hit) => {
                     hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
                     hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+                    hit["_source"]["indexRoot"] = this._getIndexRoot(hit["_source"]["index"]);
 
                     this.setHitProps(hit);
                     this.setHitTags(hit);
@@ -259,9 +166,9 @@ class Sist2Api {
         });
     }
 
-    ftsQuery(query: any): Promise<EsResult> {
+    ftsQuery(query) {
         return axios.post(`${this.baseUrl}fts/search`, query).then(resp => {
-            const res = resp.data as any;
+            const res = resp.data;
 
             if (res.hits.hits) {
                 res.hits.hits.forEach(hit => {
@@ -282,7 +189,7 @@ class Sist2Api {
         });
     }
 
-    private getMimeTypesEs(query) {
+    getMimeTypesEs(query) {
         const AGGS = {
             mimeTypes: {
                 terms: {
@@ -311,7 +218,7 @@ class Sist2Api {
         });
     }
 
-    private getMimeTypesSqlite(): Promise<[{ mime: string, count: number }]> {
+    getMimeTypesSqlite() {
         return axios.get(`${this.baseUrl}fts/mimetypes`)
             .then(resp => {
                 return resp.data;
@@ -321,15 +228,15 @@ class Sist2Api {
     async getMimeTypes(query = undefined) {
         let buckets;
 
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             buckets = await this.getMimeTypesSqlite();
         } else {
             buckets = await this.getMimeTypesEs(query);
         }
 
-        const mimeMap: any[] = [];
+        const mimeMap = [];
 
-        buckets.sort((a: any, b: any) => a.mime > b.mime).forEach((bucket: any) => {
+        buckets.sort((a, b) => a.mime > b.mime).forEach((bucket) => {
             const tmp = bucket.mime.split("/");
             const category = tmp[0];
             const mime = tmp[1];
@@ -363,7 +270,7 @@ class Sist2Api {
         return {buckets, mimeMap};
     }
 
-    _createEsTag(tag: string, count: number): EsTag {
+    _createEsTag(tag, count) {
         const tokens = tag.split(".");
 
         if (/.*\.#[0-9a-fA-F]{6}/.test(tag)) {
@@ -383,7 +290,7 @@ class Sist2Api {
         };
     }
 
-    private getTagsEs() {
+    getTagsEs() {
         return this.esQuery({
             aggs: {
                 tags: {
@@ -396,21 +303,21 @@ class Sist2Api {
             size: 0,
         }).then(resp => {
             return resp["aggregations"]["tags"]["buckets"]
-                .sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
+                .sort((a, b) => a["key"].localeCompare(b["key"]))
-                .map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
+                .map((bucket) => this._createEsTag(bucket["key"], bucket["doc_count"]));
         });
     }
 
-    private getTagsSqlite() {
+    getTagsSqlite() {
-        return axios.get(`${this.baseUrl}/fts/tags`)
+        return axios.get(`${this.baseUrl}fts/tags`)
             .then(resp => {
                 return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
             });
     }
 
-    async getTags(): Promise<EsTag[]> {
+    async getTags() {
         let tags;
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             tags = await this.getTagsSqlite();
         } else {
             tags = await this.getTagsEs();
@@ -419,7 +326,7 @@ class Sist2Api {
         // Remove duplicates (same tag with different color)
         const seen = new Set();
 
-        return tags.filter((t: EsTag) => {
+        return tags.filter((t) => {
             if (seen.has(t.id)) {
                 return false;
             }
@@ -428,31 +335,29 @@ class Sist2Api {
         });
     }
 
-    saveTag(tag: string, hit: EsHit) {
+    saveTag(tag, hit) {
-        return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
+        return axios.post(`${this.baseUrl}tag/${sid(hit)}`, {
             delete: false,
             name: tag,
-            doc_id: hit["_id"]
         });
     }
 
-    deleteTag(tag: string, hit: EsHit) {
+    deleteTag(tag, hit) {
-        return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
+        return axios.post(`${this.baseUrl}tag/${sid(hit)}`, {
             delete: true,
             name: tag,
-            doc_id: hit["_id"]
         });
     }
 
     searchPaths(indexId, minDepth, maxDepth, prefix = null) {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.searchPathsSqlite(indexId, minDepth, minDepth, prefix);
         } else {
             return this.searchPathsEs(indexId, minDepth, maxDepth, prefix);
         }
     }
 
-    private searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
+    searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
         return axios.post(`${this.baseUrl}fts/paths`, {
             indexId, minDepth, maxDepth, prefix
         }).then(resp => {
@@ -460,7 +365,7 @@ class Sist2Api {
         });
     }
 
-    private searchPathsEs(indexId, minDepth, maxDepth, prefix): Promise<[{ path: string, count: number }]> {
+    searchPathsEs(indexId, minDepth, maxDepth, prefix) {
 
         const query = {
             query: {
@@ -505,23 +410,25 @@ class Sist2Api {
         });
     }
 
-    private getDateRangeSqlite() {
+    getDateRangeSqlite() {
         return axios.get(`${this.baseUrl}fts/dateRange`)
             .then(resp => ({
                 min: resp.data.dateMin,
-                max: resp.data.dateMax,
+                max: (resp.data.dateMax === resp.data.dateMin)
+                    ? resp.data.dateMax + 1
+                    : resp.data.dateMax,
             }));
     }
 
-    getDateRange(): Promise<{ min: number, max: number }> {
+    getDateRange() {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.getDateRangeSqlite();
         } else {
             return this.getDateRangeEs();
         }
     }
 
-    private getDateRangeEs() {
+    getDateRangeEs() {
         return this.esQuery({
             // TODO: filter current selected indices
             aggs: {
@@ -538,7 +445,7 @@ class Sist2Api {
             if (range.min == null) {
                 range.min = 0;
                 range.max = 1;
-            } else if (range.min == range.max) {
+            } else if (range.min === range.max) {
                 range.max += 1;
             }
 
@@ -546,7 +453,7 @@ class Sist2Api {
         });
     }
 
-    private getPathSuggestionsSqlite(text: string) {
+    getPathSuggestionsSqlite(text) {
         return axios.post(`${this.baseUrl}fts/paths`, {
             prefix: text,
             minDepth: 1,
@@ -556,7 +463,7 @@ class Sist2Api {
         })
     }
 
-    private getPathSuggestionsEs(text) {
+    getPathSuggestionsEs(text) {
         return this.esQuery({
             suggest: {
                 path: {
@@ -574,31 +481,31 @@ class Sist2Api {
         });
     }
 
-    getPathSuggestions(text: string): Promise<string[]> {
+    getPathSuggestions(text) {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.getPathSuggestionsSqlite(text);
         } else {
             return this.getPathSuggestionsEs(text)
         }
     }
 
-    getTreemapStat(indexId: string) {
+    getTreemapStat(indexId) {
         return `${this.baseUrl}s/${indexId}/TMAP`;
     }
 
-    getMimeStat(indexId: string) {
+    getMimeStat(indexId) {
         return `${this.baseUrl}s/${indexId}/MAGG`;
     }
 
-    getSizeStat(indexId: string) {
+    getSizeStat(indexId) {
         return `${this.baseUrl}s/${indexId}/SAGG`;
     }
 
-    getDateStat(indexId: string) {
+    getDateStat(indexId) {
         return `${this.baseUrl}s/${indexId}/DAGG`;
     }
 
-    private getDocumentEs(docId: string, highlight: boolean, fuzzy: boolean) {
+    getDocumentEs(sid, highlight, fuzzy) {
         const query = Sist2Query.searchQuery();
 
         if (highlight) {
@@ -618,6 +525,15 @@ class Sist2Api {
             }
         }
 
+        if ("knn" in query) {
+            query.query = {
+                bool: {
+                    must: []
+                }
+            };
+            delete query.knn;
+        }
+
         if ("function_score" in query.query) {
             query.query = query.query.function_score.query;
         }
@@ -628,7 +544,7 @@ class Sist2Api {
             query.query.bool.must = [query.query.bool.must];
         }
 
-        query.query.bool.must.push({match: {_id: docId}});
+        query.query.bool.must.push({match: {_id: sid}});
 
         delete query["sort"];
         delete query["aggs"];
@@ -649,35 +565,35 @@ class Sist2Api {
         });
     }
 
-    private getDocumentSqlite(docId: string): Promise<EsHit> {
+    getDocumentSqlite(sid) {
-        return axios.get(`${this.baseUrl}/fts/d/${docId}`)
+        return axios.get(`${this.baseUrl}fts/d/${sid}`)
             .then(resp => ({
                 _source: resp.data
-            } as EsHit));
+            }));
     }
 
-    getDocument(docId: string, highlight: boolean, fuzzy: boolean): Promise<EsHit | null> {
+    getDocument(sid, highlight, fuzzy) {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
-            return this.getDocumentSqlite(docId);
+            return this.getDocumentSqlite(sid);
         } else {
-            return this.getDocumentEs(docId, highlight, fuzzy);
+            return this.getDocumentEs(sid, highlight, fuzzy);
         }
     }
 
-    getTagSuggestions(prefix: string): Promise<string[]> {
+    getTagSuggestions(prefix) {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.getTagSuggestionsSqlite(prefix);
         } else {
             return this.getTagSuggestionsEs(prefix);
         }
     }
 
-    private getTagSuggestionsSqlite(prefix): Promise<string[]> {
+    getTagSuggestionsSqlite(prefix) {
-        return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
+        return axios.post(`${this.baseUrl}fts/suggestTags`, prefix)
             .then(resp => (resp.data));
     }
 
-    private getTagSuggestionsEs(prefix): Promise<string[]> {
+    getTagSuggestionsEs(prefix) {
         return this.esQuery({
             suggest: {
                 tag: {
@@ -702,6 +618,11 @@ class Sist2Api {
             return result;
         });
     }
+
+    getEmbeddings(sid, modelId) {
+        return axios.post(`${this.baseUrl}e/${sid}/${modelId.toString().padStart(3, '0')}`)
+            .then(resp => (resp.data));
+    }
 }
 
 export default new Sist2Api("");

sist2-vue/src/Sist2ElasticsearchQuery.js

@@ -1,5 +1,5 @@
-import store from "./store";
+import store from "@/store";
-import {EsHit, Index} from "@/Sist2Api";
+import Sist2Api from "@/Sist2Api";
 
 const SORT_MODES = {
     score: {
@@ -7,62 +7,62 @@ const SORT_MODES = {
             {_score: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._score
+        key: (hit) => hit._score
     },
     random: {
         mode: [
             {_score: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._score
+        key: (hit) => hit._score
     },
     dateAsc: {
         mode: [
             {mtime: {order: "asc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.mtime
+        key: (hit) => hit._source.mtime
     },
     dateDesc: {
         mode: [
             {mtime: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.mtime
+        key: (hit) => hit._source.mtime
     },
     sizeAsc: {
         mode: [
             {size: {order: "asc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.size
+        key: (hit) => hit._source.size
     },
     sizeDesc: {
         mode: [
             {size: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.size
+        key: (hit) => hit._source.size
     },
     nameAsc: {
         mode: [
             {name: {order: "asc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.name
+        key: (hit) => hit._source.name
     },
     nameDesc: {
         mode: [
             {name: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.name
+        key: (hit) => hit._source.name
     }
-} as any;
+};
 
 class Sist2ElasticsearchQuery {
 
-    searchQuery(blankSearch: boolean = false): any {
+    searchQuery(blankSearch = false) {
 
         const getters = store.getters;
 
@@ -76,15 +76,17 @@ class Sist2ElasticsearchQuery {
         const fuzzy = getters.fuzzy;
         const size = getters.size;
         const after = getters.lastDoc;
-        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
+        const selectedIndexIds = getters.selectedIndices.map((idx) => idx.id)
         const selectedMimeTypes = getters.selectedMimeTypes;
         const selectedTags = getters.selectedTags;
+        const sortMode = getters.embedding ? "score" : getters.sortMode;
 
         const legacyES = store.state.sist2Info.esVersionLegacy;
+        const hasKnn = store.state.sist2Info.esVersionHasKnn;
 
         const filters = [
             {terms: {index: selectedIndexIds}}
-        ] as any[];
+        ];
 
         const fields = [
             "name^8",
@@ -115,11 +117,11 @@ class Sist2ElasticsearchQuery {
             }
 
             if (dateMin && dateMax) {
-                filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
+                filters.push({range: {mtime: {gte: dateMin, lte: dateMax, format: "epoch_second"}}})
             } else if (dateMin) {
-                filters.push({range: {mtime: {gte: dateMin}}})
+                filters.push({range: {mtime: {gte: dateMin, format: "epoch_second"}}})
             } else if (dateMax) {
-                filters.push({range: {mtime: {lte: dateMax}}})
+                filters.push({range: {mtime: {lte: dateMax, format: "epoch_second"}}})
             }
 
             const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
@@ -136,7 +138,7 @@ class Sist2ElasticsearchQuery {
                 if (getters.optTagOrOperator) {
                     filters.push({terms: {"tag": selectedTags}});
                 } else {
-                    selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
+                    selectedTags.forEach((tag) => filters.push({term: {"tag": tag}}));
                 }
             }
         }
@@ -162,16 +164,16 @@ class Sist2ElasticsearchQuery {
 
         const q = {
             _source: {
-                excludes: ["content", "_tie"]
+                excludes: ["content", "_tie", "emb.*"]
             },
             query: {
                 bool: {
                     filter: filters,
                 }
             },
-            sort: SORT_MODES[getters.sortMode].mode,
+            sort: SORT_MODES[sortMode].mode,
             size: size,
-        } as any;
+        };
 
         if (!after) {
             q.aggs = {
@@ -181,14 +183,57 @@ class Sist2ElasticsearchQuery {
         }
 
         if (!empty && !blankSearch) {
+            if (getters.embedding) {
+                filters.push(query)
+            } else {
                 q.query.bool.must = query;
             }
+        }
+
+        if (getters.embedding) {
+            delete q.query;
+
+            const field = "emb." + Sist2Api.models().find(m => m.id === getters.embeddingsModel).path;
+
+            if (hasKnn) {
+                // Use knn (8.8+)
+                q.knn = {
+                    field: field,
+                    query_vector: getters.embedding,
+
+                    k: 600,
+                    num_candidates: 600,
+
+                    filter: filters
+                }
+            } else {
+                // Use brute-force as a fallback
+
+                filters.push({exists: {field: field}});
+
+                q.query = {
+                    function_score: {
+                        query: {
+                            bool: {
+                                must: filters,
+                            }
+                        },
+                        script_score: {
+                            script: {
+                                source: `cosineSimilarity(params.query_vector, "${field}") + 1.0`,
+                                params: {query_vector: getters.embedding}
+                            }
+                        }
+                    }
+                }
+            }
+        }
 
         if (after) {
-            q.search_after = [SORT_MODES[getters.sortMode].key(after), after["_id"]];
+            q.search_after = [SORT_MODES[sortMode].key(after), after["_id"]];
         }
 
-        if (getters.optHighlight) {
+        if (getters.optHighlight && !getters.embedding) {
             q.highlight = {
                 pre_tags: ["<mark>"],
                 post_tags: ["</mark>"],
@@ -214,7 +259,7 @@ class Sist2ElasticsearchQuery {
             }
         }
 
-        if (getters.sortMode === "random") {
+        if (sortMode === "random") {
             q.query = {
                 function_score: {
                     query: {

sist2-vue/src/Sist2SqliteQuery.js

@@ -1,5 +1,4 @@
-import store from "./store";
+import store from "@/store";
-import {EsHit, Index} from "@/Sist2Api";
 
 const SORT_MODES = {
     score: {
@@ -29,18 +28,12 @@ const SORT_MODES = {
         "sort": "name",
         "sortAsc": false
     }
-} as any;
+};
-
-interface SortMode {
-    text: string
-    mode: any[]
-    key: (hit: EsHit) => any
-}
 
 
 class Sist2ElasticsearchQuery {
 
-    searchQuery(): any {
+    searchQuery() {
 
         const getters = store.getters;
 
@@ -52,7 +45,7 @@ class Sist2ElasticsearchQuery {
         const dateMax = getters.dateMax;
         const size = getters.size;
         const after = getters.lastDoc;
-        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
+        const selectedIndexIds = getters.selectedIndices.map((idx) => idx.id)
         const selectedMimeTypes = getters.selectedMimeTypes;
         const selectedTags = getters.selectedTags;
 
@@ -95,7 +88,7 @@ class Sist2ElasticsearchQuery {
         if (selectedTags.length > 0) {
             q["tags"] = selectedTags
         }
-        if (getters.sortMode == "random") {
+        if (getters.sortMode === "random") {
             q["seed"] = getters.seed;
         }
         if (getters.optHighlight) {
@@ -103,6 +96,18 @@ class Sist2ElasticsearchQuery {
             q["highlightContextSize"] = Number(getters.optFragmentSize);
         }
 
+        if (getters.embedding) {
+            q["model"] = getters.embeddingsModel;
+            q["embedding"] = getters.embedding;
+            q["sort"] = "embedding";
+            q["sortAsc"] = false;
+        } else if (getters.sortMode === "embedding") {
+            q["sort"] = "sort"
+            q["sortAsc"] = true;
+        }
+
+        q["searchInPath"] = getters.optSearchInPath;
+
         return q;
     }
 }

sist2-vue/src/components/AnalyzedContentSpan.vue

@@ -12,7 +12,7 @@ export default {
     props: ["span", "text"],
     methods: {
         getStyle() {
-            return ModelsRepo.data[this.$store.getters.mlModel.name].labelStyles[this.span.label];
+            return ModelsRepo.data[this.$store.getters.nerModel.name].labelStyles[this.span.label];
         }
     }
 }

sist2-vue/src/components/AnalyzedContentSpanContainer.vue

@@ -22,7 +22,7 @@ export default {
     props: ["spans", "text"],
     computed: {
         legend() {
-            return Object.entries(ModelsRepo.data[this.$store.state.mlModel.name].legend)
+            return Object.entries(ModelsRepo.data[this.$store.state.nerModel.name].legend)
                 .map(([label, name]) => ({
                     text: name,
                     id: label,

sist2-vue/src/components/D3Treemap.vue

@@ -9,8 +9,7 @@
 
 <script>
 import * as d3 from "d3";
-import {burrow} from "@/util-js"
+import {humanFileSize, burrow} from "@/util";
-import {humanFileSize} from "@/util";
 import Sist2Api from "@/Sist2Api";
 import domtoimage from "dom-to-image";
 

sist2-vue/src/components/DateSlider.vue

@@ -31,8 +31,7 @@
 <script>
 import noUiSlider from 'nouislider';
 import 'nouislider/dist/nouislider.css';
-import {humanDate} from "@/util";
+import {humanDate, mergeTooltips} from "@/util";
-import {mergeTooltips} from "@/util-js";
 
 export default {
   name: "DateSlider",

sist2-vue/src/components/DebugInfo.vue

@@ -45,7 +45,8 @@ export default {
                 items.push(
                     {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
                     {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
-                    {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy}
+                    {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
+                    {key: "esVersionHasKnn", value: this.$store.state.sist2Info.esVersionHasKnn},
                 );
             }
 

sist2-vue/src/components/DocCard.vue

@@ -16,7 +16,7 @@
             <!-- Audio player-->
             <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
                    :type="doc._source.mime"
-                   :src="`f/${doc._source.index}/${doc._id}`"
+                   :src="`f/${sid(doc)}`"
                    @play="onAudioPlay()"></audio>
 
             <b-card-body class="padding-03">
@@ -24,6 +24,7 @@
                 <!-- Title line -->
                 <div style="display: flex">
                     <span class="info-icon" @click="onInfoClick()"></span>
+                    <MLIcon v-if="doc._source.embedding" clickable @click="onEmbeddingClick()"></MLIcon>
                     <DocFileTitle :doc="doc"></DocFileTitle>
                 </div>
 
@@ -42,17 +43,19 @@
 </template>
 
 <script>
-import {ext, humanFileSize, humanTime} from "@/util";
+import {ext, humanFileSize, humanTime, sid} from "@/util";
 import TagContainer from "@/components/TagContainer.vue";
 import DocFileTitle from "@/components/DocFileTitle.vue";
 import DocInfoModal from "@/components/DocInfoModal.vue";
 import ContentDiv from "@/components/ContentDiv.vue";
 import FullThumbnail from "@/components/FullThumbnail";
 import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
+import MLIcon from "@/components/icons/MlIcon.vue";
+import Sist2Api from "@/Sist2Api";
 
 
 export default {
-    components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+    components: {MLIcon, FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
     props: ["doc", "width"],
     data() {
         return {
@@ -66,11 +69,19 @@ export default {
         }
     },
     methods: {
+        sid: sid,
         humanFileSize: humanFileSize,
         humanTime: humanTime,
         onInfoClick() {
             this.showInfo = true;
         },
+        onEmbeddingClick() {
+            Sist2Api.getEmbeddings(sid(this.doc), this.$store.state.embeddingsModel).then(embeddings => {
+                this.$store.commit("setEmbeddingText", "");
+                this.$store.commit("setEmbedding", embeddings);
+                this.$store.commit("setEmbeddingDoc", this.doc);
+            })
+        },
         async onThumbnailClick() {
             this.$store.commit("setUiLightboxSlide", this.doc._seq);
             await this.$store.dispatch("showLightbox");

sist2-vue/src/components/DocCardWall.vue

@@ -5,11 +5,12 @@
             @append="append"
             @layout-complete="$emit('layout-complete')"
     >
-    <DocCard v-for="doc in docs" :key="doc._id" :doc="doc" :width="width"></DocCard>
+        <DocCard v-for="doc in docs" :key="sid(doc)" :doc="doc" :width="width"></DocCard>
     </GridLayout>
 </template>
 
 <script>
+import {sid} from "@/util";
 import Vue from "vue";
 import DocCard from "@/components/DocCard";
 
@@ -39,6 +40,9 @@ export default Vue.extend({
             }
         }
     },
+    methods: {
+        sid,
+    },
     computed: {
         colCount() {
             const columns = this.$store.getters["optColumns"];

sist2-vue/src/components/DocFileTitle.vue

@@ -1,17 +1,19 @@
 <template>
-  <a :href="`f/${doc._source.index}/${doc._id}`" class="file-title-anchor" target="_blank">
+    <a :href="`f/${sid(doc)}`"
+       :class="doc._source.embedding ? 'file-title-anchor-with-embedding' : 'file-title-anchor'" target="_blank">
         <div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
              v-html="fileName() + ext(doc)"></div>
     </a>
 </template>
 
 <script>
-import {ext} from "@/util";
+import {ext, sid} from "@/util";
 
 export default {
     name: "DocFileTitle",
     props: ["doc"],
     methods: {
+        sid: sid,
         ext: ext,
         fileName() {
             if (!this.doc.highlight) {
@@ -34,8 +36,13 @@ export default {
     max-width: calc(100% - 1.2rem);
 }
 
+.file-title-anchor-with-embedding {
+    max-width: calc(100% - 2.2rem);
+}
+
 .file-title {
     width: 100%;
+    max-width: 100%;
     line-height: 1rem;
     height: 1.1rem;
     white-space: nowrap;
@@ -49,6 +56,7 @@ export default {
 .theme-black .file-title {
     color: #ddd;
 }
+
 .theme-black .file-title:hover {
     color: #fff;
 }

sist2-vue/src/components/DocInfoModal.vue

@@ -8,16 +8,16 @@
             </h5>
         </template>
 
-    <img v-if="doc._props.hasThumbnail" :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">
+        <img v-if="doc._props.hasThumbnail" :src="`t/${sid(doc)}`" alt="" class="fit card-img-top">
 
         <InfoTable :doc="doc"></InfoTable>
 
-    <LazyContentDiv :doc-id="doc._id"></LazyContentDiv>
+        <LazyContentDiv :sid="sid(doc)"></LazyContentDiv>
     </b-modal>
 </template>
 
 <script>
-import {ext} from "@/util";
+import {ext, sid} from "@/util";
 import InfoTable from "@/components/InfoTable";
 import LazyContentDiv from "@/components/LazyContentDiv";
 
@@ -27,6 +27,7 @@ export default {
     props: ["doc", "show"],
     methods: {
         ext: ext,
+        sid: sid
     }
 }
 </script>

sist2-vue/src/components/DocList.vue

@@ -1,10 +1,11 @@
 <template>
     <b-list-group class="mt-3">
-    <DocListItem v-for="doc in docs" :key="doc._id" :doc="doc"></DocListItem>
+        <DocListItem v-for="doc in docs" :key="sid(doc)" :doc="doc"></DocListItem>
     </b-list-group>
 </template>
 
-<script lang="ts">
+<script>
+import {sid} from "@/util";
 import DocListItem from "@/components/DocListItem.vue";
 import Vue from "vue";
 
@@ -21,6 +22,9 @@ export default Vue.extend({
                 this.append();
             }
         });
+    },
+    methods: {
+        sid: sid
     }
 });
 </script>

sist2-vue/src/components/DocListItem.vue

@@ -17,10 +17,10 @@
                     </div>
 
                     <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-               :src="(doc._props.isGif && hover) ? `f/${doc._source.index}/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
+                         :src="(doc._props.isGif && hover) ? `f/${sid(doc)}` : `t/${sid(doc)}`"
                          alt=""
                          class="pointer fit-sm" @click="onThumbnailClick()">
-          <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
+                    <img v-else :src="`t/${sid(doc)}`" alt=""
                          class="fit-sm">
                 </div>
             </div>
@@ -32,6 +32,7 @@
             <div class="doc-line ml-3">
                 <div style="display: flex">
                     <span class="info-icon" @click="showInfo = true"></span>
+                    <MLIcon v-if="doc._source.embedding" clickable @click="onEmbeddingClick()"></MLIcon>
                     <DocFileTitle :doc="doc"></DocFileTitle>
                 </div>
 
@@ -67,10 +68,13 @@ import DocInfoModal from "@/components/DocInfoModal";
 import ContentDiv from "@/components/ContentDiv";
 import FileIcon from "@/components/icons/FileIcon";
 import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
+import MLIcon from "@/components/icons/MlIcon.vue";
+import Sist2Api from "@/Sist2Api";
+import {sid} from "@/util";
 
 export default {
     name: "DocListItem",
-  components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
+    components: {MLIcon, FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
     props: ["doc"],
     data() {
         return {
@@ -79,10 +83,18 @@ export default {
         }
     },
     methods: {
+        sid: sid,
         async onThumbnailClick() {
             this.$store.commit("setUiLightboxSlide", this.doc._seq);
             await this.$store.dispatch("showLightbox");
         },
+        onEmbeddingClick() {
+            Sist2Api.getEmbeddings(sid(this.doc), this.$store.state.embeddingsModel).then(embeddings => {
+                this.$store.commit("setEmbeddingText", "");
+                this.$store.commit("setEmbedding", embeddings);
+                this.$store.commit("setEmbeddingDoc", this.doc);
+            })
+        },
         path() {
             if (!this.doc.highlight) {
                 return this.doc._source.path + "/"

sist2-vue/src/components/EmbeddingsSearchBar.vue

@@ -0,0 +1,155 @@
+<template>
+    <div>
+        <b-progress v-if="modelLoading && [0, 1].includes(modelLoadingProgress)" max="1" class="mb-1" variant="primary"
+                    striped animated :value="1">
+        </b-progress>
+        <b-progress v-else-if="modelLoading" :value="modelLoadingProgress" max="1" class="mb-1" variant="warning"
+                    show-progress>
+        </b-progress>
+        <div style="display: flex">
+            <b-select :options="modelOptions()" class="mr-2 input-prepend" :value="modelName"
+                      @change="onModelChange($event)"></b-select>
+
+            <b-input-group>
+                <b-form-input :value="embeddingText"
+                              :placeholder="$store.state.embeddingDoc ? ' ' : $t('embeddingsSearchPlaceholder')"
+                              @input="onInput($event)"
+                              :disabled="modelLoading"
+                              :style="{'pointer-events': $store.state.embeddingDoc ? 'none' : undefined}"
+                ></b-form-input>
+                <b-badge v-if="$store.state.embeddingDoc" pill variant="primary" class="overlay-badge" href="#"
+                         @click="onBadgeClick()">{{ docName }}
+                </b-badge>
+
+                <template #prepend>
+                </template>
+
+                <template #append>
+                    <b-input-group-text>
+                        <MLIcon class="ml-append" big></MLIcon>
+                    </b-input-group-text>
+                </template>
+
+            </b-input-group>
+        </div>
+
+    </div>
+</template>
+
+<script>
+import {mapGetters, mapMutations} from "vuex";
+import {CLIPTransformerModel} from "@/ml/CLIPTransformerModel"
+import _debounce from "lodash/debounce";
+import MLIcon from "@/components/icons/MlIcon.vue";
+import Sist2AdminApi from "@/Sist2Api";
+
+export default {
+    components: {MLIcon},
+    data() {
+        return {
+            modelLoading: false,
+            modelLoadingProgress: 0,
+            modelLoaded: false,
+            model: null,
+            modelName: null
+        }
+    },
+    computed: {
+        ...mapGetters({
+            optQueryMode: "optQueryMode",
+            embeddingText: "embeddingText",
+            fuzzy: "fuzzy",
+        }),
+        docName() {
+            const ext = this.$store.state.embeddingDoc._source.extension;
+            return this.$store.state.embeddingDoc._source.name +
+                (ext ? "." + ext : "")
+        }
+    },
+    mounted() {
+        // Set default model
+        this.modelName = Sist2AdminApi.models()[0].name;
+        this.onModelChange(this.modelName);
+
+        this.onInput = _debounce(this._onInput, 450, {leading: false});
+    },
+    methods: {
+        ...mapMutations({
+            setEmbeddingText: "setEmbeddingText",
+            setEmbedding: "setEmbedding",
+            setEmbeddingModel: "setEmbeddingsModel",
+        }),
+        async loadModel() {
+            this.modelLoading = true;
+
+            await this.model.init(async progress => {
+                this.modelLoadingProgress = progress;
+            });
+            this.modelLoading = false;
+            this.modelLoaded = true;
+        },
+        async _onInput(text) {
+            try {
+
+                if (!this.modelLoaded) {
+                    await this.loadModel();
+                }
+
+                if (text.length === 0) {
+                    this.setEmbeddingText("");
+                    this.setEmbedding(null);
+                    return;
+                }
+
+                const embeddings = await this.model.predict(text);
+
+                this.setEmbeddingText(text);
+                this.setEmbedding(embeddings);
+            } catch (e) {
+                alert(e)
+            }
+        },
+        modelOptions() {
+            return Sist2AdminApi.models().map(model => model.name);
+        },
+        onModelChange(name) {
+            this.modelLoaded = false;
+            this.modelLoadingProgress = 0;
+
+            const modelInfo = Sist2AdminApi.models().find(m => m.name === name);
+
+            if (modelInfo.name === "CLIP") {
+                const tokenizerUrl = new URL("./tokenizer.json", modelInfo.url).href;
+                this.model = new CLIPTransformerModel(modelInfo.url, tokenizerUrl)
+                this.setEmbeddingModel(modelInfo.id);
+            } else {
+                throw new Error("Unknown model: " + name);
+            }
+        },
+        onBadgeClick() {
+            this.$store.commit("setEmbedding", null);
+            this.$store.commit("setEmbeddingDoc", null);
+        }
+    }
+}
+</script>
+<style>
+.overlay-badge {
+    position: absolute;
+    z-index: 1;
+    left: 0.375rem;
+    top: 8px;
+    line-height: 1.1rem;
+    overflow: hidden;
+    max-width: 200px;
+    text-overflow: ellipsis;
+}
+
+.input-prepend {
+    max-width: 100px;
+}
+
+.theme-black .ml-append {
+    filter: brightness(0.95) !important;
+}
+</style>

sist2-vue/src/components/FullThumbnail.vue

@@ -40,7 +40,7 @@
 </template>
 
 <script>
-import {humanTime} from "@/util";
+import {humanTime, sid} from "@/util";
 import ThumbnailProgressBar from "@/components/ThumbnailProgressBar";
 
 export default {
@@ -67,15 +67,16 @@ export default {
         },
     },
     methods: {
+        sid: sid,
         getThumbnailSrc(thumbnailNum) {
             const doc = this.doc;
             const props = doc._props;
             if (props.isGif && this.hover) {
-        return `f/${doc._source.index}/${doc._id}`;
+                return `f/${sid(doc)}`;
             }
             return (this.currentThumbnailNum === 0)
-          ? `t/${doc._source.index}/${doc._id}`
+                ? `t/${sid(doc)}`
-          : `t/${doc._source.index}/${doc._id}/${String(thumbnailNum).padStart(4, "0")}`;
+                : `t/${sid(doc)}/${String(thumbnailNum).padStart(4, "0")}`;
         },
         humanTime: humanTime,
         onThumbnailClick() {

sist2-vue/src/components/IndexPicker.vue

@@ -27,11 +27,16 @@
                     @click.shift="shiftClick(idx, $event)"
                     class="d-flex justify-content-between align-items-center list-group-item-action pointer"
                     :class="{active: lastClickIndex === idx}"
+                    :key="idx.id"
             >
                 <div class="d-flex">
                     <b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
                     {{ idx.name }}
-          <span class="text-muted timestamp-text ml-2">{{ formatIdxDate(idx.timestamp) }}</span>
+                    <div v-if="hasEmbeddings(idx)" style="vertical-align: center; margin-left: 5px">
+                        <MLIcon small style="top: -1px; position: relative"></MLIcon>
+                    </div>
+                    <span class="text-muted timestamp-text ml-2"
+                          style="top: 1px; position: relative">{{ formatIdxDate(idx.timestamp) }}</span>
                 </div>
                 <b-badge class="version-badge">v{{ idx.version }}</b-badge>
             </b-list-group-item>
@@ -39,14 +44,16 @@
     </div>
 </template>
 
-<script lang="ts">
+<script>
 import SmallBadge from "./SmallBadge.vue"
 import {mapActions, mapGetters} from "vuex";
 import Vue from "vue";
 import {format} from "date-fns";
+import MLIcon from "@/components/icons/MlIcon.vue";
 
 export default Vue.extend({
     components: {
+        MLIcon,
         SmallBadge
     },
     data() {
@@ -105,7 +112,7 @@ export default Vue.extend({
         onSelect(value) {
             this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
         },
-    formatIdxDate(timestamp: number): string {
+        formatIdxDate(timestamp) {
             return format(new Date(timestamp * 1000), "yyyy-MM-dd");
         },
         toggleIndex(index, e) {
@@ -115,14 +122,17 @@ export default Vue.extend({
 
             this.lastClickIndex = index;
             if (this.isSelected(index)) {
-        this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
+                this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id !== index.id));
             } else {
                 this.setSelectedIndices([index, ...this.selectedIndices]);
             }
         },
         isSelected(index) {
-      return this.selectedIndices.find(idx => idx.id == index.id) != null;
+            return this.selectedIndices.find(idx => idx.id === index.id) != null;
-    }
+        },
+        hasEmbeddings(index) {
+            return index.models.length > 0;
+        },
     },
 })
 </script>
@@ -170,18 +180,18 @@ export default Vue.extend({
 }
 
 .theme-black .list-group-item {
-  border: 1px solid rgba(255,255,255, 0.1);
+    border: 1px solid rgba(255, 255, 255, 0.1);
 }
 
 .theme-black .list-group-item:first-child {
-  border: 1px solid rgba(255,255,255, 0.05);
+    border: 1px solid rgba(255, 255, 255, 0.05);
 }
 
 .theme-black .list-group-item.active {
     z-index: 2;
     background-color: inherit;
     color: inherit;
-  border: 1px solid rgba(255,255,255, 0.3);
+    border: 1px solid rgba(255, 255, 255, 0.3);
     border-radius: 0;
 }
 

sist2-vue/src/components/InfoTable.vue

@@ -59,7 +59,7 @@ export default {
 
       const fields = [
         "title", "duration", "audioc", "videoc",
-        "bitrate", "artist", "album", "album_artist", "genre", "font_name", "author",
+        "bitrate", "artist", "album", "album_artist", "genre", "font_name", "author", "media_comment",
         "modified_by", "pages", "tag",
         "exif_make", "exif_software", "exif_exposure_time", "exif_fnumber", "exif_focal_length",
         "exif_user_comment", "exif_iso_speed_ratings", "exif_model", "exif_datetime",

sist2-vue/src/components/LazyContentDiv.vue

@@ -9,8 +9,8 @@
             <b-button :disabled="mlPredictionsLoading || mlLoading" @click="mlAnalyze" variant="primary"
             >{{ $t("ml.analyzeText") }}
             </b-button>
-            <b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="mlModel">
+            <b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="nerModel">
-                <b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()">{{ opt.text }}
+                <b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()" :key="opt.value">{{ opt.text }}
                 </b-select-option>
             </b-select>
         </b-form>
@@ -46,7 +46,7 @@ import {mapGetters, mapMutations} from "vuex";
 export default {
     name: "LazyContentDiv",
     components: {AnalyzedContentSpansContainer, Preloader},
-    props: ["docId"],
+    props: ["sid"],
     data() {
         return {
             ModelsRepo,
@@ -57,20 +57,20 @@ export default {
             modelPredictionProgress: 0,
             mlPredictionsLoading: false,
             mlLoading: false,
-            mlModel: null,
+            nerModel: null,
             analyzedContentSpans: []
         }
     },
     mounted() {
 
         if (this.$store.getters.optMlDefaultModel) {
-            this.mlModel = this.$store.getters.optMlDefaultModel
+            this.nerModel = this.$store.getters.optMlDefaultModel
         } else {
-            this.mlModel = ModelsRepo.getDefaultModel();
+            this.nerModel = ModelsRepo.getDefaultModel();
         }
 
         Sist2Api
-            .getDocument(this.docId, this.$store.state.optHighlight, this.$store.state.fuzzy)
+            .getDocument(this.sid, this.$store.state.optHighlight, this.$store.state.fuzzy)
             .then(doc => {
                 this.loading = false;
 
@@ -86,7 +86,7 @@ export default {
     computed: {
         ...mapGetters(["optAutoAnalyze"]),
         modelSize() {
-            const modelData = ModelsRepo.data[this.mlModel];
+            const modelData = ModelsRepo.data[this.nerModel];
             if (!modelData) {
                 return 0;
             }
@@ -110,10 +110,10 @@ export default {
             }
         },
         async getMlModel() {
-            if (this.$store.getters.mlModel.name !== this.mlModel) {
+            if (this.$store.getters.nerModel.name !== this.nerModel) {
                 this.mlLoading = true;
                 this.modelLoadingProgress = 0;
-                const modelInfo = ModelsRepo.data[this.mlModel];
+                const modelInfo = ModelsRepo.data[this.nerModel];
 
                 const model = new BertNerModel(
                     modelInfo.vocabUrl,
@@ -122,25 +122,25 @@ export default {
                 )
 
                 await model.init(progress => this.modelLoadingProgress = progress);
-                this.$store.commit("setMlModel", {model, name: this.mlModel});
+                this.$store.commit("setNerModel", {model, name: this.nerModel});
 
                 this.mlLoading = false;
                 return model
             }
 
-            return this.$store.getters.mlModel.model;
+            return this.$store.getters.nerModel.model;
         },
         async mlAnalyze() {
             if (!this.content) {
                 return;
             }
 
-            const modelInfo = ModelsRepo.data[this.mlModel];
+            const modelInfo = ModelsRepo.data[this.nerModel];
             if (modelInfo === undefined) {
                 return;
             }
 
-            this.$store.commit("setOptMlDefaultModel", this.mlModel);
+            this.$store.commit("setOptMlDefaultModel", this.nerModel);
             await this.$store.dispatch("updateConfiguration");
 
             const model = await this.getMlModel();

sist2-vue/src/components/Lightbox.vue

@@ -77,6 +77,7 @@ export default {
         return listener(e);
       }
     };
+
   },
   methods: {
     keyDownListener(e) {

sist2-vue/src/components/NavBar.vue

@@ -9,7 +9,7 @@
 
     <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
       v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
-        href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
+        href="https://github.com/sist2app/sist2/blob/master/docs/USAGE.md#elasticsearch"
         target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
     </span>
 

sist2-vue/src/components/ResultsCard.vue

@@ -43,8 +43,7 @@
     </b-card>
 </template>
 
-<script lang="ts">
+<script>
-import Sist2Api, {EsResult} from "@/Sist2Api";
 import Vue from "vue";
 import {humanFileSize} from "@/util";
 import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
@@ -52,6 +51,7 @@ import SortSelect from "@/components/SortSelect.vue";
 import Preloader from "@/components/Preloader.vue";
 import Sist2Query from "@/Sist2ElasticsearchQuery";
 import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
+import Sist2Api from "@/Sist2Api";
 
 export default Vue.extend({
     name: "ResultsCard",
@@ -64,7 +64,7 @@ export default Vue.extend({
             return this.$store.state.lastQueryResults != null;
         },
         hitCount() {
-            return (this.$store.state.firstQueryResults as EsResult).aggregations.total_count.value;
+            return (this.$store.state.firstQueryResults).aggregations.total_count.value;
         },
         tableItems() {
             const items = [];
@@ -79,10 +79,10 @@ export default Vue.extend({
     },
     methods: {
         took() {
-            return (this.$store.state.lastQueryResults as EsResult).took + "ms";
+            return (this.$store.state.lastQueryResults).took + "ms";
         },
         totalSize() {
-            return humanFileSize((this.$store.state.firstQueryResults as EsResult).aggregations.total_size.value);
+            return humanFileSize((this.$store.state.firstQueryResults).aggregations.total_size.value);
         },
         onToggle() {
             const show = !document.getElementById("collapse-1").classList.contains("show");

sist2-vue/src/components/SizeSlider.vue

@@ -5,8 +5,7 @@
 <script>
 import noUiSlider from 'nouislider';
 import 'nouislider/dist/nouislider.css';
-import {humanFileSize} from "@/util";
+import {humanFileSize, mergeTooltips} from "@/util";
-import {mergeTooltips} from "@/util-js";
 
 export default {
   name: "SizeSlider",

sist2-vue/src/components/SmallBadge.vue

@@ -2,7 +2,7 @@
   <b-badge variant="secondary" :pill="pill">{{ text }}</b-badge>
 </template>
 
-<script lang="ts">
+<script>
 import Vue from "vue";
 
 export default Vue.extend({

sist2-vue/src/components/SortSelect.vue

@@ -1,5 +1,5 @@
 <template>
-  <b-dropdown variant="primary">
+  <b-dropdown variant="primary" :disabled="$store.getters.embedding !== null">
     <b-dropdown-item :class="{'dropdown-active': sort === 'score'}" @click="onSelect('score')">{{
         $t("sort.relevance")
       }}

sist2-vue/src/components/TagPicker.vue

@@ -210,4 +210,8 @@ export default {
 .theme-black .inspire-tree .matched > .wholerow {
     background: rgba(251, 191, 41, 0.25);
 }
+#tagTree {
+    max-height: 350px;
+    overflow: auto;
+}
 </style>

sist2-vue/src/components/ThumbnailProgressBar.vue

@@ -1,5 +1,5 @@
 <template>
-  <div class="thumbnail-progress-bar" :style="{width: `${percentProgress}%`}"></div>
+  <div class="thumbnail_count-progress-bar" :style="{width: `${percentProgress}%`}"></div>
 </template>
 
 <script>
@@ -16,7 +16,7 @@ export default {
 
 <style scoped>
 
-.thumbnail-progress-bar {
+.thumbnail_count-progress-bar {
   position: absolute;
   left: 0;
   bottom: 0;
@@ -27,11 +27,11 @@ export default {
   z-index: 9;
 }
 
-.theme-black .thumbnail-progress-bar {
+.theme-black .thumbnail_count-progress-bar {
   background: rgba(0, 188, 212, 0.95);
 }
 
-.sub-document .thumbnail-progress-bar {
+.sub-document .thumbnail_count-progress-bar {
   max-width: calc(100% - 8px);
   left: 4px;
 }

sist2-vue/src/components/icons/MlIcon.vue

@@ -0,0 +1,76 @@
+<template>
+    <svg class="ml-icon" :class="{'m-icon': 1, 'ml-icon-big': big, 'ml-icon-clickable': clickable}" xmlns="http://www.w3.org/2000/svg"
+         viewBox="0 0 512 512" xml:space="preserve" fill="currentColor" stroke="currentColor" @click="$emit('click')">
+                <g>
+	<path class="st0" d="M167.314,14.993C167.314,6.712,160.602,0,152.332,0h-5.514c-8.27,0-14.982,6.712-14.982,14.993v41.466h35.478
+		V14.993z"/>
+                    <path class="st0"
+                          d="M238.26,14.993C238.26,6.712,231.549,0,223.278,0h-5.504c-8.271,0-14.982,6.712-14.982,14.993v41.466h35.468 V14.993z"/>
+                    <path class="st0"
+                          d="M309.207,14.993C309.207,6.712,302.496,0,294.225,0h-5.504c-8.271,0-14.982,6.712-14.982,14.993v41.466h35.468 V14.993z"/>
+                    <path class="st0"
+                          d="M380.164,14.993C380.164,6.712,373.453,0,365.182,0h-5.514c-8.27,0-14.982,6.712-14.982,14.993v41.466h35.478 V14.993z"/>
+                    <path class="st0"
+                          d="M131.836,497.007c0,8.282,6.712,14.993,14.982,14.993h5.514c8.27,0,14.982-6.711,14.982-14.993V455.55h-35.478 V497.007z"/>
+                    <path class="st0"
+                          d="M202.792,497.007c0,8.282,6.712,14.993,14.982,14.993h5.504c8.27,0,14.982-6.711,14.982-14.993V455.55h-35.468 V497.007z"/>
+                    <path class="st0"
+                          d="M273.739,497.007c0,8.282,6.712,14.993,14.982,14.993h5.504c8.271,0,14.982-6.711,14.982-14.993V455.55 h-35.468V497.007z"/>
+                    <path class="st0"
+                          d="M344.686,497.007c0,8.282,6.712,14.993,14.982,14.993h5.514c8.271,0,14.982-6.711,14.982-14.993V455.55 h-35.478V497.007z"/>
+                    <path class="st0"
+                          d="M497.018,131.836H455.55v35.479h41.468c8.27,0,14.982-6.712,14.982-14.993v-5.493 C512,138.548,505.288,131.836,497.018,131.836z"/>
+                    <path class="st0"
+                          d="M497.018,202.793H455.55v35.468h41.468c8.27,0,14.982-6.712,14.982-14.982v-5.494 C512,209.504,505.288,202.793,497.018,202.793z"/>
+                    <path class="st0"
+                          d="M497.018,273.739H455.55v35.468h41.468c8.27,0,14.982-6.711,14.982-14.992v-5.494 C512,280.451,505.288,273.739,497.018,273.739z"/>
+                    <path class="st0"
+                          d="M497.018,344.686H455.55v35.479h41.468c8.27,0,14.982-6.712,14.982-14.993v-5.493 C512,351.398,505.288,344.686,497.018,344.686z"/>
+                    <path class="st0"
+                          d="M0,146.828v5.493c0,8.281,6.711,14.993,14.982,14.993H56.46v-35.479H14.982C6.711,131.836,0,138.548,0,146.828 z"/>
+                    <path class="st0"
+                          d="M0,217.785v5.494c0,8.27,6.711,14.982,14.982,14.982H56.46v-35.468H14.982C6.711,202.793,0,209.504,0,217.785z "/>
+                    <path class="st0"
+                          d="M0,288.721v5.494c0,8.281,6.711,14.992,14.982,14.992H56.46v-35.468H14.982C6.711,273.739,0,280.451,0,288.721 z"/>
+                    <path class="st0"
+                          d="M0,359.679v5.493c0,8.281,6.711,14.993,14.982,14.993H56.46v-35.479H14.982C6.711,344.686,0,351.398,0,359.679 z"/>
+                    <path class="st0"
+                          d="M78.628,433.382h354.753V78.628H78.628V433.382z M376.56,120.2c9.18,0,16.635,7.445,16.635,16.634 c0,9.18-7.455,16.624-16.635,16.624c-9.179,0-16.624-7.445-16.624-16.624C359.936,127.644,367.381,120.2,376.56,120.2z M376.56,361.32c9.18,0,16.635,7.445,16.635,16.635c0,9.179-7.455,16.623-16.635,16.623c-9.179,0-16.624-7.444-16.624-16.623 C359.936,368.764,367.381,361.32,376.56,361.32z M184.362,184.362h143.287v143.287H184.362V184.362z M135.439,120.2 c9.19,0,16.635,7.445,16.635,16.634c0,9.169-7.445,16.624-16.635,16.624c-9.178,0-16.623-7.455-16.623-16.624 C118.816,127.644,126.26,120.2,135.439,120.2z M135.439,361.32c9.19,0,16.635,7.445,16.635,16.635 c0,9.169-7.445,16.623-16.635,16.623c-9.178,0-16.623-7.454-16.623-16.623C118.816,368.764,126.26,361.32,135.439,361.32z"/>
+</g>
+</svg>
+</template>
+
+<script>
+export default {
+    name: "MLIcon",
+    props: {
+        "big": Boolean,
+        "clickable": Boolean
+    }
+}
+</script>
+
+<style scoped>
+.ml-icon-clickable {
+    cursor: pointer;
+}
+
+.ml-icon-big {
+    width: 24px !important;
+    height: 24px !important;
+}
+
+.ml-icon {
+    width: 1rem;
+    min-width: 1rem;
+    margin-right: 0.2rem;
+    line-height: 1rem;
+    height: 1rem;
+    min-height: 1rem;
+    filter: brightness(45%);
+}
+
+.theme-black .ml-icon {
+    filter: brightness(80%);
+}
+</style>

sist2-vue/src/i18n/messages.js

@@ -18,6 +18,7 @@ export default {
         tags: "Tags",
         tagFilter: "Filter tags",
         forExample: "For example:",
+        embeddingsSearchPlaceholder: "Embeddings search",
         help: {
             simpleSearch: "Simple search",
             advancedSearch: "Advanced search",
@@ -137,7 +138,7 @@ export default {
         },
         debug: "Debug information",
         debugDescription: "Information useful for debugging. If you encounter bugs or have suggestions for" +
-            " new features, please submit a new issue <a href='https://github.com/simon987/sist2/issues/new/choose'>here</a>.",
+            " new features, please submit a new issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>here</a>.",
         tagline: "Tagline",
         toast: {
             esConnErrTitle: "Elasticsearch connection error",
@@ -317,7 +318,7 @@ export default {
         },
         debug: "Debug Informationen",
         debugDescription: "Informationen für das Debugging. Wenn du Bugs gefunden oder Anregungen für " +
-            "neue Features hast, poste sie bitte <a href='https://github.com/simon987/sist2/issues/new/choose'>hier</a>.",
+            "neue Features hast, poste sie bitte <a href='https://github.com/sist2app/sist2/issues/new/choose'>hier</a>.",
         tagline: "Tagline",
         toast: {
             esConnErrTitle: "Elasticsearch Verbindungsfehler",
@@ -493,7 +494,7 @@ export default {
         debug: "Information de débogage",
         debugDescription: "Informations utiles pour le débogage\n" +
             "Si vous rencontrez des bogues ou si vous avez des suggestions pour de nouvelles fonctionnalités," +
-            " veuillez soumettre un nouvel Issue <a href='https://github.com/simon987/sist2/issues/new/choose'>ici</a>.",
+            " veuillez soumettre un nouvel Issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>ici</a>.",
         tagline: "Tagline",
         toast: {
             esConnErrTitle: "Erreur de connexion Elasticsearch",
@@ -667,7 +668,7 @@ export default {
         },
         debug: "调试信息",
         debugDescription: "对调试除错有用的信息。 若您遇到bug或者想建议新功能，请提交新Issue到" +
-            "<a href='https://github.com/simon987/sist2/issues/new/choose'>这里</a>.",
+            "<a href='https://github.com/sist2app/sist2/issues/new/choose'>这里</a>.",
         tagline: "标签栏",
         toast: {
             esConnErrTitle: "Elasticsearch连接错误",
@@ -845,7 +846,7 @@ export default {
         },
         debug: "Informacje dla programistów",
         debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
-            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
+            " propozycje zmian, zgłoś to proszę <a href='https://github.com/sist2app/sist2/issues/new/choose'>tutaj</a>.",
         tagline: "Slogan",
         toast: {
             esConnErrTitle: "Problem z połączeniem z Elasticsearch",

sist2-vue/src/main.js

@@ -1,4 +1,3 @@
-import '@babel/polyfill'
 import 'mutationobserver-shim'
 import Vue from 'vue'
 import './plugins/bootstrap-vue'

sist2-vue/src/ml/BPETokenizer.js

@@ -0,0 +1,118 @@
+const inf = Number.POSITIVE_INFINITY;
+const START_TOK = 49406;
+const END_TOK = 49407;
+
+function min(array, key) {
+    return array
+        .reduce((a, b) => (key(a, b) ? b : a))
+}
+
+class TupleSet extends Set {
+    add(elem) {
+        return super.add(elem.join("`"));
+    }
+
+    has(elem) {
+        return super.has(elem.join("`"));
+    }
+
+    toList() {
+        return [...this].map(x => x.split("`"))
+    }
+}
+
+export class BPETokenizer {
+
+    _encoder = null;
+    _bpeRanks = null;
+
+    constructor(encoder, bpeRanks) {
+        this._encoder = encoder;
+        this._bpeRanks = bpeRanks;
+    }
+
+    getPairs(word) {
+        const pairs = new TupleSet();
+
+        let prevChar = word[0];
+        for (let i = 1; i < word.length; i++) {
+            pairs.add([prevChar, word[i]])
+            prevChar = word[i];
+        }
+
+        return pairs.toList();
+    }
+
+    bpe(token) {
+        let word = [...token];
+        word[word.length - 1] += "</w>";
+        let pairs = this.getPairs(word)
+
+        if (pairs.length === 0) {
+            return token + "</w>"
+        }
+
+        while (true) {
+            const bigram = min(pairs, (a, b) => {
+                return (this._bpeRanks[a.join("`")] ?? inf) > (this._bpeRanks[b.join("`") ?? inf])
+            });
+
+            if (this._bpeRanks[bigram.join("`")] === undefined) {
+                break;
+            }
+
+            const [first, second] = bigram;
+            let newWord = [];
+            let i = 0;
+
+            while (i < word.length) {
+                const j = word.indexOf(first, i);
+                if (j === -1) {
+                    newWord.push(...word.slice(i));
+                    break;
+                } else {
+                    newWord.push(...word.slice(i, j));
+                    i = j;
+                }
+
+                if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
+                    newWord.push(first + second);
+                    i += 2;
+                } else {
+                    newWord.push(word[i]);
+                    i += 1;
+                }
+            }
+
+            word = [...newWord]
+            if (word.length === 1) {
+                break;
+            } else {
+                pairs = this.getPairs(word);
+            }
+        }
+
+        return word.join(" ");
+    }
+
+    encode(text) {
+        let bpeTokens = [];
+        text = text.trim();
+        text = text.replaceAll(/\s+/g, " ");
+
+        text
+            .match(/<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[a-zA-Z0-9]+/ig)
+            .forEach(token => {
+                bpeTokens.push(...this.bpe(token).split(" ").map(t => this._encoder[t]));
+            });
+
+        bpeTokens.unshift(START_TOK);
+        bpeTokens = bpeTokens.slice(0, 76);
+        bpeTokens.push(END_TOK);
+        while (bpeTokens.length < 77) {
+            bpeTokens.push(0);
+        }
+
+        return bpeTokens;
+    }
+}

sist2-vue/src/ml/BertNerModel.js

@@ -1,6 +1,8 @@
 import BertTokenizer from "@/ml/BertTokenizer";
-import * as tf from "@tensorflow/tfjs";
 import axios from "axios";
+import {chunk as _chunk} from "underscore";
+import * as ort from "onnxruntime-web";
+import {argMax, downloadToBuffer, ORT_WASM_PATHS} from "@/ml/mlUtils";
 
 export default class BertNerModel {
     vocabUrl;
@@ -29,7 +31,10 @@ export default class BertNerModel {
     }
 
     async loadModel(onProgress) {
-        this._model = await tf.loadGraphModel(this.modelUrl, {onProgress});
+        ort.env.wasm.wasmPaths = ORT_WASM_PATHS;
+        const buf = await downloadToBuffer(this.modelUrl, onProgress);
+
+        this._model = await ort.InferenceSession.create(buf.buffer, {executionProviders: ["wasm"]});
     }
 
     alignLabels(labels, wordIds, words) {
@@ -57,21 +62,28 @@ export default class BertNerModel {
 
     async predict(text, callback) {
         this._previousWordId = null;
-        const encoded = this._tokenizer.encodeText(text, this.inputSize)
+        const encoded = this._tokenizer.encodeText(text, this.inputSize);
 
+        let i = 0;
         for (let chunk of encoded.inputChunks) {
-            const rawResult = tf.tidy(() => this._model.execute({
-                input_ids: tf.tensor2d(chunk.inputIds, [1, this.inputSize], "int32"),
-                token_type_ids: tf.tensor2d(chunk.segmentIds, [1, this.inputSize], "int32"),
-                attention_mask: tf.tensor2d(chunk.inputMask, [1, this.inputSize], "int32"),
-            }));
 
-            const labelIds = await tf.argMax(rawResult, -1);
+            const results = await this._model.run({
-            const labelIdsArray = await labelIds.array();
+                input_ids: new ort.Tensor("int32", chunk.inputIds, [1, this.inputSize]),
-            const labels = labelIdsArray[0].map(id => this.id2label[id]);
+                token_type_ids: new ort.Tensor("int32", chunk.segmentIds, [1, this.inputSize]),
-            rawResult.dispose()
+                attention_mask: new ort.Tensor("int32", chunk.inputMask, [1, this.inputSize]),
+            });
 
-            callback(this.alignLabels(labels, chunk.wordIds, encoded.words))
+            const labelIds = _chunk(results["output"].data, this.id2label.length).map(argMax);
+            const labels = labelIds.map(id => this.id2label[id]);
+
+            callback(this.alignLabels(labels, chunk.wordIds, encoded.words));
+
+            i += 1;
+
+            // give browser some time to repaint
+            if (i % 2 === 0) {
+                await new Promise(resolve => setTimeout(resolve, 0));
+            }
         }
     }
 }

sist2-vue/src/ml/BertTokenizer.js

@@ -1,4 +1,5 @@
 import {zip, chunk} from "underscore";
+import {toInt64} from "@/ml/mlUtils";
 
 const UNK_INDEX = 100;
 const CLS_INDEX = 101;

sist2-vue/src/ml/CLIPTransformerModel.js

@@ -0,0 +1,59 @@
+import * as ort from "onnxruntime-web";
+import {BPETokenizer} from "@/ml/BPETokenizer";
+import axios from "axios";
+import {downloadToBuffer, ORT_WASM_PATHS} from "@/ml/mlUtils";
+import ModelStore from "@/ml/ModelStore";
+
+export class CLIPTransformerModel {
+
+    _modelUrl = null;
+    _tokenizerUrl = null;
+    _model = null;
+    _tokenizer = null;
+
+    constructor(modelUrl, tokenizerUrl) {
+        this._modelUrl = modelUrl;
+        this._tokenizerUrl = tokenizerUrl;
+    }
+
+    async init(onProgress) {
+        await Promise.all([this.loadTokenizer(), this.loadModel(onProgress)]);
+    }
+
+    async loadModel(onProgress) {
+        ort.env.wasm.wasmPaths = ORT_WASM_PATHS;
+        if (window.crossOriginIsolated) {
+            ort.env.wasm.numThreads = 2;
+        }
+
+        let buf = await ModelStore.get(this._modelUrl);
+        if (!buf) {
+            buf = await downloadToBuffer(this._modelUrl, onProgress);
+            await ModelStore.set(this._modelUrl, buf);
+        }
+
+        this._model = await ort.InferenceSession.create(buf.buffer, {
+            executionProviders: ["wasm"],
+        });
+    }
+
+    async loadTokenizer() {
+        const resp = await axios.get(this._tokenizerUrl);
+        this._tokenizer = new BPETokenizer(resp.data.encoder, resp.data.bpe_ranks)
+    }
+
+    async predict(text) {
+        const tokenized = this._tokenizer.encode(text);
+
+        const inputs = {
+            input_ids: new ort.Tensor("int32", tokenized, [1, 77])
+        };
+
+        const results = await this._model.run(inputs);
+
+        return Array.from(
+            Object.values(results)
+                .find(result => result.size === 512).data
+        );
+    }
+}

sist2-vue/src/ml/ModelStore.js

@@ -0,0 +1,67 @@
+class ModelStore {
+
+    _ok;
+    _db;
+    _resolve;
+    _loadingPromise;
+
+    constructor() {
+        const request = window.indexedDB.open("ModelStore", 1);
+
+        request.onerror = () => {
+            this._ok = false;
+        }
+
+        request.onupgradeneeded = event => {
+            const db = event.target.result;
+            db.createObjectStore("models");
+        }
+
+        request.onsuccess = () => {
+            this._ok = true;
+            this._db = request.result;
+
+            this._resolve();
+        }
+
+        this._loadingPromise = new Promise(resolve => this._resolve = resolve);
+    }
+
+    async get(key) {
+        await this._loadingPromise;
+
+        const req = this._db.transaction(["models"], "readwrite")
+            .objectStore("models")
+            .get(key);
+
+        return new Promise(resolve => {
+            req.onsuccess = event => {
+                resolve(event.target.result);
+            };
+            req.onerror = event => {
+                console.log("ERROR:");
+                console.log(event);
+                resolve(null);
+            };
+        });
+    }
+
+    async set(key, val) {
+        await this._loadingPromise;
+
+        const req = this._db.transaction(["models"], "readwrite")
+            .objectStore("models")
+            .put(val, key);
+
+        return new Promise(resolve => {
+            req.onsuccess = () => {
+                resolve(true);
+            };
+            req.onerror = () => {
+                resolve(false);
+            };
+        });
+    }
+}
+
+export default new ModelStore();

sist2-vue/src/ml/mlUtils.js

@@ -0,0 +1,46 @@
+export async function downloadToBuffer(url, onProgress) {
+    const resp = await fetch(url);
+
+    const contentLength = +resp.headers.get("Content-Length");
+    const buf = new Uint8ClampedArray(contentLength);
+    const reader = resp.body.getReader();
+    let cursor = 0;
+
+    if (onProgress) {
+        onProgress(0);
+    }
+
+    while (true) {
+        const {done, value} = await reader.read();
+
+        if (done) {
+            break;
+        }
+
+        buf.set(value, cursor);
+        cursor += value.length;
+
+        if (onProgress) {
+            onProgress(cursor / contentLength);
+        }
+    }
+
+    return buf;
+}
+
+export function argMax(array) {
+    return array
+        .map((x, i) => [x, i])
+        .reduce((r, a) => (a[0] > r[0] ? a : r))[1];
+}
+
+export function toInt64(array) {
+    return new BigInt64Array(array.map(BigInt));
+}
+
+export const ORT_WASM_PATHS = {
+    "ort-wasm-simd.wasm": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.15.1/dist/ort-wasm-simd.wasm",
+    "ort-wasm.wasm": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.15.1/dist/ort-wasm.wasm",
+    "ort-wasm-simd-threaded.wasm": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.15.1/dist/ort-wasm-simd-threaded.wasm",
+    "ort-wasm-threaded.wasm": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.15.1/dist/ort-wasm-threaded.wasm",
+}

sist2-vue/src/router/index.js

@@ -21,7 +21,7 @@ const authGuard = (to, from, next) => {
     next();
 }
 
-const routes: Array<RouteConfig> = [
+const routes = [
     {
         path: "/",
         name: "SearchPage",

sist2-vue/src/store/index.js

@@ -1,20 +1,18 @@
 import Vue from "vue"
 import Vuex from "vuex"
-import VueRouter, {Route} from "vue-router";
-import {EsHit, EsResult, EsTag, Index} from "@/Sist2Api";
 import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
 import {getInstance} from "@/plugins/auth0.js";
 
 const CONF_VERSION = 3;
 
-Vue.use(Vuex)
+Vue.use(Vuex);
 
 export default new Vuex.Store({
     state: {
         seed: 0,
-        indices: [] as Index[],
+        indices: [],
-        tags: [] as EsTag[],
+        tags: [],
-        sist2Info: null as any,
+        sist2Info: null,
 
         sizeMin: undefined,
         sizeMax: undefined,
@@ -23,6 +21,9 @@ export default new Vuex.Store({
         dateMin: undefined,
         dateMax: undefined,
         searchText: "",
+        embeddingText: "",
+        embedding: null,
+        embeddingDoc: null,
         pathText: "",
         sortMode: "score",
 
@@ -57,16 +58,16 @@ export default new Vuex.Store({
         optVidPreviewInterval: 700,
         optSimpleLightbox: true,
         optShowTagPickerFilter: true,
-        optMlRepositories: "https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json",
+        optMlRepositories: "https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json",
         optAutoAnalyze: false,
         optMlDefaultModel: null,
 
-        _onLoadSelectedIndices: [] as string[],
+        _onLoadSelectedIndices: [],
-        _onLoadSelectedMimeTypes: [] as string[],
+        _onLoadSelectedMimeTypes: [],
-        _onLoadSelectedTags: [] as string[],
+        _onLoadSelectedTags: [],
-        selectedIndices: [] as Index[],
+        selectedIndices: [],
-        selectedMimeTypes: [] as string[],
+        selectedMimeTypes: [],
-        selectedTags: [] as string[],
+        selectedTags: [],
 
         lastQueryResults: null,
         firstQueryResults: null,
@@ -77,10 +78,10 @@ export default new Vuex.Store({
         uiSqliteMode: false,
         uiLightboxIsOpen: false,
         uiShowLightbox: false,
-        uiLightboxSources: [] as string[],
+        uiLightboxSources: [],
-        uiLightboxThumbs: [] as string[],
+        uiLightboxThumbs: [],
-        uiLightboxCaptions: [] as any[],
+        uiLightboxCaptions: [],
-        uiLightboxTypes: [] as string[],
+        uiLightboxTypes: [],
         uiLightboxKey: 0,
         uiLightboxSlide: 0,
         uiReachedScrollEnd: false,
@@ -88,13 +89,14 @@ export default new Vuex.Store({
         uiDetailsMimeAgg: null,
         uiShowDetails: false,
 
-        uiMimeMap: [] as any[],
+        uiMimeMap: [],
 
         auth0Token: null,
-        mlModel: {
+        nerModel: {
             model: null,
             name: null
         },
+        embeddingsModel: null
     },
     mutations: {
         setUiShowDetails: (state, val) => state.uiShowDetails = val,
@@ -118,7 +120,7 @@ export default new Vuex.Store({
             if (state._onLoadSelectedIndices.length > 0) {
 
                 state.selectedIndices = val.filter(
-                    (idx: Index) => state._onLoadSelectedIndices.some(prefix => idx.id.startsWith(prefix))
+                    (idx) => state._onLoadSelectedIndices.some(id => id === idx.id.toString(16))
                 );
             } else {
                 state.selectedIndices = val;
@@ -129,6 +131,9 @@ export default new Vuex.Store({
         setDateBoundsMin: (state, val) => state.dateBoundsMin = val,
         setDateBoundsMax: (state, val) => state.dateBoundsMax = val,
         setSearchText: (state, val) => state.searchText = val,
+        setEmbeddingText: (state, val) => state.embeddingText = val,
+        setEmbedding: (state, val) => state.embedding = val,
+        setEmbeddingDoc: (state, val) => state.embeddingDoc = val,
         setFuzzy: (state, val) => state.fuzzy = val,
         setLastQueryResult: (state, val) => state.lastQueryResults = val,
         setFirstQueryResult: (state, val) => state.firstQueryResults = val,
@@ -138,18 +143,18 @@ export default new Vuex.Store({
         setSelectedIndices: (state, val) => state.selectedIndices = val,
         setSelectedMimeTypes: (state, val) => state.selectedMimeTypes = val,
         setSelectedTags: (state, val) => state.selectedTags = val,
-        setUiLightboxIsOpen: (state, val: boolean) => state.uiLightboxIsOpen = val,
+        setUiLightboxIsOpen: (state, val) => state.uiLightboxIsOpen = val,
-        _setUiShowLightbox: (state, val: boolean) => state.uiShowLightbox = val,
+        _setUiShowLightbox: (state, val) => state.uiShowLightbox = val,
-        setUiLightboxKey: (state, val: number) => state.uiLightboxKey = val,
+        setUiLightboxKey: (state, val) => state.uiLightboxKey = val,
-        _setKeySequence: (state, val: number) => state.keySequence = val,
+        _setKeySequence: (state, val) => state.keySequence = val,
-        _setQuerySequence: (state, val: number) => state.querySequence = val,
+        _setQuerySequence: (state, val) => state.querySequence = val,
         addLightboxSource: (state, {source, thumbnail, caption, type}) => {
             state.uiLightboxSources.push(source);
             state.uiLightboxThumbs.push(thumbnail);
             state.uiLightboxCaptions.push(caption);
             state.uiLightboxTypes.push(type);
         },
-        setUiLightboxSlide: (state, val: number) => state.uiLightboxSlide = val,
+        setUiLightboxSlide: (state, val) => state.uiLightboxSlide = val,
 
         setUiLightboxSources: (state, val) => state.uiLightboxSources = val,
         setUiLightboxThumbs: (state, val) => state.uiLightboxThumbs = val,
@@ -212,7 +217,8 @@ export default new Vuex.Store({
             // noop
         },
         setAuth0Token: (state, val) => state.auth0Token = val,
-        setMlModel: (state, val) => state.mlModel = val,
+        setNerModel: (state, val) => state.nerModel = val,
+        setEmbeddingsModel: (state, val) => state.embeddingsModel = val,
     },
     actions: {
         setSist2Info: (store, val) => {
@@ -222,7 +228,7 @@ export default new Vuex.Store({
                 store.commit("setOptLang", val.lang);
             }
         },
-        loadFromArgs({commit}, route: Route) {
+        loadFromArgs({commit}, route) {
 
             if (route.query.q) {
                 commit("setSearchText", route.query.q);
@@ -257,11 +263,11 @@ export default new Vuex.Store({
             }
 
             if (route.query.m) {
-                commit("_setOnLoadSelectedMimeTypes", deserializeMimes(route.query.m as string));
+                commit("_setOnLoadSelectedMimeTypes", deserializeMimes(route.query.m));
             }
 
             if (route.query.t) {
-                commit("_setOnLoadSelectedTags", (route.query.t as string).split(","));
+                commit("_setOnLoadSelectedTags", (route.query.t).split(","));
             }
 
             if (route.query.sort) {
@@ -272,7 +278,7 @@ export default new Vuex.Store({
                 commit("setSeed", Number(route.query.seed));
             }
         },
-        async updateArgs({state}, router: VueRouter) {
+        async updateArgs({state}, router) {
 
             if (router.currentRoute.path !== "/") {
                 return;
@@ -282,14 +288,14 @@ export default new Vuex.Store({
                 query: {
                     q: state.searchText.trim() ? state.searchText.trim().replace(/\s+/g, " ") : undefined,
                     fuzzy: state.fuzzy ? null : undefined,
-                    i: state.selectedIndices ? state.selectedIndices.map((idx: Index) => idx.idPrefix) : undefined,
+                    i: state.selectedIndices ? state.selectedIndices.map((idx) => idx.id.toString(16)) : undefined,
                     dMin: state.dateMin,
                     dMax: state.dateMax,
                     sMin: state.sizeMin,
                     sMax: state.sizeMax,
                     path: state.pathText ? state.pathText : undefined,
                     m: serializeMimes(state.selectedMimeTypes),
-                    t: state.selectedTags.length == 0 ? undefined : state.selectedTags.join(","),
+                    t: state.selectedTags.length === 0 ? undefined : state.selectedTags.join(","),
                     sort: state.sortMode === "score" ? undefined : state.sortMode,
                     seed: state.sortMode === "random" ? state.seed.toString() : undefined
                 }
@@ -298,11 +304,11 @@ export default new Vuex.Store({
             });
         },
         updateConfiguration({state}) {
-            const conf = {} as any;
+            const conf = {};
 
             Object.keys(state).forEach((key) => {
                 if (key.startsWith("opt")) {
-                    conf[key] = (state as any)[key];
+                    conf[key] = (state)[key];
                 }
             });
 
@@ -315,14 +321,14 @@ export default new Vuex.Store({
             if (confString) {
                 const conf = JSON.parse(confString);
 
-                if (!("version" in conf) || conf["version"] != CONF_VERSION) {
+                if (!("version" in conf) || conf["version"] !== CONF_VERSION) {
                     localStorage.removeItem("sist2_configuration");
                     window.location.reload();
                 }
 
                 Object.keys(state).forEach((key) => {
                     if (key.startsWith("opt")) {
-                        (state as any)[key] = conf[key];
+                        (state)[key] = conf[key];
                     }
                 });
             }
@@ -370,13 +376,15 @@ export default new Vuex.Store({
     },
     modules: {},
     getters: {
-        mlModel: (state) => state.mlModel,
+        nerModel: (state) => state.nerModel,
+        embeddingsModel: (state) => state.embeddingsModel,
+        embedding: (state) => state.embedding,
         seed: (state) => state.seed,
         getPathText: (state) => state.pathText,
         indices: state => state.indices,
         sist2Info: state => state.sist2Info,
         indexMap: state => {
-            const map = {} as any;
+            const map = {};
             state.indices.forEach(idx => map[idx.id] = idx);
             return map;
         },
@@ -389,17 +397,18 @@ export default new Vuex.Store({
         sizeMin: state => state.sizeMin,
         sizeMax: state => state.sizeMax,
         searchText: state => state.searchText,
+        embeddingText: state => state.embeddingText,
         pathText: state => state.pathText,
         fuzzy: state => state.fuzzy,
         size: state => state.optSize,
         sortMode: state => state.sortMode,
         lastQueryResult: state => state.lastQueryResults,
-        lastDoc: function (state): EsHit | null {
+        lastDoc: function (state) {
             if (state.lastQueryResults == null) {
                 return null;
             }
 
-            return (state.lastQueryResults as unknown as EsResult).hits.hits.slice(-1)[0];
+            return (state.lastQueryResults).hits.hits.slice(-1)[0];
         },
         uiShowLightbox: state => state.uiShowLightbox,
         uiLightboxSources: state => state.uiLightboxSources,
@@ -440,7 +449,7 @@ export default new Vuex.Store({
         optMlRepositories: state => state.optMlRepositories,
         mlRepositoryList: state => {
             const repos = state.optMlRepositories.split("\n")
-            return repos[0] == "" ? [] : repos;
+            return repos[0] === "" ? [] : repos;
         },
         optMlDefaultModel: state => state.optMlDefaultModel,
         optAutoAnalyze: state => state.optAutoAnalyze,

sist2-vue/src/util-js.js

@@ -1,139 +0,0 @@
-export function mergeTooltips(slider, threshold, separator, fixTooltips) {
-
-    const isMobile = window.innerWidth <= 650;
-    if (isMobile) {
-        threshold = 25;
-    }
-
-    const textIsRtl = getComputedStyle(slider).direction === 'rtl';
-    const isRtl = slider.noUiSlider.options.direction === 'rtl';
-    const isVertical = slider.noUiSlider.options.orientation === 'vertical';
-    const tooltips = slider.noUiSlider.getTooltips();
-    const origins = slider.noUiSlider.getOrigins();
-
-    // Move tooltips into the origin element. The default stylesheet handles this.
-    tooltips.forEach(function (tooltip, index) {
-        if (tooltip) {
-            origins[index].appendChild(tooltip);
-        }
-    });
-
-    slider.noUiSlider.on('update', function (values, handle, unencoded, tap, positions) {
-
-        const pools = [[]];
-        const poolPositions = [[]];
-        const poolValues = [[]];
-        let atPool = 0;
-
-        // Assign the first tooltip to the first pool, if the tooltip is configured
-        if (tooltips[0]) {
-            pools[0][0] = 0;
-            poolPositions[0][0] = positions[0];
-            poolValues[0][0] = values[0];
-        }
-
-        for (let i = 1; i < positions.length; i++) {
-            if (!tooltips[i] || (positions[i] - positions[i - 1]) > threshold) {
-                atPool++;
-                pools[atPool] = [];
-                poolValues[atPool] = [];
-                poolPositions[atPool] = [];
-            }
-
-            if (tooltips[i]) {
-                pools[atPool].push(i);
-                poolValues[atPool].push(values[i]);
-                poolPositions[atPool].push(positions[i]);
-            }
-        }
-
-        pools.forEach(function (pool, poolIndex) {
-            const handlesInPool = pool.length;
-
-            for (let j = 0; j < handlesInPool; j++) {
-                const handleNumber = pool[j];
-
-                if (j === handlesInPool - 1) {
-                    let offset = 0;
-
-                    poolPositions[poolIndex].forEach(function (value) {
-                        offset += 1000 - 10 * value;
-                    });
-
-                    const direction = isVertical ? 'bottom' : 'right';
-                    const last = isRtl ? 0 : handlesInPool - 1;
-                    const lastOffset = 1000 - 10 * poolPositions[poolIndex][last];
-                    offset = (textIsRtl && !isVertical ? 100 : 0) + (offset / handlesInPool) - lastOffset;
-
-                    // Center this tooltip over the affected handles
-                    tooltips[handleNumber].innerHTML = poolValues[poolIndex].join(separator);
-                    tooltips[handleNumber].style.display = 'block';
-
-                    tooltips[handleNumber].style[direction] = offset + '%';
-                } else {
-                    // Hide this tooltip
-                    tooltips[handleNumber].style.display = 'none';
-                }
-            }
-        });
-
-        if (fixTooltips) {
-            const isMobile = window.innerWidth <= 650;
-            const len = isMobile ? 20 : 5;
-
-            if (positions[0] < len) {
-                tooltips[0].style.right = `${(1 - ((positions[0]) / len)) * -35}px`
-            } else {
-                tooltips[0].style.right = "0"
-            }
-
-            if (positions[1] > (100 - len)) {
-                tooltips[1].style.right = `${((positions[1] - (100 - len)) / len) * 35}px`
-            } else {
-                tooltips[1].style.right = "0"
-            }
-        }
-    });
-}
-
-
-export function burrow(table, addSelfDir, rootName) {
-    const root = {};
-    table.forEach(row => {
-        let layer = root;
-
-        row.taxonomy.forEach(key => {
-            layer[key] = key in layer ? layer[key] : {};
-            layer = layer[key];
-        });
-        if (Object.keys(layer).length === 0) {
-            layer["$size$"] = row.size;
-        } else if (addSelfDir) {
-            layer["."] = {
-                "$size$": row.size,
-            };
-        }
-    });
-
-    const descend = function (obj, depth) {
-        return Object.keys(obj).filter(k => k !== "$size$").map(k => {
-            const child = {
-                name: k,
-                depth: depth,
-                value: 0,
-                children: descend(obj[k], depth + 1)
-            };
-            if ("$size$" in obj[k]) {
-                child.value = obj[k]["$size$"];
-            }
-            return child;
-        });
-    };
-
-    return {
-        name: rootName,
-        children: descend(root, 1),
-        value: 0,
-        depth: 0,
-    }
-}

sist2-vue/src/util.js

@@ -0,0 +1,329 @@
+export function mergeTooltips(slider, threshold, separator, fixTooltips) {
+
+    const isMobile = window.innerWidth <= 650;
+    if (isMobile) {
+        threshold = 25;
+    }
+
+    const textIsRtl = getComputedStyle(slider).direction === 'rtl';
+    const isRtl = slider.noUiSlider.options.direction === 'rtl';
+    const isVertical = slider.noUiSlider.options.orientation === 'vertical';
+    const tooltips = slider.noUiSlider.getTooltips();
+    const origins = slider.noUiSlider.getOrigins();
+
+    // Move tooltips into the origin element. The default stylesheet handles this.
+    tooltips.forEach(function (tooltip, index) {
+        if (tooltip) {
+            origins[index].appendChild(tooltip);
+        }
+    });
+
+    slider.noUiSlider.on('update', function (values, handle, unencoded, tap, positions) {
+
+        const pools = [[]];
+        const poolPositions = [[]];
+        const poolValues = [[]];
+        let atPool = 0;
+
+        // Assign the first tooltip to the first pool, if the tooltip is configured
+        if (tooltips[0]) {
+            pools[0][0] = 0;
+            poolPositions[0][0] = positions[0];
+            poolValues[0][0] = values[0];
+        }
+
+        for (let i = 1; i < positions.length; i++) {
+            if (!tooltips[i] || (positions[i] - positions[i - 1]) > threshold) {
+                atPool++;
+                pools[atPool] = [];
+                poolValues[atPool] = [];
+                poolPositions[atPool] = [];
+            }
+
+            if (tooltips[i]) {
+                pools[atPool].push(i);
+                poolValues[atPool].push(values[i]);
+                poolPositions[atPool].push(positions[i]);
+            }
+        }
+
+        pools.forEach(function (pool, poolIndex) {
+            const handlesInPool = pool.length;
+
+            for (let j = 0; j < handlesInPool; j++) {
+                const handleNumber = pool[j];
+
+                if (j === handlesInPool - 1) {
+                    let offset = 0;
+
+                    poolPositions[poolIndex].forEach(function (value) {
+                        offset += 1000 - 10 * value;
+                    });
+
+                    const direction = isVertical ? 'bottom' : 'right';
+                    const last = isRtl ? 0 : handlesInPool - 1;
+                    const lastOffset = 1000 - 10 * poolPositions[poolIndex][last];
+                    offset = (textIsRtl && !isVertical ? 100 : 0) + (offset / handlesInPool) - lastOffset;
+
+                    // Center this tooltip over the affected handles
+                    tooltips[handleNumber].innerHTML = poolValues[poolIndex].join(separator);
+                    tooltips[handleNumber].style.display = 'block';
+
+                    tooltips[handleNumber].style[direction] = offset + '%';
+                } else {
+                    // Hide this tooltip
+                    tooltips[handleNumber].style.display = 'none';
+                }
+            }
+        });
+
+        if (fixTooltips) {
+            const isMobile = window.innerWidth <= 650;
+            const len = isMobile ? 20 : 5;
+
+            if (positions[0] < len) {
+                tooltips[0].style.right = `${(1 - ((positions[0]) / len)) * -35}px`
+            } else {
+                tooltips[0].style.right = "0"
+            }
+
+            if (positions[1] > (100 - len)) {
+                tooltips[1].style.right = `${((positions[1] - (100 - len)) / len) * 35}px`
+            } else {
+                tooltips[1].style.right = "0"
+            }
+        }
+    });
+}
+
+
+export function burrow(table, addSelfDir, rootName) {
+    const root = {};
+    table.forEach(row => {
+        let layer = root;
+
+        row.taxonomy.forEach(key => {
+            layer[key] = key in layer ? layer[key] : {};
+            layer = layer[key];
+        });
+        if (Object.keys(layer).length === 0) {
+            layer["$size$"] = row.size;
+        } else if (addSelfDir) {
+            layer["."] = {
+                "$size$": row.size,
+            };
+        }
+    });
+
+    const descend = function (obj, depth) {
+        return Object.keys(obj).filter(k => k !== "$size$").map(k => {
+            const child = {
+                name: k,
+                depth: depth,
+                value: 0,
+                children: descend(obj[k], depth + 1)
+            };
+            if ("$size$" in obj[k]) {
+                child.value = obj[k]["$size$"];
+            }
+            return child;
+        });
+    };
+
+    return {
+        name: rootName,
+        children: descend(root, 1),
+        value: 0,
+        depth: 0,
+    }
+}
+
+
+export function ext(hit) {
+    return srcExt(hit._source)
+}
+
+export function srcExt(src) {
+    return Object.prototype.hasOwnProperty.call(src, "extension")
+    && src["extension"] !== "" ? "." + src["extension"] : "";
+}
+
+export function strUnescape(str) {
+    let result = "";
+
+    for (let i = 0; i < str.length; i++) {
+        const c = str[i];
+        const next = str[i + 1];
+
+        if (c === "]") {
+            if (next === "]") {
+                result += c;
+                i += 1;
+            } else {
+                result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
+                i += 2;
+            }
+        } else {
+            result += c;
+        }
+    }
+    return result;
+}
+
+const thresh = 1000;
+const units = ["k", "M", "G", "T", "P", "E", "Z", "Y"];
+
+export function humanFileSize(bytes) {
+    if (bytes === 0) {
+        return "0 B"
+    }
+
+    if (Math.abs(bytes) < thresh) {
+        return bytes + ' B';
+    }
+    let u = -1;
+    do {
+        bytes /= thresh;
+        ++u;
+    } while (Math.abs(bytes) >= thresh && u < units.length - 1);
+
+    return bytes.toFixed(1) + units[u];
+}
+
+export function humanTime(sec_num) {
+    sec_num = Math.floor(sec_num);
+    const hours = Math.floor(sec_num / 3600);
+    const minutes = Math.floor((sec_num - (hours * 3600)) / 60);
+    const seconds = sec_num - (hours * 3600) - (minutes * 60);
+
+    if (sec_num < 60) {
+        return `${sec_num}s`
+    }
+
+    if (sec_num < 3600) {
+        return `${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
+    }
+
+    return `${hours < 10 ? "0" : ""}${hours}:${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
+}
+
+export function humanDate(numMilis) {
+    const date = (new Date(numMilis * 1000));
+    return date.getUTCFullYear() + "-" + ("0" + (date.getUTCMonth() + 1)).slice(-2) + "-" + ("0" + date.getUTCDate()).slice(-2)
+
+}
+
+export function lum(c) {
+    c = c.substring(1);
+    const rgb = parseInt(c, 16);
+    const r = (rgb >> 16) & 0xff;
+    const g = (rgb >> 8) & 0xff;
+    const b = (rgb >> 0) & 0xff;
+
+    return 0.2126 * r + 0.7152 * g + 0.0722 * b;
+}
+
+
+export function getSelectedTreeNodes(tree) {
+    const selectedNodes = new Set();
+
+    const selected = tree.selected();
+
+    for (let i = 0; i < selected.length; i++) {
+
+        if (selected[i].id === "any") {
+            return ["any"]
+        }
+
+        //Only get children
+        if (selected[i].text.indexOf("(") !== -1) {
+            if (selected[i].values) {
+                selectedNodes.add(selected[i].values.slice(-1)[0]);
+            } else {
+                selectedNodes.add(selected[i].id);
+            }
+        }
+    }
+
+    return Array.from(selectedNodes);
+}
+
+export function getTreeNodeAttributes(tree) {
+    const nodes = tree.selectable();
+    const attributes = {};
+
+    for (let i = 0; i < nodes.length; i++) {
+
+        let id = null;
+
+        if (nodes[i].text.indexOf("(") !== -1 && nodes[i].values) {
+            id = nodes[i].values.slice(-1)[0];
+        } else {
+            id = nodes[i].id
+        }
+
+        attributes[id] = {
+            checked: nodes[i].itree.state.checked,
+            collapsed: nodes[i].itree.state.collapsed,
+        }
+    }
+
+    return attributes;
+}
+
+
+export function serializeMimes(mimes) {
+    if (mimes.length === 0) {
+        return undefined;
+    }
+    return mimes.map(mime => compressMime(mime)).join("");
+}
+
+export function deserializeMimes(mimeString) {
+    return mimeString
+        .replaceAll(/([IVATUF])/g, "$$$&")
+        .split("$")
+        .map(mime => decompressMime(mime))
+        .slice(1) // Ignore the first (empty) token
+}
+
+export function compressMime(mime) {
+    return mime
+        .replace("image/", "I")
+        .replace("video/", "V")
+        .replace("application/", "A")
+        .replace("text/", "T")
+        .replace("audio/", "U")
+        .replace("font/", "F")
+        .replace("+", ",")
+        .replace("x-", "X")
+}
+
+export function decompressMime(mime) {
+    return mime
+        .replace("I", "image/")
+        .replace("V", "video/")
+        .replace("A", "application/")
+        .replace("T", "text/")
+        .replace("U", "audio/")
+        .replace("F", "font/")
+        .replace(",", "+")
+        .replace("X", "x-")
+}
+
+export function randomSeed() {
+    return Math.round(Math.random() * 100000);
+}
+
+export function sid(doc) {
+    if (doc._id.includes(".")) {
+        return doc._id
+    }
+
+    const num = BigInt(doc._id);
+
+    const indexId = (num >> BigInt(32));
+    const docId = num & BigInt(0xFFFFFFFF);
+
+    return indexId.toString(16).padStart(8, "0") + "." + docId.toString(16).padStart(8, "0");
+}

sist2-vue/src/util.ts

@@ -1,177 +0,0 @@
-import {EsHit} from "@/Sist2Api";
-
-export function ext(hit: EsHit) {
-    return srcExt(hit._source)
-}
-
-export function srcExt(src) {
-    return Object.prototype.hasOwnProperty.call(src, "extension")
-        && src["extension"] !== "" ? "." + src["extension"] : "";
-}
-
-export function strUnescape(str: string): string {
-    let result = "";
-
-    for (let i = 0; i < str.length; i++) {
-        const c = str[i];
-        const next = str[i + 1];
-
-        if (c === "]") {
-            if (next === "]") {
-                result += c;
-                i += 1;
-            } else {
-                result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
-                i += 2;
-            }
-        } else {
-            result += c;
-        }
-    }
-    return result;
-}
-
-const thresh = 1000;
-const units = ["k", "M", "G", "T", "P", "E", "Z", "Y"];
-
-export function humanFileSize(bytes: number): string {
-    if (bytes === 0) {
-        return "0 B"
-    }
-
-    if (Math.abs(bytes) < thresh) {
-        return bytes + ' B';
-    }
-    let u = -1;
-    do {
-        bytes /= thresh;
-        ++u;
-    } while (Math.abs(bytes) >= thresh && u < units.length - 1);
-
-    return bytes.toFixed(1) + units[u];
-}
-
-export function humanTime(sec_num: number): string {
-    sec_num = Math.floor(sec_num);
-    const hours = Math.floor(sec_num / 3600);
-    const minutes = Math.floor((sec_num - (hours * 3600)) / 60);
-    const seconds = sec_num - (hours * 3600) - (minutes * 60);
-
-    if (sec_num < 60) {
-        return `${sec_num}s`
-    }
-
-    if (sec_num < 3600) {
-        return `${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
-    }
-
-    return `${hours < 10 ? "0" : ""}${hours}:${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
-}
-
-export function humanDate(numMilis: number): string {
-    const date = (new Date(numMilis * 1000));
-    return date.getUTCFullYear() + "-" + ("0" + (date.getUTCMonth() + 1)).slice(-2) + "-" + ("0" + date.getUTCDate()).slice(-2)
-
-}
-
-export function lum(c: string) {
-    c = c.substring(1);
-    const rgb = parseInt(c, 16);
-    const r = (rgb >> 16) & 0xff;
-    const g = (rgb >> 8) & 0xff;
-    const b = (rgb >> 0) & 0xff;
-
-    return 0.2126 * r + 0.7152 * g + 0.0722 * b;
-}
-
-
-export function getSelectedTreeNodes(tree: any) {
-    const selectedNodes = new Set();
-
-    const selected = tree.selected();
-
-    for (let i = 0; i < selected.length; i++) {
-
-        if (selected[i].id === "any") {
-            return ["any"]
-        }
-
-        //Only get children
-        if (selected[i].text.indexOf("(") !== -1) {
-            if (selected[i].values) {
-                selectedNodes.add(selected[i].values.slice(-1)[0]);
-            } else {
-                selectedNodes.add(selected[i].id);
-            }
-        }
-    }
-
-    return Array.from(selectedNodes);
-}
-
-export function getTreeNodeAttributes(tree: any) {
-    const nodes = tree.selectable();
-    const attributes = {};
-
-    for (let i = 0; i < nodes.length; i++) {
-
-        let id = null;
-
-        if (nodes[i].text.indexOf("(") !== -1 && nodes[i].values) {
-            id = nodes[i].values.slice(-1)[0];
-        } else {
-            id = nodes[i].id
-        }
-
-        attributes[id] = {
-            checked: nodes[i].itree.state.checked,
-            collapsed: nodes[i].itree.state.collapsed,
-        }
-    }
-
-    return attributes;
-}
-
-
-export function serializeMimes(mimes: string[]): string | undefined {
-    if (mimes.length == 0) {
-        return undefined;
-    }
-    return mimes.map(mime => compressMime(mime)).join("");
-}
-
-export function deserializeMimes(mimeString: string): string[] {
-    return mimeString
-        .replaceAll(/([IVATUF])/g, "$$$&")
-        .split("$")
-        .map(mime => decompressMime(mime))
-        .slice(1) // Ignore the first (empty) token
-}
-
-export function compressMime(mime: string): string {
-    return mime
-        .replace("image/", "I")
-        .replace("video/", "V")
-        .replace("application/", "A")
-        .replace("text/", "T")
-        .replace("audio/", "U")
-        .replace("font/", "F")
-        .replace("+", ",")
-        .replace("x-", "X")
-}
-
-export function decompressMime(mime: string): string {
-    return mime
-        .replace("I", "image/")
-        .replace("V", "video/")
-        .replace("A", "application/")
-        .replace("T", "text/")
-        .replace("U", "audio/")
-        .replace("F", "font/")
-        .replace(",", "+")
-        .replace("X", "x-")
-}
-
-export function randomSeed(): number {
-    return Math.round(Math.random() * 100000);
-}

sist2-vue/src/views/Configuration.vue

@@ -81,6 +81,7 @@
                             <li><code>doc.artist</code></li>
                             <li><code>doc.title</code></li>
                             <li><code>doc.genre</code></li>
+                            <li><code>doc.media_comment</code></li>
                             <li><code>doc.album_artist</code></li>
                             <li><code>doc.exif_make</code></li>
                             <li><code>doc.exif_model</code></li>
@@ -136,7 +137,7 @@
                         {{ $t("opt.fuzzy") }}
                     </b-form-checkbox>
 
-                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optSearchInPath" @input="setOptSearchInPath">{{
+                    <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
                         $t("opt.searchInPath")
                         }}
                     </b-form-checkbox>

sist2-vue/src/views/FilePage.vue

@@ -14,7 +14,7 @@
             <!-- Audio player-->
             <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
                    :type="doc._source.mime"
-             :src="`f/${doc._source.index}/${doc._id}`"></audio>
+                   :src="`f/${sid(doc)}`"></audio>
 
             <InfoTable :doc="doc" v-if="doc"></InfoTable>
 
@@ -32,9 +32,8 @@
 import Preloader from "@/components/Preloader.vue";
 import InfoTable from "@/components/InfoTable.vue";
 import Sist2Api from "@/Sist2Api";
-import {ext} from "@/util";
+import {ext, sid} from "@/util";
 import Vue from "vue";
-import sist2 from "@/Sist2Api";
 import FullThumbnail from "@/components/FullThumbnail";
 
 export default Vue.extend({
@@ -53,8 +52,9 @@ export default Vue.extend({
     },
     methods: {
         ext: ext,
+        sid: sid,
         onThumbnailClick() {
-      window.open(`/f/${this.doc.index}/${this.doc._id}`, "_blank");
+            window.open(`/f/${sid(this.doc)}`, "_blank");
         },
         findByCustomField(field, id) {
             return {

sist2-vue/src/views/SearchPage.vue

@@ -13,6 +13,7 @@
 
         <b-card v-show="!uiLoading && !showEsConnectionError" id="search-panel">
             <SearchBar @show-help="showHelp=true"></SearchBar>
+            <EmbeddingsSearchBar v-if="hasEmbeddings" class="mt-3"></EmbeddingsSearchBar>
             <b-row>
                 <b-col style="height: 70px;" sm="6">
                     <SizeSlider></SizeSlider>
@@ -58,16 +59,15 @@
     </div>
 </template>
 
-<script lang="ts">
+<script>
+import {sid} from "@/util";
 import Preloader from "@/components/Preloader.vue";
 import {mapActions, mapGetters, mapMutations} from "vuex";
-import sist2 from "../Sist2Api";
-import Sist2Api, {EsHit, EsResult} from "../Sist2Api";
 import SearchBar from "@/components/SearchBar.vue";
 import IndexPicker from "@/components/IndexPicker.vue";
 import Vue from "vue";
 import Sist2Query from "@/Sist2ElasticsearchQuery";
-import _debounce from "lodash/debounce";
+import {debounce as _debounce} from "underscore";
 import DocCardWall from "@/components/DocCardWall.vue";
 import Lightbox from "@/components/Lightbox.vue";
 import LightboxCaption from "@/components/LightboxCaption.vue";
@@ -79,11 +79,13 @@ import DateSlider from "@/components/DateSlider.vue";
 import TagPicker from "@/components/TagPicker.vue";
 import DocList from "@/components/DocList.vue";
 import HelpDialog from "@/components/HelpDialog.vue";
-import Sist2SqliteQuery from "@/Sist2SqliteQuery";
+import EmbeddingsSearchBar from "@/components/EmbeddingsSearchBar.vue";
+import Sist2Api from "@/Sist2Api";
 
 
 export default Vue.extend({
     components: {
+        EmbeddingsSearchBar,
         HelpDialog,
         DocList,
         TagPicker,
@@ -91,10 +93,9 @@ export default Vue.extend({
         SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
     },
     data: () => ({
-        loading: false,
         uiLoading: true,
-        search: undefined as any,
+        search: undefined,
-        docs: [] as EsHit[],
+        docs: [],
         docIds: new Set(),
         docChecksums: new Set(),
         searchBusy: false,
@@ -104,20 +105,23 @@ export default Vue.extend({
     }),
     computed: {
         ...mapGetters(["indices", "optDisplay"]),
+        hasEmbeddings() {
+            return Sist2Api.models().length > 0;
+        },
     },
     mounted() {
         // Handle touch events
         window.ontouchend = () => this.$store.commit("busTouchEnd");
-        window.ontouchcancel = this.$store.commit("busTouchEnd");
+        window.ontouchcancel = () => this.$store.commit("busTouchEnd");
 
-        this.search = _debounce(async (clear: boolean) => {
+        this.search = _debounce(async (clear) => {
             if (clear) {
                 await this.clearResults();
             }
 
             await this.searchNow();
 
-        }, 350, {leading: false});
+        }, 350, false);
 
         this.$store.dispatch("loadFromArgs", this.$route).then(() => {
             this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
@@ -126,6 +130,7 @@ export default Vue.extend({
                     "setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
                     "setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
                     "setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
+                    "setEmbedding"
                 ].includes(mutation.type)) {
                     if (this.searchBusy) {
                         return;
@@ -152,7 +157,7 @@ export default Vue.extend({
         }).catch(error => {
             console.log(error);
 
-            if (error.response.status == 503 || error.response.status == 500) {
+            if (error.response.status === 503 || error.response.status === 500) {
                 this.showEsConnectionError = true;
                 this.uiLoading = false;
             } else {
@@ -181,7 +186,7 @@ export default Vue.extend({
                     bodyClass: "toast-body-error",
                 });
         },
-        showSyntaxErrorToast: function (): void {
+        showSyntaxErrorToast: function () {
             this.$bvToast.toast(
                 this.$t("toast.esQueryErr"),
                 {
@@ -197,10 +202,11 @@ export default Vue.extend({
             await this.$store.dispatch("incrementQuerySequence");
             this.$store.commit("busSearch");
 
-            Sist2Api.search().then(async (resp: EsResult) => {
+            Sist2Api.search().then(async (resp) => {
                 await this.handleSearch(resp);
                 this.searchBusy = false;
             }).catch(err => {
+                console.log(err)
                 if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
                     this.showSyntaxErrorToast();
                 } else {
@@ -215,8 +221,8 @@ export default Vue.extend({
             await this.$store.dispatch("clearResults");
             this.$store.commit("setUiReachedScrollEnd", false);
         },
-        async handleSearch(resp: EsResult) {
+        async handleSearch(resp) {
-            if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
+            if (resp.hits.hits.length === 0 || resp.hits.hits.length < this.$store.state.optSize) {
                 this.$store.commit("setUiReachedScrollEnd", true);
             }
 
@@ -239,9 +245,9 @@ export default Vue.extend({
                 if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
                     hit._seq = await this.$store.dispatch("getKeySequence");
                     this.$store.commit("addLightboxSource", {
-                        source: `f/${hit._source.index}/${hit._id}`,
+                        source: `f/${sid(hit)}`,
-                        thumbnail: hit._props.hasThumbnail
+                        thumbnail_count: hit._props.hasThumbnail
-                            ? `t/${hit._source.index}/${hit._id}`
+                            ? `t/${sid(hit)}`
                             : null,
                         caption: {
                             component: LightboxCaption,