Update readme

fixed occasional hanging when indexing

.devcontainer/Dockerfile

@@ -1,9 +0,0 @@
-FROM simon987/sist2-build
-
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
-RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-

.devcontainer/devcontainer.json

@@ -1,16 +0,0 @@
-{
-  "name": "sist2-dev",
-  "dockerComposeFile": [
-    "docker-compose.yml"
-  ],
-  "service": "sist2-dev",
-  "customizations": {
-    "vscode": {
-      "extensions": [
-        "ms-vscode.cpptools-extension-pack"
-      ]
-    }
-  },
-  "remoteUser": "root",
-  "workspaceFolder": "/app/"
-}

.devcontainer/docker-compose.yml

@@ -1,8 +0,0 @@
-version: "3"
-
-services:
-  sist2-dev:
-    build: .
-    command: sleep infinity
-    volumes:
-      - ../:/app

.dockerignore

@@ -38,4 +38,6 @@ build/
 __pycache__/
 sist2-vue/dist
 sist2-admin/frontend/dist
-*.fts
+*.fts
+.git
+third-party/libscan/third-party/ext_*/*

.drone.yml

@@ -7,11 +7,36 @@ platform:
   arch: amd64
 
 steps:
+  - name: submodules
+    image: alpine/git
+    commands:
+      - git submodule update --init --recursive
+  - name: docker
+    image: plugins/docker
+    depends_on:
+      - submodules
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: sist2app/sist2
+      context: ./
+      dockerfile: ./Dockerfile
+      auto_tag: true
+      auto_tag_suffix: x64-linux
+      when:
+        event:
+          - tag
   - name: build
-    image: simon987/sist2-build
+    image: sist2app/sist2-build
+    depends_on:
+    - submodules
     commands:
       - ./scripts/build.sh
   - name: scp files
+    depends_on:
+      - build
     image: appleboy/drone-scp
     settings:
       host:
@@ -22,26 +47,11 @@ steps:
         from_secret: SSH_USER
       key:
         from_secret: SSH_KEY
-      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
       source:
         - ./VERSION
         - ./sist2-x64-linux
         - ./sist2-x64-linux-debug
-  - name: docker
-    image: plugins/docker
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: simon987/sist2
-      context: ./
-      dockerfile: ./Dockerfile
-      auto_tag: true
-      auto_tag_suffix: x64-linux
-      when:
-        event:
-          - tag
 
 ---
 kind: pipeline
@@ -52,11 +62,36 @@ platform:
   arch: arm64
 
 steps:
+  - name: submodules
+    image: alpine/git
+    commands:
+      - git submodule update --init --recursive
+  - name: docker
+    image: plugins/docker
+    depends_on:
+      - submodules
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: sist2app/sist2
+      context: ./
+      dockerfile: ./Dockerfile.arm64
+      auto_tag: true
+      auto_tag_suffix: arm64-linux
+      when:
+        event:
+          - tag
   - name: build
-    image: simon987/sist2-build-arm64
+    image: sist2app/sist2-build-arm64
+    depends_on:
+      - submodules
     commands:
       - ./scripts/build_arm64.sh
   - name: scp files
+    depends_on:
+      - build
     image: appleboy/drone-scp
     settings:
       host:
@@ -67,22 +102,7 @@ steps:
         from_secret: SSH_USER
       key:
         from_secret: SSH_KEY
-      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
       source:
         - ./sist2-arm64-linux
         - ./sist2-arm64-linux-debug
-  - name: docker
-    image: plugins/docker
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: simon987/sist2
-      context: ./
-      dockerfile: ./Dockerfile.arm64
-      auto_tag: true
-      auto_tag_suffix: arm64-linux
-      when:
-        event:
-          - tag

.gitignore

@@ -3,6 +3,7 @@ thumbs
 *.cbp
 CMakeCache.txt
 CMakeFiles
+cmake-build-default-event-trace
 cmake-build-debug
 cmake_install.cmake
 Makefile

CMakeLists.txt

@@ -53,7 +53,6 @@ add_executable(
         src/types.h
         src/log.c src/log.h
         src/cli.c src/cli.h
-        src/parsing/sidecar.c src/parsing/sidecar.h
         src/database/database.c src/database/database.h
         src/parsing/fs_util.h
 
@@ -63,7 +62,9 @@ add_executable(
         src/database/database_schema.c
         src/database/database_fts.c
         src/web/web_fts.c
-        src/database/database_embeddings.c)
+        src/database/database_embeddings.c
+        src/ignorelist.c
+        src/ignorelist.h)
 set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
@@ -77,6 +78,7 @@ find_package(CURL CONFIG REQUIRED)
 find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
 find_package(unofficial-sqlite3 CONFIG REQUIRED)
 find_package(OpenBLAS CONFIG REQUIRED)
+find_package(libgit2 CONFIG REQUIRED)
 
 
 target_include_directories(
@@ -90,7 +92,7 @@ target_include_directories(
 target_compile_options(
         sist2
         PRIVATE
-        -fPIC
+#        -fPIC
 )
 
 if (SIST_DEBUG)
@@ -148,7 +150,9 @@ add_dependencies(
 target_link_libraries(
         sist2
 
+#        m
         z
+        libgit2::libgit2package
         argparse
         unofficial::mongoose::mongoose
         CURL::libcurl

Dockerfile

@@ -1,5 +1,4 @@
-FROM simon987/sist2-build as build
+FROM sist2app/sist2-build as build
-MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 
@@ -48,5 +47,6 @@ COPY --from=build /build/build/sist2 /root/sist2
 # sist2-admin
 WORKDIR /root/sist2-admin
 COPY sist2-admin/requirements.txt /root/sist2-admin/
-RUN python3 -m pip install --no-cache -r /root/sist2-admin/requirements.txt
+RUN ln /usr/bin/python3 /usr/bin/python
+RUN python -m pip install --no-cache -r /root/sist2-admin/requirements.txt
 COPY --from=build /build/sist2-admin/ /root/sist2-admin/

Dockerfile.arm64

@@ -1,5 +1,4 @@
-FROM simon987/sist2-build-arm64 as build
+FROM sist2app/sist2-build-arm64 as build
-MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 

README.md

@@ -1,9 +1,11 @@
-![GitHub](https://img.shields.io/github/license/simon987/sist2.svg)
+![GitHub](https://img.shields.io/github/license/sist2app/sist2.svg)
-[![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
+[![CodeFactor](https://www.codefactor.io/repository/github/sist2app/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/sist2app/sist2)
 [![Development snapshots](https://ci.simon987.net/api/badges/simon987/sist2/status.svg)](https://files.simon987.net/.gate/sist2/simon987_sist2/)
 
 **Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
 
+**Community URL:** [Discord](https://discord.gg/2PEjDy3Rfs)
+
 # sist2
 
 sist2 (Simple incremental search tool)
@@ -36,26 +38,32 @@ sist2 (Simple incremental search tool)
 ### Using Docker Compose *(Windows/Linux/Mac)*
 
 ```yaml
-version: "3"
-
 services:
   elasticsearch:
     image: elasticsearch:7.17.9
     restart: unless-stopped
+    volumes:
+      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
+      - /data/sist2-es-data/:/usr/share/elasticsearch/data
     environment:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "PUID=1000"
+      - "PGID=1000"
   sist2-admin:
-    image: simon987/sist2:3.1.4-x64-linux
+    image: sist2app/sist2:x64-linux
     restart: unless-stopped
     volumes:
-      - ./sist2-admin-data/:/sist2-admin/
+      - /data/sist2-admin-data/:/sist2-admin/
-      - /:/host
+      - /<path to index>/:/host
     ports:
-      - 4090:4090 # sist2
+      - 4090:4090
-      - 8080:8080 # sist2-admin
+      # NOTE: Don't expose this port publicly!
+      - 8080:8080
     working_dir: /root/sist2-admin/
-    entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
+    entrypoint: python3
+    command:
+      - /root/sist2-admin/sist2_admin/app.py
 ```
 
 Navigate to http://localhost:8080/ to configure sist2-admin.
@@ -71,7 +79,7 @@ Navigate to http://localhost:8080/ to configure sist2-admin.
             ```
     * **SQLite**: No installation required
 
-2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
+2. Download the [latest sist2 release](https://github.com/sist2app/sist2/releases).
    Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
 3. See [usage guide](docs/USAGE.md) for command line usage.
 
@@ -80,28 +88,30 @@ Example usage:
 1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
 2. Prepare search index:
     * **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
-    * **SQLite**: `sist2 index --search-index ./search.sist2 ./documents.sist2`
+    * **SQLite**: `sist2 sqlite-index --search-index ./search.sist2 ./documents.sist2`
-3. Start web interface: `sist2 web ./documents.sist2`
+3. Start web interface: 
+   * **Elasticsearch**: `sist2 web ./documents.sist2`
+   * **SQLite**: `sist2 web --search-index ./search.sist2 ./documents.sist2`
 
 ## Format support
 
 | File type                                                                 | Library                                                                      | Content  | Thumbnail   | Metadata                                                                                                                               |
 |:--------------------------------------------------------------------------|:-----------------------------------------------------------------------------|:---------|:------------|:---------------------------------------------------------------------------------------------------------------------------------------|
 | pdf,xps,fb2,epub                                                          | MuPDF                                                                        | text+ocr | yes         | author, title                                                                                                                          |
-| cbz,cbr                                                                   | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | -        | yes         | -                                                                                                                                      |
+| cbz,cbr                                                                   | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | -        | yes         | -                                                                                                                                      |
 | `audio/*`                                                                 | ffmpeg                                                                       | -        | yes         | ID3 tags                                                                                                                               |
 | `video/*`                                                                 | ffmpeg                                                                       | -        | yes         | title, comment, artist                                                                                                                 |
-| `image/*`                                                                 | ffmpeg                                                                       | ocr      | yes         | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
+| `image/*`                                                                 | ffmpeg                                                                       | ocr      | yes         | [Common EXIF tags](https://github.com/sist2app/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
 | raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf | LibRaw                                                                       | no       | yes         | Common EXIF tags, GPS tags                                                                                                             |
 | ttf,ttc,cff,woff,fnt,otf                                                  | Freetype2                                                                    | -        | yes, `bmp`  | Name & style                                                                                                                           |
-| `text/plain`                                                              | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| `text/plain`                                                              | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
-| html, xml                                                                 | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| html, xml                                                                 | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
 | tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
-| docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
+| docx, xlsx, pptx                                                          | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
 | doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
 | mobi, azw, azw3                                                           | libmobi                                                                      | yes      | yes         | author, title                                                                                                                          |
 | wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
-| json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
+| json, jsonl, ndjson                                                       | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
 \* *See [Archive files](#archive-files)*
 
@@ -125,7 +135,7 @@ You can enable OCR support for ebook (pdf,xps,fb2,epub) or image file types with
 Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
-The `simon987/sist2` image comes with common languages
+The `sist2app/sist2` image comes with common languages
 (hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
@@ -153,10 +163,10 @@ indices, but it uses much less memory and is easier to set up.
 | Query syntax                                 |      [fts5](https://www.sqlite.org/fts5.html)       | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
 | Fuzzy search                                 |                                                     |                                                                   ✓                                                                   |
 | Media Types tree real-time updating          |                                                     |                                                                   ✓                                                                   |
-| Search in file `path`                        | [WIP](https://github.com/simon987/sist2/issues/402) |                                                                   ✓                                                                   |
 | Manual tagging                               |                          ✓                          |                                                                   ✓                                                                   |
 | User scripts                                 |                          ✓                          |                                                                   ✓                                                                   |
 | Media Type breakdown for search results      |                                                     |                                                                   ✓                                                                   |
+| Embeddings search                            |                      ✓ *O(n)*                       |                                                              ✓ *O(logn)*                                                              |
 
 ### NER
 
@@ -165,13 +175,13 @@ sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported re
 to enable it.
 
 The text processing is done in your browser, no data is sent to any third-party services.
-See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) for more details.
+See [sist2app/sist2-ner-models](https://github.com/sist2app/sist2-ner-models) for more details.
 
 #### List of available repositories:
 
 | URL                                                                                                     | Maintainer                              | Purpose |
 |---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
-| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
+| [sist2app/sist2-ner-models](https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json) | [sist2app](https://github.com/sist2app) | General |
 
 <details>
   <summary>Screenshot</summary>
@@ -187,7 +197,7 @@ You can compile **sist2** by yourself if you don't want to use the pre-compiled
 ### Using docker
 
 ```bash
-git clone --recursive https://github.com/simon987/sist2/
+git clone --recursive https://github.com/sist2app/sist2/
 cd sist2
 docker build . -t my-sist2-image
 # Copy sist2 executable from docker image
@@ -202,18 +212,18 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
    apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
    ```
 
-2. Install vcpkg using my fork: https://github.com/simon987/vcpkg
+2. Install vcpkg using my fork: https://github.com/sist2app/vcpkg
 3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib]
+    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5,json1] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib] libgit2[core,pcre]
     ```
 
 4. Build
     ```bash
-    git clone --recursive https://github.com/simon987/sist2/
+    git clone --recursive https://github.com/sist2app/sist2/
     (cd sist2-vue; npm install; npm run build)
     (cd sist2-admin/frontend; npm install; npm run build)
-    cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
+    cmake -DSIST_DEBUG=off -G "Unix Makefiles" -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

docker-compose.yml

@@ -4,16 +4,21 @@ services:
   elasticsearch:
     image: elasticsearch:7.17.9
     container_name: sist2-es
+    volumes:
+      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
+      - /data/sist2-es-data/:/usr/share/elasticsearch/data
     environment:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "PUID=1000"
+      - "PGID=1000"
   sist2-admin:
     build:
       context: .
     container_name: sist2-admin
     volumes:
-      - /mnt/array/sist2-admin-data/:/sist2-admin/
+      - /data/sist2-admin-data/:/sist2-admin/
-      - /:/host
+      - /<path to index>/:/host
     ports:
       - 4090:4090
       # NOTE: Don't export this port publicly!

docs/USAGE.md

@@ -16,9 +16,9 @@ Lightning-fast file system indexer and search tool.
 
 Scan options
     -t, --threads=<int>               Number of threads. DEFAULT: 1
-    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
+    -q, --thumbnail_count-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
-    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
+    --thumbnail_count-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
-    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
+    --thumbnail_count-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
     --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
     -o, --output=<str>                Output index file path. DEFAULT: index.sist2
     --incremental                     If the output file path exists, only scan new or modified files.
@@ -78,9 +78,9 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 
 #### Thumbnail database size estimation
 
-See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
+See chart below for rough estimate of thumbnail_count size vs. thumbnail_count size & quality arguments:
 
-For example, `--thumbnail-size=500`, `--thumbnail-quality=50` for a directory with 8 million images will create a thumbnail database 
+For example, `--thumbnail_count-size=500`, `--thumbnail_count-quality=50` for a directory with 8 million images will create a thumbnail_count database 
 that is about `8000000 * 11.8kB = 94.4GB`.
 
 ![thumbnail_size](thumbnail_size.png)
@@ -92,7 +92,7 @@ Simple scan
 sist2 scan ~/Documents
 
 sist2 scan \
-    --threads 4 --content-size 16000000 --thumbnail-quality 2 --archive shallow \
+    --threads 4 --content-size 16000000 --thumbnail_count-quality 2 --archive shallow \
     --name "My Documents" --rewrite-url "http://nas.domain.local/My Documents/" \
     ~/Documents -o ./documents.sist2
 ```
@@ -108,6 +108,27 @@ sist scan ~/Documents -o ./documents.sist2 --incremental
 sist scan ~/Documents -o ./documents.sist2 --incremental
 ```
 
+### Excluding files
+
+You can use the `--exclude` option to specify exclude patterns. For more complex setups, you can create a 
+`.sist2ignore` file at the root of the scan path (For example, `~/Documents/.sist2ignore` for the example above). 
+
+The syntax for sist2ignore is the same as .gitignore for Git (reference [here](https://git-scm.com/docs/gitignore)).
+
+Example:
+
+**.sist2ignore**
+```gitignore
+# Ignore all PDF files
+*.pdf
+
+# But don't ignore them for the /important_files/ directory
+!/important_files/*.pdf
+
+# Ignore all files in _staging/ directories
+_staging/
+```
+
 ### Index documents to Elasticsearch search backend
 
 ```bash
@@ -172,9 +193,39 @@ Using a version >=7.14.0 is recommended to enable the following features:
 
 - Bug fix for large documents (See #198)
 
+Using a version >=8.0.0 is recommended to enable the following features:
+
+- Approximate KNN search for Embeddings search (faster queries).
+
 When using a legacy version of ES, a notice will be displayed next to the sist2 version in the web UI.
 If you don't care about the features above, you can ignore it or disable it in the configuration page.
 
+# Embeddings search
+
+Since v3.2.0, User scripts can be used to generate _embeddings_ (vector of float32 numbers) which are stored in the .sist2 index file 
+(see [scripting](scripting.md)). Embeddings can be used for:
+
+* Nearest-neighbor queries (e.g. "return the documents most similar to this one")
+* Semantic searches (e.g. "return the documents that are most closely related to the given topic")
+
+In theory, embeddings can be created for any type of documents (image, text, audio etc.).
+
+For example, the [clip](https://github.com/sist2app/sist2-script-clip) User Script, generates 512-d embeddings of images
+(videos are also supported using the thumbnails generated by sist2). When the user enters a query in the "Embeddings Search"
+textbox, the query's embedding is generated in their browser, leveraging the ONNX web runtime.
+
+<details>
+  <summary>Screenshots</summary>
+ 
+ ![embeddings-1](embeddings-1.png)
+ ![embeddings-2](embeddings-2.png)
+
+1. Embeddings search bar. You can select the model using the dropdown on the left.
+2. This icon appears for indices with embeddings search enabled.
+3. Documents with this icon have embeddings. Click on the icon to perform KNN search.
+</details>
+
+
 # Tagging
 
 ### Manual tagging
@@ -200,42 +251,3 @@ See [Automatic tagging](#automatic-tagging) for information about tag
 ### Automatic tagging
 
 See [scripting](scripting.md) documentation.
-
-# Sidecar files
-
-When scanning, sist2 will read metadata from `.s2meta` JSON files and overwrite the 
-original document's indexed metadata (does not modify the actual file). Sidecar metadata files will also work inside archives.
-Sidecar files themselves are not saved in the index.
-
-This feature is useful to leverage third-party applications such as speech-to-text or
-OCR to add additional metadata to a file.
-
-**Example**
-
-```
-~/Documents/
-├── Video.mp4
-└── Video.mp4.s2meta
-```
-
-The sidecar file must have exactly the same file path and the `.s2meta` suffix.
-
-`Video.mp4.s2meta`:
-```json
-{
-  "content": "This sidecar file will overwrite some metadata fields of Video.mp4",
-  "author": "Some author",
-  "duration": 12345,
-  "bitrate": 67890,
-  "some_arbitrary_field": [1,2,3]
-}
-```
-
-```
-sist2 scan ~/Documents -o ./docs.sist2
-sist2 index ./docs.sist2
-```
-
-*NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,
-it is not currently possible to restore both manual tags and sidecar tags without user scripts
-while reindexing.

scripts/build.sh

@@ -2,8 +2,6 @@
 
 VCPKG_ROOT="/vcpkg"
 
-git submodule update --init --recursive
-
 (
   cd sist2-vue/
   npm install

scripts/generate_big_index.py

@@ -1,131 +0,0 @@
-import sqlite3
-import orjson as json
-import os
-import string
-from hashlib import md5
-import random
-from tqdm import tqdm
-
-schema = """
-CREATE TABLE thumbnail (
-  id TEXT NOT NULL CHECK (
-    length(id) = 32
-  ), 
-  num INTEGER NOT NULL, 
-  data BLOB NOT NULL, 
-  PRIMARY KEY(id, num)
-) WITHOUT ROWID;
-CREATE TABLE version (
-  id INTEGER PRIMARY KEY AUTOINCREMENT, 
-  date TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP)
-);
-CREATE TABLE document (
-  id TEXT PRIMARY KEY NOT NULL CHECK (
-    length(id) = 32
-  ), 
-  marked INTEGER NOT NULL DEFAULT (1), 
-  version INTEGER NOT NULL REFERENCES version(id), 
-  mtime INTEGER NOT NULL, 
-  size INTEGER NOT NULL, 
-  json_data TEXT NOT NULL CHECK (
-    json_valid(json_data)
-  )
-);
-CREATE TABLE delete_list (
-  id TEXT PRIMARY KEY CHECK (
-    length(id) = 32
-  )
-) WITHOUT ROWID;
-CREATE TABLE tag (
-  id TEXT NOT NULL, 
-  tag TEXT NOT NULL, 
-  PRIMARY KEY (id, tag)
-);
-CREATE TABLE document_sidecar (
-  id TEXT PRIMARY KEY NOT NULL, json_data TEXT NOT NULL
-) WITHOUT ROWID;
-CREATE TABLE descriptor (
-  id TEXT NOT NULL, version_major INTEGER NOT NULL, 
-  version_minor INTEGER NOT NULL, version_patch INTEGER NOT NULL, 
-  root TEXT NOT NULL, name TEXT NOT NULL, 
-  rewrite_url TEXT, timestamp INTEGER NOT NULL
-);
-CREATE TABLE stats_treemap (
-  path TEXT NOT NULL, size INTEGER NOT NULL
-);
-CREATE TABLE stats_size_agg (
-  bucket INTEGER NOT NULL, count INTEGER NOT NULL
-);
-CREATE TABLE stats_date_agg (
-  bucket INTEGER NOT NULL, count INTEGER NOT NULL
-);
-CREATE TABLE stats_mime_agg (
-  mime TEXT NOT NULL, size INTEGER NOT NULL, 
-  count INTEGER NOT NULL
-);
-CREATE TABLE embedding (
-  id TEXT REFERENCES document(id), 
-  model_id INTEGER NOT NULL references model(id), 
-  start INTEGER NOT NULL, 
-  end INTEGER, 
-  embedding BLOB NOT NULL, 
-  PRIMARY KEY (id, model_id, start)
-);
-CREATE TABLE model (
-  id INTEGER PRIMARY KEY, 
-  name TEXT NOT NULL UNIQUE CHECK (
-    length(name) < 16
-  ), 
-  url TEXT, 
-  path TEXT NOT NULL UNIQUE, 
-  size INTEGER NOT NULL, 
-  type TEXT NOT NULL CHECK (
-    type IN ('flat', 'nested')
-  )
-);
-"""
-
-content = "".join(random.choices(string.ascii_letters, k=500))
-
-
-def gen_document():
-    return [
-        md5(random.randbytes(8)).hexdigest(),
-        json.dumps({
-            "content": content,
-            "mime": "image/jpeg",
-            "extension": "jpeg",
-            "name": "test",
-            "path": "",
-        })
-    ]
-
-
-if __name__ == "__main__":
-    DB_NAME = "big_index.sist2"
-    SIZE = 30_000_000
-
-    os.remove(DB_NAME)
-    db = sqlite3.connect(DB_NAME)
-    db.executescript(schema)
-
-    db.executescript("""
-    PRAGMA journal_mode = OFF;
-    PRAGMA synchronous = 0;
-    """)
-
-    for _ in tqdm(range(SIZE), total=SIZE):
-        db.execute(
-            "INSERT INTO document (id, version, mtime, size, json_data) VALUES (?, 1, 1000000, 10000, ?)",
-            gen_document()
-        )
-
-    # 1. Enable rowid from document
-    # 2. CREATE TABLE marked (
-    #     id INTEGER PRIMARY KEY,
-    #     marked int
-    #    );
-    # 3. Set FK for document_sidecar, embedding, tag, thumbnail
-    # 4. Toggle FK if debug
-
-    db.commit()

scripts/magic_static.py

@@ -1,8 +1,16 @@
 
-try:
+MAGIC_PATHS = [
-    with open("/usr/lib/file/magic.mgc", "rb") as f:
+    "/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
-        data = f.read()
+    "/work/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
-except:
+    "/usr/lib/file/magic.mgc"
-    data = bytes([])
+]
+
+for path in MAGIC_PATHS:
+    try:
+        with open(path, "rb") as f:
+            data = f.read()
+        break
+    except:
+        continue
 
 print("char magic_database_buffer[%d] = {%s};" % (len(data), ",".join(str(int(b)) for b in data)))

scripts/mime.csv

@@ -450,4 +450,4 @@ image/x-sony-arw, arw
 image/x-sony-sr2, sr2
 image/x-sony-srf, srf
 image/x-epson-erf, erf
-sist2/sidecar, s2meta
+image/jp2, jp2

scripts/mime.py

@@ -3,6 +3,7 @@ import zlib
 mimes = {}
 noparse = set()
 ext_in_hash = set()
+mime_ids = {}
 
 major_mime = {
     "sist2": 0,
@@ -102,6 +103,9 @@ cnt = 1
 
 
 def mime_id(mime):
+    if mime in mime_ids:
+        return mime_ids[mime]
+
     global cnt
     major = mime.split("/")[0]
     mime_id = str((major_mime[major] << 16) + cnt)
@@ -127,9 +131,7 @@ def mime_id(mime):
     elif mime == "application/x-empty":
         cnt -= 1
         return "1"
-    elif mime == "sist2/sidecar":
+    mime_ids[mime] = mime_id
-        cnt -= 1
-        return "2"
     return mime_id
 
 
@@ -197,4 +199,12 @@ with open("scripts/mime.csv") as f:
         print(f"case {crc(mime)}: return {clean(mime)};")
 
     print("default: return 0;}}")
+
+    # mime list
+
+    mime_list = ",".join(mime_id(x) for x in mimes.keys()) + ",0"
+
+    print(f"unsigned int mime_ids[] = {{{mime_list}}};")
+    print("unsigned int* get_mime_ids() { return mime_ids; }")
+
     print("#endif")

sist2-admin/frontend/package.json

@@ -8,7 +8,7 @@
     "watch": "vue-cli-service build --watch"
   },
   "dependencies": {
-    "axios": "^0.27.2",
+    "axios": "^1.6.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
     "moment": "^2.29.3",

sist2-admin/frontend/src/App.vue

@@ -4,7 +4,7 @@
         <b-container class="pt-4">
             <b-alert show dismissible variant="info">
                 This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
-                to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
+                to the <a href="https://github.com/sist2app/sist2/issues/new/choose" target="_blank">issue tracker on
                 Github</a>. Thank you!
             </b-alert>
             <router-view v-if="$store.state.sist2AdminInfo"/>

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -89,9 +89,12 @@ class Sist2AdminApi {
 
     /**
      * @param {string} name
+     * @param {bool} full
      */
-    runJob(name) {
+    runJob(name, full) {
-        return axios.get(`${this.baseUrl}/api/job/${name}/run`);
+        return axios.get(`${this.baseUrl}/api/job/${name}/run`, {
+            params: {full}
+        });
     }
 
     /**

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -95,6 +95,7 @@ export default {
   methods: {
     onOcrLangChange() {
       this.options.ocr_lang = this.selectedOcrLangs.join("+");
+      this.update();
     },
     update() {
       this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;

sist2-admin/frontend/src/components/WebOptions.vue

@@ -1,59 +1,70 @@
 <template>
-    <div>
+  <div>
-        <h4>{{ $t("webOptions.title") }}</h4>
+    <h4>{{ $t("webOptions.title") }}</h4>
-        <b-card>
+    <b-card>
-            <label>{{ $t("webOptions.lang") }}</label>
+      <label>{{ $t("webOptions.lang") }}</label>
-            <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
+      <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
-                           @change="update()"></b-form-select>
+                     @change="update()"></b-form-select>
 
-            <label>{{ $t("webOptions.bind") }}</label>
+      <label>{{ $t("webOptions.bind") }}</label>
-            <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+      <b-form-input v-model="options.bind" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.tagline") }}</label>
+      <label>{{ $t("webOptions.tagline") }}</label>
-            <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+      <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
 
-            <label>{{ $t("webOptions.auth") }}</label>
+      <label>{{ $t("webOptions.auth") }}</label>
-            <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+      <b-form-input v-model="options.auth" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.tagAuth") }}</label>
+      <label>{{ $t("webOptions.tagAuth") }}</label>
-            <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+      <b-form-input v-model="options.tag_auth" @change="update()" :disabled="Boolean(options.auth)"></b-form-input>
-        </b-card>
 
-        <br>
+      <b-form-checkbox v-model="options.verbose" @change="update()">
-        <h4>Auth0 options</h4>
+        {{$t("webOptions.verbose")}}
-        <b-card>
+      </b-form-checkbox>
-            <label>{{ $t("webOptions.auth0Audience") }}</label>
+    </b-card>
-            <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.auth0Domain") }}</label>
+    <br>
-            <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
+    <h4>Auth0 options</h4>
+    <b-card>
+      <label>{{ $t("webOptions.auth0Audience") }}</label>
+      <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.auth0ClientId") }}</label>
+      <label>{{ $t("webOptions.auth0Domain") }}</label>
-            <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
+      <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
 
-            <label>{{ $t("webOptions.auth0PublicKey") }}</label>
+      <label>{{ $t("webOptions.auth0ClientId") }}</label>
-            <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
+      <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
-        </b-card>
+
-    </div>
+      <label>{{ $t("webOptions.auth0PublicKey") }}</label>
+      <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
+    </b-card>
+  </div>
 </template>
 
 <script>
 
 export default {
-    name: "WebOptions",
+  name: "WebOptions",
-    props: ["options", "frontendName"],
+  props: ["options", "frontendName"],
-    data() {
+  data() {
-        return {
+    return {
-            showEsTestAlert: false,
+      showEsTestAlert: false,
-            esTestOk: false,
+      esTestOk: false,
-            esTestMessage: "",
+      esTestMessage: ""
-        }
-    },
-    methods: {
-        update() {
-            this.$emit("change", this.options);
-        },
     }
+  },
+  methods: {
+    update() {
+
+      console.log(this.options)
+      if (this.options.auth && this.options.tag_auth) {
+        // If both are set, remove tagAuth
+        this.options.tag_auth = "";
+      }
+
+      this.$emit("change", this.options);
+    },
+  }
 }
 </script>
 

sist2-admin/frontend/src/i18n/messages.js

@@ -8,6 +8,7 @@ export default {
         view: "View",
         delete: "Delete",
         runNow: "Index now",
+        runNowFull: "Full re-index",
         create: "Create",
         cancel: "Cancel",
         test: "Test",
@@ -64,6 +65,9 @@ export default {
         gitRepository: "Git repository URL",
         extraArgs: "Extra command line arguments",
 
+        couldNotStartFrontend: "Could not start frontend",
+        couldNotStartFrontendBody: "Unable to start the frontend, check server logs for more details.",
+
         selectJobs: "Available jobs",
         selectJob: "Select a job",
         webOptions: {
@@ -77,6 +81,7 @@ export default {
             auth0Domain: "Auth0 domain",
             auth0ClientId: "Auth0 client ID",
             auth0PublicKey: "Auth0 public key",
+            verbose: "Verbose logs"
         },
         backendOptions: {
             title: "Search backend options",
@@ -88,7 +93,9 @@ export default {
             threads: "Number of threads",
             batchSize: "Index batch size",
             script: "User script",
-            searchIndex: "Search index file location"
+            searchIndex: "Search index file location",
+            esMappings: "Elasticsearch mappings file override",
+            esSettings: "Elasticsearch settings file override"
         },
         scanOptions: {
             title: "Scanning options",

sist2-admin/frontend/src/views/Frontend.vue

@@ -1,63 +1,63 @@
 <template>
-    <b-card>
+  <b-card>
-        <b-card-title>
+    <b-card-title>
-            {{ name }}
+      {{ name }}
-            <small style="vertical-align: top">
+      <small style="vertical-align: top">
-                <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
-                <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
-            </small>
+      </small>
-        </b-card-title>
+    </b-card-title>
 
-        <!-- Action buttons-->
+    <!-- Action buttons-->
-        <div class="mb-3" v-if="!loading">
+    <div class="mb-3" v-if="!loading">
-            <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
-                $t("start")
+          $t("start")
-                }}
+        }}
-            </b-button>
+      </b-button>
-            <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
-                $t("stop")
+          $t("stop")
-                }}
+        }}
-            </b-button>
+      </b-button>
-            <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
-                {{ $t("go") }}
+        {{ $t("go") }}
-            </b-button>
+      </b-button>
-            <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
-        </div>
+    </div>
 
-        <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
-        <b-card-body v-else>
+    <b-card-body v-else>
 
-            <h4>{{ $t("backendOptions.title") }}</h4>
+      <h4>{{ $t("backendOptions.title") }}</h4>
-            <b-card>
+      <b-card>
-                <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
 
-                <SearchBackendSelect :value="frontend.web_options.search_backend"
+        <SearchBackendSelect :value="frontend.web_options.search_backend"
-                                     @change="onBackendSelect($event)"></SearchBackendSelect>
+                             @change="onBackendSelect($event)"></SearchBackendSelect>
 
-                <br>
+        <br>
-                <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
-            </b-card>
+      </b-card>
 
-            <br/>
+      <br/>
 
-            <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
+      <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
-                        @change="update()"></WebOptions>
+                  @change="update()"></WebOptions>
-            <br/>
+      <br/>
 
-            <h4>{{ $t("frontendOptions.title") }}</h4>
+      <h4>{{ $t("frontendOptions.title") }}</h4>
-            <b-card>
+      <b-card>
-                <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
-                    {{ $t("autoStart") }}
+          {{ $t("autoStart") }}
-                </b-form-checkbox>
+        </b-form-checkbox>
 
-                <label>{{ $t("extraQueryArgs") }}</label>
+        <label>{{ $t("extraQueryArgs") }}</label>
-                <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
 
-                <label>{{ $t("customUrl") }}</label>
+        <label>{{ $t("customUrl") }}</label>
-                <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
-            </b-card>
+      </b-card>
-        </b-card-body>
+    </b-card-body>
-    </b-card>
+  </b-card>
 </template>
 
 <script>
@@ -68,71 +68,78 @@ import WebOptions from "@/components/WebOptions";
 import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
-    name: 'Frontend',
+  name: 'Frontend',
-    components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
+  components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
-    data() {
+  data() {
-        return {
+    return {
-            loading: true,
+      loading: true,
-            frontend: null,
+      frontend: null,
-        }
-    },
-    computed: {
-        valid() {
-            return !this.loading && this.frontend.jobs.length > 0;
-        },
-        frontendUrl() {
-            if (this.frontend.custom_url) {
-                return this.frontend.custom_url + this.args;
-            }
-
-            if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
-                return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
-            }
-
-            return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
-        },
-        name() {
-            return this.$route.params.name;
-        },
-        port() {
-            return this.frontend.web_options.bind.split(":")[1]
-        },
-        args() {
-            const args = this.frontend.extra_query_args;
-            if (args !== "") {
-                return "#" + (args.startsWith("?") ? (args) : ("?" + args));
-            }
-            return "";
-        }
-    },
-    mounted() {
-        Sist2AdminApi.getFrontend(this.name).then(resp => {
-            this.frontend = resp.data;
-            this.loading = false;
-        });
-    },
-    methods: {
-        start() {
-            this.frontend.running = true;
-            Sist2AdminApi.startFrontend(this.name)
-        },
-        stop() {
-            this.frontend.running = false;
-            Sist2AdminApi.stopFrontend(this.name)
-        },
-        deleteFrontend() {
-            Sist2AdminApi.deleteFrontend(this.name).then(() => {
-                this.$router.push("/");
-            });
-        },
-        update() {
-            Sist2AdminApi.updateFrontend(this.name, this.frontend);
-        },
-        onBackendSelect(backend) {
-            this.frontend.web_options.search_backend = backend;
-            this.frontend.jobs = [];
-            this.update();
-        }
     }
+  },
+  computed: {
+    valid() {
+      return !this.loading && this.frontend.jobs.length > 0;
+    },
+    frontendUrl() {
+      if (this.frontend.custom_url) {
+        return this.frontend.custom_url + this.args;
+      }
+
+      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+      }
+
+      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+    },
+    name() {
+      return this.$route.params.name;
+    },
+    port() {
+      return this.frontend.web_options.bind.split(":")[1]
+    },
+    args() {
+      const args = this.frontend.extra_query_args;
+      if (args !== "") {
+        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+      }
+      return "";
+    }
+  },
+  mounted() {
+    Sist2AdminApi.getFrontend(this.name).then(resp => {
+      this.frontend = resp.data;
+      this.loading = false;
+    });
+  },
+  methods: {
+    start() {
+      Sist2AdminApi.startFrontend(this.name).then(() => {
+        this.frontend.running = true;
+      }).catch(() => {
+        this.$bvToast.toast(this.$t("couldNotStartFrontendBody"), {
+          title: this.$t("couldNotStartFrontend"),
+          variant: "danger",
+          toaster: "b-toaster-bottom-right"
+        });
+      });
+    },
+    stop() {
+      this.frontend.running = false;
+      Sist2AdminApi.stopFrontend(this.name)
+    },
+    deleteFrontend() {
+      Sist2AdminApi.deleteFrontend(this.name).then(() => {
+        this.$router.push("/");
+      });
+    },
+    update() {
+      Sist2AdminApi.updateFrontend(this.name, this.frontend);
+    },
+    onBackendSelect(backend) {
+      this.frontend.web_options.search_backend = backend;
+      this.frontend.jobs = [];
+      this.update();
+    }
+  }
 }
 </script>

sist2-admin/frontend/src/views/Job.vue

@@ -6,7 +6,19 @@
         </b-card-title>
 
         <div class="mb-3">
-            <b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
+
+            <b-dropdown
+                    split
+                    split-variant="primary"
+                    variant="primary"
+                    :text="$t('runNow')"
+                    class="mr-1"
+                    :disabled="!valid"
+                    @click="runJob()"
+            >
+                <b-dropdown-item href="#" @click="runJob(true)">{{ $t("runNowFull") }}</b-dropdown-item>
+            </b-dropdown>
+
             <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
         </div>
 
@@ -69,6 +81,7 @@ export default {
         return {
             loading: true,
             job: null,
+            console: console
         }
     },
     methods: {
@@ -78,8 +91,8 @@ export default {
         update() {
             Sist2AdminApi.updateJob(this.getName(), this.job);
         },
-        runJob() {
+        runJob(full = false) {
-            Sist2AdminApi.runJob(this.getName()).then(() => {
+            Sist2AdminApi.runJob(this.getName(), full).then(() => {
                 this.$bvToast.toast(this.$t("runJobConfirmation"), {
                     title: this.$t("runJobConfirmationTitle"),
                     variant: "success",

sist2-admin/frontend/src/views/SearchBackend.vue

@@ -44,6 +44,12 @@
 
                 <label>{{ $t("backendOptions.batchSize") }}</label>
                 <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.esMappings") }}</label>
+                <b-form-textarea v-model="backend.es_mappings" rows="4" @change="update()"></b-form-textarea>
+
+                <label>{{ $t("backendOptions.esSettings") }}</label>
+                <b-form-textarea v-model="backend.es_settings" rows="4" @change="update()"></b-form-textarea>
             </template>
             <template v-else>
                 <label>{{ $t("backendOptions.searchIndex") }}</label>

sist2-admin/frontend/src/views/Tail.vue

@@ -170,6 +170,6 @@ span.ADMIN {
     margin: 3px;
     white-space: pre;
     color: #000;
-    overflow: hidden;
+    overflow-y: hidden;
 }
-</style>
+</style>

sist2-admin/frontend/src/views/Tasks.vue

@@ -81,7 +81,7 @@ function humanDuration(sec_num) {
         return `${seconds}s`;
     }
 
-    return "<0s";
+    return "<1s";
 }
 
 export default {
@@ -134,7 +134,7 @@ export default {
                     duration: this.taskDuration(row),
                     time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"),
                     logs: null,
-                    status: [0,1].includes(row.return_code) ? "ok" : "failed",
+                    status: row.return_code === 0 ? "ok" : "failed",
                     _row: row
                 }));
             });

sist2-admin/requirements.txt

@@ -4,4 +4,4 @@ uvicorn
 websockets
 pycron
 GitPython
-git+https://github.com/simon987/sist2-python.git
+git+https://github.com/sist2app/sist2-python.git@2.1

sist2-admin/sist2_admin/app.py

@@ -2,6 +2,7 @@ import asyncio
 import os
 import signal
 from datetime import datetime
+from time import sleep
 from urllib.parse import urlparse
 
 import requests
@@ -25,6 +26,7 @@ from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEM
     get_log_files_to_remove, delete_log_file, create_default_search_backends
 from web import Sist2Frontend
 from script import UserScript, SCRIPT_TEMPLATES
+from util import tail_sync, pid_is_running
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
 db = PersistentState(dbfile=os.path.join(DATA_FOLDER, "state.db"))
@@ -169,11 +171,14 @@ def _run_job(job: Sist2Job):
 
 
 @app.get("/api/job/{name:str}/run")
-async def run_job(name: str):
+async def run_job(name: str, full: bool = False):
-    job = db["jobs"][name]
+    job: Sist2Job = db["jobs"][name]
     if not job:
         raise HTTPException(status_code=404)
 
+    if full:
+        job.do_full_scan = True
+
     _run_job(job)
 
     return "ok"
@@ -222,7 +227,10 @@ async def delete_job(name: str):
 @app.delete("/api/frontend/{name:str}")
 async def delete_frontend(name: str):
     if name in RUNNING_FRONTENDS:
-        os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        try:
+            os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
+        except ProcessLookupError:
+            pass
         del RUNNING_FRONTENDS[name]
 
     frontend = db["frontends"][name]
@@ -318,7 +326,18 @@ def start_frontend_(frontend: Sist2Frontend):
     logger.debug(f"Fetched search backend options for {backend_name}")
 
     pid = sist2.web(frontend.web_options, search_backend, frontend.name)
+
+    sleep(0.2)
+    if not pid_is_running(pid):
+        frontend_log = frontend.get_log_path(LOG_FOLDER)
+        logger.error(f"Frontend exited too quickly, check {frontend_log} for more details:")
+        for line in tail_sync(frontend.get_log_path(LOG_FOLDER), 3):
+            logger.error(line.strip())
+
+        return False
+
     RUNNING_FRONTENDS[frontend.name] = pid
+    return True
 
 
 @app.post("/api/frontend/{name:str}/start")
@@ -327,7 +346,12 @@ async def start_frontend(name: str):
     if not frontend:
         raise HTTPException(status_code=404)
 
-    start_frontend_(frontend)
+    ok = start_frontend_(frontend)
+
+    if not ok:
+        raise HTTPException(status_code=500)
+
+    return "ok"
 
 
 @app.post("/api/frontend/{name:str}/stop")

sist2-admin/sist2_admin/jobs.py

@@ -120,6 +120,10 @@ class Sist2Task:
 
         logger.info(f"Started task {self.display_name}")
 
+    def set_pid(self, pid):
+        self.pid = pid
+
+
 
 class Sist2ScanTask(Sist2Task):
 
@@ -133,13 +137,10 @@ class Sist2ScanTask(Sist2Task):
         else:
             self.job.scan_options.output = None
 
-        def set_pid(pid):
+        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
-            self.pid = pid
-
-        return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
         self.ended = datetime.utcnow()
 
-        is_ok = return_code in (0, 1)
+        is_ok = (return_code in (0, 1)) if "debug" in sist2.bin_path else (return_code == 0)
 
         if not is_ok:
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
@@ -165,6 +166,9 @@ class Sist2ScanTask(Sist2Task):
             self.job.previous_index_path = self.job.index_path
             db["jobs"][self.job.name] = self.job
 
+        if is_ok:
+            return 0
+
         return return_code
 
 
@@ -185,7 +189,7 @@ class Sist2IndexTask(Sist2Task):
 
         logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
 
-        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback)
+        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback, set_pid_cb=self.set_pid)
         self.ended = datetime.utcnow()
 
         duration = self.ended - self.started
@@ -200,7 +204,7 @@ class Sist2IndexTask(Sist2Task):
         self.job.previous_index_path = self.job.index_path
         db["jobs"][self.job.name] = self.job
 
-        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
+        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}, {ok=}"}))
 
         logger.info(f"Completed {self.display_name} ({return_code=})")
 
@@ -216,7 +220,7 @@ class Sist2IndexTask(Sist2Task):
             except ProcessLookupError:
                 pass
             try:
-                os.wait()
+                os.waitpid(pid, 0)
             except ChildProcessError:
                 pass
 
@@ -249,7 +253,7 @@ class Sist2UserScriptTask(Sist2Task):
         super().run(sist2, db)
 
         try:
-            self.user_script.setup(self.log_callback)
+            self.user_script.setup(self.log_callback, self.set_pid)
         except Exception as e:
             logger.error(f"Setup for {self.user_script.name} failed: ")
             logger.exception(e)
@@ -269,7 +273,7 @@ class Sist2UserScriptTask(Sist2Task):
         self.log_callback({"sist2-admin": f"Starting user script with {executable=}, {index_path=}, {extra_args=}"})
 
         proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.user_script.script_dir())
-        self.pid = proc.pid
+        self.set_pid(proc.pid)
 
         t_stderr = Thread(target=self._consume_logs, args=(self.log_callback, proc, "stderr", False))
         t_stderr.start()
@@ -316,7 +320,7 @@ class TaskQueue:
     def _tasks_failed(self):
         done = set()
 
-        for row in self._db["task_done"].sql("WHERE return_code NOT IN (0,1)"):
+        for row in self._db["task_done"].sql("WHERE return_code != 0"):
             done.add(uuid.UUID(row["id"]))
 
         return done

sist2-admin/sist2_admin/script.py

@@ -20,7 +20,7 @@ def set_executable(file):
     os.chmod(file, os.stat(file).st_mode | stat.S_IEXEC)
 
 
-def _initialize_git_repository(url, path, log_cb, force_clone):
+def _initialize_git_repository(url, path, log_cb, force_clone, set_pid_cb):
     log_cb({"sist2-admin": f"Cloning {url}"})
 
     if force_clone or not os.path.exists(os.path.join(path, ".git")):
@@ -36,14 +36,18 @@ def _initialize_git_repository(url, path, log_cb, force_clone):
         log_cb({"sist2-admin": f"Executing setup script {setup_script}"})
 
         set_executable(setup_script)
-        result = subprocess.run([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        proc = subprocess.Popen([setup_script], cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        for line in result.stdout.split(b"\n"):
+        set_pid_cb(proc.pid)
+        proc.wait()
+        stdout = proc.stdout.read()
+
+        for line in stdout.split(b"\n"):
             if line:
                 log_cb({"stdout": line.decode()})
 
-        log_cb({"stdout": f"Executed setup script {setup_script}, return code = {result.returncode}"})
+        log_cb({"stdout": f"Executed setup script {setup_script}, return code = {proc.returncode}"})
 
-        if result.returncode != 0:
+        if proc.returncode != 0:
             raise Exception("Error when running setup script!")
 
     log_cb({"sist2-admin": f"Initialized git repository in {path}"})
@@ -60,11 +64,11 @@ class UserScript(BaseModel):
     def script_dir(self):
         return os.path.join(SCRIPT_FOLDER, self.name)
 
-    def setup(self, log_cb):
+    def setup(self, log_cb, set_pid_cb):
         os.makedirs(self.script_dir(), exist_ok=True)
 
         if self.type == ScriptType.GIT:
-            _initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone)
+            _initialize_git_repository(self.git_repository, self.script_dir(), log_cb, self.force_clone, set_pid_cb)
             self.force_clone = False
         elif self.type == ScriptType.SIMPLE:
             self._setup_simple()
@@ -92,7 +96,7 @@ SCRIPT_TEMPLATES = {
     "CLIP - Generate embeddings to predict the most relevant image based on the text prompt": lambda name: UserScript(
         name=name,
         type=ScriptType.GIT,
-        git_repository="https://github.com/simon987/sist2-script-clip",
+        git_repository="https://github.com/sist2app/sist2-script-clip",
         extra_args="--num-tags=1 --tags-file=general.txt --color=#dcd7ff"
     ),
     "Whisper - Speech to text with OpenAI Whisper": lambda name: UserScript(

sist2-admin/sist2_admin/sist2.py

@@ -2,14 +2,15 @@ import datetime
 import json
 import logging
 import os.path
+import sys
 from datetime import datetime
 from enum import Enum
 from io import TextIOWrapper
-from logging import FileHandler
+from logging import FileHandler, StreamHandler
 from subprocess import Popen, PIPE
 from tempfile import NamedTemporaryFile
 from threading import Thread
-from typing import List
+from typing import List, Optional
 
 from pydantic import BaseModel
 
@@ -39,6 +40,8 @@ class Sist2SearchBackend(BaseModel):
 
     es_url: str = "http://elasticsearch:9200"
     es_insecure_ssl: bool = False
+    es_mappings: Optional[str] = None
+    es_settings: Optional[str] = None
     es_index: str = "sist2"
     threads: int = 1
     batch_size: int = 70
@@ -56,6 +59,8 @@ class IndexOptions(BaseModel):
     path: str = None
     incremental_index: bool = True
     search_backend: str = None
+    es_mappings_file: Optional[str] = None
+    es_settings_file: Optional[str] = None
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -74,6 +79,12 @@ class IndexOptions(BaseModel):
 
             if search_backend.es_insecure_ssl:
                 args.append(f"--es-insecure-ssl")
+
+            if self.es_mappings_file:
+                args.append(f"--mappings-file={self.es_mappings_file}")
+            if self.es_settings_file:
+                args.append(f"--settings-file={self.es_settings_file}")
+
             if self.incremental_index:
                 args.append(f"--incremental-index")
 
@@ -200,6 +211,7 @@ class WebOptions(BaseModel):
     auth0_client_id: str = None
     auth0_public_key: str = None
     auth0_public_key_file: str = None
+    verbose: bool = False
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -231,6 +243,8 @@ class WebOptions(BaseModel):
             args.append(f"--tag-auth={self.tag_auth}")
         if self.dev:
             args.append(f"--dev")
+        if self.verbose:
+            args.append(f"--very-verbose")
 
         args.extend(self.indices)
 
@@ -243,7 +257,21 @@ class Sist2:
         self.bin_path = bin_path
         self._data_dir = data_directory
 
-    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
+    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb, set_pid_cb):
+
+        if search_backend.es_mappings:
+            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
+                f.write(search_backend.es_mappings)
+            options.es_mappings_file = f.name
+        else:
+            options.es_mappings_file = None
+
+        if search_backend.es_settings:
+            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
+                f.write(search_backend.es_settings)
+            options.es_settings_file = f.name
+        else:
+            options.es_settings_file = None
 
         args = [
             self.bin_path,
@@ -255,7 +283,9 @@ class Sist2:
         logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        set_pid_cb(proc.pid)
+
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
         t_stderr.start()
 
         self._consume_logs_stdout(logs_cb, proc)
@@ -282,7 +312,7 @@ class Sist2:
 
         set_pid_cb(proc.pid)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
         t_stderr.start()
 
         self._consume_logs_stdout(logs_cb, proc)
@@ -292,7 +322,7 @@ class Sist2:
         return proc.returncode
 
     @staticmethod
-    def _consume_logs_stderr(logs_cb, proc):
+    def _consume_logs_stderr(logs_cb, exit_cb, proc):
         pipe_wrapper = TextIOWrapper(proc.stderr, encoding="utf8", errors="ignore")
         try:
             for line in pipe_wrapper:
@@ -300,7 +330,9 @@ class Sist2:
                     continue
                 logs_cb({"stderr": line})
         finally:
-            proc.wait()
+            return_code = proc.wait()
+            if exit_cb:
+                exit_cb(return_code)
             pipe_wrapper.close()
 
     @staticmethod
@@ -334,15 +366,19 @@ class Sist2:
 
         web_logger = logging.Logger(name=f"sist2-frontend-{name}")
         web_logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"frontend-{name}.log")))
+        web_logger.addHandler(StreamHandler())
 
         def logs_cb(message):
             web_logger.info(json.dumps(message))
 
+        def exit_cb(return_code):
+            logger.info(f"Web frontend exited with return code {return_code}")
+
         logger.info(f"Starting frontend {' '.join(args)}")
 
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, exit_cb, proc))
         t_stderr.start()
 
         t_stdout = Thread(target=self._consume_logs_stdout, args=(logs_cb, proc))

sist2-admin/sist2_admin/util.py

@@ -0,0 +1,41 @@
+from glob import glob
+import os
+from config import DATA_FOLDER
+
+
+def get_old_index_files(name):
+    files = glob(os.path.join(DATA_FOLDER, f"scan-{name.replace('/', '_')}-*.sist2"))
+    files = list(sorted(files, key=lambda f: os.stat(f).st_mtime))
+    files = files[-1:]
+
+    return files
+
+
+def tail_sync(filename, lines=1, _buffer=4098):
+    with open(filename) as f:
+        lines_found = []
+
+        block_counter = -1
+
+        while len(lines_found) < lines:
+            try:
+                f.seek(block_counter * _buffer, os.SEEK_END)
+            except IOError:
+                f.seek(0)
+                lines_found = f.readlines()
+                break
+
+            lines_found = f.readlines()
+
+            block_counter -= 1
+
+        return lines_found[-lines:]
+
+
+def pid_is_running(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+
+    return True

sist2-vue/package.json

@@ -9,7 +9,7 @@
   "dependencies": {
     "@auth0/auth0-spa-js": "^2.0.2",
     "@egjs/vue-infinitegrid": "3.3.0",
-    "axios": "^0.25.0",
+    "axios": "^1.6.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
     "d3": "^5.6.1",
@@ -17,7 +17,7 @@
     "dom-to-image": "^2.6.0",
     "fslightbox-vue": "fslightbox-vue.tgz",
     "nouislider": "^15.2.0",
-    "onnxruntime-web": "^1.15.1",
+    "onnxruntime-web": "1.15.1",
     "underscore": "^1.13.1",
     "vue": "^2.6.12",
     "vue-color": "^2.8.1",
@@ -32,7 +32,6 @@
     "@types/underscore": "^1.11.6",
     "@vue/cli-plugin-babel": "~5.0.8",
     "@vue/cli-plugin-router": "~5.0.8",
-    "@vue/cli-plugin-typescript": "^5.0.8",
     "@vue/cli-plugin-vuex": "~5.0.8",
     "@vue/cli-service": "^5.0.8",
     "@vue/test-utils": "^1.0.3",
@@ -44,7 +43,6 @@
     "portal-vue": "^2.1.7",
     "sass": "^1.26.11",
     "sass-loader": "^10.0.2",
-    "typescript": "^4.9.5",
     "vue-cli-plugin-bootstrap-vue": "~0.8.2",
     "vue-template-compiler": "^2.6.11"
   },

sist2-vue/src/Sist2Api.js

@@ -1,116 +1,20 @@
 import axios from "axios";
-import {ext, strUnescape, lum} from "./util";
+import {strUnescape, lum, sid} from "./util";
 import Sist2Query from "@/Sist2ElasticsearchQuery";
 import store from "@/store";
 
-export interface EsTag {
-    id: string
-    count: number
-    color: string | undefined
-    isLeaf: boolean
-}
-
-export interface Tag {
-    style: string
-    text: string
-    rawText: string
-    fg: string
-    bg: string
-    userTag: boolean
-}
-
-export interface Index {
-    name: string
-    version: string
-    id: string
-    idPrefix: string
-    timestamp: number
-    models: []
-}
-
-export interface EsHit {
-    _index: string
-    _id: string
-    _score: number
-    _type: string
-    _tags: Tag[]
-    _seq: number
-    _source: {
-        path: string
-        size: number
-        mime: string
-        name: string
-        extension: string
-        index: string
-        _depth: number
-        mtime: number
-        videoc: string
-        audioc: string
-        parent: string
-        width: number
-        height: number
-        duration: number
-        tag: string[]
-        checksum: string
-        thumbnail: string
-    }
-    _props: {
-        isSubDocument: boolean
-        isImage: boolean
-        isGif: boolean
-        isVideo: boolean
-        isPlayableVideo: boolean
-        isPlayableImage: boolean
-        isAudio: boolean
-        hasThumbnail: boolean
-        hasVidPreview: boolean
-        imageAspectRatio: number
-        /** Number of thumbnails available */
-        tnNum: number
-    }
-    highlight: {
-        name: string[] | undefined,
-        content: string[] | undefined,
-    }
-}
-
-function getIdPrefix(indices: Index[], id: string): string {
-    for (let i = 4; i < 32; i++) {
-        const prefix = id.slice(0, i);
-
-        if (indices.filter(idx => idx.id.slice(0, i) == prefix).length == 1) {
-            return prefix;
-        }
-    }
-
-    return id;
-}
-
-export interface EsResult {
-    took: number
-
-    hits: {
-        // TODO: ES 6.X ?
-        total: {
-            value: number
-        }
-        hits: EsHit[]
-    }
-
-    aggregations: any
-}
 
 class Sist2Api {
 
-    private readonly baseUrl: string
+    baseUrl;
-    private sist2Info: any
+    sist2Info;
-    private queryfunc: () => EsResult;
+    queryfunc;
 
-    constructor(baseUrl: string) {
+    constructor(baseUrl) {
         this.baseUrl = baseUrl;
     }
 
-    init(queryFunc: () => EsResult) {
+    init(queryFunc) {
         this.queryfunc = queryFunc;
     }
 
@@ -127,29 +31,16 @@ class Sist2Api {
             .filter((v, i, a) => a.findIndex(v2 => (v2.id === v.id)) === i)
     }
 
-    getSist2Info(): Promise<any> {
+    getSist2Info() {
         return axios.get(`${this.baseUrl}i`).then(resp => {
-            const indices = resp.data.indices as Index[];
-
-            resp.data.indices = indices.map(idx => {
-                return {
-                    id: idx.id,
-                    name: idx.name,
-                    timestamp: idx.timestamp,
-                    version: idx.version,
-                    models: idx.models,
-                    idPrefix: getIdPrefix(indices, idx.id),
-                } as Index;
-            });
-
             this.sist2Info = resp.data;
 
             return resp.data;
         })
     }
 
-    setHitProps(hit: EsHit): void {
+    setHitProps(hit) {
-        hit["_props"] = {} as any;
+        hit["_props"] = {};
 
         const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
 
@@ -157,7 +48,7 @@ class Sist2Api {
             hit._props.isSubDocument = true;
         }
 
-        if ("thumbnail" in hit._source) {
+        if ("thumbnail" in hit._source && hit._source.thumbnail > 0) {
             hit._props.hasThumbnail = true;
 
             if (Number.isNaN(Number(hit._source.thumbnail))) {
@@ -178,7 +69,8 @@ class Sist2Api {
                     hit._props.isImage = true;
                 }
                 if ("width" in hit._source && !hit._props.isSubDocument && hit._source.videoc !== "tiff"
-                    && hit._source.videoc !== "raw" && hit._source.videoc !== "ppm") {
+                    && hit._source.videoc !== "raw" && hit._source.videoc !== "ppm"
+                    && hit._source.mime !== "image/jp2") {
                     hit._props.isPlayableImage = true;
                 }
                 if ("width" in hit._source && "height" in hit._source) {
@@ -213,8 +105,8 @@ class Sist2Api {
         }
     }
 
-    setHitTags(hit: EsHit): void {
+    setHitTags(hit) {
-        const tags = [] as Tag[];
+        const tags = [];
 
         // User tags
         if ("tag" in hit._source) {
@@ -226,10 +118,10 @@ class Sist2Api {
         hit._tags = tags;
     }
 
-    createUserTag(tag: string): Tag {
+    createUserTag(tag) {
         const tokens = tag.split(".");
 
-        const colorToken = tokens.pop() as string;
+        const colorToken = tokens.pop();
 
         const bg = colorToken;
         const fg = lum(colorToken) > 50 ? "#000" : "#fff";
@@ -241,25 +133,30 @@ class Sist2Api {
             text: tokens.join("."),
             rawText: tag,
             userTag: true,
-        } as Tag;
+        };
     }
 
-    search(): Promise<EsResult> {
+    search() {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.ftsQuery(this.queryfunc())
         } else {
             return this.esQuery(this.queryfunc());
         }
     }
 
-    esQuery(query: any): Promise<EsResult> {
+    _getIndexRoot(indexId) {
+        return this.sist2Info.indices.find(idx => idx.id === indexId).root;
+    }
+
+    esQuery(query) {
         return axios.post(`${this.baseUrl}es`, query).then(resp => {
-            const res = resp.data as EsResult;
+            const res = resp.data;
 
             if (res.hits?.hits) {
-                res.hits.hits.forEach((hit: EsHit) => {
+                res.hits.hits.forEach((hit) => {
                     hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
                     hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+                    hit["_source"]["indexRoot"] = this._getIndexRoot(hit["_source"]["index"]);
 
                     this.setHitProps(hit);
                     this.setHitTags(hit);
@@ -270,9 +167,9 @@ class Sist2Api {
         });
     }
 
-    ftsQuery(query: any): Promise<EsResult> {
+    ftsQuery(query) {
         return axios.post(`${this.baseUrl}fts/search`, query).then(resp => {
-            const res = resp.data as any;
+            const res = resp.data;
 
             if (res.hits.hits) {
                 res.hits.hits.forEach(hit => {
@@ -293,7 +190,7 @@ class Sist2Api {
         });
     }
 
-    private getMimeTypesEs(query) {
+    getMimeTypesEs(query) {
         const AGGS = {
             mimeTypes: {
                 terms: {
@@ -322,7 +219,7 @@ class Sist2Api {
         });
     }
 
-    private getMimeTypesSqlite(): Promise<[{ mime: string, count: number }]> {
+    getMimeTypesSqlite() {
         return axios.get(`${this.baseUrl}fts/mimetypes`)
             .then(resp => {
                 return resp.data;
@@ -332,15 +229,15 @@ class Sist2Api {
     async getMimeTypes(query = undefined) {
         let buckets;
 
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             buckets = await this.getMimeTypesSqlite();
         } else {
             buckets = await this.getMimeTypesEs(query);
         }
 
-        const mimeMap: any[] = [];
+        const mimeMap = [];
 
-        buckets.sort((a: any, b: any) => a.mime > b.mime).forEach((bucket: any) => {
+        buckets.sort((a, b) => a.mime > b.mime).forEach((bucket) => {
             const tmp = bucket.mime.split("/");
             const category = tmp[0];
             const mime = tmp[1];
@@ -374,7 +271,7 @@ class Sist2Api {
         return {buckets, mimeMap};
     }
 
-    _createEsTag(tag: string, count: number): EsTag {
+    _createEsTag(tag, count) {
         const tokens = tag.split(".");
 
         if (/.*\.#[0-9a-fA-F]{6}/.test(tag)) {
@@ -394,7 +291,7 @@ class Sist2Api {
         };
     }
 
-    private getTagsEs() {
+    getTagsEs() {
         return this.esQuery({
             aggs: {
                 tags: {
@@ -407,21 +304,21 @@ class Sist2Api {
             size: 0,
         }).then(resp => {
             return resp["aggregations"]["tags"]["buckets"]
-                .sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
+                .sort((a, b) => a["key"].localeCompare(b["key"]))
-                .map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
+                .map((bucket) => this._createEsTag(bucket["key"], bucket["doc_count"]));
         });
     }
 
-    private getTagsSqlite() {
+    getTagsSqlite() {
-        return axios.get(`${this.baseUrl}/fts/tags`)
+        return axios.get(`${this.baseUrl}fts/tags`)
             .then(resp => {
                 return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
             });
     }
 
-    async getTags(): Promise<EsTag[]> {
+    async getTags() {
         let tags;
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             tags = await this.getTagsSqlite();
         } else {
             tags = await this.getTagsEs();
@@ -430,7 +327,7 @@ class Sist2Api {
         // Remove duplicates (same tag with different color)
         const seen = new Set();
 
-        return tags.filter((t: EsTag) => {
+        return tags.filter((t) => {
             if (seen.has(t.id)) {
                 return false;
             }
@@ -439,31 +336,29 @@ class Sist2Api {
         });
     }
 
-    saveTag(tag: string, hit: EsHit) {
+    saveTag(tag, hit) {
-        return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
+        return axios.post(`${this.baseUrl}tag/${sid(hit)}`, {
             delete: false,
             name: tag,
-            doc_id: hit["_id"]
         });
     }
 
-    deleteTag(tag: string, hit: EsHit) {
+    deleteTag(tag, hit) {
-        return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
+        return axios.post(`${this.baseUrl}tag/${sid(hit)}`, {
             delete: true,
             name: tag,
-            doc_id: hit["_id"]
         });
     }
 
     searchPaths(indexId, minDepth, maxDepth, prefix = null) {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.searchPathsSqlite(indexId, minDepth, minDepth, prefix);
         } else {
             return this.searchPathsEs(indexId, minDepth, maxDepth, prefix);
         }
     }
 
-    private searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
+    searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
         return axios.post(`${this.baseUrl}fts/paths`, {
             indexId, minDepth, maxDepth, prefix
         }).then(resp => {
@@ -471,7 +366,7 @@ class Sist2Api {
         });
     }
 
-    private searchPathsEs(indexId, minDepth, maxDepth, prefix): Promise<[{ path: string, count: number }]> {
+    searchPathsEs(indexId, minDepth, maxDepth, prefix) {
 
         const query = {
             query: {
@@ -516,23 +411,25 @@ class Sist2Api {
         });
     }
 
-    private getDateRangeSqlite() {
+    getDateRangeSqlite() {
         return axios.get(`${this.baseUrl}fts/dateRange`)
             .then(resp => ({
                 min: resp.data.dateMin,
-                max: resp.data.dateMax,
+                max: (resp.data.dateMax === resp.data.dateMin)
+                    ? resp.data.dateMax + 1
+                    : resp.data.dateMax,
             }));
     }
 
-    getDateRange(): Promise<{ min: number, max: number }> {
+    getDateRange() {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.getDateRangeSqlite();
         } else {
             return this.getDateRangeEs();
         }
     }
 
-    private getDateRangeEs() {
+    getDateRangeEs() {
         return this.esQuery({
             // TODO: filter current selected indices
             aggs: {
@@ -549,7 +446,7 @@ class Sist2Api {
             if (range.min == null) {
                 range.min = 0;
                 range.max = 1;
-            } else if (range.min == range.max) {
+            } else if (range.min === range.max) {
                 range.max += 1;
             }
 
@@ -557,7 +454,7 @@ class Sist2Api {
         });
     }
 
-    private getPathSuggestionsSqlite(text: string) {
+    getPathSuggestionsSqlite(text) {
         return axios.post(`${this.baseUrl}fts/paths`, {
             prefix: text,
             minDepth: 1,
@@ -567,7 +464,7 @@ class Sist2Api {
         })
     }
 
-    private getPathSuggestionsEs(text) {
+    getPathSuggestionsEs(text) {
         return this.esQuery({
             suggest: {
                 path: {
@@ -585,31 +482,31 @@ class Sist2Api {
         });
     }
 
-    getPathSuggestions(text: string): Promise<string[]> {
+    getPathSuggestions(text) {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.getPathSuggestionsSqlite(text);
         } else {
             return this.getPathSuggestionsEs(text)
         }
     }
 
-    getTreemapStat(indexId: string) {
+    getTreemapStat(indexId) {
         return `${this.baseUrl}s/${indexId}/TMAP`;
     }
 
-    getMimeStat(indexId: string) {
+    getMimeStat(indexId) {
         return `${this.baseUrl}s/${indexId}/MAGG`;
     }
 
-    getSizeStat(indexId: string) {
+    getSizeStat(indexId) {
         return `${this.baseUrl}s/${indexId}/SAGG`;
     }
 
-    getDateStat(indexId: string) {
+    getDateStat(indexId) {
         return `${this.baseUrl}s/${indexId}/DAGG`;
     }
 
-    private getDocumentEs(docId: string, highlight: boolean, fuzzy: boolean) {
+    getDocumentEs(sid, highlight, fuzzy) {
         const query = Sist2Query.searchQuery();
 
         if (highlight) {
@@ -648,7 +545,7 @@ class Sist2Api {
             query.query.bool.must = [query.query.bool.must];
         }
 
-        query.query.bool.must.push({match: {_id: docId}});
+        query.query.bool.must.push({match: {_id: sid}});
 
         delete query["sort"];
         delete query["aggs"];
@@ -669,35 +566,35 @@ class Sist2Api {
         });
     }
 
-    private getDocumentSqlite(docId: string): Promise<EsHit> {
+    getDocumentSqlite(sid) {
-        return axios.get(`${this.baseUrl}/fts/d/${docId}`)
+        return axios.get(`${this.baseUrl}fts/d/${sid}`)
             .then(resp => ({
                 _source: resp.data
-            } as EsHit));
+            }));
     }
 
-    getDocument(docId: string, highlight: boolean, fuzzy: boolean): Promise<EsHit | null> {
+    getDocument(sid, highlight, fuzzy) {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
-            return this.getDocumentSqlite(docId);
+            return this.getDocumentSqlite(sid);
         } else {
-            return this.getDocumentEs(docId, highlight, fuzzy);
+            return this.getDocumentEs(sid, highlight, fuzzy);
         }
     }
 
-    getTagSuggestions(prefix: string): Promise<string[]> {
+    getTagSuggestions(prefix) {
-        if (this.backend() == "sqlite") {
+        if (this.backend() === "sqlite") {
             return this.getTagSuggestionsSqlite(prefix);
         } else {
             return this.getTagSuggestionsEs(prefix);
         }
     }
 
-    private getTagSuggestionsSqlite(prefix): Promise<string[]> {
+    getTagSuggestionsSqlite(prefix) {
-        return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
+        return axios.post(`${this.baseUrl}fts/suggestTags`, prefix)
             .then(resp => (resp.data));
     }
 
-    private getTagSuggestionsEs(prefix): Promise<string[]> {
+    getTagSuggestionsEs(prefix) {
         return this.esQuery({
             suggest: {
                 tag: {
@@ -723,8 +620,8 @@ class Sist2Api {
         });
     }
 
-    getEmbeddings(indexId, docId, modelId) {
+    getEmbeddings(sid, modelId) {
-        return axios.post(`${this.baseUrl}/e/${indexId}/${docId}/${modelId.toString().padStart(3, '0')}`)
+        return axios.post(`${this.baseUrl}e/${sid}/${modelId.toString().padStart(3, '0')}`)
             .then(resp => (resp.data));
     }
 }

sist2-vue/src/Sist2ElasticsearchQuery.js

@@ -1,5 +1,5 @@
-import store from "./store";
+import store from "@/store";
-import sist2Api, {EsHit, Index} from "@/Sist2Api";
+import Sist2Api from "@/Sist2Api";
 
 const SORT_MODES = {
     score: {
@@ -7,62 +7,62 @@ const SORT_MODES = {
             {_score: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._score
+        key: (hit) => hit._score
     },
     random: {
         mode: [
             {_score: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._score
+        key: (hit) => hit._score
     },
     dateAsc: {
         mode: [
             {mtime: {order: "asc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.mtime
+        key: (hit) => hit._source.mtime
     },
     dateDesc: {
         mode: [
             {mtime: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.mtime
+        key: (hit) => hit._source.mtime
     },
     sizeAsc: {
         mode: [
             {size: {order: "asc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.size
+        key: (hit) => hit._source.size
     },
     sizeDesc: {
         mode: [
             {size: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.size
+        key: (hit) => hit._source.size
     },
     nameAsc: {
         mode: [
             {name: {order: "asc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.name
+        key: (hit) => hit._source.name
     },
     nameDesc: {
         mode: [
             {name: {order: "desc"}},
             {_tie: {order: "asc"}}
         ],
-        key: (hit: EsHit) => hit._source.name
+        key: (hit) => hit._source.name
     }
-} as any;
+};
 
 class Sist2ElasticsearchQuery {
 
-    searchQuery(blankSearch: boolean = false): any {
+    searchQuery(blankSearch = false) {
 
         const getters = store.getters;
 
@@ -76,7 +76,7 @@ class Sist2ElasticsearchQuery {
         const fuzzy = getters.fuzzy;
         const size = getters.size;
         const after = getters.lastDoc;
-        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
+        const selectedIndexIds = getters.selectedIndices.map((idx) => idx.id)
         const selectedMimeTypes = getters.selectedMimeTypes;
         const selectedTags = getters.selectedTags;
         const sortMode = getters.embedding ? "score" : getters.sortMode;
@@ -86,7 +86,7 @@ class Sist2ElasticsearchQuery {
 
         const filters = [
             {terms: {index: selectedIndexIds}}
-        ] as any[];
+        ];
 
         const fields = [
             "name^8",
@@ -117,11 +117,11 @@ class Sist2ElasticsearchQuery {
             }
 
             if (dateMin && dateMax) {
-                filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
+                filters.push({range: {mtime: {gte: dateMin, lte: dateMax, format: "epoch_second"}}})
             } else if (dateMin) {
-                filters.push({range: {mtime: {gte: dateMin}}})
+                filters.push({range: {mtime: {gte: dateMin, format: "epoch_second"}}})
             } else if (dateMax) {
-                filters.push({range: {mtime: {lte: dateMax}}})
+                filters.push({range: {mtime: {lte: dateMax, format: "epoch_second"}}})
             }
 
             const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
@@ -138,7 +138,7 @@ class Sist2ElasticsearchQuery {
                 if (getters.optTagOrOperator) {
                     filters.push({terms: {"tag": selectedTags}});
                 } else {
-                    selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
+                    selectedTags.forEach((tag) => filters.push({term: {"tag": tag}}));
                 }
             }
         }
@@ -173,7 +173,7 @@ class Sist2ElasticsearchQuery {
             },
             sort: SORT_MODES[sortMode].mode,
             size: size,
-        } as any;
+        };
 
         if (!after) {
             q.aggs = {
@@ -193,7 +193,7 @@ class Sist2ElasticsearchQuery {
         if (getters.embedding) {
             delete q.query;
 
-            const field = "emb." + sist2Api.models().find(m => m.id == getters.embeddingsModel).path;
+            const field = "emb." + Sist2Api.models().find(m => m.id === getters.embeddingsModel).path;
 
             if (hasKnn) {
                 // Use knn (8.8+)
@@ -238,7 +238,7 @@ class Sist2ElasticsearchQuery {
                 pre_tags: ["<mark>"],
                 post_tags: ["</mark>"],
                 fragment_size: getters.optFragmentSize,
-                number_of_fragments: 1,
+                number_of_fragments: getters.optFragmentCount,
                 order: "score",
                 fields: {
                     content: {},

sist2-vue/src/Sist2SqliteQuery.js

@@ -1,5 +1,4 @@
-import store from "./store";
+import store from "@/store";
-import {EsHit, Index} from "@/Sist2Api";
 
 const SORT_MODES = {
     score: {
@@ -29,18 +28,12 @@ const SORT_MODES = {
         "sort": "name",
         "sortAsc": false
     }
-} as any;
+};
-
-interface SortMode {
-    text: string
-    mode: any[]
-    key: (hit: EsHit) => any
-}
 
 
 class Sist2ElasticsearchQuery {
 
-    searchQuery(): any {
+    searchQuery() {
 
         const getters = store.getters;
 
@@ -52,7 +45,7 @@ class Sist2ElasticsearchQuery {
         const dateMax = getters.dateMax;
         const size = getters.size;
         const after = getters.lastDoc;
-        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
+        const selectedIndexIds = getters.selectedIndices.map((idx) => idx.id)
         const selectedMimeTypes = getters.selectedMimeTypes;
         const selectedTags = getters.selectedTags;
 
@@ -95,7 +88,7 @@ class Sist2ElasticsearchQuery {
         if (selectedTags.length > 0) {
             q["tags"] = selectedTags
         }
-        if (getters.sortMode == "random") {
+        if (getters.sortMode === "random") {
             q["seed"] = getters.seed;
         }
         if (getters.optHighlight) {
@@ -108,11 +101,13 @@ class Sist2ElasticsearchQuery {
             q["embedding"] = getters.embedding;
             q["sort"] = "embedding";
             q["sortAsc"] = false;
-        } else if (getters.sortMode == "embedding") {
+        } else if (getters.sortMode === "embedding") {
             q["sort"] = "sort"
             q["sortAsc"] = true;
         }
 
+        q["searchInPath"] = getters.optSearchInPath;
+
         return q;
     }
 }

sist2-vue/src/components/ContentDiv.vue

@@ -3,6 +3,8 @@
 </template>
 
 <script>
+const FRAGMENT_SEPARATOR = "<br /><i style='line-height: 2.4'>[…]</i><br/>";
+
 export default {
   name: "ContentDiv",
   props: ["doc"],
@@ -13,10 +15,10 @@ export default {
       }
 
       if (this.doc.highlight["content.nGram"]) {
-        return this.doc.highlight["content.nGram"][0];
+        return this.doc.highlight["content.nGram"].join(FRAGMENT_SEPARATOR);
       }
       if (this.doc.highlight.content) {
-        return this.doc.highlight.content[0];
+        return this.doc.highlight.content.join(FRAGMENT_SEPARATOR);
       }
     }
   }

sist2-vue/src/components/D3Treemap.vue

@@ -9,8 +9,7 @@
 
 <script>
 import * as d3 from "d3";
-import {burrow} from "@/util-js"
+import {humanFileSize, burrow} from "@/util";
-import {humanFileSize} from "@/util";
 import Sist2Api from "@/Sist2Api";
 import domtoimage from "dom-to-image";
 

sist2-vue/src/components/DateSlider.vue

@@ -31,8 +31,7 @@
 <script>
 import noUiSlider from 'nouislider';
 import 'nouislider/dist/nouislider.css';
-import {humanDate} from "@/util";
+import {humanDate, mergeTooltips} from "@/util";
-import {mergeTooltips} from "@/util-js";
 
 export default {
   name: "DateSlider",

sist2-vue/src/components/DocCard.vue

@@ -16,7 +16,7 @@
             <!-- Audio player-->
             <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
                    :type="doc._source.mime"
-                   :src="`f/${doc._source.index}/${doc._id}`"
+                   :src="`f/${sid(doc)}`"
                    @play="onAudioPlay()"></audio>
 
             <b-card-body class="padding-03">
@@ -43,7 +43,7 @@
 </template>
 
 <script>
-import {ext, humanFileSize, humanTime} from "@/util";
+import {ext, humanFileSize, humanTime, sid} from "@/util";
 import TagContainer from "@/components/TagContainer.vue";
 import DocFileTitle from "@/components/DocFileTitle.vue";
 import DocInfoModal from "@/components/DocInfoModal.vue";
@@ -69,13 +69,14 @@ export default {
         }
     },
     methods: {
+        sid: sid,
         humanFileSize: humanFileSize,
         humanTime: humanTime,
         onInfoClick() {
             this.showInfo = true;
         },
         onEmbeddingClick() {
-            Sist2Api.getEmbeddings(this.doc._source.index, this.doc._id, this.$store.state.embeddingsModel).then(embeddings => {
+            Sist2Api.getEmbeddings(sid(this.doc), this.$store.state.embeddingsModel).then(embeddings => {
                 this.$store.commit("setEmbeddingText", "");
                 this.$store.commit("setEmbedding", embeddings);
                 this.$store.commit("setEmbeddingDoc", this.doc);

sist2-vue/src/components/DocCardWall.vue

@@ -1,15 +1,16 @@
 <template>
-  <GridLayout
+    <GridLayout
-      ref="grid-layout"
+            ref="grid-layout"
-      :options="gridOptions"
+            :options="gridOptions"
-      @append="append"
+            @append="append"
-      @layout-complete="$emit('layout-complete')"
+            @layout-complete="$emit('layout-complete')"
-  >
+    >
-    <DocCard v-for="doc in docs" :key="doc._id" :doc="doc" :width="width"></DocCard>
+        <DocCard v-for="doc in docs" :key="sid(doc)" :doc="doc" :width="width"></DocCard>
-  </GridLayout>
+    </GridLayout>
 </template>
 
 <script>
+import {sid} from "@/util";
 import Vue from "vue";
 import DocCard from "@/components/DocCard";
 
@@ -18,50 +19,53 @@ import VueInfiniteGrid from "@egjs/vue-infinitegrid";
 Vue.use(VueInfiniteGrid);
 
 export default Vue.extend({
-  components: {
+    components: {
-    DocCard,
+        DocCard,
-  },
-  props: ["docs", "append"],
-  data() {
-    return {
-      width: 0,
-      gridOptions: {
-        align: "center",
-        margin: 0,
-        transitionDuration: 0,
-        isOverflowScroll: false,
-        isConstantSize: false,
-        useFit: false,
-        // Indicates whether keep the number of DOMs is maintained. If the useRecycle value is 'true', keep the number
-        //  of DOMs is maintained. If the useRecycle value is 'false', the number of DOMs will increase as card elements
-        //  are added.
-        useRecycle: false
-      }
-    }
-  },
-  computed: {
-    colCount() {
-      const columns = this.$store.getters["optColumns"];
-
-      if (columns === "auto") {
-        return Math.round(this.$refs["grid-layout"].$el.scrollWidth / 300)
-      }
-      return columns;
     },
-  },
+    props: ["docs", "append"],
-  mounted() {
+    data() {
-    this.width = this.$refs["grid-layout"].$el.scrollWidth / this.colCount;
+        return {
+            width: 0,
+            gridOptions: {
+                align: "center",
+                margin: 0,
+                transitionDuration: 0,
+                isOverflowScroll: false,
+                isConstantSize: false,
+                useFit: false,
+                // Indicates whether keep the number of DOMs is maintained. If the useRecycle value is 'true', keep the number
+                //  of DOMs is maintained. If the useRecycle value is 'false', the number of DOMs will increase as card elements
+                //  are added.
+                useRecycle: false
+            }
+        }
+    },
+    methods: {
+        sid,
+    },
+    computed: {
+        colCount() {
+            const columns = this.$store.getters["optColumns"];
 
-    if (this.colCount === 1) {
+            if (columns === "auto") {
-      this.$refs["grid-layout"].$el.classList.add("grid-single-column");
+                return Math.round(this.$refs["grid-layout"].$el.scrollWidth / 300)
-    }
+            }
+            return columns;
+        },
+    },
+    mounted() {
+        this.width = this.$refs["grid-layout"].$el.scrollWidth / this.colCount;
 
-    this.$store.subscribe((mutation) => {
+        if (this.colCount === 1) {
-      if (mutation.type === "busUpdateWallItems" && this.$refs["grid-layout"]) {
+            this.$refs["grid-layout"].$el.classList.add("grid-single-column");
-        this.$refs["grid-layout"].updateItems();
+        }
-      }
+
-    });
+        this.$store.subscribe((mutation) => {
-  },
+            if (mutation.type === "busUpdateWallItems" && this.$refs["grid-layout"]) {
+                this.$refs["grid-layout"].updateItems();
+            }
+        });
+    },
 });
 </script>
 

sist2-vue/src/components/DocFileTitle.vue

@@ -1,5 +1,5 @@
 <template>
-    <a :href="`f/${doc._source.index}/${doc._id}`"
+    <a :href="`f/${sid(doc)}`"
        :class="doc._source.embedding ? 'file-title-anchor-with-embedding' : 'file-title-anchor'" target="_blank">
         <div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
              v-html="fileName() + ext(doc)"></div>
@@ -7,12 +7,13 @@
 </template>
 
 <script>
-import {ext} from "@/util";
+import {ext, sid} from "@/util";
 
 export default {
     name: "DocFileTitle",
     props: ["doc"],
     methods: {
+        sid: sid,
         ext: ext,
         fileName() {
             if (!this.doc.highlight) {

sist2-vue/src/components/DocInfoModal.vue

@@ -1,33 +1,34 @@
 <template>
-  <b-modal :visible="show" size="lg" :hide-footer="true" static lazy @close="$emit('close')" @hide="$emit('close')"
+    <b-modal :visible="show" size="lg" :hide-footer="true" static lazy @close="$emit('close')" @hide="$emit('close')"
-  >
+    >
-    <template #modal-title>
+        <template #modal-title>
-      <h5 class="modal-title" :title="doc._source.name + ext(doc)">
+            <h5 class="modal-title" :title="doc._source.name + ext(doc)">
-        {{ doc._source.name + ext(doc) }}
+                {{ doc._source.name + ext(doc) }}
-        <router-link :to="`/file?byId=${doc._id}`">#</router-link>
+                <router-link :to="`/file?byId=${doc._id}`">#</router-link>
-      </h5>
+            </h5>
-    </template>
+        </template>
 
-    <img v-if="doc._props.hasThumbnail" :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">
+        <img v-if="doc._props.hasThumbnail" :src="`t/${sid(doc)}`" alt="" class="fit card-img-top">
 
-    <InfoTable :doc="doc"></InfoTable>
+        <InfoTable :doc="doc"></InfoTable>
 
-    <LazyContentDiv :doc-id="doc._id"></LazyContentDiv>
+        <LazyContentDiv :sid="sid(doc)"></LazyContentDiv>
-  </b-modal>
+    </b-modal>
 </template>
 
 <script>
-import {ext} from "@/util";
+import {ext, sid} from "@/util";
 import InfoTable from "@/components/InfoTable";
 import LazyContentDiv from "@/components/LazyContentDiv";
 
 export default {
-  name: "DocInfoModal",
+    name: "DocInfoModal",
-  components: {LazyContentDiv, InfoTable},
+    components: {LazyContentDiv, InfoTable},
-  props: ["doc", "show"],
+    props: ["doc", "show"],
-  methods: {
+    methods: {
-    ext: ext,
+        ext: ext,
-  }
+        sid: sid
+    }
 }
 </script>
 

sist2-vue/src/components/DocList.vue

@@ -1,45 +1,49 @@
 <template>
-  <b-list-group class="mt-3">
+    <b-list-group class="mt-3">
-    <DocListItem v-for="doc in docs" :key="doc._id" :doc="doc"></DocListItem>
+        <DocListItem v-for="doc in docs" :key="sid(doc)" :doc="doc"></DocListItem>
-  </b-list-group>
+    </b-list-group>
 </template>
 
-<script lang="ts">
+<script>
+import {sid} from "@/util";
 import DocListItem from "@/components/DocListItem.vue";
 import Vue from "vue";
 
 export default Vue.extend({
-  name: "DocList",
+    name: "DocList",
-  components: {DocListItem},
+    components: {DocListItem},
-  props: ["docs", "append"],
+    props: ["docs", "append"],
-  mounted() {
+    mounted() {
-    window.addEventListener("scroll", () => {
+        window.addEventListener("scroll", () => {
-      const threshold = 400;
+            const threshold = 400;
-      const app = document.getElementById("app");
+            const app = document.getElementById("app");
 
-      if ((window.innerHeight + window.scrollY) >= app.offsetHeight - threshold) {
+            if ((window.innerHeight + window.scrollY) >= app.offsetHeight - threshold) {
-        this.append();
+                this.append();
-      }
+            }
-    });
+        });
-  }
+    },
+    methods: {
+        sid: sid
+    }
 });
 </script>
 
 <style>
 
 .theme-black .list-group-item {
-  background: #212121;
+    background: #212121;
-  color: #e0e0e0;
+    color: #e0e0e0;
 
-  border-bottom: none;
+    border-bottom: none;
-  border-left: none;
+    border-left: none;
-  border-right: none;
+    border-right: none;
-  border-radius: 0;
+    border-radius: 0;
-  padding: .25rem 0.5rem;
+    padding: .25rem 0.5rem;
 }
 
 .theme-black .list-group-item:first-child {
-  border-top: none;
+    border-top: none;
 }
 
 </style>

sist2-vue/src/components/DocListItem.vue

@@ -17,10 +17,10 @@
                     </div>
 
                     <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-                         :src="(doc._props.isGif && hover) ? `f/${doc._source.index}/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
+                         :src="(doc._props.isGif && hover) ? `f/${sid(doc)}` : `t/${sid(doc)}`"
                          alt=""
                          class="pointer fit-sm" @click="onThumbnailClick()">
-                    <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
+                    <img v-else :src="`t/${sid(doc)}`" alt=""
                          class="fit-sm">
                 </div>
             </div>
@@ -70,6 +70,7 @@ import FileIcon from "@/components/icons/FileIcon";
 import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
 import MLIcon from "@/components/icons/MlIcon.vue";
 import Sist2Api from "@/Sist2Api";
+import {sid} from "@/util";
 
 export default {
     name: "DocListItem",
@@ -82,12 +83,13 @@ export default {
         }
     },
     methods: {
+        sid: sid,
         async onThumbnailClick() {
             this.$store.commit("setUiLightboxSlide", this.doc._seq);
             await this.$store.dispatch("showLightbox");
         },
         onEmbeddingClick() {
-            Sist2Api.getEmbeddings(this.doc._source.index, this.doc._id, this.$store.state.embeddingsModel).then(embeddings => {
+            Sist2Api.getEmbeddings(sid(this.doc), this.$store.state.embeddingsModel).then(embeddings => {
                 this.$store.commit("setEmbeddingText", "");
                 this.$store.commit("setEmbedding", embeddings);
                 this.$store.commit("setEmbeddingDoc", this.doc);

sist2-vue/src/components/FullThumbnail.vue

@@ -1,188 +1,189 @@
 <template>
-  <div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()"
+    <div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()"
-       @touchstart="onTouchStart()">
+         @touchstart="onTouchStart()">
-    <div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
+        <div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
-      <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
+            <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
+        </div>
+
+        <div
+                v-if="doc._props.isImage && doc._props.imageAspectRatio < 5"
+                class="card-img-overlay"
+                :class="{'small-badge': smallBadge}">
+            <span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
+        </div>
+
+        <div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0"
+             class="card-img-overlay"
+             :class="{'small-badge': smallBadge}">
+            <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
+        </div>
+
+        <div v-if="doc._props.isPlayableVideo" class="play">
+            <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
+                <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
+            </svg>
+        </div>
+
+        <img ref="tn"
+             v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
+             :src="tnSrc"
+             alt=""
+             :style="{height: (doc._props.isGif && hover) ? `${tnHeight()}px` : undefined}"
+             class="pointer fit card-img-top" @click="onThumbnailClick()">
+        <img v-else :src="tnSrc" alt=""
+             class="fit card-img-top">
+
+        <ThumbnailProgressBar v-if="hover && doc._props.hasVidPreview"
+                              :progress="(currentThumbnailNum + 1) / (doc._props.tnNum)"
+        ></ThumbnailProgressBar>
     </div>
-
-    <div
-        v-if="doc._props.isImage && doc._props.imageAspectRatio < 5"
-        class="card-img-overlay"
-        :class="{'small-badge': smallBadge}">
-      <span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
-    </div>
-
-    <div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0"
-         class="card-img-overlay"
-         :class="{'small-badge': smallBadge}">
-      <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
-    </div>
-
-    <div v-if="doc._props.isPlayableVideo" class="play">
-      <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
-        <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
-      </svg>
-    </div>
-
-    <img ref="tn"
-         v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-         :src="tnSrc"
-         alt=""
-         :style="{height: (doc._props.isGif && hover) ? `${tnHeight()}px` : undefined}"
-         class="pointer fit card-img-top" @click="onThumbnailClick()">
-    <img v-else :src="tnSrc" alt=""
-         class="fit card-img-top">
-
-    <ThumbnailProgressBar v-if="hover && doc._props.hasVidPreview"
-                          :progress="(currentThumbnailNum + 1) / (doc._props.tnNum)"
-    ></ThumbnailProgressBar>
-  </div>
 </template>
 
 <script>
-import {humanTime} from "@/util";
+import {humanTime, sid} from "@/util";
 import ThumbnailProgressBar from "@/components/ThumbnailProgressBar";
 
 export default {
-  name: "FullThumbnail",
+    name: "FullThumbnail",
-  props: ["doc", "smallBadge"],
+    props: ["doc", "smallBadge"],
-  components: {ThumbnailProgressBar},
+    components: {ThumbnailProgressBar},
-  data() {
+    data() {
-    return {
+        return {
-      hover: false,
+            hover: false,
-      currentThumbnailNum: 0,
+            currentThumbnailNum: 0,
-      timeoutId: null
+            timeoutId: null
+        }
+    },
+    created() {
+        this.$store.subscribe((mutation) => {
+            if (mutation.type === "busTnTouchStart" && mutation.payload !== this.doc._id) {
+                this.onTnLeave();
+            }
+        });
+    },
+    computed: {
+        tnSrc() {
+            return this.getThumbnailSrc(this.currentThumbnailNum);
+        },
+    },
+    methods: {
+        sid: sid,
+        getThumbnailSrc(thumbnailNum) {
+            const doc = this.doc;
+            const props = doc._props;
+            if (props.isGif && this.hover) {
+                return `f/${sid(doc)}`;
+            }
+            return (this.currentThumbnailNum === 0)
+                ? `t/${sid(doc)}`
+                : `t/${sid(doc)}/${String(thumbnailNum).padStart(4, "0")}`;
+        },
+        humanTime: humanTime,
+        onThumbnailClick() {
+            this.$emit("onThumbnailClick");
+        },
+        tnHeight() {
+            return this.$refs.tn.height;
+        },
+        tnWidth() {
+            return this.$refs.tn.width;
+        },
+        onTnEnter() {
+            this.hover = true;
+            const start = Date.now()
+            if (this.doc._props.hasVidPreview) {
+                let img = new Image();
+                img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
+                img.onload = () => {
+                    this.currentThumbnailNum += 1;
+                    this.scheduleNextTnNum(Date.now() - start);
+                }
+            }
+        },
+        onTnLeave() {
+            this.currentThumbnailNum = 0;
+            this.hover = false;
+            if (this.timeoutId !== null) {
+                window.clearTimeout(this.timeoutId);
+                this.timeoutId = null;
+            }
+        },
+        scheduleNextTnNum(offset = 0) {
+            const INTERVAL = (this.$store.state.optVidPreviewInterval ?? 700) - offset;
+            this.timeoutId = window.setTimeout(() => {
+                const start = Date.now();
+                if (!this.hover) {
+                    return;
+                }
+                if (this.currentThumbnailNum === this.doc._props.tnNum - 1) {
+                    this.currentThumbnailNum = 0;
+                    this.scheduleNextTnNum();
+                } else {
+                    let img = new Image();
+                    img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
+                    img.onload = () => {
+                        this.currentThumbnailNum += 1;
+                        this.scheduleNextTnNum(Date.now() - start);
+                    }
+                }
+            }, INTERVAL);
+        },
+        onTouchStart() {
+            this.$store.commit("busTnTouchStart", this.doc._id);
+            if (!this.hover) {
+                this.onTnEnter()
+            }
+        },
     }
-  },
-  created() {
-    this.$store.subscribe((mutation) => {
-      if (mutation.type === "busTnTouchStart" && mutation.payload !== this.doc._id) {
-        this.onTnLeave();
-      }
-    });
-  },
-  computed: {
-    tnSrc() {
-      return this.getThumbnailSrc(this.currentThumbnailNum);
-    },
-  },
-  methods: {
-    getThumbnailSrc(thumbnailNum) {
-      const doc = this.doc;
-      const props = doc._props;
-      if (props.isGif && this.hover) {
-        return `f/${doc._source.index}/${doc._id}`;
-      }
-      return (this.currentThumbnailNum === 0)
-          ? `t/${doc._source.index}/${doc._id}`
-          : `t/${doc._source.index}/${doc._id}/${String(thumbnailNum).padStart(4, "0")}`;
-    },
-    humanTime: humanTime,
-    onThumbnailClick() {
-      this.$emit("onThumbnailClick");
-    },
-    tnHeight() {
-      return this.$refs.tn.height;
-    },
-    tnWidth() {
-      return this.$refs.tn.width;
-    },
-    onTnEnter() {
-      this.hover = true;
-      const start = Date.now()
-      if (this.doc._props.hasVidPreview) {
-        let img = new Image();
-        img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
-        img.onload = () => {
-          this.currentThumbnailNum += 1;
-          this.scheduleNextTnNum(Date.now() - start);
-        }
-      }
-    },
-    onTnLeave() {
-      this.currentThumbnailNum = 0;
-      this.hover = false;
-      if (this.timeoutId !== null) {
-        window.clearTimeout(this.timeoutId);
-        this.timeoutId = null;
-      }
-    },
-    scheduleNextTnNum(offset = 0) {
-      const INTERVAL = (this.$store.state.optVidPreviewInterval ?? 700) - offset;
-      this.timeoutId = window.setTimeout(() => {
-        const start = Date.now();
-        if (!this.hover) {
-          return;
-        }
-        if (this.currentThumbnailNum === this.doc._props.tnNum - 1) {
-          this.currentThumbnailNum = 0;
-          this.scheduleNextTnNum();
-        } else {
-          let img = new Image();
-          img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
-          img.onload = () => {
-            this.currentThumbnailNum += 1;
-            this.scheduleNextTnNum(Date.now() - start);
-          }
-        }
-      }, INTERVAL);
-    },
-    onTouchStart() {
-      this.$store.commit("busTnTouchStart", this.doc._id);
-      if (!this.hover) {
-        this.onTnEnter()
-      }
-    },
-  }
 }
 </script>
 
 <style scoped>
 .img-wrapper {
-  position: relative;
+    position: relative;
 }
 
 .img-wrapper:hover svg {
-  fill: rgba(0, 0, 0, 1);
+    fill: rgba(0, 0, 0, 1);
 }
 
 .card-img-top {
-  border-top-left-radius: 0;
+    border-top-left-radius: 0;
-  border-top-right-radius: 0;
+    border-top-right-radius: 0;
 }
 
 .play {
-  position: absolute;
+    position: absolute;
-  width: 25px;
+    width: 25px;
-  height: 25px;
+    height: 25px;
-  left: 50%;
+    left: 50%;
-  top: 50%;
+    top: 50%;
-  transform: translate(-50%, -50%);
+    transform: translate(-50%, -50%);
-  pointer-events: none;
+    pointer-events: none;
 }
 
 .play svg {
-  fill: rgba(0, 0, 0, 0.7);
+    fill: rgba(0, 0, 0, 0.7);
 }
 
 .badge-resolution {
-  color: #c6c6c6;
+    color: #c6c6c6;
-  background-color: #272727CC;
+    background-color: #272727CC;
-  padding: 2px 3px;
+    padding: 2px 3px;
 }
 
 .card-img-overlay {
-  pointer-events: none;
+    pointer-events: none;
-  padding: 2px 6px;
+    padding: 2px 6px;
-  bottom: 4px;
+    bottom: 4px;
-  top: unset;
+    top: unset;
-  left: unset;
+    left: unset;
-  right: 0;
+    right: 0;
 }
 
 .small-badge {
-  padding: 1px 3px;
+    padding: 1px 3px;
-  font-size: 70%;
+    font-size: 70%;
 }
 
 </style>

sist2-vue/src/components/IndexPicker.vue

@@ -27,11 +27,12 @@
                     @click.shift="shiftClick(idx, $event)"
                     class="d-flex justify-content-between align-items-center list-group-item-action pointer"
                     :class="{active: lastClickIndex === idx}"
+                    :key="idx.id"
             >
                 <div class="d-flex">
                     <b-checkbox style="pointer-events: none" :checked="isSelected(idx)"></b-checkbox>
                     {{ idx.name }}
-                    <div style="vertical-align: center; margin-left: 5px">
+                    <div v-if="hasEmbeddings(idx)" style="vertical-align: center; margin-left: 5px">
                         <MLIcon small style="top: -1px; position: relative"></MLIcon>
                     </div>
                     <span class="text-muted timestamp-text ml-2"
@@ -43,7 +44,7 @@
     </div>
 </template>
 
-<script lang="ts">
+<script>
 import SmallBadge from "./SmallBadge.vue"
 import {mapActions, mapGetters} from "vuex";
 import Vue from "vue";
@@ -111,7 +112,7 @@ export default Vue.extend({
         onSelect(value) {
             this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
         },
-        formatIdxDate(timestamp: number): string {
+        formatIdxDate(timestamp) {
             return format(new Date(timestamp * 1000), "yyyy-MM-dd");
         },
         toggleIndex(index, e) {
@@ -121,14 +122,17 @@ export default Vue.extend({
 
             this.lastClickIndex = index;
             if (this.isSelected(index)) {
-                this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
+                this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id !== index.id));
             } else {
                 this.setSelectedIndices([index, ...this.selectedIndices]);
             }
         },
         isSelected(index) {
-            return this.selectedIndices.find(idx => idx.id == index.id) != null;
+            return this.selectedIndices.find(idx => idx.id === index.id) != null;
-        }
+        },
+        hasEmbeddings(index) {
+            return index.models.length > 0;
+        },
     },
 })
 </script>

sist2-vue/src/components/InfoTable.vue

@@ -59,7 +59,7 @@ export default {
 
       const fields = [
         "title", "duration", "audioc", "videoc",
-        "bitrate", "artist", "album", "album_artist", "genre", "font_name", "author",
+        "bitrate", "artist", "album", "album_artist", "genre", "font_name", "author", "media_comment",
         "modified_by", "pages", "tag",
         "exif_make", "exif_software", "exif_exposure_time", "exif_fnumber", "exif_focal_length",
         "exif_user_comment", "exif_iso_speed_ratings", "exif_model", "exif_datetime",

sist2-vue/src/components/LazyContentDiv.vue

@@ -10,7 +10,7 @@
             >{{ $t("ml.analyzeText") }}
             </b-button>
             <b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="nerModel">
-                <b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()">{{ opt.text }}
+                <b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()" :key="opt.value">{{ opt.text }}
                 </b-select-option>
             </b-select>
         </b-form>
@@ -46,7 +46,7 @@ import {mapGetters, mapMutations} from "vuex";
 export default {
     name: "LazyContentDiv",
     components: {AnalyzedContentSpansContainer, Preloader},
-    props: ["docId"],
+    props: ["sid"],
     data() {
         return {
             ModelsRepo,
@@ -70,7 +70,7 @@ export default {
         }
 
         Sist2Api
-            .getDocument(this.docId, this.$store.state.optHighlight, this.$store.state.fuzzy)
+            .getDocument(this.sid, this.$store.state.optHighlight, this.$store.state.fuzzy)
             .then(doc => {
                 this.loading = false;
 

sist2-vue/src/components/Lightbox.vue

@@ -77,6 +77,7 @@ export default {
         return listener(e);
       }
     };
+
   },
   methods: {
     keyDownListener(e) {

sist2-vue/src/components/NavBar.vue

@@ -9,7 +9,7 @@
 
     <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
       v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
-        href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
+        href="https://github.com/sist2app/sist2/blob/master/docs/USAGE.md#elasticsearch"
         target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
     </span>
 

sist2-vue/src/components/ResultsCard.vue

@@ -43,8 +43,7 @@
     </b-card>
 </template>
 
-<script lang="ts">
+<script>
-import Sist2Api, {EsResult} from "@/Sist2Api";
 import Vue from "vue";
 import {humanFileSize} from "@/util";
 import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
@@ -52,6 +51,7 @@ import SortSelect from "@/components/SortSelect.vue";
 import Preloader from "@/components/Preloader.vue";
 import Sist2Query from "@/Sist2ElasticsearchQuery";
 import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
+import Sist2Api from "@/Sist2Api";
 
 export default Vue.extend({
     name: "ResultsCard",
@@ -64,7 +64,7 @@ export default Vue.extend({
             return this.$store.state.lastQueryResults != null;
         },
         hitCount() {
-            return (this.$store.state.firstQueryResults as EsResult).aggregations.total_count.value;
+            return (this.$store.state.firstQueryResults).aggregations.total_count.value;
         },
         tableItems() {
             const items = [];
@@ -79,10 +79,10 @@ export default Vue.extend({
     },
     methods: {
         took() {
-            return (this.$store.state.lastQueryResults as EsResult).took + "ms";
+            return (this.$store.state.lastQueryResults).took + "ms";
         },
         totalSize() {
-            return humanFileSize((this.$store.state.firstQueryResults as EsResult).aggregations.total_size.value);
+            return humanFileSize((this.$store.state.firstQueryResults).aggregations.total_size.value);
         },
         onToggle() {
             const show = !document.getElementById("collapse-1").classList.contains("show");

sist2-vue/src/components/SizeSlider.vue

@@ -5,8 +5,7 @@
 <script>
 import noUiSlider from 'nouislider';
 import 'nouislider/dist/nouislider.css';
-import {humanFileSize} from "@/util";
+import {humanFileSize, mergeTooltips} from "@/util";
-import {mergeTooltips} from "@/util-js";
 
 export default {
   name: "SizeSlider",

sist2-vue/src/components/SmallBadge.vue

@@ -2,7 +2,7 @@
   <b-badge variant="secondary" :pill="pill">{{ text }}</b-badge>
 </template>
 
-<script lang="ts">
+<script>
 import Vue from "vue";
 
 export default Vue.extend({

sist2-vue/src/components/ThumbnailProgressBar.vue

@@ -1,5 +1,5 @@
 <template>
-  <div class="thumbnail-progress-bar" :style="{width: `${percentProgress}%`}"></div>
+  <div class="thumbnail_count-progress-bar" :style="{width: `${percentProgress}%`}"></div>
 </template>
 
 <script>
@@ -16,7 +16,7 @@ export default {
 
 <style scoped>
 
-.thumbnail-progress-bar {
+.thumbnail_count-progress-bar {
   position: absolute;
   left: 0;
   bottom: 0;
@@ -27,11 +27,11 @@ export default {
   z-index: 9;
 }
 
-.theme-black .thumbnail-progress-bar {
+.theme-black .thumbnail_count-progress-bar {
   background: rgba(0, 188, 212, 0.95);
 }
 
-.sub-document .thumbnail-progress-bar {
+.sub-document .thumbnail_count-progress-bar {
   max-width: calc(100% - 8px);
   left: 4px;
 }

sist2-vue/src/i18n/messages.js

@@ -59,6 +59,7 @@ export default {
             searchInPath: "Enable matching query against document path",
             suggestPath: "Enable auto-complete in path filter bar",
             fragmentSize: "Highlight context size",
+            fragmentCount: "Number of highlight snippets",
             queryMode: "Search mode",
             displayMode: "Display",
             columns: "Column count",
@@ -138,7 +139,7 @@ export default {
         },
         debug: "Debug information",
         debugDescription: "Information useful for debugging. If you encounter bugs or have suggestions for" +
-            " new features, please submit a new issue <a href='https://github.com/simon987/sist2/issues/new/choose'>here</a>.",
+            " new features, please submit a new issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>here</a>.",
         tagline: "Tagline",
         toast: {
             esConnErrTitle: "Elasticsearch connection error",
@@ -242,6 +243,7 @@ export default {
             searchInPath: "Abgleich der Abfrage mit dem Dokumentpfad aktivieren",
             suggestPath: "Aktiviere Auto-Vervollständigung in Pfadfilter-Leiste",
             fragmentSize: "Kontextgröße",
+            fragmentCount: "Anzahl der hervorgehobenen Snippets",
             queryMode: "Such-Modus",
             displayMode: "Ansicht",
             columns: "Anzahl Spalten",
@@ -318,7 +320,7 @@ export default {
         },
         debug: "Debug Informationen",
         debugDescription: "Informationen für das Debugging. Wenn du Bugs gefunden oder Anregungen für " +
-            "neue Features hast, poste sie bitte <a href='https://github.com/simon987/sist2/issues/new/choose'>hier</a>.",
+            "neue Features hast, poste sie bitte <a href='https://github.com/sist2app/sist2/issues/new/choose'>hier</a>.",
         tagline: "Tagline",
         toast: {
             esConnErrTitle: "Elasticsearch Verbindungsfehler",
@@ -417,6 +419,7 @@ export default {
             searchInPath: "Activer la recherche dans le chemin des documents",
             suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
             fragmentSize: "Longueur du contexte de surlignage",
+            fragmentCount: "Nombre d'extraits surlignés",
             queryMode: "Mode de recherche",
             displayMode: "Affichage",
             columns: "Nombre de colonnes",
@@ -494,7 +497,7 @@ export default {
         debug: "Information de débogage",
         debugDescription: "Informations utiles pour le débogage\n" +
             "Si vous rencontrez des bogues ou si vous avez des suggestions pour de nouvelles fonctionnalités," +
-            " veuillez soumettre un nouvel Issue <a href='https://github.com/simon987/sist2/issues/new/choose'>ici</a>.",
+            " veuillez soumettre un nouvel Issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>ici</a>.",
         tagline: "Tagline",
         toast: {
             esConnErrTitle: "Erreur de connexion Elasticsearch",
@@ -592,6 +595,7 @@ export default {
             searchInPath: "匹配文档路径",
             suggestPath: "搜索框启用自动补全",
             fragmentSize: "高亮上下文大小",
+            fragmentCount: "突出显示的项目数",
             queryMode: "搜索模式",
             displayMode: "显示",
             columns: "列数",
@@ -668,7 +672,7 @@ export default {
         },
         debug: "调试信息",
         debugDescription: "对调试除错有用的信息。 若您遇到bug或者想建议新功能，请提交新Issue到" +
-            "<a href='https://github.com/simon987/sist2/issues/new/choose'>这里</a>.",
+            "<a href='https://github.com/sist2app/sist2/issues/new/choose'>这里</a>.",
         tagline: "标签栏",
         toast: {
             esConnErrTitle: "Elasticsearch连接错误",
@@ -767,6 +771,7 @@ export default {
             searchInPath: "Włącz szukanie również w ścieżce dokumentu",
             suggestPath: "Włącz auto-uzupełnianie w filtrze ścieżek",
             fragmentSize: "Podświetl wielkość kontekstu w znakach",
+            fragmentCount: "Liczba wyróżnionych fragmentów",
             queryMode: "Tryb szukania",
             displayMode: "Wyświetlanie",
             columns: "Liczba kolumn",
@@ -846,7 +851,7 @@ export default {
         },
         debug: "Informacje dla programistów",
         debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
-            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
+            " propozycje zmian, zgłoś to proszę <a href='https://github.com/sist2app/sist2/issues/new/choose'>tutaj</a>.",
         tagline: "Slogan",
         toast: {
             esConnErrTitle: "Problem z połączeniem z Elasticsearch",

sist2-vue/src/ml/CLIPTransformerModel.js

@@ -22,7 +22,9 @@ export class CLIPTransformerModel {
 
     async loadModel(onProgress) {
         ort.env.wasm.wasmPaths = ORT_WASM_PATHS;
-        ort.env.wasm.numThreads = 2;
+        if (window.crossOriginIsolated) {
+            ort.env.wasm.numThreads = 2;
+        }
 
         let buf = await ModelStore.get(this._modelUrl);
         if (!buf) {

sist2-vue/src/router/index.js

@@ -21,7 +21,7 @@ const authGuard = (to, from, next) => {
     next();
 }
 
-const routes: Array<RouteConfig> = [
+const routes = [
     {
         path: "/",
         name: "SearchPage",

sist2-vue/src/store/index.js

@@ -1,20 +1,18 @@
 import Vue from "vue"
 import Vuex from "vuex"
-import VueRouter, {Route} from "vue-router";
-import {EsHit, EsResult, EsTag, Index} from "@/Sist2Api";
 import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
 import {getInstance} from "@/plugins/auth0.js";
 
-const CONF_VERSION = 3;
+const CONF_VERSION = 4;
 
-Vue.use(Vuex)
+Vue.use(Vuex);
 
 export default new Vuex.Store({
     state: {
         seed: 0,
-        indices: [] as Index[],
+        indices: [],
-        tags: [] as EsTag[],
+        tags: [],
-        sist2Info: null as any,
+        sist2Info: null,
 
         sizeMin: undefined,
         sizeMax: undefined,
@@ -43,6 +41,7 @@ export default new Vuex.Store({
         optTagOrOperator: false,
         optFuzzy: true,
         optFragmentSize: 200,
+        optFragmentCount: 1,
         optQueryMode: "simple",
         optSearchInPath: false,
         optColumns: "auto",
@@ -60,16 +59,16 @@ export default new Vuex.Store({
         optVidPreviewInterval: 700,
         optSimpleLightbox: true,
         optShowTagPickerFilter: true,
-        optMlRepositories: "https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json",
+        optMlRepositories: "https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json",
         optAutoAnalyze: false,
         optMlDefaultModel: null,
 
-        _onLoadSelectedIndices: [] as string[],
+        _onLoadSelectedIndices: [],
-        _onLoadSelectedMimeTypes: [] as string[],
+        _onLoadSelectedMimeTypes: [],
-        _onLoadSelectedTags: [] as string[],
+        _onLoadSelectedTags: [],
-        selectedIndices: [] as Index[],
+        selectedIndices: [],
-        selectedMimeTypes: [] as string[],
+        selectedMimeTypes: [],
-        selectedTags: [] as string[],
+        selectedTags: [],
 
         lastQueryResults: null,
         firstQueryResults: null,
@@ -80,10 +79,10 @@ export default new Vuex.Store({
         uiSqliteMode: false,
         uiLightboxIsOpen: false,
         uiShowLightbox: false,
-        uiLightboxSources: [] as string[],
+        uiLightboxSources: [],
-        uiLightboxThumbs: [] as string[],
+        uiLightboxThumbs: [],
-        uiLightboxCaptions: [] as any[],
+        uiLightboxCaptions: [],
-        uiLightboxTypes: [] as string[],
+        uiLightboxTypes: [],
         uiLightboxKey: 0,
         uiLightboxSlide: 0,
         uiReachedScrollEnd: false,
@@ -91,7 +90,7 @@ export default new Vuex.Store({
         uiDetailsMimeAgg: null,
         uiShowDetails: false,
 
-        uiMimeMap: [] as any[],
+        uiMimeMap: [],
 
         auth0Token: null,
         nerModel: {
@@ -122,7 +121,7 @@ export default new Vuex.Store({
             if (state._onLoadSelectedIndices.length > 0) {
 
                 state.selectedIndices = val.filter(
-                    (idx: Index) => state._onLoadSelectedIndices.some(prefix => idx.id.startsWith(prefix))
+                    (idx) => state._onLoadSelectedIndices.some(id => id === idx.id.toString(16))
                 );
             } else {
                 state.selectedIndices = val;
@@ -145,18 +144,18 @@ export default new Vuex.Store({
         setSelectedIndices: (state, val) => state.selectedIndices = val,
         setSelectedMimeTypes: (state, val) => state.selectedMimeTypes = val,
         setSelectedTags: (state, val) => state.selectedTags = val,
-        setUiLightboxIsOpen: (state, val: boolean) => state.uiLightboxIsOpen = val,
+        setUiLightboxIsOpen: (state, val) => state.uiLightboxIsOpen = val,
-        _setUiShowLightbox: (state, val: boolean) => state.uiShowLightbox = val,
+        _setUiShowLightbox: (state, val) => state.uiShowLightbox = val,
-        setUiLightboxKey: (state, val: number) => state.uiLightboxKey = val,
+        setUiLightboxKey: (state, val) => state.uiLightboxKey = val,
-        _setKeySequence: (state, val: number) => state.keySequence = val,
+        _setKeySequence: (state, val) => state.keySequence = val,
-        _setQuerySequence: (state, val: number) => state.querySequence = val,
+        _setQuerySequence: (state, val) => state.querySequence = val,
         addLightboxSource: (state, {source, thumbnail, caption, type}) => {
             state.uiLightboxSources.push(source);
             state.uiLightboxThumbs.push(thumbnail);
             state.uiLightboxCaptions.push(caption);
             state.uiLightboxTypes.push(type);
         },
-        setUiLightboxSlide: (state, val: number) => state.uiLightboxSlide = val,
+        setUiLightboxSlide: (state, val) => state.uiLightboxSlide = val,
 
         setUiLightboxSources: (state, val) => state.uiLightboxSources = val,
         setUiLightboxThumbs: (state, val) => state.uiLightboxThumbs = val,
@@ -172,6 +171,7 @@ export default new Vuex.Store({
         setOptSearchInPath: (state, val) => state.optSearchInPath = val,
         setOptSuggestPath: (state, val) => state.optSuggestPath = val,
         setOptFragmentSize: (state, val) => state.optFragmentSize = val,
+        setOptFragmentCount: (state, val) => state.optFragmentCount = val,
         setOptQueryMode: (state, val) => state.optQueryMode = val,
         setOptResultSize: (state, val) => state.optSize = val,
         setOptTagOrOperator: (state, val) => state.optTagOrOperator = val,
@@ -230,7 +230,7 @@ export default new Vuex.Store({
                 store.commit("setOptLang", val.lang);
             }
         },
-        loadFromArgs({commit}, route: Route) {
+        loadFromArgs({commit}, route) {
 
             if (route.query.q) {
                 commit("setSearchText", route.query.q);
@@ -265,11 +265,11 @@ export default new Vuex.Store({
             }
 
             if (route.query.m) {
-                commit("_setOnLoadSelectedMimeTypes", deserializeMimes(route.query.m as string));
+                commit("_setOnLoadSelectedMimeTypes", deserializeMimes(route.query.m));
             }
 
             if (route.query.t) {
-                commit("_setOnLoadSelectedTags", (route.query.t as string).split(","));
+                commit("_setOnLoadSelectedTags", (route.query.t).split(","));
             }
 
             if (route.query.sort) {
@@ -280,7 +280,7 @@ export default new Vuex.Store({
                 commit("setSeed", Number(route.query.seed));
             }
         },
-        async updateArgs({state}, router: VueRouter) {
+        async updateArgs({state}, router) {
 
             if (router.currentRoute.path !== "/") {
                 return;
@@ -290,14 +290,14 @@ export default new Vuex.Store({
                 query: {
                     q: state.searchText.trim() ? state.searchText.trim().replace(/\s+/g, " ") : undefined,
                     fuzzy: state.fuzzy ? null : undefined,
-                    i: state.selectedIndices ? state.selectedIndices.map((idx: Index) => idx.idPrefix) : undefined,
+                    i: state.selectedIndices ? state.selectedIndices.map((idx) => idx.id.toString(16)) : undefined,
                     dMin: state.dateMin,
                     dMax: state.dateMax,
                     sMin: state.sizeMin,
                     sMax: state.sizeMax,
                     path: state.pathText ? state.pathText : undefined,
                     m: serializeMimes(state.selectedMimeTypes),
-                    t: state.selectedTags.length == 0 ? undefined : state.selectedTags.join(","),
+                    t: state.selectedTags.length === 0 ? undefined : state.selectedTags.join(","),
                     sort: state.sortMode === "score" ? undefined : state.sortMode,
                     seed: state.sortMode === "random" ? state.seed.toString() : undefined
                 }
@@ -306,11 +306,11 @@ export default new Vuex.Store({
             });
         },
         updateConfiguration({state}) {
-            const conf = {} as any;
+            const conf = {};
 
             Object.keys(state).forEach((key) => {
                 if (key.startsWith("opt")) {
-                    conf[key] = (state as any)[key];
+                    conf[key] = (state)[key];
                 }
             });
 
@@ -323,14 +323,14 @@ export default new Vuex.Store({
             if (confString) {
                 const conf = JSON.parse(confString);
 
-                if (!("version" in conf) || conf["version"] != CONF_VERSION) {
+                if (!("version" in conf) || conf["version"] !== CONF_VERSION) {
                     localStorage.removeItem("sist2_configuration");
                     window.location.reload();
                 }
 
                 Object.keys(state).forEach((key) => {
                     if (key.startsWith("opt")) {
-                        (state as any)[key] = conf[key];
+                        (state)[key] = conf[key];
                     }
                 });
             }
@@ -386,7 +386,7 @@ export default new Vuex.Store({
         indices: state => state.indices,
         sist2Info: state => state.sist2Info,
         indexMap: state => {
-            const map = {} as any;
+            const map = {};
             state.indices.forEach(idx => map[idx.id] = idx);
             return map;
         },
@@ -405,12 +405,12 @@ export default new Vuex.Store({
         size: state => state.optSize,
         sortMode: state => state.sortMode,
         lastQueryResult: state => state.lastQueryResults,
-        lastDoc: function (state): EsHit | null {
+        lastDoc: function (state) {
             if (state.lastQueryResults == null) {
                 return null;
             }
 
-            return (state.lastQueryResults as unknown as EsResult).hits.hits.slice(-1)[0];
+            return (state.lastQueryResults).hits.hits.slice(-1)[0];
         },
         uiShowLightbox: state => state.uiShowLightbox,
         uiLightboxSources: state => state.uiLightboxSources,
@@ -432,6 +432,7 @@ export default new Vuex.Store({
         optSearchInPath: state => state.optSearchInPath,
         optSuggestPath: state => state.optSuggestPath,
         optFragmentSize: state => state.optFragmentSize,
+        optFragmentCount: state => state.optFragmentCount,
         optQueryMode: state => state.optQueryMode,
         optTreemapType: state => state.optTreemapType,
         optTreemapTiling: state => state.optTreemapTiling,
@@ -451,7 +452,7 @@ export default new Vuex.Store({
         optMlRepositories: state => state.optMlRepositories,
         mlRepositoryList: state => {
             const repos = state.optMlRepositories.split("\n")
-            return repos[0] == "" ? [] : repos;
+            return repos[0] === "" ? [] : repos;
         },
         optMlDefaultModel: state => state.optMlDefaultModel,
         optAutoAnalyze: state => state.optAutoAnalyze,

sist2-vue/src/util-js.js

@@ -1,139 +0,0 @@
-export function mergeTooltips(slider, threshold, separator, fixTooltips) {
-
-    const isMobile = window.innerWidth <= 650;
-    if (isMobile) {
-        threshold = 25;
-    }
-
-    const textIsRtl = getComputedStyle(slider).direction === 'rtl';
-    const isRtl = slider.noUiSlider.options.direction === 'rtl';
-    const isVertical = slider.noUiSlider.options.orientation === 'vertical';
-    const tooltips = slider.noUiSlider.getTooltips();
-    const origins = slider.noUiSlider.getOrigins();
-
-    // Move tooltips into the origin element. The default stylesheet handles this.
-    tooltips.forEach(function (tooltip, index) {
-        if (tooltip) {
-            origins[index].appendChild(tooltip);
-        }
-    });
-
-    slider.noUiSlider.on('update', function (values, handle, unencoded, tap, positions) {
-
-        const pools = [[]];
-        const poolPositions = [[]];
-        const poolValues = [[]];
-        let atPool = 0;
-
-        // Assign the first tooltip to the first pool, if the tooltip is configured
-        if (tooltips[0]) {
-            pools[0][0] = 0;
-            poolPositions[0][0] = positions[0];
-            poolValues[0][0] = values[0];
-        }
-
-        for (let i = 1; i < positions.length; i++) {
-            if (!tooltips[i] || (positions[i] - positions[i - 1]) > threshold) {
-                atPool++;
-                pools[atPool] = [];
-                poolValues[atPool] = [];
-                poolPositions[atPool] = [];
-            }
-
-            if (tooltips[i]) {
-                pools[atPool].push(i);
-                poolValues[atPool].push(values[i]);
-                poolPositions[atPool].push(positions[i]);
-            }
-        }
-
-        pools.forEach(function (pool, poolIndex) {
-            const handlesInPool = pool.length;
-
-            for (let j = 0; j < handlesInPool; j++) {
-                const handleNumber = pool[j];
-
-                if (j === handlesInPool - 1) {
-                    let offset = 0;
-
-                    poolPositions[poolIndex].forEach(function (value) {
-                        offset += 1000 - 10 * value;
-                    });
-
-                    const direction = isVertical ? 'bottom' : 'right';
-                    const last = isRtl ? 0 : handlesInPool - 1;
-                    const lastOffset = 1000 - 10 * poolPositions[poolIndex][last];
-                    offset = (textIsRtl && !isVertical ? 100 : 0) + (offset / handlesInPool) - lastOffset;
-
-                    // Center this tooltip over the affected handles
-                    tooltips[handleNumber].innerHTML = poolValues[poolIndex].join(separator);
-                    tooltips[handleNumber].style.display = 'block';
-
-                    tooltips[handleNumber].style[direction] = offset + '%';
-                } else {
-                    // Hide this tooltip
-                    tooltips[handleNumber].style.display = 'none';
-                }
-            }
-        });
-
-        if (fixTooltips) {
-            const isMobile = window.innerWidth <= 650;
-            const len = isMobile ? 20 : 5;
-
-            if (positions[0] < len) {
-                tooltips[0].style.right = `${(1 - ((positions[0]) / len)) * -35}px`
-            } else {
-                tooltips[0].style.right = "0"
-            }
-
-            if (positions[1] > (100 - len)) {
-                tooltips[1].style.right = `${((positions[1] - (100 - len)) / len) * 35}px`
-            } else {
-                tooltips[1].style.right = "0"
-            }
-        }
-    });
-}
-
-
-export function burrow(table, addSelfDir, rootName) {
-    const root = {};
-    table.forEach(row => {
-        let layer = root;
-
-        row.taxonomy.forEach(key => {
-            layer[key] = key in layer ? layer[key] : {};
-            layer = layer[key];
-        });
-        if (Object.keys(layer).length === 0) {
-            layer["$size$"] = row.size;
-        } else if (addSelfDir) {
-            layer["."] = {
-                "$size$": row.size,
-            };
-        }
-    });
-
-    const descend = function (obj, depth) {
-        return Object.keys(obj).filter(k => k !== "$size$").map(k => {
-            const child = {
-                name: k,
-                depth: depth,
-                value: 0,
-                children: descend(obj[k], depth + 1)
-            };
-            if ("$size$" in obj[k]) {
-                child.value = obj[k]["$size$"];
-            }
-            return child;
-        });
-    };
-
-    return {
-        name: rootName,
-        children: descend(root, 1),
-        value: 0,
-        depth: 0,
-    }
-}

sist2-vue/src/util.js

@@ -0,0 +1,329 @@
+export function mergeTooltips(slider, threshold, separator, fixTooltips) {
+
+    const isMobile = window.innerWidth <= 650;
+    if (isMobile) {
+        threshold = 25;
+    }
+
+    const textIsRtl = getComputedStyle(slider).direction === 'rtl';
+    const isRtl = slider.noUiSlider.options.direction === 'rtl';
+    const isVertical = slider.noUiSlider.options.orientation === 'vertical';
+    const tooltips = slider.noUiSlider.getTooltips();
+    const origins = slider.noUiSlider.getOrigins();
+
+    // Move tooltips into the origin element. The default stylesheet handles this.
+    tooltips.forEach(function (tooltip, index) {
+        if (tooltip) {
+            origins[index].appendChild(tooltip);
+        }
+    });
+
+    slider.noUiSlider.on('update', function (values, handle, unencoded, tap, positions) {
+
+        const pools = [[]];
+        const poolPositions = [[]];
+        const poolValues = [[]];
+        let atPool = 0;
+
+        // Assign the first tooltip to the first pool, if the tooltip is configured
+        if (tooltips[0]) {
+            pools[0][0] = 0;
+            poolPositions[0][0] = positions[0];
+            poolValues[0][0] = values[0];
+        }
+
+        for (let i = 1; i < positions.length; i++) {
+            if (!tooltips[i] || (positions[i] - positions[i - 1]) > threshold) {
+                atPool++;
+                pools[atPool] = [];
+                poolValues[atPool] = [];
+                poolPositions[atPool] = [];
+            }
+
+            if (tooltips[i]) {
+                pools[atPool].push(i);
+                poolValues[atPool].push(values[i]);
+                poolPositions[atPool].push(positions[i]);
+            }
+        }
+
+        pools.forEach(function (pool, poolIndex) {
+            const handlesInPool = pool.length;
+
+            for (let j = 0; j < handlesInPool; j++) {
+                const handleNumber = pool[j];
+
+                if (j === handlesInPool - 1) {
+                    let offset = 0;
+
+                    poolPositions[poolIndex].forEach(function (value) {
+                        offset += 1000 - 10 * value;
+                    });
+
+                    const direction = isVertical ? 'bottom' : 'right';
+                    const last = isRtl ? 0 : handlesInPool - 1;
+                    const lastOffset = 1000 - 10 * poolPositions[poolIndex][last];
+                    offset = (textIsRtl && !isVertical ? 100 : 0) + (offset / handlesInPool) - lastOffset;
+
+                    // Center this tooltip over the affected handles
+                    tooltips[handleNumber].innerHTML = poolValues[poolIndex].join(separator);
+                    tooltips[handleNumber].style.display = 'block';
+
+                    tooltips[handleNumber].style[direction] = offset + '%';
+                } else {
+                    // Hide this tooltip
+                    tooltips[handleNumber].style.display = 'none';
+                }
+            }
+        });
+
+        if (fixTooltips) {
+            const isMobile = window.innerWidth <= 650;
+            const len = isMobile ? 20 : 5;
+
+            if (positions[0] < len) {
+                tooltips[0].style.right = `${(1 - ((positions[0]) / len)) * -35}px`
+            } else {
+                tooltips[0].style.right = "0"
+            }
+
+            if (positions[1] > (100 - len)) {
+                tooltips[1].style.right = `${((positions[1] - (100 - len)) / len) * 35}px`
+            } else {
+                tooltips[1].style.right = "0"
+            }
+        }
+    });
+}
+
+
+export function burrow(table, addSelfDir, rootName) {
+    const root = {};
+    table.forEach(row => {
+        let layer = root;
+
+        row.taxonomy.forEach(key => {
+            layer[key] = key in layer ? layer[key] : {};
+            layer = layer[key];
+        });
+        if (Object.keys(layer).length === 0) {
+            layer["$size$"] = row.size;
+        } else if (addSelfDir) {
+            layer["."] = {
+                "$size$": row.size,
+            };
+        }
+    });
+
+    const descend = function (obj, depth) {
+        return Object.keys(obj).filter(k => k !== "$size$").map(k => {
+            const child = {
+                name: k,
+                depth: depth,
+                value: 0,
+                children: descend(obj[k], depth + 1)
+            };
+            if ("$size$" in obj[k]) {
+                child.value = obj[k]["$size$"];
+            }
+            return child;
+        });
+    };
+
+    return {
+        name: rootName,
+        children: descend(root, 1),
+        value: 0,
+        depth: 0,
+    }
+}
+
+
+export function ext(hit) {
+    return srcExt(hit._source)
+}
+
+export function srcExt(src) {
+    return Object.prototype.hasOwnProperty.call(src, "extension")
+    && src["extension"] !== "" ? "." + src["extension"] : "";
+}
+
+export function strUnescape(str) {
+    let result = "";
+
+    for (let i = 0; i < str.length; i++) {
+        const c = str[i];
+        const next = str[i + 1];
+
+        if (c === "]") {
+            if (next === "]") {
+                result += c;
+                i += 1;
+            } else {
+                result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
+                i += 2;
+            }
+        } else {
+            result += c;
+        }
+    }
+    return result;
+}
+
+const thresh = 1000;
+const units = ["k", "M", "G", "T", "P", "E", "Z", "Y"];
+
+export function humanFileSize(bytes) {
+    if (bytes === 0) {
+        return "0 B"
+    }
+
+    if (Math.abs(bytes) < thresh) {
+        return bytes + ' B';
+    }
+    let u = -1;
+    do {
+        bytes /= thresh;
+        ++u;
+    } while (Math.abs(bytes) >= thresh && u < units.length - 1);
+
+    return bytes.toFixed(1) + units[u];
+}
+
+export function humanTime(sec_num) {
+    sec_num = Math.floor(sec_num);
+    const hours = Math.floor(sec_num / 3600);
+    const minutes = Math.floor((sec_num - (hours * 3600)) / 60);
+    const seconds = sec_num - (hours * 3600) - (minutes * 60);
+
+    if (sec_num < 60) {
+        return `${sec_num}s`
+    }
+
+    if (sec_num < 3600) {
+        return `${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
+    }
+
+    return `${hours < 10 ? "0" : ""}${hours}:${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
+}
+
+export function humanDate(numMilis) {
+    const date = (new Date(numMilis * 1000));
+    return date.getUTCFullYear() + "-" + ("0" + (date.getUTCMonth() + 1)).slice(-2) + "-" + ("0" + date.getUTCDate()).slice(-2)
+
+}
+
+export function lum(c) {
+    c = c.substring(1);
+    const rgb = parseInt(c, 16);
+    const r = (rgb >> 16) & 0xff;
+    const g = (rgb >> 8) & 0xff;
+    const b = (rgb >> 0) & 0xff;
+
+    return 0.2126 * r + 0.7152 * g + 0.0722 * b;
+}
+
+
+export function getSelectedTreeNodes(tree) {
+    const selectedNodes = new Set();
+
+    const selected = tree.selected();
+
+    for (let i = 0; i < selected.length; i++) {
+
+        if (selected[i].id === "any") {
+            return ["any"]
+        }
+
+        //Only get children
+        if (selected[i].text.indexOf("(") !== -1) {
+            if (selected[i].values) {
+                selectedNodes.add(selected[i].values.slice(-1)[0]);
+            } else {
+                selectedNodes.add(selected[i].id);
+            }
+        }
+    }
+
+    return Array.from(selectedNodes);
+}
+
+export function getTreeNodeAttributes(tree) {
+    const nodes = tree.selectable();
+    const attributes = {};
+
+    for (let i = 0; i < nodes.length; i++) {
+
+        let id = null;
+
+        if (nodes[i].text.indexOf("(") !== -1 && nodes[i].values) {
+            id = nodes[i].values.slice(-1)[0];
+        } else {
+            id = nodes[i].id
+        }
+
+        attributes[id] = {
+            checked: nodes[i].itree.state.checked,
+            collapsed: nodes[i].itree.state.collapsed,
+        }
+    }
+
+    return attributes;
+}
+
+
+export function serializeMimes(mimes) {
+    if (mimes.length === 0) {
+        return undefined;
+    }
+    return mimes.map(mime => compressMime(mime)).join("");
+}
+
+export function deserializeMimes(mimeString) {
+    return mimeString
+        .replaceAll(/([IVATUF])/g, "$$$&")
+        .split("$")
+        .map(mime => decompressMime(mime))
+        .slice(1) // Ignore the first (empty) token
+}
+
+export function compressMime(mime) {
+    return mime
+        .replace("image/", "I")
+        .replace("video/", "V")
+        .replace("application/", "A")
+        .replace("text/", "T")
+        .replace("audio/", "U")
+        .replace("font/", "F")
+        .replace("+", ",")
+        .replace("x-", "X")
+}
+
+export function decompressMime(mime) {
+    return mime
+        .replace("I", "image/")
+        .replace("V", "video/")
+        .replace("A", "application/")
+        .replace("T", "text/")
+        .replace("U", "audio/")
+        .replace("F", "font/")
+        .replace(",", "+")
+        .replace("X", "x-")
+}
+
+export function randomSeed() {
+    return Math.round(Math.random() * 100000);
+}
+
+export function sid(doc) {
+    if (doc._id.includes(".")) {
+        return doc._id
+    }
+
+    const num = BigInt(doc._id);
+
+    const indexId = (num >> BigInt(32));
+    const docId = num & BigInt(0xFFFFFFFF);
+
+    return indexId.toString(16).padStart(8, "0") + "." + docId.toString(16).padStart(8, "0");
+}

sist2-vue/src/util.ts

@@ -1,177 +0,0 @@
-import {EsHit} from "@/Sist2Api";
-
-export function ext(hit: EsHit) {
-    return srcExt(hit._source)
-}
-
-export function srcExt(src) {
-    return Object.prototype.hasOwnProperty.call(src, "extension")
-        && src["extension"] !== "" ? "." + src["extension"] : "";
-}
-
-export function strUnescape(str: string): string {
-    let result = "";
-
-    for (let i = 0; i < str.length; i++) {
-        const c = str[i];
-        const next = str[i + 1];
-
-        if (c === "]") {
-            if (next === "]") {
-                result += c;
-                i += 1;
-            } else {
-                result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
-                i += 2;
-            }
-        } else {
-            result += c;
-        }
-    }
-    return result;
-}
-
-const thresh = 1000;
-const units = ["k", "M", "G", "T", "P", "E", "Z", "Y"];
-
-export function humanFileSize(bytes: number): string {
-    if (bytes === 0) {
-        return "0 B"
-    }
-
-    if (Math.abs(bytes) < thresh) {
-        return bytes + ' B';
-    }
-    let u = -1;
-    do {
-        bytes /= thresh;
-        ++u;
-    } while (Math.abs(bytes) >= thresh && u < units.length - 1);
-
-    return bytes.toFixed(1) + units[u];
-}
-
-export function humanTime(sec_num: number): string {
-    sec_num = Math.floor(sec_num);
-    const hours = Math.floor(sec_num / 3600);
-    const minutes = Math.floor((sec_num - (hours * 3600)) / 60);
-    const seconds = sec_num - (hours * 3600) - (minutes * 60);
-
-    if (sec_num < 60) {
-        return `${sec_num}s`
-    }
-
-    if (sec_num < 3600) {
-        return `${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
-    }
-
-    return `${hours < 10 ? "0" : ""}${hours}:${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
-}
-
-export function humanDate(numMilis: number): string {
-    const date = (new Date(numMilis * 1000));
-    return date.getUTCFullYear() + "-" + ("0" + (date.getUTCMonth() + 1)).slice(-2) + "-" + ("0" + date.getUTCDate()).slice(-2)
-
-}
-
-export function lum(c: string) {
-    c = c.substring(1);
-    const rgb = parseInt(c, 16);
-    const r = (rgb >> 16) & 0xff;
-    const g = (rgb >> 8) & 0xff;
-    const b = (rgb >> 0) & 0xff;
-
-    return 0.2126 * r + 0.7152 * g + 0.0722 * b;
-}
-
-
-export function getSelectedTreeNodes(tree: any) {
-    const selectedNodes = new Set();
-
-    const selected = tree.selected();
-
-    for (let i = 0; i < selected.length; i++) {
-
-        if (selected[i].id === "any") {
-            return ["any"]
-        }
-
-        //Only get children
-        if (selected[i].text.indexOf("(") !== -1) {
-            if (selected[i].values) {
-                selectedNodes.add(selected[i].values.slice(-1)[0]);
-            } else {
-                selectedNodes.add(selected[i].id);
-            }
-        }
-    }
-
-    return Array.from(selectedNodes);
-}
-
-export function getTreeNodeAttributes(tree: any) {
-    const nodes = tree.selectable();
-    const attributes = {};
-
-    for (let i = 0; i < nodes.length; i++) {
-
-        let id = null;
-
-        if (nodes[i].text.indexOf("(") !== -1 && nodes[i].values) {
-            id = nodes[i].values.slice(-1)[0];
-        } else {
-            id = nodes[i].id
-        }
-
-        attributes[id] = {
-            checked: nodes[i].itree.state.checked,
-            collapsed: nodes[i].itree.state.collapsed,
-        }
-    }
-
-    return attributes;
-}
-
-
-export function serializeMimes(mimes: string[]): string | undefined {
-    if (mimes.length == 0) {
-        return undefined;
-    }
-    return mimes.map(mime => compressMime(mime)).join("");
-}
-
-export function deserializeMimes(mimeString: string): string[] {
-    return mimeString
-        .replaceAll(/([IVATUF])/g, "$$$&")
-        .split("$")
-        .map(mime => decompressMime(mime))
-        .slice(1) // Ignore the first (empty) token
-}
-
-export function compressMime(mime: string): string {
-    return mime
-        .replace("image/", "I")
-        .replace("video/", "V")
-        .replace("application/", "A")
-        .replace("text/", "T")
-        .replace("audio/", "U")
-        .replace("font/", "F")
-        .replace("+", ",")
-        .replace("x-", "X")
-}
-
-export function decompressMime(mime: string): string {
-    return mime
-        .replace("I", "image/")
-        .replace("V", "video/")
-        .replace("A", "application/")
-        .replace("T", "text/")
-        .replace("U", "audio/")
-        .replace("F", "font/")
-        .replace(",", "+")
-        .replace("X", "x-")
-}
-
-export function randomSeed(): number {
-    return Math.round(Math.random() * 100000);
-}

sist2-vue/src/views/Configuration.vue

@@ -81,6 +81,7 @@
                             <li><code>doc.artist</code></li>
                             <li><code>doc.title</code></li>
                             <li><code>doc.genre</code></li>
+                            <li><code>doc.media_comment</code></li>
                             <li><code>doc.album_artist</code></li>
                             <li><code>doc.exif_make</code></li>
                             <li><code>doc.exif_model</code></li>
@@ -136,7 +137,7 @@
                         {{ $t("opt.fuzzy") }}
                     </b-form-checkbox>
 
-                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optSearchInPath" @input="setOptSearchInPath">{{
+                    <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
                         $t("opt.searchInPath")
                         }}
                     </b-form-checkbox>
@@ -150,6 +151,10 @@
                     <b-form-input :value="optFragmentSize" step="10" type="number" min="0"
                                   @input="setOptFragmentSize"></b-form-input>
 
+                  <label :class="{'text-muted': uiSqliteMode}">{{ $t("opt.fragmentCount") }}</label>
+                  <b-form-input :value="optFragmentCount" :disabled="uiSqliteMode" step="1" type="number" min="1"
+                                @input="setOptFragmentCount"></b-form-input>
+
                     <label>{{ $t("opt.resultSize") }}</label>
                     <b-form-input :value="optResultSize" type="number" min="10"
                                   @input="setOptResultSize"></b-form-input>
@@ -313,6 +318,7 @@ export default {
             "optSearchInPath",
             "optSuggestPath",
             "optFragmentSize",
+            "optFragmentCount",
             "optQueryMode",
             "optTreemapType",
             "optTreemapTiling",
@@ -359,6 +365,7 @@ export default {
             "setOptSearchInPath",
             "setOptSuggestPath",
             "setOptFragmentSize",
+            "setOptFragmentCount",
             "setOptQueryMode",
             "setOptTreemapType",
             "setOptTreemapTiling",

sist2-vue/src/views/FilePage.vue

@@ -1,142 +1,142 @@
 <template>
-  <div style="margin-left: auto; margin-right: auto;" class="container">
+    <div style="margin-left: auto; margin-right: auto;" class="container">
-    <Preloader v-if="loading"></Preloader>
+        <Preloader v-if="loading"></Preloader>
-    <b-card v-else-if="!loading && found">
+        <b-card v-else-if="!loading && found">
-      <b-card-title :title="doc._source.name + ext(doc)">
+            <b-card-title :title="doc._source.name + ext(doc)">
-        {{ doc._source.name + ext(doc) }}
+                {{ doc._source.name + ext(doc) }}
-      </b-card-title>
+            </b-card-title>
 
-      <!-- Thumbnail-->
+            <!-- Thumbnail-->
-      <div style="position: relative; margin-left: auto; margin-right: auto; text-align: center">
+            <div style="position: relative; margin-left: auto; margin-right: auto; text-align: center">
-        <FullThumbnail :doc="doc" :small-badge="false" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
+                <FullThumbnail :doc="doc" :small-badge="false" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
-      </div>
+            </div>
 
-      <!-- Audio player-->
+            <!-- Audio player-->
-      <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
+            <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
-             :type="doc._source.mime"
+                   :type="doc._source.mime"
-             :src="`f/${doc._source.index}/${doc._id}`"></audio>
+                   :src="`f/${sid(doc)}`"></audio>
 
-      <InfoTable :doc="doc" v-if="doc"></InfoTable>
+            <InfoTable :doc="doc" v-if="doc"></InfoTable>
 
-      <div v-if="doc._source.content" class="content-div">{{ doc._source.content }}</div>
+            <div v-if="doc._source.content" class="content-div">{{ doc._source.content }}</div>
-    </b-card>
+        </b-card>
-    <div v-else>
+        <div v-else>
-      <b-card>
+            <b-card>
-        <b-card-title>{{ $t("filePage.notFound") }}</b-card-title>
+                <b-card-title>{{ $t("filePage.notFound") }}</b-card-title>
-      </b-card>
+            </b-card>
+        </div>
     </div>
-  </div>
 </template>
 
 <script>
 import Preloader from "@/components/Preloader.vue";
 import InfoTable from "@/components/InfoTable.vue";
 import Sist2Api from "@/Sist2Api";
-import {ext} from "@/util";
+import {ext, sid} from "@/util";
 import Vue from "vue";
-import sist2 from "@/Sist2Api";
 import FullThumbnail from "@/components/FullThumbnail";
 
 export default Vue.extend({
-  name: "FilePage",
+    name: "FilePage",
-  components: {
+    components: {
-    FullThumbnail,
+        FullThumbnail,
-    Preloader,
+        Preloader,
-    InfoTable
+        InfoTable
-  },
-  data() {
-    return {
-      loading: true,
-      found: false,
-      doc: null
-    }
-  },
-  methods: {
-    ext: ext,
-    onThumbnailClick() {
-      window.open(`/f/${this.doc.index}/${this.doc._id}`, "_blank");
     },
-    findByCustomField(field, id) {
+    data() {
-      return {
+        return {
-        query: {
+            loading: true,
-          bool: {
+            found: false,
-            must: [
+            doc: null
-              {
+        }
-                match: {
-                  [field]: id
-                }
-              }
-            ]
-          }
-        },
-        size: 1
-      }
     },
-    findById(id) {
+    methods: {
-      return {
+        ext: ext,
-        query: {
+        sid: sid,
-          bool: {
+        onThumbnailClick() {
-            must: [
+            window.open(`/f/${sid(this.doc)}`, "_blank");
-              {
-                match: {
-                  "_id": id
-                }
-              }
-            ]
-          }
         },
-        size: 1
+        findByCustomField(field, id) {
-      }
+            return {
-    },
+                query: {
-    findByName(name) {
+                    bool: {
-      return {
+                        must: [
-        query: {
+                            {
-          bool: {
+                                match: {
-            must: [
+                                    [field]: id
-              {
+                                }
-                match: {
+                            }
-                  "name": name
+                        ]
-                }
+                    }
-              }
+                },
-            ]
+                size: 1
-          }
+            }
         },
-        size: 1
+        findById(id) {
-      }
+            return {
-    }
+                query: {
-
+                    bool: {
-  },
+                        must: [
-  mounted() {
+                            {
-    let query = null;
+                                match: {
-    if (this.$route.query.byId) {
+                                    "_id": id
-      query = this.findById(this.$route.query.byId);
+                                }
-    } else if (this.$route.query.byName) {
+                            }
-      query = this.findByName(this.$route.query.byName);
+                        ]
-    } else if (this.$route.query.by && this.$route.query.q) {
+                    }
-      query = this.findByCustomField(this.$route.query.by, this.$route.query.q)
+                },
-    }
+                size: 1
-
+            }
-    if (query) {
+        },
-      Sist2Api.esQuery(query).then(result => {
+        findByName(name) {
-        if (result.hits.hits.length === 0) {
+            return {
-          this.found = false;
+                query: {
-        } else {
+                    bool: {
-          this.doc = result.hits.hits[0];
+                        must: [
-          this.found = true;
+                            {
+                                match: {
+                                    "name": name
+                                }
+                            }
+                        ]
+                    }
+                },
+                size: 1
+            }
         }
 
-        this.loading = false;
+    },
-      });
+    mounted() {
-    } else {
+        let query = null;
-      this.loading = false;
+        if (this.$route.query.byId) {
-      this.found = false;
+            query = this.findById(this.$route.query.byId);
+        } else if (this.$route.query.byName) {
+            query = this.findByName(this.$route.query.byName);
+        } else if (this.$route.query.by && this.$route.query.q) {
+            query = this.findByCustomField(this.$route.query.by, this.$route.query.q)
+        }
+
+        if (query) {
+            Sist2Api.esQuery(query).then(result => {
+                if (result.hits.hits.length === 0) {
+                    this.found = false;
+                } else {
+                    this.doc = result.hits.hits[0];
+                    this.found = true;
+                }
+
+                this.loading = false;
+            });
+        } else {
+            this.loading = false;
+            this.found = false;
+        }
     }
-  }
 });
 </script>
 
 <style scoped>
 .img-wrapper {
-  display: inline-block;
+    display: inline-block;
 }
 </style>

sist2-vue/src/views/SearchPage.vue

@@ -60,6 +60,7 @@
 </template>
 
 <script>
+import {sid} from "@/util";
 import Preloader from "@/components/Preloader.vue";
 import {mapActions, mapGetters, mapMutations} from "vuex";
 import SearchBar from "@/components/SearchBar.vue";
@@ -92,7 +93,6 @@ export default Vue.extend({
         SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
     },
     data: () => ({
-        loading: false,
         uiLoading: true,
         search: undefined,
         docs: [],
@@ -105,6 +105,9 @@ export default Vue.extend({
     }),
     computed: {
         ...mapGetters(["indices", "optDisplay"]),
+        hasEmbeddings() {
+            return Sist2Api.models().length > 0;
+        },
     },
     mounted() {
         // Handle touch events
@@ -172,12 +175,6 @@ export default Vue.extend({
             setDateBoundsMax: "setDateBoundsMax",
             setTags: "setTags",
         }),
-        hasEmbeddings() {
-            if (!this.loading) {
-                return false;
-            }
-            return Sist2Api.models().some();
-        },
         showErrorToast() {
             this.$bvToast.toast(
                 this.$t("toast.esConnErr"),
@@ -248,9 +245,9 @@ export default Vue.extend({
                 if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
                     hit._seq = await this.$store.dispatch("getKeySequence");
                     this.$store.commit("addLightboxSource", {
-                        source: `f/${hit._source.index}/${hit._id}`,
+                        source: `f/${sid(hit)}`,
-                        thumbnail: hit._props.hasThumbnail
+                        thumbnail_count: hit._props.hasThumbnail
-                            ? `t/${hit._source.index}/${hit._id}`
+                            ? `t/${sid(hit)}`
                             : null,
                         caption: {
                             component: LightboxCaption,

sist2-vue/tsconfig.json

@@ -1,40 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "esnext",
-    "module": "esnext",
-    "strict": false,
-    "jsx": "preserve",
-    "importHelpers": true,
-    "moduleResolution": "node",
-    "experimentalDecorators": true,
-    "skipLibCheck": true,
-    "esModuleInterop": true,
-    "allowSyntheticDefaultImports": true,
-    "sourceMap": false,
-    "baseUrl": ".",
-    "types": [
-      "webpack-env",
-    ],
-    "paths": {
-      "@/*": [
-        "src/*"
-      ]
-    },
-    "lib": [
-      "esnext",
-      "dom",
-      "dom.iterable",
-      "scripthost"
-    ]
-  },
-  "include": [
-    "src/**/*.ts",
-    "src/**/*.tsx",
-    "src/**/*.vue",
-    "tests/**/*.ts",
-    "tests/**/*.tsx"
-  ],
-  "exclude": [
-    "node_modules"
-  ]
-}

sist2-vue/watch.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export NODE_OPTIONS=--openssl-legacy-provider
+
 ./node_modules/@vue/cli-service/bin/vue-cli-service.js build --watch

src/cli.c

@@ -25,6 +25,7 @@ const char *TESS_DATAPATHS[] = {
         "/usr/share/tessdata/",
         "/usr/share/tesseract-ocr/tessdata/",
         "/usr/share/tesseract-ocr/4.00/tessdata/",
+        "/usr/share/tesseract-ocr/5/tessdata/",
         "./",
         NULL
 };
@@ -74,6 +75,21 @@ void sqlite_index_args_destroy(sqlite_index_args_t *args) {
     free(args);
 }
 
+char *add_trailing_slash(char *abs_path) {
+    if (strcmp(abs_path, "/") == 0) {
+        // Special case: don't add trailing slash for "/"
+        return abs_path;
+    }
+
+    char *new_abs_path = realloc(abs_path, strlen(abs_path) + 2);
+    if (new_abs_path == NULL) {
+        LOG_FATALF("cli.c", "FIXME: realloc() failed for abs_path=%s", abs_path);
+    }
+    strcat(new_abs_path, "/");
+
+    return new_abs_path;
+}
+
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (argc < 2) {
         fprintf(stderr, "Required positional argument: PATH.\n");
@@ -83,15 +99,10 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     char *abs_path = abspath(argv[1]);
     if (abs_path == NULL) {
         LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
-    } else {
-        char *new_abs_path = realloc(abs_path, strlen(abs_path) + 2);
-        if (new_abs_path == NULL) {
-            LOG_FATALF("cli.c", "FIXME: realloc() failed for argv[1]=%s, abs_path=%s", argv[1], abs_path);
-        }
-        strcat(new_abs_path, "/");
-        args->path = new_abs_path;
     }
 
+    args->path = add_trailing_slash(abs_path);
+
     if (args->tn_quality == OPTION_VALUE_UNSPECIFIED) {
         args->tn_quality = DEFAULT_QUALITY;
     } else if (args->tn_quality < 0 || args->tn_quality > 100) {

src/ctx.c

@@ -1,8 +1,6 @@
 #include "ctx.h"
 
 ScanCtx_t ScanCtx = {
-        .stat_index_size = 0,
-        .stat_tn_size = 0,
         .pool = NULL,
         .index.path = {0,},
 };

src/ctx.h

@@ -19,6 +19,7 @@
 #include "src/database/database.h"
 #include "src/index/elastic.h"
 #include "sqlite3.h"
+#include "ignorelist.h"
 
 #include <pcre.h>
 
@@ -31,12 +32,10 @@ typedef struct {
     int depth;
     int calculate_checksums;
 
-    size_t stat_tn_size;
-    size_t stat_index_size;
-
     pcre *exclude;
     pcre_extra *exclude_extra;
     int fast;
+    ignorelist_t *ignorelist;
 
     scan_arc_ctx_t arc_ctx;
     scan_comic_ctx_t comic_ctx;

src/database/database.c

@@ -4,6 +4,7 @@
 #include <string.h>
 #include <pthread.h>
 #include "src/util.h"
+#include "src/parsing/mime.h"
 
 #include <time.h>
 
@@ -64,9 +65,11 @@ static int sep_rfind(const char *str) {
 }
 
 void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+#ifdef SIST_DEBUG
     if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
         sqlite3_result_error(ctx, "Invalid parameters", -1);
     }
+#endif
 
     const char *value = (const char *) sqlite3_value_text(argv[0]);
 
@@ -82,28 +85,27 @@ void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
 }
 
 void random_func(sqlite3_context *ctx, int argc, UNUSED(sqlite3_value **argv)) {
+#ifdef SIST_DEBUG
     if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
         sqlite3_result_error(ctx, "Invalid parameters", -1);
     }
+#endif
 
-    char state_buf[128] = {0,};
+    char state_buf[8] = {0,};
-    struct random_data buf;
-    int result;
-
     long seed = sqlite3_value_int64(argv[0]);
 
-    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
+    initstate((int) seed, state_buf, sizeof(state_buf));
 
-    random_r(&buf, &result);
+    sqlite3_result_int(ctx, (int) random());
-
-    sqlite3_result_int(ctx, result);
 }
 
 
 void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+#ifdef SIST_DEBUG
     if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
         sqlite3_result_error(ctx, "Invalid parameters", -1);
     }
+#endif
 
     database_ipc_ctx_t *ipc_ctx = sqlite3_user_data(ctx);
 
@@ -112,7 +114,7 @@ void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv)
     char buf[PATH_MAX];
     strcpy(buf, current_job);
 
-    strcpy(ipc_ctx->current_job[ProcData.thread_id], current_job);
+    SET_CURRENT_JOB(ipc_ctx, current_job);
 
     sqlite3_result_text(ctx, "ok", -1, SQLITE_STATIC);
 }
@@ -146,6 +148,12 @@ void database_open(database_t *db) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA temp_store = memory;", NULL, NULL, NULL));
     }
 
+#ifdef SIST_DEBUG
+        //    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA foreign_keys = ON;", NULL, NULL, NULL));
+#else
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA ignore_check_constraints = ON;", NULL, NULL, NULL));
+#endif
+
     if (db->type == INDEX_DATABASE) {
         // Prepare statements;
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
@@ -154,16 +162,15 @@ void database_open(database_t *db) {
                 &db->select_thumbnail_stmt, NULL));
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
                 db->db,
-                "UPDATE document SET marked=1 WHERE id=? AND mtime=? RETURNING id",
+                "UPDATE marked SET marked=1 WHERE id=(SELECT ROWID FROM document WHERE path=?) AND mtime=? RETURNING id",
                 -1,
                 &db->mark_document_stmt, NULL));
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
                 db->db,
-                "REPLACE INTO document_sidecar (id, json_data) VALUES (?,?)", -1,
+                "INSERT INTO document (path, parent, mime, mtime, size, thumbnail_count, json_data, version) "
-                &db->write_document_sidecar_stmt, NULL));
+                "VALUES (?, (SELECT id FROM document WHERE path=?), ?, ?, ?, ?, ?, (SELECT max(id) FROM version)) "
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                "ON CONFLICT (path) DO UPDATE SET json_data=excluded.json_data "
-                db->db,
+                "RETURNING id;",
-                "REPLACE INTO document (id, mtime, size, json_data, version) VALUES (?, ?, ?, ?, (SELECT max(id) FROM version));",
                 -1,
                 &db->write_document_stmt, NULL));
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
@@ -173,7 +180,12 @@ void database_open(database_t *db) {
                 &db->write_thumbnail_stmt, NULL));
 
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT json_data FROM document WHERE id=?", -1,
+                db->db, "SELECT json_set(json_data, "
+                        "'$._id', CAST (doc.id AS TEXT),"
+                        "'$.thumbnail', doc.thumbnail_count,"
+                        "'$.mime', m.name,"
+                        "'$.size', doc.size"
+                        ") FROM document doc LEFT JOIN mime m ON m.id=doc.mime WHERE doc.id=?", -1,
                 &db->get_document, NULL));
 
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
@@ -184,6 +196,12 @@ void database_open(database_t *db) {
                 db->db, "SELECT embedding FROM embedding WHERE id=? AND model_id=? AND start=0", -1,
                 &db->get_embedding, NULL));
 
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "INSERT INTO tag (id, tag) VALUES (?,?) ON CONFLICT DO NOTHING;",
+                -1,
+                &db->write_tag_stmt, NULL));
+
         // Create functions
         sqlite3_create_function(
                 db->db,
@@ -228,7 +246,7 @@ void database_open(database_t *db) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
                 db->db,
                 "DELETE FROM index_job WHERE id = (SELECT MIN(id) FROM index_job)"
-                " RETURNING doc_id,type,line;",
+                " RETURNING sid,type,line;",
                 -1, &db->pop_index_job_stmt, NULL
         ));
 
@@ -243,7 +261,7 @@ void database_open(database_t *db) {
                 db->db, "INSERT INTO parse_job (filepath,mtime,st_size) VALUES (?,?,?);", -1,
                 &db->insert_parse_job_stmt, NULL));
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "INSERT INTO index_job (doc_id,type,line) VALUES (?,?,?);", -1,
+                db->db, "INSERT INTO index_job (sid,type,line) VALUES (?,?,?);", -1,
                 &db->insert_index_job_stmt, NULL));
 
     } else if (db->type == FTS_DATABASE) {
@@ -294,6 +312,12 @@ void database_open(database_t *db) {
                 db->db, "SELECT mime, sum(count) FROM mime_index WHERE mime is not NULL GROUP BY mime", -1,
                 &db->fts_get_mimetypes, NULL));
 
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "INSERT INTO tag (id, index_id, tag) VALUES (?,?,?) ON CONFLICT DO NOTHING;",
+                -1,
+                &db->fts_write_tag_stmt, NULL));
+
         sqlite3_create_function(
                 db->db,
                 "random_seeded",
@@ -340,13 +364,6 @@ void database_open(database_t *db) {
     }
 
     if (db->type == FTS_DATABASE || db->type == INDEX_DATABASE) {
-        // Tag table is the same schema for FTS database & index database
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db,
-                "INSERT INTO tag (id, tag) VALUES (?,?) ON CONFLICT DO NOTHING;",
-                -1,
-                &db->write_tag_stmt, NULL));
-
         CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
                 db->db,
                 "DELETE FROM tag WHERE id=? AND tag=?;",
@@ -356,7 +373,7 @@ void database_open(database_t *db) {
 }
 
 void database_close(database_t *db, int optimize) {
-    LOG_DEBUGF("database.c", "Closing database %s", db->filename);
+    LOG_DEBUGF("database.c", "Closing database %s (%p)", db->filename, db->db);
 
     if (optimize) {
         LOG_DEBUG("database.c", "Optimizing database");
@@ -376,8 +393,8 @@ void database_close(database_t *db, int optimize) {
     db = NULL;
 }
 
-void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *return_value_len) {
+void *database_read_thumbnail(database_t *db, int doc_id, int num, size_t *return_value_len) {
-    sqlite3_bind_text(db->select_thumbnail_stmt, 1, id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->select_thumbnail_stmt, 1, doc_id);
     sqlite3_bind_int(db->select_thumbnail_stmt, 2, num);
 
     int ret = sqlite3_step(db->select_thumbnail_stmt);
@@ -410,7 +427,7 @@ void database_write_index_descriptor(database_t *db, index_descriptor_t *desc) {
 
     sqlite3_prepare_v2(db->db, "INSERT INTO descriptor (id, version_major, version_minor, version_patch,"
                                " root, name, rewrite_url, timestamp) VALUES (?,?,?,?,?,?,?,?);", -1, &stmt, NULL);
-    sqlite3_bind_text(stmt, 1, desc->id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(stmt, 1, desc->id);
     sqlite3_bind_int(stmt, 2, desc->version_major);
     sqlite3_bind_int(stmt, 3, desc->version_minor);
     sqlite3_bind_int(stmt, 4, desc->version_patch);
@@ -433,7 +450,7 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) {
 
     CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
 
-    const char *id = (char *) sqlite3_column_text(stmt, 0);
+    int id = sqlite3_column_int(stmt, 0);
     int v_major = sqlite3_column_int(stmt, 1);
     int v_minor = sqlite3_column_int(stmt, 2);
     int v_patch = sqlite3_column_int(stmt, 3);
@@ -443,7 +460,7 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) {
     int timestamp = sqlite3_column_int(stmt, 7);
 
     index_descriptor_t *desc = malloc(sizeof(index_descriptor_t));
-    strcpy(desc->id, id);
+    desc->id = id;
     snprintf(desc->version, sizeof(desc->version), "%d.%d.%d", v_major, v_minor, v_patch);
     desc->version_major = v_major;
     desc->version_minor = v_minor;
@@ -461,7 +478,7 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) {
 database_iterator_t *database_create_delete_list_iterator(database_t *db) {
 
     sqlite3_stmt *stmt;
-    sqlite3_prepare_v2(db->db, "SELECT id FROM delete_list;", -1, &stmt, NULL);
+    sqlite3_prepare_v2(db->db, "SELECT id FROM delete_list", -1, &stmt, NULL);
 
     database_iterator_t *iter = malloc(sizeof(database_iterator_t));
 
@@ -471,14 +488,11 @@ database_iterator_t *database_create_delete_list_iterator(database_t *db) {
     return iter;
 }
 
-char *database_delete_list_iter(database_iterator_t *iter) {
+int database_delete_list_iter(database_iterator_t *iter) {
     int ret = sqlite3_step(iter->stmt);
 
     if (ret == SQLITE_ROW) {
-        const char *id = (const char *) sqlite3_column_text(iter->stmt, 0);
+        return sqlite3_column_int(iter->stmt, 0);
-        char *id_heap = malloc(strlen(id) + 1);
-        strcpy(id_heap, id);
-        return id_heap;
     }
 
     if (ret != SQLITE_DONE) {
@@ -491,7 +505,7 @@ char *database_delete_list_iter(database_iterator_t *iter) {
 
     iter->stmt = NULL;
 
-    return NULL;
+    return 0;
 }
 
 database_iterator_t *database_create_document_iterator(database_t *db) {
@@ -501,27 +515,31 @@ database_iterator_t *database_create_document_iterator(database_t *db) {
     CRASH_IF_NOT_SQLITE_OK(
             sqlite3_prepare_v2(
                     db->db,
-                    "WITH doc (j) AS (SELECT CASE"
+                    "WITH doc (id, j) AS ("
-                    " WHEN emb.embedding IS NULL THEN"
+                    "SELECT"
-                    "  json_set(document.json_data, "
+                    " document.id,"
-                    "      '$._id', document.id, "
+                    " json_set(document.json_data,"
-                    "      '$.size', document.size, "
+                    "  '$._id', document.id,"
-                    "      '$.mtime', document.mtime, "
+                    "  '$.index', (SELECT id FROM descriptor),"
-                    "      '$.tag', json_group_array((SELECT tag FROM tag WHERE document.id = tag.id)))"
+                    "  '$.size', document.size,"
-                    " ELSE"
+                    "  '$.mtime', document.mtime,"
-                    "  json_set(document.json_data,"
+                    "  '$.mime', mim.name,"
-                    "      '$._id', document.id,"
+                    "  '$.thumbnail', document.thumbnail_count,"
-                    "      '$.size', document.size,"
+                    "  '$.tag', json_group_array(t.tag))"
-                    "      '$.mtime', document.mtime,"
-                    "      '$.tag', json_group_array((SELECT tag FROM tag WHERE document.id = tag.id)),"
-                    "      '$.emb', json_group_object(m.path, json(emb_to_json(emb.embedding))),"
-                    "      '$.embedding', 1)"
-                    " END"
                     " FROM document"
-                    " LEFT JOIN embedding emb ON document.id = emb.id"
+                    "  LEFT JOIN mime mim ON mim.id = document.mime"
-                    " LEFT JOIN model m ON emb.model_id = m.id"
+                    "  LEFT JOIN tag t ON t.id = document.id"
                     " GROUP BY document.id)"
-                    " SELECT json_set(j, '$.index', (SELECT id FROM descriptor)) FROM doc",
+                    "SELECT CASE"
+                    " WHEN emb.embedding IS NULL THEN j"
+                    " ELSE json_set(j,"
+                    "  '$.emb', json_group_object(m.path, json(emb_to_json(emb.embedding))),"
+                    "  '$.embedding', 1"
+                    "     ) END"
+                    " FROM doc"
+                    " LEFT JOIN embedding emb ON doc.id = emb.id"
+                    " LEFT JOIN model m ON emb.model_id = m.id"
+                    " GROUP BY doc.id",
                     -1, &stmt, NULL));
 
     database_iterator_t *iter = malloc(sizeof(database_iterator_t));
@@ -573,43 +591,49 @@ cJSON *database_document_iter(database_iterator_t *iter) {
 
 cJSON *database_incremental_scan_begin(database_t *db) {
     LOG_DEBUG("database.c", "Preparing database for incremental scan");
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "UPDATE document SET marked=0;", NULL, NULL, NULL));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "DELETE FROM marked;", NULL, NULL, NULL));
+    LOG_DEBUG("database.c", "Preparing database for incremental scan (create marked table)");
+    CRASH_IF_NOT_SQLITE_OK(
+            sqlite3_exec(db->db, "INSERT INTO marked SELECT id, 0, mtime FROM document;", NULL, NULL, NULL));
 }
 
 cJSON *database_incremental_scan_end(database_t *db) {
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "DELETE FROM delete_list WHERE id IN (SELECT id FROM document WHERE marked=1);",
+            "DELETE FROM delete_list WHERE id IN (SELECT id FROM marked WHERE marked = 1);",
             NULL, NULL, NULL
     ));
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "DELETE FROM thumbnail WHERE id IN (SELECT id FROM document WHERE marked=0);",
+            "DELETE FROM thumbnail WHERE EXISTS ("
+            " SELECT document.id FROM document INNER JOIN marked m ON m.id = document.ROWID"
+            " WHERE marked=0 and document.id = thumbnail.id)",
             NULL, NULL, NULL
     ));
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "INSERT INTO delete_list (id) SELECT id FROM document WHERE marked=0;",
+            "INSERT INTO delete_list (id) "
+            "SELECT id FROM marked WHERE marked=0 ON CONFLICT DO NOTHING;",
             NULL, NULL, NULL
     ));
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "DELETE FROM document_sidecar WHERE id IN (SELECT id FROM document WHERE marked=0);",
+            "DELETE FROM document WHERE ROWID IN (SELECT id FROM marked WHERE marked=0);",
             NULL, NULL, NULL
     ));
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "DELETE FROM document WHERE marked=0;",
+            "DELETE FROM marked;",
             NULL, NULL, NULL
     ));
 }
 
-int database_mark_document(database_t *db, const char *id, int mtime) {
+int database_mark_document(database_t *db, const char *path, int mtime) {
-    sqlite3_bind_text(db->mark_document_stmt, 1, id, -1, SQLITE_STATIC);
+    sqlite3_bind_text(db->mark_document_stmt, 1, path, -1, SQLITE_STATIC);
     sqlite3_bind_int(db->mark_document_stmt, 2, mtime);
 
     pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
@@ -631,31 +655,38 @@ int database_mark_document(database_t *db, const char *id, int mtime) {
     CRASH_IF_STMT_FAIL(ret);
 }
 
-void database_write_document(database_t *db, document_t *doc, const char *json_data) {
+int database_write_document(database_t *db, document_t *doc, const char *json_data) {
-    sqlite3_bind_text(db->write_document_stmt, 1, doc->doc_id, -1, SQLITE_STATIC);
+
-    sqlite3_bind_int(db->write_document_stmt, 2, doc->mtime);
+    const char *rel_path = doc->filepath + ScanCtx.index.desc.root_len;
-    sqlite3_bind_int64(db->write_document_stmt, 3, (long) doc->size);
+    const char *parent_rel_path = doc->parent[0] != '\0'
-    sqlite3_bind_text(db->write_document_stmt, 4, json_data, -1, SQLITE_STATIC);
+                                  ? doc->parent + ScanCtx.index.desc.root_len
+                                  : NULL;
+
+    // path, parent, mtime, size, json_data
+    sqlite3_bind_text(db->write_document_stmt, 1, rel_path, -1, SQLITE_STATIC);
+    sqlite3_bind_text(db->write_document_stmt, 2, parent_rel_path, -1, SQLITE_STATIC);
+    sqlite3_bind_int64(db->write_document_stmt, 3, doc->mime);
+    sqlite3_bind_int(db->write_document_stmt, 4, doc->mtime);
+    sqlite3_bind_int64(db->write_document_stmt, 5, (long) doc->size);
+    sqlite3_bind_int(db->write_document_stmt, 6, doc->thumbnail_count);
+    if (json_data) {
+        sqlite3_bind_text(db->write_document_stmt, 7, json_data, -1, SQLITE_STATIC);
+    } else {
+        sqlite3_bind_null(db->write_document_stmt, 7);
+    }
 
     pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
     CRASH_IF_STMT_FAIL(sqlite3_step(db->write_document_stmt));
+    int id = sqlite3_column_int(db->write_document_stmt, 0);
     CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_document_stmt));
     pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
+
+    return id;
 }
 
 
-void database_write_document_sidecar(database_t *db, const char *id, const char *json_data) {
+void database_write_thumbnail(database_t *db, int doc_id, int num, void *data, size_t data_size) {
-    sqlite3_bind_text(db->write_document_sidecar_stmt, 1, id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->write_thumbnail_stmt, 1, doc_id);
-    sqlite3_bind_text(db->write_document_sidecar_stmt, 2, json_data, -1, SQLITE_STATIC);
-
-    pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
-    CRASH_IF_STMT_FAIL(sqlite3_step(db->write_document_sidecar_stmt));
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_document_sidecar_stmt));
-    pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
-}
-
-void database_write_thumbnail(database_t *db, const char *id, int num, void *data, size_t data_size) {
-    sqlite3_bind_text(db->write_thumbnail_stmt, 1, id, -1, SQLITE_STATIC);
     sqlite3_bind_int(db->write_thumbnail_stmt, 2, num);
     sqlite3_bind_blob(db->write_thumbnail_stmt, 3, data, (int) data_size, SQLITE_STATIC);
 
@@ -716,7 +747,7 @@ job_t *database_get_work(database_t *db, job_type_t job_type) {
         } else {
             job->bulk_line = malloc(sizeof(es_bulk_line_t));
         }
-        strcpy(job->bulk_line->doc_id, (const char *) sqlite3_column_text(db->pop_index_job_stmt, 0));
+        strcpy(job->bulk_line->sid, (const char *) sqlite3_column_text(db->pop_index_job_stmt, 0));
         job->bulk_line->type = sqlite3_column_int(db->pop_index_job_stmt, 1);
         job->bulk_line->next = NULL;
 
@@ -767,7 +798,7 @@ void database_add_work(database_t *db, job_t *job) {
         } while (ret != SQLITE_DONE && ret != SQLITE_OK);
     } else if (job->type == JOB_BULK_LINE) {
         do {
-            sqlite3_bind_text(db->insert_index_job_stmt, 1, job->bulk_line->doc_id, -1, SQLITE_STATIC);
+            sqlite3_bind_text(db->insert_index_job_stmt, 1, job->bulk_line->sid, -1, SQLITE_STATIC);
             sqlite3_bind_int(db->insert_index_job_stmt, 2, job->bulk_line->type);
             if (job->bulk_line->type != ES_BULK_LINE_DELETE) {
                 sqlite3_bind_text(db->insert_index_job_stmt, 3, job->bulk_line->line, -1, SQLITE_STATIC);
@@ -808,24 +839,25 @@ void database_add_work(database_t *db, job_t *job) {
     pthread_mutex_unlock(&db->ipc_ctx->mutex);
 }
 
-void database_write_tag(database_t *db, char *doc_id, char *tag) {
+void database_write_tag(database_t *db, long sid, char *tag) {
-    sqlite3_bind_text(db->write_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_int64(db->write_tag_stmt, 1, sid);
-    sqlite3_bind_text(db->write_tag_stmt, 2, tag, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->write_tag_stmt, 2, (int) (sid >> 32));
+    sqlite3_bind_text(db->write_tag_stmt, 3, tag, -1, SQLITE_STATIC);
 
     CRASH_IF_STMT_FAIL(sqlite3_step(db->write_tag_stmt));
     CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_tag_stmt));
 }
 
-void database_delete_tag(database_t *db, char *doc_id, char *tag) {
+void database_delete_tag(database_t *db, long sid, char *tag) {
-    sqlite3_bind_text(db->delete_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_int64(db->delete_tag_stmt, 1, sid);
     sqlite3_bind_text(db->delete_tag_stmt, 2, tag, -1, SQLITE_STATIC);
 
     CRASH_IF_STMT_FAIL(sqlite3_step(db->delete_tag_stmt));
     CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->delete_tag_stmt));
 }
 
-cJSON *database_get_document(database_t *db, char *doc_id) {
+cJSON *database_get_document(database_t *db, int doc_id) {
-    sqlite3_bind_text(db->get_document, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->get_document, 1, doc_id);
 
     int ret = sqlite3_step(db->get_document);
     CRASH_IF_STMT_FAIL(ret);
@@ -833,7 +865,7 @@ cJSON *database_get_document(database_t *db, char *doc_id) {
     cJSON *json;
 
     if (ret == SQLITE_ROW) {
-        const char *json_str = sqlite3_column_text(db->get_document, 0);
+        const char *json_str = (char *) sqlite3_column_text(db->get_document, 0);
         json = cJSON_Parse(json_str);
     } else {
         json = NULL;
@@ -847,4 +879,24 @@ cJSON *database_get_document(database_t *db, char *doc_id) {
 void database_increment_version(database_t *db) {
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db, "INSERT INTO version DEFAULT VALUES", NULL, NULL, NULL));
+}
+
+void database_sync_mime_table(database_t *db) {
+    unsigned int *cur = get_mime_ids();
+
+    sqlite3_stmt *stmt;
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare(
+            db->db,
+            "REPLACE INTO mime (id, name) VALUES (?,?)", -1, &stmt, NULL));
+
+    while (*cur != 0) {
+        sqlite3_bind_int64(stmt, 1, (long) *cur);
+        sqlite3_bind_text(stmt, 2, mime_get_mime_text(*cur), -1, NULL);
+
+        CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
+        sqlite3_reset(stmt);
+
+        cur += 1;
+    }
+    sqlite3_finalize(stmt);
 }

src/database/database.h

@@ -64,6 +64,8 @@ typedef struct {
     char current_job[MAX_THREADS][PATH_MAX * 2];
 } database_ipc_ctx_t;
 
+#define SET_CURRENT_JOB(ctx, job) (strcpy((ctx)->current_job[ProcData.thread_id], job))
+
 typedef struct {
     double date_min;
     double date_max;
@@ -81,7 +83,6 @@ typedef struct database {
 
     sqlite3_stmt *mark_document_stmt;
     sqlite3_stmt *write_document_stmt;
-    sqlite3_stmt *write_document_sidecar_stmt;
     sqlite3_stmt *write_thumbnail_stmt;
     sqlite3_stmt *get_document;
     sqlite3_stmt *get_models;
@@ -103,9 +104,9 @@ typedef struct database {
     sqlite3_stmt *fts_get_document;
     sqlite3_stmt *fts_suggest_tag;
     sqlite3_stmt *fts_get_tags;
+    sqlite3_stmt *fts_write_tag_stmt;
     sqlite3_stmt *fts_model_size;
 
-
     char **tag_array;
 
     database_ipc_ctx_t *ipc_ctx;
@@ -133,15 +134,15 @@ void database_close(database_t *, int optimize);
 
 void database_increment_version(database_t *db);
 
-void database_write_thumbnail(database_t *db, const char *id, int num, void *data, size_t data_size);
+void database_write_thumbnail(database_t *db, int doc_id, int num, void *data, size_t data_size);
 
-void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *return_value_len);
+void *database_read_thumbnail(database_t *db, int doc_id, int num, size_t *return_value_len);
 
 void database_write_index_descriptor(database_t *db, index_descriptor_t *desc);
 
 index_descriptor_t *database_read_index_descriptor(database_t *db);
 
-void database_write_document(database_t *db, document_t *doc, const char *json_data);
+int database_write_document(database_t *db, document_t *doc, const char *json_data);
 
 database_iterator_t *database_create_document_iterator(database_t *db);
 
@@ -154,10 +155,10 @@ cJSON *database_document_iter(database_iterator_t *);
 
 database_iterator_t *database_create_delete_list_iterator(database_t *db);
 
-char *database_delete_list_iter(database_iterator_t *iter);
+int database_delete_list_iter(database_iterator_t *iter);
 
 #define database_delete_list_iter_foreach(element, iter) \
-    for (char *(element) = database_delete_list_iter(iter); (element) != NULL; (element) = database_delete_list_iter(iter))
+    for (int (element) = database_delete_list_iter(iter); (element) != 0; (element) = database_delete_list_iter(iter))
 
 
 cJSON *database_incremental_scan_begin(database_t *db);
@@ -166,8 +167,6 @@ cJSON *database_incremental_scan_end(database_t *db);
 
 int database_mark_document(database_t *db, const char *id, int mtime);
 
-void database_write_document_sidecar(database_t *db, const char *id, const char *json_data);
-
 database_iterator_t *database_create_treemap_iterator(database_t *db, long threshold);
 
 treemap_row_t database_treemap_iter(database_iterator_t *iter);
@@ -206,7 +205,7 @@ void database_fts_index(database_t *db);
 
 void database_fts_optimize(database_t *db);
 
-cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
+cJSON *database_fts_get_paths(database_t *db, int index_id, int depth_min, int depth_max, const char *prefix,
                               int suggest);
 
 cJSON *database_fts_get_mimetypes(database_t *db);
@@ -215,18 +214,20 @@ database_summary_stats_t database_fts_get_date_range(database_t *db);
 
 cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
                            long size_max, long date_min, long date_max, int page_size,
-                           char **index_ids, char **mime_types, char **tags, int sort_asc,
+                           int *index_ids, char **mime_types, char **tags, int sort_asc,
                            fts_sort_t sort, int seed, char **after, int fetch_aggregations,
                            int highlight, int highlight_context_size, int model,
                            const float *embedding, int embedding_size);
 
-void database_write_tag(database_t *db, char *doc_id, char *tag);
+void database_write_tag(database_t *db, long sid, char *tag);
 
-void database_delete_tag(database_t *db, char *doc_id, char *tag);
+void database_fts_write_tag(database_t *db, long sid, char *tag);
+
+void database_delete_tag(database_t *db, long sid, char *tag);
 
 void database_fts_detach(database_t *db);
 
-cJSON *database_fts_get_document(database_t *db, char *doc_id);
+cJSON *database_fts_get_document(database_t *db, long sid);
 
 database_summary_stats_t database_fts_sync_tags(database_t *db);
 
@@ -234,7 +235,7 @@ cJSON *database_fts_suggest_tag(database_t *db, char *prefix);
 
 cJSON *database_fts_get_tags(database_t *db);
 
-cJSON *database_get_document(database_t *db, char *doc_id);
+cJSON *database_get_document(database_t *db, int doc_id);
 
 void cosine_sim_func(sqlite3_context *ctx, int argc, sqlite3_value **argv);
 
@@ -242,6 +243,8 @@ cJSON *database_get_models(database_t *db);
 
 int database_fts_get_model_size(database_t *db, int model_id);
 
-cJSON *database_get_embedding(database_t *db, char *doc_id, int model_id);
+cJSON *database_get_embedding(database_t *db, int doc_id, int model_id);
+
+void database_sync_mime_table(database_t *db);
 
 #endif

src/database/database_embeddings.c

@@ -69,9 +69,9 @@ cJSON *database_get_models(database_t *db) {
     return json;
 }
 
-cJSON *database_get_embedding(database_t *db, char *doc_id, int model_id) {
+cJSON *database_get_embedding(database_t *db, int doc_id, int model_id) {
 
-    sqlite3_bind_text(db->get_embedding, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->get_embedding, 1, doc_id);
     sqlite3_bind_int(db->get_embedding, 2, model_id);
     int ret = sqlite3_step(db->get_embedding);
     CRASH_IF_STMT_FAIL(ret);

src/database/database_fts.c

@@ -42,21 +42,23 @@ void database_fts_index(database_t *db) {
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
             "WITH docs AS ("
-            " SELECT document.id as id, (SELECT id FROM descriptor) as index_id, size,"
+            " SELECT "
+            "  ((SELECT id FROM descriptor) << 32) | document.id as id,"
+            "  (SELECT id FROM descriptor) as index_id,"
+            "  size,"
             "  document.json_data ->> 'name' as name,"
             "  document.json_data ->> 'path' as path,"
             "  mtime,"
-            "  document.json_data ->> 'mime' as mime,"
+            "  m.name as mime,"
-            "    json_set(document.json_data, "
+            "  thumbnail_count,"
-            "             '$._id',document.id,"
+            "  document.json_data"
-            "             '$.size',document.size, "
-            "             '$.mtime',document.mtime)"
             " FROM document"
+            " LEFT JOIN mime m ON m.id=document.mime"
             " )"
             " INSERT"
-            " INTO fts.document_index (id, index_id, size, name, path, mtime, mime, json_data)"
+            " INTO fts.document_index (id, index_id, size, name, path, mtime, mime, thumbnail_count, json_data)"
             " SELECT * FROM docs WHERE true"
-            " on conflict (id, index_id) do update set "
+            " on conflict (id) do update set "
             "  size=excluded.size, mtime=excluded.mtime, mime=excluded.mime, json_data=excluded.json_data;",
             NULL, NULL, NULL));
 
@@ -64,13 +66,14 @@ void database_fts_index(database_t *db) {
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "REPLACE INTO fts.embedding (id, model_id, start, end, embedding)"
+            "REPLACE INTO fts.model (id, size)"
-            " SELECT id, model_id, start, end, embedding FROM embedding", NULL, NULL, NULL));
+            " SELECT id, size FROM model", NULL, NULL, NULL));
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "INSERT INTO fts.model (id, size)"
+            "REPLACE INTO fts.embedding (id, model_id, start, end, embedding)"
-            " SELECT id, size FROM model WHERE TRUE ON CONFLICT (id) DO NOTHING", NULL, NULL, NULL));
+            " SELECT (SELECT id FROM descriptor) << 32 | id, model_id, start, end, embedding FROM embedding "
+            " WHERE TRUE ON CONFLICT (id, model_id, start) DO NOTHING;", NULL, NULL, NULL));
 
     // TODO: delete old embeddings
 
@@ -99,7 +102,9 @@ void database_fts_index(database_t *db) {
             db->db, "DELETE FROM fts.mime_index;", NULL, NULL, NULL));
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db, "INSERT INTO fts.mime_index (index_id, mime, count) "
-                    "SELECT index_id, mime, count(*) FROM fts.document_index GROUP BY index_id, mime",
+                    "SELECT index_id, mime, count(*) FROM fts.document_index "
+                    "WHERE mime IS NOT NULL "
+                    "GROUP BY index_id, mime",
             NULL, NULL, NULL));
 
     LOG_DEBUG("database_fts.c", "Generating path index");
@@ -157,7 +162,8 @@ void database_fts_index(database_t *db) {
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "INSERT INTO search(rowid, name, content, title) SELECT id, name, content, title from document_view",
+            "INSERT INTO search(rowid, name, content, title, path) "
+            "SELECT id, name, content, title, path from document_view",
             NULL, NULL, NULL));
 }
 
@@ -172,7 +178,7 @@ void database_fts_optimize(database_t *db) {
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA fts.optimize;", NULL, NULL, NULL));
 }
 
-cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
+cJSON *database_fts_get_paths(database_t *db, int index_id, int depth_min, int depth_max, const char *prefix,
                               int suggest) {
 
     sqlite3_stmt *stmt;
@@ -192,7 +198,7 @@ cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_mi
     } else if (prefix) {
         stmt = db->fts_search_paths_w_prefix;
         if (index_id) {
-            sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
+            sqlite3_bind_int(stmt, 1, index_id);
         } else {
             sqlite3_bind_null(stmt, 1);
         }
@@ -207,7 +213,7 @@ cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_mi
     } else {
         stmt = db->fts_search_paths;
         if (index_id) {
-            sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
+            sqlite3_bind_int(stmt, 1, index_id);
         } else {
             sqlite3_bind_null(stmt, 1);
         }
@@ -290,7 +296,6 @@ const char *date_where_clause(long date_min, long date_max) {
 }
 
 int array_length(char **arr) {
-
     if (arr == NULL) {
         return 0;
     }
@@ -301,6 +306,17 @@ int array_length(char **arr) {
     return count;
 }
 
+int int_array_length(const int *arr) {
+    if (arr == NULL) {
+        return 0;
+    }
+
+    int count = -1;
+    while (arr[++count] != 0);
+
+    return count;
+}
+
 #define INDEX_ID_PARAM_OFFSET (10)
 #define MIME_PARAM_OFFSET (INDEX_ID_PARAM_OFFSET + 1000)
 
@@ -351,8 +367,8 @@ char *build_where_clause(const char *path_where, const char *size_where, const c
     return where;
 }
 
-char *index_ids_where_clause(char **index_ids) {
+char *index_ids_where_clause(int *index_ids) {
-    int param_count = array_length(index_ids);
+    int param_count = int_array_length(index_ids);
 
     char *clause = malloc(13 + 2 + 6 * param_count);
 
@@ -483,7 +499,7 @@ int database_fts_get_model_size(database_t *db, int model_id) {
 
 cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
                            long size_max, long date_min, long date_max, int page_size,
-                           char **index_ids, char **mime_types, char **tags, int sort_asc,
+                           int *index_ids, char **mime_types, char **tags, int sort_asc,
                            fts_sort_t sort, int seed, char **after, int fetch_aggregations,
                            int highlight, int highlight_context_size, int model,
                            const float *embedding, int embedding_size) {
@@ -524,13 +540,21 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
     const char *json_object_sql;
     if (highlight && query_where != NULL) {
         json_object_sql = "json_set(json_remove(doc.json_data, '$.content'),"
+                          "'$._id', CAST(doc.id AS TEXT),"
                           "'$.index', doc.index_id,"
+                          "'$.thumbnail', doc.thumbnail_count,"
+                          "'$.mime', doc.mime,"
+                          "'$.size', doc.size,"
                           "'$.embedding', (CASE WHEN emb.id IS NOT NULL THEN 1 ELSE 0 END),"
                           "'$._highlight.name', snippet(search, 0, '<mark>', '</mark>', '', ?6),"
                           "'$._highlight.content', snippet(search, 1, '<mark>', '</mark>', '', ?6))";
     } else {
         json_object_sql = "json_set(json_remove(doc.json_data, '$.content'),"
+                          "'$._id', CAST(doc.id AS TEXT),"
                           "'$.index', doc.index_id,"
+                          "'$.thumbnail', doc.thumbnail_count,"
+                          "'$.mime', doc.mime,"
+                          "'$.size', doc.size,"
                           "'$.embedding', (CASE WHEN emb.id IS NOT NULL THEN 1 ELSE 0 END))";
     }
 
@@ -592,7 +616,7 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
 
     if (index_ids) {
         array_foreach(index_ids) {
-            sqlite3_bind_text(stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
+            sqlite3_bind_int(stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i]);
         }
     }
     if (mime_types) {
@@ -692,7 +716,7 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
 
         if (index_ids) {
             array_foreach(index_ids) {
-                sqlite3_bind_text(agg_stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
+                sqlite3_bind_int(agg_stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i]);
             }
         }
         if (mime_types) {
@@ -764,19 +788,20 @@ database_summary_stats_t database_fts_sync_tags(database_t *db) {
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
             "DELETE FROM fts.tag WHERE"
-            " (id, tag) NOT IN (SELECT id, tag FROM tag)",
+            " (id, index_id, tag) NOT IN (SELECT ((SELECT id FROM descriptor) << 32) | id, (SELECT id FROM descriptor), tag FROM tag)"
+            " AND index_id = (SELECT id FROM descriptor)",
             NULL, NULL, NULL));
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "INSERT INTO fts.tag (id, tag) "
+            "INSERT INTO fts.tag (id, index_id, tag) "
-            " SELECT id, tag FROM tag "
+            " SELECT (((SELECT id FROM descriptor) << 32) | id) as sid, (SELECT id FROM descriptor), tag FROM tag "
-            " WHERE (id, tag) NOT IN (SELECT * FROM fts.tag)",
+            " WHERE (sid, tag) NOT IN (SELECT id, tag FROM fts.tag)",
             NULL, NULL, NULL));
 }
 
-cJSON *database_fts_get_document(database_t *db, char *doc_id) {
+cJSON *database_fts_get_document(database_t *db, long sid) {
-    sqlite3_bind_text(db->fts_get_document, 1, doc_id, -1, NULL);
+    sqlite3_bind_int64(db->fts_get_document, 1, sid);
 
     int ret = sqlite3_step(db->fts_get_document);
     cJSON *json = NULL;
@@ -844,3 +869,11 @@ cJSON *database_fts_get_tags(database_t *db) {
 
     return json;
 }
+void database_fts_write_tag(database_t *db, long sid, char *tag) {
+    sqlite3_bind_int64(db->fts_write_tag_stmt, 1, sid);
+    sqlite3_bind_int(db->fts_write_tag_stmt, 2, (int) (sid >> 32));
+    sqlite3_bind_text(db->fts_write_tag_stmt, 3, tag, -1, SQLITE_STATIC);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->fts_write_tag_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->fts_write_tag_stmt));
+}

src/database/database_schema.c

@@ -1,57 +1,63 @@
+#ifdef SIST_DEBUG
+#define STRICT " STRICT"
+#else
+#define STRICT ""
+#endif
+
 const char *FtsDatabaseSchema =
         "CREATE TABLE IF NOT EXISTS document_index ("
-        "   id TEXT NOT NULL,"
+        "   id INTEGER PRIMARY KEY,"
-        "   index_id TEXT NOT NULL,"
+        "   index_id INTEGER NOT NULL,"
         "   size INTEGER NOT NULL,"
         "   name TEXT NOT NULL,"
         "   path TEXT NOT NULL,"
         "   mtime INTEGER NOT NULL,"
         "   mime TEXT,"
-        "   json_data TEXT NOT NULL,"
+        "   thumbnail_count INTEGER NOT NULL,"
-        "   PRIMARY KEY (id, index_id)"
+        "   json_data TEXT NOT NULL"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE IF NOT EXISTS stats ("
         "   mtime_min INTEGER,"
         "   mtime_max INTEGER"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE IF NOT EXISTS path_index ("
         "   path TEXT,"
-        "   index_id TEXT,"
+        "   index_id INTEGER,"
         "   count INTEGER NOT NULL,"
         "   depth INTEGER NOT NULL,"
         "   PRIMARY KEY (path, index_id)"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE IF NOT EXISTS mime_index ("
-        "   index_id TEXT,"
+        "   index_id INTEGER,"
         "   mime TEXT,"
-        "   count INT,"
+        "   count INTEGER,"
         "   PRIMARY KEY(index_id, mime)"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE IF NOT EXISTS tag ("
-        "   id TEXT NOT NULL,"
+        "   id INTEGER NOT NULL,"
+        "   index_id INTEGER NOT NULL,"
         "   tag TEXT NOT NULL,"
         "   PRIMARY KEY (id, tag)"
-        ");"
+        ")"STRICT";"
         "CREATE INDEX IF NOT EXISTS tag_tag_idx ON tag(tag);"
-        "CREATE INDEX IF NOT EXISTS tag_id_idx ON tag(id);"
         ""
         "CREATE TABLE IF NOT EXISTS embedding ("
-        "   id TEXT REFERENCES document(id),"
+        "   id INTEGER REFERENCES document_index(id),"
         "   model_id INTEGER NOT NULL REFERENCES model(id),"
         "   start INTEGER NOT NULL,"
         "   end INTEGER,"
         "   embedding BLOB NOT NULL,"
         "   PRIMARY KEY (id, model_id, start)"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE IF NOT EXISTS model ("
         "   id INTEGER PRIMARY KEY CHECK (id > 0 AND id < 1000),"
         "   size INTEGER NOT NULL"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TRIGGER IF NOT EXISTS tag_write_trigger"
         "    AFTER INSERT ON tag"
@@ -69,23 +75,25 @@ const char *FtsDatabaseSchema =
         "  WHERE id = OLD.id;"
         " END;"
         ""
-        "CREATE VIEW IF NOT EXISTS document_view (id, name, content, title)"
+        "CREATE VIEW IF NOT EXISTS document_view (id, name, content, title, path)"
         " AS"
-        " SELECT rowid,"
+        " SELECT id,"
         "  json_data->>'name',"
         "  json_data->>'content',"
-        "  json_data->>'title'"
+        "  json_data->>'title',"
+        "  json_data->>'path'"
         " FROM document_index;"
         ""
         "CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5 ("
         "   name,"
         "   content,"
         "   title,"
+        "   path,"
         "   content='document_view',"
         "   content_rowid='id'"
         ");"
-        // name^8, content^3, title^8
+        // name^8, content^3, title^8, path^5
-        "INSERT INTO search(search, rank) VALUES('rank', 'bm25(8, 3, 8)');"
+        "INSERT INTO search(search, rank) VALUES('rank', 'bm25(8, 3, 8, 5)');"
         "";
 
 const char *IpcDatabaseSchema =
@@ -94,18 +102,18 @@ const char *IpcDatabaseSchema =
         "   filepath TEXT NOT NULL,"
         "   mtime INTEGER NOT NULL,"
         "   st_size INTEGER NOT NULL"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE index_job ("
         "   id INTEGER PRIMARY KEY,"
-        "   doc_id TEXT NOT NULL CHECK ( length(doc_id) = 32 ),"
+        "   sid TEXT NOT NULL,"
         "   type INTEGER NOT NULL,"
         "   line TEXT"
-        ");";
+        ")"STRICT";";
 
 const char *IndexDatabaseSchema =
         "CREATE TABLE thumbnail ("
-        "   id TEXT NOT NULL CHECK ( length(id) = 32 ),"
+        "   id INTEGER REFERENCES document(id),"
         "   num INTEGER NOT NULL,"
         "   data BLOB NOT NULL,"
         "   PRIMARY KEY(id, num)"
@@ -114,34 +122,46 @@ const char *IndexDatabaseSchema =
         "CREATE TABLE version ("
         "   id INTEGER PRIMARY KEY AUTOINCREMENT,"
         "   date TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP)"
-        ");"
+        ")"STRICT";"
+        ""
+        "CREATE TABLE mime("
+        "   id INTEGER PRIMARY KEY,"
+        "   name TEXT"
+        ")"STRICT";"
+        "CREATE UNIQUE INDEX mime_name_idx ON mime(name);"
         ""
         "CREATE TABLE document ("
-        "   id TEXT PRIMARY KEY CHECK ( length(id) = 32 ),"
+        "   id INTEGER PRIMARY KEY,"
-        "   marked INTEGER NOT NULL DEFAULT (1),"
+        "   parent INTEGER REFERENCES document(id),"
+        "   mime INTEGER REFERENCES mime(id),"
+        "   path TEXT NOT NULL,"
         "   version INTEGER NOT NULL REFERENCES version(id),"
         "   mtime INTEGER NOT NULL,"
         "   size INTEGER NOT NULL,"
-        "   json_data TEXT NOT NULL CHECK ( json_valid(json_data) )"
+        "   thumbnail_count INTEGER NOT NULL,"
-        ") WITHOUT ROWID;"
+        "   json_data TEXT CHECK ( json_data IS NULL OR json_valid(json_data) )"
+        ")"STRICT";"
+        "CREATE UNIQUE INDEX document_path_idx ON document(path);"
+        "CREATE TABLE marked ("
+        "   id INTEGER PRIMARY KEY,"
+        "   marked INTEGER NOT NULL,"
+        "   mtime INTEGER NOT NULL"
+        ")"STRICT";"
+        ""
+        "CREATE INDEX marked_marked ON marked(marked);"
         ""
         "CREATE TABLE delete_list ("
-        "   id TEXT PRIMARY KEY CHECK ( length(id) = 32 )"
+        "   id INTEGER PRIMARY KEY"
-        ") WITHOUT ROWID;"
+        ")"STRICT";"
         ""
         "CREATE TABLE tag ("
-        "   id TEXT NOT NULL,"
+        "   id INTEGER NOT NULL REFERENCES document(id),"
         "   tag TEXT NOT NULL,"
         "   PRIMARY KEY (id, tag)"
-        ");"
+        ")"STRICT";"
-        ""
-        "CREATE TABLE document_sidecar ("
-        "   id TEXT PRIMARY KEY NOT NULL,"
-        "   json_data TEXT NOT NULL"
-        ") WITHOUT ROWID;"
         ""
         "CREATE TABLE descriptor ("
-        "   id TEXT NOT NULL,"
+        "   id INTEGER PRIMARY KEY,"
         "   version_major INTEGER NOT NULL,"
         "   version_minor INTEGER NOT NULL,"
         "   version_patch INTEGER NOT NULL,"
@@ -149,37 +169,37 @@ const char *IndexDatabaseSchema =
         "   name TEXT NOT NULL,"
         "   rewrite_url TEXT,"
         "   timestamp INTEGER NOT NULL"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE stats_treemap ("
         "   path TEXT NOT NULL,"
         "   size INTEGER NOT NULL"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE stats_size_agg ("
         "   bucket INTEGER NOT NULL,"
         "   count INTEGER NOT NULL"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE stats_date_agg ("
         "   bucket INTEGER NOT NULL,"
         "   count INTEGER NOT NULL"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE stats_mime_agg ("
         "   mime TEXT NOT NULL,"
         "   size INTEGER NOT NULL,"
         "   count INTEGER NOT NULL"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE embedding ("
-        "   id TEXT REFERENCES document(id),"
+        "   id INTEGER REFERENCES document(id),"
         "   model_id INTEGER NOT NULL references model(id),"
         "   start INTEGER NOT NULL,"
         "   end INTEGER,"
         "   embedding BLOB NOT NULL,"
         "   PRIMARY KEY (id, model_id, start)"
-        ");"
+        ")"STRICT";"
         ""
         "CREATE TABLE model ("
         "   id INTEGER PRIMARY KEY CHECK (id > 0 AND id < 1000),"
@@ -188,5 +208,5 @@ const char *IndexDatabaseSchema =
         "   path TEXT NOT NULL UNIQUE,"
         "   size INTEGER NOT NULL,"
         "   type TEXT NOT NULL CHECK ( type IN ('flat', 'nested') )"
-        ");";
+        ")"STRICT";";
 

src/database/database_stats.c

@@ -98,10 +98,10 @@ void database_generate_stats(database_t *db, double treemap_threshold) {
     // mime aggregation
     sqlite3_prepare_v2(db->db, "INSERT INTO stats_mime_agg"
                                " SELECT"
-                               "  (json_data->>'mime') as bucket,"
+                               "  m.name as bucket,"
                                "  sum(size),"
                                "  count(*)"
-                               " FROM document"
+                               " FROM document INNER JOIN mime m ON m.id=document.mime"
                                " WHERE bucket IS NOT NULL"
                                " GROUP BY bucket", -1, &stmt, NULL);
     CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
@@ -117,8 +117,8 @@ void database_generate_stats(database_t *db, double treemap_threshold) {
 
     // flat map
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db,
-                                        "INSERT INTO tm (path, size) SELECT json_data->>'path' as path, sum(size)"
+                                        "INSERT INTO tm (path, size) SELECT path, sum(size)"
-                                        " FROM document WHERE json_data->>'parent' IS NULL GROUP BY path;",
+                                        " FROM document WHERE parent IS NULL GROUP BY path;",
                                         NULL, NULL, NULL));
 
     // Merge up

src/ignorelist.c

@@ -0,0 +1,106 @@
+#include "ignorelist.h"
+#include "ctx.h"
+#include <git2.h>
+
+typedef struct ignorelist {
+    git_repository *repo;
+    char repo_path[PATH_MAX];
+    int has_rules;
+} ignorelist_t;
+
+char *get_tempdir() {
+    char *tempdir_env = getenv("TMPDIR");
+
+    if (tempdir_env != NULL) {
+        return tempdir_env;
+    }
+
+    return "/tmp/";
+}
+
+void ignorelist_destroy(ignorelist_t* ignorelist) {
+    git_libgit2_shutdown();
+
+    if (ignorelist->repo != NULL) {
+        git_repository_free(ignorelist->repo);
+    }
+
+    free(ignorelist);
+}
+
+ignorelist_t *ignorelist_create() {
+    git_libgit2_init();
+
+    ignorelist_t *ignorelist = malloc(sizeof(ignorelist_t));
+
+    ignorelist->repo = NULL;
+    ignorelist->has_rules = FALSE;
+
+    char *tempdir = get_tempdir();
+
+    if (tempdir[strlen(tempdir) - 1] == '/') {
+        sprintf(ignorelist->repo_path, "%ssist2-ignorelist-%d", tempdir, getpid());
+    } else {
+        sprintf(ignorelist->repo_path, "%s/sist2-ignorelist-%d", tempdir, getpid());
+    }
+
+    return ignorelist;
+}
+
+void ignorelist_load_ignore_file(ignorelist_t *ignorelist, const char *filepath) {
+
+    FILE *file;
+    char line[PATH_MAX * 2];
+
+    file = fopen(filepath, "r");
+
+    if(file == NULL) {
+        // No ignore list
+        return;
+    }
+
+    LOG_DEBUGF("ignorelist.c", "Opening temporary git repository %s", ignorelist->repo_path);
+    int init_result = git_repository_init(&ignorelist->repo, ignorelist->repo_path, TRUE);
+
+    if (init_result != 0) {
+        LOG_FATALF("ignorelist.c", "Got error code from git_repository_init(): %d", init_result);
+    }
+
+    git_ignore_clear_internal_rules(ignorelist->repo);
+
+    while(fgets(line, PATH_MAX * 2, file)){
+
+        line[strlen(line) - 1] = '\0'; // Strip trailing newline
+        char *rules = {line,};
+
+        int result = git_ignore_add_rule(ignorelist->repo, rules);
+
+        if (result == 0) {
+            LOG_DEBUGF("ignorelist.c", "Load ignore rule: %s", line);
+            ignorelist->has_rules = TRUE;
+        } else {
+            LOG_FATALF("ignorelist.c", "Invalid ignore rule: %s", line);
+        }
+    }
+
+    fclose(file);
+}
+
+int ignorelist_is_ignored(ignorelist_t *ignorelist, const char *filepath) {
+
+    if (!ignorelist->has_rules) {
+        return FALSE;
+    }
+
+    const char *rel_path = filepath + ScanCtx.index.desc.root_len;
+
+    int ignored = -1;
+
+    int result = git_ignore_path_is_ignored(&ignored, ignorelist->repo, rel_path);
+
+    if (result != 0) {
+        LOG_FATALF("ignorelist.c", "git_ignore_path_is_ignored returned error code: %d", result);
+    }
+
+    return ignored;
+}

src/ignorelist.h

@@ -0,0 +1,16 @@
+#ifndef SIST2_IGNORELIST_H
+#define SIST2_IGNORELIST_H
+
+#include "src/sist.h"
+
+typedef struct ignorelist ignorelist_t;
+
+ignorelist_t *ignorelist_create();
+
+void ignorelist_destroy(ignorelist_t* ignorelist);
+
+void ignorelist_load_ignore_file(ignorelist_t* ignorelist, const char* filepath);
+
+int ignorelist_is_ignored(ignorelist_t* ignorelist, const char* filepath);
+
+#endif //SIST2_IGNORELIST_H

src/index/elastic.c

@@ -47,7 +47,7 @@ void elastic_cleanup() {
     destroy_indexer(Indexer);
 }
 
-void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) {
+void print_json(cJSON *document, const char id_str[SIST_SID_LEN]) {
 
     cJSON *line = cJSON_CreateObject();
 
@@ -64,12 +64,12 @@ void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) {
     cJSON_Delete(line);
 }
 
-void delete_document(const char *document_id) {
+void delete_document(const char *sid) {
     es_bulk_line_t bulk_line;
 
     bulk_line.type = ES_BULK_LINE_DELETE;
     bulk_line.next = NULL;
-    strcpy(bulk_line.doc_id, document_id);
+    strcpy(bulk_line.sid, sid);
 
     tpool_add_work(IndexCtx.pool, &(job_t) {
             .type = JOB_BULK_LINE,
@@ -78,14 +78,14 @@ void delete_document(const char *document_id) {
 }
 
 
-void index_json(cJSON *document, const char doc_id[SIST_DOC_ID_LEN]) {
+void index_json(cJSON *document, const char doc_id[SIST_SID_LEN]) {
     char *json = cJSON_PrintUnformatted(document);
 
     size_t json_len = strlen(json);
     es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t) + json_len + 2);
     bulk_line->type = ES_BULK_LINE_INDEX;
     memcpy(bulk_line->line, json, json_len);
-    strcpy(bulk_line->doc_id, doc_id);
+    strcpy(bulk_line->sid, doc_id);
     *(bulk_line->line + json_len) = '\n';
     *(bulk_line->line + json_len + 1) = '\0';
     bulk_line->next = NULL;
@@ -124,13 +124,13 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len, int legacy) {
                 snprintf(
                         action_str, sizeof(action_str),
                         "{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n",
-                        line->doc_id, Indexer->es_index
+                        line->sid, Indexer->es_index
                 );
             } else {
                 snprintf(
                         action_str, sizeof(action_str),
                         "{\"index\":{\"_id\":\"%s\",\"_index\":\"%s\"}}\n",
-                        line->doc_id, Indexer->es_index
+                        line->sid, Indexer->es_index
                 );
             }
 
@@ -148,7 +148,7 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len, int legacy) {
             snprintf(
                     action_str, sizeof(action_str),
                     "{\"delete\":{\"_id\":\"%s\",\"_index\":\"%s\"}}\n",
-                    line->doc_id, Indexer->es_index
+                    line->sid, Indexer->es_index
             );
 
             size_t action_str_len = strlen(action_str);
@@ -236,7 +236,7 @@ void _elastic_flush(int max) {
     if (r->status_code == 413) {
 
         if (max <= 1) {
-            LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->doc_id);
+            LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->sid);
             free_response(r);
             free(buf);
             free_queue(1);
@@ -348,7 +348,7 @@ es_indexer_t *create_indexer(const char *url, const char *index) {
     return indexer;
 }
 
-void finish_indexer(char *index_id) {
+void finish_indexer(int index_id) {
 
     char url[4096];
 

src/index/elastic.h

@@ -8,7 +8,7 @@
 
 typedef struct es_bulk_line {
     struct es_bulk_line *next;
-    char doc_id[SIST_DOC_ID_LEN];
+    char sid[SIST_SID_LEN];
     int type;
     char line[0];
 } es_bulk_line_t;
@@ -44,16 +44,16 @@ typedef struct es_indexer es_indexer_t;
 
 void elastic_index_line(es_bulk_line_t *line);
 
-void print_json(cJSON *document, const char index_id_str[SIST_INDEX_ID_LEN]);
+void print_json(cJSON *document, const char doc_id[SIST_SID_LEN]);
 
-void index_json(cJSON *document, const char doc_id[SIST_INDEX_ID_LEN]);
+void index_json(cJSON *document, const char doc_id[SIST_SID_LEN]);
 
-void delete_document(const char *document_id);
+void delete_document(const char *sid);
 
 es_indexer_t *create_indexer(const char *url, const char *index);
 
 void elastic_cleanup();
-void finish_indexer(char *index_id);
+void finish_indexer(int index_id);
 
 void elastic_init(int force_reset, const char* user_mappings, const char* user_settings);
 

src/index/web.c

@@ -90,6 +90,7 @@ subreq_ctx_t *web_post_async(const char *url, char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, req->curl_err_buffer);
@@ -123,6 +124,7 @@ response_t *web_get(const char *url, int timeout, int insecure) {
     curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     struct curl_slist *headers = NULL;
@@ -162,6 +164,7 @@ response_t *web_post(const char *url, const char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     char err_buffer[CURL_ERROR_SIZE + 1] = {};
@@ -203,10 +206,11 @@ response_t *web_put(const char *url, const char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
     curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
-    curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
+    curl_easy_setopt(curl, CURLOPT_SHARE, 0);
     curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4);
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     struct curl_slist *headers = NULL;
@@ -241,6 +245,7 @@ response_t *web_delete(const char *url, int insecure) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "");

src/io/serialize.c

@@ -30,10 +30,10 @@ char *get_meta_key_text(enum metakey meta_key) {
             return "genre";
         case MetaTitle:
             return "title";
+        case MetaMediaComment:
+            return "media_comment";
         case MetaFontName:
             return "font_name";
-        case MetaParent:
-            return "parent";
         case MetaExifMake:
             return "exif_make";
         case MetaExifDescription:
@@ -58,8 +58,6 @@ char *get_meta_key_text(enum metakey meta_key) {
             return "author";
         case MetaModifiedBy:
             return "modified_by";
-        case MetaThumbnail:
-            return "thumbnail";
         case MetaPages:
             return "pages";
         case MetaExifGpsLongitudeRef:
@@ -81,21 +79,23 @@ char *get_meta_key_text(enum metakey meta_key) {
     }
 }
 
-char *build_json_string(document_t *doc) {
+typedef struct {
+    meta_line_t *meta_head;
+    meta_line_t *meta_tail;
+} linked_list_t;
+
+
+void write_document(document_t *doc) {
+    linked_list_t thumbnails_to_write = {.meta_head = NULL, .meta_tail = NULL};
+
     cJSON *json = cJSON_CreateObject();
     int buffer_size_guess = 8192;
 
-    const char *mime_text = mime_get_mime_text(doc->mime);
-    if (mime_text == NULL) {
-        cJSON_AddNullToObject(json, "mime");
-    } else {
-        cJSON_AddStringToObject(json, "mime", mime_text);
-    }
-
     // Ignore root directory in the file path
     doc->ext = (short) (doc->ext - ScanCtx.index.desc.root_len);
     doc->base = (short) (doc->base - ScanCtx.index.desc.root_len);
-    char *filepath = doc->filepath + ScanCtx.index.desc.root_len;
+    char filepath[PATH_MAX * 3];
+    strcpy(filepath, doc->filepath + ScanCtx.index.desc.root_len);
 
     cJSON_AddStringToObject(json, "extension", filepath + doc->ext);
 
@@ -125,7 +125,6 @@ char *build_json_string(document_t *doc) {
     while (meta != NULL) {
 
         switch (meta->key) {
-            case MetaThumbnail:
             case MetaPages:
             case MetaWidth:
             case MetaHeight:
@@ -143,7 +142,6 @@ char *build_json_string(document_t *doc) {
             case MetaAlbumArtist:
             case MetaGenre:
             case MetaFontName:
-            case MetaParent:
             case MetaExifMake:
             case MetaExifDescription:
             case MetaExifSoftware:
@@ -163,11 +161,17 @@ char *build_json_string(document_t *doc) {
             case MetaExifGpsLatitudeDec:
             case MetaExifGpsLatitudeRef:
             case MetaChecksum:
+            case MetaMediaComment:
             case MetaTitle: {
                 cJSON_AddStringToObject(json, get_meta_key_text(meta->key), meta->str_val);
                 buffer_size_guess += (int) strlen(meta->str_val);
                 break;
             }
+            case MetaThumbnail: {
+                // Keep a list of thumbnails to write after we know what the sid is
+                APPEND_THUMBNAIL(&thumbnails_to_write, meta->str_val, meta->size);
+                break;
+            }
             default:
             LOG_FATALF("serialize.c", "Invalid meta key: %x %s", meta->key, get_meta_key_text(meta->key));
         }
@@ -180,13 +184,19 @@ char *build_json_string(document_t *doc) {
     char *json_str = cJSON_PrintBuffered(json, buffer_size_guess, FALSE);
     cJSON_Delete(json);
 
-    return json_str;
+    int doc_id = database_write_document(ProcData.index_db, doc, json_str);
-}
-
-void write_document(document_t *doc) {
-    char *json_str = build_json_string(doc);
-
-    database_write_document(ProcData.index_db, doc, json_str);
     free(doc);
     free(json_str);
+
+    // Write thumbnails
+    meta = thumbnails_to_write.meta_head;
+    int index_num = 0;
+    while (meta != NULL) {
+        database_write_thumbnail(ProcData.index_db, doc_id, index_num, meta->str_val, meta->size);
+
+        meta_line_t *tmp = meta;
+        meta = meta->next;
+        free(tmp);
+        index_num += 1;
+    }
 }

src/io/walk.c

@@ -23,8 +23,17 @@ int handle_entry(const char *filepath, const struct stat *info, int typeflag, st
     if (ScanCtx.exclude != NULL && EXCLUDED(filepath)) {
         LOG_DEBUGF("walk.c", "Excluded: %s", filepath);
 
-        if (typeflag == FTW_F && S_ISREG(info->st_mode)) {
+        if (typeflag == FTW_D) {
-        } else if (typeflag == FTW_D) {
+            return FTW_SKIP_SUBTREE;
+        }
+
+        return FTW_CONTINUE;
+    }
+
+    if (ignorelist_is_ignored(ScanCtx.ignorelist, filepath)) {
+        LOG_DEBUGF("walk.c", "Ignored: %s", filepath);
+
+        if (typeflag == FTW_D) {
             return FTW_SKIP_SUBTREE;
         }
 

src/main.c

@@ -11,7 +11,7 @@
 #include "web/serve.h"
 #include "parsing/mime.h"
 #include "parsing/parse.h"
-#include "auth0/auth0_c_api.h"
+#include "ignorelist.h"
 
 #include <signal.h>
 #include <pthread.h>
@@ -39,7 +39,7 @@ void database_scan_begin(scan_args_t *args) {
         index_descriptor_t *original_desc = database_read_index_descriptor(db);
 
         // copy original index id
-        strcpy(desc->id, original_desc->id);
+        desc->id = original_desc->id;
 
         if (original_desc->version_major != VersionMajor) {
             LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc->version, Version);
@@ -67,7 +67,7 @@ void database_scan_begin(scan_args_t *args) {
         desc->version_patch = VersionPatch;
 
         // generate new index id based on timestamp
-        md5_hexdigest(&ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), ScanCtx.index.desc.id);
+        desc->id = (int) ScanCtx.index.desc.timestamp;
 
         database_initialize(db);
         database_open(db);
@@ -75,14 +75,11 @@ void database_scan_begin(scan_args_t *args) {
     }
 
     database_increment_version(db);
+    database_sync_mime_table(db);
 
     database_close(db, FALSE);
 }
 
-void write_thumbnail_callback(char *key, int num, void *buf, size_t buf_len) {
-    database_write_thumbnail(ProcData.index_db, key, num, buf, buf_len);
-}
-
 void log_callback(const char *filepath, int level, char *str) {
     if (level == LEVEL_FATAL) {
         sist_log(filepath, level, str);
@@ -140,7 +137,6 @@ void initialize_scan_context(scan_args_t *args) {
     // Comic
     ScanCtx.comic_ctx.log = log_callback;
     ScanCtx.comic_ctx.logf = logf_callback;
-    ScanCtx.comic_ctx.store = write_thumbnail_callback;
     ScanCtx.comic_ctx.enable_tn = args->tn_count > 0;
     ScanCtx.comic_ctx.tn_size = args->tn_size;
     ScanCtx.comic_ctx.tn_qscale = args->tn_quality;
@@ -157,7 +153,6 @@ void initialize_scan_context(scan_args_t *args) {
     }
     ScanCtx.ebook_ctx.log = log_callback;
     ScanCtx.ebook_ctx.logf = logf_callback;
-    ScanCtx.ebook_ctx.store = write_thumbnail_callback;
     ScanCtx.ebook_ctx.fast_epub_parse = args->fast_epub;
     ScanCtx.ebook_ctx.tn_qscale = args->tn_quality;
 
@@ -165,7 +160,6 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.font_ctx.enable_tn = args->tn_count > 0;
     ScanCtx.font_ctx.log = log_callback;
     ScanCtx.font_ctx.logf = logf_callback;
-    ScanCtx.font_ctx.store = write_thumbnail_callback;
 
     // Media
     ScanCtx.media_ctx.tn_qscale = args->tn_quality;
@@ -173,7 +167,6 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.media_ctx.tn_count = args->tn_count;
     ScanCtx.media_ctx.log = log_callback;
     ScanCtx.media_ctx.logf = logf_callback;
-    ScanCtx.media_ctx.store = write_thumbnail_callback;
     ScanCtx.media_ctx.max_media_buffer = (long) args->max_memory_buffer_mib * 1024 * 1024;
     ScanCtx.media_ctx.read_subtitles = args->read_subtitles;
     ScanCtx.media_ctx.read_subtitles = args->tn_count;
@@ -189,13 +182,11 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.ooxml_ctx.content_size = args->content_size;
     ScanCtx.ooxml_ctx.log = log_callback;
     ScanCtx.ooxml_ctx.logf = logf_callback;
-    ScanCtx.ooxml_ctx.store = write_thumbnail_callback;
 
     // MOBI
     ScanCtx.mobi_ctx.content_size = args->content_size;
     ScanCtx.mobi_ctx.log = log_callback;
     ScanCtx.mobi_ctx.logf = logf_callback;
-    ScanCtx.mobi_ctx.store = write_thumbnail_callback;
     ScanCtx.mobi_ctx.enable_tn = args->tn_count > 0;
     ScanCtx.mobi_ctx.tn_size = args->tn_size;
     ScanCtx.mobi_ctx.tn_qscale = args->tn_quality;
@@ -209,7 +200,6 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.msdoc_ctx.content_size = args->content_size;
     ScanCtx.msdoc_ctx.log = log_callback;
     ScanCtx.msdoc_ctx.logf = logf_callback;
-    ScanCtx.msdoc_ctx.store = write_thumbnail_callback;
     ScanCtx.msdoc_ctx.msdoc_mime = mime_get_mime_by_string("application/msword");
 
     ScanCtx.threads = args->threads;
@@ -228,7 +218,6 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.raw_ctx.tn_size = args->tn_size;
     ScanCtx.raw_ctx.log = log_callback;
     ScanCtx.raw_ctx.logf = logf_callback;
-    ScanCtx.raw_ctx.store = write_thumbnail_callback;
 
     // Wpd
     ScanCtx.wpd_ctx.content_size = args->content_size;
@@ -251,6 +240,13 @@ void sist2_scan(scan_args_t *args) {
 
     LOG_INFOF("main.c", "sist2 v%s", Version);
 
+    ScanCtx.ignorelist = ignorelist_create();
+
+    char ignore_filepath[PATH_MAX];
+    sprintf(ignore_filepath, "%s.sist2ignore", args->path);
+
+    ignorelist_load_ignore_file(ScanCtx.ignorelist, ignore_filepath);
+
     ScanCtx.pool = tpool_create(ScanCtx.threads, TRUE);
     tpool_start(ScanCtx.pool);
 
@@ -271,9 +267,6 @@ void sist2_scan(scan_args_t *args) {
     tpool_wait(ScanCtx.pool);
     tpool_destroy(ScanCtx.pool);
 
-    LOG_DEBUGF("main.c", "Thumbnail store size: %lu", ScanCtx.stat_tn_size);
-    LOG_DEBUGF("main.c", "Index size: %lu", ScanCtx.stat_index_size);
-
     database_t *db = database_create(args->output, INDEX_DATABASE);
     database_open(db);
 
@@ -283,6 +276,7 @@ void sist2_scan(scan_args_t *args) {
 
     database_generate_stats(db, args->treemap_threshold);
     database_close(db, args->optimize_database);
+    ignorelist_destroy(ScanCtx.ignorelist);
 }
 
 void sist2_index(index_args_t *args) {
@@ -316,16 +310,15 @@ void sist2_index(index_args_t *args) {
     database_open(db);
     database_iterator_t *iterator = database_create_document_iterator(db);
     database_document_iter_foreach(json, iterator) {
-        char doc_id[SIST_DOC_ID_LEN];
+        char sid[SIST_SID_LEN];
-        strcpy(doc_id, cJSON_GetObjectItem(json, "_id")->valuestring);
+        int doc_id = cJSON_GetObjectItem(json, "_id")->valueint;
         cJSON_DeleteItemFromObject(json, "_id");
-
+        format_sid(sid, desc->id, doc_id);
-        // TODO: delete tag if empty
 
         if (args->print) {
-            print_json(json, doc_id);
+            print_json(json, sid);
         } else {
-            index_json(json, doc_id);
+            index_json(json, sid);
             cnt += 1;
         }
         cJSON_Delete(json);
@@ -334,10 +327,12 @@ void sist2_index(index_args_t *args) {
     free(iterator);
 
     if (!args->print) {
+        char sid[SIST_SID_LEN];
+
         database_iterator_t *del_iter = database_create_delete_list_iterator(db);
-        database_delete_list_iter_foreach(id, del_iter) {
+        database_delete_list_iter_foreach(doc_id, del_iter) {
-            delete_document(id);
+            format_sid(sid, desc->id, doc_id);
-            free(id);
+            delete_document(sid);
         }
         free(del_iter);
     }
@@ -366,7 +361,6 @@ void sist2_sqlite_index(sqlite_index_args_t *args) {
     database_fts_optimize(db);
 
     database_close(db, FALSE);
-    database_close(search_db, FALSE);
 }
 
 void sist2_web(web_args_t *args) {
@@ -439,6 +433,8 @@ int set_to_negative_if_value_is_zero(UNUSED(struct argparse *self), const struct
         fprintf(stderr, "error: option `--%s` Value must be >= 0\n", option->long_name);
         exit(1);
     }
+
+    return 0;
 }
 
 int main(int argc, const char *argv[]) {
@@ -533,7 +529,8 @@ int main(int argc, const char *argv[]) {
             OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings."),
 
             OPT_GROUP("sqlite-index options"),
-            OPT_STRING(0, "search-index", &common_search_index, "Path to search index. Will be created if it does not exist yet."),
+            OPT_STRING(0, "search-index", &common_search_index,
+                       "Path to search index. Will be created if it does not exist yet."),
 
             OPT_GROUP("Web options"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT: http://localhost:9200"),
@@ -557,7 +554,7 @@ int main(int argc, const char *argv[]) {
             OPT_END(),
     };
 
-    struct argparse argparse;
+    struct argparse argparse = {};
     argparse_init(&argparse, options, usage, 0);
     argparse_describe(
             &argparse,

src/parsing/mime.h

@@ -61,4 +61,6 @@ unsigned int mime_get_mime_by_ext(const char *ext);
 
 unsigned int mime_get_mime_by_string(const char *str);
 
+unsigned int* get_mime_ids();
+
 #endif

src/parsing/mime_generated.c

@@ -295,167 +295,167 @@ image_g3fax=524575,
 image_gif=524576,
 image_heic=524577,
 image_ief=524578,
-image_jpeg=524579,
+image_jp2=524579,
-image_jutvision=524580,
+image_jpeg=524580,
-image_naplps=524581,
+image_jutvision=524581,
-image_pict=524582,
+image_naplps=524582,
-image_png=524583,
+image_pict=524583,
-image_svg=524584 | 0x80000000,
+image_png=524584,
-image_svg_xml=524585 | 0x80000000,
+image_svg=524585 | 0x80000000,
-image_tiff=524586,
+image_svg_xml=524586 | 0x80000000,
-image_vnd_adobe_photoshop=524587 | 0x80000000,
+image_tiff=524587,
-image_vnd_djvu=524588 | 0x80000000,
+image_vnd_adobe_photoshop=524588 | 0x80000000,
-image_vnd_fpx=524589,
+image_vnd_djvu=524589 | 0x80000000,
-image_vnd_microsoft_icon=524590,
+image_vnd_fpx=524590,
-image_vnd_rn_realflash=524591,
+image_vnd_microsoft_icon=524591,
-image_vnd_rn_realpix=524592,
+image_vnd_rn_realflash=524592,
-image_vnd_wap_wbmp=524593,
+image_vnd_rn_realpix=524593,
-image_vnd_xiff=524594,
+image_vnd_wap_wbmp=524594,
-image_webp=524595,
+image_vnd_xiff=524595,
-image_wmf=524596,
+image_webp=524596,
-image_x_3ds=524597,
+image_wmf=524597,
-image_x_adobe_dng=524598 | 0x00800000,
+image_x_3ds=524598,
-image_x_award_bioslogo=524599,
+image_x_adobe_dng=524599 | 0x00800000,
-image_x_canon_cr2=524600 | 0x00800000,
+image_x_award_bioslogo=524600,
-image_x_canon_crw=524601 | 0x00800000,
+image_x_canon_cr2=524601 | 0x00800000,
-image_x_cmu_raster=524602,
+image_x_canon_crw=524602 | 0x00800000,
-image_x_cur=524603,
+image_x_cmu_raster=524603,
-image_x_dcraw=524604 | 0x00800000,
+image_x_cur=524604,
-image_x_dwg=524605,
+image_x_dcraw=524605 | 0x00800000,
-image_x_eps=524606,
+image_x_dwg=524606,
-image_x_epson_erf=524607 | 0x00800000,
+image_x_eps=524607,
-image_x_exr=524608,
+image_x_epson_erf=524608 | 0x00800000,
-image_x_fuji_raf=524609 | 0x00800000,
+image_x_exr=524609,
-image_x_gem=524610,
+image_x_fuji_raf=524610 | 0x00800000,
-image_x_icns=524611,
+image_x_gem=524611,
-image_x_icon=524612 | 0x80000000,
+image_x_icns=524612,
-image_x_jg=524613,
+image_x_icon=524613 | 0x80000000,
-image_x_jps=524614,
+image_x_jg=524614,
-image_x_kodak_dcr=524615 | 0x00800000,
+image_x_jps=524615,
-image_x_kodak_k25=524616 | 0x00800000,
+image_x_kodak_dcr=524616 | 0x00800000,
-image_x_kodak_kdc=524617 | 0x00800000,
+image_x_kodak_k25=524617 | 0x00800000,
-image_x_minolta_mrw=524618 | 0x00800000,
+image_x_kodak_kdc=524618 | 0x00800000,
-image_x_ms_bmp=524619,
+image_x_minolta_mrw=524619 | 0x00800000,
-image_x_niff=524620,
+image_x_ms_bmp=524620,
-image_x_nikon_nef=524621 | 0x00800000,
+image_x_niff=524621,
-image_x_olympus_orf=524622 | 0x00800000,
+image_x_nikon_nef=524622 | 0x00800000,
-image_x_panasonic_raw=524623 | 0x00800000,
+image_x_olympus_orf=524623 | 0x00800000,
-image_x_pcx=524624,
+image_x_panasonic_raw=524624 | 0x00800000,
-image_x_pentax_pef=524625 | 0x00800000,
+image_x_pcx=524625,
-image_x_pict=524626,
+image_x_pentax_pef=524626 | 0x00800000,
-image_x_portable_bitmap=524627,
+image_x_pict=524627,
-image_x_portable_graymap=524628,
+image_x_portable_bitmap=524628,
-image_x_portable_pixmap=524629,
+image_x_portable_graymap=524629,
-image_x_quicktime=524630,
+image_x_portable_pixmap=524630,
-image_x_rgb=524631,
+image_x_quicktime=524631,
-image_x_sigma_x3f=524632 | 0x00800000,
+image_x_rgb=524632,
-image_x_sony_arw=524633 | 0x00800000,
+image_x_sigma_x3f=524633 | 0x00800000,
-image_x_sony_sr2=524634 | 0x00800000,
+image_x_sony_arw=524634 | 0x00800000,
-image_x_sony_srf=524635 | 0x00800000,
+image_x_sony_sr2=524635 | 0x00800000,
-image_x_tga=524636,
+image_x_sony_srf=524636 | 0x00800000,
-image_x_tiff=524637,
+image_x_tga=524637,
-image_x_win_bitmap=524638,
+image_x_tiff=524638,
-image_x_xcf=524639 | 0x80000000,
+image_x_win_bitmap=524639,
-image_x_xpixmap=524640 | 0x80000000,
+image_x_xcf=524640 | 0x80000000,
-image_x_xwindowdump=524641,
+image_x_xpixmap=524641 | 0x80000000,
-message_news=196962,
+image_x_xwindowdump=524642,
-message_rfc822=196963,
+message_news=196963,
-model_vnd_dwf=65892,
+message_rfc822=196964,
-model_vnd_gdl=65893,
+model_vnd_dwf=65893,
-model_vnd_gs_gdl=65894,
+model_vnd_gdl=65894,
-model_vrml=65895,
+model_vnd_gs_gdl=65895,
-model_x_pov=65896,
+model_vrml=65896,
-sist2_sidecar=2,
+model_x_pov=65897,
-text_PGP=590185,
+text_PGP=590186,
-text_asp=590186,
+text_asp=590187,
-text_css=590187,
+text_css=590188,
-text_csv=590188,
+text_csv=590189,
-text_html=590189 | 0x01000000,
+text_html=590190 | 0x01000000,
-text_javascript=590190,
+text_javascript=590191,
-text_mcf=590191,
+text_mcf=590192,
-text_pascal=590192,
+text_pascal=590193,
-text_plain=590193,
+text_plain=590194,
-text_richtext=590194,
+text_richtext=590195,
-text_rtf=590195,
+text_rtf=590196,
-text_scriplet=590196,
+text_scriplet=590197,
-text_tab_separated_values=590197,
+text_tab_separated_values=590198,
-text_troff=590198,
+text_troff=590199,
-text_uri_list=590199,
+text_uri_list=590200,
-text_vnd_abc=590200,
+text_vnd_abc=590201,
-text_vnd_fmi_flexstor=590201,
+text_vnd_fmi_flexstor=590202,
-text_vnd_wap_wml=590202,
+text_vnd_wap_wml=590203,
-text_vnd_wap_wmlscript=590203,
+text_vnd_wap_wmlscript=590204,
-text_webviewhtml=590204,
+text_webviewhtml=590205,
-text_x_Algol68=590205,
+text_x_Algol68=590206,
-text_x_asm=590206,
+text_x_asm=590207,
-text_x_audiosoft_intra=590207,
+text_x_audiosoft_intra=590208,
-text_x_awk=590208,
+text_x_awk=590209,
-text_x_bcpl=590209,
+text_x_bcpl=590210,
-text_x_c=590210,
+text_x_c=590211,
-text_x_c__=590211,
+text_x_c__=590212,
-text_x_component=590212,
+text_x_component=590213,
-text_x_diff=590213,
+text_x_diff=590214,
-text_x_fortran=590214,
+text_x_fortran=590215,
-text_x_java=590215,
+text_x_java=590216,
-text_x_la_asf=590216,
+text_x_la_asf=590217,
-text_x_lisp=590217,
+text_x_lisp=590218,
-text_x_m=590218,
+text_x_m=590219,
-text_x_m4=590219,
+text_x_m4=590220,
-text_x_makefile=590220,
+text_x_makefile=590221,
-text_x_ms_regedit=590221,
+text_x_ms_regedit=590222,
-text_x_msdos_batch=590222,
+text_x_msdos_batch=590223,
-text_x_objective_c=590223,
+text_x_objective_c=590224,
-text_x_pascal=590224,
+text_x_pascal=590225,
-text_x_perl=590225,
+text_x_perl=590226,
-text_x_php=590226,
+text_x_php=590227,
-text_x_po=590227,
+text_x_po=590228,
-text_x_python=590228,
+text_x_python=590229,
-text_x_ruby=590229,
+text_x_ruby=590230,
-text_x_sass=590230,
+text_x_sass=590231,
-text_x_script_python=590231,
+text_x_script_python=590232,
-text_x_scss=590232,
+text_x_scss=590233,
-text_x_server_parsed_html=590233,
+text_x_server_parsed_html=590234,
-text_x_setext=590234,
+text_x_setext=590235,
-text_x_sgml=590235 | 0x01000000,
+text_x_sgml=590236 | 0x01000000,
-text_x_shellscript=590236,
+text_x_shellscript=590237,
-text_x_speech=590237,
+text_x_speech=590238,
-text_x_tcl=590238,
+text_x_tcl=590239,
-text_x_tex=590239,
+text_x_tex=590240,
-text_x_uil=590240,
+text_x_uil=590241,
-text_x_uuencode=590241,
+text_x_uuencode=590242,
-text_x_vcalendar=590242,
+text_x_vcalendar=590243,
-text_x_vcard=590243,
+text_x_vcard=590244,
-text_xml=590244 | 0x01000000,
+text_xml=590245 | 0x01000000,
-video_MP2T=393637,
+video_MP2T=393638,
-video_animaflex=393638,
+video_animaflex=393639,
-video_avi=393639,
+video_avi=393640,
-video_avs_video=393640,
+video_avs_video=393641,
-video_mp4=393641,
+video_mp4=393642,
-video_mpeg=393642,
+video_mpeg=393643,
-video_quicktime=393643,
+video_quicktime=393644,
-video_vdo=393644,
+video_vdo=393645,
-video_vivo=393645,
+video_vivo=393646,
-video_vnd_rn_realvideo=393646,
+video_vnd_rn_realvideo=393647,
-video_vosaic=393647,
+video_vosaic=393648,
-video_webm=393648,
+video_webm=393649,
-video_x_amt_demorun=393649,
+video_x_amt_demorun=393650,
-video_x_amt_showrun=393650,
+video_x_amt_showrun=393651,
-video_x_atomic3d_feature=393651,
+video_x_atomic3d_feature=393652,
-video_x_dl=393652,
+video_x_dl=393653,
-video_x_dv=393653,
+video_x_dv=393654,
-video_x_fli=393654,
+video_x_fli=393655,
-video_x_flv=393655,
+video_x_flv=393656,
-video_x_isvideo=393656,
+video_x_isvideo=393657,
-video_x_jng=393657 | 0x80000000,
+video_x_jng=393658 | 0x80000000,
-video_x_m4v=393658,
+video_x_m4v=393659,
-video_x_matroska=393659,
+video_x_matroska=393660,
-video_x_mng=393660,
+video_x_mng=393661,
-video_x_motion_jpeg=393661,
+video_x_motion_jpeg=393662,
-video_x_ms_asf=393662,
+video_x_ms_asf=393663,
-video_x_msvideo=393663,
+video_x_msvideo=393664,
-video_x_qtc=393664,
+video_x_qtc=393665,
-video_x_sgi_movie=393665,
+video_x_sgi_movie=393666,
-x_epoc_x_sisx_app=721346,
+x_epoc_x_sisx_app=721347,
 };
 char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
 case application_x_matlab_data: return "application/x-matlab-data";
@@ -909,7 +909,7 @@ case image_x_sony_arw: return "image/x-sony-arw";
 case image_x_sony_sr2: return "image/x-sony-sr2";
 case image_x_sony_srf: return "image/x-sony-srf";
 case image_x_epson_erf: return "image/x-epson-erf";
-case sist2_sidecar: return "sist2/sidecar";
+case image_jp2: return "image/jp2";
 default: return NULL;}}
 unsigned int mime_extension_lookup(unsigned long extension_crc32) {switch (extension_crc32) {
 case 2495639202:return application_x_matlab_data;
@@ -1293,7 +1293,7 @@ case 1698465774:return image_x_sony_arw;
 case 2083014127:return image_x_sony_sr2;
 case 271503362:return image_x_sony_srf;
 case 142938048:return image_x_epson_erf;
-case 287571459:return sist2_sidecar;
+case 1575600018:return image_jp2;
 default: return 0;}}
 unsigned int mime_name_lookup(unsigned long mime_crc32) {switch (mime_crc32) {
 case 3272851765: return application_x_matlab_data;
@@ -1747,6 +1747,8 @@ case 3060720351: return image_x_sony_arw;
 case 2944016606: return image_x_sony_sr2;
 case 3279729971: return image_x_sony_srf;
 case 1665206815: return image_x_epson_erf;
-case 521139448: return sist2_sidecar;
+case 1849479005: return image_jp2;
 default: return 0;}}
+unsigned int mime_ids[] = {655530,655363,655364,655365,655366,655362,655361,655367,655368,655369,655370,655371,655372 | 0x40000000,655373,655374,655375,655376 | 0x08000000,655377,655378,655379,655380,655382,655381,655383,655384,655390,655385,655386,655387,655388,655389,655391,655392,655393,655394,655395 | 0x40000000,655396,655397,655398,655399,655400,655401,655402,655403,655404,655405,655406,655407,655408,655411,655412,655413,655414,655415,655416,655417,655418,655419 | 0x20000000,655421,655422,655423,655424,655425,655426,655427,655428,655429,655430,655431,655432 | 0x04000000,655433 | 0x04000000,655434 | 0x04000000,655435,655436,655437,655438,655439,655440,655441,655442,655443,655444,655445,655446 | 0x10000000,655447,655448,655449 | 0x10000000,655450,655451,655452,655453,655454,655455,655456,655457,655458,655459,655461 | 0x08000000,655460,655462,655463,655464,655465,655466,655467,655468,655469,655470,655471,655472,655473,655474,655475,655476,655477,655478,655479,655480,1,655481,655482,655483,655484,655485,655486,655487,655488,655489 | 0x20000000,655490,655491,655492,655493,655494,655495,655496,655497,655498,655499,655500,655501,655502,655503,655504,655505,655506,655507,655508,655509,655510,655511,655512,655513,655514,655515,655516,655517,655519,655518 | 0x08000000,655521,655520,655522 | 0x08000000,655523 | 0x08000000,655524 | 0x08000000,655525,655526,655527,655528,655529,655531,655532,655533,655534,655535,655599,655536 | 0x02000000,655409 | 0x02000000,655540,655537,655538,655539,655541,655542,655543,655544,655545,655546,655547,655548,655549,655550,655552,655551,655553,655554,655555,655556,655557,655558,655559,655560,655561,655562 | 0x10000000,655563,655564,655565,655566,655567,655569,655568,655570,655571,655572,655573,655574,655575,655576,655577,655578 | 0x10000000,655579,655580,655581,655583,655582,655584,655585,655586,655587,655588,655589,655590,655591,655592,655593,655594,655595 | 0x08000000,655596,655597 | 0x08000000,655600 | 0x10000000,655601,458994 | 0x80000000,458995,458996,458998,458997,458999,459000,459001,459002,459003,459004,459005,459006,459007,459008,459009,459010,459011,459012,459013,459014,459015,459016,459017,459018,459030,459019,459020,459021,459022,459023,459025,459024,459026,459027,459029 | 0x80000000,459028 | 0x80000000,327959 | 0x20000000,327960 | 0x20000000,327962 | 0x20000000,327961 | 0x20000000,524571,524572,524573,524574,524575,524576,524577,524578,524580,524581,524582,524583,524584,524585 | 0x80000000,524586 | 0x80000000,524587,524588 | 0x80000000,524589 | 0x80000000,524590,524591,524592,524593,524594,524595,524596,524597,524598,524600,524603,524604,524606,524607,524609,524611,524612,524613 | 0x80000000,524614,524615,524620,524621,524625,524627,524628,524629,524630,524631,524632,524637,524638,524639,524640 | 0x80000000,524641 | 0x80000000,524642,196963,196964,65893,65894,65895,65896,65897,590187,590188,590190 | 0x01000000,590191,590192,590193,590186,590194,590232,590189,655410,590195,590196,590197,590198,590199,590200,590201,590202,590204,590203,590205,590206,590207,590208,590209,590210,590211,590212,590213,590214,590215,590216,590217,590218,590220,590221,590245 | 0x01000000,590219,590223,590222,590224,590225,590226,590227,590228,590229,590230,590231,590233,590234,590235,590236 | 0x01000000,590237,590238,590239,590240,590241,590242,590243,590244,393639,393640,393641,393638,393642,393643,393644,393645,393646,393647,393648,393649,393650,393651,393652,393653,393654,393655,393656,393657,393658 | 0x80000000,393659,393660,393661,393662,393663,393664,393665,393666,721347,655598,655420,524623 | 0x00800000,524622 | 0x00800000,524610 | 0x00800000,524624 | 0x00800000,524599 | 0x00800000,524601 | 0x00800000,524602 | 0x00800000,524605 | 0x00800000,524616 | 0x00800000,524617 | 0x00800000,524618 | 0x00800000,524619 | 0x00800000,524626 | 0x00800000,524633 | 0x00800000,524634 | 0x00800000,524635 | 0x00800000,524636 | 0x00800000,524608 | 0x00800000,524579,0};
+unsigned int* get_mime_ids() { return mime_ids; }
 #endif

src/parsing/parse.c

@@ -4,10 +4,8 @@
 #include "src/ctx.h"
 #include "mime.h"
 #include "src/io/serialize.h"
-#include "src/parsing/sidecar.h"
 #include "src/parsing/fs_util.h"
 #include "src/parsing/magic_util.h"
-#include <pthread.h>
 
 
 #define MIN_VIDEO_SIZE (1024 * 64)
@@ -27,7 +25,6 @@ typedef enum {
     FILETYPE_OOXML,
     FILETYPE_COMIC,
     FILETYPE_MOBI,
-    FILETYPE_SIST2_SIDECAR,
     FILETYPE_MSDOC,
     FILETYPE_JSON,
     FILETYPE_NDJSON,
@@ -63,8 +60,6 @@ file_type_t get_file_type(unsigned int mime, size_t size, const char *filepath)
         return FILETYPE_COMIC;
     } else if (IS_MOBI(mime)) {
         return FILETYPE_MOBI;
-    } else if (mime == MIME_SIST2_SIDECAR) {
-        return FILETYPE_SIST2_SIDECAR;
     } else if (is_msdoc(&ScanCtx.msdoc_ctx, mime)) {
         return FILETYPE_MSDOC;
     } else if (is_json(&ScanCtx.json_ctx, mime)) {
@@ -147,6 +142,10 @@ void parse(parse_job_t *job) {
         job->vfile.calculate_checksum = ScanCtx.calculate_checksums;
     }
 
+    if (IS_SUB_JOB(job)) {
+        SET_CURRENT_JOB(ProcData.ipc_db->ipc_ctx, job->filepath);
+    }
+
     document_t *doc = malloc(sizeof(document_t));
 
     strcpy(doc->filepath, job->filepath);
@@ -157,7 +156,8 @@ void parse(parse_job_t *job) {
     doc->size = job->vfile.st_size;
     doc->mtime = MAX(job->vfile.mtime, 0);
     doc->mime = get_mime(job);
-    generate_doc_id(doc->filepath + ScanCtx.index.desc.root_len, doc->doc_id);
+    doc->thumbnail_count = 0;
+    strcpy(doc->parent, job->parent);
 
     if (doc->mime == GET_MIME_ERROR_FATAL) {
         CLOSE_FILE(job->vfile)
@@ -165,16 +165,13 @@ void parse(parse_job_t *job) {
         return;
     }
 
-    if (database_mark_document(ProcData.index_db, doc->doc_id, doc->mtime)) {
+    int document_exists = database_mark_document(ProcData.index_db, doc->filepath + ScanCtx.index.desc.root_len, doc->mtime);
+    if (document_exists) {
         CLOSE_FILE(job->vfile)
         free(doc);
         return;
     }
 
-    if (LogCtx.very_verbose) {
-        LOG_DEBUGF(job->filepath, "Starting parse job {%s}", doc->doc_id);
-    }
-
     switch (get_file_type(doc->mime, doc->size, doc->filepath)) {
         case FILETYPE_RAW:
             parse_raw(&ScanCtx.raw_ctx, &job->vfile, doc);
@@ -195,6 +192,10 @@ void parse(parse_job_t *job) {
             parse_font(&ScanCtx.font_ctx, &job->vfile, doc);
             break;
         case FILETYPE_ARCHIVE:
+
+            // Insert the document now so that the children documents can link to an existing ID
+            database_write_document(ProcData.index_db, doc, NULL);
+
             parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc, ScanCtx.exclude, ScanCtx.exclude_extra);
             break;
         case FILETYPE_OOXML:
@@ -206,11 +207,6 @@ void parse(parse_job_t *job) {
         case FILETYPE_MOBI:
             parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, doc);
             break;
-        case FILETYPE_SIST2_SIDECAR:
-            parse_sidecar(&job->vfile, doc);
-            CLOSE_FILE(job->vfile)
-            free(doc);
-            return;
         case FILETYPE_MSDOC:
             parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, doc);
             break;
@@ -225,14 +221,6 @@ void parse(parse_job_t *job) {
             break;
     }
 
-    //Parent meta
-    if (job->parent[0] != '\0') {
-        meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + SIST_INDEX_ID_LEN);
-        meta_parent->key = MetaParent;
-        strcpy(meta_parent->str_val, job->parent);
-        APPEND_META((doc), meta_parent);
-    }
-
     CLOSE_FILE(job->vfile)
 
     if (job->vfile.has_checksum) {

src/parsing/sidecar.c

@@ -1,40 +0,0 @@
-#include "sidecar.h"
-
-#include "src/ctx.h"
-
-void parse_sidecar(vfile_t *vfile, document_t *doc) {
-
-    LOG_DEBUGF("sidecar.c", "Parsing sidecar file %s", vfile->filepath);
-
-    size_t size;
-    char *buf = read_all(vfile, &size);
-    if (buf == NULL) {
-        LOG_ERRORF("sidecar.c", "Read error for %s", vfile->filepath);
-        return;
-    }
-
-    buf = realloc(buf, size + 1);
-    *(buf + size) = '\0';
-
-    cJSON *json = cJSON_Parse(buf);
-    if (json == NULL) {
-        LOG_ERRORF("sidecar.c", "Could not parse JSON sidecar %s", vfile->filepath);
-        return;
-    }
-    char *json_str = cJSON_PrintUnformatted(json);
-
-    char assoc_doc_id[SIST_DOC_ID_LEN];
-
-    char rel_path[PATH_MAX];
-    size_t rel_path_len = doc->ext - 1 - ScanCtx.index.desc.root_len;
-    memcpy(rel_path, vfile->filepath + ScanCtx.index.desc.root_len, rel_path_len);
-    *(rel_path + rel_path_len) = '\0';
-
-    generate_doc_id(rel_path, assoc_doc_id);
-
-    database_write_document_sidecar(ProcData.index_db, assoc_doc_id, json_str);
-
-    cJSON_Delete(json);
-    free(json_str);
-    free(buf);
-}

src/parsing/sidecar.h

@@ -1,8 +0,0 @@
-#ifndef SIST2_SIDECAR_H
-#define SIST2_SIDECAR_H
-
-#include "src/sist.h"
-
-void parse_sidecar(vfile_t *vfile, document_t *doc);
-
-#endif

src/sist.h

@@ -3,19 +3,19 @@
 
 #define _GNU_SOURCE
 
-#ifndef	FALSE
+#ifndef    FALSE
-#define	FALSE	(0)
+#define    FALSE    (0)
 #define BOOL int
 #endif
 
-#ifndef	TRUE
+#ifndef    TRUE
-#define	TRUE	(!FALSE)
+#define    TRUE    (!FALSE)
 #endif
 
-#undef	MAX
+#undef    MAX
 #define MAX(a, b)  (((a) > (b)) ? (a) : (b))
 
-#undef	MIN
+#undef    MIN
 #define MIN(a, b)  (((a) < (b)) ? (a) : (b))
 
 #ifndef PATH_MAX
@@ -23,7 +23,7 @@
 #endif
 
 #undef ABS
-#define ABS(a)	   (((a) < 0) ? -(a) : (a))
+#define ABS(a)       (((a) < 0) ? -(a) : (a))
 
 #define UNUSED(x) __attribute__((__unused__))  x
 
@@ -51,17 +51,17 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.2.0"
+#define VERSION "3.5.0"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
-static const int VersionMinor = 2;
+static const int VersionMinor = 5;
 static const int VersionPatch = 0;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown
 #endif
 
-#define EXPECTED_MONGOOSE_VERSION "7.7"
+#define EXPECTED_MONGOOSE_VERSION "7.16"
 
 #define Q(x) #x
 #define QUOTE(x) Q(x)