Update thumbnail-quality parameter in sist2-admin

SQLite search backend

.devcontainer/Dockerfile

@@ -0,0 +1,9 @@
+FROM simon987/sist2-build
+
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
+RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+

.devcontainer/devcontainer.json

@@ -0,0 +1,16 @@
+{
+  "name": "sist2-dev",
+  "dockerComposeFile": [
+    "docker-compose.yml"
+  ],
+  "service": "sist2-dev",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-vscode.cpptools-extension-pack"
+      ]
+    }
+  },
+  "remoteUser": "root",
+  "workspaceFolder": "/app/"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,8 @@
+version: "3"
+
+services:
+  sist2-dev:
+    build: .
+    command: sleep infinity
+    volumes:
+      - ../:/app

.dockerignore

@@ -15,7 +15,6 @@ Makefile
 **/*.cbp
 VERSION
 **/node_modules/
-.git/
 sist2-*-linux-debug
 sist2-*-linux
 sist2_debug
@@ -33,4 +32,9 @@ tmp_scan/
 Dockerfile
 Dockerfile.arm64
 docker-compose.yml
-state.db
+state.db
+*-journal
+build/
+__pycache__/
+sist2-vue/dist
+sist2-admin/frontend/dist

.gitattributes

@@ -1,3 +0,0 @@
-CMakeModules/* linguist-vendored
-**/*_generated.c linguist-vendored
-**/*_generated.h linguist-vendored

.gitignore

@@ -33,3 +33,16 @@ state.db
 *.pyc
 !sist2-admin/frontend/dist
 *.js.map
+sist2-vue/dist
+sist2-admin/frontend/dist
+.ninja_deps
+.ninja_log
+build.ninja
+src/web/static_generated.c
+src/magic_generated.c
+src/index/static_generated.c
+*.sist2
+*-shm
+*-journal
+.vscode
+*.fts

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "third-party/libscan/third-party/libmobi"]
 	path = third-party/libscan/third-party/libmobi
 	url = https://github.com/bfabiszewski/libmobi
+[submodule "third-party/libscan/libscan-test-files"]
+	path = third-party/libscan/libscan-test-files
+	url = https://github.com/simon987/libscan-test-files

CMakeLists.txt

@@ -5,7 +5,7 @@ set(CMAKE_C_STANDARD 11)
 
 option(SIST_DEBUG "Build a debug executable" on)
 option(SIST_FAST "Enable more optimisation flags" off)
-option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0)
+option(SIST_DEBUG_INFO "Turn on debug information in web interface" on)
 
 add_compile_definitions(
         "SIST_PLATFORM=${SIST_PLATFORM}"
@@ -15,36 +15,55 @@ if (SIST_DEBUG)
     add_compile_definitions(
             "SIST_DEBUG=${SIST_DEBUG}"
     )
-endif()
+    set(VCPKG_BUILD_TYPE debug)
+else ()
+    set(VCPKG_BUILD_TYPE release)
+endif ()
+
+if (SIST_DEBUG_INFO)
+    add_compile_definitions(
+            "SIST_DEBUG_INFO=${SIST_DEBUG_INFO}"
+    )
+endif ()
+
 
 add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
-add_executable(sist2
+add_executable(
+        sist2
+        # argparse
+        third-party/argparse/argparse.h third-party/argparse/argparse.c
+
         src/main.c
         src/sist.h
         src/io/walk.h src/io/walk.c
-        src/io/store.h src/io/store.c
         src/tpool.h src/tpool.c
         src/parsing/parse.h src/parsing/parse.c
+        src/parsing/magic_util.c src/parsing/magic_util.h
         src/io/serialize.h src/io/serialize.c
         src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
         src/index/web.c src/index/web.h
         src/web/serve.c src/web/serve.h
+        src/web/web_util.c src/web/web_util.h
         src/index/elastic.c src/index/elastic.h
         src/util.c src/util.h
-        src/ctx.h src/types.h
+        src/ctx.c src/ctx.h
+        src/types.h
         src/log.c src/log.h
         src/cli.c src/cli.h
-        src/stats.c src/stats.h src/ctx.c
         src/parsing/sidecar.c src/parsing/sidecar.h
+        src/database/database.c src/database/database.h
+        src/parsing/fs_util.h
 
         src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp
 
-        # argparse
-        third-party/argparse/argparse.h third-party/argparse/argparse.c
-        )
+        src/database/database_stats.c
+        src/database/database_schema.c
+        src/database/database_fts.c
+        src/web/web_fts.c
+)
 set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
@@ -52,16 +71,11 @@ set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
 
 find_package(PkgConfig REQUIRED)
 
-pkg_search_module(GLIB REQUIRED glib-2.0)
-
-find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
-find_library(MAGIC_LIB
-        NAMES libmagic.so.1 magic
-        PATHS /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/
-)
+find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
+find_package(unofficial-sqlite3 CONFIG REQUIRED)
 
 
 target_include_directories(
@@ -70,7 +84,6 @@ target_include_directories(
         ${CMAKE_SOURCE_DIR}/third-party/utf8.h/
         ${CMAKE_SOURCE_DIR}/third-party/libscan/
         ${CMAKE_SOURCE_DIR}/
-        ${GLIB_INCLUDE_DIRS}
 )
 
 target_compile_options(
@@ -88,7 +101,7 @@ if (SIST_DEBUG)
             -fno-omit-frame-pointer
             -fsanitize=address
             -fno-inline
-#            -O2
+            #            -O2
     )
     target_link_options(
             sist2
@@ -118,8 +131,10 @@ else ()
             PRIVATE
 
             -Ofast
+#            -g
             -fno-stack-protector
             -fomit-frame-pointer
+            -w
     )
 endif ()
 
@@ -133,20 +148,16 @@ target_link_libraries(
         sist2
 
         z
-        lmdb
-        cjson
         argparse
-        ${GLIB_LDFLAGS}
         unofficial::mongoose::mongoose
         CURL::libcurl
 
         pthread
 
-        c
-
         scan
 
         ${MAGIC_LIB}
+        unofficial::sqlite3::sqlite3
 )
 
 add_custom_target(

Dockerfile

@@ -1,6 +1,11 @@
 FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
+RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /build/
 
 COPY scripts scripts
@@ -9,14 +14,17 @@ COPY CMakeLists.txt .
 COPY third-party third-party
 COPY src src
 COPY sist2-vue sist2-vue
+COPY sist2-admin sist2-admin
 
-RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
-RUN make -j$(nproc)
-RUN strip sist2 || mv sist2_debug sist2
+RUN cd sist2-vue/ && npm install && npm run build
+RUN cd sist2-admin/frontend/ && npm install && npm run build
+
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN cd build && make -j$(nproc)
+RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
 
-WORKDIR /root
 
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
@@ -24,23 +32,27 @@ ENV LC_ALL C.UTF-8
 ENTRYPOINT ["/root/sist2"]
 
 RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 libmagic1 python3  \
-    python3-pip git tesseract-ocr libpq-dev && rm -rf /var/lib/apt/lists/*
+    python3-pip git tesseract-ocr && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
-    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
-COPY --from=build /build/sist2 /root/sist2
+COPY --from=build /build/build/sist2 /root/sist2
 
 # sist2-admin
-COPY sist2-admin/requirements.txt sist2-admin/
-RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
-COPY sist2-admin/ sist2-admin/
+WORKDIR /root/sist2-admin
+COPY sist2-admin/requirements.txt /root/sist2-admin/
+RUN python3 -m pip install --no-cache -r /root/sist2-admin/requirements.txt
+COPY --from=build /build/sist2-admin/ /root/sist2-admin/

Dockerfile.arm64

@@ -3,26 +3,39 @@ MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 ADD . /build/
-RUN cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
-RUN make -j$(nproc)
-RUN strip sist2
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN cd build && make -j$(nproc)
+RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
-FROM --platform="linux/arm64/v8" ubuntu:20.04
+FROM --platform=linux/arm64/v8 ubuntu@sha256:537da24818633b45fcb65e5285a68c3ec1f3db25f5ae5476a7757bc8dfae92a3
 
-RUN apt update && apt install -y curl libasan5 && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /usr/share/tessdata && \
-    cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
+WORKDIR /root
 
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 
 ENTRYPOINT ["/root/sist2"]
 
-COPY --from=build /build/sist2 /root/sist2
+RUN apt update && apt install -y curl libasan5 libmagic1 tesseract-ocr python3-pip python3 git && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /usr/share/tessdata && \
+    cd /usr/share/tessdata/ && \
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+
+# sist2
+COPY --from=build /build/build/sist2 /root/sist2
+
+# sist2-admin
+COPY sist2-admin/requirements.txt sist2-admin/
+RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
+COPY --from=build /build/sist2-admin/ sist2-admin/

README.md

@@ -10,13 +10,13 @@ sist2 (Simple incremental search tool)
 
 *Warning: sist2 is in early development*
 
-![search panel](docs/sist2.png)
+![search panel](docs/sist2.gif)
 
 ## Features
 
 * Fast, low memory usage, multi-threaded
+* Manage & schedule scan jobs with simple web interface (Docker only)
 * Mobile-friendly Web interface
-* Portable (all its features are packaged in a single executable)
 * Extracts text and metadata from common file types \*
 * Generates thumbnails \*
 * Incremental scanning
@@ -24,47 +24,64 @@ sist2 (Simple incremental search tool)
 * Recursive scan inside archive files \*\*
 * OCR support with tesseract \*\*\*
 * Stats page & disk utilisation visualization
+* Named-entity recognition (client-side) \*\*\*\*
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
-\*\*\* See [OCR](#ocr)
-
-![stats](docs/stats.png)
+\*\*\* See [OCR](#ocr)    
+\*\*\*\* See [Named-Entity Recognition](#NER)
 
 ## Getting Started
 
-1. Have an Elasticsearch (>= 6.8.X, ideally >=7.14.0) instance running
-    1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
-    1. *(or)* Run using docker:
-        ```bash
-        docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.14.0
-        ```
-    1. *(or)* Run using docker-compose:
-        ```yaml
-          elasticsearch:
-            image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0
-            environment:
-              - discovery.type=single-node
-              - "ES_JAVA_OPTS=-Xms1G -Xmx2G"
-        ```
-1. Download sist2 executable
-    1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases). 
-Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x` *
-    2. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
-       recommended!)*
-    3. *(or)* `docker pull simon987/sist2:2.12.1-x64-linux`
+### Using Docker Compose *(Windows/Linux/Mac)*
 
-1. See [Usage guide](docs/USAGE.md)
+```yaml
+version: "3"
 
-\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
+services:
+  elasticsearch:
+    image: elasticsearch:7.17.9
+    restart: unless-stopped
+    environment:
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+  sist2-admin:
+    image: simon987/sist2:3.1.0-x64-linux
+    restart: unless-stopped
+    volumes:
+      - ./sist2-admin-data/:/sist2-admin/
+      - /:/host
+    ports:
+      - 4090:4090 # sist2
+      - 8080:8080 # sist2-admin
+    working_dir: /root/sist2-admin/
+    entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
+```
 
-## Example usage
+Navigate to http://localhost:8080/ to configure sist2-admin.
 
-See [Usage guide](docs/USAGE.md) for more details
+### Using the executable file *(Linux/WSL only)*
 
-1. Scan a directory: `sist2 scan ~/Documents -o ./docs_idx`
-1. Push index to Elasticsearch: `sist2 index ./docs_idx`
-1. Start web interface: `sist2 web ./docs_idx`
+1. Choose search backend (See [comparison](#search-backends)):
+    * **Elasticsearch**: have an Elasticsearch (version >= 6.8.X, ideally >=7.14.0) instance running
+        1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
+        2. *(or)* Run using docker:
+            ```bash
+            docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
+            ```
+    * **SQLite**: No installation required
+
+2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
+   Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
+3. See [usage guide](docs/USAGE.md) for command line usage.
+
+Example usage:
+
+1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
+2. Prepare search index:
+    * **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
+    * **SQLite**: `sist2 index --search-index ./search.sist2 ./documents.sist2`
+3. Start web interface: `sist2 web ./documents.sist2`
 
 ## Format support
 
@@ -81,8 +98,8 @@ See [Usage guide](docs/USAGE.md) for more details
 | html, xml                                                                 | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
 | tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
 | docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
-| doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | yes         | author, title                                                                                                                          |
-| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | no          | author, title                                                                                                                          |
+| doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
+| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | yes         | author, title                                                                                                                          |
 | wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
 | json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
@@ -109,10 +126,10 @@ Download the language data files with your package manager (`apt install tessera
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
 The `simon987/sist2` image comes with common languages
-(hin, jpn, eng, fra, rus, spa) pre-installed.
+(hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
-name must be identical to the `*.traineddata` file installed on your system 
+name must be identical to the `*.traineddata` file installed on your system
 (use `chi_sim` rather than `chi-sim`).
 
 Examples:
@@ -123,39 +140,80 @@ sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
 sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
 ```
 
+### Search backends
+
+sist2 v3.0.7+ supports SQLite search backend. The SQLite search backend has
+fewer features and generally comparable query performance for medium-size
+indices, but it uses much less memory and is easier to set up.
+
+|                                              |                  SQLite                  |                                                             Elasticsearch                                                             |
+|----------------------------------------------|:----------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------:|
+| Requires separate search engine installation |                                          |                                                                   ✓                                                                   |
+| Memory footprint                             |                  ~20MB                   |                                                                >500MB                                                                 |
+| Query syntax                                 | [fts5](https://www.sqlite.org/fts5.html) | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
+| Fuzzy search                                 |                                          |                                                                   ✓                                                                   |
+| Media Types tree real-time updating          |                                          |                                                                   ✓                                                                   |
+| Search in file `path`                        |                                          |                                                                   ✓                                                                   |
+| Manual tagging                               |                    ✓                     |                                                                   ✓                                                                   |
+| User scripts                                 |                                          |                                                                   ✓                                                                   |
+| Media Type breakdown for search results      |                                          |                                                                   ✓                                                                   |
+
+### NER
+
+sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to
+**Configuration** > **Machine learning options** > **Model repositories**
+to enable it.
+
+The text processing is done in your browser, no data is sent to any third-party services.
+See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) for more details.
+
+#### List of available repositories:
+
+| URL                                                                                                     | Maintainer                              | Purpose |
+|---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
+| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
+
+<details>
+  <summary>Screenshot</summary>
+
+![ner](docs/ner.png)
+
+</details>
+
 ## Build from source
 
 You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries
 
-### With docker (recommended)
+### Using docker
 
 ```bash
 git clone --recursive https://github.com/simon987/sist2/
 cd sist2
-docker build . -f ./Dockerfile -t my-sist2-image
+docker build . -t my-sist2-image
+# Copy sist2 executable from docker image
 docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 ```
 
-### On a linux computer
+### Using a linux computer
 
 1. Install compile-time dependencies
 
    ```bash
-   apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git
+   apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
    ```
 
-1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
-
-1. Install vcpkg dependencies
+2. Install vcpkg using my fork: https://github.com/simon987/vcpkg
+3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl]
-    vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw jasper lcms gumbo
+    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp]
     ```
 
-1. Build
+4. Build
     ```bash
     git clone --recursive https://github.com/simon987/sist2/
+    (cd sist2-vue; npm install; npm run build)
+    (cd sist2-admin/frontend; npm install; npm run build)
     cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

contrib/systemd/sist2-update-files.sh

@@ -12,7 +12,7 @@ REWRITE_URL=""
 sist2 scan \
   --threads 14 \
   --mem-throttle 32768 \
-  --quality 1.0 \
+  --thumbnail-quality 2 \
   --name $NAME \
   --ocr-lang=eng+chi_sim \
   --ocr-ebooks \

contrib/systemd/sist2-update-nextcloud.sh

@@ -12,7 +12,7 @@ REWRITE_URL=""
 sist2 scan \
   --threads 14 \
   --mem-throttle 32768 \
-  --quality 1.0 \
+  --thumbnail-quality 2 \
   --name $NAME \
   --ocr-lang=eng+chi_sim \
   --ocr-ebooks \

docker-compose.yml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   elasticsearch:
-    image: elasticsearch:7.14.0
+    image: elasticsearch:7.17.9
     container_name: sist2-es
     environment:
       - "discovery.type=single-node"
@@ -15,9 +15,9 @@ services:
       - /mnt/array/sist2-admin-data/:/sist2-admin/
       - /:/host
     ports:
+      - 4090:4090
       # NOTE: Don't export this port publicly!
       - 8080:8080
-      - 4090:4090
     working_dir: /root/sist2-admin/
     entrypoint: python3
     command:

docs/USAGE.md

@@ -1,78 +1,69 @@
 # Usage
 
-*More examples (specifically with docker/compose) are in progress*
-
-* [scan](#scan)
-    * [options](#scan-options)
-    * [examples](#scan-examples)
-    * [index format](#index-format)
-* [index](#index)
-    * [options](#index-options)
-    * [examples](#index-examples)
-* [web](#web)
-    * [options](#web-options)
-    * [examples](#web-examples)
-    * [rewrite_url](#rewrite_url)
-* [elasticsearch](#elasticsearch)
-* [exec-script](#exec-script)
-* [tagging](#tagging)
-* [sidecar files](#sidecar-files)
-
 ```
 Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
+   or: sist2 sqlite-index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
    or: sist2 exec-script [OPTION]... INDEX
+
 Lightning-fast file system indexer and search tool.
 
     -h, --help                        show this help message and exit
-    -v, --version                     Show version and exit
-    --verbose                         Turn on logging
-    --very-verbose                    Turn on debug messages
+    -v, --version                     Print version and exit.
+    --verbose                         Turn on logging.
+    --very-verbose                    Turn on debug messages.
+    --json-logs                       Output logs in JSON format.
 
 Scan options
-    -t, --threads=<int>               Number of threads. DEFAULT=1
-    --mem-throttle=<int>              Total memory threshold in MiB for scan throttling. DEFAULT=0
-    -q, --thumbnail-quality=<flt>     Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=1
-    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT=500
-    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT=1
-    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT=32768
-    --incremental=<str>               Reuse an existing index and only scan modified files.
-    -o, --output=<str>                Output directory. DEFAULT=index.sist2/
+    -t, --threads=<int>               Number of threads. DEFAULT: 1
+    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
+    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
+    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
+    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
+    -o, --output=<str>                Output index file path. DEFAULT: index.sist2
+    --incremental                     If the output file path exists, only scan new or modified files.
+    --optimize-index                  Defragment index file after scan to reduce its file size.
     --rewrite-url=<str>               Serve files from this url instead of from disk.
-    --name=<str>                      Index display name. DEFAULT: (name of the directory)
+    --name=<str>                      Index display name. DEFAULT: index
     --depth=<int>                     Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
+    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: don't scan, list: only save file names as text, shallow: don't scan archives inside archives. DEFAULT: recurse
     --archive-passphrase=<str>        Passphrase for encrypted archive files
     --ocr-lang=<str>                  Tesseract language (use 'tesseract --list-langs' to see which are installed on your machine)
     --ocr-images                      Enable OCR'ing of image files.
     --ocr-ebooks                      Enable OCR'ing of ebook files.
-    -e, --exclude=<str>               Files that match this regex will not be scanned
-    --fast                            Only index file names & mime type
+    -e, --exclude=<str>               Files that match this regex will not be scanned.
+    --fast                            Only index file names & mime type.
     --treemap-threshold=<str>         Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
     --mem-buffer=<int>                Maximum memory buffer size per thread in MiB for files inside archives (see USAGE.md). DEFAULT: 2000
     --read-subtitles                  Read subtitles from media files.
-    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata)
+    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata).
     --checksums                       Calculate file checksums when scanning.
     --list-file=<str>                 Specify a list of newline-delimited paths to be scanned instead of normal directory traversal. Use '-' to read from stdin.
 
 Index options
-    -t, --threads=<int>               Number of threads. DEFAULT=1
-    --es-url=<str>                    Elasticsearch url with port. DEFAULT=http://localhost:9200
-    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
-    -p, --print                       Just print JSON documents to stdout.
-    --incremental-index               Conduct incremental indexing, assumes that the old index is already digested by Elasticsearch.
+    -t, --threads=<int>               Number of threads. DEFAULT: 1
+    --es-url=<str>                    Elasticsearch url with port. DEFAULT: http://localhost:9200
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
+    -p, --print                       Print JSON documents to stdout instead of indexing to elasticsearch.
+    --incremental-index               Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
     --script-file=<str>               Path to user script.
     --mappings-file=<str>             Path to Elasticsearch mappings.
     --settings-file=<str>             Path to Elasticsearch settings.
     --async-script                    Execute user script asynchronously.
-    --batch-size=<int>                Index batch size. DEFAULT: 100
-    -f, --force-reset                 Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
+    --batch-size=<int>                Index batch size. DEFAULT: 70
+    -f, --force-reset                 Reset Elasticsearch mappings and settings.
+
+sqlite-index options
+    --search-index=<str>              Path to search index. Will be created if it does not exist yet.
 
 Web options
-    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
-    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
-    --bind=<str>                      Listen on this address. DEFAULT=localhost:4090
+    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --search-index=<str>              Path to SQLite search index.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
+    --bind=<str>                      Listen for connections on this address. DEFAULT: localhost:4090
     --auth=<str>                      Basic auth in user:password format
     --auth0-audience=<str>            API audience/identifier
     --auth0-domain=<str>              Application domain
@@ -84,75 +75,23 @@ Web options
     --lang=<str>                      Default UI language. Can be changed by the user
 
 Exec-script options
-    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
-    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
     --script-file=<str>               Path to user script.
     --async-script                    Execute user script asynchronously.
+
 Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
-## Scan
+#### Thumbnail database size estimation
 
-### Scan options
+See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
 
-* `-t, --threads` 
-      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
-* `--mem-throttle`
-    Total memory threshold in MiB for scan throttling. Worker threads will not start a new parse job
-    until the total memory usage of sist2 is below this threshold. Set to 0 to disable. DEFAULT=0
-* `-q, --thumbnail-quality` 
-    Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best.
-* `--thumbnail-size` 
-    Thumbnail size in pixels.
-* `--thumbnail-count`
-    Maximum number of thumbnails to generate. When set to a value >= 2, thumbnails for video previews
-    will be generated. The actual number of thumbnails generated depends on the length of the video (maximum 1 image 
-    every ~7s). Set to 0 to completely disable thumbnails.
-* `--content-size` 
-    Number of bytes of text to be extracted from the content of files (plain text, PDFs etc.).
-    Repeated whitespace and special characters do not count toward this limit.
-    Set to 0 to completely disable content parsing.
-* `--incremental`
-    Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
-    will be copied to the new index and will not be parsed again.
-* `-o, --output` Output directory. 
-* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
-* `--name` Set the `name` option for the web module
-* `--depth` Maximum scan dept. Set to 0 only scan files directly in the root directory, set to -1 for infinite depth
-* `--archive` Archive file mode.
-    * skip: Don't parse
-    * list: Only get file names as text
-    * shallow: Don't parse archives inside archives.
-    * recurse: Scan archives recursively (default)
-* `--ocr-lang`, `--ocr-ebooks`, `--ocr-images` See [OCR](../README.md#OCR)
-* `-e, --exclude` Regex pattern to exclude files. A file is excluded if the pattern matches any 
-    part of the full absolute path.
-    
-    Examples: 
-    * `-e ".*\.ttf"`: Ignore ttf files
-    * `-e ".*\.(ttf|rar)"`: Ignore ttf and rar files
-    * `-e "^/mnt/backups/"`: Ignore all files in the `/mnt/backups/` directory
-    * `-e "^/mnt/Data[12]/"`: Ignore all files in the `/mnt/Data1/` and `/mnt/Data2/` directory
-    * `-e "(^/usr/)|(^/var/)|(^/media/DRIVE-A/tmp/)|(^/media/DRIVE-B/Trash/)"` Exclude the
-     `/usr`, `/var`, `/media/DRIVE-A/tmp`, `/media/DRIVE-B/Trash` directories
-* `--fast` Only index file names and mime type
-* `--treemap-threshold` Directories smaller than (`treemap-threshold` * `<total size of the index>`)
-    will not be considered for the disk utilisation visualization; their size will be added to
-    the parent directory. If the parent directory is still smaller than the threshold, it will also be "merged upwards"
-    and so on.
-    
-    In effect, smaller `treemap-threshold` values will yield a more detailed 
-    (but also a more cluttered and harder to read) visualization. 
-    
-* `--mem-buffer` Maximum memory buffer size in MiB (per thread) for files inside archives. Media files 
-    larger than this number will be read sequentially and no *seek* operations will be supported.
+For example, `--thumbnail-size=500`, `--thumbnail-quality=50` for a directory with 8 million images will create a thumbnail database 
+that is about `8000000 * 11.8kB = 94.4GB`.
 
-    To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
-* `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
-* `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
-* `--checksums` Calculate file checksums (SHA1) when scanning files. This option does not cause any additional read 
-  operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
-  files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
+![thumbnail_size](thumbnail_size.png)
 
 ### Scan examples
 
@@ -161,132 +100,70 @@ Simple scan
 sist2 scan ~/Documents
 
 sist2 scan \
-    --threads 4 --content-size 16000000 --quality 1.0 --archive shallow \
+    --threads 4 --content-size 16000000 --thumbnail-quality 2 --archive shallow \
     --name "My Documents" --rewrite-url "http://nas.domain.local/My Documents/" \
-    ~/Documents -o ./documents.idx/
+    ~/Documents -o ./documents.sist2
 ```
 
 Incremental scan
-```
-sist2 scan --incremental ./orig_idx/ -o ./updated_idx/ ~/Documents
-```
 
-### Index format
-
-A typical `ndjson` type index structure looks like this:
-```
-documents.idx/
-├── descriptor.json
-├── _index_main.ndjson.zst
-├── treemap.csv
-├── agg_mime.csv
-├── agg_date.csv
-├── add_size.csv
-├── thumbs/
-|   ├── data.mdb
-|   └── lock.mdb
-├── tags/
-|   ├── data.mdb
-|   └── lock.mdb
-└── meta/
-    ├── data.mdb
-    └── lock.mdb
-```
-
-The `_index_*.ndjson.zst` files contain the document data in JSON format, in a compressed newline-delemited file.
-
-The `thumbs/` folder is a [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database)
-database containing the thumbnails.
-
-The `descriptor.json` file contains general information about the index. The 
-following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rewrite_url) and `timestamp`.
-
-The `.csv` are pre-computed aggregations necessary for the stats page.
-
-*thumbs/*:
-
-LMDB key-value store. Keys are **binary** 16-byte md5 hash* (`_id` field)
-and values are raw image bytes.
-
-*\* Hash is calculated from the full path of the file, including the extension, relative to the index root*
-
-
-## Index
-### Index options
- * `--es-url` 
- Elasticsearch url and port. If you are using docker, make sure that both containers are on the
- same network.
- * `--es-index` 
-    Elasticsearch index name. DEFAULT=sist2
- * `-p, --print` 
-    Print index in JSON format to stdout.
- * `--incremental-index`
-   Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
-   Only the new changes since the last scan will be sent.
- * `--script-file` 
-    Path to user script. See [Scripting](scripting.md).
- * `--mappings-file`
-    Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used.
- * `--settings-file`
-    Path to custom Elasticsearch settings. *(See above)*
- * `--async-script` 
-    Use `wait_for_completion=false` elasticsearch option while executing user script.
-     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
- * `--batch-size=<int>` 
-    Index batch size. Indexing is generally faster with larger batches, but payloads that
-    are too large will fail and additional overhead for retrying with smaller sizes may slow
-    down the process.
- * `-f, --force-reset` 
-    Reset Elasticsearch mappings and settings.
- * `-t, --threads` Number of threads to use. Ideally, choose a number equal to the number of logical cores of the machine hosting Elasticsearch.
-    
-### Index examples
-
-**Push to elasticsearch**
+If the index file does not exist, `--incremental` has no effect.
 ```bash
-sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index/
-sist2 index ./my_index/
+sist scan ~/Documents -o ./documents.sist2
+sist scan ~/Documents -o ./documents.sist2 --incremental
+# or
+sist scan ~/Documents -o ./documents.sist2 --incremental
+sist scan ~/Documents -o ./documents.sist2 --incremental
+```
+
+### Index documents to Elasticsearch search backend
+
+```bash
+sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index.sist2
+sist2 index ./my_index.sist2
+```
+
+#### Index documents to SQLite search backend
+```bash
+# The search index will be created if it does not exist already
+sist2 sqlite-index ./index1.sist2 --search-index search.sist2
+sist2 sqlite-index ./index2.sist2 --search-index search.sist2
 ```
 
 **Save index in JSON format**
 ```bash
-sist2 index --print ./my_index/ > my_index.ndjson
+sist2 index --print ./my_index.sist2 > my_index.ndjson
 ```
 
 **Inspect contents of an index**
 ```bash
-sist2 index --print ./my_index/ | jq | less
+sist2 index --print ./my_index.sist2 | jq | less
 ```
 
 ## Web
 
-### Web options
- * `--es-url=<str>` Elasticsearch url.
- * `--es-index` 
-    Elasticsearch index name. DEFAULT=sist2
- * `--bind=<str>` Listen on this address.
- * `--auth=<str>` Basic auth in user:password format
- * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
-    `--auth` argument, but authentication is only applied the `/tag/` endpoint.
- * `--tagline=<str>` When specified, will replace the default tagline in the navbar.
- * `--dev` Serve html & js files from disk (for development, used to modify frontend files without having to recompile)
- * `--lang=<str>` Set the default web UI language (See #180 for a list of supported languages, default
-   is `en`). The user can change the language in the configuration page
- * `--auth0-audience`, `--auth0-domain`, `--auth0-client-id`, `--auth0-public-key-file` See [Authentication with Auth0](auth0.md)
-
 ### Web examples
 
-**Single index**
+**Single index (Elasticsearch backend)**
 ```bash
-sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index
+sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index.sist2
 ```
 
-**Multiple indices**
+**Multiple indices (Elasticsearch backend)**
 ```bash
 # Indices will be displayed in this order in the web interface
-sist2 web index1 index2 index3 index4
+sist2 web index1.sist2 index2.sist2 index3.sist2 index4.sist2
 ```
 
+**SQLite search backend**
+```bash
+sist2 web --search-index search.sist2 index1.sist2
+```
+
+#### Auth0 authentication
+
+See [auth0.md](auth0.md)
+
 ### rewrite_url
 
 When the `rewrite_url` field is not empty, the web module ignores the `root`
@@ -368,8 +245,8 @@ The sidecar file must have exactly the same file path and the `.s2meta` suffix.
 ```
 
 ```
-sist2 scan ~/Documents -o ./docs.idx
-sist2 index ./docs.idx
+sist2 scan ~/Documents -o ./docs.sist2
+sist2 index ./docs.sist2
 ```
 
 *NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,

scripts/before_build.sh

@@ -1,10 +1,13 @@
 #!/usr/bin/env bash
 
-rm -rf index.sist2/
+(
+  cd ..
+  rm -rf index.sist2
 
-python3 scripts/mime.py > src/parsing/mime_generated.c
-python3 scripts/serve_static.py > src/web/static_generated.c
-python3 scripts/index_static.py > src/index/static_generated.c
-python3 scripts/magic_static.py > src/magic_generated.c
+  python3 scripts/mime.py > src/parsing/mime_generated.c
+  python3 scripts/serve_static.py > src/web/static_generated.c
+  python3 scripts/index_static.py > src/index/static_generated.c
+  python3 scripts/magic_static.py > src/magic_generated.c
 
-printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h
+  printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h
+)

scripts/build.sh

@@ -4,14 +4,20 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
-rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
-make -j $(nproc)
-strip sist2
-./sist2 -v > VERSION
-mv sist2 sist2-x64-linux
+mkdir build
+(
+  cd build
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  make -j $(nproc)
+  strip sist2
+  ./sist2 -v > VERSION
+)
+mv build/sist2 sist2-x64-linux
 
-rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
-make -j  $(nproc)
-mv sist2_debug sist2-x64-linux-debug
+(
+  cd build
+  rm -rf CMakeFiles CMakeCache.txt
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  make -j  $(nproc)
+)
+mv build/sist2_debug sist2-x64-linux-debug

scripts/build_arm64.sh

@@ -4,14 +4,19 @@ VCPKG_ROOT="/vcpkg"
 
 git submodule update --init --recursive
 
-rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
-make -j $(nproc)
-strip sist2
-mv sist2 sist2-arm64-linux
+mkdir build
+(
+  cd build
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  make -j $(nproc)
+  strip sist2
+)
+mv build/sist2 sist2-arm64-linux
 
 rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
-make -j $(nproc)
-strip sist2
-mv sist2_debug sist2-arm64-linux-debug
+(
+  cd build
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  make -j $(nproc)
+)
+mv build/sist2_debug sist2-arm64-linux-debug

scripts/mime.csv

@@ -1,3 +1,4 @@
+application/x-matlab-data,mat
 application/arj, arj
 application/base64, mme
 application/binhex, hqx
@@ -29,7 +30,7 @@ application/mime, aps
 application/mspowerpoint, ppz
 application/msword, doc|dot|w6w|wiz|word
 application/netmc, mcp
-application/octet-stream, bin|dump|gpg
+application/octet-stream, bin|dump|gpg|pack|idx
 application/oda, oda
 application/ogg, ogv
 application/pdf, pdf
@@ -243,7 +244,7 @@ audio/make, funk|my|pfunk
 audio/midi, kar
 audio/mid, rmi
 audio/mp4, m4b
-audio/mpeg, m2a|mpa
+audio/mpeg, m2a|mpa|mpga
 audio/ogg, ogg
 audio/s3m, s3m
 audio/tsp-audio, tsi
@@ -346,6 +347,8 @@ text/mcf, mcf
 text/pascal, pas
 text/PGP,
 text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml|make|log|markdown|yaml
+text/x-script.python, pyx
+text/csv,
 application/vnd.coffeescript, coffee
 text/richtext, rt|rtf|rtx
 text/rtf,
@@ -382,7 +385,7 @@ text/x-pascal, p
 text/x-perl, pl
 text/x-php, php
 text/x-po, po
-text/x-python, py
+text/x-python, py|pyi
 text/x-ruby, rb
 text/x-sass, sass
 text/x-scss, scss

scripts/mime.py

@@ -1,3 +1,5 @@
+import zlib
+
 mimes = {}
 noparse = set()
 ext_in_hash = set()
@@ -135,24 +137,40 @@ def clean(t):
     return t.replace("/", "_").replace(".", "_").replace("+", "_").replace("-", "_")
 
 
+def crc(s):
+    return zlib.crc32(s.encode()) & 0xffffffff
+
+
 with open("scripts/mime.csv") as f:
     for l in f:
         mime, ext_list = l.split(",")
         if l.startswith("!"):
             mime = mime[1:]
             noparse.add(mime)
-        ext = [x.strip() for x in ext_list.split("|")]
+        ext = [x.strip() for x in ext_list.split("|") if x.strip() != ""]
         mimes[mime] = ext
 
+    seen_crc = set()
+    for ext in mimes.values():
+        for e in ext:
+            if crc(e) in seen_crc:
+                raise Exception("CRC32 collision")
+            seen_crc.add(crc(e))
+
+    seen_crc = set()
+    for mime in mimes.keys():
+        if crc(mime) in seen_crc:
+            raise Exception("CRC32 collision")
+        seen_crc.add(crc(mime))
+
     print("// **Generated by mime.py**")
     print("#ifndef MIME_GENERATED_C")
     print("#define MIME_GENERATED_C")
-    print("#include <glib.h>\n")
     print("#include <stdlib.h>\n")
     # Enum
     print("enum mime {")
     for mime, ext in sorted(mimes.items()):
-        print("    " + clean(mime) + "=" + mime_id(mime) + ",")
+        print(f"{clean(mime)}={mime_id(mime)},")
     print("};")
 
     # Enum -> string
@@ -163,20 +181,20 @@ with open("scripts/mime.csv") as f:
     print("default: return NULL;}}")
 
     # Ext -> Enum
-    print("GHashTable *mime_get_ext_table() {"
-          "GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);")
+    print("unsigned int mime_extension_lookup(unsigned long extension_crc32) {"
+          "switch (extension_crc32) {")
     for mime, ext in mimes.items():
-        for e in [e for e in ext if e]:
-            print("g_hash_table_insert(ext_table, \"" + e + "\", (gpointer)" + clean(mime) + ");")
-            if e in ext_in_hash:
-                raise Exception("extension already in hash: " + e)
-            ext_in_hash.add(e)
-    print("return ext_table;}")
+        if len(ext) > 0:
+            for e in ext:
+                print(f"case {crc(e)}:", end="")
+            print(f"return {clean(mime)};")
+    print("default: return 0;}}")
 
     # string -> Enum
-    print("GHashTable *mime_get_mime_table() {"
-          "GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);")
-    for mime, ext in mimes.items():
-        print("g_hash_table_insert(mime_table, \"" + mime + "\", (gpointer)" + clean(mime) + ");")
-    print("return mime_table;}")
+    print("unsigned int mime_name_lookup(unsigned long mime_crc32) {"
+          "switch (mime_crc32) {")
+    for mime in mimes.keys():
+        print(f"case {crc(mime)}: return {clean(mime)};")
+
+    print("default: return 0;}}")
     print("#endif")

scripts/sqlite_extension.c

@@ -0,0 +1,84 @@
+#include <sqlite3ext.h>
+#include <string.h>
+#include <stdlib.h>
+
+SQLITE_EXTENSION_INIT1
+
+static int sep_rfind(const char *str) {
+    for (int i = (int) strlen(str); i >= 0; i--) {
+        if (str[i] == '/') {
+            return i;
+        }
+    }
+    return -1;
+}
+
+void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    const char *value = (const char *) sqlite3_value_text(argv[0]);
+
+    int stop = sep_rfind(value);
+    if (stop == -1) {
+        sqlite3_result_null(ctx);
+        return;
+    }
+    char parent[4096 * 3];
+    strncpy(parent, value, stop);
+
+    sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
+}
+
+void random_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    char state_buf[32] = {0,};
+    struct random_data buf;
+    int result;
+
+    long seed = sqlite3_value_int64(argv[0]);
+
+    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
+
+    random_r(&buf, &result);
+
+    sqlite3_result_int(ctx, result);
+}
+
+
+int sqlite3_extension_init(
+        sqlite3 *db,
+        char **pzErrMsg,
+        const sqlite3_api_routines *pApi
+) {
+    SQLITE_EXTENSION_INIT2(pApi);
+
+
+    sqlite3_create_function(
+            db,
+            "path_parent",
+            1,
+            SQLITE_UTF8,
+            NULL,
+            path_parent_func,
+            NULL,
+            NULL
+    );
+
+    sqlite3_create_function(
+            db,
+            "random_seeded",
+            1,
+            SQLITE_UTF8,
+            NULL,
+            random_func,
+            NULL,
+            NULL
+    );
+
+    return SQLITE_OK;
+}

scripts/sqlite_extension_compile.sh

@@ -0,0 +1 @@
+gcc -I/mnt/work/vcpkg/installed/x64-linux/include -g -fPIC -shared sqlite_extension.c -o sist2funcs.so

scripts/start_dev_es.sh

@@ -1,3 +1,3 @@
 docker run --rm -it --name "sist2-dev-es"\
        	-p 9200:9200 -e "discovery.type=single-node" \
-	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.14.0
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:7.17.9

scripts/start_dev_es_8.sh

@@ -1,3 +1,3 @@
 docker run --rm -it --name "sist2-dev-es"\
        	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
-	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.1.2
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.7.0

sist2-admin/frontend/dist/css/app.css

@@ -1 +0,0 @@
-.navbar[data-v-27bc1d68]{box-shadow:0 .125rem .25rem rgba(0,0,0,.08)!important;border-radius:0}.theme-black .navbar[data-v-27bc1d68]{background:rgba(84,107,122,.18823529411764706);border-bottom:none}.navbar-brand[data-v-27bc1d68]{color:#222!important;font-size:1.75rem;padding:0}.navbar-brand[data-v-27bc1d68]:hover{color:#000!important}.version[data-v-27bc1d68]{color:#222!important;margin-left:-18px;margin-top:-14px;font-size:11px;font-family:monospace}.btn-link[data-v-27bc1d68]{color:#222}body,html{height:100%}#app{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;color:#2c3e50;padding-bottom:1em;min-height:100%}.info-icon{width:1rem;margin-right:.2rem;cursor:pointer;line-height:1rem;height:1rem;background-image:url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjYuNjY3IDQyNi42NjciIGZpbGw9IiNmZmYiPjxwYXRoIGQ9Ik0xOTIgMTkyaDQyLjY2N3YxMjhIMTkyeiIvPjxwYXRoIGQ9Ik0yMTMuMzMzIDBDOTUuNDY3IDAgMCA5NS40NjcgMCAyMTMuMzMzczk1LjQ2NyAyMTMuMzMzIDIxMy4zMzMgMjEzLjMzM1M0MjYuNjY3IDMzMS4yIDQyNi42NjcgMjEzLjMzMyAzMzEuMiAwIDIxMy4zMzMgMHptMCAzODRjLTk0LjA4IDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMgNDIuNjY3IDIxMy4zMzMgNDIuNjY3IDM4NCAxMTkuMjUzIDM4NCAyMTMuMzMzIDMwNy40MTMgMzg0IDIxMy4zMzMgMzg0eiIvPjxwYXRoIGQ9Ik0xOTIgMTA2LjY2N2g0Mi42Njd2NDIuNjY3SDE5MnoiLz48L3N2Zz4=);filter:brightness(45%);display:block}.tabs{margin-top:10px}.modal-title{text-overflow:ellipsis;overflow:hidden;white-space:nowrap}@media screen and (min-width:1500px){.container{max-width:1440px}}label{margin-top:.5rem;margin-bottom:0}.shrink[data-v-9b017c42]{flex-grow:inherit}#task-history[data-v-46960281]{font-family:monospace;font-size:12px}#log-tail-output span{display:block}span.DEBUG{color:#9e9e9e}span.WARNING{color:#ffb300}span.INFO{color:#039be5}span.ERROR,span.FATAL{color:#f4511e}span.ADMIN{color:#ee05ff}#log-tail-output{font-size:13px;font-family:monospace;padding:6px;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px;margin:3px;white-space:pre;color:#000;overflow:hidden}

sist2-admin/frontend/dist/index.html

@@ -1 +0,0 @@
-<!DOCTYPE html><html lang=""><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="favicon.ico"><title>sist2-admin</title><link href="css/app.css" rel="preload" as="style"><link href="css/chunk-vendors.css" rel="preload" as="style"><link href="js/app.js" rel="preload" as="script"><link href="js/chunk-vendors.js" rel="preload" as="script"><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/app.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but sist2-admin-vue doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="js/chunk-vendors.js"></script><script src="js/app.js"></script></body></html>

sist2-admin/frontend/package.json

@@ -20,14 +20,11 @@
   },
   "devDependencies": {
     "@vue/cli-plugin-babel": "~5.0.8",
-    "@vue/cli-plugin-eslint": "~5.0.8",
     "@vue/cli-plugin-router": "~5.0.8",
     "@vue/cli-plugin-vuex": "~5.0.8",
     "@vue/cli-service": "~5.0.8",
     "babel-eslint": "^10.1.0",
     "bootstrap": "^4.5.2",
-    "eslint": "^6.7.2",
-    "eslint-plugin-vue": "^6.2.2",
     "vue-template-compiler": "^2.6.11"
   },
   "eslintConfig": {

sist2-admin/frontend/public/index.html

@@ -4,7 +4,7 @@
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width,initial-scale=1.0">
-    <link rel="icon" href="<%= BASE_URL %>favicon.ico">
+    <link rel="icon" href="<%= BASE_URL %>serve_favicon_ico.ico">
     <title>sist2-admin</title>
   </head>
   <body>

sist2-admin/frontend/src/components/JobOptions.vue

@@ -28,16 +28,22 @@ export default {
       return this.$store.state.jobDesktopNotificationMap[this.job.name];
     }
   },
-  methods: {
+    mounted() {
+      this.cronValid = this.checkCron(this.job.cron_expression)
+    },
+    methods: {
+      checkCron(expression) {
+          return /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(expression);
+      },
     updateNotifications(value) {
       this.$store.dispatch("setJobDesktopNotification", {
         job: this.job.name,
         enabled: value
-      })
+      });
     },
     update() {
       if (this.job.schedule_enabled) {
-        this.cronValid = /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(this.job.cron_expression);
+        this.cronValid = this.checkCron(this.job.cron_expression);
       } else {
         this.cronValid = undefined;
       }

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -6,11 +6,8 @@
     <label>{{ $t("scanOptions.threads") }}</label>
     <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
 
-    <label>{{ $t("scanOptions.memThrottle") }}</label>
-    <b-form-input type="number" min="0" v-model="options.mem_throttle" @change="update()"></b-form-input>
-
     <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
-    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+    <b-form-input type="number" min="0" max="100" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailCount") }}</label>
     <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>
@@ -70,8 +67,9 @@
       {{ $t("scanOptions.readSubtitles") }}
     </b-form-checkbox>
 
-    <label>{{ $t("scanOptions.memBuffer") }}</label>
-    <b-form-input type="number" min="0" v-model="options.mem_buffer" @change="update()"></b-form-input>
+    <b-form-checkbox v-model="options.optimize_index" @change="update()">
+        {{ $t("scanOptions.optimizeIndex") }}
+    </b-form-checkbox>
 
     <label>{{ $t("scanOptions.treemapThreshold") }}</label>
     <b-form-input type="number" min="0" v-model="options.treemap_threshold" @change="update()"></b-form-input>

sist2-admin/frontend/src/i18n/messages.js

@@ -56,13 +56,17 @@ export default {
             tagline: "Tagline in navbar",
             auth: "Basic auth in user:password format",
             tagAuth: "Basic auth in user:password format for tagging",
+            auth0Audience: "Auth0 audience",
+            auth0Domain: "Auth0 domain",
+            auth0ClientId: "Auth0 client ID",
+            auth0PublicKey: "Auth0 public key",
         },
         scanOptions: {
             title: "Scanning options",
             path: "Path",
             threads: "Number of threads",
             memThrottle: "Total memory threshold in MiB for scan throttling",
-            thumbnailQuality: "Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best",
+            thumbnailQuality: "Thumbnail quality, on a scale of 0 to 100, 100 being the best",
             thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
             thumbnailSize: "Thumbnail size, in pixels",
             contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",
@@ -80,7 +84,8 @@ export default {
             checksums: "Calculate file checksums when scanning",
             readSubtitles: "Read subtitles from media files",
             memBuffer: "Maximum memory buffer size per thread in MiB for files inside archives",
-            treemapThreshold: "Relative size threshold for treemap"
+            treemapThreshold: "Relative size threshold for treemap",
+            optimizeIndex: "Defragment index file after scan to reduce its file size."
         },
         indexOptions: {
             title: "Indexing options",

sist2-admin/frontend/src/views/Tasks.vue

@@ -40,6 +40,39 @@ import TaskListItem from "@/components/TaskListItem";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import moment from "moment";
 
+const DAY = 3600 * 24;
+const HOUR = 3600;
+const MINUTE = 60;
+
+function humanDuration(sec_num) {
+  sec_num = sec_num / 1000;
+  const days = Math.floor(sec_num / DAY);
+  sec_num -= days * DAY;
+  const hours = Math.floor(sec_num / HOUR);
+  sec_num -= hours * HOUR;
+  const minutes = Math.floor(sec_num / MINUTE);
+  sec_num -= minutes * MINUTE;
+  const seconds = Math.floor(sec_num);
+
+  if (days > 0) {
+    return `${days} days ${hours}h ${minutes}m ${seconds}s`;
+  }
+
+  if (hours > 0) {
+    return `${hours}h ${minutes}m ${seconds}s`;
+  }
+
+  if (minutes > 0) {
+    return `${minutes}m ${seconds}s`;
+  }
+
+  if (seconds > 0) {
+    return `${seconds}s`;
+  }
+
+  return "<0s";
+}
+
 export default {
   name: 'Tasks',
   components: {TaskListItem},
@@ -100,17 +133,10 @@ export default {
       })
     },
     taskDuration(task) {
-      const start = moment(task.started);
-      const end = moment(task.ended);
+      const start = moment.utc(task.started);
+      const end = moment.utc(task.ended);
 
-      let duration = moment.utc(end.diff(start)).format("HH[h] mm[m] ss[s]");
-
-      duration = duration.replace("00h ", "");
-      duration = duration.replace(/^00m /, "");
-      duration = duration.replace(/00s/, "<1s");
-      duration = duration.replace(/^0/, "");
-
-      return duration;
+      return humanDuration(end.diff(start))
     }
   }
 }

sist2-admin/sist2_admin/app.py

@@ -21,13 +21,11 @@ from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
 from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
 from notifications import Subscribe, Notifications
 from sist2 import Sist2
-from state import PickleTable, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
+from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
 from web import Sist2Frontend
 
-VERSION = "1.0"
-
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
-db = PersistentState(table_factory=PickleTable, dbfile=os.path.join(DATA_FOLDER, "state.db"))
+db = PersistentState(dbfile=os.path.join(DATA_FOLDER, "state.db"))
 notifications = Notifications()
 task_queue = TaskQueue(sist2, db, notifications)
 
@@ -52,7 +50,6 @@ async def home():
 @app.get("/api")
 async def api():
     return {
-        "version": VERSION,
         "tesseract_langs": TESSERACT_LANGS,
         "logs_folder": LOG_FOLDER
     }
@@ -60,18 +57,17 @@ async def api():
 
 @app.get("/api/job/{name:str}")
 async def get_job(name: str):
-    row = db["jobs"][name]
-    if row:
-        return row["job"]
-    raise HTTPException(status_code=404)
+    job = db["jobs"][name]
+    if not job:
+        raise HTTPException(status_code=404)
+    return job
 
 
 @app.get("/api/frontend/{name:str}")
 async def get_frontend(name: str):
-    row = db["frontends"][name]
-    if row:
-        frontend = row["frontend"]
-        frontend: Sist2Frontend
+    frontend = db["frontends"][name]
+    frontend: Sist2Frontend
+    if frontend:
         frontend.running = frontend.name in RUNNING_FRONTENDS
         return frontend
     raise HTTPException(status_code=404)
@@ -79,16 +75,16 @@ async def get_frontend(name: str):
 
 @app.get("/api/job/")
 async def get_jobs():
-    return [row["job"] for row in db["jobs"]]
+    return list(db["jobs"])
 
 
 @app.put("/api/job/{name:str}")
-async def update_job(name: str, job: Sist2Job):
+async def update_job(name: str, new_job: Sist2Job):
     # TODO: Check etag
 
-    job.last_modified = datetime.now()
-    row = db["jobs"][name]
-    if not row:
+    new_job.last_modified = datetime.now()
+    job = db["jobs"][name]
+    if not job:
         raise HTTPException(status_code=404)
 
     args_that_trigger_full_scan = [
@@ -108,15 +104,15 @@ async def update_job(name: str, job: Sist2Job):
         "read_subtitles",
     ]
     for arg in args_that_trigger_full_scan:
-        if getattr(row["job"].scan_options, arg) != getattr(job.scan_options, arg):
-            job.do_full_scan = True
+        if getattr(new_job.scan_options, arg) != getattr(job.scan_options, arg):
+            new_job.do_full_scan = True
 
-    db["jobs"][name] = {"job": job}
+    db["jobs"][name] = new_job
 
 
 @app.put("/api/frontend/{name:str}")
 async def update_frontend(name: str, frontend: Sist2Frontend):
-    db["frontends"][name] = {"frontend": frontend}
+    db["frontends"][name] = frontend
 
     # TODO: Check etag
 
@@ -142,7 +138,7 @@ def _run_job(job: Sist2Job):
     job.last_modified = datetime.now()
     if job.status == JobStatus("created"):
         job.status = JobStatus("started")
-    db["jobs"][job.name] = {"job": job}
+    db["jobs"][job.name] = job
 
     scan_task = Sist2ScanTask(job, f"Scan [{job.name}]")
     index_task = Sist2IndexTask(job, f"Index [{job.name}]", depends_on=scan_task)
@@ -153,19 +149,19 @@ def _run_job(job: Sist2Job):
 
 @app.get("/api/job/{name:str}/run")
 async def run_job(name: str):
-    row = db["jobs"][name]
-    if not row:
+    job = db["jobs"][name]
+    if not job:
         raise HTTPException(status_code=404)
 
-    _run_job(row["job"])
+    _run_job(job)
 
     return "ok"
 
 
 @app.delete("/api/job/{name:str}")
 async def delete_job(name: str):
-    row = db["jobs"][name]
-    if row:
+    job = db["jobs"][name]
+    if job:
         del db["jobs"][name]
     else:
         raise HTTPException(status_code=404)
@@ -177,8 +173,8 @@ async def delete_frontend(name: str):
         os.kill(RUNNING_FRONTENDS[name], signal.SIGTERM)
         del RUNNING_FRONTENDS[name]
 
-    row = db["frontends"][name]
-    if row:
+    frontend = db["frontends"][name]
+    if frontend:
         del db["frontends"][name]
     else:
         raise HTTPException(status_code=404)
@@ -190,18 +186,18 @@ async def create_job(name: str):
         raise ValueError("Job with the same name already exists")
 
     job = Sist2Job.create_default(name)
-    db["jobs"][name] = {"job": job}
+    db["jobs"][name] = job
 
     return job
 
 
 @app.post("/api/frontend/{name:str}")
 async def create_frontend(name: str):
-    if db["frontend"][name]:
+    if db["frontends"][name]:
         raise ValueError("Frontend with the same name already exists")
 
     frontend = Sist2Frontend.create_default(name)
-    db["frontends"][name] = {"frontend": frontend}
+    db["frontends"][name] = frontend
 
     return frontend
 
@@ -255,7 +251,7 @@ def check_es_version(es_url: str, insecure: bool):
 
 
 def start_frontend_(frontend: Sist2Frontend):
-    frontend.web_options.indices = list(map(lambda j: db["jobs"][j]["job"].last_index, frontend.jobs))
+    frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
 
     pid = sist2.web(frontend.web_options, frontend.name)
     RUNNING_FRONTENDS[frontend.name] = pid
@@ -263,11 +259,11 @@ def start_frontend_(frontend: Sist2Frontend):
 
 @app.post("/api/frontend/{name:str}/start")
 async def start_frontend(name: str):
-    row = db["frontends"][name]
-    if not row:
+    frontend = db["frontends"][name]
+    if not frontend:
         raise HTTPException(status_code=404)
 
-    start_frontend_(row["frontend"])
+    start_frontend_(frontend)
 
 
 @app.post("/api/frontend/{name:str}/stop")
@@ -280,8 +276,7 @@ async def stop_frontend(name: str):
 @app.get("/api/frontend/")
 async def get_frontends():
     res = []
-    for row in db["frontends"]:
-        frontend = row["frontend"]
+    for frontend in db["frontends"]:
         frontend: Sist2Frontend
         frontend.running = frontend.name in RUNNING_FRONTENDS
         res.append(frontend)
@@ -364,14 +359,14 @@ def initialize_db():
     db["sist2_admin"]["info"] = {"version": DB_SCHEMA_VERSION}
 
     frontend = Sist2Frontend.create_default("default")
-    db["frontends"]["default"] = {"frontend": frontend}
+    db["frontends"]["default"] = frontend
 
     logger.info("Initialized database.")
 
 
 def start_frontends():
-    for row in db["frontends"]:
-        frontend: Sist2Frontend = row["frontend"]
+    for frontend in db["frontends"]:
+        frontend: Sist2Frontend
         if frontend.auto_start and len(frontend.jobs) > 0:
             start_frontend_(frontend)
 
@@ -380,8 +375,11 @@ if __name__ == '__main__':
 
     if not db["sist2_admin"]["info"]:
         initialize_db()
-    elif db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
-        print("Database has incompatible schema version! Delete state.db to continue.")
+    if db["sist2_admin"]["info"]["version"] == "1":
+        logger.info("Migrating to v2 database schema")
+        migrate_v1_to_v2(db)
+    if db["sist2_admin"]["info"]["version"] == "2":
+        logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.")
         exit(-1)
 
     start_frontends()

sist2-admin/sist2_admin/cron.py

@@ -10,7 +10,7 @@ from jobs import Sist2Job
 
 
 def _check_schedule(db: PersistentState, run_job):
-    for job in (row["job"] for row in db["jobs"]):
+    for job in db["jobs"]:
         job: Sist2Job
 
         if job.schedule_enabled:

sist2-admin/sist2_admin/jobs.py

@@ -1,23 +1,21 @@
 import json
 import logging
 import os.path
-import shutil
 import signal
 import uuid
 from datetime import datetime
 from enum import Enum
-from hashlib import md5
 from logging import FileHandler
 from threading import Lock, Thread
 from time import sleep
 from uuid import uuid4, UUID
 
 from hexlib.db import PersistentState
-from pydantic import BaseModel, validator
+from pydantic import BaseModel
 
 from config import logger, LOG_FOLDER
 from notifications import Notifications
-from sist2 import ScanOptions, IndexOptions, Sist2, Sist2Index
+from sist2 import ScanOptions, IndexOptions, Sist2
 from state import RUNNING_FRONTENDS
 from web import Sist2Frontend
 
@@ -38,7 +36,8 @@ class Sist2Job(BaseModel):
     schedule_enabled: bool = False
 
     previous_index: str = None
-    last_index: str = None
+    index_path: str = None
+    previous_index_path: str = None
     last_index_date: datetime = None
     status: JobStatus = JobStatus("created")
     last_modified: datetime
@@ -58,10 +57,10 @@ class Sist2Job(BaseModel):
             cron_expression="0 0 * * *"
         )
 
-    @validator("etag", always=True)
-    def validate_etag(cls, value, values):
-        s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
-        return md5(s.encode()).hexdigest()
+    # @validator("etag", always=True)
+    # def validate_etag(cls, value, values):
+    #     s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
+    #     return md5(s.encode()).hexdigest()
 
 
 class Sist2TaskProgress:
@@ -124,10 +123,10 @@ class Sist2ScanTask(Sist2Task):
 
         self.job.scan_options.name = self.job.name
 
-        if self.job.last_index and os.path.exists(self.job.last_index) and not self.job.do_full_scan:
-            self.job.scan_options.incremental = self.job.last_index
+        if self.job.index_path is not None and not self.job.do_full_scan:
+            self.job.scan_options.output = self.job.index_path
         else:
-            self.job.scan_options.incremental = None
+            self.job.scan_options.output = None
 
         def set_pid(pid):
             self.pid = pid
@@ -139,19 +138,26 @@ class Sist2ScanTask(Sist2Task):
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
             logger.info(f"Task {self.display_name} failed ({return_code})")
         else:
-            index = Sist2Index(self.job.scan_options.output)
-
-            # Save latest index
-            self.job.previous_index = self.job.last_index
-
-            self.job.last_index = index.path
+            self.job.index_path = self.job.scan_options.output
             self.job.last_index_date = datetime.now()
             self.job.do_full_scan = False
-            db["jobs"][self.job.name] = {"job": self.job}
-            self._logger.info(json.dumps({"sist2-admin": f"Save last_index={self.job.last_index}"}))
+            db["jobs"][self.job.name] = self.job
+            self._logger.info(json.dumps({"sist2-admin": f"Save last_index_date={self.job.last_index_date}"}))
 
         logger.info(f"Completed {self.display_name} ({return_code=})")
 
+        # Remove old index
+        if return_code == 0:
+            if self.job.previous_index_path is not None and self.job.previous_index_path != self.job.index_path:
+                self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index_path=}"}))
+                try:
+                    os.remove(self.job.previous_index_path)
+                except FileNotFoundError:
+                    pass
+
+            self.job.previous_index_path = self.job.index_path
+            db["jobs"][self.job.name] = self.job
+
         return return_code
 
 
@@ -173,19 +179,12 @@ class Sist2IndexTask(Sist2Task):
         ok = return_code == 0
 
         if ok:
-            # Remove old index
-            if self.job.previous_index is not None:
-                self._logger.info(json.dumps({"sist2-admin": f"Remove {self.job.previous_index=}"}))
-                try:
-                    shutil.rmtree(self.job.previous_index)
-                except FileNotFoundError:
-                    pass
-
             self.restart_running_frontends(db, sist2)
 
         # Update status
         self.job.status = JobStatus("indexed") if ok else JobStatus("failed")
-        db["jobs"][self.job.name] = {"job": self.job}
+        self.job.previous_index_path = self.job.index_path
+        db["jobs"][self.job.name] = self.job
 
         self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
 
@@ -195,16 +194,19 @@ class Sist2IndexTask(Sist2Task):
 
     def restart_running_frontends(self, db: PersistentState, sist2: Sist2):
         for frontend_name, pid in RUNNING_FRONTENDS.items():
-            frontend = db["frontends"][frontend_name]["frontend"]
+            frontend = db["frontends"][frontend_name]
             frontend: Sist2Frontend
 
-            os.kill(pid, signal.SIGTERM)
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
             try:
                 os.wait()
             except ChildProcessError:
                 pass
 
-            frontend.web_options.indices = map(lambda j: db["jobs"][j]["job"].last_index, frontend.jobs)
+            frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
 
             pid = sist2.web(frontend.web_options, frontend.name)
             RUNNING_FRONTENDS[frontend_name] = pid

sist2-admin/sist2_admin/sist2.py

@@ -2,7 +2,6 @@ import datetime
 import json
 import logging
 import os.path
-import traceback
 from datetime import datetime
 from io import TextIOWrapper
 from logging import FileHandler
@@ -47,7 +46,7 @@ class WebOptions(BaseModel):
         super().__init__(**kwargs)
 
     def args(self):
-        args = ["web", f"--es-url={self.es_url}", f"--bind={self.bind}",
+        args = ["web", f"--es-url={self.es_url}", f"--es-index={self.es_index}", f"--bind={self.bind}",
                 f"--tagline={self.tagline}", f"--lang={self.lang}"]
 
         if self.auth0_audience:
@@ -63,7 +62,7 @@ class WebOptions(BaseModel):
         if self.auth:
             args.append(f"--auth={self.auth}")
         if self.tag_auth:
-            args.append(f"--tag_auth={self.tag_auth}")
+            args.append(f"--tag-auth={self.tag_auth}")
         if self.dev:
             args.append(f"--dev")
 
@@ -78,10 +77,10 @@ class IndexOptions(BaseModel):
     es_url: str = "http://elasticsearch:9200"
     es_insecure_ssl: bool = False
     es_index: str = "sist2"
-    incremental_index: bool = False
+    incremental_index: bool = True
     script: str = ""
     script_file: str = None
-    batch_size: int = 100
+    batch_size: int = 70
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -110,15 +109,14 @@ ARCHIVE_RECURSE = "recurse"
 class ScanOptions(BaseModel):
     path: str
     threads: int = 1
-    mem_throttle: int = 0
-    thumbnail_quality: float = 1.0
-    thumbnail_size: int = 500
+    thumbnail_quality: int = 50
+    thumbnail_size: int = 552
     thumbnail_count: int = 1
     content_size: int = 32768
     depth: int = -1
     archive: str = ARCHIVE_RECURSE
     archive_passphrase: str = None
-    ocr_lang: bool = None
+    ocr_lang: str = None
     ocr_images: bool = False
     ocr_ebooks: bool = False
     exclude: str = None
@@ -128,7 +126,8 @@ class ScanOptions(BaseModel):
     read_subtitles: bool = False
     fast_epub: bool = False
     checksums: bool = False
-    incremental: str = None
+    incremental: bool = True
+    optimize_index: bool = False
     output: str = None
     name: str = None
     rewrite_url: str = None
@@ -138,13 +137,15 @@ class ScanOptions(BaseModel):
         super().__init__(**kwargs)
 
     def args(self):
-        args = ["scan", self.path, f"--threads={self.threads}", f"--mem-throttle={self.mem_throttle}",
-                f"--thumbnail-quality={self.thumbnail_quality}", f"--thumbnail-count={self.thumbnail_count}",
+        args = ["scan", self.path, f"--threads={self.threads}", f"--thumbnail-quality={self.thumbnail_quality}",
+                f"--thumbnail-count={self.thumbnail_count}", f"--thumbnail-size={self.thumbnail_size}",
                 f"--content-size={self.content_size}", f"--output={self.output}", f"--depth={self.depth}",
                 f"--archive={self.archive}", f"--mem-buffer={self.mem_buffer}"]
 
         if self.incremental:
-            args.append(f"--incremental={self.incremental}")
+            args.append(f"--incremental")
+        if self.optimize_index:
+            args.append(f"--optimize-index")
         if self.rewrite_url:
             args.append(f"--rewrite-url={self.rewrite_url}")
         if self.name:
@@ -234,11 +235,11 @@ class Sist2:
 
     def scan(self, options: ScanOptions, logs_cb, set_pid_cb):
 
-        output_dir = os.path.join(
-            self._data_dir,
-            f"scan-{datetime.now()}.sist2"
-        )
-        options.output = output_dir
+        if options.output is None:
+            options.output = os.path.join(
+                self._data_dir,
+                f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2"
+            )
 
         args = [
             self._bin_path,
@@ -277,23 +278,17 @@ class Sist2:
     @staticmethod
     def _consume_logs_stdout(logs_cb, proc):
         pipe_wrapper = TextIOWrapper(proc.stdout, encoding="utf8", errors="ignore")
-        try:
-            for line in pipe_wrapper:
+        for line in pipe_wrapper:
+            try:
                 if line.strip() == "":
                     continue
                 log_object = json.loads(line)
                 logs_cb(log_object)
-        except Exception as e:
-            proc.kill()
-            try:
-                print(line)
-            except NameError:
-                pass
-            print(traceback.format_exc())
-        finally:
-            pass
-            # proc.wait()
-            # pipe_wrapper.close()
+            except Exception as e:
+                try:
+                    logs_cb({"sist2-admin": f"Could not decode log line: {line}; {e}"})
+                except NameError:
+                    pass
 
     def web(self, options: WebOptions, name: str):
 

sist2-admin/sist2_admin/state.py

@@ -1,6 +1,7 @@
 from typing import Dict
+import shutil
 
-from hexlib.db import Table
+from hexlib.db import Table, PersistentState
 import pickle
 
 from tesseract import get_tesseract_langs
@@ -9,7 +10,7 @@ RUNNING_FRONTENDS: Dict[str, int] = {}
 
 TESSERACT_LANGS = get_tesseract_langs()
 
-DB_SCHEMA_VERSION = "1"
+DB_SCHEMA_VERSION = "3"
 
 from pydantic import BaseModel
 
@@ -48,3 +49,31 @@ class PickleTable(Table):
         for row in super().sql(where_clause, *params):
             yield dict((k, _deserialize(v)) for k, v in row.items())
 
+
+def migrate_v1_to_v2(db: PersistentState):
+
+    shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v2.bak")
+
+    # Frontends
+    db._table_factory = PickleTable
+    frontends = [row["frontend"] for row in db["frontends"]]
+    del db["frontends"]
+
+    db._table_factory = Table
+    for frontend in frontends:
+        db["frontends"][frontend.name] = frontend
+    list(db["frontends"])
+
+    # Jobs
+    db._table_factory = PickleTable
+    jobs = [row["job"] for row in db["jobs"]]
+    del db["jobs"]
+
+    db._table_factory = Table
+    for job in jobs:
+        db["jobs"][job.name] = job
+    list(db["jobs"])
+
+    db["sist2_admin"]["info"] = {
+        "version": "2"
+    }

sist2-vue/dist/index.html

@@ -1,3 +0,0 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"/><title>sist2</title><script defer="defer" src="js/chunk-vendors.js"></script><script defer="defer" src="js/index.js"></script><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/index.css" rel="stylesheet"></head><body><noscript><style>body {
-            height: initial;
-        }</style><div style="text-align: center; margin-top: 100px"><strong>We're sorry but sist2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong><br/><strong>Nous sommes désolés mais sist2 ne fonctionne pas correctement si JavaScript est activé. Veuillez l'activer pour continuer.</strong></div></noscript><div id="app"></div></body></html>

sist2-vue/package.json

@@ -9,13 +9,14 @@
   "dependencies": {
     "@auth0/auth0-spa-js": "^2.0.2",
     "@egjs/vue-infinitegrid": "3.3.0",
+    "@tensorflow/tfjs": "^4.4.0",
     "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "d3": "^5.6.1",
+    "d3": "^7.8.4",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",
-    "fslightbox-vue": "file:../../../../mnt/Hatchery/projects/sist2/fslightbox-vue-pro-1.3.1.tgz",
+    "fslightbox-vue": "fslightbox-vue.tgz",
     "nouislider": "^15.2.0",
     "underscore": "^1.13.1",
     "vue": "^2.6.12",

sist2-vue/public/index.html

@@ -19,12 +19,6 @@
             We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled.
             Please enable it to continue.
         </strong>
-        <br/>
-        <strong>
-            Nous sommes désolés mais <%= htmlWebpackPlugin.options.title %> ne fonctionne pas correctement
-            si JavaScript est activé.
-            Veuillez l'activer pour continuer.
-        </strong>
     </div>
 </noscript>
 <div id="app"></div>

sist2-vue/src/App.vue

@@ -1,383 +1,404 @@
 <template>
-  <div id="app" :class="getClass()" v-if="!authLoading">
-    <NavBar></NavBar>
-    <router-view v-if="!configLoading"/>
-  </div>
-  <div class="loading-page" v-else>
-    <div class="loading-spinners">
-      <b-spinner type="grow" variant="primary"></b-spinner>
-      <b-spinner type="grow" variant="primary"></b-spinner>
-      <b-spinner type="grow" variant="primary"></b-spinner>
+    <div id="app" :class="getClass()" v-if="!authLoading">
+        <NavBar></NavBar>
+        <router-view v-if="!configLoading"/>
     </div>
-    <div class="loading-text">
-      Loading • Chargement • 装载
+    <div class="loading-page" v-else>
+        <div class="loading-spinners">
+            <b-spinner type="grow" variant="primary"></b-spinner>
+            <b-spinner type="grow" variant="primary"></b-spinner>
+            <b-spinner type="grow" variant="primary"></b-spinner>
+        </div>
+        <div class="loading-text">
+            Loading • Chargement • 装载 • Wird geladen • Ładowanie
+        </div>
     </div>
-  </div>
 </template>
 
 <script>
 import NavBar from "@/components/NavBar";
 import {mapActions, mapGetters, mapMutations} from "vuex";
 import Sist2Api from "@/Sist2Api";
+import ModelsRepo from "@/ml/modelsRepo";
 import {setupAuth0} from "@/main";
+import Sist2ElasticsearchQuery from "@/Sist2ElasticsearchQuery";
+import Sist2SqliteQuery from "@/Sist2SqliteQuery";
 
 export default {
-  components: {NavBar},
-  data() {
-    return {
-      configLoading: false,
-      authLoading: true,
-      sist2InfoLoading: true
-    }
-  },
-  computed: {
-    ...mapGetters(["optTheme"]),
-  },
-  mounted() {
-    this.$store.dispatch("loadConfiguration").then(() => {
-      this.$root.$i18n.locale = this.$store.state.optLang;
-    });
+    components: {NavBar},
+    data() {
+        return {
+            configLoading: false,
+            authLoading: true,
+            sist2InfoLoading: true
+        }
+    },
+    computed: {
+        ...mapGetters(["optTheme"]),
+    },
+    mounted() {
+        this.$store.dispatch("loadConfiguration").then(() => {
+            this.$root.$i18n.locale = this.$store.state.optLang;
+            ModelsRepo.init(this.$store.getters.mlRepositoryList).catch(err => {
+                this.$bvToast.toast(
+                    this.$t("ml.repoFetchError"),
+                    {
+                        title: this.$t("ml.repoFetchErrorTitle"),
+                        noAutoHide: true,
+                        toaster: "b-toaster-bottom-right",
+                        headerClass: "toast-header-warning",
+                        bodyClass: "toast-body-warning",
+                    });
+            });
+        });
 
-    this.$store.subscribe((mutation) => {
-      if (mutation.type === "setOptLang") {
-        this.$root.$i18n.locale = mutation.payload;
-        this.configLoading = true;
-        window.setTimeout(() => this.configLoading = false, 10);
-      }
-
-      if (mutation.type === "setAuth0Token") {
-        this.authLoading = false;
-      }
-    });
-
-    Sist2Api.getSist2Info().then(data => {
-
-      if (data.auth0Enabled) {
-        this.authLoading = true;
-        setupAuth0(data.auth0Domain, data.auth0ClientId, data.auth0Audience)
-
-        this.$auth.$watch("loading", loading => {
-          if (loading === false) {
-
-            if (!this.$auth.isAuthenticated) {
-              this.$auth.loginWithRedirect();
-              return;
+        this.$store.subscribe((mutation) => {
+            if (mutation.type === "setOptLang") {
+                this.$root.$i18n.locale = mutation.payload;
+                this.configLoading = true;
+                window.setTimeout(() => this.configLoading = false, 10);
             }
 
-            // Remove "code" param
-            window.history.replaceState({}, "", "/" + window.location.hash);
-
-            this.$store.dispatch("loadAuth0Token");
-          }
+            if (mutation.type === "setAuth0Token") {
+                this.authLoading = false;
+            }
         });
-      } else {
-        this.authLoading = false;
-      }
 
-      this.setSist2Info(data);
-      this.setIndices(data.indices)
-    });
-  },
-  methods: {
-    ...mapActions(["setSist2Info",]),
-    ...mapMutations(["setIndices",]),
-    getClass() {
-      return {
-        "theme-light": this.optTheme === "light",
-        "theme-black": this.optTheme === "black",
-      }
+        Sist2Api.getSist2Info().then(data => {
+
+            if (data.auth0Enabled) {
+                this.authLoading = true;
+                setupAuth0(data.auth0Domain, data.auth0ClientId, data.auth0Audience)
+
+                this.$auth.$watch("loading", loading => {
+                    if (loading === false) {
+
+                        if (!this.$auth.isAuthenticated) {
+                            this.$auth.loginWithRedirect();
+                            return;
+                        }
+
+                        // Remove "code" param
+                        window.history.replaceState({}, "", "/" + window.location.hash);
+
+                        this.$store.dispatch("loadAuth0Token");
+                    }
+                });
+            } else {
+                this.authLoading = false;
+            }
+
+            this.setSist2Info(data);
+            this.setIndices(data.indices)
+
+            if (Sist2Api.backend() === "sqlite") {
+                Sist2Api.init(Sist2SqliteQuery.searchQuery);
+                this.$store.commit("setUiSqliteMode", true);
+            } else {
+                Sist2Api.init(Sist2ElasticsearchQuery.searchQuery);
+            }
+        });
+    },
+    methods: {
+        ...mapActions(["setSist2Info",]),
+        ...mapMutations(["setIndices",]),
+        getClass() {
+            return {
+                "theme-light": this.optTheme === "light",
+                "theme-black": this.optTheme === "black",
+            }
+        }
     }
-  }
-  ,
+    ,
 }
 </script>
 
 <style>
 html, body {
-  height: 100%;
+    height: 100%;
 }
 
 #app {
-  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-  /*text-align: center;*/
-  color: #2c3e50;
-  padding-bottom: 1em;
-  min-height: 100%;
+    /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+    -webkit-font-smoothing: antialiased;
+    -moz-osx-font-smoothing: grayscale;
+    /*text-align: center;*/
+    color: #2c3e50;
+    padding-bottom: 1em;
+    min-height: 100%;
 }
 
 /*Black theme*/
 .theme-black {
-  background-color: #000;
+    background-color: #000;
 }
 
 .theme-black .card, .theme-black .modal-content {
-  background: #212121;
-  color: #e0e0e0;
-  border-radius: 1px;
-  border: none;
+    background: #212121;
+    color: #e0e0e0;
+    border-radius: 1px;
+    border: none;
 }
 
 
 .theme-black .table {
-  color: #e0e0e0;
+    color: #e0e0e0;
 }
 
 .theme-black .table td, .theme-black .table th {
-  border: none;
+    border: none;
 }
 
 .theme-black .table thead th {
-  border-bottom: 1px solid #646464;
+    border-bottom: 1px solid #646464;
 }
 
 .theme-black .custom-select {
-  overflow: auto;
-  background-color: #37474F;
-  border: 1px solid #616161;
-  color: #bdbdbd;
+    overflow: auto;
+    background-color: #37474F;
+    border: 1px solid #616161;
+    color: #bdbdbd;
 }
 
 .theme-black .custom-select:focus {
-  border-color: #757575;
-  outline: 0;
-  box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
+    border-color: #757575;
+    outline: 0;
+    box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
 }
 
 .theme-black .inspire-tree .selected > .wholerow, .theme-black .inspire-tree .selected > .title-wrap:hover + .wholerow {
-  background: none !important;
+    background: none !important;
 }
 
 .theme-black .inspire-tree .icon-expand::before, .theme-black .inspire-tree .icon-collapse::before {
-  background-color: black !important;
+    background-color: black !important;
 }
 
 .theme-black .inspire-tree .title {
-  color: #eee;
+    color: #eee;
 }
 
 .theme-black .inspire-tree {
-  font-weight: 400;
-  font-size: 14px;
-  font-family: Helvetica, Nueue, Verdana, sans-serif;
-  max-height: 350px;
-  overflow: auto;
+    font-weight: 400;
+    font-size: 14px;
+    font-family: Helvetica, Nueue, Verdana, sans-serif;
+    max-height: 350px;
+    overflow: auto;
 }
 
 .inspire-tree [type=checkbox] {
-  left: 22px !important;
-  top: 7px !important;
+    left: 22px !important;
+    top: 7px !important;
 }
 
 .theme-black .form-control {
-  background-color: #37474F;
-  border: 1px solid #616161;
-  color: #dbdbdb !important;
+    background-color: #37474F;
+    border: 1px solid #616161;
+    color: #dbdbdb !important;
 }
 
 .theme-black .form-control:focus {
-  background-color: #546E7A;
-  color: #fff;
+    background-color: #546E7A;
+    color: #fff;
 }
 
 .theme-black .input-group-text, .theme-black .default-input {
-  background: #37474F !important;
-  border: 1px solid #616161 !important;
-  color: #dbdbdb !important;
+    background: #37474F !important;
+    border: 1px solid #616161 !important;
+    color: #dbdbdb !important;
 }
 
 .theme-black ::placeholder {
-  color: #BDBDBD !important;
-  opacity: 1;
+    color: #BDBDBD !important;
+    opacity: 1;
 }
 
 .theme-black .nav-tabs .nav-link {
-  color: #e0e0e0;
-  border-radius: 0;
+    color: #e0e0e0;
+    border-radius: 0;
 }
 
 .theme-black .nav-tabs .nav-item.show .nav-link, .theme-black .nav-tabs .nav-link.active {
-  background-color: #212121;
-  border-color: #616161 #616161 #212121;
-  color: #e0e0e0;
+    background-color: #212121;
+    border-color: #616161 #616161 #212121;
+    color: #e0e0e0;
 }
 
 .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:focus {
-  border-color: #616161 #616161 #212121;
-  color: #e0e0e0;
+    border-color: #616161 #616161 #212121;
+    color: #e0e0e0;
 }
 
 .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:hover {
-  border-color: #e0e0e0 #e0e0e0 #212121;
-  color: #e0e0e0;
+    border-color: #e0e0e0 #e0e0e0 #212121;
+    color: #e0e0e0;
 }
 
 .theme-black .nav-tabs {
-  border-bottom: #616161;
+    border-bottom: #616161;
 }
 
 .theme-black a:hover, .theme-black .btn:hover {
-  color: #fff;
+    color: #fff;
 }
 
 .theme-black .b-dropdown a:hover {
-  color: inherit;
+    color: inherit;
 }
 
 .theme-black .btn {
-  color: #eee;
+    color: #eee;
 }
 
 .theme-black .modal-header .close {
-  color: #e0e0e0;
-  text-shadow: none;
+    color: #e0e0e0;
+    text-shadow: none;
 }
 
 .theme-black .modal-header {
-  border-bottom: 1px solid #646464;
+    border-bottom: 1px solid #646464;
 }
 
 /* -------------------------- */
 
 #nav {
-  padding: 30px;
+    padding: 30px;
 }
 
 #nav a {
-  font-weight: bold;
-  color: #2c3e50;
+    font-weight: bold;
+    color: #2c3e50;
 }
 
 #nav a.router-link-exact-active {
-  color: #42b983;
+    color: #42b983;
 }
 
 .mobile {
-  display: none;
+    display: none;
 }
 
 .container {
-  padding-top: 1em;
+    padding-top: 1em;
 }
 
 @media (max-width: 650px) {
-  .mobile {
-    display: initial;
-  }
+    .mobile {
+        display: initial;
+    }
 
-  .not-mobile {
-    display: none;
-  }
+    .not-mobile {
+        display: none;
+    }
 
-  .grid-single-column .fit {
-    max-height: none !important;
-  }
+    .grid-single-column .fit {
+        max-height: none !important;
+    }
 
-  .container {
-    padding-left: 0;
-    padding-right: 0;
-    padding-top: 0
-  }
+    .container {
+        padding-left: 0;
+        padding-right: 0;
+        padding-top: 0
+    }
 
-  .lightbox-caption {
-    display: none;
-  }
+    .lightbox-caption {
+        display: none;
+    }
 }
 
 .info-icon {
-  width: 1rem;
-  margin-right: 0.2rem;
-  cursor: pointer;
-  line-height: 1rem;
-  height: 1rem;
-  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
-  filter: brightness(45%);
-  display: block;
+    width: 1rem;
+    margin-right: 0.2rem;
+    cursor: pointer;
+    line-height: 1rem;
+    height: 1rem;
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+    filter: brightness(45%);
+    display: block;
 }
 
 .tabs {
-  margin-top: 10px;
+    margin-top: 10px;
 }
 
 .modal-title {
-  text-overflow: ellipsis;
-  overflow: hidden;
-  white-space: nowrap;
+    text-overflow: ellipsis;
+    overflow: hidden;
+    white-space: nowrap;
 }
 
 @media screen and (min-width: 1500px) {
-  .container {
-    max-width: 1440px;
-  }
+    .container {
+        max-width: 1440px;
+    }
 }
 
 .noUi-connects {
-  border-radius: 1px !important;
+    border-radius: 1px !important;
 }
 
 mark {
-  background: #fff217;
-  border-radius: 0;
-  padding: 1px 0;
-  color: inherit;
+    background: #fff217;
+    border-radius: 0;
+    padding: 1px 0;
+    color: inherit;
 }
 
 .theme-black mark {
-  background: rgba(251, 191, 41, 0.25);
-  border-radius: 0;
-  padding: 1px 0;
-  color: inherit;
+    background: rgba(251, 191, 41, 0.25);
+    border-radius: 0;
+    padding: 1px 0;
+    color: inherit;
 }
 
 .theme-black .content-div mark {
-  background: rgba(251, 191, 41, 0.40);
-  color: white;
+    background: rgba(251, 191, 41, 0.40);
+    color: white;
 }
 
 .content-div {
-  font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
-  font-size: 13px;
-  padding: 1em;
-  background-color: #f5f5f5;
-  border: 1px solid #ccc;
-  border-radius: 4px;
-  margin: 3px;
-  white-space: normal;
-  color: #000;
-  overflow: hidden;
+    font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+    font-size: 13px;
+    padding: 1em;
+    background-color: #f5f5f5;
+    border: 1px solid #ccc;
+    border-radius: 4px;
+    margin: 3px;
+    white-space: normal;
+    color: #000;
+    overflow: hidden;
 }
 
 .theme-black .content-div {
-  background-color: #37474F;
-  border: 1px solid #616161;
-  color: #E0E0E0FF;
+    background-color: #37474F;
+    border: 1px solid #616161;
+    color: #E0E0E0FF;
 }
 
 .graph {
-  display: inline-block;
-  width: 40%;
+    display: inline-block;
+    width: 40%;
 }
 
 .pointer {
-  cursor: pointer;
+    cursor: pointer;
 }
 
 .loading-page {
-  display: flex;
-  justify-content: center;
-  align-items: center;
-  flex-direction: column;
-  height: 100%;
-  gap: 15px
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    flex-direction: column;
+    height: 100%;
+    gap: 15px
 }
 
 .loading-spinners {
-  display: flex;
-  gap: 10px;
+    display: flex;
+    gap: 10px;
 }
 
 .loading-text {
-  text-align: center;
+    text-align: center;
 }
 </style>

sist2-vue/src/Sist2Api.ts

@@ -1,5 +1,7 @@
 import axios from "axios";
 import {ext, strUnescape, lum} from "./util";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
+import store from "@/store";
 
 export interface EsTag {
     id: string
@@ -61,6 +63,7 @@ export interface EsHit {
         isAudio: boolean
         hasThumbnail: boolean
         hasVidPreview: boolean
+        imageAspectRatio: number
         /** Number of thumbnails available */
         tnNum: number
     }
@@ -98,12 +101,22 @@ export interface EsResult {
 
 class Sist2Api {
 
-    private baseUrl: string
+    private readonly baseUrl: string
+    private sist2Info: any
+    private queryfunc: () => EsResult;
 
     constructor(baseUrl: string) {
         this.baseUrl = baseUrl;
     }
 
+    init(queryFunc: () => EsResult) {
+        this.queryfunc = queryFunc;
+    }
+
+    backend() {
+        return this.sist2Info.searchBackend;
+    }
+
     getSist2Info(): Promise<any> {
         return axios.get(`${this.baseUrl}i`).then(resp => {
             const indices = resp.data.indices as Index[];
@@ -118,6 +131,8 @@ class Sist2Api {
                 } as Index;
             });
 
+            this.sist2Info = resp.data;
+
             return resp.data;
         })
     }
@@ -155,6 +170,9 @@ class Sist2Api {
                     && hit._source.videoc !== "raw" && hit._source.videoc !== "ppm") {
                     hit._props.isPlayableImage = true;
                 }
+                if ("width" in hit._source && "height" in hit._source) {
+                    hit._props.imageAspectRatio = hit._source.width / hit._source.height;
+                }
                 break;
             case "video":
                 if ("videoc" in hit._source) {
@@ -187,30 +205,6 @@ class Sist2Api {
     setHitTags(hit: EsHit): void {
         const tags = [] as Tag[];
 
-        const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
-
-        switch (mimeCategory) {
-            case "image":
-            case "video":
-                if ("videoc" in hit._source && hit._source.videoc) {
-                    tags.push({
-                        style: "video",
-                        text: hit._source.videoc.replace(" ", ""),
-                        userTag: false
-                    } as Tag);
-                }
-                break
-            case "audio":
-                if ("audioc" in hit._source && hit._source.audioc) {
-                    tags.push({
-                        style: "audio",
-                        text: hit._source.audioc,
-                        userTag: false
-                    } as Tag);
-                }
-                break;
-        }
-
         // User tags
         if ("tag" in hit._source) {
             hit._source.tag.forEach(tag => {
@@ -239,6 +233,14 @@ class Sist2Api {
         } as Tag;
     }
 
+    search(): Promise<EsResult> {
+        if (this.backend() == "sqlite") {
+            return this.ftsQuery(this.queryfunc())
+        } else {
+            return this.esQuery(this.queryfunc());
+        }
+    }
+
     esQuery(query: any): Promise<EsResult> {
         return axios.post(`${this.baseUrl}es`, query).then(resp => {
             const res = resp.data as EsResult;
@@ -257,7 +259,30 @@ class Sist2Api {
         });
     }
 
-    getMimeTypes(query = undefined) {
+    ftsQuery(query: any): Promise<EsResult> {
+        return axios.post(`${this.baseUrl}fts/search`, query).then(resp => {
+            const res = resp.data as any;
+
+            if (res.hits.hits) {
+                res.hits.hits.forEach(hit => {
+                    hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
+                    hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+
+                    this.setHitProps(hit);
+                    this.setHitTags(hit);
+
+                    if ("highlight" in hit) {
+                        hit["highlight"]["name"] = [hit["highlight"]["name"]];
+                        hit["highlight"]["content"] = [hit["highlight"]["content"]];
+                    }
+                });
+            }
+
+            return res;
+        });
+    }
+
+    private getMimeTypesEs(query) {
         const AGGS = {
             mimeTypes: {
                 terms: {
@@ -278,48 +303,70 @@ class Sist2Api {
         }
 
         return this.esQuery(query).then(resp => {
-            const mimeMap: any[] = [];
-            const buckets = resp["aggregations"]["mimeTypes"]["buckets"];
+            return resp["aggregations"]["mimeTypes"]["buckets"].map(bucket => ({
+                mime: bucket.key,
+                count: bucket.doc_count
+            }));
 
-            buckets.sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
-                const tmp = bucket["key"].split("/");
-                const category = tmp[0];
-                const mime = tmp[1];
+        });
+    }
 
-                let category_exists = false;
+    private getMimeTypesSqlite(): Promise<[{ mime: string, count: number }]> {
+        return axios.get(`${this.baseUrl}fts/mimetypes`)
+            .then(resp => {
+                return resp.data;
+            });
+    }
 
-                const child = {
-                    "id": bucket["key"],
-                    "text": `${mime} (${bucket["doc_count"]})`
-                };
+    async getMimeTypes(query = undefined) {
+        let buckets;
 
-                mimeMap.forEach(node => {
-                    if (node.text === category) {
-                        node.children.push(child);
-                        category_exists = true;
-                    }
-                });
+        if (this.backend() == "sqlite") {
+            buckets = await this.getMimeTypesSqlite();
+        } else {
+            buckets = await this.getMimeTypesEs(query);
+        }
 
-                if (!category_exists) {
-                    mimeMap.push({text: category, children: [child], id: category});
-                }
-            })
+        const mimeMap: any[] = [];
+
+        buckets.sort((a: any, b: any) => a.mime > b.mime).forEach((bucket: any) => {
+            const tmp = bucket.mime.split("/");
+            const category = tmp[0];
+            const mime = tmp[1];
+
+            let category_exists = false;
+
+            const child = {
+                "id": bucket.mime,
+                "text": `${mime} (${bucket.count})`
+            };
 
             mimeMap.forEach(node => {
-                if (node.children) {
-                    node.children.sort((a, b) => a.id.localeCompare(b.id));
+                if (node.text === category) {
+                    node.children.push(child);
+                    category_exists = true;
                 }
-            })
-            mimeMap.sort((a, b) => a.id.localeCompare(b.id))
+            });
 
-            return {buckets, mimeMap};
-        });
+            if (!category_exists) {
+                mimeMap.push({text: category, children: [child], id: category});
+            }
+        })
+
+        mimeMap.forEach(node => {
+            if (node.children) {
+                node.children.sort((a, b) => a.id.localeCompare(b.id));
+            }
+        })
+        mimeMap.sort((a, b) => a.id.localeCompare(b.id))
+
+        return {buckets, mimeMap};
     }
 
     _createEsTag(tag: string, count: number): EsTag {
         const tokens = tag.split(".");
 
-        if (/.*\.#[0-9a-f]{6}/.test(tag)) {
+        if (/.*\.#[0-9a-fA-F]{6}/.test(tag)) {
             return {
                 id: tokens.slice(0, -1).join("."),
                 color: tokens.pop(),
@@ -336,32 +383,48 @@ class Sist2Api {
         };
     }
 
-    getTags() {
+    private getTagsEs() {
         return this.esQuery({
             aggs: {
                 tags: {
                     terms: {
                         field: "tag",
-                        size: 10000
+                        size: 65535
                     }
                 }
             },
             size: 0,
         }).then(resp => {
-            const seen = new Set();
-
-            const tags = resp["aggregations"]["tags"]["buckets"]
+            return resp["aggregations"]["tags"]["buckets"]
                 .sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
                 .map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
+        });
+    }
 
-            // Remove duplicates (same tag with different color)
-            return tags.filter((t: EsTag) => {
-                if (seen.has(t.id)) {
-                    return false;
-                }
-                seen.add(t.id);
-                return true;
+    private getTagsSqlite() {
+        return axios.get(`${this.baseUrl}/fts/tags`)
+            .then(resp => {
+                return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
             });
+    }
+
+    async getTags(): Promise<EsTag[]> {
+        let tags;
+        if (this.backend() == "sqlite") {
+            tags = await this.getTagsSqlite();
+        } else {
+            tags = await this.getTagsEs();
+        }
+
+        // Remove duplicates (same tag with different color)
+        const seen = new Set();
+
+        return tags.filter((t: EsTag) => {
+            if (seen.has(t.id)) {
+                return false;
+            }
+            seen.add(t.id);
+            return true;
         });
     }
 
@@ -381,20 +444,263 @@ class Sist2Api {
         });
     }
 
-    getTreemapCsvUrl(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/1`;
+    searchPaths(indexId, minDepth, maxDepth, prefix = null) {
+        if (this.backend() == "sqlite") {
+            return this.searchPathsSqlite(indexId, minDepth, minDepth, prefix);
+        } else {
+            return this.searchPathsEs(indexId, minDepth, maxDepth, prefix);
+        }
     }
 
-    getMimeCsvUrl(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/2`;
+    private searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
+        return axios.post(`${this.baseUrl}fts/paths`, {
+            indexId, minDepth, maxDepth, prefix
+        }).then(resp => {
+            return resp.data;
+        });
     }
 
-    getSizeCsv(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/3`;
+    private searchPathsEs(indexId, minDepth, maxDepth, prefix): Promise<[{ path: string, count: number }]> {
+
+        const query = {
+            query: {
+                bool: {
+                    filter: [
+                        {term: {index: indexId}},
+                        {range: {_depth: {gte: minDepth, lte: maxDepth}}},
+                    ]
+                }
+            },
+            aggs: {
+                paths: {
+                    terms: {
+                        field: "path",
+                        size: 10000
+                    }
+                }
+            },
+            size: 0
+        };
+
+        if (prefix != null) {
+            query["query"]["bool"]["must"] = {
+                prefix: {
+                    path: prefix,
+                }
+            };
+        }
+
+        return this.esQuery(query).then(resp => {
+            const buckets = resp["aggregations"]["paths"]["buckets"];
+
+            if (!buckets) {
+                return [];
+            }
+
+            return buckets
+                .map(bucket => ({
+                    path: bucket.key,
+                    count: bucket.doc_count
+                }));
+        });
     }
 
-    getDateCsv(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/4`;
+    private getDateRangeSqlite() {
+        return axios.get(`${this.baseUrl}fts/dateRange`)
+            .then(resp => ({
+                min: resp.data.dateMin,
+                max: resp.data.dateMax,
+            }));
+    }
+
+    getDateRange(): Promise<{ min: number, max: number }> {
+        if (this.backend() == "sqlite") {
+            return this.getDateRangeSqlite();
+        } else {
+            return this.getDateRangeEs();
+        }
+    }
+
+    private getDateRangeEs() {
+        return this.esQuery({
+            // TODO: filter current selected indices
+            aggs: {
+                dateMin: {min: {field: "mtime"}},
+                dateMax: {max: {field: "mtime"}},
+            },
+            size: 0
+        }).then(res => {
+            const range = {
+                min: res.aggregations.dateMin.value,
+                max: res.aggregations.dateMax.value,
+            }
+
+            if (range.min == null) {
+                range.min = 0;
+                range.max = 1;
+            } else if (range.min == range.max) {
+                range.max += 1;
+            }
+
+            return range;
+        });
+    }
+
+    private getPathSuggestionsSqlite(text: string) {
+        return axios.post(`${this.baseUrl}fts/paths`, {
+            prefix: text,
+            minDepth: 1,
+            maxDepth: 10000
+        }).then(resp => {
+            return resp.data.map(bucket => bucket.path);
+        })
+    }
+
+    private getPathSuggestionsEs(text) {
+        return this.esQuery({
+            suggest: {
+                path: {
+                    prefix: text,
+                    completion: {
+                        field: "suggest-path",
+                        skip_duplicates: true,
+                        size: 10000
+                    }
+                }
+            }
+        }).then(resp => {
+            return resp["suggest"]["path"][0]["options"]
+                .map(opt => opt["_source"]["path"]);
+        });
+    }
+
+    getPathSuggestions(text: string): Promise<string[]> {
+        if (this.backend() == "sqlite") {
+            return this.getPathSuggestionsSqlite(text);
+        } else {
+            return this.getPathSuggestionsEs(text)
+        }
+    }
+
+    getTreemapStat(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/TMAP`;
+    }
+
+    getMimeStat(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/MAGG`;
+    }
+
+    getSizeStat(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/SAGG`;
+    }
+
+    getDateStat(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/DAGG`;
+    }
+
+    private getDocumentEs(docId: string, highlight: boolean, fuzzy: boolean) {
+        const query = Sist2Query.searchQuery();
+
+        if (highlight) {
+            const fields = fuzzy
+                ? {"content.nGram": {}}
+                : {content: {}};
+
+            query.highlight = {
+                pre_tags: ["<mark>"],
+                post_tags: ["</mark>"],
+                number_of_fragments: 0,
+                fields,
+            };
+
+            if (!store.state.sist2Info.esVersionLegacy) {
+                query.highlight.max_analyzed_offset = 999_999;
+            }
+        }
+
+        if ("function_score" in query.query) {
+            query.query = query.query.function_score.query;
+        }
+
+        if (!("must" in query.query.bool)) {
+            query.query.bool.must = [];
+        } else if (!Array.isArray(query.query.bool.must)) {
+            query.query.bool.must = [query.query.bool.must];
+        }
+
+        query.query.bool.must.push({match: {_id: docId}});
+
+        delete query["sort"];
+        delete query["aggs"];
+        delete query["search_after"];
+        delete query.query["function_score"];
+
+        query._source = {
+            includes: ["content", "name", "path", "extension"]
+        }
+
+        query.size = 1;
+
+        return this.esQuery(query).then(resp => {
+            if (resp.hits.hits.length === 1) {
+                return resp.hits.hits[0];
+            }
+            return null;
+        });
+    }
+
+    private getDocumentSqlite(docId: string): Promise<EsHit> {
+        return axios.get(`${this.baseUrl}/fts/d/${docId}`)
+            .then(resp => ({
+                _source: resp.data
+            } as EsHit));
+    }
+
+    getDocument(docId: string, highlight: boolean, fuzzy: boolean): Promise<EsHit | null> {
+        if (this.backend() == "sqlite") {
+            return this.getDocumentSqlite(docId);
+        } else {
+            return this.getDocumentEs(docId, highlight, fuzzy);
+        }
+    }
+
+    getTagSuggestions(prefix: string): Promise<string[]> {
+        if (this.backend() == "sqlite") {
+            return this.getTagSuggestionsSqlite(prefix);
+        } else {
+            return this.getTagSuggestionsEs(prefix);
+        }
+    }
+
+    private getTagSuggestionsSqlite(prefix): Promise<string[]> {
+        return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
+            .then(resp => (resp.data));
+    }
+
+    private getTagSuggestionsEs(prefix): Promise<string[]> {
+        return this.esQuery({
+            suggest: {
+                tag: {
+                    prefix: prefix,
+                    completion: {
+                        field: "suggest-tag",
+                        skip_duplicates: true,
+                        size: 10000
+                    }
+                }
+            }
+        }).then(resp => {
+            const result = [];
+            resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
+                tags.forEach(tag => {
+                    const t = tag.slice(0, -8);
+                    if (!result.find(x => x.slice(0, -8) === t)) {
+                        result.push(tag);
+                    }
+                });
+            });
+            return result;
+        });
     }
 }
 

sist2-vue/src/Sist2ElasticsearchQuery.ts

@@ -60,14 +60,7 @@ const SORT_MODES = {
     }
 } as any;
 
-interface SortMode {
-    text: string
-    mode: any[]
-    key: (hit: EsHit) => any
-}
-
-
-class Sist2Query {
+class Sist2ElasticsearchQuery {
 
     searchQuery(blankSearch: boolean = false): any {
 
@@ -177,14 +170,16 @@ class Sist2Query {
                 }
             },
             sort: SORT_MODES[getters.sortMode].mode,
-            aggs:
-                {
-                    total_size: {"sum": {"field": "size"}},
-                    total_count: {"value_count": {"field": "size"}}
-                },
             size: size,
         } as any;
 
+        if (!after) {
+            q.aggs = {
+                total_size: {"sum": {"field": "size"}},
+                total_count: {"value_count": {"field": "size"}}
+            };
+        }
+
         if (!empty && !blankSearch) {
             q.query.bool.must = query;
         }
@@ -249,4 +244,5 @@ class Sist2Query {
     }
 }
 
-export default new Sist2Query();
+
+export default new Sist2ElasticsearchQuery();

sist2-vue/src/Sist2SqliteQuery.ts

@@ -0,0 +1,111 @@
+import store from "./store";
+import {EsHit, Index} from "@/Sist2Api";
+
+const SORT_MODES = {
+    score: {
+        "sort": "score",
+    },
+    random: {
+        "sort": "random"
+    },
+    dateAsc: {
+        "sort": "mtime"
+    },
+    dateDesc: {
+        "sort": "mtime",
+        "sortAsc": false
+    },
+    sizeAsc: {
+        "sort": "size",
+    },
+    sizeDesc: {
+        "sort": "size",
+        "sortAsc": false
+    },
+    nameAsc: {
+        "sort": "name",
+    },
+    nameDesc: {
+        "sort": "name",
+        "sortAsc": false
+    }
+} as any;
+
+interface SortMode {
+    text: string
+    mode: any[]
+    key: (hit: EsHit) => any
+}
+
+
+class Sist2ElasticsearchQuery {
+
+    searchQuery(): any {
+
+        const getters = store.getters;
+
+        const searchText = getters.searchText;
+        const pathText = getters.pathText;
+        const sizeMin = getters.sizeMin;
+        const sizeMax = getters.sizeMax;
+        const dateMin = getters.dateMin;
+        const dateMax = getters.dateMax;
+        const size = getters.size;
+        const after = getters.lastDoc;
+        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
+        const selectedMimeTypes = getters.selectedMimeTypes;
+        const selectedTags = getters.selectedTags;
+
+        const q = {
+            "pageSize": size
+        }
+
+        Object.assign(q, SORT_MODES[getters.sortMode]);
+
+        if (!after) {
+            q["fetchAggregations"] = true;
+        }
+        if (searchText) {
+            q["query"] = searchText;
+        }
+        if (pathText) {
+            q["path"] = pathText.endsWith("/") ? pathText.slice(0, -1) : pathText;
+        }
+        if (sizeMin) {
+            q["sizeMin"] = sizeMin;
+        }
+        if (sizeMax) {
+            q["sizeMax"] = sizeMax;
+        }
+        if (dateMin) {
+            q["dateMin"] = dateMin;
+        }
+        if (dateMax) {
+            q["dateMax"] = dateMax;
+        }
+        if (after) {
+            q["after"] = after.sort;
+        }
+        if (selectedIndexIds.length > 0) {
+            q["indexIds"] = selectedIndexIds;
+        }
+        if (selectedMimeTypes.length > 0) {
+            q["mimeTypes"] = selectedMimeTypes;
+        }
+        if (selectedTags.length > 0) {
+            q["tags"] = selectedTags
+        }
+        if (getters.sortMode == "random") {
+            q["seed"] = getters.seed;
+        }
+        if (getters.optHighlight) {
+            q["highlight"] = true;
+            q["highlightContextSize"] = Number(getters.optFragmentSize);
+        }
+
+        return q;
+    }
+}
+
+
+export default new Sist2ElasticsearchQuery();

sist2-vue/src/components/AnalyzedContentSpan.vue

@@ -0,0 +1,21 @@
+<template>
+    <span :style="getStyle()">{{span.text}}</span>
+</template>
+
+<script>
+
+
+import ModelsRepo from "@/ml/modelsRepo";
+
+export default {
+    name: "AnalyzedContentSpan",
+    props: ["span", "text"],
+    methods: {
+        getStyle() {
+            return ModelsRepo.data[this.$store.getters.mlModel.name].labelStyles[this.span.label];
+        }
+    }
+}
+</script>
+
+<style scoped></style>

sist2-vue/src/components/AnalyzedContentSpanContainer.vue

@@ -0,0 +1,75 @@
+<template>
+    <div>
+        <b-card class="mb-2">
+            <AnalyzedContentSpan v-for="span of legend" :key="span.id" :span="span"
+                                 class="mr-2"></AnalyzedContentSpan>
+        </b-card>
+        <div class="content-div">
+            <AnalyzedContentSpan v-for="span of mergedSpans" :key="span.id" :span="span"></AnalyzedContentSpan>
+        </div>
+    </div>
+</template>
+
+<script>
+
+
+import AnalyzedContentSpan from "@/components/AnalyzedContentSpan.vue";
+import ModelsRepo from "@/ml/modelsRepo";
+
+export default {
+    name: "AnalyzedContentSpanContainer",
+    components: {AnalyzedContentSpan},
+    props: ["spans", "text"],
+    computed: {
+        legend() {
+            return Object.entries(ModelsRepo.data[this.$store.state.mlModel.name].legend)
+                .map(([label, name]) => ({
+                    text: name,
+                    id: label,
+                    label: label
+                }));
+        },
+        mergedSpans() {
+            const spans = this.spans;
+
+            const merged = [];
+
+            let lastLabel = null;
+            let fixSpace = false;
+            for (let i = 0; i < spans.length; i++) {
+
+                if (spans[i].label !== lastLabel) {
+                    let start = spans[i].wordIndex;
+                    const nextSpan = spans.slice(i + 1).find(s => s.label !== spans[i].label)
+                    let end = nextSpan ? nextSpan.wordIndex : undefined;
+
+                    if (end !== undefined && this.text[end - 1] === " ") {
+                        end -= 1;
+                        fixSpace = true;
+                    }
+
+                    merged.push({
+                        text: this.text.slice(start, end),
+                        label: spans[i].label,
+                        id: spans[i].wordIndex
+                    });
+
+                    if (fixSpace) {
+                        merged.push({
+                            text: " ",
+                            label: "O",
+                            id: end
+                        });
+                        fixSpace = false;
+                    }
+                    lastLabel = spans[i].label;
+                }
+            }
+
+            return merged;
+        },
+    },
+}
+</script>
+
+<style scoped></style>

sist2-vue/src/components/D3DateHistogram.vue

@@ -120,7 +120,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#date-histogram");
 
-      d3.csv(Sist2Api.getDateCsv(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getDateStat(indexId)).then(tabularData => {
         dateHistogram(tabularData.slice(), svg, this.$t("d3.dateHistogram"));
       });
     }

sist2-vue/src/components/D3MimeBarCount.vue

@@ -91,7 +91,7 @@ export default {
       const mimeSvgCount = d3.select("#agg-mime-count");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
         mimeBarCount(tabularData.slice(), mimeSvgCount, fillOpacity, this.$t("d3.mimeCount"));
       });
     }

sist2-vue/src/components/D3MimeBarSize.vue

@@ -90,7 +90,7 @@ export default {
       const mimeSvgSize = d3.select("#agg-mime-size");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
         mimeBarSize(tabularData.slice(), mimeSvgSize, fillOpacity, this.$t("d3.mimeSize"));
       });
     }

sist2-vue/src/components/D3SizeHistogram.vue

@@ -117,7 +117,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#size-histogram");
 
-      d3.csv(Sist2Api.getSizeCsv(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getSizeStat(indexId)).then(tabularData => {
         sizeHistogram(tabularData.slice(), svg, this.$t("d3.sizeHistogram"));
       });
     }

sist2-vue/src/components/D3Treemap.vue

@@ -240,7 +240,7 @@ export default {
           .style("overflow", "visible")
           .style("font", "10px sans-serif");
 
-      d3.csv(Sist2Api.getTreemapCsvUrl(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
         tabularData.forEach(row => {
           row.taxonomy = row.path.split("/");
           row.size = Number(row.size);

sist2-vue/src/components/DebugInfo.vue

@@ -1,41 +1,56 @@
 <template>
-  <b-card class="mb-4 mt-4">
-    <b-card-title><DebugIcon class="mr-1"></DebugIcon>{{ $t("debug") }}</b-card-title>
-    <p v-html="$t('debugDescription')"></p>
+    <b-card v-if="$store.state.sist2Info.showDebugInfo" class="mb-4 mt-4">
+        <b-card-title>
+            <DebugIcon class="mr-1"></DebugIcon>
+            {{ $t("debug") }}
+        </b-card-title>
+        <p v-html="$t('debugDescription')"></p>
 
-    <b-card-body>
+        <b-card-body>
 
-      <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
+            <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
 
-      <hr />
-      <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx" class="mt-2"></IndexDebugInfo>
-    </b-card-body>
-  </b-card>
+            <hr/>
+            <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx"
+                            class="mt-2"></IndexDebugInfo>
+        </b-card-body>
+    </b-card>
 </template>
 
 <script>
 import IndexDebugInfo from "@/components/IndexDebugInfo";
 import DebugIcon from "@/components/icons/DebugIcon";
+import {mapGetters} from "vuex";
 
 export default {
-  name: "DebugInfo.vue",
-  components: {DebugIcon, IndexDebugInfo},
-  computed: {
-    tableItems() {
-      return [
-        {key: "version", value: this.$store.state.sist2Info.version},
-        {key: "platform", value: this.$store.state.sist2Info.platform},
-        {key: "debugBinary", value: this.$store.state.sist2Info.debug},
-        {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
-        {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
-        {key: "tagline", value: this.$store.state.sist2Info.tagline},
-        {key: "dev", value: this.$store.state.sist2Info.dev},
-        {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
-        {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
-        {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
-        {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
-      ]
+    name: "DebugInfo.vue",
+    components: {DebugIcon, IndexDebugInfo},
+    computed: {
+        ...mapGetters([
+            "uiSqliteMode",
+        ]),
+        tableItems() {
+            const items = [
+                {key: "version", value: this.$store.state.sist2Info.version},
+                {key: "platform", value: this.$store.state.sist2Info.platform},
+                {key: "debugBinary", value: this.$store.state.sist2Info.debug},
+                {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
+                {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
+                {key: "tagline", value: this.$store.state.sist2Info.tagline},
+                {key: "dev", value: this.$store.state.sist2Info.dev},
+                {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
+            ];
+
+            if (!this.uiSqliteMode) {
+                items.push(
+                    {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
+                    {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
+                    {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy}
+                );
+            }
+
+            return items;
+        }
     }
-  }
 }
 </script>

sist2-vue/src/components/DocCard.vue

@@ -1,39 +1,44 @@
 <template>
-  <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
-       @click="$store.commit('busTnTouchStart', null)">
-    <b-card
-        no-body
-        img-top
-    >
-      <!-- Info modal-->
-      <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
+    <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
+         @click="$store.commit('busTnTouchStart', null)">
+        <b-card
+                no-body
+                img-top
+        >
+            <!-- Info modal-->
+            <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
 
-      <ContentDiv :doc="doc"></ContentDiv>
+            <ContentDiv :doc="doc"></ContentDiv>
 
-      <!-- Thumbnail-->
-      <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
+            <!-- Thumbnail-->
+            <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
 
-      <!-- Audio player-->
-      <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
-             :type="doc._source.mime"
-             :src="`f/${doc._id}`"
-             @play="onAudioPlay()"></audio>
+            <!-- Audio player-->
+            <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
+                   :type="doc._source.mime"
+                   :src="`f/${doc._source.index}/${doc._id}`"
+                   @play="onAudioPlay()"></audio>
 
-      <b-card-body class="padding-03">
+            <b-card-body class="padding-03">
 
-        <!-- Title line -->
-        <div style="display: flex">
-          <span class="info-icon" @click="onInfoClick()"></span>
-          <DocFileTitle :doc="doc"></DocFileTitle>
-        </div>
+                <!-- Title line -->
+                <div style="display: flex">
+                    <span class="info-icon" @click="onInfoClick()"></span>
+                    <DocFileTitle :doc="doc"></DocFileTitle>
+                </div>
 
-        <!-- Tags -->
-        <div class="card-text">
-          <TagContainer :hit="doc"></TagContainer>
-        </div>
-      </b-card-body>
-    </b-card>
-  </div>
+                <!-- Featured line -->
+                <div style="display: flex">
+                    <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
+                </div>
+
+                <!-- Tags -->
+                <div class="card-text">
+                    <TagContainer :hit="doc"></TagContainer>
+                </div>
+            </b-card-body>
+        </b-card>
+    </div>
 </template>
 
 <script>
@@ -43,94 +48,95 @@ import DocFileTitle from "@/components/DocFileTitle.vue";
 import DocInfoModal from "@/components/DocInfoModal.vue";
 import ContentDiv from "@/components/ContentDiv.vue";
 import FullThumbnail from "@/components/FullThumbnail";
+import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
 
 
 export default {
-  components: {FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
-  props: ["doc", "width"],
-  data() {
-    return {
-      ext: ext,
-      showInfo: false,
-    }
-  },
-  computed: {
-    smallBadge() {
-      return this.width < 150;
-    }
-  },
-  methods: {
-    humanFileSize: humanFileSize,
-    humanTime: humanTime,
-    onInfoClick() {
-      this.showInfo = true;
-    },
-    async onThumbnailClick() {
-      this.$store.commit("setUiLightboxSlide", this.doc._seq);
-      await this.$store.dispatch("showLightbox");
-    },
-    onAudioPlay() {
-      document.getElementsByTagName("audio").forEach((el) => {
-        if (el !== this.$refs["audio"]) {
-          el.pause();
+    components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+    props: ["doc", "width"],
+    data() {
+        return {
+            ext: ext,
+            showInfo: false,
         }
-      });
     },
-  },
+    computed: {
+        smallBadge() {
+            return this.width < 150;
+        }
+    },
+    methods: {
+        humanFileSize: humanFileSize,
+        humanTime: humanTime,
+        onInfoClick() {
+            this.showInfo = true;
+        },
+        async onThumbnailClick() {
+            this.$store.commit("setUiLightboxSlide", this.doc._seq);
+            await this.$store.dispatch("showLightbox");
+        },
+        onAudioPlay() {
+            Array.prototype.slice.call(document.getElementsByTagName("audio")).forEach((el) => {
+                if (el !== this.$refs["audio"]) {
+                    el.pause();
+                }
+            });
+        },
+    },
 }
 </script>
 <style>
 .fit {
-  display: block;
-  min-width: 64px;
-  max-width: 100%;
-  /*max-height: 400px;*/
-  margin: 0 auto 0;
-  width: auto;
-  height: auto;
+    display: block;
+    min-width: 64px;
+    max-width: 100%;
+    /*max-height: 400px;*/
+    margin: 0 auto 0;
+    width: auto;
+    height: auto;
 }
 
 .audio-fit {
-  height: 39px;
-  vertical-align: bottom;
-  display: inline;
-  width: 100%;
+    height: 39px;
+    vertical-align: bottom;
+    display: inline;
+    width: 100%;
 }
 </style>
 
 <style scoped>
 
 .padding-03 {
-  padding: 0.3rem;
+    padding: 0.3rem;
 }
 
 .card {
-  margin-top: 1em;
-  margin-left: 0;
-  margin-right: 0;
-  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-  border-radius: 0;
-  border: none;
+    margin-top: 1em;
+    margin-left: 0;
+    margin-right: 0;
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
 }
 
 .card-body {
-  padding: 0.3rem;
+    padding: 0.3rem;
 }
 
 .doc-card {
-  padding-left: 3px;
-  padding-right: 3px;
+    padding-left: 3px;
+    padding-right: 3px;
 }
 
 .sub-document .card {
-  background: #AB47BC1F !important;
+    background: #AB47BC1F !important;
 }
 
 .theme-black .sub-document .card {
-  background: #37474F !important;
+    background: #37474F !important;
 }
 
 .sub-document .fit {
-  padding: 4px 4px 0 4px;
+    padding: 4px 4px 0 4px;
 }
 </style>

sist2-vue/src/components/DocFileTitle.vue

@@ -1,5 +1,5 @@
 <template>
-  <a :href="`f/${doc._id}`" class="file-title-anchor" target="_blank">
+  <a :href="`f/${doc._source.index}/${doc._id}`" class="file-title-anchor" target="_blank">
     <div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
          v-html="fileName() + ext(doc)"></div>
   </a>

sist2-vue/src/components/DocListItem.vue

@@ -17,7 +17,7 @@
           </div>
 
           <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-               :src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
+               :src="(doc._props.isGif && hover) ? `f/${doc._source.index}/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
                alt=""
                class="pointer fit-sm" @click="onThumbnailClick()">
           <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
@@ -50,6 +50,11 @@
           <span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
           <span v-if="doc._source.author">{{ doc._source.author }}</span>
         </div>
+
+        <!-- Featured line -->
+        <div style="display: flex">
+          <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
+        </div>
       </div>
     </div>
   </b-list-group-item>
@@ -61,10 +66,11 @@ import DocFileTitle from "@/components/DocFileTitle";
 import DocInfoModal from "@/components/DocInfoModal";
 import ContentDiv from "@/components/ContentDiv";
 import FileIcon from "@/components/icons/FileIcon";
+import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
 
 export default {
   name: "DocListItem",
-  components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+  components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer, FeaturedFieldsLine},
   props: ["doc"],
   data() {
     return {

sist2-vue/src/components/FeaturedFieldsLine.vue

@@ -0,0 +1,46 @@
+<template>
+    <div class="featured-line" v-html="featuredLineHtml"></div>
+</template>
+
+<script>
+import {humanDate, humanFileSize} from "@/util";
+
+function scopedEval(context, expr) {
+    const evaluator = Function.apply(null, [...Object.keys(context), "expr", "return eval(expr)"]);
+    return evaluator.apply(null, [...Object.values(context), expr]);
+}
+
+
+export default {
+    name: "FeaturedFieldsLine",
+    props: ["doc"],
+    computed: {
+        featuredLineHtml() {
+            if (this.$store.getters.optFeaturedFields === undefined) {
+                return "";
+            }
+
+            const scope = {doc: this.doc._source, humanDate: humanDate, humanFileSize: humanFileSize};
+
+            return this.$store.getters.optFeaturedFields
+                .replaceAll(/\$\{([^}]*)}/g, (match, g1) => {
+                    return scopedEval(scope, g1);
+                });
+        }
+    }
+}
+</script>
+
+<style scoped>
+
+.featured-line {
+    font-size: 90%;
+    font-family: 'Source Sans Pro', 'Helvetica Neue', Arial, sans-serif;
+    color: #424242;
+    padding-left: 2px;
+}
+
+.theme-black .featured-line {
+    color: #bebebe;
+}
+</style>

sist2-vue/src/components/FullThumbnail.vue

@@ -6,13 +6,13 @@
     </div>
 
     <div
-        v-if="doc._props.isImage && !hover && doc._props.tnW / doc._props.tnH < 5"
+        v-if="doc._props.isImage && doc._props.imageAspectRatio < 5"
         class="card-img-overlay"
         :class="{'small-badge': smallBadge}">
       <span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
     </div>
 
-    <div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0 && !hover"
+    <div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0"
          class="card-img-overlay"
          :class="{'small-badge': smallBadge}">
       <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
@@ -63,17 +63,20 @@ export default {
   },
   computed: {
     tnSrc() {
-      const doc = this.doc;
-      const props = doc._props;
-      if (props.isGif && this.hover) {
-        return `f/${doc._id}`;
-      }
-      return (this.currentThumbnailNum === 0)
-          ? `t/${doc._source.index}/${doc._id}`
-          : `t/${doc._source.index}/${doc._id}${String(this.currentThumbnailNum).padStart(4, "0")}`;
+      return this.getThumbnailSrc(this.currentThumbnailNum);
     },
   },
   methods: {
+    getThumbnailSrc(thumbnailNum) {
+      const doc = this.doc;
+      const props = doc._props;
+      if (props.isGif && this.hover) {
+        return `f/${doc._source.index}/${doc._id}`;
+      }
+      return (this.currentThumbnailNum === 0)
+          ? `t/${doc._source.index}/${doc._id}`
+          : `t/${doc._source.index}/${doc._id}/${String(thumbnailNum).padStart(4, "0")}`;
+    },
     humanTime: humanTime,
     onThumbnailClick() {
       this.$emit("onThumbnailClick");
@@ -86,9 +89,14 @@ export default {
     },
     onTnEnter() {
       this.hover = true;
+      const start = Date.now()
       if (this.doc._props.hasVidPreview) {
-        this.currentThumbnailNum += 1;
-        this.scheduleNextTnNum();
+        let img = new Image();
+        img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
+        img.onload = () => {
+          this.currentThumbnailNum += 1;
+          this.scheduleNextTnNum(Date.now() - start);
+        }
       }
     },
     onTnLeave() {
@@ -99,17 +107,23 @@ export default {
         this.timeoutId = null;
       }
     },
-    scheduleNextTnNum() {
-      const INTERVAL = this.$store.state.optVidPreviewInterval ?? 700;
+    scheduleNextTnNum(offset = 0) {
+      const INTERVAL = (this.$store.state.optVidPreviewInterval ?? 700) - offset;
       this.timeoutId = window.setTimeout(() => {
+        const start = Date.now();
         if (!this.hover) {
           return;
         }
-        this.scheduleNextTnNum();
         if (this.currentThumbnailNum === this.doc._props.tnNum - 1) {
           this.currentThumbnailNum = 0;
+          this.scheduleNextTnNum();
         } else {
-          this.currentThumbnailNum += 1;
+          let img = new Image();
+          img.src = this.getThumbnailSrc(this.currentThumbnailNum + 1);
+          img.onload = () => {
+            this.currentThumbnailNum += 1;
+            this.scheduleNextTnNum(Date.now() - start);
+          }
         }
       }, INTERVAL);
     },
@@ -152,17 +166,18 @@ export default {
 }
 
 .badge-resolution {
-  color: #212529;
-  background-color: #FFC107;
+  color: #c6c6c6;
+  background-color: #272727CC;
+  padding: 2px 3px;
 }
 
 .card-img-overlay {
   pointer-events: none;
-  padding: 0.75rem;
-  bottom: unset;
-  top: 0;
+  padding: 2px 6px;
+  bottom: 4px;
+  top: unset;
   left: unset;
-  right: unset;
+  right: 0;
 }
 
 .small-badge {

sist2-vue/src/components/LazyContentDiv.vue

@@ -1,94 +1,167 @@
 <template>
-  <Preloader v-if="loading"></Preloader>
-  <div v-else-if="content" class="content-div" v-html="content"></div>
+    <Preloader v-if="loading"></Preloader>
+    <div v-else-if="content">
+        <b-form inline class="my-2" v-if="ModelsRepo.getOptions().length > 0">
+            <b-checkbox class="ml-auto mr-2" :checked="optAutoAnalyze"
+                        @input="setOptAutoAnalyze($event); $store.dispatch('updateConfiguration')">
+                {{ $t("ml.auto") }}
+            </b-checkbox>
+            <b-button :disabled="mlPredictionsLoading || mlLoading" @click="mlAnalyze" variant="primary"
+            >{{ $t("ml.analyzeText") }}
+            </b-button>
+            <b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="mlModel">
+                <b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()">{{ opt.text }}
+                </b-select-option>
+            </b-select>
+        </b-form>
+
+        <b-progress v-if="mlLoading" variant="warning" show-progress :max="1" class="mb-3"
+        >
+            <b-progress-bar :value="modelLoadingProgress">
+                <strong>{{ ((modelLoadingProgress * modelSize) / (1024 * 1024)).toFixed(1) }}MB / {{
+                    (modelSize / (1024 * 1024)).toFixed(1)
+                    }}MB</strong>
+            </b-progress-bar>
+        </b-progress>
+
+        <b-progress v-if="mlPredictionsLoading" variant="primary" :value="modelPredictionProgress"
+                    :max="content.length" class="mb-3"></b-progress>
+
+        <AnalyzedContentSpansContainer v-if="analyzedContentSpans.length > 0"
+                                       :spans="analyzedContentSpans" :text="rawContent"></AnalyzedContentSpansContainer>
+        <div v-else class="content-div" v-html="content"></div>
+    </div>
 </template>
 
 <script>
 import Sist2Api from "@/Sist2Api";
 import Preloader from "@/components/Preloader";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 import store from "@/store";
+import BertNerModel from "@/ml/BertNerModel";
+import AnalyzedContentSpansContainer from "@/components/AnalyzedContentSpanContainer.vue";
+import ModelsRepo from "@/ml/modelsRepo";
+import {mapGetters, mapMutations} from "vuex";
 
 export default {
-  name: "LazyContentDiv",
-  components: {Preloader},
-  props: ["docId"],
-  data() {
-    return {
-      content: "",
-      loading: true
+    name: "LazyContentDiv",
+    components: {AnalyzedContentSpansContainer, Preloader},
+    props: ["docId"],
+    data() {
+        return {
+            ModelsRepo,
+            content: "",
+            rawContent: "",
+            loading: true,
+            modelLoadingProgress: 0,
+            modelPredictionProgress: 0,
+            mlPredictionsLoading: false,
+            mlLoading: false,
+            mlModel: null,
+            analyzedContentSpans: []
+        }
+    },
+    mounted() {
+
+        if (this.$store.getters.optMlDefaultModel) {
+            this.mlModel = this.$store.getters.optMlDefaultModel
+        } else {
+            this.mlModel = ModelsRepo.getDefaultModel();
+        }
+
+        Sist2Api
+            .getDocument(this.docId, this.$store.state.optHighlight, this.$store.state.fuzzy)
+            .then(doc => {
+                this.loading = false;
+
+                if (doc) {
+                    this.content = this.getContent(doc)
+                }
+
+                if (this.optAutoAnalyze) {
+                    this.mlAnalyze();
+                }
+            });
+    },
+    computed: {
+        ...mapGetters(["optAutoAnalyze"]),
+        modelSize() {
+            const modelData = ModelsRepo.data[this.mlModel];
+            if (!modelData) {
+                return 0;
+            }
+            return modelData.size;
+        }
+    },
+    methods: {
+        ...mapMutations(["setOptAutoAnalyze"]),
+        getContent(doc) {
+            this.rawContent = doc._source.content;
+
+            if (!doc.highlight) {
+                return doc._source.content;
+            }
+
+            if (doc.highlight["content.nGram"]) {
+                return doc.highlight["content.nGram"][0];
+            }
+            if (doc.highlight.content) {
+                return doc.highlight.content[0];
+            }
+        },
+        async getMlModel() {
+            if (this.$store.getters.mlModel.name !== this.mlModel) {
+                this.mlLoading = true;
+                this.modelLoadingProgress = 0;
+                const modelInfo = ModelsRepo.data[this.mlModel];
+
+                const model = new BertNerModel(
+                    modelInfo.vocabUrl,
+                    modelInfo.modelUrl,
+                    modelInfo.id2label,
+                )
+
+                await model.init(progress => this.modelLoadingProgress = progress);
+                this.$store.commit("setMlModel", {model, name: this.mlModel});
+
+                this.mlLoading = false;
+                return model
+            }
+
+            return this.$store.getters.mlModel.model;
+        },
+        async mlAnalyze() {
+            if (!this.content) {
+                return;
+            }
+
+            const modelInfo = ModelsRepo.data[this.mlModel];
+            if (modelInfo === undefined) {
+                return;
+            }
+
+            this.$store.commit("setOptMlDefaultModel", this.mlModel);
+            await this.$store.dispatch("updateConfiguration");
+
+            const model = await this.getMlModel();
+
+            this.analyzedContentSpans = [];
+
+            this.mlPredictionsLoading = true;
+
+            await model.predict(this.rawContent, results => {
+                results.forEach(result => result.label = modelInfo.humanLabels[result.label]);
+                this.analyzedContentSpans.push(...results);
+                this.modelPredictionProgress = results[results.length - 1].wordIndex;
+            });
+            this.mlPredictionsLoading = false;
+        }
     }
-  },
-  mounted() {
-    const query = Sist2Query.searchQuery();
-
-    if (this.$store.state.optHighlight) {
-
-      const fields = this.$store.state.fuzzy
-          ? {"content.nGram": {}}
-          : {content: {}};
-
-      query.highlight = {
-        pre_tags: ["<mark>"],
-        post_tags: ["</mark>"],
-        number_of_fragments: 0,
-        fields,
-      };
-
-      if (!store.state.sist2Info.esVersionLegacy) {
-        query.highlight.max_analyzed_offset = 999_999;
-      }
-    }
-
-    if ("function_score" in query.query) {
-      query.query = query.query.function_score.query;
-    }
-
-    if (!("must" in query.query.bool)) {
-      query.query.bool.must = [];
-    } else if (!Array.isArray(query.query.bool.must)) {
-      query.query.bool.must = [query.query.bool.must];
-    }
-
-    query.query.bool.must.push({match: {_id: this.docId}});
-
-    delete query["sort"];
-    delete query["aggs"];
-    delete query["search_after"];
-    delete query.query["function_score"];
-
-    query._source = {
-      includes: ["content", "name", "path", "extension"]
-    }
-
-    query.size = 1;
-
-    Sist2Api.esQuery(query).then(resp => {
-      this.loading = false;
-      if (resp.hits.hits.length === 1) {
-        this.content = this.getContent(resp.hits.hits[0]);
-      } else {
-        console.log("FIXME: could not get content")
-        console.log(resp)
-      }
-    });
-  },
-  methods: {
-    getContent(doc) {
-      if (!doc.highlight) {
-        return doc._source.content;
-      }
-
-      if (doc.highlight["content.nGram"]) {
-        return doc.highlight["content.nGram"][0];
-      }
-      if (doc.highlight.content) {
-        return doc.highlight.content[0];
-      }
-    }
-  }
 }
 </script>
 
-<style scoped>
+<style>
+.progress-bar {
+    transition: none;
+}
 </style>

sist2-vue/src/components/Lightbox.vue

@@ -160,9 +160,13 @@ export default {
     },
     onSlideChange() {
       // Pause all videos when changing slide
-      document.getElementsByTagName("video").forEach((el) => {
+      const videos = document.getElementsByTagName("video");
+      if (videos.length === 0) {
+        return
+      }
+      for (let el of videos) {
         el.pause();
-      });
+      }
     },
   }
 

sist2-vue/src/components/MimePicker.vue

@@ -9,7 +9,7 @@ import InspireTreeDOM from "inspire-tree-dom";
 import "inspire-tree-dom/dist/inspire-tree-light.min.css";
 import {getSelectedTreeNodes, getTreeNodeAttributes} from "@/util";
 import Sist2Api from "@/Sist2Api";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 
 export default {
   name: "MimePicker",

sist2-vue/src/components/NavBar.vue

@@ -10,7 +10,7 @@
     <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
       v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
         href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
-        target="_blank">legacyES</a></span>
+        target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
     </span>
 
     <span v-if="$store && $store.state.sist2Info" class="tagline" v-html="tagline()"></span>

sist2-vue/src/components/PathTree.vue

@@ -1,40 +1,41 @@
 <template>
-  <div>
-    <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
-      <div class="input-group-prepend">
+    <div>
+        <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
+            <div class="input-group-prepend">
 
-        <b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
-          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
-            <path
-                fill="currentColor"
-                d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
-            />
-          </svg>
-        </b-button>
-      </div>
+                <b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
+                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
+                        <path
+                                fill="currentColor"
+                                d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
+                        />
+                    </svg>
+                </b-button>
+            </div>
+
+            <VueSimpleSuggest
+                    class="form-control-fix-flex"
+                    @input="setPathText"
+                    :value="getPathText"
+                    :list="suggestPath"
+                    :max-suggestions="0"
+                    :placeholder="$t('pathBar.placeholder')"
+                    :debounce="200"
+            >
+                <!-- Suggestion item template-->
+                <div slot="suggestion-item" slot-scope="{ suggestion, query }">
+                    <div class="suggestion-line" :title="suggestion">
+                        <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
+                    </div>
+                </div>
+            </VueSimpleSuggest>
 
-      <VueSimpleSuggest
-          class="form-control-fix-flex"
-          @input="setPathText"
-          :value="getPathText"
-          :list="suggestPath"
-          :max-suggestions="0"
-          :placeholder="$t('pathBar.placeholder')"
-      >
-        <!-- Suggestion item template-->
-        <div slot="suggestion-item" slot-scope="{ suggestion, query }">
-          <div class="suggestion-line" :title="suggestion">
-            <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
-          </div>
         </div>
-      </VueSimpleSuggest>
 
+        <b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
+            <div id="pathTree"></div>
+        </b-modal>
     </div>
-
-    <b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
-      <div id="pathTree"></div>
-    </b-modal>
-  </div>
 </template>
 
 <script>
@@ -48,198 +49,153 @@ import VueSimpleSuggest from 'vue-simple-suggest'
 import 'vue-simple-suggest/dist/styles.css' // Optional CSS
 
 export default {
-  name: "PathTree",
-  components: {
-    VueSimpleSuggest
-  },
-  data() {
-    return {
-      mimeTree: null,
-      pathItems: [],
-      tmpPath: ""
-
-    }
-  },
-  computed: {
-    ...mapGetters(["getPathText"])
-  },
-  mounted() {
-    this.$store.subscribe((mutation) => {
-      // Wait until indices are loaded to get the root paths
-      if (mutation.type === "setIndices") {
-        let pathTree = new InspireTree({
-          data: (node, resolve, reject) => {
-            return this.getNextDepth(node);
-          },
-          sort: "text"
-        });
-
-        this.$store.state.indices.forEach(idx => {
-          pathTree.addNode({
-            id: "/" + idx.id,
-            values: ["/" + idx.id],
-            text: `/[${idx.name}]`,
-            index: idx.id,
-            depth: 0,
-            children: true
-          })
-        });
-
-        new InspireTreeDOM(pathTree, {
-          target: "#pathTree"
-        });
-
-        pathTree.on("node.click", this.handleTreeClick);
-        pathTree.expand();
-      }
-    });
-  },
-  methods: {
-    ...mapMutations(["setPathText"]),
-    getSuggestionWithoutQueryPrefix(suggestion, query) {
-      return suggestion.slice(query.length)
+    name: "PathTree",
+    components: {
+        VueSimpleSuggest
     },
-    async getPathChoices() {
-      return new Promise(getPaths => {
-        const q = {
-          suggest: {
-            path: {
-              prefix: this.getPathText,
-              completion: {
-                field: "suggest-path",
-                skip_duplicates: true,
-                size: 10000
-              }
-            }
-          }
-        };
+    data() {
+        return {
+            mimeTree: null,
+            pathItems: [],
+            tmpPath: ""
 
-        Sist2Api.esQuery(q)
-            .then(resp => getPaths(resp["suggest"]["path"][0]["options"].map(opt => opt["_source"]["path"])));
-      })
-    },
-    async suggestPath(term) {
-      if (!this.$store.state.optSuggestPath) {
-        return []
-      }
-
-      term = term.toLowerCase();
-
-      const choices = await this.getPathChoices();
-
-      let matches = [];
-      for (let i = 0; i < choices.length; i++) {
-        if (~choices[i].toLowerCase().indexOf(term)) {
-          matches.push(choices[i]);
         }
-      }
-      return matches.sort((a, b) => a.length - b.length);
     },
-    getNextDepth(node) {
-      const q = {
-        query: {
-          bool: {
-            filter: [
-              {term: {index: node.index}},
-              {range: {_depth: {gte: node.depth + 1, lte: node.depth + 3}}},
-            ]
-          }
-        },
-        aggs: {
-          paths: {
-            terms: {
-              field: "path",
-              size: 10000
+    computed: {
+        ...mapGetters(["getPathText"])
+    },
+    mounted() {
+        this.$store.subscribe((mutation) => {
+            // Wait until indices are loaded to get the root paths
+            if (mutation.type === "setIndices") {
+                let pathTree = new InspireTree({
+                    data: (node, resolve, reject) => {
+                        return this.getNextDepth(node);
+                    },
+                    sort: "text"
+                });
+
+                this.$store.state.indices.forEach(idx => {
+                    pathTree.addNode({
+                        id: "/" + idx.id,
+                        values: ["/" + idx.id],
+                        text: `/[${idx.name}]`,
+                        index: idx.id,
+                        depth: 0,
+                        children: true
+                    })
+                });
+
+                new InspireTreeDOM(pathTree, {
+                    target: "#pathTree"
+                });
+
+                pathTree.on("node.click", this.handleTreeClick);
+                pathTree.expand();
             }
-          }
+        });
+    },
+    methods: {
+        ...mapMutations(["setPathText"]),
+        getSuggestionWithoutQueryPrefix(suggestion, query) {
+            return suggestion.slice(query.length)
         },
-        size: 0
-      };
+        async getPathChoices() {
+            return new Promise(getPaths => {
+                Sist2Api.getPathSuggestions(this.getPathText).then(getPaths);
+            });
+        },
+        async suggestPath(term) {
+            if (!this.$store.state.optSuggestPath) {
+                return []
+            }
 
-      if (node.depth > 0) {
-        q.query.bool.must = {
-          prefix: {
-            path: node.id,
-          }
-        };
-      }
+            term = term.toLowerCase();
 
-      return Sist2Api.esQuery(q).then(resp => {
-        const buckets = resp["aggregations"]["paths"]["buckets"];
-        if (!buckets) {
-          return false;
-        }
+            const choices = await this.getPathChoices();
 
-        const paths = [];
+            let matches = [];
+            for (let i = 0; i < choices.length; i++) {
+                if (~choices[i].toLowerCase().indexOf(term)) {
+                    matches.push(choices[i]);
+                }
+            }
+            return matches.sort((a, b) => a.length - b.length);
+        },
+        getNextDepth(node) {
+            return Sist2Api
+                .searchPaths(node.index, node.depth + 1, node.depth + 3, node.depth > 0 ? node.id : null)
+                .then(buckets => {
+                    const paths = [];
 
-        return buckets
-            .filter(bucket => bucket.key.length > node.id.length || node.id.startsWith("/"))
-            .sort((a, b) => a.key > b.key)
-            .map(bucket => {
+                    return buckets
+                        .filter(bucket => bucket.path.length > node.id.length || node.id.startsWith("/"))
+                        .sort((a, b) => a.path > b.path ? 1 : -1)
+                        .map(bucket => {
+                            if (paths.some(n => bucket.path.startsWith(n))) {
+                                return null;
+                            }
 
-              if (paths.some(n => bucket.key.startsWith(n))) {
-                return null;
-              }
+                            const name = node.id.startsWith("/") ? bucket.path : bucket.path.slice(node.id.length + 1);
 
-              const name = node.id.startsWith("/") ? bucket.key : bucket.key.slice(node.id.length + 1);
+                            paths.push(bucket.path);
 
-              paths.push(bucket.key);
+                            return {
+                                id: bucket.path,
+                                text: `${name}/ (${bucket.count})`,
+                                depth: node.depth + 1,
+                                index: node.index,
+                                values: [bucket.path],
+                                children: true,
+                            }
+                        })
+                        .filter(bucket => bucket !== null);
+                });
+        },
+        handleTreeClick(e, node, handler) {
+            if (node.depth !== 0) {
+                this.setPathText(node.id);
+                this.$refs['path-modal'].hide()
 
-              return {
-                id: bucket.key,
-                text: `${name}/ (${bucket.doc_count})`,
-                depth: node.depth + 1,
-                index: node.index,
-                values: [bucket.key],
-                children: true,
-              }
-            }).filter(x => x !== null)
-      });
+                this.$emit("search");
+            }
+
+            handler();
+        },
     },
-    handleTreeClick(e, node, handler) {
-      if (node.depth !== 0) {
-        this.setPathText(node.id);
-        this.$refs['path-modal'].hide()
-
-        this.$emit("search");
-      }
-
-      handler();
-    },
-  },
 }
 </script>
 
 <style scoped>
 #mimeTree {
-  max-height: 350px;
-  overflow: auto;
+    max-height: 350px;
+    overflow: auto;
 }
 
 .form-control-fix-flex {
-  flex: 1 1 auto;
-  width: 1%;
-  min-width: 0;
-  margin-bottom: 0;
+    flex: 1 1 auto;
+    width: 1%;
+    min-width: 0;
+    margin-bottom: 0;
 }
 
 .suggestion-line {
-  max-width: 100%;
-  white-space: nowrap;
-  overflow: hidden;
-  text-overflow: ellipsis;
-  line-height: 1.1;
+    max-width: 100%;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    line-height: 1.1;
 }
 
 </style>
 
 <style>
 .suggestions {
-  max-height: 250px;
-  overflow-y: auto;
+    max-height: 250px;
+    overflow-y: auto;
 }
 
 .theme-black .suggestions {
-  color: black
+    color: black
 }
 </style>

sist2-vue/src/components/ResultsCard.vue

@@ -1,40 +1,46 @@
 <template>
-  <b-card v-if="lastResultsLoaded" id="results">
-    <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
+    <b-card v-if="lastResultsLoaded" id="results">
+        <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
 
-    <div style="float: right">
-      <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
-          $t("details")
-        }}
-      </b-button>
+        <div style="float: right">
+            <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
+                $t("details")
+                }}
+            </b-button>
 
-      <template v-if="hitCount !== 0">
-        <SortSelect class="ml-2"></SortSelect>
+            <template v-if="hitCount !== 0">
+                <SortSelect class="ml-2"></SortSelect>
 
-        <DisplayModeToggle class="ml-2"></DisplayModeToggle>
-      </template>
-    </div>
+                <DisplayModeToggle class="ml-2"></DisplayModeToggle>
+            </template>
+        </div>
 
-    <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
-      <b-card>
-        <b-table :items="tableItems" small borderless bordered thead-class="hidden" class="mb-0"></b-table>
+        <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
+            <b-card>
+                <b-table :items="tableItems" small borderless thead-class="hidden" class="mb-0"></b-table>
 
-        <br/>
-        <h4>
-          {{$t("mimeTypes")}}
-          <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick"><ClipboardIcon/></b-button>
-        </h4>
-        <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
-        <b-table
-            v-else
-            sort-by="doc_count"
-            :sort-desc="true"
-            thead-class="hidden"
-            :items="$store.state.uiDetailsMimeAgg" small bordered class="mb-0"
-        ></b-table>
-      </b-card>
-    </b-collapse>
-  </b-card>
+                <template v-if="!$store.state.uiSqliteMode">
+
+                    <br/>
+                    <h4>
+                        {{ $t("mimeTypes") }}
+                        <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick">
+                            <ClipboardIcon/>
+                        </b-button>
+                    </h4>
+                    <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
+                    <b-table
+                            v-else
+                            sort-by="doc_count"
+                            :sort-desc="true"
+                            thead-class="hidden"
+                            bordered
+                            :items="$store.state.uiDetailsMimeAgg" small class="mb-0"
+                    ></b-table>
+                </template>
+            </b-card>
+        </b-collapse>
+    </b-card>
 </template>
 
 <script lang="ts">
@@ -44,91 +50,96 @@ import {humanFileSize} from "@/util";
 import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
 import SortSelect from "@/components/SortSelect.vue";
 import Preloader from "@/components/Preloader.vue";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
 
 export default Vue.extend({
-  name: "ResultsCard",
-  components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
-  created() {
+    name: "ResultsCard",
+    components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
+    created() {
 
-  },
-  computed: {
-    lastResultsLoaded() {
-      return this.$store.state.lastQueryResults != null;
     },
-    hitCount() {
-      return (this.$store.state.lastQueryResults as EsResult).aggregations.total_count.value;
+    computed: {
+        lastResultsLoaded() {
+            return this.$store.state.lastQueryResults != null;
+        },
+        hitCount() {
+            return (this.$store.state.firstQueryResults as EsResult).aggregations.total_count.value;
+        },
+        tableItems() {
+            const items = [];
+
+            if (!this.$store.state.uiSqliteMode) {
+                items.push({key: this.$t("queryTime"), value: this.took()});
+            }
+            items.push({key: this.$t("totalSize"), value: this.totalSize()});
+
+            return items;
+        }
     },
-    tableItems() {
-      const items = [];
+    methods: {
+        took() {
+            return (this.$store.state.lastQueryResults as EsResult).took + "ms";
+        },
+        totalSize() {
+            return humanFileSize((this.$store.state.firstQueryResults as EsResult).aggregations.total_size.value);
+        },
+        onToggle() {
+            const show = !document.getElementById("collapse-1").classList.contains("show");
+            this.$store.commit("setUiShowDetails", show);
 
+            if (this.$store.state.uiSqliteMode) {
+                return;
+            }
 
-      items.push({key: this.$t("queryTime"), value: this.took()});
-      items.push({key: this.$t("totalSize"), value: this.totalSize()});
+            if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
+                // Mime aggs are not updated automatically, update now
+                this.forceUpdateMimeAgg();
+            }
+        },
+        onCopyClick() {
+            let tsvString = "";
+            this.$store.state.uiDetailsMimeAgg.slice().sort((a, b) => b["doc_count"] - a["doc_count"]).forEach(row => {
+                tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
+            });
 
-      return items;
-    }
-  },
-  methods: {
-    took() {
-      return (this.$store.state.lastQueryResults as EsResult).took + "ms";
+            navigator.clipboard.writeText(tsvString);
+
+            this.$bvToast.toast(
+                this.$t("toast.copiedToClipboard"),
+                {
+                    title: null,
+                    noAutoHide: false,
+                    toaster: "b-toaster-bottom-right",
+                    headerClass: "hidden",
+                    bodyClass: "toast-body-info",
+                });
+        },
+        forceUpdateMimeAgg() {
+            const query = Sist2Query.searchQuery();
+            Sist2Api.getMimeTypes(query).then(({buckets}) => {
+                this.$store.commit("setUiDetailsMimeAgg", buckets);
+            });
+        }
     },
-    totalSize() {
-      return humanFileSize((this.$store.state.lastQueryResults as EsResult).aggregations.total_size.value);
-    },
-    onToggle() {
-      const show = !document.getElementById("collapse-1").classList.contains("show");
-      this.$store.commit("setUiShowDetails", show);
-
-      if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
-        // Mime aggs are not updated automatically, update now
-        this.forceUpdateMimeAgg();
-      }
-    },
-    onCopyClick() {
-      let tsvString = "";
-      this.$store.state.uiDetailsMimeAgg.slice().sort((a,b) => b["doc_count"] - a["doc_count"]).forEach(row => {
-        tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
-      });
-
-      navigator.clipboard.writeText(tsvString);
-
-      this.$bvToast.toast(
-          this.$t("toast.copiedToClipboard"),
-          {
-            title: null,
-            noAutoHide: false,
-            toaster: "b-toaster-bottom-right",
-            headerClass: "hidden",
-            bodyClass: "toast-body-info",
-          });
-    },
-    forceUpdateMimeAgg() {
-      const query = Sist2Query.searchQuery();
-      Sist2Api.getMimeTypes(query).then(({buckets}) => {
-        this.$store.commit("setUiDetailsMimeAgg", buckets);
-      });
-    }
-  },
 });
 
 </script>
 
 <style>
 #results {
-  margin-top: 1em;
+    margin-top: 1em;
 
-  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-  border-radius: 0;
-  border: none;
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
 }
 
 #results .card-body {
-  padding: 0.7em 1.25em;
+    padding: 0.7em 1.25em;
 }
 
 .hidden {
-  display: none;
+    display: none;
 }
 </style>

sist2-vue/src/components/SearchBar.vue

@@ -6,7 +6,7 @@
                     @input="setSearchText($event)"></b-form-input>
 
       <template #prepend>
-        <b-input-group-text>
+        <b-input-group-text v-if="!$store.state.uiSqliteMode">
           <b-form-checkbox :checked="fuzzy" title="Toggle fuzzy searching" @change="setFuzzy($event)">
             {{ $t("searchBar.fuzzy") }}
           </b-form-checkbox>

sist2-vue/src/components/TagContainer.vue

@@ -1,75 +1,78 @@
 <template>
-  <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
+    <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
 
-    <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static lazy>
-      <b-row>
-        <b-col style="flex-grow: 2" sm>
-          <VueSimpleSuggest
-              ref="suggest"
-              :value="tagText"
-              @select="setTagText($event)"
-              @input="setTagText($event)"
-              class="form-control-fix-flex"
-              style="margin-top: 17px"
-              :list="suggestTag"
-              :max-suggestions="0"
-              :placeholder="$t('saveTagPlaceholder')"
-          >
-            <!-- Suggestion item template-->
-            <div slot="suggestion-item" slot-scope="{ suggestion, query}"
-            >
-              <div class="suggestion-line">
+        <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static
+                 lazy>
+            <b-row>
+                <b-col style="flex-grow: 2" sm>
+                    <VueSimpleSuggest
+                            ref="suggest"
+                            :value="tagText"
+                            @select="setTagText($event)"
+                            @input="setTagText($event)"
+                            class="form-control-fix-flex"
+                            style="margin-top: 17px"
+                            :list="suggestTag"
+                            :max-suggestions="0"
+                            :placeholder="$t('saveTagPlaceholder')"
+                    >
+                        <!-- Suggestion item template-->
+                        <div slot="suggestion-item" slot-scope="{ suggestion, query}"
+                        >
+                            <div class="suggestion-line">
                 <span
-                    class="badge badge-suggestion"
-                    :style="{background: getBg(suggestion), color: getFg(suggestion)}"
+                        class="badge badge-suggestion"
+                        :style="{background: getBg(suggestion), color: getFg(suggestion)}"
                 >
                 <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
                 </span>
-              </div>
-            </div>
-          </VueSimpleSuggest>
-        </b-col>
-        <b-col class="mt-4">
-          <TwitterColorPicker v-model="color" triangle="hide" :width="252" class="mr-auto ml-auto"></TwitterColorPicker>
-        </b-col>
-      </b-row>
+                            </div>
+                        </div>
+                    </VueSimpleSuggest>
+                </b-col>
+                <b-col class="mt-4">
+                    <TwitterColorPicker v-model="color" triangle="hide" :width="252"
+                                        class="mr-auto ml-auto"></TwitterColorPicker>
+                </b-col>
+            </b-row>
 
-      <b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
-      </b-button>
-    </b-modal>
+            <b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
+            </b-button>
+        </b-modal>
 
 
-    <template v-for="tag in hit._tags">
-      <div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
+        <template v-for="tag in hit._tags">
+            <!-- User tag-->
+            <div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
         <span
-            :id="hit._id+tag.rawText"
-            :title="tag.text"
-            tabindex="-1"
-            class="badge pointer"
-            :style="badgeStyle(tag)" :class="badgeClass(tag)"
-            @click.right="onTagRightClick(tag, $event)"
+                :id="hit._id+tag.rawText"
+                :title="tag.text"
+                tabindex="-1"
+                class="badge pointer"
+                :style="badgeStyle(tag)" :class="badgeClass(tag)"
+                @click.right="onTagRightClick(tag, $event)"
         >{{ tag.text.split(".").pop() }}</span>
 
-        <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
-          <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{$t("deleteTag")}}</b-button>
-        </b-popover>
-      </div>
+                <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
+                    <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{ $t("deleteTag") }}</b-button>
+                </b-popover>
+            </div>
 
-      <span
-          v-else :key="tag.text"
-          class="badge"
-          :style="badgeStyle(tag)" :class="badgeClass(tag)"
-      >{{ tag.text.split(".").pop() }}</span>
-    </template>
+            <span
+                    v-else :key="tag.text"
+                    class="badge"
+                    :style="badgeStyle(tag)" :class="badgeClass(tag)"
+            >{{ tag.text.split(".").pop() }}</span>
+        </template>
 
-    <!-- Add button -->
-    <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{$t("addTag")}}</small>
+        <!-- Add button -->
+        <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{ $t("addTag") }}</small>
 
-    <!-- Size tag-->
-    <small v-else class="text-muted badge-size">{{
-        humanFileSize(hit._source.size)
-      }}</small>
-  </div>
+        <!-- Size tag-->
+        <small v-else class="text-muted badge-size" style="padding-left: 2px">{{
+            humanFileSize(hit._source.size)
+            }}</small>
+    </div>
 </template>
 
 <script>
@@ -80,170 +83,136 @@ import Sist2Api from "@/Sist2Api";
 import VueSimpleSuggest from 'vue-simple-suggest'
 
 export default Vue.extend({
-  components: {
-    "TwitterColorPicker": Twitter,
-    VueSimpleSuggest
-  },
-  props: ["hit"],
-  data() {
-    return {
-      showAddButton: false,
-      showModal: false,
-      tagText: null,
-      color: {
-        hex: "#e0e0e0",
-      },
-    }
-  },
-  computed: {
-    tagHover() {
-      return this.$store.getters["uiTagHover"];
-    }
-  },
-  methods: {
-    humanFileSize: humanFileSize,
-    getSuggestionWithoutQueryPrefix(suggestion, query) {
-      return suggestion.id.slice(query.length, -8)
+    components: {
+        "TwitterColorPicker": Twitter,
+        VueSimpleSuggest
     },
-    getBg(suggestion) {
-      return suggestion.id.slice(-7);
-    },
-    getFg(suggestion) {
-      return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
-    },
-    setTagText(value) {
-      this.$refs.suggest.clearSuggestions();
-
-      if (typeof value === "string") {
-        this.tagText = {
-          id: value,
-          title: value
-        };
-        return;
-      }
-
-      this.color = {
-        hex: "#" + value.id.split("#")[1]
-      }
-
-      this.tagText = value;
-    },
-    badgeClass(tag) {
-      return `badge-${tag.style}`;
-    },
-    badgeStyle(tag) {
-      return {
-        background: tag.bg,
-        color: tag.fg,
-      };
-    },
-    onTagHover(tag) {
-      if (tag.userTag) {
-        this.$store.commit("setUiTagHover", tag);
-      }
-    },
-    onTagLeave() {
-      this.$store.commit("setUiTagHover", null);
-    },
-    onTagDeleteClick(tag, e) {
-      this.hit._tags = this.hit._tags.filter(t => t !== tag);
-
-      Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
-        //toast
-        this.$store.commit("busUpdateWallItems");
-        this.$store.commit("busUpdateTags");
-      });
-    },
-    tagAdd() {
-      this.showModal = true;
-    },
-    saveTag() {
-      if (this.tagText.id.includes("#")) {
-        this.$bvToast.toast(
-            this.$t("toast.invalidTag"),
-            {
-              title: this.$t("toast.invalidTagTitle"),
-              noAutoHide: true,
-              toaster: "b-toaster-bottom-right",
-              headerClass: "toast-header-error",
-              bodyClass: "toast-body-error",
-            });
-        return;
-      }
-
-      let tag = this.tagText.id + this.color.hex.replace("#", ".#");
-      const userTags = this.hit._tags.filter(t => t.userTag);
-
-      if (userTags.find(t => t.rawText === tag) != null) {
-        this.$bvToast.toast(
-            this.$t("toast.dupeTag"),
-            {
-              title: this.$t("toast.dupeTagTitle"),
-              noAutoHide: true,
-              toaster: "b-toaster-bottom-right",
-              headerClass: "toast-header-error",
-              bodyClass: "toast-body-error",
-            });
-        return;
-      }
-
-      this.hit._tags.push(Sist2Api.createUserTag(tag));
-
-      Sist2Api.saveTag(tag, this.hit).then(() => {
-        this.tagText = null;
-        this.showModal = false;
-        this.$store.commit("busUpdateWallItems");
-        this.$store.commit("busUpdateTags");
-        // TODO: toast
-      });
-    },
-    async suggestTag(term) {
-      term = term.toLowerCase();
-
-      const choices = await this.getTagChoices(term);
-
-      let matches = [];
-      for (let i = 0; i < choices.length; i++) {
-        if (~choices[i].toLowerCase().indexOf(term)) {
-          matches.push(choices[i]);
-        }
-      }
-
-      return matches.sort().map(match => {
+    props: ["hit"],
+    data() {
         return {
-          title: match.split(".").slice(0,-1).join("."),
-          id: match
+            showAddButton: false,
+            showModal: false,
+            tagText: null,
+            color: {
+                hex: "#e0e0e0",
+            },
         }
-      });
     },
-    getTagChoices(prefix) {
-      return new Promise(getPaths => {
-        Sist2Api.esQuery({
-          suggest: {
-            tag: {
-              prefix: prefix,
-              completion: {
-                field: "suggest-tag",
-                skip_duplicates: true,
-                size: 10000
-              }
+    methods: {
+        humanFileSize: humanFileSize,
+        getSuggestionWithoutQueryPrefix(suggestion, query) {
+            return suggestion.id.slice(query.length, -8)
+        },
+        getBg(suggestion) {
+            return suggestion.id.slice(-7);
+        },
+        getFg(suggestion) {
+            return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
+        },
+        setTagText(value) {
+            this.$refs.suggest.clearSuggestions();
+
+            if (typeof value === "string") {
+                this.tagText = {
+                    id: value,
+                    title: value
+                };
+                return;
             }
-          }
-        }).then(resp => {
-          const result = [];
-          resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
-            tags.forEach(tag => {
-              const t = tag.slice(0, -8);
-              if (!result.find(x => x.slice(0, -8) === t)) {
-                result.push(tag);
-              }
+
+            this.color = {
+                hex: "#" + value.id.split("#")[1]
+            }
+
+            this.tagText = value;
+        },
+        badgeClass(tag) {
+            return `badge-${tag.style}`;
+        },
+        badgeStyle(tag) {
+            return {
+                background: tag.bg,
+                color: tag.fg,
+            };
+        },
+        onTagDeleteClick(tag, e) {
+            this.hit._tags = this.hit._tags.filter(t => t !== tag);
+
+            Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
+                //toast
+                this.$store.commit("busUpdateWallItems");
+                this.$store.commit("busUpdateTags");
             });
-          });
-          getPaths(result);
-        });
-      });
-    }
-  },
+        },
+        tagAdd() {
+            this.showModal = true;
+        },
+        saveTag() {
+            if (this.tagText.id.includes("#")) {
+                this.$bvToast.toast(
+                    this.$t("toast.invalidTag"),
+                    {
+                        title: this.$t("toast.invalidTagTitle"),
+                        noAutoHide: true,
+                        toaster: "b-toaster-bottom-right",
+                        headerClass: "toast-header-error",
+                        bodyClass: "toast-body-error",
+                    });
+                return;
+            }
+
+            let tag = this.tagText.id + this.color.hex.replace("#", ".#");
+            const userTags = this.hit._tags.filter(t => t.userTag);
+
+            if (userTags.find(t => t.rawText === tag) != null) {
+                this.$bvToast.toast(
+                    this.$t("toast.dupeTag"),
+                    {
+                        title: this.$t("toast.dupeTagTitle"),
+                        noAutoHide: true,
+                        toaster: "b-toaster-bottom-right",
+                        headerClass: "toast-header-error",
+                        bodyClass: "toast-body-error",
+                    });
+                return;
+            }
+
+            this.hit._tags.push(Sist2Api.createUserTag(tag));
+
+            Sist2Api.saveTag(tag, this.hit).then(() => {
+                this.tagText = null;
+                this.showModal = false;
+                this.$store.commit("busUpdateWallItems");
+                this.$store.commit("busUpdateTags");
+                // TODO: toast
+            });
+        },
+        async suggestTag(term) {
+            term = term.toLowerCase();
+
+            const choices = await this.getTagChoices(term);
+
+            let matches = [];
+            for (let i = 0; i < choices.length; i++) {
+                if (~choices[i].toLowerCase().indexOf(term)) {
+                    matches.push(choices[i]);
+                }
+            }
+
+            return matches.sort().map(match => {
+                return {
+                    title: match.split(".").slice(0, -1).join("."),
+                    id: match
+                }
+            });
+        },
+        getTagChoices(prefix) {
+            return new Promise(getPaths => {
+                Sist2Api.getTagSuggestions(prefix)
+                    .then(paths => getPaths(paths))
+            });
+        }
+    },
 });
 </script>
 
@@ -251,87 +220,87 @@ export default Vue.extend({
 
 
 .badge-video {
-  color: #FFFFFF;
-  background-color: #F27761;
+    color: #FFFFFF;
+    background-color: #F27761;
 }
 
 .badge-image {
-  color: #FFFFFF;
-  background-color: #AA99C9;
+    color: #FFFFFF;
+    background-color: #AA99C9;
 }
 
 .badge-audio {
-  color: #FFFFFF;
-  background-color: #00ADEF;
+    color: #FFFFFF;
+    background-color: #00ADEF;
 }
 
 .badge-user {
-  color: #212529;
-  background-color: #e0e0e0;
+    color: #212529;
+    background-color: #e0e0e0;
 }
 
 .badge-user:hover, .add-tag-button:hover {
-  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
+    box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
 }
 
 .badge-text {
-  color: #FFFFFF;
-  background-color: #FAAB3C;
+    color: #FFFFFF;
+    background-color: #FAAB3C;
 }
 
 .badge {
-  margin-right: 3px;
+    margin-right: 3px;
 }
 
 .badge-delete {
-  margin-right: -2px;
-  margin-left: 2px;
-  margin-top: -1px;
-  font-family: monospace;
-  font-size: 90%;
-  background: rgba(0, 0, 0, 0.2);
-  padding: 0.1em 0.4em;
-  color: white;
-  cursor: pointer;
+    margin-right: -2px;
+    margin-left: 2px;
+    margin-top: -1px;
+    font-family: monospace;
+    font-size: 90%;
+    background: rgba(0, 0, 0, 0.2);
+    padding: 0.1em 0.4em;
+    color: white;
+    cursor: pointer;
 }
 
 .badge-size {
-  width: 50px;
-  display: inline-block;
+    width: 50px;
+    display: inline-block;
 }
 
 .add-tag-button {
-  cursor: pointer;
-  color: #212529;
-  background-color: #e0e0e0;
-  width: 50px;
+    cursor: pointer;
+    color: #212529;
+    background-color: #e0e0e0;
+    width: 50px;
 }
 
 .badge {
-  user-select: none;
+    user-select: none;
 }
 
 .badge-suggestion {
-  font-size: 90%;
-  font-weight: normal;
+    font-size: 90%;
+    font-weight: normal;
 }
 </style>
 
 <style>
 .vc-twitter-body {
-  padding: 0 !important;
+    padding: 0 !important;
 }
 
 .vc-twitter {
-  box-shadow: none !important;
-  background: none !important;
+    box-shadow: none !important;
+    background: none !important;
 }
 
 .tooltip {
-  user-select: none;
+    user-select: none;
 }
 
 .toast {
-  border: none;
+    border: none;
 }
 </style>

sist2-vue/src/components/TagPicker.vue

@@ -1,13 +1,13 @@
 <template>
-  <div>
-    <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
-      <b-form-input :value="filter"
-                    :placeholder="$t('tagFilter')"
-                    @input="onFilter($event)"></b-form-input>
-    </b-input-group>
+    <div>
+        <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
+            <b-form-input :value="filter"
+                          :placeholder="$t('tagFilter')"
+                          @input="onFilter($event)"></b-form-input>
+        </b-input-group>
 
-    <div id="tagTree"></div>
-  </div>
+        <div id="tagTree"></div>
+    </div>
 </template>
 
 <script>
@@ -19,191 +19,195 @@ import {getSelectedTreeNodes} from "@/util";
 import Sist2Api from "@/Sist2Api";
 
 function resetState(node) {
-  node._tree.defaultState.forEach(function (val, prop) {
-    node.state(prop, val);
-  });
+    node._tree.defaultState.forEach(function (val, prop) {
+        node.state(prop, val);
+    });
 
-  return node;
+    return node;
 }
 
 function baseStateChange(prop, value, verb, node, deep) {
-  if (node.state(prop) !== value) {
-    node._tree.batch();
+    if (node.state(prop) !== value) {
+        node._tree.batch();
 
-    if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
-      resetState(node);
+        if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
+            resetState(node);
+        }
+
+        node.state(prop, value);
+
+        node._tree.emit('node.' + verb, node, false);
+
+        if (deep && node.hasChildren()) {
+            node.children.recurseDown(function (child) {
+                baseStateChange(prop, value, verb, child);
+            });
+        }
+
+        node.markDirty();
+        node._tree.end();
     }
 
-    node.state(prop, value);
-
-    node._tree.emit('node.' + verb, node, false);
-
-    if (deep && node.hasChildren()) {
-      node.children.recurseDown(function (child) {
-        baseStateChange(prop, value, verb, child);
-      });
-    }
-
-    node.markDirty();
-    node._tree.end();
-  }
-
-  return node;
+    return node;
 }
 
 function addTag(map, tag, id, count) {
-  const tags = tag.split(".");
+    const tags = tag.split(".");
 
-  const child = {
-    id: id,
-    count: count,
-    text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
-    name: tags[0],
-    children: [],
-    // Overwrite base functions
-    blur: function () {
-      // noop
-    },
-    select: function () {
-      this.state("selected", true);
-      return this.check()
-    },
-    deselect: function () {
-      this.state("selected", false);
-      return this.uncheck()
-    },
-    uncheck: function () {
-      baseStateChange('checked', false, 'unchecked', this, false);
-      this.state('indeterminate', false);
+    const child = {
+        id: id,
+        count: count,
+        text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
+        name: tags[0],
+        children: [],
+        // Overwrite base functions
+        blur: function () {
+            // noop
+        },
+        select: function () {
+            this.state("selected", true);
+            return this.check()
+        },
+        deselect: function () {
+            this.state("selected", false);
+            return this.uncheck()
+        },
+        uncheck: function () {
+            baseStateChange('checked', false, 'unchecked', this, false);
+            this.state('indeterminate', false);
 
-      if (this.hasParent()) {
-        this.getParent().refreshIndeterminateState();
-      }
+            if (this.hasParent()) {
+                this.getParent().refreshIndeterminateState();
+            }
 
-      this._tree.end();
-      return this;
-    },
-    check: function () {
-      baseStateChange('checked', true, 'checked', this, false);
+            this._tree.end();
+            return this;
+        },
+        check: function () {
+            baseStateChange('checked', true, 'checked', this, false);
 
-      if (this.hasParent()) {
-        this.getParent().refreshIndeterminateState();
-      }
+            if (this.hasParent()) {
+                this.getParent().refreshIndeterminateState();
+            }
 
-      this._tree.end();
-      return this;
+            this._tree.end();
+            return this;
+        }
+    };
+
+    let found = false;
+    map.forEach(node => {
+        if (node.name === child.name) {
+            found = true;
+            if (tags.length !== 1) {
+                addTag(node.children, tags.slice(1).join("."), id, count);
+            } else {
+                // Same name, different color
+                console.error("FIXME: Duplicate tag?")
+                console.trace(node)
+            }
+        }
+    });
+    if (!found) {
+        if (tags.length !== 1) {
+            addTag(child.children, tags.slice(1).join("."), id, count);
+            map.push(child);
+        } else {
+            map.push(child);
+        }
     }
-  };
-
-  let found = false;
-  map.forEach(node => {
-    if (node.name === child.name) {
-      found = true;
-      if (tags.length !== 1) {
-        addTag(node.children, tags.slice(1).join("."), id, count);
-      } else {
-        // Same name, different color
-        console.error("FIXME: Duplicate tag?")
-        console.trace(node)
-      }
-    }
-  });
-  if (!found) {
-    if (tags.length !== 1) {
-      addTag(child.children, tags.slice(1).join("."), id, count);
-      map.push(child);
-    } else {
-      map.push(child);
-    }
-  }
 }
 
 export default {
-  name: "TagPicker",
-  props: ["showSearchBar"],
-  data() {
-    return {
-      tagTree: null,
-      loadedFromArgs: false,
-      filter: ""
-    }
-  },
-  mounted() {
-    this.$store.subscribe((mutation) => {
-      if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
-        this.initializeTree();
-        this.updateTree();
-      } else if (mutation.type === "busUpdateTags") {
-        window.setTimeout(this.updateTree, 2000);
-      }
-    });
-  },
-  methods: {
-    onFilter(value) {
-      this.filter = value;
-      this.tagTree.search(value);
-    },
-    initializeTree() {
-      const tagMap = [];
-      this.tagTree = new InspireTree({
-        selection: {
-          mode: "checkbox",
-          autoDeselect: false,
-        },
-        checkbox: {
-          autoCheckChildren: false,
-        },
-        data: tagMap
-      });
-      new InspireTreeDOM(this.tagTree, {
-        target: '#tagTree'
-      });
-      this.tagTree.on("node.state.changed", this.handleTreeClick);
-    },
-    updateTree() {
-      // TODO: remember which tags are selected and restore?
-      const tagMap = [];
-      Sist2Api.getTags().then(tags => {
-        tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
-        this.tagTree.removeAll();
-        this.tagTree.addNodes(tagMap);
-
-        if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
-          this.$store.state._onLoadSelectedTags.forEach(mime => {
-            this.tagTree.node(mime).select();
-            this.loadedFromArgs = true;
-          });
+    name: "TagPicker",
+    props: ["showSearchBar"],
+    data() {
+        return {
+            tagTree: null,
+            loadedFromArgs: false,
+            filter: ""
         }
-      });
     },
-    handleTreeClick(node, e) {
-      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
-          || e === "matched" || e === "hidden") {
-        return;
-      }
+    mounted() {
+        this.$store.subscribe((mutation) => {
+            if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
+                this.initializeTree();
+                this.updateTree();
+            } else if (mutation.type === "busUpdateTags") {
+                if (this.$store.state.uiSqliteMode) {
+                    this.updateTree();
+                } else {
+                    window.setTimeout(this.updateTree, 2000);
+                }
+            }
+        });
+    },
+    methods: {
+        onFilter(value) {
+            this.filter = value;
+            this.tagTree.search(value);
+        },
+        initializeTree() {
+            const tagMap = [];
+            this.tagTree = new InspireTree({
+                selection: {
+                    mode: "checkbox",
+                    autoDeselect: false,
+                },
+                checkbox: {
+                    autoCheckChildren: false,
+                },
+                data: tagMap
+            });
+            new InspireTreeDOM(this.tagTree, {
+                target: '#tagTree'
+            });
+            this.tagTree.on("node.state.changed", this.handleTreeClick);
+        },
+        updateTree() {
+            // TODO: remember which tags are selected and restore?
+            const tagMap = [];
+            Sist2Api.getTags().then(tags => {
+                tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
+                this.tagTree.removeAll();
+                this.tagTree.addNodes(tagMap);
 
-      this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
-    },
-  }
+                if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
+                    this.$store.state._onLoadSelectedTags.forEach(mime => {
+                        this.tagTree.node(mime).select();
+                        this.loadedFromArgs = true;
+                    });
+                }
+            });
+        },
+        handleTreeClick(node, e) {
+            if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
+                || e === "matched" || e === "hidden") {
+                return;
+            }
+
+            this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
+        },
+    }
 }
 </script>
 
 <style scoped>
 #mimeTree {
-  max-height: 350px;
-  overflow: auto;
+    max-height: 350px;
+    overflow: auto;
 }
 </style>
 <style>
 .inspire-tree .focused > .wholerow {
-  border: none;
+    border: none;
 }
 
 #tag-picker-filter-bar {
-  padding: 10px 4px 4px;
+    padding: 10px 4px 4px;
 }
 
 .theme-black .inspire-tree .matched > .wholerow {
-  background: rgba(251, 191, 41, 0.25);
+    background: rgba(251, 191, 41, 0.25);
 }
 </style>

sist2-vue/src/i18n/messages.ts

@@ -1,14 +1,14 @@
 export default {
     en: {
         filePage: {
-          notFound: "Not found"
+            notFound: "Not found"
         },
         searchBar: {
             simple: "Search",
             advanced: "Advanced search",
             fuzzy: "Fuzzy"
         },
-        addTag: "Add",
+        addTag: "Tag",
         deleteTag: "Delete",
         download: "Download",
         and: "and",
@@ -17,6 +17,7 @@ export default {
         mimeTypes: "Media types",
         tags: "Tags",
         tagFilter: "Filter tags",
+        forExample: "For example:",
         help: {
             simpleSearch: "Simple search",
             advancedSearch: "Advanced search",
@@ -48,6 +49,7 @@ export default {
         configReset: "Reset configuration",
         searchOptions: "Search options",
         treemapOptions: "Treemap options",
+        mlOptions: "Machine learning options",
         displayOptions: "Display options",
         opt: {
             lang: "Language",
@@ -55,7 +57,7 @@ export default {
             fuzzy: "Set fuzzy search by default",
             searchInPath: "Enable matching query against document path",
             suggestPath: "Enable auto-complete in path filter bar",
-            fragmentSize: "Highlight context size in characters",
+            fragmentSize: "Highlight context size",
             queryMode: "Search mode",
             displayMode: "Display",
             columns: "Column count",
@@ -75,7 +77,12 @@ export default {
             useDatePicker: "Use a Date Picker component rather than a slider",
             vidPreviewInterval: "Video preview frame duration in ms",
             simpleLightbox: "Disable animations in image viewer",
-            showTagPickerFilter: "Display the tag filter bar"
+            showTagPickerFilter: "Display the tag filter bar",
+            featuredFields: "Featured fields Javascript template string. Will appear in the search results.",
+            featuredFieldsList: "Available variables",
+            autoAnalyze: "Automatically analyze text",
+            defaultModel: "Default model",
+            mlRepositories: "Model repositories (one per line)"
         },
         queryMode: {
             simple: "Simple",
@@ -83,7 +90,9 @@ export default {
         },
         lang: {
             en: "English",
+            de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -167,6 +176,186 @@ export default {
             selectedIndex: "selected index",
             selectedIndices: "selected indices",
         },
+        ml: {
+            analyzeText: "Analyze",
+            auto: "Auto",
+            repoFetchError: "Failed to get list of models. Check browser console for more details.",
+            repoFetchErrorTitle: "Could not fetch model repositories",
+        }
+    },
+    de: {
+        filePage: {
+            notFound: "Nicht gefunden"
+        },
+        searchBar: {
+            simple: "Suche",
+            advanced: "Erweiterte Suche",
+            fuzzy: "Fuzzy"
+        },
+        addTag: "Tag",
+        deleteTag: "Löschen",
+        download: "Herunterladen",
+        and: "und",
+        page: "Seite",
+        pages: "Seiten",
+        mimeTypes: "Medientypen",
+        tags: "Tags",
+        tagFilter: "Tags filtern",
+        forExample: "Zum Beispiel:",
+        help: {
+            simpleSearch: "Einfache Suche",
+            advancedSearch: "Erweiterte Suche",
+            help: "Hilfe",
+            term: "<BEGRIFF>",
+            and: "UND Operator",
+            or: "ODER Operator",
+            not: "negiert einen einzelnen Begriff",
+            quotes: "liefert Treffer, wenn die Abfolge in der genauen Reihenfolge gefunden wird",
+            prefix: "liefert Treffer, wenn die Abfolge einen solchen Präfix hat",
+            parens: "gruppiert Ausdrücke",
+            tildeTerm: "liefert Treffer, im gegebenen 'Editierabstand'",
+            tildePhrase: "liefert Treffer, mit dem Ausdruck. Erfolgt die gegebene Anzahl zwischenstehnde Nicht-Treffer-Wörter.",
+            example1:
+                "Zum Beispiel: <code>\"fried eggs\" +(eggplant | potato) -frittata</code> wird " +
+                "<i>fried eggs</i> und <i>eggplant</i> oder <i>potato</i> finden, aber keine Ergebnisse, " +
+                "die <i>frittata</i> enthalten.",
+            defaultOperator:
+                "Wenn weder <code>+</code> noch <code>|</code> angegeben sind, ist " +
+                "<code>+</code> (and) der Standard.",
+            fuzzy:
+                "Wenn <b>Fuzzy</b> aktiviert ist, werden Teil-Treffer (3-grams) ebenfalls akzeptiert.",
+            moreInfoSimple: "Für weitere Informationen s.<a target=\"_blank\" " +
+                "rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">Elasticsearch Dokumentation</a>",
+            moreInfoAdvanced: "Für die Dokumentation der erweiterten Suche s. <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">Elasticsearch Dokumentation</a>"
+        },
+        config: "Konfiguration",
+        configDescription: "Konfiguration wird in Echtzeit für diesen Browser gespeichert.",
+        configReset: "Konfiguration zurücksetzen",
+        searchOptions: "Such-Optionen",
+        treemapOptions: "Kacheldiagramm-Optionen",
+        displayOptions: "Anzeige-Optionen",
+        opt: {
+            lang: "Sprache",
+            highlight: "Aktiviere Hervorhebung von Treffern",
+            fuzzy: "Aktiviere Fuzzy-Suche standardmäßig",
+            searchInPath: "Abgleich der Abfrage mit dem Dokumentpfad aktivieren",
+            suggestPath: "Aktiviere Auto-Vervollständigung in Pfadfilter-Leiste",
+            fragmentSize: "Kontextgröße",
+            queryMode: "Such-Modus",
+            displayMode: "Ansicht",
+            columns: "Anzahl Spalten",
+            treemapType: "Kacheldiagramme Typ",
+            treemapTiling: "Kacheldiagramm Tiling",
+            treemapColorGroupingDepth: "Kacheldiagramme Gruppierungsfarbe Tiefe (flach)",
+            treemapColor: "Kacheldiagramme Farbe (kaskadiert)",
+            treemapSize: "Kacheldiagramm Größe",
+            theme: "Theme",
+            lightboxLoadOnlyCurrent: "keine Bilder in voller Größe für benachbachte Slides im Image-Viewer vorab laden.",
+            slideDuration: "Slide Dauer",
+            resultSize: "Anzahl Treffer pro Seite",
+            tagOrOperator: "Verwende ODER Operator bei der Angabe mehrere Tags.",
+            hideDuplicates: "Verstecke Duplikate basierend auf der Prüfsumme",
+            hideLegacy: "Verstecke die 'legacyES' Elasticsearch Notiz",
+            updateMimeMap: "Aktualisiere Medientyp-Baum in Echtzeit",
+            useDatePicker: "Benutze Datumswähler statt Schieber",
+            vidPreviewInterval: "Videovorschau Framedauer in ms",
+            simpleLightbox: "Schalte Animationen im Image-Viewer ab",
+            showTagPickerFilter: "Zeige die Tag-Filter-Leiste",
+            featuredFields: "Variablen, welche zusätzlich in den Suchergebnissen angezeigt werden können.",
+            featuredFieldsList: "verfügbare Variablen"
+        },
+        queryMode: {
+            simple: "Einfach",
+            advanced: "Erweitert",
+        },
+        lang: {
+            en: "English",
+            de: "Deutsch",
+            fr: "Français",
+            pl: "Polski",
+            "zh-CN": "简体中文",
+        },
+        displayMode: {
+            grid: "Gitter",
+            list: "Liste",
+        },
+        columns: {
+            auto: "Auto"
+        },
+        treemapType: {
+            cascaded: "kaskadiert",
+            flat: "flach (kompakt)"
+        },
+        treemapSize: {
+            small: "klein",
+            medium: "mittel",
+            large: "groß",
+            xLarge: "sehr groß",
+            xxLarge: "riesig",
+            custom: "eigene",
+        },
+        treemapTiling: {
+            binary: "binär",
+            squarify: "quadratisch",
+            slice: "Slice",
+            dice: "Dice",
+            sliceDice: "Slice & Dice",
+        },
+        theme: {
+            light: "Hell",
+            black: "Dunkel"
+        },
+        hit: "Treffer",
+        hits: "Treffer",
+        details: "Details",
+        stats: "Statistiken",
+        queryTime: "Abfragedauer",
+        totalSize: "Gesamtgröße",
+        pathBar: {
+            placeholder: "Filter Pfad",
+            modalTitle: "Wähle Pfad"
+        },
+        debug: "Debug Informationen",
+        debugDescription: "Informationen für das Debugging. Wenn du Bugs gefunden oder Anregungen für " +
+            "neue Features hast, poste sie bitte <a href='https://github.com/simon987/sist2/issues/new/choose'>hier</a>.",
+        tagline: "Tagline",
+        toast: {
+            esConnErrTitle: "Elasticsearch Verbindungsfehler",
+            esConnErr: "sist2 Web-Modul stellte einen Fehler beim Verbinden mit Elasticsearch fest. " +
+                "Schau in die Server-Logs für weitere Informationen.",
+            esQueryErrTitle: "Query Fehler",
+            esQueryErr: "Konnte Query nicht verarbeiten/ausführen, bitte schaue in die Dokumentation zur erweiterten Suche. " +
+                "Schau in die Server-Logs für weitere Informationen.",
+            dupeTagTitle: "Tag Duplikat",
+            dupeTag: "Dieser Tag existiert bereits für das Dokument.",
+            copiedToClipboard: "In die Zwischenablage kopiert."
+        },
+        saveTagModalTitle: "Tag hinzufügen",
+        saveTagPlaceholder: "Tag Name",
+        confirm: "Bestätigen",
+        indexPickerPlaceholder: "Index auswählen",
+        sort: {
+            relevance: "Relevanz",
+            dateAsc: "Datum (älteste zuerst)",
+            dateDesc: "Datum (neuste zuerst)",
+            sizeAsc: "Größe (kleinste zuerst)",
+            sizeDesc: "Größe (größte zuerst)",
+            nameAsc: "Name (A-z)",
+            nameDesc: "Name (Z-a)",
+            random: "zufällig",
+        },
+        d3: {
+            mimeCount: "Anzahl nach Medientyp",
+            mimeSize: "Größen nach Medientyp",
+            dateHistogram: "Änderungszeiten",
+            sizeHistogram: "Dateigrößen",
+        },
+        indexPicker: {
+            selectNone: "keinen auswählen",
+            selectAll: "alle auswählen",
+            selectedIndex: "ausgewählter Index",
+            selectedIndices: "ausgewählte Indizes",
+        },
     },
     fr: {
         filePage: {
@@ -177,7 +366,7 @@ export default {
             advanced: "Recherche avancée",
             fuzzy: "Approximatif"
         },
-        addTag: "Ajouter",
+        addTag: "Taguer",
         deleteTag: "Supprimer",
         download: "Télécharger",
         and: "et",
@@ -186,6 +375,7 @@ export default {
         mimeTypes: "Types de médias",
         tags: "Tags",
         tagFilter: "Filtrer les tags",
+        forExample: "Par exemple:",
         help: {
             simpleSearch: "Recherche simple",
             advancedSearch: "Recherche avancée",
@@ -225,7 +415,7 @@ export default {
             fuzzy: "Activer la recherche approximative par défaut",
             searchInPath: "Activer la recherche dans le chemin des documents",
             suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
-            fragmentSize: "Longueur du contexte de surlignage, en nombre de caractères",
+            fragmentSize: "Longueur du contexte de surlignage",
             queryMode: "Mode de recherche",
             displayMode: "Affichage",
             columns: "Nombre de colonnes",
@@ -245,7 +435,9 @@ export default {
             useDatePicker: "Afficher un composant « Date Picker » plutôt qu'un slider",
             vidPreviewInterval: "Durée des images d'aperçu video en millisecondes",
             simpleLightbox: "Désactiver les animations du visualiseur d'images",
-            showTagPickerFilter: "Afficher le filtre dans l'onglet Tags"
+            showTagPickerFilter: "Afficher le filtre dans l'onglet Tags",
+            featuredFields: "Expression Javascript pour les variables mises en évidence. Sera affiché dans les résultats de recherche.",
+            featuredFieldsList: "Variables disponibles"
         },
         queryMode: {
             simple: "Simple",
@@ -253,7 +445,9 @@ export default {
         },
         lang: {
             en: "English",
+            de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -348,7 +542,7 @@ export default {
             advanced: "高级搜索",
             fuzzy: "模糊搜索"
         },
-        addTag: "添加",
+        addTag: "签条",
         deleteTag: "删除",
         download: "下载",
         and: "与",
@@ -357,6 +551,7 @@ export default {
         mimeTypes: "文件类型",
         tags: "标签",
         tagFilter: "筛选标签",
+        forExample: "例如:",
         help: {
             simpleSearch: "简易搜索",
             advancedSearch: "高级搜索",
@@ -415,7 +610,9 @@ export default {
             useDatePicker: "使用日期选择器组件而不是滑块",
             vidPreviewInterval: "视频预览帧的持续时间，以毫秒为单位",
             simpleLightbox: "在图片查看器中，禁用动画",
-            showTagPickerFilter: "显示标签过滤栏"
+            showTagPickerFilter: "显示标签过滤栏",
+            featuredFields: "特色领域的Javascript模板字符串。将出现在搜索结果中。",
+            featuredFieldsList: "可利用的变量"
         },
         queryMode: {
             simple: "简单",
@@ -423,7 +620,9 @@ export default {
         },
         lang: {
             en: "English",
+            de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -508,4 +707,188 @@ export default {
             selectedIndices: "选中索引",
         },
     },
+    pl: {
+        filePage: {
+            notFound: "Nie znaleziono"
+        },
+        searchBar: {
+            simple: "Szukaj",
+            advanced: "Zaawansowane szukanie",
+            fuzzy: "Również podobne"
+        },
+        addTag: "Tag",
+        deleteTag: "Usuń",
+        download: "Pobierz",
+        and: "i",
+        page: "strona",
+        pages: "stron",
+        mimeTypes: "Typy danych",
+        tags: "Tagi",
+        tagFilter: "Filtruj tagi",
+        forExample: "Na przykład:",
+        help: {
+            simpleSearch: "Proste szukanie",
+            advancedSearch: "Zaawansowane szukanie",
+            help: "Pomoc",
+            term: "<WYRAZ>",
+            and: "operator I",
+            or: "operator LUB",
+            not: "zabrania danego wyrazu",
+            quotes: "znajdzie objętą sekwencję wyrazów w podanej kolejności",
+            prefix: "znajdzie dowolny wyraz rozpoczynający się na takie litery, jeśli zastosowane na końcu wyrazu",
+            parens: "używane do grupowania wyrażeń",
+            tildeTerm: "znajdzie wyraz w podanej odległości",
+            tildePhrase: "znajdzie frazę przeplecioną podaną liczbą niepasujących wyrazów",
+            example1:
+                "Na przykład: <code>\"pieczone jajko\" +(kiełbasa | ziemniak) -frytki</code> znajdzie frazę " +
+                "<i>pieczone jajko</i> gdzie występuje też: <i>kiełbasa</i> albo <i>ziemniak</i>, ale zignoruje rezultat " +
+                "zawierający <i>frytki</i>.",
+            defaultOperator:
+                "Kiedy nie podano ani <code>+</code>, ani <code>|</code>, to domyślnym operatorem jest " +
+                "<code>+</code> (i).",
+            fuzzy:
+                "Kiedy opcja <b>Również podobne</b> jest zaznaczona, częściowo zgodne wyrazy są również znajdywane.",
+            moreInfoSimple: "Po więcej informacji sięgnij do <a target=\"_blank\" " +
+                "rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">dokumentacji Elasticsearch</a>",
+            moreInfoAdvanced: "Aby uzyskać więcej informacji o zaawansowanym szukaniu, przeczytaj <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">dokumentację Elasticsearch</a>"
+        },
+        config: "Ustawienia",
+        configDescription: "Ustawienia są zapisywane na żywo w tej przeglądarce.",
+        configReset: "Zresetuj ustawienia",
+        searchOptions: "Opcje szukania",
+        treemapOptions: "Opcje mapy",
+        mlOptions: "Opcje uczenia maszynowego",
+        displayOptions: "Opcje wyświetlania",
+        opt: {
+            lang: "Język",
+            highlight: "Zaznaczaj znalezione fragmenty",
+            fuzzy: "Ustaw szukanie również podobnych jako domyślne",
+            searchInPath: "Włącz szukanie również w ścieżce dokumentu",
+            suggestPath: "Włącz auto-uzupełnianie w filtrze ścieżek",
+            fragmentSize: "Podświetl wielkość kontekstu w znakach",
+            queryMode: "Tryb szukania",
+            displayMode: "Wyświetlanie",
+            columns: "Liczba kolumn",
+            treemapType: "Typ mapy",
+            treemapTiling: "Układanie mapy",
+            treemapColorGroupingDepth: "Jak głęboko grupować kolory mapy (na płasko)",
+            treemapColor: "Kolor mapy (kaskadowo)",
+            treemapSize: "Wielkość mapy",
+            theme: "Styl graficzny",
+            lightboxLoadOnlyCurrent: "Nie pobieraj od razu obrazów w pełnej wielkości dla sąsiednich obrazów podglądu.",
+            slideDuration: "Czas trwania jednego slajdu w pokazie slajdów",
+            resultSize: "Liczba wyników na stronę",
+            tagOrOperator: "Użyj operatora LUB przy wyborze kilku tagów",
+            hideDuplicates: "Ukryj zduplikowane wyniki (według sumy kontrolnej)",
+            hideLegacy: "Ukryj powiadomienie Elasticsearch 'legacyES'",
+            updateMimeMap: "Uaktualniaj drzewo typów mediów na żywo",
+            useDatePicker: "Używaj kalendarza do wyboru dat, zamiast suwaka",
+            vidPreviewInterval: "Czas trwania jednej klatki w podglądzie wideo (w ms)",
+            simpleLightbox: "Wyłącz animacje w podglądzie obrazów",
+            showTagPickerFilter: "Pokazuj pole filtrowania tagów",
+            featuredFields: "Wybrane pola szablonu Javascript. Będą pojawiać się przy wynikach wyszukiwania.",
+            featuredFieldsList: "Dostępne zmienne",
+            autoAnalyze: "Automatycznie analizuj tekst",
+            defaultModel: "Domyślny model",
+            mlRepositories: "Repozytoria modeli (każde w osobnej linii)"
+        },
+        queryMode: {
+            simple: "Proste",
+            advanced: "Zaawansowane",
+        },
+        lang: {
+            en: "English",
+            de: "Deutsch",
+            fr: "Français",
+            pl: "Polski",
+            "zh-CN": "简体中文",
+        },
+        displayMode: {
+            grid: "Siatka",
+            list: "Lista",
+        },
+        columns: {
+            auto: "Automatyczna"
+        },
+        treemapType: {
+            cascaded: "Kaskadowa",
+            flat: "Płaska (kompaktowa)"
+        },
+        treemapSize: {
+            small: "Mała",
+            medium: "Średnia",
+            large: "Duża",
+            xLarge: "Bardzo duża",
+            xxLarge: "Ogromna",
+            custom: "Inna",
+        },
+        treemapTiling: {
+            binary: "Binarnie",
+            squarify: "Kwadratowo",
+            slice: "Wycinek",
+            dice: "Kostka",
+            sliceDice: "Wycinek i kostka",
+        },
+        theme: {
+            light: "Jasny",
+            black: "Czarny"
+        },
+        hit: "traf",
+        hits: "trafień",
+        details: "Szczegóły",
+        stats: "Statystyki",
+        queryTime: "Czas szukania",
+        totalSize: "Całkowita wielkość",
+        pathBar: {
+            placeholder: "Filtruj ścieżki",
+            modalTitle: "Wybierz ścieżkę"
+        },
+        debug: "Informacje dla programistów",
+        debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
+            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
+        tagline: "Slogan",
+        toast: {
+            esConnErrTitle: "Problem z połączeniem z Elasticsearch",
+            esConnErr: "Moduł strony internetowej sist2 napotkał problem przy połączeniu z Elasticsearch." +
+                " Zobacz logi serwera, aby uzyskać więcej informacji.",
+            esQueryErrTitle: "Problem z kwerendą",
+            esQueryErr: "Kwerenda szukania jest niezrozumiała albo nie udało się jej przesłać. Sprawdź dokumentację zaawansowanego szukania. " +
+                "Zobacz logi serwera, aby uzyskać więcej informacji.",
+            dupeTagTitle: "Zduplikowany tag",
+            dupeTag: "Ten dokument już ma taki tag.",
+            copiedToClipboard: "Skopiowano do schowka"
+        },
+        saveTagModalTitle: "Dodaj tag",
+        saveTagPlaceholder: "Nazwa",
+        confirm: "Zatwierdź",
+        indexPickerPlaceholder: "Wybierz indeks",
+        sort: {
+            relevance: "Zgodność z szukanym",
+            dateAsc: "Data (najpierw starsze)",
+            dateDesc: "Data (najpierw nowsze)",
+            sizeAsc: "Wielkość (najpierw mniejsze)",
+            sizeDesc: "Wielkość (najpierw większe)",
+            nameAsc: "Nazwa (A-z)",
+            nameDesc: "Nazwa (Z-a)",
+            random: "Losowo",
+        },
+        d3: {
+            mimeCount: "Dystrybucja liczby plików według typów mediów",
+            mimeSize: "Dystrybucja wielkości plików według typów mediów",
+            dateHistogram: "Dystrybucja dat modyfikacji plików",
+            sizeHistogram: "Dystrybucja wielkości plików",
+        },
+        indexPicker: {
+            selectNone: "Zaznacz nic",
+            selectAll: "Zaznacz wszystko",
+            selectedIndex: "wybrany indeks",
+            selectedIndices: "wybrane indeksy",
+        },
+        ml: {
+            analyzeText: "Analizuj",
+            auto: "Automatycznie",
+            repoFetchError: "Nie udało się uzyskać listy modeli. Zobacz konsolę przeglądarki, aby uzyskać więcej informacji.",
+            repoFetchErrorTitle: "Nie udało się pobrać repozytoriów modeli",
+        }
+    }
 }

sist2-vue/src/ml/BertNerModel.js

@@ -0,0 +1,77 @@
+import BertTokenizer from "@/ml/BertTokenizer";
+import * as tf from "@tensorflow/tfjs";
+import axios from "axios";
+
+export default class BertNerModel {
+    vocabUrl;
+    modelUrl;
+
+    id2label;
+    _tokenizer;
+    _model;
+    inputSize = 128;
+
+    _previousWordId = null;
+
+    constructor(vocabUrl, modelUrl, id2label) {
+        this.vocabUrl = vocabUrl;
+        this.modelUrl = modelUrl;
+        this.id2label = id2label;
+    }
+
+    async init(onProgress) {
+        await Promise.all([this.loadTokenizer(), this.loadModel(onProgress)]);
+    }
+
+    async loadTokenizer() {
+        const vocab = (await axios.get(this.vocabUrl)).data;
+        this._tokenizer = new BertTokenizer(vocab);
+    }
+
+    async loadModel(onProgress) {
+        this._model = await tf.loadGraphModel(this.modelUrl, {onProgress});
+    }
+
+    alignLabels(labels, wordIds, words) {
+        const result = [];
+
+        for (let i = 0; i < this.inputSize; i++) {
+            const label = labels[i];
+            const wordId = wordIds[i];
+
+            if (wordId === -1) {
+                continue;
+            }
+            if (wordId === this._previousWordId) {
+                continue;
+            }
+
+            result.push({
+                word: words[wordId].text, wordIndex: words[wordId].index, label: label
+            });
+            this._previousWordId = wordId;
+        }
+
+        return result;
+    }
+
+    async predict(text, callback) {
+        this._previousWordId = null;
+        const encoded = this._tokenizer.encodeText(text, this.inputSize)
+
+        for (let chunk of encoded.inputChunks) {
+            const rawResult = tf.tidy(() => this._model.execute({
+                input_ids: tf.tensor2d(chunk.inputIds, [1, this.inputSize], "int32"),
+                token_type_ids: tf.tensor2d(chunk.segmentIds, [1, this.inputSize], "int32"),
+                attention_mask: tf.tensor2d(chunk.inputMask, [1, this.inputSize], "int32"),
+            }));
+
+            const labelIds = await tf.argMax(rawResult, -1);
+            const labelIdsArray = await labelIds.array();
+            const labels = labelIdsArray[0].map(id => this.id2label[id]);
+            rawResult.dispose()
+
+            callback(this.alignLabels(labels, chunk.wordIds, encoded.words))
+        }
+    }
+}

sist2-vue/src/ml/BertTokenizer.js

@@ -0,0 +1,184 @@
+import {zip, chunk} from "underscore";
+
+const UNK_INDEX = 100;
+const CLS_INDEX = 101;
+const SEP_INDEX = 102;
+const CONTINUING_SUBWORD_PREFIX = "##";
+
+function isWhitespace(ch) {
+    return /\s/.test(ch);
+}
+
+function isInvalid(ch) {
+    return (ch.charCodeAt(0) === 0 || ch.charCodeAt(0) === 0xfffd);
+}
+
+const punctuations = '[~`!@#$%^&*(){}[];:"\'<,.>?/\\|-_+=';
+
+/** To judge whether it's a punctuation. */
+function isPunctuation(ch) {
+    return punctuations.indexOf(ch) !== -1;
+}
+
+export default class BertTokenizer {
+    vocab;
+
+    constructor(vocab) {
+        this.vocab = vocab;
+    }
+
+    tokenize(text) {
+        const charOriginalIndex = [];
+        const cleanedText = this.cleanText(text, charOriginalIndex);
+        const origTokens = cleanedText.split(' ');
+
+        let charCount = 0;
+        const tokens = origTokens.map((token) => {
+            token = token.toLowerCase();
+            const tokens = this.runSplitOnPunctuation(token, charCount, charOriginalIndex);
+            charCount += token.length + 1;
+            return tokens;
+        });
+
+        let flattenTokens = [];
+        for (let index = 0; index < tokens.length; index++) {
+            flattenTokens = flattenTokens.concat(tokens[index]);
+        }
+        return flattenTokens;
+    }
+
+    /* Performs invalid character removal and whitespace cleanup on text. */
+    cleanText(text, charOriginalIndex) {
+        text = text.replace(/\?/g, "").trim();
+
+        const stringBuilder = [];
+        let originalCharIndex = 0;
+        let newCharIndex = 0;
+
+        for (const ch of text) {
+            // Skip the characters that cannot be used.
+            if (isInvalid(ch)) {
+                originalCharIndex += ch.length;
+                continue;
+            }
+            if (isWhitespace(ch)) {
+                if (stringBuilder.length > 0 && stringBuilder[stringBuilder.length - 1] !== ' ') {
+                    stringBuilder.push(' ');
+                    charOriginalIndex[newCharIndex] = originalCharIndex;
+                    originalCharIndex += ch.length;
+                } else {
+                    originalCharIndex += ch.length;
+                    continue;
+                }
+            } else {
+                stringBuilder.push(ch);
+                charOriginalIndex[newCharIndex] = originalCharIndex;
+                originalCharIndex += ch.length;
+            }
+            newCharIndex++;
+        }
+        return stringBuilder.join('');
+    }
+
+    /* Splits punctuation on a piece of text. */
+    runSplitOnPunctuation(text, count, charOriginalIndex) {
+        const tokens = [];
+        let startNewWord = true;
+        for (const ch of text) {
+            if (isPunctuation(ch)) {
+                tokens.push({text: ch, index: charOriginalIndex[count]});
+                count += ch.length;
+                startNewWord = true;
+            } else {
+                if (startNewWord) {
+                    tokens.push({text: '', index: charOriginalIndex[count]});
+                    startNewWord = false;
+                }
+                tokens[tokens.length - 1].text += ch;
+                count += ch.length;
+            }
+        }
+        return tokens;
+    }
+
+    encode(words) {
+        let outputTokens = [];
+        const wordIds = [];
+
+        for (let i = 0; i < words.length; i++) {
+            let chars = [...words[i].text];
+
+            let isUnknown = false;
+            let start = 0;
+            let subTokens = [];
+
+            while (start < chars.length) {
+                let end = chars.length;
+                let currentSubstring = null;
+                while (start < end) {
+                    let substr = chars.slice(start, end).join('');
+
+                    if (start > 0) {
+                        substr = CONTINUING_SUBWORD_PREFIX + substr;
+                    }
+                    if (this.vocab.includes(substr)) {
+                        currentSubstring = this.vocab.indexOf(substr);
+                        break;
+                    }
+
+                    --end;
+                }
+                if (currentSubstring == null) {
+                    isUnknown = true;
+                    break;
+                }
+                subTokens.push(currentSubstring);
+                start = end;
+            }
+
+            if (isUnknown) {
+                outputTokens.push(UNK_INDEX);
+                wordIds.push(i);
+            } else {
+                subTokens.forEach(tok => {
+                    outputTokens.push(tok);
+                    wordIds.push(i)
+                });
+            }
+        }
+
+        return {tokens: outputTokens, wordIds};
+    }
+
+    encodeText(inputText, inputSize) {
+
+        const tokenized = this.tokenize(inputText);
+        const encoded = this.encode(tokenized);
+
+        const encodedTokenChunks = chunk(encoded.tokens, inputSize - 2);
+        const encodedWordIdChunks = chunk(encoded.wordIds, inputSize - 2);
+
+        const chunks = [];
+
+        zip(encodedTokenChunks, encodedWordIdChunks).forEach(([tokens, wordIds]) => {
+            const inputIds = [CLS_INDEX, ...tokens, SEP_INDEX];
+            const segmentIds = Array(inputIds.length).fill(0);
+            const inputMask = Array(inputIds.length).fill(1);
+            wordIds = [-1, ...wordIds, -1];
+
+            while (inputIds.length < inputSize) {
+                inputIds.push(0);
+                inputMask.push(0);
+                segmentIds.push(0);
+                wordIds.push(-1);
+            }
+
+            chunks.push({inputIds, inputMask, segmentIds, wordIds})
+        });
+
+        return {
+            inputChunks: chunks,
+            words: tokenized
+        };
+    }
+}

sist2-vue/src/ml/modelsRepo.js

@@ -0,0 +1,43 @@
+import axios from "axios";
+
+class ModelsRepo {
+    _repositories;
+    data = {};
+
+    async init(repositories) {
+        this._repositories = repositories;
+
+        const data = await Promise.all(this._repositories.map(this._loadRepository));
+
+        data.forEach(models => {
+            models.forEach(model => {
+                this.data[model.name] = model;
+            })
+        });
+    }
+
+    async _loadRepository(repository) {
+        const data = (await axios.get(repository)).data;
+        data.forEach(model => {
+            model["modelUrl"] = new URL(model["modelPath"], repository).href;
+            model["vocabUrl"] = new URL(model["vocabPath"], repository).href;
+        });
+        return data;
+    }
+
+    getOptions() {
+        return Object.values(this.data).map(model => ({
+            text: `${model.name} (${Math.round(model.size / (1024*1024))}MB)`,
+            value: model.name
+        }));
+    }
+
+    getDefaultModel() {
+        if (Object.values(this.data).length === 0) {
+            return null;
+        }
+        return Object.values(this.data).find(model => model.default).name;
+    }
+}
+
+export default new ModelsRepo();

sist2-vue/src/store/index.ts

@@ -1,11 +1,11 @@
 import Vue from "vue"
 import Vuex from "vuex"
 import VueRouter, {Route} from "vue-router";
-import {EsHit, EsResult, EsTag, Index, Tag} from "@/Sist2Api";
+import {EsHit, EsResult, EsTag, Index} from "@/Sist2Api";
 import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
 import {getInstance} from "@/plugins/auth0.js";
 
-const CONF_VERSION = 2;
+const CONF_VERSION = 3;
 
 Vue.use(Vuex)
 
@@ -33,6 +33,7 @@ export default new Vuex.Store({
         optHideDuplicates: true,
         optTheme: "light",
         optDisplay: "grid",
+        optFeaturedFields: "",
 
         optSize: 60,
         optHighlight: true,
@@ -56,6 +57,9 @@ export default new Vuex.Store({
         optVidPreviewInterval: 700,
         optSimpleLightbox: true,
         optShowTagPickerFilter: true,
+        optMlRepositories: "https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json",
+        optAutoAnalyze: false,
+        optMlDefaultModel: null,
 
         _onLoadSelectedIndices: [] as string[],
         _onLoadSelectedMimeTypes: [] as string[],
@@ -65,11 +69,12 @@ export default new Vuex.Store({
         selectedTags: [] as string[],
 
         lastQueryResults: null,
+        firstQueryResults: null,
 
         keySequence: 0,
         querySequence: 0,
 
-        uiTagHover: null as Tag | null,
+        uiSqliteMode: false,
         uiLightboxIsOpen: false,
         uiShowLightbox: false,
         uiLightboxSources: [] as string[],
@@ -85,7 +90,11 @@ export default new Vuex.Store({
 
         uiMimeMap: [] as any[],
 
-        auth0Token: null
+        auth0Token: null,
+        mlModel: {
+            model: null,
+            name: null
+        },
     },
     mutations: {
         setUiShowDetails: (state, val) => state.uiShowDetails = val,
@@ -122,13 +131,13 @@ export default new Vuex.Store({
         setSearchText: (state, val) => state.searchText = val,
         setFuzzy: (state, val) => state.fuzzy = val,
         setLastQueryResult: (state, val) => state.lastQueryResults = val,
+        setFirstQueryResult: (state, val) => state.firstQueryResults = val,
         _setOnLoadSelectedIndices: (state, val) => state._onLoadSelectedIndices = val,
         _setOnLoadSelectedMimeTypes: (state, val) => state._onLoadSelectedMimeTypes = val,
         _setOnLoadSelectedTags: (state, val) => state._onLoadSelectedTags = val,
         setSelectedIndices: (state, val) => state.selectedIndices = val,
         setSelectedMimeTypes: (state, val) => state.selectedMimeTypes = val,
         setSelectedTags: (state, val) => state.selectedTags = val,
-        setUiTagHover: (state, val: Tag | null) => state.uiTagHover = val,
         setUiLightboxIsOpen: (state, val: boolean) => state.uiLightboxIsOpen = val,
         _setUiShowLightbox: (state, val: boolean) => state.uiShowLightbox = val,
         setUiLightboxKey: (state, val: number) => state.uiLightboxKey = val,
@@ -146,6 +155,7 @@ export default new Vuex.Store({
         setUiLightboxThumbs: (state, val) => state.uiLightboxThumbs = val,
         setUiLightboxTypes: (state, val) => state.uiLightboxTypes = val,
         setUiLightboxCaptions: (state, val) => state.uiLightboxCaptions = val,
+        setUiSqliteMode: (state, val) => state.uiSqliteMode = val,
 
         setOptTheme: (state, val) => state.optTheme = val,
         setOptDisplay: (state, val) => state.optDisplay = val,
@@ -158,6 +168,7 @@ export default new Vuex.Store({
         setOptQueryMode: (state, val) => state.optQueryMode = val,
         setOptResultSize: (state, val) => state.optSize = val,
         setOptTagOrOperator: (state, val) => state.optTagOrOperator = val,
+        setOptFeaturedFields: (state, val) => state.optFeaturedFields = val,
 
         setOptTreemapType: (state, val) => state.optTreemapType = val,
         setOptTreemapTiling: (state, val) => state.optTreemapTiling = val,
@@ -170,6 +181,15 @@ export default new Vuex.Store({
         setOptVidPreviewInterval: (state, val) => state.optVidPreviewInterval = val,
         setOptSimpleLightbox: (state, val) => state.optSimpleLightbox = val,
         setOptShowTagPickerFilter: (state, val) => state.optShowTagPickerFilter = val,
+        setOptAutoAnalyze: (state, val) => {
+            state.optAutoAnalyze = val
+        },
+        setOptMlRepositories: (state, val) => {
+            state.optMlRepositories = val
+        },
+        setOptMlDefaultModel: (state, val) => {
+            state.optMlDefaultModel = val
+        },
 
         setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
         setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val,
@@ -192,6 +212,7 @@ export default new Vuex.Store({
             // noop
         },
         setAuth0Token: (state, val) => state.auth0Token = val,
+        setMlModel: (state, val) => state.mlModel = val,
     },
     actions: {
         setSist2Info: (store, val) => {
@@ -327,6 +348,7 @@ export default new Vuex.Store({
             commit("_setUiShowLightbox", !state.uiShowLightbox);
         },
         clearResults({commit}) {
+            commit("setFirstQueryResult", null);
             commit("setLastQueryResult", null);
             commit("_setKeySequence", 0);
             commit("_setUiShowLightbox", false);
@@ -348,6 +370,7 @@ export default new Vuex.Store({
     },
     modules: {},
     getters: {
+        mlModel: (state) => state.mlModel,
         seed: (state) => state.seed,
         getPathText: (state) => state.pathText,
         indices: state => state.indices,
@@ -378,7 +401,6 @@ export default new Vuex.Store({
 
             return (state.lastQueryResults as unknown as EsResult).hits.hits.slice(-1)[0];
         },
-        uiTagHover: state => state.uiTagHover,
         uiShowLightbox: state => state.uiShowLightbox,
         uiLightboxSources: state => state.uiLightboxSources,
         uiLightboxThumbs: state => state.uiLightboxThumbs,
@@ -386,6 +408,7 @@ export default new Vuex.Store({
         uiLightboxTypes: state => state.uiLightboxTypes,
         uiLightboxKey: state => state.uiLightboxKey,
         uiLightboxSlide: state => state.uiLightboxSlide,
+        uiSqliteMode: state => state.uiSqliteMode,
 
         optHideDuplicates: state => state.optHideDuplicates,
         optLang: state => state.optLang,
@@ -413,5 +436,13 @@ export default new Vuex.Store({
         optVidPreviewInterval: state => state.optVidPreviewInterval,
         optSimpleLightbox: state => state.optSimpleLightbox,
         optShowTagPickerFilter: state => state.optShowTagPickerFilter,
+        optFeaturedFields: state => state.optFeaturedFields,
+        optMlRepositories: state => state.optMlRepositories,
+        mlRepositoryList: state => {
+            const repos = state.optMlRepositories.split("\n")
+            return repos[0] == "" ? [] : repos;
+        },
+        optMlDefaultModel: state => state.optMlDefaultModel,
+        optAutoAnalyze: state => state.optAutoAnalyze,
     }
 })

sist2-vue/src/util.ts

@@ -57,6 +57,14 @@ export function humanTime(sec_num: number): string {
     const minutes = Math.floor((sec_num - (hours * 3600)) / 60);
     const seconds = sec_num - (hours * 3600) - (minutes * 60);
 
+    if (sec_num < 60) {
+        return `${sec_num}s`
+    }
+
+    if (sec_num < 3600) {
+        return `${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
+    }
+
     return `${hours < 10 ? "0" : ""}${hours}:${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
 }
 

sist2-vue/src/views/Configuration.vue

@@ -1,144 +1,221 @@
 <template>
   <!--  <div :style="{width: `${$store.getters.optContainerWidth}px`}"-->
-  <div
-      v-if="!configLoading"
-      style="margin-left: auto; margin-right: auto;" class="container">
-
-    <b-card>
-      <b-card-title>
-        <GearIcon></GearIcon>
-        {{ $t("config") }}
-      </b-card-title>
-      <p>{{ $t("configDescription") }}</p>
-
-      <b-card-body>
-        <h4>{{ $t("displayOptions") }}</h4>
+    <div
+            v-if="!configLoading"
+            style="margin-left: auto; margin-right: auto;" class="container">
 
         <b-card>
+            <b-card-title>
+                <GearIcon></GearIcon>
+                {{ $t("config") }}
+            </b-card-title>
+            <p>{{ $t("configDescription") }}</p>
 
-          <label><LanguageIcon/><span style="vertical-align: middle">&nbsp;{{ $t("opt.lang") }}</span></label>
-          <b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
+            <b-card-body>
+                <h4>{{ $t("displayOptions") }}</h4>
 
-          <label>{{ $t("opt.theme") }}</label>
-          <b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
+                <b-card>
 
-          <label>{{ $t("opt.displayMode") }}</label>
-          <b-form-select :options="displayModeOptions" :value="optDisplay" @input="setOptDisplay"></b-form-select>
+                    <label>
+                        <LanguageIcon/>
+                        <span style="vertical-align: middle">&nbsp;{{ $t("opt.lang") }}</span></label>
+                    <b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
 
-          <label>{{ $t("opt.columns") }}</label>
-          <b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select>
+                    <label>{{ $t("opt.theme") }}</label>
+                    <b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
 
-          <div style="height: 10px"></div>
+                    <label>{{ $t("opt.displayMode") }}</label>
+                    <b-form-select :options="displayModeOptions" :value="optDisplay"
+                                   @input="setOptDisplay"></b-form-select>
 
-          <b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent">
-            {{ $t("opt.lightboxLoadOnlyCurrent") }}
-          </b-form-checkbox>
+                    <label>{{ $t("opt.columns") }}</label>
+                    <b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select>
 
-          <b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy">
-            {{ $t("opt.hideLegacy") }}
-          </b-form-checkbox>
+                    <div style="height: 10px"></div>
 
-          <b-form-checkbox :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
-            {{ $t("opt.updateMimeMap") }}
-          </b-form-checkbox>
+                    <b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent">
+                        {{ $t("opt.lightboxLoadOnlyCurrent") }}
+                    </b-form-checkbox>
 
-          <b-form-checkbox :checked="optUseDatePicker" @input="setOptUseDatePicker">
-            {{ $t("opt.useDatePicker") }}
-          </b-form-checkbox>
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optHideLegacy" @input="setOptHideLegacy">
+                        {{ $t("opt.hideLegacy") }}
+                    </b-form-checkbox>
 
-          <b-form-checkbox :checked="optSimpleLightbox" @input="setOptSimpleLightbox">{{
-              $t("opt.simpleLightbox")
-            }}
-          </b-form-checkbox>
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
+                        {{ $t("opt.updateMimeMap") }}
+                    </b-form-checkbox>
 
-          <b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{
-              $t("opt.showTagPickerFilter")
-            }}
-          </b-form-checkbox>
+                    <b-form-checkbox :checked="optUseDatePicker" @input="setOptUseDatePicker">
+                        {{ $t("opt.useDatePicker") }}
+                    </b-form-checkbox>
+
+                    <b-form-checkbox :checked="optSimpleLightbox" @input="setOptSimpleLightbox">{{
+                        $t("opt.simpleLightbox")
+                        }}
+                    </b-form-checkbox>
+
+                    <b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{
+                        $t("opt.showTagPickerFilter")
+                        }}
+                    </b-form-checkbox>
+
+                    <br/>
+                    <label>{{ $t("opt.featuredFields") }}</label>
+
+                    <br>
+                    <b-button v-b-toggle.collapse-1 variant="secondary" class="dropdown-toggle">{{
+                        $t("opt.featuredFieldsList")
+                        }}
+                    </b-button>
+                    <b-collapse id="collapse-1" class="mt-2">
+                        <ul>
+                            <li><code>doc.checksum</code></li>
+                            <li><code>doc.path</code></li>
+                            <li><code>doc.mime</code></li>
+                            <li><code>doc.videoc</code></li>
+                            <li><code>doc.audioc</code></li>
+                            <li><code>doc.pages</code></li>
+                            <li><code>doc.mtime</code></li>
+                            <li><code>doc.font_name</code></li>
+                            <li><code>doc.album</code></li>
+                            <li><code>doc.artist</code></li>
+                            <li><code>doc.title</code></li>
+                            <li><code>doc.genre</code></li>
+                            <li><code>doc.album_artist</code></li>
+                            <li><code>doc.exif_make</code></li>
+                            <li><code>doc.exif_model</code></li>
+                            <li><code>doc.exif_software</code></li>
+                            <li><code>doc.exif_exposure_time</code></li>
+                            <li><code>doc.exif_fnumber</code></li>
+                            <li><code>doc.exif_iso_speed_ratings</code></li>
+                            <li><code>doc.exif_focal_length</code></li>
+                            <li><code>doc.exif_user_comment</code></li>
+                            <li><code>doc.exif_user_comment</code></li>
+                            <li><code>doc.exif_gps_longitude_ref</code></li>
+                            <li><code>doc.exif_gps_longitude_dms</code></li>
+                            <li><code>doc.exif_gps_longitude_dec</code></li>
+                            <li><code>doc.exif_gps_latitude_ref</code></li>
+                            <li><code>doc.exif_gps_latitude_dec</code></li>
+                            <li><code>humanDate()</code></li>
+                            <li><code>humanFileSize()</code></li>
+                        </ul>
+
+                        <p>{{ $t("forExample") }}</p>
+
+                        <ul>
+                            <li>
+                                <code>&lt;b&gt;${humanDate(doc.mtime)}&lt;/b&gt; • ${doc.videoc || ''}</code>
+                            </li>
+                            <li>
+                                <code>${doc.pages ? (doc.pages + ' pages') : ''}</code>
+                            </li>
+                        </ul>
+                    </b-collapse>
+                    <br/>
+                    <br/>
+                    <b-textarea rows="3" :value="optFeaturedFields" @input="setOptFeaturedFields"></b-textarea>
+                </b-card>
+
+                <br/>
+                <h4>{{ $t("searchOptions") }}</h4>
+                <b-card>
+                    <b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
+                        $t("opt.hideDuplicates")
+                        }}
+                    </b-form-checkbox>
+
+                    <b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{
+                        $t("opt.highlight")
+                        }}
+                    </b-form-checkbox>
+                    <b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
+                        $t("opt.tagOrOperator")
+                        }}
+                    </b-form-checkbox>
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optFuzzy" @input="setOptFuzzy">
+                        {{ $t("opt.fuzzy") }}
+                    </b-form-checkbox>
+
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optSearchInPath" @input="setOptSearchInPath">{{
+                        $t("opt.searchInPath")
+                        }}
+                    </b-form-checkbox>
+                    <b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
+                        $t("opt.suggestPath")
+                        }}
+                    </b-form-checkbox>
+
+                    <br/>
+                    <label>{{ $t("opt.fragmentSize") }}</label>
+                    <b-form-input :value="optFragmentSize" step="10" type="number" min="0"
+                                  @input="setOptFragmentSize"></b-form-input>
+
+                    <label>{{ $t("opt.resultSize") }}</label>
+                    <b-form-input :value="optResultSize" type="number" min="10"
+                                  @input="setOptResultSize"></b-form-input>
+
+                    <label :class="{'text-muted': uiSqliteMode}">{{ $t("opt.queryMode") }}</label>
+                    <b-form-select :disabled="uiSqliteMode" :options="queryModeOptions" :value="optQueryMode"
+                                   @input="setOptQueryMode"></b-form-select>
+
+                    <label>{{ $t("opt.slideDuration") }}</label>
+                    <b-form-input :value="optLightboxSlideDuration" type="number" min="1"
+                                  @input="setOptLightboxSlideDuration"></b-form-input>
+
+                    <label>{{ $t("opt.vidPreviewInterval") }}</label>
+                    <b-form-input :value="optVidPreviewInterval" type="number" min="50"
+                                  @input="setOptVidPreviewInterval"></b-form-input>
+                </b-card>
+
+                <h4 class="mt-3">{{ $t("mlOptions") }}</h4>
+                <b-card>
+                    <label>{{ $t("opt.mlRepositories") }}</label>
+                    <b-textarea rows="3" :value="optMlRepositories" @input="setOptMlRepositories"></b-textarea>
+                    <br>
+                    <b-form-checkbox :checked="optAutoAnalyze" @input="setOptAutoAnalyze">{{
+                        $t("opt.autoAnalyze")
+                        }}
+                    </b-form-checkbox>
+                </b-card>
+
+                <h4 class="mt-3">{{ $t("treemapOptions") }}</h4>
+                <b-card>
+                    <label>{{ $t("opt.treemapType") }}</label>
+                    <b-form-select :value="optTreemapType" :options="treemapTypeOptions"
+                                   @input="setOptTreemapType"></b-form-select>
+
+                    <label>{{ $t("opt.treemapTiling") }}</label>
+                    <b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
+                                   @input="setOptTreemapTiling"></b-form-select>
+
+                    <label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
+                    <b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
+                                  @input="setOptTreemapColorGroupingDepth"></b-form-input>
+
+                    <label>{{ $t("opt.treemapSize") }}</label>
+                    <b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
+                                   @input="setOptTreemapSize"></b-form-select>
+
+                    <template v-if="$store.getters.optTreemapSize === 'custom'">
+                        <!-- TODO Width/Height input -->
+                        <b-form-input type="number" min="0" step="10"></b-form-input>
+                        <b-form-input type="number" min="0" step="10"></b-form-input>
+                    </template>
+
+                    <label>{{ $t("opt.treemapColor") }}</label>
+                    <b-form-select :value="optTreemapColor" :options="treemapColorOptions"
+                                   @input="setOptTreemapColor"></b-form-select>
+                </b-card>
+
+                <b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
+            </b-card-body>
         </b-card>
 
-        <br/>
-        <h4>{{ $t("searchOptions") }}</h4>
-        <b-card>
-          <b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
-              $t("opt.hideDuplicates")
-            }}
-          </b-form-checkbox>
-
-          <b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{ $t("opt.highlight") }}</b-form-checkbox>
-          <b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
-              $t("opt.tagOrOperator")
-            }}
-          </b-form-checkbox>
-          <b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
-          <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
-              $t("opt.searchInPath")
-            }}
-          </b-form-checkbox>
-          <b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
-              $t("opt.suggestPath")
-            }}
-          </b-form-checkbox>
-
-          <br/>
-          <label>{{ $t("opt.fragmentSize") }}</label>
-          <b-form-input :value="optFragmentSize" step="10" type="number" min="0"
-                        @input="setOptFragmentSize"></b-form-input>
-
-          <label>{{ $t("opt.resultSize") }}</label>
-          <b-form-input :value="optResultSize" type="number" min="10"
-                        @input="setOptResultSize"></b-form-input>
-
-          <label>{{ $t("opt.queryMode") }}</label>
-          <b-form-select :options="queryModeOptions" :value="optQueryMode" @input="setOptQueryMode"></b-form-select>
-
-          <label>{{ $t("opt.slideDuration") }}</label>
-          <b-form-input :value="optLightboxSlideDuration" type="number" min="1"
-                        @input="setOptLightboxSlideDuration"></b-form-input>
-
-          <label>{{ $t("opt.vidPreviewInterval") }}</label>
-          <b-form-input :value="optVidPreviewInterval" type="number" min="50"
-                        @input="setOptVidPreviewInterval"></b-form-input>
+        <b-card v-if="loading" class="mt-4">
+            <Preloader></Preloader>
         </b-card>
-
-        <h4 class="mt-3">{{ $t("treemapOptions") }}</h4>
-        <b-card>
-          <label>{{ $t("opt.treemapType") }}</label>
-          <b-form-select :value="optTreemapType" :options="treemapTypeOptions"
-                         @input="setOptTreemapType"></b-form-select>
-
-          <label>{{ $t("opt.treemapTiling") }}</label>
-          <b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
-                         @input="setOptTreemapTiling"></b-form-select>
-
-          <label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
-          <b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
-                        @input="setOptTreemapColorGroupingDepth"></b-form-input>
-
-          <label>{{ $t("opt.treemapSize") }}</label>
-          <b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
-                         @input="setOptTreemapSize"></b-form-select>
-
-          <template v-if="$store.getters.optTreemapSize === 'custom'">
-            <!-- TODO Width/Height input -->
-            <b-form-input type="number" min="0" step="10"></b-form-input>
-            <b-form-input type="number" min="0" step="10"></b-form-input>
-          </template>
-
-          <label>{{ $t("opt.treemapColor") }}</label>
-          <b-form-select :value="optTreemapColor" :options="treemapColorOptions"
-                         @input="setOptTreemapColor"></b-form-select>
-        </b-card>
-
-        <b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
-      </b-card-body>
-    </b-card>
-
-    <b-card v-if="loading" class="mt-4">
-      <Preloader></Preloader>
-    </b-card>
-    <DebugInfo v-else></DebugInfo>
-  </div>
+        <DebugInfo v-else></DebugInfo>
+    </div>
 </template>
 
 <script>
@@ -150,161 +227,170 @@ import GearIcon from "@/components/icons/GearIcon.vue";
 import LanguageIcon from "@/components/icons/LanguageIcon";
 
 export default {
-  components: {LanguageIcon, GearIcon, DebugInfo, Preloader},
-  data() {
-    return {
-      loading: false,
-      configLoading: false,
-      langOptions: [
-        {value: "en", text: this.$t("lang.en")},
-        {value: "fr", text: this.$t("lang.fr")},
-        {value: "zh-CN", text: this.$t("lang.zh-CN")},
-      ],
-      queryModeOptions: [
-        {value: "simple", text: this.$t("queryMode.simple")},
-        {value: "advanced", text: this.$t("queryMode.advanced")}
-      ],
-      displayModeOptions: [
-        {value: "grid", text: this.$t("displayMode.grid")},
-        {value: "list", text: this.$t("displayMode.list")}
-      ],
-      columnsOptions: [
-        {value: "auto", text: this.$t("columns.auto")},
-        {value: 1, text: "1"},
-        {value: 2, text: "2"},
-        {value: 3, text: "3"},
-        {value: 4, text: "4"},
-        {value: 5, text: "5"},
-        {value: 6, text: "6"},
-        {value: 7, text: "7"},
-        {value: 8, text: "8"},
-        {value: 9, text: "9"},
-        {value: 10, text: "10"},
-        {value: 11, text: "11"},
-        {value: 12, text: "12"},
-      ],
-      treemapTypeOptions: [
-        {value: "cascaded", text: this.$t("treemapType.cascaded")},
-        {value: "flat", text: this.$t("treemapType.flat")}
-      ],
-      treemapTilingOptions: [
-        {value: "binary", text: this.$t("treemapTiling.binary")},
-        {value: "squarify", text: this.$t("treemapTiling.squarify")},
-        {value: "slice", text: this.$t("treemapTiling.slice")},
-        {value: "dice", text: this.$t("treemapTiling.dice")},
-        {value: "sliceDice", text: this.$t("treemapTiling.sliceDice")},
-      ],
-      treemapSizeOptions: [
-        {value: "small", text: this.$t("treemapSize.small")},
-        {value: "medium", text: this.$t("treemapSize.medium")},
-        {value: "large", text: this.$t("treemapSize.large")},
-        {value: "x-large", text: this.$t("treemapSize.xLarge")},
-        {value: "xx-large", text: this.$t("treemapSize.xxLarge")},
-        // {value: "custom", text: this.$t("treemapSize.custom")},
-      ],
-      treemapColorOptions: [
-        {value: "PuBuGn", text: "Purple-Blue-Green"},
-        {value: "PuRd", text: "Purple-Red"},
-        {value: "PuBu", text: "Purple-Blue"},
-        {value: "YlOrBr", text: "Yellow-Orange-Brown"},
-        {value: "YlOrRd", text: "Yellow-Orange-Red"},
-        {value: "YlGn", text: "Yellow-Green"},
-        {value: "YlGnBu", text: "Yellow-Green-Blue"},
-        {value: "Plasma", text: "Plasma"},
-        {value: "Magma", text: "Magma"},
-        {value: "Inferno", text: "Inferno"},
-        {value: "Viridis", text: "Viridis"},
-        {value: "Turbo", text: "Turbo"},
-      ],
-      themeOptions: [
-        {value: "light", text: this.$t("theme.light")},
-        {value: "black", text: this.$t("theme.black")}
-      ]
+    components: {LanguageIcon, GearIcon, DebugInfo, Preloader},
+    data() {
+        return {
+            loading: false,
+            configLoading: false,
+            langOptions: [
+                {value: "en", text: this.$t("lang.en")},
+                {value: "fr", text: this.$t("lang.fr")},
+                {value: "zh-CN", text: this.$t("lang.zh-CN")},
+                {value: "de", text: this.$t("lang.de")},
+                {value: "pl", text: this.$t("lang.pl")},
+            ],
+            queryModeOptions: [
+                {value: "simple", text: this.$t("queryMode.simple")},
+                {value: "advanced", text: this.$t("queryMode.advanced")}
+            ],
+            displayModeOptions: [
+                {value: "grid", text: this.$t("displayMode.grid")},
+                {value: "list", text: this.$t("displayMode.list")}
+            ],
+            columnsOptions: [
+                {value: "auto", text: this.$t("columns.auto")},
+                {value: 1, text: "1"},
+                {value: 2, text: "2"},
+                {value: 3, text: "3"},
+                {value: 4, text: "4"},
+                {value: 5, text: "5"},
+                {value: 6, text: "6"},
+                {value: 7, text: "7"},
+                {value: 8, text: "8"},
+                {value: 9, text: "9"},
+                {value: 10, text: "10"},
+                {value: 11, text: "11"},
+                {value: 12, text: "12"},
+            ],
+            treemapTypeOptions: [
+                {value: "cascaded", text: this.$t("treemapType.cascaded")},
+                {value: "flat", text: this.$t("treemapType.flat")}
+            ],
+            treemapTilingOptions: [
+                {value: "binary", text: this.$t("treemapTiling.binary")},
+                {value: "squarify", text: this.$t("treemapTiling.squarify")},
+                {value: "slice", text: this.$t("treemapTiling.slice")},
+                {value: "dice", text: this.$t("treemapTiling.dice")},
+                {value: "sliceDice", text: this.$t("treemapTiling.sliceDice")},
+            ],
+            treemapSizeOptions: [
+                {value: "small", text: this.$t("treemapSize.small")},
+                {value: "medium", text: this.$t("treemapSize.medium")},
+                {value: "large", text: this.$t("treemapSize.large")},
+                {value: "x-large", text: this.$t("treemapSize.xLarge")},
+                {value: "xx-large", text: this.$t("treemapSize.xxLarge")},
+                // {value: "custom", text: this.$t("treemapSize.custom")},
+            ],
+            treemapColorOptions: [
+                {value: "PuBuGn", text: "Purple-Blue-Green"},
+                {value: "PuRd", text: "Purple-Red"},
+                {value: "PuBu", text: "Purple-Blue"},
+                {value: "YlOrBr", text: "Yellow-Orange-Brown"},
+                {value: "YlOrRd", text: "Yellow-Orange-Red"},
+                {value: "YlGn", text: "Yellow-Green"},
+                {value: "YlGnBu", text: "Yellow-Green-Blue"},
+                {value: "Plasma", text: "Plasma"},
+                {value: "Magma", text: "Magma"},
+                {value: "Inferno", text: "Inferno"},
+                {value: "Viridis", text: "Viridis"},
+                {value: "Turbo", text: "Turbo"},
+            ],
+            themeOptions: [
+                {value: "light", text: this.$t("theme.light")},
+                {value: "black", text: this.$t("theme.black")}
+            ]
 
-    }
-  },
-  computed: {
-    ...mapGetters([
-      "optTheme",
-      "optDisplay",
-      "optColumns",
-      "optHighlight",
-      "optFuzzy",
-      "optSearchInPath",
-      "optSuggestPath",
-      "optFragmentSize",
-      "optQueryMode",
-      "optTreemapType",
-      "optTreemapTiling",
-      "optTreemapColorGroupingDepth",
-      "optTreemapColor",
-      "optTreemapSize",
-      "optLightboxLoadOnlyCurrent",
-      "optLightboxSlideDuration",
-      "optResultSize",
-      "optTagOrOperator",
-      "optLang",
-      "optHideDuplicates",
-      "optHideLegacy",
-      "optUpdateMimeMap",
-      "optUseDatePicker",
-      "optVidPreviewInterval",
-      "optSimpleLightbox",
-      "optShowTagPickerFilter",
-    ]),
-    clientWidth() {
-      return window.innerWidth;
-    }
-  },
-  mounted() {
-    this.$store.subscribe((mutation) => {
-      if (mutation.type.startsWith("setOpt")) {
-        this.$store.dispatch("updateConfiguration");
-      }
-    });
-  },
-  methods: {
-    ...mapActions({
-      setSist2Info: "setSist2Info",
-    }),
-    ...mapMutations([
-      "setOptTheme",
-      "setOptDisplay",
-      "setOptColumns",
-      "setOptHighlight",
-      "setOptFuzzy",
-      "setOptSearchInPath",
-      "setOptSuggestPath",
-      "setOptFragmentSize",
-      "setOptQueryMode",
-      "setOptTreemapType",
-      "setOptTreemapTiling",
-      "setOptTreemapColorGroupingDepth",
-      "setOptTreemapColor",
-      "setOptTreemapSize",
-      "setOptLightboxLoadOnlyCurrent",
-      "setOptLightboxSlideDuration",
-      "setOptResultSize",
-      "setOptTagOrOperator",
-      "setOptLang",
-      "setOptHideDuplicates",
-      "setOptHideLegacy",
-      "setOptUpdateMimeMap",
-      "setOptUseDatePicker",
-      "setOptVidPreviewInterval",
-      "setOptSimpleLightbox",
-      "setOptShowTagPickerFilter",
-    ]),
-    onResetClick() {
-      localStorage.removeItem("sist2_configuration");
-      window.location.reload();
-    }
-  },
+        }
+    },
+    computed: {
+        ...mapGetters([
+            "uiSqliteMode",
+            "optTheme",
+            "optDisplay",
+            "optColumns",
+            "optHighlight",
+            "optFuzzy",
+            "optSearchInPath",
+            "optSuggestPath",
+            "optFragmentSize",
+            "optQueryMode",
+            "optTreemapType",
+            "optTreemapTiling",
+            "optTreemapColorGroupingDepth",
+            "optTreemapColor",
+            "optTreemapSize",
+            "optLightboxLoadOnlyCurrent",
+            "optLightboxSlideDuration",
+            "optResultSize",
+            "optTagOrOperator",
+            "optLang",
+            "optHideDuplicates",
+            "optHideLegacy",
+            "optUpdateMimeMap",
+            "optUseDatePicker",
+            "optVidPreviewInterval",
+            "optSimpleLightbox",
+            "optShowTagPickerFilter",
+            "optFeaturedFields",
+            "optMlRepositories",
+            "optAutoAnalyze",
+        ]),
+        clientWidth() {
+            return window.innerWidth;
+        }
+    },
+    mounted() {
+        this.$store.subscribe((mutation) => {
+            if (mutation.type.startsWith("setOpt")) {
+                this.$store.dispatch("updateConfiguration");
+            }
+        });
+    },
+    methods: {
+        ...mapActions({
+            setSist2Info: "setSist2Info",
+        }),
+        ...mapMutations([
+            "setOptTheme",
+            "setOptDisplay",
+            "setOptColumns",
+            "setOptHighlight",
+            "setOptFuzzy",
+            "setOptSearchInPath",
+            "setOptSuggestPath",
+            "setOptFragmentSize",
+            "setOptQueryMode",
+            "setOptTreemapType",
+            "setOptTreemapTiling",
+            "setOptTreemapColorGroupingDepth",
+            "setOptTreemapColor",
+            "setOptTreemapSize",
+            "setOptLightboxLoadOnlyCurrent",
+            "setOptLightboxSlideDuration",
+            "setOptResultSize",
+            "setOptTagOrOperator",
+            "setOptLang",
+            "setOptHideDuplicates",
+            "setOptHideLegacy",
+            "setOptUpdateMimeMap",
+            "setOptUseDatePicker",
+            "setOptVidPreviewInterval",
+            "setOptSimpleLightbox",
+            "setOptShowTagPickerFilter",
+            "setOptFeaturedFields",
+            "setOptMlRepositories",
+            "setOptAutoAnalyze",
+        ]),
+        onResetClick() {
+            localStorage.removeItem("sist2_configuration");
+            window.location.reload();
+        }
+    },
 }
 </script>
 
 <style>
 .shrink {
-  flex-grow: inherit;
+    flex-grow: inherit;
 }
 </style>

sist2-vue/src/views/FilePage.vue

@@ -14,7 +14,7 @@
       <!-- Audio player-->
       <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
              :type="doc._source.mime"
-             :src="`f/${doc._id}`"></audio>
+             :src="`f/${doc._source.index}/${doc._id}`"></audio>
 
       <InfoTable :doc="doc" v-if="doc"></InfoTable>
 
@@ -54,7 +54,7 @@ export default Vue.extend({
   methods: {
     ext: ext,
     onThumbnailClick() {
-      window.open(`/f/${this.doc._id}`, "_blank");
+      window.open(`/f/${this.doc.index}/${this.doc._id}`, "_blank");
     },
     findByCustomField(field, id) {
       return {

sist2-vue/src/views/SearchPage.vue

@@ -1,57 +1,61 @@
 <template>
-  <div class="container">
-    <Lightbox></Lightbox>
-    <HelpDialog :show="showHelp" @close="showHelp = false"></HelpDialog>
+    <div class="container">
+        <Lightbox></Lightbox>
+        <HelpDialog :show="showHelp" @close="showHelp = false"></HelpDialog>
 
-    <b-card v-if="uiLoading">
-      <Preloader></Preloader>
-    </b-card>
+        <b-card v-if="uiLoading">
+            <Preloader></Preloader>
+        </b-card>
 
-    <b-card v-show="!uiLoading" id="search-panel">
-      <SearchBar @show-help="showHelp=true"></SearchBar>
-      <b-row>
-        <b-col style="height: 70px;" sm="6">
-          <SizeSlider></SizeSlider>
-        </b-col>
-        <b-col>
-          <PathTree @search="search(true)"></PathTree>
-        </b-col>
-      </b-row>
-      <b-row>
-        <b-col sm="6">
-          <DateSlider></DateSlider>
-          <b-row>
-            <b-col>
-              <IndexPicker></IndexPicker>
-            </b-col>
-          </b-row>
-        </b-col>
-        <b-col>
-          <b-tabs justified>
-            <b-tab :title="$t('mimeTypes')">
-              <MimePicker></MimePicker>
-            </b-tab>
-            <b-tab :title="$t('tags')">
-              <TagPicker :show-search-bar="$store.state.optShowTagPickerFilter"></TagPicker>
-            </b-tab>
-          </b-tabs>
-        </b-col>
-      </b-row>
-    </b-card>
+        <b-alert v-show="!uiLoading && showEsConnectionError" show variant="danger" class="mt-2">
+            {{ $t("toast.esConnErr") }}
+        </b-alert>
 
-    <div v-show="docs.length === 0 && !uiLoading">
-      <Preloader v-if="searchBusy" class="mt-3"></Preloader>
+        <b-card v-show="!uiLoading && !showEsConnectionError" id="search-panel">
+            <SearchBar @show-help="showHelp=true"></SearchBar>
+            <b-row>
+                <b-col style="height: 70px;" sm="6">
+                    <SizeSlider></SizeSlider>
+                </b-col>
+                <b-col>
+                    <PathTree @search="search(true)"></PathTree>
+                </b-col>
+            </b-row>
+            <b-row>
+                <b-col sm="6">
+                    <DateSlider></DateSlider>
+                    <b-row>
+                        <b-col>
+                            <IndexPicker></IndexPicker>
+                        </b-col>
+                    </b-row>
+                </b-col>
+                <b-col>
+                    <b-tabs justified>
+                        <b-tab :title="$t('mimeTypes')">
+                            <MimePicker></MimePicker>
+                        </b-tab>
+                        <b-tab :title="$t('tags')">
+                            <TagPicker :show-search-bar="$store.state.optShowTagPickerFilter"></TagPicker>
+                        </b-tab>
+                    </b-tabs>
+                </b-col>
+            </b-row>
+        </b-card>
 
-      <ResultsCard></ResultsCard>
+        <div v-show="docs.length === 0 && !uiLoading">
+            <Preloader v-if="searchBusy" class="mt-3"></Preloader>
+
+            <ResultsCard></ResultsCard>
+        </div>
+
+        <div v-if="docs.length > 0">
+            <ResultsCard></ResultsCard>
+
+            <DocCardWall v-if="optDisplay==='grid'" :docs="docs" :append="appendFunc"></DocCardWall>
+            <DocList v-else :docs="docs" :append="appendFunc"></DocList>
+        </div>
     </div>
-
-    <div v-if="docs.length > 0">
-      <ResultsCard></ResultsCard>
-
-      <DocCardWall v-if="optDisplay==='grid'" :docs="docs" :append="appendFunc"></DocCardWall>
-      <DocList v-else :docs="docs" :append="appendFunc"></DocList>
-    </div>
-  </div>
 </template>
 
 <script lang="ts">
@@ -62,7 +66,7 @@ import Sist2Api, {EsHit, EsResult} from "../Sist2Api";
 import SearchBar from "@/components/SearchBar.vue";
 import IndexPicker from "@/components/IndexPicker.vue";
 import Vue from "vue";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 import _debounce from "lodash/debounce";
 import DocCardWall from "@/components/DocCardWall.vue";
 import Lightbox from "@/components/Lightbox.vue";
@@ -75,237 +79,236 @@ import DateSlider from "@/components/DateSlider.vue";
 import TagPicker from "@/components/TagPicker.vue";
 import DocList from "@/components/DocList.vue";
 import HelpDialog from "@/components/HelpDialog.vue";
+import Sist2SqliteQuery from "@/Sist2SqliteQuery";
 
 
 export default Vue.extend({
-  components: {
-    HelpDialog,
-    DocList,
-    TagPicker,
-    DateSlider,
-    SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
-  },
-  data: () => ({
-    loading: false,
-    uiLoading: true,
-    search: undefined as any,
-    docs: [] as EsHit[],
-    docIds: new Set(),
-    docChecksums: new Set(),
-    searchBusy: false,
-    Sist2Query: Sist2Query,
-    showHelp: false
-  }),
-  computed: {
-    ...mapGetters(["indices", "optDisplay"]),
-  },
-  mounted() {
-    // Handle touch events
-    window.ontouchend = () => this.$store.commit("busTouchEnd");
-    window.ontouchcancel = this.$store.commit("busTouchEnd");
-
-    this.search = _debounce(async (clear: boolean) => {
-      if (clear) {
-        await this.clearResults();
-      }
-
-      await this.searchNow(Sist2Query.searchQuery());
-
-    }, 350, {leading: false});
-
-    this.$store.dispatch("loadFromArgs", this.$route).then(() => {
-      this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
-      this.$store.subscribe((mutation) => {
-        if ([
-          "setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
-          "setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
-          "setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
-        ].includes(mutation.type)) {
-          if (this.searchBusy) {
-            return;
-          }
-
-          this.search(true);
-        }
-      });
-    });
-
-    this.setIndices(this.$store.getters["sist2Info"].indices)
-
-    this.getDateRange().then((range: { min: number, max: number }) => {
-      this.setDateBoundsMin(range.min);
-      this.setDateBoundsMax(range.max);
-
-      const doBlankSearch = !this.$store.state.optUpdateMimeMap;
-
-      Sist2Api.getMimeTypes(Sist2Query.searchQuery(doBlankSearch)).then(({mimeMap}) => {
-        this.$store.commit("setUiMimeMap", mimeMap);
-        this.uiLoading = false;
-        this.search(true);
-      });
-    });
-  },
-  methods: {
-    ...mapActions({
-      setSist2Info: "setSist2Info",
+    components: {
+        HelpDialog,
+        DocList,
+        TagPicker,
+        DateSlider,
+        SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
+    },
+    data: () => ({
+        loading: false,
+        uiLoading: true,
+        search: undefined as any,
+        docs: [] as EsHit[],
+        docIds: new Set(),
+        docChecksums: new Set(),
+        searchBusy: false,
+        Sist2Query: Sist2Query,
+        showHelp: false,
+        showEsConnectionError: false
     }),
-    ...mapMutations({
-      setIndices: "setIndices",
-      setDateBoundsMin: "setDateBoundsMin",
-      setDateBoundsMax: "setDateBoundsMax",
-      setTags: "setTags",
-    }),
-    showErrorToast() {
-      this.$bvToast.toast(
-          this.$t("toast.esConnErr"),
-          {
-            title: this.$t("toast.esConnErrTitle"),
-            noAutoHide: true,
-            toaster: "b-toaster-bottom-right",
-            headerClass: "toast-header-error",
-            bodyClass: "toast-body-error",
-          });
+    computed: {
+        ...mapGetters(["indices", "optDisplay"]),
     },
-    showSyntaxErrorToast: function (): void {
-      this.$bvToast.toast(
-          this.$t("toast.esQueryErr"),
-          {
-            title: this.$t("toast.esQueryErrTitle"),
-            noAutoHide: true,
-            toaster: "b-toaster-bottom-right",
-            headerClass: "toast-header-warning",
-            bodyClass: "toast-body-warning",
-          });
-    },
-    async searchNow(q: any) {
-      this.searchBusy = true;
-      await this.$store.dispatch("incrementQuerySequence");
-      this.$store.commit("busSearch");
+    mounted() {
+        // Handle touch events
+        window.ontouchend = () => this.$store.commit("busTouchEnd");
+        window.ontouchcancel = this.$store.commit("busTouchEnd");
 
-      Sist2Api.esQuery(q).then(async (resp: EsResult) => {
-        await this.handleSearch(resp);
-        this.searchBusy = false;
-      }).catch(err => {
-        if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
-          this.showSyntaxErrorToast();
-        } else {
-          this.showErrorToast();
-        }
-      });
-    },
-    async clearResults() {
-      this.docs = [];
-      this.docIds.clear();
-      this.docChecksums.clear();
-      await this.$store.dispatch("clearResults");
-      this.$store.commit("setUiReachedScrollEnd", false);
-    },
-    async handleSearch(resp: EsResult) {
-      if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
-        this.$store.commit("setUiReachedScrollEnd", true);
-      }
+        this.search = _debounce(async (clear: boolean) => {
+            if (clear) {
+                await this.clearResults();
+            }
 
-      resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
+            await this.searchNow();
 
-      if (this.$store.state.optHideDuplicates) {
-        resp.hits.hits = resp.hits.hits.filter(hit => {
+        }, 350, {leading: false});
 
-          if (!("checksum" in hit._source)) {
-            return true;
-          }
+        this.$store.dispatch("loadFromArgs", this.$route).then(() => {
+            this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
+            this.$store.subscribe((mutation) => {
+                if ([
+                    "setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
+                    "setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
+                    "setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
+                ].includes(mutation.type)) {
+                    if (this.searchBusy) {
+                        return;
+                    }
 
-          const isDupe = !this.docChecksums.has(hit._source.checksum);
-          this.docChecksums.add(hit._source.checksum);
-          return isDupe;
+                    this.search(true);
+                }
+            });
         });
-      }
 
-      for (const hit of resp.hits.hits) {
-        if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
-          hit._seq = await this.$store.dispatch("getKeySequence");
-          this.$store.commit("addLightboxSource", {
-            source: `f/${hit._id}`,
-            thumbnail: hit._props.hasThumbnail
-                ? `t/${hit._source.index}/${hit._id}`
-                : null,
-            caption: {
-              component: LightboxCaption,
-              props: {hit: hit}
-            },
-            type: hit._props.isVideo ? "video" : "image"
-          });
-        }
-      }
+        this.setIndices(this.$store.getters["sist2Info"].indices)
 
-      await this.$store.dispatch("remountLightbox");
-      this.$store.commit("setLastQueryResult", resp);
+        Sist2Api.getDateRange().then((range) => {
+            this.setDateBoundsMin(range.min);
+            this.setDateBoundsMax(range.max);
 
-      this.docs.push(...resp.hits.hits);
+            const doBlankSearch = !this.$store.state.optUpdateMimeMap;
 
-      resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
+            Sist2Api.getMimeTypes(Sist2Query.searchQuery(doBlankSearch)).then(({mimeMap}) => {
+                this.$store.commit("setUiMimeMap", mimeMap);
+                this.uiLoading = false;
+                this.search(true);
+            });
+        }).catch(error => {
+            console.log(error);
+
+            if (error.response.status == 503 || error.response.status == 500) {
+                this.showEsConnectionError = true;
+                this.uiLoading = false;
+            } else {
+                this.showErrorToast();
+            }
+        });
     },
-    getDateRange(): Promise<{ min: number, max: number }> {
-      return sist2.esQuery({
-        // TODO: filter current selected indices
-        aggs: {
-          dateMin: {min: {field: "mtime"}},
-          dateMax: {max: {field: "mtime"}},
+    methods: {
+        ...mapActions({
+            setSist2Info: "setSist2Info",
+        }),
+        ...mapMutations({
+            setIndices: "setIndices",
+            setDateBoundsMin: "setDateBoundsMin",
+            setDateBoundsMax: "setDateBoundsMax",
+            setTags: "setTags",
+        }),
+        showErrorToast() {
+            this.$bvToast.toast(
+                this.$t("toast.esConnErr"),
+                {
+                    title: this.$t("toast.esConnErrTitle"),
+                    noAutoHide: true,
+                    toaster: "b-toaster-bottom-right",
+                    headerClass: "toast-header-error",
+                    bodyClass: "toast-body-error",
+                });
         },
-        size: 0
-      }).then(res => {
-        return {
-          min: res.aggregations.dateMin.value,
-          max: res.aggregations.dateMax.value,
+        showSyntaxErrorToast: function (): void {
+            this.$bvToast.toast(
+                this.$t("toast.esQueryErr"),
+                {
+                    title: this.$t("toast.esQueryErrTitle"),
+                    noAutoHide: true,
+                    toaster: "b-toaster-bottom-right",
+                    headerClass: "toast-header-warning",
+                    bodyClass: "toast-body-warning",
+                });
+        },
+        async searchNow() {
+            this.searchBusy = true;
+            await this.$store.dispatch("incrementQuerySequence");
+            this.$store.commit("busSearch");
+
+            Sist2Api.search().then(async (resp: EsResult) => {
+                await this.handleSearch(resp);
+                this.searchBusy = false;
+            }).catch(err => {
+                if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
+                    this.showSyntaxErrorToast();
+                } else {
+                    this.showErrorToast();
+                }
+            });
+        },
+        async clearResults() {
+            this.docs = [];
+            this.docIds.clear();
+            this.docChecksums.clear();
+            await this.$store.dispatch("clearResults");
+            this.$store.commit("setUiReachedScrollEnd", false);
+        },
+        async handleSearch(resp: EsResult) {
+            if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
+                this.$store.commit("setUiReachedScrollEnd", true);
+            }
+
+            resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
+
+            if (this.$store.state.optHideDuplicates) {
+                resp.hits.hits = resp.hits.hits.filter(hit => {
+
+                    if (!("checksum" in hit._source)) {
+                        return true;
+                    }
+
+                    const isDupe = !this.docChecksums.has(hit._source.checksum);
+                    this.docChecksums.add(hit._source.checksum);
+                    return isDupe;
+                });
+            }
+
+            for (const hit of resp.hits.hits) {
+                if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
+                    hit._seq = await this.$store.dispatch("getKeySequence");
+                    this.$store.commit("addLightboxSource", {
+                        source: `f/${hit._source.index}/${hit._id}`,
+                        thumbnail: hit._props.hasThumbnail
+                            ? `t/${hit._source.index}/${hit._id}`
+                            : null,
+                        caption: {
+                            component: LightboxCaption,
+                            props: {hit: hit}
+                        },
+                        type: hit._props.isVideo ? "video" : "image"
+                    });
+                }
+            }
+
+            await this.$store.dispatch("remountLightbox");
+            this.$store.commit("setLastQueryResult", resp);
+            if (this.$store.state.firstQueryResults == null) {
+                this.$store.commit("setFirstQueryResult", resp);
+            }
+
+            this.docs.push(...resp.hits.hits);
+
+            resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
+        },
+        appendFunc() {
+            if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
+                this.searchNow();
+            }
+        }
+    },
+    beforeRouteUpdate(to, from, next) {
+        if (this.$store.state.uiLightboxIsOpen) {
+            this.$store.commit("_setUiShowLightbox", false);
+            next(false);
+        } else {
+            next();
         }
-      })
     },
-    appendFunc() {
-      if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
-        this.searchNow(Sist2Query.searchQuery());
-      }
-    }
-  },
-  beforeRouteUpdate(to, from, next) {
-    if (this.$store.state.uiLightboxIsOpen) {
-      this.$store.commit("_setUiShowLightbox", false);
-      next(false);
-    } else {
-      next();
-    }
-  },
 })
 </script>
 
 <style>
 
 #search-panel {
-  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-  border-radius: 0;
-  border: none;
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
 }
 
 .toast-header-info, .toast-body-info {
-  background: #2196f3;
-  color: #fff !important;
+    background: #2196f3;
+    color: #fff !important;
 }
 
 .toast-header-error, .toast-body-error {
-  background: #a94442;
-  color: #f2dede !important;
+    background: #a94442;
+    color: #f2dede !important;
 }
 
 .toast-header-error {
-  color: #fff !important;
-  border-bottom: none;
-  margin-bottom: -1em;
+    color: #fff !important;
+    border-bottom: none;
+    margin-bottom: -1em;
 }
 
 .toast-header-error .close {
-  text-shadow: none;
+    text-shadow: none;
 }
 
 .toast-header-warning, .toast-body-warning {
-  background: #FF8F00;
-  color: #FFF3E0 !important;
+    background: #FF8F00;
+    color: #FFF3E0 !important;
 }
 </style>

sist2-vue/src/views/StatsPage.vue

@@ -1,37 +1,36 @@
 <template>
-  <b-container>
+    <b-container>
 
-    <b-card v-if="loading">
-      <Preloader></Preloader>
-    </b-card>
+        <template>
+            <b-card>
+                <b-card-body>
+                    <b-select v-model="selectedIndex" :options="indexOptions">
+                        <template #first>
+                            <b-form-select-option :value="null" disabled>{{
+                                $t("indexPickerPlaceholder")
+                                }}
+                            </b-form-select-option>
+                        </template>
+                    </b-select>
+                </b-card-body>
+            </b-card>
 
-    <template v-else>
-      <b-card>
-        <b-card-body>
-          <b-select v-model="selectedIndex" :options="indexOptions">
-            <template #first>
-              <b-form-select-option :value="null" disabled>{{ $t("indexPickerPlaceholder") }}</b-form-select-option>
-            </template>
-          </b-select>
-        </b-card-body>
-      </b-card>
-
-      <b-card v-if="selectedIndex !== null" class="mt-3">
-        <b-card-body>
-          <D3Treemap :index-id="selectedIndex"></D3Treemap>
+            <b-card v-if="selectedIndex !== null" class="mt-3">
+                <b-card-body>
+                    <D3Treemap :index-id="selectedIndex"></D3Treemap>
 
 
-        </b-card-body>
-      </b-card>
+                </b-card-body>
+            </b-card>
 
-      <b-card v-if="selectedIndex !== null" class="stats-card mt-3">
-        <D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
-        <D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
-        <D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
-        <D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
-      </b-card>
-    </template>
-  </b-container>
+            <b-card v-if="selectedIndex !== null" class="stats-card mt-3">
+                <D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
+                <D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
+                <D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
+                <D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
+            </b-card>
+        </template>
+    </b-container>
 </template>
 <script>
 import D3Treemap from "@/components/D3Treemap";
@@ -43,37 +42,30 @@ import D3DateHistogram from "@/components/D3DateHistogram";
 import D3SizeHistogram from "@/components/D3SizeHistogram";
 
 export default {
-  components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
-  data() {
-    return {
-      loading: true,
-      selectedIndex: null,
-      indices: []
-    }
-  },
-  computed: {
-    indexOptions() {
-      return this.indices.map(idx => {
+    components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
+    data() {
         return {
-          text: idx.name,
-          value: idx.id
-        };
-      })
+            selectedIndex: null,
+        }
+    },
+    computed: {
+        indexOptions() {
+            return this.indices.map(idx => {
+                return {
+                    text: idx.name,
+                    value: idx.id
+                };
+            })
+        },
+        indices: () => this.$store.state.indices
     }
-  },
-  mounted() {
-    Sist2Api.getSist2Info().then(data => {
-      this.indices = data.indices;
-      this.loading = false;
-    })
-  }
 }
 </script>
 
 <style>
 
 .stats-card {
-  text-align: center;
-  padding: 1em;
+    text-align: center;
+    padding: 1em;
 }
 </style>

src/auth0/auth0_c_api.h

@@ -1,12 +1,13 @@
 #ifndef SIST2_AUTH0_C_API_H
 #define SIST2_AUTH0_C_API_H
 
-#include "stdlib.h"
 
 #ifdef __cplusplus
 #define EXTERNC extern "C"
+#include "cstdlib"
 #else
 #define EXTERNC
+#include "stdlib.h"
 #endif
 
 #define AUTH0_OK (0)

src/cli.c

@@ -2,16 +2,17 @@
 #include "ctx.h"
 #include <tesseract/capi.h>
 
-#define DEFAULT_OUTPUT "index.sist2/"
+#define DEFAULT_OUTPUT "index.sist2"
+#define DEFAULT_NAME "index"
 #define DEFAULT_CONTENT_SIZE 32768
-#define DEFAULT_QUALITY 1
-#define DEFAULT_THUMBNAIL_SIZE 500
+#define DEFAULT_QUALITY 50
+#define DEFAULT_THUMBNAIL_SIZE 552
 #define DEFAULT_THUMBNAIL_COUNT 1
 #define DEFAULT_REWRITE_URL ""
 
 #define DEFAULT_ES_URL "http://localhost:9200"
 #define DEFAULT_ES_INDEX "sist2"
-#define DEFAULT_BATCH_SIZE 100
+#define DEFAULT_BATCH_SIZE 70
 #define DEFAULT_TAGLINE "Lightning-fast file system indexer and search tool"
 #define DEFAULT_LANG "en"
 
@@ -20,8 +21,6 @@
 
 #define DEFAULT_MAX_MEM_BUFFER 2000
 
-#define DEFAULT_THROTTLE_MEMORY_THRESHOLD 0
-
 const char *TESS_DATAPATHS[] = {
         "/usr/share/tessdata/",
         "/usr/share/tesseract-ocr/tessdata/",
@@ -48,9 +47,6 @@ void scan_args_destroy(scan_args_t *args) {
     if (args->name != NULL) {
         free(args->name);
     }
-    if (args->incremental != NULL) {
-        free(args->incremental);
-    }
     if (args->path != NULL) {
         free(args->path);
     }
@@ -61,7 +57,6 @@ void scan_args_destroy(scan_args_t *args) {
 }
 
 void index_args_destroy(index_args_t *args) {
-    //todo
     if (args->es_mappings_path) {
         free(args->es_mappings);
     }
@@ -76,7 +71,6 @@ void index_args_destroy(index_args_t *args) {
 }
 
 void web_args_destroy(web_args_t *args) {
-    //todo
     free(args);
 }
 
@@ -97,23 +91,17 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     char *abs_path = abspath(argv[1]);
     if (abs_path == NULL) {
-        LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1])
+        LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
     } else {
+        abs_path = realloc(abs_path, strlen(abs_path) + 2);
+        strcat(abs_path, "/");
         args->path = abs_path;
     }
 
-    if (args->incremental != OPTION_VALUE_UNSPECIFIED) {
-        args->incremental = abspath(args->incremental);
-        if (abs_path == NULL) {
-            sist_log("main.c", LOG_SIST_WARNING, "Could not open original index! Disabled incremental scan feature.");
-            args->incremental = NULL;
-        }
-    }
-
     if (args->tn_quality == OPTION_VALUE_UNSPECIFIED) {
         args->tn_quality = DEFAULT_QUALITY;
-    } else if (args->tn_quality < 1.0f || args->tn_quality > 31.0f) {
-        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %f. Must be within [1.0, 31.0].\n",
+    } else if (args->tn_quality < 0 || args->tn_quality > 100) {
+        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [0, 100].\n",
                 args->tn_quality);
         return 1;
     }
@@ -121,7 +109,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (args->tn_size == OPTION_VALUE_UNSPECIFIED) {
         args->tn_size = DEFAULT_THUMBNAIL_SIZE;
     } else if (args->tn_size < 32) {
-        printf("Invalid value --thumbnail-size argument: %d. Must be greater than 32 pixels.\n", args->tn_size);
+        printf("Invalid value --thumbnail-size argument: %d. Must be >= 32 pixels.\n", args->tn_size);
         return 1;
     }
 
@@ -140,8 +128,8 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     if (args->threads == 0) {
         args->threads = 1;
-    } else if (args->threads < 0) {
-        fprintf(stderr, "Invalid value for --threads: %d. Must be a positive number\n", args->threads);
+    } else if (args->threads < 0 || args->threads > 256) {
+        fprintf(stderr, "Invalid value for --threads: %d. Must be a positive number <= 256\n", args->threads);
         return 1;
     }
 
@@ -152,20 +140,24 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         args->output = expandpath(args->output);
     }
 
-    int ret = mkdir(args->output, S_IRUSR | S_IWUSR | S_IXUSR);
-    if (ret != 0) {
-        fprintf(stderr, "Invalid output: '%s' (%s).\n", args->output, strerror(errno));
-        return 1;
+    char *abs_output = abspath(args->output);
+    if (args->incremental && abs_output == NULL) {
+        LOG_WARNINGF("main.c", "Could not open original index for incremental scan: %s. Will not perform incremental scan.", args->output);
+        args->incremental = FALSE;
+    } else if (!args->incremental && abs_output != NULL) {
+        LOG_FATALF("main.c", "Index already exists: %s. If you wish to perform incremental scan, you must specify --incremental", abs_output);
     }
+    free(abs_output);
 
     if (args->depth <= 0) {
-        args->depth = G_MAXINT32;
+        args->depth = 2147483647;
     } else {
         args->depth += 1;
     }
 
     if (args->name == OPTION_VALUE_UNSPECIFIED) {
-        args->name = g_path_get_basename(args->output);
+        args->name = malloc(strlen(DEFAULT_NAME) + 1);
+        strcpy(args->name, DEFAULT_NAME);
     } else {
         char *tmp = malloc(strlen(args->name) + 1);
         strcpy(tmp, args->name);
@@ -224,7 +216,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
             }
             if (trained_data_path != NULL && path != trained_data_path) {
                 LOG_FATAL("cli.c", "When specifying more than one tesseract language, all the traineddata "
-                                   "files must be in the same folder")
+                                   "files must be in the same folder");
             }
             trained_data_path = path;
 
@@ -232,7 +224,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         }
         free(lang);
 
-        ret = TessBaseAPIInit3(api, trained_data_path, args->tesseract_lang);
+        int ret = TessBaseAPIInit3(api, trained_data_path, args->tesseract_lang);
         if (ret != 0) {
             fprintf(stderr, "Could not initialize tesseract with lang '%s'\n", args->tesseract_lang);
             return 1;
@@ -249,12 +241,12 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
         pcre *re = pcre_compile(args->exclude_regex, 0, &error, &error_offset, 0);
         if (error != NULL) {
-            LOG_FATALF("cli.c", "pcre_compile returned error: %s (offset:%d)", error, error_offset)
+            LOG_FATALF("cli.c", "pcre_compile returned error: %s (offset:%d)", error, error_offset);
         }
 
         pcre_extra *re_extra = pcre_study(re, 0, &error);
         if (error != NULL) {
-            LOG_FATALF("cli.c", "pcre_study returned error: %s", error)
+            LOG_FATALF("cli.c", "pcre_study returned error: %s", error);
         }
 
         ScanCtx.exclude = re;
@@ -273,14 +265,10 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         args->max_memory_buffer_mib = DEFAULT_MAX_MEM_BUFFER;
     }
 
-    if (args->scan_mem_limit_mib == OPTION_VALUE_UNSPECIFIED || args->scan_mem_limit_mib == OPTION_VALUE_DISABLE) {
-        args->scan_mem_limit_mib = DEFAULT_THROTTLE_MEMORY_THRESHOLD;
-    }
-
     if (args->list_path != OPTION_VALUE_UNSPECIFIED) {
         if (strcmp(args->list_path, "-") == 0) {
             args->list_file = stdin;
-            LOG_DEBUG("cli.c", "Using stdin as list file")
+            LOG_DEBUG("cli.c", "Using stdin as list file");
         } else {
             args->list_file = fopen(args->list_path, "r");
 
@@ -290,27 +278,27 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         }
     }
 
-    LOG_DEBUGF("cli.c", "arg tn_quality=%f", args->tn_quality)
-    LOG_DEBUGF("cli.c", "arg tn_size=%d", args->tn_size)
-    LOG_DEBUGF("cli.c", "arg tn_count=%d", args->tn_count)
-    LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
-    LOG_DEBUGF("cli.c", "arg threads=%d", args->threads)
-    LOG_DEBUGF("cli.c", "arg incremental=%s", args->incremental)
-    LOG_DEBUGF("cli.c", "arg output=%s", args->output)
-    LOG_DEBUGF("cli.c", "arg rewrite_url=%s", args->rewrite_url)
-    LOG_DEBUGF("cli.c", "arg name=%s", args->name)
-    LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
-    LOG_DEBUGF("cli.c", "arg path=%s", args->path)
-    LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
-    LOG_DEBUGF("cli.c", "arg archive_passphrase=%s", args->archive_passphrase)
-    LOG_DEBUGF("cli.c", "arg tesseract_lang=%s", args->tesseract_lang)
-    LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path)
-    LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex)
-    LOG_DEBUGF("cli.c", "arg fast=%d", args->fast)
-    LOG_DEBUGF("cli.c", "arg fast_epub=%d", args->fast_epub)
-    LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold)
-    LOG_DEBUGF("cli.c", "arg max_memory_buffer_mib=%d", args->max_memory_buffer_mib)
-    LOG_DEBUGF("cli.c", "arg list_path=%s", args->list_path)
+    LOG_DEBUGF("cli.c", "arg tn_quality=%f", args->tn_quality);
+    LOG_DEBUGF("cli.c", "arg tn_size=%d", args->tn_size);
+    LOG_DEBUGF("cli.c", "arg tn_count=%d", args->tn_count);
+    LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size);
+    LOG_DEBUGF("cli.c", "arg threads=%d", args->threads);
+    LOG_DEBUGF("cli.c", "arg incremental=%d", args->incremental);
+    LOG_DEBUGF("cli.c", "arg output=%s", args->output);
+    LOG_DEBUGF("cli.c", "arg rewrite_url=%s", args->rewrite_url);
+    LOG_DEBUGF("cli.c", "arg name=%s", args->name);
+    LOG_DEBUGF("cli.c", "arg depth=%d", args->depth);
+    LOG_DEBUGF("cli.c", "arg path=%s", args->path);
+    LOG_DEBUGF("cli.c", "arg archive=%s", args->archive);
+    LOG_DEBUGF("cli.c", "arg archive_passphrase=%s", args->archive_passphrase);
+    LOG_DEBUGF("cli.c", "arg tesseract_lang=%s", args->tesseract_lang);
+    LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path);
+    LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex);
+    LOG_DEBUGF("cli.c", "arg fast=%d", args->fast);
+    LOG_DEBUGF("cli.c", "arg fast_epub=%d", args->fast_epub);
+    LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold);
+    LOG_DEBUGF("cli.c", "arg max_memory_buffer_mib=%d", args->max_memory_buffer_mib);
+    LOG_DEBUGF("cli.c", "arg list_path=%s", args->list_path);
 
     return 0;
 }
@@ -320,20 +308,20 @@ int load_external_file(const char *file_path, char **dst) {
     int res = stat(file_path, &info);
 
     if (res == -1) {
-        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno));
         return 1;
     }
 
     int fd = open(file_path, O_RDONLY);
     if (fd == -1) {
-        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno));
         return 1;
     }
 
     *dst = malloc(info.st_size + 1);
     res = read(fd, *dst, info.st_size);
     if (res < 0) {
-        LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno))
+        LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno));
         return 1;
     }
 
@@ -361,7 +349,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
 
     char *index_path = abspath(argv[1]);
     if (index_path == NULL) {
-        LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1])
+        LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
     } else {
         args->index_path = index_path;
     }
@@ -396,28 +384,54 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
         args->batch_size = DEFAULT_BATCH_SIZE;
     }
 
-    LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
-    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
-    LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl)
-    LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path)
-    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
-    LOG_DEBUGF("cli.c", "arg async_script=%d", args->async_script)
+    LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url);
+    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index);
+    LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl);
+    LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path);
+    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path);
+    LOG_DEBUGF("cli.c", "arg async_script=%d", args->async_script);
 
     if (args->script) {
         char log_buf[5000];
 
         strncpy(log_buf, args->script, sizeof(log_buf));
         *(log_buf + sizeof(log_buf) - 1) = '\0';
-        LOG_DEBUGF("cli.c", "arg script=%s", log_buf)
+        LOG_DEBUGF("cli.c", "arg script=%s", log_buf);
     }
 
-    LOG_DEBUGF("cli.c", "arg print=%d", args->print)
-    LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path)
-    LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings)
-    LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path)
-    LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings)
-    LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size)
-    LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset)
+    LOG_DEBUGF("cli.c", "arg print=%d", args->print);
+    LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path);
+    LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings);
+    LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path);
+    LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings);
+    LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size);
+    LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset);
+
+    return 0;
+}
+
+int sqlite_index_args_validate(sqlite_index_args_t *args, int argc, const char **argv) {
+
+    LogCtx.verbose = 1;
+
+    if (argc < 2) {
+        fprintf(stderr, "Required positional argument: PATH.\n");
+        return 1;
+    }
+
+    char *index_path = abspath(argv[1]);
+    if (index_path == NULL) {
+        LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
+    } else {
+        args->index_path = index_path;
+    }
+
+    if (args->search_index_path == NULL) {
+        LOG_FATAL("cli.c", "Missing required argument --search-index");
+    }
+
+    LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path);
+    LOG_DEBUGF("cli.c", "arg search_index_path=%s", args->search_index_path);
 
     return 0;
 }
@@ -431,6 +445,16 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
+    if (args->search_index_path != NULL && args->es_url != NULL) {
+        LOG_FATAL("cli.c", "--search-index and --es-url arguments are mutually exclusive.");
+    }
+    if (args->search_index_path != NULL && args->es_index != NULL) {
+        LOG_FATAL("cli.c", "--search-index and --es-index arguments are mutually exclusive.");
+    }
+    if (args->search_index_path != NULL && args->es_insecure_ssl == TRUE) {
+        LOG_FATAL("cli.c", "--search-index and --es-insecure_ssl arguments are mutually exclusive.");
+    }
+
     if (args->es_url == NULL) {
         args->es_url = DEFAULT_ES_URL;
     }
@@ -538,23 +562,40 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
     for (int i = 0; i < args->index_count; i++) {
         char *abs_path = abspath(args->indices[i]);
         if (abs_path == NULL) {
-            LOG_FATALF("cli.c", "Index not found: %s", args->indices[i])
+            LOG_FATALF("cli.c", "Index not found: %s", args->indices[i]);
         }
+        free(abs_path);
     }
 
-    LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
-    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
-    LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl)
-    LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline)
-    LOG_DEBUGF("cli.c", "arg dev=%d", args->dev)
-    LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address)
-    LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials)
-    LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials)
-    LOG_DEBUGF("cli.c", "arg auth_user=%s", args->auth_user)
-    LOG_DEBUGF("cli.c", "arg auth_pass=%s", args->auth_pass)
-    LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count)
+    if (args->search_index_path != NULL) {
+        char *abs_path = abspath(args->search_index_path);
+        if (abs_path == NULL) {
+            LOG_FATALF("cli.c", "Search index not found: %s", args->search_index_path);
+        }
+
+        args->es_index = NULL;
+        args->es_url = NULL;
+        args->es_insecure_ssl = FALSE;
+        args->search_backend = SQLITE_SEARCH_BACKEND;
+    } else {
+        args->search_backend = ES_SEARCH_BACKEND;
+    }
+
+    LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url);
+    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index);
+    LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl);
+    LOG_DEBUGF("cli.c", "arg search_index_path=%s", args->search_index_path);
+    LOG_DEBUGF("cli.c", "arg search_backend=%d", args->search_backend);
+    LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline);
+    LOG_DEBUGF("cli.c", "arg dev=%d", args->dev);
+    LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address);
+    LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials);
+    LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials);
+    LOG_DEBUGF("cli.c", "arg auth_user=%s", args->auth_user);
+    LOG_DEBUGF("cli.c", "arg auth_pass=%s", args->auth_pass);
+    LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count);
     for (int i = 0; i < args->index_count; i++) {
-        LOG_DEBUGF("cli.c", "arg indices[%d]=%s", i, args->indices[i])
+        LOG_DEBUGF("cli.c", "arg indices[%d]=%s", i, args->indices[i]);
     }
 
     return 0;
@@ -565,6 +606,11 @@ index_args_t *index_args_create() {
     return args;
 }
 
+sqlite_index_args_t *sqlite_index_args_create() {
+    sqlite_index_args_t *args = calloc(sizeof(sqlite_index_args_t), 1);
+    return args;
+}
+
 web_args_t *web_args_create() {
     web_args_t *args = calloc(sizeof(web_args_t), 1);
     return args;
@@ -579,7 +625,7 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
 
     char *index_path = abspath(argv[1]);
     if (index_path == NULL) {
-        LOG_FATALF("cli.c", "Invalid index PATH argument. File not found: %s", argv[1])
+        LOG_FATALF("cli.c", "Invalid index PATH argument. File not found: %s", argv[1]);
     } else {
         args->index_path = index_path;
     }
@@ -600,12 +646,12 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
-    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
+    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path);
 
     char log_buf[5000];
     strncpy(log_buf, args->script, sizeof(log_buf));
     *(log_buf + sizeof(log_buf) - 1) = '\0';
-    LOG_DEBUGF("cli.c", "arg script=%s", log_buf)
+    LOG_DEBUGF("cli.c", "arg script=%s", log_buf);
 
     return 0;
 }

src/cli.h

@@ -9,12 +9,12 @@
 #define OPTION_VALUE_UNSPECIFIED (0)
 
 typedef struct scan_args {
-    float tn_quality;
+    int tn_quality;
     int tn_size;
     int content_size;
     int threads;
-    int scan_mem_limit_mib;
-    char *incremental;
+    int incremental;
+    int optimize_database;
     char *output;
     char *rewrite_url;
     char *name;
@@ -66,10 +66,21 @@ typedef struct index_args {
     int incremental;
 } index_args_t;
 
+typedef struct {
+    char *index_path;
+    char *search_index_path;
+} sqlite_index_args_t;
+
+typedef enum {
+    ES_SEARCH_BACKEND,
+    SQLITE_SEARCH_BACKEND,
+} search_backend_t;
+
 typedef struct web_args {
     char *es_url;
     char *es_index;
     int es_insecure_ssl;
+    char *search_index_path;
     char *listen_address;
     char *credentials;
     char *tag_credentials;
@@ -88,6 +99,7 @@ typedef struct web_args {
     int index_count;
     int dev;
     const char **indices;
+    search_backend_t search_backend;
 } web_args_t;
 
 typedef struct exec_args {
@@ -102,6 +114,8 @@ typedef struct exec_args {
 
 index_args_t *index_args_create();
 
+sqlite_index_args_t *sqlite_index_args_create();
+
 void index_args_destroy(index_args_t *args);
 
 web_args_t *web_args_create();
@@ -110,6 +124,8 @@ void web_args_destroy(web_args_t *args);
 
 int index_args_validate(index_args_t *args, int argc, const char **argv);
 
+int sqlite_index_args_validate(sqlite_index_args_t *args, int argc, const char **argv);
+
 int web_args_validate(web_args_t *args, int argc, const char **argv);
 
 exec_args_t *exec_args_create();
@@ -118,4 +134,5 @@ void exec_args_destroy(exec_args_t *args);
 
 int exec_args_validate(exec_args_t *args, int argc, const char **argv);
 
+
 #endif

src/ctx.c

@@ -3,9 +3,10 @@
 ScanCtx_t ScanCtx = {
         .stat_index_size = 0,
         .stat_tn_size = 0,
-        .dbg_current_files = NULL,
-        .pool = NULL
+        .pool = NULL,
+        .index.path = {0,},
 };
 WebCtx_t WebCtx;
 IndexCtx_t IndexCtx;
 LogCtx_t LogCtx;
+__thread ProcData_t ProcData;

src/ctx.h

@@ -16,47 +16,28 @@
 #include "libscan/msdoc/msdoc.h"
 #include "libscan/wpd/wpd.h"
 #include "libscan/json/json.h"
-#include "src/io/store.h"
+#include "src/database/database.h"
 #include "src/index/elastic.h"
+#include "sqlite3.h"
 
-#include <glib.h>
 #include <pcre.h>
 
 typedef struct {
     struct index_t index;
 
-    GHashTable *mime_table;
-    GHashTable *ext_table;
-
     tpool_t *pool;
 
-    tpool_t *writer_pool;
-
     int threads;
     int depth;
     int calculate_checksums;
-    size_t mem_limit;
 
     size_t stat_tn_size;
     size_t stat_index_size;
 
-    GHashTable *original_table;
-    GHashTable *copy_table;
-    GHashTable *new_table;
-    pthread_mutex_t copy_table_mu;
-
     pcre *exclude;
     pcre_extra *exclude_extra;
     int fast;
 
-    GHashTable *dbg_current_files;
-    pthread_mutex_t dbg_current_files_mu;
-
-    int dbg_failed_files_count;
-    int dbg_skipped_files_count;
-    int dbg_excluded_files_count;
-    pthread_mutex_t dbg_file_counts_mu;
-
     scan_arc_ctx_t arc_ctx;
     scan_comic_ctx_t comic_ctx;
     scan_ebook_ctx_t ebook_ctx;
@@ -85,10 +66,6 @@ typedef struct {
     char *es_index;
     int batch_size;
     tpool_t *pool;
-    store_t *tag_store;
-    GHashTable *tags;
-    store_t *meta_store;
-    GHashTable *meta;
     /**
      * Set to false when using --print
      */
@@ -99,6 +76,7 @@ typedef struct {
     char *es_url;
     es_version_t *es_version;
     char *es_index;
+    database_t *search_db;
     int es_insecure_ssl;
     int index_count;
     char *auth_user;
@@ -116,12 +94,21 @@ typedef struct {
     struct index_t indices[256];
     char lang[10];
     int dev;
+    int search_backend;
 } WebCtx_t;
 
+
+typedef struct {
+    int thread_id;
+    database_t *ipc_db;
+    database_t *index_db;
+} ProcData_t;
+
 extern ScanCtx_t ScanCtx;
 extern WebCtx_t WebCtx;
 extern IndexCtx_t IndexCtx;
 extern LogCtx_t LogCtx;
+extern __thread ProcData_t ProcData;
 
 
 #endif

src/database/database.c

@@ -0,0 +1,798 @@
+#include "database.h"
+#include "malloc.h"
+#include "src/ctx.h"
+#include <string.h>
+#include <pthread.h>
+#include "src/util.h"
+
+#include <time.h>
+
+
+database_t *database_create(const char *filename, database_type_t type) {
+    database_t *db = malloc(sizeof(database_t));
+
+    strcpy(db->filename, filename);
+    db->type = type;
+    db->select_thumbnail_stmt = NULL;
+    db->db = NULL;
+    db->tag_array = NULL;
+
+    db->ipc_ctx = NULL;
+
+    return db;
+}
+
+int tag_matches(const char *query, const char *tag) {
+    size_t query_len = strlen(query);
+    size_t tag_len = strlen(tag);
+
+    if (query_len >= tag_len) {
+        return FALSE;
+    }
+
+    return strncmp(tag, query, query_len) == 0 && *(tag + query_len) == '.';
+}
+
+void tag_matches_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    const char *tag = (const char *) sqlite3_value_text(argv[0]);
+
+    char **tags = *(char ***) sqlite3_user_data(ctx);
+
+    array_foreach(tags) {
+        if (tag_matches(tags[i], tag)) {
+            sqlite3_result_int(ctx, TRUE);
+            return;
+        }
+    }
+
+    sqlite3_result_int(ctx, FALSE);
+}
+
+__always_inline
+static int sep_rfind(const char *str) {
+    for (int i = (int) strlen(str); i >= 0; i--) {
+        if (str[i] == '/') {
+            return i;
+        }
+    }
+    return -1;
+}
+
+void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    const char *value = (const char *) sqlite3_value_text(argv[0]);
+
+    int stop = sep_rfind(value);
+    if (stop == -1) {
+        sqlite3_result_null(ctx);
+        return;
+    }
+    char parent[PATH_MAX * 3];
+    strncpy(parent, value, stop);
+
+    sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
+}
+
+void random_func(sqlite3_context *ctx, int argc, UNUSED(sqlite3_value **argv)) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    char state_buf[128] = {0,};
+    struct random_data buf;
+    int result;
+
+    long seed = sqlite3_value_int64(argv[0]);
+
+    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
+
+    random_r(&buf, &result);
+
+    sqlite3_result_int(ctx, result);
+}
+
+
+void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    database_ipc_ctx_t *ipc_ctx = sqlite3_user_data(ctx);
+
+    const char *current_job = (const char *) sqlite3_value_text(argv[0]);
+
+    char buf[PATH_MAX];
+    strcpy(buf, current_job);
+
+    strcpy(ipc_ctx->current_job[ProcData.thread_id], current_job);
+
+    sqlite3_result_text(ctx, "ok", -1, SQLITE_STATIC);
+}
+
+void database_initialize(database_t *db) {
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_open(db->filename, &db->db));
+
+    LOG_DEBUGF("database.c", "Initializing database %s", db->filename);
+    if (db->type == INDEX_DATABASE) {
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IndexDatabaseSchema, NULL, NULL, NULL));
+    } else if (db->type == IPC_CONSUMER_DATABASE || db->type == IPC_PRODUCER_DATABASE) {
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IpcDatabaseSchema, NULL, NULL, NULL));
+    } else if (db->type == FTS_DATABASE) {
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, FtsDatabaseSchema, NULL, NULL, NULL));
+    }
+
+    sqlite3_close(db->db);
+}
+
+void database_open(database_t *db) {
+    LOG_DEBUGF("database.c", "Opening database %s (%d)", db->filename, db->type);
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_open(db->filename, &db->db));
+    sqlite3_busy_timeout(db->db, 1000);
+
+    // TODO: Optional argument?
+//    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA cache_size = -200000;", NULL, NULL, NULL));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL));
+
+    if (db->type == INDEX_DATABASE) {
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA temp_store = memory;", NULL, NULL, NULL));
+    }
+
+    if (db->type == INDEX_DATABASE) {
+        // Prepare statements;
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "SELECT data FROM thumbnail WHERE id=? AND num=? LIMIT 1;", -1,
+                &db->select_thumbnail_stmt, NULL));
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "UPDATE document SET marked=1 WHERE id=? AND mtime=? RETURNING id",
+                -1,
+                &db->mark_document_stmt, NULL));
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "REPLACE INTO document_sidecar (id, json_data) VALUES (?,?)", -1,
+                &db->write_document_sidecar_stmt, NULL));
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "REPLACE INTO document (id, mtime, size, json_data) VALUES (?, ?, ?, ?);", -1,
+                &db->write_document_stmt, NULL));
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "INSERT INTO thumbnail (id, num, data) VALUES (?,?,?) ON CONFLICT DO UPDATE SET data=excluded.data;",
+                -1,
+                &db->write_thumbnail_stmt, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT json_data FROM document WHERE id=?", -1,
+                &db->get_document, NULL));
+
+        // Create functions
+        sqlite3_create_function(
+                db->db,
+                "path_parent",
+                1,
+                SQLITE_UTF8,
+                NULL,
+                path_parent_func,
+                NULL,
+                NULL
+        );
+    } else if (db->type == IPC_CONSUMER_DATABASE) {
+
+        sqlite3_create_function(
+                db->db,
+                "save_current_job_info",
+                1,
+                SQLITE_UTF8,
+                db->ipc_ctx,
+                save_current_job_info,
+                NULL,
+                NULL
+        );
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "DELETE FROM parse_job WHERE id = (SELECT MIN(id) FROM parse_job)"
+                " RETURNING filepath,mtime,st_size,save_current_job_info(filepath);",
+                -1, &db->pop_parse_job_stmt, NULL
+        ));
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "DELETE FROM index_job WHERE id = (SELECT MIN(id) FROM index_job)"
+                " RETURNING doc_id,type,line;",
+                -1, &db->pop_index_job_stmt, NULL
+        ));
+
+    } else if (db->type == IPC_PRODUCER_DATABASE) {
+        char sql[40];
+        int max_size_mb = 10; // TODO: read from args.
+
+        snprintf(sql, sizeof(sql), "PRAGMA max_page_count=%d", (max_size_mb * 1024 * 1024) / 4096);
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, sql, NULL, NULL, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "INSERT INTO parse_job (filepath,mtime,st_size) VALUES (?,?,?);", -1,
+                &db->insert_parse_job_stmt, NULL));
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "INSERT INTO index_job (doc_id,type,line) VALUES (?,?,?);", -1,
+                &db->insert_index_job_stmt, NULL));
+
+    } else if (db->type == FTS_DATABASE) {
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT path, count FROM path_index"
+                        " WHERE index_id=? AND depth BETWEEN ? AND ?"
+                        " LIMIT 65536", -1,
+                &db->fts_search_paths, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT json_data FROM document_index"
+                        " WHERE id=?", -1,
+                &db->fts_get_document, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT DISTINCT tag FROM tag"
+                        " WHERE tag GLOB (? || '*') ORDER BY tag LIMIT 100", -1,
+                &db->fts_suggest_tag, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT tag, count(*) FROM tag GROUP BY tag", -1,
+                &db->fts_get_tags, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT path, count FROM path_index"
+                        " WHERE (index_id=?1 OR ?1 IS NULL) AND depth BETWEEN ? AND ?"
+                        " AND (path = ?4 or path GLOB ?5)"
+                        " LIMIT 65536", -1,
+                &db->fts_search_paths_w_prefix, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT path, count FROM path_index"
+                        " WHERE depth BETWEEN ? AND ?"
+                        " AND path GLOB ?"
+                        " LIMIT 65536", -1,
+                &db->fts_suggest_paths, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT * FROM stats", -1,
+                &db->fts_date_range, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT mime, sum(count) FROM mime_index WHERE mime is not NULL GROUP BY mime", -1,
+                &db->fts_get_mimetypes, NULL));
+
+        sqlite3_create_function(
+                db->db,
+                "random_seeded",
+                1,
+                SQLITE_UTF8,
+                NULL,
+                random_func,
+                NULL,
+                NULL
+        );
+
+        sqlite3_create_function(
+                db->db,
+                "path_parent",
+                1,
+                SQLITE_UTF8,
+                NULL,
+                path_parent_func,
+                NULL,
+                NULL
+        );
+
+        sqlite3_create_function(
+                db->db,
+                "tag_matches",
+                1,
+                SQLITE_UTF8,
+                &db->tag_array,
+                tag_matches_func,
+                NULL,
+                NULL
+        );
+    }
+
+    if (db->type == FTS_DATABASE || db->type == INDEX_DATABASE) {
+        // Tag table is the same schema for FTS database & index database
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "INSERT INTO tag (id, tag) VALUES (?,?) ON CONFLICT DO NOTHING;",
+                -1,
+                &db->write_tag_stmt, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "DELETE FROM tag WHERE id=? AND tag=?;",
+                -1,
+                &db->delete_tag_stmt, NULL));
+    }
+}
+
+void database_close(database_t *db, int optimize) {
+    LOG_DEBUGF("database.c", "Closing database %s", db->filename);
+
+    if (optimize) {
+        LOG_DEBUG("database.c", "Optimizing database");
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "VACUUM;", NULL, NULL, NULL));
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA optimize;", NULL, NULL, NULL));
+    }
+
+    if (db->db) {
+        sqlite3_close(db->db);
+    }
+
+    if (db->type == IPC_PRODUCER_DATABASE) {
+        remove(db->filename);
+    }
+
+    free(db);
+    db = NULL;
+}
+
+void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *return_value_len) {
+    sqlite3_bind_text(db->select_thumbnail_stmt, 1, id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->select_thumbnail_stmt, 2, num);
+
+    int ret = sqlite3_step(db->select_thumbnail_stmt);
+
+    if (ret == SQLITE_DONE) {
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->select_thumbnail_stmt));
+        *return_value_len = 0;
+        return NULL;
+    }
+
+    CRASH_IF_STMT_FAIL(ret);
+
+    const void *blob = sqlite3_column_blob(db->select_thumbnail_stmt, 0);
+    const int blob_size = sqlite3_column_bytes(db->select_thumbnail_stmt, 0);
+
+    *return_value_len = blob_size;
+    void *return_data = malloc(blob_size);
+    memcpy(return_data, blob, blob_size);
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->select_thumbnail_stmt));
+
+    return return_data;
+}
+
+void database_write_index_descriptor(database_t *db, index_descriptor_t *desc) {
+
+    sqlite3_exec(db->db, "DELETE FROM descriptor;", NULL, NULL, NULL);
+
+    sqlite3_stmt *stmt;
+
+    sqlite3_prepare_v2(db->db, "INSERT INTO descriptor (id, version_major, version_minor, version_patch,"
+                               " root, name, rewrite_url, timestamp) VALUES (?,?,?,?,?,?,?,?);", -1, &stmt, NULL);
+    sqlite3_bind_text(stmt, 1, desc->id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(stmt, 2, desc->version_major);
+    sqlite3_bind_int(stmt, 3, desc->version_minor);
+    sqlite3_bind_int(stmt, 4, desc->version_patch);
+    sqlite3_bind_text(stmt, 5, desc->root, -1, SQLITE_STATIC);
+    sqlite3_bind_text(stmt, 6, desc->name, -1, SQLITE_STATIC);
+    sqlite3_bind_text(stmt, 7, desc->rewrite_url, -1, SQLITE_STATIC);
+    sqlite3_bind_int64(stmt, 8, desc->timestamp);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
+
+    sqlite3_finalize(stmt);
+}
+
+index_descriptor_t *database_read_index_descriptor(database_t *db) {
+
+    sqlite3_stmt *stmt;
+
+    sqlite3_prepare_v2(db->db, "SELECT id, version_major, version_minor, version_patch,"
+                               " root, name, rewrite_url, timestamp FROM descriptor;", -1, &stmt, NULL);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
+
+    const char *id = (char *) sqlite3_column_text(stmt, 0);
+    int v_major = sqlite3_column_int(stmt, 1);
+    int v_minor = sqlite3_column_int(stmt, 2);
+    int v_patch = sqlite3_column_int(stmt, 3);
+    const char *root = (char *) sqlite3_column_text(stmt, 4);
+    const char *name = (char *) sqlite3_column_text(stmt, 5);
+    const char *rewrite_url = (char *) sqlite3_column_text(stmt, 6);
+    int timestamp = sqlite3_column_int(stmt, 7);
+
+    index_descriptor_t *desc = malloc(sizeof(index_descriptor_t));
+    strcpy(desc->id, id);
+    snprintf(desc->version, sizeof(desc->version), "%d.%d.%d", v_major, v_minor, v_patch);
+    desc->version_major = v_major;
+    desc->version_minor = v_minor;
+    desc->version_patch = v_patch;
+    strcpy(desc->root, root);
+    strcpy(desc->name, name);
+    strcpy(desc->rewrite_url, rewrite_url);
+    desc->timestamp = timestamp;
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_finalize(stmt));
+
+    return desc;
+}
+
+database_iterator_t *database_create_delete_list_iterator(database_t *db) {
+
+    sqlite3_stmt *stmt;
+    sqlite3_prepare_v2(db->db, "SELECT id FROM delete_list;", -1, &stmt, NULL);
+
+    database_iterator_t *iter = malloc(sizeof(database_iterator_t));
+
+    iter->stmt = stmt;
+    iter->db = db;
+
+    return iter;
+}
+
+char *database_delete_list_iter(database_iterator_t *iter) {
+    int ret = sqlite3_step(iter->stmt);
+
+    if (ret == SQLITE_ROW) {
+        const char *id = (const char *) sqlite3_column_text(iter->stmt, 0);
+        char *id_heap = malloc(strlen(id) + 1);
+        strcpy(id_heap, id);
+        return id_heap;
+    }
+
+    if (ret != SQLITE_DONE) {
+        LOG_FATALF("database.c", "FIXME: delete iter returned %s", sqlite3_errmsg(iter->db->db));
+    }
+
+    if (sqlite3_finalize(iter->stmt) != SQLITE_OK) {
+        LOG_FATALF("database.c", "FIXME: delete iter returned %s", sqlite3_errmsg(iter->db->db));
+    }
+
+    iter->stmt = NULL;
+
+    return NULL;
+}
+
+database_iterator_t *database_create_document_iterator(database_t *db) {
+
+    sqlite3_stmt *stmt;
+
+    // TODO optimization: remove mtime, size, _id from json_data
+
+    sqlite3_prepare_v2(db->db, "WITH doc (j) AS (SELECT CASE"
+                               " WHEN sc.json_data IS NULL THEN"
+                               "  CASE"
+                               "   WHEN t.tag IS NULL THEN"
+                               "    json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime)"
+                               "   ELSE"
+                               "    json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime, '$.tag', json_group_array(t.tag))"
+                               "   END"
+                               " ELSE"
+                               "  CASE"
+                               "   WHEN t.tag IS NULL THEN"
+                               "    json_patch(json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime), sc.json_data)"
+                               "   ELSE"
+                               //   This will overwrite any tags specified in the sidecar file!
+                               //   TODO: concatenate the two arrays?
+                               "    json_set(json_patch(document.json_data, sc.json_data), '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime, '$.tag', json_group_array(t.tag))"
+                               "   END"
+                               " END"
+                               " FROM document"
+                               " LEFT JOIN document_sidecar sc ON document.id = sc.id"
+                               " LEFT JOIN tag t ON document.id = t.id"
+                               " GROUP BY document.id)"
+                               " SELECT json_set(j, '$.index', (SELECT id FROM descriptor)) FROM doc", -1, &stmt, NULL);
+
+    database_iterator_t *iter = malloc(sizeof(database_iterator_t));
+
+    iter->stmt = stmt;
+    iter->db = db;
+
+    return iter;
+}
+
+cJSON *database_document_iter(database_iterator_t *iter) {
+
+    if (iter->stmt == NULL) {
+        LOG_ERROR("database.c", "FIXME: database_document_iter() called after iteration stopped");
+        return NULL;
+    }
+
+    int ret = sqlite3_step(iter->stmt);
+
+    if (ret == SQLITE_ROW) {
+        const char *json_string = (const char *) sqlite3_column_text(iter->stmt, 0);
+        return cJSON_Parse(json_string);
+    }
+
+    if (ret != SQLITE_DONE) {
+        LOG_FATALF("database.c", "FIXME: doc iter returned %s", sqlite3_errmsg(iter->db->db));
+    }
+
+    if (sqlite3_finalize(iter->stmt) != SQLITE_OK) {
+        LOG_FATALF("database.c", "FIXME: doc iter returned %s", sqlite3_errmsg(iter->db->db));
+    }
+
+    iter->stmt = NULL;
+
+    return NULL;
+}
+
+cJSON *database_incremental_scan_begin(database_t *db) {
+    LOG_DEBUG("database.c", "Preparing database for incremental scan");
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "UPDATE document SET marked=0;", NULL, NULL, NULL));
+}
+
+cJSON *database_incremental_scan_end(database_t *db) {
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM delete_list WHERE id IN (SELECT id FROM document WHERE marked=1);",
+            NULL, NULL, NULL
+    ));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM thumbnail WHERE id IN (SELECT id FROM document WHERE marked=0);",
+            NULL, NULL, NULL
+    ));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "INSERT INTO delete_list (id) SELECT id FROM document WHERE marked=0;",
+            NULL, NULL, NULL
+    ));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM document_sidecar WHERE id IN (SELECT id FROM document WHERE marked=0);",
+            NULL, NULL, NULL
+    ));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM document WHERE marked=0;",
+            NULL, NULL, NULL
+    ));
+}
+
+int database_mark_document(database_t *db, const char *id, int mtime) {
+    sqlite3_bind_text(db->mark_document_stmt, 1, id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->mark_document_stmt, 2, mtime);
+
+    pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
+    int ret = sqlite3_step(db->mark_document_stmt);
+
+    if (ret == SQLITE_ROW) {
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->mark_document_stmt));
+        pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
+        return TRUE;
+    }
+
+    if (ret == SQLITE_DONE) {
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->mark_document_stmt));
+        pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
+        return FALSE;
+    }
+    pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
+
+    CRASH_IF_STMT_FAIL(ret);
+}
+
+void database_write_document(database_t *db, document_t *doc, const char *json_data) {
+    sqlite3_bind_text(db->write_document_stmt, 1, doc->doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->write_document_stmt, 2, doc->mtime);
+    sqlite3_bind_int64(db->write_document_stmt, 3, (long) doc->size);
+    sqlite3_bind_text(db->write_document_stmt, 4, json_data, -1, SQLITE_STATIC);
+
+    pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->write_document_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_document_stmt));
+    pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
+}
+
+
+void database_write_document_sidecar(database_t *db, const char *id, const char *json_data) {
+    sqlite3_bind_text(db->write_document_sidecar_stmt, 1, id, -1, SQLITE_STATIC);
+    sqlite3_bind_text(db->write_document_sidecar_stmt, 2, json_data, -1, SQLITE_STATIC);
+
+    pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->write_document_sidecar_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_document_sidecar_stmt));
+    pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
+}
+
+void database_write_thumbnail(database_t *db, const char *id, int num, void *data, size_t data_size) {
+    sqlite3_bind_text(db->write_thumbnail_stmt, 1, id, -1, SQLITE_STATIC);
+    sqlite3_bind_int(db->write_thumbnail_stmt, 2, num);
+    sqlite3_bind_blob(db->write_thumbnail_stmt, 3, data, (int) data_size, SQLITE_STATIC);
+
+    pthread_mutex_lock(&db->ipc_ctx->index_db_mutex);
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->write_thumbnail_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_thumbnail_stmt));
+    pthread_mutex_unlock(&db->ipc_ctx->index_db_mutex);
+}
+
+
+job_t *database_get_work(database_t *db, job_type_t job_type) {
+    job_t *job;
+
+    pthread_mutex_lock(&db->ipc_ctx->mutex);
+    while (db->ipc_ctx->job_count == 0 && !db->ipc_ctx->no_more_jobs) {
+        pthread_cond_timedwait_ms(&db->ipc_ctx->has_work_cond, &db->ipc_ctx->mutex, 10);
+    }
+    pthread_mutex_unlock(&db->ipc_ctx->mutex);
+
+    pthread_mutex_lock(&db->ipc_ctx->db_mutex);
+
+    if (job_type == JOB_PARSE_JOB) {
+        int ret = sqlite3_step(db->pop_parse_job_stmt);
+        if (ret == SQLITE_DONE) {
+            CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_parse_job_stmt));
+            pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
+            return NULL;
+        } else {
+            CRASH_IF_STMT_FAIL(ret);
+        }
+
+        job = malloc(sizeof(*job));
+
+        job->parse_job = create_parse_job(
+                (const char *) sqlite3_column_text(db->pop_parse_job_stmt, 0),
+                sqlite3_column_int(db->pop_parse_job_stmt, 1),
+                sqlite3_column_int64(db->pop_parse_job_stmt, 2));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_parse_job_stmt));
+    } else {
+
+        int ret = sqlite3_step(db->pop_index_job_stmt);
+
+        if (ret == SQLITE_DONE) {
+            CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_index_job_stmt));
+            pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
+            return NULL;
+        }
+
+        CRASH_IF_STMT_FAIL(ret);
+
+        job = malloc(sizeof(*job));
+
+        const char *line = (const char *) sqlite3_column_text(db->pop_index_job_stmt, 2);
+        if (line != NULL) {
+            job->bulk_line = malloc(sizeof(es_bulk_line_t) + strlen(line) + 1);
+            strcpy(job->bulk_line->line, line);
+        } else {
+            job->bulk_line = malloc(sizeof(es_bulk_line_t));
+        }
+        strcpy(job->bulk_line->doc_id, (const char *) sqlite3_column_text(db->pop_index_job_stmt, 0));
+        job->bulk_line->type = sqlite3_column_int(db->pop_index_job_stmt, 1);
+        job->bulk_line->next = NULL;
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->pop_index_job_stmt));
+    }
+
+    pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
+
+    pthread_mutex_lock(&db->ipc_ctx->mutex);
+    db->ipc_ctx->job_count -= 1;
+    pthread_mutex_unlock(&db->ipc_ctx->mutex);
+
+    job->type = job_type;
+    return job;
+}
+
+void database_add_work(database_t *db, job_t *job) {
+    int ret;
+
+    pthread_mutex_lock(&db->ipc_ctx->db_mutex);
+
+    if (job->type == JOB_PARSE_JOB) {
+        do {
+            sqlite3_bind_text(db->insert_parse_job_stmt, 1, job->parse_job->filepath, -1, SQLITE_STATIC);
+            sqlite3_bind_int(db->insert_parse_job_stmt, 2, job->parse_job->vfile.mtime);
+            sqlite3_bind_int64(db->insert_parse_job_stmt, 3, (long) job->parse_job->vfile.st_size);
+
+            ret = sqlite3_step(db->insert_parse_job_stmt);
+
+            if (ret == SQLITE_FULL) {
+                sqlite3_reset(db->insert_parse_job_stmt);
+                pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
+                usleep(1000000);
+                pthread_mutex_lock(&db->ipc_ctx->db_mutex);
+                continue;
+            } else {
+                CRASH_IF_STMT_FAIL(ret);
+            }
+
+            ret = sqlite3_reset(db->insert_parse_job_stmt);
+            if (ret == SQLITE_FULL) {
+                pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
+                usleep(100000);
+                pthread_mutex_lock(&db->ipc_ctx->db_mutex);
+            } else if (ret != SQLITE_OK) {
+                LOG_FATALF("database.c", "sqlite3_reset returned error %d", ret);
+            }
+        } while (ret != SQLITE_DONE && ret != SQLITE_OK);
+    } else if (job->type == JOB_BULK_LINE) {
+        do {
+            sqlite3_bind_text(db->insert_index_job_stmt, 1, job->bulk_line->doc_id, -1, SQLITE_STATIC);
+            sqlite3_bind_int(db->insert_index_job_stmt, 2, job->bulk_line->type);
+            if (job->bulk_line->type != ES_BULK_LINE_DELETE) {
+                sqlite3_bind_text(db->insert_index_job_stmt, 3, job->bulk_line->line, -1, SQLITE_STATIC);
+            } else {
+                sqlite3_bind_null(db->insert_index_job_stmt, 3);
+            }
+
+            ret = sqlite3_step(db->insert_index_job_stmt);
+
+            if (ret == SQLITE_FULL) {
+                sqlite3_reset(db->insert_index_job_stmt);
+                pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
+                usleep(100000);
+                pthread_mutex_lock(&db->ipc_ctx->db_mutex);
+                continue;
+            } else {
+                CRASH_IF_STMT_FAIL(ret);
+            }
+
+            ret = sqlite3_reset(db->insert_index_job_stmt);
+            if (ret == SQLITE_FULL) {
+                pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
+                usleep(100000);
+                pthread_mutex_lock(&db->ipc_ctx->db_mutex);
+            } else if (ret != SQLITE_OK) {
+                LOG_FATALF("database.c", "sqlite3_reset returned error %d", ret);
+            }
+
+        } while (ret != SQLITE_DONE && ret != SQLITE_OK);
+    } else {
+        LOG_FATAL("database.c", "FIXME: invalid job type");
+    }
+    pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
+
+    pthread_mutex_lock(&db->ipc_ctx->mutex);
+    db->ipc_ctx->job_count += 1;
+    pthread_cond_signal(&db->ipc_ctx->has_work_cond);
+    pthread_mutex_unlock(&db->ipc_ctx->mutex);
+}
+
+void database_write_tag(database_t *db, char *doc_id, char *tag) {
+    sqlite3_bind_text(db->write_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_text(db->write_tag_stmt, 2, tag, -1, SQLITE_STATIC);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->write_tag_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_tag_stmt));
+}
+
+void database_delete_tag(database_t *db, char *doc_id, char *tag) {
+    sqlite3_bind_text(db->delete_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_text(db->delete_tag_stmt, 2, tag, -1, SQLITE_STATIC);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->delete_tag_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->delete_tag_stmt));
+}
+
+cJSON *database_get_document(database_t *db, char *doc_id) {
+    sqlite3_bind_text(db->get_document, 1, doc_id, -1, SQLITE_STATIC);
+
+    int ret = sqlite3_step(db->get_document);
+    CRASH_IF_STMT_FAIL(ret);
+
+    cJSON *json;
+
+    if (ret == SQLITE_ROW) {
+        const char *json_str = sqlite3_column_text(db->get_document, 0);
+        json = cJSON_Parse(json_str);
+    } else {
+        json = NULL;
+    }
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->get_document));
+
+    return json;
+}

src/database/database.h

@@ -0,0 +1,229 @@
+#ifndef SIST2_DATABASE_H
+#define SIST2_DATABASE_H
+
+#include <sqlite3.h>
+#include <cjson/cJSON.h>
+#include "src/sist.h"
+#include "src/index/elastic.h"
+
+typedef struct index_descriptor index_descriptor_t;
+
+extern const char *IpcDatabaseSchema;
+extern const char *IndexDatabaseSchema;
+extern const char *FtsDatabaseSchema;
+
+typedef enum {
+    INDEX_DATABASE,
+    IPC_CONSUMER_DATABASE,
+    IPC_PRODUCER_DATABASE,
+    FTS_DATABASE
+} database_type_t;
+
+typedef enum {
+    DATABASE_STAT_INVALID,
+    DATABASE_STAT_TREEMAP,
+    DATABASE_STAT_MIME_AGG,
+    DATABASE_STAT_SIZE_AGG,
+    DATABASE_STAT_DATE_AGG,
+} database_stat_type_d;
+
+typedef enum {
+    JOB_UNDEFINED,
+    JOB_BULK_LINE,
+    JOB_PARSE_JOB
+} job_type_t;
+
+typedef enum {
+    FTS_SORT_INVALID,
+    FTS_SORT_SCORE,
+    FTS_SORT_SIZE,
+    FTS_SORT_MTIME,
+    FTS_SORT_RANDOM,
+    FTS_SORT_NAME,
+    FTS_SORT_ID,
+} fts_sort_t;
+
+typedef struct {
+    job_type_t type;
+    union {
+        parse_job_t *parse_job;
+        es_bulk_line_t *bulk_line;
+    };
+} job_t;
+
+typedef struct {
+    int job_count;
+    int no_more_jobs;
+    int completed_job_count;
+
+    pthread_mutex_t mutex;
+    pthread_mutex_t db_mutex;
+    pthread_mutex_t index_db_mutex;
+    pthread_cond_t has_work_cond;
+    char current_job[MAX_THREADS][PATH_MAX * 2];
+} database_ipc_ctx_t;
+
+typedef struct {
+    double date_min;
+    double date_max;
+} database_summary_stats_t;
+
+typedef struct database {
+    char filename[PATH_MAX];
+    database_type_t type;
+    sqlite3 *db;
+
+    // Prepared statements
+    sqlite3_stmt *select_thumbnail_stmt;
+    sqlite3_stmt *treemap_merge_up_update_stmt;
+    sqlite3_stmt *treemap_merge_up_delete_stmt;
+
+    sqlite3_stmt *mark_document_stmt;
+    sqlite3_stmt *write_document_stmt;
+    sqlite3_stmt *write_document_sidecar_stmt;
+    sqlite3_stmt *write_thumbnail_stmt;
+    sqlite3_stmt *get_document;
+
+    sqlite3_stmt *delete_tag_stmt;
+    sqlite3_stmt *write_tag_stmt;
+
+    sqlite3_stmt *insert_parse_job_stmt;
+    sqlite3_stmt *insert_index_job_stmt;
+    sqlite3_stmt *pop_parse_job_stmt;
+    sqlite3_stmt *pop_index_job_stmt;
+
+    sqlite3_stmt *fts_search_paths;
+    sqlite3_stmt *fts_search_paths_w_prefix;
+    sqlite3_stmt *fts_suggest_paths;
+    sqlite3_stmt *fts_date_range;
+    sqlite3_stmt *fts_get_mimetypes;
+    sqlite3_stmt *fts_get_document;
+    sqlite3_stmt *fts_suggest_tag;
+    sqlite3_stmt *fts_get_tags;
+
+    char **tag_array;
+
+    database_ipc_ctx_t *ipc_ctx;
+} database_t;
+
+typedef struct {
+    database_t *db;
+    sqlite3_stmt *stmt;
+} database_iterator_t;
+
+typedef struct {
+    const char *path;
+    const char *parent;
+    long size;
+} treemap_row_t;
+
+
+database_t *database_create(const char *filename, database_type_t type);
+
+void database_initialize(database_t *db);
+
+void database_open(database_t *db);
+
+void database_close(database_t *, int optimize);
+
+void database_write_thumbnail(database_t *db, const char *id, int num, void *data, size_t data_size);
+
+void *database_read_thumbnail(database_t *db, const char *id, int num, size_t *return_value_len);
+
+void database_write_index_descriptor(database_t *db, index_descriptor_t *desc);
+
+index_descriptor_t *database_read_index_descriptor(database_t *db);
+
+void database_write_document(database_t *db, document_t *doc, const char *json_data);
+
+database_iterator_t *database_create_document_iterator(database_t *db);
+
+cJSON *database_document_iter(database_iterator_t *);
+
+#define database_document_iter_foreach(element, iter) \
+    for (cJSON *(element) = database_document_iter(iter); (element) != NULL; (element) = database_document_iter(iter))
+
+database_iterator_t *database_create_delete_list_iterator(database_t *db);
+
+char *database_delete_list_iter(database_iterator_t *iter);
+
+#define database_delete_list_iter_foreach(element, iter) \
+    for (char *(element) = database_delete_list_iter(iter); (element) != NULL; (element) = database_delete_list_iter(iter))
+
+
+cJSON *database_incremental_scan_begin(database_t *db);
+
+cJSON *database_incremental_scan_end(database_t *db);
+
+int database_mark_document(database_t *db, const char *id, int mtime);
+
+void database_write_document_sidecar(database_t *db, const char *id, const char *json_data);
+
+database_iterator_t *database_create_treemap_iterator(database_t *db, long threshold);
+
+treemap_row_t database_treemap_iter(database_iterator_t *iter);
+
+#define database_treemap_iter_foreach(element, iter) \
+    for (treemap_row_t element = database_treemap_iter(iter); (element).path != NULL; (element) = database_treemap_iter(iter))
+
+
+void database_generate_stats(database_t *db, double treemap_threshold);
+
+database_stat_type_d database_get_stat_type_by_mnemonic(const char *name);
+
+job_t *database_get_work(database_t *db, job_type_t job_type);
+
+void database_add_work(database_t *db, job_t *job);
+
+cJSON *database_get_stats(database_t *db, database_stat_type_d type);
+
+#define CRASH_IF_STMT_FAIL(x) do { \
+        int return_value = x;                \
+        if (return_value != SQLITE_DONE && return_value != SQLITE_ROW) {     \
+            LOG_FATALF("database.c", "Sqlite error @ %s:%d : (%d) %s", __BASE_FILE__, __LINE__, return_value, sqlite3_errmsg(db->db)); \
+        }                           \
+    } while (0)
+
+#define CRASH_IF_NOT_SQLITE_OK(x) do { \
+        int return_value = x;                \
+        if (return_value != SQLITE_OK) {     \
+            LOG_FATALF("database.c", "Sqlite error @ %s:%d : (%d) %s", __BASE_FILE__, __LINE__, return_value, sqlite3_errmsg(db->db)); \
+        }                           \
+    } while (0)
+
+void database_fts_attach(database_t *db, const char *fts_database_path);
+
+void database_fts_index(database_t *db);
+
+void database_fts_optimize(database_t *db);
+
+cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
+                              int suggest);
+
+cJSON *database_fts_get_mimetypes(database_t *db);
+
+database_summary_stats_t database_fts_get_date_range(database_t *db);
+
+cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
+                           long size_max, long date_min, long date_max, int page_size,
+                           char **index_ids, char **mime_types, char **tags, int sort_asc,
+                           fts_sort_t sort, int seed, char **after, int fetch_aggregations,
+                           int highlight, int highlight_context_size);
+
+void database_write_tag(database_t *db, char *doc_id, char *tag);
+
+void database_delete_tag(database_t *db, char *doc_id, char *tag);
+
+void database_fts_detach(database_t *db);
+
+cJSON *database_fts_get_document(database_t *db, char *doc_id);
+
+database_summary_stats_t database_fts_sync_tags(database_t *db);
+
+cJSON *database_fts_suggest_tag(database_t *db, char *prefix);
+
+cJSON *database_fts_get_tags(database_t *db);
+
+cJSON *database_get_document(database_t *db, char *doc_id);
+
+#endif

src/database/database_fts.c

@@ -0,0 +1,804 @@
+#include "database.h"
+#include "src/ctx.h"
+
+void database_fts_detach(database_t *db) {
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "DETACH DATABASE fts",
+            NULL, NULL, NULL
+    ));
+}
+
+void database_fts_attach(database_t *db, const char *fts_database_path) {
+
+    LOG_DEBUGF("database_fts.c", "Attaching to %s", fts_database_path);
+
+    sqlite3_stmt *stmt;
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+            db->db, "ATTACH DATABASE ? AS fts"
+                    "", -1, &stmt, NULL));
+
+    sqlite3_bind_text(stmt, 1, fts_database_path, -1, SQLITE_STATIC);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
+    sqlite3_finalize(stmt);
+}
+
+int database_fts_get_max_path_depth(database_t *db) {
+    sqlite3_stmt *stmt;
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+            db->db, "SELECT MAX(depth) FROM path_tmp", -1, &stmt, NULL));
+    CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
+
+    int max_depth = sqlite3_column_int(stmt, 0);
+    sqlite3_finalize(stmt);
+
+    return max_depth;
+}
+
+void database_fts_index(database_t *db) {
+
+    LOG_INFO("database_fts.c", "Creating content table.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "WITH docs AS ("
+            " SELECT document.id as id, (SELECT id FROM descriptor) as index_id, size,"
+            "  document.json_data ->> 'name' as name,"
+            "  document.json_data ->> 'path' as path,"
+            "  mtime,"
+            "  document.json_data ->> 'mime' as mime,"
+            "  CASE"
+            "   WHEN sc.json_data IS NULL THEN"
+            "    json_set(document.json_data, "
+            "             '$._id',document.id,"
+            "             '$.size',document.size, "
+            "             '$.mtime',document.mtime)"
+            "    ELSE json_patch("
+            "             json_set(document.json_data,"
+            "                      '$._id',document.id,"
+            "                      '$.size',document.size,"
+            "                      '$.mtime', document.mtime),"
+            "             sc.json_data) END"
+            " FROM document"
+            "  LEFT JOIN document_sidecar sc ON document.id = sc.id"
+            " GROUP BY document.id)"
+            " INSERT"
+            " INTO fts.document_index (id, index_id, size, name, path, mtime, mime, json_data)"
+            " SELECT * FROM docs WHERE true"
+            " on conflict (id, index_id) do update set "
+            "  size=excluded.size, mtime=excluded.mtime, mime=excluded.mime, json_data=excluded.json_data;",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Deleting old documents.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM fts.document_index"
+            " WHERE id IN (SELECT id FROM delete_list)"
+            "  AND index_id = (SELECT id FROM descriptor);",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Generating summary stats");
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM fts.stats", NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "INSERT INTO fts.stats "
+                    "SELECT min(mtime), max(mtime) FROM fts.document_index",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Generating mime index");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "DELETE FROM fts.mime_index;", NULL, NULL, NULL));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "INSERT INTO fts.mime_index (index_id, mime, count) "
+                    "SELECT index_id, mime, count(*) FROM fts.document_index GROUP BY index_id, mime",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Generating path index");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "CREATE TEMP TABLE path_tmp ("
+            " path TEXT,"
+            " index_id TEXT,"
+            " count INTEGER NOT NULL,"
+            " depth INTEGER NOT NULL,"
+            " children INTEGER NOT NULL DEFAULT(0),"
+            " total INTEGER AS (count + children),"
+            " PRIMARY KEY (path, index_id)"
+            ");", NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "INSERT INTO path_tmp (path, index_id, count, depth)"
+            " SELECT path, index_id, count(*), CASE WHEN length(json_data->>'path') == 0 THEN 0"
+            " ELSE 1 + length(json_data->>'path') - length(REPLACE(json_data->>'path', '/', ''))"
+            " END as depth FROM document_index WHERE depth > 0"
+            " GROUP BY path", NULL, NULL, NULL));
+
+    int max_depth = database_fts_get_max_path_depth(db);
+
+    for (int i = max_depth; i > 1; i--) {
+        sqlite3_stmt *stmt;
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "INSERT INTO path_tmp (path, index_id, children, depth, count)"
+                " SELECT path_parent(path) parent, index_id, (SELECT COALESCE(sum(count), 0) FROM path_tmp WHERE path "
+                " BETWEEN path_parent(p.path) || '/' AND path_parent(p.path) || '/𘚟' AND index_id = p.index_id) as cnt, depth-1, 0 "
+                " FROM path_tmp p WHERE depth=? GROUP BY parent"
+                " ON CONFLICT(path, index_id) DO UPDATE SET children=excluded.children",
+                -1, &stmt, NULL));
+        sqlite3_bind_int(stmt, 1, i);
+        CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
+
+        LOG_DEBUGF("database_fts.c", "Path index depth %d (%d)", i, sqlite3_changes(db->db));
+    }
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM path_index;"
+            "INSERT INTO path_index (path, index_id, count, depth) SELECT path, index_id, total, depth FROM path_tmp",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Generating search index.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "INSERT INTO search(search) VALUES ('delete-all')",
+            NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "INSERT INTO search(rowid, name, content, title) SELECT id, name, content, title from document_view",
+            NULL, NULL, NULL));
+}
+
+void database_fts_optimize(database_t *db) {
+    LOG_INFO("database_fts.c", "Optimizing search index.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "INSERT INTO search(search) VALUES('optimize');",
+            NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA fts.optimize;", NULL, NULL, NULL));
+}
+
+cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
+                              int suggest) {
+
+    sqlite3_stmt *stmt;
+
+    if (suggest) {
+        stmt = db->fts_suggest_paths;
+        sqlite3_bind_int(stmt, 1, depth_min);
+        sqlite3_bind_int(stmt, 2, depth_max);
+
+        if (prefix) {
+            char *prefix_glob = malloc(strlen(prefix) + 2);
+            sprintf(prefix_glob, "%s*", prefix);
+            sqlite3_bind_text(stmt, 3, prefix_glob, -1, SQLITE_TRANSIENT);
+            free(prefix_glob);
+        }
+
+    } else if (prefix) {
+        stmt = db->fts_search_paths_w_prefix;
+        if (index_id) {
+            sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
+        } else {
+            sqlite3_bind_null(stmt, 1);
+        }
+        sqlite3_bind_int(stmt, 2, depth_min);
+        sqlite3_bind_int(stmt, 3, depth_max);
+
+        char *prefix_glob = malloc(strlen(prefix) + 3);
+        sprintf(prefix_glob, "%s/*", prefix);
+        sqlite3_bind_text(stmt, 4, prefix, -1, SQLITE_STATIC);
+        sqlite3_bind_text(stmt, 5, prefix_glob, -1, SQLITE_TRANSIENT);
+        free(prefix_glob);
+    } else {
+        stmt = db->fts_search_paths;
+        if (index_id) {
+            sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
+        } else {
+            sqlite3_bind_null(stmt, 1);
+        }
+        sqlite3_bind_int(stmt, 2, depth_min);
+        sqlite3_bind_int(stmt, 3, depth_max);
+    }
+
+    cJSON *json = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(stmt);
+        CRASH_IF_STMT_FAIL(ret);
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON *row = cJSON_CreateObject();
+
+        cJSON_AddStringToObject(row, "path", (const char *) sqlite3_column_text(stmt, 0));
+        cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 1));
+
+        cJSON_AddItemToArray(json, row);
+    } while (TRUE);
+
+    sqlite3_reset(stmt);
+
+    return json;
+}
+
+cJSON *database_fts_get_mimetypes(database_t *db) {
+
+    cJSON *json = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(db->fts_get_mimetypes);
+        CRASH_IF_STMT_FAIL(ret);
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON *row = cJSON_CreateObject();
+
+        cJSON_AddStringToObject(row, "mime", (const char *) sqlite3_column_text(db->fts_get_mimetypes, 0));
+        cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(db->fts_get_mimetypes, 1));
+
+        cJSON_AddItemToArray(json, row);
+    } while (TRUE);
+
+    sqlite3_reset(db->fts_get_mimetypes);
+
+    return json;
+}
+
+const char *size_where_clause(long size_min, long size_max) {
+    if (size_min > 0 && size_max > 0) {
+        return "size BETWEEN @size_min AND @size_max";
+    } else if (size_min > 0) {
+        return "size >= @size_min";
+    } else if (size_max > 0) {
+        return "size <= @size_max";
+    }
+
+    return NULL;
+}
+
+const char *date_where_clause(long date_min, long date_max) {
+    if (date_min > 0 && date_max > 0) {
+        return "mtime BETWEEN @date_min AND @date_max";
+    } else if (date_min > 0) {
+        return "mtime >= @date_min";
+    } else if (date_max > 0) {
+        return "mtime <= @date_max";
+    }
+
+    return NULL;
+}
+
+int array_length(char **arr) {
+
+    if (arr == NULL) {
+        return 0;
+    }
+
+    int count = -1;
+    while (arr[++count] != NULL);
+
+    return count;
+}
+
+#define INDEX_ID_PARAM_OFFSET (10)
+#define MIME_PARAM_OFFSET (INDEX_ID_PARAM_OFFSET + 1000)
+
+char *build_where_clause(const char *path_where, const char *size_where, const char *date_where,
+                         const char *index_id_where, const char *mime_where, const char *query_where,
+                         const char *after_where, const char *tags_where) {
+    char *where = calloc(
+            strlen(index_id_where)
+            + (query_where ? strlen(query_where) + sizeof(" AND ") : 0)
+            + (path_where ? strlen(path_where) + sizeof(" AND ") : 0)
+            + (size_where ? strlen(size_where) + sizeof(" AND ") : 0)
+            + (date_where ? strlen(date_where) + sizeof(" AND ") : 0)
+            + (after_where ? strlen(after_where) + sizeof(" AND ") : 0)
+            + (tags_where ? strlen(tags_where) + sizeof(" AND ") : 0)
+            + (mime_where ? strlen(mime_where) + sizeof(" AND ") : 0) + 1,
+            sizeof(char)
+    );
+
+    strcat(where, index_id_where);
+    if (query_where) {
+        strcat(where, " AND ");
+        strcat(where, query_where);
+    }
+    if (path_where) {
+        strcat(where, " AND ");
+        strcat(where, path_where);
+    }
+    if (size_where) {
+        strcat(where, " AND ");
+        strcat(where, size_where);
+    }
+    if (date_where) {
+        strcat(where, " AND ");
+        strcat(where, date_where);
+    }
+    if (mime_where) {
+        strcat(where, " AND ");
+        strcat(where, mime_where);
+    }
+    if (after_where) {
+        strcat(where, " AND ");
+        strcat(where, after_where);
+    }
+    if (tags_where) {
+        strcat(where, " AND ");
+        strcat(where, tags_where);
+    }
+    return where;
+}
+
+char *index_ids_where_clause(char **index_ids) {
+    int param_count = array_length(index_ids);
+
+    char *clause = malloc(13 + 2 + 6 * param_count);
+
+    strcpy(clause, "index_id IN (");
+    for (int i = 0; i < param_count; i++) {
+        char param[10];
+        snprintf(param, sizeof(param), "?%d%s",
+                 INDEX_ID_PARAM_OFFSET + i, i == param_count - 1 ? "" : ",");
+        strcat(clause, param);
+    }
+    strcat(clause, ")");
+
+    return clause;
+}
+
+char *mime_types_where_clause(char **mime_types) {
+    int param_count = array_length(mime_types);
+
+    if (param_count == 0) {
+        return NULL;
+    }
+
+    char *clause = malloc(9 + 2 + 6 * param_count);
+
+    strcpy(clause, "mime IN (");
+    for (int i = 0; i < param_count; i++) {
+        char param[10];
+        snprintf(param, sizeof(param), "?%d%s",
+                 MIME_PARAM_OFFSET + i, i == param_count - 1 ? "" : ",");
+        strcat(clause, param);
+    }
+    strcat(clause, ")");
+
+    return clause;
+}
+
+const char *path_where_clause(const char *path) {
+    if (path == NULL || strlen(path) == 0) {
+        return NULL;
+    }
+
+    return "(path = @path or path GLOB @path_glob)";
+}
+
+const char *get_sort_var(fts_sort_t sort) {
+
+    switch (sort) {
+        case FTS_SORT_SCORE:
+            // Round to 14 decimal places to avoid precision problems when converting to JSON...
+            return "round(rank, 14)";
+        case FTS_SORT_SIZE:
+            return "size";
+        case FTS_SORT_MTIME:
+            return "mtime";
+        case FTS_SORT_RANDOM:
+            return "random_seeded(doc.ROWID + ?5)";
+        case FTS_SORT_NAME:
+            return "doc.name";
+        case FTS_SORT_ID:
+            return "doc.id";
+        default:
+            return NULL;
+    }
+}
+
+const char *match_where(const char *query) {
+    if (query == NULL || strlen(query) == 0) {
+        return NULL;
+    } else {
+        return "search MATCH ?1";
+    }
+}
+
+char *tags_where_clause(char **tags) {
+    if (tags == NULL) {
+        return NULL;
+    }
+
+    return "EXISTS (SELECT 1 FROM tag WHERE id=doc.id AND tag_matches(tag))";
+}
+
+database_summary_stats_t database_fts_get_date_range(database_t *db) {
+
+    int ret = sqlite3_step(db->fts_date_range);
+    CRASH_IF_STMT_FAIL(ret);
+
+    if (ret == SQLITE_DONE) {
+        return (database_summary_stats_t) {0, 0};
+    }
+
+    database_summary_stats_t stats;
+    stats.date_min = (double) sqlite3_column_int64(db->fts_date_range, 0);
+    stats.date_max = (double) sqlite3_column_int64(db->fts_date_range, 1);
+
+    sqlite3_reset(db->fts_date_range);
+
+    return stats;
+}
+
+char *get_after_where(char **after, fts_sort_t sort) {
+    if (after == NULL) {
+        return NULL;
+    }
+
+    return "(sort_var, doc.ROWID) > (?3, ?4)";
+}
+
+cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
+                           long size_max, long date_min, long date_max, int page_size,
+                           char **index_ids, char **mime_types, char **tags, int sort_asc,
+                           fts_sort_t sort, int seed, char **after, int fetch_aggregations,
+                           int highlight, int highlight_context_size) {
+
+    char path_glob[PATH_MAX * 2];
+    snprintf(path_glob, sizeof(path_glob), "%s/*", path);
+    const char *path_where = path_where_clause(path);
+    const char *size_where = size_where_clause(size_min, size_max);
+    const char *date_where = date_where_clause(date_min, date_max);
+    const char *index_id_where = index_ids_where_clause(index_ids);
+    const char *mime_where = mime_types_where_clause(mime_types);
+    const char *query_where = match_where(query);
+    const char *after_where = get_after_where(after, sort);
+    const char *tags_where = tags_where_clause(tags);
+
+    if (!query_where && sort == FTS_SORT_SCORE) {
+        // If query is NULL, then sort by id instead
+        sort = FTS_SORT_ID;
+    }
+
+    char *agg_where;
+    char *where = build_where_clause(path_where, size_where, date_where, index_id_where, mime_where, query_where,
+                                     after_where, tags_where);
+    if (fetch_aggregations) {
+        agg_where = build_where_clause(path_where, size_where, date_where, index_id_where, mime_where, query_where,
+                                       NULL, tags_where);
+    }
+
+    const char *json_object_sql;
+    if (highlight && query_where != NULL) {
+        json_object_sql = "json_remove(json_set(doc.json_data,"
+                          "'$.index', doc.index_id,"
+                          "'$._highlight.name', snippet(search, 0, '<mark>', '</mark>', '', ?6),"
+                          "'$._highlight.content', snippet(search, 1, '<mark>', '</mark>', '', ?6)),"
+                          "'$.content')";
+    } else {
+        json_object_sql = "json_remove(json_set(doc.json_data,"
+                          "'$.index', doc.index_id),"
+                          "'$.content')";
+    }
+
+    char *sql;
+    char *agg_sql;
+
+    if (query_where) {
+        asprintf(
+                &sql,
+                "SELECT"
+                " %s, %s as sort_var, doc.ROWID"
+                " FROM search"
+                " INNER JOIN document_index doc on doc.ROWID = search.ROWID"
+                " WHERE %s"
+                " ORDER BY sort_var%s, doc.ROWID"
+                " LIMIT ?2",
+                json_object_sql, get_sort_var(sort),
+                where,
+                sort_asc ? "" : "DESC");
+
+        if (fetch_aggregations) {
+            asprintf(&agg_sql,
+                     "SELECT count(*), sum(size)"
+                     " FROM search"
+                     "  INNER JOIN document_index doc on doc.ROWID = search.ROWID"
+                     " WHERE search MATCH ?1"
+                     " AND %s", agg_where);
+        }
+    } else {
+        asprintf(
+                &sql,
+                "SELECT"
+                " %s, %s as sort_var, doc.ROWID"
+                " FROM document_index doc"
+                " WHERE %s"
+                " ORDER BY sort_var%s,doc.ROWID"
+                " LIMIT ?2",
+                json_object_sql, get_sort_var(sort),
+                where,
+                sort_asc ? "" : " DESC");
+
+        if (fetch_aggregations) {
+            asprintf(&agg_sql,
+                     "SELECT count(*), sum(size)"
+                     " FROM document_index doc"
+                     " WHERE %s", agg_where);
+        }
+    }
+
+    sqlite3_stmt *stmt;
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(db->db, sql, -1, &stmt, NULL));
+
+    if (query_where) {
+        sqlite3_bind_text(stmt, 1, query, -1, SQLITE_STATIC);
+    }
+    sqlite3_bind_int(stmt, 2, page_size);
+
+    if (index_ids) {
+        array_foreach(index_ids) {
+            sqlite3_bind_text(stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
+        }
+    }
+    if (mime_types) {
+        array_foreach(mime_types) {
+            sqlite3_bind_text(stmt, MIME_PARAM_OFFSET + i, mime_types[i], -1, SQLITE_STATIC);
+        }
+    }
+    if (tags) {
+        db->tag_array = tags;
+    }
+
+    if (size_min > 0) {
+        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@size_min"), size_min);
+    }
+    if (size_max > 0) {
+        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@size_max"), size_max);
+    }
+    if (date_min > 0) {
+        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@date_min"), date_min);
+    }
+    if (date_max > 0) {
+        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@date_max"), date_max);
+    }
+    if (path_where) {
+        sqlite3_bind_text(stmt, sqlite3_bind_parameter_index(stmt, "@path"), path, -1, SQLITE_STATIC);
+        sqlite3_bind_text(stmt, sqlite3_bind_parameter_index(stmt, "@path_glob"), path_glob, -1, SQLITE_STATIC);
+    }
+    if (after_where) {
+        if (sort == FTS_SORT_NAME || sort == FTS_SORT_ID) {
+            sqlite3_bind_text(stmt, 3, after[0], -1, SQLITE_STATIC);
+        } else if (sort == FTS_SORT_SCORE) {
+            sqlite3_bind_double(stmt, 3, strtod(after[0], NULL));
+        } else {
+            sqlite3_bind_int64(stmt, 3, strtol(after[0], NULL, 10));
+        }
+        sqlite3_bind_int64(stmt, 4, strtol(after[1], NULL, 10));
+    }
+    if (sort == FTS_SORT_RANDOM) {
+        sqlite3_bind_int(stmt, 5, seed);
+    }
+    if (highlight) {
+        sqlite3_bind_int(stmt, 6, highlight_context_size);
+    }
+
+    cJSON *json = cJSON_CreateObject();
+    cJSON *hits_hits = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(stmt);
+        if (ret != SQLITE_DONE && ret != SQLITE_ROW) {
+            break;
+        }
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        const char *json_str = (const char *) sqlite3_column_text(stmt, 0);
+        cJSON *row = cJSON_CreateObject();
+        cJSON *source = cJSON_Parse(json_str);
+        if (highlight) {
+            cJSON *hl = cJSON_DetachItemFromObject(source, "_highlight");
+            cJSON_AddItemToObject(row, "highlight", hl);
+        }
+        cJSON *id = cJSON_DetachItemFromObject(source, "_id");
+        cJSON_AddItemToObject(row, "_id", id);
+        cJSON_AddItemToObject(row, "_source", source);
+
+        cJSON *sort_info = cJSON_AddArrayToObject(row, "sort");
+        cJSON_AddItemToArray(
+                sort_info,
+                cJSON_CreateString((char *) sqlite3_column_text(stmt, 1))
+        );
+        cJSON_AddItemToArray(
+                sort_info,
+                cJSON_CreateString((char *) sqlite3_column_text(stmt, 2))
+        );
+
+        cJSON_AddItemToArray(hits_hits, row);
+    } while (TRUE);
+
+    sqlite3_finalize(stmt);
+
+    cJSON *hits = cJSON_AddObjectToObject(json, "hits");
+    cJSON_AddItemToObject(hits, "hits", hits_hits);
+
+    // Aggregations
+    if (fetch_aggregations) {
+
+        sqlite3_stmt *agg_stmt;
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(db->db, agg_sql, -1, &agg_stmt, NULL));
+
+        if (index_ids) {
+            array_foreach(index_ids) {
+                sqlite3_bind_text(agg_stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
+            }
+        }
+        if (mime_types) {
+            array_foreach(mime_types) {
+                sqlite3_bind_text(agg_stmt, MIME_PARAM_OFFSET + i, mime_types[i], -1, SQLITE_STATIC);
+            }
+        }
+
+        if (query_where) {
+            sqlite3_bind_text(agg_stmt, 1, query, -1, SQLITE_STATIC);
+        }
+        if (size_min > 0) {
+            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@size_min"), size_min);
+        }
+        if (size_max > 0) {
+            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@size_max"), size_max);
+        }
+        if (date_min > 0) {
+            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@date_min"), date_min);
+        }
+        if (date_max > 0) {
+            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@date_max"), date_max);
+        }
+        if (path_where) {
+            sqlite3_bind_text(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@path"), path, -1, SQLITE_STATIC);
+            sqlite3_bind_text(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@path_glob"), path_glob, -1,
+                              SQLITE_STATIC);
+        }
+
+        int agg_ret = sqlite3_step(agg_stmt);
+
+        if (agg_ret == SQLITE_ROW) {
+            cJSON *aggregations = cJSON_AddObjectToObject(json, "aggregations");
+            cJSON *total_count = cJSON_AddObjectToObject(aggregations, "total_count");
+            cJSON_AddNumberToObject(total_count, "value", sqlite3_column_double(agg_stmt, 0));
+            cJSON *total_size = cJSON_AddObjectToObject(aggregations, "total_size");
+            cJSON_AddNumberToObject(total_size, "value", sqlite3_column_double(agg_stmt, 1));
+        } else {
+            cJSON *aggregations = cJSON_AddObjectToObject(json, "aggregations");
+            cJSON *total_count = cJSON_AddObjectToObject(aggregations, "total_count");
+            cJSON_AddNumberToObject(total_count, "value", 0);
+            cJSON *total_size = cJSON_AddObjectToObject(aggregations, "total_size");
+            cJSON_AddNumberToObject(total_size, "value", 0);
+        }
+        sqlite3_finalize(agg_stmt);
+    }
+
+    // Cleanup
+    if (index_id_where) {
+        free(index_id_where);
+    }
+    if (mime_where) {
+        free(mime_where);
+    }
+    free(where);
+    free(sql);
+    if (fetch_aggregations) {
+        free(agg_where);
+        free(agg_sql);
+    }
+
+    return json;
+}
+
+database_summary_stats_t database_fts_sync_tags(database_t *db) {
+
+    LOG_INFO("database_fts.c", "Syncing tags.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM fts.tag WHERE"
+            " (id, tag) NOT IN (SELECT id, tag FROM tag)",
+            NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "INSERT INTO fts.tag (id, tag) "
+            " SELECT id, tag FROM tag "
+            " WHERE (id, tag) NOT IN (SELECT * FROM fts.tag)",
+            NULL, NULL, NULL));
+}
+
+cJSON *database_fts_get_document(database_t *db, char *doc_id) {
+    sqlite3_bind_text(db->fts_get_document, 1, doc_id, -1, NULL);
+
+    int ret = sqlite3_step(db->fts_get_document);
+    cJSON *json = NULL;
+
+    if (ret == SQLITE_ROW) {
+        const char *json_data = (const char *) sqlite3_column_text(db->fts_get_document, 0);
+        json = cJSON_Parse(json_data);
+    } else {
+        CRASH_IF_STMT_FAIL(ret);
+    }
+
+    sqlite3_reset(db->fts_get_document);
+
+    return json;
+}
+
+cJSON *database_fts_suggest_tag(database_t *db, char *prefix) {
+    sqlite3_bind_text(db->fts_suggest_tag, 1, prefix, -1, NULL);
+
+    cJSON *json = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(db->fts_suggest_tag);
+        CRASH_IF_STMT_FAIL(ret);
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON_AddItemToArray(
+                json,
+                cJSON_CreateString((const char *) sqlite3_column_text(db->fts_suggest_tag, 0))
+        );
+
+    } while (TRUE);
+
+    sqlite3_reset(db->fts_suggest_tag);
+
+    return json;
+}
+
+
+cJSON *database_fts_get_tags(database_t *db) {
+    cJSON *json = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(db->fts_get_tags);
+        CRASH_IF_STMT_FAIL(ret);
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON *row = cJSON_CreateObject();
+
+        cJSON_AddStringToObject(row, "tag", (const char *) sqlite3_column_text(db->fts_get_tags, 0));
+        cJSON_AddNumberToObject(row, "count", sqlite3_column_int(db->fts_get_tags, 1));
+
+        cJSON_AddItemToArray(json, row);
+    } while (TRUE);
+
+    sqlite3_reset(db->fts_get_tags);
+
+    return json;
+}