Bug fixes

Bump socket.io-parser from 4.2.2 to 4.2.3 in /sist2-admin/frontend

.devcontainer/Dockerfile

@@ -0,0 +1,9 @@
+FROM simon987/sist2-build
+
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
+RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+

.devcontainer/devcontainer.json

@@ -0,0 +1,16 @@
+{
+  "name": "sist2-dev",
+  "dockerComposeFile": [
+    "docker-compose.yml"
+  ],
+  "service": "sist2-dev",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-vscode.cpptools-extension-pack"
+      ]
+    }
+  },
+  "remoteUser": "root",
+  "workspaceFolder": "/app/"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,8 @@
+version: "3"
+
+services:
+  sist2-dev:
+    build: .
+    command: sleep infinity
+    volumes:
+      - ../:/app

.dockerignore

@@ -37,4 +37,5 @@ state.db
 build/
 __pycache__/
 sist2-vue/dist
-sist2-admin/frontend/dist
+sist2-admin/frontend/dist
+*.fts

.gitignore

@@ -43,4 +43,6 @@ src/magic_generated.c
 src/index/static_generated.c
 *.sist2
 *-shm
-*-journal
+*-journal
+.vscode
+*.fts

CMakeLists.txt

@@ -31,7 +31,8 @@ add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
-add_executable(sist2
+add_executable(
+        sist2
         # argparse
         third-party/argparse/argparse.h third-party/argparse/argparse.c
 
@@ -58,7 +59,11 @@ add_executable(sist2
 
         src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp
 
-        src/database/database_stats.c src/database/database_schema.c)
+        src/database/database_stats.c
+        src/database/database_schema.c
+        src/database/database_fts.c
+        src/web/web_fts.c
+)
 set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
@@ -126,6 +131,7 @@ else ()
             PRIVATE
 
             -Ofast
+#            -g
             -fno-stack-protector
             -fomit-frame-pointer
             -w

Dockerfile

@@ -36,16 +36,17 @@ RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 li
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
-    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
-    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
-    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
 COPY --from=build /build/build/sist2 /root/sist2

Dockerfile.arm64

@@ -20,16 +20,17 @@ RUN apt update && apt install -y curl libasan5 libmagic1 tesseract-ocr python3-p
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
-    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
-    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
-    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/pol.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/pol.traineddata &&\
+    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
 COPY --from=build /build/build/sist2 /root/sist2

README.md

@@ -28,7 +28,7 @@ sist2 (Simple incremental search tool)
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
-\*\*\* See [OCR](#ocr)
+\*\*\* See [OCR](#ocr)    
 \*\*\*\* See [Named-Entity Recognition](#NER)
 
 ## Getting Started
@@ -46,7 +46,7 @@ services:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
   sist2-admin:
-    image: simon987/sist2:3.0.3
+    image: simon987/sist2:3.1.0-x64-linux
     restart: unless-stopped
     volumes:
       - ./sist2-admin-data/:/sist2-admin/
@@ -62,12 +62,14 @@ Navigate to http://localhost:8080/ to configure sist2-admin.
 
 ### Using the executable file *(Linux/WSL only)*
 
-1. Have an Elasticsearch (>= 6.8.X, ideally >=7.14.0) instance running
-    1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
-    2. *(or)* Run using docker:
-        ```bash
-        docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
-        ```
+1. Choose search backend (See [comparison](#search-backends)):
+    * **Elasticsearch**: have an Elasticsearch (version >= 6.8.X, ideally >=7.14.0) instance running
+        1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
+        2. *(or)* Run using docker:
+            ```bash
+            docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
+            ```
+    * **SQLite**: No installation required
 
 2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
    Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
@@ -76,7 +78,9 @@ Navigate to http://localhost:8080/ to configure sist2-admin.
 Example usage:
 
 1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
-2. Push index to Elasticsearch: `sist2 index ./documents.sist2`
+2. Prepare search index:
+    * **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
+    * **SQLite**: `sist2 index --search-index ./search.sist2 ./documents.sist2`
 3. Start web interface: `sist2 web ./documents.sist2`
 
 ## Format support
@@ -122,7 +126,7 @@ Download the language data files with your package manager (`apt install tessera
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
 The `simon987/sist2` image comes with common languages
-(hin, jpn, eng, fra, rus, spa, chi_sim, deu) pre-installed.
+(hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
 name must be identical to the `*.traineddata` file installed on your system
@@ -136,9 +140,27 @@ sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
 sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
 ```
 
+### Search backends
+
+sist2 v3.0.7+ supports SQLite search backend. The SQLite search backend has
+fewer features and generally comparable query performance for medium-size
+indices, but it uses much less memory and is easier to set up.
+
+|                                              |                  SQLite                  |                                                             Elasticsearch                                                             |
+|----------------------------------------------|:----------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------:|
+| Requires separate search engine installation |                                          |                                                                   ✓                                                                   |
+| Memory footprint                             |                  ~20MB                   |                                                                >500MB                                                                 |
+| Query syntax                                 | [fts5](https://www.sqlite.org/fts5.html) | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
+| Fuzzy search                                 |                                          |                                                                   ✓                                                                   |
+| Media Types tree real-time updating          |                                          |                                                                   ✓                                                                   |
+| Search in file `path`                        |                                          |                                                                   ✓                                                                   |
+| Manual tagging                               |                    ✓                     |                                                                   ✓                                                                   |
+| User scripts                                 |                                          |                                                                   ✓                                                                   |
+| Media Type breakdown for search results      |                                          |                                                                   ✓                                                                   |
+
 ### NER
 
-sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to 
+sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to
 **Configuration** > **Machine learning options** > **Model repositories**
 to enable it.
 
@@ -151,7 +173,6 @@ See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) fo
 |---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
 | [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
 
-
 <details>
   <summary>Screenshot</summary>
 
@@ -185,7 +206,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl] sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
+    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp]
     ```
 
 4. Build

docs/USAGE.md

@@ -3,6 +3,7 @@
 ```
 Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
+   or: sist2 sqlite-index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
    or: sist2 exec-script [OPTION]... INDEX
 
@@ -16,7 +17,7 @@ Lightning-fast file system indexer and search tool.
 
 Scan options
     -t, --threads=<int>               Number of threads. DEFAULT: 1
-    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2
+    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50
     --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
     --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
     --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
@@ -54,9 +55,13 @@ Index options
     --batch-size=<int>                Index batch size. DEFAULT: 70
     -f, --force-reset                 Reset Elasticsearch mappings and settings.
 
+sqlite-index options
+    --search-index=<str>              Path to search index. Will be created if it does not exist yet.
+
 Web options
     --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
     --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --search-index=<str>              Path to SQLite search index.
     --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
     --bind=<str>                      Listen for connections on this address. DEFAULT: localhost:4090
     --auth=<str>                      Basic auth in user:password format
@@ -83,8 +88,8 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 
 See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
 
-For example, `--thumbnail-size=500`, `--thumbnail-quality=2` for a directory with 8 million images will create a thumbnail database 
-that is about `8000000 * 36kB = 288GB`.
+For example, `--thumbnail-size=500`, `--thumbnail-quality=50` for a directory with 8 million images will create a thumbnail database 
+that is about `8000000 * 11.8kB = 94.4GB`.
 
 ![thumbnail_size](thumbnail_size.png)
 
@@ -111,53 +116,54 @@ sist scan ~/Documents -o ./documents.sist2 --incremental
 sist scan ~/Documents -o ./documents.sist2 --incremental
 ```
 
-### Index examples
+### Index documents to Elasticsearch search backend
 
-**Push to elasticsearch**
 ```bash
-sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index/
-sist2 index ./my_index/
+sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index.sist2
+sist2 index ./my_index.sist2
+```
+
+#### Index documents to SQLite search backend
+```bash
+# The search index will be created if it does not exist already
+sist2 sqlite-index ./index1.sist2 --search-index search.sist2
+sist2 sqlite-index ./index2.sist2 --search-index search.sist2
 ```
 
 **Save index in JSON format**
 ```bash
-sist2 index --print ./my_index/ > my_index.ndjson
+sist2 index --print ./my_index.sist2 > my_index.ndjson
 ```
 
 **Inspect contents of an index**
 ```bash
-sist2 index --print ./my_index/ | jq | less
+sist2 index --print ./my_index.sist2 | jq | less
 ```
 
 ## Web
 
-### Web options
- * `--es-url=<str>` Elasticsearch url.
- * `--es-index` 
-    Elasticsearch index name. DEFAULT=sist2
- * `--bind=<str>` Listen on this address.
- * `--auth=<str>` Basic auth in user:password format
- * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
-    `--auth` argument, but authentication is only applied the `/tag/` endpoint.
- * `--tagline=<str>` When specified, will replace the default tagline in the navbar.
- * `--dev` Serve html & js files from disk (for development, used to modify frontend files without having to recompile)
- * `--lang=<str>` Set the default web UI language (See #180 for a list of supported languages, default
-   is `en`). The user can change the language in the configuration page
- * `--auth0-audience`, `--auth0-domain`, `--auth0-client-id`, `--auth0-public-key-file` See [Authentication with Auth0](auth0.md)
-
 ### Web examples
 
-**Single index**
+**Single index (Elasticsearch backend)**
 ```bash
-sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index
+sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index.sist2
 ```
 
-**Multiple indices**
+**Multiple indices (Elasticsearch backend)**
 ```bash
 # Indices will be displayed in this order in the web interface
-sist2 web index1 index2 index3 index4
+sist2 web index1.sist2 index2.sist2 index3.sist2 index4.sist2
 ```
 
+**SQLite search backend**
+```bash
+sist2 web --search-index search.sist2 index1.sist2
+```
+
+#### Auth0 authentication
+
+See [auth0.md](auth0.md)
+
 ### rewrite_url
 
 When the `rewrite_url` field is not empty, the web module ignores the `root`

schema/mappings.json

@@ -68,7 +68,7 @@
     },
     "mtime": {
       "type": "date",
-      "format": "epoch_millis"
+      "format": "epoch_second"
     },
     "size": {
       "type": "long"

scripts/sqlite_extension.c

@@ -0,0 +1,84 @@
+#include <sqlite3ext.h>
+#include <string.h>
+#include <stdlib.h>
+
+SQLITE_EXTENSION_INIT1
+
+static int sep_rfind(const char *str) {
+    for (int i = (int) strlen(str); i >= 0; i--) {
+        if (str[i] == '/') {
+            return i;
+        }
+    }
+    return -1;
+}
+
+void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    const char *value = (const char *) sqlite3_value_text(argv[0]);
+
+    int stop = sep_rfind(value);
+    if (stop == -1) {
+        sqlite3_result_null(ctx);
+        return;
+    }
+    char parent[4096 * 3];
+    strncpy(parent, value, stop);
+
+    sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
+}
+
+void random_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    char state_buf[32] = {0,};
+    struct random_data buf;
+    int result;
+
+    long seed = sqlite3_value_int64(argv[0]);
+
+    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
+
+    random_r(&buf, &result);
+
+    sqlite3_result_int(ctx, result);
+}
+
+
+int sqlite3_extension_init(
+        sqlite3 *db,
+        char **pzErrMsg,
+        const sqlite3_api_routines *pApi
+) {
+    SQLITE_EXTENSION_INIT2(pApi);
+
+
+    sqlite3_create_function(
+            db,
+            "path_parent",
+            1,
+            SQLITE_UTF8,
+            NULL,
+            path_parent_func,
+            NULL,
+            NULL
+    );
+
+    sqlite3_create_function(
+            db,
+            "random_seeded",
+            1,
+            SQLITE_UTF8,
+            NULL,
+            random_func,
+            NULL,
+            NULL
+    );
+
+    return SQLITE_OK;
+}

scripts/sqlite_extension_compile.sh

@@ -0,0 +1 @@
+gcc -I/mnt/work/vcpkg/installed/x64-linux/include -g -fPIC -shared sqlite_extension.c -o sist2funcs.so

scripts/test_in_docker.sh

@@ -0,0 +1,7 @@
+docker build . -t tmp
+
+docker run --rm -it\
+  -v $(pwd):/host \
+  tmp \
+  scan --ocr-lang eng --ocr-ebooks -t6 --incremental --very-verbose \
+  -o /host/docker.sist2 /host/third-party/libscan/libscan-test-files/test_files/

sist2-admin/frontend/package-lock.json

@@ -9528,9 +9528,9 @@
       }
     },
     "node_modules/socket.io-parser": {
-      "version": "4.2.2",
-      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.2.tgz",
-      "integrity": "sha512-DJtziuKypFkMMHCm2uIshOYC7QaylbtzQwiMYDuCKy3OPkjLzu4B2vAhTlqipRHHzrI0NJeBAizTK7X+6m1jVw==",
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.3.tgz",
+      "integrity": "sha512-JMafRntWVO2DCJimKsRTh/wnqVvO4hrfwOqtO7f+uzwsQMuxO6VwImtYxaQ+ieoyshWOTJyV0fA21lccEXRPpQ==",
       "dependencies": {
         "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.1"
@@ -17996,9 +17996,9 @@
       }
     },
     "socket.io-parser": {
-      "version": "4.2.2",
-      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.2.tgz",
-      "integrity": "sha512-DJtziuKypFkMMHCm2uIshOYC7QaylbtzQwiMYDuCKy3OPkjLzu4B2vAhTlqipRHHzrI0NJeBAizTK7X+6m1jVw==",
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.3.tgz",
+      "integrity": "sha512-JMafRntWVO2DCJimKsRTh/wnqVvO4hrfwOqtO7f+uzwsQMuxO6VwImtYxaQ+ieoyshWOTJyV0fA21lccEXRPpQ==",
       "requires": {
         "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.1"

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -33,9 +33,26 @@ class Sist2AdminApi {
         return axios.get(`${this.baseUrl}/api/job/${name}`);
     }
 
-    /**
-     * @param {string} name
-     */
+    getSearchBackend(name) {
+        return axios.get(`${this.baseUrl}/api/search_backend/${name}`);
+    }
+
+    updateSearchBackend(name, data) {
+        return axios.put(`${this.baseUrl}/api/search_backend/${name}`, data);
+    }
+
+    getSearchBackends() {
+        return axios.get(`${this.baseUrl}/api/search_backend/`);
+    }
+
+    deleteBackend(name) {
+        return axios.delete(`${this.baseUrl}/api/search_backend/${name}`)
+    }
+
+    createBackend(name) {
+        return axios.post(`${this.baseUrl}/api/search_backend/${name}`);
+    }
+
     getFrontend(name) {
         return axios.get(`${this.baseUrl}/api/frontend/${name}`);
     }
@@ -112,6 +129,16 @@ class Sist2AdminApi {
     getSist2AdminInfo() {
         return axios.get(`${this.baseUrl}/api/`);
     }
+
+    getLogsToDelete(jobName, n) {
+        return axios.get(`${this.baseUrl}/api/job/${jobName}/logs_to_delete`, {
+            params: {n: n}
+        });
+    }
+
+    deleteTaskLogs(taskId) {
+        return axios.post(`${this.baseUrl}/api/task/${taskId}/delete_logs`);
+    }
 }
 
 export default new Sist2AdminApi()

sist2-admin/frontend/src/components/IndexOptions.vue

@@ -1,64 +0,0 @@
-<template>
-  <div>
-    <label>{{ $t("indexOptions.threads") }}</label>
-    <b-form-input v-model="options.threads" type="number" min="1" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.esUrl") }}</label>
-    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
-      {{ esTestMessage }}
-    </b-alert>
-    <b-input-group>
-      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
-      <b-input-group-append>
-        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
-      </b-input-group-append>
-    </b-input-group>
-
-    <label>{{ $t("indexOptions.esIndex") }}</label>
-    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
-
-    <br>
-    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!options.es_url.startsWith('https')" @change="update()">
-      {{ $t("webOptions.esInsecure") }}
-    </b-form-checkbox>
-
-    <label>{{ $t("indexOptions.batchSize") }}</label>
-    <b-form-input v-model="options.batch_size" type="number" min="1" @change="update()"></b-form-input>
-
-    <label>{{ $t("indexOptions.script") }}</label>
-    <b-form-textarea v-model="options.script" rows="6" @change="update()"></b-form-textarea>
-  </div>
-
-</template>
-
-<script>
-import sist2AdminApi from "@/Sist2AdminApi";
-
-export default {
-  name: "IndexOptions",
-  props: ["options"],
-  data() {
-    return {
-      showEsTestAlert: false,
-      esTestOk: false,
-      esTestMessage: "",
-    }
-  },
-  methods: {
-    update() {
-      this.$emit("change", this.options);
-    },
-    testEs() {
-      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
-        this.showEsTestAlert = true;
-        this.esTestOk = resp.data.ok;
-        this.esTestMessage = resp.data.message;
-      });
-    }
-  },
-}
-</script>
-
-<style scoped>
-
-</style>

sist2-admin/frontend/src/components/JobListItem.vue

@@ -44,8 +44,7 @@ export default {
         return "";
       }
 
-      const date = Date.parse(dateString);
-      return moment(date).fromNow();
+      return moment.utc(dateString).local().fromNow();
     }
   }
 }

sist2-admin/frontend/src/components/JobOptions.vue

@@ -1,57 +1,94 @@
 <template>
-  <div>
-    <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
-      {{ $t("jobOptions.desktopNotifications") }}
-    </b-form-checkbox>
+    <div>
+        <b-form-checkbox :checked="desktopNotificationsEnabled" @change="updateNotifications($event)">
+            {{ $t("jobOptions.desktopNotifications") }}
+        </b-form-checkbox>
 
-    <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
-      {{ $t("jobOptions.scheduleEnabled") }}
-    </b-form-checkbox>
+        <b-form-checkbox v-model="job.schedule_enabled" @change="update()">
+            {{ $t("jobOptions.scheduleEnabled") }}
+        </b-form-checkbox>
 
-    <label>{{ $t("jobOptions.cron") }}</label>
-    <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression" :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
-  </div>
+        <label>{{ $t("jobOptions.cron") }}</label>
+        <b-form-input class="text-monospace" :state="cronValid" v-model="job.cron_expression"
+                      :disabled="!job.schedule_enabled" @change="update()"></b-form-input>
+
+        <label>{{ $t("jobOptions.keepNLogs") }}</label>
+        <b-input-group>
+            <b-form-input type="number" v-model="job.keep_last_n_logs" @change="update()"></b-form-input>
+            <b-input-group-append>
+                <b-button variant="danger" @click="onDeleteNowClick()">{{ $t("jobOptions.deleteNow") }}</b-button>
+            </b-input-group-append>
+        </b-input-group>
+
+    </div>
 </template>
 
 <script>
 
+import Sist2AdminApi from "@/Sist2AdminApi";
+
 export default {
-  name: "JobOptions",
-  props: ["job"],
-  data() {
-    return {
-      cronValid: undefined
-    }
-  },
-  computed: {
-    desktopNotificationsEnabled() {
-      return this.$store.state.jobDesktopNotificationMap[this.job.name];
-    }
-  },
+    name: "JobOptions",
+    props: ["job"],
+    data() {
+        return {
+            cronValid: undefined,
+            logsToDelete: null
+        }
+    },
+    computed: {
+        desktopNotificationsEnabled() {
+            return this.$store.state.jobDesktopNotificationMap[this.job.name];
+        }
+    },
     mounted() {
-      this.cronValid = this.checkCron(this.job.cron_expression)
+        this.cronValid = this.checkCron(this.job.cron_expression)
     },
     methods: {
-      checkCron(expression) {
-          return /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(expression);
-      },
-    updateNotifications(value) {
-      this.$store.dispatch("setJobDesktopNotification", {
-        job: this.job.name,
-        enabled: value
-      });
-    },
-    update() {
-      if (this.job.schedule_enabled) {
-        this.cronValid = this.checkCron(this.job.cron_expression);
-      } else {
-        this.cronValid = undefined;
-      }
+        checkCron(expression) {
+            return /((((\d+,)+\d+|(\d+([/-])\d+)|\d+|\*) ?){5,7})/.test(expression);
+        },
+        updateNotifications(value) {
+            this.$store.dispatch("setJobDesktopNotification", {
+                job: this.job.name,
+                enabled: value
+            });
+        },
+        update() {
+            if (this.job.schedule_enabled) {
+                this.cronValid = this.checkCron(this.job.cron_expression);
+            } else {
+                this.cronValid = undefined;
+            }
 
-      if (this.cronValid !== false) {
-        this.$emit("change", this.job);
-      }
+            if (this.cronValid !== false) {
+                this.$emit("change", this.job);
+            }
+        },
+        onDeleteNowClick() {
+            Sist2AdminApi.getLogsToDelete(this.job.name, this.job.keep_last_n_logs).then(resp => {
+                const toDelete = resp.data;
+                const message = `Delete ${toDelete.length} log files?`;
+
+                this.$bvModal.msgBoxConfirm(message, {
+                    title: this.$t("confirmation"),
+                    size: "sm",
+                    buttonSize: "sm",
+                    okVariant: "danger",
+                    okTitle: this.$t("delete"),
+                    cancelTitle: this.$t("cancel"),
+                    footerClass: "p-2",
+                    hideHeaderClose: false,
+                    centered: true
+                }).then(value => {
+                    if (value) {
+                        toDelete.forEach(row => {
+                            Sist2AdminApi.deleteTaskLogs(row["id"]);
+                        });
+                    }
+                });
+            })
+        }
     },
-  },
 }
 </script>

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -7,7 +7,7 @@
     <b-form-input type="number" min="1" v-model="options.threads" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailQuality") }}</label>
-    <b-form-input type="number" min="1" max="31" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
+    <b-form-input type="number" min="0" max="100" v-model="options.thumbnail_quality" @change="update()"></b-form-input>
 
     <label>{{ $t("scanOptions.thumbnailCount") }}</label>
     <b-form-input type="number" min="0" max="1000" v-model="options.thumbnail_count" @change="update()"></b-form-input>

sist2-admin/frontend/src/components/SearchBackendListItem.vue

@@ -0,0 +1,24 @@
+<template>
+    <b-list-group-item action :to="`/searchBackend/${backend.name}`">
+
+        <div class="d-flex w-100 justify-content-between">
+            <h5 class="mb-1">
+                {{ backend.name }}
+            </h5>
+
+            <div>
+                <b-badge v-if="backend.backend_type === 'sqlite'" variant="info">SQLite</b-badge>
+                <b-badge v-else variant="info">Elasticsearch</b-badge>
+            </div>
+        </div>
+
+    </b-list-group-item>
+</template>
+
+<script>
+
+export default {
+    name: "SearchBackendListItem",
+    props: ["backend"],
+}
+</script>

sist2-admin/frontend/src/components/SearchBackendSelect.vue

@@ -0,0 +1,37 @@
+<template>
+    <b-progress v-if="loading" striped animated value="100"></b-progress>
+    <div v-else>
+        <label>{{$t("backendOptions.searchBackend")}}</label>
+        <b-select :options="options" :value="value" @change="$emit('change', $event)"></b-select>
+    </div>
+</template>
+
+<script>
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "SearchBackendSelect",
+    props: ["value"],
+    data() {
+        return {
+            loading: true,
+            backends: null,
+        }
+    },
+    computed: {
+        options() {
+            return this.backends.map(backend => backend.name)
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getSearchBackends().then(resp => {
+            this.loading = false;
+            this.backends = resp.data
+        })
+    }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/components/WebOptions.vue

@@ -1,56 +1,35 @@
 <template>
-  <div>
-    <label>{{ $t("webOptions.esUrl") }}</label>
-    <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
-      {{ esTestMessage }}
-    </b-alert>
+    <div>
 
-    <b-input-group>
-      <b-form-input v-model="options.es_url" @change="update()"></b-form-input>
-      <b-input-group-append>
-        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
-      </b-input-group-append>
-    </b-input-group>
+        <label>{{ $t("webOptions.lang") }}</label>
+        <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
 
-    <b-form-checkbox v-model="options.es_insecure_ssl" :disabled="!this.options.es_url.startsWith('https')" @change="update()">
-      {{ $t("webOptions.esInsecure") }}
-    </b-form-checkbox>
+        <label>{{ $t("webOptions.bind") }}</label>
+        <b-form-input v-model="options.bind" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.esIndex") }}</label>
-    <b-form-input v-model="options.es_index" @change="update()"></b-form-input>
+        <label>{{ $t("webOptions.tagline") }}</label>
+        <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
 
-    <label>{{ $t("webOptions.lang") }}</label>
-    <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN']" @change="update()"></b-form-select>
+        <label>{{ $t("webOptions.auth") }}</label>
+        <b-form-input v-model="options.auth" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.bind") }}</label>
-    <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+        <label>{{ $t("webOptions.tagAuth") }}</label>
+        <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.tagline") }}</label>
-    <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+        <br>
+        <h5>Auth0 options</h5>
+        <label>{{ $t("webOptions.auth0Audience") }}</label>
+        <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.auth") }}</label>
-    <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+        <label>{{ $t("webOptions.auth0Domain") }}</label>
+        <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
 
-    <label>{{ $t("webOptions.tagAuth") }}</label>
-    <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+        <label>{{ $t("webOptions.auth0ClientId") }}</label>
+        <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
 
-    <br>
-    <h5>Auth0 options</h5>
-    <label>{{ $t("webOptions.auth0Audience") }}</label>
-    <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.auth0Domain") }}</label>
-    <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.auth0ClientId") }}</label>
-    <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
-
-    <label>{{ $t("webOptions.auth0PublicKey") }}</label>
-    <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
-
-
-
-  </div>
+        <label>{{ $t("webOptions.auth0PublicKey") }}</label>
+        <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
+    </div>
 </template>
 
 <script>
@@ -58,31 +37,24 @@
 import sist2AdminApi from "@/Sist2AdminApi";
 
 export default {
-  name: "WebOptions",
-  props: ["options", "frontendName"],
-  data() {
-    return {
-      showEsTestAlert: false,
-      esTestOk: false,
-      esTestMessage: "",
-    }
-  },
-  methods: {
-    update() {
-      if (!this.options.es_url.startsWith("https")) {
-        this.options.es_insecure_ssl = false;
-      }
-
-      this.$emit("change", this.options);
+    name: "WebOptions",
+    props: ["options", "frontendName"],
+    data() {
+        return {
+            showEsTestAlert: false,
+            esTestOk: false,
+            esTestMessage: "",
+        }
     },
-    testEs() {
-      sist2AdminApi.pingEs(this.options.es_url, this.options.es_insecure_ssl).then((resp) => {
-        this.showEsTestAlert = true;
-        this.esTestOk = resp.data.ok;
-        this.esTestMessage = resp.data.message;
-      });
+    methods: {
+        update() {
+            if (!this.options.es_url.startsWith("https")) {
+                this.options.es_insecure_ssl = false;
+            }
+
+            this.$emit("change", this.options);
+        },
     }
-  }
 }
 </script>
 

sist2-admin/frontend/src/i18n/messages.js

@@ -5,10 +5,13 @@ export default {
         go: "Go",
         online: "online",
         offline: "offline",
+        view: "View",
         delete: "Delete",
         runNow: "Index now",
         create: "Create",
+        cancel: "Cancel",
         test: "Test",
+        confirmation: "Confirmation",
 
         jobTitle: "job configuration",
         tasks: "Tasks",
@@ -45,12 +48,13 @@ export default {
         extraQueryArgs: "Extra query arguments when launching from sist2-admin",
         customUrl: "Custom URL when launching from sist2-admin",
 
+        searchBackends: "Search backends",
+        searchBackendTitle: "search backend configuration",
+        newBackendName: "New search backend name",
+
         selectJobs: "Select jobs",
         webOptions: {
             title: "Web options",
-            esUrl: "Elasticsearch URL",
-            esIndex: "Elasticsearch index name",
-            esInsecure: "Do not verify SSL connections to Elasticsearch.",
             lang: "UI Language",
             bind: "Listen address",
             tagline: "Tagline in navbar",
@@ -61,12 +65,24 @@ export default {
             auth0ClientId: "Auth0 client ID",
             auth0PublicKey: "Auth0 public key",
         },
+        backendOptions: {
+            title: "Search backend options",
+            searchBackend: "Search backend",
+            type: "Search backend type",
+            esUrl: "Elasticsearch URL",
+            esIndex: "Elasticsearch index name",
+            esInsecure: "Do not verify SSL connections to Elasticsearch.",
+            threads: "Number of threads",
+            batchSize: "Index batch size",
+            script: "User script",
+            searchIndex: "Search index file location"
+        },
         scanOptions: {
             title: "Scanning options",
             path: "Path",
             threads: "Number of threads",
             memThrottle: "Total memory threshold in MiB for scan throttling",
-            thumbnailQuality: "Thumbnail quality, on a scale of 2 to 32, 2 being the best",
+            thumbnailQuality: "Thumbnail quality, on a scale of 0 to 100, 100 being the best",
             thumbnailCount: "Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails.",
             thumbnailSize: "Thumbnail size, in pixels",
             contentSize: "Number of bytes to be extracted from text documents. Set to 0 to disable",
@@ -87,20 +103,14 @@ export default {
             treemapThreshold: "Relative size threshold for treemap",
             optimizeIndex: "Defragment index file after scan to reduce its file size."
         },
-        indexOptions: {
-            title: "Indexing options",
-            threads: "Number of threads",
-            esUrl: "Elasticsearch URL",
-            esIndex: "Elasticsearch index name",
-            esInsecure: "Do not verify SSL connections to Elasticsearch.",
-            batchSize: "Index batch size",
-            script: "User script"
-        },
         jobOptions: {
             title: "Job options",
             cron: "Job schedule",
+            keepNLogs: "Keep last N log files. Set to -1 to keep all logs.",
+            deleteNow: "Delete now",
             scheduleEnabled: "Enable scheduled re-scan",
             noJobAvailable: "No jobs available.",
+            noBackendError: "You must select a search backend to run this job",
             desktopNotifications: "Desktop notifications"
         },
         frontendOptions: {

sist2-admin/frontend/src/router/index.js

@@ -5,6 +5,7 @@ import Job from "@/views/Job";
 import Tasks from "@/views/Tasks";
 import Frontend from "@/views/Frontend";
 import Tail from "@/views/Tail";
+import SearchBackend from "@/views/SearchBackend.vue";
 
 Vue.use(VueRouter);
 
@@ -29,6 +30,11 @@ const routes = [
     name: "Frontend",
     component: Frontend
   },
+  {
+    path: "/searchBackend/:name",
+    name: "SearchBackend",
+    component: SearchBackend
+  },
   {
     path: "/log/:taskId",
     name: "Tail",

sist2-admin/frontend/src/views/Frontend.vue

@@ -1,60 +1,70 @@
 <template>
-  <b-card>
-    <b-card-title>
-      {{ name }}
-      <small style="vertical-align: top">
-        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
-        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
-      </small>
-    </b-card-title>
+    <b-card>
+        <b-card-title>
+            {{ name }}
+            <small style="vertical-align: top">
+                <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+                <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+            </small>
+        </b-card-title>
 
-    <div class="mb-3" v-if="!loading">
-      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
-          $t("start")
-        }}
-      </b-button>
-      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
-          $t("stop")
-        }}
-      </b-button>
-      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
-        {{ $t("go") }}
-      </b-button>
-      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
-    </div>
+        <div class="mb-3" v-if="!loading">
+            <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+                $t("start")
+                }}
+            </b-button>
+            <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+                $t("stop")
+                }}
+            </b-button>
+            <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+                {{ $t("go") }}
+            </b-button>
+            <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+        </div>
 
 
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <b-card-body v-else>
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-card-body v-else>
 
-      <h4>{{ $t("frontendOptions.title") }}</h4>
-      <b-card>
-        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
-          {{ $t("autoStart") }}
-        </b-form-checkbox>
+            <h4>{{ $t("frontendOptions.title") }}</h4>
+            <b-card>
+                <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+                    {{ $t("autoStart") }}
+                </b-form-checkbox>
 
-        <label>{{ $t("extraQueryArgs") }}</label>
-        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+                <label>{{ $t("extraQueryArgs") }}</label>
+                <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
 
-        <label>{{ $t("customUrl") }}</label>
-        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+                <label>{{ $t("customUrl") }}</label>
+                <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
 
-        <br/>
+                <br/>
 
-        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+                <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
 
-        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
-      </b-card>
+                <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+            </b-card>
 
-      <br/>
+            <br/>
 
-      <h4>{{ $t("webOptions.title") }}</h4>
-      <b-card>
-        <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name" @change="update()"></WebOptions>
-      </b-card>
-    </b-card-body>
+            <h4>{{ $t("webOptions.title") }}</h4>
+            <b-card>
+                <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
+                            @change="update()"></WebOptions>
+            </b-card>
 
-  </b-card>
+            <br>
+
+            <h4>{{ $t("backendOptions.title") }}</h4>
+            <b-card>
+                <SearchBackendSelect :value="frontend.web_options.search_backend"
+                                     @change="onBackendSelect($event)"></SearchBackendSelect>
+            </b-card>
+        </b-card-body>
+
+
+    </b-card>
 </template>
 
 <script>
@@ -62,68 +72,73 @@
 import Sist2AdminApi from "@/Sist2AdminApi";
 import JobCheckboxGroup from "@/components/JobCheckboxGroup";
 import WebOptions from "@/components/WebOptions";
+import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
-  name: 'Frontend',
-  components: {JobCheckboxGroup, WebOptions},
-  data() {
-    return {
-      loading: true,
-      frontend: null,
-    }
-  },
-  computed: {
-    valid() {
-      return !this.loading && this.frontend.jobs.length > 0;
+    name: 'Frontend',
+    components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
+    data() {
+        return {
+            loading: true,
+            frontend: null,
+        }
     },
-    frontendUrl() {
-      if (this.frontend.custom_url) {
-        return this.frontend.custom_url + this.args;
-      }
+    computed: {
+        valid() {
+            return !this.loading && this.frontend.jobs.length > 0;
+        },
+        frontendUrl() {
+            if (this.frontend.custom_url) {
+                return this.frontend.custom_url + this.args;
+            }
 
-      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
-        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
-      }
+            if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+                return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+            }
 
-      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+            return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+        },
+        name() {
+            return this.$route.params.name;
+        },
+        port() {
+            return this.frontend.web_options.bind.split(":")[1]
+        },
+        args() {
+            const args = this.frontend.extra_query_args;
+            if (args !== "") {
+                return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+            }
+            return "";
+        }
     },
-    name() {
-      return this.$route.params.name;
+    mounted() {
+        Sist2AdminApi.getFrontend(this.name).then(resp => {
+            this.frontend = resp.data;
+            this.loading = false;
+        });
     },
-    port() {
-      return this.frontend.web_options.bind.split(":")[1]
-    },
-    args() {
-      const args = this.frontend.extra_query_args;
-      if (args !== "") {
-        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
-      }
-      return "";
+    methods: {
+        start() {
+            this.frontend.running = true;
+            Sist2AdminApi.startFrontend(this.name)
+        },
+        stop() {
+            this.frontend.running = false;
+            Sist2AdminApi.stopFrontend(this.name)
+        },
+        deleteFrontend() {
+            Sist2AdminApi.deleteFrontend(this.name).then(() => {
+                this.$router.push("/");
+            });
+        },
+        update() {
+            Sist2AdminApi.updateFrontend(this.name, this.frontend);
+        },
+        onBackendSelect(backend) {
+            this.frontend.web_options.search_backend = backend;
+            this.update();
+        }
     }
-  },
-  mounted() {
-    Sist2AdminApi.getFrontend(this.name).then(resp => {
-      this.frontend = resp.data;
-      this.loading = false;
-    });
-  },
-  methods: {
-    start() {
-      this.frontend.running = true;
-      Sist2AdminApi.startFrontend(this.name)
-    },
-    stop() {
-      this.frontend.running = false;
-      Sist2AdminApi.stopFrontend(this.name)
-    },
-    deleteFrontend() {
-      Sist2AdminApi.deleteFrontend(this.name).then(() => {
-        this.$router.push("/frontends");
-      });
-    },
-    update() {
-      Sist2AdminApi.updateFrontend(this.name, this.frontend);
-    },
-  }
 }
 </script>

sist2-admin/frontend/src/views/Home.vue

@@ -1,60 +1,89 @@
 <template>
-  <div>
-    <b-card>
-      <b-card-title>{{ $t("jobs") }}</b-card-title>
-      <b-row>
-        <b-col>
-          <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
-          <b-popover
-              :show.sync="showHelp"
-              target="new-job"
-              placement="top"
-              triggers="manual"
-              variant="primary"
-              :content="$t('newJobHelp')"
-          ></b-popover>
-        </b-col>
-        <b-col>
-          <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">{{ $t("create") }}
-          </b-button>
-        </b-col>
-      </b-row>
+    <div>
+        <b-card>
+            <b-card-title>{{ $t("jobs") }}</b-card-title>
+            <b-row>
+                <b-col>
+                    <b-input id="new-job" v-model="newJobName" :placeholder="$t('newJobName')"></b-input>
+                    <b-popover
+                            :show.sync="showHelp"
+                            target="new-job"
+                            placement="top"
+                            triggers="manual"
+                            variant="primary"
+                            :content="$t('newJobHelp')"
+                    ></b-popover>
+                </b-col>
+                <b-col>
+                    <b-button variant="primary" @click="createJob()" :disabled="!jobNameValid(newJobName)">
+                        {{ $t("create") }}
+                    </b-button>
+                </b-col>
+            </b-row>
 
-      <hr/>
+            <hr/>
 
-      <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
-      <b-list-group v-else>
-        <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
-      </b-list-group>
-    </b-card>
+            <b-progress v-if="jobsLoading" striped animated value="100"></b-progress>
+            <b-list-group v-else>
+                <JobListItem v-for="job in jobs" :key="job.name" :job="job"></JobListItem>
+            </b-list-group>
+        </b-card>
 
-    <br/>
+        <br/>
 
-    <b-card>
+        <b-card>
 
-      <b-card-title>{{ $t("frontends") }}</b-card-title>
+            <b-card-title>{{ $t("frontends") }}</b-card-title>
 
-      <b-row>
-        <b-col>
-          <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
-        </b-col>
-        <b-col>
-          <b-button variant="primary" @click="createFrontend()" :disabled="!frontendNameValid(newFrontendName)">
-            {{ $t("create") }}
-          </b-button>
-        </b-col>
-      </b-row>
+            <b-row>
+                <b-col>
+                    <b-input v-model="newFrontendName" :placeholder="$t('newFrontendName')"></b-input>
+                </b-col>
+                <b-col>
+                    <b-button variant="primary" @click="createFrontend()"
+                              :disabled="!frontendNameValid(newFrontendName)">
+                        {{ $t("create") }}
+                    </b-button>
+                </b-col>
+            </b-row>
 
-      <hr/>
+            <hr/>
 
-      <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
-      <b-list-group v-else>
-        <FrontendListItem v-for="frontend in frontends"
-                          :key="frontend.name" :frontend="frontend"></FrontendListItem>
-      </b-list-group>
+            <b-progress v-if="frontendsLoading" striped animated value="100"></b-progress>
+            <b-list-group v-else>
+                <FrontendListItem v-for="frontend in frontends"
+                                  :key="frontend.name" :frontend="frontend"></FrontendListItem>
+            </b-list-group>
 
-    </b-card>
-  </div>
+        </b-card>
+
+        <br/>
+
+        <b-card>
+            <b-card-title>{{ $t("searchBackends") }}</b-card-title>
+
+            <b-row>
+                <b-col>
+                    <b-input v-model="newBackendName" :placeholder="$t('newBackendName')"></b-input>
+                </b-col>
+                <b-col>
+                    <b-button variant="primary" @click="createBackend()"
+                              :disabled="!backendNameValid(newBackendName)">
+                        {{ $t("create") }}
+                    </b-button>
+                </b-col>
+            </b-row>
+
+            <hr/>
+
+            <b-progress v-if="backendsLoading" striped animated value="100"></b-progress>
+            <b-list-group v-else>
+                <SearchBackendListItem v-for="backend in backends"
+                                  :key="backend.name" :backend="backend"></SearchBackendListItem>
+            </b-list-group>
+
+        </b-card>
+    </div>
 </template>
 
 <script>
@@ -62,61 +91,80 @@ import JobListItem from "@/components/JobListItem";
 import {formatBindAddress} from "@/util";
 import Sist2AdminApi from "@/Sist2AdminApi";
 import FrontendListItem from "@/components/FrontendListItem";
+import SearchBackendListItem from "@/components/SearchBackendListItem.vue";
 
 export default {
-  name: "Jobs",
-  components: {JobListItem, FrontendListItem},
-  data() {
-    return {
-      jobsLoading: true,
-      newJobName: "",
-      jobs: [],
+    name: "Jobs",
+    components: {SearchBackendListItem, JobListItem, FrontendListItem},
+    data() {
+        return {
+            jobsLoading: true,
+            newJobName: "",
+            jobs: [],
 
-      frontendsLoading: true,
-      frontends: [],
-      formatBindAddress,
-      newFrontendName: "",
+            frontendsLoading: true,
+            frontends: [],
+            formatBindAddress,
+            newFrontendName: "",
 
-      showHelp: false
+            backends: [],
+            backendsLoading: true,
+            newBackendName: "",
+
+            showHelp: false
+        }
+    },
+    mounted() {
+        this.loading = true;
+        this.reload();
+    },
+    methods: {
+        jobNameValid(name) {
+            if (this.jobs.some(job => job.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        frontendNameValid(name) {
+            if (this.frontends.some(frontend => frontend.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        backendNameValid(name) {
+            if (this.backends.some(backend => backend.name === name)) {
+                return false;
+            }
+
+            return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
+        },
+        reload() {
+            Sist2AdminApi.getJobs().then(resp => {
+                this.jobs = resp.data;
+                this.jobsLoading = false;
+
+                this.showHelp = this.jobs.length === 0;
+            });
+            Sist2AdminApi.getFrontends().then(resp => {
+                this.frontends = resp.data;
+                this.frontendsLoading = false;
+            });
+            Sist2AdminApi.getSearchBackends().then(resp => {
+                this.backends = resp.data;
+                this.backendsLoading = false;
+            })
+        },
+        createJob() {
+            Sist2AdminApi.createJob(this.newJobName).then(this.reload);
+        },
+        createFrontend() {
+            Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
+        },
+        createBackend() {
+            Sist2AdminApi.createBackend(this.newBackendName).then(this.reload);
+        }
     }
-  },
-  mounted() {
-    this.loading = true;
-    this.reload();
-  },
-  methods: {
-    jobNameValid(name) {
-      if (this.jobs.some(job => job.name === name)) {
-        return false;
-      }
-
-      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
-    },
-    frontendNameValid(name) {
-      if (this.frontends.some(job => job.name === name)) {
-        return false;
-      }
-
-      return /^[a-zA-Z0-9-_,.; ]+$/.test(name);
-    },
-    reload() {
-      Sist2AdminApi.getJobs().then(resp => {
-        this.jobs = resp.data;
-        this.jobsLoading = false;
-
-        this.showHelp = this.jobs.length === 0;
-      });
-      Sist2AdminApi.getFrontends().then(resp => {
-        this.frontends = resp.data;
-        this.frontendsLoading = false;
-      });
-    },
-    createJob() {
-      Sist2AdminApi.createJob(this.newJobName).then(this.reload);
-    },
-    createFrontend() {
-      Sist2AdminApi.createFrontend(this.newFrontendName).then(this.reload)
-    }
-  }
 }
 </script>

sist2-admin/frontend/src/views/Job.vue

@@ -1,92 +1,112 @@
 <template>
-  <b-card>
-    <b-card-title>
-      [{{ getName() }}]
-      {{ $t("jobTitle") }}
-    </b-card-title>
+    <b-card>
+        <b-card-title>
+            [{{ getName() }}]
+            {{ $t("jobTitle") }}
+        </b-card-title>
 
-    <div class="mb-3">
-      <b-button class="mr-1" variant="primary" @click="runJob()">{{ $t("runNow") }}</b-button>
-      <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
-    </div>
+        <div class="mb-3">
+            <b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
+            <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
+        </div>
 
-    <div v-if="job">
-      {{ $t("status") }}: <code>{{ job.status }}</code>
-    </div>
+        <div v-if="job">
+            {{ $t("status") }}: <code>{{ job.status }}</code>
+        </div>
 
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <b-card-body v-else>
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-card-body v-else>
 
-      <h4>{{ $t("jobOptions.title") }}</h4>
-      <b-card>
-        <JobOptions :job="job" @change="update"></JobOptions>
-      </b-card>
+            <h4>{{ $t("jobOptions.title") }}</h4>
+            <b-card>
+                <JobOptions :job="job" @change="update"></JobOptions>
+            </b-card>
 
-      <br/>
+            <br/>
 
-      <h4>{{ $t("scanOptions.title") }}</h4>
-      <b-card>
-        <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
-      </b-card>
+            <h4>{{ $t("scanOptions.title") }}</h4>
+            <b-card>
+                <ScanOptions :options="job.scan_options" @change="update()"></ScanOptions>
+            </b-card>
 
-      <br/>
+            <br/>
 
-      <h4>{{ $t("indexOptions.title") }}</h4>
-      <b-card>
-        <IndexOptions :options="job.index_options" @change="update()"></IndexOptions>
-      </b-card>
+            <h4>{{ $t("backendOptions.title") }}</h4>
+            <b-card>
+                <b-alert v-if="!valid" variant="warning" show>{{ $t("jobOptions.noBackendError") }}</b-alert>
+                <SearchBackendSelect :value="job.index_options.search_backend"
+                                     @change="onBackendSelect($event)"></SearchBackendSelect>
+            </b-card>
 
-    </b-card-body>
+        </b-card-body>
 
-  </b-card>
+    </b-card>
 </template>
 
 <script>
 import ScanOptions from "@/components/ScanOptions";
 import Sist2AdminApi from "@/Sist2AdminApi";
-import IndexOptions from "@/components/IndexOptions";
 import JobOptions from "@/components/JobOptions";
+import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
-  name: "Job",
-  components: {
-    IndexOptions,
-    ScanOptions,
-    JobOptions
-  },
-  data() {
-    return {
-      loading: true,
-      job: null
+    name: "Job",
+    components: {
+        SearchBackendSelect,
+        ScanOptions,
+        JobOptions
+    },
+    data() {
+        return {
+            loading: true,
+            job: null,
+        }
+    },
+    methods: {
+        getName() {
+            return this.$route.params.name;
+        },
+        update() {
+            Sist2AdminApi.updateJob(this.getName(), this.job);
+        },
+        runJob() {
+            Sist2AdminApi.runJob(this.getName()).then(() => {
+                this.$bvToast.toast(this.$t("runJobConfirmation"), {
+                    title: this.$t("runJobConfirmationTitle"),
+                    variant: "success",
+                    toaster: "b-toaster-bottom-right"
+                });
+            });
+        },
+        deleteJob() {
+            Sist2AdminApi.deleteJob(this.getName())
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete job " +
+                        "because it is referenced by a frontend", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        },
+        onBackendSelect(backend) {
+            this.job.index_options.search_backend = backend;
+            this.update();
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getJob(this.getName()).then(resp => {
+            this.loading = false;
+            this.job = resp.data;
+        })
+    },
+    computed: {
+        valid() {
+            return this.job?.index_options.search_backend != null;
+        }
     }
-  },
-  methods: {
-    getName() {
-      return this.$route.params.name;
-    },
-    update() {
-      Sist2AdminApi.updateJob(this.getName(), this.job);
-    },
-    runJob() {
-      Sist2AdminApi.runJob(this.getName()).then(() => {
-        this.$bvToast.toast(this.$t("runJobConfirmation"), {
-          title: this.$t("runJobConfirmationTitle"),
-          variant: "success",
-          toaster: "b-toaster-bottom-right"
-        });
-      });
-    },
-    deleteJob() {
-      Sist2AdminApi.deleteJob(this.getName()).then(() => {
-        this.$router.push("/");
-      })
-    }
-  },
-  mounted() {
-    Sist2AdminApi.getJob(this.getName()).then(resp => {
-      this.loading = false;
-      this.job = resp.data;
-    })
-  }
 }
 </script>

sist2-admin/frontend/src/views/SearchBackend.vue

@@ -0,0 +1,126 @@
+<template>
+
+    <b-card>
+        <b-card-title>
+            <span class="text-monospace">{{ getName() }}</span>
+            {{ $t("searchBackendTitle") }}
+        </b-card-title>
+
+        <div class="mb-3">
+            <b-button variant="danger" @click="deleteBackend()">{{ $t("delete") }}</b-button>
+        </div>
+
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-card-body v-else>
+
+            <label>{{ $t("backendOptions.type") }}</label>
+            <b-select :options="backendTypeOptions" v-model="backend.backend_type" @change="update()"></b-select>
+
+            <hr/>
+
+            <template v-if="backend.backend_type === 'elasticsearch'">
+                <b-alert :variant="esTestOk ? 'success' : 'danger'" :show="showEsTestAlert" class="mt-1">
+                    {{ esTestMessage }}
+                </b-alert>
+
+                <label>{{ $t("backendOptions.esUrl") }}</label>
+                <b-input-group>
+                    <b-form-input v-model="backend.es_url" @change="update()"></b-form-input>
+                    <b-input-group-append>
+                        <b-button variant="outline-primary" @click="testEs()">{{ $t("test") }}</b-button>
+                    </b-input-group-append>
+                </b-input-group>
+
+                <b-form-checkbox v-model="backend.es_insecure_ssl" :disabled="!this.backend.es_url.startsWith('https')"
+                                 @change="update()">
+                    {{ $t("backendOptions.esInsecure") }}
+                </b-form-checkbox>
+
+                <label>{{ $t("backendOptions.esIndex") }}</label>
+                <b-form-input v-model="backend.es_index" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.threads") }}</label>
+                <b-form-input v-model="backend.threads" type="number" min="1" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.batchSize") }}</label>
+                <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
+
+                <label>{{ $t("backendOptions.script") }}</label>
+                <b-form-textarea v-model="backend.script" rows="6" @change="update()"></b-form-textarea>
+            </template>
+            <template v-else>
+                <label>{{ $t("backendOptions.searchIndex") }}</label>
+                <b-form-input v-model="backend.search_index" disabled></b-form-input>
+            </template>
+        </b-card-body>
+
+    </b-card>
+</template>
+
+<script>
+import sist2AdminApi from "@/Sist2AdminApi";
+import Sist2AdminApi from "@/Sist2AdminApi";
+
+export default {
+    name: "SearchBackend",
+    data() {
+        return {
+            showEsTestAlert: false,
+            esTestOk: false,
+            esTestMessage: "",
+            loading: true,
+            backend: null,
+            backendTypeOptions: [
+                {
+                    text: "Elasticsearch",
+                    value: "elasticsearch"
+                },
+                {
+                    text: "SQLite",
+                    value: "sqlite"
+                }
+            ]
+        }
+    },
+    mounted() {
+        Sist2AdminApi.getSearchBackend(this.getName()).then(resp => {
+            this.backend = resp.data;
+            this.loading = false;
+        });
+    },
+    methods: {
+        getName() {
+            return this.$route.params.name;
+        },
+        testEs() {
+            sist2AdminApi.pingEs(this.backend.es_url, this.backend.es_insecure_ssl)
+                .then((resp) => {
+                    this.showEsTestAlert = true;
+                    this.esTestOk = resp.data.ok;
+                    this.esTestMessage = resp.data.message;
+                });
+        },
+        update() {
+            Sist2AdminApi.updateSearchBackend(this.getName(), this.backend);
+        },
+        deleteBackend() {
+            Sist2AdminApi.deleteBackend(this.getName())
+                .then(() => {
+                    this.$router.push("/");
+                })
+                .catch(err => {
+                    this.$bvToast.toast("Cannot delete search backend " +
+                        "because it is referenced by a job or frontend", {
+                        title: "Error",
+                        variant: "danger",
+                        toaster: "b-toaster-bottom-right"
+                    });
+                })
+        }
+    }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-admin/frontend/src/views/Tasks.vue

@@ -1,38 +1,49 @@
 <template>
-  <div>
+    <div>
 
-    <b-card v-if="tasks.length > 0">
-      <h2>{{ $t("runningTasks") }}</h2>
-      <b-list-group>
-        <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
-      </b-list-group>
-    </b-card>
+        <b-card v-if="tasks.length > 0">
+            <h2>{{ $t("runningTasks") }}</h2>
+            <b-list-group>
+                <TaskListItem v-for="task in tasks" :key="task.id" :task="task"></TaskListItem>
+            </b-list-group>
+        </b-card>
 
-    <b-card class="mt-4">
+        <b-card class="mt-4">
 
-      <b-card-title>{{ $t("taskHistory") }}</b-card-title>
+            <b-card-title>{{ $t("taskHistory") }}</b-card-title>
 
-      <br/>
+            <br/>
 
-      <b-table
-          id="task-history"
-          :items="historyItems"
-          :fields="historyFields"
-          :current-page="historyCurrentPage"
-          :tbody-tr-class="rowClass"
-          :per-page="10"
-      >
-        <template #cell(logs)="data">
-          <router-link :to="`/log/${data.item.logs}`">{{ $t("logs") }}</router-link>
-        </template>
+            <b-table
+                    id="task-history"
+                    :items="historyItems"
+                    :fields="historyFields"
+                    :current-page="historyCurrentPage"
+                    :tbody-tr-class="rowClass"
+                    :per-page="10"
+            >
+                <template #cell(logs)="data">
+                    <template v-if="data.item._row.has_logs">
+                        <b-button variant="link" size="sm" :to="`/log/${data.item.id}`">
+                            {{ $t("view") }}
+                        </b-button>
+                        /
+                        <b-button variant="link" size="sm" @click="deleteLogs(data.item.id)">
+                            {{ $t("delete") }}
+                        </b-button>
+                    </template>
+                </template>
 
-      </b-table>
+                <template #cell(delete)="data">
+                </template>
 
-      <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
-                    :per-page="10"></b-pagination>
+            </b-table>
 
-    </b-card>
-  </div>
+            <b-pagination limit="20" v-model="historyCurrentPage" :total-rows="historyItems.length"
+                          :per-page="10"></b-pagination>
+
+        </b-card>
+    </div>
 </template>
 
 <script>
@@ -45,106 +56,116 @@ const HOUR = 3600;
 const MINUTE = 60;
 
 function humanDuration(sec_num) {
-  sec_num = sec_num / 1000;
-  const days = Math.floor(sec_num / DAY);
-  sec_num -= days * DAY;
-  const hours = Math.floor(sec_num / HOUR);
-  sec_num -= hours * HOUR;
-  const minutes = Math.floor(sec_num / MINUTE);
-  sec_num -= minutes * MINUTE;
-  const seconds = Math.floor(sec_num);
+    sec_num = sec_num / 1000;
+    const days = Math.floor(sec_num / DAY);
+    sec_num -= days * DAY;
+    const hours = Math.floor(sec_num / HOUR);
+    sec_num -= hours * HOUR;
+    const minutes = Math.floor(sec_num / MINUTE);
+    sec_num -= minutes * MINUTE;
+    const seconds = Math.floor(sec_num);
 
-  if (days > 0) {
-    return `${days} days ${hours}h ${minutes}m ${seconds}s`;
-  }
+    if (days > 0) {
+        return `${days} days ${hours}h ${minutes}m ${seconds}s`;
+    }
 
-  if (hours > 0) {
-    return `${hours}h ${minutes}m ${seconds}s`;
-  }
+    if (hours > 0) {
+        return `${hours}h ${minutes}m ${seconds}s`;
+    }
 
-  if (minutes > 0) {
-    return `${minutes}m ${seconds}s`;
-  }
+    if (minutes > 0) {
+        return `${minutes}m ${seconds}s`;
+    }
 
-  if (seconds > 0) {
-    return `${seconds}s`;
-  }
+    if (seconds > 0) {
+        return `${seconds}s`;
+    }
 
-  return "<0s";
+    return "<0s";
 }
 
 export default {
-  name: 'Tasks',
-  components: {TaskListItem},
-  data() {
-    return {
-      loading: true,
-      tasks: [],
-      taskHistory: [],
-      timerId: null,
-      historyFields: [
-        {key: "name", label: this.$t("taskName")},
-        {key: "time", label: this.$t("taskStarted")},
-        {key: "duration", label: this.$t("taskDuration")},
-        {key: "status", label: this.$t("taskStatus")},
-        {key: "logs", label: this.$t("logs")},
-      ],
-      historyCurrentPage: 1,
-      historyItems: []
-    }
-  },
-  props: {
-    msg: String
-  },
-  mounted() {
-    this.loading = true;
-    this.update().then(() => this.loading = false);
+    name: 'Tasks',
+    components: {TaskListItem},
+    data() {
+        return {
+            loading: true,
+            tasks: [],
+            taskHistory: [],
+            timerId: null,
+            historyFields: [
+                {key: "name", label: this.$t("taskName")},
+                {key: "time", label: this.$t("taskStarted")},
+                {key: "duration", label: this.$t("taskDuration")},
+                {key: "status", label: this.$t("taskStatus")},
+                {key: "logs", label: this.$t("logs")},
+            ],
+            historyCurrentPage: 1,
+            historyItems: []
+        }
+    },
+    props: {
+        msg: String
+    },
+    mounted() {
+        this.loading = true;
+        this.update().then(() => this.loading = false);
 
-    this.timerId = window.setInterval(this.update, 1000);
-    this.updateHistory();
-  },
-  destroyed() {
-    if (this.timerId) {
-      window.clearInterval(this.timerId);
-    }
-  },
-  methods: {
-    rowClass(row) {
-      if (row.status === "failed") {
-        return "table-danger";
-      }
-      return null;
+        this.timerId = window.setInterval(this.update, 1000);
+        this.updateHistory();
     },
-    updateHistory() {
-      Sist2AdminApi.getTaskHistory().then(resp => {
-        this.historyItems = resp.data.map(row => ({
-          id: row.id,
-          name: row.name,
-          duration: this.taskDuration(row),
-          time: moment(row.started).format("dd, MMM Do YYYY, HH:mm:ss"),
-          logs: row.id,
-          status: row.return_code === 0 ? "ok" : "failed"
-        }));
-      });
+    destroyed() {
+        if (this.timerId) {
+            window.clearInterval(this.timerId);
+        }
     },
-    update() {
-      return Sist2AdminApi.getTasks().then(resp => {
-        this.tasks = resp.data;
-      })
-    },
-    taskDuration(task) {
-      const start = moment.utc(task.started);
-      const end = moment.utc(task.ended);
+    methods: {
+        rowClass(row) {
+            if (row.status === "failed") {
+                return "table-danger";
+            }
+            return null;
+        },
+        updateHistory() {
+            Sist2AdminApi.getTaskHistory().then(resp => {
+                this.historyItems = resp.data.map(row => ({
+                    id: row.id,
+                    name: row.name,
+                    duration: this.taskDuration(row),
+                    time: moment.utc(row.started).local().format("dd, MMM Do YYYY, HH:mm:ss"),
+                    logs: null,
+                    status: row.return_code === 0 ? "ok" : "failed",
+                    _row: row
+                }));
+            });
+        },
+        update() {
+            return Sist2AdminApi.getTasks().then(resp => {
+                this.tasks = resp.data;
+            })
+        },
+        taskDuration(task) {
+            const start = moment.utc(task.started);
+            const end = moment.utc(task.ended);
 
-      return humanDuration(end.diff(start))
+            return humanDuration(end.diff(start))
+        },
+        deleteLogs(taskId) {
+            Sist2AdminApi.deleteTaskLogs(taskId).then(() => {
+                this.updateHistory();
+            })
+        }
     }
-  }
 }
 </script>
 
 <style scoped>
 #task-history {
-  font-family: monospace;
-  font-size: 12px;
+    font-family: monospace;
+    font-size: 12px;
+}
+
+.btn-link {
+    padding: 0;
 }
 </style>

sist2-admin/frontend/yarn.lock

@@ -4913,9 +4913,9 @@ socket.io-client@^4.5.1:
     socket.io-parser "~4.2.0"
 
 socket.io-parser@~4.2.0:
-  version "4.2.2"
-  resolved "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.2.tgz"
-  integrity sha512-DJtziuKypFkMMHCm2uIshOYC7QaylbtzQwiMYDuCKy3OPkjLzu4B2vAhTlqipRHHzrI0NJeBAizTK7X+6m1jVw==
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.3.tgz#926bcc6658e2ae0883dc9dee69acbdc76e4e3667"
+  integrity sha512-JMafRntWVO2DCJimKsRTh/wnqVvO4hrfwOqtO7f+uzwsQMuxO6VwImtYxaQ+ieoyshWOTJyV0fA21lccEXRPpQ==
   dependencies:
     "@socket.io/component-emitter" "~3.1.0"
     debug "~4.3.1"

sist2-admin/sist2_admin/app.py

@@ -20,8 +20,9 @@ import cron
 from config import LOG_FOLDER, logger, WEBSERVER_PORT, DATA_FOLDER, SIST2_BINARY
 from jobs import Sist2Job, Sist2ScanTask, TaskQueue, Sist2IndexTask, JobStatus
 from notifications import Subscribe, Notifications
-from sist2 import Sist2
-from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION
+from sist2 import Sist2, Sist2SearchBackend
+from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEMA_VERSION, migrate_v3_to_v4, \
+    get_log_files_to_remove, delete_log_file, create_default_search_backends
 from web import Sist2Frontend
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
@@ -80,9 +81,7 @@ async def get_jobs():
 
 @app.put("/api/job/{name:str}")
 async def update_job(name: str, new_job: Sist2Job):
-    # TODO: Check etag
-
-    new_job.last_modified = datetime.now()
+    new_job.last_modified = datetime.utcnow()
     job = db["jobs"][name]
     if not job:
         raise HTTPException(status_code=404)
@@ -134,8 +133,18 @@ async def kill_job(task_id: str):
     return task_queue.kill_task(task_id)
 
 
+@app.post("/api/task/{task_id:str}/delete_logs")
+async def delete_task_logs(task_id: str):
+    if not db["task_done"][task_id]:
+        raise HTTPException(status_code=404)
+
+    delete_log_file(db, task_id)
+
+    return "ok"
+
+
 def _run_job(job: Sist2Job):
-    job.last_modified = datetime.now()
+    job.last_modified = datetime.utcnow()
     if job.status == JobStatus("created"):
         job.status = JobStatus("started")
     db["jobs"][job.name] = job
@@ -158,14 +167,29 @@ async def run_job(name: str):
     return "ok"
 
 
+@app.get("/api/job/{name:str}/logs_to_delete")
+async def task_history(n: int, name: str):
+    return get_log_files_to_remove(db, name, n)
+
+
 @app.delete("/api/job/{name:str}")
 async def delete_job(name: str):
-    job = db["jobs"][name]
-    if job:
-        del db["jobs"][name]
-    else:
+    job: Sist2Job = db["jobs"][name]
+    if not job:
         raise HTTPException(status_code=404)
 
+    if any(name in frontend.jobs for frontend in db["frontends"]):
+        raise HTTPException(status_code=400, detail="in use (frontend)")
+
+    try:
+        os.remove(job.previous_index)
+    except:
+        pass
+
+    del db["jobs"][name]
+
+    return "ok"
+
 
 @app.delete("/api/frontend/{name:str}")
 async def delete_frontend(name: str):
@@ -253,7 +277,16 @@ def check_es_version(es_url: str, insecure: bool):
 def start_frontend_(frontend: Sist2Frontend):
     frontend.web_options.indices = list(map(lambda j: db["jobs"][j].index_path, frontend.jobs))
 
-    pid = sist2.web(frontend.web_options, frontend.name)
+    backend_name = frontend.web_options.search_backend
+    search_backend = db["search_backends"][backend_name]
+    if search_backend is None:
+        logger.error(
+            f"Error while running task: search backend not found: {backend_name}")
+        return -1
+
+    logger.debug(f"Fetched search backend options for {backend_name}")
+
+    pid = sist2.web(frontend.web_options, search_backend, frontend.name)
     RUNNING_FRONTENDS[frontend.name] = pid
 
 
@@ -283,6 +316,62 @@ async def get_frontends():
     return res
 
 
+@app.get("/api/search_backend/")
+async def get_search_backends():
+    return list(db["search_backends"])
+
+
+@app.put("/api/search_backend/{name:str}")
+async def update_search_backend(name: str, backend: Sist2SearchBackend):
+    if not db["search_backends"][name]:
+        raise HTTPException(status_code=404)
+
+    db["search_backends"][name] = backend
+    return "ok"
+
+
+@app.get("/api/search_backend/{name:str}")
+def get_search_backend(name: str):
+    backend = db["search_backends"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    return backend
+
+
+@app.delete("/api/search_backend/{name:str}")
+def delete_search_backend(name: str):
+    backend: Sist2SearchBackend = db["search_backends"][name]
+    if not backend:
+        raise HTTPException(status_code=404)
+
+    if any(frontend.web_options.search_backend == name for frontend in db["frontends"]):
+        raise HTTPException(status_code=400, detail="in use (frontend)")
+
+    if any(job.index_options.search_backend == name for job in db["jobs"]):
+        raise HTTPException(status_code=400, detail="in use (job)")
+
+    del db["search_backends"][name]
+
+    try:
+        os.remove(backend.search_index)
+    except:
+        pass
+
+    return "ok"
+
+
+@app.post("/api/search_backend/{name:str}")
+def create_search_backend(name: str):
+    if db["search_backends"][name] is not None:
+        return HTTPException(status_code=400, detail="already exists")
+
+    backend = Sist2SearchBackend.create_default(name)
+    db["search_backends"][name] = backend
+
+    return backend
+
+
 def tail(filepath: str, n: int):
     with open(filepath) as file:
 
@@ -321,7 +410,6 @@ async def ws_tail_log(websocket: WebSocket):
         async with Subscribe(notifications) as ob:
             async for notification in ob.notifications():
                 await websocket.send_json(notification)
-                print(notification)
 
     except ConnectionClosed:
         return
@@ -352,7 +440,7 @@ async def ws_tail_log(websocket: WebSocket, task_id: str, n: int):
 
 
 def main():
-    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0")
+    uvicorn.run(app, port=WEBSERVER_PORT, host="0.0.0.0", timeout_graceful_shutdown=0)
 
 
 def initialize_db():
@@ -361,6 +449,8 @@ def initialize_db():
     frontend = Sist2Frontend.create_default("default")
     db["frontends"]["default"] = frontend
 
+    create_default_search_backends(db)
+
     logger.info("Initialized database.")
 
 
@@ -381,6 +471,12 @@ if __name__ == '__main__':
     if db["sist2_admin"]["info"]["version"] == "2":
         logger.error("Cannot migrate database from v2 to v3. Delete state.db to proceed.")
         exit(-1)
+    if db["sist2_admin"]["info"]["version"] == "3":
+        logger.info("Migrating to v4 database schema")
+        migrate_v3_to_v4(db)
+
+    if db["sist2_admin"]["info"]["version"] != DB_SCHEMA_VERSION:
+        raise Exception(f"Incompatible database version for {db.dbfile}")
 
     start_frontends()
     cron.initialize(db, _run_job)

sist2-admin/sist2_admin/jobs.py

@@ -16,7 +16,7 @@ from pydantic import BaseModel
 from config import logger, LOG_FOLDER
 from notifications import Notifications
 from sist2 import ScanOptions, IndexOptions, Sist2
-from state import RUNNING_FRONTENDS
+from state import RUNNING_FRONTENDS, get_log_files_to_remove, delete_log_file
 from web import Sist2Frontend
 
 
@@ -35,6 +35,8 @@ class Sist2Job(BaseModel):
     cron_expression: str
     schedule_enabled: bool = False
 
+    keep_last_n_logs: int = -1
+
     previous_index: str = None
     index_path: str = None
     previous_index_path: str = None
@@ -53,15 +55,10 @@ class Sist2Job(BaseModel):
             name=name,
             scan_options=ScanOptions(path="/"),
             index_options=IndexOptions(),
-            last_modified=datetime.now(),
+            last_modified=datetime.utcnow(),
             cron_expression="0 0 * * *"
         )
 
-    # @validator("etag", always=True)
-    # def validate_etag(cls, value, values):
-    #     s = values["name"] + values["scan_options"].json() + values["index_options"].json() + values["cron_expression"]
-    #     return md5(s.encode()).hexdigest()
-
 
 class Sist2TaskProgress:
 
@@ -111,7 +108,7 @@ class Sist2Task:
             self._logger.info(json.dumps(log_json))
 
     def run(self, sist2: Sist2, db: PersistentState):
-        self.started = datetime.now()
+        self.started = datetime.utcnow()
 
         logger.info(f"Started task {self.display_name}")
 
@@ -132,14 +129,14 @@ class Sist2ScanTask(Sist2Task):
             self.pid = pid
 
         return_code = sist2.scan(self.job.scan_options, logs_cb=self.log_callback, set_pid_cb=set_pid)
-        self.ended = datetime.now()
+        self.ended = datetime.utcnow()
 
         if return_code != 0:
             self._logger.error(json.dumps({"sist2-admin": f"Process returned non-zero exit code ({return_code})"}))
             logger.info(f"Task {self.display_name} failed ({return_code})")
         else:
             self.job.index_path = self.job.scan_options.output
-            self.job.last_index_date = datetime.now()
+            self.job.last_index_date = datetime.utcnow()
             self.job.do_full_scan = False
             db["jobs"][self.job.name] = self.job
             self._logger.info(json.dumps({"sist2-admin": f"Save last_index_date={self.job.last_index_date}"}))
@@ -171,8 +168,15 @@ class Sist2IndexTask(Sist2Task):
 
         self.job.index_options.path = self.job.scan_options.output
 
-        return_code = sist2.index(self.job.index_options, logs_cb=self.log_callback)
-        self.ended = datetime.now()
+        search_backend = db["search_backends"][self.job.index_options.search_backend]
+        if search_backend is None:
+            logger.error(f"Error while running task: search backend not found: {self.job.index_options.search_backend}")
+            return -1
+
+        logger.debug(f"Fetched search backend options for {self.job.index_options.search_backend}")
+
+        return_code = sist2.index(self.job.index_options, search_backend, logs_cb=self.log_callback)
+        self.ended = datetime.utcnow()
 
         duration = self.ended - self.started
 
@@ -206,9 +210,17 @@ class Sist2IndexTask(Sist2Task):
             except ChildProcessError:
                 pass
 
+            backend_name = frontend.web_options.search_backend
+            search_backend = db["search_backends"][backend_name]
+            if search_backend is None:
+                logger.error(f"Error while running task: search backend not found: {backend_name}")
+                return -1
+
+            logger.debug(f"Fetched search backend options for {backend_name}")
+
             frontend.web_options.indices = map(lambda j: db["jobs"][j].index_path, frontend.jobs)
 
-            pid = sist2.web(frontend.web_options, frontend.name)
+            pid = sist2.web(frontend.web_options, search_backend, frontend.name)
             RUNNING_FRONTENDS[frontend_name] = pid
 
             self._logger.info(json.dumps({"sist2-admin": f"Restart frontend {pid=} {frontend_name=}"}))
@@ -301,8 +313,14 @@ class TaskQueue:
                 "ended": task.ended,
                 "started": task.started,
                 "name": task.display_name,
-                "return_code": task_result
+                "return_code": task_result,
+                "has_logs": 1
             }
+
+            logs_to_delete = get_log_files_to_remove(self._db, task.job.name, task.job.keep_last_n_logs)
+            for row in logs_to_delete:
+                delete_log_file(self._db, row["id"])
+
         if isinstance(task, Sist2IndexTask):
             self._notifications.notify({
                 "message": "notifications.indexCompleted",

sist2-admin/sist2_admin/sist2.py

@@ -3,6 +3,7 @@ import json
 import logging
 import os.path
 from datetime import datetime
+from enum import Enum
 from io import TextIOWrapper
 from logging import FileHandler
 from subprocess import Popen, PIPE
@@ -12,7 +13,7 @@ from typing import List
 
 from pydantic import BaseModel
 
-from config import logger, LOG_FOLDER
+from config import logger, LOG_FOLDER, DATA_FOLDER
 
 
 class Sist2Version:
@@ -25,77 +26,57 @@ class Sist2Version:
         return f"{self.major}.{self.minor}.{self.patch}"
 
 
-class WebOptions(BaseModel):
-    indices: List[str] = []
+class SearchBackendType(Enum):
+    SQLITE = "sqlite"
+    ELASTICSEARCH = "elasticsearch"
+
+
+class Sist2SearchBackend(BaseModel):
+    backend_type: SearchBackendType = SearchBackendType("elasticsearch")
+    name: str
+
+    search_index: str = ""
+
     es_url: str = "http://elasticsearch:9200"
     es_insecure_ssl: bool = False
     es_index: str = "sist2"
-    bind: str = "0.0.0.0:4090"
-    auth: str = None
-    tag_auth: str = None
-    tagline: str = "Lightning-fast file system indexer and search tool"
-    dev: bool = False
-    lang: str = "en"
-    auth0_audience: str = None
-    auth0_domain: str = None
-    auth0_client_id: str = None
-    auth0_public_key: str = None
-    auth0_public_key_file: str = None
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def args(self):
-        args = ["web", f"--es-url={self.es_url}", f"--bind={self.bind}",
-                f"--tagline={self.tagline}", f"--lang={self.lang}"]
-
-        if self.auth0_audience:
-            args.append(f"--auth0-audience={self.auth0_audience}")
-        if self.auth0_domain:
-            args.append(f"--auth0-domain={self.auth0_domain}")
-        if self.auth0_client_id:
-            args.append(f"--auth0-client-id={self.auth0_client_id}")
-        if self.auth0_public_key_file:
-            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
-        if self.es_insecure_ssl:
-            args.append(f"--es-insecure-ssl")
-        if self.auth:
-            args.append(f"--auth={self.auth}")
-        if self.tag_auth:
-            args.append(f"--tag-auth={self.tag_auth}")
-        if self.dev:
-            args.append(f"--dev")
-
-        args.extend(self.indices)
-
-        return args
-
-
-class IndexOptions(BaseModel):
-    path: str = None
     threads: int = 1
-    es_url: str = "http://elasticsearch:9200"
-    es_insecure_ssl: bool = False
-    es_index: str = "sist2"
-    incremental_index: bool = True
     script: str = ""
     script_file: str = None
     batch_size: int = 70
 
+    @staticmethod
+    def create_default(name: str, backend_type: SearchBackendType = SearchBackendType("elasticsearch")):
+        return Sist2SearchBackend(
+            name=name,
+            search_index=os.path.join(DATA_FOLDER, f"search-index-{name.replace('/', '_')}.sist2"),
+            backend_type=backend_type
+        )
+
+
+class IndexOptions(BaseModel):
+    path: str = None
+    incremental_index: bool = True
+    search_backend: str = None
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def args(self):
+    def args(self, search_backend):
+        if search_backend.backend_type == SearchBackendType("sqlite"):
+            args = ["sqlite-index", self.path, "--search-index", search_backend.search_index]
+        else:
+            args = ["index", self.path, f"--threads={search_backend.threads}",
+                    f"--es-url={search_backend.es_url}",
+                    f"--es-index={search_backend.es_index}",
+                    f"--batch-size={search_backend.batch_size}"]
 
-        args = ["index", self.path, f"--threads={self.threads}", f"--es-url={self.es_url}",
-                f"--es-index={self.es_index}", f"--batch-size={self.batch_size}"]
-
-        if self.script_file:
-            args.append(f"--script-file={self.script_file}")
-        if self.es_insecure_ssl:
-            args.append(f"--es-insecure-ssl")
-        if self.incremental_index:
-            args.append(f"--incremental-index")
+            if search_backend.script_file:
+                args.append(f"--script-file={search_backend.script_file}")
+            if search_backend.es_insecure_ssl:
+                args.append(f"--es-insecure-ssl")
+            if self.incremental_index:
+                args.append(f"--incremental-index")
 
         return args
 
@@ -109,7 +90,7 @@ ARCHIVE_RECURSE = "recurse"
 class ScanOptions(BaseModel):
     path: str
     threads: int = 1
-    thumbnail_quality: int = 2
+    thumbnail_quality: int = 50
     thumbnail_size: int = 552
     thumbnail_count: int = 1
     content_size: int = 32768
@@ -200,6 +181,56 @@ class Sist2Index:
     def name(self) -> str:
         return self._descriptor["name"]
 
+class WebOptions(BaseModel):
+    indices: List[str] = []
+
+    search_backend: str = "elasticsearch"
+
+    bind: str = "0.0.0.0:4090"
+    auth: str = None
+    tag_auth: str = None
+    tagline: str = "Lightning-fast file system indexer and search tool"
+    dev: bool = False
+    lang: str = "en"
+    auth0_audience: str = None
+    auth0_domain: str = None
+    auth0_client_id: str = None
+    auth0_public_key: str = None
+    auth0_public_key_file: str = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def args(self, search_backend: Sist2SearchBackend):
+        args = ["web", f"--bind={self.bind}", f"--tagline={self.tagline}",
+                f"--lang={self.lang}"]
+
+        if search_backend.backend_type == SearchBackendType("sqlite"):
+            args.append(f"--search-index={search_backend.search_index}")
+        else:
+            args.append(f"--es-url={search_backend.es_url}")
+            args.append(f"--es-index={search_backend.es_index}")
+            if search_backend.es_insecure_ssl:
+                args.append(f"--es-insecure-ssl")
+
+        if self.auth0_audience:
+            args.append(f"--auth0-audience={self.auth0_audience}")
+        if self.auth0_domain:
+            args.append(f"--auth0-domain={self.auth0_domain}")
+        if self.auth0_client_id:
+            args.append(f"--auth0-client-id={self.auth0_client_id}")
+        if self.auth0_public_key_file:
+            args.append(f"--auth0-public-key-file={self.auth0_public_key_file}")
+        if self.auth:
+            args.append(f"--auth={self.auth}")
+        if self.tag_auth:
+            args.append(f"--tag-auth={self.tag_auth}")
+        if self.dev:
+            args.append(f"--dev")
+
+        args.extend(self.indices)
+
+        return args
 
 class Sist2:
 
@@ -207,21 +238,23 @@ class Sist2:
         self._bin_path = bin_path
         self._data_dir = data_directory
 
-    def index(self, options: IndexOptions, logs_cb):
+    def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb):
 
-        if options.script:
+        if search_backend.script and search_backend.backend_type == SearchBackendType("elasticsearch"):
             with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".painless", delete=False) as f:
-                f.write(options.script)
-            options.script_file = f.name
+                f.write(search_backend.script)
+            search_backend.script_file = f.name
         else:
-            options.script_file = None
+            search_backend.script_file = None
 
         args = [
             self._bin_path,
-            *options.args(),
+            *options.args(search_backend),
             "--json-logs",
             "--very-verbose"
         ]
+
+        logs_cb({"sist2-admin": f"Starting sist2 command with args {args}"})
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
         t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
@@ -238,7 +271,7 @@ class Sist2:
         if options.output is None:
             options.output = os.path.join(
                 self._data_dir,
-                f"scan-{options.name.replace('/', '_')}-{datetime.now()}.sist2"
+                f"scan-{options.name.replace('/', '_')}-{datetime.utcnow()}.sist2"
             )
 
         args = [
@@ -290,7 +323,7 @@ class Sist2:
                 except NameError:
                     pass
 
-    def web(self, options: WebOptions, name: str):
+    def web(self, options: WebOptions, search_backend: Sist2SearchBackend, name: str):
 
         if options.auth0_public_key:
             with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
@@ -301,7 +334,7 @@ class Sist2:
 
         args = [
             self._bin_path,
-            *options.args()
+            *options.args(search_backend)
         ]
 
         web_logger = logging.Logger(name=f"sist2-frontend-{name}")
@@ -321,3 +354,5 @@ class Sist2:
         t_stdout.start()
 
         return proc.pid
+
+

sist2-admin/sist2_admin/state.py

@@ -1,16 +1,20 @@
 from typing import Dict
+import os
 import shutil
 
 from hexlib.db import Table, PersistentState
 import pickle
 
 from tesseract import get_tesseract_langs
+import sqlite3
+from config import LOG_FOLDER, logger
+from sist2 import SearchBackendType, Sist2SearchBackend
 
 RUNNING_FRONTENDS: Dict[str, int] = {}
 
 TESSERACT_LANGS = get_tesseract_langs()
 
-DB_SCHEMA_VERSION = "3"
+DB_SCHEMA_VERSION = "4"
 
 from pydantic import BaseModel
 
@@ -50,8 +54,35 @@ class PickleTable(Table):
             yield dict((k, _deserialize(v)) for k, v in row.items())
 
 
-def migrate_v1_to_v2(db: PersistentState):
+def get_log_files_to_remove(db: PersistentState, job_name: str, n: int):
+    if n < 0:
+        return []
 
+    counter = 0
+    to_remove = []
+
+    for row in db["task_done"].sql("WHERE has_logs=1 ORDER BY started DESC"):
+        if row["name"].endswith(f"[{job_name}]"):
+            counter += 1
+
+        if counter > n:
+            to_remove.append(row)
+
+    return to_remove
+
+
+def delete_log_file(db: PersistentState, task_id: str):
+    db["task_done"][task_id] = {
+        "has_logs": 0
+    }
+
+    try:
+        os.remove(os.path.join(LOG_FOLDER, f"sist2-{task_id}.log"))
+    except:
+        pass
+
+
+def migrate_v1_to_v2(db: PersistentState):
     shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v2.bak")
 
     # Frontends
@@ -77,3 +108,29 @@ def migrate_v1_to_v2(db: PersistentState):
     db["sist2_admin"]["info"] = {
         "version": "2"
     }
+
+
+def create_default_search_backends(db: PersistentState):
+    es_backend = Sist2SearchBackend.create_default(name="elasticsearch",
+                                                   backend_type=SearchBackendType("elasticsearch"))
+    db["search_backends"]["elasticsearch"] = es_backend
+    sqlite_backend = Sist2SearchBackend.create_default(name="sqlite", backend_type=SearchBackendType("sqlite"))
+    db["search_backends"]["sqlite"] = sqlite_backend
+
+
+def migrate_v3_to_v4(db: PersistentState):
+    shutil.copy(db.dbfile, db.dbfile + "-before-migrate-v4.bak")
+
+    create_default_search_backends(db)
+
+    try:
+        conn = sqlite3.connect(db.dbfile)
+        conn.execute("ALTER TABLE task_done ADD COLUMN has_logs INTEGER DEFAULT 1")
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        logger.exception(e)
+
+    db["sist2_admin"]["info"] = {
+        "version": "4"
+    }

sist2-vue/public/index.html

@@ -19,12 +19,6 @@
             We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled.
             Please enable it to continue.
         </strong>
-        <br/>
-        <strong>
-            Nous sommes désolés mais <%= htmlWebpackPlugin.options.title %> ne fonctionne pas correctement
-            si JavaScript est activé.
-            Veuillez l'activer pour continuer.
-        </strong>
     </div>
 </noscript>
 <div id="app"></div>

sist2-vue/src/App.vue

@@ -10,7 +10,7 @@
             <b-spinner type="grow" variant="primary"></b-spinner>
         </div>
         <div class="loading-text">
-            Loading • Chargement • 装载 • Wird geladen
+            Loading • Chargement • 装载 • Wird geladen • Ładowanie
         </div>
     </div>
 </template>
@@ -21,6 +21,8 @@ import {mapActions, mapGetters, mapMutations} from "vuex";
 import Sist2Api from "@/Sist2Api";
 import ModelsRepo from "@/ml/modelsRepo";
 import {setupAuth0} from "@/main";
+import Sist2ElasticsearchQuery from "@/Sist2ElasticsearchQuery";
+import Sist2SqliteQuery from "@/Sist2SqliteQuery";
 
 export default {
     components: {NavBar},
@@ -88,6 +90,13 @@ export default {
 
             this.setSist2Info(data);
             this.setIndices(data.indices)
+
+            if (Sist2Api.backend() === "sqlite") {
+                Sist2Api.init(Sist2SqliteQuery.searchQuery);
+                this.$store.commit("setUiSqliteMode", true);
+            } else {
+                Sist2Api.init(Sist2ElasticsearchQuery.searchQuery);
+            }
         });
     },
     methods: {

sist2-vue/src/Sist2Api.ts

@@ -1,5 +1,7 @@
 import axios from "axios";
 import {ext, strUnescape, lum} from "./util";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
+import store from "@/store";
 
 export interface EsTag {
     id: string
@@ -99,12 +101,22 @@ export interface EsResult {
 
 class Sist2Api {
 
-    private baseUrl: string
+    private readonly baseUrl: string
+    private sist2Info: any
+    private queryfunc: () => EsResult;
 
     constructor(baseUrl: string) {
         this.baseUrl = baseUrl;
     }
 
+    init(queryFunc: () => EsResult) {
+        this.queryfunc = queryFunc;
+    }
+
+    backend() {
+        return this.sist2Info.searchBackend;
+    }
+
     getSist2Info(): Promise<any> {
         return axios.get(`${this.baseUrl}i`).then(resp => {
             const indices = resp.data.indices as Index[];
@@ -119,6 +131,8 @@ class Sist2Api {
                 } as Index;
             });
 
+            this.sist2Info = resp.data;
+
             return resp.data;
         })
     }
@@ -219,6 +233,14 @@ class Sist2Api {
         } as Tag;
     }
 
+    search(): Promise<EsResult> {
+        if (this.backend() == "sqlite") {
+            return this.ftsQuery(this.queryfunc())
+        } else {
+            return this.esQuery(this.queryfunc());
+        }
+    }
+
     esQuery(query: any): Promise<EsResult> {
         return axios.post(`${this.baseUrl}es`, query).then(resp => {
             const res = resp.data as EsResult;
@@ -237,7 +259,30 @@ class Sist2Api {
         });
     }
 
-    getMimeTypes(query = undefined) {
+    ftsQuery(query: any): Promise<EsResult> {
+        return axios.post(`${this.baseUrl}fts/search`, query).then(resp => {
+            const res = resp.data as any;
+
+            if (res.hits.hits) {
+                res.hits.hits.forEach(hit => {
+                    hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
+                    hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+
+                    this.setHitProps(hit);
+                    this.setHitTags(hit);
+
+                    if ("highlight" in hit) {
+                        hit["highlight"]["name"] = [hit["highlight"]["name"]];
+                        hit["highlight"]["content"] = [hit["highlight"]["content"]];
+                    }
+                });
+            }
+
+            return res;
+        });
+    }
+
+    private getMimeTypesEs(query) {
         const AGGS = {
             mimeTypes: {
                 terms: {
@@ -258,48 +303,70 @@ class Sist2Api {
         }
 
         return this.esQuery(query).then(resp => {
-            const mimeMap: any[] = [];
-            const buckets = resp["aggregations"]["mimeTypes"]["buckets"];
+            return resp["aggregations"]["mimeTypes"]["buckets"].map(bucket => ({
+                mime: bucket.key,
+                count: bucket.doc_count
+            }));
 
-            buckets.sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
-                const tmp = bucket["key"].split("/");
-                const category = tmp[0];
-                const mime = tmp[1];
+        });
+    }
 
-                let category_exists = false;
+    private getMimeTypesSqlite(): Promise<[{ mime: string, count: number }]> {
+        return axios.get(`${this.baseUrl}fts/mimetypes`)
+            .then(resp => {
+                return resp.data;
+            });
+    }
 
-                const child = {
-                    "id": bucket["key"],
-                    "text": `${mime} (${bucket["doc_count"]})`
-                };
+    async getMimeTypes(query = undefined) {
+        let buckets;
 
-                mimeMap.forEach(node => {
-                    if (node.text === category) {
-                        node.children.push(child);
-                        category_exists = true;
-                    }
-                });
+        if (this.backend() == "sqlite") {
+            buckets = await this.getMimeTypesSqlite();
+        } else {
+            buckets = await this.getMimeTypesEs(query);
+        }
 
-                if (!category_exists) {
-                    mimeMap.push({text: category, children: [child], id: category});
-                }
-            })
+        const mimeMap: any[] = [];
+
+        buckets.sort((a: any, b: any) => a.mime > b.mime).forEach((bucket: any) => {
+            const tmp = bucket.mime.split("/");
+            const category = tmp[0];
+            const mime = tmp[1];
+
+            let category_exists = false;
+
+            const child = {
+                "id": bucket.mime,
+                "text": `${mime} (${bucket.count})`
+            };
 
             mimeMap.forEach(node => {
-                if (node.children) {
-                    node.children.sort((a, b) => a.id.localeCompare(b.id));
+                if (node.text === category) {
+                    node.children.push(child);
+                    category_exists = true;
                 }
-            })
-            mimeMap.sort((a, b) => a.id.localeCompare(b.id))
+            });
 
-            return {buckets, mimeMap};
-        });
+            if (!category_exists) {
+                mimeMap.push({text: category, children: [child], id: category});
+            }
+        })
+
+        mimeMap.forEach(node => {
+            if (node.children) {
+                node.children.sort((a, b) => a.id.localeCompare(b.id));
+            }
+        })
+        mimeMap.sort((a, b) => a.id.localeCompare(b.id))
+
+        return {buckets, mimeMap};
     }
 
     _createEsTag(tag: string, count: number): EsTag {
         const tokens = tag.split(".");
 
-        if (/.*\.#[0-9a-f]{6}/.test(tag)) {
+        if (/.*\.#[0-9a-fA-F]{6}/.test(tag)) {
             return {
                 id: tokens.slice(0, -1).join("."),
                 color: tokens.pop(),
@@ -316,32 +383,48 @@ class Sist2Api {
         };
     }
 
-    getTags() {
+    private getTagsEs() {
         return this.esQuery({
             aggs: {
                 tags: {
                     terms: {
                         field: "tag",
-                        size: 10000
+                        size: 65535
                     }
                 }
             },
             size: 0,
         }).then(resp => {
-            const seen = new Set();
-
-            const tags = resp["aggregations"]["tags"]["buckets"]
+            return resp["aggregations"]["tags"]["buckets"]
                 .sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
                 .map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
+        });
+    }
 
-            // Remove duplicates (same tag with different color)
-            return tags.filter((t: EsTag) => {
-                if (seen.has(t.id)) {
-                    return false;
-                }
-                seen.add(t.id);
-                return true;
+    private getTagsSqlite() {
+        return axios.get(`${this.baseUrl}/fts/tags`)
+            .then(resp => {
+                return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
             });
+    }
+
+    async getTags(): Promise<EsTag[]> {
+        let tags;
+        if (this.backend() == "sqlite") {
+            tags = await this.getTagsSqlite();
+        } else {
+            tags = await this.getTagsEs();
+        }
+
+        // Remove duplicates (same tag with different color)
+        const seen = new Set();
+
+        return tags.filter((t: EsTag) => {
+            if (seen.has(t.id)) {
+                return false;
+            }
+            seen.add(t.id);
+            return true;
         });
     }
 
@@ -361,6 +444,144 @@ class Sist2Api {
         });
     }
 
+    searchPaths(indexId, minDepth, maxDepth, prefix = null) {
+        if (this.backend() == "sqlite") {
+            return this.searchPathsSqlite(indexId, minDepth, minDepth, prefix);
+        } else {
+            return this.searchPathsEs(indexId, minDepth, maxDepth, prefix);
+        }
+    }
+
+    private searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
+        return axios.post(`${this.baseUrl}fts/paths`, {
+            indexId, minDepth, maxDepth, prefix
+        }).then(resp => {
+            return resp.data;
+        });
+    }
+
+    private searchPathsEs(indexId, minDepth, maxDepth, prefix): Promise<[{ path: string, count: number }]> {
+
+        const query = {
+            query: {
+                bool: {
+                    filter: [
+                        {term: {index: indexId}},
+                        {range: {_depth: {gte: minDepth, lte: maxDepth}}},
+                    ]
+                }
+            },
+            aggs: {
+                paths: {
+                    terms: {
+                        field: "path",
+                        size: 10000
+                    }
+                }
+            },
+            size: 0
+        };
+
+        if (prefix != null) {
+            query["query"]["bool"]["must"] = {
+                prefix: {
+                    path: prefix,
+                }
+            };
+        }
+
+        return this.esQuery(query).then(resp => {
+            const buckets = resp["aggregations"]["paths"]["buckets"];
+
+            if (!buckets) {
+                return [];
+            }
+
+            return buckets
+                .map(bucket => ({
+                    path: bucket.key,
+                    count: bucket.doc_count
+                }));
+        });
+    }
+
+    private getDateRangeSqlite() {
+        return axios.get(`${this.baseUrl}fts/dateRange`)
+            .then(resp => ({
+                min: resp.data.dateMin,
+                max: resp.data.dateMax,
+            }));
+    }
+
+    getDateRange(): Promise<{ min: number, max: number }> {
+        if (this.backend() == "sqlite") {
+            return this.getDateRangeSqlite();
+        } else {
+            return this.getDateRangeEs();
+        }
+    }
+
+    private getDateRangeEs() {
+        return this.esQuery({
+            // TODO: filter current selected indices
+            aggs: {
+                dateMin: {min: {field: "mtime"}},
+                dateMax: {max: {field: "mtime"}},
+            },
+            size: 0
+        }).then(res => {
+            const range = {
+                min: res.aggregations.dateMin.value,
+                max: res.aggregations.dateMax.value,
+            }
+
+            if (range.min == null) {
+                range.min = 0;
+                range.max = 1;
+            } else if (range.min == range.max) {
+                range.max += 1;
+            }
+
+            return range;
+        });
+    }
+
+    private getPathSuggestionsSqlite(text: string) {
+        return axios.post(`${this.baseUrl}fts/paths`, {
+            prefix: text,
+            minDepth: 1,
+            maxDepth: 10000
+        }).then(resp => {
+            return resp.data.map(bucket => bucket.path);
+        })
+    }
+
+    private getPathSuggestionsEs(text) {
+        return this.esQuery({
+            suggest: {
+                path: {
+                    prefix: text,
+                    completion: {
+                        field: "suggest-path",
+                        skip_duplicates: true,
+                        size: 10000
+                    }
+                }
+            }
+        }).then(resp => {
+            return resp["suggest"]["path"][0]["options"]
+                .map(opt => opt["_source"]["path"]);
+        });
+    }
+
+    getPathSuggestions(text: string): Promise<string[]> {
+        if (this.backend() == "sqlite") {
+            return this.getPathSuggestionsSqlite(text);
+        } else {
+            return this.getPathSuggestionsEs(text)
+        }
+    }
+
     getTreemapStat(indexId: string) {
         return `${this.baseUrl}s/${indexId}/TMAP`;
     }
@@ -376,6 +597,111 @@ class Sist2Api {
     getDateStat(indexId: string) {
         return `${this.baseUrl}s/${indexId}/DAGG`;
     }
+
+    private getDocumentEs(docId: string, highlight: boolean, fuzzy: boolean) {
+        const query = Sist2Query.searchQuery();
+
+        if (highlight) {
+            const fields = fuzzy
+                ? {"content.nGram": {}}
+                : {content: {}};
+
+            query.highlight = {
+                pre_tags: ["<mark>"],
+                post_tags: ["</mark>"],
+                number_of_fragments: 0,
+                fields,
+            };
+
+            if (!store.state.sist2Info.esVersionLegacy) {
+                query.highlight.max_analyzed_offset = 999_999;
+            }
+        }
+
+        if ("function_score" in query.query) {
+            query.query = query.query.function_score.query;
+        }
+
+        if (!("must" in query.query.bool)) {
+            query.query.bool.must = [];
+        } else if (!Array.isArray(query.query.bool.must)) {
+            query.query.bool.must = [query.query.bool.must];
+        }
+
+        query.query.bool.must.push({match: {_id: docId}});
+
+        delete query["sort"];
+        delete query["aggs"];
+        delete query["search_after"];
+        delete query.query["function_score"];
+
+        query._source = {
+            includes: ["content", "name", "path", "extension"]
+        }
+
+        query.size = 1;
+
+        return this.esQuery(query).then(resp => {
+            if (resp.hits.hits.length === 1) {
+                return resp.hits.hits[0];
+            }
+            return null;
+        });
+    }
+
+    private getDocumentSqlite(docId: string): Promise<EsHit> {
+        return axios.get(`${this.baseUrl}/fts/d/${docId}`)
+            .then(resp => ({
+                _source: resp.data
+            } as EsHit));
+    }
+
+    getDocument(docId: string, highlight: boolean, fuzzy: boolean): Promise<EsHit | null> {
+        if (this.backend() == "sqlite") {
+            return this.getDocumentSqlite(docId);
+        } else {
+            return this.getDocumentEs(docId, highlight, fuzzy);
+        }
+    }
+
+    getTagSuggestions(prefix: string): Promise<string[]> {
+        if (this.backend() == "sqlite") {
+            return this.getTagSuggestionsSqlite(prefix);
+        } else {
+            return this.getTagSuggestionsEs(prefix);
+        }
+    }
+
+    private getTagSuggestionsSqlite(prefix): Promise<string[]> {
+        return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
+            .then(resp => (resp.data));
+    }
+
+    private getTagSuggestionsEs(prefix): Promise<string[]> {
+        return this.esQuery({
+            suggest: {
+                tag: {
+                    prefix: prefix,
+                    completion: {
+                        field: "suggest-tag",
+                        skip_duplicates: true,
+                        size: 10000
+                    }
+                }
+            }
+        }).then(resp => {
+            const result = [];
+            resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
+                tags.forEach(tag => {
+                    const t = tag.slice(0, -8);
+                    if (!result.find(x => x.slice(0, -8) === t)) {
+                        result.push(tag);
+                    }
+                });
+            });
+            return result;
+        });
+    }
 }
 
 export default new Sist2Api("");

sist2-vue/src/Sist2ElasticsearchQuery.ts

@@ -60,14 +60,7 @@ const SORT_MODES = {
     }
 } as any;
 
-interface SortMode {
-    text: string
-    mode: any[]
-    key: (hit: EsHit) => any
-}
-
-
-class Sist2Query {
+class Sist2ElasticsearchQuery {
 
     searchQuery(blankSearch: boolean = false): any {
 
@@ -177,14 +170,16 @@ class Sist2Query {
                 }
             },
             sort: SORT_MODES[getters.sortMode].mode,
-            aggs:
-                {
-                    total_size: {"sum": {"field": "size"}},
-                    total_count: {"value_count": {"field": "size"}}
-                },
             size: size,
         } as any;
 
+        if (!after) {
+            q.aggs = {
+                total_size: {"sum": {"field": "size"}},
+                total_count: {"value_count": {"field": "size"}}
+            };
+        }
+
         if (!empty && !blankSearch) {
             q.query.bool.must = query;
         }
@@ -249,4 +244,5 @@ class Sist2Query {
     }
 }
 
-export default new Sist2Query();
+
+export default new Sist2ElasticsearchQuery();

sist2-vue/src/Sist2SqliteQuery.ts

@@ -0,0 +1,111 @@
+import store from "./store";
+import {EsHit, Index} from "@/Sist2Api";
+
+const SORT_MODES = {
+    score: {
+        "sort": "score",
+    },
+    random: {
+        "sort": "random"
+    },
+    dateAsc: {
+        "sort": "mtime"
+    },
+    dateDesc: {
+        "sort": "mtime",
+        "sortAsc": false
+    },
+    sizeAsc: {
+        "sort": "size",
+    },
+    sizeDesc: {
+        "sort": "size",
+        "sortAsc": false
+    },
+    nameAsc: {
+        "sort": "name",
+    },
+    nameDesc: {
+        "sort": "name",
+        "sortAsc": false
+    }
+} as any;
+
+interface SortMode {
+    text: string
+    mode: any[]
+    key: (hit: EsHit) => any
+}
+
+
+class Sist2ElasticsearchQuery {
+
+    searchQuery(): any {
+
+        const getters = store.getters;
+
+        const searchText = getters.searchText;
+        const pathText = getters.pathText;
+        const sizeMin = getters.sizeMin;
+        const sizeMax = getters.sizeMax;
+        const dateMin = getters.dateMin;
+        const dateMax = getters.dateMax;
+        const size = getters.size;
+        const after = getters.lastDoc;
+        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
+        const selectedMimeTypes = getters.selectedMimeTypes;
+        const selectedTags = getters.selectedTags;
+
+        const q = {
+            "pageSize": size
+        }
+
+        Object.assign(q, SORT_MODES[getters.sortMode]);
+
+        if (!after) {
+            q["fetchAggregations"] = true;
+        }
+        if (searchText) {
+            q["query"] = searchText;
+        }
+        if (pathText) {
+            q["path"] = pathText.endsWith("/") ? pathText.slice(0, -1) : pathText;
+        }
+        if (sizeMin) {
+            q["sizeMin"] = sizeMin;
+        }
+        if (sizeMax) {
+            q["sizeMax"] = sizeMax;
+        }
+        if (dateMin) {
+            q["dateMin"] = dateMin;
+        }
+        if (dateMax) {
+            q["dateMax"] = dateMax;
+        }
+        if (after) {
+            q["after"] = after.sort;
+        }
+        if (selectedIndexIds.length > 0) {
+            q["indexIds"] = selectedIndexIds;
+        }
+        if (selectedMimeTypes.length > 0) {
+            q["mimeTypes"] = selectedMimeTypes;
+        }
+        if (selectedTags.length > 0) {
+            q["tags"] = selectedTags
+        }
+        if (getters.sortMode == "random") {
+            q["seed"] = getters.seed;
+        }
+        if (getters.optHighlight) {
+            q["highlight"] = true;
+            q["highlightContextSize"] = Number(getters.optFragmentSize);
+        }
+
+        return q;
+    }
+}
+
+
+export default new Sist2ElasticsearchQuery();

sist2-vue/src/components/DebugInfo.vue

@@ -1,41 +1,56 @@
 <template>
-  <b-card v-if="$store.state.sist2Info.showDebugInfo" class="mb-4 mt-4">
-    <b-card-title><DebugIcon class="mr-1"></DebugIcon>{{ $t("debug") }}</b-card-title>
-    <p v-html="$t('debugDescription')"></p>
+    <b-card v-if="$store.state.sist2Info.showDebugInfo" class="mb-4 mt-4">
+        <b-card-title>
+            <DebugIcon class="mr-1"></DebugIcon>
+            {{ $t("debug") }}
+        </b-card-title>
+        <p v-html="$t('debugDescription')"></p>
 
-    <b-card-body>
+        <b-card-body>
 
-      <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
+            <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
 
-      <hr />
-      <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx" class="mt-2"></IndexDebugInfo>
-    </b-card-body>
-  </b-card>
+            <hr/>
+            <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx"
+                            class="mt-2"></IndexDebugInfo>
+        </b-card-body>
+    </b-card>
 </template>
 
 <script>
 import IndexDebugInfo from "@/components/IndexDebugInfo";
 import DebugIcon from "@/components/icons/DebugIcon";
+import {mapGetters} from "vuex";
 
 export default {
-  name: "DebugInfo.vue",
-  components: {DebugIcon, IndexDebugInfo},
-  computed: {
-    tableItems() {
-      return [
-        {key: "version", value: this.$store.state.sist2Info.version},
-        {key: "platform", value: this.$store.state.sist2Info.platform},
-        {key: "debugBinary", value: this.$store.state.sist2Info.debug},
-        {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
-        {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
-        {key: "tagline", value: this.$store.state.sist2Info.tagline},
-        {key: "dev", value: this.$store.state.sist2Info.dev},
-        {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
-        {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
-        {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
-        {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
-      ]
+    name: "DebugInfo.vue",
+    components: {DebugIcon, IndexDebugInfo},
+    computed: {
+        ...mapGetters([
+            "uiSqliteMode",
+        ]),
+        tableItems() {
+            const items = [
+                {key: "version", value: this.$store.state.sist2Info.version},
+                {key: "platform", value: this.$store.state.sist2Info.platform},
+                {key: "debugBinary", value: this.$store.state.sist2Info.debug},
+                {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
+                {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
+                {key: "tagline", value: this.$store.state.sist2Info.tagline},
+                {key: "dev", value: this.$store.state.sist2Info.dev},
+                {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
+            ];
+
+            if (!this.uiSqliteMode) {
+                items.push(
+                    {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
+                    {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
+                    {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy}
+                );
+            }
+
+            return items;
+        }
     }
-  }
 }
 </script>

sist2-vue/src/components/DocCard.vue

@@ -1,44 +1,44 @@
 <template>
-  <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
-       @click="$store.commit('busTnTouchStart', null)">
-    <b-card
-        no-body
-        img-top
-    >
-      <!-- Info modal-->
-      <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
+    <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
+         @click="$store.commit('busTnTouchStart', null)">
+        <b-card
+                no-body
+                img-top
+        >
+            <!-- Info modal-->
+            <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
 
-      <ContentDiv :doc="doc"></ContentDiv>
+            <ContentDiv :doc="doc"></ContentDiv>
 
-      <!-- Thumbnail-->
-      <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
+            <!-- Thumbnail-->
+            <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
 
-      <!-- Audio player-->
-      <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
-             :type="doc._source.mime"
-             :src="`f/${doc._id}`"
-             @play="onAudioPlay()"></audio>
+            <!-- Audio player-->
+            <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
+                   :type="doc._source.mime"
+                   :src="`f/${doc._source.index}/${doc._id}`"
+                   @play="onAudioPlay()"></audio>
 
-      <b-card-body class="padding-03">
+            <b-card-body class="padding-03">
 
-        <!-- Title line -->
-        <div style="display: flex">
-          <span class="info-icon" @click="onInfoClick()"></span>
-          <DocFileTitle :doc="doc"></DocFileTitle>
-        </div>
+                <!-- Title line -->
+                <div style="display: flex">
+                    <span class="info-icon" @click="onInfoClick()"></span>
+                    <DocFileTitle :doc="doc"></DocFileTitle>
+                </div>
 
-        <!-- Featured line -->
-        <div style="display: flex">
-          <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
-        </div>
+                <!-- Featured line -->
+                <div style="display: flex">
+                    <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
+                </div>
 
-        <!-- Tags -->
-        <div class="card-text">
-          <TagContainer :hit="doc"></TagContainer>
-        </div>
-      </b-card-body>
-    </b-card>
-  </div>
+                <!-- Tags -->
+                <div class="card-text">
+                    <TagContainer :hit="doc"></TagContainer>
+                </div>
+            </b-card-body>
+        </b-card>
+    </div>
 </template>
 
 <script>
@@ -52,91 +52,91 @@ import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
 
 
 export default {
-  components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
-  props: ["doc", "width"],
-  data() {
-    return {
-      ext: ext,
-      showInfo: false,
-    }
-  },
-  computed: {
-    smallBadge() {
-      return this.width < 150;
-    }
-  },
-  methods: {
-    humanFileSize: humanFileSize,
-    humanTime: humanTime,
-    onInfoClick() {
-      this.showInfo = true;
-    },
-    async onThumbnailClick() {
-      this.$store.commit("setUiLightboxSlide", this.doc._seq);
-      await this.$store.dispatch("showLightbox");
-    },
-    onAudioPlay() {
-      document.getElementsByTagName("audio").forEach((el) => {
-        if (el !== this.$refs["audio"]) {
-          el.pause();
+    components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+    props: ["doc", "width"],
+    data() {
+        return {
+            ext: ext,
+            showInfo: false,
         }
-      });
     },
-  },
+    computed: {
+        smallBadge() {
+            return this.width < 150;
+        }
+    },
+    methods: {
+        humanFileSize: humanFileSize,
+        humanTime: humanTime,
+        onInfoClick() {
+            this.showInfo = true;
+        },
+        async onThumbnailClick() {
+            this.$store.commit("setUiLightboxSlide", this.doc._seq);
+            await this.$store.dispatch("showLightbox");
+        },
+        onAudioPlay() {
+            Array.prototype.slice.call(document.getElementsByTagName("audio")).forEach((el) => {
+                if (el !== this.$refs["audio"]) {
+                    el.pause();
+                }
+            });
+        },
+    },
 }
 </script>
 <style>
 .fit {
-  display: block;
-  min-width: 64px;
-  max-width: 100%;
-  /*max-height: 400px;*/
-  margin: 0 auto 0;
-  width: auto;
-  height: auto;
+    display: block;
+    min-width: 64px;
+    max-width: 100%;
+    /*max-height: 400px;*/
+    margin: 0 auto 0;
+    width: auto;
+    height: auto;
 }
 
 .audio-fit {
-  height: 39px;
-  vertical-align: bottom;
-  display: inline;
-  width: 100%;
+    height: 39px;
+    vertical-align: bottom;
+    display: inline;
+    width: 100%;
 }
 </style>
 
 <style scoped>
 
 .padding-03 {
-  padding: 0.3rem;
+    padding: 0.3rem;
 }
 
 .card {
-  margin-top: 1em;
-  margin-left: 0;
-  margin-right: 0;
-  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-  border-radius: 0;
-  border: none;
+    margin-top: 1em;
+    margin-left: 0;
+    margin-right: 0;
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
 }
 
 .card-body {
-  padding: 0.3rem;
+    padding: 0.3rem;
 }
 
 .doc-card {
-  padding-left: 3px;
-  padding-right: 3px;
+    padding-left: 3px;
+    padding-right: 3px;
 }
 
 .sub-document .card {
-  background: #AB47BC1F !important;
+    background: #AB47BC1F !important;
 }
 
 .theme-black .sub-document .card {
-  background: #37474F !important;
+    background: #37474F !important;
 }
 
 .sub-document .fit {
-  padding: 4px 4px 0 4px;
+    padding: 4px 4px 0 4px;
 }
 </style>

sist2-vue/src/components/DocFileTitle.vue

@@ -1,5 +1,5 @@
 <template>
-  <a :href="`f/${doc._id}`" class="file-title-anchor" target="_blank">
+  <a :href="`f/${doc._source.index}/${doc._id}`" class="file-title-anchor" target="_blank">
     <div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
          v-html="fileName() + ext(doc)"></div>
   </a>

sist2-vue/src/components/DocListItem.vue

@@ -17,7 +17,7 @@
           </div>
 
           <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-               :src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
+               :src="(doc._props.isGif && hover) ? `f/${doc._source.index}/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
                alt=""
                class="pointer fit-sm" @click="onThumbnailClick()">
           <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""

sist2-vue/src/components/FullThumbnail.vue

@@ -71,7 +71,7 @@ export default {
       const doc = this.doc;
       const props = doc._props;
       if (props.isGif && this.hover) {
-        return `f/${doc._id}`;
+        return `f/${doc._source.index}/${doc._id}`;
       }
       return (this.currentThumbnailNum === 0)
           ? `t/${doc._source.index}/${doc._id}`

sist2-vue/src/components/LazyContentDiv.vue

@@ -18,7 +18,7 @@
         <b-progress v-if="mlLoading" variant="warning" show-progress :max="1" class="mb-3"
         >
             <b-progress-bar :value="modelLoadingProgress">
-                <strong>{{ ((modelLoadingProgress * modelSize) / (1024*1024)).toFixed(1) }}MB / {{
+                <strong>{{ ((modelLoadingProgress * modelSize) / (1024 * 1024)).toFixed(1) }}MB / {{
                     (modelSize / (1024 * 1024)).toFixed(1)
                     }}MB</strong>
             </b-progress-bar>
@@ -36,7 +36,7 @@
 <script>
 import Sist2Api from "@/Sist2Api";
 import Preloader from "@/components/Preloader";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 import store from "@/store";
 import BertNerModel from "@/ml/BertNerModel";
 import AnalyzedContentSpansContainer from "@/components/AnalyzedContentSpanContainer.vue";
@@ -69,58 +69,19 @@ export default {
             this.mlModel = ModelsRepo.getDefaultModel();
         }
 
-        const query = Sist2Query.searchQuery();
+        Sist2Api
+            .getDocument(this.docId, this.$store.state.optHighlight, this.$store.state.fuzzy)
+            .then(doc => {
+                this.loading = false;
 
-        if (this.$store.state.optHighlight) {
-            const fields = this.$store.state.fuzzy
-                ? {"content.nGram": {}}
-                : {content: {}};
+                if (doc) {
+                    this.content = this.getContent(doc)
+                }
 
-            query.highlight = {
-                pre_tags: ["<mark>"],
-                post_tags: ["</mark>"],
-                number_of_fragments: 0,
-                fields,
-            };
-
-            if (!store.state.sist2Info.esVersionLegacy) {
-                query.highlight.max_analyzed_offset = 999_999;
-            }
-        }
-
-        if ("function_score" in query.query) {
-            query.query = query.query.function_score.query;
-        }
-
-        if (!("must" in query.query.bool)) {
-            query.query.bool.must = [];
-        } else if (!Array.isArray(query.query.bool.must)) {
-            query.query.bool.must = [query.query.bool.must];
-        }
-
-        query.query.bool.must.push({match: {_id: this.docId}});
-
-        delete query["sort"];
-        delete query["aggs"];
-        delete query["search_after"];
-        delete query.query["function_score"];
-
-        query._source = {
-            includes: ["content", "name", "path", "extension"]
-        }
-
-        query.size = 1;
-
-        Sist2Api.esQuery(query).then(resp => {
-            this.loading = false;
-            if (resp.hits.hits.length === 1) {
-                this.content = this.getContent(resp.hits.hits[0]);
-            }
-
-            if (this.optAutoAnalyze) {
-                this.mlAnalyze();
-            }
-        });
+                if (this.optAutoAnalyze) {
+                    this.mlAnalyze();
+                }
+            });
     },
     computed: {
         ...mapGetters(["optAutoAnalyze"]),

sist2-vue/src/components/MimePicker.vue

@@ -9,7 +9,7 @@ import InspireTreeDOM from "inspire-tree-dom";
 import "inspire-tree-dom/dist/inspire-tree-light.min.css";
 import {getSelectedTreeNodes, getTreeNodeAttributes} from "@/util";
 import Sist2Api from "@/Sist2Api";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 
 export default {
   name: "MimePicker",

sist2-vue/src/components/NavBar.vue

@@ -10,7 +10,7 @@
     <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
       v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
         href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
-        target="_blank">legacyES</a></span>
+        target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
     </span>
 
     <span v-if="$store && $store.state.sist2Info" class="tagline" v-html="tagline()"></span>

sist2-vue/src/components/PathTree.vue

@@ -1,40 +1,41 @@
 <template>
-  <div>
-    <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
-      <div class="input-group-prepend">
+    <div>
+        <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
+            <div class="input-group-prepend">
 
-        <b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
-          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
-            <path
-                fill="currentColor"
-                d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
-            />
-          </svg>
-        </b-button>
-      </div>
+                <b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
+                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
+                        <path
+                                fill="currentColor"
+                                d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
+                        />
+                    </svg>
+                </b-button>
+            </div>
+
+            <VueSimpleSuggest
+                    class="form-control-fix-flex"
+                    @input="setPathText"
+                    :value="getPathText"
+                    :list="suggestPath"
+                    :max-suggestions="0"
+                    :placeholder="$t('pathBar.placeholder')"
+                    :debounce="200"
+            >
+                <!-- Suggestion item template-->
+                <div slot="suggestion-item" slot-scope="{ suggestion, query }">
+                    <div class="suggestion-line" :title="suggestion">
+                        <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
+                    </div>
+                </div>
+            </VueSimpleSuggest>
 
-      <VueSimpleSuggest
-          class="form-control-fix-flex"
-          @input="setPathText"
-          :value="getPathText"
-          :list="suggestPath"
-          :max-suggestions="0"
-          :placeholder="$t('pathBar.placeholder')"
-      >
-        <!-- Suggestion item template-->
-        <div slot="suggestion-item" slot-scope="{ suggestion, query }">
-          <div class="suggestion-line" :title="suggestion">
-            <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
-          </div>
         </div>
-      </VueSimpleSuggest>
 
+        <b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
+            <div id="pathTree"></div>
+        </b-modal>
     </div>
-
-    <b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
-      <div id="pathTree"></div>
-    </b-modal>
-  </div>
 </template>
 
 <script>
@@ -48,198 +49,153 @@ import VueSimpleSuggest from 'vue-simple-suggest'
 import 'vue-simple-suggest/dist/styles.css' // Optional CSS
 
 export default {
-  name: "PathTree",
-  components: {
-    VueSimpleSuggest
-  },
-  data() {
-    return {
-      mimeTree: null,
-      pathItems: [],
-      tmpPath: ""
-
-    }
-  },
-  computed: {
-    ...mapGetters(["getPathText"])
-  },
-  mounted() {
-    this.$store.subscribe((mutation) => {
-      // Wait until indices are loaded to get the root paths
-      if (mutation.type === "setIndices") {
-        let pathTree = new InspireTree({
-          data: (node, resolve, reject) => {
-            return this.getNextDepth(node);
-          },
-          sort: "text"
-        });
-
-        this.$store.state.indices.forEach(idx => {
-          pathTree.addNode({
-            id: "/" + idx.id,
-            values: ["/" + idx.id],
-            text: `/[${idx.name}]`,
-            index: idx.id,
-            depth: 0,
-            children: true
-          })
-        });
-
-        new InspireTreeDOM(pathTree, {
-          target: "#pathTree"
-        });
-
-        pathTree.on("node.click", this.handleTreeClick);
-        pathTree.expand();
-      }
-    });
-  },
-  methods: {
-    ...mapMutations(["setPathText"]),
-    getSuggestionWithoutQueryPrefix(suggestion, query) {
-      return suggestion.slice(query.length)
+    name: "PathTree",
+    components: {
+        VueSimpleSuggest
     },
-    async getPathChoices() {
-      return new Promise(getPaths => {
-        const q = {
-          suggest: {
-            path: {
-              prefix: this.getPathText,
-              completion: {
-                field: "suggest-path",
-                skip_duplicates: true,
-                size: 10000
-              }
-            }
-          }
-        };
+    data() {
+        return {
+            mimeTree: null,
+            pathItems: [],
+            tmpPath: ""
 
-        Sist2Api.esQuery(q)
-            .then(resp => getPaths(resp["suggest"]["path"][0]["options"].map(opt => opt["_source"]["path"])));
-      })
-    },
-    async suggestPath(term) {
-      if (!this.$store.state.optSuggestPath) {
-        return []
-      }
-
-      term = term.toLowerCase();
-
-      const choices = await this.getPathChoices();
-
-      let matches = [];
-      for (let i = 0; i < choices.length; i++) {
-        if (~choices[i].toLowerCase().indexOf(term)) {
-          matches.push(choices[i]);
         }
-      }
-      return matches.sort((a, b) => a.length - b.length);
     },
-    getNextDepth(node) {
-      const q = {
-        query: {
-          bool: {
-            filter: [
-              {term: {index: node.index}},
-              {range: {_depth: {gte: node.depth + 1, lte: node.depth + 3}}},
-            ]
-          }
-        },
-        aggs: {
-          paths: {
-            terms: {
-              field: "path",
-              size: 10000
+    computed: {
+        ...mapGetters(["getPathText"])
+    },
+    mounted() {
+        this.$store.subscribe((mutation) => {
+            // Wait until indices are loaded to get the root paths
+            if (mutation.type === "setIndices") {
+                let pathTree = new InspireTree({
+                    data: (node, resolve, reject) => {
+                        return this.getNextDepth(node);
+                    },
+                    sort: "text"
+                });
+
+                this.$store.state.indices.forEach(idx => {
+                    pathTree.addNode({
+                        id: "/" + idx.id,
+                        values: ["/" + idx.id],
+                        text: `/[${idx.name}]`,
+                        index: idx.id,
+                        depth: 0,
+                        children: true
+                    })
+                });
+
+                new InspireTreeDOM(pathTree, {
+                    target: "#pathTree"
+                });
+
+                pathTree.on("node.click", this.handleTreeClick);
+                pathTree.expand();
             }
-          }
+        });
+    },
+    methods: {
+        ...mapMutations(["setPathText"]),
+        getSuggestionWithoutQueryPrefix(suggestion, query) {
+            return suggestion.slice(query.length)
         },
-        size: 0
-      };
+        async getPathChoices() {
+            return new Promise(getPaths => {
+                Sist2Api.getPathSuggestions(this.getPathText).then(getPaths);
+            });
+        },
+        async suggestPath(term) {
+            if (!this.$store.state.optSuggestPath) {
+                return []
+            }
 
-      if (node.depth > 0) {
-        q.query.bool.must = {
-          prefix: {
-            path: node.id,
-          }
-        };
-      }
+            term = term.toLowerCase();
 
-      return Sist2Api.esQuery(q).then(resp => {
-        const buckets = resp["aggregations"]["paths"]["buckets"];
-        if (!buckets) {
-          return false;
-        }
+            const choices = await this.getPathChoices();
 
-        const paths = [];
+            let matches = [];
+            for (let i = 0; i < choices.length; i++) {
+                if (~choices[i].toLowerCase().indexOf(term)) {
+                    matches.push(choices[i]);
+                }
+            }
+            return matches.sort((a, b) => a.length - b.length);
+        },
+        getNextDepth(node) {
+            return Sist2Api
+                .searchPaths(node.index, node.depth + 1, node.depth + 3, node.depth > 0 ? node.id : null)
+                .then(buckets => {
+                    const paths = [];
 
-        return buckets
-            .filter(bucket => bucket.key.length > node.id.length || node.id.startsWith("/"))
-            .sort((a, b) => a.key > b.key)
-            .map(bucket => {
+                    return buckets
+                        .filter(bucket => bucket.path.length > node.id.length || node.id.startsWith("/"))
+                        .sort((a, b) => a.path > b.path ? 1 : -1)
+                        .map(bucket => {
+                            if (paths.some(n => bucket.path.startsWith(n))) {
+                                return null;
+                            }
 
-              if (paths.some(n => bucket.key.startsWith(n))) {
-                return null;
-              }
+                            const name = node.id.startsWith("/") ? bucket.path : bucket.path.slice(node.id.length + 1);
 
-              const name = node.id.startsWith("/") ? bucket.key : bucket.key.slice(node.id.length + 1);
+                            paths.push(bucket.path);
 
-              paths.push(bucket.key);
+                            return {
+                                id: bucket.path,
+                                text: `${name}/ (${bucket.count})`,
+                                depth: node.depth + 1,
+                                index: node.index,
+                                values: [bucket.path],
+                                children: true,
+                            }
+                        })
+                        .filter(bucket => bucket !== null);
+                });
+        },
+        handleTreeClick(e, node, handler) {
+            if (node.depth !== 0) {
+                this.setPathText(node.id);
+                this.$refs['path-modal'].hide()
 
-              return {
-                id: bucket.key,
-                text: `${name}/ (${bucket.doc_count})`,
-                depth: node.depth + 1,
-                index: node.index,
-                values: [bucket.key],
-                children: true,
-              }
-            }).filter(x => x !== null)
-      });
+                this.$emit("search");
+            }
+
+            handler();
+        },
     },
-    handleTreeClick(e, node, handler) {
-      if (node.depth !== 0) {
-        this.setPathText(node.id);
-        this.$refs['path-modal'].hide()
-
-        this.$emit("search");
-      }
-
-      handler();
-    },
-  },
 }
 </script>
 
 <style scoped>
 #mimeTree {
-  max-height: 350px;
-  overflow: auto;
+    max-height: 350px;
+    overflow: auto;
 }
 
 .form-control-fix-flex {
-  flex: 1 1 auto;
-  width: 1%;
-  min-width: 0;
-  margin-bottom: 0;
+    flex: 1 1 auto;
+    width: 1%;
+    min-width: 0;
+    margin-bottom: 0;
 }
 
 .suggestion-line {
-  max-width: 100%;
-  white-space: nowrap;
-  overflow: hidden;
-  text-overflow: ellipsis;
-  line-height: 1.1;
+    max-width: 100%;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    line-height: 1.1;
 }
 
 </style>
 
 <style>
 .suggestions {
-  max-height: 250px;
-  overflow-y: auto;
+    max-height: 250px;
+    overflow-y: auto;
 }
 
 .theme-black .suggestions {
-  color: black
+    color: black
 }
 </style>

sist2-vue/src/components/ResultsCard.vue

@@ -1,40 +1,46 @@
 <template>
-  <b-card v-if="lastResultsLoaded" id="results">
-    <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
+    <b-card v-if="lastResultsLoaded" id="results">
+        <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
 
-    <div style="float: right">
-      <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
-          $t("details")
-        }}
-      </b-button>
+        <div style="float: right">
+            <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
+                $t("details")
+                }}
+            </b-button>
 
-      <template v-if="hitCount !== 0">
-        <SortSelect class="ml-2"></SortSelect>
+            <template v-if="hitCount !== 0">
+                <SortSelect class="ml-2"></SortSelect>
 
-        <DisplayModeToggle class="ml-2"></DisplayModeToggle>
-      </template>
-    </div>
+                <DisplayModeToggle class="ml-2"></DisplayModeToggle>
+            </template>
+        </div>
 
-    <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
-      <b-card>
-        <b-table :items="tableItems" small borderless bordered thead-class="hidden" class="mb-0"></b-table>
+        <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
+            <b-card>
+                <b-table :items="tableItems" small borderless thead-class="hidden" class="mb-0"></b-table>
 
-        <br/>
-        <h4>
-          {{$t("mimeTypes")}}
-          <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick"><ClipboardIcon/></b-button>
-        </h4>
-        <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
-        <b-table
-            v-else
-            sort-by="doc_count"
-            :sort-desc="true"
-            thead-class="hidden"
-            :items="$store.state.uiDetailsMimeAgg" small bordered class="mb-0"
-        ></b-table>
-      </b-card>
-    </b-collapse>
-  </b-card>
+                <template v-if="!$store.state.uiSqliteMode">
+
+                    <br/>
+                    <h4>
+                        {{ $t("mimeTypes") }}
+                        <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick">
+                            <ClipboardIcon/>
+                        </b-button>
+                    </h4>
+                    <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
+                    <b-table
+                            v-else
+                            sort-by="doc_count"
+                            :sort-desc="true"
+                            thead-class="hidden"
+                            bordered
+                            :items="$store.state.uiDetailsMimeAgg" small class="mb-0"
+                    ></b-table>
+                </template>
+            </b-card>
+        </b-collapse>
+    </b-card>
 </template>
 
 <script lang="ts">
@@ -44,91 +50,96 @@ import {humanFileSize} from "@/util";
 import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
 import SortSelect from "@/components/SortSelect.vue";
 import Preloader from "@/components/Preloader.vue";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
 
 export default Vue.extend({
-  name: "ResultsCard",
-  components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
-  created() {
+    name: "ResultsCard",
+    components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
+    created() {
 
-  },
-  computed: {
-    lastResultsLoaded() {
-      return this.$store.state.lastQueryResults != null;
     },
-    hitCount() {
-      return (this.$store.state.lastQueryResults as EsResult).aggregations.total_count.value;
+    computed: {
+        lastResultsLoaded() {
+            return this.$store.state.lastQueryResults != null;
+        },
+        hitCount() {
+            return (this.$store.state.firstQueryResults as EsResult).aggregations.total_count.value;
+        },
+        tableItems() {
+            const items = [];
+
+            if (!this.$store.state.uiSqliteMode) {
+                items.push({key: this.$t("queryTime"), value: this.took()});
+            }
+            items.push({key: this.$t("totalSize"), value: this.totalSize()});
+
+            return items;
+        }
     },
-    tableItems() {
-      const items = [];
+    methods: {
+        took() {
+            return (this.$store.state.lastQueryResults as EsResult).took + "ms";
+        },
+        totalSize() {
+            return humanFileSize((this.$store.state.firstQueryResults as EsResult).aggregations.total_size.value);
+        },
+        onToggle() {
+            const show = !document.getElementById("collapse-1").classList.contains("show");
+            this.$store.commit("setUiShowDetails", show);
 
+            if (this.$store.state.uiSqliteMode) {
+                return;
+            }
 
-      items.push({key: this.$t("queryTime"), value: this.took()});
-      items.push({key: this.$t("totalSize"), value: this.totalSize()});
+            if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
+                // Mime aggs are not updated automatically, update now
+                this.forceUpdateMimeAgg();
+            }
+        },
+        onCopyClick() {
+            let tsvString = "";
+            this.$store.state.uiDetailsMimeAgg.slice().sort((a, b) => b["doc_count"] - a["doc_count"]).forEach(row => {
+                tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
+            });
 
-      return items;
-    }
-  },
-  methods: {
-    took() {
-      return (this.$store.state.lastQueryResults as EsResult).took + "ms";
+            navigator.clipboard.writeText(tsvString);
+
+            this.$bvToast.toast(
+                this.$t("toast.copiedToClipboard"),
+                {
+                    title: null,
+                    noAutoHide: false,
+                    toaster: "b-toaster-bottom-right",
+                    headerClass: "hidden",
+                    bodyClass: "toast-body-info",
+                });
+        },
+        forceUpdateMimeAgg() {
+            const query = Sist2Query.searchQuery();
+            Sist2Api.getMimeTypes(query).then(({buckets}) => {
+                this.$store.commit("setUiDetailsMimeAgg", buckets);
+            });
+        }
     },
-    totalSize() {
-      return humanFileSize((this.$store.state.lastQueryResults as EsResult).aggregations.total_size.value);
-    },
-    onToggle() {
-      const show = !document.getElementById("collapse-1").classList.contains("show");
-      this.$store.commit("setUiShowDetails", show);
-
-      if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
-        // Mime aggs are not updated automatically, update now
-        this.forceUpdateMimeAgg();
-      }
-    },
-    onCopyClick() {
-      let tsvString = "";
-      this.$store.state.uiDetailsMimeAgg.slice().sort((a,b) => b["doc_count"] - a["doc_count"]).forEach(row => {
-        tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
-      });
-
-      navigator.clipboard.writeText(tsvString);
-
-      this.$bvToast.toast(
-          this.$t("toast.copiedToClipboard"),
-          {
-            title: null,
-            noAutoHide: false,
-            toaster: "b-toaster-bottom-right",
-            headerClass: "hidden",
-            bodyClass: "toast-body-info",
-          });
-    },
-    forceUpdateMimeAgg() {
-      const query = Sist2Query.searchQuery();
-      Sist2Api.getMimeTypes(query).then(({buckets}) => {
-        this.$store.commit("setUiDetailsMimeAgg", buckets);
-      });
-    }
-  },
 });
 
 </script>
 
 <style>
 #results {
-  margin-top: 1em;
+    margin-top: 1em;
 
-  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-  border-radius: 0;
-  border: none;
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
 }
 
 #results .card-body {
-  padding: 0.7em 1.25em;
+    padding: 0.7em 1.25em;
 }
 
 .hidden {
-  display: none;
+    display: none;
 }
 </style>

sist2-vue/src/components/SearchBar.vue

@@ -6,7 +6,7 @@
                     @input="setSearchText($event)"></b-form-input>
 
       <template #prepend>
-        <b-input-group-text>
+        <b-input-group-text v-if="!$store.state.uiSqliteMode">
           <b-form-checkbox :checked="fuzzy" title="Toggle fuzzy searching" @change="setFuzzy($event)">
             {{ $t("searchBar.fuzzy") }}
           </b-form-checkbox>

sist2-vue/src/components/TagContainer.vue

@@ -1,76 +1,78 @@
 <template>
-  <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
+    <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
 
-    <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static lazy>
-      <b-row>
-        <b-col style="flex-grow: 2" sm>
-          <VueSimpleSuggest
-              ref="suggest"
-              :value="tagText"
-              @select="setTagText($event)"
-              @input="setTagText($event)"
-              class="form-control-fix-flex"
-              style="margin-top: 17px"
-              :list="suggestTag"
-              :max-suggestions="0"
-              :placeholder="$t('saveTagPlaceholder')"
-          >
-            <!-- Suggestion item template-->
-            <div slot="suggestion-item" slot-scope="{ suggestion, query}"
-            >
-              <div class="suggestion-line">
+        <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static
+                 lazy>
+            <b-row>
+                <b-col style="flex-grow: 2" sm>
+                    <VueSimpleSuggest
+                            ref="suggest"
+                            :value="tagText"
+                            @select="setTagText($event)"
+                            @input="setTagText($event)"
+                            class="form-control-fix-flex"
+                            style="margin-top: 17px"
+                            :list="suggestTag"
+                            :max-suggestions="0"
+                            :placeholder="$t('saveTagPlaceholder')"
+                    >
+                        <!-- Suggestion item template-->
+                        <div slot="suggestion-item" slot-scope="{ suggestion, query}"
+                        >
+                            <div class="suggestion-line">
                 <span
-                    class="badge badge-suggestion"
-                    :style="{background: getBg(suggestion), color: getFg(suggestion)}"
+                        class="badge badge-suggestion"
+                        :style="{background: getBg(suggestion), color: getFg(suggestion)}"
                 >
                 <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
                 </span>
-              </div>
-            </div>
-          </VueSimpleSuggest>
-        </b-col>
-        <b-col class="mt-4">
-          <TwitterColorPicker v-model="color" triangle="hide" :width="252" class="mr-auto ml-auto"></TwitterColorPicker>
-        </b-col>
-      </b-row>
+                            </div>
+                        </div>
+                    </VueSimpleSuggest>
+                </b-col>
+                <b-col class="mt-4">
+                    <TwitterColorPicker v-model="color" triangle="hide" :width="252"
+                                        class="mr-auto ml-auto"></TwitterColorPicker>
+                </b-col>
+            </b-row>
 
-      <b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
-      </b-button>
-    </b-modal>
+            <b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
+            </b-button>
+        </b-modal>
 
 
-    <template v-for="tag in hit._tags">
-      <!-- User tag-->
-      <div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
+        <template v-for="tag in hit._tags">
+            <!-- User tag-->
+            <div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
         <span
-            :id="hit._id+tag.rawText"
-            :title="tag.text"
-            tabindex="-1"
-            class="badge pointer"
-            :style="badgeStyle(tag)" :class="badgeClass(tag)"
-            @click.right="onTagRightClick(tag, $event)"
+                :id="hit._id+tag.rawText"
+                :title="tag.text"
+                tabindex="-1"
+                class="badge pointer"
+                :style="badgeStyle(tag)" :class="badgeClass(tag)"
+                @click.right="onTagRightClick(tag, $event)"
         >{{ tag.text.split(".").pop() }}</span>
 
-        <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
-          <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{ $t("deleteTag") }}</b-button>
-        </b-popover>
-      </div>
+                <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
+                    <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{ $t("deleteTag") }}</b-button>
+                </b-popover>
+            </div>
 
-      <span
-          v-else :key="tag.text"
-          class="badge"
-          :style="badgeStyle(tag)" :class="badgeClass(tag)"
-      >{{ tag.text.split(".").pop() }}</span>
-    </template>
+            <span
+                    v-else :key="tag.text"
+                    class="badge"
+                    :style="badgeStyle(tag)" :class="badgeClass(tag)"
+            >{{ tag.text.split(".").pop() }}</span>
+        </template>
 
-    <!-- Add button -->
-    <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{$t("addTag")}}</small>
+        <!-- Add button -->
+        <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{ $t("addTag") }}</small>
 
-    <!-- Size tag-->
-    <small v-else class="text-muted badge-size" style="padding-left: 2px">{{
-        humanFileSize(hit._source.size)
-      }}</small>
-  </div>
+        <!-- Size tag-->
+        <small v-else class="text-muted badge-size" style="padding-left: 2px">{{
+            humanFileSize(hit._source.size)
+            }}</small>
+    </div>
 </template>
 
 <script>
@@ -81,170 +83,136 @@ import Sist2Api from "@/Sist2Api";
 import VueSimpleSuggest from 'vue-simple-suggest'
 
 export default Vue.extend({
-  components: {
-    "TwitterColorPicker": Twitter,
-    VueSimpleSuggest
-  },
-  props: ["hit"],
-  data() {
-    return {
-      showAddButton: false,
-      showModal: false,
-      tagText: null,
-      color: {
-        hex: "#e0e0e0",
-      },
-    }
-  },
-  computed: {
-    tagHover() {
-      return this.$store.getters["uiTagHover"];
-    }
-  },
-  methods: {
-    humanFileSize: humanFileSize,
-    getSuggestionWithoutQueryPrefix(suggestion, query) {
-      return suggestion.id.slice(query.length, -8)
+    components: {
+        "TwitterColorPicker": Twitter,
+        VueSimpleSuggest
     },
-    getBg(suggestion) {
-      return suggestion.id.slice(-7);
-    },
-    getFg(suggestion) {
-      return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
-    },
-    setTagText(value) {
-      this.$refs.suggest.clearSuggestions();
-
-      if (typeof value === "string") {
-        this.tagText = {
-          id: value,
-          title: value
-        };
-        return;
-      }
-
-      this.color = {
-        hex: "#" + value.id.split("#")[1]
-      }
-
-      this.tagText = value;
-    },
-    badgeClass(tag) {
-      return `badge-${tag.style}`;
-    },
-    badgeStyle(tag) {
-      return {
-        background: tag.bg,
-        color: tag.fg,
-      };
-    },
-    onTagHover(tag) {
-      if (tag.userTag) {
-        this.$store.commit("setUiTagHover", tag);
-      }
-    },
-    onTagLeave() {
-      this.$store.commit("setUiTagHover", null);
-    },
-    onTagDeleteClick(tag, e) {
-      this.hit._tags = this.hit._tags.filter(t => t !== tag);
-
-      Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
-        //toast
-        this.$store.commit("busUpdateWallItems");
-        this.$store.commit("busUpdateTags");
-      });
-    },
-    tagAdd() {
-      this.showModal = true;
-    },
-    saveTag() {
-      if (this.tagText.id.includes("#")) {
-        this.$bvToast.toast(
-            this.$t("toast.invalidTag"),
-            {
-              title: this.$t("toast.invalidTagTitle"),
-              noAutoHide: true,
-              toaster: "b-toaster-bottom-right",
-              headerClass: "toast-header-error",
-              bodyClass: "toast-body-error",
-            });
-        return;
-      }
-
-      let tag = this.tagText.id + this.color.hex.replace("#", ".#");
-      const userTags = this.hit._tags.filter(t => t.userTag);
-
-      if (userTags.find(t => t.rawText === tag) != null) {
-        this.$bvToast.toast(
-            this.$t("toast.dupeTag"),
-            {
-              title: this.$t("toast.dupeTagTitle"),
-              noAutoHide: true,
-              toaster: "b-toaster-bottom-right",
-              headerClass: "toast-header-error",
-              bodyClass: "toast-body-error",
-            });
-        return;
-      }
-
-      this.hit._tags.push(Sist2Api.createUserTag(tag));
-
-      Sist2Api.saveTag(tag, this.hit).then(() => {
-        this.tagText = null;
-        this.showModal = false;
-        this.$store.commit("busUpdateWallItems");
-        this.$store.commit("busUpdateTags");
-        // TODO: toast
-      });
-    },
-    async suggestTag(term) {
-      term = term.toLowerCase();
-
-      const choices = await this.getTagChoices(term);
-
-      let matches = [];
-      for (let i = 0; i < choices.length; i++) {
-        if (~choices[i].toLowerCase().indexOf(term)) {
-          matches.push(choices[i]);
-        }
-      }
-
-      return matches.sort().map(match => {
+    props: ["hit"],
+    data() {
         return {
-          title: match.split(".").slice(0, -1).join("."),
-          id: match
+            showAddButton: false,
+            showModal: false,
+            tagText: null,
+            color: {
+                hex: "#e0e0e0",
+            },
         }
-      });
     },
-    getTagChoices(prefix) {
-      return new Promise(getPaths => {
-        Sist2Api.esQuery({
-          suggest: {
-            tag: {
-              prefix: prefix,
-              completion: {
-                field: "suggest-tag",
-                skip_duplicates: true,
-                size: 10000
-              }
+    methods: {
+        humanFileSize: humanFileSize,
+        getSuggestionWithoutQueryPrefix(suggestion, query) {
+            return suggestion.id.slice(query.length, -8)
+        },
+        getBg(suggestion) {
+            return suggestion.id.slice(-7);
+        },
+        getFg(suggestion) {
+            return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
+        },
+        setTagText(value) {
+            this.$refs.suggest.clearSuggestions();
+
+            if (typeof value === "string") {
+                this.tagText = {
+                    id: value,
+                    title: value
+                };
+                return;
             }
-          }
-        }).then(resp => {
-          const result = [];
-          resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
-            tags.forEach(tag => {
-              const t = tag.slice(0, -8);
-              if (!result.find(x => x.slice(0, -8) === t)) {
-                result.push(tag);
-              }
+
+            this.color = {
+                hex: "#" + value.id.split("#")[1]
+            }
+
+            this.tagText = value;
+        },
+        badgeClass(tag) {
+            return `badge-${tag.style}`;
+        },
+        badgeStyle(tag) {
+            return {
+                background: tag.bg,
+                color: tag.fg,
+            };
+        },
+        onTagDeleteClick(tag, e) {
+            this.hit._tags = this.hit._tags.filter(t => t !== tag);
+
+            Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
+                //toast
+                this.$store.commit("busUpdateWallItems");
+                this.$store.commit("busUpdateTags");
             });
-          });
-          getPaths(result);
-        });
-      });
-    }
-  },
+        },
+        tagAdd() {
+            this.showModal = true;
+        },
+        saveTag() {
+            if (this.tagText.id.includes("#")) {
+                this.$bvToast.toast(
+                    this.$t("toast.invalidTag"),
+                    {
+                        title: this.$t("toast.invalidTagTitle"),
+                        noAutoHide: true,
+                        toaster: "b-toaster-bottom-right",
+                        headerClass: "toast-header-error",
+                        bodyClass: "toast-body-error",
+                    });
+                return;
+            }
+
+            let tag = this.tagText.id + this.color.hex.replace("#", ".#");
+            const userTags = this.hit._tags.filter(t => t.userTag);
+
+            if (userTags.find(t => t.rawText === tag) != null) {
+                this.$bvToast.toast(
+                    this.$t("toast.dupeTag"),
+                    {
+                        title: this.$t("toast.dupeTagTitle"),
+                        noAutoHide: true,
+                        toaster: "b-toaster-bottom-right",
+                        headerClass: "toast-header-error",
+                        bodyClass: "toast-body-error",
+                    });
+                return;
+            }
+
+            this.hit._tags.push(Sist2Api.createUserTag(tag));
+
+            Sist2Api.saveTag(tag, this.hit).then(() => {
+                this.tagText = null;
+                this.showModal = false;
+                this.$store.commit("busUpdateWallItems");
+                this.$store.commit("busUpdateTags");
+                // TODO: toast
+            });
+        },
+        async suggestTag(term) {
+            term = term.toLowerCase();
+
+            const choices = await this.getTagChoices(term);
+
+            let matches = [];
+            for (let i = 0; i < choices.length; i++) {
+                if (~choices[i].toLowerCase().indexOf(term)) {
+                    matches.push(choices[i]);
+                }
+            }
+
+            return matches.sort().map(match => {
+                return {
+                    title: match.split(".").slice(0, -1).join("."),
+                    id: match
+                }
+            });
+        },
+        getTagChoices(prefix) {
+            return new Promise(getPaths => {
+                Sist2Api.getTagSuggestions(prefix)
+                    .then(paths => getPaths(paths))
+            });
+        }
+    },
 });
 </script>
 
@@ -252,87 +220,87 @@ export default Vue.extend({
 
 
 .badge-video {
-  color: #FFFFFF;
-  background-color: #F27761;
+    color: #FFFFFF;
+    background-color: #F27761;
 }
 
 .badge-image {
-  color: #FFFFFF;
-  background-color: #AA99C9;
+    color: #FFFFFF;
+    background-color: #AA99C9;
 }
 
 .badge-audio {
-  color: #FFFFFF;
-  background-color: #00ADEF;
+    color: #FFFFFF;
+    background-color: #00ADEF;
 }
 
 .badge-user {
-  color: #212529;
-  background-color: #e0e0e0;
+    color: #212529;
+    background-color: #e0e0e0;
 }
 
 .badge-user:hover, .add-tag-button:hover {
-  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
+    box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
 }
 
 .badge-text {
-  color: #FFFFFF;
-  background-color: #FAAB3C;
+    color: #FFFFFF;
+    background-color: #FAAB3C;
 }
 
 .badge {
-  margin-right: 3px;
+    margin-right: 3px;
 }
 
 .badge-delete {
-  margin-right: -2px;
-  margin-left: 2px;
-  margin-top: -1px;
-  font-family: monospace;
-  font-size: 90%;
-  background: rgba(0, 0, 0, 0.2);
-  padding: 0.1em 0.4em;
-  color: white;
-  cursor: pointer;
+    margin-right: -2px;
+    margin-left: 2px;
+    margin-top: -1px;
+    font-family: monospace;
+    font-size: 90%;
+    background: rgba(0, 0, 0, 0.2);
+    padding: 0.1em 0.4em;
+    color: white;
+    cursor: pointer;
 }
 
 .badge-size {
-  width: 50px;
-  display: inline-block;
+    width: 50px;
+    display: inline-block;
 }
 
 .add-tag-button {
-  cursor: pointer;
-  color: #212529;
-  background-color: #e0e0e0;
-  width: 50px;
+    cursor: pointer;
+    color: #212529;
+    background-color: #e0e0e0;
+    width: 50px;
 }
 
 .badge {
-  user-select: none;
+    user-select: none;
 }
 
 .badge-suggestion {
-  font-size: 90%;
-  font-weight: normal;
+    font-size: 90%;
+    font-weight: normal;
 }
 </style>
 
 <style>
 .vc-twitter-body {
-  padding: 0 !important;
+    padding: 0 !important;
 }
 
 .vc-twitter {
-  box-shadow: none !important;
-  background: none !important;
+    box-shadow: none !important;
+    background: none !important;
 }
 
 .tooltip {
-  user-select: none;
+    user-select: none;
 }
 
 .toast {
-  border: none;
+    border: none;
 }
 </style>

sist2-vue/src/components/TagPicker.vue

@@ -1,13 +1,13 @@
 <template>
-  <div>
-    <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
-      <b-form-input :value="filter"
-                    :placeholder="$t('tagFilter')"
-                    @input="onFilter($event)"></b-form-input>
-    </b-input-group>
+    <div>
+        <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
+            <b-form-input :value="filter"
+                          :placeholder="$t('tagFilter')"
+                          @input="onFilter($event)"></b-form-input>
+        </b-input-group>
 
-    <div id="tagTree"></div>
-  </div>
+        <div id="tagTree"></div>
+    </div>
 </template>
 
 <script>
@@ -19,191 +19,195 @@ import {getSelectedTreeNodes} from "@/util";
 import Sist2Api from "@/Sist2Api";
 
 function resetState(node) {
-  node._tree.defaultState.forEach(function (val, prop) {
-    node.state(prop, val);
-  });
+    node._tree.defaultState.forEach(function (val, prop) {
+        node.state(prop, val);
+    });
 
-  return node;
+    return node;
 }
 
 function baseStateChange(prop, value, verb, node, deep) {
-  if (node.state(prop) !== value) {
-    node._tree.batch();
+    if (node.state(prop) !== value) {
+        node._tree.batch();
 
-    if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
-      resetState(node);
+        if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
+            resetState(node);
+        }
+
+        node.state(prop, value);
+
+        node._tree.emit('node.' + verb, node, false);
+
+        if (deep && node.hasChildren()) {
+            node.children.recurseDown(function (child) {
+                baseStateChange(prop, value, verb, child);
+            });
+        }
+
+        node.markDirty();
+        node._tree.end();
     }
 
-    node.state(prop, value);
-
-    node._tree.emit('node.' + verb, node, false);
-
-    if (deep && node.hasChildren()) {
-      node.children.recurseDown(function (child) {
-        baseStateChange(prop, value, verb, child);
-      });
-    }
-
-    node.markDirty();
-    node._tree.end();
-  }
-
-  return node;
+    return node;
 }
 
 function addTag(map, tag, id, count) {
-  const tags = tag.split(".");
+    const tags = tag.split(".");
 
-  const child = {
-    id: id,
-    count: count,
-    text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
-    name: tags[0],
-    children: [],
-    // Overwrite base functions
-    blur: function () {
-      // noop
-    },
-    select: function () {
-      this.state("selected", true);
-      return this.check()
-    },
-    deselect: function () {
-      this.state("selected", false);
-      return this.uncheck()
-    },
-    uncheck: function () {
-      baseStateChange('checked', false, 'unchecked', this, false);
-      this.state('indeterminate', false);
+    const child = {
+        id: id,
+        count: count,
+        text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
+        name: tags[0],
+        children: [],
+        // Overwrite base functions
+        blur: function () {
+            // noop
+        },
+        select: function () {
+            this.state("selected", true);
+            return this.check()
+        },
+        deselect: function () {
+            this.state("selected", false);
+            return this.uncheck()
+        },
+        uncheck: function () {
+            baseStateChange('checked', false, 'unchecked', this, false);
+            this.state('indeterminate', false);
 
-      if (this.hasParent()) {
-        this.getParent().refreshIndeterminateState();
-      }
+            if (this.hasParent()) {
+                this.getParent().refreshIndeterminateState();
+            }
 
-      this._tree.end();
-      return this;
-    },
-    check: function () {
-      baseStateChange('checked', true, 'checked', this, false);
+            this._tree.end();
+            return this;
+        },
+        check: function () {
+            baseStateChange('checked', true, 'checked', this, false);
 
-      if (this.hasParent()) {
-        this.getParent().refreshIndeterminateState();
-      }
+            if (this.hasParent()) {
+                this.getParent().refreshIndeterminateState();
+            }
 
-      this._tree.end();
-      return this;
+            this._tree.end();
+            return this;
+        }
+    };
+
+    let found = false;
+    map.forEach(node => {
+        if (node.name === child.name) {
+            found = true;
+            if (tags.length !== 1) {
+                addTag(node.children, tags.slice(1).join("."), id, count);
+            } else {
+                // Same name, different color
+                console.error("FIXME: Duplicate tag?")
+                console.trace(node)
+            }
+        }
+    });
+    if (!found) {
+        if (tags.length !== 1) {
+            addTag(child.children, tags.slice(1).join("."), id, count);
+            map.push(child);
+        } else {
+            map.push(child);
+        }
     }
-  };
-
-  let found = false;
-  map.forEach(node => {
-    if (node.name === child.name) {
-      found = true;
-      if (tags.length !== 1) {
-        addTag(node.children, tags.slice(1).join("."), id, count);
-      } else {
-        // Same name, different color
-        console.error("FIXME: Duplicate tag?")
-        console.trace(node)
-      }
-    }
-  });
-  if (!found) {
-    if (tags.length !== 1) {
-      addTag(child.children, tags.slice(1).join("."), id, count);
-      map.push(child);
-    } else {
-      map.push(child);
-    }
-  }
 }
 
 export default {
-  name: "TagPicker",
-  props: ["showSearchBar"],
-  data() {
-    return {
-      tagTree: null,
-      loadedFromArgs: false,
-      filter: ""
-    }
-  },
-  mounted() {
-    this.$store.subscribe((mutation) => {
-      if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
-        this.initializeTree();
-        this.updateTree();
-      } else if (mutation.type === "busUpdateTags") {
-        window.setTimeout(this.updateTree, 2000);
-      }
-    });
-  },
-  methods: {
-    onFilter(value) {
-      this.filter = value;
-      this.tagTree.search(value);
-    },
-    initializeTree() {
-      const tagMap = [];
-      this.tagTree = new InspireTree({
-        selection: {
-          mode: "checkbox",
-          autoDeselect: false,
-        },
-        checkbox: {
-          autoCheckChildren: false,
-        },
-        data: tagMap
-      });
-      new InspireTreeDOM(this.tagTree, {
-        target: '#tagTree'
-      });
-      this.tagTree.on("node.state.changed", this.handleTreeClick);
-    },
-    updateTree() {
-      // TODO: remember which tags are selected and restore?
-      const tagMap = [];
-      Sist2Api.getTags().then(tags => {
-        tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
-        this.tagTree.removeAll();
-        this.tagTree.addNodes(tagMap);
-
-        if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
-          this.$store.state._onLoadSelectedTags.forEach(mime => {
-            this.tagTree.node(mime).select();
-            this.loadedFromArgs = true;
-          });
+    name: "TagPicker",
+    props: ["showSearchBar"],
+    data() {
+        return {
+            tagTree: null,
+            loadedFromArgs: false,
+            filter: ""
         }
-      });
     },
-    handleTreeClick(node, e) {
-      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
-          || e === "matched" || e === "hidden") {
-        return;
-      }
+    mounted() {
+        this.$store.subscribe((mutation) => {
+            if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
+                this.initializeTree();
+                this.updateTree();
+            } else if (mutation.type === "busUpdateTags") {
+                if (this.$store.state.uiSqliteMode) {
+                    this.updateTree();
+                } else {
+                    window.setTimeout(this.updateTree, 2000);
+                }
+            }
+        });
+    },
+    methods: {
+        onFilter(value) {
+            this.filter = value;
+            this.tagTree.search(value);
+        },
+        initializeTree() {
+            const tagMap = [];
+            this.tagTree = new InspireTree({
+                selection: {
+                    mode: "checkbox",
+                    autoDeselect: false,
+                },
+                checkbox: {
+                    autoCheckChildren: false,
+                },
+                data: tagMap
+            });
+            new InspireTreeDOM(this.tagTree, {
+                target: '#tagTree'
+            });
+            this.tagTree.on("node.state.changed", this.handleTreeClick);
+        },
+        updateTree() {
+            // TODO: remember which tags are selected and restore?
+            const tagMap = [];
+            Sist2Api.getTags().then(tags => {
+                tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
+                this.tagTree.removeAll();
+                this.tagTree.addNodes(tagMap);
 
-      this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
-    },
-  }
+                if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
+                    this.$store.state._onLoadSelectedTags.forEach(mime => {
+                        this.tagTree.node(mime).select();
+                        this.loadedFromArgs = true;
+                    });
+                }
+            });
+        },
+        handleTreeClick(node, e) {
+            if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
+                || e === "matched" || e === "hidden") {
+                return;
+            }
+
+            this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
+        },
+    }
 }
 </script>
 
 <style scoped>
 #mimeTree {
-  max-height: 350px;
-  overflow: auto;
+    max-height: 350px;
+    overflow: auto;
 }
 </style>
 <style>
 .inspire-tree .focused > .wholerow {
-  border: none;
+    border: none;
 }
 
 #tag-picker-filter-bar {
-  padding: 10px 4px 4px;
+    padding: 10px 4px 4px;
 }
 
 .theme-black .inspire-tree .matched > .wholerow {
-  background: rgba(251, 191, 41, 0.25);
+    background: rgba(251, 191, 41, 0.25);
 }
 </style>

sist2-vue/src/i18n/messages.ts

@@ -1,7 +1,7 @@
 export default {
     en: {
         filePage: {
-          notFound: "Not found"
+            notFound: "Not found"
         },
         searchBar: {
             simple: "Search",
@@ -57,7 +57,7 @@ export default {
             fuzzy: "Set fuzzy search by default",
             searchInPath: "Enable matching query against document path",
             suggestPath: "Enable auto-complete in path filter bar",
-            fragmentSize: "Highlight context size in characters",
+            fragmentSize: "Highlight context size",
             queryMode: "Search mode",
             displayMode: "Display",
             columns: "Column count",
@@ -92,6 +92,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -184,7 +185,7 @@ export default {
     },
     de: {
         filePage: {
-          notFound: "Nicht gefunden"
+            notFound: "Nicht gefunden"
         },
         searchBar: {
             simple: "Suche",
@@ -239,7 +240,7 @@ export default {
             fuzzy: "Aktiviere Fuzzy-Suche standardmäßig",
             searchInPath: "Abgleich der Abfrage mit dem Dokumentpfad aktivieren",
             suggestPath: "Aktiviere Auto-Vervollständigung in Pfadfilter-Leiste",
-            fragmentSize: "Kontextgröße in Zeichen hervorheben",
+            fragmentSize: "Kontextgröße",
             queryMode: "Such-Modus",
             displayMode: "Ansicht",
             columns: "Anzahl Spalten",
@@ -271,6 +272,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -413,7 +415,7 @@ export default {
             fuzzy: "Activer la recherche approximative par défaut",
             searchInPath: "Activer la recherche dans le chemin des documents",
             suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
-            fragmentSize: "Longueur du contexte de surlignage, en nombre de caractères",
+            fragmentSize: "Longueur du contexte de surlignage",
             queryMode: "Mode de recherche",
             displayMode: "Affichage",
             columns: "Nombre de colonnes",
@@ -445,6 +447,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -619,6 +622,7 @@ export default {
             en: "English",
             de: "Deutsch",
             fr: "Français",
+            pl: "Polski",
             "zh-CN": "简体中文",
         },
         displayMode: {
@@ -703,4 +707,188 @@ export default {
             selectedIndices: "选中索引",
         },
     },
+    pl: {
+        filePage: {
+            notFound: "Nie znaleziono"
+        },
+        searchBar: {
+            simple: "Szukaj",
+            advanced: "Zaawansowane szukanie",
+            fuzzy: "Również podobne"
+        },
+        addTag: "Tag",
+        deleteTag: "Usuń",
+        download: "Pobierz",
+        and: "i",
+        page: "strona",
+        pages: "stron",
+        mimeTypes: "Typy danych",
+        tags: "Tagi",
+        tagFilter: "Filtruj tagi",
+        forExample: "Na przykład:",
+        help: {
+            simpleSearch: "Proste szukanie",
+            advancedSearch: "Zaawansowane szukanie",
+            help: "Pomoc",
+            term: "<WYRAZ>",
+            and: "operator I",
+            or: "operator LUB",
+            not: "zabrania danego wyrazu",
+            quotes: "znajdzie objętą sekwencję wyrazów w podanej kolejności",
+            prefix: "znajdzie dowolny wyraz rozpoczynający się na takie litery, jeśli zastosowane na końcu wyrazu",
+            parens: "używane do grupowania wyrażeń",
+            tildeTerm: "znajdzie wyraz w podanej odległości",
+            tildePhrase: "znajdzie frazę przeplecioną podaną liczbą niepasujących wyrazów",
+            example1:
+                "Na przykład: <code>\"pieczone jajko\" +(kiełbasa | ziemniak) -frytki</code> znajdzie frazę " +
+                "<i>pieczone jajko</i> gdzie występuje też: <i>kiełbasa</i> albo <i>ziemniak</i>, ale zignoruje rezultat " +
+                "zawierający <i>frytki</i>.",
+            defaultOperator:
+                "Kiedy nie podano ani <code>+</code>, ani <code>|</code>, to domyślnym operatorem jest " +
+                "<code>+</code> (i).",
+            fuzzy:
+                "Kiedy opcja <b>Również podobne</b> jest zaznaczona, częściowo zgodne wyrazy są również znajdywane.",
+            moreInfoSimple: "Po więcej informacji sięgnij do <a target=\"_blank\" " +
+                "rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">dokumentacji Elasticsearch</a>",
+            moreInfoAdvanced: "Aby uzyskać więcej informacji o zaawansowanym szukaniu, przeczytaj <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">dokumentację Elasticsearch</a>"
+        },
+        config: "Ustawienia",
+        configDescription: "Ustawienia są zapisywane na żywo w tej przeglądarce.",
+        configReset: "Zresetuj ustawienia",
+        searchOptions: "Opcje szukania",
+        treemapOptions: "Opcje mapy",
+        mlOptions: "Opcje uczenia maszynowego",
+        displayOptions: "Opcje wyświetlania",
+        opt: {
+            lang: "Język",
+            highlight: "Zaznaczaj znalezione fragmenty",
+            fuzzy: "Ustaw szukanie również podobnych jako domyślne",
+            searchInPath: "Włącz szukanie również w ścieżce dokumentu",
+            suggestPath: "Włącz auto-uzupełnianie w filtrze ścieżek",
+            fragmentSize: "Podświetl wielkość kontekstu w znakach",
+            queryMode: "Tryb szukania",
+            displayMode: "Wyświetlanie",
+            columns: "Liczba kolumn",
+            treemapType: "Typ mapy",
+            treemapTiling: "Układanie mapy",
+            treemapColorGroupingDepth: "Jak głęboko grupować kolory mapy (na płasko)",
+            treemapColor: "Kolor mapy (kaskadowo)",
+            treemapSize: "Wielkość mapy",
+            theme: "Styl graficzny",
+            lightboxLoadOnlyCurrent: "Nie pobieraj od razu obrazów w pełnej wielkości dla sąsiednich obrazów podglądu.",
+            slideDuration: "Czas trwania jednego slajdu w pokazie slajdów",
+            resultSize: "Liczba wyników na stronę",
+            tagOrOperator: "Użyj operatora LUB przy wyborze kilku tagów",
+            hideDuplicates: "Ukryj zduplikowane wyniki (według sumy kontrolnej)",
+            hideLegacy: "Ukryj powiadomienie Elasticsearch 'legacyES'",
+            updateMimeMap: "Uaktualniaj drzewo typów mediów na żywo",
+            useDatePicker: "Używaj kalendarza do wyboru dat, zamiast suwaka",
+            vidPreviewInterval: "Czas trwania jednej klatki w podglądzie wideo (w ms)",
+            simpleLightbox: "Wyłącz animacje w podglądzie obrazów",
+            showTagPickerFilter: "Pokazuj pole filtrowania tagów",
+            featuredFields: "Wybrane pola szablonu Javascript. Będą pojawiać się przy wynikach wyszukiwania.",
+            featuredFieldsList: "Dostępne zmienne",
+            autoAnalyze: "Automatycznie analizuj tekst",
+            defaultModel: "Domyślny model",
+            mlRepositories: "Repozytoria modeli (każde w osobnej linii)"
+        },
+        queryMode: {
+            simple: "Proste",
+            advanced: "Zaawansowane",
+        },
+        lang: {
+            en: "English",
+            de: "Deutsch",
+            fr: "Français",
+            pl: "Polski",
+            "zh-CN": "简体中文",
+        },
+        displayMode: {
+            grid: "Siatka",
+            list: "Lista",
+        },
+        columns: {
+            auto: "Automatyczna"
+        },
+        treemapType: {
+            cascaded: "Kaskadowa",
+            flat: "Płaska (kompaktowa)"
+        },
+        treemapSize: {
+            small: "Mała",
+            medium: "Średnia",
+            large: "Duża",
+            xLarge: "Bardzo duża",
+            xxLarge: "Ogromna",
+            custom: "Inna",
+        },
+        treemapTiling: {
+            binary: "Binarnie",
+            squarify: "Kwadratowo",
+            slice: "Wycinek",
+            dice: "Kostka",
+            sliceDice: "Wycinek i kostka",
+        },
+        theme: {
+            light: "Jasny",
+            black: "Czarny"
+        },
+        hit: "traf",
+        hits: "trafień",
+        details: "Szczegóły",
+        stats: "Statystyki",
+        queryTime: "Czas szukania",
+        totalSize: "Całkowita wielkość",
+        pathBar: {
+            placeholder: "Filtruj ścieżki",
+            modalTitle: "Wybierz ścieżkę"
+        },
+        debug: "Informacje dla programistów",
+        debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
+            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
+        tagline: "Slogan",
+        toast: {
+            esConnErrTitle: "Problem z połączeniem z Elasticsearch",
+            esConnErr: "Moduł strony internetowej sist2 napotkał problem przy połączeniu z Elasticsearch." +
+                " Zobacz logi serwera, aby uzyskać więcej informacji.",
+            esQueryErrTitle: "Problem z kwerendą",
+            esQueryErr: "Kwerenda szukania jest niezrozumiała albo nie udało się jej przesłać. Sprawdź dokumentację zaawansowanego szukania. " +
+                "Zobacz logi serwera, aby uzyskać więcej informacji.",
+            dupeTagTitle: "Zduplikowany tag",
+            dupeTag: "Ten dokument już ma taki tag.",
+            copiedToClipboard: "Skopiowano do schowka"
+        },
+        saveTagModalTitle: "Dodaj tag",
+        saveTagPlaceholder: "Nazwa",
+        confirm: "Zatwierdź",
+        indexPickerPlaceholder: "Wybierz indeks",
+        sort: {
+            relevance: "Zgodność z szukanym",
+            dateAsc: "Data (najpierw starsze)",
+            dateDesc: "Data (najpierw nowsze)",
+            sizeAsc: "Wielkość (najpierw mniejsze)",
+            sizeDesc: "Wielkość (najpierw większe)",
+            nameAsc: "Nazwa (A-z)",
+            nameDesc: "Nazwa (Z-a)",
+            random: "Losowo",
+        },
+        d3: {
+            mimeCount: "Dystrybucja liczby plików według typów mediów",
+            mimeSize: "Dystrybucja wielkości plików według typów mediów",
+            dateHistogram: "Dystrybucja dat modyfikacji plików",
+            sizeHistogram: "Dystrybucja wielkości plików",
+        },
+        indexPicker: {
+            selectNone: "Zaznacz nic",
+            selectAll: "Zaznacz wszystko",
+            selectedIndex: "wybrany indeks",
+            selectedIndices: "wybrane indeksy",
+        },
+        ml: {
+            analyzeText: "Analizuj",
+            auto: "Automatycznie",
+            repoFetchError: "Nie udało się uzyskać listy modeli. Zobacz konsolę przeglądarki, aby uzyskać więcej informacji.",
+            repoFetchErrorTitle: "Nie udało się pobrać repozytoriów modeli",
+        }
+    }
 }

sist2-vue/src/store/index.ts

@@ -1,11 +1,11 @@
 import Vue from "vue"
 import Vuex from "vuex"
 import VueRouter, {Route} from "vue-router";
-import {EsHit, EsResult, EsTag, Index, Tag} from "@/Sist2Api";
+import {EsHit, EsResult, EsTag, Index} from "@/Sist2Api";
 import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
 import {getInstance} from "@/plugins/auth0.js";
 
-const CONF_VERSION = 2;
+const CONF_VERSION = 3;
 
 Vue.use(Vuex)
 
@@ -69,11 +69,12 @@ export default new Vuex.Store({
         selectedTags: [] as string[],
 
         lastQueryResults: null,
+        firstQueryResults: null,
 
         keySequence: 0,
         querySequence: 0,
 
-        uiTagHover: null as Tag | null,
+        uiSqliteMode: false,
         uiLightboxIsOpen: false,
         uiShowLightbox: false,
         uiLightboxSources: [] as string[],
@@ -130,13 +131,13 @@ export default new Vuex.Store({
         setSearchText: (state, val) => state.searchText = val,
         setFuzzy: (state, val) => state.fuzzy = val,
         setLastQueryResult: (state, val) => state.lastQueryResults = val,
+        setFirstQueryResult: (state, val) => state.firstQueryResults = val,
         _setOnLoadSelectedIndices: (state, val) => state._onLoadSelectedIndices = val,
         _setOnLoadSelectedMimeTypes: (state, val) => state._onLoadSelectedMimeTypes = val,
         _setOnLoadSelectedTags: (state, val) => state._onLoadSelectedTags = val,
         setSelectedIndices: (state, val) => state.selectedIndices = val,
         setSelectedMimeTypes: (state, val) => state.selectedMimeTypes = val,
         setSelectedTags: (state, val) => state.selectedTags = val,
-        setUiTagHover: (state, val: Tag | null) => state.uiTagHover = val,
         setUiLightboxIsOpen: (state, val: boolean) => state.uiLightboxIsOpen = val,
         _setUiShowLightbox: (state, val: boolean) => state.uiShowLightbox = val,
         setUiLightboxKey: (state, val: number) => state.uiLightboxKey = val,
@@ -154,6 +155,7 @@ export default new Vuex.Store({
         setUiLightboxThumbs: (state, val) => state.uiLightboxThumbs = val,
         setUiLightboxTypes: (state, val) => state.uiLightboxTypes = val,
         setUiLightboxCaptions: (state, val) => state.uiLightboxCaptions = val,
+        setUiSqliteMode: (state, val) => state.uiSqliteMode = val,
 
         setOptTheme: (state, val) => state.optTheme = val,
         setOptDisplay: (state, val) => state.optDisplay = val,
@@ -179,9 +181,15 @@ export default new Vuex.Store({
         setOptVidPreviewInterval: (state, val) => state.optVidPreviewInterval = val,
         setOptSimpleLightbox: (state, val) => state.optSimpleLightbox = val,
         setOptShowTagPickerFilter: (state, val) => state.optShowTagPickerFilter = val,
-        setOptAutoAnalyze: (state, val) => {state.optAutoAnalyze = val},
-        setOptMlRepositories: (state, val) => {state.optMlRepositories = val},
-        setOptMlDefaultModel: (state, val) => {state.optMlDefaultModel = val},
+        setOptAutoAnalyze: (state, val) => {
+            state.optAutoAnalyze = val
+        },
+        setOptMlRepositories: (state, val) => {
+            state.optMlRepositories = val
+        },
+        setOptMlDefaultModel: (state, val) => {
+            state.optMlDefaultModel = val
+        },
 
         setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
         setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val,
@@ -340,6 +348,7 @@ export default new Vuex.Store({
             commit("_setUiShowLightbox", !state.uiShowLightbox);
         },
         clearResults({commit}) {
+            commit("setFirstQueryResult", null);
             commit("setLastQueryResult", null);
             commit("_setKeySequence", 0);
             commit("_setUiShowLightbox", false);
@@ -392,7 +401,6 @@ export default new Vuex.Store({
 
             return (state.lastQueryResults as unknown as EsResult).hits.hits.slice(-1)[0];
         },
-        uiTagHover: state => state.uiTagHover,
         uiShowLightbox: state => state.uiShowLightbox,
         uiLightboxSources: state => state.uiLightboxSources,
         uiLightboxThumbs: state => state.uiLightboxThumbs,
@@ -400,6 +408,7 @@ export default new Vuex.Store({
         uiLightboxTypes: state => state.uiLightboxTypes,
         uiLightboxKey: state => state.uiLightboxKey,
         uiLightboxSlide: state => state.uiLightboxSlide,
+        uiSqliteMode: state => state.uiSqliteMode,
 
         optHideDuplicates: state => state.optHideDuplicates,
         optLang: state => state.optLang,

sist2-vue/src/views/Configuration.vue

@@ -37,11 +37,11 @@
                         {{ $t("opt.lightboxLoadOnlyCurrent") }}
                     </b-form-checkbox>
 
-                    <b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy">
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optHideLegacy" @input="setOptHideLegacy">
                         {{ $t("opt.hideLegacy") }}
                     </b-form-checkbox>
 
-                    <b-form-checkbox :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
                         {{ $t("opt.updateMimeMap") }}
                     </b-form-checkbox>
 
@@ -132,8 +132,11 @@
                         $t("opt.tagOrOperator")
                         }}
                     </b-form-checkbox>
-                    <b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
-                    <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optFuzzy" @input="setOptFuzzy">
+                        {{ $t("opt.fuzzy") }}
+                    </b-form-checkbox>
+
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optSearchInPath" @input="setOptSearchInPath">{{
                         $t("opt.searchInPath")
                         }}
                     </b-form-checkbox>
@@ -151,8 +154,8 @@
                     <b-form-input :value="optResultSize" type="number" min="10"
                                   @input="setOptResultSize"></b-form-input>
 
-                    <label>{{ $t("opt.queryMode") }}</label>
-                    <b-form-select :options="queryModeOptions" :value="optQueryMode"
+                    <label :class="{'text-muted': uiSqliteMode}">{{ $t("opt.queryMode") }}</label>
+                    <b-form-select :disabled="uiSqliteMode" :options="queryModeOptions" :value="optQueryMode"
                                    @input="setOptQueryMode"></b-form-select>
 
                     <label>{{ $t("opt.slideDuration") }}</label>
@@ -170,7 +173,7 @@
                     <b-textarea rows="3" :value="optMlRepositories" @input="setOptMlRepositories"></b-textarea>
                     <br>
                     <b-form-checkbox :checked="optAutoAnalyze" @input="setOptAutoAnalyze">{{
-                            $t("opt.autoAnalyze")
+                        $t("opt.autoAnalyze")
                         }}
                     </b-form-checkbox>
                 </b-card>
@@ -234,6 +237,7 @@ export default {
                 {value: "fr", text: this.$t("lang.fr")},
                 {value: "zh-CN", text: this.$t("lang.zh-CN")},
                 {value: "de", text: this.$t("lang.de")},
+                {value: "pl", text: this.$t("lang.pl")},
             ],
             queryModeOptions: [
                 {value: "simple", text: this.$t("queryMode.simple")},
@@ -300,6 +304,7 @@ export default {
     },
     computed: {
         ...mapGetters([
+            "uiSqliteMode",
             "optTheme",
             "optDisplay",
             "optColumns",

sist2-vue/src/views/FilePage.vue

@@ -14,7 +14,7 @@
       <!-- Audio player-->
       <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
              :type="doc._source.mime"
-             :src="`f/${doc._id}`"></audio>
+             :src="`f/${doc._source.index}/${doc._id}`"></audio>
 
       <InfoTable :doc="doc" v-if="doc"></InfoTable>
 
@@ -54,7 +54,7 @@ export default Vue.extend({
   methods: {
     ext: ext,
     onThumbnailClick() {
-      window.open(`/f/${this.doc._id}`, "_blank");
+      window.open(`/f/${this.doc.index}/${this.doc._id}`, "_blank");
     },
     findByCustomField(field, id) {
       return {

sist2-vue/src/views/SearchPage.vue

@@ -66,7 +66,7 @@ import Sist2Api, {EsHit, EsResult} from "../Sist2Api";
 import SearchBar from "@/components/SearchBar.vue";
 import IndexPicker from "@/components/IndexPicker.vue";
 import Vue from "vue";
-import Sist2Query from "@/Sist2Query";
+import Sist2Query from "@/Sist2ElasticsearchQuery";
 import _debounce from "lodash/debounce";
 import DocCardWall from "@/components/DocCardWall.vue";
 import Lightbox from "@/components/Lightbox.vue";
@@ -79,6 +79,7 @@ import DateSlider from "@/components/DateSlider.vue";
 import TagPicker from "@/components/TagPicker.vue";
 import DocList from "@/components/DocList.vue";
 import HelpDialog from "@/components/HelpDialog.vue";
+import Sist2SqliteQuery from "@/Sist2SqliteQuery";
 
 
 export default Vue.extend({
@@ -114,7 +115,7 @@ export default Vue.extend({
                 await this.clearResults();
             }
 
-            await this.searchNow(Sist2Query.searchQuery());
+            await this.searchNow();
 
         }, 350, {leading: false});
 
@@ -137,7 +138,7 @@ export default Vue.extend({
 
         this.setIndices(this.$store.getters["sist2Info"].indices)
 
-        this.getDateRange().then((range: { min: number, max: number }) => {
+        Sist2Api.getDateRange().then((range) => {
             this.setDateBoundsMin(range.min);
             this.setDateBoundsMax(range.max);
 
@@ -191,12 +192,12 @@ export default Vue.extend({
                     bodyClass: "toast-body-warning",
                 });
         },
-        async searchNow(q: any) {
+        async searchNow() {
             this.searchBusy = true;
             await this.$store.dispatch("incrementQuerySequence");
             this.$store.commit("busSearch");
 
-            Sist2Api.esQuery(q).then(async (resp: EsResult) => {
+            Sist2Api.search().then(async (resp: EsResult) => {
                 await this.handleSearch(resp);
                 this.searchBusy = false;
             }).catch(err => {
@@ -238,7 +239,7 @@ export default Vue.extend({
                 if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
                     hit._seq = await this.$store.dispatch("getKeySequence");
                     this.$store.commit("addLightboxSource", {
-                        source: `f/${hit._id}`,
+                        source: `f/${hit._source.index}/${hit._id}`,
                         thumbnail: hit._props.hasThumbnail
                             ? `t/${hit._source.index}/${hit._id}`
                             : null,
@@ -253,38 +254,17 @@ export default Vue.extend({
 
             await this.$store.dispatch("remountLightbox");
             this.$store.commit("setLastQueryResult", resp);
+            if (this.$store.state.firstQueryResults == null) {
+                this.$store.commit("setFirstQueryResult", resp);
+            }
 
             this.docs.push(...resp.hits.hits);
 
             resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
         },
-        getDateRange(): Promise<{ min: number, max: number }> {
-            return sist2.esQuery({
-                // TODO: filter current selected indices
-                aggs: {
-                    dateMin: {min: {field: "mtime"}},
-                    dateMax: {max: {field: "mtime"}},
-                },
-                size: 0
-            }).then(res => {
-                const range = {
-                    min: res.aggregations.dateMin.value,
-                    max: res.aggregations.dateMax.value,
-                }
-
-                if (range.min == null) {
-                    range.min = 0;
-                    range.max = 1;
-                } else if (range.min == range.max) {
-                    range.max += 1;
-                }
-
-                return range;
-            });
-        },
         appendFunc() {
             if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
-                this.searchNow(Sist2Query.searchQuery());
+                this.searchNow();
             }
         }
     },

sist2-vue/src/views/StatsPage.vue

@@ -1,37 +1,36 @@
 <template>
-  <b-container>
+    <b-container>
 
-    <b-card v-if="loading">
-      <Preloader></Preloader>
-    </b-card>
+        <template>
+            <b-card>
+                <b-card-body>
+                    <b-select v-model="selectedIndex" :options="indexOptions">
+                        <template #first>
+                            <b-form-select-option :value="null" disabled>{{
+                                $t("indexPickerPlaceholder")
+                                }}
+                            </b-form-select-option>
+                        </template>
+                    </b-select>
+                </b-card-body>
+            </b-card>
 
-    <template v-else>
-      <b-card>
-        <b-card-body>
-          <b-select v-model="selectedIndex" :options="indexOptions">
-            <template #first>
-              <b-form-select-option :value="null" disabled>{{ $t("indexPickerPlaceholder") }}</b-form-select-option>
-            </template>
-          </b-select>
-        </b-card-body>
-      </b-card>
-
-      <b-card v-if="selectedIndex !== null" class="mt-3">
-        <b-card-body>
-          <D3Treemap :index-id="selectedIndex"></D3Treemap>
+            <b-card v-if="selectedIndex !== null" class="mt-3">
+                <b-card-body>
+                    <D3Treemap :index-id="selectedIndex"></D3Treemap>
 
 
-        </b-card-body>
-      </b-card>
+                </b-card-body>
+            </b-card>
 
-      <b-card v-if="selectedIndex !== null" class="stats-card mt-3">
-        <D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
-        <D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
-        <D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
-        <D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
-      </b-card>
-    </template>
-  </b-container>
+            <b-card v-if="selectedIndex !== null" class="stats-card mt-3">
+                <D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
+                <D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
+                <D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
+                <D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
+            </b-card>
+        </template>
+    </b-container>
 </template>
 <script>
 import D3Treemap from "@/components/D3Treemap";
@@ -43,37 +42,30 @@ import D3DateHistogram from "@/components/D3DateHistogram";
 import D3SizeHistogram from "@/components/D3SizeHistogram";
 
 export default {
-  components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
-  data() {
-    return {
-      loading: true,
-      selectedIndex: null,
-      indices: []
-    }
-  },
-  computed: {
-    indexOptions() {
-      return this.indices.map(idx => {
+    components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
+    data() {
         return {
-          text: idx.name,
-          value: idx.id
-        };
-      })
+            selectedIndex: null,
+        }
+    },
+    computed: {
+        indexOptions() {
+            return this.indices.map(idx => {
+                return {
+                    text: idx.name,
+                    value: idx.id
+                };
+            })
+        },
+        indices: () => this.$store.state.indices
     }
-  },
-  mounted() {
-    Sist2Api.getSist2Info().then(data => {
-      this.indices = data.indices;
-      this.loading = false;
-    })
-  }
 }
 </script>
 
 <style>
 
 .stats-card {
-  text-align: center;
-  padding: 1em;
+    text-align: center;
+    padding: 1em;
 }
 </style>

src/cli.c

@@ -5,7 +5,7 @@
 #define DEFAULT_OUTPUT "index.sist2"
 #define DEFAULT_NAME "index"
 #define DEFAULT_CONTENT_SIZE 32768
-#define DEFAULT_QUALITY 2
+#define DEFAULT_QUALITY 50
 #define DEFAULT_THUMBNAIL_SIZE 552
 #define DEFAULT_THUMBNAIL_COUNT 1
 #define DEFAULT_REWRITE_URL ""
@@ -93,15 +93,18 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (abs_path == NULL) {
         LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
     } else {
-        abs_path = realloc(abs_path, strlen(abs_path) + 2);
-        strcat(abs_path, "/");
-        args->path = abs_path;
+        char *new_abs_path = realloc(abs_path, strlen(abs_path) + 2);
+        if (new_abs_path == NULL) {
+            LOG_FATALF("cli.c", "FIXME: realloc() failed for argv[1]=%s, abs_path=%s", argv[1], abs_path);
+        }
+        strcat(new_abs_path, "/");
+        args->path = new_abs_path;
     }
 
     if (args->tn_quality == OPTION_VALUE_UNSPECIFIED) {
         args->tn_quality = DEFAULT_QUALITY;
-    } else if (args->tn_quality < 2 || args->tn_quality > 31) {
-        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [2, 31].\n",
+    } else if (args->tn_quality < 0 || args->tn_quality > 100) {
+        fprintf(stderr, "Invalid value for --thumbnail-quality argument: %d. Must be within [0, 100].\n",
                 args->tn_quality);
         return 1;
     }
@@ -109,7 +112,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (args->tn_size == OPTION_VALUE_UNSPECIFIED) {
         args->tn_size = DEFAULT_THUMBNAIL_SIZE;
     } else if (args->tn_size < 32) {
-        printf("Invalid value --thumbnail-size argument: %d. Must be greater than 32 pixels.\n", args->tn_size);
+        printf("Invalid value --thumbnail-size argument: %d. Must be >= 32 pixels.\n", args->tn_size);
         return 1;
     }
 
@@ -142,10 +145,14 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     char *abs_output = abspath(args->output);
     if (args->incremental && abs_output == NULL) {
-        LOG_WARNINGF("main.c", "Could not open original index for incremental scan: %s. Will not perform incremental scan.", args->output);
+        LOG_WARNINGF("main.c",
+                     "Could not open original index for incremental scan: %s. Will not perform incremental scan.",
+                     args->output);
         args->incremental = FALSE;
     } else if (!args->incremental && abs_output != NULL) {
-        LOG_FATALF("main.c", "Index already exists: %s. If you wish to perform incremental scan, you must specify --incremental", abs_output);
+        LOG_FATALF("main.c",
+                   "Index already exists: %s. If you wish to perform incremental scan, you must specify --incremental",
+                   abs_output);
     }
     free(abs_output);
 
@@ -410,6 +417,32 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
     return 0;
 }
 
+int sqlite_index_args_validate(sqlite_index_args_t *args, int argc, const char **argv) {
+
+    LogCtx.verbose = 1;
+
+    if (argc < 2) {
+        fprintf(stderr, "Required positional argument: PATH.\n");
+        return 1;
+    }
+
+    char *index_path = abspath(argv[1]);
+    if (index_path == NULL) {
+        LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
+    } else {
+        args->index_path = index_path;
+    }
+
+    if (args->search_index_path == NULL) {
+        LOG_FATAL("cli.c", "Missing required argument --search-index");
+    }
+
+    LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path);
+    LOG_DEBUGF("cli.c", "arg search_index_path=%s", args->search_index_path);
+
+    return 0;
+}
+
 int web_args_validate(web_args_t *args, int argc, const char **argv) {
 
     LogCtx.verbose = 1;
@@ -419,6 +452,16 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
+    if (args->search_index_path != NULL && args->es_url != NULL) {
+        LOG_FATAL("cli.c", "--search-index and --es-url arguments are mutually exclusive.");
+    }
+    if (args->search_index_path != NULL && args->es_index != NULL) {
+        LOG_FATAL("cli.c", "--search-index and --es-index arguments are mutually exclusive.");
+    }
+    if (args->search_index_path != NULL && args->es_insecure_ssl == TRUE) {
+        LOG_FATAL("cli.c", "--search-index and --es-insecure_ssl arguments are mutually exclusive.");
+    }
+
     if (args->es_url == NULL) {
         args->es_url = DEFAULT_ES_URL;
     }
@@ -531,9 +574,25 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         free(abs_path);
     }
 
+    if (args->search_index_path != NULL) {
+        char *abs_path = abspath(args->search_index_path);
+        if (abs_path == NULL) {
+            LOG_FATALF("cli.c", "Search index not found: %s", args->search_index_path);
+        }
+
+        args->es_index = NULL;
+        args->es_url = NULL;
+        args->es_insecure_ssl = FALSE;
+        args->search_backend = SQLITE_SEARCH_BACKEND;
+    } else {
+        args->search_backend = ES_SEARCH_BACKEND;
+    }
+
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url);
     LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index);
     LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl);
+    LOG_DEBUGF("cli.c", "arg search_index_path=%s", args->search_index_path);
+    LOG_DEBUGF("cli.c", "arg search_backend=%d", args->search_backend);
     LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline);
     LOG_DEBUGF("cli.c", "arg dev=%d", args->dev);
     LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address);
@@ -554,6 +613,11 @@ index_args_t *index_args_create() {
     return args;
 }
 
+sqlite_index_args_t *sqlite_index_args_create() {
+    sqlite_index_args_t *args = calloc(sizeof(sqlite_index_args_t), 1);
+    return args;
+}
+
 web_args_t *web_args_create() {
     web_args_t *args = calloc(sizeof(web_args_t), 1);
     return args;

src/cli.h

@@ -66,10 +66,21 @@ typedef struct index_args {
     int incremental;
 } index_args_t;
 
+typedef struct {
+    char *index_path;
+    char *search_index_path;
+} sqlite_index_args_t;
+
+typedef enum {
+    ES_SEARCH_BACKEND,
+    SQLITE_SEARCH_BACKEND,
+} search_backend_t;
+
 typedef struct web_args {
     char *es_url;
     char *es_index;
     int es_insecure_ssl;
+    char *search_index_path;
     char *listen_address;
     char *credentials;
     char *tag_credentials;
@@ -88,6 +99,7 @@ typedef struct web_args {
     int index_count;
     int dev;
     const char **indices;
+    search_backend_t search_backend;
 } web_args_t;
 
 typedef struct exec_args {
@@ -102,6 +114,8 @@ typedef struct exec_args {
 
 index_args_t *index_args_create();
 
+sqlite_index_args_t *sqlite_index_args_create();
+
 void index_args_destroy(index_args_t *args);
 
 web_args_t *web_args_create();
@@ -110,6 +124,8 @@ void web_args_destroy(web_args_t *args);
 
 int index_args_validate(index_args_t *args, int argc, const char **argv);
 
+int sqlite_index_args_validate(sqlite_index_args_t *args, int argc, const char **argv);
+
 int web_args_validate(web_args_t *args, int argc, const char **argv);
 
 exec_args_t *exec_args_create();
@@ -118,4 +134,5 @@ void exec_args_destroy(exec_args_t *args);
 
 int exec_args_validate(exec_args_t *args, int argc, const char **argv);
 
+
 #endif

src/ctx.h

@@ -76,6 +76,7 @@ typedef struct {
     char *es_url;
     es_version_t *es_version;
     char *es_index;
+    database_t *search_db;
     int es_insecure_ssl;
     int index_count;
     char *auth_user;
@@ -93,6 +94,7 @@ typedef struct {
     struct index_t indices[256];
     char lang[10];
     int dev;
+    int search_backend;
 } WebCtx_t;
 
 

src/database/database.c

@@ -14,12 +14,45 @@ database_t *database_create(const char *filename, database_type_t type) {
     strcpy(db->filename, filename);
     db->type = type;
     db->select_thumbnail_stmt = NULL;
+    db->db = NULL;
+    db->tag_array = NULL;
 
     db->ipc_ctx = NULL;
 
     return db;
 }
 
+int tag_matches(const char *query, const char *tag) {
+    size_t query_len = strlen(query);
+    size_t tag_len = strlen(tag);
+
+    if (query_len >= tag_len) {
+        return FALSE;
+    }
+
+    return strncmp(tag, query, query_len) == 0 && *(tag + query_len) == '.';
+}
+
+void tag_matches_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
+
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    const char *tag = (const char *) sqlite3_value_text(argv[0]);
+
+    char **tags = *(char ***) sqlite3_user_data(ctx);
+
+    array_foreach(tags) {
+        if (tag_matches(tags[i], tag)) {
+            sqlite3_result_int(ctx, TRUE);
+            return;
+        }
+    }
+
+    sqlite3_result_int(ctx, FALSE);
+}
+
 __always_inline
 static int sep_rfind(const char *str) {
     for (int i = (int) strlen(str); i >= 0; i--) {
@@ -48,6 +81,24 @@ void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
     sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
 }
 
+void random_func(sqlite3_context *ctx, int argc, UNUSED(sqlite3_value **argv)) {
+    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
+        sqlite3_result_error(ctx, "Invalid parameters", -1);
+    }
+
+    char state_buf[128] = {0,};
+    struct random_data buf;
+    int result;
+
+    long seed = sqlite3_value_int64(argv[0]);
+
+    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
+
+    random_r(&buf, &result);
+
+    sqlite3_result_int(ctx, result);
+}
+
 
 void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
     if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
@@ -74,6 +125,8 @@ void database_initialize(database_t *db) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IndexDatabaseSchema, NULL, NULL, NULL));
     } else if (db->type == IPC_CONSUMER_DATABASE || db->type == IPC_PRODUCER_DATABASE) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IpcDatabaseSchema, NULL, NULL, NULL));
+    } else if (db->type == FTS_DATABASE) {
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, FtsDatabaseSchema, NULL, NULL, NULL));
     }
 
     sqlite3_close(db->db);
@@ -85,7 +138,8 @@ void database_open(database_t *db) {
     CRASH_IF_NOT_SQLITE_OK(sqlite3_open(db->filename, &db->db));
     sqlite3_busy_timeout(db->db, 1000);
 
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA cache_size = -200000;", NULL, NULL, NULL));
+    // TODO: Optional argument?
+//    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA cache_size = -200000;", NULL, NULL, NULL));
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL));
 
     if (db->type == INDEX_DATABASE) {
@@ -117,6 +171,10 @@ void database_open(database_t *db) {
                 -1,
                 &db->write_thumbnail_stmt, NULL));
 
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT json_data FROM document WHERE id=?", -1,
+                &db->get_document, NULL));
+
         // Create functions
         sqlite3_create_function(
                 db->db,
@@ -168,6 +226,61 @@ void database_open(database_t *db) {
                 db->db, "INSERT INTO index_job (doc_id,type,line) VALUES (?,?,?);", -1,
                 &db->insert_index_job_stmt, NULL));
 
+    } else if (db->type == FTS_DATABASE) {
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT path, count FROM path_index"
+                        " WHERE index_id=? AND depth BETWEEN ? AND ?"
+                        " LIMIT 65536", -1,
+                &db->fts_search_paths, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT json_data FROM document_index"
+                        " WHERE id=?", -1,
+                &db->fts_get_document, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT DISTINCT tag FROM tag"
+                        " WHERE tag GLOB (? || '*') ORDER BY tag LIMIT 100", -1,
+                &db->fts_suggest_tag, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT tag, count(*) FROM tag GROUP BY tag", -1,
+                &db->fts_get_tags, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT path, count FROM path_index"
+                        " WHERE (index_id=?1 OR ?1 IS NULL) AND depth BETWEEN ? AND ?"
+                        " AND (path = ?4 or path GLOB ?5)"
+                        " LIMIT 65536", -1,
+                &db->fts_search_paths_w_prefix, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT path, count FROM path_index"
+                        " WHERE depth BETWEEN ? AND ?"
+                        " AND path GLOB ?"
+                        " LIMIT 65536", -1,
+                &db->fts_suggest_paths, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT * FROM stats", -1,
+                &db->fts_date_range, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db, "SELECT mime, sum(count) FROM mime_index WHERE mime is not NULL GROUP BY mime", -1,
+                &db->fts_get_mimetypes, NULL));
+
+        sqlite3_create_function(
+                db->db,
+                "random_seeded",
+                1,
+                SQLITE_UTF8,
+                NULL,
+                random_func,
+                NULL,
+                NULL
+        );
+
         sqlite3_create_function(
                 db->db,
                 "path_parent",
@@ -178,8 +291,33 @@ void database_open(database_t *db) {
                 NULL,
                 NULL
         );
+
+        sqlite3_create_function(
+                db->db,
+                "tag_matches",
+                1,
+                SQLITE_UTF8,
+                &db->tag_array,
+                tag_matches_func,
+                NULL,
+                NULL
+        );
     }
 
+    if (db->type == FTS_DATABASE || db->type == INDEX_DATABASE) {
+        // Tag table is the same schema for FTS database & index database
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "INSERT INTO tag (id, tag) VALUES (?,?) ON CONFLICT DO NOTHING;",
+                -1,
+                &db->write_tag_stmt, NULL));
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "DELETE FROM tag WHERE id=? AND tag=?;",
+                -1,
+                &db->delete_tag_stmt, NULL));
+    }
 }
 
 void database_close(database_t *db, int optimize) {
@@ -191,7 +329,9 @@ void database_close(database_t *db, int optimize) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA optimize;", NULL, NULL, NULL));
     }
 
-    sqlite3_close(db->db);
+    if (db->db) {
+        sqlite3_close(db->db);
+    }
 
     if (db->type == IPC_PRODUCER_DATABASE) {
         remove(db->filename);
@@ -479,28 +619,6 @@ void database_write_thumbnail(database_t *db, const char *id, int num, void *dat
 }
 
 
-//void database_create_fts_index(database_t *db, database_t *fts_db) {
-//    // In a separate file,
-//
-//    // use database_initialize() to create FTS schema
-//    // if --force-reset, then truncate the tables first
-//
-//    /*
-//     * create/append fts table
-//     *
-//     * create/append scalar index table with
-//     *  id,index,size,mtime,mime
-//     *
-//     * create/append path index table with
-//     *  index,path,depth
-//     *
-//     * content table is a view with SELECT UNION for all attached tables
-//     *  random_seed column
-//     */
-//
-//    // INSERT INTO ft(ft) VALUES('optimize');
-//}
-
 job_t *database_get_work(database_t *db, job_type_t job_type) {
     job_t *job;
 
@@ -642,3 +760,39 @@ void database_add_work(database_t *db, job_t *job) {
     pthread_cond_signal(&db->ipc_ctx->has_work_cond);
     pthread_mutex_unlock(&db->ipc_ctx->mutex);
 }
+
+void database_write_tag(database_t *db, char *doc_id, char *tag) {
+    sqlite3_bind_text(db->write_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_text(db->write_tag_stmt, 2, tag, -1, SQLITE_STATIC);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->write_tag_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_tag_stmt));
+}
+
+void database_delete_tag(database_t *db, char *doc_id, char *tag) {
+    sqlite3_bind_text(db->delete_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
+    sqlite3_bind_text(db->delete_tag_stmt, 2, tag, -1, SQLITE_STATIC);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(db->delete_tag_stmt));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->delete_tag_stmt));
+}
+
+cJSON *database_get_document(database_t *db, char *doc_id) {
+    sqlite3_bind_text(db->get_document, 1, doc_id, -1, SQLITE_STATIC);
+
+    int ret = sqlite3_step(db->get_document);
+    CRASH_IF_STMT_FAIL(ret);
+
+    cJSON *json;
+
+    if (ret == SQLITE_ROW) {
+        const char *json_str = sqlite3_column_text(db->get_document, 0);
+        json = cJSON_Parse(json_str);
+    } else {
+        json = NULL;
+    }
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->get_document));
+
+    return json;
+}

src/database/database.h

@@ -10,6 +10,7 @@ typedef struct index_descriptor index_descriptor_t;
 
 extern const char *IpcDatabaseSchema;
 extern const char *IndexDatabaseSchema;
+extern const char *FtsDatabaseSchema;
 
 typedef enum {
     INDEX_DATABASE,
@@ -32,6 +33,16 @@ typedef enum {
     JOB_PARSE_JOB
 } job_type_t;
 
+typedef enum {
+    FTS_SORT_INVALID,
+    FTS_SORT_SCORE,
+    FTS_SORT_SIZE,
+    FTS_SORT_MTIME,
+    FTS_SORT_RANDOM,
+    FTS_SORT_NAME,
+    FTS_SORT_ID,
+} fts_sort_t;
+
 typedef struct {
     job_type_t type;
     union {
@@ -52,6 +63,11 @@ typedef struct {
     char current_job[MAX_THREADS][PATH_MAX * 2];
 } database_ipc_ctx_t;
 
+typedef struct {
+    double date_min;
+    double date_max;
+} database_summary_stats_t;
+
 typedef struct database {
     char filename[PATH_MAX];
     database_type_t type;
@@ -66,12 +82,27 @@ typedef struct database {
     sqlite3_stmt *write_document_stmt;
     sqlite3_stmt *write_document_sidecar_stmt;
     sqlite3_stmt *write_thumbnail_stmt;
+    sqlite3_stmt *get_document;
+
+    sqlite3_stmt *delete_tag_stmt;
+    sqlite3_stmt *write_tag_stmt;
 
     sqlite3_stmt *insert_parse_job_stmt;
     sqlite3_stmt *insert_index_job_stmt;
     sqlite3_stmt *pop_parse_job_stmt;
     sqlite3_stmt *pop_index_job_stmt;
 
+    sqlite3_stmt *fts_search_paths;
+    sqlite3_stmt *fts_search_paths_w_prefix;
+    sqlite3_stmt *fts_suggest_paths;
+    sqlite3_stmt *fts_date_range;
+    sqlite3_stmt *fts_get_mimetypes;
+    sqlite3_stmt *fts_get_document;
+    sqlite3_stmt *fts_suggest_tag;
+    sqlite3_stmt *fts_get_tags;
+
+    char **tag_array;
+
     database_ipc_ctx_t *ipc_ctx;
 } database_t;
 
@@ -86,8 +117,6 @@ typedef struct {
     long size;
 } treemap_row_t;
 
-static treemap_row_t null_treemap_row = {0, 0, 0};
-
 
 database_t *database_create(const char *filename, database_type_t type);
 
@@ -116,7 +145,7 @@ cJSON *database_document_iter(database_iterator_t *);
 
 database_iterator_t *database_create_delete_list_iterator(database_t *db);
 
-char * database_delete_list_iter(database_iterator_t *iter);
+char *database_delete_list_iter(database_iterator_t *iter);
 
 #define database_delete_list_iter_foreach(element, iter) \
     for (char *(element) = database_delete_list_iter(iter); (element) != NULL; (element) = database_delete_list_iter(iter))
@@ -135,7 +164,7 @@ database_iterator_t *database_create_treemap_iterator(database_t *db, long thres
 treemap_row_t database_treemap_iter(database_iterator_t *iter);
 
 #define database_treemap_iter_foreach(element, iter) \
-    for (treemap_row_t element = database_treemap_iter(iter); element.path != NULL; element = database_treemap_iter(iter))
+    for (treemap_row_t element = database_treemap_iter(iter); (element).path != NULL; (element) = database_treemap_iter(iter))
 
 
 void database_generate_stats(database_t *db, double treemap_threshold);
@@ -146,22 +175,55 @@ job_t *database_get_work(database_t *db, job_type_t job_type);
 
 void database_add_work(database_t *db, job_t *job);
 
-//void database_index(database_t *db);
-
 cJSON *database_get_stats(database_t *db, database_stat_type_d type);
 
 #define CRASH_IF_STMT_FAIL(x) do { \
         int return_value = x;                \
         if (return_value != SQLITE_DONE && return_value != SQLITE_ROW) {     \
-            LOG_FATALF("database.c", "Sqlite error @ database.c:%d : (%d) %s", __LINE__, return_value, sqlite3_errmsg(db->db)); \
+            LOG_FATALF("database.c", "Sqlite error @ %s:%d : (%d) %s", __BASE_FILE__, __LINE__, return_value, sqlite3_errmsg(db->db)); \
         }                           \
     } while (0)
 
 #define CRASH_IF_NOT_SQLITE_OK(x) do { \
         int return_value = x;                \
         if (return_value != SQLITE_OK) {     \
-            LOG_FATALF("database.c", "Sqlite error @ database.c:%d : (%d) %s", __LINE__, return_value, sqlite3_errmsg(db->db)); \
+            LOG_FATALF("database.c", "Sqlite error @ %s:%d : (%d) %s", __BASE_FILE__, __LINE__, return_value, sqlite3_errmsg(db->db)); \
         }                           \
     } while (0)
 
-#endif //SIST2_DATABASE_H
+void database_fts_attach(database_t *db, const char *fts_database_path);
+
+void database_fts_index(database_t *db);
+
+void database_fts_optimize(database_t *db);
+
+cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
+                              int suggest);
+
+cJSON *database_fts_get_mimetypes(database_t *db);
+
+database_summary_stats_t database_fts_get_date_range(database_t *db);
+
+cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
+                           long size_max, long date_min, long date_max, int page_size,
+                           char **index_ids, char **mime_types, char **tags, int sort_asc,
+                           fts_sort_t sort, int seed, char **after, int fetch_aggregations,
+                           int highlight, int highlight_context_size);
+
+void database_write_tag(database_t *db, char *doc_id, char *tag);
+
+void database_delete_tag(database_t *db, char *doc_id, char *tag);
+
+void database_fts_detach(database_t *db);
+
+cJSON *database_fts_get_document(database_t *db, char *doc_id);
+
+database_summary_stats_t database_fts_sync_tags(database_t *db);
+
+cJSON *database_fts_suggest_tag(database_t *db, char *prefix);
+
+cJSON *database_fts_get_tags(database_t *db);
+
+cJSON *database_get_document(database_t *db, char *doc_id);
+
+#endif

src/database/database_fts.c

@@ -0,0 +1,804 @@
+#include "database.h"
+#include "src/ctx.h"
+
+void database_fts_detach(database_t *db) {
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "DETACH DATABASE fts",
+            NULL, NULL, NULL
+    ));
+}
+
+void database_fts_attach(database_t *db, const char *fts_database_path) {
+
+    LOG_DEBUGF("database_fts.c", "Attaching to %s", fts_database_path);
+
+    sqlite3_stmt *stmt;
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+            db->db, "ATTACH DATABASE ? AS fts"
+                    "", -1, &stmt, NULL));
+
+    sqlite3_bind_text(stmt, 1, fts_database_path, -1, SQLITE_STATIC);
+
+    CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
+    sqlite3_finalize(stmt);
+}
+
+int database_fts_get_max_path_depth(database_t *db) {
+    sqlite3_stmt *stmt;
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+            db->db, "SELECT MAX(depth) FROM path_tmp", -1, &stmt, NULL));
+    CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
+
+    int max_depth = sqlite3_column_int(stmt, 0);
+    sqlite3_finalize(stmt);
+
+    return max_depth;
+}
+
+void database_fts_index(database_t *db) {
+
+    LOG_INFO("database_fts.c", "Creating content table.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "WITH docs AS ("
+            " SELECT document.id as id, (SELECT id FROM descriptor) as index_id, size,"
+            "  document.json_data ->> 'name' as name,"
+            "  document.json_data ->> 'path' as path,"
+            "  mtime,"
+            "  document.json_data ->> 'mime' as mime,"
+            "  CASE"
+            "   WHEN sc.json_data IS NULL THEN"
+            "    json_set(document.json_data, "
+            "             '$._id',document.id,"
+            "             '$.size',document.size, "
+            "             '$.mtime',document.mtime)"
+            "    ELSE json_patch("
+            "             json_set(document.json_data,"
+            "                      '$._id',document.id,"
+            "                      '$.size',document.size,"
+            "                      '$.mtime', document.mtime),"
+            "             sc.json_data) END"
+            " FROM document"
+            "  LEFT JOIN document_sidecar sc ON document.id = sc.id"
+            " GROUP BY document.id)"
+            " INSERT"
+            " INTO fts.document_index (id, index_id, size, name, path, mtime, mime, json_data)"
+            " SELECT * FROM docs WHERE true"
+            " on conflict (id, index_id) do update set "
+            "  size=excluded.size, mtime=excluded.mtime, mime=excluded.mime, json_data=excluded.json_data;",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Deleting old documents.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM fts.document_index"
+            " WHERE id IN (SELECT id FROM delete_list)"
+            "  AND index_id = (SELECT id FROM descriptor);",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Generating summary stats");
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM fts.stats", NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "INSERT INTO fts.stats "
+                    "SELECT min(mtime), max(mtime) FROM fts.document_index",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Generating mime index");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "DELETE FROM fts.mime_index;", NULL, NULL, NULL));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "INSERT INTO fts.mime_index (index_id, mime, count) "
+                    "SELECT index_id, mime, count(*) FROM fts.document_index GROUP BY index_id, mime",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Generating path index");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "CREATE TEMP TABLE path_tmp ("
+            " path TEXT,"
+            " index_id TEXT,"
+            " count INTEGER NOT NULL,"
+            " depth INTEGER NOT NULL,"
+            " children INTEGER NOT NULL DEFAULT(0),"
+            " total INTEGER AS (count + children),"
+            " PRIMARY KEY (path, index_id)"
+            ");", NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "INSERT INTO path_tmp (path, index_id, count, depth)"
+            " SELECT path, index_id, count(*), CASE WHEN length(json_data->>'path') == 0 THEN 0"
+            " ELSE 1 + length(json_data->>'path') - length(REPLACE(json_data->>'path', '/', ''))"
+            " END as depth FROM document_index WHERE depth > 0"
+            " GROUP BY path", NULL, NULL, NULL));
+
+    int max_depth = database_fts_get_max_path_depth(db);
+
+    for (int i = max_depth; i > 1; i--) {
+        sqlite3_stmt *stmt;
+
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                db->db,
+                "INSERT INTO path_tmp (path, index_id, children, depth, count)"
+                " SELECT path_parent(path) parent, index_id, (SELECT COALESCE(sum(count), 0) FROM path_tmp WHERE path "
+                " BETWEEN path_parent(p.path) || '/' AND path_parent(p.path) || '/𘚟' AND index_id = p.index_id) as cnt, depth-1, 0 "
+                " FROM path_tmp p WHERE depth=? GROUP BY parent"
+                " ON CONFLICT(path, index_id) DO UPDATE SET children=excluded.children",
+                -1, &stmt, NULL));
+        sqlite3_bind_int(stmt, 1, i);
+        CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
+
+        LOG_DEBUGF("database_fts.c", "Path index depth %d (%d)", i, sqlite3_changes(db->db));
+    }
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM path_index;"
+            "INSERT INTO path_index (path, index_id, count, depth) SELECT path, index_id, total, depth FROM path_tmp",
+            NULL, NULL, NULL));
+
+    LOG_DEBUG("database_fts.c", "Generating search index.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db, "INSERT INTO search(search) VALUES ('delete-all')",
+            NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "INSERT INTO search(rowid, name, content, title) SELECT id, name, content, title from document_view",
+            NULL, NULL, NULL));
+}
+
+void database_fts_optimize(database_t *db) {
+    LOG_INFO("database_fts.c", "Optimizing search index.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "INSERT INTO search(search) VALUES('optimize');",
+            NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA fts.optimize;", NULL, NULL, NULL));
+}
+
+cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
+                              int suggest) {
+
+    sqlite3_stmt *stmt;
+
+    if (suggest) {
+        stmt = db->fts_suggest_paths;
+        sqlite3_bind_int(stmt, 1, depth_min);
+        sqlite3_bind_int(stmt, 2, depth_max);
+
+        if (prefix) {
+            char *prefix_glob = malloc(strlen(prefix) + 2);
+            sprintf(prefix_glob, "%s*", prefix);
+            sqlite3_bind_text(stmt, 3, prefix_glob, -1, SQLITE_TRANSIENT);
+            free(prefix_glob);
+        }
+
+    } else if (prefix) {
+        stmt = db->fts_search_paths_w_prefix;
+        if (index_id) {
+            sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
+        } else {
+            sqlite3_bind_null(stmt, 1);
+        }
+        sqlite3_bind_int(stmt, 2, depth_min);
+        sqlite3_bind_int(stmt, 3, depth_max);
+
+        char *prefix_glob = malloc(strlen(prefix) + 3);
+        sprintf(prefix_glob, "%s/*", prefix);
+        sqlite3_bind_text(stmt, 4, prefix, -1, SQLITE_STATIC);
+        sqlite3_bind_text(stmt, 5, prefix_glob, -1, SQLITE_TRANSIENT);
+        free(prefix_glob);
+    } else {
+        stmt = db->fts_search_paths;
+        if (index_id) {
+            sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
+        } else {
+            sqlite3_bind_null(stmt, 1);
+        }
+        sqlite3_bind_int(stmt, 2, depth_min);
+        sqlite3_bind_int(stmt, 3, depth_max);
+    }
+
+    cJSON *json = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(stmt);
+        CRASH_IF_STMT_FAIL(ret);
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON *row = cJSON_CreateObject();
+
+        cJSON_AddStringToObject(row, "path", (const char *) sqlite3_column_text(stmt, 0));
+        cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 1));
+
+        cJSON_AddItemToArray(json, row);
+    } while (TRUE);
+
+    sqlite3_reset(stmt);
+
+    return json;
+}
+
+cJSON *database_fts_get_mimetypes(database_t *db) {
+
+    cJSON *json = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(db->fts_get_mimetypes);
+        CRASH_IF_STMT_FAIL(ret);
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON *row = cJSON_CreateObject();
+
+        cJSON_AddStringToObject(row, "mime", (const char *) sqlite3_column_text(db->fts_get_mimetypes, 0));
+        cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(db->fts_get_mimetypes, 1));
+
+        cJSON_AddItemToArray(json, row);
+    } while (TRUE);
+
+    sqlite3_reset(db->fts_get_mimetypes);
+
+    return json;
+}
+
+const char *size_where_clause(long size_min, long size_max) {
+    if (size_min > 0 && size_max > 0) {
+        return "size BETWEEN @size_min AND @size_max";
+    } else if (size_min > 0) {
+        return "size >= @size_min";
+    } else if (size_max > 0) {
+        return "size <= @size_max";
+    }
+
+    return NULL;
+}
+
+const char *date_where_clause(long date_min, long date_max) {
+    if (date_min > 0 && date_max > 0) {
+        return "mtime BETWEEN @date_min AND @date_max";
+    } else if (date_min > 0) {
+        return "mtime >= @date_min";
+    } else if (date_max > 0) {
+        return "mtime <= @date_max";
+    }
+
+    return NULL;
+}
+
+int array_length(char **arr) {
+
+    if (arr == NULL) {
+        return 0;
+    }
+
+    int count = -1;
+    while (arr[++count] != NULL);
+
+    return count;
+}
+
+#define INDEX_ID_PARAM_OFFSET (10)
+#define MIME_PARAM_OFFSET (INDEX_ID_PARAM_OFFSET + 1000)
+
+char *build_where_clause(const char *path_where, const char *size_where, const char *date_where,
+                         const char *index_id_where, const char *mime_where, const char *query_where,
+                         const char *after_where, const char *tags_where) {
+    char *where = calloc(
+            strlen(index_id_where)
+            + (query_where ? strlen(query_where) + sizeof(" AND ") : 0)
+            + (path_where ? strlen(path_where) + sizeof(" AND ") : 0)
+            + (size_where ? strlen(size_where) + sizeof(" AND ") : 0)
+            + (date_where ? strlen(date_where) + sizeof(" AND ") : 0)
+            + (after_where ? strlen(after_where) + sizeof(" AND ") : 0)
+            + (tags_where ? strlen(tags_where) + sizeof(" AND ") : 0)
+            + (mime_where ? strlen(mime_where) + sizeof(" AND ") : 0) + 1,
+            sizeof(char)
+    );
+
+    strcat(where, index_id_where);
+    if (query_where) {
+        strcat(where, " AND ");
+        strcat(where, query_where);
+    }
+    if (path_where) {
+        strcat(where, " AND ");
+        strcat(where, path_where);
+    }
+    if (size_where) {
+        strcat(where, " AND ");
+        strcat(where, size_where);
+    }
+    if (date_where) {
+        strcat(where, " AND ");
+        strcat(where, date_where);
+    }
+    if (mime_where) {
+        strcat(where, " AND ");
+        strcat(where, mime_where);
+    }
+    if (after_where) {
+        strcat(where, " AND ");
+        strcat(where, after_where);
+    }
+    if (tags_where) {
+        strcat(where, " AND ");
+        strcat(where, tags_where);
+    }
+    return where;
+}
+
+char *index_ids_where_clause(char **index_ids) {
+    int param_count = array_length(index_ids);
+
+    char *clause = malloc(13 + 2 + 6 * param_count);
+
+    strcpy(clause, "index_id IN (");
+    for (int i = 0; i < param_count; i++) {
+        char param[10];
+        snprintf(param, sizeof(param), "?%d%s",
+                 INDEX_ID_PARAM_OFFSET + i, i == param_count - 1 ? "" : ",");
+        strcat(clause, param);
+    }
+    strcat(clause, ")");
+
+    return clause;
+}
+
+char *mime_types_where_clause(char **mime_types) {
+    int param_count = array_length(mime_types);
+
+    if (param_count == 0) {
+        return NULL;
+    }
+
+    char *clause = malloc(9 + 2 + 6 * param_count);
+
+    strcpy(clause, "mime IN (");
+    for (int i = 0; i < param_count; i++) {
+        char param[10];
+        snprintf(param, sizeof(param), "?%d%s",
+                 MIME_PARAM_OFFSET + i, i == param_count - 1 ? "" : ",");
+        strcat(clause, param);
+    }
+    strcat(clause, ")");
+
+    return clause;
+}
+
+const char *path_where_clause(const char *path) {
+    if (path == NULL || strlen(path) == 0) {
+        return NULL;
+    }
+
+    return "(path = @path or path GLOB @path_glob)";
+}
+
+const char *get_sort_var(fts_sort_t sort) {
+
+    switch (sort) {
+        case FTS_SORT_SCORE:
+            // Round to 14 decimal places to avoid precision problems when converting to JSON...
+            return "round(rank, 14)";
+        case FTS_SORT_SIZE:
+            return "size";
+        case FTS_SORT_MTIME:
+            return "mtime";
+        case FTS_SORT_RANDOM:
+            return "random_seeded(doc.ROWID + ?5)";
+        case FTS_SORT_NAME:
+            return "doc.name";
+        case FTS_SORT_ID:
+            return "doc.id";
+        default:
+            return NULL;
+    }
+}
+
+const char *match_where(const char *query) {
+    if (query == NULL || strlen(query) == 0) {
+        return NULL;
+    } else {
+        return "search MATCH ?1";
+    }
+}
+
+char *tags_where_clause(char **tags) {
+    if (tags == NULL) {
+        return NULL;
+    }
+
+    return "EXISTS (SELECT 1 FROM tag WHERE id=doc.id AND tag_matches(tag))";
+}
+
+database_summary_stats_t database_fts_get_date_range(database_t *db) {
+
+    int ret = sqlite3_step(db->fts_date_range);
+    CRASH_IF_STMT_FAIL(ret);
+
+    if (ret == SQLITE_DONE) {
+        return (database_summary_stats_t) {0, 0};
+    }
+
+    database_summary_stats_t stats;
+    stats.date_min = (double) sqlite3_column_int64(db->fts_date_range, 0);
+    stats.date_max = (double) sqlite3_column_int64(db->fts_date_range, 1);
+
+    sqlite3_reset(db->fts_date_range);
+
+    return stats;
+}
+
+char *get_after_where(char **after, fts_sort_t sort) {
+    if (after == NULL) {
+        return NULL;
+    }
+
+    return "(sort_var, doc.ROWID) > (?3, ?4)";
+}
+
+cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
+                           long size_max, long date_min, long date_max, int page_size,
+                           char **index_ids, char **mime_types, char **tags, int sort_asc,
+                           fts_sort_t sort, int seed, char **after, int fetch_aggregations,
+                           int highlight, int highlight_context_size) {
+
+    char path_glob[PATH_MAX * 2];
+    snprintf(path_glob, sizeof(path_glob), "%s/*", path);
+    const char *path_where = path_where_clause(path);
+    const char *size_where = size_where_clause(size_min, size_max);
+    const char *date_where = date_where_clause(date_min, date_max);
+    char *index_id_where = index_ids_where_clause(index_ids);
+    char *mime_where = mime_types_where_clause(mime_types);
+    const char *query_where = match_where(query);
+    const char *after_where = get_after_where(after, sort);
+    const char *tags_where = tags_where_clause(tags);
+
+    if (!query_where && sort == FTS_SORT_SCORE) {
+        // If query is NULL, then sort by id instead
+        sort = FTS_SORT_ID;
+    }
+
+    char *agg_where;
+    char *where = build_where_clause(path_where, size_where, date_where, index_id_where, mime_where, query_where,
+                                     after_where, tags_where);
+    if (fetch_aggregations) {
+        agg_where = build_where_clause(path_where, size_where, date_where, index_id_where, mime_where, query_where,
+                                       NULL, tags_where);
+    }
+
+    const char *json_object_sql;
+    if (highlight && query_where != NULL) {
+        json_object_sql = "json_remove(json_set(doc.json_data,"
+                          "'$.index', doc.index_id,"
+                          "'$._highlight.name', snippet(search, 0, '<mark>', '</mark>', '', ?6),"
+                          "'$._highlight.content', snippet(search, 1, '<mark>', '</mark>', '', ?6)),"
+                          "'$.content')";
+    } else {
+        json_object_sql = "json_remove(json_set(doc.json_data,"
+                          "'$.index', doc.index_id),"
+                          "'$.content')";
+    }
+
+    char *sql;
+    char *agg_sql;
+
+    if (query_where) {
+        asprintf(
+                &sql,
+                "SELECT"
+                " %s, %s as sort_var, doc.ROWID"
+                " FROM search"
+                " INNER JOIN document_index doc on doc.ROWID = search.ROWID"
+                " WHERE %s"
+                " ORDER BY sort_var%s, doc.ROWID"
+                " LIMIT ?2",
+                json_object_sql, get_sort_var(sort),
+                where,
+                sort_asc ? "" : "DESC");
+
+        if (fetch_aggregations) {
+            asprintf(&agg_sql,
+                     "SELECT count(*), sum(size)"
+                     " FROM search"
+                     "  INNER JOIN document_index doc on doc.ROWID = search.ROWID"
+                     " WHERE search MATCH ?1"
+                     " AND %s", agg_where);
+        }
+    } else {
+        asprintf(
+                &sql,
+                "SELECT"
+                " %s, %s as sort_var, doc.ROWID"
+                " FROM document_index doc"
+                " WHERE %s"
+                " ORDER BY sort_var%s,doc.ROWID"
+                " LIMIT ?2",
+                json_object_sql, get_sort_var(sort),
+                where,
+                sort_asc ? "" : " DESC");
+
+        if (fetch_aggregations) {
+            asprintf(&agg_sql,
+                     "SELECT count(*), sum(size)"
+                     " FROM document_index doc"
+                     " WHERE %s", agg_where);
+        }
+    }
+
+    sqlite3_stmt *stmt;
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(db->db, sql, -1, &stmt, NULL));
+
+    if (query_where) {
+        sqlite3_bind_text(stmt, 1, query, -1, SQLITE_STATIC);
+    }
+    sqlite3_bind_int(stmt, 2, page_size);
+
+    if (index_ids) {
+        array_foreach(index_ids) {
+            sqlite3_bind_text(stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
+        }
+    }
+    if (mime_types) {
+        array_foreach(mime_types) {
+            sqlite3_bind_text(stmt, MIME_PARAM_OFFSET + i, mime_types[i], -1, SQLITE_STATIC);
+        }
+    }
+    if (tags) {
+        db->tag_array = tags;
+    }
+
+    if (size_min > 0) {
+        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@size_min"), size_min);
+    }
+    if (size_max > 0) {
+        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@size_max"), size_max);
+    }
+    if (date_min > 0) {
+        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@date_min"), date_min);
+    }
+    if (date_max > 0) {
+        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@date_max"), date_max);
+    }
+    if (path_where) {
+        sqlite3_bind_text(stmt, sqlite3_bind_parameter_index(stmt, "@path"), path, -1, SQLITE_STATIC);
+        sqlite3_bind_text(stmt, sqlite3_bind_parameter_index(stmt, "@path_glob"), path_glob, -1, SQLITE_STATIC);
+    }
+    if (after_where) {
+        if (sort == FTS_SORT_NAME || sort == FTS_SORT_ID) {
+            sqlite3_bind_text(stmt, 3, after[0], -1, SQLITE_STATIC);
+        } else if (sort == FTS_SORT_SCORE) {
+            sqlite3_bind_double(stmt, 3, strtod(after[0], NULL));
+        } else {
+            sqlite3_bind_int64(stmt, 3, strtol(after[0], NULL, 10));
+        }
+        sqlite3_bind_int64(stmt, 4, strtol(after[1], NULL, 10));
+    }
+    if (sort == FTS_SORT_RANDOM) {
+        sqlite3_bind_int(stmt, 5, seed);
+    }
+    if (highlight) {
+        sqlite3_bind_int(stmt, 6, highlight_context_size);
+    }
+
+    cJSON *json = cJSON_CreateObject();
+    cJSON *hits_hits = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(stmt);
+        if (ret != SQLITE_DONE && ret != SQLITE_ROW) {
+            break;
+        }
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        const char *json_str = (const char *) sqlite3_column_text(stmt, 0);
+        cJSON *row = cJSON_CreateObject();
+        cJSON *source = cJSON_Parse(json_str);
+        if (highlight) {
+            cJSON *hl = cJSON_DetachItemFromObject(source, "_highlight");
+            cJSON_AddItemToObject(row, "highlight", hl);
+        }
+        cJSON *id = cJSON_DetachItemFromObject(source, "_id");
+        cJSON_AddItemToObject(row, "_id", id);
+        cJSON_AddItemToObject(row, "_source", source);
+
+        cJSON *sort_info = cJSON_AddArrayToObject(row, "sort");
+        cJSON_AddItemToArray(
+                sort_info,
+                cJSON_CreateString((char *) sqlite3_column_text(stmt, 1))
+        );
+        cJSON_AddItemToArray(
+                sort_info,
+                cJSON_CreateString((char *) sqlite3_column_text(stmt, 2))
+        );
+
+        cJSON_AddItemToArray(hits_hits, row);
+    } while (TRUE);
+
+    sqlite3_finalize(stmt);
+
+    cJSON *hits = cJSON_AddObjectToObject(json, "hits");
+    cJSON_AddItemToObject(hits, "hits", hits_hits);
+
+    // Aggregations
+    if (fetch_aggregations) {
+
+        sqlite3_stmt *agg_stmt;
+        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(db->db, agg_sql, -1, &agg_stmt, NULL));
+
+        if (index_ids) {
+            array_foreach(index_ids) {
+                sqlite3_bind_text(agg_stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
+            }
+        }
+        if (mime_types) {
+            array_foreach(mime_types) {
+                sqlite3_bind_text(agg_stmt, MIME_PARAM_OFFSET + i, mime_types[i], -1, SQLITE_STATIC);
+            }
+        }
+
+        if (query_where) {
+            sqlite3_bind_text(agg_stmt, 1, query, -1, SQLITE_STATIC);
+        }
+        if (size_min > 0) {
+            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@size_min"), size_min);
+        }
+        if (size_max > 0) {
+            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@size_max"), size_max);
+        }
+        if (date_min > 0) {
+            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@date_min"), date_min);
+        }
+        if (date_max > 0) {
+            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@date_max"), date_max);
+        }
+        if (path_where) {
+            sqlite3_bind_text(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@path"), path, -1, SQLITE_STATIC);
+            sqlite3_bind_text(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@path_glob"), path_glob, -1,
+                              SQLITE_STATIC);
+        }
+
+        int agg_ret = sqlite3_step(agg_stmt);
+
+        if (agg_ret == SQLITE_ROW) {
+            cJSON *aggregations = cJSON_AddObjectToObject(json, "aggregations");
+            cJSON *total_count = cJSON_AddObjectToObject(aggregations, "total_count");
+            cJSON_AddNumberToObject(total_count, "value", sqlite3_column_double(agg_stmt, 0));
+            cJSON *total_size = cJSON_AddObjectToObject(aggregations, "total_size");
+            cJSON_AddNumberToObject(total_size, "value", sqlite3_column_double(agg_stmt, 1));
+        } else {
+            cJSON *aggregations = cJSON_AddObjectToObject(json, "aggregations");
+            cJSON *total_count = cJSON_AddObjectToObject(aggregations, "total_count");
+            cJSON_AddNumberToObject(total_count, "value", 0);
+            cJSON *total_size = cJSON_AddObjectToObject(aggregations, "total_size");
+            cJSON_AddNumberToObject(total_size, "value", 0);
+        }
+        sqlite3_finalize(agg_stmt);
+    }
+
+    // Cleanup
+    if (index_id_where) {
+        free(index_id_where);
+    }
+    if (mime_where) {
+        free(mime_where);
+    }
+    free(where);
+    free(sql);
+    if (fetch_aggregations) {
+        free(agg_where);
+        free(agg_sql);
+    }
+
+    return json;
+}
+
+database_summary_stats_t database_fts_sync_tags(database_t *db) {
+
+    LOG_INFO("database_fts.c", "Syncing tags.");
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "DELETE FROM fts.tag WHERE"
+            " (id, tag) NOT IN (SELECT id, tag FROM tag)",
+            NULL, NULL, NULL));
+
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
+            db->db,
+            "INSERT INTO fts.tag (id, tag) "
+            " SELECT id, tag FROM tag "
+            " WHERE (id, tag) NOT IN (SELECT * FROM fts.tag)",
+            NULL, NULL, NULL));
+}
+
+cJSON *database_fts_get_document(database_t *db, char *doc_id) {
+    sqlite3_bind_text(db->fts_get_document, 1, doc_id, -1, NULL);
+
+    int ret = sqlite3_step(db->fts_get_document);
+    cJSON *json = NULL;
+
+    if (ret == SQLITE_ROW) {
+        const char *json_data = (const char *) sqlite3_column_text(db->fts_get_document, 0);
+        json = cJSON_Parse(json_data);
+    } else {
+        CRASH_IF_STMT_FAIL(ret);
+    }
+
+    sqlite3_reset(db->fts_get_document);
+
+    return json;
+}
+
+cJSON *database_fts_suggest_tag(database_t *db, char *prefix) {
+    sqlite3_bind_text(db->fts_suggest_tag, 1, prefix, -1, NULL);
+
+    cJSON *json = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(db->fts_suggest_tag);
+        CRASH_IF_STMT_FAIL(ret);
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON_AddItemToArray(
+                json,
+                cJSON_CreateString((const char *) sqlite3_column_text(db->fts_suggest_tag, 0))
+        );
+
+    } while (TRUE);
+
+    sqlite3_reset(db->fts_suggest_tag);
+
+    return json;
+}
+
+
+cJSON *database_fts_get_tags(database_t *db) {
+    cJSON *json = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(db->fts_get_tags);
+        CRASH_IF_STMT_FAIL(ret);
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON *row = cJSON_CreateObject();
+
+        cJSON_AddStringToObject(row, "tag", (const char *) sqlite3_column_text(db->fts_get_tags, 0));
+        cJSON_AddNumberToObject(row, "count", sqlite3_column_int(db->fts_get_tags, 1));
+
+        cJSON_AddItemToArray(json, row);
+    } while (TRUE);
+
+    sqlite3_reset(db->fts_get_tags);
+
+    return json;
+}

src/database/database_schema.c

@@ -1,3 +1,77 @@
+const char *FtsDatabaseSchema =
+        "CREATE TABLE IF NOT EXISTS document_index ("
+        "   id TEXT NOT NULL,"
+        "   index_id TEXT NOT NULL,"
+        "   size INTEGER NOT NULL,"
+        "   name TEXT NOT NULL,"
+        "   path TEXT NOT NULL,"
+        "   mtime INTEGER NOT NULL,"
+        "   mime TEXT,"
+        "   json_data TEXT NOT NULL,"
+        "   PRIMARY KEY (id, index_id)"
+        ");"
+        ""
+        "CREATE TABLE IF NOT EXISTS stats ("
+        "   mtime_min INTEGER,"
+        "   mtime_max INTEGER"
+        ");"
+        ""
+        "CREATE TABLE IF NOT EXISTS path_index ("
+        "   path TEXT,"
+        "   index_id TEXT,"
+        "   count INTEGER NOT NULL,"
+        "   depth INTEGER NOT NULL,"
+        "   PRIMARY KEY (path, index_id)"
+        ");"
+        ""
+        "CREATE TABLE IF NOT EXISTS mime_index ("
+        "   index_id TEXT,"
+        "   mime TEXT,"
+        "   count INT,"
+        "   PRIMARY KEY(index_id, mime)"
+        ");"
+        ""
+        "CREATE TABLE IF NOT EXISTS tag ("
+        "   id TEXT NOT NULL,"
+        "   tag TEXT NOT NULL,"
+        "   PRIMARY KEY (id, tag)"
+        ");"
+        "CREATE INDEX IF NOT EXISTS tag_tag_idx ON tag(tag);"
+        "CREATE INDEX IF NOT EXISTS tag_id_idx ON tag(id);"
+        "CREATE TRIGGER IF NOT EXISTS tag_write_trigger"
+        "    AFTER INSERT ON tag"
+        " BEGIN"
+        "  UPDATE document_index"
+        "  SET json_data = json_set(json_data, '$.tag', (SELECT json_group_array(tag) FROM tag WHERE id = NEW.id))"
+        "  WHERE id = NEW.id;"
+        " END;"
+        ""
+        "CREATE TRIGGER IF NOT EXISTS tag_delete_trigger"
+        " AFTER DELETE ON tag"
+        " BEGIN"
+        "  UPDATE document_index"
+        "  SET json_data = json_set(json_data, '$.tag', (SELECT json_group_array(tag) FROM tag WHERE id = OLD.id))"
+        "  WHERE id = OLD.id;"
+        " END;"
+        ""
+        "CREATE VIEW IF NOT EXISTS document_view (id, name, content, title)"
+        " AS"
+        " SELECT rowid,"
+        "  json_data->>'name',"
+        "  json_data->>'content',"
+        "  json_data->>'title'"
+        " FROM document_index;"
+        ""
+        "CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5 ("
+        "   name,"
+        "   content,"
+        "   title,"
+        "   content='document_view',"
+        "   content_rowid='id'"
+        ");"
+        // name^8, content^3, title^8
+        "INSERT INTO search(search, rank) VALUES('rank', 'bm25(8, 3, 8)');"
+        "";
 
 const char *IpcDatabaseSchema =
         "CREATE TABLE parse_job ("
@@ -36,7 +110,8 @@ const char *IndexDatabaseSchema =
         ""
         "CREATE TABLE tag ("
         "   id TEXT NOT NULL,"
-        "   tag TEXT NOT NULL"
+        "   tag TEXT NOT NULL,"
+        "   PRIMARY KEY (id, tag)"
         ");"
         ""
         "CREATE TABLE document_sidecar ("

src/database/database_stats.c

@@ -238,5 +238,7 @@ cJSON *database_get_stats(database_t *db, database_stat_type_d type) {
         cJSON_AddItemToArray(json, row);
     } while (TRUE);
 
+    sqlite3_finalize(stmt);
+
     return json;
 }

src/index/elastic.c

@@ -92,8 +92,8 @@ void index_json(cJSON *document, const char doc_id[SIST_DOC_ID_LEN]) {
 
     cJSON_free(json);
     tpool_add_work(IndexCtx.pool, &(job_t) {
-        .type = JOB_BULK_LINE,
-        .bulk_line = bulk_line,
+            .type = JOB_BULK_LINE,
+            .bulk_line = bulk_line,
     });
     free(bulk_line);
 }
@@ -240,7 +240,10 @@ void print_errors(response_t *r) {
     } else if (errors->valueint != 0) {
         cJSON *err;
         cJSON_ArrayForEach(err, cJSON_GetObjectItem(ret_json, "items")) {
-            if (cJSON_GetObjectItem(cJSON_GetObjectItem(err, "index"), "status")->valueint != 201) {
+
+            int status = cJSON_GetObjectItem(cJSON_GetObjectItem(err, "index"), "status")->valueint;
+
+            if (status != 201 && status != 200) {
                 char *str = cJSON_Print(err);
                 LOG_ERRORF("elastic.c", "%s\n", str);
                 cJSON_free(str);
@@ -606,4 +609,4 @@ char *elastic_get_status() {
     free_response(r);
     cJSON_Delete(json);
     return status;
-}
+}

src/log.h

@@ -3,6 +3,7 @@
 
 
 #include <signal.h>
+
 #define LOG_MAX_LENGTH 8192
 
 #define LOG_SIST_DEBUG 0
@@ -33,11 +34,12 @@
 
 #define LOG_FATALF(filepath, fmt, ...)\
     sist_logf(filepath, LOG_SIST_FATAL, fmt, __VA_ARGS__);\
-    raise(SIGUSR1)
+    raise(SIGUSR1);                   \
+    exit(-1)
 #define LOG_FATAL(filepath, str) \
     sist_log(filepath, LOG_SIST_FATAL, str);\
-    exit(SIGUSR1)
-
+    raise(SIGUSR1);                   \
+    exit(-1)
 #define LOG_FATALF_NO_EXIT(filepath, fmt, ...) \
     sist_logf(filepath, LOG_SIST_FATAL, fmt, __VA_ARGS__)
 #define LOG_FATAL_NO_EXIT(filepath, str) \
@@ -46,6 +48,7 @@
 #include "sist.h"
 
 void sist_logf(const char *filepath, int level, char *format, ...);
+
 void vsist_logf(const char *filepath, int level, char *format, va_list ap);
 
 void sist_log(const char *filepath, int level, char *str);

src/main.c

@@ -22,6 +22,7 @@
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
+        "sist2 sqlite-index [OPTION]... INDEX",
         "sist2 web [OPTION]... INDEX...",
         "sist2 exec-script [OPTION]... INDEX",
         NULL,
@@ -351,6 +352,22 @@ void sist2_index(index_args_t *args) {
     free(desc);
 }
 
+void sist2_sqlite_index(sqlite_index_args_t *args) {
+    database_t *db = database_create(args->index_path, INDEX_DATABASE);
+    database_open(db);
+
+    database_t *search_db = database_create(args->search_index_path, FTS_DATABASE);
+    database_initialize(search_db);
+
+    database_fts_attach(db, args->search_index_path);
+
+    database_fts_index(db);
+    database_fts_optimize(db);
+
+    database_close(db, FALSE);
+    database_close(search_db, FALSE);
+}
+
 void sist2_exec_script(exec_args_t *args) {
     LogCtx.verbose = TRUE;
 
@@ -373,6 +390,7 @@ void sist2_exec_script(exec_args_t *args) {
 void sist2_web(web_args_t *args) {
 
     WebCtx.es_url = args->es_url;
+    WebCtx.search_backend = args->search_backend;
     WebCtx.es_index = args->es_index;
     WebCtx.es_insecure_ssl = args->es_insecure_ssl;
     WebCtx.index_count = args->index_count;
@@ -389,15 +407,27 @@ void sist2_web(web_args_t *args) {
     WebCtx.auth0_audience = args->auth0_audience;
     strcpy(WebCtx.lang, args->lang);
 
+    if (args->search_backend == SQLITE_SEARCH_BACKEND) {
+        WebCtx.search_db = database_create(args->search_index_path, FTS_DATABASE);
+        database_open(WebCtx.search_db);
+    }
+
     for (int i = 0; i < args->index_count; i++) {
         char *abs_path = abspath(args->indices[i]);
 
         strcpy(WebCtx.indices[i].path, abs_path);
 
-        WebCtx.indices[i].db = database_create(abs_path, INDEX_DATABASE);
-        database_open(WebCtx.indices[i].db);
+        database_t *db = database_create(abs_path, INDEX_DATABASE);
+        database_open(db);
+        if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
+            database_fts_attach(db, args->search_index_path);
+            database_fts_sync_tags(db);
+            database_fts_detach(db);
+        }
 
-        index_descriptor_t *desc = database_read_index_descriptor(WebCtx.indices[i].db);
+        WebCtx.indices[i].db = db;
+
+        index_descriptor_t *desc = database_read_index_descriptor(db);
         WebCtx.indices[i].desc = *desc;
         free(desc);
 
@@ -436,6 +466,7 @@ int main(int argc, const char *argv[]) {
     index_args_t *index_args = index_args_create();
     web_args_t *web_args = web_args_create();
     exec_args_t *exec_args = exec_args_create();
+    sqlite_index_args_t *sqlite_index_args = sqlite_index_args_create();
 
     int arg_version = 0;
 
@@ -445,6 +476,8 @@ int main(int argc, const char *argv[]) {
     char *common_script_path = NULL;
     int common_async_script = 0;
     int common_threads = 0;
+    int common_optimize_database = 0;
+    char *common_search_index = NULL;
 
     struct argparse_option options[] = {
             OPT_HELP(),
@@ -457,7 +490,7 @@ int main(int argc, const char *argv[]) {
             OPT_GROUP("Scan options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT: 1"),
             OPT_INTEGER('q', "thumbnail-quality", &scan_args->tn_quality,
-                        "Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2",
+                        "Thumbnail quality, on a scale of 0 to 100, 100 being the best. DEFAULT: 50",
                         set_to_negative_if_value_is_zero, (intptr_t) &scan_args->tn_quality),
             OPT_INTEGER(0, "thumbnail-size", &scan_args->tn_size,
                         "Thumbnail size, in pixels. DEFAULT: 552",
@@ -471,7 +504,7 @@ int main(int argc, const char *argv[]) {
             OPT_STRING('o', "output", &scan_args->output, "Output index file path. DEFAULT: index.sist2"),
             OPT_BOOLEAN(0, "incremental", &scan_args->incremental,
                         "If the output file path exists, only scan new or modified files."),
-            OPT_BOOLEAN(0, "optimize-index", &scan_args->optimize_database,
+            OPT_BOOLEAN(0, "optimize-index", &common_optimize_database,
                         "Defragment index file after scan to reduce its file size."),
             OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."),
             OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: index"),
@@ -520,10 +553,15 @@ int main(int argc, const char *argv[]) {
             OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 70"),
             OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings."),
 
+            OPT_GROUP("sqlite-index options"),
+            OPT_STRING(0, "search-index", &common_search_index, "Path to search index. Will be created if it does not exist yet."),
+
             OPT_GROUP("Web options"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT: http://localhost:9200"),
             OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl,
                         "Do not verify SSL connections to Elasticsearch."),
+    // TODO: change arg name (?)
+            OPT_STRING(0, "search-index", &common_search_index, "Path to SQLite search index."),
             OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT: sist2"),
             OPT_STRING(0, "bind", &web_args->listen_address,
                        "Listen for connections on this address. DEFAULT: localhost:4090"),
@@ -559,7 +597,7 @@ int main(int argc, const char *argv[]) {
     argc = argparse_parse(&argparse, argc, argv);
 
     if (arg_version) {
-        printf(Version);
+        printf("%s", Version);
         goto end;
     }
 
@@ -586,6 +624,11 @@ int main(int argc, const char *argv[]) {
     exec_args->async_script = common_async_script;
     index_args->async_script = common_async_script;
 
+    scan_args->optimize_database = common_optimize_database;
+
+    sqlite_index_args->search_index_path = common_search_index;
+    web_args->search_index_path = common_search_index;
+
     if (argc == 0) {
         argparse_usage(&argparse);
         goto end;
@@ -605,6 +648,14 @@ int main(int argc, const char *argv[]) {
         }
         sist2_index(index_args);
 
+    } else if (strcmp(argv[0], "sqlite-index") == 0) {
+
+        int err = sqlite_index_args_validate(sqlite_index_args, argc, argv);
+        if (err != 0) {
+            goto end;
+        }
+        sist2_sqlite_index(sqlite_index_args);
+
     } else if (strcmp(argv[0], "web") == 0) {
 
         int err = web_args_validate(web_args, argc, argv);

src/sist.h

@@ -51,11 +51,11 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.0.4"
+#define VERSION "3.1.1"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
-static const int VersionMinor = 0;
-static const int VersionPatch = 4;
+static const int VersionMinor = 1;
+static const int VersionPatch = 1;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown

src/util.c

@@ -162,9 +162,9 @@ const char *find_file_in_paths(const char *paths[], const char *filename) {
         }
 
         char path[PATH_MAX];
-        snprintf(path, sizeof(path), "%s%s", apath, filename);
+        snprintf(path, sizeof(path), "%s/%s", apath, filename);
 
-        LOG_DEBUGF("util.c", "Looking for '%s' in folder '%s'", filename, apath);
+        LOG_DEBUGF("util.c", "Looking for '%s' in folder '%s' (%s)", filename, apath, path);
         free(apath);
 
         struct stat info;

src/util.h

@@ -114,4 +114,7 @@ struct timespec timespec_add(struct timespec ts1, long usec);
         pthread_cond_timedwait(cond, mutex, &end_time); \
     } while (0)
 
+#define array_foreach(arr) \
+    for (int i = 0; (arr)[i] != NULL; i++)
+
 #endif

src/web/serve.c

@@ -5,11 +5,24 @@
 #include "src/index/web.h"
 #include "src/auth0/auth0_c_api.h"
 #include "src/web/web_util.h"
+#include "src/cli.h"
+#include <time.h>
 
 #include <src/ctx.h>
 
-#define HTTP_TEXT_TYPE_HEADER "Content-Type: text/plain;charset=utf-8\r\n"
-#define HTTP_REPLY_NOT_FOUND mg_http_reply(nc, 404, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Not found");
+void fts_search_paths(struct mg_connection *nc, struct mg_http_message *hm);
+
+void fts_search_mimetypes(struct mg_connection *nc, struct mg_http_message *hm);
+
+void fts_search_summary_stats(struct mg_connection *nc, struct mg_http_message *hm);
+
+void fts_search(struct mg_connection *nc, struct mg_http_message *hm);
+
+void fts_get_document(struct mg_connection *nc, struct mg_http_message *hm);
+
+void fts_suggest_tag(struct mg_connection *nc, struct mg_http_message *hm);
+
+void fts_get_tags(struct mg_connection *nc, struct mg_http_message *hm);
 
 static struct mg_http_serve_opts DefaultServeOpts = {
         .fs = NULL,
@@ -47,12 +60,8 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
     }
 
     cJSON *json = database_get_stats(db, stat_type);
-    char *json_str = cJSON_PrintUnformatted(json);
+    mg_send_json(nc, json);
 
-    web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
-    mg_send(nc, json_str, strlen(json_str));
-
-    free(json_str);
     cJSON_Delete(json);
 }
 
@@ -93,7 +102,7 @@ void serve_chunk_vendors_css(struct mg_connection *nc, struct mg_http_message *h
 }
 
 void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, const char *arg_index,
-        const char *arg_doc_id, int arg_num) {
+                     const char *arg_doc_id, int arg_num) {
 
     database_t *db = web_get_database(arg_index);
     if (db == NULL) {
@@ -252,6 +261,10 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
 }
 
 void cache_es_version() {
+    if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
+        return;
+    }
+
     static int is_cached = FALSE;
 
     if (is_cached == TRUE) {
@@ -321,6 +334,12 @@ void index_info(struct mg_connection *nc) {
         cJSON_AddItemToArray(arr, idx_json);
     }
 
+    if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
+        cJSON_AddStringToObject(json, "searchBackend", "sqlite");
+    } else {
+        cJSON_AddStringToObject(json, "searchBackend", "elasticsearch");
+    }
+
     char *json_str = cJSON_PrintUnformatted(json);
 
     web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
@@ -329,54 +348,63 @@ void index_info(struct mg_connection *nc) {
     cJSON_Delete(json);
 }
 
+cJSON *get_root_document_by_id(const char *index_id, const char *doc_id) {
+
+    database_t *db = web_get_database(index_id);
+    if (!db) {
+        return NULL;
+    }
+
+    char next_id[SIST_DOC_ID_LEN];
+    strcpy(next_id, doc_id);
+
+    while (TRUE) {
+        cJSON *doc = database_get_document(db, next_id);
+
+        if (doc == NULL) {
+            return NULL;
+        }
+
+        cJSON *parent = cJSON_GetObjectItem(doc, "parent");
+        if (parent == NULL || cJSON_IsNull(parent)) {
+            return doc;
+        }
+
+        strcpy(next_id, parent->valuestring);
+        cJSON_Delete(parent);
+    }
+}
 
 void file(struct mg_connection *nc, struct mg_http_message *hm) {
 
-    if (hm->uri.len != SIST_DOC_ID_LEN + 2) {
+    if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2) {
         LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr);
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
     char arg_doc_id[SIST_DOC_ID_LEN];
-    memcpy(arg_doc_id, hm->uri.ptr + 3, SIST_DOC_ID_LEN);
+    char arg_index[SIST_INDEX_ID_LEN];
+
+    memcpy(arg_index, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
+    *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
+    memcpy(arg_doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN);
     *(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0';
 
-    const char *next = arg_doc_id;
-    cJSON *doc = NULL;
-    cJSON *index_id = NULL;
-    cJSON *source = NULL;
-
-    while (true) {
-        doc = elastic_get_document(next);
-        source = cJSON_GetObjectItem(doc, "_source");
-        index_id = cJSON_GetObjectItem(source, "index");
-        if (index_id == NULL) {
-            cJSON_Delete(doc);
-            HTTP_REPLY_NOT_FOUND
-            return;
-        }
-        cJSON *parent = cJSON_GetObjectItem(source, "parent");
-        if (parent == NULL) {
-            break;
-        }
-        next = parent->valuestring;
-    }
-
-    index_t *idx = web_get_index_by_id(index_id->valuestring);
-
+    index_t *idx = web_get_index_by_id(arg_index);
     if (idx == NULL) {
-        cJSON_Delete(doc);
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
+    cJSON *source = get_root_document_by_id(arg_index, arg_doc_id);
+
     if (strlen(idx->desc.rewrite_url) == 0) {
         serve_file_from_disk(source, idx, nc, hm);
     } else {
         serve_file_from_url(source, idx, nc);
     }
-    cJSON_Delete(doc);
+    cJSON_Delete(source);
 }
 
 void status(struct mg_connection *nc) {
@@ -398,6 +426,10 @@ typedef struct {
 
 tag_req_t *parse_tag_request(cJSON *json) {
 
+    if (json == NULL) {
+        return NULL;
+    }
+
     if (!cJSON_IsObject(json)) {
         return NULL;
     }
@@ -425,115 +457,101 @@ tag_req_t *parse_tag_request(cJSON *json) {
     return req;
 }
 
+subreq_ctx_t *elastic_delete_tag(const tag_req_t *req) {
+    char *buf = malloc(sizeof(char) * 8192);
+    snprintf(buf, 8192,
+             "{"
+             "    \"script\" : {"
+             "        \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\","
+             "        \"lang\": \"painless\","
+             "        \"params\" : {"
+             "            \"tag\" : \"%s\""
+             "        }"
+             "    }"
+             "}", req->name
+    );
+
+    char url[4096];
+    snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, req->doc_id);
+
+    return web_post_async(url, buf, WebCtx.es_insecure_ssl);
+}
+
+subreq_ctx_t *elastic_write_tag(const tag_req_t *req) {
+    char *buf = malloc(sizeof(char) * 8192);
+    snprintf(buf, 8192,
+             "{"
+             "    \"script\" : {"
+             "        \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\","
+             "        \"lang\": \"painless\","
+             "        \"params\" : {"
+             "            \"tag\" : \"%s\""
+             "        }"
+             "    }"
+             "}", req->name
+    );
+
+    char url[4096];
+    snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, req->doc_id);
+    return web_post_async(url, buf, WebCtx.es_insecure_ssl);
+}
+
 void tag(struct mg_connection *nc, struct mg_http_message *hm) {
-//    if (hm->uri.len != SIST_INDEX_ID_LEN + 4) {
-//        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr)
-//        HTTP_REPLY_NOT_FOUND
-//        return;
-//    }
-//
-//    char arg_index[SIST_INDEX_ID_LEN];
-//    memcpy(arg_index, hm->uri.ptr + 5, SIST_INDEX_ID_LEN);
-//    *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
-//
-//    if (hm->body.len < 2 || hm->method.len != 4 || memcmp(&hm->method, "POST", 4) == 0) {
-//        LOG_DEBUG("serve.c", "Invalid tag request")
-//        HTTP_REPLY_NOT_FOUND
-//        return;
-//    }
-//
-//    store_t *store = get_tag_store(arg_index);
-//    if (store == NULL) {
-//        LOG_DEBUGF("serve.c", "Could not get tag store for index: %s", arg_index)
-//        HTTP_REPLY_NOT_FOUND
-//        return;
-//    }
-//
-//    char *body = malloc(hm->body.len + 1);
-//    memcpy(body, hm->body.ptr, hm->body.len);
-//    *(body + hm->body.len) = '\0';
-//    cJSON *json = cJSON_Parse(body);
-//
-//    tag_req_t *arg_req = parse_tag_request(json);
-//    if (arg_req == NULL) {
-//        LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index)
-//        cJSON_Delete(json);
-//        free(body);
-//        mg_http_reply(nc, 400, "", "Invalid request");
-//        return;
-//    }
-//
-//    cJSON *arr = NULL;
-//
-//    size_t data_len = 0;
-//    const char *data = store_read(store, arg_req->doc_id, SIST_DOC_ID_LEN, &data_len);
-//    if (data_len == 0) {
-//        arr = cJSON_CreateArray();
-//    } else {
-//        arr = cJSON_Parse(data);
-//    }
-//
-//    if (arg_req->delete) {
-//
-//        if (data_len > 0) {
-//            cJSON *element = NULL;
-//            int i = 0;
-//            cJSON_ArrayForEach(element, arr) {
-//                if (strcmp(element->valuestring, arg_req->name) == 0) {
-//                    cJSON_DeleteItemFromArray(arr, i);
-//                    break;
-//                }
-//                i++;
-//            }
-//        }
-//
-//        char *buf = malloc(sizeof(char) * 8192);
-//        snprintf(buf, 8192,
-//                 "{"
-//                 "    \"script\" : {"
-//                 "        \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\","
-//                 "        \"lang\": \"painless\","
-//                 "        \"params\" : {"
-//                 "            \"tag\" : \"%s\""
-//                 "        }"
-//                 "    }"
-//                 "}", arg_req->name
-//        );
-//
-//        char url[4096];
-//        snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
-//        nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl);
-//
-//    } else {
-//        cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name));
-//
-//        char *buf = malloc(sizeof(char) * 8192);
-//        snprintf(buf, 8192,
-//                 "{"
-//                 "    \"script\" : {"
-//                 "        \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\","
-//                 "        \"lang\": \"painless\","
-//                 "        \"params\" : {"
-//                 "            \"tag\" : \"%s\""
-//                 "        }"
-//                 "    }"
-//                 "}", arg_req->name
-//        );
-//
-//        char url[4096];
-//        snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
-//        nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl);
-//    }
-//
-//    char *json_str = cJSON_PrintUnformatted(arr);
-//    store_write(store, arg_req->doc_id, SIST_DOC_ID_LEN, json_str, strlen(json_str) + 1);
-//    store_flush(store);
-//
-//    free(arg_req);
-//    free(json_str);
-//    cJSON_Delete(json);
-//    cJSON_Delete(arr);
-//    free(body);
+    if (hm->uri.len != SIST_INDEX_ID_LEN + 4) {
+        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr);
+        HTTP_REPLY_NOT_FOUND
+        return;
+    }
+
+    char arg_index[SIST_INDEX_ID_LEN];
+    memcpy(arg_index, hm->uri.ptr + 5, SIST_INDEX_ID_LEN);
+    *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
+
+    char *body = malloc(hm->body.len + 1);
+    memcpy(body, hm->body.ptr, hm->body.len);
+    *(body + hm->body.len) = '\0';
+    cJSON *json = cJSON_Parse(body);
+    free(body);
+
+    if (json == NULL) {
+        HTTP_REPLY_BAD_REQUEST
+        return;
+    }
+
+    database_t *db = web_get_database(arg_index);
+    if (db == NULL) {
+        LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index);
+        HTTP_REPLY_NOT_FOUND
+        return;
+    }
+
+    tag_req_t *req = parse_tag_request(json);
+    cJSON_Delete(json);
+    if (req == NULL) {
+        LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index);
+        HTTP_REPLY_BAD_REQUEST
+        return;
+    }
+
+    if (req->delete) {
+        database_delete_tag(db, req->doc_id, req->name);
+        if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
+            database_delete_tag(WebCtx.search_db, req->doc_id, req->name);
+            HTTP_REPLY_OK
+        } else {
+            nc->fn_data = elastic_delete_tag(req);
+        }
+    } else {
+        database_write_tag(db, req->doc_id, req->name);
+        if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
+            database_write_tag(WebCtx.search_db, req->doc_id, req->name);
+            HTTP_REPLY_OK
+        } else {
+            nc->fn_data = elastic_write_tag(req);
+        }
+    }
+
+    free(req);
 }
 
 int validate_auth(struct mg_connection *nc, struct mg_http_message *hm) {
@@ -631,11 +649,39 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
             return;
         }
 
-        if (mg_http_match_uri(hm, "/es")) {
-            search(nc, hm);
-        } else if (mg_http_match_uri(hm, "/status")) {
+        if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
+            if (mg_http_match_uri(hm, "/fts/paths")) {
+                fts_search_paths(nc, hm);
+                return;
+            } else if (mg_http_match_uri(hm, "/fts/mimetypes")) {
+                fts_search_mimetypes(nc, hm);
+                return;
+            } else if (mg_http_match_uri(hm, "/fts/dateRange")) {
+                fts_search_summary_stats(nc, hm);
+                return;
+            } else if (mg_http_match_uri(hm, "/fts/search")) {
+                fts_search(nc, hm);
+                return;
+            } else if (mg_http_match_uri(hm, "/fts/d/*")) {
+                fts_get_document(nc, hm);
+                return;
+            } else if (mg_http_match_uri(hm, "/fts/suggestTags")) {
+                fts_suggest_tag(nc, hm);
+                return;
+            } else if (mg_http_match_uri(hm, "/fts/tags")) {
+                fts_get_tags(nc, hm);
+                return;
+            }
+        } else if (WebCtx.search_backend == ES_SEARCH_BACKEND) {
+            if (mg_http_match_uri(hm, "/es")) {
+                search(nc, hm);
+                return;
+            }
+        }
+
+        if (mg_http_match_uri(hm, "/status")) {
             status(nc);
-        } else if (mg_http_match_uri(hm, "/f/*")) {
+        } else if (mg_http_match_uri(hm, "/f/*/*")) {
             file(nc, hm);
         } else if (mg_http_match_uri(hm, "/t/*/*/*")) {
             thumbnail_with_num(nc, hm);
@@ -702,14 +748,12 @@ void serve(const char *listen_address) {
     struct mg_mgr mgr;
     mg_mgr_init(&mgr);
 
-    int ok = 1;
-
     struct mg_connection *nc = mg_http_listen(&mgr, listen_address, ev_router, NULL);
     if (nc == NULL) {
         LOG_FATALF("serve.c", "Couldn't bind web server on address %s", listen_address);
     }
 
-    while (ok) {
+    while (TRUE) {
         mg_mgr_poll(&mgr, 10);
     }
     mg_mgr_free(&mgr);

src/web/serve.h

@@ -3,6 +3,11 @@
 
 #include "src/sist.h"
 
+#define HTTP_TEXT_TYPE_HEADER "Content-Type: text/plain;charset=utf-8\r\n"
+#define HTTP_REPLY_NOT_FOUND mg_http_reply(nc, 404, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Not found");
+#define HTTP_REPLY_BAD_REQUEST mg_http_reply(nc, 400, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Invalid request");
+#define HTTP_REPLY_OK mg_http_reply(nc, 200, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "ok");
+
 void serve(const char *listen_address);
 
 #endif

src/web/web_fts.c

@@ -0,0 +1,378 @@
+#include "serve.h"
+#include <mongoose.h>
+#include "src/web/web_util.h"
+
+typedef struct {
+    char *index_id;
+    char *prefix;
+    int min_depth;
+    int max_depth;
+} fts_search_paths_req_t;
+
+typedef struct {
+    cJSON *val;
+    int invalid;
+} json_value;
+
+typedef struct {
+    char *query;
+    char *path;
+    fts_sort_t sort;
+    double size_min;
+    double size_max;
+    double date_min;
+    double date_max;
+    int page_size;
+    char **index_ids;
+    char **mime_types;
+    char **tags;
+    int sort_asc;
+    int seed;
+    char **after;
+    int fetch_aggregations;
+    int highlight;
+    int highlight_context_size;
+} fts_search_req_t;
+
+fts_sort_t get_sort_mode(const cJSON *req_sort) {
+    if (strcmp(req_sort->valuestring, "score") == 0) {
+        return FTS_SORT_SCORE;
+    } else if (strcmp(req_sort->valuestring, "size") == 0) {
+        return FTS_SORT_SIZE;
+    } else if (strcmp(req_sort->valuestring, "mtime") == 0) {
+        return FTS_SORT_MTIME;
+    } else if (strcmp(req_sort->valuestring, "random") == 0) {
+        return FTS_SORT_RANDOM;
+    } else if (strcmp(req_sort->valuestring, "name") == 0) {
+        return FTS_SORT_NAME;
+    }
+
+    return FTS_SORT_INVALID;
+}
+
+
+static json_value get_json_string(cJSON *object, const char *name) {
+
+    cJSON *item = cJSON_GetObjectItem(object, name);
+    if (item == NULL || cJSON_IsNull(item)) {
+        return (json_value) {NULL, FALSE};
+    }
+    if (!cJSON_IsString(item)) {
+        return (json_value) {NULL, TRUE};
+    }
+
+    return (json_value) {item, FALSE};
+}
+
+static json_value get_json_number(cJSON *object, const char *name) {
+
+    cJSON *item = cJSON_GetObjectItem(object, name);
+    if (item == NULL || cJSON_IsNull(item)) {
+        return (json_value) {NULL, FALSE};
+    }
+    if (!cJSON_IsNumber(item)) {
+        return (json_value) {NULL, TRUE};
+    }
+
+    return (json_value) {item, FALSE};
+}
+
+static json_value get_json_bool(cJSON *object, const char *name) {
+    cJSON *item = cJSON_GetObjectItem(object, name);
+    if (item == NULL || cJSON_IsNull(item)) {
+        return (json_value) {NULL, FALSE};
+    }
+    if (!cJSON_IsBool(item)) {
+        return (json_value) {NULL, TRUE};
+    }
+
+    return (json_value) {item, FALSE};
+}
+
+static json_value get_json_array(cJSON *object, const char *name) {
+    cJSON *item = cJSON_GetObjectItem(object, name);
+    if (item == NULL || cJSON_IsNull(item)) {
+        return (json_value) {NULL, FALSE};
+    }
+    if (!cJSON_IsArray(item) || cJSON_GetArraySize(item) == 0) {
+        return (json_value) {NULL, TRUE};
+    }
+
+    cJSON *elem;
+    cJSON_ArrayForEach(elem, item) {
+        if (!cJSON_IsString(elem)) {
+            return (json_value) {NULL, TRUE};
+        }
+    }
+
+    return (json_value) {item, FALSE};
+}
+
+char **json_array_to_c_array(cJSON *json) {
+
+    cJSON *element;
+    char **arr = calloc(cJSON_GetArraySize(json) + 1, sizeof(char *));
+    int i = 0;
+    cJSON_ArrayForEach(element, json) {
+        arr[i++] = strdup(element->valuestring);
+    }
+
+    return arr;
+}
+
+#define DEFAULT_HIGHLIGHT_CONTEXT_SIZE 20
+
+fts_search_req_t *get_search_req(struct mg_http_message *hm) {
+    cJSON *json = web_get_json_body(hm);
+
+    if (json == NULL) {
+        return NULL;
+    }
+
+    json_value req_query, req_path, req_size_min, req_size_max, req_date_min, req_date_max, req_page_size,
+            req_index_ids, req_mime_types, req_tags, req_sort_asc, req_sort, req_seed, req_after,
+            req_fetch_aggregations, req_highlight, req_highlight_context_size;
+
+    if (!cJSON_IsObject(json) ||
+        (req_query = get_json_string(json, "query")).invalid ||
+        (req_path = get_json_string(json, "path")).invalid ||
+        (req_sort = get_json_string(json, "sort")).val == NULL ||
+        (req_size_min = get_json_number(json, "sizeMin")).invalid ||
+        (req_size_max = get_json_number(json, "sizeMax")).invalid ||
+        (req_date_min = get_json_number(json, "dateMin")).invalid ||
+        (req_date_max = get_json_number(json, "dateMax")).invalid ||
+        (req_page_size = get_json_number(json, "pageSize")).val == NULL ||
+        (req_after = get_json_array(json, "after")).invalid ||
+        (req_seed = get_json_number(json, "seed")).invalid ||
+        (req_fetch_aggregations = get_json_bool(json, "fetchAggregations")).invalid ||
+        (req_sort_asc = get_json_bool(json, "sortAsc")).invalid ||
+        (req_index_ids = get_json_array(json, "indexIds")).invalid ||
+        (req_mime_types = get_json_array(json, "mimeTypes")).invalid ||
+        (req_highlight = get_json_bool(json, "highlight")).invalid ||
+        (req_highlight_context_size = get_json_number(json, "highlightContextSize")).invalid ||
+        (req_tags = get_json_array(json, "tags")).invalid) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+
+    int index_id_count = cJSON_GetArraySize(req_index_ids.val);
+    if (index_id_count > 999) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+    int mime_count = req_mime_types.val ? 0 : cJSON_GetArraySize(req_mime_types.val);
+    if (mime_count > 999) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+    int tag_count = req_tags.val ? 0 : cJSON_GetArraySize(req_tags.val);
+    if (tag_count > 9999) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+    if (req_path.val && (strstr(req_path.val->valuestring, "*") || strlen(req_path.val->valuestring) >= PATH_MAX)) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+
+    fts_sort_t sort = get_sort_mode(req_sort.val);
+    if (sort == FTS_SORT_INVALID) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+
+    if (req_after.val && cJSON_GetArraySize(req_after.val) != 2) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+
+    if (req_page_size.val->valueint > 1000 || req_page_size.val->valueint < 0) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+    if (req_highlight_context_size.val->valueint < 0) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+
+    fts_search_req_t *req = malloc(sizeof(fts_search_req_t));
+
+    req->sort = sort;
+    req->query = req_query.val ? strdup(req_query.val->valuestring) : NULL;
+    req->path = req_path.val ? strdup(req_path.val->valuestring) : NULL;
+    req->size_min = req_size_min.val ? req_size_min.val->valuedouble : 0;
+    req->size_max = req_size_max.val ? req_size_max.val->valuedouble : 0;
+    req->seed = (int) (req_seed.val ? req_seed.val->valuedouble : 0);
+    req->date_min = req_date_min.val ? req_date_min.val->valuedouble : 0;
+    req->date_max = req_date_max.val ? req_date_max.val->valuedouble : 0;
+    req->page_size = (int) req_page_size.val->valuedouble;
+    req->sort_asc = req_sort_asc.val ? req_sort_asc.val->valueint : TRUE;
+    req->index_ids = req_index_ids.val ? json_array_to_c_array(req_index_ids.val) : NULL;
+    req->after = req_after.val ? json_array_to_c_array(req_after.val) : NULL;
+    req->mime_types = req_mime_types.val ? json_array_to_c_array(req_mime_types.val) : NULL;
+    req->tags = req_tags.val ? json_array_to_c_array(req_tags.val) : NULL;
+    req->fetch_aggregations = req_fetch_aggregations.val ? req_fetch_aggregations.val->valueint : FALSE;
+    req->highlight = req_highlight.val ? req_highlight.val->valueint : FALSE;
+    req->highlight_context_size = req_highlight_context_size.val
+                                  ? req_highlight_context_size.val->valueint
+                                  : DEFAULT_HIGHLIGHT_CONTEXT_SIZE;
+
+    cJSON_Delete(json);
+
+    return req;
+}
+
+void destroy_array(char **array) {
+    if (array == NULL) {
+        return;
+    }
+    array_foreach(array) { free(array[i]); }
+    free(array);
+}
+
+void destroy_search_req(fts_search_req_t *req) {
+    free(req->query);
+    free(req->path);
+
+    destroy_array(req->index_ids);
+    destroy_array(req->mime_types);
+    destroy_array(req->tags);
+
+    free(req);
+}
+
+fts_search_paths_req_t *get_search_paths_req(struct mg_http_message *hm) {
+    cJSON *json = web_get_json_body(hm);
+
+    if (json == NULL) {
+        return NULL;
+    }
+
+    json_value req_index_id, req_min_depth, req_max_depth, req_prefix;
+
+    if (!cJSON_IsObject(json) ||
+        (req_index_id = get_json_string(json, "indexId")).invalid ||
+        (req_prefix = get_json_string(json, "prefix")).invalid ||
+        (req_min_depth = get_json_number(json, "minDepth")).val == NULL ||
+        (req_max_depth = get_json_number(json, "maxDepth")).val == NULL) {
+        cJSON_Delete(json);
+        return NULL;
+    }
+
+    fts_search_paths_req_t *req = malloc(sizeof(fts_search_paths_req_t));
+
+    req->index_id = req_index_id.val ? strdup(req_index_id.val->valuestring) : NULL;
+    req->min_depth = req_min_depth.val->valueint;
+    req->max_depth = req_max_depth.val->valueint;
+    req->prefix = req_prefix.val ? strdup(req_prefix.val->valuestring) : NULL;
+
+    cJSON_Delete(json);
+    return req;
+}
+
+void destroy_search_paths_req(fts_search_paths_req_t *req) {
+    if (req->index_id) {
+        free(req->index_id);
+    }
+    if (req->prefix) {
+        free(req->prefix);
+    }
+    free(req);
+}
+
+void fts_search_paths(struct mg_connection *nc, struct mg_http_message *hm) {
+
+    fts_search_paths_req_t *req = get_search_paths_req(hm);
+    if (req == NULL) {
+        HTTP_REPLY_BAD_REQUEST
+        return;
+    }
+
+    cJSON *json = database_fts_get_paths(WebCtx.search_db, req->index_id, req->min_depth,
+                                         req->max_depth, req->prefix, req->max_depth == 10000);
+
+    destroy_search_paths_req(req);
+    mg_send_json(nc, json);
+    cJSON_Delete(json);
+}
+
+void fts_search_mimetypes(struct mg_connection *nc, struct mg_http_message *hm) {
+
+    cJSON *json = database_fts_get_mimetypes(WebCtx.search_db);
+
+    mg_send_json(nc, json);
+    cJSON_Delete(json);
+}
+
+void fts_search_summary_stats(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
+
+    database_summary_stats_t stats = database_fts_get_date_range(WebCtx.search_db);
+
+    cJSON *json = cJSON_CreateObject();
+
+    cJSON_AddNumberToObject(json, "dateMin", stats.date_min);
+    cJSON_AddNumberToObject(json, "dateMax", stats.date_max);
+
+    mg_send_json(nc, json);
+    cJSON_Delete(json);
+}
+
+void fts_search(struct mg_connection *nc, struct mg_http_message *hm) {
+
+    fts_search_req_t *req = get_search_req(hm);
+    if (req == NULL) {
+        HTTP_REPLY_BAD_REQUEST
+        return;
+    }
+
+    cJSON *json = database_fts_search(WebCtx.search_db, req->query, req->path,
+                                      (long) req->size_min, (long) req->size_max,
+                                      (long) req->date_min, (long) req->date_max,
+                                      req->page_size, req->index_ids, req->mime_types,
+                                      req->tags, req->sort_asc, req->sort, req->seed,
+                                      req->after, req->fetch_aggregations, req->highlight,
+                                      req->highlight_context_size);
+
+    destroy_search_req(req);
+    mg_send_json(nc, json);
+    cJSON_Delete(json);
+}
+
+void fts_get_document(struct mg_connection *nc, struct mg_http_message *hm) {
+
+    char doc_id[SIST_DOC_ID_LEN];
+    memcpy(doc_id, hm->uri.ptr + 7, SIST_INDEX_ID_LEN);
+    *(doc_id + SIST_INDEX_ID_LEN - 1) = '\0';
+
+    cJSON *json = database_fts_get_document(WebCtx.search_db, doc_id);
+
+    if (!json) {
+        HTTP_REPLY_NOT_FOUND
+        return;
+    }
+
+    mg_send_json(nc, json);
+    cJSON_Delete(json);
+}
+
+void fts_suggest_tag(struct mg_connection *nc, struct mg_http_message *hm) {
+    char *body = web_get_string_body(hm);
+
+    if (body == NULL) {
+        HTTP_REPLY_BAD_REQUEST
+        return;
+    }
+
+    cJSON *json = database_fts_suggest_tag(WebCtx.search_db, body);
+
+    mg_send_json(nc, json);
+    cJSON_Delete(json);
+    free(body);
+}
+
+void fts_get_tags(struct mg_connection *nc, struct mg_http_message *hm) {
+    cJSON *json = database_fts_get_tags(WebCtx.search_db);
+
+    mg_send_json(nc, json);
+    cJSON_Delete(json);
+}

src/web/web_util.c

@@ -61,3 +61,38 @@ void web_send_headers(struct mg_connection *nc, int status_code, size_t length,
             extra_headers
     );
 }
+cJSON *web_get_json_body(struct mg_http_message *hm) {
+    if (hm->body.len == 0) {
+        return NULL;
+    }
+
+    char *body = malloc(hm->body.len + 1);
+    memcpy(body, hm->body.ptr, hm->body.len);
+    *(body + hm->body.len) = '\0';
+    cJSON *json = cJSON_Parse(body);
+    free(body);
+
+    return json;
+}
+
+char *web_get_string_body(struct mg_http_message *hm) {
+    if (hm->body.len == 0) {
+        return NULL;
+    }
+
+    char *body = malloc(hm->body.len + 1);
+    memcpy(body, hm->body.ptr, hm->body.len);
+    *(body + hm->body.len) = '\0';
+
+    return body;
+}
+
+void mg_send_json(struct mg_connection *nc, const cJSON *json) {
+    char *json_str = cJSON_PrintUnformatted(json);
+
+    web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
+    mg_send(nc, json_str, strlen(json_str));
+
+    free(json_str);
+}
+

src/web/web_util.h

@@ -14,10 +14,9 @@ database_t *web_get_database(const char *index_id);
 
 __always_inline
 static char *web_address_to_string(struct mg_addr *addr) {
-    return "TODO";
-//    static char address_to_string_buf[INET6_ADDRSTRLEN];
-//
-//    return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf));
+    static char address_to_string_buf[INET6_ADDRSTRLEN];
+
+    return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf));
 }
 
 void web_send_headers(struct mg_connection *nc, int status_code, size_t length, char *extra_headers);
@@ -29,4 +28,8 @@ void web_serve_asset_favicon_ico(struct mg_connection *nc);
 void web_serve_asset_style_css(struct mg_connection *nc);
 void web_serve_asset_chunk_vendors_css(struct mg_connection *nc);
 
+cJSON *web_get_json_body(struct mg_http_message *hm);
+char *web_get_string_body(struct mg_http_message *hm);
+void mg_send_json(struct mg_connection *nc, const cJSON *json);
+
 #endif //SIST2_WEB_UTIL_H

third-party/libscan/libscan/arc/arc.c

@@ -4,7 +4,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <fcntl.h>
-#include <openssl/evp.h>
 #include <pcre.h>
 
 #define MAX_DECOMPRESSED_SIZE_RATIO 40.0
@@ -211,11 +210,20 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
 
                 double decompressed_size_ratio = (double) sub_job->vfile.st_size / (double) f->st_size;
                 if (decompressed_size_ratio > MAX_DECOMPRESSED_SIZE_RATIO) {
-                    CTX_LOG_DEBUGF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)", sub_job->filepath,
+                    CTX_LOG_ERRORF("arc.c", "Skipped %s, possible zip bomb (decompressed_size_ratio=%f)",
+                                   sub_job->filepath,
                                    decompressed_size_ratio);
                     break;
                 }
 
+                if ((archive_entry_is_encrypted(entry) || archive_entry_is_data_encrypted(entry) ||
+                     archive_entry_is_metadata_encrypted(entry)) && ctx->passphrase[0] == 0) {
+                    // Is encrypted but no password is specified, skip
+                    CTX_LOG_ERRORF("arc.c", "Skipped %s, archive is encrypted but no passphrase is supplied",
+                                   doc->filepath);
+                    break;
+                }
+
                 // Handle excludes
                 if (exclude != NULL && EXCLUDED(sub_job->filepath)) {
                     CTX_LOG_DEBUGF("arc.c", "Excluded: %s", sub_job->filepath);

third-party/libscan/libscan/ebook/ebook.c

@@ -153,22 +153,23 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
 
     sws_freeContext(sws_ctx);
 
-    // YUV420p -> JPEG
-    AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(pixmap->w, pixmap->h, ctx->tn_qscale);
-    avcodec_send_frame(jpeg_encoder, scaled_frame);
+    // YUV420p -> JPEG/WEBP
+    AVCodecContext *thumbnail_encoder = alloc_webp_encoder(pixmap->w, pixmap->h, ctx->tn_qscale);
+    avcodec_send_frame(thumbnail_encoder, scaled_frame);
+    avcodec_send_frame(thumbnail_encoder, NULL); // Send EOF
 
-    AVPacket jpeg_packet;
-    av_init_packet(&jpeg_packet);
-    avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
+    AVPacket thumbnail_packet;
+    av_init_packet(&thumbnail_packet);
+    avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
 
     APPEND_LONG_META(doc, MetaThumbnail, 1);
-    ctx->store(doc->doc_id, 0, (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store(doc->doc_id, 0, (char *) thumbnail_packet.data, thumbnail_packet.size);
 
     free(samples);
-    av_packet_unref(&jpeg_packet);
+    av_packet_unref(&thumbnail_packet);
     av_free(*scaled_frame->data);
     av_frame_free(&scaled_frame);
-    avcodec_free_context(&jpeg_encoder);
+    avcodec_free_context(&thumbnail_encoder);
 
     fz_drop_pixmap(fzctx, pixmap);
     fz_drop_page(fzctx, cover);

third-party/libscan/libscan/media/media.c

@@ -68,7 +68,7 @@ void *scale_frame(const AVCodecContext *decoder, const AVFrame *frame, int size)
 
     struct SwsContext *sws_ctx = sws_getContext(
             decoder->width, decoder->height, decoder->pix_fmt,
-            dstW, dstH, AV_PIX_FMT_YUVJ420P,
+            dstW, dstH, AV_PIX_FMT_YUV420P,
             SIST_SWS_ALGO, 0, 0, 0
     );
 
@@ -436,7 +436,8 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
         }
 
         if (seek_ok == FALSE && thumbnail_index != 0) {
-            CTX_LOG_WARNING(doc->filepath, "(media.c) Could not seek media file. Can't generate additional thumbnails.");
+            CTX_LOG_WARNING(doc->filepath,
+                            "(media.c) Could not seek media file. Can't generate additional thumbnails.");
             return SAVE_THUMBNAIL_FAILED;
         }
     }
@@ -470,18 +471,19 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
 
         ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size);
     } else {
-        // Encode frame to jpeg
-        AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
-                                                          ctx->tn_qscale);
-        avcodec_send_frame(jpeg_encoder, scaled_frame);
+        // Encode frame
+        AVCodecContext *thumbnail_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height,
+                                                               ctx->tn_qscale);
+        avcodec_send_frame(thumbnail_encoder, scaled_frame);
+        avcodec_send_frame(thumbnail_encoder, NULL); // send EOF
 
-        AVPacket jpeg_packet;
-        av_init_packet(&jpeg_packet);
-        avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
+        AVPacket thumbnail_packet;
+        av_init_packet(&thumbnail_packet);
+        avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
 
         // Save thumbnail
         if (thumbnail_index == 0) {
-            ctx->store(doc->doc_id, 0, jpeg_packet.data, jpeg_packet.size);
+            ctx->store(doc->doc_id, 0, thumbnail_packet.data, thumbnail_packet.size);
             return_value = SAVE_THUMBNAIL_OK;
 
         } else if (thumbnail_index > 1) {
@@ -489,15 +491,15 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
             //  I figure out a better fix.
             thumbnail_index -= 1;
 
-            ctx->store(doc->doc_id, thumbnail_index, jpeg_packet.data, jpeg_packet.size);
+            ctx->store(doc->doc_id, thumbnail_index, thumbnail_packet.data, thumbnail_packet.size);
 
             return_value = SAVE_THUMBNAIL_OK;
         } else {
             return_value = SAVE_THUMBNAIL_SKIPPED;
         }
 
-        avcodec_free_context(&jpeg_encoder);
-        av_packet_unref(&jpeg_packet);
+        avcodec_free_context(&thumbnail_encoder);
+        av_packet_unref(&thumbnail_packet);
         av_free(*scaled_frame->data);
         av_frame_free(&scaled_frame);
     }
@@ -854,9 +856,10 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, docu
         ctx->store(doc->doc_id, 0, frame_and_packet->packet->data, frame_and_packet->packet->size);
     } else {
         // Encode frame to jpeg
-        AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
+        AVCodecContext *jpeg_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height,
                                                           ctx->tn_qscale);
         avcodec_send_frame(jpeg_encoder, scaled_frame);
+        avcodec_send_frame(jpeg_encoder, NULL); // Send EOF
 
         AVPacket jpeg_packet;
         av_init_packet(&jpeg_packet);

third-party/libscan/libscan/media/media.h

@@ -48,6 +48,28 @@ static AVCodecContext *alloc_jpeg_encoder(int w, int h, int qscale) {
     return jpeg;
 }
 
+static AVCodecContext *alloc_webp_encoder(int w, int h, int qscale) {
+
+    const AVCodec *webp_codec = avcodec_find_encoder(AV_CODEC_ID_WEBP);
+    AVCodecContext *webp = avcodec_alloc_context3(webp_codec);
+    webp->width = w;
+    webp->height = h;
+    webp->time_base.den = 1000000;
+    webp->time_base.num = 1;
+    webp->compression_level = 6;
+    webp->global_quality = FF_QP2LAMBDA * qscale;
+
+    webp->pix_fmt = AV_PIX_FMT_YUV420P;
+    webp->color_range = AVCOL_RANGE_JPEG;
+    int ret = avcodec_open2(webp, webp_codec, NULL);
+
+    if (ret != 0) {
+        return NULL;
+    }
+
+    return webp;
+}
+
 
 void parse_media(scan_media_ctx_t *ctx, vfile_t *f, document_t *doc, const char *mime_str);
 

third-party/libscan/libscan/raw/raw.c

@@ -52,7 +52,7 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
 
     struct SwsContext *sws_ctx = sws_getContext(
             img->width, img->height, AV_PIX_FMT_RGB24,
-            dstW, dstH, AV_PIX_FMT_YUVJ420P,
+            dstW, dstH, AV_PIX_FMT_YUV420P,
             SIST_SWS_ALGO, 0, 0, 0
     );
 
@@ -76,20 +76,21 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
 
     sws_freeContext(sws_ctx);
 
-    AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height, 1.0f);
-    avcodec_send_frame(jpeg_encoder, scaled_frame);
+    AVCodecContext *thumbnail_encoder = alloc_webp_encoder(scaled_frame->width, scaled_frame->height, ctx->tn_qscale);
+    avcodec_send_frame(thumbnail_encoder, scaled_frame);
+    avcodec_send_frame(thumbnail_encoder, NULL); // Send EOF
 
-    AVPacket jpeg_packet;
-    av_init_packet(&jpeg_packet);
-    avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
+    AVPacket thumbnail_packet;
+    av_init_packet(&thumbnail_packet);
+    avcodec_receive_packet(thumbnail_encoder, &thumbnail_packet);
 
     APPEND_LONG_META(doc, MetaThumbnail, 1);
-    ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store((char *) doc->doc_id, sizeof(doc->doc_id), (char *) thumbnail_packet.data, thumbnail_packet.size);
 
-    av_packet_unref(&jpeg_packet);
+    av_packet_unref(&thumbnail_packet);
     av_free(*scaled_frame->data);
     av_frame_free(&scaled_frame);
-    avcodec_free_context(&jpeg_encoder);
+    avcodec_free_context(&thumbnail_encoder);
 
     return TRUE;
 }