Update CI build script

.gitattributes

@@ -0,0 +1,3 @@
+CMakeModules/* linguist-vendored
+**/*_generated.c linguist-vendored
+**/*_generated.h linguist-vendored

.gitignore

@@ -43,6 +43,4 @@ src/magic_generated.c
 src/index/static_generated.c
 *.sist2
 *-shm
-*-journal
-.vscode
-*.fts
+*-journal

CMakeLists.txt

@@ -5,7 +5,6 @@ set(CMAKE_C_STANDARD 11)
 
 option(SIST_DEBUG "Build a debug executable" on)
 option(SIST_FAST "Enable more optimisation flags" off)
-option(SIST_DEBUG_INFO "Turn on debug information in web interface" on)
 
 add_compile_definitions(
         "SIST_PLATFORM=${SIST_PLATFORM}"
@@ -15,24 +14,13 @@ if (SIST_DEBUG)
     add_compile_definitions(
             "SIST_DEBUG=${SIST_DEBUG}"
     )
-    set(VCPKG_BUILD_TYPE debug)
-else ()
-    set(VCPKG_BUILD_TYPE release)
-endif ()
-
-if (SIST_DEBUG_INFO)
-    add_compile_definitions(
-            "SIST_DEBUG_INFO=${SIST_DEBUG_INFO}"
-    )
-endif ()
-
+endif()
 
 add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
-add_executable(
-        sist2
+add_executable(sist2
         # argparse
         third-party/argparse/argparse.h third-party/argparse/argparse.c
 
@@ -59,11 +47,7 @@ add_executable(
 
         src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp
 
-        src/database/database_stats.c
-        src/database/database_schema.c
-        src/database/database_fts.c
-        src/web/web_fts.c
-)
+        src/database/database_stats.c src/database/database_stats.h src/database/database_schema.c)
 set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
@@ -101,7 +85,7 @@ if (SIST_DEBUG)
             -fno-omit-frame-pointer
             -fsanitize=address
             -fno-inline
-            #            -O2
+#            -O2
     )
     target_link_options(
             sist2
@@ -131,7 +115,6 @@ else ()
             PRIVATE
 
             -Ofast
-#            -g
             -fno-stack-protector
             -fomit-frame-pointer
             -w

Dockerfile

@@ -19,12 +19,13 @@ COPY sist2-admin sist2-admin
 RUN cd sist2-vue/ && npm install && npm run build
 RUN cd sist2-admin/frontend/ && npm install && npm run build
 
-RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
 RUN cd build && make -j$(nproc)
 RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
 
+WORKDIR /root
 
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
@@ -36,22 +37,21 @@ RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libasan5 li
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
-    curl -o /usr/share/tesseract-ocr/4.00/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
+    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tessdata/osd.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/osd.traineddata &&\
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata &&\
+    curl -o /usr/share/tessdata/deu.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/deu.traineddata &&\
+    curl -o /usr/share/tessdata/equ.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/equ.traineddata &&\
+    curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
 COPY --from=build /build/build/sist2 /root/sist2
 
 # sist2-admin
-WORKDIR /root/sist2-admin
-COPY sist2-admin/requirements.txt /root/sist2-admin/
-RUN python3 -m pip install --no-cache -r /root/sist2-admin/requirements.txt
-COPY --from=build /build/sist2-admin/ /root/sist2-admin/
+COPY sist2-admin/requirements.txt sist2-admin/
+RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
+COPY --from=build /build/sist2-admin/ sist2-admin/

Dockerfile.arm64

@@ -3,7 +3,7 @@ MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 ADD . /build/
-RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
 RUN cd build && make -j$(nproc)
 RUN strip build/sist2 || mv build/sist2_debug build/sist2
 

README.md

@@ -10,13 +10,13 @@ sist2 (Simple incremental search tool)
 
 *Warning: sist2 is in early development*
 
-![search panel](docs/sist2.gif)
+![search panel](docs/sist2.png)
 
 ## Features
 
 * Fast, low memory usage, multi-threaded
-* Manage & schedule scan jobs with simple web interface (Docker only)
 * Mobile-friendly Web interface
+* Portable (all its features are packaged in a single executable)
 * Extracts text and metadata from common file types \*
 * Generates thumbnails \*
 * Incremental scanning
@@ -24,64 +24,47 @@ sist2 (Simple incremental search tool)
 * Recursive scan inside archive files \*\*
 * OCR support with tesseract \*\*\*
 * Stats page & disk utilisation visualization
-* Named-entity recognition (client-side) \*\*\*\*
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
 \*\*\* See [OCR](#ocr)
-\*\*\*\* See [Named-Entity Recognition](#NER)
+
+![stats](docs/stats.png)
 
 ## Getting Started
 
-### Using Docker Compose *(Windows/Linux/Mac)*
+1. Have an Elasticsearch (>= 6.8.X, ideally >=7.14.0) instance running
+    1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
+    1. *(or)* Run using docker:
+        ```bash
+        docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
+        ```
+    1. *(or)* Run using docker-compose:
+        ```yaml
+          elasticsearch:
+            image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9
+            environment:
+              - discovery.type=single-node
+              - "ES_JAVA_OPTS=-Xms1G -Xmx2G"
+        ```
+1. Download sist2 executable
+    1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases). 
+Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x` *
+    2. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
+       recommended!)*
+    3. *(or)* `docker pull simon987/sist2:2.12.1-x64-linux`
 
-```yaml
-version: "3"
+1. See [Usage guide](docs/USAGE.md)
 
-services:
-  elasticsearch:
-    image: elasticsearch:7.17.9
-    restart: unless-stopped
-    environment:
-      - "discovery.type=single-node"
-      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
-  sist2-admin:
-    image: simon987/sist2:3.0.7-x64-linux
-    restart: unless-stopped
-    volumes:
-      - ./sist2-admin-data/:/sist2-admin/
-      - /:/host
-    ports:
-      - 4090:4090 # sist2
-      - 8080:8080 # sist2-admin
-    working_dir: /root/sist2-admin/
-    entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
-```
+\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
 
-Navigate to http://localhost:8080/ to configure sist2-admin.
+## Example usage
 
-### Using the executable file *(Linux/WSL only)*
+See [Usage guide](docs/USAGE.md) for more details
 
-1. Choose search backend (See [comparison](#search-backends)):
-    * **Elasticsearch**: have an Elasticsearch (version >= 6.8.X, ideally >=7.14.0) instance running
-        1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
-        2. *(or)* Run using docker:
-            ```bash
-            docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
-            ```
-    * **SQLite**: No installation required
-
-2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
-   Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
-3. See [usage guide](docs/USAGE.md) for command line usage.
-
-Example usage:
-
-1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
-2. Prepare search index:
-    * **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
-    * **SQLite**: `sist2 index --search-index ./search.sist2 ./documents.sist2`
-3. Start web interface: `sist2 web ./documents.sist2`
+1. Scan a directory: `sist2 scan ~/Documents -o ./docs_idx`
+1. Push index to Elasticsearch: `sist2 index ./docs_idx`
+1. Start web interface: `sist2 web ./docs_idx`
 
 ## Format support
 
@@ -99,7 +82,7 @@ Example usage:
 | tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
 | docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
 | doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
-| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | yes         | author, title                                                                                                                          |
+| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | no          | author, title                                                                                                                          |
 | wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
 | json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
@@ -129,7 +112,7 @@ The `simon987/sist2` image comes with common languages
 (hin, jpn, eng, fra, rus, spa, chi_sim, deu) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
-name must be identical to the `*.traineddata` file installed on your system
+name must be identical to the `*.traineddata` file installed on your system 
 (use `chi_sim` rather than `chi-sim`).
 
 Examples:
@@ -140,61 +123,20 @@ sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
 sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
 ```
 
-### Search backends
-
-sist2 v3.0.7+ supports SQLite search backend. The SQLite search backend has
-fewer features and generally comparable query performance for medium-size
-indices, but it uses much less memory and is easier to set up.
-
-|                                              |                  SQLite                  |                                                             Elasticsearch                                                             |
-|----------------------------------------------|:----------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------:|
-| Requires separate search engine installation |                                          |                                                                   ✓                                                                   |
-| Memory footprint                             |                  ~20MB                   |                                                                >500MB                                                                 |
-| Query syntax                                 | [fts5](https://www.sqlite.org/fts5.html) | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
-| Fuzzy search                                 |                                          |                                                                   ✓                                                                   |
-| Media Types tree real-time updating          |                                          |                                                                   ✓                                                                   |
-| Search in file `path`                        |                                          |                                                                   ✓                                                                   |
-| Manual tagging                               |                    ✓                     |                                                                   ✓                                                                   |
-| User scripts                                 |                                          |                                                                   ✓                                                                   |
-| Media Type breakdown for search results      |                                          |                                                                   ✓                                                                   |
-
-### NER
-
-sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to
-**Configuration** > **Machine learning options** > **Model repositories**
-to enable it.
-
-The text processing is done in your browser, no data is sent to any third-party services.
-See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) for more details.
-
-#### List of available repositories:
-
-| URL                                                                                                     | Maintainer                              | Purpose |
-|---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
-| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
-
-<details>
-  <summary>Screenshot</summary>
-
-![ner](docs/ner.png)
-
-</details>
-
 ## Build from source
 
 You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries
 
-### Using docker
+### With docker (recommended)
 
 ```bash
 git clone --recursive https://github.com/simon987/sist2/
 cd sist2
-docker build . -t my-sist2-image
-# Copy sist2 executable from docker image
+docker build . -f ./Dockerfile -t my-sist2-image
 docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 ```
 
-### Using a linux computer
+### On a linux computer
 
 1. Install compile-time dependencies
 
@@ -202,14 +144,15 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
    apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
    ```
 
-2. Install vcpkg using my fork: https://github.com/simon987/vcpkg
-3. Install vcpkg dependencies
+1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
+
+1. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
+    vcpkg install curl[core,openssl] sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
     ```
 
-4. Build
+1. Build
     ```bash
     git clone --recursive https://github.com/simon987/sist2/
     (cd sist2-vue; npm install; npm run build)

docs/USAGE.md

@@ -1,69 +1,78 @@
 # Usage
 
+*More examples (specifically with docker/compose) are in progress*
+
+* [scan](#scan)
+    * [options](#scan-options)
+    * [examples](#scan-examples)
+    * [index format](#index-format)
+* [index](#index)
+    * [options](#index-options)
+    * [examples](#index-examples)
+* [web](#web)
+    * [options](#web-options)
+    * [examples](#web-examples)
+    * [rewrite_url](#rewrite_url)
+* [elasticsearch](#elasticsearch)
+* [exec-script](#exec-script)
+* [tagging](#tagging)
+* [sidecar files](#sidecar-files)
+
 ```
 Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
-   or: sist2 sqlite-index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
    or: sist2 exec-script [OPTION]... INDEX
-
 Lightning-fast file system indexer and search tool.
 
     -h, --help                        show this help message and exit
-    -v, --version                     Print version and exit.
-    --verbose                         Turn on logging.
-    --very-verbose                    Turn on debug messages.
-    --json-logs                       Output logs in JSON format.
+    -v, --version                     Show version and exit
+    --verbose                         Turn on logging
+    --very-verbose                    Turn on debug messages
 
 Scan options
-    -t, --threads=<int>               Number of threads. DEFAULT: 1
-    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2
-    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
-    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
-    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
-    -o, --output=<str>                Output index file path. DEFAULT: index.sist2
-    --incremental                     If the output file path exists, only scan new or modified files.
-    --optimize-index                  Defragment index file after scan to reduce its file size.
+    -t, --threads=<int>               Number of threads. DEFAULT=1
+    --mem-throttle=<int>              Total memory threshold in MiB for scan throttling. DEFAULT=0
+    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT=2
+    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT=500
+    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT=1
+    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT=32768
+    --incremental=<str>               Reuse an existing index and only scan modified files.
+    -o, --output=<str>                Output directory. DEFAULT=index.sist2/
     --rewrite-url=<str>               Serve files from this url instead of from disk.
-    --name=<str>                      Index display name. DEFAULT: index
+    --name=<str>                      Index display name. DEFAULT: (name of the directory)
     --depth=<int>                     Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: don't scan, list: only save file names as text, shallow: don't scan archives inside archives. DEFAULT: recurse
+    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
     --archive-passphrase=<str>        Passphrase for encrypted archive files
     --ocr-lang=<str>                  Tesseract language (use 'tesseract --list-langs' to see which are installed on your machine)
     --ocr-images                      Enable OCR'ing of image files.
     --ocr-ebooks                      Enable OCR'ing of ebook files.
-    -e, --exclude=<str>               Files that match this regex will not be scanned.
-    --fast                            Only index file names & mime type.
+    -e, --exclude=<str>               Files that match this regex will not be scanned
+    --fast                            Only index file names & mime type
     --treemap-threshold=<str>         Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
     --mem-buffer=<int>                Maximum memory buffer size per thread in MiB for files inside archives (see USAGE.md). DEFAULT: 2000
     --read-subtitles                  Read subtitles from media files.
-    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata).
+    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata)
     --checksums                       Calculate file checksums when scanning.
     --list-file=<str>                 Specify a list of newline-delimited paths to be scanned instead of normal directory traversal. Use '-' to read from stdin.
 
 Index options
-    -t, --threads=<int>               Number of threads. DEFAULT: 1
-    --es-url=<str>                    Elasticsearch url with port. DEFAULT: http://localhost:9200
-    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
-    -p, --print                       Print JSON documents to stdout instead of indexing to elasticsearch.
-    --incremental-index               Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
+    -t, --threads=<int>               Number of threads. DEFAULT=1
+    --es-url=<str>                    Elasticsearch url with port. DEFAULT=http://localhost:9200
+    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    -p, --print                       Just print JSON documents to stdout.
+    --incremental-index               Conduct incremental indexing, assumes that the old index is already digested by Elasticsearch.
     --script-file=<str>               Path to user script.
     --mappings-file=<str>             Path to Elasticsearch mappings.
     --settings-file=<str>             Path to Elasticsearch settings.
     --async-script                    Execute user script asynchronously.
-    --batch-size=<int>                Index batch size. DEFAULT: 70
-    -f, --force-reset                 Reset Elasticsearch mappings and settings.
-
-sqlite-index options
-    --search-index=<str>              Path to search index. Will be created if it does not exist yet.
+    --batch-size=<int>                Index batch size. DEFAULT: 100
+    -f, --force-reset                 Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
 
 Web options
-    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
-    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    --search-index=<str>              Path to SQLite search index.
-    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
-    --bind=<str>                      Listen for connections on this address. DEFAULT: localhost:4090
+    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --bind=<str>                      Listen on this address. DEFAULT=localhost:4090
     --auth=<str>                      Basic auth in user:password format
     --auth0-audience=<str>            API audience/identifier
     --auth0-domain=<str>              Application domain
@@ -75,15 +84,77 @@ Web options
     --lang=<str>                      Default UI language. Can be changed by the user
 
 Exec-script options
-    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
-    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
+    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
     --script-file=<str>               Path to user script.
     --async-script                    Execute user script asynchronously.
-
 Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
+## Scan
+
+### Scan options
+
+* `-t, --threads` 
+      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
+* `--mem-throttle`
+    Total memory threshold in MiB for scan throttling. Worker threads will not start a new parse job
+    until the total memory usage of sist2 is below this threshold. Set to 0 to disable. DEFAULT=0
+* `-q, --thumbnail-quality` 
+    Thumbnail quality, on a scale of 2 to 32, 2 being the best. See section below for a rough estimate of thumbnail database size
+* `--thumbnail-size` 
+    Thumbnail size in pixels.
+* `--thumbnail-count`
+    Maximum number of thumbnails to generate. When set to a value >= 2, thumbnails for video previews
+    will be generated. The actual number of thumbnails generated depends on the length of the video (maximum 1 image 
+    every ~7s). Set to 0 to completely disable thumbnails.
+* `--content-size` 
+    Number of bytes of text to be extracted from the content of files (plain text, PDFs etc.).
+    Repeated whitespace and special characters do not count toward this limit.
+    Set to 0 to completely disable content parsing.
+* `--incremental`
+    Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
+    will be copied to the new index and will not be parsed again.
+* `-o, --output` Output directory. 
+* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
+* `--name` Set the `name` option for the web module
+* `--depth` Maximum scan dept. Set to 0 only scan files directly in the root directory, set to -1 for infinite depth
+* `--archive` Archive file mode.
+    * skip: Don't parse
+    * list: Only get file names as text
+    * shallow: Don't parse archives inside archives.
+    * recurse: Scan archives recursively (default)
+* `--ocr-lang`, `--ocr-ebooks`, `--ocr-images` See [OCR](../README.md#OCR)
+* `-e, --exclude` Regex pattern to exclude files. A file is excluded if the pattern matches any 
+    part of the full absolute path.
+    
+    Examples: 
+    * `-e ".*\.ttf"`: Ignore ttf files
+    * `-e ".*\.(ttf|rar)"`: Ignore ttf and rar files
+    * `-e "^/mnt/backups/"`: Ignore all files in the `/mnt/backups/` directory
+    * `-e "^/mnt/Data[12]/"`: Ignore all files in the `/mnt/Data1/` and `/mnt/Data2/` directory
+    * `-e "(^/usr/)|(^/var/)|(^/media/DRIVE-A/tmp/)|(^/media/DRIVE-B/Trash/)"` Exclude the
+     `/usr`, `/var`, `/media/DRIVE-A/tmp`, `/media/DRIVE-B/Trash` directories
+* `--fast` Only index file names and mime type
+* `--treemap-threshold` Directories smaller than (`treemap-threshold` * `<total size of the index>`)
+    will not be considered for the disk utilisation visualization; their size will be added to
+    the parent directory. If the parent directory is still smaller than the threshold, it will also be "merged upwards"
+    and so on.
+    
+    In effect, smaller `treemap-threshold` values will yield a more detailed 
+    (but also a more cluttered and harder to read) visualization. 
+    
+* `--mem-buffer` Maximum memory buffer size in MiB (per thread) for files inside archives. Media files 
+    larger than this number will be read sequentially and no *seek* operations will be supported.
+
+    To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
+* `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
+* `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
+* `--checksums` Calculate file checksums (SHA1) when scanning files. This option does not cause any additional read 
+  operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
+  files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
+
+
 #### Thumbnail database size estimation
 
 See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
@@ -93,6 +164,8 @@ that is about `8000000 * 36kB = 288GB`.
 
 ![thumbnail_size](thumbnail_size.png)
 
+// TODO: add note about LMDB page size 4096
+
 ### Scan examples
 
 Simple scan
@@ -102,68 +175,130 @@ sist2 scan ~/Documents
 sist2 scan \
     --threads 4 --content-size 16000000 --thumbnail-quality 2 --archive shallow \
     --name "My Documents" --rewrite-url "http://nas.domain.local/My Documents/" \
-    ~/Documents -o ./documents.sist2
+    ~/Documents -o ./documents.idx/
 ```
 
 Incremental scan
-
-If the index file does not exist, `--incremental` has no effect.
-```bash
-sist scan ~/Documents -o ./documents.sist2
-sist scan ~/Documents -o ./documents.sist2 --incremental
-# or
-sist scan ~/Documents -o ./documents.sist2 --incremental
-sist scan ~/Documents -o ./documents.sist2 --incremental
+```
+sist2 scan --incremental ./orig_idx/ -o ./updated_idx/ ~/Documents
 ```
 
-### Index documents to Elasticsearch search backend
+### Index format
 
-```bash
-sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index.sist2
-sist2 index ./my_index.sist2
+A typical `ndjson` type index structure looks like this:
+```
+documents.idx/
+├── descriptor.json
+├── _index_main.ndjson.zst
+├── treemap.csv
+├── agg_mime.csv
+├── agg_date.csv
+├── add_size.csv
+├── thumbs/
+|   ├── data.mdb
+|   └── lock.mdb
+├── tags/
+|   ├── data.mdb
+|   └── lock.mdb
+└── meta/
+    ├── data.mdb
+    └── lock.mdb
 ```
 
-#### Index documents to SQLite search backend
+The `_index_*.ndjson.zst` files contain the document data in JSON format, in a compressed newline-delemited file.
+
+The `thumbs/` folder is a [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database)
+database containing the thumbnails.
+
+The `descriptor.json` file contains general information about the index. The 
+following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rewrite_url) and `timestamp`.
+
+The `.csv` are pre-computed aggregations necessary for the stats page.
+
+*thumbs/*:
+
+LMDB key-value store. Keys are **binary** 16-byte md5 hash* (`_id` field)
+and values are raw image bytes.
+
+*\* Hash is calculated from the full path of the file, including the extension, relative to the index root*
+
+
+## Index
+### Index options
+ * `--es-url` 
+ Elasticsearch url and port. If you are using docker, make sure that both containers are on the
+ same network.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
+ * `-p, --print` 
+    Print index in JSON format to stdout.
+ * `--incremental-index`
+   Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
+   Only the new changes since the last scan will be sent.
+ * `--script-file` 
+    Path to user script. See [Scripting](scripting.md).
+ * `--mappings-file`
+    Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used.
+ * `--settings-file`
+    Path to custom Elasticsearch settings. *(See above)*
+ * `--async-script` 
+    Use `wait_for_completion=false` elasticsearch option while executing user script.
+     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
+ * `--batch-size=<int>` 
+    Index batch size. Indexing is generally faster with larger batches, but payloads that
+    are too large will fail and additional overhead for retrying with smaller sizes may slow
+    down the process.
+ * `-f, --force-reset` 
+    Reset Elasticsearch mappings and settings.
+ * `-t, --threads` Number of threads to use. Ideally, choose a number equal to the number of logical cores of the machine hosting Elasticsearch.
+    
+### Index examples
+
+**Push to elasticsearch**
 ```bash
-# The search index will be created if it does not exist already
-sist2 sqlite-index ./index1.sist2 --search-index search.sist2
-sist2 sqlite-index ./index2.sist2 --search-index search.sist2
+sist2 index --force-reset --batch-size 1000 --es-url http://localhost:9200 ./my_index/
+sist2 index ./my_index/
 ```
 
 **Save index in JSON format**
 ```bash
-sist2 index --print ./my_index.sist2 > my_index.ndjson
+sist2 index --print ./my_index/ > my_index.ndjson
 ```
 
 **Inspect contents of an index**
 ```bash
-sist2 index --print ./my_index.sist2 | jq | less
+sist2 index --print ./my_index/ | jq | less
 ```
 
 ## Web
 
+### Web options
+ * `--es-url=<str>` Elasticsearch url.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
+ * `--bind=<str>` Listen on this address.
+ * `--auth=<str>` Basic auth in user:password format
+ * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
+    `--auth` argument, but authentication is only applied the `/tag/` endpoint.
+ * `--tagline=<str>` When specified, will replace the default tagline in the navbar.
+ * `--dev` Serve html & js files from disk (for development, used to modify frontend files without having to recompile)
+ * `--lang=<str>` Set the default web UI language (See #180 for a list of supported languages, default
+   is `en`). The user can change the language in the configuration page
+ * `--auth0-audience`, `--auth0-domain`, `--auth0-client-id`, `--auth0-public-key-file` See [Authentication with Auth0](auth0.md)
+
 ### Web examples
 
-**Single index (Elasticsearch backend)**
+**Single index**
 ```bash
-sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index.sist2
+sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index
 ```
 
-**Multiple indices (Elasticsearch backend)**
+**Multiple indices**
 ```bash
 # Indices will be displayed in this order in the web interface
-sist2 web index1.sist2 index2.sist2 index3.sist2 index4.sist2
+sist2 web index1 index2 index3 index4
 ```
 
-**SQLite search backend**
-```bash
-sist2 web --search-index search.sist2 index1.sist2
-```
-
-#### Auth0 authentication
-
-See [auth0.md](auth0.md)
-
 ### rewrite_url
 
 When the `rewrite_url` field is not empty, the web module ignores the `root`
@@ -245,8 +380,8 @@ The sidecar file must have exactly the same file path and the `.s2meta` suffix.
 ```
 
 ```
-sist2 scan ~/Documents -o ./docs.sist2
-sist2 index ./docs.sist2
+sist2 scan ~/Documents -o ./docs.idx
+sist2 index ./docs.idx
 ```
 
 *NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,

scripts/build.sh

@@ -7,7 +7,7 @@ git submodule update --init --recursive
 mkdir build
 (
   cd build
-  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
   make -j $(nproc)
   strip sist2
   ./sist2 -v > VERSION
@@ -17,7 +17,7 @@ mv build/sist2 sist2-x64-linux
 (
   cd build
   rm -rf CMakeFiles CMakeCache.txt
-  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
   make -j  $(nproc)
 )
 mv build/sist2_debug sist2-x64-linux-debug

scripts/build_arm64.sh

@@ -7,7 +7,7 @@ git submodule update --init --recursive
 mkdir build
 (
   cd build
-  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
   make -j $(nproc)
   strip sist2
 )
@@ -16,7 +16,7 @@ mv build/sist2 sist2-arm64-linux
 rm -rf CMakeFiles CMakeCache.txt
 (
   cd build
-  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
   make -j $(nproc)
 )
 mv build/sist2_debug sist2-arm64-linux-debug

scripts/sqlite_extension.c

@@ -1,84 +0,0 @@
-#include <sqlite3ext.h>
-#include <string.h>
-#include <stdlib.h>
-
-SQLITE_EXTENSION_INIT1
-
-static int sep_rfind(const char *str) {
-    for (int i = (int) strlen(str); i >= 0; i--) {
-        if (str[i] == '/') {
-            return i;
-        }
-    }
-    return -1;
-}
-
-void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
-    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
-        sqlite3_result_error(ctx, "Invalid parameters", -1);
-    }
-
-    const char *value = (const char *) sqlite3_value_text(argv[0]);
-
-    int stop = sep_rfind(value);
-    if (stop == -1) {
-        sqlite3_result_null(ctx);
-        return;
-    }
-    char parent[4096 * 3];
-    strncpy(parent, value, stop);
-
-    sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
-}
-
-void random_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
-    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
-        sqlite3_result_error(ctx, "Invalid parameters", -1);
-    }
-
-    char state_buf[32] = {0,};
-    struct random_data buf;
-    int result;
-
-    long seed = sqlite3_value_int64(argv[0]);
-
-    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
-
-    random_r(&buf, &result);
-
-    sqlite3_result_int(ctx, result);
-}
-
-
-int sqlite3_extension_init(
-        sqlite3 *db,
-        char **pzErrMsg,
-        const sqlite3_api_routines *pApi
-) {
-    SQLITE_EXTENSION_INIT2(pApi);
-
-
-    sqlite3_create_function(
-            db,
-            "path_parent",
-            1,
-            SQLITE_UTF8,
-            NULL,
-            path_parent_func,
-            NULL,
-            NULL
-    );
-
-    sqlite3_create_function(
-            db,
-            "random_seeded",
-            1,
-            SQLITE_UTF8,
-            NULL,
-            random_func,
-            NULL,
-            NULL
-    );
-
-    return SQLITE_OK;
-}

scripts/sqlite_extension_compile.sh

@@ -1 +0,0 @@
-gcc -I/mnt/work/vcpkg/installed/x64-linux/include -g -fPIC -shared sqlite_extension.c -o sist2funcs.so

scripts/start_dev_es_8.sh

@@ -1,3 +1,3 @@
 docker run --rm -it --name "sist2-dev-es"\
        	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
-	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.7.0
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.1.2

sist2-admin/frontend/package-lock.json

@@ -10491,9 +10491,9 @@
       "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
     },
     "node_modules/webpack": {
-      "version": "5.78.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.78.0.tgz",
-      "integrity": "sha512-gT5DP72KInmE/3azEaQrISjTvLYlSM0j1Ezhht/KLVkrqtv10JoP/RXhwmX/frrutOPuSq3o5Vq0ehR/4Vmd1g==",
+      "version": "5.75.0",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz",
+      "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==",
       "dev": true,
       "dependencies": {
         "@types/eslint-scope": "^3.7.3",
@@ -18719,9 +18719,9 @@
       "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
     },
     "webpack": {
-      "version": "5.78.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.78.0.tgz",
-      "integrity": "sha512-gT5DP72KInmE/3azEaQrISjTvLYlSM0j1Ezhht/KLVkrqtv10JoP/RXhwmX/frrutOPuSq3o5Vq0ehR/4Vmd1g==",
+      "version": "5.75.0",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz",
+      "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==",
       "dev": true,
       "requires": {
         "@types/eslint-scope": "^3.7.3",

sist2-admin/frontend/yarn.lock

@@ -1390,14 +1390,14 @@
     thread-loader "^3.0.0"
     webpack "^5.54.0"
 
-"@vue/cli-plugin-router@^5.0.8", "@vue/cli-plugin-router@~5.0.8":
+"@vue/cli-plugin-router@~5.0.8":
   version "5.0.8"
   resolved "https://registry.npmjs.org/@vue/cli-plugin-router/-/cli-plugin-router-5.0.8.tgz"
   integrity sha512-Gmv4dsGdAsWPqVijz3Ux2OS2HkMrWi1ENj2cYL75nUeL+Xj5HEstSqdtfZ0b1q9NCce+BFB6QnHfTBXc/fCvMg==
   dependencies:
     "@vue/cli-shared-utils" "^5.0.8"
 
-"@vue/cli-plugin-vuex@^5.0.8", "@vue/cli-plugin-vuex@~5.0.8":
+"@vue/cli-plugin-vuex@~5.0.8":
   version "5.0.8"
   resolved "https://registry.npmjs.org/@vue/cli-plugin-vuex/-/cli-plugin-vuex-5.0.8.tgz"
   integrity sha512-HSYWPqrunRE5ZZs8kVwiY6oWcn95qf/OQabwLfprhdpFWAGtLStShjsGED2aDpSSeGAskQETrtR/5h7VqgIlBA==
@@ -5492,9 +5492,9 @@ webpack-virtual-modules@^0.4.2:
   integrity sha512-5tyDlKLqPfMqjT3Q9TAqf2YqjwmnUleZwzJi1A5qXnlBCdj2AtOJ6wAWdglTIDOPgOiOrXeBeFcsQ8+aGQ6QbA==
 
 webpack@^5.54.0:
-  version "5.78.0"
-  resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.78.0.tgz#836452a12416af2a7beae906b31644cb2562f9e6"
-  integrity sha512-gT5DP72KInmE/3azEaQrISjTvLYlSM0j1Ezhht/KLVkrqtv10JoP/RXhwmX/frrutOPuSq3o5Vq0ehR/4Vmd1g==
+  version "5.75.0"
+  resolved "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz"
+  integrity sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==
   dependencies:
     "@types/eslint-scope" "^3.7.3"
     "@types/estree" "^0.0.51"

sist2-vue/package.json

@@ -9,11 +9,10 @@
   "dependencies": {
     "@auth0/auth0-spa-js": "^2.0.2",
     "@egjs/vue-infinitegrid": "3.3.0",
-    "@tensorflow/tfjs": "^4.4.0",
     "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "d3": "^7.8.4",
+    "d3": "^5.6.1",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",
     "fslightbox-vue": "fslightbox-vue.tgz",

sist2-vue/src/App.vue

@@ -1,404 +1,383 @@
 <template>
-    <div id="app" :class="getClass()" v-if="!authLoading">
-        <NavBar></NavBar>
-        <router-view v-if="!configLoading"/>
+  <div id="app" :class="getClass()" v-if="!authLoading">
+    <NavBar></NavBar>
+    <router-view v-if="!configLoading"/>
+  </div>
+  <div class="loading-page" v-else>
+    <div class="loading-spinners">
+      <b-spinner type="grow" variant="primary"></b-spinner>
+      <b-spinner type="grow" variant="primary"></b-spinner>
+      <b-spinner type="grow" variant="primary"></b-spinner>
     </div>
-    <div class="loading-page" v-else>
-        <div class="loading-spinners">
-            <b-spinner type="grow" variant="primary"></b-spinner>
-            <b-spinner type="grow" variant="primary"></b-spinner>
-            <b-spinner type="grow" variant="primary"></b-spinner>
-        </div>
-        <div class="loading-text">
-            Loading • Chargement • 装载 • Wird geladen
-        </div>
+    <div class="loading-text">
+      Loading • Chargement • 装载 • Wird geladen
     </div>
+  </div>
 </template>
 
 <script>
 import NavBar from "@/components/NavBar";
 import {mapActions, mapGetters, mapMutations} from "vuex";
 import Sist2Api from "@/Sist2Api";
-import ModelsRepo from "@/ml/modelsRepo";
 import {setupAuth0} from "@/main";
-import Sist2ElasticsearchQuery from "@/Sist2ElasticsearchQuery";
-import Sist2SqliteQuery from "@/Sist2SqliteQuery";
 
 export default {
-    components: {NavBar},
-    data() {
-        return {
-            configLoading: false,
-            authLoading: true,
-            sist2InfoLoading: true
-        }
-    },
-    computed: {
-        ...mapGetters(["optTheme"]),
-    },
-    mounted() {
-        this.$store.dispatch("loadConfiguration").then(() => {
-            this.$root.$i18n.locale = this.$store.state.optLang;
-            ModelsRepo.init(this.$store.getters.mlRepositoryList).catch(err => {
-                this.$bvToast.toast(
-                    this.$t("ml.repoFetchError"),
-                    {
-                        title: this.$t("ml.repoFetchErrorTitle"),
-                        noAutoHide: true,
-                        toaster: "b-toaster-bottom-right",
-                        headerClass: "toast-header-warning",
-                        bodyClass: "toast-body-warning",
-                    });
-            });
-        });
-
-        this.$store.subscribe((mutation) => {
-            if (mutation.type === "setOptLang") {
-                this.$root.$i18n.locale = mutation.payload;
-                this.configLoading = true;
-                window.setTimeout(() => this.configLoading = false, 10);
-            }
-
-            if (mutation.type === "setAuth0Token") {
-                this.authLoading = false;
-            }
-        });
-
-        Sist2Api.getSist2Info().then(data => {
-
-            if (data.auth0Enabled) {
-                this.authLoading = true;
-                setupAuth0(data.auth0Domain, data.auth0ClientId, data.auth0Audience)
-
-                this.$auth.$watch("loading", loading => {
-                    if (loading === false) {
-
-                        if (!this.$auth.isAuthenticated) {
-                            this.$auth.loginWithRedirect();
-                            return;
-                        }
-
-                        // Remove "code" param
-                        window.history.replaceState({}, "", "/" + window.location.hash);
-
-                        this.$store.dispatch("loadAuth0Token");
-                    }
-                });
-            } else {
-                this.authLoading = false;
-            }
-
-            this.setSist2Info(data);
-            this.setIndices(data.indices)
-
-            if (Sist2Api.backend() === "sqlite") {
-                Sist2Api.init(Sist2SqliteQuery.searchQuery);
-                this.$store.commit("setUiSqliteMode", true);
-            } else {
-                Sist2Api.init(Sist2ElasticsearchQuery.searchQuery);
-            }
-        });
-    },
-    methods: {
-        ...mapActions(["setSist2Info",]),
-        ...mapMutations(["setIndices",]),
-        getClass() {
-            return {
-                "theme-light": this.optTheme === "light",
-                "theme-black": this.optTheme === "black",
-            }
-        }
+  components: {NavBar},
+  data() {
+    return {
+      configLoading: false,
+      authLoading: true,
+      sist2InfoLoading: true
     }
-    ,
+  },
+  computed: {
+    ...mapGetters(["optTheme"]),
+  },
+  mounted() {
+    this.$store.dispatch("loadConfiguration").then(() => {
+      this.$root.$i18n.locale = this.$store.state.optLang;
+    });
+
+    this.$store.subscribe((mutation) => {
+      if (mutation.type === "setOptLang") {
+        this.$root.$i18n.locale = mutation.payload;
+        this.configLoading = true;
+        window.setTimeout(() => this.configLoading = false, 10);
+      }
+
+      if (mutation.type === "setAuth0Token") {
+        this.authLoading = false;
+      }
+    });
+
+    Sist2Api.getSist2Info().then(data => {
+
+      if (data.auth0Enabled) {
+        this.authLoading = true;
+        setupAuth0(data.auth0Domain, data.auth0ClientId, data.auth0Audience)
+
+        this.$auth.$watch("loading", loading => {
+          if (loading === false) {
+
+            if (!this.$auth.isAuthenticated) {
+              this.$auth.loginWithRedirect();
+              return;
+            }
+
+            // Remove "code" param
+            window.history.replaceState({}, "", "/" + window.location.hash);
+
+            this.$store.dispatch("loadAuth0Token");
+          }
+        });
+      } else {
+        this.authLoading = false;
+      }
+
+      this.setSist2Info(data);
+      this.setIndices(data.indices)
+    });
+  },
+  methods: {
+    ...mapActions(["setSist2Info",]),
+    ...mapMutations(["setIndices",]),
+    getClass() {
+      return {
+        "theme-light": this.optTheme === "light",
+        "theme-black": this.optTheme === "black",
+      }
+    }
+  }
+  ,
 }
 </script>
 
 <style>
 html, body {
-    height: 100%;
+  height: 100%;
 }
 
 #app {
-    /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
-    -webkit-font-smoothing: antialiased;
-    -moz-osx-font-smoothing: grayscale;
-    /*text-align: center;*/
-    color: #2c3e50;
-    padding-bottom: 1em;
-    min-height: 100%;
+  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  /*text-align: center;*/
+  color: #2c3e50;
+  padding-bottom: 1em;
+  min-height: 100%;
 }
 
 /*Black theme*/
 .theme-black {
-    background-color: #000;
+  background-color: #000;
 }
 
 .theme-black .card, .theme-black .modal-content {
-    background: #212121;
-    color: #e0e0e0;
-    border-radius: 1px;
-    border: none;
+  background: #212121;
+  color: #e0e0e0;
+  border-radius: 1px;
+  border: none;
 }
 
 
 .theme-black .table {
-    color: #e0e0e0;
+  color: #e0e0e0;
 }
 
 .theme-black .table td, .theme-black .table th {
-    border: none;
+  border: none;
 }
 
 .theme-black .table thead th {
-    border-bottom: 1px solid #646464;
+  border-bottom: 1px solid #646464;
 }
 
 .theme-black .custom-select {
-    overflow: auto;
-    background-color: #37474F;
-    border: 1px solid #616161;
-    color: #bdbdbd;
+  overflow: auto;
+  background-color: #37474F;
+  border: 1px solid #616161;
+  color: #bdbdbd;
 }
 
 .theme-black .custom-select:focus {
-    border-color: #757575;
-    outline: 0;
-    box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
+  border-color: #757575;
+  outline: 0;
+  box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
 }
 
 .theme-black .inspire-tree .selected > .wholerow, .theme-black .inspire-tree .selected > .title-wrap:hover + .wholerow {
-    background: none !important;
+  background: none !important;
 }
 
 .theme-black .inspire-tree .icon-expand::before, .theme-black .inspire-tree .icon-collapse::before {
-    background-color: black !important;
+  background-color: black !important;
 }
 
 .theme-black .inspire-tree .title {
-    color: #eee;
+  color: #eee;
 }
 
 .theme-black .inspire-tree {
-    font-weight: 400;
-    font-size: 14px;
-    font-family: Helvetica, Nueue, Verdana, sans-serif;
-    max-height: 350px;
-    overflow: auto;
+  font-weight: 400;
+  font-size: 14px;
+  font-family: Helvetica, Nueue, Verdana, sans-serif;
+  max-height: 350px;
+  overflow: auto;
 }
 
 .inspire-tree [type=checkbox] {
-    left: 22px !important;
-    top: 7px !important;
+  left: 22px !important;
+  top: 7px !important;
 }
 
 .theme-black .form-control {
-    background-color: #37474F;
-    border: 1px solid #616161;
-    color: #dbdbdb !important;
+  background-color: #37474F;
+  border: 1px solid #616161;
+  color: #dbdbdb !important;
 }
 
 .theme-black .form-control:focus {
-    background-color: #546E7A;
-    color: #fff;
+  background-color: #546E7A;
+  color: #fff;
 }
 
 .theme-black .input-group-text, .theme-black .default-input {
-    background: #37474F !important;
-    border: 1px solid #616161 !important;
-    color: #dbdbdb !important;
+  background: #37474F !important;
+  border: 1px solid #616161 !important;
+  color: #dbdbdb !important;
 }
 
 .theme-black ::placeholder {
-    color: #BDBDBD !important;
-    opacity: 1;
+  color: #BDBDBD !important;
+  opacity: 1;
 }
 
 .theme-black .nav-tabs .nav-link {
-    color: #e0e0e0;
-    border-radius: 0;
+  color: #e0e0e0;
+  border-radius: 0;
 }
 
 .theme-black .nav-tabs .nav-item.show .nav-link, .theme-black .nav-tabs .nav-link.active {
-    background-color: #212121;
-    border-color: #616161 #616161 #212121;
-    color: #e0e0e0;
+  background-color: #212121;
+  border-color: #616161 #616161 #212121;
+  color: #e0e0e0;
 }
 
 .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:focus {
-    border-color: #616161 #616161 #212121;
-    color: #e0e0e0;
+  border-color: #616161 #616161 #212121;
+  color: #e0e0e0;
 }
 
 .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:hover {
-    border-color: #e0e0e0 #e0e0e0 #212121;
-    color: #e0e0e0;
+  border-color: #e0e0e0 #e0e0e0 #212121;
+  color: #e0e0e0;
 }
 
 .theme-black .nav-tabs {
-    border-bottom: #616161;
+  border-bottom: #616161;
 }
 
 .theme-black a:hover, .theme-black .btn:hover {
-    color: #fff;
+  color: #fff;
 }
 
 .theme-black .b-dropdown a:hover {
-    color: inherit;
+  color: inherit;
 }
 
 .theme-black .btn {
-    color: #eee;
+  color: #eee;
 }
 
 .theme-black .modal-header .close {
-    color: #e0e0e0;
-    text-shadow: none;
+  color: #e0e0e0;
+  text-shadow: none;
 }
 
 .theme-black .modal-header {
-    border-bottom: 1px solid #646464;
+  border-bottom: 1px solid #646464;
 }
 
 /* -------------------------- */
 
 #nav {
-    padding: 30px;
+  padding: 30px;
 }
 
 #nav a {
-    font-weight: bold;
-    color: #2c3e50;
+  font-weight: bold;
+  color: #2c3e50;
 }
 
 #nav a.router-link-exact-active {
-    color: #42b983;
+  color: #42b983;
 }
 
 .mobile {
-    display: none;
+  display: none;
 }
 
 .container {
-    padding-top: 1em;
+  padding-top: 1em;
 }
 
 @media (max-width: 650px) {
-    .mobile {
-        display: initial;
-    }
+  .mobile {
+    display: initial;
+  }
 
-    .not-mobile {
-        display: none;
-    }
+  .not-mobile {
+    display: none;
+  }
 
-    .grid-single-column .fit {
-        max-height: none !important;
-    }
+  .grid-single-column .fit {
+    max-height: none !important;
+  }
 
-    .container {
-        padding-left: 0;
-        padding-right: 0;
-        padding-top: 0
-    }
+  .container {
+    padding-left: 0;
+    padding-right: 0;
+    padding-top: 0
+  }
 
-    .lightbox-caption {
-        display: none;
-    }
+  .lightbox-caption {
+    display: none;
+  }
 }
 
 .info-icon {
-    width: 1rem;
-    margin-right: 0.2rem;
-    cursor: pointer;
-    line-height: 1rem;
-    height: 1rem;
-    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
-    filter: brightness(45%);
-    display: block;
+  width: 1rem;
+  margin-right: 0.2rem;
+  cursor: pointer;
+  line-height: 1rem;
+  height: 1rem;
+  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+  filter: brightness(45%);
+  display: block;
 }
 
 .tabs {
-    margin-top: 10px;
+  margin-top: 10px;
 }
 
 .modal-title {
-    text-overflow: ellipsis;
-    overflow: hidden;
-    white-space: nowrap;
+  text-overflow: ellipsis;
+  overflow: hidden;
+  white-space: nowrap;
 }
 
 @media screen and (min-width: 1500px) {
-    .container {
-        max-width: 1440px;
-    }
+  .container {
+    max-width: 1440px;
+  }
 }
 
 .noUi-connects {
-    border-radius: 1px !important;
+  border-radius: 1px !important;
 }
 
 mark {
-    background: #fff217;
-    border-radius: 0;
-    padding: 1px 0;
-    color: inherit;
+  background: #fff217;
+  border-radius: 0;
+  padding: 1px 0;
+  color: inherit;
 }
 
 .theme-black mark {
-    background: rgba(251, 191, 41, 0.25);
-    border-radius: 0;
-    padding: 1px 0;
-    color: inherit;
+  background: rgba(251, 191, 41, 0.25);
+  border-radius: 0;
+  padding: 1px 0;
+  color: inherit;
 }
 
 .theme-black .content-div mark {
-    background: rgba(251, 191, 41, 0.40);
-    color: white;
+  background: rgba(251, 191, 41, 0.40);
+  color: white;
 }
 
 .content-div {
-    font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
-    font-size: 13px;
-    padding: 1em;
-    background-color: #f5f5f5;
-    border: 1px solid #ccc;
-    border-radius: 4px;
-    margin: 3px;
-    white-space: normal;
-    color: #000;
-    overflow: hidden;
+  font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+  font-size: 13px;
+  padding: 1em;
+  background-color: #f5f5f5;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  margin: 3px;
+  white-space: normal;
+  color: #000;
+  overflow: hidden;
 }
 
 .theme-black .content-div {
-    background-color: #37474F;
-    border: 1px solid #616161;
-    color: #E0E0E0FF;
+  background-color: #37474F;
+  border: 1px solid #616161;
+  color: #E0E0E0FF;
 }
 
 .graph {
-    display: inline-block;
-    width: 40%;
+  display: inline-block;
+  width: 40%;
 }
 
 .pointer {
-    cursor: pointer;
+  cursor: pointer;
 }
 
 .loading-page {
-    display: flex;
-    justify-content: center;
-    align-items: center;
-    flex-direction: column;
-    height: 100%;
-    gap: 15px
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  flex-direction: column;
+  height: 100%;
+  gap: 15px
 }
 
 .loading-spinners {
-    display: flex;
-    gap: 10px;
+  display: flex;
+  gap: 10px;
 }
 
 .loading-text {
-    text-align: center;
+  text-align: center;
 }
 </style>

sist2-vue/src/Sist2Api.ts

@@ -1,7 +1,5 @@
 import axios from "axios";
 import {ext, strUnescape, lum} from "./util";
-import Sist2Query from "@/Sist2ElasticsearchQuery";
-import store from "@/store";
 
 export interface EsTag {
     id: string
@@ -101,22 +99,12 @@ export interface EsResult {
 
 class Sist2Api {
 
-    private readonly baseUrl: string
-    private sist2Info: any
-    private queryfunc: () => EsResult;
+    private baseUrl: string
 
     constructor(baseUrl: string) {
         this.baseUrl = baseUrl;
     }
 
-    init(queryFunc: () => EsResult) {
-        this.queryfunc = queryFunc;
-    }
-
-    backend() {
-        return this.sist2Info.searchBackend;
-    }
-
     getSist2Info(): Promise<any> {
         return axios.get(`${this.baseUrl}i`).then(resp => {
             const indices = resp.data.indices as Index[];
@@ -131,8 +119,6 @@ class Sist2Api {
                 } as Index;
             });
 
-            this.sist2Info = resp.data;
-
             return resp.data;
         })
     }
@@ -233,14 +219,6 @@ class Sist2Api {
         } as Tag;
     }
 
-    search(): Promise<EsResult> {
-        if (this.backend() == "sqlite") {
-            return this.ftsQuery(this.queryfunc())
-        } else {
-            return this.esQuery(this.queryfunc());
-        }
-    }
-
     esQuery(query: any): Promise<EsResult> {
         return axios.post(`${this.baseUrl}es`, query).then(resp => {
             const res = resp.data as EsResult;
@@ -259,30 +237,7 @@ class Sist2Api {
         });
     }
 
-    ftsQuery(query: any): Promise<EsResult> {
-        return axios.post(`${this.baseUrl}fts/search`, query).then(resp => {
-            const res = resp.data as any;
-
-            if (res.hits.hits) {
-                res.hits.hits.forEach(hit => {
-                    hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
-                    hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
-
-                    this.setHitProps(hit);
-                    this.setHitTags(hit);
-
-                    if ("highlight" in hit) {
-                        hit["highlight"]["name"] = [hit["highlight"]["name"]];
-                        hit["highlight"]["content"] = [hit["highlight"]["content"]];
-                    }
-                });
-            }
-
-            return res;
-        });
-    }
-
-    private getMimeTypesEs(query) {
+    getMimeTypes(query = undefined) {
         const AGGS = {
             mimeTypes: {
                 terms: {
@@ -303,70 +258,48 @@ class Sist2Api {
         }
 
         return this.esQuery(query).then(resp => {
-            return resp["aggregations"]["mimeTypes"]["buckets"].map(bucket => ({
-                mime: bucket.key,
-                count: bucket.doc_count
-            }));
+            const mimeMap: any[] = [];
+            const buckets = resp["aggregations"]["mimeTypes"]["buckets"];
 
-        });
-    }
+            buckets.sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
+                const tmp = bucket["key"].split("/");
+                const category = tmp[0];
+                const mime = tmp[1];
 
-    private getMimeTypesSqlite(): Promise<[{ mime: string, count: number }]> {
-        return axios.get(`${this.baseUrl}fts/mimetypes`)
-            .then(resp => {
-                return resp.data;
-            });
-    }
+                let category_exists = false;
 
-    async getMimeTypes(query = undefined) {
-        let buckets;
+                const child = {
+                    "id": bucket["key"],
+                    "text": `${mime} (${bucket["doc_count"]})`
+                };
 
-        if (this.backend() == "sqlite") {
-            buckets = await this.getMimeTypesSqlite();
-        } else {
-            buckets = await this.getMimeTypesEs(query);
-        }
+                mimeMap.forEach(node => {
+                    if (node.text === category) {
+                        node.children.push(child);
+                        category_exists = true;
+                    }
+                });
 
-        const mimeMap: any[] = [];
-
-        buckets.sort((a: any, b: any) => a.mime > b.mime).forEach((bucket: any) => {
-            const tmp = bucket.mime.split("/");
-            const category = tmp[0];
-            const mime = tmp[1];
-
-            let category_exists = false;
-
-            const child = {
-                "id": bucket.mime,
-                "text": `${mime} (${bucket.count})`
-            };
+                if (!category_exists) {
+                    mimeMap.push({text: category, children: [child], id: category});
+                }
+            })
 
             mimeMap.forEach(node => {
-                if (node.text === category) {
-                    node.children.push(child);
-                    category_exists = true;
+                if (node.children) {
+                    node.children.sort((a, b) => a.id.localeCompare(b.id));
                 }
-            });
+            })
+            mimeMap.sort((a, b) => a.id.localeCompare(b.id))
 
-            if (!category_exists) {
-                mimeMap.push({text: category, children: [child], id: category});
-            }
-        })
-
-        mimeMap.forEach(node => {
-            if (node.children) {
-                node.children.sort((a, b) => a.id.localeCompare(b.id));
-            }
-        })
-        mimeMap.sort((a, b) => a.id.localeCompare(b.id))
-
-        return {buckets, mimeMap};
+            return {buckets, mimeMap};
+        });
     }
 
     _createEsTag(tag: string, count: number): EsTag {
         const tokens = tag.split(".");
 
-        if (/.*\.#[0-9a-fA-F]{6}/.test(tag)) {
+        if (/.*\.#[0-9a-f]{6}/.test(tag)) {
             return {
                 id: tokens.slice(0, -1).join("."),
                 color: tokens.pop(),
@@ -383,48 +316,32 @@ class Sist2Api {
         };
     }
 
-    private getTagsEs() {
+    getTags() {
         return this.esQuery({
             aggs: {
                 tags: {
                     terms: {
                         field: "tag",
-                        size: 65535
+                        size: 10000
                     }
                 }
             },
             size: 0,
         }).then(resp => {
-            return resp["aggregations"]["tags"]["buckets"]
+            const seen = new Set();
+
+            const tags = resp["aggregations"]["tags"]["buckets"]
                 .sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
                 .map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
-        });
-    }
 
-    private getTagsSqlite() {
-        return axios.get(`${this.baseUrl}/fts/tags`)
-            .then(resp => {
-                return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
+            // Remove duplicates (same tag with different color)
+            return tags.filter((t: EsTag) => {
+                if (seen.has(t.id)) {
+                    return false;
+                }
+                seen.add(t.id);
+                return true;
             });
-    }
-
-    async getTags(): Promise<EsTag[]> {
-        let tags;
-        if (this.backend() == "sqlite") {
-            tags = await this.getTagsSqlite();
-        } else {
-            tags = await this.getTagsEs();
-        }
-
-        // Remove duplicates (same tag with different color)
-        const seen = new Set();
-
-        return tags.filter((t: EsTag) => {
-            if (seen.has(t.id)) {
-                return false;
-            }
-            seen.add(t.id);
-            return true;
         });
     }
 
@@ -444,263 +361,20 @@ class Sist2Api {
         });
     }
 
-    searchPaths(indexId, minDepth, maxDepth, prefix = null) {
-        if (this.backend() == "sqlite") {
-            return this.searchPathsSqlite(indexId, minDepth, minDepth, prefix);
-        } else {
-            return this.searchPathsEs(indexId, minDepth, maxDepth, prefix);
-        }
+    getTreemapCsvUrl(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/1`;
     }
 
-    private searchPathsSqlite(indexId, minDepth, maxDepth, prefix) {
-        return axios.post(`${this.baseUrl}fts/paths`, {
-            indexId, minDepth, maxDepth, prefix
-        }).then(resp => {
-            return resp.data;
-        });
+    getMimeCsvUrl(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/2`;
     }
 
-    private searchPathsEs(indexId, minDepth, maxDepth, prefix): Promise<[{ path: string, count: number }]> {
-
-        const query = {
-            query: {
-                bool: {
-                    filter: [
-                        {term: {index: indexId}},
-                        {range: {_depth: {gte: minDepth, lte: maxDepth}}},
-                    ]
-                }
-            },
-            aggs: {
-                paths: {
-                    terms: {
-                        field: "path",
-                        size: 10000
-                    }
-                }
-            },
-            size: 0
-        };
-
-        if (prefix != null) {
-            query["query"]["bool"]["must"] = {
-                prefix: {
-                    path: prefix,
-                }
-            };
-        }
-
-        return this.esQuery(query).then(resp => {
-            const buckets = resp["aggregations"]["paths"]["buckets"];
-
-            if (!buckets) {
-                return [];
-            }
-
-            return buckets
-                .map(bucket => ({
-                    path: bucket.key,
-                    count: bucket.doc_count
-                }));
-        });
+    getSizeCsv(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/3`;
     }
 
-    private getDateRangeSqlite() {
-        return axios.get(`${this.baseUrl}fts/dateRange`)
-            .then(resp => ({
-                min: resp.data.dateMin,
-                max: resp.data.dateMax,
-            }));
-    }
-
-    getDateRange(): Promise<{ min: number, max: number }> {
-        if (this.backend() == "sqlite") {
-            return this.getDateRangeSqlite();
-        } else {
-            return this.getDateRangeEs();
-        }
-    }
-
-    private getDateRangeEs() {
-        return this.esQuery({
-            // TODO: filter current selected indices
-            aggs: {
-                dateMin: {min: {field: "mtime"}},
-                dateMax: {max: {field: "mtime"}},
-            },
-            size: 0
-        }).then(res => {
-            const range = {
-                min: res.aggregations.dateMin.value,
-                max: res.aggregations.dateMax.value,
-            }
-
-            if (range.min == null) {
-                range.min = 0;
-                range.max = 1;
-            } else if (range.min == range.max) {
-                range.max += 1;
-            }
-
-            return range;
-        });
-    }
-
-    private getPathSuggestionsSqlite(text: string) {
-        return axios.post(`${this.baseUrl}fts/paths`, {
-            prefix: text,
-            minDepth: 1,
-            maxDepth: 10000
-        }).then(resp => {
-            return resp.data.map(bucket => bucket.path);
-        })
-    }
-
-    private getPathSuggestionsEs(text) {
-        return this.esQuery({
-            suggest: {
-                path: {
-                    prefix: text,
-                    completion: {
-                        field: "suggest-path",
-                        skip_duplicates: true,
-                        size: 10000
-                    }
-                }
-            }
-        }).then(resp => {
-            return resp["suggest"]["path"][0]["options"]
-                .map(opt => opt["_source"]["path"]);
-        });
-    }
-
-    getPathSuggestions(text: string): Promise<string[]> {
-        if (this.backend() == "sqlite") {
-            return this.getPathSuggestionsSqlite(text);
-        } else {
-            return this.getPathSuggestionsEs(text)
-        }
-    }
-
-    getTreemapStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/TMAP`;
-    }
-
-    getMimeStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/MAGG`;
-    }
-
-    getSizeStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/SAGG`;
-    }
-
-    getDateStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/DAGG`;
-    }
-
-    private getDocumentEs(docId: string, highlight: boolean, fuzzy: boolean) {
-        const query = Sist2Query.searchQuery();
-
-        if (highlight) {
-            const fields = fuzzy
-                ? {"content.nGram": {}}
-                : {content: {}};
-
-            query.highlight = {
-                pre_tags: ["<mark>"],
-                post_tags: ["</mark>"],
-                number_of_fragments: 0,
-                fields,
-            };
-
-            if (!store.state.sist2Info.esVersionLegacy) {
-                query.highlight.max_analyzed_offset = 999_999;
-            }
-        }
-
-        if ("function_score" in query.query) {
-            query.query = query.query.function_score.query;
-        }
-
-        if (!("must" in query.query.bool)) {
-            query.query.bool.must = [];
-        } else if (!Array.isArray(query.query.bool.must)) {
-            query.query.bool.must = [query.query.bool.must];
-        }
-
-        query.query.bool.must.push({match: {_id: docId}});
-
-        delete query["sort"];
-        delete query["aggs"];
-        delete query["search_after"];
-        delete query.query["function_score"];
-
-        query._source = {
-            includes: ["content", "name", "path", "extension"]
-        }
-
-        query.size = 1;
-
-        return this.esQuery(query).then(resp => {
-            if (resp.hits.hits.length === 1) {
-                return resp.hits.hits[0];
-            }
-            return null;
-        });
-    }
-
-    private getDocumentSqlite(docId: string): Promise<EsHit> {
-        return axios.get(`${this.baseUrl}/fts/d/${docId}`)
-            .then(resp => ({
-                _source: resp.data
-            } as EsHit));
-    }
-
-    getDocument(docId: string, highlight: boolean, fuzzy: boolean): Promise<EsHit | null> {
-        if (this.backend() == "sqlite") {
-            return this.getDocumentSqlite(docId);
-        } else {
-            return this.getDocumentEs(docId, highlight, fuzzy);
-        }
-    }
-
-    getTagSuggestions(prefix: string): Promise<string[]> {
-        if (this.backend() == "sqlite") {
-            return this.getTagSuggestionsSqlite(prefix);
-        } else {
-            return this.getTagSuggestionsEs(prefix);
-        }
-    }
-
-    private getTagSuggestionsSqlite(prefix): Promise<string[]> {
-        return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
-            .then(resp => (resp.data));
-    }
-
-    private getTagSuggestionsEs(prefix): Promise<string[]> {
-        return this.esQuery({
-            suggest: {
-                tag: {
-                    prefix: prefix,
-                    completion: {
-                        field: "suggest-tag",
-                        skip_duplicates: true,
-                        size: 10000
-                    }
-                }
-            }
-        }).then(resp => {
-            const result = [];
-            resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
-                tags.forEach(tag => {
-                    const t = tag.slice(0, -8);
-                    if (!result.find(x => x.slice(0, -8) === t)) {
-                        result.push(tag);
-                    }
-                });
-            });
-            return result;
-        });
+    getDateCsv(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/4`;
     }
 }
 

sist2-vue/src/Sist2Query.ts

@@ -60,7 +60,14 @@ const SORT_MODES = {
     }
 } as any;
 
-class Sist2ElasticsearchQuery {
+interface SortMode {
+    text: string
+    mode: any[]
+    key: (hit: EsHit) => any
+}
+
+
+class Sist2Query {
 
     searchQuery(blankSearch: boolean = false): any {
 
@@ -242,5 +249,4 @@ class Sist2ElasticsearchQuery {
     }
 }
 
-
-export default new Sist2ElasticsearchQuery();
+export default new Sist2Query();

sist2-vue/src/Sist2SqliteQuery.ts

@@ -1,111 +0,0 @@
-import store from "./store";
-import {EsHit, Index} from "@/Sist2Api";
-
-const SORT_MODES = {
-    score: {
-        "sort": "score",
-    },
-    random: {
-        "sort": "random"
-    },
-    dateAsc: {
-        "sort": "mtime"
-    },
-    dateDesc: {
-        "sort": "mtime",
-        "sortAsc": false
-    },
-    sizeAsc: {
-        "sort": "size",
-    },
-    sizeDesc: {
-        "sort": "size",
-        "sortAsc": false
-    },
-    nameAsc: {
-        "sort": "name",
-    },
-    nameDesc: {
-        "sort": "name",
-        "sortAsc": false
-    }
-} as any;
-
-interface SortMode {
-    text: string
-    mode: any[]
-    key: (hit: EsHit) => any
-}
-
-
-class Sist2ElasticsearchQuery {
-
-    searchQuery(): any {
-
-        const getters = store.getters;
-
-        const searchText = getters.searchText;
-        const pathText = getters.pathText;
-        const sizeMin = getters.sizeMin;
-        const sizeMax = getters.sizeMax;
-        const dateMin = getters.dateMin;
-        const dateMax = getters.dateMax;
-        const size = getters.size;
-        const after = getters.lastDoc;
-        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
-        const selectedMimeTypes = getters.selectedMimeTypes;
-        const selectedTags = getters.selectedTags;
-
-        const q = {
-            "pageSize": size
-        }
-
-        Object.assign(q, SORT_MODES[getters.sortMode]);
-
-        if (!after) {
-            q["fetchAggregations"] = true;
-        }
-        if (searchText) {
-            q["query"] = searchText;
-        }
-        if (pathText) {
-            q["path"] = pathText.endsWith("/") ? pathText.slice(0, -1) : pathText;
-        }
-        if (sizeMin) {
-            q["sizeMin"] = sizeMin;
-        }
-        if (sizeMax) {
-            q["sizeMax"] = sizeMax;
-        }
-        if (dateMin) {
-            q["dateMin"] = dateMin;
-        }
-        if (dateMax) {
-            q["dateMax"] = dateMax;
-        }
-        if (after) {
-            q["after"] = after.sort;
-        }
-        if (selectedIndexIds.length > 0) {
-            q["indexIds"] = selectedIndexIds;
-        }
-        if (selectedMimeTypes.length > 0) {
-            q["mimeTypes"] = selectedMimeTypes;
-        }
-        if (selectedTags.length > 0) {
-            q["tags"] = selectedTags
-        }
-        if (getters.sortMode == "random") {
-            q["seed"] = getters.seed;
-        }
-        if (getters.optHighlight) {
-            q["highlight"] = true;
-            q["highlightContextSize"] = Number(getters.optFragmentSize);
-        }
-
-        return q;
-    }
-}
-
-
-export default new Sist2ElasticsearchQuery();

sist2-vue/src/components/AnalyzedContentSpan.vue

@@ -1,21 +0,0 @@
-<template>
-    <span :style="getStyle()">{{span.text}}</span>
-</template>
-
-<script>
-
-
-import ModelsRepo from "@/ml/modelsRepo";
-
-export default {
-    name: "AnalyzedContentSpan",
-    props: ["span", "text"],
-    methods: {
-        getStyle() {
-            return ModelsRepo.data[this.$store.getters.mlModel.name].labelStyles[this.span.label];
-        }
-    }
-}
-</script>
-
-<style scoped></style>

sist2-vue/src/components/AnalyzedContentSpanContainer.vue

@@ -1,75 +0,0 @@
-<template>
-    <div>
-        <b-card class="mb-2">
-            <AnalyzedContentSpan v-for="span of legend" :key="span.id" :span="span"
-                                 class="mr-2"></AnalyzedContentSpan>
-        </b-card>
-        <div class="content-div">
-            <AnalyzedContentSpan v-for="span of mergedSpans" :key="span.id" :span="span"></AnalyzedContentSpan>
-        </div>
-    </div>
-</template>
-
-<script>
-
-
-import AnalyzedContentSpan from "@/components/AnalyzedContentSpan.vue";
-import ModelsRepo from "@/ml/modelsRepo";
-
-export default {
-    name: "AnalyzedContentSpanContainer",
-    components: {AnalyzedContentSpan},
-    props: ["spans", "text"],
-    computed: {
-        legend() {
-            return Object.entries(ModelsRepo.data[this.$store.state.mlModel.name].legend)
-                .map(([label, name]) => ({
-                    text: name,
-                    id: label,
-                    label: label
-                }));
-        },
-        mergedSpans() {
-            const spans = this.spans;
-
-            const merged = [];
-
-            let lastLabel = null;
-            let fixSpace = false;
-            for (let i = 0; i < spans.length; i++) {
-
-                if (spans[i].label !== lastLabel) {
-                    let start = spans[i].wordIndex;
-                    const nextSpan = spans.slice(i + 1).find(s => s.label !== spans[i].label)
-                    let end = nextSpan ? nextSpan.wordIndex : undefined;
-
-                    if (end !== undefined && this.text[end - 1] === " ") {
-                        end -= 1;
-                        fixSpace = true;
-                    }
-
-                    merged.push({
-                        text: this.text.slice(start, end),
-                        label: spans[i].label,
-                        id: spans[i].wordIndex
-                    });
-
-                    if (fixSpace) {
-                        merged.push({
-                            text: " ",
-                            label: "O",
-                            id: end
-                        });
-                        fixSpace = false;
-                    }
-                    lastLabel = spans[i].label;
-                }
-            }
-
-            return merged;
-        },
-    },
-}
-</script>
-
-<style scoped></style>

sist2-vue/src/components/D3DateHistogram.vue

@@ -120,7 +120,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#date-histogram");
 
-      d3.json(Sist2Api.getDateStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getDateCsv(indexId)).then(tabularData => {
         dateHistogram(tabularData.slice(), svg, this.$t("d3.dateHistogram"));
       });
     }

sist2-vue/src/components/D3MimeBarCount.vue

@@ -91,7 +91,7 @@ export default {
       const mimeSvgCount = d3.select("#agg-mime-count");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
         mimeBarCount(tabularData.slice(), mimeSvgCount, fillOpacity, this.$t("d3.mimeCount"));
       });
     }

sist2-vue/src/components/D3MimeBarSize.vue

@@ -90,7 +90,7 @@ export default {
       const mimeSvgSize = d3.select("#agg-mime-size");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
         mimeBarSize(tabularData.slice(), mimeSvgSize, fillOpacity, this.$t("d3.mimeSize"));
       });
     }

sist2-vue/src/components/D3SizeHistogram.vue

@@ -117,7 +117,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#size-histogram");
 
-      d3.json(Sist2Api.getSizeStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getSizeCsv(indexId)).then(tabularData => {
         sizeHistogram(tabularData.slice(), svg, this.$t("d3.sizeHistogram"));
       });
     }

sist2-vue/src/components/D3Treemap.vue

@@ -240,7 +240,7 @@ export default {
           .style("overflow", "visible")
           .style("font", "10px sans-serif");
 
-      d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
+      d3.csv(Sist2Api.getTreemapCsvUrl(indexId)).then(tabularData => {
         tabularData.forEach(row => {
           row.taxonomy = row.path.split("/");
           row.size = Number(row.size);

sist2-vue/src/components/DebugInfo.vue

@@ -1,56 +1,41 @@
 <template>
-    <b-card v-if="$store.state.sist2Info.showDebugInfo" class="mb-4 mt-4">
-        <b-card-title>
-            <DebugIcon class="mr-1"></DebugIcon>
-            {{ $t("debug") }}
-        </b-card-title>
-        <p v-html="$t('debugDescription')"></p>
+  <b-card class="mb-4 mt-4">
+    <b-card-title><DebugIcon class="mr-1"></DebugIcon>{{ $t("debug") }}</b-card-title>
+    <p v-html="$t('debugDescription')"></p>
 
-        <b-card-body>
+    <b-card-body>
 
-            <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
+      <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
 
-            <hr/>
-            <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx"
-                            class="mt-2"></IndexDebugInfo>
-        </b-card-body>
-    </b-card>
+      <hr />
+      <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx" class="mt-2"></IndexDebugInfo>
+    </b-card-body>
+  </b-card>
 </template>
 
 <script>
 import IndexDebugInfo from "@/components/IndexDebugInfo";
 import DebugIcon from "@/components/icons/DebugIcon";
-import {mapGetters} from "vuex";
 
 export default {
-    name: "DebugInfo.vue",
-    components: {DebugIcon, IndexDebugInfo},
-    computed: {
-        ...mapGetters([
-            "uiSqliteMode",
-        ]),
-        tableItems() {
-            const items = [
-                {key: "version", value: this.$store.state.sist2Info.version},
-                {key: "platform", value: this.$store.state.sist2Info.platform},
-                {key: "debugBinary", value: this.$store.state.sist2Info.debug},
-                {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
-                {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
-                {key: "tagline", value: this.$store.state.sist2Info.tagline},
-                {key: "dev", value: this.$store.state.sist2Info.dev},
-                {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
-            ];
-
-            if (!this.uiSqliteMode) {
-                items.push(
-                    {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
-                    {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
-                    {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy}
-                );
-            }
-
-            return items;
-        }
+  name: "DebugInfo.vue",
+  components: {DebugIcon, IndexDebugInfo},
+  computed: {
+    tableItems() {
+      return [
+        {key: "version", value: this.$store.state.sist2Info.version},
+        {key: "platform", value: this.$store.state.sist2Info.platform},
+        {key: "debugBinary", value: this.$store.state.sist2Info.debug},
+        {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
+        {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
+        {key: "tagline", value: this.$store.state.sist2Info.tagline},
+        {key: "dev", value: this.$store.state.sist2Info.dev},
+        {key: "mongooseVersion", value: this.$store.state.sist2Info.mongooseVersion},
+        {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
+        {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
+        {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
+      ]
     }
+  }
 }
 </script>

sist2-vue/src/components/DocCard.vue

@@ -1,44 +1,44 @@
 <template>
-    <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
-         @click="$store.commit('busTnTouchStart', null)">
-        <b-card
-                no-body
-                img-top
-        >
-            <!-- Info modal-->
-            <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
+  <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`"
+       @click="$store.commit('busTnTouchStart', null)">
+    <b-card
+        no-body
+        img-top
+    >
+      <!-- Info modal-->
+      <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
 
-            <ContentDiv :doc="doc"></ContentDiv>
+      <ContentDiv :doc="doc"></ContentDiv>
 
-            <!-- Thumbnail-->
-            <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
+      <!-- Thumbnail-->
+      <FullThumbnail :doc="doc" :small-badge="smallBadge" @onThumbnailClick="onThumbnailClick()"></FullThumbnail>
 
-            <!-- Audio player-->
-            <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
-                   :type="doc._source.mime"
-                   :src="`f/${doc._source.index}/${doc._id}`"
-                   @play="onAudioPlay()"></audio>
+      <!-- Audio player-->
+      <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
+             :type="doc._source.mime"
+             :src="`f/${doc._id}`"
+             @play="onAudioPlay()"></audio>
 
-            <b-card-body class="padding-03">
+      <b-card-body class="padding-03">
 
-                <!-- Title line -->
-                <div style="display: flex">
-                    <span class="info-icon" @click="onInfoClick()"></span>
-                    <DocFileTitle :doc="doc"></DocFileTitle>
-                </div>
+        <!-- Title line -->
+        <div style="display: flex">
+          <span class="info-icon" @click="onInfoClick()"></span>
+          <DocFileTitle :doc="doc"></DocFileTitle>
+        </div>
 
-                <!-- Featured line -->
-                <div style="display: flex">
-                    <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
-                </div>
+        <!-- Featured line -->
+        <div style="display: flex">
+          <FeaturedFieldsLine :doc="doc"></FeaturedFieldsLine>
+        </div>
 
-                <!-- Tags -->
-                <div class="card-text">
-                    <TagContainer :hit="doc"></TagContainer>
-                </div>
-            </b-card-body>
-        </b-card>
-    </div>
+        <!-- Tags -->
+        <div class="card-text">
+          <TagContainer :hit="doc"></TagContainer>
+        </div>
+      </b-card-body>
+    </b-card>
+  </div>
 </template>
 
 <script>
@@ -52,91 +52,91 @@ import FeaturedFieldsLine from "@/components/FeaturedFieldsLine";
 
 
 export default {
-    components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
-    props: ["doc", "width"],
-    data() {
-        return {
-            ext: ext,
-            showInfo: false,
+  components: {FeaturedFieldsLine, FullThumbnail, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+  props: ["doc", "width"],
+  data() {
+    return {
+      ext: ext,
+      showInfo: false,
+    }
+  },
+  computed: {
+    smallBadge() {
+      return this.width < 150;
+    }
+  },
+  methods: {
+    humanFileSize: humanFileSize,
+    humanTime: humanTime,
+    onInfoClick() {
+      this.showInfo = true;
+    },
+    async onThumbnailClick() {
+      this.$store.commit("setUiLightboxSlide", this.doc._seq);
+      await this.$store.dispatch("showLightbox");
+    },
+    onAudioPlay() {
+      document.getElementsByTagName("audio").forEach((el) => {
+        if (el !== this.$refs["audio"]) {
+          el.pause();
         }
+      });
     },
-    computed: {
-        smallBadge() {
-            return this.width < 150;
-        }
-    },
-    methods: {
-        humanFileSize: humanFileSize,
-        humanTime: humanTime,
-        onInfoClick() {
-            this.showInfo = true;
-        },
-        async onThumbnailClick() {
-            this.$store.commit("setUiLightboxSlide", this.doc._seq);
-            await this.$store.dispatch("showLightbox");
-        },
-        onAudioPlay() {
-            Array.prototype.slice.call(document.getElementsByTagName("audio")).forEach((el) => {
-                if (el !== this.$refs["audio"]) {
-                    el.pause();
-                }
-            });
-        },
-    },
+  },
 }
 </script>
 <style>
 .fit {
-    display: block;
-    min-width: 64px;
-    max-width: 100%;
-    /*max-height: 400px;*/
-    margin: 0 auto 0;
-    width: auto;
-    height: auto;
+  display: block;
+  min-width: 64px;
+  max-width: 100%;
+  /*max-height: 400px;*/
+  margin: 0 auto 0;
+  width: auto;
+  height: auto;
 }
 
 .audio-fit {
-    height: 39px;
-    vertical-align: bottom;
-    display: inline;
-    width: 100%;
+  height: 39px;
+  vertical-align: bottom;
+  display: inline;
+  width: 100%;
 }
 </style>
 
 <style scoped>
 
 .padding-03 {
-    padding: 0.3rem;
+  padding: 0.3rem;
 }
 
 .card {
-    margin-top: 1em;
-    margin-left: 0;
-    margin-right: 0;
-    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-    border-radius: 0;
-    border: none;
+  margin-top: 1em;
+  margin-left: 0;
+  margin-right: 0;
+  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+  border-radius: 0;
+  border: none;
 }
 
 .card-body {
-    padding: 0.3rem;
+  padding: 0.3rem;
 }
 
 .doc-card {
-    padding-left: 3px;
-    padding-right: 3px;
+  padding-left: 3px;
+  padding-right: 3px;
 }
 
 .sub-document .card {
-    background: #AB47BC1F !important;
+  background: #AB47BC1F !important;
 }
 
 .theme-black .sub-document .card {
-    background: #37474F !important;
+  background: #37474F !important;
 }
 
 .sub-document .fit {
-    padding: 4px 4px 0 4px;
+  padding: 4px 4px 0 4px;
 }
 </style>

sist2-vue/src/components/DocFileTitle.vue

@@ -1,5 +1,5 @@
 <template>
-  <a :href="`f/${doc._source.index}/${doc._id}`" class="file-title-anchor" target="_blank">
+  <a :href="`f/${doc._id}`" class="file-title-anchor" target="_blank">
     <div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
          v-html="fileName() + ext(doc)"></div>
   </a>

sist2-vue/src/components/DocListItem.vue

@@ -17,7 +17,7 @@
           </div>
 
           <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
-               :src="(doc._props.isGif && hover) ? `f/${doc._source.index}/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
+               :src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
                alt=""
                class="pointer fit-sm" @click="onThumbnailClick()">
           <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""

sist2-vue/src/components/FeaturedFieldsLine.vue

@@ -1,46 +1,42 @@
 <template>
-    <div class="featured-line" v-html="featuredLineHtml"></div>
+  <div class="featured-line" v-html="featuredLineHtml"></div>
 </template>
 
 <script>
 import {humanDate, humanFileSize} from "@/util";
 
 function scopedEval(context, expr) {
-    const evaluator = Function.apply(null, [...Object.keys(context), "expr", "return eval(expr)"]);
-    return evaluator.apply(null, [...Object.values(context), expr]);
+  const evaluator = Function.apply(null, [...Object.keys(context), "expr", "return eval(expr)"]);
+  return evaluator.apply(null, [...Object.values(context), expr]);
 }
 
 
 export default {
-    name: "FeaturedFieldsLine",
-    props: ["doc"],
-    computed: {
-        featuredLineHtml() {
-            if (this.$store.getters.optFeaturedFields === undefined) {
-                return "";
-            }
+  name: "FeaturedFieldsLine",
+  props: ["doc"],
+  computed: {
+    featuredLineHtml() {
+      const scope = {doc: this.doc._source, humanDate: humanDate, humanFileSize: humanFileSize};
 
-            const scope = {doc: this.doc._source, humanDate: humanDate, humanFileSize: humanFileSize};
-
-            return this.$store.getters.optFeaturedFields
-                .replaceAll(/\$\{([^}]*)}/g, (match, g1) => {
-                    return scopedEval(scope, g1);
-                });
-        }
+      return this.$store.getters.optFeaturedFields
+          .replaceAll(/\$\{([^}]*)}/g, (match, g1) => {
+            return scopedEval(scope, g1);
+          });
     }
+  }
 }
 </script>
 
 <style scoped>
 
 .featured-line {
-    font-size: 90%;
-    font-family: 'Source Sans Pro', 'Helvetica Neue', Arial, sans-serif;
-    color: #424242;
-    padding-left: 2px;
+  font-size: 90%;
+  font-family: 'Source Sans Pro', 'Helvetica Neue', Arial, sans-serif;
+  color: #424242;
+  padding-left: 2px;
 }
 
 .theme-black .featured-line {
-    color: #bebebe;
+  color: #bebebe;
 }
 </style>

sist2-vue/src/components/FullThumbnail.vue

@@ -71,7 +71,7 @@ export default {
       const doc = this.doc;
       const props = doc._props;
       if (props.isGif && this.hover) {
-        return `f/${doc._source.index}/${doc._id}`;
+        return `f/${doc._id}`;
       }
       return (this.currentThumbnailNum === 0)
           ? `t/${doc._source.index}/${doc._id}`

sist2-vue/src/components/LazyContentDiv.vue

@@ -1,167 +1,94 @@
 <template>
-    <Preloader v-if="loading"></Preloader>
-    <div v-else-if="content">
-        <b-form inline class="my-2" v-if="ModelsRepo.getOptions().length > 0">
-            <b-checkbox class="ml-auto mr-2" :checked="optAutoAnalyze"
-                        @input="setOptAutoAnalyze($event); $store.dispatch('updateConfiguration')">
-                {{ $t("ml.auto") }}
-            </b-checkbox>
-            <b-button :disabled="mlPredictionsLoading || mlLoading" @click="mlAnalyze" variant="primary"
-            >{{ $t("ml.analyzeText") }}
-            </b-button>
-            <b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="mlModel">
-                <b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()">{{ opt.text }}
-                </b-select-option>
-            </b-select>
-        </b-form>
-
-        <b-progress v-if="mlLoading" variant="warning" show-progress :max="1" class="mb-3"
-        >
-            <b-progress-bar :value="modelLoadingProgress">
-                <strong>{{ ((modelLoadingProgress * modelSize) / (1024 * 1024)).toFixed(1) }}MB / {{
-                    (modelSize / (1024 * 1024)).toFixed(1)
-                    }}MB</strong>
-            </b-progress-bar>
-        </b-progress>
-
-        <b-progress v-if="mlPredictionsLoading" variant="primary" :value="modelPredictionProgress"
-                    :max="content.length" class="mb-3"></b-progress>
-
-        <AnalyzedContentSpansContainer v-if="analyzedContentSpans.length > 0"
-                                       :spans="analyzedContentSpans" :text="rawContent"></AnalyzedContentSpansContainer>
-        <div v-else class="content-div" v-html="content"></div>
-    </div>
+  <Preloader v-if="loading"></Preloader>
+  <div v-else-if="content" class="content-div" v-html="content"></div>
 </template>
 
 <script>
 import Sist2Api from "@/Sist2Api";
 import Preloader from "@/components/Preloader";
-import Sist2Query from "@/Sist2ElasticsearchQuery";
+import Sist2Query from "@/Sist2Query";
 import store from "@/store";
-import BertNerModel from "@/ml/BertNerModel";
-import AnalyzedContentSpansContainer from "@/components/AnalyzedContentSpanContainer.vue";
-import ModelsRepo from "@/ml/modelsRepo";
-import {mapGetters, mapMutations} from "vuex";
 
 export default {
-    name: "LazyContentDiv",
-    components: {AnalyzedContentSpansContainer, Preloader},
-    props: ["docId"],
-    data() {
-        return {
-            ModelsRepo,
-            content: "",
-            rawContent: "",
-            loading: true,
-            modelLoadingProgress: 0,
-            modelPredictionProgress: 0,
-            mlPredictionsLoading: false,
-            mlLoading: false,
-            mlModel: null,
-            analyzedContentSpans: []
-        }
-    },
-    mounted() {
-
-        if (this.$store.getters.optMlDefaultModel) {
-            this.mlModel = this.$store.getters.optMlDefaultModel
-        } else {
-            this.mlModel = ModelsRepo.getDefaultModel();
-        }
-
-        Sist2Api
-            .getDocument(this.docId, this.$store.state.optHighlight, this.$store.state.fuzzy)
-            .then(doc => {
-                this.loading = false;
-
-                if (doc) {
-                    this.content = this.getContent(doc)
-                }
-
-                if (this.optAutoAnalyze) {
-                    this.mlAnalyze();
-                }
-            });
-    },
-    computed: {
-        ...mapGetters(["optAutoAnalyze"]),
-        modelSize() {
-            const modelData = ModelsRepo.data[this.mlModel];
-            if (!modelData) {
-                return 0;
-            }
-            return modelData.size;
-        }
-    },
-    methods: {
-        ...mapMutations(["setOptAutoAnalyze"]),
-        getContent(doc) {
-            this.rawContent = doc._source.content;
-
-            if (!doc.highlight) {
-                return doc._source.content;
-            }
-
-            if (doc.highlight["content.nGram"]) {
-                return doc.highlight["content.nGram"][0];
-            }
-            if (doc.highlight.content) {
-                return doc.highlight.content[0];
-            }
-        },
-        async getMlModel() {
-            if (this.$store.getters.mlModel.name !== this.mlModel) {
-                this.mlLoading = true;
-                this.modelLoadingProgress = 0;
-                const modelInfo = ModelsRepo.data[this.mlModel];
-
-                const model = new BertNerModel(
-                    modelInfo.vocabUrl,
-                    modelInfo.modelUrl,
-                    modelInfo.id2label,
-                )
-
-                await model.init(progress => this.modelLoadingProgress = progress);
-                this.$store.commit("setMlModel", {model, name: this.mlModel});
-
-                this.mlLoading = false;
-                return model
-            }
-
-            return this.$store.getters.mlModel.model;
-        },
-        async mlAnalyze() {
-            if (!this.content) {
-                return;
-            }
-
-            const modelInfo = ModelsRepo.data[this.mlModel];
-            if (modelInfo === undefined) {
-                return;
-            }
-
-            this.$store.commit("setOptMlDefaultModel", this.mlModel);
-            await this.$store.dispatch("updateConfiguration");
-
-            const model = await this.getMlModel();
-
-            this.analyzedContentSpans = [];
-
-            this.mlPredictionsLoading = true;
-
-            await model.predict(this.rawContent, results => {
-                results.forEach(result => result.label = modelInfo.humanLabels[result.label]);
-                this.analyzedContentSpans.push(...results);
-                this.modelPredictionProgress = results[results.length - 1].wordIndex;
-            });
-            this.mlPredictionsLoading = false;
-        }
+  name: "LazyContentDiv",
+  components: {Preloader},
+  props: ["docId"],
+  data() {
+    return {
+      content: "",
+      loading: true
     }
+  },
+  mounted() {
+    const query = Sist2Query.searchQuery();
+
+    if (this.$store.state.optHighlight) {
+
+      const fields = this.$store.state.fuzzy
+          ? {"content.nGram": {}}
+          : {content: {}};
+
+      query.highlight = {
+        pre_tags: ["<mark>"],
+        post_tags: ["</mark>"],
+        number_of_fragments: 0,
+        fields,
+      };
+
+      if (!store.state.sist2Info.esVersionLegacy) {
+        query.highlight.max_analyzed_offset = 999_999;
+      }
+    }
+
+    if ("function_score" in query.query) {
+      query.query = query.query.function_score.query;
+    }
+
+    if (!("must" in query.query.bool)) {
+      query.query.bool.must = [];
+    } else if (!Array.isArray(query.query.bool.must)) {
+      query.query.bool.must = [query.query.bool.must];
+    }
+
+    query.query.bool.must.push({match: {_id: this.docId}});
+
+    delete query["sort"];
+    delete query["aggs"];
+    delete query["search_after"];
+    delete query.query["function_score"];
+
+    query._source = {
+      includes: ["content", "name", "path", "extension"]
+    }
+
+    query.size = 1;
+
+    Sist2Api.esQuery(query).then(resp => {
+      this.loading = false;
+      if (resp.hits.hits.length === 1) {
+        this.content = this.getContent(resp.hits.hits[0]);
+      } else {
+        console.log("FIXME: could not get content")
+        console.log(resp)
+      }
+    });
+  },
+  methods: {
+    getContent(doc) {
+      if (!doc.highlight) {
+        return doc._source.content;
+      }
+
+      if (doc.highlight["content.nGram"]) {
+        return doc.highlight["content.nGram"][0];
+      }
+      if (doc.highlight.content) {
+        return doc.highlight.content[0];
+      }
+    }
+  }
 }
 </script>
 
-<style>
-.progress-bar {
-    transition: none;
-}
+<style scoped>
 </style>

sist2-vue/src/components/MimePicker.vue

@@ -9,7 +9,7 @@ import InspireTreeDOM from "inspire-tree-dom";
 import "inspire-tree-dom/dist/inspire-tree-light.min.css";
 import {getSelectedTreeNodes, getTreeNodeAttributes} from "@/util";
 import Sist2Api from "@/Sist2Api";
-import Sist2Query from "@/Sist2ElasticsearchQuery";
+import Sist2Query from "@/Sist2Query";
 
 export default {
   name: "MimePicker",

sist2-vue/src/components/NavBar.vue

@@ -10,7 +10,7 @@
     <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
       v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
         href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
-        target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
+        target="_blank">legacyES</a></span>
     </span>
 
     <span v-if="$store && $store.state.sist2Info" class="tagline" v-html="tagline()"></span>

sist2-vue/src/components/PathTree.vue

@@ -1,41 +1,40 @@
 <template>
-    <div>
-        <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
-            <div class="input-group-prepend">
+  <div>
+    <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
+      <div class="input-group-prepend">
 
-                <b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
-                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
-                        <path
-                                fill="currentColor"
-                                d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
-                        />
-                    </svg>
-                </b-button>
-            </div>
-
-            <VueSimpleSuggest
-                    class="form-control-fix-flex"
-                    @input="setPathText"
-                    :value="getPathText"
-                    :list="suggestPath"
-                    :max-suggestions="0"
-                    :placeholder="$t('pathBar.placeholder')"
-                    :debounce="200"
-            >
-                <!-- Suggestion item template-->
-                <div slot="suggestion-item" slot-scope="{ suggestion, query }">
-                    <div class="suggestion-line" :title="suggestion">
-                        <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
-                    </div>
-                </div>
-            </VueSimpleSuggest>
+        <b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
+            <path
+                fill="currentColor"
+                d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
+            />
+          </svg>
+        </b-button>
+      </div>
 
+      <VueSimpleSuggest
+          class="form-control-fix-flex"
+          @input="setPathText"
+          :value="getPathText"
+          :list="suggestPath"
+          :max-suggestions="0"
+          :placeholder="$t('pathBar.placeholder')"
+      >
+        <!-- Suggestion item template-->
+        <div slot="suggestion-item" slot-scope="{ suggestion, query }">
+          <div class="suggestion-line" :title="suggestion">
+            <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
+          </div>
         </div>
+      </VueSimpleSuggest>
 
-        <b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
-            <div id="pathTree"></div>
-        </b-modal>
     </div>
+
+    <b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
+      <div id="pathTree"></div>
+    </b-modal>
+  </div>
 </template>
 
 <script>
@@ -49,153 +48,198 @@ import VueSimpleSuggest from 'vue-simple-suggest'
 import 'vue-simple-suggest/dist/styles.css' // Optional CSS
 
 export default {
-    name: "PathTree",
-    components: {
-        VueSimpleSuggest
-    },
-    data() {
-        return {
-            mimeTree: null,
-            pathItems: [],
-            tmpPath: ""
+  name: "PathTree",
+  components: {
+    VueSimpleSuggest
+  },
+  data() {
+    return {
+      mimeTree: null,
+      pathItems: [],
+      tmpPath: ""
 
-        }
-    },
-    computed: {
-        ...mapGetters(["getPathText"])
-    },
-    mounted() {
-        this.$store.subscribe((mutation) => {
-            // Wait until indices are loaded to get the root paths
-            if (mutation.type === "setIndices") {
-                let pathTree = new InspireTree({
-                    data: (node, resolve, reject) => {
-                        return this.getNextDepth(node);
-                    },
-                    sort: "text"
-                });
-
-                this.$store.state.indices.forEach(idx => {
-                    pathTree.addNode({
-                        id: "/" + idx.id,
-                        values: ["/" + idx.id],
-                        text: `/[${idx.name}]`,
-                        index: idx.id,
-                        depth: 0,
-                        children: true
-                    })
-                });
-
-                new InspireTreeDOM(pathTree, {
-                    target: "#pathTree"
-                });
-
-                pathTree.on("node.click", this.handleTreeClick);
-                pathTree.expand();
-            }
+    }
+  },
+  computed: {
+    ...mapGetters(["getPathText"])
+  },
+  mounted() {
+    this.$store.subscribe((mutation) => {
+      // Wait until indices are loaded to get the root paths
+      if (mutation.type === "setIndices") {
+        let pathTree = new InspireTree({
+          data: (node, resolve, reject) => {
+            return this.getNextDepth(node);
+          },
+          sort: "text"
         });
+
+        this.$store.state.indices.forEach(idx => {
+          pathTree.addNode({
+            id: "/" + idx.id,
+            values: ["/" + idx.id],
+            text: `/[${idx.name}]`,
+            index: idx.id,
+            depth: 0,
+            children: true
+          })
+        });
+
+        new InspireTreeDOM(pathTree, {
+          target: "#pathTree"
+        });
+
+        pathTree.on("node.click", this.handleTreeClick);
+        pathTree.expand();
+      }
+    });
+  },
+  methods: {
+    ...mapMutations(["setPathText"]),
+    getSuggestionWithoutQueryPrefix(suggestion, query) {
+      return suggestion.slice(query.length)
     },
-    methods: {
-        ...mapMutations(["setPathText"]),
-        getSuggestionWithoutQueryPrefix(suggestion, query) {
-            return suggestion.slice(query.length)
-        },
-        async getPathChoices() {
-            return new Promise(getPaths => {
-                Sist2Api.getPathSuggestions(this.getPathText).then(getPaths);
-            });
-        },
-        async suggestPath(term) {
-            if (!this.$store.state.optSuggestPath) {
-                return []
+    async getPathChoices() {
+      return new Promise(getPaths => {
+        const q = {
+          suggest: {
+            path: {
+              prefix: this.getPathText,
+              completion: {
+                field: "suggest-path",
+                skip_duplicates: true,
+                size: 10000
+              }
             }
+          }
+        };
 
-            term = term.toLowerCase();
-
-            const choices = await this.getPathChoices();
-
-            let matches = [];
-            for (let i = 0; i < choices.length; i++) {
-                if (~choices[i].toLowerCase().indexOf(term)) {
-                    matches.push(choices[i]);
-                }
-            }
-            return matches.sort((a, b) => a.length - b.length);
-        },
-        getNextDepth(node) {
-            return Sist2Api
-                .searchPaths(node.index, node.depth + 1, node.depth + 3, node.depth > 0 ? node.id : null)
-                .then(buckets => {
-                    const paths = [];
-
-                    return buckets
-                        .filter(bucket => bucket.path.length > node.id.length || node.id.startsWith("/"))
-                        .sort((a, b) => a.path > b.path ? 1 : -1)
-                        .map(bucket => {
-                            if (paths.some(n => bucket.path.startsWith(n))) {
-                                return null;
-                            }
-
-                            const name = node.id.startsWith("/") ? bucket.path : bucket.path.slice(node.id.length + 1);
-
-                            paths.push(bucket.path);
-
-                            return {
-                                id: bucket.path,
-                                text: `${name}/ (${bucket.count})`,
-                                depth: node.depth + 1,
-                                index: node.index,
-                                values: [bucket.path],
-                                children: true,
-                            }
-                        })
-                        .filter(bucket => bucket !== null);
-                });
-        },
-        handleTreeClick(e, node, handler) {
-            if (node.depth !== 0) {
-                this.setPathText(node.id);
-                this.$refs['path-modal'].hide()
-
-                this.$emit("search");
-            }
-
-            handler();
-        },
+        Sist2Api.esQuery(q)
+            .then(resp => getPaths(resp["suggest"]["path"][0]["options"].map(opt => opt["_source"]["path"])));
+      })
     },
+    async suggestPath(term) {
+      if (!this.$store.state.optSuggestPath) {
+        return []
+      }
+
+      term = term.toLowerCase();
+
+      const choices = await this.getPathChoices();
+
+      let matches = [];
+      for (let i = 0; i < choices.length; i++) {
+        if (~choices[i].toLowerCase().indexOf(term)) {
+          matches.push(choices[i]);
+        }
+      }
+      return matches.sort((a, b) => a.length - b.length);
+    },
+    getNextDepth(node) {
+      const q = {
+        query: {
+          bool: {
+            filter: [
+              {term: {index: node.index}},
+              {range: {_depth: {gte: node.depth + 1, lte: node.depth + 3}}},
+            ]
+          }
+        },
+        aggs: {
+          paths: {
+            terms: {
+              field: "path",
+              size: 10000
+            }
+          }
+        },
+        size: 0
+      };
+
+      if (node.depth > 0) {
+        q.query.bool.must = {
+          prefix: {
+            path: node.id,
+          }
+        };
+      }
+
+      return Sist2Api.esQuery(q).then(resp => {
+        const buckets = resp["aggregations"]["paths"]["buckets"];
+        if (!buckets) {
+          return false;
+        }
+
+        const paths = [];
+
+        return buckets
+            .filter(bucket => bucket.key.length > node.id.length || node.id.startsWith("/"))
+            .sort((a, b) => a.key > b.key)
+            .map(bucket => {
+
+              if (paths.some(n => bucket.key.startsWith(n))) {
+                return null;
+              }
+
+              const name = node.id.startsWith("/") ? bucket.key : bucket.key.slice(node.id.length + 1);
+
+              paths.push(bucket.key);
+
+              return {
+                id: bucket.key,
+                text: `${name}/ (${bucket.doc_count})`,
+                depth: node.depth + 1,
+                index: node.index,
+                values: [bucket.key],
+                children: true,
+              }
+            }).filter(x => x !== null)
+      });
+    },
+    handleTreeClick(e, node, handler) {
+      if (node.depth !== 0) {
+        this.setPathText(node.id);
+        this.$refs['path-modal'].hide()
+
+        this.$emit("search");
+      }
+
+      handler();
+    },
+  },
 }
 </script>
 
 <style scoped>
 #mimeTree {
-    max-height: 350px;
-    overflow: auto;
+  max-height: 350px;
+  overflow: auto;
 }
 
 .form-control-fix-flex {
-    flex: 1 1 auto;
-    width: 1%;
-    min-width: 0;
-    margin-bottom: 0;
+  flex: 1 1 auto;
+  width: 1%;
+  min-width: 0;
+  margin-bottom: 0;
 }
 
 .suggestion-line {
-    max-width: 100%;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-    line-height: 1.1;
+  max-width: 100%;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  line-height: 1.1;
 }
 
 </style>
 
 <style>
 .suggestions {
-    max-height: 250px;
-    overflow-y: auto;
+  max-height: 250px;
+  overflow-y: auto;
 }
 
 .theme-black .suggestions {
-    color: black
+  color: black
 }
 </style>

sist2-vue/src/components/ResultsCard.vue

@@ -1,46 +1,40 @@
 <template>
-    <b-card v-if="lastResultsLoaded" id="results">
-        <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
+  <b-card v-if="lastResultsLoaded" id="results">
+    <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
 
-        <div style="float: right">
-            <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
-                $t("details")
-                }}
-            </b-button>
+    <div style="float: right">
+      <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile" @click="onToggle()">{{
+          $t("details")
+        }}
+      </b-button>
 
-            <template v-if="hitCount !== 0">
-                <SortSelect class="ml-2"></SortSelect>
+      <template v-if="hitCount !== 0">
+        <SortSelect class="ml-2"></SortSelect>
 
-                <DisplayModeToggle class="ml-2"></DisplayModeToggle>
-            </template>
-        </div>
+        <DisplayModeToggle class="ml-2"></DisplayModeToggle>
+      </template>
+    </div>
 
-        <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
-            <b-card>
-                <b-table :items="tableItems" small borderless thead-class="hidden" class="mb-0"></b-table>
+    <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
+      <b-card>
+        <b-table :items="tableItems" small borderless bordered thead-class="hidden" class="mb-0"></b-table>
 
-                <template v-if="!$store.state.uiSqliteMode">
-
-                    <br/>
-                    <h4>
-                        {{ $t("mimeTypes") }}
-                        <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick">
-                            <ClipboardIcon/>
-                        </b-button>
-                    </h4>
-                    <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
-                    <b-table
-                            v-else
-                            sort-by="doc_count"
-                            :sort-desc="true"
-                            thead-class="hidden"
-                            bordered
-                            :items="$store.state.uiDetailsMimeAgg" small class="mb-0"
-                    ></b-table>
-                </template>
-            </b-card>
-        </b-collapse>
-    </b-card>
+        <br/>
+        <h4>
+          {{$t("mimeTypes")}}
+          <b-button size="sm" variant="primary" class="float-right" @click="onCopyClick"><ClipboardIcon/></b-button>
+        </h4>
+        <Preloader v-if="$store.state.uiDetailsMimeAgg == null"></Preloader>
+        <b-table
+            v-else
+            sort-by="doc_count"
+            :sort-desc="true"
+            thead-class="hidden"
+            :items="$store.state.uiDetailsMimeAgg" small bordered class="mb-0"
+        ></b-table>
+      </b-card>
+    </b-collapse>
+  </b-card>
 </template>
 
 <script lang="ts">
@@ -50,96 +44,91 @@ import {humanFileSize} from "@/util";
 import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
 import SortSelect from "@/components/SortSelect.vue";
 import Preloader from "@/components/Preloader.vue";
-import Sist2Query from "@/Sist2ElasticsearchQuery";
+import Sist2Query from "@/Sist2Query";
 import ClipboardIcon from "@/components/icons/ClipboardIcon.vue";
 
 export default Vue.extend({
-    name: "ResultsCard",
-    components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
-    created() {
+  name: "ResultsCard",
+  components: {ClipboardIcon, Preloader, SortSelect, DisplayModeToggle},
+  created() {
 
+  },
+  computed: {
+    lastResultsLoaded() {
+      return this.$store.state.lastQueryResults != null;
     },
-    computed: {
-        lastResultsLoaded() {
-            return this.$store.state.lastQueryResults != null;
-        },
-        hitCount() {
-            return (this.$store.state.firstQueryResults as EsResult).aggregations.total_count.value;
-        },
-        tableItems() {
-            const items = [];
-
-            if (!this.$store.state.uiSqliteMode) {
-                items.push({key: this.$t("queryTime"), value: this.took()});
-            }
-            items.push({key: this.$t("totalSize"), value: this.totalSize()});
-
-            return items;
-        }
+    hitCount() {
+      return (this.$store.state.lastQueryResults as EsResult).aggregations.total_count.value;
     },
-    methods: {
-        took() {
-            return (this.$store.state.lastQueryResults as EsResult).took + "ms";
-        },
-        totalSize() {
-            return humanFileSize((this.$store.state.firstQueryResults as EsResult).aggregations.total_size.value);
-        },
-        onToggle() {
-            const show = !document.getElementById("collapse-1").classList.contains("show");
-            this.$store.commit("setUiShowDetails", show);
+    tableItems() {
+      const items = [];
 
-            if (this.$store.state.uiSqliteMode) {
-                return;
-            }
 
-            if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
-                // Mime aggs are not updated automatically, update now
-                this.forceUpdateMimeAgg();
-            }
-        },
-        onCopyClick() {
-            let tsvString = "";
-            this.$store.state.uiDetailsMimeAgg.slice().sort((a, b) => b["doc_count"] - a["doc_count"]).forEach(row => {
-                tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
-            });
+      items.push({key: this.$t("queryTime"), value: this.took()});
+      items.push({key: this.$t("totalSize"), value: this.totalSize()});
 
-            navigator.clipboard.writeText(tsvString);
-
-            this.$bvToast.toast(
-                this.$t("toast.copiedToClipboard"),
-                {
-                    title: null,
-                    noAutoHide: false,
-                    toaster: "b-toaster-bottom-right",
-                    headerClass: "hidden",
-                    bodyClass: "toast-body-info",
-                });
-        },
-        forceUpdateMimeAgg() {
-            const query = Sist2Query.searchQuery();
-            Sist2Api.getMimeTypes(query).then(({buckets}) => {
-                this.$store.commit("setUiDetailsMimeAgg", buckets);
-            });
-        }
+      return items;
+    }
+  },
+  methods: {
+    took() {
+      return (this.$store.state.lastQueryResults as EsResult).took + "ms";
     },
+    totalSize() {
+      return humanFileSize((this.$store.state.lastQueryResults as EsResult).aggregations.total_size.value);
+    },
+    onToggle() {
+      const show = !document.getElementById("collapse-1").classList.contains("show");
+      this.$store.commit("setUiShowDetails", show);
+
+      if (show && this.$store.state.uiDetailsMimeAgg == null && !this.$store.state.optUpdateMimeMap) {
+        // Mime aggs are not updated automatically, update now
+        this.forceUpdateMimeAgg();
+      }
+    },
+    onCopyClick() {
+      let tsvString = "";
+      this.$store.state.uiDetailsMimeAgg.slice().sort((a,b) => b["doc_count"] - a["doc_count"]).forEach(row => {
+        tsvString += `${row["key"]}\t${row["doc_count"]}\n`;
+      });
+
+      navigator.clipboard.writeText(tsvString);
+
+      this.$bvToast.toast(
+          this.$t("toast.copiedToClipboard"),
+          {
+            title: null,
+            noAutoHide: false,
+            toaster: "b-toaster-bottom-right",
+            headerClass: "hidden",
+            bodyClass: "toast-body-info",
+          });
+    },
+    forceUpdateMimeAgg() {
+      const query = Sist2Query.searchQuery();
+      Sist2Api.getMimeTypes(query).then(({buckets}) => {
+        this.$store.commit("setUiDetailsMimeAgg", buckets);
+      });
+    }
+  },
 });
 
 </script>
 
 <style>
 #results {
-    margin-top: 1em;
+  margin-top: 1em;
 
-    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-    border-radius: 0;
-    border: none;
+  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+  border-radius: 0;
+  border: none;
 }
 
 #results .card-body {
-    padding: 0.7em 1.25em;
+  padding: 0.7em 1.25em;
 }
 
 .hidden {
-    display: none;
+  display: none;
 }
 </style>

sist2-vue/src/components/SearchBar.vue

@@ -6,7 +6,7 @@
                     @input="setSearchText($event)"></b-form-input>
 
       <template #prepend>
-        <b-input-group-text v-if="!$store.state.uiSqliteMode">
+        <b-input-group-text>
           <b-form-checkbox :checked="fuzzy" title="Toggle fuzzy searching" @change="setFuzzy($event)">
             {{ $t("searchBar.fuzzy") }}
           </b-form-checkbox>

sist2-vue/src/components/TagContainer.vue

@@ -1,78 +1,76 @@
 <template>
-    <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
+  <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
 
-        <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static
-                 lazy>
-            <b-row>
-                <b-col style="flex-grow: 2" sm>
-                    <VueSimpleSuggest
-                            ref="suggest"
-                            :value="tagText"
-                            @select="setTagText($event)"
-                            @input="setTagText($event)"
-                            class="form-control-fix-flex"
-                            style="margin-top: 17px"
-                            :list="suggestTag"
-                            :max-suggestions="0"
-                            :placeholder="$t('saveTagPlaceholder')"
-                    >
-                        <!-- Suggestion item template-->
-                        <div slot="suggestion-item" slot-scope="{ suggestion, query}"
-                        >
-                            <div class="suggestion-line">
+    <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static lazy>
+      <b-row>
+        <b-col style="flex-grow: 2" sm>
+          <VueSimpleSuggest
+              ref="suggest"
+              :value="tagText"
+              @select="setTagText($event)"
+              @input="setTagText($event)"
+              class="form-control-fix-flex"
+              style="margin-top: 17px"
+              :list="suggestTag"
+              :max-suggestions="0"
+              :placeholder="$t('saveTagPlaceholder')"
+          >
+            <!-- Suggestion item template-->
+            <div slot="suggestion-item" slot-scope="{ suggestion, query}"
+            >
+              <div class="suggestion-line">
                 <span
-                        class="badge badge-suggestion"
-                        :style="{background: getBg(suggestion), color: getFg(suggestion)}"
+                    class="badge badge-suggestion"
+                    :style="{background: getBg(suggestion), color: getFg(suggestion)}"
                 >
                 <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
                 </span>
-                            </div>
-                        </div>
-                    </VueSimpleSuggest>
-                </b-col>
-                <b-col class="mt-4">
-                    <TwitterColorPicker v-model="color" triangle="hide" :width="252"
-                                        class="mr-auto ml-auto"></TwitterColorPicker>
-                </b-col>
-            </b-row>
+              </div>
+            </div>
+          </VueSimpleSuggest>
+        </b-col>
+        <b-col class="mt-4">
+          <TwitterColorPicker v-model="color" triangle="hide" :width="252" class="mr-auto ml-auto"></TwitterColorPicker>
+        </b-col>
+      </b-row>
 
-            <b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
-            </b-button>
-        </b-modal>
+      <b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
+      </b-button>
+    </b-modal>
 
 
-        <template v-for="tag in hit._tags">
-            <!-- User tag-->
-            <div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
+    <template v-for="tag in hit._tags">
+      <!-- User tag-->
+      <div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
         <span
-                :id="hit._id+tag.rawText"
-                :title="tag.text"
-                tabindex="-1"
-                class="badge pointer"
-                :style="badgeStyle(tag)" :class="badgeClass(tag)"
-                @click.right="onTagRightClick(tag, $event)"
+            :id="hit._id+tag.rawText"
+            :title="tag.text"
+            tabindex="-1"
+            class="badge pointer"
+            :style="badgeStyle(tag)" :class="badgeClass(tag)"
+            @click.right="onTagRightClick(tag, $event)"
         >{{ tag.text.split(".").pop() }}</span>
 
-                <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
-                    <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{ $t("deleteTag") }}</b-button>
-                </b-popover>
-            </div>
+        <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
+          <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">{{ $t("deleteTag") }}</b-button>
+        </b-popover>
+      </div>
 
-            <span
-                    v-else :key="tag.text"
-                    class="badge"
-                    :style="badgeStyle(tag)" :class="badgeClass(tag)"
-            >{{ tag.text.split(".").pop() }}</span>
-        </template>
+      <span
+          v-else :key="tag.text"
+          class="badge"
+          :style="badgeStyle(tag)" :class="badgeClass(tag)"
+      >{{ tag.text.split(".").pop() }}</span>
+    </template>
 
-        <!-- Add button -->
-        <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{ $t("addTag") }}</small>
+    <!-- Add button -->
+    <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">{{$t("addTag")}}</small>
 
-        <!-- Size tag-->
-        <small v-else class="text-muted badge-size" style="padding-left: 2px">{{
-            humanFileSize(hit._source.size)
-            }}</small>
-    </div>
+    <!-- Size tag-->
+    <small v-else class="text-muted badge-size" style="padding-left: 2px">{{
+        humanFileSize(hit._source.size)
+      }}</small>
+  </div>
 </template>
 
 <script>
@@ -83,136 +81,170 @@ import Sist2Api from "@/Sist2Api";
 import VueSimpleSuggest from 'vue-simple-suggest'
 
 export default Vue.extend({
-    components: {
-        "TwitterColorPicker": Twitter,
-        VueSimpleSuggest
+  components: {
+    "TwitterColorPicker": Twitter,
+    VueSimpleSuggest
+  },
+  props: ["hit"],
+  data() {
+    return {
+      showAddButton: false,
+      showModal: false,
+      tagText: null,
+      color: {
+        hex: "#e0e0e0",
+      },
+    }
+  },
+  computed: {
+    tagHover() {
+      return this.$store.getters["uiTagHover"];
+    }
+  },
+  methods: {
+    humanFileSize: humanFileSize,
+    getSuggestionWithoutQueryPrefix(suggestion, query) {
+      return suggestion.id.slice(query.length, -8)
     },
-    props: ["hit"],
-    data() {
+    getBg(suggestion) {
+      return suggestion.id.slice(-7);
+    },
+    getFg(suggestion) {
+      return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
+    },
+    setTagText(value) {
+      this.$refs.suggest.clearSuggestions();
+
+      if (typeof value === "string") {
+        this.tagText = {
+          id: value,
+          title: value
+        };
+        return;
+      }
+
+      this.color = {
+        hex: "#" + value.id.split("#")[1]
+      }
+
+      this.tagText = value;
+    },
+    badgeClass(tag) {
+      return `badge-${tag.style}`;
+    },
+    badgeStyle(tag) {
+      return {
+        background: tag.bg,
+        color: tag.fg,
+      };
+    },
+    onTagHover(tag) {
+      if (tag.userTag) {
+        this.$store.commit("setUiTagHover", tag);
+      }
+    },
+    onTagLeave() {
+      this.$store.commit("setUiTagHover", null);
+    },
+    onTagDeleteClick(tag, e) {
+      this.hit._tags = this.hit._tags.filter(t => t !== tag);
+
+      Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
+        //toast
+        this.$store.commit("busUpdateWallItems");
+        this.$store.commit("busUpdateTags");
+      });
+    },
+    tagAdd() {
+      this.showModal = true;
+    },
+    saveTag() {
+      if (this.tagText.id.includes("#")) {
+        this.$bvToast.toast(
+            this.$t("toast.invalidTag"),
+            {
+              title: this.$t("toast.invalidTagTitle"),
+              noAutoHide: true,
+              toaster: "b-toaster-bottom-right",
+              headerClass: "toast-header-error",
+              bodyClass: "toast-body-error",
+            });
+        return;
+      }
+
+      let tag = this.tagText.id + this.color.hex.replace("#", ".#");
+      const userTags = this.hit._tags.filter(t => t.userTag);
+
+      if (userTags.find(t => t.rawText === tag) != null) {
+        this.$bvToast.toast(
+            this.$t("toast.dupeTag"),
+            {
+              title: this.$t("toast.dupeTagTitle"),
+              noAutoHide: true,
+              toaster: "b-toaster-bottom-right",
+              headerClass: "toast-header-error",
+              bodyClass: "toast-body-error",
+            });
+        return;
+      }
+
+      this.hit._tags.push(Sist2Api.createUserTag(tag));
+
+      Sist2Api.saveTag(tag, this.hit).then(() => {
+        this.tagText = null;
+        this.showModal = false;
+        this.$store.commit("busUpdateWallItems");
+        this.$store.commit("busUpdateTags");
+        // TODO: toast
+      });
+    },
+    async suggestTag(term) {
+      term = term.toLowerCase();
+
+      const choices = await this.getTagChoices(term);
+
+      let matches = [];
+      for (let i = 0; i < choices.length; i++) {
+        if (~choices[i].toLowerCase().indexOf(term)) {
+          matches.push(choices[i]);
+        }
+      }
+
+      return matches.sort().map(match => {
         return {
-            showAddButton: false,
-            showModal: false,
-            tagText: null,
-            color: {
-                hex: "#e0e0e0",
-            },
+          title: match.split(".").slice(0, -1).join("."),
+          id: match
         }
+      });
     },
-    methods: {
-        humanFileSize: humanFileSize,
-        getSuggestionWithoutQueryPrefix(suggestion, query) {
-            return suggestion.id.slice(query.length, -8)
-        },
-        getBg(suggestion) {
-            return suggestion.id.slice(-7);
-        },
-        getFg(suggestion) {
-            return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
-        },
-        setTagText(value) {
-            this.$refs.suggest.clearSuggestions();
-
-            if (typeof value === "string") {
-                this.tagText = {
-                    id: value,
-                    title: value
-                };
-                return;
+    getTagChoices(prefix) {
+      return new Promise(getPaths => {
+        Sist2Api.esQuery({
+          suggest: {
+            tag: {
+              prefix: prefix,
+              completion: {
+                field: "suggest-tag",
+                skip_duplicates: true,
+                size: 10000
+              }
             }
-
-            this.color = {
-                hex: "#" + value.id.split("#")[1]
-            }
-
-            this.tagText = value;
-        },
-        badgeClass(tag) {
-            return `badge-${tag.style}`;
-        },
-        badgeStyle(tag) {
-            return {
-                background: tag.bg,
-                color: tag.fg,
-            };
-        },
-        onTagDeleteClick(tag, e) {
-            this.hit._tags = this.hit._tags.filter(t => t !== tag);
-
-            Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
-                //toast
-                this.$store.commit("busUpdateWallItems");
-                this.$store.commit("busUpdateTags");
+          }
+        }).then(resp => {
+          const result = [];
+          resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
+            tags.forEach(tag => {
+              const t = tag.slice(0, -8);
+              if (!result.find(x => x.slice(0, -8) === t)) {
+                result.push(tag);
+              }
             });
-        },
-        tagAdd() {
-            this.showModal = true;
-        },
-        saveTag() {
-            if (this.tagText.id.includes("#")) {
-                this.$bvToast.toast(
-                    this.$t("toast.invalidTag"),
-                    {
-                        title: this.$t("toast.invalidTagTitle"),
-                        noAutoHide: true,
-                        toaster: "b-toaster-bottom-right",
-                        headerClass: "toast-header-error",
-                        bodyClass: "toast-body-error",
-                    });
-                return;
-            }
-
-            let tag = this.tagText.id + this.color.hex.replace("#", ".#");
-            const userTags = this.hit._tags.filter(t => t.userTag);
-
-            if (userTags.find(t => t.rawText === tag) != null) {
-                this.$bvToast.toast(
-                    this.$t("toast.dupeTag"),
-                    {
-                        title: this.$t("toast.dupeTagTitle"),
-                        noAutoHide: true,
-                        toaster: "b-toaster-bottom-right",
-                        headerClass: "toast-header-error",
-                        bodyClass: "toast-body-error",
-                    });
-                return;
-            }
-
-            this.hit._tags.push(Sist2Api.createUserTag(tag));
-
-            Sist2Api.saveTag(tag, this.hit).then(() => {
-                this.tagText = null;
-                this.showModal = false;
-                this.$store.commit("busUpdateWallItems");
-                this.$store.commit("busUpdateTags");
-                // TODO: toast
-            });
-        },
-        async suggestTag(term) {
-            term = term.toLowerCase();
-
-            const choices = await this.getTagChoices(term);
-
-            let matches = [];
-            for (let i = 0; i < choices.length; i++) {
-                if (~choices[i].toLowerCase().indexOf(term)) {
-                    matches.push(choices[i]);
-                }
-            }
-
-            return matches.sort().map(match => {
-                return {
-                    title: match.split(".").slice(0, -1).join("."),
-                    id: match
-                }
-            });
-        },
-        getTagChoices(prefix) {
-            return new Promise(getPaths => {
-                Sist2Api.getTagSuggestions(prefix)
-                    .then(paths => getPaths(paths))
-            });
-        }
-    },
+          });
+          getPaths(result);
+        });
+      });
+    }
+  },
 });
 </script>
 
@@ -220,87 +252,87 @@ export default Vue.extend({
 
 
 .badge-video {
-    color: #FFFFFF;
-    background-color: #F27761;
+  color: #FFFFFF;
+  background-color: #F27761;
 }
 
 .badge-image {
-    color: #FFFFFF;
-    background-color: #AA99C9;
+  color: #FFFFFF;
+  background-color: #AA99C9;
 }
 
 .badge-audio {
-    color: #FFFFFF;
-    background-color: #00ADEF;
+  color: #FFFFFF;
+  background-color: #00ADEF;
 }
 
 .badge-user {
-    color: #212529;
-    background-color: #e0e0e0;
+  color: #212529;
+  background-color: #e0e0e0;
 }
 
 .badge-user:hover, .add-tag-button:hover {
-    box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
+  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
 }
 
 .badge-text {
-    color: #FFFFFF;
-    background-color: #FAAB3C;
+  color: #FFFFFF;
+  background-color: #FAAB3C;
 }
 
 .badge {
-    margin-right: 3px;
+  margin-right: 3px;
 }
 
 .badge-delete {
-    margin-right: -2px;
-    margin-left: 2px;
-    margin-top: -1px;
-    font-family: monospace;
-    font-size: 90%;
-    background: rgba(0, 0, 0, 0.2);
-    padding: 0.1em 0.4em;
-    color: white;
-    cursor: pointer;
+  margin-right: -2px;
+  margin-left: 2px;
+  margin-top: -1px;
+  font-family: monospace;
+  font-size: 90%;
+  background: rgba(0, 0, 0, 0.2);
+  padding: 0.1em 0.4em;
+  color: white;
+  cursor: pointer;
 }
 
 .badge-size {
-    width: 50px;
-    display: inline-block;
+  width: 50px;
+  display: inline-block;
 }
 
 .add-tag-button {
-    cursor: pointer;
-    color: #212529;
-    background-color: #e0e0e0;
-    width: 50px;
+  cursor: pointer;
+  color: #212529;
+  background-color: #e0e0e0;
+  width: 50px;
 }
 
 .badge {
-    user-select: none;
+  user-select: none;
 }
 
 .badge-suggestion {
-    font-size: 90%;
-    font-weight: normal;
+  font-size: 90%;
+  font-weight: normal;
 }
 </style>
 
 <style>
 .vc-twitter-body {
-    padding: 0 !important;
+  padding: 0 !important;
 }
 
 .vc-twitter {
-    box-shadow: none !important;
-    background: none !important;
+  box-shadow: none !important;
+  background: none !important;
 }
 
 .tooltip {
-    user-select: none;
+  user-select: none;
 }
 
 .toast {
-    border: none;
+  border: none;
 }
 </style>

sist2-vue/src/components/TagPicker.vue

@@ -1,13 +1,13 @@
 <template>
-    <div>
-        <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
-            <b-form-input :value="filter"
-                          :placeholder="$t('tagFilter')"
-                          @input="onFilter($event)"></b-form-input>
-        </b-input-group>
+  <div>
+    <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
+      <b-form-input :value="filter"
+                    :placeholder="$t('tagFilter')"
+                    @input="onFilter($event)"></b-form-input>
+    </b-input-group>
 
-        <div id="tagTree"></div>
-    </div>
+    <div id="tagTree"></div>
+  </div>
 </template>
 
 <script>
@@ -19,195 +19,191 @@ import {getSelectedTreeNodes} from "@/util";
 import Sist2Api from "@/Sist2Api";
 
 function resetState(node) {
-    node._tree.defaultState.forEach(function (val, prop) {
-        node.state(prop, val);
-    });
+  node._tree.defaultState.forEach(function (val, prop) {
+    node.state(prop, val);
+  });
 
-    return node;
+  return node;
 }
 
 function baseStateChange(prop, value, verb, node, deep) {
-    if (node.state(prop) !== value) {
-        node._tree.batch();
+  if (node.state(prop) !== value) {
+    node._tree.batch();
 
-        if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
-            resetState(node);
-        }
-
-        node.state(prop, value);
-
-        node._tree.emit('node.' + verb, node, false);
-
-        if (deep && node.hasChildren()) {
-            node.children.recurseDown(function (child) {
-                baseStateChange(prop, value, verb, child);
-            });
-        }
-
-        node.markDirty();
-        node._tree.end();
+    if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
+      resetState(node);
     }
 
-    return node;
+    node.state(prop, value);
+
+    node._tree.emit('node.' + verb, node, false);
+
+    if (deep && node.hasChildren()) {
+      node.children.recurseDown(function (child) {
+        baseStateChange(prop, value, verb, child);
+      });
+    }
+
+    node.markDirty();
+    node._tree.end();
+  }
+
+  return node;
 }
 
 function addTag(map, tag, id, count) {
-    const tags = tag.split(".");
+  const tags = tag.split(".");
 
-    const child = {
-        id: id,
-        count: count,
-        text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
-        name: tags[0],
-        children: [],
-        // Overwrite base functions
-        blur: function () {
-            // noop
-        },
-        select: function () {
-            this.state("selected", true);
-            return this.check()
-        },
-        deselect: function () {
-            this.state("selected", false);
-            return this.uncheck()
-        },
-        uncheck: function () {
-            baseStateChange('checked', false, 'unchecked', this, false);
-            this.state('indeterminate', false);
+  const child = {
+    id: id,
+    count: count,
+    text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
+    name: tags[0],
+    children: [],
+    // Overwrite base functions
+    blur: function () {
+      // noop
+    },
+    select: function () {
+      this.state("selected", true);
+      return this.check()
+    },
+    deselect: function () {
+      this.state("selected", false);
+      return this.uncheck()
+    },
+    uncheck: function () {
+      baseStateChange('checked', false, 'unchecked', this, false);
+      this.state('indeterminate', false);
 
-            if (this.hasParent()) {
-                this.getParent().refreshIndeterminateState();
-            }
+      if (this.hasParent()) {
+        this.getParent().refreshIndeterminateState();
+      }
 
-            this._tree.end();
-            return this;
-        },
-        check: function () {
-            baseStateChange('checked', true, 'checked', this, false);
+      this._tree.end();
+      return this;
+    },
+    check: function () {
+      baseStateChange('checked', true, 'checked', this, false);
 
-            if (this.hasParent()) {
-                this.getParent().refreshIndeterminateState();
-            }
+      if (this.hasParent()) {
+        this.getParent().refreshIndeterminateState();
+      }
 
-            this._tree.end();
-            return this;
-        }
-    };
-
-    let found = false;
-    map.forEach(node => {
-        if (node.name === child.name) {
-            found = true;
-            if (tags.length !== 1) {
-                addTag(node.children, tags.slice(1).join("."), id, count);
-            } else {
-                // Same name, different color
-                console.error("FIXME: Duplicate tag?")
-                console.trace(node)
-            }
-        }
-    });
-    if (!found) {
-        if (tags.length !== 1) {
-            addTag(child.children, tags.slice(1).join("."), id, count);
-            map.push(child);
-        } else {
-            map.push(child);
-        }
+      this._tree.end();
+      return this;
     }
+  };
+
+  let found = false;
+  map.forEach(node => {
+    if (node.name === child.name) {
+      found = true;
+      if (tags.length !== 1) {
+        addTag(node.children, tags.slice(1).join("."), id, count);
+      } else {
+        // Same name, different color
+        console.error("FIXME: Duplicate tag?")
+        console.trace(node)
+      }
+    }
+  });
+  if (!found) {
+    if (tags.length !== 1) {
+      addTag(child.children, tags.slice(1).join("."), id, count);
+      map.push(child);
+    } else {
+      map.push(child);
+    }
+  }
 }
 
 export default {
-    name: "TagPicker",
-    props: ["showSearchBar"],
-    data() {
-        return {
-            tagTree: null,
-            loadedFromArgs: false,
-            filter: ""
-        }
-    },
-    mounted() {
-        this.$store.subscribe((mutation) => {
-            if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
-                this.initializeTree();
-                this.updateTree();
-            } else if (mutation.type === "busUpdateTags") {
-                if (this.$store.state.uiSqliteMode) {
-                    this.updateTree();
-                } else {
-                    window.setTimeout(this.updateTree, 2000);
-                }
-            }
-        });
-    },
-    methods: {
-        onFilter(value) {
-            this.filter = value;
-            this.tagTree.search(value);
-        },
-        initializeTree() {
-            const tagMap = [];
-            this.tagTree = new InspireTree({
-                selection: {
-                    mode: "checkbox",
-                    autoDeselect: false,
-                },
-                checkbox: {
-                    autoCheckChildren: false,
-                },
-                data: tagMap
-            });
-            new InspireTreeDOM(this.tagTree, {
-                target: '#tagTree'
-            });
-            this.tagTree.on("node.state.changed", this.handleTreeClick);
-        },
-        updateTree() {
-            // TODO: remember which tags are selected and restore?
-            const tagMap = [];
-            Sist2Api.getTags().then(tags => {
-                tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
-                this.tagTree.removeAll();
-                this.tagTree.addNodes(tagMap);
-
-                if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
-                    this.$store.state._onLoadSelectedTags.forEach(mime => {
-                        this.tagTree.node(mime).select();
-                        this.loadedFromArgs = true;
-                    });
-                }
-            });
-        },
-        handleTreeClick(node, e) {
-            if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
-                || e === "matched" || e === "hidden") {
-                return;
-            }
-
-            this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
-        },
+  name: "TagPicker",
+  props: ["showSearchBar"],
+  data() {
+    return {
+      tagTree: null,
+      loadedFromArgs: false,
+      filter: ""
     }
+  },
+  mounted() {
+    this.$store.subscribe((mutation) => {
+      if (mutation.type === "setUiMimeMap" && this.tagTree === null) {
+        this.initializeTree();
+        this.updateTree();
+      } else if (mutation.type === "busUpdateTags") {
+        window.setTimeout(this.updateTree, 2000);
+      }
+    });
+  },
+  methods: {
+    onFilter(value) {
+      this.filter = value;
+      this.tagTree.search(value);
+    },
+    initializeTree() {
+      const tagMap = [];
+      this.tagTree = new InspireTree({
+        selection: {
+          mode: "checkbox",
+          autoDeselect: false,
+        },
+        checkbox: {
+          autoCheckChildren: false,
+        },
+        data: tagMap
+      });
+      new InspireTreeDOM(this.tagTree, {
+        target: '#tagTree'
+      });
+      this.tagTree.on("node.state.changed", this.handleTreeClick);
+    },
+    updateTree() {
+      // TODO: remember which tags are selected and restore?
+      const tagMap = [];
+      Sist2Api.getTags().then(tags => {
+        tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
+        this.tagTree.removeAll();
+        this.tagTree.addNodes(tagMap);
+
+        if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
+          this.$store.state._onLoadSelectedTags.forEach(mime => {
+            this.tagTree.node(mime).select();
+            this.loadedFromArgs = true;
+          });
+        }
+      });
+    },
+    handleTreeClick(node, e) {
+      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
+          || e === "matched" || e === "hidden") {
+        return;
+      }
+
+      this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
+    },
+  }
 }
 </script>
 
 <style scoped>
 #mimeTree {
-    max-height: 350px;
-    overflow: auto;
+  max-height: 350px;
+  overflow: auto;
 }
 </style>
 <style>
 .inspire-tree .focused > .wholerow {
-    border: none;
+  border: none;
 }
 
 #tag-picker-filter-bar {
-    padding: 10px 4px 4px;
+  padding: 10px 4px 4px;
 }
 
 .theme-black .inspire-tree .matched > .wholerow {
-    background: rgba(251, 191, 41, 0.25);
+  background: rgba(251, 191, 41, 0.25);
 }
 </style>

sist2-vue/src/i18n/messages.ts

@@ -49,7 +49,6 @@ export default {
         configReset: "Reset configuration",
         searchOptions: "Search options",
         treemapOptions: "Treemap options",
-        mlOptions: "Machine learning options",
         displayOptions: "Display options",
         opt: {
             lang: "Language",
@@ -57,7 +56,7 @@ export default {
             fuzzy: "Set fuzzy search by default",
             searchInPath: "Enable matching query against document path",
             suggestPath: "Enable auto-complete in path filter bar",
-            fragmentSize: "Highlight context size",
+            fragmentSize: "Highlight context size in characters",
             queryMode: "Search mode",
             displayMode: "Display",
             columns: "Column count",
@@ -79,10 +78,7 @@ export default {
             simpleLightbox: "Disable animations in image viewer",
             showTagPickerFilter: "Display the tag filter bar",
             featuredFields: "Featured fields Javascript template string. Will appear in the search results.",
-            featuredFieldsList: "Available variables",
-            autoAnalyze: "Automatically analyze text",
-            defaultModel: "Default model",
-            mlRepositories: "Model repositories (one per line)"
+            featuredFieldsList: "Available variables"
         },
         queryMode: {
             simple: "Simple",
@@ -175,12 +171,6 @@ export default {
             selectedIndex: "selected index",
             selectedIndices: "selected indices",
         },
-        ml: {
-            analyzeText: "Analyze",
-            auto: "Auto",
-            repoFetchError: "Failed to get list of models. Check browser console for more details.",
-            repoFetchErrorTitle: "Could not fetch model repositories",
-        }
     },
     de: {
         filePage: {
@@ -239,7 +229,7 @@ export default {
             fuzzy: "Aktiviere Fuzzy-Suche standardmäßig",
             searchInPath: "Abgleich der Abfrage mit dem Dokumentpfad aktivieren",
             suggestPath: "Aktiviere Auto-Vervollständigung in Pfadfilter-Leiste",
-            fragmentSize: "Kontextgröße",
+            fragmentSize: "Kontextgröße in Zeichen hervorheben",
             queryMode: "Such-Modus",
             displayMode: "Ansicht",
             columns: "Anzahl Spalten",
@@ -260,8 +250,8 @@ export default {
             vidPreviewInterval: "Videovorschau Framedauer in ms",
             simpleLightbox: "Schalte Animationen im Image-Viewer ab",
             showTagPickerFilter: "Zeige die Tag-Filter-Leiste",
-            featuredFields: "Variablen, welche zusätzlich in den Suchergebnissen angezeigt werden können.",
-            featuredFieldsList: "verfügbare Variablen"
+            featuredFields: "Ausgewählte Felder Javascript Vorlage String. Wird in den Suchergebnissen angezeigt.",
+            featuredFieldsList: "Verfügbare Variablen"
         },
         queryMode: {
             simple: "Einfach",
@@ -343,10 +333,10 @@ export default {
             random: "zufällig",
         },
         d3: {
-            mimeCount: "Anzahl nach Medientyp",
-            mimeSize: "Größen nach Medientyp",
-            dateHistogram: "Änderungszeiten",
-            sizeHistogram: "Dateigrößen",
+            mimeCount: "Anzahlverteilung nach Medientyp",
+            mimeSize: "Größenverteilung nach Medientyp",
+            dateHistogram: "Verteilung der Änderungszeiten",
+            sizeHistogram: "Verteilung der Dateigrößen",
         },
         indexPicker: {
             selectNone: "keinen auswählen",
@@ -413,7 +403,7 @@ export default {
             fuzzy: "Activer la recherche approximative par défaut",
             searchInPath: "Activer la recherche dans le chemin des documents",
             suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
-            fragmentSize: "Longueur du contexte de surlignage",
+            fragmentSize: "Longueur du contexte de surlignage, en nombre de caractères",
             queryMode: "Mode de recherche",
             displayMode: "Affichage",
             columns: "Nombre de colonnes",

sist2-vue/src/ml/BertNerModel.js

@@ -1,77 +0,0 @@
-import BertTokenizer from "@/ml/BertTokenizer";
-import * as tf from "@tensorflow/tfjs";
-import axios from "axios";
-
-export default class BertNerModel {
-    vocabUrl;
-    modelUrl;
-
-    id2label;
-    _tokenizer;
-    _model;
-    inputSize = 128;
-
-    _previousWordId = null;
-
-    constructor(vocabUrl, modelUrl, id2label) {
-        this.vocabUrl = vocabUrl;
-        this.modelUrl = modelUrl;
-        this.id2label = id2label;
-    }
-
-    async init(onProgress) {
-        await Promise.all([this.loadTokenizer(), this.loadModel(onProgress)]);
-    }
-
-    async loadTokenizer() {
-        const vocab = (await axios.get(this.vocabUrl)).data;
-        this._tokenizer = new BertTokenizer(vocab);
-    }
-
-    async loadModel(onProgress) {
-        this._model = await tf.loadGraphModel(this.modelUrl, {onProgress});
-    }
-
-    alignLabels(labels, wordIds, words) {
-        const result = [];
-
-        for (let i = 0; i < this.inputSize; i++) {
-            const label = labels[i];
-            const wordId = wordIds[i];
-
-            if (wordId === -1) {
-                continue;
-            }
-            if (wordId === this._previousWordId) {
-                continue;
-            }
-
-            result.push({
-                word: words[wordId].text, wordIndex: words[wordId].index, label: label
-            });
-            this._previousWordId = wordId;
-        }
-
-        return result;
-    }
-
-    async predict(text, callback) {
-        this._previousWordId = null;
-        const encoded = this._tokenizer.encodeText(text, this.inputSize)
-
-        for (let chunk of encoded.inputChunks) {
-            const rawResult = tf.tidy(() => this._model.execute({
-                input_ids: tf.tensor2d(chunk.inputIds, [1, this.inputSize], "int32"),
-                token_type_ids: tf.tensor2d(chunk.segmentIds, [1, this.inputSize], "int32"),
-                attention_mask: tf.tensor2d(chunk.inputMask, [1, this.inputSize], "int32"),
-            }));
-
-            const labelIds = await tf.argMax(rawResult, -1);
-            const labelIdsArray = await labelIds.array();
-            const labels = labelIdsArray[0].map(id => this.id2label[id]);
-            rawResult.dispose()
-
-            callback(this.alignLabels(labels, chunk.wordIds, encoded.words))
-        }
-    }
-}

sist2-vue/src/ml/BertTokenizer.js

@@ -1,184 +0,0 @@
-import {zip, chunk} from "underscore";
-
-const UNK_INDEX = 100;
-const CLS_INDEX = 101;
-const SEP_INDEX = 102;
-const CONTINUING_SUBWORD_PREFIX = "##";
-
-function isWhitespace(ch) {
-    return /\s/.test(ch);
-}
-
-function isInvalid(ch) {
-    return (ch.charCodeAt(0) === 0 || ch.charCodeAt(0) === 0xfffd);
-}
-
-const punctuations = '[~`!@#$%^&*(){}[];:"\'<,.>?/\\|-_+=';
-
-/** To judge whether it's a punctuation. */
-function isPunctuation(ch) {
-    return punctuations.indexOf(ch) !== -1;
-}
-
-export default class BertTokenizer {
-    vocab;
-
-    constructor(vocab) {
-        this.vocab = vocab;
-    }
-
-    tokenize(text) {
-        const charOriginalIndex = [];
-        const cleanedText = this.cleanText(text, charOriginalIndex);
-        const origTokens = cleanedText.split(' ');
-
-        let charCount = 0;
-        const tokens = origTokens.map((token) => {
-            token = token.toLowerCase();
-            const tokens = this.runSplitOnPunctuation(token, charCount, charOriginalIndex);
-            charCount += token.length + 1;
-            return tokens;
-        });
-
-        let flattenTokens = [];
-        for (let index = 0; index < tokens.length; index++) {
-            flattenTokens = flattenTokens.concat(tokens[index]);
-        }
-        return flattenTokens;
-    }
-
-    /* Performs invalid character removal and whitespace cleanup on text. */
-    cleanText(text, charOriginalIndex) {
-        text = text.replace(/\?/g, "").trim();
-
-        const stringBuilder = [];
-        let originalCharIndex = 0;
-        let newCharIndex = 0;
-
-        for (const ch of text) {
-            // Skip the characters that cannot be used.
-            if (isInvalid(ch)) {
-                originalCharIndex += ch.length;
-                continue;
-            }
-            if (isWhitespace(ch)) {
-                if (stringBuilder.length > 0 && stringBuilder[stringBuilder.length - 1] !== ' ') {
-                    stringBuilder.push(' ');
-                    charOriginalIndex[newCharIndex] = originalCharIndex;
-                    originalCharIndex += ch.length;
-                } else {
-                    originalCharIndex += ch.length;
-                    continue;
-                }
-            } else {
-                stringBuilder.push(ch);
-                charOriginalIndex[newCharIndex] = originalCharIndex;
-                originalCharIndex += ch.length;
-            }
-            newCharIndex++;
-        }
-        return stringBuilder.join('');
-    }
-
-    /* Splits punctuation on a piece of text. */
-    runSplitOnPunctuation(text, count, charOriginalIndex) {
-        const tokens = [];
-        let startNewWord = true;
-        for (const ch of text) {
-            if (isPunctuation(ch)) {
-                tokens.push({text: ch, index: charOriginalIndex[count]});
-                count += ch.length;
-                startNewWord = true;
-            } else {
-                if (startNewWord) {
-                    tokens.push({text: '', index: charOriginalIndex[count]});
-                    startNewWord = false;
-                }
-                tokens[tokens.length - 1].text += ch;
-                count += ch.length;
-            }
-        }
-        return tokens;
-    }
-
-    encode(words) {
-        let outputTokens = [];
-        const wordIds = [];
-
-        for (let i = 0; i < words.length; i++) {
-            let chars = [...words[i].text];
-
-            let isUnknown = false;
-            let start = 0;
-            let subTokens = [];
-
-            while (start < chars.length) {
-                let end = chars.length;
-                let currentSubstring = null;
-                while (start < end) {
-                    let substr = chars.slice(start, end).join('');
-
-                    if (start > 0) {
-                        substr = CONTINUING_SUBWORD_PREFIX + substr;
-                    }
-                    if (this.vocab.includes(substr)) {
-                        currentSubstring = this.vocab.indexOf(substr);
-                        break;
-                    }
-
-                    --end;
-                }
-                if (currentSubstring == null) {
-                    isUnknown = true;
-                    break;
-                }
-                subTokens.push(currentSubstring);
-                start = end;
-            }
-
-            if (isUnknown) {
-                outputTokens.push(UNK_INDEX);
-                wordIds.push(i);
-            } else {
-                subTokens.forEach(tok => {
-                    outputTokens.push(tok);
-                    wordIds.push(i)
-                });
-            }
-        }
-
-        return {tokens: outputTokens, wordIds};
-    }
-
-    encodeText(inputText, inputSize) {
-
-        const tokenized = this.tokenize(inputText);
-        const encoded = this.encode(tokenized);
-
-        const encodedTokenChunks = chunk(encoded.tokens, inputSize - 2);
-        const encodedWordIdChunks = chunk(encoded.wordIds, inputSize - 2);
-
-        const chunks = [];
-
-        zip(encodedTokenChunks, encodedWordIdChunks).forEach(([tokens, wordIds]) => {
-            const inputIds = [CLS_INDEX, ...tokens, SEP_INDEX];
-            const segmentIds = Array(inputIds.length).fill(0);
-            const inputMask = Array(inputIds.length).fill(1);
-            wordIds = [-1, ...wordIds, -1];
-
-            while (inputIds.length < inputSize) {
-                inputIds.push(0);
-                inputMask.push(0);
-                segmentIds.push(0);
-                wordIds.push(-1);
-            }
-
-            chunks.push({inputIds, inputMask, segmentIds, wordIds})
-        });
-
-        return {
-            inputChunks: chunks,
-            words: tokenized
-        };
-    }
-}

sist2-vue/src/ml/modelsRepo.js

@@ -1,43 +0,0 @@
-import axios from "axios";
-
-class ModelsRepo {
-    _repositories;
-    data = {};
-
-    async init(repositories) {
-        this._repositories = repositories;
-
-        const data = await Promise.all(this._repositories.map(this._loadRepository));
-
-        data.forEach(models => {
-            models.forEach(model => {
-                this.data[model.name] = model;
-            })
-        });
-    }
-
-    async _loadRepository(repository) {
-        const data = (await axios.get(repository)).data;
-        data.forEach(model => {
-            model["modelUrl"] = new URL(model["modelPath"], repository).href;
-            model["vocabUrl"] = new URL(model["vocabPath"], repository).href;
-        });
-        return data;
-    }
-
-    getOptions() {
-        return Object.values(this.data).map(model => ({
-            text: `${model.name} (${Math.round(model.size / (1024*1024))}MB)`,
-            value: model.name
-        }));
-    }
-
-    getDefaultModel() {
-        if (Object.values(this.data).length === 0) {
-            return null;
-        }
-        return Object.values(this.data).find(model => model.default).name;
-    }
-}
-
-export default new ModelsRepo();

sist2-vue/src/store/index.ts

@@ -1,11 +1,11 @@
 import Vue from "vue"
 import Vuex from "vuex"
 import VueRouter, {Route} from "vue-router";
-import {EsHit, EsResult, EsTag, Index} from "@/Sist2Api";
+import {EsHit, EsResult, EsTag, Index, Tag} from "@/Sist2Api";
 import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
 import {getInstance} from "@/plugins/auth0.js";
 
-const CONF_VERSION = 3;
+const CONF_VERSION = 2;
 
 Vue.use(Vuex)
 
@@ -57,9 +57,6 @@ export default new Vuex.Store({
         optVidPreviewInterval: 700,
         optSimpleLightbox: true,
         optShowTagPickerFilter: true,
-        optMlRepositories: "https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json",
-        optAutoAnalyze: false,
-        optMlDefaultModel: null,
 
         _onLoadSelectedIndices: [] as string[],
         _onLoadSelectedMimeTypes: [] as string[],
@@ -69,12 +66,11 @@ export default new Vuex.Store({
         selectedTags: [] as string[],
 
         lastQueryResults: null,
-        firstQueryResults: null,
 
         keySequence: 0,
         querySequence: 0,
 
-        uiSqliteMode: false,
+        uiTagHover: null as Tag | null,
         uiLightboxIsOpen: false,
         uiShowLightbox: false,
         uiLightboxSources: [] as string[],
@@ -90,11 +86,7 @@ export default new Vuex.Store({
 
         uiMimeMap: [] as any[],
 
-        auth0Token: null,
-        mlModel: {
-            model: null,
-            name: null
-        },
+        auth0Token: null
     },
     mutations: {
         setUiShowDetails: (state, val) => state.uiShowDetails = val,
@@ -131,13 +123,13 @@ export default new Vuex.Store({
         setSearchText: (state, val) => state.searchText = val,
         setFuzzy: (state, val) => state.fuzzy = val,
         setLastQueryResult: (state, val) => state.lastQueryResults = val,
-        setFirstQueryResult: (state, val) => state.firstQueryResults = val,
         _setOnLoadSelectedIndices: (state, val) => state._onLoadSelectedIndices = val,
         _setOnLoadSelectedMimeTypes: (state, val) => state._onLoadSelectedMimeTypes = val,
         _setOnLoadSelectedTags: (state, val) => state._onLoadSelectedTags = val,
         setSelectedIndices: (state, val) => state.selectedIndices = val,
         setSelectedMimeTypes: (state, val) => state.selectedMimeTypes = val,
         setSelectedTags: (state, val) => state.selectedTags = val,
+        setUiTagHover: (state, val: Tag | null) => state.uiTagHover = val,
         setUiLightboxIsOpen: (state, val: boolean) => state.uiLightboxIsOpen = val,
         _setUiShowLightbox: (state, val: boolean) => state.uiShowLightbox = val,
         setUiLightboxKey: (state, val: number) => state.uiLightboxKey = val,
@@ -155,7 +147,6 @@ export default new Vuex.Store({
         setUiLightboxThumbs: (state, val) => state.uiLightboxThumbs = val,
         setUiLightboxTypes: (state, val) => state.uiLightboxTypes = val,
         setUiLightboxCaptions: (state, val) => state.uiLightboxCaptions = val,
-        setUiSqliteMode: (state, val) => state.uiSqliteMode = val,
 
         setOptTheme: (state, val) => state.optTheme = val,
         setOptDisplay: (state, val) => state.optDisplay = val,
@@ -181,15 +172,6 @@ export default new Vuex.Store({
         setOptVidPreviewInterval: (state, val) => state.optVidPreviewInterval = val,
         setOptSimpleLightbox: (state, val) => state.optSimpleLightbox = val,
         setOptShowTagPickerFilter: (state, val) => state.optShowTagPickerFilter = val,
-        setOptAutoAnalyze: (state, val) => {
-            state.optAutoAnalyze = val
-        },
-        setOptMlRepositories: (state, val) => {
-            state.optMlRepositories = val
-        },
-        setOptMlDefaultModel: (state, val) => {
-            state.optMlDefaultModel = val
-        },
 
         setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
         setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val,
@@ -212,7 +194,6 @@ export default new Vuex.Store({
             // noop
         },
         setAuth0Token: (state, val) => state.auth0Token = val,
-        setMlModel: (state, val) => state.mlModel = val,
     },
     actions: {
         setSist2Info: (store, val) => {
@@ -348,7 +329,6 @@ export default new Vuex.Store({
             commit("_setUiShowLightbox", !state.uiShowLightbox);
         },
         clearResults({commit}) {
-            commit("setFirstQueryResult", null);
             commit("setLastQueryResult", null);
             commit("_setKeySequence", 0);
             commit("_setUiShowLightbox", false);
@@ -370,7 +350,6 @@ export default new Vuex.Store({
     },
     modules: {},
     getters: {
-        mlModel: (state) => state.mlModel,
         seed: (state) => state.seed,
         getPathText: (state) => state.pathText,
         indices: state => state.indices,
@@ -401,6 +380,7 @@ export default new Vuex.Store({
 
             return (state.lastQueryResults as unknown as EsResult).hits.hits.slice(-1)[0];
         },
+        uiTagHover: state => state.uiTagHover,
         uiShowLightbox: state => state.uiShowLightbox,
         uiLightboxSources: state => state.uiLightboxSources,
         uiLightboxThumbs: state => state.uiLightboxThumbs,
@@ -408,7 +388,6 @@ export default new Vuex.Store({
         uiLightboxTypes: state => state.uiLightboxTypes,
         uiLightboxKey: state => state.uiLightboxKey,
         uiLightboxSlide: state => state.uiLightboxSlide,
-        uiSqliteMode: state => state.uiSqliteMode,
 
         optHideDuplicates: state => state.optHideDuplicates,
         optLang: state => state.optLang,
@@ -437,12 +416,5 @@ export default new Vuex.Store({
         optSimpleLightbox: state => state.optSimpleLightbox,
         optShowTagPickerFilter: state => state.optShowTagPickerFilter,
         optFeaturedFields: state => state.optFeaturedFields,
-        optMlRepositories: state => state.optMlRepositories,
-        mlRepositoryList: state => {
-            const repos = state.optMlRepositories.split("\n")
-            return repos[0] == "" ? [] : repos;
-        },
-        optMlDefaultModel: state => state.optMlDefaultModel,
-        optAutoAnalyze: state => state.optAutoAnalyze,
     }
 })

sist2-vue/src/views/Configuration.vue

@@ -1,221 +1,202 @@
 <template>
   <!--  <div :style="{width: `${$store.getters.optContainerWidth}px`}"-->
-    <div
-            v-if="!configLoading"
-            style="margin-left: auto; margin-right: auto;" class="container">
+  <div
+      v-if="!configLoading"
+      style="margin-left: auto; margin-right: auto;" class="container">
+
+    <b-card>
+      <b-card-title>
+        <GearIcon></GearIcon>
+        {{ $t("config") }}
+      </b-card-title>
+      <p>{{ $t("configDescription") }}</p>
+
+      <b-card-body>
+        <h4>{{ $t("displayOptions") }}</h4>
 
         <b-card>
-            <b-card-title>
-                <GearIcon></GearIcon>
-                {{ $t("config") }}
-            </b-card-title>
-            <p>{{ $t("configDescription") }}</p>
 
-            <b-card-body>
-                <h4>{{ $t("displayOptions") }}</h4>
+          <label>
+            <LanguageIcon/>
+            <span style="vertical-align: middle">&nbsp;{{ $t("opt.lang") }}</span></label>
+          <b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
 
-                <b-card>
+          <label>{{ $t("opt.theme") }}</label>
+          <b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
 
-                    <label>
-                        <LanguageIcon/>
-                        <span style="vertical-align: middle">&nbsp;{{ $t("opt.lang") }}</span></label>
-                    <b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
+          <label>{{ $t("opt.displayMode") }}</label>
+          <b-form-select :options="displayModeOptions" :value="optDisplay" @input="setOptDisplay"></b-form-select>
 
-                    <label>{{ $t("opt.theme") }}</label>
-                    <b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
+          <label>{{ $t("opt.columns") }}</label>
+          <b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select>
 
-                    <label>{{ $t("opt.displayMode") }}</label>
-                    <b-form-select :options="displayModeOptions" :value="optDisplay"
-                                   @input="setOptDisplay"></b-form-select>
+          <div style="height: 10px"></div>
 
-                    <label>{{ $t("opt.columns") }}</label>
-                    <b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select>
+          <b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent">
+            {{ $t("opt.lightboxLoadOnlyCurrent") }}
+          </b-form-checkbox>
 
-                    <div style="height: 10px"></div>
+          <b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy">
+            {{ $t("opt.hideLegacy") }}
+          </b-form-checkbox>
 
-                    <b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent">
-                        {{ $t("opt.lightboxLoadOnlyCurrent") }}
-                    </b-form-checkbox>
+          <b-form-checkbox :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
+            {{ $t("opt.updateMimeMap") }}
+          </b-form-checkbox>
 
-                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optHideLegacy" @input="setOptHideLegacy">
-                        {{ $t("opt.hideLegacy") }}
-                    </b-form-checkbox>
+          <b-form-checkbox :checked="optUseDatePicker" @input="setOptUseDatePicker">
+            {{ $t("opt.useDatePicker") }}
+          </b-form-checkbox>
 
-                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
-                        {{ $t("opt.updateMimeMap") }}
-                    </b-form-checkbox>
+          <b-form-checkbox :checked="optSimpleLightbox" @input="setOptSimpleLightbox">{{
+              $t("opt.simpleLightbox")
+            }}
+          </b-form-checkbox>
 
-                    <b-form-checkbox :checked="optUseDatePicker" @input="setOptUseDatePicker">
-                        {{ $t("opt.useDatePicker") }}
-                    </b-form-checkbox>
+          <b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{
+              $t("opt.showTagPickerFilter")
+            }}
+          </b-form-checkbox>
 
-                    <b-form-checkbox :checked="optSimpleLightbox" @input="setOptSimpleLightbox">{{
-                        $t("opt.simpleLightbox")
-                        }}
-                    </b-form-checkbox>
+          <br/>
+          <label>{{ $t("opt.featuredFields") }}</label>
 
-                    <b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{
-                        $t("opt.showTagPickerFilter")
-                        }}
-                    </b-form-checkbox>
+          <br>
+          <b-button v-b-toggle.collapse-1 variant="secondary" class="dropdown-toggle">{{
+              $t("opt.featuredFieldsList")
+            }}
+          </b-button>
+          <b-collapse id="collapse-1" class="mt-2">
+            <ul>
+              <li><code>doc.checksum</code></li>
+              <li><code>doc.path</code></li>
+              <li><code>doc.mime</code></li>
+              <li><code>doc.videoc</code></li>
+              <li><code>doc.audioc</code></li>
+              <li><code>doc.pages</code></li>
+              <li><code>doc.mtime</code></li>
+              <li><code>doc.font_name</code></li>
+              <li><code>doc.album</code></li>
+              <li><code>doc.artist</code></li>
+              <li><code>doc.title</code></li>
+              <li><code>doc.genre</code></li>
+              <li><code>doc.album_artist</code></li>
+              <li><code>doc.exif_make</code></li>
+              <li><code>doc.exif_model</code></li>
+              <li><code>doc.exif_software</code></li>
+              <li><code>doc.exif_exposure_time</code></li>
+              <li><code>doc.exif_fnumber</code></li>
+              <li><code>doc.exif_iso_speed_ratings</code></li>
+              <li><code>doc.exif_focal_length</code></li>
+              <li><code>doc.exif_user_comment</code></li>
+              <li><code>doc.exif_user_comment</code></li>
+              <li><code>doc.exif_gps_longitude_ref</code></li>
+              <li><code>doc.exif_gps_longitude_dms</code></li>
+              <li><code>doc.exif_gps_longitude_dec</code></li>
+              <li><code>doc.exif_gps_latitude_ref</code></li>
+              <li><code>doc.exif_gps_latitude_dec</code></li>
+              <li><code>humanDate()</code></li>
+              <li><code>humanFileSize()</code></li>
+            </ul>
 
-                    <br/>
-                    <label>{{ $t("opt.featuredFields") }}</label>
+            <p>{{ $t("forExample") }}</p>
 
-                    <br>
-                    <b-button v-b-toggle.collapse-1 variant="secondary" class="dropdown-toggle">{{
-                        $t("opt.featuredFieldsList")
-                        }}
-                    </b-button>
-                    <b-collapse id="collapse-1" class="mt-2">
-                        <ul>
-                            <li><code>doc.checksum</code></li>
-                            <li><code>doc.path</code></li>
-                            <li><code>doc.mime</code></li>
-                            <li><code>doc.videoc</code></li>
-                            <li><code>doc.audioc</code></li>
-                            <li><code>doc.pages</code></li>
-                            <li><code>doc.mtime</code></li>
-                            <li><code>doc.font_name</code></li>
-                            <li><code>doc.album</code></li>
-                            <li><code>doc.artist</code></li>
-                            <li><code>doc.title</code></li>
-                            <li><code>doc.genre</code></li>
-                            <li><code>doc.album_artist</code></li>
-                            <li><code>doc.exif_make</code></li>
-                            <li><code>doc.exif_model</code></li>
-                            <li><code>doc.exif_software</code></li>
-                            <li><code>doc.exif_exposure_time</code></li>
-                            <li><code>doc.exif_fnumber</code></li>
-                            <li><code>doc.exif_iso_speed_ratings</code></li>
-                            <li><code>doc.exif_focal_length</code></li>
-                            <li><code>doc.exif_user_comment</code></li>
-                            <li><code>doc.exif_user_comment</code></li>
-                            <li><code>doc.exif_gps_longitude_ref</code></li>
-                            <li><code>doc.exif_gps_longitude_dms</code></li>
-                            <li><code>doc.exif_gps_longitude_dec</code></li>
-                            <li><code>doc.exif_gps_latitude_ref</code></li>
-                            <li><code>doc.exif_gps_latitude_dec</code></li>
-                            <li><code>humanDate()</code></li>
-                            <li><code>humanFileSize()</code></li>
-                        </ul>
-
-                        <p>{{ $t("forExample") }}</p>
-
-                        <ul>
-                            <li>
-                                <code>&lt;b&gt;${humanDate(doc.mtime)}&lt;/b&gt; • ${doc.videoc || ''}</code>
-                            </li>
-                            <li>
-                                <code>${doc.pages ? (doc.pages + ' pages') : ''}</code>
-                            </li>
-                        </ul>
-                    </b-collapse>
-                    <br/>
-                    <br/>
-                    <b-textarea rows="3" :value="optFeaturedFields" @input="setOptFeaturedFields"></b-textarea>
-                </b-card>
-
-                <br/>
-                <h4>{{ $t("searchOptions") }}</h4>
-                <b-card>
-                    <b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
-                        $t("opt.hideDuplicates")
-                        }}
-                    </b-form-checkbox>
-
-                    <b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{
-                        $t("opt.highlight")
-                        }}
-                    </b-form-checkbox>
-                    <b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
-                        $t("opt.tagOrOperator")
-                        }}
-                    </b-form-checkbox>
-                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optFuzzy" @input="setOptFuzzy">
-                        {{ $t("opt.fuzzy") }}
-                    </b-form-checkbox>
-
-                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optSearchInPath" @input="setOptSearchInPath">{{
-                        $t("opt.searchInPath")
-                        }}
-                    </b-form-checkbox>
-                    <b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
-                        $t("opt.suggestPath")
-                        }}
-                    </b-form-checkbox>
-
-                    <br/>
-                    <label>{{ $t("opt.fragmentSize") }}</label>
-                    <b-form-input :value="optFragmentSize" step="10" type="number" min="0"
-                                  @input="setOptFragmentSize"></b-form-input>
-
-                    <label>{{ $t("opt.resultSize") }}</label>
-                    <b-form-input :value="optResultSize" type="number" min="10"
-                                  @input="setOptResultSize"></b-form-input>
-
-                    <label :class="{'text-muted': uiSqliteMode}">{{ $t("opt.queryMode") }}</label>
-                    <b-form-select :disabled="uiSqliteMode" :options="queryModeOptions" :value="optQueryMode"
-                                   @input="setOptQueryMode"></b-form-select>
-
-                    <label>{{ $t("opt.slideDuration") }}</label>
-                    <b-form-input :value="optLightboxSlideDuration" type="number" min="1"
-                                  @input="setOptLightboxSlideDuration"></b-form-input>
-
-                    <label>{{ $t("opt.vidPreviewInterval") }}</label>
-                    <b-form-input :value="optVidPreviewInterval" type="number" min="50"
-                                  @input="setOptVidPreviewInterval"></b-form-input>
-                </b-card>
-
-                <h4 class="mt-3">{{ $t("mlOptions") }}</h4>
-                <b-card>
-                    <label>{{ $t("opt.mlRepositories") }}</label>
-                    <b-textarea rows="3" :value="optMlRepositories" @input="setOptMlRepositories"></b-textarea>
-                    <br>
-                    <b-form-checkbox :checked="optAutoAnalyze" @input="setOptAutoAnalyze">{{
-                        $t("opt.autoAnalyze")
-                        }}
-                    </b-form-checkbox>
-                </b-card>
-
-                <h4 class="mt-3">{{ $t("treemapOptions") }}</h4>
-                <b-card>
-                    <label>{{ $t("opt.treemapType") }}</label>
-                    <b-form-select :value="optTreemapType" :options="treemapTypeOptions"
-                                   @input="setOptTreemapType"></b-form-select>
-
-                    <label>{{ $t("opt.treemapTiling") }}</label>
-                    <b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
-                                   @input="setOptTreemapTiling"></b-form-select>
-
-                    <label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
-                    <b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
-                                  @input="setOptTreemapColorGroupingDepth"></b-form-input>
-
-                    <label>{{ $t("opt.treemapSize") }}</label>
-                    <b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
-                                   @input="setOptTreemapSize"></b-form-select>
-
-                    <template v-if="$store.getters.optTreemapSize === 'custom'">
-                        <!-- TODO Width/Height input -->
-                        <b-form-input type="number" min="0" step="10"></b-form-input>
-                        <b-form-input type="number" min="0" step="10"></b-form-input>
-                    </template>
-
-                    <label>{{ $t("opt.treemapColor") }}</label>
-                    <b-form-select :value="optTreemapColor" :options="treemapColorOptions"
-                                   @input="setOptTreemapColor"></b-form-select>
-                </b-card>
-
-                <b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
-            </b-card-body>
+            <ul>
+              <li>
+                <code>&lt;b&gt;${humanDate(doc.mtime)}&lt;/b&gt; • ${doc.videoc || ''}</code>
+              </li>
+              <li>
+                <code>${doc.pages ? (doc.pages + ' pages') : ''}</code>
+              </li>
+            </ul>
+          </b-collapse>
+          <br/>
+          <br/>
+          <b-textarea rows="3" :value="optFeaturedFields" @input="setOptFeaturedFields"></b-textarea>
         </b-card>
 
-        <b-card v-if="loading" class="mt-4">
-            <Preloader></Preloader>
+        <br/>
+        <h4>{{ $t("searchOptions") }}</h4>
+        <b-card>
+          <b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
+              $t("opt.hideDuplicates")
+            }}
+          </b-form-checkbox>
+
+          <b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{ $t("opt.highlight") }}</b-form-checkbox>
+          <b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
+              $t("opt.tagOrOperator")
+            }}
+          </b-form-checkbox>
+          <b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
+          <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
+              $t("opt.searchInPath")
+            }}
+          </b-form-checkbox>
+          <b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
+              $t("opt.suggestPath")
+            }}
+          </b-form-checkbox>
+
+          <br/>
+          <label>{{ $t("opt.fragmentSize") }}</label>
+          <b-form-input :value="optFragmentSize" step="10" type="number" min="0"
+                        @input="setOptFragmentSize"></b-form-input>
+
+          <label>{{ $t("opt.resultSize") }}</label>
+          <b-form-input :value="optResultSize" type="number" min="10"
+                        @input="setOptResultSize"></b-form-input>
+
+          <label>{{ $t("opt.queryMode") }}</label>
+          <b-form-select :options="queryModeOptions" :value="optQueryMode" @input="setOptQueryMode"></b-form-select>
+
+          <label>{{ $t("opt.slideDuration") }}</label>
+          <b-form-input :value="optLightboxSlideDuration" type="number" min="1"
+                        @input="setOptLightboxSlideDuration"></b-form-input>
+
+          <label>{{ $t("opt.vidPreviewInterval") }}</label>
+          <b-form-input :value="optVidPreviewInterval" type="number" min="50"
+                        @input="setOptVidPreviewInterval"></b-form-input>
         </b-card>
-        <DebugInfo v-else></DebugInfo>
-    </div>
+
+        <h4 class="mt-3">{{ $t("treemapOptions") }}</h4>
+        <b-card>
+          <label>{{ $t("opt.treemapType") }}</label>
+          <b-form-select :value="optTreemapType" :options="treemapTypeOptions"
+                         @input="setOptTreemapType"></b-form-select>
+
+          <label>{{ $t("opt.treemapTiling") }}</label>
+          <b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
+                         @input="setOptTreemapTiling"></b-form-select>
+
+          <label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
+          <b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
+                        @input="setOptTreemapColorGroupingDepth"></b-form-input>
+
+          <label>{{ $t("opt.treemapSize") }}</label>
+          <b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
+                         @input="setOptTreemapSize"></b-form-select>
+
+          <template v-if="$store.getters.optTreemapSize === 'custom'">
+            <!-- TODO Width/Height input -->
+            <b-form-input type="number" min="0" step="10"></b-form-input>
+            <b-form-input type="number" min="0" step="10"></b-form-input>
+          </template>
+
+          <label>{{ $t("opt.treemapColor") }}</label>
+          <b-form-select :value="optTreemapColor" :options="treemapColorOptions"
+                         @input="setOptTreemapColor"></b-form-select>
+        </b-card>
+
+        <b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
+      </b-card-body>
+    </b-card>
+
+    <b-card v-if="loading" class="mt-4">
+      <Preloader></Preloader>
+    </b-card>
+    <DebugInfo v-else></DebugInfo>
+  </div>
 </template>
 
 <script>
@@ -227,169 +208,164 @@ import GearIcon from "@/components/icons/GearIcon.vue";
 import LanguageIcon from "@/components/icons/LanguageIcon";
 
 export default {
-    components: {LanguageIcon, GearIcon, DebugInfo, Preloader},
-    data() {
-        return {
-            loading: false,
-            configLoading: false,
-            langOptions: [
-                {value: "en", text: this.$t("lang.en")},
-                {value: "fr", text: this.$t("lang.fr")},
-                {value: "zh-CN", text: this.$t("lang.zh-CN")},
-                {value: "de", text: this.$t("lang.de")},
-            ],
-            queryModeOptions: [
-                {value: "simple", text: this.$t("queryMode.simple")},
-                {value: "advanced", text: this.$t("queryMode.advanced")}
-            ],
-            displayModeOptions: [
-                {value: "grid", text: this.$t("displayMode.grid")},
-                {value: "list", text: this.$t("displayMode.list")}
-            ],
-            columnsOptions: [
-                {value: "auto", text: this.$t("columns.auto")},
-                {value: 1, text: "1"},
-                {value: 2, text: "2"},
-                {value: 3, text: "3"},
-                {value: 4, text: "4"},
-                {value: 5, text: "5"},
-                {value: 6, text: "6"},
-                {value: 7, text: "7"},
-                {value: 8, text: "8"},
-                {value: 9, text: "9"},
-                {value: 10, text: "10"},
-                {value: 11, text: "11"},
-                {value: 12, text: "12"},
-            ],
-            treemapTypeOptions: [
-                {value: "cascaded", text: this.$t("treemapType.cascaded")},
-                {value: "flat", text: this.$t("treemapType.flat")}
-            ],
-            treemapTilingOptions: [
-                {value: "binary", text: this.$t("treemapTiling.binary")},
-                {value: "squarify", text: this.$t("treemapTiling.squarify")},
-                {value: "slice", text: this.$t("treemapTiling.slice")},
-                {value: "dice", text: this.$t("treemapTiling.dice")},
-                {value: "sliceDice", text: this.$t("treemapTiling.sliceDice")},
-            ],
-            treemapSizeOptions: [
-                {value: "small", text: this.$t("treemapSize.small")},
-                {value: "medium", text: this.$t("treemapSize.medium")},
-                {value: "large", text: this.$t("treemapSize.large")},
-                {value: "x-large", text: this.$t("treemapSize.xLarge")},
-                {value: "xx-large", text: this.$t("treemapSize.xxLarge")},
-                // {value: "custom", text: this.$t("treemapSize.custom")},
-            ],
-            treemapColorOptions: [
-                {value: "PuBuGn", text: "Purple-Blue-Green"},
-                {value: "PuRd", text: "Purple-Red"},
-                {value: "PuBu", text: "Purple-Blue"},
-                {value: "YlOrBr", text: "Yellow-Orange-Brown"},
-                {value: "YlOrRd", text: "Yellow-Orange-Red"},
-                {value: "YlGn", text: "Yellow-Green"},
-                {value: "YlGnBu", text: "Yellow-Green-Blue"},
-                {value: "Plasma", text: "Plasma"},
-                {value: "Magma", text: "Magma"},
-                {value: "Inferno", text: "Inferno"},
-                {value: "Viridis", text: "Viridis"},
-                {value: "Turbo", text: "Turbo"},
-            ],
-            themeOptions: [
-                {value: "light", text: this.$t("theme.light")},
-                {value: "black", text: this.$t("theme.black")}
-            ]
+  components: {LanguageIcon, GearIcon, DebugInfo, Preloader},
+  data() {
+    return {
+      loading: false,
+      configLoading: false,
+      langOptions: [
+        {value: "en", text: this.$t("lang.en")},
+        {value: "fr", text: this.$t("lang.fr")},
+        {value: "zh-CN", text: this.$t("lang.zh-CN")},
+        {value: "de", text: this.$t("lang.de")},
+      ],
+      queryModeOptions: [
+        {value: "simple", text: this.$t("queryMode.simple")},
+        {value: "advanced", text: this.$t("queryMode.advanced")}
+      ],
+      displayModeOptions: [
+        {value: "grid", text: this.$t("displayMode.grid")},
+        {value: "list", text: this.$t("displayMode.list")}
+      ],
+      columnsOptions: [
+        {value: "auto", text: this.$t("columns.auto")},
+        {value: 1, text: "1"},
+        {value: 2, text: "2"},
+        {value: 3, text: "3"},
+        {value: 4, text: "4"},
+        {value: 5, text: "5"},
+        {value: 6, text: "6"},
+        {value: 7, text: "7"},
+        {value: 8, text: "8"},
+        {value: 9, text: "9"},
+        {value: 10, text: "10"},
+        {value: 11, text: "11"},
+        {value: 12, text: "12"},
+      ],
+      treemapTypeOptions: [
+        {value: "cascaded", text: this.$t("treemapType.cascaded")},
+        {value: "flat", text: this.$t("treemapType.flat")}
+      ],
+      treemapTilingOptions: [
+        {value: "binary", text: this.$t("treemapTiling.binary")},
+        {value: "squarify", text: this.$t("treemapTiling.squarify")},
+        {value: "slice", text: this.$t("treemapTiling.slice")},
+        {value: "dice", text: this.$t("treemapTiling.dice")},
+        {value: "sliceDice", text: this.$t("treemapTiling.sliceDice")},
+      ],
+      treemapSizeOptions: [
+        {value: "small", text: this.$t("treemapSize.small")},
+        {value: "medium", text: this.$t("treemapSize.medium")},
+        {value: "large", text: this.$t("treemapSize.large")},
+        {value: "x-large", text: this.$t("treemapSize.xLarge")},
+        {value: "xx-large", text: this.$t("treemapSize.xxLarge")},
+        // {value: "custom", text: this.$t("treemapSize.custom")},
+      ],
+      treemapColorOptions: [
+        {value: "PuBuGn", text: "Purple-Blue-Green"},
+        {value: "PuRd", text: "Purple-Red"},
+        {value: "PuBu", text: "Purple-Blue"},
+        {value: "YlOrBr", text: "Yellow-Orange-Brown"},
+        {value: "YlOrRd", text: "Yellow-Orange-Red"},
+        {value: "YlGn", text: "Yellow-Green"},
+        {value: "YlGnBu", text: "Yellow-Green-Blue"},
+        {value: "Plasma", text: "Plasma"},
+        {value: "Magma", text: "Magma"},
+        {value: "Inferno", text: "Inferno"},
+        {value: "Viridis", text: "Viridis"},
+        {value: "Turbo", text: "Turbo"},
+      ],
+      themeOptions: [
+        {value: "light", text: this.$t("theme.light")},
+        {value: "black", text: this.$t("theme.black")}
+      ]
 
-        }
-    },
-    computed: {
-        ...mapGetters([
-            "uiSqliteMode",
-            "optTheme",
-            "optDisplay",
-            "optColumns",
-            "optHighlight",
-            "optFuzzy",
-            "optSearchInPath",
-            "optSuggestPath",
-            "optFragmentSize",
-            "optQueryMode",
-            "optTreemapType",
-            "optTreemapTiling",
-            "optTreemapColorGroupingDepth",
-            "optTreemapColor",
-            "optTreemapSize",
-            "optLightboxLoadOnlyCurrent",
-            "optLightboxSlideDuration",
-            "optResultSize",
-            "optTagOrOperator",
-            "optLang",
-            "optHideDuplicates",
-            "optHideLegacy",
-            "optUpdateMimeMap",
-            "optUseDatePicker",
-            "optVidPreviewInterval",
-            "optSimpleLightbox",
-            "optShowTagPickerFilter",
-            "optFeaturedFields",
-            "optMlRepositories",
-            "optAutoAnalyze",
-        ]),
-        clientWidth() {
-            return window.innerWidth;
-        }
-    },
-    mounted() {
-        this.$store.subscribe((mutation) => {
-            if (mutation.type.startsWith("setOpt")) {
-                this.$store.dispatch("updateConfiguration");
-            }
-        });
-    },
-    methods: {
-        ...mapActions({
-            setSist2Info: "setSist2Info",
-        }),
-        ...mapMutations([
-            "setOptTheme",
-            "setOptDisplay",
-            "setOptColumns",
-            "setOptHighlight",
-            "setOptFuzzy",
-            "setOptSearchInPath",
-            "setOptSuggestPath",
-            "setOptFragmentSize",
-            "setOptQueryMode",
-            "setOptTreemapType",
-            "setOptTreemapTiling",
-            "setOptTreemapColorGroupingDepth",
-            "setOptTreemapColor",
-            "setOptTreemapSize",
-            "setOptLightboxLoadOnlyCurrent",
-            "setOptLightboxSlideDuration",
-            "setOptResultSize",
-            "setOptTagOrOperator",
-            "setOptLang",
-            "setOptHideDuplicates",
-            "setOptHideLegacy",
-            "setOptUpdateMimeMap",
-            "setOptUseDatePicker",
-            "setOptVidPreviewInterval",
-            "setOptSimpleLightbox",
-            "setOptShowTagPickerFilter",
-            "setOptFeaturedFields",
-            "setOptMlRepositories",
-            "setOptAutoAnalyze",
-        ]),
-        onResetClick() {
-            localStorage.removeItem("sist2_configuration");
-            window.location.reload();
-        }
-    },
+    }
+  },
+  computed: {
+    ...mapGetters([
+      "optTheme",
+      "optDisplay",
+      "optColumns",
+      "optHighlight",
+      "optFuzzy",
+      "optSearchInPath",
+      "optSuggestPath",
+      "optFragmentSize",
+      "optQueryMode",
+      "optTreemapType",
+      "optTreemapTiling",
+      "optTreemapColorGroupingDepth",
+      "optTreemapColor",
+      "optTreemapSize",
+      "optLightboxLoadOnlyCurrent",
+      "optLightboxSlideDuration",
+      "optResultSize",
+      "optTagOrOperator",
+      "optLang",
+      "optHideDuplicates",
+      "optHideLegacy",
+      "optUpdateMimeMap",
+      "optUseDatePicker",
+      "optVidPreviewInterval",
+      "optSimpleLightbox",
+      "optShowTagPickerFilter",
+      "optFeaturedFields",
+    ]),
+    clientWidth() {
+      return window.innerWidth;
+    }
+  },
+  mounted() {
+    this.$store.subscribe((mutation) => {
+      if (mutation.type.startsWith("setOpt")) {
+        this.$store.dispatch("updateConfiguration");
+      }
+    });
+  },
+  methods: {
+    ...mapActions({
+      setSist2Info: "setSist2Info",
+    }),
+    ...mapMutations([
+      "setOptTheme",
+      "setOptDisplay",
+      "setOptColumns",
+      "setOptHighlight",
+      "setOptFuzzy",
+      "setOptSearchInPath",
+      "setOptSuggestPath",
+      "setOptFragmentSize",
+      "setOptQueryMode",
+      "setOptTreemapType",
+      "setOptTreemapTiling",
+      "setOptTreemapColorGroupingDepth",
+      "setOptTreemapColor",
+      "setOptTreemapSize",
+      "setOptLightboxLoadOnlyCurrent",
+      "setOptLightboxSlideDuration",
+      "setOptResultSize",
+      "setOptTagOrOperator",
+      "setOptLang",
+      "setOptHideDuplicates",
+      "setOptHideLegacy",
+      "setOptUpdateMimeMap",
+      "setOptUseDatePicker",
+      "setOptVidPreviewInterval",
+      "setOptSimpleLightbox",
+      "setOptShowTagPickerFilter",
+      "setOptFeaturedFields",
+    ]),
+    onResetClick() {
+      localStorage.removeItem("sist2_configuration");
+      window.location.reload();
+    }
+  },
 }
 </script>
 
 <style>
 .shrink {
-    flex-grow: inherit;
+  flex-grow: inherit;
 }
 </style>

sist2-vue/src/views/FilePage.vue

@@ -14,7 +14,7 @@
       <!-- Audio player-->
       <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls
              :type="doc._source.mime"
-             :src="`f/${doc._source.index}/${doc._id}`"></audio>
+             :src="`f/${doc._id}`"></audio>
 
       <InfoTable :doc="doc" v-if="doc"></InfoTable>
 
@@ -54,7 +54,7 @@ export default Vue.extend({
   methods: {
     ext: ext,
     onThumbnailClick() {
-      window.open(`/f/${this.doc.index}/${this.doc._id}`, "_blank");
+      window.open(`/f/${this.doc._id}`, "_blank");
     },
     findByCustomField(field, id) {
       return {

sist2-vue/src/views/SearchPage.vue

@@ -1,61 +1,57 @@
 <template>
-    <div class="container">
-        <Lightbox></Lightbox>
-        <HelpDialog :show="showHelp" @close="showHelp = false"></HelpDialog>
+  <div class="container">
+    <Lightbox></Lightbox>
+    <HelpDialog :show="showHelp" @close="showHelp = false"></HelpDialog>
 
-        <b-card v-if="uiLoading">
-            <Preloader></Preloader>
-        </b-card>
+    <b-card v-if="uiLoading">
+      <Preloader></Preloader>
+    </b-card>
 
-        <b-alert v-show="!uiLoading && showEsConnectionError" show variant="danger" class="mt-2">
-            {{ $t("toast.esConnErr") }}
-        </b-alert>
+    <b-card v-show="!uiLoading" id="search-panel">
+      <SearchBar @show-help="showHelp=true"></SearchBar>
+      <b-row>
+        <b-col style="height: 70px;" sm="6">
+          <SizeSlider></SizeSlider>
+        </b-col>
+        <b-col>
+          <PathTree @search="search(true)"></PathTree>
+        </b-col>
+      </b-row>
+      <b-row>
+        <b-col sm="6">
+          <DateSlider></DateSlider>
+          <b-row>
+            <b-col>
+              <IndexPicker></IndexPicker>
+            </b-col>
+          </b-row>
+        </b-col>
+        <b-col>
+          <b-tabs justified>
+            <b-tab :title="$t('mimeTypes')">
+              <MimePicker></MimePicker>
+            </b-tab>
+            <b-tab :title="$t('tags')">
+              <TagPicker :show-search-bar="$store.state.optShowTagPickerFilter"></TagPicker>
+            </b-tab>
+          </b-tabs>
+        </b-col>
+      </b-row>
+    </b-card>
 
-        <b-card v-show="!uiLoading && !showEsConnectionError" id="search-panel">
-            <SearchBar @show-help="showHelp=true"></SearchBar>
-            <b-row>
-                <b-col style="height: 70px;" sm="6">
-                    <SizeSlider></SizeSlider>
-                </b-col>
-                <b-col>
-                    <PathTree @search="search(true)"></PathTree>
-                </b-col>
-            </b-row>
-            <b-row>
-                <b-col sm="6">
-                    <DateSlider></DateSlider>
-                    <b-row>
-                        <b-col>
-                            <IndexPicker></IndexPicker>
-                        </b-col>
-                    </b-row>
-                </b-col>
-                <b-col>
-                    <b-tabs justified>
-                        <b-tab :title="$t('mimeTypes')">
-                            <MimePicker></MimePicker>
-                        </b-tab>
-                        <b-tab :title="$t('tags')">
-                            <TagPicker :show-search-bar="$store.state.optShowTagPickerFilter"></TagPicker>
-                        </b-tab>
-                    </b-tabs>
-                </b-col>
-            </b-row>
-        </b-card>
+    <div v-show="docs.length === 0 && !uiLoading">
+      <Preloader v-if="searchBusy" class="mt-3"></Preloader>
 
-        <div v-show="docs.length === 0 && !uiLoading">
-            <Preloader v-if="searchBusy" class="mt-3"></Preloader>
-
-            <ResultsCard></ResultsCard>
-        </div>
-
-        <div v-if="docs.length > 0">
-            <ResultsCard></ResultsCard>
-
-            <DocCardWall v-if="optDisplay==='grid'" :docs="docs" :append="appendFunc"></DocCardWall>
-            <DocList v-else :docs="docs" :append="appendFunc"></DocList>
-        </div>
+      <ResultsCard></ResultsCard>
     </div>
+
+    <div v-if="docs.length > 0">
+      <ResultsCard></ResultsCard>
+
+      <DocCardWall v-if="optDisplay==='grid'" :docs="docs" :append="appendFunc"></DocCardWall>
+      <DocList v-else :docs="docs" :append="appendFunc"></DocList>
+    </div>
+  </div>
 </template>
 
 <script lang="ts">
@@ -66,7 +62,7 @@ import Sist2Api, {EsHit, EsResult} from "../Sist2Api";
 import SearchBar from "@/components/SearchBar.vue";
 import IndexPicker from "@/components/IndexPicker.vue";
 import Vue from "vue";
-import Sist2Query from "@/Sist2ElasticsearchQuery";
+import Sist2Query from "@/Sist2Query";
 import _debounce from "lodash/debounce";
 import DocCardWall from "@/components/DocCardWall.vue";
 import Lightbox from "@/components/Lightbox.vue";
@@ -79,236 +75,237 @@ import DateSlider from "@/components/DateSlider.vue";
 import TagPicker from "@/components/TagPicker.vue";
 import DocList from "@/components/DocList.vue";
 import HelpDialog from "@/components/HelpDialog.vue";
-import Sist2SqliteQuery from "@/Sist2SqliteQuery";
 
 
 export default Vue.extend({
-    components: {
-        HelpDialog,
-        DocList,
-        TagPicker,
-        DateSlider,
-        SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
-    },
-    data: () => ({
-        loading: false,
-        uiLoading: true,
-        search: undefined as any,
-        docs: [] as EsHit[],
-        docIds: new Set(),
-        docChecksums: new Set(),
-        searchBusy: false,
-        Sist2Query: Sist2Query,
-        showHelp: false,
-        showEsConnectionError: false
+  components: {
+    HelpDialog,
+    DocList,
+    TagPicker,
+    DateSlider,
+    SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
+  },
+  data: () => ({
+    loading: false,
+    uiLoading: true,
+    search: undefined as any,
+    docs: [] as EsHit[],
+    docIds: new Set(),
+    docChecksums: new Set(),
+    searchBusy: false,
+    Sist2Query: Sist2Query,
+    showHelp: false
+  }),
+  computed: {
+    ...mapGetters(["indices", "optDisplay"]),
+  },
+  mounted() {
+    // Handle touch events
+    window.ontouchend = () => this.$store.commit("busTouchEnd");
+    window.ontouchcancel = this.$store.commit("busTouchEnd");
+
+    this.search = _debounce(async (clear: boolean) => {
+      if (clear) {
+        await this.clearResults();
+      }
+
+      await this.searchNow(Sist2Query.searchQuery());
+
+    }, 350, {leading: false});
+
+    this.$store.dispatch("loadFromArgs", this.$route).then(() => {
+      this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
+      this.$store.subscribe((mutation) => {
+        if ([
+          "setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
+          "setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
+          "setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
+        ].includes(mutation.type)) {
+          if (this.searchBusy) {
+            return;
+          }
+
+          this.search(true);
+        }
+      });
+    });
+
+    this.setIndices(this.$store.getters["sist2Info"].indices)
+
+    this.getDateRange().then((range: { min: number, max: number }) => {
+      this.setDateBoundsMin(range.min);
+      this.setDateBoundsMax(range.max);
+
+      const doBlankSearch = !this.$store.state.optUpdateMimeMap;
+
+      Sist2Api.getMimeTypes(Sist2Query.searchQuery(doBlankSearch)).then(({mimeMap}) => {
+        this.$store.commit("setUiMimeMap", mimeMap);
+        this.uiLoading = false;
+        this.search(true);
+      });
+    });
+  },
+  methods: {
+    ...mapActions({
+      setSist2Info: "setSist2Info",
     }),
-    computed: {
-        ...mapGetters(["indices", "optDisplay"]),
+    ...mapMutations({
+      setIndices: "setIndices",
+      setDateBoundsMin: "setDateBoundsMin",
+      setDateBoundsMax: "setDateBoundsMax",
+      setTags: "setTags",
+    }),
+    showErrorToast() {
+      this.$bvToast.toast(
+          this.$t("toast.esConnErr"),
+          {
+            title: this.$t("toast.esConnErrTitle"),
+            noAutoHide: true,
+            toaster: "b-toaster-bottom-right",
+            headerClass: "toast-header-error",
+            bodyClass: "toast-body-error",
+          });
     },
-    mounted() {
-        // Handle touch events
-        window.ontouchend = () => this.$store.commit("busTouchEnd");
-        window.ontouchcancel = this.$store.commit("busTouchEnd");
-
-        this.search = _debounce(async (clear: boolean) => {
-            if (clear) {
-                await this.clearResults();
-            }
-
-            await this.searchNow();
-
-        }, 350, {leading: false});
-
-        this.$store.dispatch("loadFromArgs", this.$route).then(() => {
-            this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
-            this.$store.subscribe((mutation) => {
-                if ([
-                    "setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
-                    "setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
-                    "setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
-                ].includes(mutation.type)) {
-                    if (this.searchBusy) {
-                        return;
-                    }
-
-                    this.search(true);
-                }
-            });
-        });
-
-        this.setIndices(this.$store.getters["sist2Info"].indices)
-
-        Sist2Api.getDateRange().then((range) => {
-            this.setDateBoundsMin(range.min);
-            this.setDateBoundsMax(range.max);
-
-            const doBlankSearch = !this.$store.state.optUpdateMimeMap;
-
-            Sist2Api.getMimeTypes(Sist2Query.searchQuery(doBlankSearch)).then(({mimeMap}) => {
-                this.$store.commit("setUiMimeMap", mimeMap);
-                this.uiLoading = false;
-                this.search(true);
-            });
-        }).catch(error => {
-            console.log(error);
-
-            if (error.response.status == 503 || error.response.status == 500) {
-                this.showEsConnectionError = true;
-                this.uiLoading = false;
-            } else {
-                this.showErrorToast();
-            }
-        });
+    showSyntaxErrorToast: function (): void {
+      this.$bvToast.toast(
+          this.$t("toast.esQueryErr"),
+          {
+            title: this.$t("toast.esQueryErrTitle"),
+            noAutoHide: true,
+            toaster: "b-toaster-bottom-right",
+            headerClass: "toast-header-warning",
+            bodyClass: "toast-body-warning",
+          });
     },
-    methods: {
-        ...mapActions({
-            setSist2Info: "setSist2Info",
-        }),
-        ...mapMutations({
-            setIndices: "setIndices",
-            setDateBoundsMin: "setDateBoundsMin",
-            setDateBoundsMax: "setDateBoundsMax",
-            setTags: "setTags",
-        }),
-        showErrorToast() {
-            this.$bvToast.toast(
-                this.$t("toast.esConnErr"),
-                {
-                    title: this.$t("toast.esConnErrTitle"),
-                    noAutoHide: true,
-                    toaster: "b-toaster-bottom-right",
-                    headerClass: "toast-header-error",
-                    bodyClass: "toast-body-error",
-                });
-        },
-        showSyntaxErrorToast: function (): void {
-            this.$bvToast.toast(
-                this.$t("toast.esQueryErr"),
-                {
-                    title: this.$t("toast.esQueryErrTitle"),
-                    noAutoHide: true,
-                    toaster: "b-toaster-bottom-right",
-                    headerClass: "toast-header-warning",
-                    bodyClass: "toast-body-warning",
-                });
-        },
-        async searchNow() {
-            this.searchBusy = true;
-            await this.$store.dispatch("incrementQuerySequence");
-            this.$store.commit("busSearch");
+    async searchNow(q: any) {
+      this.searchBusy = true;
+      await this.$store.dispatch("incrementQuerySequence");
+      this.$store.commit("busSearch");
 
-            Sist2Api.search().then(async (resp: EsResult) => {
-                await this.handleSearch(resp);
-                this.searchBusy = false;
-            }).catch(err => {
-                if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
-                    this.showSyntaxErrorToast();
-                } else {
-                    this.showErrorToast();
-                }
-            });
-        },
-        async clearResults() {
-            this.docs = [];
-            this.docIds.clear();
-            this.docChecksums.clear();
-            await this.$store.dispatch("clearResults");
-            this.$store.commit("setUiReachedScrollEnd", false);
-        },
-        async handleSearch(resp: EsResult) {
-            if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
-                this.$store.commit("setUiReachedScrollEnd", true);
-            }
-
-            resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
-
-            if (this.$store.state.optHideDuplicates) {
-                resp.hits.hits = resp.hits.hits.filter(hit => {
-
-                    if (!("checksum" in hit._source)) {
-                        return true;
-                    }
-
-                    const isDupe = !this.docChecksums.has(hit._source.checksum);
-                    this.docChecksums.add(hit._source.checksum);
-                    return isDupe;
-                });
-            }
-
-            for (const hit of resp.hits.hits) {
-                if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
-                    hit._seq = await this.$store.dispatch("getKeySequence");
-                    this.$store.commit("addLightboxSource", {
-                        source: `f/${hit._source.index}/${hit._id}`,
-                        thumbnail: hit._props.hasThumbnail
-                            ? `t/${hit._source.index}/${hit._id}`
-                            : null,
-                        caption: {
-                            component: LightboxCaption,
-                            props: {hit: hit}
-                        },
-                        type: hit._props.isVideo ? "video" : "image"
-                    });
-                }
-            }
-
-            await this.$store.dispatch("remountLightbox");
-            this.$store.commit("setLastQueryResult", resp);
-            if (this.$store.state.firstQueryResults == null) {
-                this.$store.commit("setFirstQueryResult", resp);
-            }
-
-            this.docs.push(...resp.hits.hits);
-
-            resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
-        },
-        appendFunc() {
-            if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
-                this.searchNow();
-            }
-        }
-    },
-    beforeRouteUpdate(to, from, next) {
-        if (this.$store.state.uiLightboxIsOpen) {
-            this.$store.commit("_setUiShowLightbox", false);
-            next(false);
+      Sist2Api.esQuery(q).then(async (resp: EsResult) => {
+        await this.handleSearch(resp);
+        this.searchBusy = false;
+      }).catch(err => {
+        if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
+          this.showSyntaxErrorToast();
         } else {
-            next();
+          this.showErrorToast();
         }
+      });
     },
+    async clearResults() {
+      this.docs = [];
+      this.docIds.clear();
+      this.docChecksums.clear();
+      await this.$store.dispatch("clearResults");
+      this.$store.commit("setUiReachedScrollEnd", false);
+    },
+    async handleSearch(resp: EsResult) {
+      if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
+        this.$store.commit("setUiReachedScrollEnd", true);
+      }
+
+      resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
+
+      if (this.$store.state.optHideDuplicates) {
+        resp.hits.hits = resp.hits.hits.filter(hit => {
+
+          if (!("checksum" in hit._source)) {
+            return true;
+          }
+
+          const isDupe = !this.docChecksums.has(hit._source.checksum);
+          this.docChecksums.add(hit._source.checksum);
+          return isDupe;
+        });
+      }
+
+      for (const hit of resp.hits.hits) {
+        if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
+          hit._seq = await this.$store.dispatch("getKeySequence");
+          this.$store.commit("addLightboxSource", {
+            source: `f/${hit._id}`,
+            thumbnail: hit._props.hasThumbnail
+                ? `t/${hit._source.index}/${hit._id}`
+                : null,
+            caption: {
+              component: LightboxCaption,
+              props: {hit: hit}
+            },
+            type: hit._props.isVideo ? "video" : "image"
+          });
+        }
+      }
+
+      await this.$store.dispatch("remountLightbox");
+      this.$store.commit("setLastQueryResult", resp);
+
+      this.docs.push(...resp.hits.hits);
+
+      resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
+    },
+    getDateRange(): Promise<{ min: number, max: number }> {
+      return sist2.esQuery({
+        // TODO: filter current selected indices
+        aggs: {
+          dateMin: {min: {field: "mtime"}},
+          dateMax: {max: {field: "mtime"}},
+        },
+        size: 0
+      }).then(res => {
+        return {
+          min: res.aggregations.dateMin.value,
+          max: res.aggregations.dateMax.value,
+        }
+      })
+    },
+    appendFunc() {
+      if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
+        this.searchNow(Sist2Query.searchQuery());
+      }
+    }
+  },
+  beforeRouteUpdate(to, from, next) {
+    if (this.$store.state.uiLightboxIsOpen) {
+      this.$store.commit("_setUiShowLightbox", false);
+      next(false);
+    } else {
+      next();
+    }
+  },
 })
 </script>
 
 <style>
 
 #search-panel {
-    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-    border-radius: 0;
-    border: none;
+  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+  border-radius: 0;
+  border: none;
 }
 
 .toast-header-info, .toast-body-info {
-    background: #2196f3;
-    color: #fff !important;
+  background: #2196f3;
+  color: #fff !important;
 }
 
 .toast-header-error, .toast-body-error {
-    background: #a94442;
-    color: #f2dede !important;
+  background: #a94442;
+  color: #f2dede !important;
 }
 
 .toast-header-error {
-    color: #fff !important;
-    border-bottom: none;
-    margin-bottom: -1em;
+  color: #fff !important;
+  border-bottom: none;
+  margin-bottom: -1em;
 }
 
 .toast-header-error .close {
-    text-shadow: none;
+  text-shadow: none;
 }
 
 .toast-header-warning, .toast-body-warning {
-    background: #FF8F00;
-    color: #FFF3E0 !important;
+  background: #FF8F00;
+  color: #FFF3E0 !important;
 }
 </style>

sist2-vue/src/views/StatsPage.vue

@@ -1,36 +1,37 @@
 <template>
-    <b-container>
+  <b-container>
 
-        <template>
-            <b-card>
-                <b-card-body>
-                    <b-select v-model="selectedIndex" :options="indexOptions">
-                        <template #first>
-                            <b-form-select-option :value="null" disabled>{{
-                                $t("indexPickerPlaceholder")
-                                }}
-                            </b-form-select-option>
-                        </template>
-                    </b-select>
-                </b-card-body>
-            </b-card>
+    <b-card v-if="loading">
+      <Preloader></Preloader>
+    </b-card>
 
-            <b-card v-if="selectedIndex !== null" class="mt-3">
-                <b-card-body>
-                    <D3Treemap :index-id="selectedIndex"></D3Treemap>
+    <template v-else>
+      <b-card>
+        <b-card-body>
+          <b-select v-model="selectedIndex" :options="indexOptions">
+            <template #first>
+              <b-form-select-option :value="null" disabled>{{ $t("indexPickerPlaceholder") }}</b-form-select-option>
+            </template>
+          </b-select>
+        </b-card-body>
+      </b-card>
+
+      <b-card v-if="selectedIndex !== null" class="mt-3">
+        <b-card-body>
+          <D3Treemap :index-id="selectedIndex"></D3Treemap>
 
 
-                </b-card-body>
-            </b-card>
+        </b-card-body>
+      </b-card>
 
-            <b-card v-if="selectedIndex !== null" class="stats-card mt-3">
-                <D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
-                <D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
-                <D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
-                <D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
-            </b-card>
-        </template>
-    </b-container>
+      <b-card v-if="selectedIndex !== null" class="stats-card mt-3">
+        <D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
+        <D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
+        <D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
+        <D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
+      </b-card>
+    </template>
+  </b-container>
 </template>
 <script>
 import D3Treemap from "@/components/D3Treemap";
@@ -42,30 +43,37 @@ import D3DateHistogram from "@/components/D3DateHistogram";
 import D3SizeHistogram from "@/components/D3SizeHistogram";
 
 export default {
-    components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
-    data() {
-        return {
-            selectedIndex: null,
-        }
-    },
-    computed: {
-        indexOptions() {
-            return this.indices.map(idx => {
-                return {
-                    text: idx.name,
-                    value: idx.id
-                };
-            })
-        },
-        indices: () => this.$store.state.indices
+  components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
+  data() {
+    return {
+      loading: true,
+      selectedIndex: null,
+      indices: []
     }
+  },
+  computed: {
+    indexOptions() {
+      return this.indices.map(idx => {
+        return {
+          text: idx.name,
+          value: idx.id
+        };
+      })
+    }
+  },
+  mounted() {
+    Sist2Api.getSist2Info().then(data => {
+      this.indices = data.indices;
+      this.loading = false;
+    })
+  }
 }
 </script>
 
 <style>
 
 .stats-card {
-    text-align: center;
-    padding: 1em;
+  text-align: center;
+  padding: 1em;
 }
 </style>

src/cli.c

@@ -410,32 +410,6 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
     return 0;
 }
 
-int sqlite_index_args_validate(sqlite_index_args_t *args, int argc, const char **argv) {
-
-    LogCtx.verbose = 1;
-
-    if (argc < 2) {
-        fprintf(stderr, "Required positional argument: PATH.\n");
-        return 1;
-    }
-
-    char *index_path = abspath(argv[1]);
-    if (index_path == NULL) {
-        LOG_FATALF("cli.c", "Invalid PATH argument. File not found: %s", argv[1]);
-    } else {
-        args->index_path = index_path;
-    }
-
-    if (args->search_index_path == NULL) {
-        LOG_FATAL("cli.c", "Missing required argument --search-index");
-    }
-
-    LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path);
-    LOG_DEBUGF("cli.c", "arg search_index_path=%s", args->search_index_path);
-
-    return 0;
-}
-
 int web_args_validate(web_args_t *args, int argc, const char **argv) {
 
     LogCtx.verbose = 1;
@@ -445,16 +419,6 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
-    if (args->search_index_path != NULL && args->es_url != NULL) {
-        LOG_FATAL("cli.c", "--search-index and --es-url arguments are mutually exclusive.");
-    }
-    if (args->search_index_path != NULL && args->es_index != NULL) {
-        LOG_FATAL("cli.c", "--search-index and --es-index arguments are mutually exclusive.");
-    }
-    if (args->search_index_path != NULL && args->es_insecure_ssl == TRUE) {
-        LOG_FATAL("cli.c", "--search-index and --es-insecure_ssl arguments are mutually exclusive.");
-    }
-
     if (args->es_url == NULL) {
         args->es_url = DEFAULT_ES_URL;
     }
@@ -567,25 +531,9 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         free(abs_path);
     }
 
-    if (args->search_index_path != NULL) {
-        char *abs_path = abspath(args->search_index_path);
-        if (abs_path == NULL) {
-            LOG_FATALF("cli.c", "Search index not found: %s", args->search_index_path);
-        }
-
-        args->es_index = NULL;
-        args->es_url = NULL;
-        args->es_insecure_ssl = FALSE;
-        args->search_backend = SQLITE_SEARCH_BACKEND;
-    } else {
-        args->search_backend = ES_SEARCH_BACKEND;
-    }
-
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url);
     LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index);
     LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl);
-    LOG_DEBUGF("cli.c", "arg search_index_path=%s", args->search_index_path);
-    LOG_DEBUGF("cli.c", "arg search_backend=%d", args->search_backend);
     LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline);
     LOG_DEBUGF("cli.c", "arg dev=%d", args->dev);
     LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address);
@@ -606,11 +554,6 @@ index_args_t *index_args_create() {
     return args;
 }
 
-sqlite_index_args_t *sqlite_index_args_create() {
-    sqlite_index_args_t *args = calloc(sizeof(sqlite_index_args_t), 1);
-    return args;
-}
-
 web_args_t *web_args_create() {
     web_args_t *args = calloc(sizeof(web_args_t), 1);
     return args;

src/cli.h

@@ -66,21 +66,10 @@ typedef struct index_args {
     int incremental;
 } index_args_t;
 
-typedef struct {
-    char *index_path;
-    char *search_index_path;
-} sqlite_index_args_t;
-
-typedef enum {
-    ES_SEARCH_BACKEND,
-    SQLITE_SEARCH_BACKEND,
-} search_backend_t;
-
 typedef struct web_args {
     char *es_url;
     char *es_index;
     int es_insecure_ssl;
-    char *search_index_path;
     char *listen_address;
     char *credentials;
     char *tag_credentials;
@@ -99,7 +88,6 @@ typedef struct web_args {
     int index_count;
     int dev;
     const char **indices;
-    search_backend_t search_backend;
 } web_args_t;
 
 typedef struct exec_args {
@@ -114,8 +102,6 @@ typedef struct exec_args {
 
 index_args_t *index_args_create();
 
-sqlite_index_args_t *sqlite_index_args_create();
-
 void index_args_destroy(index_args_t *args);
 
 web_args_t *web_args_create();
@@ -124,8 +110,6 @@ void web_args_destroy(web_args_t *args);
 
 int index_args_validate(index_args_t *args, int argc, const char **argv);
 
-int sqlite_index_args_validate(sqlite_index_args_t *args, int argc, const char **argv);
-
 int web_args_validate(web_args_t *args, int argc, const char **argv);
 
 exec_args_t *exec_args_create();
@@ -134,5 +118,4 @@ void exec_args_destroy(exec_args_t *args);
 
 int exec_args_validate(exec_args_t *args, int argc, const char **argv);
 
-
 #endif

src/ctx.h

@@ -76,7 +76,6 @@ typedef struct {
     char *es_url;
     es_version_t *es_version;
     char *es_index;
-    database_t *search_db;
     int es_insecure_ssl;
     int index_count;
     char *auth_user;
@@ -94,7 +93,6 @@ typedef struct {
     struct index_t indices[256];
     char lang[10];
     int dev;
-    int search_backend;
 } WebCtx_t;
 
 

src/database/database.c

@@ -14,45 +14,12 @@ database_t *database_create(const char *filename, database_type_t type) {
     strcpy(db->filename, filename);
     db->type = type;
     db->select_thumbnail_stmt = NULL;
-    db->db = NULL;
-    db->tag_array = NULL;
 
     db->ipc_ctx = NULL;
 
     return db;
 }
 
-int tag_matches(const char *query, const char *tag) {
-    size_t query_len = strlen(query);
-    size_t tag_len = strlen(tag);
-
-    if (query_len >= tag_len) {
-        return FALSE;
-    }
-
-    return strncmp(tag, query, query_len) == 0 && *(tag + query_len) == '.';
-}
-
-void tag_matches_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
-
-    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
-        sqlite3_result_error(ctx, "Invalid parameters", -1);
-    }
-
-    const char *tag = (const char *) sqlite3_value_text(argv[0]);
-
-    char **tags = *(char ***) sqlite3_user_data(ctx);
-
-    array_foreach(tags) {
-        if (tag_matches(tags[i], tag)) {
-            sqlite3_result_int(ctx, TRUE);
-            return;
-        }
-    }
-
-    sqlite3_result_int(ctx, FALSE);
-}
-
 __always_inline
 static int sep_rfind(const char *str) {
     for (int i = (int) strlen(str); i >= 0; i--) {
@@ -81,24 +48,6 @@ void path_parent_func(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
     sqlite3_result_text(ctx, parent, stop, SQLITE_TRANSIENT);
 }
 
-void random_func(sqlite3_context *ctx, int argc, UNUSED(sqlite3_value **argv)) {
-    if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
-        sqlite3_result_error(ctx, "Invalid parameters", -1);
-    }
-
-    char state_buf[128] = {0,};
-    struct random_data buf;
-    int result;
-
-    long seed = sqlite3_value_int64(argv[0]);
-
-    initstate_r((int) seed, state_buf, sizeof(state_buf), &buf);
-
-    random_r(&buf, &result);
-
-    sqlite3_result_int(ctx, result);
-}
-
 
 void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
     if (argc != 1 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
@@ -125,8 +74,6 @@ void database_initialize(database_t *db) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IndexDatabaseSchema, NULL, NULL, NULL));
     } else if (db->type == IPC_CONSUMER_DATABASE || db->type == IPC_PRODUCER_DATABASE) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, IpcDatabaseSchema, NULL, NULL, NULL));
-    } else if (db->type == FTS_DATABASE) {
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, FtsDatabaseSchema, NULL, NULL, NULL));
     }
 
     sqlite3_close(db->db);
@@ -136,10 +83,8 @@ void database_open(database_t *db) {
     LOG_DEBUGF("database.c", "Opening database %s (%d)", db->filename, db->type);
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_open(db->filename, &db->db));
-    sqlite3_busy_timeout(db->db, 1000);
 
-    // TODO: Optional argument?
-//    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA cache_size = -200000;", NULL, NULL, NULL));
+    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA cache_size = -200000;", NULL, NULL, NULL));
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL));
 
     if (db->type == INDEX_DATABASE) {
@@ -171,10 +116,6 @@ void database_open(database_t *db) {
                 -1,
                 &db->write_thumbnail_stmt, NULL));
 
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT json_data FROM document WHERE id=?", -1,
-                &db->get_document, NULL));
-
         // Create functions
         sqlite3_create_function(
                 db->db,
@@ -226,61 +167,6 @@ void database_open(database_t *db) {
                 db->db, "INSERT INTO index_job (doc_id,type,line) VALUES (?,?,?);", -1,
                 &db->insert_index_job_stmt, NULL));
 
-    } else if (db->type == FTS_DATABASE) {
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT path, count FROM path_index"
-                        " WHERE index_id=? AND depth BETWEEN ? AND ?"
-                        " LIMIT 65536", -1,
-                &db->fts_search_paths, NULL));
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT json_data FROM document_index"
-                        " WHERE id=?", -1,
-                &db->fts_get_document, NULL));
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT DISTINCT tag FROM tag"
-                        " WHERE tag GLOB (? || '*') ORDER BY tag LIMIT 100", -1,
-                &db->fts_suggest_tag, NULL));
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT tag, count(*) FROM tag GROUP BY tag", -1,
-                &db->fts_get_tags, NULL));
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT path, count FROM path_index"
-                        " WHERE (index_id=?1 OR ?1 IS NULL) AND depth BETWEEN ? AND ?"
-                        " AND (path = ?4 or path GLOB ?5)"
-                        " LIMIT 65536", -1,
-                &db->fts_search_paths_w_prefix, NULL));
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT path, count FROM path_index"
-                        " WHERE depth BETWEEN ? AND ?"
-                        " AND path GLOB ?"
-                        " LIMIT 65536", -1,
-                &db->fts_suggest_paths, NULL));
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT * FROM stats", -1,
-                &db->fts_date_range, NULL));
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db, "SELECT mime, sum(count) FROM mime_index WHERE mime is not NULL GROUP BY mime", -1,
-                &db->fts_get_mimetypes, NULL));
-
-        sqlite3_create_function(
-                db->db,
-                "random_seeded",
-                1,
-                SQLITE_UTF8,
-                NULL,
-                random_func,
-                NULL,
-                NULL
-        );
-
         sqlite3_create_function(
                 db->db,
                 "path_parent",
@@ -291,33 +177,8 @@ void database_open(database_t *db) {
                 NULL,
                 NULL
         );
-
-        sqlite3_create_function(
-                db->db,
-                "tag_matches",
-                1,
-                SQLITE_UTF8,
-                &db->tag_array,
-                tag_matches_func,
-                NULL,
-                NULL
-        );
     }
 
-    if (db->type == FTS_DATABASE || db->type == INDEX_DATABASE) {
-        // Tag table is the same schema for FTS database & index database
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db,
-                "INSERT INTO tag (id, tag) VALUES (?,?) ON CONFLICT DO NOTHING;",
-                -1,
-                &db->write_tag_stmt, NULL));
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db,
-                "DELETE FROM tag WHERE id=? AND tag=?;",
-                -1,
-                &db->delete_tag_stmt, NULL));
-    }
 }
 
 void database_close(database_t *db, int optimize) {
@@ -329,9 +190,7 @@ void database_close(database_t *db, int optimize) {
         CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA optimize;", NULL, NULL, NULL));
     }
 
-    if (db->db) {
-        sqlite3_close(db->db);
-    }
+    sqlite3_close(db->db);
 
     if (db->type == IPC_PRODUCER_DATABASE) {
         remove(db->filename);
@@ -469,18 +328,18 @@ database_iterator_t *database_create_document_iterator(database_t *db) {
                                " WHEN sc.json_data IS NULL THEN"
                                "  CASE"
                                "   WHEN t.tag IS NULL THEN"
-                               "    json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime)"
+                               "    document.json_data"
                                "   ELSE"
-                               "    json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime, '$.tag', json_group_array(t.tag))"
+                               "    json_set(document.json_data, '$.tag', json_group_array(t.tag))"
                                "   END"
                                " ELSE"
                                "  CASE"
                                "   WHEN t.tag IS NULL THEN"
-                               "    json_patch(json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime), sc.json_data)"
+                               "    json_patch(document.json_data, sc.json_data)"
                                "   ELSE"
                                //   This will overwrite any tags specified in the sidecar file!
                                //   TODO: concatenate the two arrays?
-                               "    json_set(json_patch(document.json_data, sc.json_data), '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime, '$.tag', json_group_array(t.tag))"
+                               "    json_set(json_patch(document.json_data, sc.json_data), '$.tag', json_group_array(t.tag))"
                                "   END"
                                " END"
                                " FROM document"
@@ -619,6 +478,28 @@ void database_write_thumbnail(database_t *db, const char *id, int num, void *dat
 }
 
 
+//void database_create_fts_index(database_t *db, database_t *fts_db) {
+//    // In a separate file,
+//
+//    // use database_initialize() to create FTS schema
+//    // if --force-reset, then truncate the tables first
+//
+//    /*
+//     * create/append fts table
+//     *
+//     * create/append scalar index table with
+//     *  id,index,size,mtime,mime
+//     *
+//     * create/append path index table with
+//     *  index,path,depth
+//     *
+//     * content table is a view with SELECT UNION for all attached tables
+//     *  random_seed column
+//     */
+//
+//    // INSERT INTO ft(ft) VALUES('optimize');
+//}
+
 job_t *database_get_work(database_t *db, job_type_t job_type) {
     job_t *job;
 
@@ -700,33 +581,18 @@ void database_add_work(database_t *db, job_t *job) {
             ret = sqlite3_step(db->insert_parse_job_stmt);
 
             if (ret == SQLITE_FULL) {
-                sqlite3_reset(db->insert_parse_job_stmt);
-                pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
                 usleep(1000000);
-                pthread_mutex_lock(&db->ipc_ctx->db_mutex);
-                continue;
             } else {
                 CRASH_IF_STMT_FAIL(ret);
             }
 
-            ret = sqlite3_reset(db->insert_parse_job_stmt);
-            if (ret == SQLITE_FULL) {
-                pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
-                usleep(100000);
-                pthread_mutex_lock(&db->ipc_ctx->db_mutex);
-            } else if (ret != SQLITE_OK) {
-                LOG_FATALF("database.c", "sqlite3_reset returned error %d", ret);
-            }
-        } while (ret != SQLITE_DONE && ret != SQLITE_OK);
+            CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->insert_parse_job_stmt));
+        } while (ret != SQLITE_DONE);
     } else if (job->type == JOB_BULK_LINE) {
         do {
             sqlite3_bind_text(db->insert_index_job_stmt, 1, job->bulk_line->doc_id, -1, SQLITE_STATIC);
             sqlite3_bind_int(db->insert_index_job_stmt, 2, job->bulk_line->type);
-            if (job->bulk_line->type != ES_BULK_LINE_DELETE) {
-                sqlite3_bind_text(db->insert_index_job_stmt, 3, job->bulk_line->line, -1, SQLITE_STATIC);
-            } else {
-                sqlite3_bind_null(db->insert_index_job_stmt, 3);
-            }
+            sqlite3_bind_text(db->insert_index_job_stmt, 3, job->bulk_line->line, -1, SQLITE_STATIC);
 
             ret = sqlite3_step(db->insert_index_job_stmt);
 
@@ -745,8 +611,6 @@ void database_add_work(database_t *db, job_t *job) {
                 pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
                 usleep(100000);
                 pthread_mutex_lock(&db->ipc_ctx->db_mutex);
-            } else if (ret != SQLITE_OK) {
-                LOG_FATALF("database.c", "sqlite3_reset returned error %d", ret);
             }
 
         } while (ret != SQLITE_DONE && ret != SQLITE_OK);
@@ -760,39 +624,3 @@ void database_add_work(database_t *db, job_t *job) {
     pthread_cond_signal(&db->ipc_ctx->has_work_cond);
     pthread_mutex_unlock(&db->ipc_ctx->mutex);
 }
-
-void database_write_tag(database_t *db, char *doc_id, char *tag) {
-    sqlite3_bind_text(db->write_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
-    sqlite3_bind_text(db->write_tag_stmt, 2, tag, -1, SQLITE_STATIC);
-
-    CRASH_IF_STMT_FAIL(sqlite3_step(db->write_tag_stmt));
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->write_tag_stmt));
-}
-
-void database_delete_tag(database_t *db, char *doc_id, char *tag) {
-    sqlite3_bind_text(db->delete_tag_stmt, 1, doc_id, -1, SQLITE_STATIC);
-    sqlite3_bind_text(db->delete_tag_stmt, 2, tag, -1, SQLITE_STATIC);
-
-    CRASH_IF_STMT_FAIL(sqlite3_step(db->delete_tag_stmt));
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->delete_tag_stmt));
-}
-
-cJSON *database_get_document(database_t *db, char *doc_id) {
-    sqlite3_bind_text(db->get_document, 1, doc_id, -1, SQLITE_STATIC);
-
-    int ret = sqlite3_step(db->get_document);
-    CRASH_IF_STMT_FAIL(ret);
-
-    cJSON *json;
-
-    if (ret == SQLITE_ROW) {
-        const char *json_str = sqlite3_column_text(db->get_document, 0);
-        json = cJSON_Parse(json_str);
-    } else {
-        json = NULL;
-    }
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->get_document));
-
-    return json;
-}

src/database/database.h

@@ -10,7 +10,6 @@ typedef struct index_descriptor index_descriptor_t;
 
 extern const char *IpcDatabaseSchema;
 extern const char *IndexDatabaseSchema;
-extern const char *FtsDatabaseSchema;
 
 typedef enum {
     INDEX_DATABASE,
@@ -19,30 +18,12 @@ typedef enum {
     FTS_DATABASE
 } database_type_t;
 
-typedef enum {
-    DATABASE_STAT_INVALID,
-    DATABASE_STAT_TREEMAP,
-    DATABASE_STAT_MIME_AGG,
-    DATABASE_STAT_SIZE_AGG,
-    DATABASE_STAT_DATE_AGG,
-} database_stat_type_d;
-
 typedef enum {
     JOB_UNDEFINED,
     JOB_BULK_LINE,
     JOB_PARSE_JOB
 } job_type_t;
 
-typedef enum {
-    FTS_SORT_INVALID,
-    FTS_SORT_SCORE,
-    FTS_SORT_SIZE,
-    FTS_SORT_MTIME,
-    FTS_SORT_RANDOM,
-    FTS_SORT_NAME,
-    FTS_SORT_ID,
-} fts_sort_t;
-
 typedef struct {
     job_type_t type;
     union {
@@ -63,11 +44,6 @@ typedef struct {
     char current_job[MAX_THREADS][PATH_MAX * 2];
 } database_ipc_ctx_t;
 
-typedef struct {
-    double date_min;
-    double date_max;
-} database_summary_stats_t;
-
 typedef struct database {
     char filename[PATH_MAX];
     database_type_t type;
@@ -82,27 +58,12 @@ typedef struct database {
     sqlite3_stmt *write_document_stmt;
     sqlite3_stmt *write_document_sidecar_stmt;
     sqlite3_stmt *write_thumbnail_stmt;
-    sqlite3_stmt *get_document;
-
-    sqlite3_stmt *delete_tag_stmt;
-    sqlite3_stmt *write_tag_stmt;
 
     sqlite3_stmt *insert_parse_job_stmt;
     sqlite3_stmt *insert_index_job_stmt;
     sqlite3_stmt *pop_parse_job_stmt;
     sqlite3_stmt *pop_index_job_stmt;
 
-    sqlite3_stmt *fts_search_paths;
-    sqlite3_stmt *fts_search_paths_w_prefix;
-    sqlite3_stmt *fts_suggest_paths;
-    sqlite3_stmt *fts_date_range;
-    sqlite3_stmt *fts_get_mimetypes;
-    sqlite3_stmt *fts_get_document;
-    sqlite3_stmt *fts_suggest_tag;
-    sqlite3_stmt *fts_get_tags;
-
-    char **tag_array;
-
     database_ipc_ctx_t *ipc_ctx;
 } database_t;
 
@@ -117,6 +78,8 @@ typedef struct {
     long size;
 } treemap_row_t;
 
+static treemap_row_t null_treemap_row = {0, 0, 0};
+
 
 database_t *database_create(const char *filename, database_type_t type);
 
@@ -141,14 +104,14 @@ database_iterator_t *database_create_document_iterator(database_t *db);
 cJSON *database_document_iter(database_iterator_t *);
 
 #define database_document_iter_foreach(element, iter) \
-    for (cJSON *(element) = database_document_iter(iter); (element) != NULL; (element) = database_document_iter(iter))
+    for (cJSON *element = database_document_iter(iter); element != NULL; element = database_document_iter(iter))
 
 database_iterator_t *database_create_delete_list_iterator(database_t *db);
 
-char *database_delete_list_iter(database_iterator_t *iter);
+char * database_delete_list_iter(database_iterator_t *iter);
 
 #define database_delete_list_iter_foreach(element, iter) \
-    for (char *(element) = database_delete_list_iter(iter); (element) != NULL; (element) = database_delete_list_iter(iter))
+    for (char *element = database_delete_list_iter(iter); element != NULL; element = database_delete_list_iter(iter))
 
 
 cJSON *database_incremental_scan_begin(database_t *db);
@@ -164,66 +127,29 @@ database_iterator_t *database_create_treemap_iterator(database_t *db, long thres
 treemap_row_t database_treemap_iter(database_iterator_t *iter);
 
 #define database_treemap_iter_foreach(element, iter) \
-    for (treemap_row_t element = database_treemap_iter(iter); (element).path != NULL; (element) = database_treemap_iter(iter))
+    for (treemap_row_t element = database_treemap_iter(iter); element.path != NULL; element = database_treemap_iter(iter))
 
 
 void database_generate_stats(database_t *db, double treemap_threshold);
 
-database_stat_type_d database_get_stat_type_by_mnemonic(const char *name);
-
 job_t *database_get_work(database_t *db, job_type_t job_type);
 
 void database_add_work(database_t *db, job_t *job);
 
-cJSON *database_get_stats(database_t *db, database_stat_type_d type);
+//void database_index(database_t *db);
 
 #define CRASH_IF_STMT_FAIL(x) do { \
         int return_value = x;                \
         if (return_value != SQLITE_DONE && return_value != SQLITE_ROW) {     \
-            LOG_FATALF("database.c", "Sqlite error @ %s:%d : (%d) %s", __BASE_FILE__, __LINE__, return_value, sqlite3_errmsg(db->db)); \
+            LOG_FATALF("database.c", "Sqlite error @ database.c:%d : (%d) %s", __LINE__, return_value, sqlite3_errmsg(db->db)); \
         }                           \
     } while (0)
 
 #define CRASH_IF_NOT_SQLITE_OK(x) do { \
         int return_value = x;                \
         if (return_value != SQLITE_OK) {     \
-            LOG_FATALF("database.c", "Sqlite error @ %s:%d : (%d) %s", __BASE_FILE__, __LINE__, return_value, sqlite3_errmsg(db->db)); \
+            LOG_FATALF("database.c", "Sqlite error @ database.c:%d : (%d) %s", __LINE__, return_value, sqlite3_errmsg(db->db)); \
         }                           \
     } while (0)
 
-void database_fts_attach(database_t *db, const char *fts_database_path);
-
-void database_fts_index(database_t *db);
-
-void database_fts_optimize(database_t *db);
-
-cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
-                              int suggest);
-
-cJSON *database_fts_get_mimetypes(database_t *db);
-
-database_summary_stats_t database_fts_get_date_range(database_t *db);
-
-cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
-                           long size_max, long date_min, long date_max, int page_size,
-                           char **index_ids, char **mime_types, char **tags, int sort_asc,
-                           fts_sort_t sort, int seed, char **after, int fetch_aggregations,
-                           int highlight, int highlight_context_size);
-
-void database_write_tag(database_t *db, char *doc_id, char *tag);
-
-void database_delete_tag(database_t *db, char *doc_id, char *tag);
-
-void database_fts_detach(database_t *db);
-
-cJSON *database_fts_get_document(database_t *db, char *doc_id);
-
-database_summary_stats_t database_fts_sync_tags(database_t *db);
-
-cJSON *database_fts_suggest_tag(database_t *db, char *prefix);
-
-cJSON *database_fts_get_tags(database_t *db);
-
-cJSON *database_get_document(database_t *db, char *doc_id);
-
-#endif
+#endif //SIST2_DATABASE_H

src/database/database_fts.c

@@ -1,804 +0,0 @@
-#include "database.h"
-#include "src/ctx.h"
-
-void database_fts_detach(database_t *db) {
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db, "DETACH DATABASE fts",
-            NULL, NULL, NULL
-    ));
-}
-
-void database_fts_attach(database_t *db, const char *fts_database_path) {
-
-    LOG_DEBUGF("database_fts.c", "Attaching to %s", fts_database_path);
-
-    sqlite3_stmt *stmt;
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-            db->db, "ATTACH DATABASE ? AS fts"
-                    "", -1, &stmt, NULL));
-
-    sqlite3_bind_text(stmt, 1, fts_database_path, -1, SQLITE_STATIC);
-
-    CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
-    sqlite3_finalize(stmt);
-}
-
-int database_fts_get_max_path_depth(database_t *db) {
-    sqlite3_stmt *stmt;
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-            db->db, "SELECT MAX(depth) FROM path_tmp", -1, &stmt, NULL));
-    CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
-
-    int max_depth = sqlite3_column_int(stmt, 0);
-    sqlite3_finalize(stmt);
-
-    return max_depth;
-}
-
-void database_fts_index(database_t *db) {
-
-    LOG_INFO("database_fts.c", "Creating content table.");
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "WITH docs AS ("
-            " SELECT document.id as id, (SELECT id FROM descriptor) as index_id, size,"
-            "  document.json_data ->> 'name' as name,"
-            "  document.json_data ->> 'path' as path,"
-            "  mtime,"
-            "  document.json_data ->> 'mime' as mime,"
-            "  CASE"
-            "   WHEN sc.json_data IS NULL THEN"
-            "    json_set(document.json_data, "
-            "             '$._id',document.id,"
-            "             '$.size',document.size, "
-            "             '$.mtime',document.mtime)"
-            "    ELSE json_patch("
-            "             json_set(document.json_data,"
-            "                      '$._id',document.id,"
-            "                      '$.size',document.size,"
-            "                      '$.mtime', document.mtime),"
-            "             sc.json_data) END"
-            " FROM document"
-            "  LEFT JOIN document_sidecar sc ON document.id = sc.id"
-            " GROUP BY document.id)"
-            " INSERT"
-            " INTO fts.document_index (id, index_id, size, name, path, mtime, mime, json_data)"
-            " SELECT * FROM docs WHERE true"
-            " on conflict (id, index_id) do update set "
-            "  size=excluded.size, mtime=excluded.mtime, mime=excluded.mime, json_data=excluded.json_data;",
-            NULL, NULL, NULL));
-
-    LOG_DEBUG("database_fts.c", "Deleting old documents.");
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "DELETE FROM fts.document_index"
-            " WHERE id IN (SELECT id FROM delete_list)"
-            "  AND index_id = (SELECT id FROM descriptor);",
-            NULL, NULL, NULL));
-
-    LOG_DEBUG("database_fts.c", "Generating summary stats");
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "DELETE FROM fts.stats", NULL, NULL, NULL));
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db, "INSERT INTO fts.stats "
-                    "SELECT min(mtime), max(mtime) FROM fts.document_index",
-            NULL, NULL, NULL));
-
-    LOG_DEBUG("database_fts.c", "Generating mime index");
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db, "DELETE FROM fts.mime_index;", NULL, NULL, NULL));
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db, "INSERT INTO fts.mime_index (index_id, mime, count) "
-                    "SELECT index_id, mime, count(*) FROM fts.document_index GROUP BY index_id, mime",
-            NULL, NULL, NULL));
-
-    LOG_DEBUG("database_fts.c", "Generating path index");
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "CREATE TEMP TABLE path_tmp ("
-            " path TEXT,"
-            " index_id TEXT,"
-            " count INTEGER NOT NULL,"
-            " depth INTEGER NOT NULL,"
-            " children INTEGER NOT NULL DEFAULT(0),"
-            " total INTEGER AS (count + children),"
-            " PRIMARY KEY (path, index_id)"
-            ");", NULL, NULL, NULL));
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "INSERT INTO path_tmp (path, index_id, count, depth)"
-            " SELECT path, index_id, count(*), CASE WHEN length(json_data->>'path') == 0 THEN 0"
-            " ELSE 1 + length(json_data->>'path') - length(REPLACE(json_data->>'path', '/', ''))"
-            " END as depth FROM document_index WHERE depth > 0"
-            " GROUP BY path", NULL, NULL, NULL));
-
-    int max_depth = database_fts_get_max_path_depth(db);
-
-    for (int i = max_depth; i > 1; i--) {
-        sqlite3_stmt *stmt;
-
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                db->db,
-                "INSERT INTO path_tmp (path, index_id, children, depth, count)"
-                " SELECT path_parent(path) parent, index_id, (SELECT COALESCE(sum(count), 0) FROM path_tmp WHERE path "
-                " BETWEEN path_parent(p.path) || '/' AND path_parent(p.path) || '/𘚟' AND index_id = p.index_id) as cnt, depth-1, 0 "
-                " FROM path_tmp p WHERE depth=? GROUP BY parent"
-                " ON CONFLICT(path, index_id) DO UPDATE SET children=excluded.children",
-                -1, &stmt, NULL));
-        sqlite3_bind_int(stmt, 1, i);
-        CRASH_IF_STMT_FAIL(sqlite3_step(stmt));
-
-        LOG_DEBUGF("database_fts.c", "Path index depth %d (%d)", i, sqlite3_changes(db->db));
-    }
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "DELETE FROM path_index;"
-            "INSERT INTO path_index (path, index_id, count, depth) SELECT path, index_id, total, depth FROM path_tmp",
-            NULL, NULL, NULL));
-
-    LOG_DEBUG("database_fts.c", "Generating search index.");
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db, "INSERT INTO search(search) VALUES ('delete-all')",
-            NULL, NULL, NULL));
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "INSERT INTO search(rowid, name, content, title) SELECT id, name, content, title from document_view",
-            NULL, NULL, NULL));
-}
-
-void database_fts_optimize(database_t *db) {
-    LOG_INFO("database_fts.c", "Optimizing search index.");
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "INSERT INTO search(search) VALUES('optimize');",
-            NULL, NULL, NULL));
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA fts.optimize;", NULL, NULL, NULL));
-}
-
-cJSON *database_fts_get_paths(database_t *db, const char *index_id, int depth_min, int depth_max, const char *prefix,
-                              int suggest) {
-
-    sqlite3_stmt *stmt;
-
-    if (suggest) {
-        stmt = db->fts_suggest_paths;
-        sqlite3_bind_int(stmt, 1, depth_min);
-        sqlite3_bind_int(stmt, 2, depth_max);
-
-        if (prefix) {
-            char *prefix_glob = malloc(strlen(prefix) + 2);
-            sprintf(prefix_glob, "%s*", prefix);
-            sqlite3_bind_text(stmt, 3, prefix_glob, -1, SQLITE_TRANSIENT);
-            free(prefix_glob);
-        }
-
-    } else if (prefix) {
-        stmt = db->fts_search_paths_w_prefix;
-        if (index_id) {
-            sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
-        } else {
-            sqlite3_bind_null(stmt, 1);
-        }
-        sqlite3_bind_int(stmt, 2, depth_min);
-        sqlite3_bind_int(stmt, 3, depth_max);
-
-        char *prefix_glob = malloc(strlen(prefix) + 3);
-        sprintf(prefix_glob, "%s/*", prefix);
-        sqlite3_bind_text(stmt, 4, prefix, -1, SQLITE_STATIC);
-        sqlite3_bind_text(stmt, 5, prefix_glob, -1, SQLITE_TRANSIENT);
-        free(prefix_glob);
-    } else {
-        stmt = db->fts_search_paths;
-        if (index_id) {
-            sqlite3_bind_text(stmt, 1, index_id, -1, SQLITE_STATIC);
-        } else {
-            sqlite3_bind_null(stmt, 1);
-        }
-        sqlite3_bind_int(stmt, 2, depth_min);
-        sqlite3_bind_int(stmt, 3, depth_max);
-    }
-
-    cJSON *json = cJSON_CreateArray();
-
-    int ret;
-    do {
-        ret = sqlite3_step(stmt);
-        CRASH_IF_STMT_FAIL(ret);
-
-        if (ret == SQLITE_DONE) {
-            break;
-        }
-
-        cJSON *row = cJSON_CreateObject();
-
-        cJSON_AddStringToObject(row, "path", (const char *) sqlite3_column_text(stmt, 0));
-        cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 1));
-
-        cJSON_AddItemToArray(json, row);
-    } while (TRUE);
-
-    sqlite3_reset(stmt);
-
-    return json;
-}
-
-cJSON *database_fts_get_mimetypes(database_t *db) {
-
-    cJSON *json = cJSON_CreateArray();
-
-    int ret;
-    do {
-        ret = sqlite3_step(db->fts_get_mimetypes);
-        CRASH_IF_STMT_FAIL(ret);
-
-        if (ret == SQLITE_DONE) {
-            break;
-        }
-
-        cJSON *row = cJSON_CreateObject();
-
-        cJSON_AddStringToObject(row, "mime", (const char *) sqlite3_column_text(db->fts_get_mimetypes, 0));
-        cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(db->fts_get_mimetypes, 1));
-
-        cJSON_AddItemToArray(json, row);
-    } while (TRUE);
-
-    sqlite3_reset(db->fts_get_mimetypes);
-
-    return json;
-}
-
-const char *size_where_clause(long size_min, long size_max) {
-    if (size_min > 0 && size_max > 0) {
-        return "size BETWEEN @size_min AND @size_max";
-    } else if (size_min > 0) {
-        return "size >= @size_min";
-    } else if (size_max > 0) {
-        return "size <= @size_max";
-    }
-
-    return NULL;
-}
-
-const char *date_where_clause(long date_min, long date_max) {
-    if (date_min > 0 && date_max > 0) {
-        return "mtime BETWEEN @date_min AND @date_max";
-    } else if (date_min > 0) {
-        return "mtime >= @date_min";
-    } else if (date_max > 0) {
-        return "mtime <= @date_max";
-    }
-
-    return NULL;
-}
-
-int array_length(char **arr) {
-
-    if (arr == NULL) {
-        return 0;
-    }
-
-    int count = -1;
-    while (arr[++count] != NULL);
-
-    return count;
-}
-
-#define INDEX_ID_PARAM_OFFSET (10)
-#define MIME_PARAM_OFFSET (INDEX_ID_PARAM_OFFSET + 1000)
-
-char *build_where_clause(const char *path_where, const char *size_where, const char *date_where,
-                         const char *index_id_where, const char *mime_where, const char *query_where,
-                         const char *after_where, const char *tags_where) {
-    char *where = calloc(
-            strlen(index_id_where)
-            + (query_where ? strlen(query_where) + sizeof(" AND ") : 0)
-            + (path_where ? strlen(path_where) + sizeof(" AND ") : 0)
-            + (size_where ? strlen(size_where) + sizeof(" AND ") : 0)
-            + (date_where ? strlen(date_where) + sizeof(" AND ") : 0)
-            + (after_where ? strlen(after_where) + sizeof(" AND ") : 0)
-            + (tags_where ? strlen(tags_where) + sizeof(" AND ") : 0)
-            + (mime_where ? strlen(mime_where) + sizeof(" AND ") : 0) + 1,
-            sizeof(char)
-    );
-
-    strcat(where, index_id_where);
-    if (query_where) {
-        strcat(where, " AND ");
-        strcat(where, query_where);
-    }
-    if (path_where) {
-        strcat(where, " AND ");
-        strcat(where, path_where);
-    }
-    if (size_where) {
-        strcat(where, " AND ");
-        strcat(where, size_where);
-    }
-    if (date_where) {
-        strcat(where, " AND ");
-        strcat(where, date_where);
-    }
-    if (mime_where) {
-        strcat(where, " AND ");
-        strcat(where, mime_where);
-    }
-    if (after_where) {
-        strcat(where, " AND ");
-        strcat(where, after_where);
-    }
-    if (tags_where) {
-        strcat(where, " AND ");
-        strcat(where, tags_where);
-    }
-    return where;
-}
-
-char *index_ids_where_clause(char **index_ids) {
-    int param_count = array_length(index_ids);
-
-    char *clause = malloc(13 + 2 + 6 * param_count);
-
-    strcpy(clause, "index_id IN (");
-    for (int i = 0; i < param_count; i++) {
-        char param[10];
-        snprintf(param, sizeof(param), "?%d%s",
-                 INDEX_ID_PARAM_OFFSET + i, i == param_count - 1 ? "" : ",");
-        strcat(clause, param);
-    }
-    strcat(clause, ")");
-
-    return clause;
-}
-
-char *mime_types_where_clause(char **mime_types) {
-    int param_count = array_length(mime_types);
-
-    if (param_count == 0) {
-        return NULL;
-    }
-
-    char *clause = malloc(9 + 2 + 6 * param_count);
-
-    strcpy(clause, "mime IN (");
-    for (int i = 0; i < param_count; i++) {
-        char param[10];
-        snprintf(param, sizeof(param), "?%d%s",
-                 MIME_PARAM_OFFSET + i, i == param_count - 1 ? "" : ",");
-        strcat(clause, param);
-    }
-    strcat(clause, ")");
-
-    return clause;
-}
-
-const char *path_where_clause(const char *path) {
-    if (path == NULL || strlen(path) == 0) {
-        return NULL;
-    }
-
-    return "(path = @path or path GLOB @path_glob)";
-}
-
-const char *get_sort_var(fts_sort_t sort) {
-
-    switch (sort) {
-        case FTS_SORT_SCORE:
-            // Round to 14 decimal places to avoid precision problems when converting to JSON...
-            return "round(rank, 14)";
-        case FTS_SORT_SIZE:
-            return "size";
-        case FTS_SORT_MTIME:
-            return "mtime";
-        case FTS_SORT_RANDOM:
-            return "random_seeded(doc.ROWID + ?5)";
-        case FTS_SORT_NAME:
-            return "doc.name";
-        case FTS_SORT_ID:
-            return "doc.id";
-        default:
-            return NULL;
-    }
-}
-
-const char *match_where(const char *query) {
-    if (query == NULL || strlen(query) == 0) {
-        return NULL;
-    } else {
-        return "search MATCH ?1";
-    }
-}
-
-char *tags_where_clause(char **tags) {
-    if (tags == NULL) {
-        return NULL;
-    }
-
-    return "EXISTS (SELECT 1 FROM tag WHERE id=doc.id AND tag_matches(tag))";
-}
-
-database_summary_stats_t database_fts_get_date_range(database_t *db) {
-
-    int ret = sqlite3_step(db->fts_date_range);
-    CRASH_IF_STMT_FAIL(ret);
-
-    if (ret == SQLITE_DONE) {
-        return (database_summary_stats_t) {0, 0};
-    }
-
-    database_summary_stats_t stats;
-    stats.date_min = (double) sqlite3_column_int64(db->fts_date_range, 0);
-    stats.date_max = (double) sqlite3_column_int64(db->fts_date_range, 1);
-
-    sqlite3_reset(db->fts_date_range);
-
-    return stats;
-}
-
-char *get_after_where(char **after, fts_sort_t sort) {
-    if (after == NULL) {
-        return NULL;
-    }
-
-    return "(sort_var, doc.ROWID) > (?3, ?4)";
-}
-
-cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
-                           long size_max, long date_min, long date_max, int page_size,
-                           char **index_ids, char **mime_types, char **tags, int sort_asc,
-                           fts_sort_t sort, int seed, char **after, int fetch_aggregations,
-                           int highlight, int highlight_context_size) {
-
-    char path_glob[PATH_MAX * 2];
-    snprintf(path_glob, sizeof(path_glob), "%s/*", path);
-    const char *path_where = path_where_clause(path);
-    const char *size_where = size_where_clause(size_min, size_max);
-    const char *date_where = date_where_clause(date_min, date_max);
-    const char *index_id_where = index_ids_where_clause(index_ids);
-    const char *mime_where = mime_types_where_clause(mime_types);
-    const char *query_where = match_where(query);
-    const char *after_where = get_after_where(after, sort);
-    const char *tags_where = tags_where_clause(tags);
-
-    if (!query_where && sort == FTS_SORT_SCORE) {
-        // If query is NULL, then sort by id instead
-        sort = FTS_SORT_ID;
-    }
-
-    char *agg_where;
-    char *where = build_where_clause(path_where, size_where, date_where, index_id_where, mime_where, query_where,
-                                     after_where, tags_where);
-    if (fetch_aggregations) {
-        agg_where = build_where_clause(path_where, size_where, date_where, index_id_where, mime_where, query_where,
-                                       NULL, tags_where);
-    }
-
-    const char *json_object_sql;
-    if (highlight && query_where != NULL) {
-        json_object_sql = "json_remove(json_set(doc.json_data,"
-                          "'$.index', doc.index_id,"
-                          "'$._highlight.name', snippet(search, 0, '<mark>', '</mark>', '', ?6),"
-                          "'$._highlight.content', snippet(search, 1, '<mark>', '</mark>', '', ?6)),"
-                          "'$.content')";
-    } else {
-        json_object_sql = "json_remove(json_set(doc.json_data,"
-                          "'$.index', doc.index_id),"
-                          "'$.content')";
-    }
-
-    char *sql;
-    char *agg_sql;
-
-    if (query_where) {
-        asprintf(
-                &sql,
-                "SELECT"
-                " %s, %s as sort_var, doc.ROWID"
-                " FROM search"
-                " INNER JOIN document_index doc on doc.ROWID = search.ROWID"
-                " WHERE %s"
-                " ORDER BY sort_var%s, doc.ROWID"
-                " LIMIT ?2",
-                json_object_sql, get_sort_var(sort),
-                where,
-                sort_asc ? "" : "DESC");
-
-        if (fetch_aggregations) {
-            asprintf(&agg_sql,
-                     "SELECT count(*), sum(size)"
-                     " FROM search"
-                     "  INNER JOIN document_index doc on doc.ROWID = search.ROWID"
-                     " WHERE search MATCH ?1"
-                     " AND %s", agg_where);
-        }
-    } else {
-        asprintf(
-                &sql,
-                "SELECT"
-                " %s, %s as sort_var, doc.ROWID"
-                " FROM document_index doc"
-                " WHERE %s"
-                " ORDER BY sort_var%s,doc.ROWID"
-                " LIMIT ?2",
-                json_object_sql, get_sort_var(sort),
-                where,
-                sort_asc ? "" : " DESC");
-
-        if (fetch_aggregations) {
-            asprintf(&agg_sql,
-                     "SELECT count(*), sum(size)"
-                     " FROM document_index doc"
-                     " WHERE %s", agg_where);
-        }
-    }
-
-    sqlite3_stmt *stmt;
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(db->db, sql, -1, &stmt, NULL));
-
-    if (query_where) {
-        sqlite3_bind_text(stmt, 1, query, -1, SQLITE_STATIC);
-    }
-    sqlite3_bind_int(stmt, 2, page_size);
-
-    if (index_ids) {
-        array_foreach(index_ids) {
-            sqlite3_bind_text(stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
-        }
-    }
-    if (mime_types) {
-        array_foreach(mime_types) {
-            sqlite3_bind_text(stmt, MIME_PARAM_OFFSET + i, mime_types[i], -1, SQLITE_STATIC);
-        }
-    }
-    if (tags) {
-        db->tag_array = tags;
-    }
-
-    if (size_min > 0) {
-        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@size_min"), size_min);
-    }
-    if (size_max > 0) {
-        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@size_max"), size_max);
-    }
-    if (date_min > 0) {
-        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@date_min"), date_min);
-    }
-    if (date_max > 0) {
-        sqlite3_bind_int64(stmt, sqlite3_bind_parameter_index(stmt, "@date_max"), date_max);
-    }
-    if (path_where) {
-        sqlite3_bind_text(stmt, sqlite3_bind_parameter_index(stmt, "@path"), path, -1, SQLITE_STATIC);
-        sqlite3_bind_text(stmt, sqlite3_bind_parameter_index(stmt, "@path_glob"), path_glob, -1, SQLITE_STATIC);
-    }
-    if (after_where) {
-        if (sort == FTS_SORT_NAME || sort == FTS_SORT_ID) {
-            sqlite3_bind_text(stmt, 3, after[0], -1, SQLITE_STATIC);
-        } else if (sort == FTS_SORT_SCORE) {
-            sqlite3_bind_double(stmt, 3, strtod(after[0], NULL));
-        } else {
-            sqlite3_bind_int64(stmt, 3, strtol(after[0], NULL, 10));
-        }
-        sqlite3_bind_int64(stmt, 4, strtol(after[1], NULL, 10));
-    }
-    if (sort == FTS_SORT_RANDOM) {
-        sqlite3_bind_int(stmt, 5, seed);
-    }
-    if (highlight) {
-        sqlite3_bind_int(stmt, 6, highlight_context_size);
-    }
-
-    cJSON *json = cJSON_CreateObject();
-    cJSON *hits_hits = cJSON_CreateArray();
-
-    int ret;
-    do {
-        ret = sqlite3_step(stmt);
-        if (ret != SQLITE_DONE && ret != SQLITE_ROW) {
-            break;
-        }
-
-        if (ret == SQLITE_DONE) {
-            break;
-        }
-
-        const char *json_str = (const char *) sqlite3_column_text(stmt, 0);
-        cJSON *row = cJSON_CreateObject();
-        cJSON *source = cJSON_Parse(json_str);
-        if (highlight) {
-            cJSON *hl = cJSON_DetachItemFromObject(source, "_highlight");
-            cJSON_AddItemToObject(row, "highlight", hl);
-        }
-        cJSON *id = cJSON_DetachItemFromObject(source, "_id");
-        cJSON_AddItemToObject(row, "_id", id);
-        cJSON_AddItemToObject(row, "_source", source);
-
-        cJSON *sort_info = cJSON_AddArrayToObject(row, "sort");
-        cJSON_AddItemToArray(
-                sort_info,
-                cJSON_CreateString((char *) sqlite3_column_text(stmt, 1))
-        );
-        cJSON_AddItemToArray(
-                sort_info,
-                cJSON_CreateString((char *) sqlite3_column_text(stmt, 2))
-        );
-
-        cJSON_AddItemToArray(hits_hits, row);
-    } while (TRUE);
-
-    sqlite3_finalize(stmt);
-
-    cJSON *hits = cJSON_AddObjectToObject(json, "hits");
-    cJSON_AddItemToObject(hits, "hits", hits_hits);
-
-    // Aggregations
-    if (fetch_aggregations) {
-
-        sqlite3_stmt *agg_stmt;
-        CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(db->db, agg_sql, -1, &agg_stmt, NULL));
-
-        if (index_ids) {
-            array_foreach(index_ids) {
-                sqlite3_bind_text(agg_stmt, INDEX_ID_PARAM_OFFSET + i, index_ids[i], -1, SQLITE_STATIC);
-            }
-        }
-        if (mime_types) {
-            array_foreach(mime_types) {
-                sqlite3_bind_text(agg_stmt, MIME_PARAM_OFFSET + i, mime_types[i], -1, SQLITE_STATIC);
-            }
-        }
-
-        if (query_where) {
-            sqlite3_bind_text(agg_stmt, 1, query, -1, SQLITE_STATIC);
-        }
-        if (size_min > 0) {
-            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@size_min"), size_min);
-        }
-        if (size_max > 0) {
-            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@size_max"), size_max);
-        }
-        if (date_min > 0) {
-            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@date_min"), date_min);
-        }
-        if (date_max > 0) {
-            sqlite3_bind_int64(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@date_max"), date_max);
-        }
-        if (path_where) {
-            sqlite3_bind_text(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@path"), path, -1, SQLITE_STATIC);
-            sqlite3_bind_text(agg_stmt, sqlite3_bind_parameter_index(agg_stmt, "@path_glob"), path_glob, -1,
-                              SQLITE_STATIC);
-        }
-
-        int agg_ret = sqlite3_step(agg_stmt);
-
-        if (agg_ret == SQLITE_ROW) {
-            cJSON *aggregations = cJSON_AddObjectToObject(json, "aggregations");
-            cJSON *total_count = cJSON_AddObjectToObject(aggregations, "total_count");
-            cJSON_AddNumberToObject(total_count, "value", sqlite3_column_double(agg_stmt, 0));
-            cJSON *total_size = cJSON_AddObjectToObject(aggregations, "total_size");
-            cJSON_AddNumberToObject(total_size, "value", sqlite3_column_double(agg_stmt, 1));
-        } else {
-            cJSON *aggregations = cJSON_AddObjectToObject(json, "aggregations");
-            cJSON *total_count = cJSON_AddObjectToObject(aggregations, "total_count");
-            cJSON_AddNumberToObject(total_count, "value", 0);
-            cJSON *total_size = cJSON_AddObjectToObject(aggregations, "total_size");
-            cJSON_AddNumberToObject(total_size, "value", 0);
-        }
-        sqlite3_finalize(agg_stmt);
-    }
-
-    // Cleanup
-    if (index_id_where) {
-        free(index_id_where);
-    }
-    if (mime_where) {
-        free(mime_where);
-    }
-    free(where);
-    free(sql);
-    if (fetch_aggregations) {
-        free(agg_where);
-        free(agg_sql);
-    }
-
-    return json;
-}
-
-database_summary_stats_t database_fts_sync_tags(database_t *db) {
-
-    LOG_INFO("database_fts.c", "Syncing tags.");
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "DELETE FROM fts.tag WHERE"
-            " (id, tag) NOT IN (SELECT id, tag FROM tag)",
-            NULL, NULL, NULL));
-
-    CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
-            db->db,
-            "INSERT INTO fts.tag (id, tag) "
-            " SELECT id, tag FROM tag "
-            " WHERE (id, tag) NOT IN (SELECT * FROM fts.tag)",
-            NULL, NULL, NULL));
-}
-
-cJSON *database_fts_get_document(database_t *db, char *doc_id) {
-    sqlite3_bind_text(db->fts_get_document, 1, doc_id, -1, NULL);
-
-    int ret = sqlite3_step(db->fts_get_document);
-    cJSON *json = NULL;
-
-    if (ret == SQLITE_ROW) {
-        const char *json_data = (const char *) sqlite3_column_text(db->fts_get_document, 0);
-        json = cJSON_Parse(json_data);
-    } else {
-        CRASH_IF_STMT_FAIL(ret);
-    }
-
-    sqlite3_reset(db->fts_get_document);
-
-    return json;
-}
-
-cJSON *database_fts_suggest_tag(database_t *db, char *prefix) {
-    sqlite3_bind_text(db->fts_suggest_tag, 1, prefix, -1, NULL);
-
-    cJSON *json = cJSON_CreateArray();
-
-    int ret;
-    do {
-        ret = sqlite3_step(db->fts_suggest_tag);
-        CRASH_IF_STMT_FAIL(ret);
-
-        if (ret == SQLITE_DONE) {
-            break;
-        }
-
-        cJSON_AddItemToArray(
-                json,
-                cJSON_CreateString((const char *) sqlite3_column_text(db->fts_suggest_tag, 0))
-        );
-
-    } while (TRUE);
-
-    sqlite3_reset(db->fts_suggest_tag);
-
-    return json;
-}
-
-
-cJSON *database_fts_get_tags(database_t *db) {
-    cJSON *json = cJSON_CreateArray();
-
-    int ret;
-    do {
-        ret = sqlite3_step(db->fts_get_tags);
-        CRASH_IF_STMT_FAIL(ret);
-
-        if (ret == SQLITE_DONE) {
-            break;
-        }
-
-        cJSON *row = cJSON_CreateObject();
-
-        cJSON_AddStringToObject(row, "tag", (const char *) sqlite3_column_text(db->fts_get_tags, 0));
-        cJSON_AddNumberToObject(row, "count", sqlite3_column_int(db->fts_get_tags, 1));
-
-        cJSON_AddItemToArray(json, row);
-    } while (TRUE);
-
-    sqlite3_reset(db->fts_get_tags);
-
-    return json;
-}

src/database/database_schema.c

@@ -1,77 +1,3 @@
-const char *FtsDatabaseSchema =
-        "CREATE TABLE IF NOT EXISTS document_index ("
-        "   id TEXT NOT NULL,"
-        "   index_id TEXT NOT NULL,"
-        "   size INTEGER NOT NULL,"
-        "   name TEXT NOT NULL,"
-        "   path TEXT NOT NULL,"
-        "   mtime INTEGER NOT NULL,"
-        "   mime TEXT,"
-        "   json_data TEXT NOT NULL,"
-        "   PRIMARY KEY (id, index_id)"
-        ");"
-        ""
-        "CREATE TABLE IF NOT EXISTS stats ("
-        "   mtime_min INTEGER,"
-        "   mtime_max INTEGER"
-        ");"
-        ""
-        "CREATE TABLE IF NOT EXISTS path_index ("
-        "   path TEXT,"
-        "   index_id TEXT,"
-        "   count INTEGER NOT NULL,"
-        "   depth INTEGER NOT NULL,"
-        "   PRIMARY KEY (path, index_id)"
-        ");"
-        ""
-        "CREATE TABLE IF NOT EXISTS mime_index ("
-        "   index_id TEXT,"
-        "   mime TEXT,"
-        "   count INT,"
-        "   PRIMARY KEY(index_id, mime)"
-        ");"
-        ""
-        "CREATE TABLE IF NOT EXISTS tag ("
-        "   id TEXT NOT NULL,"
-        "   tag TEXT NOT NULL,"
-        "   PRIMARY KEY (id, tag)"
-        ");"
-        "CREATE INDEX IF NOT EXISTS tag_tag_idx ON tag(tag);"
-        "CREATE INDEX IF NOT EXISTS tag_id_idx ON tag(id);"
-        "CREATE TRIGGER IF NOT EXISTS tag_write_trigger"
-        "    AFTER INSERT ON tag"
-        " BEGIN"
-        "  UPDATE document_index"
-        "  SET json_data = json_set(json_data, '$.tag', (SELECT json_group_array(tag) FROM tag WHERE id = NEW.id))"
-        "  WHERE id = NEW.id;"
-        " END;"
-        ""
-        "CREATE TRIGGER IF NOT EXISTS tag_delete_trigger"
-        " AFTER DELETE ON tag"
-        " BEGIN"
-        "  UPDATE document_index"
-        "  SET json_data = json_set(json_data, '$.tag', (SELECT json_group_array(tag) FROM tag WHERE id = OLD.id))"
-        "  WHERE id = OLD.id;"
-        " END;"
-        ""
-        "CREATE VIEW IF NOT EXISTS document_view (id, name, content, title)"
-        " AS"
-        " SELECT rowid,"
-        "  json_data->>'name',"
-        "  json_data->>'content',"
-        "  json_data->>'title'"
-        " FROM document_index;"
-        ""
-        "CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5 ("
-        "   name,"
-        "   content,"
-        "   title,"
-        "   content='document_view',"
-        "   content_rowid='id'"
-        ");"
-        // name^8, content^3, title^8
-        "INSERT INTO search(search, rank) VALUES('rank', 'bm25(8, 3, 8)');"
-        "";
 
 const char *IpcDatabaseSchema =
         "CREATE TABLE parse_job ("
@@ -110,8 +36,7 @@ const char *IndexDatabaseSchema =
         ""
         "CREATE TABLE tag ("
         "   id TEXT NOT NULL,"
-        "   tag TEXT NOT NULL,"
-        "   PRIMARY KEY (id, tag)"
+        "   tag TEXT NOT NULL"
         ");"
         ""
         "CREATE TABLE document_sidecar ("

src/database/database_stats.c

@@ -6,7 +6,6 @@
 #define SIZE_BUCKET (long)(5 * 1000 * 1000)
 #define DATE_BUCKET (long)(2629800) // ~30 days
 
-
 database_iterator_t *database_create_treemap_iterator(database_t *db, long threshold) {
 
     sqlite3_stmt *stmt;
@@ -158,87 +157,3 @@ void database_generate_stats(database_t *db, double treemap_threshold) {
     LOG_INFO("database.c", "Done!");
 }
 
-database_stat_type_d database_get_stat_type_by_mnemonic(const char *name) {
-    if (strcmp(name, "TMAP") == 0) {
-        return DATABASE_STAT_TREEMAP;
-    }
-    if (strcmp(name, "MAGG") == 0) {
-        return DATABASE_STAT_MIME_AGG;
-    }
-    if (strcmp(name, "SAGG") == 0) {
-        return DATABASE_STAT_SIZE_AGG;
-    }
-    if (strcmp(name, "DAGG") == 0) {
-        return DATABASE_STAT_DATE_AGG;
-    }
-
-    return DATABASE_STAT_INVALID;
-}
-
-cJSON *database_get_stats(database_t *db, database_stat_type_d type) {
-
-    sqlite3_stmt *stmt;
-
-    switch (type) {
-        case DATABASE_STAT_TREEMAP:
-            CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                    db->db, "SELECT path,size FROM stats_treemap", -1, &stmt, NULL
-            ));
-            break;
-        case DATABASE_STAT_DATE_AGG:
-            CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                    db->db, "SELECT bucket,count FROM stats_date_agg", -1, &stmt, NULL
-            ));
-            break;
-        case DATABASE_STAT_SIZE_AGG:
-            CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                    db->db, "SELECT bucket,count FROM stats_size_agg", -1, &stmt, NULL
-            ));
-            break;
-        case DATABASE_STAT_MIME_AGG:
-            CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
-                    db->db, "SELECT mime,size,count FROM stats_mime_agg", -1, &stmt, NULL
-            ));
-            break;
-        case DATABASE_STAT_INVALID:
-        default:
-        LOG_FATALF("database_stats.c", "Invalid stat type: %d", type);
-    }
-
-    cJSON *json = cJSON_CreateArray();
-
-    int ret;
-    do {
-        ret = sqlite3_step(stmt);
-        CRASH_IF_STMT_FAIL(ret);
-
-        if (ret == SQLITE_DONE) {
-            break;
-        }
-
-        cJSON *row = cJSON_CreateObject();
-
-        switch (type) {
-            case DATABASE_STAT_TREEMAP:
-                cJSON_AddStringToObject(row, "path", (const char *) sqlite3_column_text(stmt, 0));
-                cJSON_AddNumberToObject(row, "size", (double) sqlite3_column_int64(stmt, 1));
-                break;
-            case DATABASE_STAT_DATE_AGG:
-            case DATABASE_STAT_SIZE_AGG:
-                cJSON_AddNumberToObject(row, "bucket", (double) sqlite3_column_int64(stmt, 0));
-                cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 1));
-                break;
-            case DATABASE_STAT_MIME_AGG:
-                cJSON_AddStringToObject(row, "mime", (const char *) sqlite3_column_text(stmt, 0));
-                cJSON_AddNumberToObject(row, "size", (double) sqlite3_column_int64(stmt, 1));
-                cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 2));
-                break;
-        }
-
-        cJSON_AddItemToArray(json, row);
-    } while (TRUE);
-
-    sqlite3_finalize(stmt);
-
-    return json;
-}

src/database/database_stats.h

@@ -0,0 +1,5 @@
+#ifndef SIST2_DATABASE_STATS_H
+#define SIST2_DATABASE_STATS_H
+
+
+#endif //SIST2_DATABASE_STATS_H

src/index/elastic.c

@@ -64,16 +64,20 @@ void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) {
     cJSON_Delete(line);
 }
 
-void delete_document(const char *document_id) {
-    es_bulk_line_t bulk_line;
+void index_json_func(job_t *job) {
+    elastic_index_line(job->bulk_line);
+}
 
-    bulk_line.type = ES_BULK_LINE_DELETE;
-    bulk_line.next = NULL;
-    strcpy(bulk_line.doc_id, document_id);
+void delete_document(const char *document_id) {
+    es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t));
+
+    bulk_line->type = ES_BULK_LINE_DELETE;
+    bulk_line->next = NULL;
+    strcpy(bulk_line->doc_id, document_id);
 
     tpool_add_work(IndexCtx.pool, &(job_t) {
             .type = JOB_BULK_LINE,
-            .bulk_line = &bulk_line,
+            .bulk_line = bulk_line,
     });
 }
 
@@ -92,10 +96,9 @@ void index_json(cJSON *document, const char doc_id[SIST_DOC_ID_LEN]) {
 
     cJSON_free(json);
     tpool_add_work(IndexCtx.pool, &(job_t) {
-            .type = JOB_BULK_LINE,
-            .bulk_line = bulk_line,
+        .type = JOB_BULK_LINE,
+        .bulk_line = bulk_line,
     });
-    free(bulk_line);
 }
 
 void execute_update_script(const char *script, int async, const char index_id[SIST_INDEX_ID_LEN]) {
@@ -606,4 +609,4 @@ char *elastic_get_status() {
     free_response(r);
     cJSON_Delete(json);
     return status;
-}
+}

src/io/serialize.c

@@ -91,6 +91,8 @@ char *build_json_string(document_t *doc) {
     } else {
         cJSON_AddStringToObject(json, "mime", mime_text);
     }
+    cJSON_AddNumberToObject(json, "size", (double) doc->size);
+    cJSON_AddNumberToObject(json, "mtime", doc->mtime);
 
     // Ignore root directory in the file path
     doc->ext = (short) (doc->ext - ScanCtx.index.desc.root_len);
@@ -120,6 +122,8 @@ char *build_json_string(document_t *doc) {
         cJSON_AddStringToObject(json, "path", "");
     }
 
+    cJSON_AddStringToObject(json, "_id", doc->doc_id);
+
     // Metadata
     meta_line_t *meta = doc->meta_head;
     while (meta != NULL) {

src/main.c

@@ -22,7 +22,6 @@
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
-        "sist2 sqlite-index [OPTION]... INDEX",
         "sist2 web [OPTION]... INDEX...",
         "sist2 exec-script [OPTION]... INDEX",
         NULL,
@@ -196,10 +195,6 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.mobi_ctx.content_size = args->content_size;
     ScanCtx.mobi_ctx.log = log_callback;
     ScanCtx.mobi_ctx.logf = logf_callback;
-    ScanCtx.mobi_ctx.store = write_thumbnail_callback;
-    ScanCtx.mobi_ctx.enable_tn = args->tn_count > 0;
-    ScanCtx.mobi_ctx.tn_size = args->tn_size;
-    ScanCtx.mobi_ctx.tn_qscale = args->tn_quality;
 
     // TEXT
     ScanCtx.text_ctx.content_size = args->content_size;
@@ -317,20 +312,17 @@ void sist2_index(index_args_t *args) {
     database_open(db);
     database_iterator_t *iterator = database_create_document_iterator(db);
     database_document_iter_foreach(json, iterator) {
-        char doc_id[SIST_DOC_ID_LEN];
-        strcpy(doc_id, cJSON_GetObjectItem(json, "_id")->valuestring);
-        cJSON_DeleteItemFromObject(json, "_id");
-
+        const char *doc_id = cJSON_GetObjectItem(json, "_id")->valuestring;
         if (args->print) {
             print_json(json, doc_id);
         } else {
             index_json(json, doc_id);
             cnt += 1;
         }
-        cJSON_Delete(json);
     }
 
     free(iterator);
+    database_close(db, FALSE);
 
     if (!args->print) {
         database_iterator_t *del_iter = database_create_delete_list_iterator(db);
@@ -338,11 +330,8 @@ void sist2_index(index_args_t *args) {
             delete_document(id);
             free(id);
         }
-        free(del_iter);
     }
 
-    database_close(db, FALSE);
-
     tpool_wait(IndexCtx.pool);
     tpool_destroy(IndexCtx.pool);
 
@@ -352,22 +341,6 @@ void sist2_index(index_args_t *args) {
     free(desc);
 }
 
-void sist2_sqlite_index(sqlite_index_args_t *args) {
-    database_t *db = database_create(args->index_path, INDEX_DATABASE);
-    database_open(db);
-
-    database_t *search_db = database_create(args->search_index_path, FTS_DATABASE);
-    database_initialize(search_db);
-
-    database_fts_attach(db, args->search_index_path);
-
-    database_fts_index(db);
-    database_fts_optimize(db);
-
-    database_close(db, FALSE);
-    database_close(search_db, FALSE);
-}
-
 void sist2_exec_script(exec_args_t *args) {
     LogCtx.verbose = TRUE;
 
@@ -390,7 +363,6 @@ void sist2_exec_script(exec_args_t *args) {
 void sist2_web(web_args_t *args) {
 
     WebCtx.es_url = args->es_url;
-    WebCtx.search_backend = args->search_backend;
     WebCtx.es_index = args->es_index;
     WebCtx.es_insecure_ssl = args->es_insecure_ssl;
     WebCtx.index_count = args->index_count;
@@ -407,27 +379,15 @@ void sist2_web(web_args_t *args) {
     WebCtx.auth0_audience = args->auth0_audience;
     strcpy(WebCtx.lang, args->lang);
 
-    if (args->search_backend == SQLITE_SEARCH_BACKEND) {
-        WebCtx.search_db = database_create(args->search_index_path, FTS_DATABASE);
-        database_open(WebCtx.search_db);
-    }
-
     for (int i = 0; i < args->index_count; i++) {
         char *abs_path = abspath(args->indices[i]);
 
         strcpy(WebCtx.indices[i].path, abs_path);
 
-        database_t *db = database_create(abs_path, INDEX_DATABASE);
-        database_open(db);
-        if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
-            database_fts_attach(db, args->search_index_path);
-            database_fts_sync_tags(db);
-            database_fts_detach(db);
-        }
+        WebCtx.indices[i].db = database_create(abs_path, INDEX_DATABASE);
+        database_open(WebCtx.indices[i].db);
 
-        WebCtx.indices[i].db = db;
-
-        index_descriptor_t *desc = database_read_index_descriptor(db);
+        index_descriptor_t *desc = database_read_index_descriptor(WebCtx.indices[i].db);
         WebCtx.indices[i].desc = *desc;
         free(desc);
 
@@ -466,7 +426,6 @@ int main(int argc, const char *argv[]) {
     index_args_t *index_args = index_args_create();
     web_args_t *web_args = web_args_create();
     exec_args_t *exec_args = exec_args_create();
-    sqlite_index_args_t *sqlite_index_args = sqlite_index_args_create();
 
     int arg_version = 0;
 
@@ -476,8 +435,6 @@ int main(int argc, const char *argv[]) {
     char *common_script_path = NULL;
     int common_async_script = 0;
     int common_threads = 0;
-    int common_optimize_database = 0;
-    char *common_search_index = NULL;
 
     struct argparse_option options[] = {
             OPT_HELP(),
@@ -504,7 +461,7 @@ int main(int argc, const char *argv[]) {
             OPT_STRING('o', "output", &scan_args->output, "Output index file path. DEFAULT: index.sist2"),
             OPT_BOOLEAN(0, "incremental", &scan_args->incremental,
                         "If the output file path exists, only scan new or modified files."),
-            OPT_BOOLEAN(0, "optimize-index", &common_optimize_database,
+            OPT_BOOLEAN(0, "optimize-index", &scan_args->optimize_database,
                         "Defragment index file after scan to reduce its file size."),
             OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."),
             OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: index"),
@@ -553,15 +510,10 @@ int main(int argc, const char *argv[]) {
             OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 70"),
             OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings."),
 
-            OPT_GROUP("sqlite-index options"),
-            OPT_STRING(0, "search-index", &common_search_index, "Path to search index. Will be created if it does not exist yet."),
-
             OPT_GROUP("Web options"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT: http://localhost:9200"),
             OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl,
                         "Do not verify SSL connections to Elasticsearch."),
-    // TODO: change arg name (?)
-            OPT_STRING(0, "search-index", &common_search_index, "Path to SQLite search index."),
             OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT: sist2"),
             OPT_STRING(0, "bind", &web_args->listen_address,
                        "Listen for connections on this address. DEFAULT: localhost:4090"),
@@ -597,7 +549,7 @@ int main(int argc, const char *argv[]) {
     argc = argparse_parse(&argparse, argc, argv);
 
     if (arg_version) {
-        printf("%s", Version);
+        printf(Version);
         goto end;
     }
 
@@ -624,11 +576,6 @@ int main(int argc, const char *argv[]) {
     exec_args->async_script = common_async_script;
     index_args->async_script = common_async_script;
 
-    scan_args->optimize_database = common_optimize_database;
-
-    sqlite_index_args->search_index_path = common_search_index;
-    web_args->search_index_path = common_search_index;
-
     if (argc == 0) {
         argparse_usage(&argparse);
         goto end;
@@ -648,14 +595,6 @@ int main(int argc, const char *argv[]) {
         }
         sist2_index(index_args);
 
-    } else if (strcmp(argv[0], "sqlite-index") == 0) {
-
-        int err = sqlite_index_args_validate(sqlite_index_args, argc, argv);
-        if (err != 0) {
-            goto end;
-        }
-        sist2_sqlite_index(sqlite_index_args);
-
     } else if (strcmp(argv[0], "web") == 0) {
 
         int err = web_args_validate(web_args, argc, argv);

src/sist.h

@@ -51,11 +51,11 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.0.7"
+#define VERSION "3.0.0"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
 static const int VersionMinor = 0;
-static const int VersionPatch = 7;
+static const int VersionPatch = 0;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown

src/tpool.c

@@ -149,11 +149,6 @@ void worker_proc_cleanup(tpool_t *pool) {
     if (ProcData.index_db != NULL) {
         database_close(ProcData.index_db, FALSE);
     }
-
-    if (IndexCtx.needs_es_connection) {
-        elastic_cleanup();
-    }
-
     database_close(ProcData.ipc_db, FALSE);
 }
 
@@ -247,7 +242,6 @@ static void *tpool_worker(void *arg) {
     pthread_mutex_lock(&pool->shm->mutex);
     pthread_cond_signal(&pool->shm->done_working_cond);
     pthread_mutex_unlock(&pool->shm->mutex);
-    worker_proc_cleanup(pool);
 #endif
 
     return NULL;

src/util.c

@@ -162,9 +162,9 @@ const char *find_file_in_paths(const char *paths[], const char *filename) {
         }
 
         char path[PATH_MAX];
-        snprintf(path, sizeof(path), "%s/%s", apath, filename);
+        snprintf(path, sizeof(path), "%s%s", apath, filename);
 
-        LOG_DEBUGF("util.c", "Looking for '%s' in folder '%s' (%s)", filename, apath, path);
+        LOG_DEBUGF("util.c", "Looking for '%s' in folder '%s'", filename, apath);
         free(apath);
 
         struct stat info;

src/util.h

@@ -114,7 +114,4 @@ struct timespec timespec_add(struct timespec ts1, long usec);
         pthread_cond_timedwait(cond, mutex, &end_time); \
     } while (0)
 
-#define array_foreach(arr) \
-    for (int i = 0; (arr)[i] != NULL; i++)
-
 #endif

src/web/serve.c

@@ -5,24 +5,11 @@
 #include "src/index/web.h"
 #include "src/auth0/auth0_c_api.h"
 #include "src/web/web_util.h"
-#include "src/cli.h"
-#include <time.h>
 
 #include <src/ctx.h>
 
-void fts_search_paths(struct mg_connection *nc, struct mg_http_message *hm);
-
-void fts_search_mimetypes(struct mg_connection *nc, struct mg_http_message *hm);
-
-void fts_search_summary_stats(struct mg_connection *nc, struct mg_http_message *hm);
-
-void fts_search(struct mg_connection *nc, struct mg_http_message *hm);
-
-void fts_get_document(struct mg_connection *nc, struct mg_http_message *hm);
-
-void fts_suggest_tag(struct mg_connection *nc, struct mg_http_message *hm);
-
-void fts_get_tags(struct mg_connection *nc, struct mg_http_message *hm);
+#define HTTP_TEXT_TYPE_HEADER "Content-Type: text/plain;charset=utf-8\r\n"
+#define HTTP_REPLY_NOT_FOUND mg_http_reply(nc, 404, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Not found");
 
 static struct mg_http_serve_opts DefaultServeOpts = {
         .fs = NULL,
@@ -33,36 +20,49 @@ static struct mg_http_serve_opts DefaultServeOpts = {
 
 void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
 
-    if (hm->uri.len != SIST_INDEX_ID_LEN + 7) {
+    if (hm->uri.len != SIST_INDEX_ID_LEN + 4) {
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
     char arg_index_id[SIST_INDEX_ID_LEN];
-    char arg_stat_type[5];
-
     memcpy(arg_index_id, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
     *(arg_index_id + SIST_INDEX_ID_LEN - 1) = '\0';
-    memcpy(arg_stat_type, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, 4);
-    *(arg_stat_type + sizeof(arg_stat_type) - 1) = '\0';
 
-    database_stat_type_d stat_type = database_get_stat_type_by_mnemonic(arg_stat_type);
-    if (stat_type == DATABASE_STAT_INVALID) {
+    index_t *index = web_get_index_by_id(arg_index_id);
+    if (index == NULL) {
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
-    database_t *db = web_get_database(arg_index_id);
-    if (db == NULL) {
-        LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index_id);
-        HTTP_REPLY_NOT_FOUND
-        return;
+    const char *file;
+    switch (atoi(hm->uri.ptr + 3 + SIST_INDEX_ID_LEN)) {
+        case 1:
+            file = "treemap.csv";
+            break;
+        case 2:
+            file = "mime_agg.csv";
+            break;
+        case 3:
+            file = "size_agg.csv";
+            break;
+        case 4:
+            file = "date_agg.csv";
+            break;
+        default:
+            return;
     }
 
-    cJSON *json = database_get_stats(db, stat_type);
-    mg_send_json(nc, json);
+    char disposition[8192];
+    snprintf(disposition, sizeof(disposition),
+             "Content-Disposition: inline; filename=\"%s\"\r\nCache-Control: max-age=31536000\r\n", file);
 
-    cJSON_Delete(json);
+    char full_path[PATH_MAX];
+    strcpy(full_path, index->path);
+    strcat(full_path, file);
+
+    struct mg_http_serve_opts opts = {};
+    mg_http_serve_file(nc, hm, full_path, &opts);
 }
 
 void serve_index_html(struct mg_connection *nc, struct mg_http_message *hm) {
@@ -102,7 +102,7 @@ void serve_chunk_vendors_css(struct mg_connection *nc, struct mg_http_message *h
 }
 
 void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, const char *arg_index,
-                     const char *arg_doc_id, int arg_num) {
+        const char *arg_doc_id, int arg_num) {
 
     database_t *db = web_get_database(arg_index);
     if (db == NULL) {
@@ -261,10 +261,6 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
 }
 
 void cache_es_version() {
-    if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
-        return;
-    }
-
     static int is_cached = FALSE;
 
     if (is_cached == TRUE) {
@@ -290,23 +286,16 @@ void index_info(struct mg_connection *nc) {
     cJSON *json = cJSON_CreateObject();
     cJSON *arr = cJSON_AddArrayToObject(json, "indices");
 
+    cJSON_AddStringToObject(json, "mongooseVersion", MG_VERSION);
     cJSON_AddStringToObject(json, "esIndex", WebCtx.es_index);
     cJSON_AddStringToObject(json, "version", Version);
-
-#ifdef SIST_DEBUG_INFO
-    cJSON_AddStringToObject(json, "mongooseVersion", MG_VERSION);
     cJSON_AddStringToObject(json, "esVersion", es_version);
-    cJSON_AddStringToObject(json, "platform", QUOTE(SIST_PLATFORM));
-    cJSON_AddStringToObject(json, "sist2Hash", Sist2CommitHash);
-    cJSON_AddBoolToObject(json, "dev", WebCtx.dev);
-    cJSON_AddBoolToObject(json, "showDebugInfo", TRUE);
-#else
-    cJSON_AddBoolToObject(json, "showDebugInfo", FALSE);
-#endif
-
     cJSON_AddBoolToObject(json, "esVersionSupported", IS_SUPPORTED_ES_VERSION(WebCtx.es_version));
     cJSON_AddBoolToObject(json, "esVersionLegacy", IS_LEGACY_VERSION(WebCtx.es_version));
+    cJSON_AddStringToObject(json, "platform", QUOTE(SIST_PLATFORM));
+    cJSON_AddStringToObject(json, "sist2Hash", Sist2CommitHash);
     cJSON_AddStringToObject(json, "lang", WebCtx.lang);
+    cJSON_AddBoolToObject(json, "dev", WebCtx.dev);
 
     cJSON_AddBoolToObject(json, "auth0Enabled", WebCtx.auth0_enabled);
     if (WebCtx.auth0_enabled) {
@@ -334,12 +323,6 @@ void index_info(struct mg_connection *nc) {
         cJSON_AddItemToArray(arr, idx_json);
     }
 
-    if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
-        cJSON_AddStringToObject(json, "searchBackend", "sqlite");
-    } else {
-        cJSON_AddStringToObject(json, "searchBackend", "elasticsearch");
-    }
-
     char *json_str = cJSON_PrintUnformatted(json);
 
     web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
@@ -348,63 +331,54 @@ void index_info(struct mg_connection *nc) {
     cJSON_Delete(json);
 }
 
-cJSON *get_root_document_by_id(const char *index_id, const char *doc_id) {
-
-    database_t *db = web_get_database(index_id);
-    if (!db) {
-        return NULL;
-    }
-
-    char next_id[SIST_DOC_ID_LEN];
-    strcpy(next_id, doc_id);
-
-    while (TRUE) {
-        cJSON *doc = database_get_document(db, next_id);
-
-        if (doc == NULL) {
-            return NULL;
-        }
-
-        cJSON *parent = cJSON_GetObjectItem(doc, "parent");
-        if (parent == NULL || cJSON_IsNull(parent)) {
-            return doc;
-        }
-
-        strcpy(next_id, parent->valuestring);
-        cJSON_Delete(parent);
-    }
-}
 
 void file(struct mg_connection *nc, struct mg_http_message *hm) {
 
-    if (hm->uri.len != SIST_INDEX_ID_LEN + SIST_DOC_ID_LEN + 2) {
+    if (hm->uri.len != SIST_DOC_ID_LEN + 2) {
         LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr);
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
     char arg_doc_id[SIST_DOC_ID_LEN];
-    char arg_index[SIST_INDEX_ID_LEN];
-
-    memcpy(arg_index, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
-    *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
-    memcpy(arg_doc_id, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, SIST_DOC_ID_LEN);
+    memcpy(arg_doc_id, hm->uri.ptr + 3, SIST_DOC_ID_LEN);
     *(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0';
 
-    index_t *idx = web_get_index_by_id(arg_index);
+    const char *next = arg_doc_id;
+    cJSON *doc = NULL;
+    cJSON *index_id = NULL;
+    cJSON *source = NULL;
+
+    while (true) {
+        doc = elastic_get_document(next);
+        source = cJSON_GetObjectItem(doc, "_source");
+        index_id = cJSON_GetObjectItem(source, "index");
+        if (index_id == NULL) {
+            cJSON_Delete(doc);
+            HTTP_REPLY_NOT_FOUND
+            return;
+        }
+        cJSON *parent = cJSON_GetObjectItem(source, "parent");
+        if (parent == NULL) {
+            break;
+        }
+        next = parent->valuestring;
+    }
+
+    index_t *idx = web_get_index_by_id(index_id->valuestring);
+
     if (idx == NULL) {
+        cJSON_Delete(doc);
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
-    cJSON *source = get_root_document_by_id(arg_index, arg_doc_id);
-
     if (strlen(idx->desc.rewrite_url) == 0) {
         serve_file_from_disk(source, idx, nc, hm);
     } else {
         serve_file_from_url(source, idx, nc);
     }
-    cJSON_Delete(source);
+    cJSON_Delete(doc);
 }
 
 void status(struct mg_connection *nc) {
@@ -426,10 +400,6 @@ typedef struct {
 
 tag_req_t *parse_tag_request(cJSON *json) {
 
-    if (json == NULL) {
-        return NULL;
-    }
-
     if (!cJSON_IsObject(json)) {
         return NULL;
     }
@@ -457,101 +427,115 @@ tag_req_t *parse_tag_request(cJSON *json) {
     return req;
 }
 
-subreq_ctx_t *elastic_delete_tag(const tag_req_t *req) {
-    char *buf = malloc(sizeof(char) * 8192);
-    snprintf(buf, 8192,
-             "{"
-             "    \"script\" : {"
-             "        \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\","
-             "        \"lang\": \"painless\","
-             "        \"params\" : {"
-             "            \"tag\" : \"%s\""
-             "        }"
-             "    }"
-             "}", req->name
-    );
-
-    char url[4096];
-    snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, req->doc_id);
-
-    return web_post_async(url, buf, WebCtx.es_insecure_ssl);
-}
-
-subreq_ctx_t *elastic_write_tag(const tag_req_t *req) {
-    char *buf = malloc(sizeof(char) * 8192);
-    snprintf(buf, 8192,
-             "{"
-             "    \"script\" : {"
-             "        \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\","
-             "        \"lang\": \"painless\","
-             "        \"params\" : {"
-             "            \"tag\" : \"%s\""
-             "        }"
-             "    }"
-             "}", req->name
-    );
-
-    char url[4096];
-    snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, req->doc_id);
-    return web_post_async(url, buf, WebCtx.es_insecure_ssl);
-}
-
 void tag(struct mg_connection *nc, struct mg_http_message *hm) {
-    if (hm->uri.len != SIST_INDEX_ID_LEN + 4) {
-        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr);
-        HTTP_REPLY_NOT_FOUND
-        return;
-    }
-
-    char arg_index[SIST_INDEX_ID_LEN];
-    memcpy(arg_index, hm->uri.ptr + 5, SIST_INDEX_ID_LEN);
-    *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
-
-    char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.ptr, hm->body.len);
-    *(body + hm->body.len) = '\0';
-    cJSON *json = cJSON_Parse(body);
-    free(body);
-
-    if (json == NULL) {
-        HTTP_REPLY_BAD_REQUEST
-        return;
-    }
-
-    database_t *db = web_get_database(arg_index);
-    if (db == NULL) {
-        LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index);
-        HTTP_REPLY_NOT_FOUND
-        return;
-    }
-
-    tag_req_t *req = parse_tag_request(json);
-    cJSON_Delete(json);
-    if (req == NULL) {
-        LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index);
-        HTTP_REPLY_BAD_REQUEST
-        return;
-    }
-
-    if (req->delete) {
-        database_delete_tag(db, req->doc_id, req->name);
-        if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
-            database_delete_tag(WebCtx.search_db, req->doc_id, req->name);
-            HTTP_REPLY_OK
-        } else {
-            nc->fn_data = elastic_delete_tag(req);
-        }
-    } else {
-        database_write_tag(db, req->doc_id, req->name);
-        if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
-            database_write_tag(WebCtx.search_db, req->doc_id, req->name);
-            HTTP_REPLY_OK
-        } else {
-            nc->fn_data = elastic_write_tag(req);
-        }
-    }
-
-    free(req);
+//    if (hm->uri.len != SIST_INDEX_ID_LEN + 4) {
+//        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr)
+//        HTTP_REPLY_NOT_FOUND
+//        return;
+//    }
+//
+//    char arg_index[SIST_INDEX_ID_LEN];
+//    memcpy(arg_index, hm->uri.ptr + 5, SIST_INDEX_ID_LEN);
+//    *(arg_index + SIST_INDEX_ID_LEN - 1) = '\0';
+//
+//    if (hm->body.len < 2 || hm->method.len != 4 || memcmp(&hm->method, "POST", 4) == 0) {
+//        LOG_DEBUG("serve.c", "Invalid tag request")
+//        HTTP_REPLY_NOT_FOUND
+//        return;
+//    }
+//
+//    store_t *store = get_tag_store(arg_index);
+//    if (store == NULL) {
+//        LOG_DEBUGF("serve.c", "Could not get tag store for index: %s", arg_index)
+//        HTTP_REPLY_NOT_FOUND
+//        return;
+//    }
+//
+//    char *body = malloc(hm->body.len + 1);
+//    memcpy(body, hm->body.ptr, hm->body.len);
+//    *(body + hm->body.len) = '\0';
+//    cJSON *json = cJSON_Parse(body);
+//
+//    tag_req_t *arg_req = parse_tag_request(json);
+//    if (arg_req == NULL) {
+//        LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index)
+//        cJSON_Delete(json);
+//        free(body);
+//        mg_http_reply(nc, 400, "", "Invalid request");
+//        return;
+//    }
+//
+//    cJSON *arr = NULL;
+//
+//    size_t data_len = 0;
+//    const char *data = store_read(store, arg_req->doc_id, SIST_DOC_ID_LEN, &data_len);
+//    if (data_len == 0) {
+//        arr = cJSON_CreateArray();
+//    } else {
+//        arr = cJSON_Parse(data);
+//    }
+//
+//    if (arg_req->delete) {
+//
+//        if (data_len > 0) {
+//            cJSON *element = NULL;
+//            int i = 0;
+//            cJSON_ArrayForEach(element, arr) {
+//                if (strcmp(element->valuestring, arg_req->name) == 0) {
+//                    cJSON_DeleteItemFromArray(arr, i);
+//                    break;
+//                }
+//                i++;
+//            }
+//        }
+//
+//        char *buf = malloc(sizeof(char) * 8192);
+//        snprintf(buf, 8192,
+//                 "{"
+//                 "    \"script\" : {"
+//                 "        \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\","
+//                 "        \"lang\": \"painless\","
+//                 "        \"params\" : {"
+//                 "            \"tag\" : \"%s\""
+//                 "        }"
+//                 "    }"
+//                 "}", arg_req->name
+//        );
+//
+//        char url[4096];
+//        snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
+//        nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl);
+//
+//    } else {
+//        cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name));
+//
+//        char *buf = malloc(sizeof(char) * 8192);
+//        snprintf(buf, 8192,
+//                 "{"
+//                 "    \"script\" : {"
+//                 "        \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\","
+//                 "        \"lang\": \"painless\","
+//                 "        \"params\" : {"
+//                 "            \"tag\" : \"%s\""
+//                 "        }"
+//                 "    }"
+//                 "}", arg_req->name
+//        );
+//
+//        char url[4096];
+//        snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
+//        nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl);
+//    }
+//
+//    char *json_str = cJSON_PrintUnformatted(arr);
+//    store_write(store, arg_req->doc_id, SIST_DOC_ID_LEN, json_str, strlen(json_str) + 1);
+//    store_flush(store);
+//
+//    free(arg_req);
+//    free(json_str);
+//    cJSON_Delete(json);
+//    cJSON_Delete(arr);
+//    free(body);
 }
 
 int validate_auth(struct mg_connection *nc, struct mg_http_message *hm) {
@@ -649,39 +633,11 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
             return;
         }
 
-        if (WebCtx.search_backend == SQLITE_SEARCH_BACKEND) {
-            if (mg_http_match_uri(hm, "/fts/paths")) {
-                fts_search_paths(nc, hm);
-                return;
-            } else if (mg_http_match_uri(hm, "/fts/mimetypes")) {
-                fts_search_mimetypes(nc, hm);
-                return;
-            } else if (mg_http_match_uri(hm, "/fts/dateRange")) {
-                fts_search_summary_stats(nc, hm);
-                return;
-            } else if (mg_http_match_uri(hm, "/fts/search")) {
-                fts_search(nc, hm);
-                return;
-            } else if (mg_http_match_uri(hm, "/fts/d/*")) {
-                fts_get_document(nc, hm);
-                return;
-            } else if (mg_http_match_uri(hm, "/fts/suggestTags")) {
-                fts_suggest_tag(nc, hm);
-                return;
-            } else if (mg_http_match_uri(hm, "/fts/tags")) {
-                fts_get_tags(nc, hm);
-                return;
-            }
-        } else if (WebCtx.search_backend == ES_SEARCH_BACKEND) {
-            if (mg_http_match_uri(hm, "/es")) {
-                search(nc, hm);
-                return;
-            }
-        }
-
-        if (mg_http_match_uri(hm, "/status")) {
+        if (mg_http_match_uri(hm, "/es")) {
+            search(nc, hm);
+        } else if (mg_http_match_uri(hm, "/status")) {
             status(nc);
-        } else if (mg_http_match_uri(hm, "/f/*/*")) {
+        } else if (mg_http_match_uri(hm, "/f/*")) {
             file(nc, hm);
         } else if (mg_http_match_uri(hm, "/t/*/*/*")) {
             thumbnail_with_num(nc, hm);
@@ -712,9 +668,6 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
                     mg_send(nc, r->body, r->size);
                 } else if (r->status_code == 0) {
                     sist_log("serve.c", LOG_SIST_ERROR, "Could not connect to elasticsearch!");
-
-                    mg_http_reply(nc, 503, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER,
-                                  "Elasticsearch connection error, see server logs.");
                 } else {
                     sist_logf("serve.c", LOG_SIST_WARNING, "ElasticSearch error during query (%d)", r->status_code);
                     if (r->size != 0) {
@@ -748,12 +701,14 @@ void serve(const char *listen_address) {
     struct mg_mgr mgr;
     mg_mgr_init(&mgr);
 
+    int ok = 1;
+
     struct mg_connection *nc = mg_http_listen(&mgr, listen_address, ev_router, NULL);
     if (nc == NULL) {
         LOG_FATALF("serve.c", "Couldn't bind web server on address %s", listen_address);
     }
 
-    while (TRUE) {
+    while (ok) {
         mg_mgr_poll(&mgr, 10);
     }
     mg_mgr_free(&mgr);

src/web/serve.h

@@ -3,11 +3,6 @@
 
 #include "src/sist.h"
 
-#define HTTP_TEXT_TYPE_HEADER "Content-Type: text/plain;charset=utf-8\r\n"
-#define HTTP_REPLY_NOT_FOUND mg_http_reply(nc, 404, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Not found");
-#define HTTP_REPLY_BAD_REQUEST mg_http_reply(nc, 400, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Invalid request");
-#define HTTP_REPLY_OK mg_http_reply(nc, 200, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "ok");
-
 void serve(const char *listen_address);
 
 #endif

src/web/web_fts.c

@@ -1,378 +0,0 @@
-#include "serve.h"
-#include <mongoose.h>
-#include "src/web/web_util.h"
-
-typedef struct {
-    char *index_id;
-    char *prefix;
-    int min_depth;
-    int max_depth;
-} fts_search_paths_req_t;
-
-typedef struct {
-    cJSON *val;
-    int invalid;
-} json_value;
-
-typedef struct {
-    char *query;
-    char *path;
-    fts_sort_t sort;
-    double size_min;
-    double size_max;
-    double date_min;
-    double date_max;
-    int page_size;
-    char **index_ids;
-    char **mime_types;
-    char **tags;
-    int sort_asc;
-    int seed;
-    char **after;
-    int fetch_aggregations;
-    int highlight;
-    int highlight_context_size;
-} fts_search_req_t;
-
-fts_sort_t get_sort_mode(const cJSON *req_sort) {
-    if (strcmp(req_sort->valuestring, "score") == 0) {
-        return FTS_SORT_SCORE;
-    } else if (strcmp(req_sort->valuestring, "size") == 0) {
-        return FTS_SORT_SIZE;
-    } else if (strcmp(req_sort->valuestring, "mtime") == 0) {
-        return FTS_SORT_MTIME;
-    } else if (strcmp(req_sort->valuestring, "random") == 0) {
-        return FTS_SORT_RANDOM;
-    } else if (strcmp(req_sort->valuestring, "name") == 0) {
-        return FTS_SORT_NAME;
-    }
-
-    return FTS_SORT_INVALID;
-}
-
-
-static json_value get_json_string(cJSON *object, const char *name) {
-
-    cJSON *item = cJSON_GetObjectItem(object, name);
-    if (item == NULL || cJSON_IsNull(item)) {
-        return (json_value) {NULL, FALSE};
-    }
-    if (!cJSON_IsString(item)) {
-        return (json_value) {NULL, TRUE};
-    }
-
-    return (json_value) {item, FALSE};
-}
-
-static json_value get_json_number(cJSON *object, const char *name) {
-
-    cJSON *item = cJSON_GetObjectItem(object, name);
-    if (item == NULL || cJSON_IsNull(item)) {
-        return (json_value) {NULL, FALSE};
-    }
-    if (!cJSON_IsNumber(item)) {
-        return (json_value) {NULL, TRUE};
-    }
-
-    return (json_value) {item, FALSE};
-}
-
-static json_value get_json_bool(cJSON *object, const char *name) {
-    cJSON *item = cJSON_GetObjectItem(object, name);
-    if (item == NULL || cJSON_IsNull(item)) {
-        return (json_value) {NULL, FALSE};
-    }
-    if (!cJSON_IsBool(item)) {
-        return (json_value) {NULL, TRUE};
-    }
-
-    return (json_value) {item, FALSE};
-}
-
-static json_value get_json_array(cJSON *object, const char *name) {
-    cJSON *item = cJSON_GetObjectItem(object, name);
-    if (item == NULL || cJSON_IsNull(item)) {
-        return (json_value) {NULL, FALSE};
-    }
-    if (!cJSON_IsArray(item) || cJSON_GetArraySize(item) == 0) {
-        return (json_value) {NULL, TRUE};
-    }
-
-    cJSON *elem;
-    cJSON_ArrayForEach(elem, item) {
-        if (!cJSON_IsString(elem)) {
-            return (json_value) {NULL, TRUE};
-        }
-    }
-
-    return (json_value) {item, FALSE};
-}
-
-char **json_array_to_c_array(cJSON *json) {
-
-    cJSON *element;
-    char **arr = calloc(cJSON_GetArraySize(json) + 1, sizeof(char *));
-    int i = 0;
-    cJSON_ArrayForEach(element, json) {
-        arr[i++] = strdup(element->valuestring);
-    }
-
-    return arr;
-}
-
-#define DEFAULT_HIGHLIGHT_CONTEXT_SIZE 20
-
-fts_search_req_t *get_search_req(struct mg_http_message *hm) {
-    cJSON *json = web_get_json_body(hm);
-
-    if (json == NULL) {
-        return NULL;
-    }
-
-    json_value req_query, req_path, req_size_min, req_size_max, req_date_min, req_date_max, req_page_size,
-            req_index_ids, req_mime_types, req_tags, req_sort_asc, req_sort, req_seed, req_after,
-            req_fetch_aggregations, req_highlight, req_highlight_context_size;
-
-    if (!cJSON_IsObject(json) ||
-        (req_query = get_json_string(json, "query")).invalid ||
-        (req_path = get_json_string(json, "path")).invalid ||
-        (req_sort = get_json_string(json, "sort")).val == NULL ||
-        (req_size_min = get_json_number(json, "sizeMin")).invalid ||
-        (req_size_max = get_json_number(json, "sizeMax")).invalid ||
-        (req_date_min = get_json_number(json, "dateMin")).invalid ||
-        (req_date_max = get_json_number(json, "dateMax")).invalid ||
-        (req_page_size = get_json_number(json, "pageSize")).val == NULL ||
-        (req_after = get_json_array(json, "after")).invalid ||
-        (req_seed = get_json_number(json, "seed")).invalid ||
-        (req_fetch_aggregations = get_json_bool(json, "fetchAggregations")).invalid ||
-        (req_sort_asc = get_json_bool(json, "sortAsc")).invalid ||
-        (req_index_ids = get_json_array(json, "indexIds")).invalid ||
-        (req_mime_types = get_json_array(json, "mimeTypes")).invalid ||
-        (req_highlight = get_json_bool(json, "highlight")).invalid ||
-        (req_highlight_context_size = get_json_number(json, "highlightContextSize")).invalid ||
-        (req_tags = get_json_array(json, "tags")).invalid) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-
-    int index_id_count = cJSON_GetArraySize(req_index_ids.val);
-    if (index_id_count > 999) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-    int mime_count = req_mime_types.val ? 0 : cJSON_GetArraySize(req_mime_types.val);
-    if (mime_count > 999) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-    int tag_count = req_tags.val ? 0 : cJSON_GetArraySize(req_tags.val);
-    if (tag_count > 9999) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-    if (req_path.val && (strstr(req_path.val->valuestring, "*") || strlen(req_path.val) >= PATH_MAX)) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-
-    fts_sort_t sort = get_sort_mode(req_sort.val);
-    if (sort == FTS_SORT_INVALID) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-
-    if (req_after.val && cJSON_GetArraySize(req_after.val) != 2) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-
-    if (req_page_size.val->valueint > 1000 || req_page_size.val->valueint < 0) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-    if (req_highlight_context_size.val->valueint < 0) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-
-    fts_search_req_t *req = malloc(sizeof(fts_search_req_t));
-
-    req->sort = sort;
-    req->query = req_query.val ? strdup(req_query.val->valuestring) : NULL;
-    req->path = req_path.val ? strdup(req_path.val->valuestring) : NULL;
-    req->size_min = req_size_min.val ? req_size_min.val->valuedouble : 0;
-    req->size_max = req_size_max.val ? req_size_max.val->valuedouble : 0;
-    req->seed = (int) (req_seed.val ? req_seed.val->valuedouble : 0);
-    req->date_min = req_date_min.val ? req_date_min.val->valuedouble : 0;
-    req->date_max = req_date_max.val ? req_date_max.val->valuedouble : 0;
-    req->page_size = (int) req_page_size.val->valuedouble;
-    req->sort_asc = req_sort_asc.val ? req_sort_asc.val->valueint : TRUE;
-    req->index_ids = req_index_ids.val ? json_array_to_c_array(req_index_ids.val) : NULL;
-    req->after = req_after.val ? json_array_to_c_array(req_after.val) : NULL;
-    req->mime_types = req_mime_types.val ? json_array_to_c_array(req_mime_types.val) : NULL;
-    req->tags = req_tags.val ? json_array_to_c_array(req_tags.val) : NULL;
-    req->fetch_aggregations = req_fetch_aggregations.val ? req_fetch_aggregations.val->valueint : FALSE;
-    req->highlight = req_highlight.val ? req_highlight.val->valueint : FALSE;
-    req->highlight_context_size = req_highlight_context_size.val
-                                  ? req_highlight_context_size.val->valueint
-                                  : DEFAULT_HIGHLIGHT_CONTEXT_SIZE;
-
-    cJSON_Delete(json);
-
-    return req;
-}
-
-void destroy_array(char **array) {
-    if (array == NULL) {
-        return;
-    }
-    array_foreach(array) { free(array[i]); }
-    free(array);
-}
-
-void destroy_search_req(fts_search_req_t *req) {
-    free(req->query);
-    free(req->path);
-
-    destroy_array(req->index_ids);
-    destroy_array(req->mime_types);
-    destroy_array(req->tags);
-
-    free(req);
-}
-
-fts_search_paths_req_t *get_search_paths_req(struct mg_http_message *hm) {
-    cJSON *json = web_get_json_body(hm);
-
-    if (json == NULL) {
-        return NULL;
-    }
-
-    json_value req_index_id, req_min_depth, req_max_depth, req_prefix;
-
-    if (!cJSON_IsObject(json) ||
-        (req_index_id = get_json_string(json, "indexId")).invalid ||
-        (req_prefix = get_json_string(json, "prefix")).invalid ||
-        (req_min_depth = get_json_number(json, "minDepth")).val == NULL ||
-        (req_max_depth = get_json_number(json, "maxDepth")).val == NULL) {
-        cJSON_Delete(json);
-        return NULL;
-    }
-
-    fts_search_paths_req_t *req = malloc(sizeof(fts_search_paths_req_t));
-
-    req->index_id = req_index_id.val ? strdup(req_index_id.val->valuestring) : NULL;
-    req->min_depth = req_min_depth.val->valueint;
-    req->max_depth = req_max_depth.val->valueint;
-    req->prefix = req_prefix.val ? strdup(req_prefix.val->valuestring) : NULL;
-
-    cJSON_Delete(json);
-    return req;
-}
-
-void destroy_search_paths_req(fts_search_paths_req_t *req) {
-    if (req->index_id) {
-        free(req->index_id);
-    }
-    if (req->prefix) {
-        free(req->prefix);
-    }
-    free(req);
-}
-
-void fts_search_paths(struct mg_connection *nc, struct mg_http_message *hm) {
-
-    fts_search_paths_req_t *req = get_search_paths_req(hm);
-    if (req == NULL) {
-        HTTP_REPLY_BAD_REQUEST
-        return;
-    }
-
-    cJSON *json = database_fts_get_paths(WebCtx.search_db, req->index_id, req->min_depth,
-                                         req->max_depth, req->prefix, req->max_depth == 10000);
-
-    destroy_search_paths_req(req);
-    mg_send_json(nc, json);
-    cJSON_Delete(json);
-}
-
-void fts_search_mimetypes(struct mg_connection *nc, struct mg_http_message *hm) {
-
-    cJSON *json = database_fts_get_mimetypes(WebCtx.search_db);
-
-    mg_send_json(nc, json);
-    cJSON_Delete(json);
-}
-
-void fts_search_summary_stats(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
-
-    database_summary_stats_t stats = database_fts_get_date_range(WebCtx.search_db);
-
-    cJSON *json = cJSON_CreateObject();
-
-    cJSON_AddNumberToObject(json, "dateMin", stats.date_min);
-    cJSON_AddNumberToObject(json, "dateMax", stats.date_max);
-
-    mg_send_json(nc, json);
-    cJSON_Delete(json);
-}
-
-void fts_search(struct mg_connection *nc, struct mg_http_message *hm) {
-
-    fts_search_req_t *req = get_search_req(hm);
-    if (req == NULL) {
-        HTTP_REPLY_BAD_REQUEST
-        return;
-    }
-
-    cJSON *json = database_fts_search(WebCtx.search_db, req->query, req->path,
-                                      (long) req->size_min, (long) req->size_max,
-                                      (long) req->date_min, (long) req->date_max,
-                                      req->page_size, req->index_ids, req->mime_types,
-                                      req->tags, req->sort_asc, req->sort, req->seed,
-                                      req->after, req->fetch_aggregations, req->highlight,
-                                      req->highlight_context_size);
-
-    destroy_search_req(req);
-    mg_send_json(nc, json);
-    cJSON_Delete(json);
-}
-
-void fts_get_document(struct mg_connection *nc, struct mg_http_message *hm) {
-
-    char doc_id[SIST_DOC_ID_LEN];
-    memcpy(doc_id, hm->uri.ptr + 7, SIST_INDEX_ID_LEN);
-    *(doc_id + SIST_INDEX_ID_LEN - 1) = '\0';
-
-    cJSON *json = database_fts_get_document(WebCtx.search_db, doc_id);
-
-    if (!json) {
-        HTTP_REPLY_NOT_FOUND
-        return;
-    }
-
-    mg_send_json(nc, json);
-    cJSON_Delete(json);
-}
-
-void fts_suggest_tag(struct mg_connection *nc, struct mg_http_message *hm) {
-    char *body = web_get_string_body(hm);
-
-    if (body == NULL) {
-        HTTP_REPLY_BAD_REQUEST
-        return;
-    }
-
-    cJSON *json = database_fts_suggest_tag(WebCtx.search_db, body);
-
-    mg_send_json(nc, json);
-    cJSON_Delete(json);
-    free(body);
-}
-
-void fts_get_tags(struct mg_connection *nc, struct mg_http_message *hm) {
-    cJSON *json = database_fts_get_tags(WebCtx.search_db);
-
-    mg_send_json(nc, json);
-    cJSON_Delete(json);
-}

src/web/web_util.c

@@ -61,38 +61,3 @@ void web_send_headers(struct mg_connection *nc, int status_code, size_t length,
             extra_headers
     );
 }
-cJSON *web_get_json_body(struct mg_http_message *hm) {
-    if (hm->body.len == 0) {
-        return NULL;
-    }
-
-    char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.ptr, hm->body.len);
-    *(body + hm->body.len) = '\0';
-    cJSON *json = cJSON_Parse(body);
-    free(body);
-
-    return json;
-}
-
-char *web_get_string_body(struct mg_http_message *hm) {
-    if (hm->body.len == 0) {
-        return NULL;
-    }
-
-    char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.ptr, hm->body.len);
-    *(body + hm->body.len) = '\0';
-
-    return body;
-}
-
-void mg_send_json(struct mg_connection *nc, const cJSON *json) {
-    char *json_str = cJSON_PrintUnformatted(json);
-
-    web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
-    mg_send(nc, json_str, strlen(json_str));
-
-    free(json_str);
-}
-

src/web/web_util.h

@@ -14,9 +14,10 @@ database_t *web_get_database(const char *index_id);
 
 __always_inline
 static char *web_address_to_string(struct mg_addr *addr) {
-    static char address_to_string_buf[INET6_ADDRSTRLEN];
-
-    return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf));
+    return "TODO";
+//    static char address_to_string_buf[INET6_ADDRSTRLEN];
+//
+//    return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf));
 }
 
 void web_send_headers(struct mg_connection *nc, int status_code, size_t length, char *extra_headers);
@@ -28,8 +29,4 @@ void web_serve_asset_favicon_ico(struct mg_connection *nc);
 void web_serve_asset_style_css(struct mg_connection *nc);
 void web_serve_asset_chunk_vendors_css(struct mg_connection *nc);
 
-cJSON *web_get_json_body(struct mg_http_message *hm);
-char *web_get_string_body(struct mg_http_message *hm);
-void mg_send_json(struct mg_connection *nc, const cJSON *json);
-
 #endif //SIST2_WEB_UTIL_H

third-party/libscan/CMakeLists.txt

@@ -106,7 +106,7 @@ find_library(MUPDF_LIB NAMES liblibmupdf.a)
 find_library(CMS_LIB NAMES lcms2)
 find_library(JAS_LIB NAMES jasper)
 find_library(GUMBO_LIB NAMES gumbo)
-find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/8/ /usr/lib/gcc/aarch64-linux-gnu/8/)
+find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/11/)
 find_package(Leptonica CONFIG REQUIRED)
 find_package(FFMPEG REQUIRED)
 find_package(libraw CONFIG REQUIRED)

third-party/libscan/libscan/mobi/scan_mobi.c

@@ -1,44 +1,9 @@
 #include "scan_mobi.h"
 
 #include "../../third-party/libmobi/src/mobi.h"
-#include "../media/media.h"
 #include <errno.h>
 #include "stdlib.h"
 
-int store_cover(scan_mobi_ctx_t *ctx, document_t *doc, MOBIData *m) {
-    MOBIExthHeader *exth = mobi_get_exthrecord_by_tag(m, EXTH_COVEROFFSET);
-
-    if (exth == NULL) {
-        return FALSE;
-    }
-
-    uint32_t offset = mobi_decode_exthvalue(exth->data, exth->size);
-    size_t first_resource = mobi_get_first_resource_record(m);
-    size_t uid = first_resource + offset;
-    MOBIPdbRecord *record = mobi_get_record_by_seqnumber(m, uid);
-
-    if (record == NULL || record->size < 4) {
-        return FALSE;
-    }
-
-    scan_media_ctx_t media_ctx = {
-            .tn_count = TRUE,
-            .tn_size = ctx->tn_size,
-            .tn_qscale = ctx->tn_qscale,
-            .tesseract_lang = NULL,
-            .tesseract_path = NULL,
-            .read_subtitles = FALSE,
-            .max_media_buffer = 0,
-            .log = ctx->log,
-            .logf = ctx->logf,
-            .store = ctx->store,
-    };
-
-    store_image_thumbnail(&media_ctx, record->data, record->size, doc, "img.jpg");
-
-    return TRUE;
-}
-
 void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) {
 
     MOBIData *m = mobi_init();
@@ -107,10 +72,6 @@ void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) {
 
     APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf);
 
-    if (ctx->enable_tn) {
-        store_cover(ctx, doc, m);
-    }
-
     free(content_str);
     free(buf);
     text_buffer_destroy(&tex);

third-party/libscan/libscan/mobi/scan_mobi.h

@@ -7,11 +7,6 @@ typedef struct {
     long content_size;
     log_callback_t log;
     logf_callback_t logf;
-    store_callback_t store;
-
-    int tn_qscale;
-    int tn_size;
-    int enable_tn;
 } scan_mobi_ctx_t;
 
 void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc);