Bug fixes

Bump d3-color and d3 in /sist2-vue

.gitattributes

@@ -1,3 +0,0 @@
-CMakeModules/* linguist-vendored
-**/*_generated.c linguist-vendored
-**/*_generated.h linguist-vendored

CMakeLists.txt

@@ -5,6 +5,7 @@ set(CMAKE_C_STANDARD 11)
 
 option(SIST_DEBUG "Build a debug executable" on)
 option(SIST_FAST "Enable more optimisation flags" off)
+option(SIST_DEBUG_INFO "Turn on debug information in web interface" on)
 
 add_compile_definitions(
         "SIST_PLATFORM=${SIST_PLATFORM}"
@@ -14,7 +15,17 @@ if (SIST_DEBUG)
     add_compile_definitions(
             "SIST_DEBUG=${SIST_DEBUG}"
     )
-endif()
+    set(VCPKG_BUILD_TYPE debug)
+else ()
+    set(VCPKG_BUILD_TYPE release)
+endif ()
+
+if (SIST_DEBUG_INFO)
+    add_compile_definitions(
+            "SIST_DEBUG_INFO=${SIST_DEBUG_INFO}"
+    )
+endif ()
+
 
 add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
@@ -47,7 +58,7 @@ add_executable(sist2
 
         src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp
 
-        src/database/database_stats.c src/database/database_stats.h src/database/database_schema.c)
+        src/database/database_stats.c src/database/database_schema.c)
 set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
@@ -85,7 +96,7 @@ if (SIST_DEBUG)
             -fno-omit-frame-pointer
             -fsanitize=address
             -fno-inline
-#            -O2
+            #            -O2
     )
     target_link_options(
             sist2

Dockerfile

@@ -19,13 +19,12 @@ COPY sist2-admin sist2-admin
 RUN cd sist2-vue/ && npm install && npm run build
 RUN cd sist2-admin/frontend/ && npm install && npm run build
 
-RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
 RUN cd build && make -j$(nproc)
 RUN strip build/sist2 || mv build/sist2_debug build/sist2
 
 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
 
-WORKDIR /root
 
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
@@ -52,6 +51,7 @@ RUN mkdir -p /usr/share/tessdata && \
 COPY --from=build /build/build/sist2 /root/sist2
 
 # sist2-admin
-COPY sist2-admin/requirements.txt sist2-admin/
+WORKDIR /root/sist2-admin
-RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
+COPY sist2-admin/requirements.txt /root/sist2-admin/
-COPY --from=build /build/sist2-admin/ sist2-admin/
+RUN python3 -m pip install --no-cache -r /root/sist2-admin/requirements.txt
+COPY --from=build /build/sist2-admin/ /root/sist2-admin/

Dockerfile.arm64

@@ -3,7 +3,7 @@ MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 ADD . /build/
-RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=arm64_linux_docker -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
 RUN cd build && make -j$(nproc)
 RUN strip build/sist2 || mv build/sist2_debug build/sist2
 

README.md

@@ -10,13 +10,13 @@ sist2 (Simple incremental search tool)
 
 *Warning: sist2 is in early development*
 
-![search panel](docs/sist2.png)
+![search panel](docs/sist2.gif)
 
 ## Features
 
 * Fast, low memory usage, multi-threaded
+* Manage & schedule scan jobs with simple web interface (Docker only)
 * Mobile-friendly Web interface
-* Portable (all its features are packaged in a single executable)
 * Extracts text and metadata from common file types \*
 * Generates thumbnails \*
 * Incremental scanning
@@ -24,47 +24,60 @@ sist2 (Simple incremental search tool)
 * Recursive scan inside archive files \*\*
 * OCR support with tesseract \*\*\*
 * Stats page & disk utilisation visualization
+* Named-entity recognition (client-side) \*\*\*\*
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
 \*\*\* See [OCR](#ocr)
-
+\*\*\*\* See [Named-Entity Recognition](#NER)
-![stats](docs/stats.png)
 
 ## Getting Started
 
+### Using Docker Compose *(Windows/Linux/Mac)*
+
+```yaml
+version: "3"
+
+services:
+  elasticsearch:
+    image: elasticsearch:7.17.9
+    restart: unless-stopped
+    environment:
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+  sist2-admin:
+    image: simon987/sist2:3.0.3
+    restart: unless-stopped
+    volumes:
+      - ./sist2-admin-data/:/sist2-admin/
+      - /:/host
+    ports:
+      - 4090:4090 # sist2
+      - 8080:8080 # sist2-admin
+    working_dir: /root/sist2-admin/
+    entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
+```
+
+Navigate to http://localhost:8080/ to configure sist2-admin.
+
+### Using the executable file *(Linux/WSL only)*
+
 1. Have an Elasticsearch (>= 6.8.X, ideally >=7.14.0) instance running
     1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
-    1. *(or)* Run using docker:
+    2. *(or)* Run using docker:
         ```bash
         docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.17.9
         ```
-    1. *(or)* Run using docker-compose:
-        ```yaml
-          elasticsearch:
-            image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9
-            environment:
-              - discovery.type=single-node
-              - "ES_JAVA_OPTS=-Xms1G -Xmx2G"
-        ```
-1. Download sist2 executable
-    1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases). 
-Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x` *
-    2. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
-       recommended!)*
-    3. *(or)* `docker pull simon987/sist2:2.12.1-x64-linux`
 
-1. See [Usage guide](docs/USAGE.md)
+2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
+   Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
+3. See [usage guide](docs/USAGE.md) for command line usage.
 
-\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
+Example usage:
 
-## Example usage
+1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
-
+2. Push index to Elasticsearch: `sist2 index ./documents.sist2`
-See [Usage guide](docs/USAGE.md) for more details
+3. Start web interface: `sist2 web ./documents.sist2`
-
-1. Scan a directory: `sist2 scan ~/Documents -o ./docs_idx`
-1. Push index to Elasticsearch: `sist2 index ./docs_idx`
-1. Start web interface: `sist2 web ./docs_idx`
 
 ## Format support
 
@@ -82,7 +95,7 @@ See [Usage guide](docs/USAGE.md) for more details
 | tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
 | docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
 | doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
-| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | no          | author, title                                                                                                                          |
+| mobi, azw, azw3                                                           | libmobi                                                                      | yes      | yes         | author, title                                                                                                                          |
 | wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
 | json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
@@ -112,7 +125,7 @@ The `simon987/sist2` image comes with common languages
 (hin, jpn, eng, fra, rus, spa, chi_sim, deu) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
-name must be identical to the `*.traineddata` file installed on your system 
+name must be identical to the `*.traineddata` file installed on your system
 (use `chi_sim` rather than `chi-sim`).
 
 Examples:
@@ -123,20 +136,44 @@ sist2 scan --ocr-images --ocr-lang eng ~/Images/Screenshots/
 sist2 scan --ocr-ebooks --ocr-images --ocr-lang eng+chi_sim ~/Chinese-Bilingual/
 ```
 
+### NER
+
+sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported repository URL to 
+**Configuration** > **Machine learning options** > **Model repositories**
+to enable it.
+
+The text processing is done in your browser, no data is sent to any third-party services.
+See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) for more details.
+
+#### List of available repositories:
+
+| URL                                                                                                     | Maintainer                              | Purpose |
+|---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
+| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
+
+
+<details>
+  <summary>Screenshot</summary>
+
+![ner](docs/ner.png)
+
+</details>
+
 ## Build from source
 
 You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries
 
-### With docker (recommended)
+### Using docker
 
 ```bash
 git clone --recursive https://github.com/simon987/sist2/
 cd sist2
-docker build . -f ./Dockerfile -t my-sist2-image
+docker build . -t my-sist2-image
+# Copy sist2 executable from docker image
 docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 ```
 
-### On a linux computer
+### Using a linux computer
 
 1. Install compile-time dependencies
 
@@ -144,15 +181,14 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
    apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
    ```
 
-1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
+2. Install vcpkg using my fork: https://github.com/simon987/vcpkg
-
+3. Install vcpkg dependencies
-1. Install vcpkg dependencies
 
     ```bash
     vcpkg install curl[core,openssl] sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
     ```
 
-1. Build
+4. Build
     ```bash
     git clone --recursive https://github.com/simon987/sist2/
     (cd sist2-vue; npm install; npm run build)

docs/USAGE.md

@@ -1,78 +1,64 @@
 # Usage
 
-*More examples (specifically with docker/compose) are in progress*
-
-* [scan](#scan)
-    * [options](#scan-options)
-    * [examples](#scan-examples)
-    * [index format](#index-format)
-* [index](#index)
-    * [options](#index-options)
-    * [examples](#index-examples)
-* [web](#web)
-    * [options](#web-options)
-    * [examples](#web-examples)
-    * [rewrite_url](#rewrite_url)
-* [elasticsearch](#elasticsearch)
-* [exec-script](#exec-script)
-* [tagging](#tagging)
-* [sidecar files](#sidecar-files)
-
 ```
 Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
    or: sist2 exec-script [OPTION]... INDEX
+
 Lightning-fast file system indexer and search tool.
 
     -h, --help                        show this help message and exit
-    -v, --version                     Show version and exit
+    -v, --version                     Print version and exit.
-    --verbose                         Turn on logging
+    --verbose                         Turn on logging.
-    --very-verbose                    Turn on debug messages
+    --very-verbose                    Turn on debug messages.
+    --json-logs                       Output logs in JSON format.
 
 Scan options
-    -t, --threads=<int>               Number of threads. DEFAULT=1
+    -t, --threads=<int>               Number of threads. DEFAULT: 1
-    --mem-throttle=<int>              Total memory threshold in MiB for scan throttling. DEFAULT=0
+    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT: 2
-    -q, --thumbnail-quality=<int>     Thumbnail quality, on a scale of 2 to 31, 2 being the best. DEFAULT=2
+    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT: 552
-    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT=500
+    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT: 1
-    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT=1
+    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT: 32768
-    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT=32768
+    -o, --output=<str>                Output index file path. DEFAULT: index.sist2
-    --incremental=<str>               Reuse an existing index and only scan modified files.
+    --incremental                     If the output file path exists, only scan new or modified files.
-    -o, --output=<str>                Output directory. DEFAULT=index.sist2/
+    --optimize-index                  Defragment index file after scan to reduce its file size.
     --rewrite-url=<str>               Serve files from this url instead of from disk.
-    --name=<str>                      Index display name. DEFAULT: (name of the directory)
+    --name=<str>                      Index display name. DEFAULT: index
     --depth=<int>                     Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
+    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: don't scan, list: only save file names as text, shallow: don't scan archives inside archives. DEFAULT: recurse
     --archive-passphrase=<str>        Passphrase for encrypted archive files
     --ocr-lang=<str>                  Tesseract language (use 'tesseract --list-langs' to see which are installed on your machine)
     --ocr-images                      Enable OCR'ing of image files.
     --ocr-ebooks                      Enable OCR'ing of ebook files.
-    -e, --exclude=<str>               Files that match this regex will not be scanned
+    -e, --exclude=<str>               Files that match this regex will not be scanned.
-    --fast                            Only index file names & mime type
+    --fast                            Only index file names & mime type.
     --treemap-threshold=<str>         Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
     --mem-buffer=<int>                Maximum memory buffer size per thread in MiB for files inside archives (see USAGE.md). DEFAULT: 2000
     --read-subtitles                  Read subtitles from media files.
-    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata)
+    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata).
     --checksums                       Calculate file checksums when scanning.
     --list-file=<str>                 Specify a list of newline-delimited paths to be scanned instead of normal directory traversal. Use '-' to read from stdin.
 
 Index options
-    -t, --threads=<int>               Number of threads. DEFAULT=1
+    -t, --threads=<int>               Number of threads. DEFAULT: 1
-    --es-url=<str>                    Elasticsearch url with port. DEFAULT=http://localhost:9200
+    --es-url=<str>                    Elasticsearch url with port. DEFAULT: http://localhost:9200
-    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    -p, --print                       Just print JSON documents to stdout.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
-    --incremental-index               Conduct incremental indexing, assumes that the old index is already digested by Elasticsearch.
+    -p, --print                       Print JSON documents to stdout instead of indexing to elasticsearch.
+    --incremental-index               Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
     --script-file=<str>               Path to user script.
     --mappings-file=<str>             Path to Elasticsearch mappings.
     --settings-file=<str>             Path to Elasticsearch settings.
     --async-script                    Execute user script asynchronously.
-    --batch-size=<int>                Index batch size. DEFAULT: 100
+    --batch-size=<int>                Index batch size. DEFAULT: 70
-    -f, --force-reset                 Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
+    -f, --force-reset                 Reset Elasticsearch mappings and settings.
 
 Web options
-    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
-    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
-    --bind=<str>                      Listen on this address. DEFAULT=localhost:4090
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
+    --bind=<str>                      Listen for connections on this address. DEFAULT: localhost:4090
     --auth=<str>                      Basic auth in user:password format
     --auth0-audience=<str>            API audience/identifier
     --auth0-domain=<str>              Application domain
@@ -84,77 +70,15 @@ Web options
     --lang=<str>                      Default UI language. Can be changed by the user
 
 Exec-script options
-    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-url=<str>                    Elasticsearch url. DEFAULT: http://localhost:9200
-    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --es-insecure-ssl                 Do not verify SSL connections to Elasticsearch.
+    --es-index=<str>                  Elasticsearch index name. DEFAULT: sist2
     --script-file=<str>               Path to user script.
     --async-script                    Execute user script asynchronously.
+
 Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
-## Scan
-
-### Scan options
-
-* `-t, --threads` 
-      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
-* `--mem-throttle`
-    Total memory threshold in MiB for scan throttling. Worker threads will not start a new parse job
-    until the total memory usage of sist2 is below this threshold. Set to 0 to disable. DEFAULT=0
-* `-q, --thumbnail-quality` 
-    Thumbnail quality, on a scale of 2 to 32, 2 being the best. See section below for a rough estimate of thumbnail database size
-* `--thumbnail-size` 
-    Thumbnail size in pixels.
-* `--thumbnail-count`
-    Maximum number of thumbnails to generate. When set to a value >= 2, thumbnails for video previews
-    will be generated. The actual number of thumbnails generated depends on the length of the video (maximum 1 image 
-    every ~7s). Set to 0 to completely disable thumbnails.
-* `--content-size` 
-    Number of bytes of text to be extracted from the content of files (plain text, PDFs etc.).
-    Repeated whitespace and special characters do not count toward this limit.
-    Set to 0 to completely disable content parsing.
-* `--incremental`
-    Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
-    will be copied to the new index and will not be parsed again.
-* `-o, --output` Output directory. 
-* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
-* `--name` Set the `name` option for the web module
-* `--depth` Maximum scan dept. Set to 0 only scan files directly in the root directory, set to -1 for infinite depth
-* `--archive` Archive file mode.
-    * skip: Don't parse
-    * list: Only get file names as text
-    * shallow: Don't parse archives inside archives.
-    * recurse: Scan archives recursively (default)
-* `--ocr-lang`, `--ocr-ebooks`, `--ocr-images` See [OCR](../README.md#OCR)
-* `-e, --exclude` Regex pattern to exclude files. A file is excluded if the pattern matches any 
-    part of the full absolute path.
-    
-    Examples: 
-    * `-e ".*\.ttf"`: Ignore ttf files
-    * `-e ".*\.(ttf|rar)"`: Ignore ttf and rar files
-    * `-e "^/mnt/backups/"`: Ignore all files in the `/mnt/backups/` directory
-    * `-e "^/mnt/Data[12]/"`: Ignore all files in the `/mnt/Data1/` and `/mnt/Data2/` directory
-    * `-e "(^/usr/)|(^/var/)|(^/media/DRIVE-A/tmp/)|(^/media/DRIVE-B/Trash/)"` Exclude the
-     `/usr`, `/var`, `/media/DRIVE-A/tmp`, `/media/DRIVE-B/Trash` directories
-* `--fast` Only index file names and mime type
-* `--treemap-threshold` Directories smaller than (`treemap-threshold` * `<total size of the index>`)
-    will not be considered for the disk utilisation visualization; their size will be added to
-    the parent directory. If the parent directory is still smaller than the threshold, it will also be "merged upwards"
-    and so on.
-    
-    In effect, smaller `treemap-threshold` values will yield a more detailed 
-    (but also a more cluttered and harder to read) visualization. 
-    
-* `--mem-buffer` Maximum memory buffer size in MiB (per thread) for files inside archives. Media files 
-    larger than this number will be read sequentially and no *seek* operations will be supported.
-
-    To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
-* `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
-* `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
-* `--checksums` Calculate file checksums (SHA1) when scanning files. This option does not cause any additional read 
-  operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
-  files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
-
-
 #### Thumbnail database size estimation
 
 See chart below for rough estimate of thumbnail size vs. thumbnail size & quality arguments:
@@ -164,8 +88,6 @@ that is about `8000000 * 36kB = 288GB`.
 
 ![thumbnail_size](thumbnail_size.png)
 
-// TODO: add note about LMDB page size 4096
-
 ### Scan examples
 
 Simple scan
@@ -175,83 +97,20 @@ sist2 scan ~/Documents
 sist2 scan \
     --threads 4 --content-size 16000000 --thumbnail-quality 2 --archive shallow \
     --name "My Documents" --rewrite-url "http://nas.domain.local/My Documents/" \
-    ~/Documents -o ./documents.idx/
+    ~/Documents -o ./documents.sist2
 ```
 
 Incremental scan
-```
+
-sist2 scan --incremental ./orig_idx/ -o ./updated_idx/ ~/Documents
+If the index file does not exist, `--incremental` has no effect.
+```bash
+sist scan ~/Documents -o ./documents.sist2
+sist scan ~/Documents -o ./documents.sist2 --incremental
+# or
+sist scan ~/Documents -o ./documents.sist2 --incremental
+sist scan ~/Documents -o ./documents.sist2 --incremental
 ```
 
-### Index format
-
-A typical `ndjson` type index structure looks like this:
-```
-documents.idx/
-├── descriptor.json
-├── _index_main.ndjson.zst
-├── treemap.csv
-├── agg_mime.csv
-├── agg_date.csv
-├── add_size.csv
-├── thumbs/
-|   ├── data.mdb
-|   └── lock.mdb
-├── tags/
-|   ├── data.mdb
-|   └── lock.mdb
-└── meta/
-    ├── data.mdb
-    └── lock.mdb
-```
-
-The `_index_*.ndjson.zst` files contain the document data in JSON format, in a compressed newline-delemited file.
-
-The `thumbs/` folder is a [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database)
-database containing the thumbnails.
-
-The `descriptor.json` file contains general information about the index. The 
-following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rewrite_url) and `timestamp`.
-
-The `.csv` are pre-computed aggregations necessary for the stats page.
-
-*thumbs/*:
-
-LMDB key-value store. Keys are **binary** 16-byte md5 hash* (`_id` field)
-and values are raw image bytes.
-
-*\* Hash is calculated from the full path of the file, including the extension, relative to the index root*
-
-
-## Index
-### Index options
- * `--es-url` 
- Elasticsearch url and port. If you are using docker, make sure that both containers are on the
- same network.
- * `--es-index` 
-    Elasticsearch index name. DEFAULT=sist2
- * `-p, --print` 
-    Print index in JSON format to stdout.
- * `--incremental-index`
-   Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
-   Only the new changes since the last scan will be sent.
- * `--script-file` 
-    Path to user script. See [Scripting](scripting.md).
- * `--mappings-file`
-    Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used.
- * `--settings-file`
-    Path to custom Elasticsearch settings. *(See above)*
- * `--async-script` 
-    Use `wait_for_completion=false` elasticsearch option while executing user script.
-     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
- * `--batch-size=<int>` 
-    Index batch size. Indexing is generally faster with larger batches, but payloads that
-    are too large will fail and additional overhead for retrying with smaller sizes may slow
-    down the process.
- * `-f, --force-reset` 
-    Reset Elasticsearch mappings and settings.
- * `-t, --threads` Number of threads to use. Ideally, choose a number equal to the number of logical cores of the machine hosting Elasticsearch.
-    
 ### Index examples
 
 **Push to elasticsearch**
@@ -380,8 +239,8 @@ The sidecar file must have exactly the same file path and the `.s2meta` suffix.
 ```
 
 ```
-sist2 scan ~/Documents -o ./docs.idx
+sist2 scan ~/Documents -o ./docs.sist2
-sist2 index ./docs.idx
+sist2 index ./docs.sist2
 ```
 
 *NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,

scripts/build.sh

@@ -7,7 +7,7 @@ git submodule update --init --recursive
 mkdir build
 (
   cd build
-  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
   make -j $(nproc)
   strip sist2
   ./sist2 -v > VERSION
@@ -17,7 +17,7 @@ mv build/sist2 sist2-x64-linux
 (
   cd build
   rm -rf CMakeFiles CMakeCache.txt
-  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
+  cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
   make -j  $(nproc)
 )
 mv build/sist2_debug sist2-x64-linux-debug

scripts/build_arm64.sh

@@ -7,7 +7,7 @@ git submodule update --init --recursive
 mkdir build
 (
   cd build
-  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
   make -j $(nproc)
   strip sist2
 )
@@ -16,7 +16,7 @@ mv build/sist2 sist2-arm64-linux
 rm -rf CMakeFiles CMakeCache.txt
 (
   cd build
-  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+  cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG_INFO=on -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" ..
   make -j $(nproc)
 )
 mv build/sist2_debug sist2-arm64-linux-debug

scripts/start_dev_es_8.sh

@@ -1,3 +1,3 @@
 docker run --rm -it --name "sist2-dev-es"\
        	-p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \
-	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.1.2
+	-e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.7.0

sist2-admin/frontend/package-lock.json

@@ -10491,9 +10491,9 @@
       "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
     },
     "node_modules/webpack": {
-      "version": "5.75.0",
+      "version": "5.78.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.78.0.tgz",
-      "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==",
+      "integrity": "sha512-gT5DP72KInmE/3azEaQrISjTvLYlSM0j1Ezhht/KLVkrqtv10JoP/RXhwmX/frrutOPuSq3o5Vq0ehR/4Vmd1g==",
       "dev": true,
       "dependencies": {
         "@types/eslint-scope": "^3.7.3",
@@ -18719,9 +18719,9 @@
       "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
     },
     "webpack": {
-      "version": "5.75.0",
+      "version": "5.78.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.78.0.tgz",
-      "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==",
+      "integrity": "sha512-gT5DP72KInmE/3azEaQrISjTvLYlSM0j1Ezhht/KLVkrqtv10JoP/RXhwmX/frrutOPuSq3o5Vq0ehR/4Vmd1g==",
       "dev": true,
       "requires": {
         "@types/eslint-scope": "^3.7.3",

sist2-admin/frontend/yarn.lock

@@ -1390,14 +1390,14 @@
     thread-loader "^3.0.0"
     webpack "^5.54.0"
 
-"@vue/cli-plugin-router@~5.0.8":
+"@vue/cli-plugin-router@^5.0.8", "@vue/cli-plugin-router@~5.0.8":
   version "5.0.8"
   resolved "https://registry.npmjs.org/@vue/cli-plugin-router/-/cli-plugin-router-5.0.8.tgz"
   integrity sha512-Gmv4dsGdAsWPqVijz3Ux2OS2HkMrWi1ENj2cYL75nUeL+Xj5HEstSqdtfZ0b1q9NCce+BFB6QnHfTBXc/fCvMg==
   dependencies:
     "@vue/cli-shared-utils" "^5.0.8"
 
-"@vue/cli-plugin-vuex@~5.0.8":
+"@vue/cli-plugin-vuex@^5.0.8", "@vue/cli-plugin-vuex@~5.0.8":
   version "5.0.8"
   resolved "https://registry.npmjs.org/@vue/cli-plugin-vuex/-/cli-plugin-vuex-5.0.8.tgz"
   integrity sha512-HSYWPqrunRE5ZZs8kVwiY6oWcn95qf/OQabwLfprhdpFWAGtLStShjsGED2aDpSSeGAskQETrtR/5h7VqgIlBA==
@@ -5492,9 +5492,9 @@ webpack-virtual-modules@^0.4.2:
   integrity sha512-5tyDlKLqPfMqjT3Q9TAqf2YqjwmnUleZwzJi1A5qXnlBCdj2AtOJ6wAWdglTIDOPgOiOrXeBeFcsQ8+aGQ6QbA==
 
 webpack@^5.54.0:
-  version "5.75.0"
+  version "5.78.0"
-  resolved "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz"
+  resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.78.0.tgz#836452a12416af2a7beae906b31644cb2562f9e6"
-  integrity sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==
+  integrity sha512-gT5DP72KInmE/3azEaQrISjTvLYlSM0j1Ezhht/KLVkrqtv10JoP/RXhwmX/frrutOPuSq3o5Vq0ehR/4Vmd1g==
   dependencies:
     "@types/eslint-scope" "^3.7.3"
     "@types/estree" "^0.0.51"

sist2-vue/package.json

@@ -9,10 +9,11 @@
   "dependencies": {
     "@auth0/auth0-spa-js": "^2.0.2",
     "@egjs/vue-infinitegrid": "3.3.0",
+    "@tensorflow/tfjs": "^4.4.0",
     "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
-    "d3": "^5.6.1",
+    "d3": "^7.8.4",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",
     "fslightbox-vue": "fslightbox-vue.tgz",

sist2-vue/src/App.vue

@@ -1,383 +1,395 @@
 <template>
-  <div id="app" :class="getClass()" v-if="!authLoading">
+    <div id="app" :class="getClass()" v-if="!authLoading">
-    <NavBar></NavBar>
+        <NavBar></NavBar>
-    <router-view v-if="!configLoading"/>
+        <router-view v-if="!configLoading"/>
-  </div>
-  <div class="loading-page" v-else>
-    <div class="loading-spinners">
-      <b-spinner type="grow" variant="primary"></b-spinner>
-      <b-spinner type="grow" variant="primary"></b-spinner>
-      <b-spinner type="grow" variant="primary"></b-spinner>
     </div>
-    <div class="loading-text">
+    <div class="loading-page" v-else>
-      Loading • Chargement • 装载 • Wird geladen
+        <div class="loading-spinners">
+            <b-spinner type="grow" variant="primary"></b-spinner>
+            <b-spinner type="grow" variant="primary"></b-spinner>
+            <b-spinner type="grow" variant="primary"></b-spinner>
+        </div>
+        <div class="loading-text">
+            Loading • Chargement • 装载 • Wird geladen
+        </div>
     </div>
-  </div>
 </template>
 
 <script>
 import NavBar from "@/components/NavBar";
 import {mapActions, mapGetters, mapMutations} from "vuex";
 import Sist2Api from "@/Sist2Api";
+import ModelsRepo from "@/ml/modelsRepo";
 import {setupAuth0} from "@/main";
 
 export default {
-  components: {NavBar},
+    components: {NavBar},
-  data() {
+    data() {
-    return {
+        return {
-      configLoading: false,
+            configLoading: false,
-      authLoading: true,
+            authLoading: true,
-      sist2InfoLoading: true
+            sist2InfoLoading: true
-    }
+        }
-  },
+    },
-  computed: {
+    computed: {
-    ...mapGetters(["optTheme"]),
+        ...mapGetters(["optTheme"]),
-  },
+    },
-  mounted() {
+    mounted() {
-    this.$store.dispatch("loadConfiguration").then(() => {
+        this.$store.dispatch("loadConfiguration").then(() => {
-      this.$root.$i18n.locale = this.$store.state.optLang;
+            this.$root.$i18n.locale = this.$store.state.optLang;
-    });
+            ModelsRepo.init(this.$store.getters.mlRepositoryList).catch(err => {
+                this.$bvToast.toast(
+                    this.$t("ml.repoFetchError"),
+                    {
+                        title: this.$t("ml.repoFetchErrorTitle"),
+                        noAutoHide: true,
+                        toaster: "b-toaster-bottom-right",
+                        headerClass: "toast-header-warning",
+                        bodyClass: "toast-body-warning",
+                    });
+            });
+        });
 
-    this.$store.subscribe((mutation) => {
+        this.$store.subscribe((mutation) => {
-      if (mutation.type === "setOptLang") {
+            if (mutation.type === "setOptLang") {
-        this.$root.$i18n.locale = mutation.payload;
+                this.$root.$i18n.locale = mutation.payload;
-        this.configLoading = true;
+                this.configLoading = true;
-        window.setTimeout(() => this.configLoading = false, 10);
+                window.setTimeout(() => this.configLoading = false, 10);
-      }
-
-      if (mutation.type === "setAuth0Token") {
-        this.authLoading = false;
-      }
-    });
-
-    Sist2Api.getSist2Info().then(data => {
-
-      if (data.auth0Enabled) {
-        this.authLoading = true;
-        setupAuth0(data.auth0Domain, data.auth0ClientId, data.auth0Audience)
-
-        this.$auth.$watch("loading", loading => {
-          if (loading === false) {
-
-            if (!this.$auth.isAuthenticated) {
-              this.$auth.loginWithRedirect();
-              return;
             }
 
-            // Remove "code" param
+            if (mutation.type === "setAuth0Token") {
-            window.history.replaceState({}, "", "/" + window.location.hash);
+                this.authLoading = false;
-
+            }
-            this.$store.dispatch("loadAuth0Token");
-          }
         });
-      } else {
-        this.authLoading = false;
-      }
 
-      this.setSist2Info(data);
+        Sist2Api.getSist2Info().then(data => {
-      this.setIndices(data.indices)
+
-    });
+            if (data.auth0Enabled) {
-  },
+                this.authLoading = true;
-  methods: {
+                setupAuth0(data.auth0Domain, data.auth0ClientId, data.auth0Audience)
-    ...mapActions(["setSist2Info",]),
+
-    ...mapMutations(["setIndices",]),
+                this.$auth.$watch("loading", loading => {
-    getClass() {
+                    if (loading === false) {
-      return {
+
-        "theme-light": this.optTheme === "light",
+                        if (!this.$auth.isAuthenticated) {
-        "theme-black": this.optTheme === "black",
+                            this.$auth.loginWithRedirect();
-      }
+                            return;
+                        }
+
+                        // Remove "code" param
+                        window.history.replaceState({}, "", "/" + window.location.hash);
+
+                        this.$store.dispatch("loadAuth0Token");
+                    }
+                });
+            } else {
+                this.authLoading = false;
+            }
+
+            this.setSist2Info(data);
+            this.setIndices(data.indices)
+        });
+    },
+    methods: {
+        ...mapActions(["setSist2Info",]),
+        ...mapMutations(["setIndices",]),
+        getClass() {
+            return {
+                "theme-light": this.optTheme === "light",
+                "theme-black": this.optTheme === "black",
+            }
+        }
     }
-  }
+    ,
-  ,
 }
 </script>
 
 <style>
 html, body {
-  height: 100%;
+    height: 100%;
 }
 
 #app {
-  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+    /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
-  -webkit-font-smoothing: antialiased;
+    -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
+    -moz-osx-font-smoothing: grayscale;
-  /*text-align: center;*/
+    /*text-align: center;*/
-  color: #2c3e50;
+    color: #2c3e50;
-  padding-bottom: 1em;
+    padding-bottom: 1em;
-  min-height: 100%;
+    min-height: 100%;
 }
 
 /*Black theme*/
 .theme-black {
-  background-color: #000;
+    background-color: #000;
 }
 
 .theme-black .card, .theme-black .modal-content {
-  background: #212121;
+    background: #212121;
-  color: #e0e0e0;
+    color: #e0e0e0;
-  border-radius: 1px;
+    border-radius: 1px;
-  border: none;
+    border: none;
 }
 
 
 .theme-black .table {
-  color: #e0e0e0;
+    color: #e0e0e0;
 }
 
 .theme-black .table td, .theme-black .table th {
-  border: none;
+    border: none;
 }
 
 .theme-black .table thead th {
-  border-bottom: 1px solid #646464;
+    border-bottom: 1px solid #646464;
 }
 
 .theme-black .custom-select {
-  overflow: auto;
+    overflow: auto;
-  background-color: #37474F;
+    background-color: #37474F;
-  border: 1px solid #616161;
+    border: 1px solid #616161;
-  color: #bdbdbd;
+    color: #bdbdbd;
 }
 
 .theme-black .custom-select:focus {
-  border-color: #757575;
+    border-color: #757575;
-  outline: 0;
+    outline: 0;
-  box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
+    box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
 }
 
 .theme-black .inspire-tree .selected > .wholerow, .theme-black .inspire-tree .selected > .title-wrap:hover + .wholerow {
-  background: none !important;
+    background: none !important;
 }
 
 .theme-black .inspire-tree .icon-expand::before, .theme-black .inspire-tree .icon-collapse::before {
-  background-color: black !important;
+    background-color: black !important;
 }
 
 .theme-black .inspire-tree .title {
-  color: #eee;
+    color: #eee;
 }
 
 .theme-black .inspire-tree {
-  font-weight: 400;
+    font-weight: 400;
-  font-size: 14px;
+    font-size: 14px;
-  font-family: Helvetica, Nueue, Verdana, sans-serif;
+    font-family: Helvetica, Nueue, Verdana, sans-serif;
-  max-height: 350px;
+    max-height: 350px;
-  overflow: auto;
+    overflow: auto;
 }
 
 .inspire-tree [type=checkbox] {
-  left: 22px !important;
+    left: 22px !important;
-  top: 7px !important;
+    top: 7px !important;
 }
 
 .theme-black .form-control {
-  background-color: #37474F;
+    background-color: #37474F;
-  border: 1px solid #616161;
+    border: 1px solid #616161;
-  color: #dbdbdb !important;
+    color: #dbdbdb !important;
 }
 
 .theme-black .form-control:focus {
-  background-color: #546E7A;
+    background-color: #546E7A;
-  color: #fff;
+    color: #fff;
 }
 
 .theme-black .input-group-text, .theme-black .default-input {
-  background: #37474F !important;
+    background: #37474F !important;
-  border: 1px solid #616161 !important;
+    border: 1px solid #616161 !important;
-  color: #dbdbdb !important;
+    color: #dbdbdb !important;
 }
 
 .theme-black ::placeholder {
-  color: #BDBDBD !important;
+    color: #BDBDBD !important;
-  opacity: 1;
+    opacity: 1;
 }
 
 .theme-black .nav-tabs .nav-link {
-  color: #e0e0e0;
+    color: #e0e0e0;
-  border-radius: 0;
+    border-radius: 0;
 }
 
 .theme-black .nav-tabs .nav-item.show .nav-link, .theme-black .nav-tabs .nav-link.active {
-  background-color: #212121;
+    background-color: #212121;
-  border-color: #616161 #616161 #212121;
+    border-color: #616161 #616161 #212121;
-  color: #e0e0e0;
+    color: #e0e0e0;
 }
 
 .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:focus {
-  border-color: #616161 #616161 #212121;
+    border-color: #616161 #616161 #212121;
-  color: #e0e0e0;
+    color: #e0e0e0;
 }
 
 .theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:hover {
-  border-color: #e0e0e0 #e0e0e0 #212121;
+    border-color: #e0e0e0 #e0e0e0 #212121;
-  color: #e0e0e0;
+    color: #e0e0e0;
 }
 
 .theme-black .nav-tabs {
-  border-bottom: #616161;
+    border-bottom: #616161;
 }
 
 .theme-black a:hover, .theme-black .btn:hover {
-  color: #fff;
+    color: #fff;
 }
 
 .theme-black .b-dropdown a:hover {
-  color: inherit;
+    color: inherit;
 }
 
 .theme-black .btn {
-  color: #eee;
+    color: #eee;
 }
 
 .theme-black .modal-header .close {
-  color: #e0e0e0;
+    color: #e0e0e0;
-  text-shadow: none;
+    text-shadow: none;
 }
 
 .theme-black .modal-header {
-  border-bottom: 1px solid #646464;
+    border-bottom: 1px solid #646464;
 }
 
 /* -------------------------- */
 
 #nav {
-  padding: 30px;
+    padding: 30px;
 }
 
 #nav a {
-  font-weight: bold;
+    font-weight: bold;
-  color: #2c3e50;
+    color: #2c3e50;
 }
 
 #nav a.router-link-exact-active {
-  color: #42b983;
+    color: #42b983;
 }
 
 .mobile {
-  display: none;
+    display: none;
 }
 
 .container {
-  padding-top: 1em;
+    padding-top: 1em;
 }
 
 @media (max-width: 650px) {
-  .mobile {
+    .mobile {
-    display: initial;
+        display: initial;
-  }
+    }
 
-  .not-mobile {
+    .not-mobile {
-    display: none;
+        display: none;
-  }
+    }
 
-  .grid-single-column .fit {
+    .grid-single-column .fit {
-    max-height: none !important;
+        max-height: none !important;
-  }
+    }
 
-  .container {
+    .container {
-    padding-left: 0;
+        padding-left: 0;
-    padding-right: 0;
+        padding-right: 0;
-    padding-top: 0
+        padding-top: 0
-  }
+    }
 
-  .lightbox-caption {
+    .lightbox-caption {
-    display: none;
+        display: none;
-  }
+    }
 }
 
 .info-icon {
-  width: 1rem;
+    width: 1rem;
-  margin-right: 0.2rem;
+    margin-right: 0.2rem;
-  cursor: pointer;
+    cursor: pointer;
-  line-height: 1rem;
+    line-height: 1rem;
-  height: 1rem;
+    height: 1rem;
-  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+    background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
-  filter: brightness(45%);
+    filter: brightness(45%);
-  display: block;
+    display: block;
 }
 
 .tabs {
-  margin-top: 10px;
+    margin-top: 10px;
 }
 
 .modal-title {
-  text-overflow: ellipsis;
+    text-overflow: ellipsis;
-  overflow: hidden;
+    overflow: hidden;
-  white-space: nowrap;
+    white-space: nowrap;
 }
 
 @media screen and (min-width: 1500px) {
-  .container {
+    .container {
-    max-width: 1440px;
+        max-width: 1440px;
-  }
+    }
 }
 
 .noUi-connects {
-  border-radius: 1px !important;
+    border-radius: 1px !important;
 }
 
 mark {
-  background: #fff217;
+    background: #fff217;
-  border-radius: 0;
+    border-radius: 0;
-  padding: 1px 0;
+    padding: 1px 0;
-  color: inherit;
+    color: inherit;
 }
 
 .theme-black mark {
-  background: rgba(251, 191, 41, 0.25);
+    background: rgba(251, 191, 41, 0.25);
-  border-radius: 0;
+    border-radius: 0;
-  padding: 1px 0;
+    padding: 1px 0;
-  color: inherit;
+    color: inherit;
 }
 
 .theme-black .content-div mark {
-  background: rgba(251, 191, 41, 0.40);
+    background: rgba(251, 191, 41, 0.40);
-  color: white;
+    color: white;
 }
 
 .content-div {
-  font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+    font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
-  font-size: 13px;
+    font-size: 13px;
-  padding: 1em;
+    padding: 1em;
-  background-color: #f5f5f5;
+    background-color: #f5f5f5;
-  border: 1px solid #ccc;
+    border: 1px solid #ccc;
-  border-radius: 4px;
+    border-radius: 4px;
-  margin: 3px;
+    margin: 3px;
-  white-space: normal;
+    white-space: normal;
-  color: #000;
+    color: #000;
-  overflow: hidden;
+    overflow: hidden;
 }
 
 .theme-black .content-div {
-  background-color: #37474F;
+    background-color: #37474F;
-  border: 1px solid #616161;
+    border: 1px solid #616161;
-  color: #E0E0E0FF;
+    color: #E0E0E0FF;
 }
 
 .graph {
-  display: inline-block;
+    display: inline-block;
-  width: 40%;
+    width: 40%;
 }
 
 .pointer {
-  cursor: pointer;
+    cursor: pointer;
 }
 
 .loading-page {
-  display: flex;
+    display: flex;
-  justify-content: center;
+    justify-content: center;
-  align-items: center;
+    align-items: center;
-  flex-direction: column;
+    flex-direction: column;
-  height: 100%;
+    height: 100%;
-  gap: 15px
+    gap: 15px
 }
 
 .loading-spinners {
-  display: flex;
+    display: flex;
-  gap: 10px;
+    gap: 10px;
 }
 
 .loading-text {
-  text-align: center;
+    text-align: center;
 }
 </style>

sist2-vue/src/Sist2Api.ts

@@ -361,20 +361,20 @@ class Sist2Api {
         });
     }
 
-    getTreemapCsvUrl(indexId: string) {
+    getTreemapStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/1`;
+        return `${this.baseUrl}s/${indexId}/TMAP`;
     }
 
-    getMimeCsvUrl(indexId: string) {
+    getMimeStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/2`;
+        return `${this.baseUrl}s/${indexId}/MAGG`;
     }
 
-    getSizeCsv(indexId: string) {
+    getSizeStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/3`;
+        return `${this.baseUrl}s/${indexId}/SAGG`;
     }
 
-    getDateCsv(indexId: string) {
+    getDateStat(indexId: string) {
-        return `${this.baseUrl}s/${indexId}/4`;
+        return `${this.baseUrl}s/${indexId}/DAGG`;
     }
 }
 

sist2-vue/src/components/AnalyzedContentSpan.vue

@@ -0,0 +1,21 @@
+<template>
+    <span :style="getStyle()">{{span.text}}</span>
+</template>
+
+<script>
+
+
+import ModelsRepo from "@/ml/modelsRepo";
+
+export default {
+    name: "AnalyzedContentSpan",
+    props: ["span", "text"],
+    methods: {
+        getStyle() {
+            return ModelsRepo.data[this.$store.getters.mlModel.name].labelStyles[this.span.label];
+        }
+    }
+}
+</script>
+
+<style scoped></style>

sist2-vue/src/components/AnalyzedContentSpanContainer.vue

@@ -0,0 +1,75 @@
+<template>
+    <div>
+        <b-card class="mb-2">
+            <AnalyzedContentSpan v-for="span of legend" :key="span.id" :span="span"
+                                 class="mr-2"></AnalyzedContentSpan>
+        </b-card>
+        <div class="content-div">
+            <AnalyzedContentSpan v-for="span of mergedSpans" :key="span.id" :span="span"></AnalyzedContentSpan>
+        </div>
+    </div>
+</template>
+
+<script>
+
+
+import AnalyzedContentSpan from "@/components/AnalyzedContentSpan.vue";
+import ModelsRepo from "@/ml/modelsRepo";
+
+export default {
+    name: "AnalyzedContentSpanContainer",
+    components: {AnalyzedContentSpan},
+    props: ["spans", "text"],
+    computed: {
+        legend() {
+            return Object.entries(ModelsRepo.data[this.$store.state.mlModel.name].legend)
+                .map(([label, name]) => ({
+                    text: name,
+                    id: label,
+                    label: label
+                }));
+        },
+        mergedSpans() {
+            const spans = this.spans;
+
+            const merged = [];
+
+            let lastLabel = null;
+            let fixSpace = false;
+            for (let i = 0; i < spans.length; i++) {
+
+                if (spans[i].label !== lastLabel) {
+                    let start = spans[i].wordIndex;
+                    const nextSpan = spans.slice(i + 1).find(s => s.label !== spans[i].label)
+                    let end = nextSpan ? nextSpan.wordIndex : undefined;
+
+                    if (end !== undefined && this.text[end - 1] === " ") {
+                        end -= 1;
+                        fixSpace = true;
+                    }
+
+                    merged.push({
+                        text: this.text.slice(start, end),
+                        label: spans[i].label,
+                        id: spans[i].wordIndex
+                    });
+
+                    if (fixSpace) {
+                        merged.push({
+                            text: " ",
+                            label: "O",
+                            id: end
+                        });
+                        fixSpace = false;
+                    }
+                    lastLabel = spans[i].label;
+                }
+            }
+
+            return merged;
+        },
+    },
+}
+</script>
+
+<style scoped></style>

sist2-vue/src/components/D3DateHistogram.vue

@@ -120,7 +120,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#date-histogram");
 
-      d3.csv(Sist2Api.getDateCsv(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getDateStat(indexId)).then(tabularData => {
         dateHistogram(tabularData.slice(), svg, this.$t("d3.dateHistogram"));
       });
     }

sist2-vue/src/components/D3MimeBarCount.vue

@@ -91,7 +91,7 @@ export default {
       const mimeSvgCount = d3.select("#agg-mime-count");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
         mimeBarCount(tabularData.slice(), mimeSvgCount, fillOpacity, this.$t("d3.mimeCount"));
       });
     }

sist2-vue/src/components/D3MimeBarSize.vue

@@ -90,7 +90,7 @@ export default {
       const mimeSvgSize = d3.select("#agg-mime-size");
       const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
 
-      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
         mimeBarSize(tabularData.slice(), mimeSvgSize, fillOpacity, this.$t("d3.mimeSize"));
       });
     }

sist2-vue/src/components/D3SizeHistogram.vue

@@ -117,7 +117,7 @@ export default {
     update(indexId) {
       const svg = d3.select("#size-histogram");
 
-      d3.csv(Sist2Api.getSizeCsv(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getSizeStat(indexId)).then(tabularData => {
         sizeHistogram(tabularData.slice(), svg, this.$t("d3.sizeHistogram"));
       });
     }

sist2-vue/src/components/D3Treemap.vue

@@ -240,7 +240,7 @@ export default {
           .style("overflow", "visible")
           .style("font", "10px sans-serif");
 
-      d3.csv(Sist2Api.getTreemapCsvUrl(indexId)).then(tabularData => {
+      d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
         tabularData.forEach(row => {
           row.taxonomy = row.path.split("/");
           row.size = Number(row.size);

sist2-vue/src/components/DebugInfo.vue

@@ -1,5 +1,5 @@
 <template>
-  <b-card class="mb-4 mt-4">
+  <b-card v-if="$store.state.sist2Info.showDebugInfo" class="mb-4 mt-4">
     <b-card-title><DebugIcon class="mr-1"></DebugIcon>{{ $t("debug") }}</b-card-title>
     <p v-html="$t('debugDescription')"></p>
 

sist2-vue/src/components/FeaturedFieldsLine.vue

@@ -1,42 +1,46 @@
 <template>
-  <div class="featured-line" v-html="featuredLineHtml"></div>
+    <div class="featured-line" v-html="featuredLineHtml"></div>
 </template>
 
 <script>
 import {humanDate, humanFileSize} from "@/util";
 
 function scopedEval(context, expr) {
-  const evaluator = Function.apply(null, [...Object.keys(context), "expr", "return eval(expr)"]);
+    const evaluator = Function.apply(null, [...Object.keys(context), "expr", "return eval(expr)"]);
-  return evaluator.apply(null, [...Object.values(context), expr]);
+    return evaluator.apply(null, [...Object.values(context), expr]);
 }
 
 
 export default {
-  name: "FeaturedFieldsLine",
+    name: "FeaturedFieldsLine",
-  props: ["doc"],
+    props: ["doc"],
-  computed: {
+    computed: {
-    featuredLineHtml() {
+        featuredLineHtml() {
-      const scope = {doc: this.doc._source, humanDate: humanDate, humanFileSize: humanFileSize};
+            if (this.$store.getters.optFeaturedFields === undefined) {
+                return "";
+            }
 
-      return this.$store.getters.optFeaturedFields
+            const scope = {doc: this.doc._source, humanDate: humanDate, humanFileSize: humanFileSize};
-          .replaceAll(/\$\{([^}]*)}/g, (match, g1) => {
+
-            return scopedEval(scope, g1);
+            return this.$store.getters.optFeaturedFields
-          });
+                .replaceAll(/\$\{([^}]*)}/g, (match, g1) => {
+                    return scopedEval(scope, g1);
+                });
+        }
     }
-  }
 }
 </script>
 
 <style scoped>
 
 .featured-line {
-  font-size: 90%;
+    font-size: 90%;
-  font-family: 'Source Sans Pro', 'Helvetica Neue', Arial, sans-serif;
+    font-family: 'Source Sans Pro', 'Helvetica Neue', Arial, sans-serif;
-  color: #424242;
+    color: #424242;
-  padding-left: 2px;
+    padding-left: 2px;
 }
 
 .theme-black .featured-line {
-  color: #bebebe;
+    color: #bebebe;
 }
 </style>

sist2-vue/src/components/LazyContentDiv.vue

@@ -1,6 +1,36 @@
 <template>
-  <Preloader v-if="loading"></Preloader>
+    <Preloader v-if="loading"></Preloader>
-  <div v-else-if="content" class="content-div" v-html="content"></div>
+    <div v-else-if="content">
+        <b-form inline class="my-2" v-if="ModelsRepo.getOptions().length > 0">
+            <b-checkbox class="ml-auto mr-2" :checked="optAutoAnalyze"
+                        @input="setOptAutoAnalyze($event); $store.dispatch('updateConfiguration')">
+                {{ $t("ml.auto") }}
+            </b-checkbox>
+            <b-button :disabled="mlPredictionsLoading || mlLoading" @click="mlAnalyze" variant="primary"
+            >{{ $t("ml.analyzeText") }}
+            </b-button>
+            <b-select :disabled="mlPredictionsLoading || mlLoading" class="ml-2" v-model="mlModel">
+                <b-select-option :value="opt.value" v-for="opt of ModelsRepo.getOptions()">{{ opt.text }}
+                </b-select-option>
+            </b-select>
+        </b-form>
+
+        <b-progress v-if="mlLoading" variant="warning" show-progress :max="1" class="mb-3"
+        >
+            <b-progress-bar :value="modelLoadingProgress">
+                <strong>{{ ((modelLoadingProgress * modelSize) / (1024*1024)).toFixed(1) }}MB / {{
+                    (modelSize / (1024 * 1024)).toFixed(1)
+                    }}MB</strong>
+            </b-progress-bar>
+        </b-progress>
+
+        <b-progress v-if="mlPredictionsLoading" variant="primary" :value="modelPredictionProgress"
+                    :max="content.length" class="mb-3"></b-progress>
+
+        <AnalyzedContentSpansContainer v-if="analyzedContentSpans.length > 0"
+                                       :spans="analyzedContentSpans" :text="rawContent"></AnalyzedContentSpansContainer>
+        <div v-else class="content-div" v-html="content"></div>
+    </div>
 </template>
 
 <script>
@@ -8,87 +38,169 @@ import Sist2Api from "@/Sist2Api";
 import Preloader from "@/components/Preloader";
 import Sist2Query from "@/Sist2Query";
 import store from "@/store";
+import BertNerModel from "@/ml/BertNerModel";
+import AnalyzedContentSpansContainer from "@/components/AnalyzedContentSpanContainer.vue";
+import ModelsRepo from "@/ml/modelsRepo";
+import {mapGetters, mapMutations} from "vuex";
 
 export default {
-  name: "LazyContentDiv",
+    name: "LazyContentDiv",
-  components: {Preloader},
+    components: {AnalyzedContentSpansContainer, Preloader},
-  props: ["docId"],
+    props: ["docId"],
-  data() {
+    data() {
-    return {
+        return {
-      content: "",
+            ModelsRepo,
-      loading: true
+            content: "",
+            rawContent: "",
+            loading: true,
+            modelLoadingProgress: 0,
+            modelPredictionProgress: 0,
+            mlPredictionsLoading: false,
+            mlLoading: false,
+            mlModel: null,
+            analyzedContentSpans: []
+        }
+    },
+    mounted() {
+
+        if (this.$store.getters.optMlDefaultModel) {
+            this.mlModel = this.$store.getters.optMlDefaultModel
+        } else {
+            this.mlModel = ModelsRepo.getDefaultModel();
+        }
+
+        const query = Sist2Query.searchQuery();
+
+        if (this.$store.state.optHighlight) {
+            const fields = this.$store.state.fuzzy
+                ? {"content.nGram": {}}
+                : {content: {}};
+
+            query.highlight = {
+                pre_tags: ["<mark>"],
+                post_tags: ["</mark>"],
+                number_of_fragments: 0,
+                fields,
+            };
+
+            if (!store.state.sist2Info.esVersionLegacy) {
+                query.highlight.max_analyzed_offset = 999_999;
+            }
+        }
+
+        if ("function_score" in query.query) {
+            query.query = query.query.function_score.query;
+        }
+
+        if (!("must" in query.query.bool)) {
+            query.query.bool.must = [];
+        } else if (!Array.isArray(query.query.bool.must)) {
+            query.query.bool.must = [query.query.bool.must];
+        }
+
+        query.query.bool.must.push({match: {_id: this.docId}});
+
+        delete query["sort"];
+        delete query["aggs"];
+        delete query["search_after"];
+        delete query.query["function_score"];
+
+        query._source = {
+            includes: ["content", "name", "path", "extension"]
+        }
+
+        query.size = 1;
+
+        Sist2Api.esQuery(query).then(resp => {
+            this.loading = false;
+            if (resp.hits.hits.length === 1) {
+                this.content = this.getContent(resp.hits.hits[0]);
+            }
+
+            if (this.optAutoAnalyze) {
+                this.mlAnalyze();
+            }
+        });
+    },
+    computed: {
+        ...mapGetters(["optAutoAnalyze"]),
+        modelSize() {
+            const modelData = ModelsRepo.data[this.mlModel];
+            if (!modelData) {
+                return 0;
+            }
+            return modelData.size;
+        }
+    },
+    methods: {
+        ...mapMutations(["setOptAutoAnalyze"]),
+        getContent(doc) {
+            this.rawContent = doc._source.content;
+
+            if (!doc.highlight) {
+                return doc._source.content;
+            }
+
+            if (doc.highlight["content.nGram"]) {
+                return doc.highlight["content.nGram"][0];
+            }
+            if (doc.highlight.content) {
+                return doc.highlight.content[0];
+            }
+        },
+        async getMlModel() {
+            if (this.$store.getters.mlModel.name !== this.mlModel) {
+                this.mlLoading = true;
+                this.modelLoadingProgress = 0;
+                const modelInfo = ModelsRepo.data[this.mlModel];
+
+                const model = new BertNerModel(
+                    modelInfo.vocabUrl,
+                    modelInfo.modelUrl,
+                    modelInfo.id2label,
+                )
+
+                await model.init(progress => this.modelLoadingProgress = progress);
+                this.$store.commit("setMlModel", {model, name: this.mlModel});
+
+                this.mlLoading = false;
+                return model
+            }
+
+            return this.$store.getters.mlModel.model;
+        },
+        async mlAnalyze() {
+            if (!this.content) {
+                return;
+            }
+
+            const modelInfo = ModelsRepo.data[this.mlModel];
+            if (modelInfo === undefined) {
+                return;
+            }
+
+            this.$store.commit("setOptMlDefaultModel", this.mlModel);
+            await this.$store.dispatch("updateConfiguration");
+
+            const model = await this.getMlModel();
+
+            this.analyzedContentSpans = [];
+
+            this.mlPredictionsLoading = true;
+
+            await model.predict(this.rawContent, results => {
+                results.forEach(result => result.label = modelInfo.humanLabels[result.label]);
+                this.analyzedContentSpans.push(...results);
+                this.modelPredictionProgress = results[results.length - 1].wordIndex;
+            });
+            this.mlPredictionsLoading = false;
+        }
     }
-  },
-  mounted() {
-    const query = Sist2Query.searchQuery();
-
-    if (this.$store.state.optHighlight) {
-
-      const fields = this.$store.state.fuzzy
-          ? {"content.nGram": {}}
-          : {content: {}};
-
-      query.highlight = {
-        pre_tags: ["<mark>"],
-        post_tags: ["</mark>"],
-        number_of_fragments: 0,
-        fields,
-      };
-
-      if (!store.state.sist2Info.esVersionLegacy) {
-        query.highlight.max_analyzed_offset = 999_999;
-      }
-    }
-
-    if ("function_score" in query.query) {
-      query.query = query.query.function_score.query;
-    }
-
-    if (!("must" in query.query.bool)) {
-      query.query.bool.must = [];
-    } else if (!Array.isArray(query.query.bool.must)) {
-      query.query.bool.must = [query.query.bool.must];
-    }
-
-    query.query.bool.must.push({match: {_id: this.docId}});
-
-    delete query["sort"];
-    delete query["aggs"];
-    delete query["search_after"];
-    delete query.query["function_score"];
-
-    query._source = {
-      includes: ["content", "name", "path", "extension"]
-    }
-
-    query.size = 1;
-
-    Sist2Api.esQuery(query).then(resp => {
-      this.loading = false;
-      if (resp.hits.hits.length === 1) {
-        this.content = this.getContent(resp.hits.hits[0]);
-      } else {
-        console.log("FIXME: could not get content")
-        console.log(resp)
-      }
-    });
-  },
-  methods: {
-    getContent(doc) {
-      if (!doc.highlight) {
-        return doc._source.content;
-      }
-
-      if (doc.highlight["content.nGram"]) {
-        return doc.highlight["content.nGram"][0];
-      }
-      if (doc.highlight.content) {
-        return doc.highlight.content[0];
-      }
-    }
-  }
 }
 </script>
 
-<style scoped>
+<style>
+.progress-bar {
+    transition: none;
+}
 </style>

sist2-vue/src/i18n/messages.ts

@@ -49,6 +49,7 @@ export default {
         configReset: "Reset configuration",
         searchOptions: "Search options",
         treemapOptions: "Treemap options",
+        mlOptions: "Machine learning options",
         displayOptions: "Display options",
         opt: {
             lang: "Language",
@@ -78,7 +79,10 @@ export default {
             simpleLightbox: "Disable animations in image viewer",
             showTagPickerFilter: "Display the tag filter bar",
             featuredFields: "Featured fields Javascript template string. Will appear in the search results.",
-            featuredFieldsList: "Available variables"
+            featuredFieldsList: "Available variables",
+            autoAnalyze: "Automatically analyze text",
+            defaultModel: "Default model",
+            mlRepositories: "Model repositories (one per line)"
         },
         queryMode: {
             simple: "Simple",
@@ -171,6 +175,12 @@ export default {
             selectedIndex: "selected index",
             selectedIndices: "selected indices",
         },
+        ml: {
+            analyzeText: "Analyze",
+            auto: "Auto",
+            repoFetchError: "Failed to get list of models. Check browser console for more details.",
+            repoFetchErrorTitle: "Could not fetch model repositories",
+        }
     },
     de: {
         filePage: {
@@ -250,8 +260,8 @@ export default {
             vidPreviewInterval: "Videovorschau Framedauer in ms",
             simpleLightbox: "Schalte Animationen im Image-Viewer ab",
             showTagPickerFilter: "Zeige die Tag-Filter-Leiste",
-            featuredFields: "Ausgewählte Felder Javascript Vorlage String. Wird in den Suchergebnissen angezeigt.",
+            featuredFields: "Variablen, welche zusätzlich in den Suchergebnissen angezeigt werden können.",
-            featuredFieldsList: "Verfügbare Variablen"
+            featuredFieldsList: "verfügbare Variablen"
         },
         queryMode: {
             simple: "Einfach",
@@ -333,10 +343,10 @@ export default {
             random: "zufällig",
         },
         d3: {
-            mimeCount: "Anzahlverteilung nach Medientyp",
+            mimeCount: "Anzahl nach Medientyp",
-            mimeSize: "Größenverteilung nach Medientyp",
+            mimeSize: "Größen nach Medientyp",
-            dateHistogram: "Verteilung der Änderungszeiten",
+            dateHistogram: "Änderungszeiten",
-            sizeHistogram: "Verteilung der Dateigrößen",
+            sizeHistogram: "Dateigrößen",
         },
         indexPicker: {
             selectNone: "keinen auswählen",

sist2-vue/src/ml/BertNerModel.js

@@ -0,0 +1,77 @@
+import BertTokenizer from "@/ml/BertTokenizer";
+import * as tf from "@tensorflow/tfjs";
+import axios from "axios";
+
+export default class BertNerModel {
+    vocabUrl;
+    modelUrl;
+
+    id2label;
+    _tokenizer;
+    _model;
+    inputSize = 128;
+
+    _previousWordId = null;
+
+    constructor(vocabUrl, modelUrl, id2label) {
+        this.vocabUrl = vocabUrl;
+        this.modelUrl = modelUrl;
+        this.id2label = id2label;
+    }
+
+    async init(onProgress) {
+        await Promise.all([this.loadTokenizer(), this.loadModel(onProgress)]);
+    }
+
+    async loadTokenizer() {
+        const vocab = (await axios.get(this.vocabUrl)).data;
+        this._tokenizer = new BertTokenizer(vocab);
+    }
+
+    async loadModel(onProgress) {
+        this._model = await tf.loadGraphModel(this.modelUrl, {onProgress});
+    }
+
+    alignLabels(labels, wordIds, words) {
+        const result = [];
+
+        for (let i = 0; i < this.inputSize; i++) {
+            const label = labels[i];
+            const wordId = wordIds[i];
+
+            if (wordId === -1) {
+                continue;
+            }
+            if (wordId === this._previousWordId) {
+                continue;
+            }
+
+            result.push({
+                word: words[wordId].text, wordIndex: words[wordId].index, label: label
+            });
+            this._previousWordId = wordId;
+        }
+
+        return result;
+    }
+
+    async predict(text, callback) {
+        this._previousWordId = null;
+        const encoded = this._tokenizer.encodeText(text, this.inputSize)
+
+        for (let chunk of encoded.inputChunks) {
+            const rawResult = tf.tidy(() => this._model.execute({
+                input_ids: tf.tensor2d(chunk.inputIds, [1, this.inputSize], "int32"),
+                token_type_ids: tf.tensor2d(chunk.segmentIds, [1, this.inputSize], "int32"),
+                attention_mask: tf.tensor2d(chunk.inputMask, [1, this.inputSize], "int32"),
+            }));
+
+            const labelIds = await tf.argMax(rawResult, -1);
+            const labelIdsArray = await labelIds.array();
+            const labels = labelIdsArray[0].map(id => this.id2label[id]);
+            rawResult.dispose()
+
+            callback(this.alignLabels(labels, chunk.wordIds, encoded.words))
+        }
+    }
+}

sist2-vue/src/ml/BertTokenizer.js

@@ -0,0 +1,184 @@
+import {zip, chunk} from "underscore";
+
+const UNK_INDEX = 100;
+const CLS_INDEX = 101;
+const SEP_INDEX = 102;
+const CONTINUING_SUBWORD_PREFIX = "##";
+
+function isWhitespace(ch) {
+    return /\s/.test(ch);
+}
+
+function isInvalid(ch) {
+    return (ch.charCodeAt(0) === 0 || ch.charCodeAt(0) === 0xfffd);
+}
+
+const punctuations = '[~`!@#$%^&*(){}[];:"\'<,.>?/\\|-_+=';
+
+/** To judge whether it's a punctuation. */
+function isPunctuation(ch) {
+    return punctuations.indexOf(ch) !== -1;
+}
+
+export default class BertTokenizer {
+    vocab;
+
+    constructor(vocab) {
+        this.vocab = vocab;
+    }
+
+    tokenize(text) {
+        const charOriginalIndex = [];
+        const cleanedText = this.cleanText(text, charOriginalIndex);
+        const origTokens = cleanedText.split(' ');
+
+        let charCount = 0;
+        const tokens = origTokens.map((token) => {
+            token = token.toLowerCase();
+            const tokens = this.runSplitOnPunctuation(token, charCount, charOriginalIndex);
+            charCount += token.length + 1;
+            return tokens;
+        });
+
+        let flattenTokens = [];
+        for (let index = 0; index < tokens.length; index++) {
+            flattenTokens = flattenTokens.concat(tokens[index]);
+        }
+        return flattenTokens;
+    }
+
+    /* Performs invalid character removal and whitespace cleanup on text. */
+    cleanText(text, charOriginalIndex) {
+        text = text.replace(/\?/g, "").trim();
+
+        const stringBuilder = [];
+        let originalCharIndex = 0;
+        let newCharIndex = 0;
+
+        for (const ch of text) {
+            // Skip the characters that cannot be used.
+            if (isInvalid(ch)) {
+                originalCharIndex += ch.length;
+                continue;
+            }
+            if (isWhitespace(ch)) {
+                if (stringBuilder.length > 0 && stringBuilder[stringBuilder.length - 1] !== ' ') {
+                    stringBuilder.push(' ');
+                    charOriginalIndex[newCharIndex] = originalCharIndex;
+                    originalCharIndex += ch.length;
+                } else {
+                    originalCharIndex += ch.length;
+                    continue;
+                }
+            } else {
+                stringBuilder.push(ch);
+                charOriginalIndex[newCharIndex] = originalCharIndex;
+                originalCharIndex += ch.length;
+            }
+            newCharIndex++;
+        }
+        return stringBuilder.join('');
+    }
+
+    /* Splits punctuation on a piece of text. */
+    runSplitOnPunctuation(text, count, charOriginalIndex) {
+        const tokens = [];
+        let startNewWord = true;
+        for (const ch of text) {
+            if (isPunctuation(ch)) {
+                tokens.push({text: ch, index: charOriginalIndex[count]});
+                count += ch.length;
+                startNewWord = true;
+            } else {
+                if (startNewWord) {
+                    tokens.push({text: '', index: charOriginalIndex[count]});
+                    startNewWord = false;
+                }
+                tokens[tokens.length - 1].text += ch;
+                count += ch.length;
+            }
+        }
+        return tokens;
+    }
+
+    encode(words) {
+        let outputTokens = [];
+        const wordIds = [];
+
+        for (let i = 0; i < words.length; i++) {
+            let chars = [...words[i].text];
+
+            let isUnknown = false;
+            let start = 0;
+            let subTokens = [];
+
+            while (start < chars.length) {
+                let end = chars.length;
+                let currentSubstring = null;
+                while (start < end) {
+                    let substr = chars.slice(start, end).join('');
+
+                    if (start > 0) {
+                        substr = CONTINUING_SUBWORD_PREFIX + substr;
+                    }
+                    if (this.vocab.includes(substr)) {
+                        currentSubstring = this.vocab.indexOf(substr);
+                        break;
+                    }
+
+                    --end;
+                }
+                if (currentSubstring == null) {
+                    isUnknown = true;
+                    break;
+                }
+                subTokens.push(currentSubstring);
+                start = end;
+            }
+
+            if (isUnknown) {
+                outputTokens.push(UNK_INDEX);
+                wordIds.push(i);
+            } else {
+                subTokens.forEach(tok => {
+                    outputTokens.push(tok);
+                    wordIds.push(i)
+                });
+            }
+        }
+
+        return {tokens: outputTokens, wordIds};
+    }
+
+    encodeText(inputText, inputSize) {
+
+        const tokenized = this.tokenize(inputText);
+        const encoded = this.encode(tokenized);
+
+        const encodedTokenChunks = chunk(encoded.tokens, inputSize - 2);
+        const encodedWordIdChunks = chunk(encoded.wordIds, inputSize - 2);
+
+        const chunks = [];
+
+        zip(encodedTokenChunks, encodedWordIdChunks).forEach(([tokens, wordIds]) => {
+            const inputIds = [CLS_INDEX, ...tokens, SEP_INDEX];
+            const segmentIds = Array(inputIds.length).fill(0);
+            const inputMask = Array(inputIds.length).fill(1);
+            wordIds = [-1, ...wordIds, -1];
+
+            while (inputIds.length < inputSize) {
+                inputIds.push(0);
+                inputMask.push(0);
+                segmentIds.push(0);
+                wordIds.push(-1);
+            }
+
+            chunks.push({inputIds, inputMask, segmentIds, wordIds})
+        });
+
+        return {
+            inputChunks: chunks,
+            words: tokenized
+        };
+    }
+}

sist2-vue/src/ml/modelsRepo.js

@@ -0,0 +1,43 @@
+import axios from "axios";
+
+class ModelsRepo {
+    _repositories;
+    data = {};
+
+    async init(repositories) {
+        this._repositories = repositories;
+
+        const data = await Promise.all(this._repositories.map(this._loadRepository));
+
+        data.forEach(models => {
+            models.forEach(model => {
+                this.data[model.name] = model;
+            })
+        });
+    }
+
+    async _loadRepository(repository) {
+        const data = (await axios.get(repository)).data;
+        data.forEach(model => {
+            model["modelUrl"] = new URL(model["modelPath"], repository).href;
+            model["vocabUrl"] = new URL(model["vocabPath"], repository).href;
+        });
+        return data;
+    }
+
+    getOptions() {
+        return Object.values(this.data).map(model => ({
+            text: `${model.name} (${Math.round(model.size / (1024*1024))}MB)`,
+            value: model.name
+        }));
+    }
+
+    getDefaultModel() {
+        if (Object.values(this.data).length === 0) {
+            return null;
+        }
+        return Object.values(this.data).find(model => model.default).name;
+    }
+}
+
+export default new ModelsRepo();

sist2-vue/src/store/index.ts

@@ -57,6 +57,9 @@ export default new Vuex.Store({
         optVidPreviewInterval: 700,
         optSimpleLightbox: true,
         optShowTagPickerFilter: true,
+        optMlRepositories: "https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json",
+        optAutoAnalyze: false,
+        optMlDefaultModel: null,
 
         _onLoadSelectedIndices: [] as string[],
         _onLoadSelectedMimeTypes: [] as string[],
@@ -86,7 +89,11 @@ export default new Vuex.Store({
 
         uiMimeMap: [] as any[],
 
-        auth0Token: null
+        auth0Token: null,
+        mlModel: {
+            model: null,
+            name: null
+        },
     },
     mutations: {
         setUiShowDetails: (state, val) => state.uiShowDetails = val,
@@ -172,6 +179,9 @@ export default new Vuex.Store({
         setOptVidPreviewInterval: (state, val) => state.optVidPreviewInterval = val,
         setOptSimpleLightbox: (state, val) => state.optSimpleLightbox = val,
         setOptShowTagPickerFilter: (state, val) => state.optShowTagPickerFilter = val,
+        setOptAutoAnalyze: (state, val) => {state.optAutoAnalyze = val},
+        setOptMlRepositories: (state, val) => {state.optMlRepositories = val},
+        setOptMlDefaultModel: (state, val) => {state.optMlDefaultModel = val},
 
         setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
         setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val,
@@ -194,6 +204,7 @@ export default new Vuex.Store({
             // noop
         },
         setAuth0Token: (state, val) => state.auth0Token = val,
+        setMlModel: (state, val) => state.mlModel = val,
     },
     actions: {
         setSist2Info: (store, val) => {
@@ -350,6 +361,7 @@ export default new Vuex.Store({
     },
     modules: {},
     getters: {
+        mlModel: (state) => state.mlModel,
         seed: (state) => state.seed,
         getPathText: (state) => state.pathText,
         indices: state => state.indices,
@@ -416,5 +428,12 @@ export default new Vuex.Store({
         optSimpleLightbox: state => state.optSimpleLightbox,
         optShowTagPickerFilter: state => state.optShowTagPickerFilter,
         optFeaturedFields: state => state.optFeaturedFields,
+        optMlRepositories: state => state.optMlRepositories,
+        mlRepositoryList: state => {
+            const repos = state.optMlRepositories.split("\n")
+            return repos[0] == "" ? [] : repos;
+        },
+        optMlDefaultModel: state => state.optMlDefaultModel,
+        optAutoAnalyze: state => state.optAutoAnalyze,
     }
 })

sist2-vue/src/views/Configuration.vue

@@ -1,202 +1,218 @@
 <template>
   <!--  <div :style="{width: `${$store.getters.optContainerWidth}px`}"-->
-  <div
+    <div
-      v-if="!configLoading"
+            v-if="!configLoading"
-      style="margin-left: auto; margin-right: auto;" class="container">
+            style="margin-left: auto; margin-right: auto;" class="container">
-
-    <b-card>
-      <b-card-title>
-        <GearIcon></GearIcon>
-        {{ $t("config") }}
-      </b-card-title>
-      <p>{{ $t("configDescription") }}</p>
-
-      <b-card-body>
-        <h4>{{ $t("displayOptions") }}</h4>
 
         <b-card>
+            <b-card-title>
+                <GearIcon></GearIcon>
+                {{ $t("config") }}
+            </b-card-title>
+            <p>{{ $t("configDescription") }}</p>
 
-          <label>
+            <b-card-body>
-            <LanguageIcon/>
+                <h4>{{ $t("displayOptions") }}</h4>
-            <span style="vertical-align: middle">&nbsp;{{ $t("opt.lang") }}</span></label>
-          <b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
 
-          <label>{{ $t("opt.theme") }}</label>
+                <b-card>
-          <b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
 
-          <label>{{ $t("opt.displayMode") }}</label>
+                    <label>
-          <b-form-select :options="displayModeOptions" :value="optDisplay" @input="setOptDisplay"></b-form-select>
+                        <LanguageIcon/>
+                        <span style="vertical-align: middle">&nbsp;{{ $t("opt.lang") }}</span></label>
+                    <b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
 
-          <label>{{ $t("opt.columns") }}</label>
+                    <label>{{ $t("opt.theme") }}</label>
-          <b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select>
+                    <b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
 
-          <div style="height: 10px"></div>
+                    <label>{{ $t("opt.displayMode") }}</label>
+                    <b-form-select :options="displayModeOptions" :value="optDisplay"
+                                   @input="setOptDisplay"></b-form-select>
 
-          <b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent">
+                    <label>{{ $t("opt.columns") }}</label>
-            {{ $t("opt.lightboxLoadOnlyCurrent") }}
+                    <b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select>
-          </b-form-checkbox>
 
-          <b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy">
+                    <div style="height: 10px"></div>
-            {{ $t("opt.hideLegacy") }}
-          </b-form-checkbox>
 
-          <b-form-checkbox :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
+                    <b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent">
-            {{ $t("opt.updateMimeMap") }}
+                        {{ $t("opt.lightboxLoadOnlyCurrent") }}
-          </b-form-checkbox>
+                    </b-form-checkbox>
 
-          <b-form-checkbox :checked="optUseDatePicker" @input="setOptUseDatePicker">
+                    <b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy">
-            {{ $t("opt.useDatePicker") }}
+                        {{ $t("opt.hideLegacy") }}
-          </b-form-checkbox>
+                    </b-form-checkbox>
 
-          <b-form-checkbox :checked="optSimpleLightbox" @input="setOptSimpleLightbox">{{
+                    <b-form-checkbox :checked="optUpdateMimeMap" @input="setOptUpdateMimeMap">
-              $t("opt.simpleLightbox")
+                        {{ $t("opt.updateMimeMap") }}
-            }}
+                    </b-form-checkbox>
-          </b-form-checkbox>
 
-          <b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{
+                    <b-form-checkbox :checked="optUseDatePicker" @input="setOptUseDatePicker">
-              $t("opt.showTagPickerFilter")
+                        {{ $t("opt.useDatePicker") }}
-            }}
+                    </b-form-checkbox>
-          </b-form-checkbox>
 
-          <br/>
+                    <b-form-checkbox :checked="optSimpleLightbox" @input="setOptSimpleLightbox">{{
-          <label>{{ $t("opt.featuredFields") }}</label>
+                        $t("opt.simpleLightbox")
+                        }}
+                    </b-form-checkbox>
 
-          <br>
+                    <b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{
-          <b-button v-b-toggle.collapse-1 variant="secondary" class="dropdown-toggle">{{
+                        $t("opt.showTagPickerFilter")
-              $t("opt.featuredFieldsList")
+                        }}
-            }}
+                    </b-form-checkbox>
-          </b-button>
-          <b-collapse id="collapse-1" class="mt-2">
-            <ul>
-              <li><code>doc.checksum</code></li>
-              <li><code>doc.path</code></li>
-              <li><code>doc.mime</code></li>
-              <li><code>doc.videoc</code></li>
-              <li><code>doc.audioc</code></li>
-              <li><code>doc.pages</code></li>
-              <li><code>doc.mtime</code></li>
-              <li><code>doc.font_name</code></li>
-              <li><code>doc.album</code></li>
-              <li><code>doc.artist</code></li>
-              <li><code>doc.title</code></li>
-              <li><code>doc.genre</code></li>
-              <li><code>doc.album_artist</code></li>
-              <li><code>doc.exif_make</code></li>
-              <li><code>doc.exif_model</code></li>
-              <li><code>doc.exif_software</code></li>
-              <li><code>doc.exif_exposure_time</code></li>
-              <li><code>doc.exif_fnumber</code></li>
-              <li><code>doc.exif_iso_speed_ratings</code></li>
-              <li><code>doc.exif_focal_length</code></li>
-              <li><code>doc.exif_user_comment</code></li>
-              <li><code>doc.exif_user_comment</code></li>
-              <li><code>doc.exif_gps_longitude_ref</code></li>
-              <li><code>doc.exif_gps_longitude_dms</code></li>
-              <li><code>doc.exif_gps_longitude_dec</code></li>
-              <li><code>doc.exif_gps_latitude_ref</code></li>
-              <li><code>doc.exif_gps_latitude_dec</code></li>
-              <li><code>humanDate()</code></li>
-              <li><code>humanFileSize()</code></li>
-            </ul>
 
-            <p>{{ $t("forExample") }}</p>
+                    <br/>
+                    <label>{{ $t("opt.featuredFields") }}</label>
 
-            <ul>
+                    <br>
-              <li>
+                    <b-button v-b-toggle.collapse-1 variant="secondary" class="dropdown-toggle">{{
-                <code>&lt;b&gt;${humanDate(doc.mtime)}&lt;/b&gt; • ${doc.videoc || ''}</code>
+                        $t("opt.featuredFieldsList")
-              </li>
+                        }}
-              <li>
+                    </b-button>
-                <code>${doc.pages ? (doc.pages + ' pages') : ''}</code>
+                    <b-collapse id="collapse-1" class="mt-2">
-              </li>
+                        <ul>
-            </ul>
+                            <li><code>doc.checksum</code></li>
-          </b-collapse>
+                            <li><code>doc.path</code></li>
-          <br/>
+                            <li><code>doc.mime</code></li>
-          <br/>
+                            <li><code>doc.videoc</code></li>
-          <b-textarea rows="3" :value="optFeaturedFields" @input="setOptFeaturedFields"></b-textarea>
+                            <li><code>doc.audioc</code></li>
+                            <li><code>doc.pages</code></li>
+                            <li><code>doc.mtime</code></li>
+                            <li><code>doc.font_name</code></li>
+                            <li><code>doc.album</code></li>
+                            <li><code>doc.artist</code></li>
+                            <li><code>doc.title</code></li>
+                            <li><code>doc.genre</code></li>
+                            <li><code>doc.album_artist</code></li>
+                            <li><code>doc.exif_make</code></li>
+                            <li><code>doc.exif_model</code></li>
+                            <li><code>doc.exif_software</code></li>
+                            <li><code>doc.exif_exposure_time</code></li>
+                            <li><code>doc.exif_fnumber</code></li>
+                            <li><code>doc.exif_iso_speed_ratings</code></li>
+                            <li><code>doc.exif_focal_length</code></li>
+                            <li><code>doc.exif_user_comment</code></li>
+                            <li><code>doc.exif_user_comment</code></li>
+                            <li><code>doc.exif_gps_longitude_ref</code></li>
+                            <li><code>doc.exif_gps_longitude_dms</code></li>
+                            <li><code>doc.exif_gps_longitude_dec</code></li>
+                            <li><code>doc.exif_gps_latitude_ref</code></li>
+                            <li><code>doc.exif_gps_latitude_dec</code></li>
+                            <li><code>humanDate()</code></li>
+                            <li><code>humanFileSize()</code></li>
+                        </ul>
+
+                        <p>{{ $t("forExample") }}</p>
+
+                        <ul>
+                            <li>
+                                <code>&lt;b&gt;${humanDate(doc.mtime)}&lt;/b&gt; • ${doc.videoc || ''}</code>
+                            </li>
+                            <li>
+                                <code>${doc.pages ? (doc.pages + ' pages') : ''}</code>
+                            </li>
+                        </ul>
+                    </b-collapse>
+                    <br/>
+                    <br/>
+                    <b-textarea rows="3" :value="optFeaturedFields" @input="setOptFeaturedFields"></b-textarea>
+                </b-card>
+
+                <br/>
+                <h4>{{ $t("searchOptions") }}</h4>
+                <b-card>
+                    <b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
+                        $t("opt.hideDuplicates")
+                        }}
+                    </b-form-checkbox>
+
+                    <b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{
+                        $t("opt.highlight")
+                        }}
+                    </b-form-checkbox>
+                    <b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
+                        $t("opt.tagOrOperator")
+                        }}
+                    </b-form-checkbox>
+                    <b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
+                    <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
+                        $t("opt.searchInPath")
+                        }}
+                    </b-form-checkbox>
+                    <b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
+                        $t("opt.suggestPath")
+                        }}
+                    </b-form-checkbox>
+
+                    <br/>
+                    <label>{{ $t("opt.fragmentSize") }}</label>
+                    <b-form-input :value="optFragmentSize" step="10" type="number" min="0"
+                                  @input="setOptFragmentSize"></b-form-input>
+
+                    <label>{{ $t("opt.resultSize") }}</label>
+                    <b-form-input :value="optResultSize" type="number" min="10"
+                                  @input="setOptResultSize"></b-form-input>
+
+                    <label>{{ $t("opt.queryMode") }}</label>
+                    <b-form-select :options="queryModeOptions" :value="optQueryMode"
+                                   @input="setOptQueryMode"></b-form-select>
+
+                    <label>{{ $t("opt.slideDuration") }}</label>
+                    <b-form-input :value="optLightboxSlideDuration" type="number" min="1"
+                                  @input="setOptLightboxSlideDuration"></b-form-input>
+
+                    <label>{{ $t("opt.vidPreviewInterval") }}</label>
+                    <b-form-input :value="optVidPreviewInterval" type="number" min="50"
+                                  @input="setOptVidPreviewInterval"></b-form-input>
+                </b-card>
+
+                <h4 class="mt-3">{{ $t("mlOptions") }}</h4>
+                <b-card>
+                    <label>{{ $t("opt.mlRepositories") }}</label>
+                    <b-textarea rows="3" :value="optMlRepositories" @input="setOptMlRepositories"></b-textarea>
+                    <br>
+                    <b-form-checkbox :checked="optAutoAnalyze" @input="setOptAutoAnalyze">{{
+                            $t("opt.autoAnalyze")
+                        }}
+                    </b-form-checkbox>
+                </b-card>
+
+                <h4 class="mt-3">{{ $t("treemapOptions") }}</h4>
+                <b-card>
+                    <label>{{ $t("opt.treemapType") }}</label>
+                    <b-form-select :value="optTreemapType" :options="treemapTypeOptions"
+                                   @input="setOptTreemapType"></b-form-select>
+
+                    <label>{{ $t("opt.treemapTiling") }}</label>
+                    <b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
+                                   @input="setOptTreemapTiling"></b-form-select>
+
+                    <label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
+                    <b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
+                                  @input="setOptTreemapColorGroupingDepth"></b-form-input>
+
+                    <label>{{ $t("opt.treemapSize") }}</label>
+                    <b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
+                                   @input="setOptTreemapSize"></b-form-select>
+
+                    <template v-if="$store.getters.optTreemapSize === 'custom'">
+                        <!-- TODO Width/Height input -->
+                        <b-form-input type="number" min="0" step="10"></b-form-input>
+                        <b-form-input type="number" min="0" step="10"></b-form-input>
+                    </template>
+
+                    <label>{{ $t("opt.treemapColor") }}</label>
+                    <b-form-select :value="optTreemapColor" :options="treemapColorOptions"
+                                   @input="setOptTreemapColor"></b-form-select>
+                </b-card>
+
+                <b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
+            </b-card-body>
         </b-card>
 
-        <br/>
+        <b-card v-if="loading" class="mt-4">
-        <h4>{{ $t("searchOptions") }}</h4>
+            <Preloader></Preloader>
-        <b-card>
-          <b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
-              $t("opt.hideDuplicates")
-            }}
-          </b-form-checkbox>
-
-          <b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{ $t("opt.highlight") }}</b-form-checkbox>
-          <b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
-              $t("opt.tagOrOperator")
-            }}
-          </b-form-checkbox>
-          <b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
-          <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
-              $t("opt.searchInPath")
-            }}
-          </b-form-checkbox>
-          <b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
-              $t("opt.suggestPath")
-            }}
-          </b-form-checkbox>
-
-          <br/>
-          <label>{{ $t("opt.fragmentSize") }}</label>
-          <b-form-input :value="optFragmentSize" step="10" type="number" min="0"
-                        @input="setOptFragmentSize"></b-form-input>
-
-          <label>{{ $t("opt.resultSize") }}</label>
-          <b-form-input :value="optResultSize" type="number" min="10"
-                        @input="setOptResultSize"></b-form-input>
-
-          <label>{{ $t("opt.queryMode") }}</label>
-          <b-form-select :options="queryModeOptions" :value="optQueryMode" @input="setOptQueryMode"></b-form-select>
-
-          <label>{{ $t("opt.slideDuration") }}</label>
-          <b-form-input :value="optLightboxSlideDuration" type="number" min="1"
-                        @input="setOptLightboxSlideDuration"></b-form-input>
-
-          <label>{{ $t("opt.vidPreviewInterval") }}</label>
-          <b-form-input :value="optVidPreviewInterval" type="number" min="50"
-                        @input="setOptVidPreviewInterval"></b-form-input>
         </b-card>
-
+        <DebugInfo v-else></DebugInfo>
-        <h4 class="mt-3">{{ $t("treemapOptions") }}</h4>
+    </div>
-        <b-card>
-          <label>{{ $t("opt.treemapType") }}</label>
-          <b-form-select :value="optTreemapType" :options="treemapTypeOptions"
-                         @input="setOptTreemapType"></b-form-select>
-
-          <label>{{ $t("opt.treemapTiling") }}</label>
-          <b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
-                         @input="setOptTreemapTiling"></b-form-select>
-
-          <label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
-          <b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
-                        @input="setOptTreemapColorGroupingDepth"></b-form-input>
-
-          <label>{{ $t("opt.treemapSize") }}</label>
-          <b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
-                         @input="setOptTreemapSize"></b-form-select>
-
-          <template v-if="$store.getters.optTreemapSize === 'custom'">
-            <!-- TODO Width/Height input -->
-            <b-form-input type="number" min="0" step="10"></b-form-input>
-            <b-form-input type="number" min="0" step="10"></b-form-input>
-          </template>
-
-          <label>{{ $t("opt.treemapColor") }}</label>
-          <b-form-select :value="optTreemapColor" :options="treemapColorOptions"
-                         @input="setOptTreemapColor"></b-form-select>
-        </b-card>
-
-        <b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
-      </b-card-body>
-    </b-card>
-
-    <b-card v-if="loading" class="mt-4">
-      <Preloader></Preloader>
-    </b-card>
-    <DebugInfo v-else></DebugInfo>
-  </div>
 </template>
 
 <script>
@@ -208,164 +224,168 @@ import GearIcon from "@/components/icons/GearIcon.vue";
 import LanguageIcon from "@/components/icons/LanguageIcon";
 
 export default {
-  components: {LanguageIcon, GearIcon, DebugInfo, Preloader},
+    components: {LanguageIcon, GearIcon, DebugInfo, Preloader},
-  data() {
+    data() {
-    return {
+        return {
-      loading: false,
+            loading: false,
-      configLoading: false,
+            configLoading: false,
-      langOptions: [
+            langOptions: [
-        {value: "en", text: this.$t("lang.en")},
+                {value: "en", text: this.$t("lang.en")},
-        {value: "fr", text: this.$t("lang.fr")},
+                {value: "fr", text: this.$t("lang.fr")},
-        {value: "zh-CN", text: this.$t("lang.zh-CN")},
+                {value: "zh-CN", text: this.$t("lang.zh-CN")},
-        {value: "de", text: this.$t("lang.de")},
+                {value: "de", text: this.$t("lang.de")},
-      ],
+            ],
-      queryModeOptions: [
+            queryModeOptions: [
-        {value: "simple", text: this.$t("queryMode.simple")},
+                {value: "simple", text: this.$t("queryMode.simple")},
-        {value: "advanced", text: this.$t("queryMode.advanced")}
+                {value: "advanced", text: this.$t("queryMode.advanced")}
-      ],
+            ],
-      displayModeOptions: [
+            displayModeOptions: [
-        {value: "grid", text: this.$t("displayMode.grid")},
+                {value: "grid", text: this.$t("displayMode.grid")},
-        {value: "list", text: this.$t("displayMode.list")}
+                {value: "list", text: this.$t("displayMode.list")}
-      ],
+            ],
-      columnsOptions: [
+            columnsOptions: [
-        {value: "auto", text: this.$t("columns.auto")},
+                {value: "auto", text: this.$t("columns.auto")},
-        {value: 1, text: "1"},
+                {value: 1, text: "1"},
-        {value: 2, text: "2"},
+                {value: 2, text: "2"},
-        {value: 3, text: "3"},
+                {value: 3, text: "3"},
-        {value: 4, text: "4"},
+                {value: 4, text: "4"},
-        {value: 5, text: "5"},
+                {value: 5, text: "5"},
-        {value: 6, text: "6"},
+                {value: 6, text: "6"},
-        {value: 7, text: "7"},
+                {value: 7, text: "7"},
-        {value: 8, text: "8"},
+                {value: 8, text: "8"},
-        {value: 9, text: "9"},
+                {value: 9, text: "9"},
-        {value: 10, text: "10"},
+                {value: 10, text: "10"},
-        {value: 11, text: "11"},
+                {value: 11, text: "11"},
-        {value: 12, text: "12"},
+                {value: 12, text: "12"},
-      ],
+            ],
-      treemapTypeOptions: [
+            treemapTypeOptions: [
-        {value: "cascaded", text: this.$t("treemapType.cascaded")},
+                {value: "cascaded", text: this.$t("treemapType.cascaded")},
-        {value: "flat", text: this.$t("treemapType.flat")}
+                {value: "flat", text: this.$t("treemapType.flat")}
-      ],
+            ],
-      treemapTilingOptions: [
+            treemapTilingOptions: [
-        {value: "binary", text: this.$t("treemapTiling.binary")},
+                {value: "binary", text: this.$t("treemapTiling.binary")},
-        {value: "squarify", text: this.$t("treemapTiling.squarify")},
+                {value: "squarify", text: this.$t("treemapTiling.squarify")},
-        {value: "slice", text: this.$t("treemapTiling.slice")},
+                {value: "slice", text: this.$t("treemapTiling.slice")},
-        {value: "dice", text: this.$t("treemapTiling.dice")},
+                {value: "dice", text: this.$t("treemapTiling.dice")},
-        {value: "sliceDice", text: this.$t("treemapTiling.sliceDice")},
+                {value: "sliceDice", text: this.$t("treemapTiling.sliceDice")},
-      ],
+            ],
-      treemapSizeOptions: [
+            treemapSizeOptions: [
-        {value: "small", text: this.$t("treemapSize.small")},
+                {value: "small", text: this.$t("treemapSize.small")},
-        {value: "medium", text: this.$t("treemapSize.medium")},
+                {value: "medium", text: this.$t("treemapSize.medium")},
-        {value: "large", text: this.$t("treemapSize.large")},
+                {value: "large", text: this.$t("treemapSize.large")},
-        {value: "x-large", text: this.$t("treemapSize.xLarge")},
+                {value: "x-large", text: this.$t("treemapSize.xLarge")},
-        {value: "xx-large", text: this.$t("treemapSize.xxLarge")},
+                {value: "xx-large", text: this.$t("treemapSize.xxLarge")},
-        // {value: "custom", text: this.$t("treemapSize.custom")},
+                // {value: "custom", text: this.$t("treemapSize.custom")},
-      ],
+            ],
-      treemapColorOptions: [
+            treemapColorOptions: [
-        {value: "PuBuGn", text: "Purple-Blue-Green"},
+                {value: "PuBuGn", text: "Purple-Blue-Green"},
-        {value: "PuRd", text: "Purple-Red"},
+                {value: "PuRd", text: "Purple-Red"},
-        {value: "PuBu", text: "Purple-Blue"},
+                {value: "PuBu", text: "Purple-Blue"},
-        {value: "YlOrBr", text: "Yellow-Orange-Brown"},
+                {value: "YlOrBr", text: "Yellow-Orange-Brown"},
-        {value: "YlOrRd", text: "Yellow-Orange-Red"},
+                {value: "YlOrRd", text: "Yellow-Orange-Red"},
-        {value: "YlGn", text: "Yellow-Green"},
+                {value: "YlGn", text: "Yellow-Green"},
-        {value: "YlGnBu", text: "Yellow-Green-Blue"},
+                {value: "YlGnBu", text: "Yellow-Green-Blue"},
-        {value: "Plasma", text: "Plasma"},
+                {value: "Plasma", text: "Plasma"},
-        {value: "Magma", text: "Magma"},
+                {value: "Magma", text: "Magma"},
-        {value: "Inferno", text: "Inferno"},
+                {value: "Inferno", text: "Inferno"},
-        {value: "Viridis", text: "Viridis"},
+                {value: "Viridis", text: "Viridis"},
-        {value: "Turbo", text: "Turbo"},
+                {value: "Turbo", text: "Turbo"},
-      ],
+            ],
-      themeOptions: [
+            themeOptions: [
-        {value: "light", text: this.$t("theme.light")},
+                {value: "light", text: this.$t("theme.light")},
-        {value: "black", text: this.$t("theme.black")}
+                {value: "black", text: this.$t("theme.black")}
-      ]
+            ]
 
-    }
+        }
-  },
+    },
-  computed: {
+    computed: {
-    ...mapGetters([
+        ...mapGetters([
-      "optTheme",
+            "optTheme",
-      "optDisplay",
+            "optDisplay",
-      "optColumns",
+            "optColumns",
-      "optHighlight",
+            "optHighlight",
-      "optFuzzy",
+            "optFuzzy",
-      "optSearchInPath",
+            "optSearchInPath",
-      "optSuggestPath",
+            "optSuggestPath",
-      "optFragmentSize",
+            "optFragmentSize",
-      "optQueryMode",
+            "optQueryMode",
-      "optTreemapType",
+            "optTreemapType",
-      "optTreemapTiling",
+            "optTreemapTiling",
-      "optTreemapColorGroupingDepth",
+            "optTreemapColorGroupingDepth",
-      "optTreemapColor",
+            "optTreemapColor",
-      "optTreemapSize",
+            "optTreemapSize",
-      "optLightboxLoadOnlyCurrent",
+            "optLightboxLoadOnlyCurrent",
-      "optLightboxSlideDuration",
+            "optLightboxSlideDuration",
-      "optResultSize",
+            "optResultSize",
-      "optTagOrOperator",
+            "optTagOrOperator",
-      "optLang",
+            "optLang",
-      "optHideDuplicates",
+            "optHideDuplicates",
-      "optHideLegacy",
+            "optHideLegacy",
-      "optUpdateMimeMap",
+            "optUpdateMimeMap",
-      "optUseDatePicker",
+            "optUseDatePicker",
-      "optVidPreviewInterval",
+            "optVidPreviewInterval",
-      "optSimpleLightbox",
+            "optSimpleLightbox",
-      "optShowTagPickerFilter",
+            "optShowTagPickerFilter",
-      "optFeaturedFields",
+            "optFeaturedFields",
-    ]),
+            "optMlRepositories",
-    clientWidth() {
+            "optAutoAnalyze",
-      return window.innerWidth;
+        ]),
-    }
+        clientWidth() {
-  },
+            return window.innerWidth;
-  mounted() {
+        }
-    this.$store.subscribe((mutation) => {
+    },
-      if (mutation.type.startsWith("setOpt")) {
+    mounted() {
-        this.$store.dispatch("updateConfiguration");
+        this.$store.subscribe((mutation) => {
-      }
+            if (mutation.type.startsWith("setOpt")) {
-    });
+                this.$store.dispatch("updateConfiguration");
-  },
+            }
-  methods: {
+        });
-    ...mapActions({
+    },
-      setSist2Info: "setSist2Info",
+    methods: {
-    }),
+        ...mapActions({
-    ...mapMutations([
+            setSist2Info: "setSist2Info",
-      "setOptTheme",
+        }),
-      "setOptDisplay",
+        ...mapMutations([
-      "setOptColumns",
+            "setOptTheme",
-      "setOptHighlight",
+            "setOptDisplay",
-      "setOptFuzzy",
+            "setOptColumns",
-      "setOptSearchInPath",
+            "setOptHighlight",
-      "setOptSuggestPath",
+            "setOptFuzzy",
-      "setOptFragmentSize",
+            "setOptSearchInPath",
-      "setOptQueryMode",
+            "setOptSuggestPath",
-      "setOptTreemapType",
+            "setOptFragmentSize",
-      "setOptTreemapTiling",
+            "setOptQueryMode",
-      "setOptTreemapColorGroupingDepth",
+            "setOptTreemapType",
-      "setOptTreemapColor",
+            "setOptTreemapTiling",
-      "setOptTreemapSize",
+            "setOptTreemapColorGroupingDepth",
-      "setOptLightboxLoadOnlyCurrent",
+            "setOptTreemapColor",
-      "setOptLightboxSlideDuration",
+            "setOptTreemapSize",
-      "setOptResultSize",
+            "setOptLightboxLoadOnlyCurrent",
-      "setOptTagOrOperator",
+            "setOptLightboxSlideDuration",
-      "setOptLang",
+            "setOptResultSize",
-      "setOptHideDuplicates",
+            "setOptTagOrOperator",
-      "setOptHideLegacy",
+            "setOptLang",
-      "setOptUpdateMimeMap",
+            "setOptHideDuplicates",
-      "setOptUseDatePicker",
+            "setOptHideLegacy",
-      "setOptVidPreviewInterval",
+            "setOptUpdateMimeMap",
-      "setOptSimpleLightbox",
+            "setOptUseDatePicker",
-      "setOptShowTagPickerFilter",
+            "setOptVidPreviewInterval",
-      "setOptFeaturedFields",
+            "setOptSimpleLightbox",
-    ]),
+            "setOptShowTagPickerFilter",
-    onResetClick() {
+            "setOptFeaturedFields",
-      localStorage.removeItem("sist2_configuration");
+            "setOptMlRepositories",
-      window.location.reload();
+            "setOptAutoAnalyze",
-    }
+        ]),
-  },
+        onResetClick() {
+            localStorage.removeItem("sist2_configuration");
+            window.location.reload();
+        }
+    },
 }
 </script>
 
 <style>
 .shrink {
-  flex-grow: inherit;
+    flex-grow: inherit;
 }
 </style>

sist2-vue/src/views/SearchPage.vue

@@ -1,57 +1,61 @@
 <template>
-  <div class="container">
+    <div class="container">
-    <Lightbox></Lightbox>
+        <Lightbox></Lightbox>
-    <HelpDialog :show="showHelp" @close="showHelp = false"></HelpDialog>
+        <HelpDialog :show="showHelp" @close="showHelp = false"></HelpDialog>
 
-    <b-card v-if="uiLoading">
+        <b-card v-if="uiLoading">
-      <Preloader></Preloader>
+            <Preloader></Preloader>
-    </b-card>
+        </b-card>
 
-    <b-card v-show="!uiLoading" id="search-panel">
+        <b-alert v-show="!uiLoading && showEsConnectionError" show variant="danger" class="mt-2">
-      <SearchBar @show-help="showHelp=true"></SearchBar>
+            {{ $t("toast.esConnErr") }}
-      <b-row>
+        </b-alert>
-        <b-col style="height: 70px;" sm="6">
-          <SizeSlider></SizeSlider>
-        </b-col>
-        <b-col>
-          <PathTree @search="search(true)"></PathTree>
-        </b-col>
-      </b-row>
-      <b-row>
-        <b-col sm="6">
-          <DateSlider></DateSlider>
-          <b-row>
-            <b-col>
-              <IndexPicker></IndexPicker>
-            </b-col>
-          </b-row>
-        </b-col>
-        <b-col>
-          <b-tabs justified>
-            <b-tab :title="$t('mimeTypes')">
-              <MimePicker></MimePicker>
-            </b-tab>
-            <b-tab :title="$t('tags')">
-              <TagPicker :show-search-bar="$store.state.optShowTagPickerFilter"></TagPicker>
-            </b-tab>
-          </b-tabs>
-        </b-col>
-      </b-row>
-    </b-card>
 
-    <div v-show="docs.length === 0 && !uiLoading">
+        <b-card v-show="!uiLoading && !showEsConnectionError" id="search-panel">
-      <Preloader v-if="searchBusy" class="mt-3"></Preloader>
+            <SearchBar @show-help="showHelp=true"></SearchBar>
+            <b-row>
+                <b-col style="height: 70px;" sm="6">
+                    <SizeSlider></SizeSlider>
+                </b-col>
+                <b-col>
+                    <PathTree @search="search(true)"></PathTree>
+                </b-col>
+            </b-row>
+            <b-row>
+                <b-col sm="6">
+                    <DateSlider></DateSlider>
+                    <b-row>
+                        <b-col>
+                            <IndexPicker></IndexPicker>
+                        </b-col>
+                    </b-row>
+                </b-col>
+                <b-col>
+                    <b-tabs justified>
+                        <b-tab :title="$t('mimeTypes')">
+                            <MimePicker></MimePicker>
+                        </b-tab>
+                        <b-tab :title="$t('tags')">
+                            <TagPicker :show-search-bar="$store.state.optShowTagPickerFilter"></TagPicker>
+                        </b-tab>
+                    </b-tabs>
+                </b-col>
+            </b-row>
+        </b-card>
 
-      <ResultsCard></ResultsCard>
+        <div v-show="docs.length === 0 && !uiLoading">
+            <Preloader v-if="searchBusy" class="mt-3"></Preloader>
+
+            <ResultsCard></ResultsCard>
+        </div>
+
+        <div v-if="docs.length > 0">
+            <ResultsCard></ResultsCard>
+
+            <DocCardWall v-if="optDisplay==='grid'" :docs="docs" :append="appendFunc"></DocCardWall>
+            <DocList v-else :docs="docs" :append="appendFunc"></DocList>
+        </div>
     </div>
-
-    <div v-if="docs.length > 0">
-      <ResultsCard></ResultsCard>
-
-      <DocCardWall v-if="optDisplay==='grid'" :docs="docs" :append="appendFunc"></DocCardWall>
-      <DocList v-else :docs="docs" :append="appendFunc"></DocList>
-    </div>
-  </div>
 </template>
 
 <script lang="ts">
@@ -78,234 +82,253 @@ import HelpDialog from "@/components/HelpDialog.vue";
 
 
 export default Vue.extend({
-  components: {
+    components: {
-    HelpDialog,
+        HelpDialog,
-    DocList,
+        DocList,
-    TagPicker,
+        TagPicker,
-    DateSlider,
+        DateSlider,
-    SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
+        SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
-  },
+    },
-  data: () => ({
+    data: () => ({
-    loading: false,
+        loading: false,
-    uiLoading: true,
+        uiLoading: true,
-    search: undefined as any,
+        search: undefined as any,
-    docs: [] as EsHit[],
+        docs: [] as EsHit[],
-    docIds: new Set(),
+        docIds: new Set(),
-    docChecksums: new Set(),
+        docChecksums: new Set(),
-    searchBusy: false,
+        searchBusy: false,
-    Sist2Query: Sist2Query,
+        Sist2Query: Sist2Query,
-    showHelp: false
+        showHelp: false,
-  }),
+        showEsConnectionError: false
-  computed: {
-    ...mapGetters(["indices", "optDisplay"]),
-  },
-  mounted() {
-    // Handle touch events
-    window.ontouchend = () => this.$store.commit("busTouchEnd");
-    window.ontouchcancel = this.$store.commit("busTouchEnd");
-
-    this.search = _debounce(async (clear: boolean) => {
-      if (clear) {
-        await this.clearResults();
-      }
-
-      await this.searchNow(Sist2Query.searchQuery());
-
-    }, 350, {leading: false});
-
-    this.$store.dispatch("loadFromArgs", this.$route).then(() => {
-      this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
-      this.$store.subscribe((mutation) => {
-        if ([
-          "setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
-          "setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
-          "setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
-        ].includes(mutation.type)) {
-          if (this.searchBusy) {
-            return;
-          }
-
-          this.search(true);
-        }
-      });
-    });
-
-    this.setIndices(this.$store.getters["sist2Info"].indices)
-
-    this.getDateRange().then((range: { min: number, max: number }) => {
-      this.setDateBoundsMin(range.min);
-      this.setDateBoundsMax(range.max);
-
-      const doBlankSearch = !this.$store.state.optUpdateMimeMap;
-
-      Sist2Api.getMimeTypes(Sist2Query.searchQuery(doBlankSearch)).then(({mimeMap}) => {
-        this.$store.commit("setUiMimeMap", mimeMap);
-        this.uiLoading = false;
-        this.search(true);
-      });
-    });
-  },
-  methods: {
-    ...mapActions({
-      setSist2Info: "setSist2Info",
     }),
-    ...mapMutations({
+    computed: {
-      setIndices: "setIndices",
+        ...mapGetters(["indices", "optDisplay"]),
-      setDateBoundsMin: "setDateBoundsMin",
-      setDateBoundsMax: "setDateBoundsMax",
-      setTags: "setTags",
-    }),
-    showErrorToast() {
-      this.$bvToast.toast(
-          this.$t("toast.esConnErr"),
-          {
-            title: this.$t("toast.esConnErrTitle"),
-            noAutoHide: true,
-            toaster: "b-toaster-bottom-right",
-            headerClass: "toast-header-error",
-            bodyClass: "toast-body-error",
-          });
     },
-    showSyntaxErrorToast: function (): void {
+    mounted() {
-      this.$bvToast.toast(
+        // Handle touch events
-          this.$t("toast.esQueryErr"),
+        window.ontouchend = () => this.$store.commit("busTouchEnd");
-          {
+        window.ontouchcancel = this.$store.commit("busTouchEnd");
-            title: this.$t("toast.esQueryErrTitle"),
-            noAutoHide: true,
-            toaster: "b-toaster-bottom-right",
-            headerClass: "toast-header-warning",
-            bodyClass: "toast-body-warning",
-          });
-    },
-    async searchNow(q: any) {
-      this.searchBusy = true;
-      await this.$store.dispatch("incrementQuerySequence");
-      this.$store.commit("busSearch");
 
-      Sist2Api.esQuery(q).then(async (resp: EsResult) => {
+        this.search = _debounce(async (clear: boolean) => {
-        await this.handleSearch(resp);
+            if (clear) {
-        this.searchBusy = false;
+                await this.clearResults();
-      }).catch(err => {
+            }
-        if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
-          this.showSyntaxErrorToast();
-        } else {
-          this.showErrorToast();
-        }
-      });
-    },
-    async clearResults() {
-      this.docs = [];
-      this.docIds.clear();
-      this.docChecksums.clear();
-      await this.$store.dispatch("clearResults");
-      this.$store.commit("setUiReachedScrollEnd", false);
-    },
-    async handleSearch(resp: EsResult) {
-      if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
-        this.$store.commit("setUiReachedScrollEnd", true);
-      }
 
-      resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
+            await this.searchNow(Sist2Query.searchQuery());
 
-      if (this.$store.state.optHideDuplicates) {
+        }, 350, {leading: false});
-        resp.hits.hits = resp.hits.hits.filter(hit => {
 
-          if (!("checksum" in hit._source)) {
+        this.$store.dispatch("loadFromArgs", this.$route).then(() => {
-            return true;
+            this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
-          }
+            this.$store.subscribe((mutation) => {
+                if ([
+                    "setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
+                    "setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
+                    "setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
+                ].includes(mutation.type)) {
+                    if (this.searchBusy) {
+                        return;
+                    }
 
-          const isDupe = !this.docChecksums.has(hit._source.checksum);
+                    this.search(true);
-          this.docChecksums.add(hit._source.checksum);
+                }
-          return isDupe;
+            });
         });
-      }
 
-      for (const hit of resp.hits.hits) {
+        this.setIndices(this.$store.getters["sist2Info"].indices)
-        if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
-          hit._seq = await this.$store.dispatch("getKeySequence");
-          this.$store.commit("addLightboxSource", {
-            source: `f/${hit._id}`,
-            thumbnail: hit._props.hasThumbnail
-                ? `t/${hit._source.index}/${hit._id}`
-                : null,
-            caption: {
-              component: LightboxCaption,
-              props: {hit: hit}
-            },
-            type: hit._props.isVideo ? "video" : "image"
-          });
-        }
-      }
 
-      await this.$store.dispatch("remountLightbox");
+        this.getDateRange().then((range: { min: number, max: number }) => {
-      this.$store.commit("setLastQueryResult", resp);
+            this.setDateBoundsMin(range.min);
+            this.setDateBoundsMax(range.max);
 
-      this.docs.push(...resp.hits.hits);
+            const doBlankSearch = !this.$store.state.optUpdateMimeMap;
 
-      resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
+            Sist2Api.getMimeTypes(Sist2Query.searchQuery(doBlankSearch)).then(({mimeMap}) => {
+                this.$store.commit("setUiMimeMap", mimeMap);
+                this.uiLoading = false;
+                this.search(true);
+            });
+        }).catch(error => {
+            console.log(error);
+
+            if (error.response.status == 503 || error.response.status == 500) {
+                this.showEsConnectionError = true;
+                this.uiLoading = false;
+            } else {
+                this.showErrorToast();
+            }
+        });
     },
-    getDateRange(): Promise<{ min: number, max: number }> {
+    methods: {
-      return sist2.esQuery({
+        ...mapActions({
-        // TODO: filter current selected indices
+            setSist2Info: "setSist2Info",
-        aggs: {
+        }),
-          dateMin: {min: {field: "mtime"}},
+        ...mapMutations({
-          dateMax: {max: {field: "mtime"}},
+            setIndices: "setIndices",
+            setDateBoundsMin: "setDateBoundsMin",
+            setDateBoundsMax: "setDateBoundsMax",
+            setTags: "setTags",
+        }),
+        showErrorToast() {
+            this.$bvToast.toast(
+                this.$t("toast.esConnErr"),
+                {
+                    title: this.$t("toast.esConnErrTitle"),
+                    noAutoHide: true,
+                    toaster: "b-toaster-bottom-right",
+                    headerClass: "toast-header-error",
+                    bodyClass: "toast-body-error",
+                });
         },
-        size: 0
+        showSyntaxErrorToast: function (): void {
-      }).then(res => {
+            this.$bvToast.toast(
-        return {
+                this.$t("toast.esQueryErr"),
-          min: res.aggregations.dateMin.value,
+                {
-          max: res.aggregations.dateMax.value,
+                    title: this.$t("toast.esQueryErrTitle"),
+                    noAutoHide: true,
+                    toaster: "b-toaster-bottom-right",
+                    headerClass: "toast-header-warning",
+                    bodyClass: "toast-body-warning",
+                });
+        },
+        async searchNow(q: any) {
+            this.searchBusy = true;
+            await this.$store.dispatch("incrementQuerySequence");
+            this.$store.commit("busSearch");
+
+            Sist2Api.esQuery(q).then(async (resp: EsResult) => {
+                await this.handleSearch(resp);
+                this.searchBusy = false;
+            }).catch(err => {
+                if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
+                    this.showSyntaxErrorToast();
+                } else {
+                    this.showErrorToast();
+                }
+            });
+        },
+        async clearResults() {
+            this.docs = [];
+            this.docIds.clear();
+            this.docChecksums.clear();
+            await this.$store.dispatch("clearResults");
+            this.$store.commit("setUiReachedScrollEnd", false);
+        },
+        async handleSearch(resp: EsResult) {
+            if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
+                this.$store.commit("setUiReachedScrollEnd", true);
+            }
+
+            resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
+
+            if (this.$store.state.optHideDuplicates) {
+                resp.hits.hits = resp.hits.hits.filter(hit => {
+
+                    if (!("checksum" in hit._source)) {
+                        return true;
+                    }
+
+                    const isDupe = !this.docChecksums.has(hit._source.checksum);
+                    this.docChecksums.add(hit._source.checksum);
+                    return isDupe;
+                });
+            }
+
+            for (const hit of resp.hits.hits) {
+                if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
+                    hit._seq = await this.$store.dispatch("getKeySequence");
+                    this.$store.commit("addLightboxSource", {
+                        source: `f/${hit._id}`,
+                        thumbnail: hit._props.hasThumbnail
+                            ? `t/${hit._source.index}/${hit._id}`
+                            : null,
+                        caption: {
+                            component: LightboxCaption,
+                            props: {hit: hit}
+                        },
+                        type: hit._props.isVideo ? "video" : "image"
+                    });
+                }
+            }
+
+            await this.$store.dispatch("remountLightbox");
+            this.$store.commit("setLastQueryResult", resp);
+
+            this.docs.push(...resp.hits.hits);
+
+            resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
+        },
+        getDateRange(): Promise<{ min: number, max: number }> {
+            return sist2.esQuery({
+                // TODO: filter current selected indices
+                aggs: {
+                    dateMin: {min: {field: "mtime"}},
+                    dateMax: {max: {field: "mtime"}},
+                },
+                size: 0
+            }).then(res => {
+                const range = {
+                    min: res.aggregations.dateMin.value,
+                    max: res.aggregations.dateMax.value,
+                }
+
+                if (range.min == null) {
+                    range.min = 0;
+                    range.max = 1;
+                } else if (range.min == range.max) {
+                    range.max += 1;
+                }
+
+                return range;
+            });
+        },
+        appendFunc() {
+            if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
+                this.searchNow(Sist2Query.searchQuery());
+            }
+        }
+    },
+    beforeRouteUpdate(to, from, next) {
+        if (this.$store.state.uiLightboxIsOpen) {
+            this.$store.commit("_setUiShowLightbox", false);
+            next(false);
+        } else {
+            next();
         }
-      })
     },
-    appendFunc() {
-      if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
-        this.searchNow(Sist2Query.searchQuery());
-      }
-    }
-  },
-  beforeRouteUpdate(to, from, next) {
-    if (this.$store.state.uiLightboxIsOpen) {
-      this.$store.commit("_setUiShowLightbox", false);
-      next(false);
-    } else {
-      next();
-    }
-  },
 })
 </script>
 
 <style>
 
 #search-panel {
-  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
-  border-radius: 0;
+    border-radius: 0;
-  border: none;
+    border: none;
 }
 
 .toast-header-info, .toast-body-info {
-  background: #2196f3;
+    background: #2196f3;
-  color: #fff !important;
+    color: #fff !important;
 }
 
 .toast-header-error, .toast-body-error {
-  background: #a94442;
+    background: #a94442;
-  color: #f2dede !important;
+    color: #f2dede !important;
 }
 
 .toast-header-error {
-  color: #fff !important;
+    color: #fff !important;
-  border-bottom: none;
+    border-bottom: none;
-  margin-bottom: -1em;
+    margin-bottom: -1em;
 }
 
 .toast-header-error .close {
-  text-shadow: none;
+    text-shadow: none;
 }
 
 .toast-header-warning, .toast-body-warning {
-  background: #FF8F00;
+    background: #FF8F00;
-  color: #FFF3E0 !important;
+    color: #FFF3E0 !important;
 }
 </style>

src/database/database.c

@@ -83,6 +83,7 @@ void database_open(database_t *db) {
     LOG_DEBUGF("database.c", "Opening database %s (%d)", db->filename, db->type);
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_open(db->filename, &db->db));
+    sqlite3_busy_timeout(db->db, 1000);
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA cache_size = -200000;", NULL, NULL, NULL));
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(db->db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL));
@@ -328,18 +329,18 @@ database_iterator_t *database_create_document_iterator(database_t *db) {
                                " WHEN sc.json_data IS NULL THEN"
                                "  CASE"
                                "   WHEN t.tag IS NULL THEN"
-                               "    document.json_data"
+                               "    json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime)"
                                "   ELSE"
-                               "    json_set(document.json_data, '$.tag', json_group_array(t.tag))"
+                               "    json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime, '$.tag', json_group_array(t.tag))"
                                "   END"
                                " ELSE"
                                "  CASE"
                                "   WHEN t.tag IS NULL THEN"
-                               "    json_patch(document.json_data, sc.json_data)"
+                               "    json_patch(json_set(document.json_data, '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime), sc.json_data)"
                                "   ELSE"
                                //   This will overwrite any tags specified in the sidecar file!
                                //   TODO: concatenate the two arrays?
-                               "    json_set(json_patch(document.json_data, sc.json_data), '$.tag', json_group_array(t.tag))"
+                               "    json_set(json_patch(document.json_data, sc.json_data), '$._id', document.id, '$.size', document.size, '$.mtime', document.mtime, '$.tag', json_group_array(t.tag))"
                                "   END"
                                " END"
                                " FROM document"
@@ -581,18 +582,33 @@ void database_add_work(database_t *db, job_t *job) {
             ret = sqlite3_step(db->insert_parse_job_stmt);
 
             if (ret == SQLITE_FULL) {
+                sqlite3_reset(db->insert_parse_job_stmt);
+                pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
                 usleep(1000000);
+                pthread_mutex_lock(&db->ipc_ctx->db_mutex);
+                continue;
             } else {
                 CRASH_IF_STMT_FAIL(ret);
             }
 
-            CRASH_IF_NOT_SQLITE_OK(sqlite3_reset(db->insert_parse_job_stmt));
+            ret = sqlite3_reset(db->insert_parse_job_stmt);
-        } while (ret != SQLITE_DONE);
+            if (ret == SQLITE_FULL) {
+                pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
+                usleep(100000);
+                pthread_mutex_lock(&db->ipc_ctx->db_mutex);
+            } else if (ret != SQLITE_OK) {
+                LOG_FATALF("database.c", "sqlite3_reset returned error %d", ret);
+            }
+        } while (ret != SQLITE_DONE && ret != SQLITE_OK);
     } else if (job->type == JOB_BULK_LINE) {
         do {
             sqlite3_bind_text(db->insert_index_job_stmt, 1, job->bulk_line->doc_id, -1, SQLITE_STATIC);
             sqlite3_bind_int(db->insert_index_job_stmt, 2, job->bulk_line->type);
-            sqlite3_bind_text(db->insert_index_job_stmt, 3, job->bulk_line->line, -1, SQLITE_STATIC);
+            if (job->bulk_line->type != ES_BULK_LINE_DELETE) {
+                sqlite3_bind_text(db->insert_index_job_stmt, 3, job->bulk_line->line, -1, SQLITE_STATIC);
+            } else {
+                sqlite3_bind_null(db->insert_index_job_stmt, 3);
+            }
 
             ret = sqlite3_step(db->insert_index_job_stmt);
 
@@ -611,6 +627,8 @@ void database_add_work(database_t *db, job_t *job) {
                 pthread_mutex_unlock(&db->ipc_ctx->db_mutex);
                 usleep(100000);
                 pthread_mutex_lock(&db->ipc_ctx->db_mutex);
+            } else if (ret != SQLITE_OK) {
+                LOG_FATALF("database.c", "sqlite3_reset returned error %d", ret);
             }
 
         } while (ret != SQLITE_DONE && ret != SQLITE_OK);

src/database/database.h

@@ -18,6 +18,14 @@ typedef enum {
     FTS_DATABASE
 } database_type_t;
 
+typedef enum {
+    DATABASE_STAT_INVALID,
+    DATABASE_STAT_TREEMAP,
+    DATABASE_STAT_MIME_AGG,
+    DATABASE_STAT_SIZE_AGG,
+    DATABASE_STAT_DATE_AGG,
+} database_stat_type_d;
+
 typedef enum {
     JOB_UNDEFINED,
     JOB_BULK_LINE,
@@ -104,14 +112,14 @@ database_iterator_t *database_create_document_iterator(database_t *db);
 cJSON *database_document_iter(database_iterator_t *);
 
 #define database_document_iter_foreach(element, iter) \
-    for (cJSON *element = database_document_iter(iter); element != NULL; element = database_document_iter(iter))
+    for (cJSON *(element) = database_document_iter(iter); (element) != NULL; (element) = database_document_iter(iter))
 
 database_iterator_t *database_create_delete_list_iterator(database_t *db);
 
 char * database_delete_list_iter(database_iterator_t *iter);
 
 #define database_delete_list_iter_foreach(element, iter) \
-    for (char *element = database_delete_list_iter(iter); element != NULL; element = database_delete_list_iter(iter))
+    for (char *(element) = database_delete_list_iter(iter); (element) != NULL; (element) = database_delete_list_iter(iter))
 
 
 cJSON *database_incremental_scan_begin(database_t *db);
@@ -132,12 +140,16 @@ treemap_row_t database_treemap_iter(database_iterator_t *iter);
 
 void database_generate_stats(database_t *db, double treemap_threshold);
 
+database_stat_type_d database_get_stat_type_by_mnemonic(const char *name);
+
 job_t *database_get_work(database_t *db, job_type_t job_type);
 
 void database_add_work(database_t *db, job_t *job);
 
 //void database_index(database_t *db);
 
+cJSON *database_get_stats(database_t *db, database_stat_type_d type);
+
 #define CRASH_IF_STMT_FAIL(x) do { \
         int return_value = x;                \
         if (return_value != SQLITE_DONE && return_value != SQLITE_ROW) {     \

src/database/database_stats.c

@@ -6,6 +6,7 @@
 #define SIZE_BUCKET (long)(5 * 1000 * 1000)
 #define DATE_BUCKET (long)(2629800) // ~30 days
 
+
 database_iterator_t *database_create_treemap_iterator(database_t *db, long threshold) {
 
     sqlite3_stmt *stmt;
@@ -157,3 +158,85 @@ void database_generate_stats(database_t *db, double treemap_threshold) {
     LOG_INFO("database.c", "Done!");
 }
 
+database_stat_type_d database_get_stat_type_by_mnemonic(const char *name) {
+    if (strcmp(name, "TMAP") == 0) {
+        return DATABASE_STAT_TREEMAP;
+    }
+    if (strcmp(name, "MAGG") == 0) {
+        return DATABASE_STAT_MIME_AGG;
+    }
+    if (strcmp(name, "SAGG") == 0) {
+        return DATABASE_STAT_SIZE_AGG;
+    }
+    if (strcmp(name, "DAGG") == 0) {
+        return DATABASE_STAT_DATE_AGG;
+    }
+
+    return DATABASE_STAT_INVALID;
+}
+
+cJSON *database_get_stats(database_t *db, database_stat_type_d type) {
+
+    sqlite3_stmt *stmt;
+
+    switch (type) {
+        case DATABASE_STAT_TREEMAP:
+            CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                    db->db, "SELECT path,size FROM stats_treemap", -1, &stmt, NULL
+            ));
+            break;
+        case DATABASE_STAT_DATE_AGG:
+            CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                    db->db, "SELECT bucket,count FROM stats_date_agg", -1, &stmt, NULL
+            ));
+            break;
+        case DATABASE_STAT_SIZE_AGG:
+            CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                    db->db, "SELECT bucket,count FROM stats_size_agg", -1, &stmt, NULL
+            ));
+            break;
+        case DATABASE_STAT_MIME_AGG:
+            CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
+                    db->db, "SELECT mime,size,count FROM stats_mime_agg", -1, &stmt, NULL
+            ));
+            break;
+        case DATABASE_STAT_INVALID:
+        default:
+        LOG_FATALF("database_stats.c", "Invalid stat type: %d", type);
+    }
+
+    cJSON *json = cJSON_CreateArray();
+
+    int ret;
+    do {
+        ret = sqlite3_step(stmt);
+        CRASH_IF_STMT_FAIL(ret);
+
+        if (ret == SQLITE_DONE) {
+            break;
+        }
+
+        cJSON *row = cJSON_CreateObject();
+
+        switch (type) {
+            case DATABASE_STAT_TREEMAP:
+                cJSON_AddStringToObject(row, "path", (const char *) sqlite3_column_text(stmt, 0));
+                cJSON_AddNumberToObject(row, "size", (double) sqlite3_column_int64(stmt, 1));
+                break;
+            case DATABASE_STAT_DATE_AGG:
+            case DATABASE_STAT_SIZE_AGG:
+                cJSON_AddNumberToObject(row, "bucket", (double) sqlite3_column_int64(stmt, 0));
+                cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 1));
+                break;
+            case DATABASE_STAT_MIME_AGG:
+                cJSON_AddStringToObject(row, "mime", (const char *) sqlite3_column_text(stmt, 0));
+                cJSON_AddNumberToObject(row, "size", (double) sqlite3_column_int64(stmt, 1));
+                cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 2));
+                break;
+        }
+
+        cJSON_AddItemToArray(json, row);
+    } while (TRUE);
+
+    return json;
+}

src/database/database_stats.h

@@ -1,5 +0,0 @@
-#ifndef SIST2_DATABASE_STATS_H
-#define SIST2_DATABASE_STATS_H
-
-
-#endif //SIST2_DATABASE_STATS_H

src/index/elastic.c

@@ -64,20 +64,16 @@ void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) {
     cJSON_Delete(line);
 }
 
-void index_json_func(job_t *job) {
-    elastic_index_line(job->bulk_line);
-}
-
 void delete_document(const char *document_id) {
-    es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t));
+    es_bulk_line_t bulk_line;
 
-    bulk_line->type = ES_BULK_LINE_DELETE;
+    bulk_line.type = ES_BULK_LINE_DELETE;
-    bulk_line->next = NULL;
+    bulk_line.next = NULL;
-    strcpy(bulk_line->doc_id, document_id);
+    strcpy(bulk_line.doc_id, document_id);
 
     tpool_add_work(IndexCtx.pool, &(job_t) {
             .type = JOB_BULK_LINE,
-            .bulk_line = bulk_line,
+            .bulk_line = &bulk_line,
     });
 }
 
@@ -99,6 +95,7 @@ void index_json(cJSON *document, const char doc_id[SIST_DOC_ID_LEN]) {
         .type = JOB_BULK_LINE,
         .bulk_line = bulk_line,
     });
+    free(bulk_line);
 }
 
 void execute_update_script(const char *script, int async, const char index_id[SIST_INDEX_ID_LEN]) {

src/io/serialize.c

@@ -91,8 +91,6 @@ char *build_json_string(document_t *doc) {
     } else {
         cJSON_AddStringToObject(json, "mime", mime_text);
     }
-    cJSON_AddNumberToObject(json, "size", (double) doc->size);
-    cJSON_AddNumberToObject(json, "mtime", doc->mtime);
 
     // Ignore root directory in the file path
     doc->ext = (short) (doc->ext - ScanCtx.index.desc.root_len);
@@ -122,8 +120,6 @@ char *build_json_string(document_t *doc) {
         cJSON_AddStringToObject(json, "path", "");
     }
 
-    cJSON_AddStringToObject(json, "_id", doc->doc_id);
-
     // Metadata
     meta_line_t *meta = doc->meta_head;
     while (meta != NULL) {

src/main.c

@@ -195,6 +195,10 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.mobi_ctx.content_size = args->content_size;
     ScanCtx.mobi_ctx.log = log_callback;
     ScanCtx.mobi_ctx.logf = logf_callback;
+    ScanCtx.mobi_ctx.store = write_thumbnail_callback;
+    ScanCtx.mobi_ctx.enable_tn = args->tn_count > 0;
+    ScanCtx.mobi_ctx.tn_size = args->tn_size;
+    ScanCtx.mobi_ctx.tn_qscale = args->tn_quality;
 
     // TEXT
     ScanCtx.text_ctx.content_size = args->content_size;
@@ -312,17 +316,20 @@ void sist2_index(index_args_t *args) {
     database_open(db);
     database_iterator_t *iterator = database_create_document_iterator(db);
     database_document_iter_foreach(json, iterator) {
-        const char *doc_id = cJSON_GetObjectItem(json, "_id")->valuestring;
+        char doc_id[SIST_DOC_ID_LEN];
+        strcpy(doc_id, cJSON_GetObjectItem(json, "_id")->valuestring);
+        cJSON_DeleteItemFromObject(json, "_id");
+
         if (args->print) {
             print_json(json, doc_id);
         } else {
             index_json(json, doc_id);
             cnt += 1;
         }
+        cJSON_Delete(json);
     }
 
     free(iterator);
-    database_close(db, FALSE);
 
     if (!args->print) {
         database_iterator_t *del_iter = database_create_delete_list_iterator(db);
@@ -330,8 +337,11 @@ void sist2_index(index_args_t *args) {
             delete_document(id);
             free(id);
         }
+        free(del_iter);
     }
 
+    database_close(db, FALSE);
+
     tpool_wait(IndexCtx.pool);
     tpool_destroy(IndexCtx.pool);
 

src/sist.h

@@ -51,11 +51,11 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.0.0"
+#define VERSION "3.0.4"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
 static const int VersionMinor = 0;
-static const int VersionPatch = 0;
+static const int VersionPatch = 4;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown

src/tpool.c

@@ -149,6 +149,11 @@ void worker_proc_cleanup(tpool_t *pool) {
     if (ProcData.index_db != NULL) {
         database_close(ProcData.index_db, FALSE);
     }
+
+    if (IndexCtx.needs_es_connection) {
+        elastic_cleanup();
+    }
+
     database_close(ProcData.ipc_db, FALSE);
 }
 
@@ -242,6 +247,7 @@ static void *tpool_worker(void *arg) {
     pthread_mutex_lock(&pool->shm->mutex);
     pthread_cond_signal(&pool->shm->done_working_cond);
     pthread_mutex_unlock(&pool->shm->mutex);
+    worker_proc_cleanup(pool);
 #endif
 
     return NULL;

src/web/serve.c

@@ -20,49 +20,40 @@ static struct mg_http_serve_opts DefaultServeOpts = {
 
 void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
 
-    if (hm->uri.len != SIST_INDEX_ID_LEN + 4) {
+    if (hm->uri.len != SIST_INDEX_ID_LEN + 7) {
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
     char arg_index_id[SIST_INDEX_ID_LEN];
+    char arg_stat_type[5];
+
     memcpy(arg_index_id, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
     *(arg_index_id + SIST_INDEX_ID_LEN - 1) = '\0';
+    memcpy(arg_stat_type, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, 4);
+    *(arg_stat_type + sizeof(arg_stat_type) - 1) = '\0';
 
-    index_t *index = web_get_index_by_id(arg_index_id);
+    database_stat_type_d stat_type = database_get_stat_type_by_mnemonic(arg_stat_type);
-    if (index == NULL) {
+    if (stat_type == DATABASE_STAT_INVALID) {
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
-    const char *file;
+    database_t *db = web_get_database(arg_index_id);
-    switch (atoi(hm->uri.ptr + 3 + SIST_INDEX_ID_LEN)) {
+    if (db == NULL) {
-        case 1:
+        LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index_id);
-            file = "treemap.csv";
+        HTTP_REPLY_NOT_FOUND
-            break;
+        return;
-        case 2:
-            file = "mime_agg.csv";
-            break;
-        case 3:
-            file = "size_agg.csv";
-            break;
-        case 4:
-            file = "date_agg.csv";
-            break;
-        default:
-            return;
     }
 
-    char disposition[8192];
+    cJSON *json = database_get_stats(db, stat_type);
-    snprintf(disposition, sizeof(disposition),
+    char *json_str = cJSON_PrintUnformatted(json);
-             "Content-Disposition: inline; filename=\"%s\"\r\nCache-Control: max-age=31536000\r\n", file);
 
-    char full_path[PATH_MAX];
+    web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
-    strcpy(full_path, index->path);
+    mg_send(nc, json_str, strlen(json_str));
-    strcat(full_path, file);
 
-    struct mg_http_serve_opts opts = {};
+    free(json_str);
-    mg_http_serve_file(nc, hm, full_path, &opts);
+    cJSON_Delete(json);
 }
 
 void serve_index_html(struct mg_connection *nc, struct mg_http_message *hm) {
@@ -286,16 +277,23 @@ void index_info(struct mg_connection *nc) {
     cJSON *json = cJSON_CreateObject();
     cJSON *arr = cJSON_AddArrayToObject(json, "indices");
 
-    cJSON_AddStringToObject(json, "mongooseVersion", MG_VERSION);
     cJSON_AddStringToObject(json, "esIndex", WebCtx.es_index);
     cJSON_AddStringToObject(json, "version", Version);
+
+#ifdef SIST_DEBUG_INFO
+    cJSON_AddStringToObject(json, "mongooseVersion", MG_VERSION);
     cJSON_AddStringToObject(json, "esVersion", es_version);
-    cJSON_AddBoolToObject(json, "esVersionSupported", IS_SUPPORTED_ES_VERSION(WebCtx.es_version));
-    cJSON_AddBoolToObject(json, "esVersionLegacy", IS_LEGACY_VERSION(WebCtx.es_version));
     cJSON_AddStringToObject(json, "platform", QUOTE(SIST_PLATFORM));
     cJSON_AddStringToObject(json, "sist2Hash", Sist2CommitHash);
-    cJSON_AddStringToObject(json, "lang", WebCtx.lang);
     cJSON_AddBoolToObject(json, "dev", WebCtx.dev);
+    cJSON_AddBoolToObject(json, "showDebugInfo", TRUE);
+#else
+    cJSON_AddBoolToObject(json, "showDebugInfo", FALSE);
+#endif
+
+    cJSON_AddBoolToObject(json, "esVersionSupported", IS_SUPPORTED_ES_VERSION(WebCtx.es_version));
+    cJSON_AddBoolToObject(json, "esVersionLegacy", IS_LEGACY_VERSION(WebCtx.es_version));
+    cJSON_AddStringToObject(json, "lang", WebCtx.lang);
 
     cJSON_AddBoolToObject(json, "auth0Enabled", WebCtx.auth0_enabled);
     if (WebCtx.auth0_enabled) {
@@ -668,6 +666,9 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
                     mg_send(nc, r->body, r->size);
                 } else if (r->status_code == 0) {
                     sist_log("serve.c", LOG_SIST_ERROR, "Could not connect to elasticsearch!");
+
+                    mg_http_reply(nc, 503, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER,
+                                  "Elasticsearch connection error, see server logs.");
                 } else {
                     sist_logf("serve.c", LOG_SIST_WARNING, "ElasticSearch error during query (%d)", r->status_code);
                     if (r->size != 0) {

third-party/libscan/CMakeLists.txt

@@ -106,7 +106,7 @@ find_library(MUPDF_LIB NAMES liblibmupdf.a)
 find_library(CMS_LIB NAMES lcms2)
 find_library(JAS_LIB NAMES jasper)
 find_library(GUMBO_LIB NAMES gumbo)
-find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/11/)
+find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/8/ /usr/lib/gcc/aarch64-linux-gnu/8/)
 find_package(Leptonica CONFIG REQUIRED)
 find_package(FFMPEG REQUIRED)
 find_package(libraw CONFIG REQUIRED)

third-party/libscan/libscan/mobi/scan_mobi.c

@@ -1,9 +1,44 @@
 #include "scan_mobi.h"
 
 #include "../../third-party/libmobi/src/mobi.h"
+#include "../media/media.h"
 #include <errno.h>
 #include "stdlib.h"
 
+int store_cover(scan_mobi_ctx_t *ctx, document_t *doc, MOBIData *m) {
+    MOBIExthHeader *exth = mobi_get_exthrecord_by_tag(m, EXTH_COVEROFFSET);
+
+    if (exth == NULL) {
+        return FALSE;
+    }
+
+    uint32_t offset = mobi_decode_exthvalue(exth->data, exth->size);
+    size_t first_resource = mobi_get_first_resource_record(m);
+    size_t uid = first_resource + offset;
+    MOBIPdbRecord *record = mobi_get_record_by_seqnumber(m, uid);
+
+    if (record == NULL || record->size < 4) {
+        return FALSE;
+    }
+
+    scan_media_ctx_t media_ctx = {
+            .tn_count = TRUE,
+            .tn_size = ctx->tn_size,
+            .tn_qscale = ctx->tn_qscale,
+            .tesseract_lang = NULL,
+            .tesseract_path = NULL,
+            .read_subtitles = FALSE,
+            .max_media_buffer = 0,
+            .log = ctx->log,
+            .logf = ctx->logf,
+            .store = ctx->store,
+    };
+
+    store_image_thumbnail(&media_ctx, record->data, record->size, doc, "img.jpg");
+
+    return TRUE;
+}
+
 void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) {
 
     MOBIData *m = mobi_init();
@@ -72,6 +107,10 @@ void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc) {
 
     APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf);
 
+    if (ctx->enable_tn) {
+        store_cover(ctx, doc, m);
+    }
+
     free(content_str);
     free(buf);
     text_buffer_destroy(&tex);

third-party/libscan/libscan/mobi/scan_mobi.h

@@ -7,6 +7,11 @@ typedef struct {
     long content_size;
     log_callback_t log;
     logf_callback_t logf;
+    store_callback_t store;
+
+    int tn_qscale;
+    int tn_size;
+    int enable_tn;
 } scan_mobi_ctx_t;
 
 void parse_mobi(scan_mobi_ctx_t *ctx, vfile_t *f, document_t *doc);