Merge pull request #200 from simon987/dev

v2.11.3

.dockerignore

@@ -0,0 +1,30 @@
+.idea/
+*.sist2
+docs/
+test_i/
+test_i_inc/
+Testing/
+.drone.yml
+**/cmake_install.cmake
+**/CMakeCache.txt
+**/CMakeFiles/
+LICENSE
+Makefile
+**/*.md
+**/*.cbp
+VERSION
+**/node_modules/
+.git/
+sist2-*-linux-debug
+sist2-*-linux
+sist2_debug
+sist2
+**/libscan-test-files
+**/scan_ub_test
+**/scan_a_test
+**/scan_test
+**/ext_ffmpeg
+**/ext_libmobi
+**/ext_libwpd
+**/core
+*.a

.drone.yml

@@ -0,0 +1,88 @@
+kind: pipeline
+type: docker
+name: amd64
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+  - name: build
+    image: simon987/sist2-build
+    commands:
+      - ./ci/build.sh
+  - name: docker
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: simon987/sist2
+      context: ./
+      dockerfile: ./Dockerfile
+      auto_tag: true
+      auto_tag_suffix: x64-linux
+      when:
+        event:
+          - tag
+  - name: scp files
+    image: appleboy/drone-scp
+    settings:
+      host:
+        from_secret: SSH_HOST
+      port:
+        from_secret: SSH_PORT
+      user:
+        from_secret: SSH_USER
+      key:
+        from_secret: SSH_KEY
+      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      source:
+        - ./VERSION
+        - ./sist2-x64-linux
+        - ./sist2-x64-linux-debug
+
+---
+kind: pipeline
+type: docker
+name: arm64
+
+platform:
+  arch: arm64
+
+steps:
+  - name: build
+    image: simon987/sist2-build-arm64
+    commands:
+      - ./ci/build_arm64.sh
+  - name: scp files
+    image: appleboy/drone-scp
+    settings:
+      host:
+        from_secret: SSH_HOST
+      port:
+        from_secret: SSH_PORT
+      user:
+        from_secret: SSH_USER
+      key:
+        from_secret: SSH_KEY
+      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      source:
+        - ./sist2-arm64-linux
+        - ./sist2-arm64-linux-debug
+  - name: docker
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: simon987/sist2
+      context: ./
+      dockerfile: ./Dockerfile.arm64
+      auto_tag: true
+      auto_tag_suffix: arm64-linux
+      when:
+        event:
+          - tag

.gitattributes

@@ -1,3 +1,3 @@
 CMakeModules/* linguist-vendored
-web/js/*.min.js linguist-vendored
-web/css/*.min.css linguist-vendored
+**/*_generated.c linguist-vendored
+**/*_generated.h linguist-vendored

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,40 @@
+---
+name: "🐞 Bug Report"
+about: Submit a bug report 
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Device Information (please complete the following information):**
+ - OS: `[e.g., Ubuntu 20.04, WSL2]`
+ - Deployment: `[Linux, Linux ARM64 or Docker]`
+ - Browser *(if relevant)*: `[e.g., chrome, safari]`
+ - SIST2 Version: `[e.g., v2.9.0]`
+ - Elasticsearch Version *(if relevant)* : ``
+
+**Command with arguments** 
+<!-- `ex: "scan ~/Documents -o ./i2 --threads 3 -q 1.0` -->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**Steps To Reproduce**
+Please be specific!
+1. Go to '...'
+2. Click on '....'
+3. etc.
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Actual Behavior**
+<!-- A clear and concise description of what actually happens. -->
+
+**Screenshots**
+<!-- If applicable, add screenshots to help explain your problem. -->
+
+**Additional context**
+<!-- Add any other context about the problem here. If applicable, please include why you think the bug is occurring and/or troubleshooting you have already performed. -->
+<!-- If the issue is related to the `scan` module, please attach the files necessary to reproduce the error or email them to me[at]simon987.net. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: SIST2 Documentation
+    url: https://github.com/simon987/sist2/blob/master/docs/USAGE.md
+    about: Check out the SIST2 documentation for answers to common questions

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: "🚀 Feature Request"
+about: Suggest an idea for SIST2
+title: ''
+assignees: ''
+
+---
+**Which SIST2 component is your Feature Request related to?**
+<!-- e.g., Scan, Index, or Web? -->
+
+**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what the problem is. e.g., "I'm always frustrated when [...]" -->
+
+**What would you like to see happen?**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/issue-template.md

@@ -9,7 +9,9 @@ assignees: ''
 
 sist2 version:
 
-Platform (please indicate if you're using Docker):
+Platform (Linux or Docker, x86-64 or arm64):
+
+Elasticsearch version:
 
 Command with arguments:  `ex: "scan ~/Documents -o ./i2 --threads 3 -q 1.0`
 

.gitignore

@@ -1,6 +1,5 @@
 .idea
 thumbs
-test
 *.cbp
 CMakeCache.txt
 CMakeFiles
@@ -10,6 +9,7 @@ Makefile
 *.out
 LOG
 sist2*
+!sist2-vue/
 index.sist2/
 bundle*.css
 bundle.js
@@ -17,3 +17,10 @@ bundle.js
 vgcore.*
 build/
 third-party/
+*.idx/
+VERSION
+git_hash.h
+Testing/
+test_i
+test_i_inc
+node_modules/

.teamcity/settings.kts

@@ -1,69 +0,0 @@
-import jetbrains.buildServer.configs.kotlin.v2019_2.*
-import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.ExecBuildStep
-import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.exec
-import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs
-import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot
-
-/*
-The settings script is an entry point for defining a TeamCity
-project hierarchy. The script should contain a single call to the
-project() function with a Project instance or an init function as
-an argument.
-
-VcsRoots, BuildTypes, Templates, and subprojects can be
-registered inside the project using the vcsRoot(), buildType(),
-template(), and subProject() methods respectively.
-
-To debug settings scripts in command-line, run the
-
-    mvnDebug org.jetbrains.teamcity:teamcity-configs-maven-plugin:generate
-
-command and attach your debugger to the port 8000.
-
-To debug in IntelliJ Idea, open the 'Maven Projects' tool window (View
--> Tool Windows -> Maven Projects), find the generate task node
-(Plugins -> teamcity-configs -> teamcity-configs:generate), the
-'Debug' option is available in the context menu for the task.
-*/
-
-version = "2019.2"
-
-project {
-
-    vcsRoot(HttpsGithubComSimon987sist2refsHeadsMaster)
-
-    buildType(Build)
-}
-
-object Build : BuildType({
-    name = "Build"
-
-    artifactRules = """
-        sist2
-        sist2_scan
-    """.trimIndent()
-
-    vcs {
-        root(HttpsGithubComSimon987sist2refsHeadsMaster)
-    }
-
-    steps {
-        exec {
-            name = "Build"
-            path = "./ci/build.sh"
-            dockerImage = "simon987/general_ci"
-            dockerImagePlatform = ExecBuildStep.ImagePlatform.Linux
-            dockerPull = true
-        }
-    }
-
-    triggers {
-        vcs {
-        }
-    }
-})
-
-object HttpsGithubComSimon987sist2refsHeadsMaster : GitVcsRoot({
-    name = "https://github.com/simon987/sist2#refs/heads/master"
-    url = "https://github.com/simon987/sist2"
-})

CMakeLists.txt

@@ -4,13 +4,27 @@ set(CMAKE_C_STANDARD 11)
 project(sist2 C)
 
 option(SIST_DEBUG "Build a debug executable" on)
+option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0)
+
+add_compile_definitions(
+        "SIST_PLATFORM=${SIST_PLATFORM}"
+)
+
+if (SIST_DEBUG)
+    add_compile_definitions(
+            "SIST_DEBUG=${SIST_DEBUG}"
+    )
+endif()
 
 add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
-add_executable(
-        sist2
+add_executable(sist2
+
+        # argparse
+        third-party/argparse/argparse.h third-party/argparse/argparse.c
+
         src/main.c
         src/sist.h
         src/io/walk.h src/io/walk.c
@@ -25,23 +39,21 @@ add_executable(
         src/util.c src/util.h
         src/ctx.h src/types.h
         src/log.c src/log.h
-
-        # argparse
-        third-party/argparse/argparse.h third-party/argparse/argparse.c
-
         src/cli.c src/cli.h
-)
+        src/stats.c src/stats.h src/ctx.c
+        src/parsing/sidecar.c src/parsing/sidecar.h)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
 set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
 
+find_package(PkgConfig REQUIRED)
+
+pkg_search_module(GLIB REQUIRED glib-2.0)
+
 find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
-find_package(unofficial-glib CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
-find_library(UUID_LIB NAMES uuid)
-
-#find_package(OpenSSL REQUIRED)
+find_package(CURL CONFIG REQUIRED)
 
 
 target_include_directories(
@@ -50,13 +62,13 @@ target_include_directories(
         ${CMAKE_SOURCE_DIR}/third-party/utf8.h/
         ${CMAKE_SOURCE_DIR}/third-party/libscan/
         ${CMAKE_SOURCE_DIR}/
+        ${GLIB_INCLUDE_DIRS}
 )
 
 target_compile_options(
         sist2
         PRIVATE
         -fPIC
-        -Werror
 )
 
 if (SIST_DEBUG)
@@ -67,12 +79,13 @@ if (SIST_DEBUG)
             -fstack-protector
             -fno-omit-frame-pointer
             -fsanitize=address
+            -fno-inline
+#            -O2
     )
     target_link_options(
             sist2
             PRIVATE
             -fsanitize=address
-#            -static
     )
     set_target_properties(
             sist2
@@ -102,14 +115,15 @@ target_link_libraries(
         lmdb
         cjson
         argparse
-        unofficial::glib::glib
+        ${GLIB_LDFLAGS}
         unofficial::mongoose::mongoose
-#        OpenSSL::SSL OpenSSL::Crypto
+        CURL::libcurl
 
-        ${UUID_LIB}
         pthread
         magic
 
+        c
+
         scan
 )
 

Docker/build.sh

@@ -1,14 +0,0 @@
-rm ./sist2 sist2_debug
-cp ../sist2.gz .
-gzip -d sist2.gz
-strip sist2
-
-version=$(./sist2 --version)
-
-echo "Version ${version}"
-docker build . -t simon987/sist2:${version} -t simon987/sist2:latest
-
-docker push simon987/sist2:${version}
-docker push simon987/sist2:latest
-
-docker run --rm simon987/sist2 -v

Dockerfile

@@ -1,9 +1,17 @@
-FROM ubuntu:19.10
+FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
-RUN apt update
-RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 \
- curl libtiff5 libpng16-16 libpcre3
+WORKDIR /build/
+COPY . .
+RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
+RUN make -j$(nproc)
+RUN strip sist2
+RUN ls -lh
+RUN ls -lh sist2-vue/dist/
+
+FROM ubuntu:20.10
+
+RUN apt update && apt install -y curl libasan5
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -12,9 +20,9 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
     curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
 
-ADD sist2 /root/sist2
+COPY --from=build /build/sist2 /root/sist2
 
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8

Dockerfile.arm64

@@ -0,0 +1,28 @@
+FROM simon987/sist2-build-arm64 as build
+MAINTAINER simon987 <me@simon987.net>
+
+WORKDIR /build/
+ADD . /build/
+RUN cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
+RUN make -j$(nproc)
+RUN strip sist2
+
+FROM ubuntu:20.10
+
+RUN apt update && apt install -y curl libasan5
+
+RUN mkdir -p /usr/share/tessdata && \
+    cd /usr/share/tessdata/ && \
+    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
+
+COPY --from=build /build/sist2 /root/sist2
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["/root/sist2"]

README.md

@@ -1,6 +1,8 @@
 ![GitHub](https://img.shields.io/github/license/simon987/sist2.svg)
 [![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
-[![Development snapshots](https://ci.simon987.net/app/rest/builds/buildType(Sist2_Build)/statusIcon)](https://files.simon987.net/artifacts/Sist2/Build/)
+[![Development snapshots](https://ci.simon987.net/api/badges/simon987/sist2/status.svg)](https://files.simon987.net/.gate/sist2/simon987_sist2/)
+
+**Demo**: [sist2.simon987.net](https://sist2.simon987.net/?i=Demo%20files)
 
 # sist2
 
@@ -15,17 +17,19 @@ sist2 (Simple incremental search tool)
 * Fast, low memory usage, multi-threaded
 * Mobile-friendly Web interface
 * Portable (all its features are packaged in a single executable)
-* Extracts text from common file types \*
+* Extracts text and metadata from common file types \*
 * Generates thumbnails \*
 * Incremental scanning
-* Automatic tagging from file attributes via [user scripts](scripting/README.md)
+* Manual tagging from the UI and automatic tagging based on file attributes via [user scripts](docs/scripting.md)
 * Recursive scan inside archive files \*\*
 * OCR support with tesseract \*\*\*
-
+* Stats page & disk utilisation visualization
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
-\*\*\* See [OCR](#ocr)    
+\*\*\* See [OCR](#ocr)
+
+![stats](docs/stats.png)
 
 ## Getting Started
 
@@ -34,98 +38,118 @@ sist2 (Simple incremental search tool)
     1. *(or)* Run using docker:
         ```bash
        docker run -d --name es1 --net sist2_net -p 9200:9200 \
-            -e "discovery.type=single-node" elasticsearch:7.5.2
+            -e "discovery.type=single-node" elasticsearch:7.14.0
         ```
     1. *(or)* Run using docker-compose:
         ```yaml
           elasticsearch:
-            image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
+            image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0
             environment:
               - discovery.type=single-node
               - "ES_JAVA_OPTS=-Xms1G -Xmx2G"
         ```
 1. Download sist2 executable
     1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) *
-    1. *(or)* Download a [development snapshot](https://files.simon987.net/artifacts/Sist2/Build/) *(Not recommended!)*
-    1. *(or)* `docker pull simon987/sist2:latest`
+    1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
+       recommended!)*
+    1. *(or)* `docker pull simon987/sist2:2.11.2-x64-linux`
 
-1. See [Usage guide](DOCS/USAGE.md)
-   
-
-\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)    
+1. See [Usage guide](docs/USAGE.md)
 
+\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
 
 ## Example usage
 
-See [Usage guide](DOCS/USAGE.md) for more details
+See [Usage guide](docs/USAGE.md) for more details
 
 1. Scan a directory: `sist2 scan ~/Documents -o ./docs_idx`
 1. Push index to Elasticsearch: `sist2 index ./docs_idx`
 1. Start web interface: `sist2 web ./docs_idx`
 
-
 ## Format support
 
-File type | Library | Content | Thumbnail | Metadata
+File type | Library | Content | Thumbnail | Metadata
 :---|:---|:---|:---|:---
-pdf,xps,cbz,cbr,fb2,epub | MuPDF | text+ocr | yes, `png` | title |
-`audio/*` | ffmpeg | - | yes, `jpeg` | ID3 tags |
-`video/*` | ffmpeg | - | yes, `jpeg` | title, comment, artist |
-`image/*` | ffmpeg | - | yes, `jpeg` | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
+pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title |
+cbz,cbr | [libscan](https://github.com/simon987/libscan) | - | yes | - |
+`audio/*` | ffmpeg | - | yes | ID3 tags |
+`video/*` | ffmpeg | - | yes | title, comment, artist |
+`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
+raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags, GPS tags |
 ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
-`text/plain` | *(none)* | yes | no | - |
-html, xml | *(none)* | yes | no | - |
+`text/plain` | [libscan](https://github.com/simon987/libscan) | yes | no | - |
+html, xml | [libscan](https://github.com/simon987/libscan) | yes | no | - |
 tar, zip, rar, 7z, ar ...  | Libarchive | yes\* | - | no |
-docx, xlsx, pptx | *(none)* | yes | no | creator, modified_by, title |
+docx, xlsx, pptx | [libscan](https://github.com/simon987/libscan) | yes | if embedded | creator, modified_by, title |
+doc (MS Word 97-2003) | antiword | yes | yes | author, title |
 mobi, azw, azw3 | libmobi | yes | no | author, title |
+wpd (WordPerfect) | libwpd | yes | no | *planned* |
+json, jsonl, ndjson | [libscan](https://github.com/simon987/libscan) | yes | - | - |
 
 \* *See [Archive files](#archive-files)*
- 
+
 ### Archive files
-**sist2** will scan files stored into archive files (zip, tar, 7z...) as if
-they were directly in the file system. Recursive (archives inside archives)
+
+**sist2** will scan files stored into archive files (zip, tar, 7z...) as if they were directly in the file system.
+Recursive (archives inside archives)
 scan is also supported.
 
 **Limitations**:
-* Parsing media files with formats that require
-*seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.) is not supported.
-* Archive files are scanned sequentially, by a single thread. On systems where
-**sist2** is not I/O bound, scans might be faster when larger archives are split
- into smaller parts.
 
-To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
- 
- 
+* Support for parsing media files with formats that require *seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.)
+  is limitted (see `--mem-buffer` option)
+* Archive files are scanned sequentially, by a single thread. On systems where
+  **sist2** is not I/O bound, scans might be faster when larger archives are split into smaller parts.
+
 ### OCR
 
-You can enable OCR support for pdf,xps,cbz,cbr,fb2,epub file types with the
-`--ocr <lang>` option. Download the language data files with your
-package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
+You can enable OCR support for pdf,xps,fb2,epub file types with the
+`--ocr <lang>` option. Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
+directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
-The `simon987/sist2` image comes with common languages 
+The `simon987/sist2` image comes with common languages
 (hin, jpn, eng, fra, rus, spa) pre-installed.
 
 Examples
+
 ```bash
 sist2 scan --ocr jpn ~/Books/Manga/
 sist2 scan --ocr eng ~/Books/Textbooks/
 ```
 
-
 ## Build from source
 
-You can compile **sist2** by yourself if you don't want to use the pre-compiled
-binaries (GCC 7+ required).
+You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries
+
+### With docker (recommended)
+
+```bash
+git clone --recursive https://github.com/simon987/sist2/
+cd sist2
+docker build . -f ./Dockerfile -t my-sist2-image
+docker run --rm my-sist2-image cat /root/sist2 > sist2-x64-linux
+```
+
+### On a linux computer
 
 1. Install compile-time dependencies
 
    ```bash
-   vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libuuid libmagic
+   apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git
    ```
 
-2. Build
+1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
+
+1. Install vcpkg dependencies
+
+    ```bash
+    vcpkg install curl[core,openssl]
+    vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libuuid libmagic libraw jasper lcms gumbo
+    ```
+
+1. Build
     ```bash
     git clone --recursive https://github.com/simon987/sist2/
-    cmake -D <VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
+    cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

ci/build.sh

@@ -1,16 +1,17 @@
 #!/usr/bin/env bash
 
+VCPKG_ROOT="/vcpkg"
 
-rm *.gz
+git submodule update --init --recursive
 
 rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
-make
+cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j $(nproc)
 strip sist2
-gzip -9 sist2
+./sist2 -v > VERSION
+mv sist2 sist2-x64-linux
 
 rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_DEBUG=on -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
-make
-cp /usr/lib/x86_64-linux-gnu/libasan.so.2.0.0 libasan.so.2
-tar -czf sist2_debug.tar.gz sist2_debug libasan.so.2
+cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j  $(nproc)
+mv sist2_debug sist2-x64-linux-debug

ci/build_arm64.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+VCPKG_ROOT="/vcpkg"
+
+git submodule update --init --recursive
+
+rm -rf CMakeFiles CMakeCache.txt
+cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j $(nproc)
+strip sist2
+mv sist2 sist2-arm64-linux
+
+rm -rf CMakeFiles CMakeCache.txt
+cmake -DSIST_PLATFORM=arm64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j $(nproc)
+strip sist2
+mv sist2_debug sist2-arm64-linux-debug

docs/USAGE.md

@@ -14,47 +14,69 @@
     * [examples](#web-examples)
     * [rewrite_url](#rewrite_url)
     * [link to specific indices](#link-to-specific-indices)
+* [exec-script](#exec-script)
+* [tagging](#tagging)
+* [sidecar files](#sidecar-files)
 
 ```
 Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
+   or: sist2 exec-script [OPTION]... INDEX
 Lightning-fast file system indexer and search tool.
 
-    -h, --help            show this help message and exit
-    -v, --version         Show version and exit
-    --verbose             Turn on logging
-    --very-verbose        Turn on debug messages
+    -h, --help                    show this help message and exit
+    -v, --version                 Show version and exit
+    --verbose                     Turn on logging
+    --very-verbose                Turn on debug messages
 
 Scan options
-    -t, --threads=<int>   Number of threads. DEFAULT=1
-    -q, --quality=<flt>   Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=5
-    --size=<int>          Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
-    --content-size=<int>  Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
-    --incremental=<str>   Reuse an existing index and only scan modified files.
-    -o, --output=<str>    Output directory. DEFAULT=index.sist2/
-    --rewrite-url=<str>   Serve files from this url instead of from disk.
-    --name=<str>          Index display name. DEFAULT: (name of the directory)
-    --depth=<int>         Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --archive=<str>       Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
-    --ocr=<str>           Tesseract language (use tesseract --list-langs to see which are installed on your machine)
-    -e, --exclude=<str>   Files that match this regex will not be scanned
-    --fast                Only index file names & mime type
+    -t, --threads=<int>           Number of threads. DEFAULT=1
+    -q, --quality=<flt>           Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=3
+    --size=<int>                  Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
+    --content-size=<int>          Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
+    --incremental=<str>           Reuse an existing index and only scan modified files.
+    -o, --output=<str>            Output directory. DEFAULT=index.sist2/
+    --rewrite-url=<str>           Serve files from this url instead of from disk.
+    --name=<str>                  Index display name. DEFAULT: (name of the directory)
+    --depth=<int>                 Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
+    --archive=<str>               Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
+    --archive-passphrase=<str>    Passphrase for encrypted archive files
+    --ocr=<str>                   Tesseract language (use tesseract --list-langs to see which are installed on your machine)
+    -e, --exclude=<str>           Files that match this regex will not be scanned
+    --fast                        Only index file names & mime type
+    --treemap-threshold=<str>     Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
+    --mem-buffer=<int>            Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000
+    --read-subtitles              Read subtitles from media files.
+    --fast-epub                   Faster but less accurate EPUB parsing (no thumbnails, metadata)
+    --checksums                   Calculate file checksums when scanning.
 
 Index options
-    --es-url=<str>        Elasticsearch url with port. DEFAULT=http://localhost:9200
-    -p, --print           Just print JSON documents to stdout.
-    --script-file=<str>   Path to user script.
-    --batch-size=<int>    Index batch size. DEFAULT: 100
-    -f, --force-reset     Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
-
+    -t, --threads=<int>           Number of threads. DEFAULT=1
+    --es-url=<str>                Elasticsearch url with port. DEFAULT=http://localhost:9200
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
+    -p, --print                   Just print JSON documents to stdout.
+    --script-file=<str>           Path to user script.
+    --mappings-file=<str>         Path to Elasticsearch mappings.
+    --settings-file=<str>         Path to Elasticsearch settings.
+    --async-script                Execute user script asynchronously.
+    --batch-size=<int>            Index batch size. DEFAULT: 100
+    -f, --force-reset             Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
 
 Web options
-    --es-url=<str>        Elasticsearch url. DEFAULT=http://localhost:9200
-    --bind=<str>          Listen on this address. DEFAULT=localhost:4090
-    --auth=<str>          Basic auth in user:password format
-Made by simon987 <me@simon987.net>. Released under GPL-3.0
+    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
+    --bind=<str>                  Listen on this address. DEFAULT=localhost:4090
+    --auth=<str>                  Basic auth in user:password format
+    --tag-auth=<str>              Basic auth in user:password format for tagging
+    --tagline=<str>               Tagline in navbar
+    --dev                         Serve html & js files from disk (for development)
 
+Exec-script options
+    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
+    --script-file=<str>           Path to user script.
+    --async-script                Execute user script asynchronously.
 ```
 
 ## Scan
@@ -62,9 +84,9 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ### Scan options
 
 * `-t, --threads` 
-      Number of threads for file parsing. **Do not set a number higher than `$(nproc)`!**.
+      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-WmiObject Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
 * `-q, --quality` 
-    Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. *Does not affect PDF thumbnails quality*
+    Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best.
 * `--size` 
     Thumbnail size in pixels.
 * `--content-size` 
@@ -74,7 +96,7 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
     Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
     will be copied to the new index and will not be parsed again.
 * `-o, --output` Output directory. 
-* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
+* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
 * `--name` Set the `name` option for the web module
 * `--depth` Maximum scan dept. Set to 0 only scan files directly in the root directory, set to -1 for infinite depth
 * `--archive` Archive file mode.
@@ -94,6 +116,23 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
     * `-e "(^/usr/)|(^/var/)|(^/media/DRIVE-A/tmp/)|(^/media/DRIVE-B/Trash/)"` Exclude the
      `/usr`, `/var`, `/media/DRIVE-A/tmp`, `/media/DRIVE-B/Trash` directories
 * `--fast` Only index file names and mime type
+* `--treemap-threshold` Directories smaller than (`treemap-threshold` * `<total size of the index>`)
+    will not be considered for the disk utilisation visualization; their size will be added to
+    the parent directory. If the parent directory is still smaller than the threshold, it will also be "merged upwards"
+    and so on.
+    
+    In effect, smaller `treemap-threshold` values will yield a more detailed 
+    (but also a more cluttered and harder to read) visualization. 
+    
+* `--mem-buffer` Maximum memory buffer size in MB (per thread) for files inside archives. Media files 
+    larger than this number will be read sequentially and no *seek* operations will be supported.
+
+    To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
+* `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
+* `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
+* `--checksums` Calculate file checksums (sha1) when scanning files. This option does not cause any additional read 
+  operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
+  files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
 
 ### Scan examples
 
@@ -114,22 +153,27 @@ sist2 scan --incremental ./orig_idx/ -o ./updated_idx/ ~/Documents
 
 ### Index format
 
-A typical `binary` type index structure looks like this:
+A typical `ndjson` type index structure looks like this:
 ```
 documents.idx/
 ├── descriptor.json
-├── _index_139965416830720
-├── _index_139965425223424
-├── _index_139965433616128
-├── _index_139965442008832
-└── thumbs
+├── _index_main.ndjson.zst
+├── treemap.csv
+├── agg_mime.csv
+├── agg_date.csv
+├── add_size.csv
+├── thumbs/
+|   ├── data.mdb
+|   └── lock.mdb
+├── tags/
+|   ├── data.mdb
+|   └── lock.mdb
+└── meta/
     ├── data.mdb
     └── lock.mdb
 ```
 
-The `_index_*` files contain the raw binary index data and are not meant to be
-read by other applications. The format is generally compatible across different 
-sist2 versions.
+The `_index_*.ndjson.zst` files contain the document data in JSON format, in a compressed newline-delemited file.
 
 The `thumbs/` folder is a [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database)
 database containing the thumbnails.
@@ -137,71 +181,14 @@ database containing the thumbnails.
 The `descriptor.json` file contains general information about the index. The 
 following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rewrite_url) and `timestamp`.
 
-
-*Advanced usage*
-
-Instead of using the `scan` module, you can also import an index generated
-by a third party application. The 'external' index must have the following format:
-
-```
-my_index/
-├── descriptor.json
-├── _index_0
-└── thumbs
-    ├── data.mdb
-    └── lock.mdb
-```
-
-*descriptor.json*:
-```json
-{
-    "uuid": "<valid UUID4>",
-    "version": "_external_v1",
-    "root": "(optional)",
-    "name": "<name>",
-    "rewrite_url": "(optional)",
-    "type": "json",
-    "timestamp": 1578971024
-}
-```
-
-*_index_0*: NDJSON format (One json object per line)
-
-```json
-{
-  "_id": "unique uuid for the file",
-  "index": "index uuid4 (same one as descriptor.json!)",
-  "mime": "application/x-cbz",
-  "size": 14341204,
-  "mtime": 1578882996,
-  "extension": "cbz",
-  "name": "my_book",
-  "path": "path/to/books",
-  "content": "text contents of the book",
-  "title": "Title of the book",
-  "tag": ["genre.fiction", "author.someguy", "etc..."],
-  "_keyword": [
-    {"k": "ISBN", "v": "ABCD34789231"}
-  ],
-  "_text": [
-    {"k": "other", "v": "This will be indexed as text"}
-  ]
-}
-```
-
-You can find the full list of supported fields [here](../src/io/serialize.c#L90)
-
-The `_keyword.*` items will be indexed and searchable as **keyword** fields (only full matches allowed).
-The `_text.*` items will be indexed and searchable as **text** fields (fuzzy searching allowed)
-
+The `.csv` are pre-computed aggregations necessary for the stats page.
 
 *thumbs/*:
 
-LMDB key-value store. Keys are **binary** 128-bit UUID4s (`_id` field)
+LMDB key-value store. Keys are **binary** 16-byte md5 hash* (`_id` field)
 and values are raw image bytes.
 
-Importing an external `binary` type index is technically possible but
-it is currently unsupported and has no guaranties of back/forward compatibility.
+*\* Hash is calculated from the full path of the file, including the extension, relative to the index root*
 
 
 ## Index
@@ -209,17 +196,26 @@ it is currently unsupported and has no guaranties of back/forward compatibility.
  * `--es-url` 
  Elasticsearch url and port. If you are using docker, make sure that both containers are on the
  same network.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
  * `-p, --print` 
     Print index in JSON format to stdout.
  * `--script-file` 
-    Path to user script. See [Scripting](scripting/README.md).
+    Path to user script. See [Scripting](scripting.md).
+ * `--mappings-file`
+    Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used.
+ * `--settings-file`
+    Path to custom Elasticsearch settings. *(See above)*
+ * `--async-script` 
+    Use `wait_for_completion=false` elasticsearch option while executing user script.
+     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
  * `--batch-size=<int>` 
     Index batch size. Indexing is generally faster with larger batches, but payloads that
     are too large will fail and additional overhead for retrying with smaller sizes may slow
     down the process.
  * `-f, --force-reset` 
     Reset Elasticsearch mappings and settings.
-    **(You must use this option the first time you use the index command)**.
+ * `-t, --threads` Number of threads to use. Ideally, choose a number equal to the number of logical cores of the machine hosting Elasticsearch.
     
 ### Index examples
 
@@ -243,8 +239,14 @@ sist2 index --print ./my_index/ | jq | less
 
 ### Web options
  * `--es-url=<str>` Elasticsearch url.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
  * `--bind=<str>` Listen on this address.
  * `--auth=<str>` Basic auth in user:password format
+ * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
+    `--auth` argument, but authentication is only applied the `/tag/` endpoint.
+ * `--tagline=<str>` When specified, will replace the default tagline in the navbar.
+ * `--dev` Serve html & js files from disk (for development, used to modify frontend files without having to recompile)
  
 ### Web examples
 
@@ -267,8 +269,72 @@ instead of serving the file from disk.
 Both the `root` and `rewrite_url` fields are safe to manually modify from the 
 `descriptor.json` file.
 
-### Link to specific indices
+## exec-script
 
-To link to specific indices, you can add a list of comma-separated index name to 
-the URL: `?i=<name>,<name>`. By default, indices with `"(nsfw)"` in their name are
-not displayed.
+The `exec-script` command is used to execute a user script for an index that has already been imported to Elasticsearch with the `index` command. Note that the documents will not be reset to their default state before each execution as the `index` command does: if you make undesired changes to the documents by accident, you will need to run `index` again to revert to the original state.
+
+
+# Tagging
+
+### Manual tagging
+
+You can modify tags of individual documents directly from the 
+ `web` interface. Note that you can setup authentication for this feature
+ with the `--tag-auth` option (See [web options](#web-options))
+
+![manual_tag](manual_tag.png)
+
+Tags that are manually added are saved both in the 
+ index folder (in `/tags/`) and in Elasticsearch*. When re-`index`ing, 
+ they are read from the index and automatically applied.
+ 
+You can safely copy the `/tags/` database to another index.
+
+See [Automatic tagging](#automatic-tagging) for information about tag 
+ hierarchies and tag colors.
+
+\* *It can take a few seconds to take effect in new search queries.*
+
+
+### Automatic tagging
+
+See [scripting](scripting.md) documentation.
+
+# Sidecar files
+
+When scanning, sist2 will read metadata from `.s2meta` JSON files and overwrite the 
+original document's metadata. Sidecar metadata files will also work inside archives.
+Sidecar files themselves are not saved in the index.
+
+This feature is useful to leverage third-party applications such as speech-to-text or
+OCR to add additional metadata to a file.
+
+**Example**
+
+```
+~/Documents/
+├── Video.mp4
+└── Video.mp4.s2meta
+```
+
+The sidecar file must have exactly the same file path and the `.s2meta` suffix.
+
+`Video.mp4.s2meta`:
+```json
+{
+  "content": "This sidecar file will overwrite some metadata fields of Video.mp4",
+  "author": "Some author",
+  "duration": 12345,
+  "bitrate": 67890,
+  "some_arbitrary_field": [1,2,3]
+}
+```
+
+```
+sist2 scan ~/Documents -o ./docs.idx
+sist2 index ./docs.idx
+```
+
+*NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,
+it is not currently possible to restore both manual tags and sidecar tags without user scripts
+while reindexing.

docs/scripting.md

@@ -39,7 +39,7 @@ it adds the `genre.<genre>` tag.
 ArrayList tags = ctx._source.tag = new ArrayList();
 
 if (ctx._source?.genre != null) {
-    tags.add("genre." + ctx._source.genre.toLowerCase())
+    tags.add("genre." + ctx._source.genre.toLowerCase());
 }
 ```
 
@@ -67,7 +67,7 @@ ArrayList tags = ctx._source.tag = new ArrayList();
 
 Matcher m = /[\(\.+](20[0-9]{2})[\)\.+]/.matcher(ctx._source.name);
 if (m.find()) {
-    tags.add("year." + m.group(1))
+    tags.add("year." + m.group(1));
 }
 ```
 
@@ -111,16 +111,6 @@ if (ctx._source.path != "") {
 }
 ```
 
-Set the name of the last folder (`/path/to/<studio>/file.mp4`) to `studio.<studio>` tag
-```Java
-ArrayList tags = ctx._source.tag = new ArrayList();
-
-if (ctx._source.path != "") {
-    String[] names = ctx._source.path.splitOnToken('/');
-    tags.add("studio." + names[names.length-1]);
-}
-```
-
 Parse `EXIF:F Number` tag
 ```Java
 if (ctx._source?.exif_fnumber != null) {

schema/mappings.json

@@ -4,12 +4,17 @@
       "type": "keyword",
       "doc_values": true
     },
+    "checksum": {
+      "type": "keyword",
+      "index": false
+    },
     "_depth": {
       "type": "integer"
     },
     "path": {
       "type": "text",
       "analyzer": "path_analyzer",
+      "copy_to": "suggest-path",
       "fielddata": true,
       "fields": {
         "nGram": {
@@ -22,9 +27,17 @@
         }
       }
     },
+    "suggest-path": {
+      "type": "completion",
+      "analyzer": "case_insensitive_kw_analyzer"
+    },
     "mime": {
       "type": "keyword"
     },
+    "parent": {
+      "type": "keyword",
+      "index": false
+    },
     "thumbnail": {
       "type": "keyword",
       "index": false
@@ -38,7 +51,7 @@
       "index": false
     },
     "duration": {
-      "type": "float",
+      "type": "integer",
       "index": false
     },
     "width": {
@@ -49,6 +62,10 @@
       "type": "integer",
       "index": false
     },
+    "pages": {
+      "type": "integer",
+      "index": false
+    },
     "mtime": {
       "type": "integer"
     },
@@ -92,10 +109,10 @@
       "analyzer": "my_nGram",
       "type": "text"
     },
-    "_keyword.*":  {
+    "_keyword.*": {
       "type": "keyword"
     },
-    "_text.*":  {
+    "_text.*": {
       "analyzer": "content_analyzer",
       "type": "text",
       "fields": {
@@ -121,7 +138,14 @@
       }
     },
     "tag": {
-      "type": "keyword"
+      "type": "text",
+      "fielddata": true,
+      "analyzer": "tag_analyzer",
+      "copy_to": "suggest-tag"
+    },
+    "suggest-tag": {
+      "type": "completion",
+      "analyzer": "case_insensitive_kw_analyzer"
     },
     "exif_make": {
       "type": "text"
@@ -147,6 +171,30 @@
     "exif_user_comment": {
       "type": "text"
     },
+    "exif_gps_longitude_ref": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_longitude_dms": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_longitude_dec": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_latitude_ref": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_latitude_dms": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_latitude_dec": {
+      "type": "keyword",
+      "index": false
+    },
     "author": {
       "type": "text"
     },

schema/settings.json

@@ -1,12 +1,19 @@
 {
   "index": {
     "refresh_interval": "30s",
-    "codec": "best_compression"
+    "codec": "best_compression",
+    "number_of_replicas": 0,
+    "highlight.max_analyzed_offset": 10000000
   },
   "analysis": {
     "tokenizer": {
       "path_tokenizer": {
-        "type": "path_hierarchy"
+        "type": "path_hierarchy",
+        "delimiter": "/"
+      },
+      "tag_tokenizer": {
+        "type": "path_hierarchy",
+        "delimiter": "."
       },
       "my_nGram_tokenizer": {
         "type": "nGram",
@@ -21,6 +28,12 @@
           "lowercase"
         ]
       },
+      "tag_analyzer": {
+        "tokenizer": "tag_tokenizer",
+        "filter": [
+          "lowercase"
+        ]
+      },
       "case_insensitive_kw_analyzer": {
         "tokenizer": "keyword",
         "filter": [

scripts/before_build.sh

@@ -2,16 +2,9 @@
 
 rm -rf index.sist2/
 
-rm src/static/js/bundle.js 2> /dev/null
-cat `ls src/static/js/*.min.js` > src/static/js/bundle.js
-cat src/static/js/{util,dom,search}.js >> src/static/js/bundle.js
-
-rm src/static/css/bundle*.css 2> /dev/null
-cat src/static/css/*.min.css > src/static/css/bundle.css
-cat src/static/css/light.css >> src/static/css/bundle.css
-cat src/static/css/*.min.css > src/static/css/bundle_dark.css
-cat src/static/css/dark.css >> src/static/css/bundle_dark.css
-
 python3 scripts/mime.py > src/parsing/mime_generated.c
 python3 scripts/serve_static.py > src/web/static_generated.c
 python3 scripts/index_static.py > src/index/static_generated.c
+
+printf "static const char *const Sist2CommitHash = \"%s\";\n" $(git rev-parse HEAD) > src/git_hash.h
+printf "static const char *const LibScanCommitHash = \"%s\";\n" $(cd third-party/libscan/ && git rev-parse HEAD) >> src/git_hash.h

scripts/mime.csv

@@ -13,7 +13,7 @@ application/epub+zip, epub
 application/freeloader, frl
 application/futuresplash, spl
 application/groupwise, vew
-application/gzip, gz
+application/gzip, gz|tgz
 application/hta, hta
 application/i-deas, unv
 application/iges, iges|igs
@@ -22,6 +22,7 @@ application/java-archive, jar
 application/java, class
 application/javascript,
 application/json, json
+application/ndjson, jsonl|ndjson
 application/marc, mrc
 application/mbedlet, mbd
 application/mime, aps
@@ -78,9 +79,7 @@ application/vocaltec-media-desc, vmd
 application/vocaltec-media-file, vmf
 application/warc, warc
 application/winhelp, hlp
-application/wordperfect6.0, w60
-application/wordperfect6.1, w61
-application/wordperfect, wp|wp5|wp6|wpd
+application/wordperfect, wp|wp5|wp6|wpd|w60|w61
 application/x-123, wk1
 application/x-7z-compressed, 7z
 application/x-aim, aim
@@ -111,7 +110,7 @@ application/x-dbf, dbf
 application/x-dbt,
 application/x-debian-package, deb
 application/x-deepv, deepv
-application/x-director, dcr|dir|dxr
+application/x-director, dir|dxr
 application/x-dmp, dmp
 application/x-dosdriver,
 application/x-dosexec, dll
@@ -157,7 +156,6 @@ application/x-livescreen, ivy
 application/x-lotus, wq1
 application/x-lz4+json, jsonlz4
 application/x-lz4, lz4
-application/x-lz4, lz4
 application/x-lzh-compressed,
 application/x-lzh, lzh
 application/x-lzip, lz
@@ -347,7 +345,8 @@ text/javascript, js
 text/mcf, mcf
 text/pascal, pas
 text/PGP,
-text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml
+text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml|make|log|markdown|yaml
+application/vnd.coffeescript, coffee
 text/richtext, rt|rtf|rtx
 text/rtf,
 text/scriplet, wsc
@@ -429,3 +428,23 @@ video/x-qtc, qtc
 video/x-sgi-movie, movie|mv
 x-epoc/x-sisx-app,
 application/x-zstd-dictionary,
+application/vnd.ms-outlook, msg
+image/x-olympus-orf, orf
+image/x-nikon-nef, nef
+image/x-fuji-raf, raf
+image/x-panasonic-raw, rw2|raw
+image/x-adobe-dng, dng
+image/x-canon-cr2, cr2
+image/x-canon-crw, crw
+image/x-dcraw,
+image/x-kodak-dcr, dcr
+image/x-kodak-k25, k25
+image/x-kodak-kdc, kdc
+image/x-minolta-mrw, mrw
+image/x-pentax-pef, pef
+image/x-sigma-x3f, xf3
+image/x-sony-arw, arw
+image/x-sony-sr2, sr2
+image/x-sony-srf, srf
+image/x-epson-erf, erf
+sist2/sidecar, s2meta

scripts/mime.py

@@ -3,6 +3,7 @@ noparse = set()
 ext_in_hash = set()
 
 major_mime = {
+    "sist2": 0,
     "model": 1,
     "example": 2,
     "message": 3,
@@ -18,7 +19,6 @@ major_mime = {
 
 pdf = (
     "application/pdf",
-    "application/x-cbz",
     "application/epub+zip",
     "application/vnd.ms-xpsdocument",
 )
@@ -73,6 +73,29 @@ markup = (
     "text/x-sgml"
 )
 
+raw = (
+    "image/x-olympus-orf",
+    "image/x-nikon-nef",
+    "image/x-fuji-raf",
+    "image/x-panasonic-raw",
+    "image/x-adobe-dng",
+    "image/x-canon-cr2",
+    "image/x-canon-crw",
+    "image/x-dcraw",
+    "image/x-kodak-dcr",
+    "image/x-kodak-k25",
+    "image/x-kodak-kdc",
+    "image/x-minolta-mrw",
+    "image/x-pentax-pef",
+    "image/x-sigma-x3f",
+    "image/x-sony-arw",
+    "image/x-sony-sr2",
+    "image/x-sony-srf",
+    "image/x-minolta-mrw",
+    "image/x-pentax-pef",
+    "image/x-epson-erf",
+)
+
 cnt = 1
 
 
@@ -97,8 +120,14 @@ def mime_id(mime):
         mime_id += " | 0x02000000"
     elif mime in markup:
         mime_id += " | 0x01000000"
+    elif mime in raw:
+        mime_id += " | 0x00800000"
     elif mime == "application/x-empty":
+        cnt -= 1
         return "1"
+    elif mime == "sist2/sidecar":
+        cnt -= 1
+        return "2"
     return mime_id
 
 

scripts/reset.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+make clean
+rm -rf CMakeFiles/ CMakeCache.txt Makefile \
+  third-party/libscan/CMakeFiles third-party/libscan/CMakeCache.txt third-party/libscan/third-party/ext_ffmpeg \
+  third-party/libscan/third-party/ext_libmobi third-party/libscan/Makefile

scripts/serve_static.py

@@ -1,10 +1,10 @@
 files = [
-    "src/static/css/bundle.css",
-    "src/static/css/bundle_dark.css",
-    "src/static/js/bundle.js",
-    "src/static/img/sprite-skin-flat.png",
-    "src/static/img/sprite-skin-flat-dark.png",
-    "src/static/search.html",
+    "sist2-vue/src/assets/favicon.ico",
+    "sist2-vue/dist/css/chunk-vendors.css",
+    "sist2-vue/dist/css/index.css",
+    "sist2-vue/dist/js/chunk-vendors.js",
+    "sist2-vue/dist/js/index.js",
+    "sist2-vue/dist/index.html",
 ]
 
 
@@ -13,6 +13,10 @@ def clean(filepath):
 
 
 for file in files:
-    with open(file, "rb") as f:
-        data = f.read()
+    try:
+        with open(file, "rb") as f:
+            data = f.read()
+    except:
+        data = bytes([])
+
     print("char %s[%d] = {%s};" % (clean(file), len(data), ",".join(str(int(b)) for b in data)))

sist2-vue/.gitignore

@@ -0,0 +1,23 @@
+.DS_Store
+node_modules
+
+
+# local env files
+.env.local
+.env.*.local
+
+# Log files
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# Editor directories and files
+.idea
+.vscode
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+*.iml

sist2-vue/babel.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+    "presets": [
+        "@vue/cli-plugin-babel/preset"
+    ]
+}

sist2-vue/dist/index.html

@@ -0,0 +1,3 @@
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>sist2</title><link href="css/chunk-vendors.css" rel="preload" as="style"><link href="css/index.css" rel="preload" as="style"><link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/index.css" rel="stylesheet"></head><body><noscript><style>body {
+            height: initial;
+        }</style><div style="text-align: center; margin-top: 100px"><strong>We're sorry but sist2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong><br><strong>Nous sommes désolés mais sist2 ne fonctionne pas correctement si JavaScript est activé. Veuillez l'activer pour continuer.</strong></div></noscript><div id="app"></div><script src="js/chunk-vendors.js"></script><script src="js/index.js"></script></body></html>

sist2-vue/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "sist2",
+  "version": "2.11.0",
+  "private": true,
+  "scripts": {
+    "serve": "vue-cli-service serve",
+    "build": "vue-cli-service build --mode production"
+  },
+  "dependencies": {
+    "@egjs/vue-infinitegrid": "3.3.0",
+    "axios": "^0.21.1",
+    "bootstrap-vue": "^2.21.2",
+    "core-js": "^3.6.5",
+    "crypto-es": "^1.2.7",
+    "d3": "^5.16.0",
+    "date-fns": "^2.21.3",
+    "dom-to-image": "^2.6.0",
+    "fslightbox-vue": "file:../../../mnt/Hatchery/main/projects/sist2/fslightbox-vue-pro-1.3.1.tgz",
+    "nouislider": "^15.2.0",
+    "underscore": "^1.13.1",
+    "vue": "^2.6.12",
+    "vue-color": "^2.8.1",
+    "vue-i18n": "^8.24.4",
+    "vue-masonry-wall": "^0.3.2",
+    "vue-router": "^3.2.0",
+    "vue-simple-suggest": "^1.11.1",
+    "vuex": "^3.4.0"
+  },
+  "devDependencies": {
+    "@babel/polyfill": "^7.11.5",
+    "@vue/cli-plugin-babel": "~4.5.0",
+    "@vue/cli-plugin-router": "~4.5.0",
+    "@vue/cli-plugin-typescript": "~4.5.0",
+    "@vue/cli-plugin-vuex": "~4.5.0",
+    "@vue/cli-service": "~4.5.0",
+    "@vue/test-utils": "^1.0.3",
+    "bootstrap": "^4.5.2",
+    "inspire-tree": "^4.3.1",
+    "inspire-tree-dom": "^4.0.6",
+    "mutationobserver-shim": "^0.3.7",
+    "popper.js": "^1.16.1",
+    "portal-vue": "^2.1.7",
+    "sass": "^1.26.11",
+    "sass-loader": "^10.0.2",
+    "typescript": "~4.1.5",
+    "vue-cli-plugin-bootstrap-vue": "~0.7.0",
+    "vue-template-compiler": "^2.6.11"
+  },
+  "browserslist": [
+    "> 1%",
+    "last 2 versions",
+    "not dead"
+  ]
+}

sist2-vue/public/index.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
+
+    <title><%= htmlWebpackPlugin.options.title %></title>
+</head>
+<body>
+<noscript>
+    <style>
+        body {
+            height: initial;
+        }
+    </style>
+    <div style="text-align: center; margin-top: 100px">
+        <strong>
+            We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled.
+            Please enable it to continue.
+        </strong>
+        <br/>
+        <strong>
+            Nous sommes désolés mais <%= htmlWebpackPlugin.options.title %> ne fonctionne pas correctement
+            si JavaScript est activé.
+            Veuillez l'activer pour continuer.
+        </strong>
+    </div>
+</noscript>
+<div id="app"></div>
+</body>
+</html>

sist2-vue/src/App.vue

@@ -0,0 +1,312 @@
+<template>
+  <div id="app" :class="getClass()">
+    <NavBar></NavBar>
+    <router-view v-if="!configLoading"/>
+  </div>
+</template>
+
+<script>
+import NavBar from "@/components/NavBar";
+import {mapGetters} from "vuex";
+
+export default {
+  components: {NavBar},
+  data() {
+    return {
+      configLoading: false
+    }
+  },
+  computed: {
+    ...mapGetters(["optTheme"]),
+  },
+  mounted() {
+    this.$store.dispatch("loadConfiguration").then(() => {
+      this.$root.$i18n.locale = this.$store.state.optLang;
+    });
+
+    this.$store.subscribe((mutation) => {
+      if (mutation.type === "setOptLang") {
+        this.$root.$i18n.locale = mutation.payload;
+        this.configLoading = true;
+        window.setTimeout(() => this.configLoading = false, 10);
+      }
+    });
+  },
+  methods: {
+    getClass() {
+      return {
+        "theme-light": this.optTheme === "light",
+        "theme-black": this.optTheme === "black",
+      }
+    }
+  }
+  ,
+}
+</script>
+
+<style>
+html, body {
+  height: 100%;
+}
+
+#app {
+  /*font-family: Avenir, Helvetica, Arial, sans-serif;*/
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  /*text-align: center;*/
+  color: #2c3e50;
+  padding-bottom: 1em;
+  min-height: 100%;
+}
+
+/*Black theme*/
+.theme-black {
+  background-color: #000;
+}
+
+.theme-black .card, .theme-black .modal-content {
+  background: #212121;
+  color: #e0e0e0;
+  border-radius: 1px;
+  border: none;
+}
+
+
+.theme-black .table {
+  color: #e0e0e0;
+}
+
+.theme-black .table td, .theme-black .table th {
+  border: none;
+}
+
+.theme-black .table thead th {
+  border-bottom: 1px solid #646464;
+}
+
+.theme-black .custom-select {
+  overflow: auto;
+  background-color: #37474F;
+  border: 1px solid #616161;
+  color: #bdbdbd;
+}
+
+.theme-black .custom-select:focus {
+  border-color: #757575;
+  outline: 0;
+  box-shadow: 0 0 0 .2rem rgba(0, 123, 255, .25);
+}
+
+.theme-black .inspire-tree .selected > .wholerow, .theme-black .inspire-tree .selected > .title-wrap:hover + .wholerow {
+  background: none !important;
+}
+
+.theme-black .inspire-tree .icon-expand::before, .theme-black .inspire-tree .icon-collapse::before {
+  background-color: black !important;
+}
+
+.theme-black .inspire-tree .title {
+  color: #eee;
+}
+
+.theme-black .inspire-tree {
+  font-weight: 400;
+  font-size: 14px;
+  font-family: Helvetica, Nueue, Verdana, sans-serif;
+  max-height: 350px;
+  overflow: auto;
+}
+
+.inspire-tree [type=checkbox] {
+  left: 22px !important;
+  top: 7px !important;
+}
+
+.theme-black .form-control {
+  background-color: #37474F;
+  border: 1px solid #616161;
+  color: #dbdbdb !important;
+}
+
+.theme-black .form-control:focus {
+  background-color: #546E7A;
+  color: #fff;
+}
+
+.theme-black .input-group-text, .theme-black .default-input {
+  background: #37474F !important;
+  border: 1px solid #616161 !important;
+  color: #dbdbdb !important;
+}
+
+.theme-black ::placeholder {
+  color: #BDBDBD !important;
+  opacity: 1;
+}
+
+.theme-black .nav-tabs .nav-link {
+  color: #e0e0e0;
+}
+
+.theme-black .nav-tabs .nav-item.show .nav-link, .theme-black .nav-tabs .nav-link.active {
+  background-color: #212121;
+  border-color: #616161 #616161 #212121;
+  color: #e0e0e0;
+}
+
+.theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:focus {
+  border-color: #616161 #616161 #212121;
+  color: #e0e0e0;
+}
+
+.theme-black .nav-tabs .nav-link:focus, .theme-black .nav-tabs .nav-link:hover {
+  border-color: #e0e0e0 #e0e0e0 #212121;
+  color: #e0e0e0;
+}
+
+.theme-black .nav-tabs {
+  border-bottom: #616161;
+}
+
+.theme-black a:hover, .theme-black .btn:hover {
+  color: #fff;
+}
+
+.theme-black .b-dropdown a:hover {
+  color: inherit;
+}
+
+.theme-black .btn {
+  color: #eee;
+}
+
+.theme-black .modal-header .close {
+  color: #e0e0e0;
+  text-shadow: none;
+}
+
+.theme-black .modal-header {
+  border-bottom: 1px solid #646464;
+}
+
+/* -------------------------- */
+
+#nav {
+  padding: 30px;
+}
+
+#nav a {
+  font-weight: bold;
+  color: #2c3e50;
+}
+
+#nav a.router-link-exact-active {
+  color: #42b983;
+}
+
+.mobile {
+  display: none;
+}
+
+.container {
+  padding-top: 1em;
+}
+
+@media (max-width: 650px) {
+  .mobile {
+    display: initial;
+  }
+
+  .not-mobile {
+    display: none;
+  }
+
+  .grid-single-column .fit {
+    max-height: none !important;
+  }
+
+  .container {
+    padding-left: 0;
+    padding-right: 0;
+    padding-top: 0
+  }
+
+  .lightbox-caption {
+    display: none;
+  }
+}
+
+.info-icon {
+  width: 1rem;
+  margin-right: 0.2rem;
+  cursor: pointer;
+  line-height: 1rem;
+  height: 1rem;
+  background-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCIKICAgICB2aWV3Qm94PSIwIDAgNDI2LjY2NyA0MjYuNjY3IiBzdHlsZT0iZW5hYmxlLWJhY2tncm91bmQ6bmV3IDAgMCA0MjYuNjY3IDQyNi42Njc7IiBmaWxsPSIjZmZmIj4KPGc+CiAgICA8Zz4KICAgICAgICA8Zz4KICAgICAgICAgICAgPHJlY3QgeD0iMTkyIiB5PSIxOTIiIHdpZHRoPSI0Mi42NjciIGhlaWdodD0iMTI4Ii8+CiAgICAgICAgICAgIDxwYXRoIGQ9Ik0yMTMuMzMzLDBDOTUuNDY3LDAsMCw5NS40NjcsMCwyMTMuMzMzczk1LjQ2NywyMTMuMzMzLDIxMy4zMzMsMjEzLjMzM1M0MjYuNjY3LDMzMS4yLDQyNi42NjcsMjEzLjMzMwogICAgICAgICAgICAgICAgUzMzMS4yLDAsMjEzLjMzMywweiBNMjEzLjMzMywzODRjLTk0LjA4LDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMsNDIuNjY3LDIxMy4zMzMsNDIuNjY3CiAgICAgICAgICAgICAgICBTMzg0LDExOS4yNTMsMzg0LDIxMy4zMzNTMzA3LjQxMywzODQsMjEzLjMzMywzODR6Ii8+CiAgICAgICAgICAgIDxyZWN0IHg9IjE5MiIgeT0iMTA2LjY2NyIgd2lkdGg9IjQyLjY2NyIgaGVpZ2h0PSI0Mi42NjciLz4KICAgICAgICA8L2c+CiAgICA8L2c+CjwvZz4KPC9zdmc+Cg==);
+  filter: brightness(45%);
+  display: block;
+}
+
+.tabs {
+  margin-top: 10px;
+}
+
+.modal-title {
+  text-overflow: ellipsis;
+  overflow: hidden;
+  white-space: nowrap;
+}
+
+@media screen and (min-width: 1500px) {
+  .container {
+    max-width: 1440px;
+  }
+}
+
+.noUi-connects {
+  border-radius: 1px !important;
+}
+
+mark {
+  background: #fff217;
+  border-radius: 0;
+  padding: 1px 0;
+  color: inherit;
+}
+
+.theme-black mark {
+  background: rgba(251, 191, 41, 0.25);
+  border-radius: 0;
+  padding: 1px 0;
+  color: inherit;
+}
+
+.theme-black .content-div mark {
+  background: rgba(251, 191, 41, 0.40);
+  color: white;
+}
+
+.content-div {
+  font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+  font-size: 13px;
+  padding: 1em;
+  background-color: #f5f5f5;
+  border: 1px solid #ccc;
+  border-radius: 4px;
+  margin: 3px;
+  white-space: normal;
+  color: #000;
+  overflow: hidden;
+}
+
+.theme-black .content-div {
+  background-color: #37474F;
+  border: 1px solid #616161;
+  color: #E0E0E0FF;
+}
+
+.graph {
+  display: inline-block;
+  width: 40%;
+}
+</style>

sist2-vue/src/Sist2Api.ts

@@ -0,0 +1,383 @@
+import axios from "axios";
+import {ext, strUnescape, lum} from "./util";
+import CryptoES from 'crypto-es';
+
+export interface EsTag {
+    id: string
+    count: number
+    color: string | undefined
+    isLeaf: boolean
+}
+
+export interface Tag {
+    style: string
+    text: string
+    rawText: string
+    fg: string
+    bg: string
+    userTag: boolean
+}
+
+export interface Index {
+    name: string
+    version: string
+    id: string
+    idPrefix: string
+    timestamp: number
+}
+
+export interface EsHit {
+    _index: string
+    _id: string
+    _score: number
+    _path_md5: string
+    _type: string
+    _tags: Tag[]
+    _seq: number
+    _source: {
+        path: string
+        size: number
+        mime: string
+        name: string
+        extension: string
+        index: string
+        _depth: number
+        mtime: number
+        videoc: string
+        audioc: string
+        parent: string
+        width: number
+        height: number
+        duration: number
+        tag: string[]
+        checksum: string
+    }
+    _props: {
+        isSubDocument: boolean
+        isImage: boolean
+        isGif: boolean
+        isVideo: boolean
+        isPlayableVideo: boolean
+        isPlayableImage: boolean
+        isAudio: boolean
+        hasThumbnail: boolean
+    }
+    highlight: {
+        name: string[] | undefined,
+        content: string[] | undefined,
+    }
+}
+
+function getIdPrefix(indices: Index[], id: string): string {
+    for (let i = 4; i < 32; i++) {
+        const prefix = id.slice(0, i);
+
+        if (indices.filter(idx => idx.id.slice(0, i) == prefix).length == 1) {
+            return prefix;
+        }
+    }
+
+    return id;
+}
+
+export interface EsResult {
+    took: number
+
+    hits: {
+        // TODO: ES 6.X ?
+        total: {
+            value: number
+        }
+        hits: EsHit[]
+    }
+
+    aggregations: any
+}
+
+class Sist2Api {
+
+    private baseUrl: string
+
+    constructor(baseUrl: string) {
+        this.baseUrl = baseUrl;
+    }
+
+    getSist2Info(): Promise<any> {
+        return axios.get(`${this.baseUrl}i`).then(resp => {
+            const indices = resp.data.indices as Index[];
+
+            resp.data.indices = indices.map(idx => {
+                return {
+                    id: idx.id,
+                    name: idx.name,
+                    timestamp: idx.timestamp,
+                    version: idx.version,
+                    idPrefix: getIdPrefix(indices, idx.id)
+                } as Index;
+            });
+
+            return resp.data;
+        })
+    }
+
+    setHitProps(hit: EsHit): void {
+        hit["_props"] = {} as any;
+
+        const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
+
+        if ("parent" in hit._source) {
+            hit._props.isSubDocument = true;
+        }
+
+        if ("thumbnail" in hit._source) {
+            hit._props.hasThumbnail = true;
+        }
+
+        switch (mimeCategory) {
+            case "image":
+                if (hit._source.videoc === "gif") {
+                    hit._props.isGif = true;
+                } else {
+                    hit._props.isImage = true;
+                }
+                if ("width" in hit._source && !hit._props.isSubDocument && hit._source.videoc !== "tiff"
+                    && hit._source.videoc !== "raw" && hit._source.videoc !== "ppm") {
+                    hit._props.isPlayableImage = true;
+                }
+                break;
+            case "video":
+                if ("videoc" in hit._source) {
+                    hit._props.isVideo = true;
+                }
+                if (hit._props.isVideo) {
+                    const videoc = hit._source.videoc;
+                    const mime = hit._source.mime;
+
+                    hit._props.isPlayableVideo = mime != null &&
+                        mime.startsWith("video/") &&
+                        !hit._props.isSubDocument &&
+                        hit._source.extension !== "mkv" &&
+                        hit._source.extension !== "avi" &&
+                        hit._source.extension !== "mov" &&
+                        videoc !== "hevc" &&
+                        videoc !== "mpeg1video" &&
+                        videoc !== "mpeg2video" &&
+                        videoc !== "wmv3";
+                }
+                break;
+            case "audio":
+                if ("audioc" in hit._source && !hit._props.isSubDocument) {
+                    hit._props.isAudio = true;
+                }
+                break;
+        }
+    }
+
+    setHitTags(hit: EsHit): void {
+        const tags = [] as Tag[];
+
+        const mimeCategory = hit._source.mime == null ? null : hit._source.mime.split("/")[0];
+
+        switch (mimeCategory) {
+            case "image":
+            case "video":
+                if ("videoc" in hit._source && hit._source.videoc) {
+                    tags.push({
+                        style: "video",
+                        text: hit._source.videoc.replace(" ", ""),
+                        userTag: false
+                    } as Tag);
+                }
+                break
+            case "audio":
+                if ("audioc" in hit._source && hit._source.audioc) {
+                    tags.push({
+                        style: "audio",
+                        text: hit._source.audioc,
+                        userTag: false
+                    } as Tag);
+                }
+                break;
+        }
+
+        // User tags
+        if ("tag" in hit._source) {
+            hit._source.tag.forEach(tag => {
+                tags.push(this.createUserTag(tag));
+            })
+        }
+
+        hit._tags = tags;
+    }
+
+    createUserTag(tag: string): Tag {
+        const tokens = tag.split(".");
+
+        const colorToken = tokens.pop() as string;
+
+        const bg = colorToken;
+        const fg = lum(colorToken) > 50 ? "#000" : "#fff";
+
+        return {
+            style: "user",
+            fg: fg,
+            bg: bg,
+            text: tokens.join("."),
+            rawText: tag,
+            userTag: true,
+        } as Tag;
+    }
+
+    esQuery(query: any): Promise<EsResult> {
+        return axios.post(`${this.baseUrl}es`, query).then(resp => {
+            const res = resp.data as EsResult;
+
+            if (res.hits?.hits) {
+                res.hits.hits.forEach((hit: EsHit) => {
+                    hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
+                    hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+                    hit["_path_md5"] = CryptoES.MD5(
+                        hit["_source"]["path"] +
+                        (hit["_source"]["path"] ? "/" : "") +
+                        hit["_source"]["name"] + ext(hit)
+                    ).toString();
+
+                    this.setHitProps(hit);
+                    this.setHitTags(hit);
+                });
+            }
+
+            return res;
+        });
+    }
+
+    getMimeTypes() {
+        return this.esQuery({
+            aggs: {
+                mimeTypes: {
+                    terms: {
+                        field: "mime",
+                        size: 10000
+                    }
+                }
+            },
+            size: 0,
+        }).then(resp => {
+            const mimeMap: any[] = [];
+            resp["aggregations"]["mimeTypes"]["buckets"].sort((a: any, b: any) => a.key > b.key).forEach((bucket: any) => {
+                const tmp = bucket["key"].split("/");
+                const category = tmp[0];
+                const mime = tmp[1];
+
+                let category_exists = false;
+
+                const child = {
+                    "id": bucket["key"],
+                    "text": `${mime} (${bucket["doc_count"]})`
+                };
+
+                mimeMap.forEach(node => {
+                    if (node.text === category) {
+                        node.children.push(child);
+                        category_exists = true;
+                    }
+                });
+
+                if (!category_exists) {
+                    mimeMap.push({"text": category, children: [child]});
+                }
+            })
+
+            return mimeMap;
+        });
+    }
+
+    _createEsTag(tag: string, count: number): EsTag {
+        const tokens = tag.split(".");
+
+        if (/.*\.#[0-9a-f]{6}/.test(tag)) {
+            return {
+                id: tokens.slice(0, -1).join("."),
+                color: tokens.pop(),
+                isLeaf: true,
+                count: count
+            };
+        }
+
+        return {
+            id: tag,
+            count: count,
+            isLeaf: false,
+            color: undefined
+        };
+    }
+
+    getDocInfo(docId: string) {
+        return axios.get(`${this.baseUrl}d/${docId}`);
+    }
+
+    getTags() {
+        return this.esQuery({
+            aggs: {
+                tags: {
+                    terms: {
+                        field: "tag",
+                        size: 10000
+                    }
+                }
+            },
+            size: 0,
+        }).then(resp => {
+            const seen = new Set();
+
+            const tags = resp["aggregations"]["tags"]["buckets"]
+                .sort((a: any, b: any) => a["key"].localeCompare(b["key"]))
+                .map((bucket: any) => this._createEsTag(bucket["key"], bucket["doc_count"]));
+
+            // Remove duplicates (same tag with different color)
+            return tags.filter((t: EsTag) => {
+                if (seen.has(t.id)) {
+                    return false;
+                }
+                seen.add(t.id);
+                return true;
+            });
+        });
+    }
+
+    saveTag(tag: string, hit: EsHit) {
+        return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
+            delete: false,
+            name: tag,
+            doc_id: hit["_id"],
+            path_md5: hit._path_md5
+        });
+    }
+
+    deleteTag(tag: string, hit: EsHit) {
+        return axios.post(`${this.baseUrl}tag/` + hit["_source"]["index"], {
+            delete: true,
+            name: tag,
+            doc_id: hit["_id"],
+            path_md5: hit._path_md5
+        });
+    }
+
+    getTreemapCsvUrl(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/1`;
+    }
+
+    getMimeCsvUrl(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/2`;
+    }
+
+    getSizeCsv(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/3`;
+    }
+
+    getDateCsv(indexId: string) {
+        return `${this.baseUrl}s/${indexId}/4`;
+    }
+}
+
+export default new Sist2Api("");

sist2-vue/src/Sist2Query.ts

@@ -0,0 +1,229 @@
+import store from "./store";
+import {EsHit, Index} from "@/Sist2Api";
+
+const SORT_MODES = {
+    score: {
+        mode: [
+            {_score: {order: "desc"}},
+            {_tie: {order: "asc"}}
+        ],
+        key: (hit: EsHit) => hit._score
+    },
+    random: {
+        mode: [
+            {_score: {order: "desc"}},
+            {_tie: {order: "asc"}}
+        ],
+        key: (hit: EsHit) => hit._score
+    },
+    dateAsc: {
+        mode: [
+            {mtime: {order: "asc"}},
+            {_tie: {order: "asc"}}
+        ],
+        key: (hit: EsHit) => hit._source.mtime
+    },
+    dateDesc: {
+        mode: [
+            {mtime: {order: "desc"}},
+            {_tie: {order: "asc"}}
+        ],
+        key: (hit: EsHit) => hit._source.mtime
+    },
+    sizeAsc: {
+        mode: [
+            {size: {order: "asc"}},
+            {_tie: {order: "asc"}}
+        ],
+        key: (hit: EsHit) => hit._source.size
+    },
+    sizeDesc: {
+        mode: [
+            {size: {order: "desc"}},
+            {_tie: {order: "asc"}}
+        ],
+        key: (hit: EsHit) => hit._source.size
+    }
+} as any;
+
+interface SortMode {
+    text: string
+    mode: any[]
+    key: (hit: EsHit) => any
+}
+
+
+class Sist2Query {
+
+    searchQuery(): any {
+
+        const getters = store.getters;
+
+        const searchText = getters.searchText;
+        const pathText = getters.pathText;
+        const empty = searchText === "";
+        const sizeMin = getters.sizeMin;
+        const sizeMax = getters.sizeMax;
+        const dateMin = getters.dateMin;
+        const dateMax = getters.dateMax;
+        const fuzzy = getters.fuzzy;
+        const size = getters.size;
+        const after = getters.lastDoc;
+        const selectedIndexIds = getters.selectedIndices.map((idx: Index) => idx.id)
+        const selectedMimeTypes = getters.selectedMimeTypes;
+        const selectedTags = getters.selectedTags;
+
+        const filters = [
+            {terms: {index: selectedIndexIds}}
+        ] as any[];
+
+        if (sizeMin && sizeMax) {
+            filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
+        } else if (sizeMin) {
+            filters.push({range: {size: {gte: sizeMin}}})
+        } else if (sizeMax) {
+            filters.push({range: {size: {lte: sizeMax}}})
+        }
+
+        if (dateMin && dateMax) {
+            filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
+        } else if (dateMin) {
+            filters.push({range: {mtime: {gte: dateMin}}})
+        } else if (dateMax) {
+            filters.push({range: {mtime: {lte: dateMax}}})
+        }
+
+        const fields = [
+            "name^8",
+            "content^3",
+            "album^8", "artist^8", "title^8", "genre^2", "album_artist^8",
+            "font_name^6"
+        ];
+
+        if (getters.optSearchInPath) {
+            fields.push("path.text^5");
+        }
+
+        if (fuzzy) {
+            fields.push("content.nGram");
+            if (getters.optSearchInPath) {
+                fields.push("path.nGram");
+            }
+            fields.push("name.nGram^3");
+        }
+
+        const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
+        if (path !== "") {
+            filters.push({term: {path: path}})
+        }
+
+        if (selectedMimeTypes.length > 0) {
+            filters.push({terms: {"mime": selectedMimeTypes}});
+        }
+
+        if (selectedTags.length > 0) {
+            if (getters.optTagOrOperator) {
+                filters.push({terms: {"tag": selectedTags}});
+            } else {
+                selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
+            }
+        }
+
+        let query;
+        if (getters.optQueryMode === "simple") {
+            query = {
+                simple_query_string: {
+                    query: searchText,
+                    fields: fields,
+                    default_operator: "and"
+                }
+            }
+        } else {
+            query = {
+                query_string: {
+                    query: searchText,
+                    default_field: "name",
+                    default_operator: "and"
+                }
+            }
+        }
+
+        const q = {
+            _source: {
+                excludes: ["content", "_tie"]
+            },
+            query: {
+                bool: {
+                    filter: filters,
+                }
+            },
+            sort: SORT_MODES[getters.sortMode].mode,
+            aggs:
+                {
+                    total_size: {"sum": {"field": "size"}},
+                    total_count: {"value_count": {"field": "size"}}
+                },
+            size: size,
+        } as any;
+
+        if (!empty) {
+            q.query.bool.must = query;
+        }
+
+        if (after) {
+            q.search_after = [SORT_MODES[getters.sortMode].key(after), after["_id"]];
+        }
+
+        if (getters.optHighlight) {
+            q.highlight = {
+                pre_tags: ["<mark>"],
+                post_tags: ["</mark>"],
+                fragment_size: getters.optFragmentSize,
+                number_of_fragments: 1,
+                order: "score",
+                fields: {
+                    content: {},
+                    name: {},
+                    "name.nGram": {},
+                    "content.nGram": {},
+                    font_name: {},
+                },
+                max_analyzed_offset: 9_999_999
+            };
+            if (getters.optSearchInPath) {
+                q.highlight.fields["path.text"] = {};
+                q.highlight.fields["path.nGram"] = {};
+            }
+        }
+
+        if (getters.sortMode === "random") {
+            q.query = {
+                function_score: {
+                    query: {
+                        bool: {
+                            must: filters,
+                        }
+                    },
+                    functions: [
+                        {
+                            random_score: {
+                                seed: getters.seed,
+                                field: "_seq_no",
+                            },
+                            weight: 1000
+                        }
+                    ],
+                    boost_mode: "sum"
+                }
+            }
+
+            if (!empty) {
+                q.query.function_score.query.bool.must.push(query);
+            }
+        }
+
+        return q;
+    }
+}
+
+export default new Sist2Query();

sist2-vue/src/components/ContentDiv.vue

@@ -0,0 +1,28 @@
+<template>
+  <div class="content-div" v-html="content()" v-if="content()"></div>
+</template>
+
+<script>
+export default {
+  name: "ContentDiv",
+  props: ["doc"],
+  methods: {
+    content() {
+      if (!this.doc.highlight) {
+        return null;
+      }
+
+      if (this.doc.highlight["content.nGram"]) {
+        return this.doc.highlight["content.nGram"][0];
+      }
+      if (this.doc.highlight.content) {
+        return this.doc.highlight.content[0];
+      }
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-vue/src/components/D3DateHistogram.vue

@@ -0,0 +1,129 @@
+<template>
+  <div class="graph">
+    <svg id="date-histogram"></svg>
+  </div>
+</template>
+
+<script>
+import * as d3 from "d3";
+import Sist2Api from "@/Sist2Api";
+
+const formatSI = d3.format("~s");
+
+
+function dateHistogram(data, svg, title) {
+  let bins = data.map(d => {
+    return {
+      length: Number(d.count),
+      x0: Number(d.bucket),
+      x1: Number(d.bucket) + 2629800
+    }
+  });
+  bins.sort((a, b) => a.length - b.length);
+
+  const margin = {
+    top: 50,
+    right: 20,
+    bottom: 70,
+    left: 40
+  };
+
+  const thresh = d3.quantile(bins, 0.9, d => d.length);
+  bins = bins.filter(d => d.length > thresh);
+
+  const width = 550;
+  const height = 450;
+
+  svg.selectAll("*").remove();
+  svg.attr("viewBox", [0, 0, width, height]);
+
+  const y = d3.scaleLinear()
+      .domain([0, d3.max(bins, d => d.length)]).nice()
+      .range([height - margin.bottom, margin.top]);
+
+  const x = d3.scaleLinear()
+      .domain(d3.extent(bins, d => d.x0)).nice()
+      .range([margin.left, width - margin.right]);
+
+  svg.append("g")
+      .attr("fill", "steelblue")
+      .selectAll("rect")
+      .data(bins)
+      .join("rect")
+      .attr("x", d => x(d.x0) + 1)
+      .attr("width", d => Math.max(1, x(d.x1) - x(d.x0) - 1))
+      .attr("y", d => y(d.length))
+      .attr("height", d => y(0) - y(d.length))
+      .call(g => g
+          .append("title")
+          .text(d => d.length)
+      );
+
+  svg.append("g")
+      .attr("transform", `translate(0,${height - margin.bottom})`)
+      .call(
+          d3.axisBottom(x)
+              .ticks(width / 30)
+              .tickSizeOuter(0)
+              .tickFormat(t => d3.timeFormat("%Y-%m-%d")(d3.utcParse("%s")(t)))
+      )
+      .call(g => g
+          .selectAll("text")
+          .style("text-anchor", "end")
+          .attr("dx", "-.8em")
+          .attr("dy", ".15em")
+          .attr("transform", "rotate(-65)")
+      )
+      .call(g => g.append("text")
+          .attr("x", width - margin.right)
+          .attr("y", -4)
+          .attr("fill", "currentColor")
+          .attr("font-weight", "bold")
+          .attr("text-anchor", "end")
+          .text("mtime")
+      );
+
+  svg.append("g")
+      .attr("transform", `translate(${margin.left},0)`)
+      .call(
+          d3.axisLeft(y)
+              .ticks(height / 40)
+              .tickFormat(t => formatSI(t))
+      )
+      .call(g => g.select(".domain").remove())
+      .call(g => g.select(".tick:last-of-type text").clone()
+          .attr("x", 4)
+          .attr("text-anchor", "start")
+          .attr("font-weight", "bold")
+          .text("File count"));
+
+  svg.append("text")
+      .attr("x", (width / 2))
+      .attr("y", (margin.top / 2))
+      .attr("text-anchor", "middle")
+      .style("font-size", "16px")
+      .text(title);
+}
+
+export default {
+  name: "D3DateHistogram",
+  props: ["indexId"],
+  mounted() {
+    this.update(this.indexId);
+  },
+  watch: {
+    indexId: function () {
+      this.update(this.indexId);
+    },
+  },
+  methods: {
+    update(indexId) {
+      const svg = d3.select("#date-histogram");
+
+      d3.csv(Sist2Api.getDateCsv(indexId)).then(tabularData => {
+        dateHistogram(tabularData.slice(), svg, this.$t("d3.dateHistogram"));
+      });
+    }
+  }
+}
+</script>

sist2-vue/src/components/D3MimeBarCount.vue

@@ -0,0 +1,100 @@
+<template>
+  <div class="graph">
+    <svg id="agg-mime-count"></svg>
+  </div>
+</template>
+
+<script>
+import * as d3 from "d3";
+import Sist2Api from "@/Sist2Api";
+
+const formatSI = d3.format("~s");
+const barHeight = 20;
+const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
+
+function mimeBarCount(data, svg, fillOpacity, title) {
+
+  const margin = {
+    top: 50,
+    right: 0,
+    bottom: 10,
+    left: Math.max(
+        d3.max(data.sort((a, b) => b.count - a.count).slice(0, 15), d => d.mime.length) * 6,
+        d3.max(data.sort((a, b) => b.size - a.size).slice(0, 15), d => d.mime.length) * 6,
+    )
+  };
+
+  data.forEach(d => {
+    d.name = d.mime;
+    d.value = Number(d.count);
+  });
+
+  data = data.sort((a, b) => b.value - a.value).slice(0, 15);
+
+  const width = 550;
+  const height = Math.ceil((data.length + 0.1) * barHeight) + margin.top + margin.bottom;
+
+  svg.selectAll("*").remove();
+  svg.attr("viewBox", [0, 0, width, height]);
+
+  const y = d3.scaleBand()
+      .domain(d3.range(data.length))
+      .rangeRound([margin.top, height - margin.bottom]);
+
+  const x = d3.scaleLinear()
+      .domain([0, d3.max(data, d => d.value)])
+      .range([margin.left, width - margin.right]);
+
+  svg.append("g")
+      .attr("fill-opacity", fillOpacity)
+      .selectAll("rect")
+      .data(data)
+      .join("rect")
+      .attr("fill", d => ordinalColor(d.name))
+      .attr("x", x(0))
+      .attr("y", (d, i) => y(i))
+      .attr("width", d => x(d.value) - x(0))
+      .attr("height", y.bandwidth())
+      .append("title")
+      .text(d => d3.format(",")(d.value));
+
+  svg.append("g")
+      .attr("transform", `translate(0,${margin.top})`)
+      .call(d3.axisTop(x).ticks(width / 80, data.format).tickFormat(formatSI))
+      .call(g => g.select(".domain").remove());
+
+  svg.append("g")
+      .attr("transform", `translate(${margin.left},0)`)
+      .call(d3.axisLeft(y).tickFormat(i => data[i].name).tickSizeOuter(0));
+
+  svg.append("text")
+      .attr("x", (width / 2))
+      .attr("y", (margin.top / 2))
+      .attr("text-anchor", "middle")
+      .style("font-size", "16px")
+      .text(title);
+}
+
+export default {
+  name: "D3MimeBarSize",
+  props: ["indexId"],
+  mounted() {
+    this.update(this.indexId);
+  },
+  watch: {
+    indexId: function () {
+      this.update(this.indexId);
+    },
+  },
+  methods: {
+    update(indexId) {
+      const mimeSvgCount = d3.select("#agg-mime-count");
+      const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
+
+      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
+        mimeBarCount(tabularData.slice(), mimeSvgCount, fillOpacity, this.$t("d3.mimeCount"));
+      });
+    }
+  }
+}
+</script>

sist2-vue/src/components/D3MimeBarSize.vue

@@ -0,0 +1,99 @@
+<template>
+  <div class="graph">
+    <svg id="agg-mime-size"></svg>
+  </div>
+</template>
+
+<script>
+import * as d3 from "d3";
+import Sist2Api from "@/Sist2Api";
+
+const formatSI = d3.format("~s");
+const barHeight = 20;
+const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
+
+function mimeBarSize(data, svg, fillOpacity, title) {
+
+  const margin = {
+    top: 50,
+    right: 0,
+    bottom: 10,
+    left: Math.max(
+        d3.max(data.sort((a, b) => b.count - a.count).slice(0, 15), d => d.mime.length) * 6,
+        d3.max(data.sort((a, b) => b.size - a.size).slice(0, 15), d => d.mime.length) * 6,
+    )
+  };
+
+  data.forEach(d => {
+    d.name = d.mime;
+    d.value = Number(d.size);
+  });
+  data = data.sort((a, b) => b.value - a.value).slice(0, 15);
+
+  const width = 550;
+  const height = Math.ceil((data.length + 0.1) * barHeight) + margin.top + margin.bottom;
+
+  svg.selectAll("*").remove();
+  svg.attr("viewBox", [0, 0, width, height]);
+
+  const y = d3.scaleBand()
+      .domain(d3.range(data.length))
+      .rangeRound([margin.top, height - margin.bottom]);
+
+  const x = d3.scaleLinear()
+      .domain([0, d3.max(data, d => d.value)])
+      .range([margin.left, width - margin.right]);
+
+  svg.append("g")
+      .attr("fill-opacity", fillOpacity)
+      .selectAll("rect")
+      .data(data)
+      .join("rect")
+      .attr("fill", d => ordinalColor(d.name))
+      .attr("x", x(0))
+      .attr("y", (d, i) => y(i))
+      .attr("width", d => x(d.value) - x(0))
+      .attr("height", y.bandwidth())
+      .append("title")
+      .text(d => formatSI(d.value));
+
+  svg.append("g")
+      .attr("transform", `translate(0,${margin.top})`)
+      .call(d3.axisTop(x).ticks(width / 80, data.format).tickFormat(formatSI))
+      .call(g => g.select(".domain").remove());
+
+  svg.append("g")
+      .attr("transform", `translate(${margin.left},0)`)
+      .call(d3.axisLeft(y).tickFormat(i => data[i].name).tickSizeOuter(0));
+
+  svg.append("text")
+      .attr("x", (width / 2))
+      .attr("y", (margin.top / 2))
+      .attr("text-anchor", "middle")
+      .style("font-size", "16px")
+      .text(title);
+}
+
+export default {
+  name: "D3MimeBarSize",
+  props: ["indexId"],
+  mounted() {
+    this.update(this.indexId);
+  },
+  watch: {
+    indexId: function () {
+      this.update(this.indexId);
+    },
+  },
+  methods: {
+    update(indexId) {
+      const mimeSvgSize = d3.select("#agg-mime-size");
+      const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
+
+      d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
+        mimeBarSize(tabularData.slice(), mimeSvgSize, fillOpacity, this.$t("d3.mimeSize"));
+      });
+    }
+  }
+}
+</script>

sist2-vue/src/components/D3SizeHistogram.vue

@@ -0,0 +1,126 @@
+<template>
+  <div class="graph">
+    <svg id="size-histogram"></svg>
+  </div>
+</template>
+
+<script>
+import * as d3 from "d3";
+import Sist2Api from "@/Sist2Api";
+
+const formatSI = d3.format("~s");
+
+function sizeHistogram(data, svg, title) {
+
+  let bins = data.map(d => {
+    return {
+      length: Number(d.count),
+      x0: Number(d.bucket),
+      x1: Number(d.bucket) + (5 * 1024 * 1024)
+    }
+  });
+  bins = bins.sort((a, b) => b.length - a.length).slice(0, 25);
+
+  const margin = {
+    top: 50,
+    right: 20,
+    bottom: 70,
+    left: 40
+  };
+
+  const width = 550;
+  const height = 450;
+
+  svg.selectAll("*").remove();
+  svg.attr("viewBox", [0, 0, width, height]);
+
+  const y = d3.scaleLinear()
+      .domain([0, d3.max(bins, d => d.length)])
+      .range([height - margin.bottom, margin.top]);
+
+  const x = d3.scaleLinear()
+      .domain(d3.extent(bins, d => d.x0)).nice()
+      .range([margin.left, width - margin.right]);
+
+  svg.append("g")
+      .attr("fill", "steelblue")
+      .selectAll("rect")
+      .data(bins)
+      .join("rect")
+      .attr("x", d => x(d.x0) + 1)
+      .attr("width", d => Math.max(1, x(d.x1) - x(d.x0) - 1))
+      .attr("y", d => y(d.length))
+      .attr("height", d => y(0) - y(d.length))
+      .call(g => g
+          .append("title")
+          .text(d => d.length)
+      );
+
+  svg.append("g")
+      .attr("transform", `translate(0,${height - margin.bottom})`)
+      .call(
+          d3.axisBottom(x)
+              .ticks(width / 30)
+              .tickSizeOuter(0)
+              .tickFormat(formatSI)
+      )
+      .call(g => g
+          .selectAll("text")
+          .style("text-anchor", "end")
+          .attr("dx", "-.8em")
+          .attr("dy", ".15em")
+          .attr("transform", "rotate(-65)")
+      )
+      .call(g => g.append("text")
+          .attr("x", width - margin.right)
+          .attr("y", -4)
+          .attr("fill", "currentColor")
+          .attr("font-weight", "bold")
+          .attr("text-anchor", "end")
+          .text("size (bytes)")
+      );
+
+  svg.append("g")
+      .attr("transform", `translate(${margin.left},0)`)
+      .call(
+          d3.axisLeft(y)
+              .ticks(height / 40)
+              .tickFormat(t => formatSI(t))
+      )
+      .call(g => g.select(".domain").remove())
+      .call(g => g.select(".tick:last-of-type text").clone()
+          .attr("x", 4)
+          .attr("text-anchor", "start")
+          .attr("font-weight", "bold")
+          .text("File count"));
+
+  svg.append("text")
+      .attr("x", (width / 2))
+      .attr("y", (margin.top / 2))
+      .attr("text-anchor", "middle")
+      .style("font-size", "16px")
+      .text(title);
+}
+
+export default {
+  name: "D3SizeHistogram",
+  props: ["indexId"],
+  mounted() {
+    this.update(this.indexId);
+  },
+  watch: {
+    indexId: function () {
+      this.update(this.indexId);
+    },
+  },
+  methods: {
+    update(indexId) {
+      const svg = d3.select("#size-histogram");
+
+      d3.csv(Sist2Api.getSizeCsv(indexId)).then(tabularData => {
+        sizeHistogram(tabularData.slice(), svg, this.$t("d3.sizeHistogram"));
+      });
+    }
+  }
+}
+</script>

sist2-vue/src/components/D3Treemap.vue

@@ -0,0 +1,267 @@
+<template>
+  <div>
+    <b-btn style="float:right;margin-bottom: 10px" @click="downloadTreemap()" variant="primary">
+      {{ $t("download") }}
+    </b-btn>
+    <svg id="treemap"></svg>
+  </div>
+</template>
+
+<script>
+import * as d3 from "d3";
+import {burrow} from "@/util-js"
+import {humanFileSize} from "@/util";
+import Sist2Api from "@/Sist2Api";
+import domtoimage from "dom-to-image";
+
+
+const TILING_MODES = {
+  "squarify": d3.treemapSquarify,
+  "binary": d3.treemapBinary,
+  "sliceDice": d3.treemapSliceDice,
+  "slice": d3.treemapSlice,
+  "dice": d3.treemapDice,
+};
+
+const COLORS = {
+  "PuBuGn": d3.interpolatePuBuGn,
+  "PuRd": d3.interpolatePuRd,
+  "PuBu": d3.interpolatePuBu,
+  "YlOrBr": d3.interpolateYlOrBr,
+  "YlOrRd": d3.interpolateYlOrRd,
+  "YlGn": d3.interpolateYlGn,
+  "YlGnBu": d3.interpolateYlGnBu,
+  "Plasma": d3.interpolatePlasma,
+  "Magma": d3.interpolateMagma,
+  "Inferno": d3.interpolateInferno,
+  "Viridis": d3.interpolateViridis,
+  "Turbo": d3.interpolateTurbo,
+};
+
+const SIZES = {
+  "small": [800, 600],
+  "medium": [1300, 750],
+  "large": [1900, 900],
+  "x-large": [2800, 1700],
+  "xx-large": [3600, 2000],
+};
+
+
+const uids = {};
+
+function uid(name) {
+  let id = uids[name] || 0;
+  uids[name] = id + 1;
+  return name + id;
+}
+
+function cascade(root, offset) {
+  const x = new Map;
+  const y = new Map;
+  return root.eachAfter(d => {
+    if (d.children && d.children.length !== 0) {
+      x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
+      y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
+    } else {
+      x.set(d, 0);
+      y.set(d, 0);
+    }
+  }).eachBefore(d => {
+    d.x1 -= 2 * offset * x.get(d);
+    d.y1 -= 2 * offset * y.get(d);
+  });
+}
+
+function cascadeTreemap(data, svg, width, height, tilingMode, treemapColor) {
+  const root = cascade(
+      d3.treemap()
+          .size([width, height])
+          .tile(TILING_MODES[tilingMode])
+          .paddingOuter(3)
+          .paddingTop(16)
+          .paddingInner(1)
+          .round(true)(
+              d3.hierarchy(data)
+                  .sum(d => d.value)
+                  .sort((a, b) => b.value - a.value)
+          ),
+      3 // treemap.paddingOuter
+  );
+
+  const maxDepth = Math.max(...root.descendants().map(d => d.depth));
+  const color = d3.scaleSequential([maxDepth, -1], COLORS[treemapColor]);
+
+  svg.append("filter")
+      .attr("id", "shadow")
+      .append("feDropShadow")
+      .attr("flood-opacity", 0.3)
+      .attr("dx", 0)
+      .attr("stdDeviation", 3);
+
+  const node = svg.selectAll("g")
+      .data(
+          d3.nest()
+              .key(d => d.depth).sortKeys(d3.ascending)
+              .entries(root.descendants())
+      )
+      .join("g")
+      .attr("filter", "url(#shadow)")
+      .selectAll("g")
+      .data(d => d.values)
+      .join("g")
+      .attr("transform", d => `translate(${d.x0},${d.y0})`);
+
+  node.append("title")
+      .text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+
+  node.append("rect")
+      .attr("id", d => (d.nodeUid = uid("node")))
+      .attr("fill", d => color(d.depth))
+      .attr("width", d => d.x1 - d.x0)
+      .attr("height", d => d.y1 - d.y0);
+
+  node.append("clipPath")
+      .attr("id", d => (d.clipUid = uid("clip")))
+      .append("use")
+      .attr("href", d => `#${d.nodeUid}`);
+
+  node.append("text")
+      .attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
+      .attr("clip-path", d => `url(#${d.clipUid})`)
+      .selectAll("tspan")
+      .data(d => [d.data.name, humanFileSize(d.value)])
+      .join("tspan")
+      .text(d => d);
+
+  node.filter(d => d.children).selectAll("tspan")
+      .attr("dx", 3)
+      .attr("y", 13);
+
+  node.filter(d => !d.children).selectAll("tspan")
+      .attr("x", 3)
+      .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`);
+}
+
+function flatTreemap(data, svg, width, height, groupingDepth, tilingMode, fillOpacity) {
+  const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
+
+  const root = d3.treemap()
+      .tile(TILING_MODES[tilingMode])
+      .size([width, height])
+      .padding(1)
+      .round(true)(
+          d3.hierarchy(data)
+              .sum(d => d.value)
+              .sort((a, b) => b.value - a.value)
+      );
+
+  const leaf = svg.selectAll("g")
+      .data(root.leaves())
+      .join("g")
+      .attr("transform", d => `translate(${d.x0},${d.y0})`);
+
+  leaf.append("title")
+      .text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+
+  leaf.append("rect")
+      .attr("id", d => (d.leafUid = uid("leaf")))
+      .attr("fill", d => {
+        while (d.depth > groupingDepth) d = d.parent;
+        return ordinalColor(d.data.name);
+      })
+      .attr("fill-opacity", fillOpacity)
+      .attr("width", d => d.x1 - d.x0)
+      .attr("height", d => d.y1 - d.y0);
+
+  leaf.append("clipPath")
+      .attr("id", d => (d.clipUid = uid("clip")))
+      .append("use")
+      .attr("href", d => `#${d.leafUid}`);
+
+  leaf.append("text")
+      .attr("clip-path", d => `url(#${d.clipUid})`)
+      .selectAll("tspan")
+      .data(d => {
+        if (d.data.name === ".") {
+          d = d.parent;
+        }
+        return [d.data.name, humanFileSize(d.value)]
+      })
+      .join("tspan")
+      .attr("x", 2)
+      .attr("y", (d, i) => `${i === 0 ? 1.1 : 2.3}em`)
+      .text(d => d);
+}
+
+function exportTreemap(indexName, width, height) {
+  domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
+      .then(function (blob) {
+        let a = document.createElement("a");
+        let url = URL.createObjectURL(blob);
+
+        a.href = url;
+        a.download = `${indexName}_treemap.png`;
+        document.body.appendChild(a);
+        a.click();
+        setTimeout(function () {
+          document.body.removeChild(a);
+          window.URL.revokeObjectURL(url);
+        }, 0);
+      });
+}
+
+export default {
+  name: "D3Treemap",
+  props: ["indexId"],
+  watch: {
+    indexId: function () {
+      this.update(this.indexId);
+    }
+  },
+  mounted() {
+    this.update(this.indexId);
+  },
+  methods: {
+    update(indexId) {
+      const width = SIZES[this.$store.state.optTreemapSize][0];
+      const height = SIZES[this.$store.state.optTreemapSize][1];
+      const tilingMode = this.$store.state.optTreemapTiling;
+      const groupingDepth = this.$store.state.optTreemapColorGroupingDepth;
+      const treemapColor = this.$store.state.optTreemapColor;
+      const treemapType = this.$store.state.optTreemapType;
+
+      const treemapSvg = d3.select("#treemap");
+
+      treemapSvg.selectAll("*").remove();
+      treemapSvg.attr("viewBox", [0, 0, width, height])
+          .attr("xmlns", "http://www.w3.org/2000/svg")
+          .attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
+          .attr("version", "1.1")
+          .style("overflow", "visible")
+          .style("font", "10px sans-serif");
+
+      d3.csv(Sist2Api.getTreemapCsvUrl(indexId)).then(tabularData => {
+        tabularData.forEach(row => {
+          row.taxonomy = row.path.split("/");
+          row.size = Number(row.size);
+        });
+
+        if (treemapType === "cascaded") {
+          const data = burrow(tabularData, false);
+          cascadeTreemap(data, treemapSvg, width, height, tilingMode, treemapColor);
+        } else {
+          const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
+          const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
+          flatTreemap(data, treemapSvg, width, height, groupingDepth, tilingMode, fillOpacity);
+        }
+      });
+    },
+    downloadTreemap() {
+      const width = SIZES[this.$store.state.optTreemapSize][0];
+      const height = SIZES[this.$store.state.optTreemapSize][1];
+
+      exportTreemap(this.indexId, width, height);
+    }
+  }
+}
+</script>

sist2-vue/src/components/DateSlider.vue

@@ -0,0 +1,66 @@
+<template>
+  <div id="dateSlider"></div>
+</template>
+
+<script>
+import noUiSlider from 'nouislider';
+import 'nouislider/dist/nouislider.css';
+import {humanDate} from "@/util";
+import {mergeTooltips} from "@/util-js";
+
+export default {
+  name: "DateSlider",
+  mounted() {
+    this.$store.subscribe((mutation) => {
+      if (mutation.type === "setDateBoundsMax") {
+        const elem = document.getElementById("dateSlider");
+
+        if (elem.children.length > 0) {
+          return;
+        }
+
+        const dateMax = this.$store.state.dateBoundsMax;
+        const dateMin = this.$store.state.dateBoundsMin;
+
+        const slider = noUiSlider.create(elem, {
+          start: [
+            this.$store.state.dateMin ? this.$store.state.dateMin : dateMin,
+            this.$store.state.dateMax ? this.$store.state.dateMax : dateMax
+          ],
+
+          tooltips: [true, true],
+          behaviour: "drag-tap",
+          connect: true,
+          range: {
+            "min": dateMin,
+            "max": dateMax,
+          },
+          format: {
+            to: x => humanDate(x),
+            from: x => x
+          }
+        });
+
+        mergeTooltips(elem, 10, " - ", true)
+
+        elem.querySelectorAll('.noUi-connect')[0].classList.add("slider-color0")
+
+        slider.on("set", (values, handle, unencoded) => {
+          if (handle === 0) {
+            this.$store.commit("setDateMin", unencoded[0] === dateMin ? undefined : Math.round(unencoded[0]));
+          } else {
+            this.$store.commit("setDateMax", unencoded[1] >= dateMax ? undefined : Math.round(unencoded[1]));
+          }
+        });
+      }
+    });
+  }
+}
+</script>
+
+<style>
+#dateSlider {
+  margin-top: 34px;
+  height: 12px;
+}
+</style>

sist2-vue/src/components/DebugIcon.vue

@@ -0,0 +1,22 @@
+<template>
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 309.998 309.998" fill="currentColor">
+    <path
+        d="M294.998,155.03H250v-48.82l39.714-39.715c5.858-5.857,5.858-15.356,0-21.213c-5.857-5.857-15.355-5.857-21.213,0 l-23.7,23.701c-12.885-37.2-48.274-63.984-89.802-63.984c-41.528,0-76.913,26.787-89.797,63.989L41.497,45.282 c-5.856-5.859-15.354-5.857-21.213,0s-5.858,15.355,0,21.213L60,106.212v48.818H15c-8.284,0-15,6.716-15,15c0,8.284,6.716,15,15,15 h45.134c0.855,16.314,5.849,31.551,13.944,44.68l-49.685,49.683c-5.858,5.857-5.858,15.354,0,21.213 c2.929,2.93,6.768,4.394,10.607,4.394c3.838,0,7.678-1.465,10.606-4.394l48.095-48.093c16.558,14.018,37.957,22.486,61.301,22.486 c0.019,0,0.037-0.001,0.057-0.001c0.011,0,0.022,0.002,0.033,0.002c0.019,0,0.037-0.003,0.056-0.003 c23.285-0.035,44.629-8.494,61.15-22.483l48.094,48.092c2.929,2.929,6.768,4.394,10.606,4.394c3.839,0,7.678-1.465,10.607-4.394 c5.858-5.858,5.858-15.355,0-21.213l-49.683-49.681c8.096-13.131,13.089-28.366,13.944-44.682h45.132c8.284,0,15-6.716,15-15 C309.998,161.746,303.282,155.03,294.998,155.03z M154.999,34.999c30.681,0,56.465,21.365,63.254,50H91.747 C98.535,56.364,124.318,34.999,154.999,34.999z M90,179.999v-9.272c0.011-0.232,0.035-0.462,0.035-0.696 c0-0.234-0.024-0.464-0.035-0.695v-54.336h50.092v128.254C111.415,236.494,90,210.708,90,179.999z M170.092,243.212V114.999H220 v54.297c-0.012,0.244-0.037,0.486-0.037,0.734c0,0.248,0.025,0.49,0.037,0.734v9.234C220,210.645,198.676,236.388,170.092,243.212z"/>
+  </svg>
+
+</template>
+
+<script>
+export default {
+  name: "DebugIcon"
+}
+</script>
+
+<style scoped>
+svg {
+  display: inline-block;
+  width: 20px;
+  height: 20px;
+  vertical-align: middle;
+}
+</style>

sist2-vue/src/components/DebugInfo.vue

@@ -0,0 +1,39 @@
+<template>
+  <b-card class="mb-4 mt-4">
+    <b-card-title><DebugIcon class="mr-1"></DebugIcon>{{ $t("debug") }}</b-card-title>
+    <p v-html="$t('debugDescription')"></p>
+
+    <b-card-body>
+
+      <!-- TODO: ES connectivity, Link to GH page -->
+      <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
+
+      <hr />
+      <IndexDebugInfo v-for="idx of $store.state.sist2Info.indices" :key="idx.id" :index="idx" class="mt-2"></IndexDebugInfo>
+    </b-card-body>
+  </b-card>
+</template>
+
+<script>
+import IndexDebugInfo from "@/components/IndexDebugInfo";
+import DebugIcon from "@/components/DebugIcon";
+
+export default {
+  name: "DebugInfo.vue",
+  components: {DebugIcon, IndexDebugInfo},
+  computed: {
+    tableItems() {
+      return [
+        {key: "version", value: this.$store.state.sist2Info.version},
+        {key: "platform", value: this.$store.state.sist2Info.platform},
+        {key: "debugBinary", value: this.$store.state.sist2Info.debug},
+        {key: "sist2CommitHash", value: this.$store.state.sist2Info.sist2Hash},
+        {key: "libscanCommitHash", value: this.$store.state.sist2Info.libscanHash},
+        {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
+        {key: "tagline", value: this.$store.state.sist2Info.tagline},
+        {key: "dev", value: this.$store.state.sist2Info.dev},
+      ]
+    }
+  }
+}
+</script>

sist2-vue/src/components/DisplayModeToggle.vue

@@ -0,0 +1,37 @@
+<template>
+  <b-button-group>
+    <b-button variant="primary" :title="$t('displayMode.list')" :pressed="optDisplay==='list'"
+              @click="setOptDisplay('list')">
+      <svg width="20px" height="20px" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512">
+        <path fill="currentColor"
+              d="M80 368H16a16 16 0 0 0-16 16v64a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16v-64a16 16 0 0 0-16-16zm0-320H16A16 16 0 0 0 0 64v64a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16V64a16 16 0 0 0-16-16zm0 160H16a16 16 0 0 0-16 16v64a16 16 0 0 0 16 16h64a16 16 0 0 0 16-16v-64a16 16 0 0 0-16-16zm416 176H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16zm0-320H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16V80a16 16 0 0 0-16-16zm0 160H176a16 16 0 0 0-16 16v32a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16v-32a16 16 0 0 0-16-16z"></path>
+      </svg>
+    </b-button>
+
+    <b-button variant="primary" :title="$t('displayMode.grid')" :pressed="optDisplay==='grid'"
+              @click="setOptDisplay('grid')">
+      <svg width="20px" height="20px" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512">
+        <path fill="currentColor"
+              d="M149.333 56v80c0 13.255-10.745 24-24 24H24c-13.255 0-24-10.745-24-24V56c0-13.255 10.745-24 24-24h101.333c13.255 0 24 10.745 24 24zm181.334 240v-80c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24h101.333c13.256 0 24.001-10.745 24.001-24zm32-240v80c0 13.255 10.745 24 24 24H488c13.255 0 24-10.745 24-24V56c0-13.255-10.745-24-24-24H386.667c-13.255 0-24 10.745-24 24zm-32 80V56c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24h101.333c13.256 0 24.001-10.745 24.001-24zm-205.334 56H24c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24zM0 376v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H24c-13.255 0-24 10.745-24 24zm386.667-56H488c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H386.667c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24zm0 160H488c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H386.667c-13.255 0-24 10.745-24 24v80c0 13.255 10.745 24 24 24zM181.333 376v80c0 13.255 10.745 24 24 24h101.333c13.255 0 24-10.745 24-24v-80c0-13.255-10.745-24-24-24H205.333c-13.255 0-24 10.745-24 24z"></path>
+      </svg>
+    </b-button>
+  </b-button-group>
+</template>
+
+<script>
+import {mapGetters, mapMutations} from "vuex";
+
+export default {
+  name: "DisplayModeToggle",
+  computed: {
+    ...mapGetters(["optDisplay"])
+  },
+  methods: {
+    ...mapMutations(["setOptDisplay"])
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-vue/src/components/DocCard.vue

@@ -0,0 +1,232 @@
+<template>
+  <div class="doc-card" :class="{'sub-document': doc._props.isSubDocument}" :style="`width: ${width}px`">
+    <b-card
+        no-body
+        img-top
+    >
+      <!-- Info modal-->
+      <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
+
+      <ContentDiv :doc="doc"></ContentDiv>
+
+      <!-- Thumbnail-->
+      <div v-if="doc._props.hasThumbnail" class="img-wrapper" @mouseenter="onTnEnter()" @mouseleave="onTnLeave()">
+        <div v-if="doc._props.isAudio" class="card-img-overlay" :class="{'small-badge': smallBadge}">
+          <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
+        </div>
+
+        <div v-if="doc._props.isImage && !hover" class="card-img-overlay" :class="{'small-badge': smallBadge}">
+          <span class="badge badge-resolution">{{ `${doc._source.width}x${doc._source.height}` }}</span>
+        </div>
+
+        <div v-if="(doc._props.isVideo || doc._props.isGif) && doc._source.duration > 0 && !hover" class="card-img-overlay"
+             :class="{'small-badge': smallBadge}">
+          <span class="badge badge-resolution">{{ humanTime(doc._source.duration) }}</span>
+        </div>
+
+        <div v-if="doc._props.isPlayableVideo" class="play">
+          <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
+            <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
+          </svg>
+        </div>
+
+        <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
+             :src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
+             alt=""
+             class="pointer fit card-img-top" @click="onThumbnailClick()">
+        <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
+             class="fit card-img-top">
+      </div>
+
+      <!-- Audio player-->
+      <audio v-if="doc._props.isAudio" ref="audio" preload="none" class="audio-fit fit" controls :type="doc._source.mime"
+             :src="`f/${doc._id}`"
+             @play="onAudioPlay()"></audio>
+
+      <b-card-body class="padding-03">
+
+        <!-- Title line -->
+        <div style="display: flex">
+          <span class="info-icon" @click="onInfoClick()"></span>
+          <DocFileTitle :doc="doc"></DocFileTitle>
+        </div>
+
+        <!-- Tags -->
+        <div class="card-text">
+          <TagContainer :hit="doc"></TagContainer>
+        </div>
+      </b-card-body>
+    </b-card>
+  </div>
+</template>
+
+<script>
+import {ext, humanFileSize, humanTime} from "@/util";
+import TagContainer from "@/components/TagContainer.vue";
+import DocFileTitle from "@/components/DocFileTitle.vue";
+import DocInfoModal from "@/components/DocInfoModal.vue";
+import ContentDiv from "@/components/ContentDiv.vue";
+
+
+export default {
+  components: {ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+  props: ["doc", "width"],
+  data() {
+    return {
+      ext: ext,
+      showInfo: false,
+      hover: false
+    }
+  },
+  computed: {
+    placeHolderStyle() {
+
+      const tokens = this.doc._source.thumbnail.split(",");
+      const w = Number(tokens[0]);
+      const h = Number(tokens[1]);
+
+      const MAX_HEIGHT = 400;
+
+      return {
+        height: `${Math.min((h / w) * this.width, MAX_HEIGHT)}px`,
+      }
+    },
+    smallBadge() {
+      return this.width < 150;
+    }
+  },
+  methods: {
+    humanFileSize: humanFileSize,
+    humanTime: humanTime,
+    onInfoClick() {
+      this.showInfo = true;
+    },
+    async onThumbnailClick() {
+      this.$store.commit("setUiLightboxSlide", this.doc._seq);
+      await this.$store.dispatch("showLightbox");
+    },
+    onAudioPlay() {
+      document.getElementsByTagName("audio").forEach((el) => {
+        if (el !== this.$refs["audio"]) {
+          el.pause();
+        }
+      });
+    },
+    onTnEnter() {
+      this.hover = true;
+    },
+    onTnLeave() {
+      this.hover = false;
+    }
+  },
+}
+</script>
+<style>
+.img-wrapper {
+  position: relative;
+}
+
+.img-wrapper:hover svg {
+  fill: rgba(0, 0, 0, 1);
+}
+
+.pointer {
+  cursor: pointer;
+}
+
+.fit {
+  display: block;
+  min-width: 64px;
+  max-width: 100%;
+  /*max-height: 400px;*/
+  margin: 0 auto 0;
+  width: auto;
+  height: auto;
+}
+</style>
+
+<style scoped>
+
+.card-img-top {
+  border-top-left-radius: 0;
+  border-top-right-radius: 0;
+}
+
+.padding-03 {
+  padding: 0.3rem;
+}
+
+.card {
+  margin-top: 1em;
+  margin-left: 0;
+  margin-right: 0;
+  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+  border-radius: 0;
+  border: none;
+}
+
+.card-body {
+  padding: 0.3rem;
+}
+
+.thumbnail-placeholder {
+
+}
+
+.card-img-overlay {
+  pointer-events: none;
+  padding: 0.75rem;
+  bottom: unset;
+  top: 0;
+  left: unset;
+  right: unset;
+}
+
+.badge-resolution {
+  color: #212529;
+  background-color: #FFC107;
+}
+
+.play {
+  position: absolute;
+  width: 25px;
+  height: 25px;
+  left: 50%;
+  top: 50%;
+  transform: translate(-50%, -50%);
+  pointer-events: none;
+}
+
+.play svg {
+  fill: rgba(0, 0, 0, 0.7);
+}
+
+.doc-card {
+  padding-left: 3px;
+  padding-right: 3px;
+}
+
+.small-badge {
+  padding: 1px 3px;
+  font-size: 70%;
+}
+
+.audio-fit {
+  height: 39px;
+  vertical-align: bottom;
+  display: inline;
+  width: 100%;
+}
+
+.sub-document .card {
+  background: #AB47BC1F !important;
+}
+
+.theme-black .sub-document .card {
+  background: #37474F !important;
+}
+
+.sub-document .fit {
+  padding: 4px 4px 0 4px;
+}
+</style>

sist2-vue/src/components/DocCardWall.vue

@@ -0,0 +1,69 @@
+<template>
+  <GridLayout
+      ref="grid-layout"
+      :options="gridOptions"
+      @append="append"
+      @layout-complete="$emit('layout-complete')"
+  >
+    <DocCard v-for="doc in docs" :key="doc._id" :doc="doc" :width="width"></DocCard>
+  </GridLayout>
+</template>
+
+<script>
+import Vue from "vue";
+import DocCard from "@/components/DocCard";
+
+import VueInfiniteGrid from "@egjs/vue-infinitegrid";
+
+Vue.use(VueInfiniteGrid);
+
+export default Vue.extend({
+  components: {
+    DocCard,
+  },
+  props: ["docs", "append"],
+  data() {
+    return {
+      width: 0,
+      gridOptions: {
+        align: "center",
+        margin: 0,
+        transitionDuration: 0,
+        isOverflowScroll: false,
+        isConstantSize: false,
+        useFit: false,
+        // Indicates whether keep the number of DOMs is maintained. If the useRecycle value is 'true', keep the number
+        //  of DOMs is maintained. If the useRecycle value is 'false', the number of DOMs will increase as card elements
+        //  are added.
+        useRecycle: false
+      }
+    }
+  },
+  computed: {
+    colCount() {
+      const columns = this.$store.getters["optColumns"];
+
+      if (columns === "auto") {
+        return Math.round(this.$refs["grid-layout"].$el.scrollWidth / 300)
+      }
+      return columns;
+    },
+  },
+  mounted() {
+    this.width = this.$refs["grid-layout"].$el.scrollWidth / this.colCount;
+
+    if (this.colCount === 1) {
+      this.$refs["grid-layout"].$el.classList.add("grid-single-column");
+    }
+
+    this.$store.subscribe((mutation) => {
+      if (mutation.type === "busUpdateWallItems" && this.$refs["grid-layout"]) {
+        this.$refs["grid-layout"].updateItems();
+      }
+    });
+  },
+});
+</script>
+
+<style>
+</style>

sist2-vue/src/components/DocFileTitle.vue

@@ -0,0 +1,63 @@
+<template>
+  <a :href="`f/${doc._id}`" class="file-title-anchor" target="_blank">
+    <div class="file-title" :title="doc._source.path + '/' + doc._source.name + ext(doc)"
+         v-html="fileName() + ext(doc)"></div>
+  </a>
+</template>
+
+<script>
+import {ext} from "@/util";
+
+export default {
+  name: "DocFileTitle",
+  props: ["doc"],
+  methods: {
+    ext: ext,
+    fileName() {
+      if (!this.doc.highlight) {
+        return this.doc._source.name;
+      }
+      if (this.doc.highlight["name.nGram"]) {
+        return this.doc.highlight["name.nGram"];
+      }
+      if (this.doc.highlight.name) {
+        return this.doc.highlight.name;
+      }
+      return this.doc._source.name;
+    }
+  }
+}
+</script>
+
+<style scoped>
+.file-title-anchor {
+  max-width: calc(100% - 1.2rem);
+}
+
+.file-title {
+  width: 100%;
+  line-height: 1rem;
+  height: 1.1rem;
+  white-space: nowrap;
+  text-overflow: ellipsis;
+  overflow: hidden;
+  font-size: 16px;
+  font-family: "Source Sans Pro", sans-serif;
+  font-weight: bold;
+}
+
+.theme-black .file-title {
+  color: #ddd;
+}
+.theme-black .file-title:hover {
+  color: #fff;
+}
+
+.theme-light .file-title {
+  color: black;
+}
+
+.doc-card .file-title {
+  font-size: 12px;
+}
+</style>

sist2-vue/src/components/DocInfoModal.vue

@@ -0,0 +1,33 @@
+<template>
+  <b-modal :visible="show" size="lg" :hide-footer="true" static lazy @close="$emit('close')" @hide="$emit('close')"
+  >
+    <template #modal-title>
+      <h5 class="modal-title" :title="doc._source.name + ext(doc)">{{ doc._source.name + ext(doc) }}</h5>
+    </template>
+
+    <img v-if="doc._props.hasThumbnail" :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">
+
+    <InfoTable :doc="doc"></InfoTable>
+
+    <LazyContentDiv :doc-id="doc._id"></LazyContentDiv>
+  </b-modal>
+</template>
+
+<script>
+import {ext} from "@/util";
+import InfoTable from "@/components/InfoTable";
+import LazyContentDiv from "@/components/LazyContentDiv";
+
+export default {
+  name: "DocInfoModal",
+  components: {LazyContentDiv, InfoTable},
+  props: ["doc", "show"],
+  methods: {
+    ext: ext,
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-vue/src/components/DocList.vue

@@ -0,0 +1,45 @@
+<template>
+  <b-list-group class="mt-3">
+    <DocListItem v-for="doc in docs" :key="doc._id" :doc="doc"></DocListItem>
+  </b-list-group>
+</template>
+
+<script lang="ts">
+import DocListItem from "@/components/DocListItem.vue";
+import Vue from "vue";
+
+export default Vue.extend({
+  name: "DocList",
+  components: {DocListItem},
+  props: ["docs", "append"],
+  mounted() {
+    window.addEventListener("scroll", () => {
+      const threshold = 400;
+      const app = document.getElementById("app");
+
+      if ((window.innerHeight + window.scrollY) >= app.offsetHeight - threshold) {
+        this.append();
+      }
+    });
+  }
+});
+</script>
+
+<style>
+
+.theme-black .list-group-item {
+  background: #212121;
+  color: #e0e0e0;
+
+  border-bottom: none;
+  border-left: none;
+  border-right: none;
+  border-radius: 0;
+  padding: .25rem 0.5rem;
+}
+
+.theme-black .list-group-item:first-child {
+  border-top: none;
+}
+
+</style>

sist2-vue/src/components/DocListItem.vue

@@ -0,0 +1,180 @@
+<template>
+  <b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}">
+
+    <!-- Info modal-->
+    <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
+
+    <div class="media ml-2">
+      <div v-if="doc._props.hasThumbnail" class="align-self-start mr-2 wrapper-sm">
+        <div class="img-wrapper">
+          <div v-if="doc._props.isPlayableVideo" class="play">
+            <svg viewBox="0 0 494.942 494.942" xmlns="http://www.w3.org/2000/svg">
+              <path d="m35.353 0 424.236 247.471-424.236 247.471z"/>
+            </svg>
+          </div>
+
+          <img v-if="doc._props.isPlayableImage || doc._props.isPlayableVideo"
+               :src="(doc._props.isGif && hover) ? `f/${doc._id}` : `t/${doc._source.index}/${doc._id}`"
+               alt=""
+               class="pointer fit-sm" @click="onThumbnailClick()">
+          <img v-else :src="`t/${doc._source.index}/${doc._id}`" alt=""
+               class="fit-sm">
+        </div>
+      </div>
+      <div v-else class="file-icon-wrapper" style="">
+        <FileIcon></FileIcon>
+      </div>
+
+      <div class="doc-line ml-3">
+        <div style="display: flex">
+          <span class="info-icon" @click="showInfo = true"></span>
+          <DocFileTitle :doc="doc"></DocFileTitle>
+        </div>
+
+        <!-- Content highlight -->
+        <ContentDiv :doc="doc"></ContentDiv>
+
+        <div class="path-row">
+          <div class="path-line" v-html="path()"></div>
+          <TagContainer :hit="doc"></TagContainer>
+        </div>
+
+        <div v-if="doc._source.pages || doc._source.author" class="path-row text-muted">
+          <span v-if="doc._source.pages">{{ doc._source.pages }} {{
+              doc._source.pages > 1 ? $t("pages") : $t("page")
+            }}</span>
+          <span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
+          <span v-if="doc._source.author">{{ doc._source.author }}</span>
+        </div>
+      </div>
+    </div>
+  </b-list-group-item>
+</template>
+
+<script>
+import TagContainer from "@/components/TagContainer";
+import DocFileTitle from "@/components/DocFileTitle";
+import DocInfoModal from "@/components/DocInfoModal";
+import ContentDiv from "@/components/ContentDiv";
+import FileIcon from "@/components/FileIcon";
+
+export default {
+  name: "DocListItem",
+  components: {FileIcon, ContentDiv, DocInfoModal, DocFileTitle, TagContainer},
+  props: ["doc"],
+  data() {
+    return {
+      hover: false,
+      showInfo: false
+    }
+  },
+  methods: {
+    async onThumbnailClick() {
+      this.$store.commit("setUiLightboxSlide", this.doc._seq);
+      await this.$store.dispatch("showLightbox");
+    },
+    path() {
+      if (!this.doc.highlight) {
+        return this.doc._source.path + "/"
+      }
+      if (this.doc.highlight["path.text"]) {
+        return this.doc.highlight["path.text"] + "/"
+      }
+
+      if (this.doc.highlight["path.nGram"]) {
+        return this.doc.highlight["path.nGram"] + "/"
+      }
+      return this.doc._source.path + "/"
+    }
+  }
+}
+</script>
+
+<style scoped>
+.sub-document {
+  background: #AB47BC1F !important;
+}
+
+.theme-black .sub-document {
+  background: #37474F !important;
+}
+
+.list-group {
+  margin-top: 1em;
+}
+
+.list-group-item {
+  padding: .25rem 0.5rem;
+
+  box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
+  border-radius: 0;
+  border: none;
+}
+
+.path-row {
+  display: -ms-flexbox;
+  display: flex;
+  -ms-flex-align: start;
+  align-items: flex-start;
+}
+
+.path-line {
+  color: #808080;
+  text-overflow: ellipsis;
+  overflow: hidden;
+  white-space: nowrap;
+  margin-right: 0.3em;
+}
+
+.theme-black .path-line {
+  color: #bbb;
+}
+
+.play {
+  position: absolute;
+  width: 18px;
+  height: 18px;
+  left: 50%;
+  top: 50%;
+  transform: translate(-50%, -50%);
+  pointer-events: none;
+}
+
+.play svg {
+  fill: rgba(0, 0, 0, 0.7);
+}
+
+.list-group-item .img-wrapper {
+  width: 88px;
+  height: 88px;
+}
+
+.fit-sm {
+  max-height: 100%;
+  max-width: 100%;
+  width: auto;
+  height: auto;
+  position: absolute;
+  top: 0;
+  bottom: 0;
+  left: 0;
+  right: 0;
+  margin: auto;
+
+  /*box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.12);*/
+}
+
+.doc-line {
+  max-width: calc(100% - 88px - 1.5rem);
+  flex: 1;
+  vertical-align: middle;
+  margin-top: auto;
+  margin-bottom: auto;
+}
+
+.file-icon-wrapper {
+  width: calc(88px + .5rem);
+  height: 88px;
+  position: relative;
+}
+</style>

sist2-vue/src/components/FileIcon.vue

@@ -0,0 +1,33 @@
+<template>
+  <svg class="file-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 50 50">
+    <path
+        fill="currentColor"
+        d="M 7 2 L 7 48 L 43 48 L 43 14.59375 L 42.71875 14.28125 L 30.71875 2.28125 L 30.40625 2 Z M 9 4 L 29 4 L 29 16 L 41 16 L 41 46 L 9 46 Z M 31 5.4375 L 39.5625 14 L 31 14 Z"/>
+  </svg>
+</template>
+
+<script>
+export default {
+  name: "FileIcon"
+}
+</script>
+
+<style scoped>
+.file-icon {
+  position: absolute;
+  max-height: 100%;
+  max-width: 100%;
+  top: 0;
+  bottom: 0;
+  left: 0;
+  right: 0;
+}
+
+.theme-black .file-icon {
+  color: #ffffff50;
+}
+
+.theme-light .file-icon {
+  color: #000000a0;
+}
+</style>

sist2-vue/src/components/GearIcon.vue

@@ -0,0 +1,23 @@
+<template>
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 1000" fill="currentColor">
+    <g transform="translate(0.000000,512.000000) scale(0.100000,-0.100000)">
+      <path
+          d="M4568.5,5011c-73.2-7.7-154-25-177.1-36.6c-84.7-46.2-102-119.4-159.8-689.2s-65.5-610.3-159.8-670c-23.1-15.4-125.1-55.8-225.3-90.5c-100.1-32.7-290.7-111.7-423.6-175.2c-319.6-152.1-315.8-152.1-619.9,94.3c-718.1,583.3-650.7,535.2-747,535.2c-77,0-104-11.6-184.8-77c-157.9-127.1-410.1-375.4-567.9-558.3c-155.9-177.1-190.6-250.3-159.8-344.6c9.6-27,165.6-227.2,344.6-446.7c181-219.5,342.7-425.5,360-458.2c52-88.6,42.3-150.2-50.1-335c-73.2-148.3-144.4-325.4-252.2-623.8c-17.3-50-57.8-113.6-88.6-138.6c-63.5-53.9-59.7-53.9-695-117.4c-527.5-52-577.6-65.5-627.6-179c-46.2-105.9-46.2-1057,0-1162.9c50-113.6,98.2-127.1,646.9-181c271.5-25,523.7-52,560.2-57.8c111.7-17.3,179.1-107.8,259.9-344.6c38.5-115.5,119.4-310,177.1-431.3c57.8-119.4,104-240.7,104-269.5c0-78.9-42.4-140.5-394.7-568c-179-219.5-335-419.7-344.6-446.6c-30.8-94.3,3.9-167.5,159.8-344.6c157.9-181,410.1-429.3,564.1-554.5c96.3-78.9,188.7-105.9,265.7-75.1c26.9,11.6,234.9,173.3,462.1,360c227.2,188.7,433.2,348.5,458.2,358.1c82.8,30.8,136.7,17.3,354.3-86.6c119.4-57.8,308-136.7,419.7-175.2c111.7-38.5,221.4-82.8,244.5-98.2c94.3-59.7,102-100.1,159.8-670c61.6-606.5,73.2-648.8,188.7-700.8c105.9-46.2,1057-46.2,1162.9,0c115.5,52,127.1,94.3,188.7,700.8c57.8,569.9,65.4,610.3,159.8,670c23.1,15.4,132.9,59.7,244.5,98.2s300.3,117.4,417.8,175.2c219.5,104,273.4,117.5,356.2,86.6c25-9.6,231-169.4,458.2-358.1c227.2-186.8,435.1-348.5,462.1-360c77-28.9,169.4-3.9,265.7,75.1c152.1,121.3,442.8,410.1,583.4,577.6c140.6,163.6,173.3,242.6,136.7,333.1c-11.6,27-173.3,234.9-360,462.1c-188.7,227.2-348.5,433.2-358.1,458.2c-30.8,82.8-17.3,136.7,86.6,356.2c57.8,117.4,138.6,311.9,177.1,427.4c80.9,236.8,148.3,327.3,259.9,344.6c36.6,5.8,288.8,32.7,562.2,59.7c308,28.9,517.9,59.7,550.6,77c30.8,15.4,71.2,59.7,90.5,100.1c32.8,65.4,36.6,123.2,34.7,573.7c0,562.2-11.5,627.6-115.5,687.3c-46.2,27-188.7,48.1-612.2,90.5c-573.7,59.7-614.2,67.4-673.8,161.7c-15.4,23.1-59.7,132.9-98.2,244.5s-117.4,300.3-175.2,417.8c-57.8,119.4-104,240.7-104,271.5c0,80.9,40.4,138.6,394.7,569.9c181,219.5,335,419.7,344.6,446.7c30.8,94.3-3.9,167.5-159.8,344.6c-157.9,181-410.1,429.3-564.1,554.5c-96.3,78.9-188.7,104-265.7,75.1c-27-11.6-234.9-173.3-462.1-360c-227.2-188.7-433.2-348.5-458.2-358.1c-80.9-30.8-130.9-19.2-371.6,96.3c-130.9,61.6-325.4,142.5-431.3,177.1c-217.5,71.2-308,140.5-325.4,250.3c-5.8,36.6-32.7,288.8-57.8,560.3c-53.9,550.6-67.4,596.8-181,645C5502.3,5018.7,4807.3,5036,4568.5,5011z M5463.8,1897.8c502.5-127.1,954.9-494.8,1184-960.7c446.7-914.5,78.9-2011.9-824-2460.5c-1053.1-521.8-2308.4,52-2604.9,1189.8c-71.2,277.2-71.2,629.6,0,904.9c192.5,737.4,814.4,1284.2,1569.1,1376.6C4974.8,1971,5255.9,1949.8,5463.8,1897.8z"/>
+    </g>
+  </svg>
+</template>
+
+<script>
+export default {
+  name: "GearIcon"
+}
+</script>
+
+<style scoped>
+svg {
+  display: inline-block;
+  width: 20px;
+  height: 20px;
+  margin-top: -4px;
+}
+</style>

sist2-vue/src/components/HelpDialog.vue

@@ -0,0 +1,72 @@
+<template>
+  <b-modal :visible="show" size="lg" :hide-footer="true" static :title="$t('help.help')"
+           @close="$emit('close')"
+           @hide="$emit('close')"
+  >
+    <h2>{{$t("help.simpleSearch")}}</h2>
+
+    <table class="table">
+      <tbody>
+      <tr>
+        <td><code>+</code></td>
+        <td>{{$t("help.and")}}</td>
+      </tr>
+      <tr>
+        <td><code>|</code></td>
+        <td>{{$t("help.or")}}</td>
+      </tr>
+      <tr>
+        <td><code>-</code></td>
+        <td>{{$t("help.not")}}</td>
+      </tr>
+      <tr>
+        <td><code>""</code></td>
+        <td>{{$t("help.quotes")}}</td>
+      </tr>
+      <tr>
+        <td><code>{{$t("help.term")}}*</code></td>
+        <td>{{$t("help.prefix")}}</td>
+      </tr>
+      <tr>
+        <td><code>(</code> {{$t("and")}} <code>)</code></td>
+        <td>{{$t("help.parens")}}</td>
+      </tr>
+      <tr>
+        <td><code>{{$t("help.term")}}~N</code></td>
+        <td>{{$t("help.tildeTerm")}}</td>
+      </tr>
+      <tr>
+        <td><code>"..."~N</code></td>
+        <td>{{$t("help.tildePhrase")}}</td>
+      </tr>
+      </tbody>
+    </table>
+
+    <p v-html="$t('help.example1')"></p>
+
+    <p v-html="$t('help.defaultOperator')"></p>
+
+    <p v-html="$t('help.fuzzy')"></p>
+
+    <br>
+
+    <p v-html="$t('help.moreInfoSimple')"></p>
+
+    <p></p>
+
+    <h2>{{$t("help.advancedSearch")}}</h2>
+    <p v-html="$t('help.moreInfoAdvanced')"></p>
+
+  </b-modal>
+</template>
+
+<script>
+export default {
+  name: "HelpDialog",
+  props: ["show"]
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-vue/src/components/IndexDebugInfo.vue

@@ -0,0 +1,30 @@
+<template>
+  <div>
+    <h4>[{{ index.name }}]</h4>
+    <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
+  </div>
+</template>
+
+<script>
+import {humanDate} from "@/util";
+
+export default {
+  name: "IndexDebugInfo",
+  props: ["index"],
+  computed: {
+    tableItems() {
+      return [
+        {key: this.$t("name"), value: this.index.name},
+        {key: this.$t("id"), value: this.index.id},
+        {key: this.$t("indexVersion"), value: this.index.version},
+        {key: this.$t("rewriteUrl"), value: this.index.rewriteUrl},
+        {key: this.$t("timestamp"), value: humanDate(this.index.timestamp)},
+      ]
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-vue/src/components/IndexPicker.vue

@@ -0,0 +1,95 @@
+<template>
+  <div v-if="isMobile">
+    <b-form-select
+        :value="selectedIndicesIds"
+        @change="onSelect($event)"
+        :options="indices" multiple :select-size="6" text-field="name"
+        value-field="id"></b-form-select>
+  </div>
+  <div v-else>
+    <b-list-group id="index-picker-desktop">
+      <b-list-group-item
+          v-for="idx in indices"
+          @click="toggleIndex(idx)"
+          class="d-flex justify-content-between align-items-center list-group-item-action pointer">
+        <div class="d-flex">
+          <b-checkbox @change="toggleIndex(idx)" :checked="isSelected(idx)"></b-checkbox>
+          {{ idx.name }}
+          <span class="text-muted timestamp-text ml-2">{{ formatIdxDate(idx.timestamp) }}</span>
+        </div>
+        <b-badge class="version-badge">v{{ idx.version }}</b-badge>
+      </b-list-group-item>
+    </b-list-group>
+  </div>
+</template>
+
+<script lang="ts">
+import SmallBadge from "./SmallBadge.vue"
+import {mapActions, mapGetters} from "vuex";
+import Vue from "vue";
+import {format} from "date-fns";
+
+export default Vue.extend({
+  components: {
+    SmallBadge
+  },
+  data() {
+    return {
+      loading: true,
+    }
+  },
+  computed: {
+    ...mapGetters([
+      "indices", "selectedIndices"
+    ]),
+    selectedIndicesIds() {
+      return this.selectedIndices.map(idx => idx.id)
+    },
+    isMobile() {
+      return window.innerWidth <= 650;
+    }
+  },
+  methods: {
+    ...mapActions({
+      setSelectedIndices: "setSelectedIndices"
+    }),
+    onSelect(value) {
+      this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
+    },
+    formatIdxDate(timestamp: number): string {
+      return format(new Date(timestamp * 1000), "yyyy-MM-dd");
+    },
+    toggleIndex(index) {
+      if (this.isSelected(index)) {
+        this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
+      } else {
+        this.setSelectedIndices([index, ...this.selectedIndices]);
+      }
+    },
+    isSelected(index) {
+      return this.selectedIndices.find(idx => idx.id == index.id) != null;
+    }
+  },
+})
+</script>
+
+<style scoped>
+.timestamp-text {
+  line-height: 24px;
+  font-size: 80%;
+}
+
+.version-badge {
+  color: #222 !important;
+  background: none;
+}
+
+.list-group-item {
+  padding: 0.2em 0.4em;
+}
+
+#index-picker-desktop {
+  overflow-y: auto;
+  max-height: 132px;
+}
+</style>

sist2-vue/src/components/InfoTable.vue

@@ -0,0 +1,94 @@
+<template>
+  <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0 mt-4">
+
+    <template #cell(value)="data">
+      <span v-if="'html' in data.item" v-html="data.item.html"></span>
+      <span v-else>{{ data.value }}</span>
+    </template>
+  </b-table>
+</template>
+
+<script>
+import {humanDate, humanFileSize} from "@/util";
+
+function makeGpsLink(latitude, longitude) {
+
+  if (isNaN(latitude) || isNaN(longitude)) {
+    return "";
+  }
+
+  return `<a target="_blank" href="https://maps.google.com/?q=${latitude},${longitude}&ll=${latitude},${longitude}&t=k&z=17">${latitude}, ${longitude}</a>`;
+}
+
+function dmsToDecimal(dms, ref) {
+  const tokens = dms.split(",")
+
+  const d = Number(tokens[0].trim().split(":")[0]) / Number(tokens[0].trim().split(":")[1])
+  const m = Number(tokens[1].trim().split(":")[0]) / Number(tokens[1].trim().split(":")[1])
+  const s = Number(tokens[2].trim().split(":")[0]) / Number(tokens[2].trim().split(":")[1])
+
+  return (d + (m / 60) + (s / 3600)) * (ref === "S" || ref === "W" ? -1 : 1)
+}
+
+export default {
+  name: "InfoTable",
+  props: ["doc"],
+  computed: {
+    tableItems() {
+      const src = this.doc._source;
+
+      const items = [
+        {key: "index", value: `[${this.$store.getters.indexMap[src.index].name}]`},
+        {key: "mtime", value: humanDate(src.mtime)},
+        {key: "mime", value: src.mime},
+        {key: "size", value: humanFileSize(src.size)},
+        {key: "path", value: src.path},
+      ];
+
+      if ("width" in this.doc._source) {
+        items.push({
+          key: "image size",
+          value: `${src.width}x${src.height}`
+        })
+      }
+
+      const fields = [
+        "title", "duration", "audioc", "videoc",
+        "bitrate", "artist", "album", "album_artist", "genre", "font_name", "author",
+        "modified_by", "pages", "tag",
+        "exif_make", "exif_software", "exif_exposure_time", "exif_fnumber", "exif_focal_length",
+        "exif_user_comment", "exif_iso_speed_ratings", "exif_model", "exif_datetime",
+        "checksum"
+      ];
+
+      fields.forEach(field => {
+        if (field in src) {
+          items.push({key: field, value: src[field]});
+        }
+      });
+
+      // Exif GPS
+      if ("exif_gps_longitude_dec" in src) {
+        items.push({
+          key: "Exif GPS",
+          html: makeGpsLink(src["exif_gps_latitude_dec"], src["exif_gps_longitude_dec"]),
+        });
+      } else if ("exif_gps_longitude_dms" in src) {
+        items.push({
+          key: "Exif GPS",
+          html: makeGpsLink(
+              dmsToDecimal(src["exif_gps_latitude_dms"], src["exif_gps_latitude_ref"]),
+              dmsToDecimal(src["exif_gps_longitude_dms"], src["exif_gps_longitude_ref"]),
+          ),
+        });
+      }
+
+      return items;
+    }
+  }
+}
+</script>
+
+<style scoped>
+
+</style>

sist2-vue/src/components/LazyContentDiv.vue

@@ -0,0 +1,30 @@
+<template>
+  <Preloader v-if="loading"></Preloader>
+  <div v-else-if="content" class="content-div">{{ content }}</div>
+</template>
+
+<script>
+import Sist2Api from "@/Sist2Api";
+import Preloader from "@/components/Preloader";
+
+export default {
+  name: "LazyContentDiv",
+  components: {Preloader},
+  props: ["docId"],
+  data() {
+    return {
+      content: "",
+      loading: true
+    }
+  },
+  mounted() {
+    Sist2Api.getDocInfo(this.docId).then(src => {
+      this.content = src.data.content;
+      this.loading = false;
+    })
+  }
+}
+</script>
+
+<style scoped>
+</style>

sist2-vue/src/components/Lightbox.vue

@@ -0,0 +1,129 @@
+<template>
+  <div>
+    <!-- TODO: Set slideshowTime as a configurable option-->
+    <FsLightbox
+        :key="lightboxKey"
+        :toggler="showLightbox"
+        :sources="lightboxSources"
+        :thumbs="lightboxThumbs"
+        :captions="lightboxCaptions"
+        :types="lightboxTypes"
+        :source-index="lightboxSlide"
+        :custom-toolbar-buttons="customButtons"
+        :slideshow-time="1000 * 10"
+        :zoom-increment="0.5"
+        :load-only-current-source="$store.getters.optLightboxLoadOnlyCurrent"
+        :on-close="onClose"
+        :on-open="onShow"
+        :on-slide-change="onSlideChange"
+    ></FsLightbox>
+
+    <a id="lightbox-download" style="display: none"></a>
+  </div>
+</template>
+
+<script>
+import FsLightbox from "fslightbox-vue";
+
+export default {
+  name: "Lightbox",
+  components: {FsLightbox},
+  data() {
+    return {
+      customButtons: [
+        {
+          viewBox: "0 0 384.928 384.928",
+          d: "M321.339,245.334c-4.74-4.692-12.439-4.704-17.179,0l-99.551,98.564V12.03 c0-6.641-5.438-12.03-12.151-12.03s-12.151,5.39-12.151,12.03v331.868l-99.551-98.552c-4.74-4.704-12.439-4.704-17.179,0 s-4.74,12.319,0,17.011l120.291,119.088c4.692,4.644,12.499,4.644,17.191,0l120.291-119.088 C326.091,257.653,326.091,250.038,321.339,245.334C316.599,240.642,326.091,250.038,321.339,245.334z",
+          width: "17px",
+          height: "17px",
+          title: "Download",
+          onClick: this.onDownloadClick
+        }
+      ]
+    }
+  },
+  computed: {
+    showLightbox() {
+      return this.$store.getters["uiShowLightbox"];
+    },
+    lightboxSources() {
+      return this.$store.getters["uiLightboxSources"];
+    },
+    lightboxThumbs() {
+      return this.$store.getters["uiLightboxThumbs"];
+    },
+    lightboxKey() {
+      return this.$store.getters["uiLightboxKey"];
+    },
+    lightboxSlide() {
+      return this.$store.getters["uiLightboxSlide"];
+    },
+    lightboxCaptions() {
+      return this.$store.getters["uiLightboxCaptions"];
+    },
+    lightboxTypes() {
+      return this.$store.getters["uiLightboxTypes"];
+    }
+  },
+  methods: {
+    onDownloadClick() {
+      const url = this.lightboxSources[this.lightboxSlide];
+
+      const a = document.getElementById("lightbox-download");
+      a.setAttribute("href", url);
+      a.setAttribute("download", "");
+      a.click();
+    },
+    onShow() {
+      this.$store.commit("setUiLightboxIsOpen", true);
+    },
+    onClose() {
+      this.$store.commit("setUiLightboxIsOpen", false);
+    },
+    onSlideChange() {
+      // Pause all videos when changing slide
+      document.getElementsByTagName("video").forEach((el) => {
+        el.pause();
+      });
+    },
+  }
+
+}
+</script>
+
+<style>
+.fslightbox-toolbar-button:nth-child(2) {
+  order: 1;
+}
+
+.fslightbox-toolbar-button:nth-child(1) {
+  order: 2;
+}
+
+.fslightbox-toolbar-button:nth-child(3) {
+  order: 3;
+}
+
+.fslightbox-toolbar-button:nth-child(4) {
+  order: 4;
+}
+
+.fslightbox-toolbar-button:nth-child(5) {
+  order: 5;
+}
+
+@media (max-width: 650px) {
+  /* Disable fullscreen on mobile because it's buggy */
+  .fslightbox-toolbar-button:nth-child(6) {
+    display: none;
+  }
+}
+
+.fslightbox-toolbar-button:nth-child(6) {
+  order: 6;
+}
+
+.fslightbox-toolbar-button:nth-child(7) {
+  order: 7;
+}
+</style>

sist2-vue/src/components/LightboxCaption.vue

@@ -0,0 +1,26 @@
+<template>
+  <div class="lightbox-caption">
+    <p>
+      <b>{{
+          `[${$store.getters.indices.find(i => i.id === hit._source.index).name}]`
+        }}</b>{{ `/${hit._source.path}/${hit._source.name}${ext(hit)}` }}
+    </p>
+    <p style="margin-top: -1em">
+      <span v-if="hit._source.width">{{ `${hit._source.width}x${hit._source.height}`}}</span>
+      {{ ` (${humanFileSize(hit._source.size)})` }}
+    </p>
+  </div>
+</template>
+
+<script>
+import {ext, humanFileSize} from "@/util";
+
+export default {
+  name: "LightboxCaption",
+  props: ["hit"],
+  methods: {
+    humanFileSize: humanFileSize,
+    ext: ext
+  }
+}
+</script>

sist2-vue/src/components/MimePicker.vue

@@ -0,0 +1,61 @@
+<template>
+  <div id="mimeTree"></div>
+</template>
+
+<script>
+import InspireTree from "inspire-tree";
+import InspireTreeDOM from "inspire-tree-dom";
+
+import "inspire-tree-dom/dist/inspire-tree-light.min.css";
+import {getSelectedTreeNodes} from "@/util";
+
+export default {
+  name: "MimePicker",
+  data() {
+    return {
+      mimeTree: null,
+    }
+  },
+  mounted() {
+    this.$store.subscribe((mutation) => {
+      if (mutation.type === "setUiMimeMap") {
+        const mimeMap = mutation.payload.slice();
+
+        this.mimeTree = new InspireTree({
+          selection: {
+            mode: 'checkbox'
+          },
+          data: mimeMap
+        });
+        new InspireTreeDOM(this.mimeTree, {
+          target: '#mimeTree'
+        });
+        this.mimeTree.on("node.state.changed", this.handleTreeClick);
+        this.mimeTree.deselect();
+
+        if (this.$store.state._onLoadSelectedMimeTypes.length > 0) {
+          this.$store.state._onLoadSelectedMimeTypes.forEach(mime => {
+            this.mimeTree.node(mime).select();
+          });
+        }
+      }
+    });
+  },
+  methods: {
+    handleTreeClick(node, e) {
+      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused") {
+        return;
+      }
+
+      this.$store.commit("setSelectedMimeTypes", getSelectedTreeNodes(this.mimeTree));
+    },
+  }
+}
+</script>
+
+<style scoped>
+#mimeTree {
+  max-height: 350px;
+  overflow: auto;
+}
+</style>

sist2-vue/src/components/NavBar.vue

@@ -0,0 +1,101 @@
+<template>
+  <b-navbar>
+    <b-navbar-brand v-if="$route.path !== '/'" to="/">
+      <Sist2Icon></Sist2Icon>
+    </b-navbar-brand>
+    <b-navbar-brand v-else href=".">
+      <Sist2Icon></Sist2Icon>
+    </b-navbar-brand>
+
+    <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
+      v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span>
+    </span>
+
+    <span v-if="$store && $store.state.sist2Info" class="tagline" v-html="tagline()"></span>
+
+    <b-button class="ml-auto" to="stats" variant="link">{{ $t("stats") }}</b-button>
+    <b-button to="config" variant="link">{{ $t("config") }}</b-button>
+  </b-navbar>
+</template>
+
+<script>
+import Sist2Icon from "@/components/Sist2Icon";
+export default {
+  name: "NavBar",
+  components: {Sist2Icon},
+  methods: {
+    tagline() {
+      return this.$store.state.sist2Info.tagline;
+    },
+    sist2Version() {
+      return this.$store.state.sist2Info.version;
+    },
+    isDebug() {
+      return this.$store.state.sist2Info.debug;
+    }
+  }
+}
+</script>
+
+<style scoped>
+.navbar {
+  box-shadow: 0 0.125rem 0.25rem rgb(0 0 0 / 8%) !important;
+  border-radius: 0;
+}
+
+.theme-black .navbar {
+  background: #546b7a30;
+  border-bottom: none;
+}
+
+.navbar-brand {
+  color: #222 !important;
+  font-size: 1.75rem;
+  padding: 0;
+  font-family: Hack;
+}
+
+.navbar-brand:hover {
+  color: #000 !important;
+}
+
+.version {
+  color: #222 !important;
+  margin-left: -18px;
+  margin-top: -14px;
+  font-size: 11px;
+  font-family: monospace;
+}
+
+.theme-black .version {
+  color: #f5f5f5 !important;
+}
+
+.theme-black .navbar-brand {
+  font-size: 1.75rem;
+  padding: 0;
+  color: #f5f5f5 !important;
+}
+
+.theme-black a:hover, .theme-black .btn:hover {
+  color: #fff;
+}
+
+.theme-black .navbar span {
+  color: #eee;
+}
+
+@media (max-width: 650px) {
+  .tagline {
+    display: none;
+  }
+
+  .version {
+    display: none;
+  }
+}
+
+.theme-light .btn-link{
+  color: #222;
+}
+</style>

sist2-vue/src/components/PathTree.vue

@@ -0,0 +1,245 @@
+<template>
+  <div>
+    <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
+      <div class="input-group-prepend">
+
+        <b-button variant="outline-secondary" @click="$refs['path-modal'].show()">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
+            <path
+                fill="currentColor"
+                d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"
+            />
+          </svg>
+        </b-button>
+      </div>
+
+      <VueSimpleSuggest
+          class="form-control-fix-flex"
+          @input="setPathText"
+          :value="getPathText"
+          :list="suggestPath"
+          :max-suggestions="0"
+          :placeholder="$t('pathBar.placeholder')"
+      >
+        <!-- Suggestion item template-->
+        <div slot="suggestion-item" slot-scope="{ suggestion, query }">
+          <div class="suggestion-line" :title="suggestion">
+            <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
+          </div>
+        </div>
+      </VueSimpleSuggest>
+
+    </div>
+
+    <b-modal ref="path-modal" :title="$t('pathBar.modalTitle')" size="lg" :hide-footer="true" static>
+      <div id="pathTree"></div>
+    </b-modal>
+  </div>
+</template>
+
+<script>
+import InspireTree from "inspire-tree";
+import InspireTreeDOM from "inspire-tree-dom";
+
+import "inspire-tree-dom/dist/inspire-tree-light.min.css";
+import Sist2Api from "@/Sist2Api";
+import {mapGetters, mapMutations} from "vuex";
+import VueSimpleSuggest from 'vue-simple-suggest'
+import 'vue-simple-suggest/dist/styles.css' // Optional CSS
+
+export default {
+  name: "PathTree",
+  components: {
+    VueSimpleSuggest
+  },
+  data() {
+    return {
+      mimeTree: null,
+      pathItems: [],
+      tmpPath: ""
+
+    }
+  },
+  computed: {
+    ...mapGetters(["getPathText"])
+  },
+  mounted() {
+    this.$store.subscribe((mutation) => {
+      // Wait until indices are loaded to get the root paths
+      if (mutation.type === "setIndices") {
+        let pathTree = new InspireTree({
+          data: (node, resolve, reject) => {
+            return this.getNextDepth(node);
+          },
+          sort: "text"
+        });
+
+        this.$store.state.indices.forEach(idx => {
+          pathTree.addNode({
+            id: "/" + idx.id,
+            values: ["/" + idx.id],
+            text: `/[${idx.name}]`,
+            index: idx.id,
+            depth: 0,
+            children: true
+          })
+        });
+
+        new InspireTreeDOM(pathTree, {
+          target: "#pathTree"
+        });
+
+        pathTree.on("node.click", this.handleTreeClick);
+        pathTree.expand();
+      }
+    });
+  },
+  methods: {
+    ...mapMutations(["setPathText"]),
+    getSuggestionWithoutQueryPrefix(suggestion, query) {
+      return suggestion.slice(query.length)
+    },
+    async getPathChoices() {
+      return new Promise(getPaths => {
+        const q = {
+          suggest: {
+            path: {
+              prefix: this.getPathText,
+              completion: {
+                field: "suggest-path",
+                skip_duplicates: true,
+                size: 10000
+              }
+            }
+          }
+        };
+
+        Sist2Api.esQuery(q)
+            .then(resp => getPaths(resp["suggest"]["path"][0]["options"].map(opt => opt["_source"]["path"])));
+      })
+    },
+    async suggestPath(term) {
+      if (!this.$store.state.optSuggestPath) {
+        return []
+      }
+
+      term = term.toLowerCase();
+
+      const choices = await this.getPathChoices();
+
+      let matches = [];
+      for (let i = 0; i < choices.length; i++) {
+        if (~choices[i].toLowerCase().indexOf(term)) {
+          matches.push(choices[i]);
+        }
+      }
+      return matches.sort((a, b) => a.length - b.length);
+    },
+    getNextDepth(node) {
+      const q = {
+        query: {
+          bool: {
+            filter: [
+              {term: {index: node.index}},
+              {range: {_depth: {gte: node.depth + 1, lte: node.depth + 3}}},
+            ]
+          }
+        },
+        aggs: {
+          paths: {
+            terms: {
+              field: "path",
+              size: 10000
+            }
+          }
+        },
+        size: 0
+      };
+
+      if (node.depth > 0) {
+        q.query.bool.must = {
+          prefix: {
+            path: node.id,
+          }
+        };
+      }
+
+      return Sist2Api.esQuery(q).then(resp => {
+        const buckets = resp["aggregations"]["paths"]["buckets"];
+        if (!buckets) {
+          return false;
+        }
+
+        const paths = [];
+
+        return buckets
+            .filter(bucket => bucket.key.length > node.id.length || node.id.startsWith("/"))
+            .sort((a, b) => a.key > b.key)
+            .map(bucket => {
+
+              if (paths.some(n => bucket.key.startsWith(n))) {
+                return null;
+              }
+
+              const name = node.id.startsWith("/") ? bucket.key : bucket.key.slice(node.id.length + 1);
+
+              paths.push(bucket.key);
+
+              return {
+                id: bucket.key,
+                text: `${name}/ (${bucket.doc_count})`,
+                depth: node.depth + 1,
+                index: node.index,
+                values: [bucket.key],
+                children: true,
+              }
+            }).filter(x => x !== null)
+      });
+    },
+    handleTreeClick(e, node, handler) {
+      if (node.depth !== 0) {
+        this.setPathText(node.id);
+        this.$refs['path-modal'].hide()
+
+        this.$emit("search");
+      }
+
+      handler();
+    },
+  },
+}
+</script>
+
+<style scoped>
+#mimeTree {
+  max-height: 350px;
+  overflow: auto;
+}
+
+.form-control-fix-flex {
+  flex: 1 1 auto;
+  width: 1%;
+  min-width: 0;
+  margin-bottom: 0;
+}
+
+.suggestion-line {
+  max-width: 100%;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  line-height: 1.1;
+}
+
+</style>
+
+<style>
+.suggestions {
+  max-height: 250px;
+  overflow-y: auto;
+}
+
+.theme-black .suggestions {
+  color: black
+}
+</style>

sist2-vue/src/components/Preloader.vue

@@ -0,0 +1,3 @@
+<template>
+  <b-progress value="1" max="1" animated></b-progress>
+</template>

sist2-vue/src/components/ResultsCard.vue

@@ -0,0 +1,76 @@
+<template>
+  <b-card v-if="lastResultsLoaded" id="results">
+    <span>{{ hitCount }} {{ hitCount === 1 ? $t("hit") : $t("hits") }}</span>
+
+    <div style="float: right">
+      <b-button v-b-toggle.collapse-1 variant="primary" class="not-mobile">{{ $t("details") }}</b-button>
+
+      <SortSelect class="ml-2"></SortSelect>
+
+      <DisplayModeToggle class="ml-2"></DisplayModeToggle>
+    </div>
+
+    <b-collapse id="collapse-1" class="pt-2" style="clear:both;">
+      <b-card>
+        <b-table :items="tableItems" small borderless thead-class="hidden" class="mb-0"></b-table>
+      </b-card>
+    </b-collapse>
+  </b-card>
+</template>
+
+<script lang="ts">
+import {EsResult} from "@/Sist2Api";
+import Vue from "vue";
+import {humanFileSize, humanTime} from "@/util";
+import DisplayModeToggle from "@/components/DisplayModeToggle.vue";
+import SortSelect from "@/components/SortSelect.vue";
+
+export default Vue.extend({
+  name: "ResultsCard",
+  components: {SortSelect, DisplayModeToggle},
+  computed: {
+    lastResultsLoaded() {
+      return this.$store.state.lastQueryResults != null;
+    },
+    hitCount() {
+      return (this.$store.state.lastQueryResults as EsResult).aggregations.total_count.value;
+    },
+    tableItems() {
+      const items = [];
+
+
+      items.push({key: this.$t("queryTime"), value: this.took()});
+      items.push({key: this.$t("totalSize"), value: this.totalSize()});
+
+      return items;
+    }
+  },
+  methods: {
+    took() {
+      return (this.$store.state.lastQueryResults as EsResult).took + "ms";
+    },
+    totalSize() {
+      return humanFileSize((this.$store.state.lastQueryResults as EsResult).aggregations.total_size.value);
+    },
+  },
+});
+
+</script>
+
+<style>
+#results {
+  margin-top: 1em;
+
+  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+  border-radius: 0;
+  border: none;
+}
+
+#results .card-body {
+  padding: 0.7em 1.25em;
+}
+
+.hidden {
+  display: none;
+}
+</style>

sist2-vue/src/components/SearchBar.vue

@@ -0,0 +1,46 @@
+<template>
+  <div>
+    <b-input-group>
+      <b-form-input :value="searchText"
+                    :placeholder="advanced() ? $t('searchBar.advanced') : $t('searchBar.simple')"
+                    @input="setSearchText($event)"></b-form-input>
+
+      <template #prepend>
+        <b-input-group-text>
+          <b-form-checkbox :checked="fuzzy" title="Toggle fuzzy searching" @change="setFuzzy($event)">
+            {{ $t("searchBar.fuzzy") }}
+          </b-form-checkbox>
+        </b-input-group-text>
+      </template>
+
+      <template #append>
+        <b-button variant="outline-secondary" @click="$emit('show-help')">{{$t("help.help")}}</b-button>
+      </template>
+    </b-input-group>
+  </div>
+</template>
+
+<script>
+import {mapGetters, mapMutations} from "vuex";
+
+export default {
+  computed: {
+    ...mapGetters({
+      optQueryMode: "optQueryMode",
+      searchText: "searchText",
+      fuzzy: "fuzzy",
+    }),
+  },
+  methods: {
+    ...mapMutations({
+      setSearchText: "setSearchText",
+      setFuzzy: "setFuzzy"
+    }),
+    advanced() {
+      return this.optQueryMode === "advanced"
+    }
+  }
+}
+</script>
+<style>
+</style>

sist2-vue/src/components/Sist2Icon.vue

@@ -0,0 +1,40 @@
+<template>
+  <svg
+      xmlns="http://www.w3.org/2000/svg"
+      width="27.868069mm"
+      height="7.6446671mm"
+      viewBox="0 0 27.868069 7.6446671"
+  >
+    <g transform="translate(-4.5018313,-4.1849793)">
+      <g
+          style="fill: currentColor;fill-opacity:1;stroke:none;stroke-width:0.26458332">
+        <path
+            d="m 6.3153296,11.829646 q -0.7717014,0 -1.8134983,-0.337619 v -0.916395 q 1.0128581,0.511252 1.803852,0.511252 0.5643067,0 0.901926,-0.236334 0.3376194,-0.236333 0.3376194,-0.63183 0,-0.3424428 -0.2845649,-0.5498376 Q 6.980922,9.4566645 6.3635609,9.3264399 L 5.9921796,9.2492698 Q 5.2301245,9.0949295 4.8732126,8.7428407 4.5211238,8.3859288 4.5211238,7.7733908 q 0,-0.7765245 0.5305447,-1.1961372 0.5305447,-0.4196126 1.5096409,-0.4196126 0.829579,0 1.6061036,0.3183268 V 7.3441319 Q 7.4101809,6.9004036 6.5854251,6.9004036 q -1.1671984,0 -1.1671984,0.7958171 0,0.2604492 0.1012858,0.4147895 0.1012858,0.1495171 0.3858507,0.2556261 0.2845649,0.1012858 0.8392253,0.2122179 l 0.3569119,0.067524 q 1.3408312,0.2652724 1.3408312,1.4614098 0,0.80064 -0.5691298,1.263661 -0.5691298,0.458197 -1.5578722,0.458197 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 11.943927,5.3087694 q -0.144694,0 -0.144694,-0.144694 V 4.3296733 q 0,-0.144694 0.144694,-0.144694 h 0.694531 q 0.144694,0 0.144694,0.144694 v 0.8344021 q 0,0.144694 -0.144694,0.144694 z M 13.5645,11.728361 q -0.795817,0 -1.234722,-0.511253 -0.434082,-0.516075 -0.434082,-1.4469398 V 6.9823969 H 10.714028 V 6.2878656 h 2.069124 v 3.4823026 q 0,0.5884228 0.221864,0.8971028 0.221865,0.308681 0.6463,0.308681 h 1.036974 v 0.752409 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 18.209178,11.829646 q -0.771701,0 -1.813498,-0.337619 v -0.916395 q 1.012858,0.511252 1.803852,0.511252 0.564306,0 0.901926,-0.236334 0.337619,-0.236333 0.337619,-0.63183 0,-0.3424428 -0.284565,-0.5498376 Q 18.87477,9.4566645 18.257409,9.3264399 l -0.371381,-0.07717 Q 17.123973,9.0949295 16.767061,8.7428407 16.414972,8.3859288 16.414972,7.7733908 q 0,-0.7765245 0.530545,-1.1961372 0.530545,-0.4196126 1.509641,-0.4196126 0.829579,0 1.606103,0.3183268 v 0.8681641 q -0.757232,-0.4437283 -1.581988,-0.4437283 -1.167198,0 -1.167198,0.7958171 0,0.2604492 0.101286,0.4147895 0.101286,0.1495171 0.385851,0.2556261 0.284565,0.1012858 0.839225,0.2122179 l 0.356912,0.067524 q 1.340831,0.2652724 1.340831,1.4614098 0,0.80064 -0.56913,1.263661 -0.56913,0.458197 -1.557872,0.458197 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 25.207545,11.709068 q -0.993565,0 -1.408355,-0.40032 -0.409966,-0.405143 -0.409966,-1.3794164 V 6.9775737 H 21.947107 V 6.2878656 h 1.442117 V 4.8746874 l 0.887457,-0.3858507 v 1.7990289 h 2.016069 v 0.6897081 h -2.016069 v 2.9517579 q 0,0.5932454 0.226687,0.8344024 0.226687,0.236333 0.790994,0.236333 h 0.998388 v 0.709001 z"
+            style="stroke-width:0.26458332"
+        />
+        <path
+            d="m 27.995317,11.043476 q 0,-0.178456 0.120578,-0.299035 0.274919,-0.289388 0.651123,-0.684885 0.376205,-0.4003199 0.805464,-0.8681638 0.327973,-0.356912 0.491959,-0.5353679 0.16881,-0.1832791 0.255626,-0.2845649 0.09164,-0.1012858 0.178456,-0.2073948 0.255626,-0.3086805 0.405144,-0.5257215 0.15434,-0.2170411 0.250803,-0.4292589 0.168809,-0.3762045 0.168809,-0.7524089 0,-0.5980686 -0.352089,-0.935688 -0.356911,-0.3424425 -0.979096,-0.3424425 -0.863341,0 -1.938899,0.6414768 V 4.8361023 q 0.491959,-0.2363335 0.979096,-0.3569119 0.47749,-0.1205783 0.945334,-0.1205783 0.501606,0 0.940511,0.1350477 0.438905,0.1350478 0.766878,0.4244358 0.289388,0.2556261 0.463021,0.6270074 0.173633,0.3665582 0.173633,0.829579 0,0.4726671 -0.212218,0.9501574 -0.106109,0.2411567 -0.274919,0.4726671 -0.163986,0.2266873 -0.424435,0.540191 Q 31.270225,8.501684 31.077299,8.718725 30.884374,8.9357661 30.628748,9.2106847 30.445469,9.4084332 30.286305,9.5675966 30.131965,9.72676 29.958332,9.9003928 29.7847,10.069203 29.558012,10.300713 29.336148,10.5274 29.012998,10.869843 h 3.356901 v 0.819932 h -4.374582 z"
+            style="stroke-width:0.26458332"
+        />
+      </g>
+    </g>
+  </svg>
+</template>
+
+<script>
+export default {
+  name: "Sist2Icon"
+}
+</script>

sist2-vue/src/components/SizeSlider.vue

@@ -0,0 +1,135 @@
+<template>
+  <div id="sizeSlider"></div>
+</template>
+
+<script>
+import noUiSlider from 'nouislider';
+import 'nouislider/dist/nouislider.css';
+import {humanFileSize} from "@/util";
+import {mergeTooltips} from "@/util-js";
+
+export default {
+  name: "SizeSlider",
+  mounted() {
+    const elem = document.getElementById("sizeSlider");
+
+    const slider = noUiSlider.create(elem, {
+      start: [
+        this.$store.state.sizeMin ? this.$store.state.sizeMin : 0,
+        this.$store.state.sizeMax ? this.$store.state.sizeMax: 1000 * 1000 * 50000
+      ],
+
+      tooltips: [true, true],
+      behaviour: "drag-tap",
+      connect: true,
+      range: {
+        'min': 0,
+        "10%": 1000 * 1000,
+        "20%": 1000 * 1000 * 10,
+        "50%": 1000 * 1000 * 5000,
+        "max": 1000 * 1000 * 50000,
+      },
+      format: {
+        to: x => x >= 1000 * 1000 * 50000 ? "50G+" : humanFileSize(Math.round(x)),
+        from: x => x
+      }
+    });
+
+    mergeTooltips(elem, 10, " - ")
+
+    elem.querySelectorAll('.noUi-connect')[0].classList.add("slider-color0")
+
+    slider.on("set", (values, handle, unencoded) => {
+
+      if (handle === 0) {
+        this.$store.commit("setSizeMin", unencoded[0] === 0 ? undefined : Math.round(unencoded[0]))
+      } else {
+        this.$store.commit("setSizeMax", unencoded[1] >= 1000 * 1000 * 50000 ? undefined : Math.round(unencoded[1]))
+      }
+    });
+  }
+}
+</script>
+
+<style>
+#sizeSlider {
+  margin-top: 34px;
+  height: 12px;
+}
+
+.slider-color0 {
+  background: #2196f3;
+  box-shadow: none;
+}
+
+.theme-black .slider-color0 {
+  background: #00bcd4;
+}
+
+.noUi-horizontal .noUi-handle {
+  width: 2px;
+  height: 18px;
+  right: -1px;
+  top: -3px;
+  border: none;
+  background-color: #1976d2;
+  box-shadow: none;
+  cursor: ew-resize;
+  border-radius: 0;
+}
+
+.theme-black .noUi-horizontal .noUi-handle {
+  background-color: #2168ac;
+}
+
+.noUi-handle:before {
+  top: -6px;
+  left: 3px;
+
+  width: 10px;
+  height: 28px;
+  background-color: transparent;
+}
+
+.noUi-handle:after {
+  top: -6px;
+  left: -10px;
+
+  width: 10px;
+  height: 28px;
+  background-color: transparent;
+}
+
+.noUi-draggable {
+  cursor: move;
+}
+
+.noUi-tooltip {
+  color: #fff;
+  font-size: 13px;
+  line-height: 1.333;
+  text-shadow: none;
+  padding: 0 2px;
+  background: #2196F3;
+  border-radius: 4px;
+  border: none;
+}
+
+.theme-black .noUi-tooltip {
+  background: #00bcd4;
+}
+
+.theme-black .noUi-connects {
+  background-color: #37474f;
+}
+
+.noUi-horizontal .noUi-origin > .noUi-tooltip {
+  bottom: 7px;
+}
+
+.noUi-target {
+  background-color: #e1e4e9;
+  border: none;
+  box-shadow: none;
+}
+</style>

sist2-vue/src/components/SmallBadge.vue

@@ -0,0 +1,21 @@
+<template>
+  <b-badge variant="secondary" :pill="pill">{{ text }}</b-badge>
+</template>
+
+<script lang="ts">
+import Vue from "vue";
+
+export default Vue.extend({
+  props: {
+    text: String,
+    pill: Boolean
+  }
+})
+</script>
+
+<style scoped>
+.badge-pill {
+  padding: 0.3em .4em 0.1em;
+  border-radius: 6rem;
+}
+</style>

sist2-vue/src/components/SortSelect.vue

@@ -0,0 +1,59 @@
+<template>
+  <b-dropdown variant="primary">
+    <b-dropdown-item :class="{'dropdown-active': sort === 'score'}" @click="onSelect('score')">{{
+        $t("sort.relevance")
+      }}
+    </b-dropdown-item>
+    <b-dropdown-item :class="{'dropdown-active': sort === 'dateAsc'}" @click="onSelect('dateAsc')">{{
+        $t("sort.dateAsc")
+      }}
+    </b-dropdown-item>
+    <b-dropdown-item :class="{'dropdown-active': sort === 'dateDesc'}" @click="onSelect('dateDesc')">
+      {{ $t("sort.dateDesc") }}
+    </b-dropdown-item>
+    <b-dropdown-item :class="{'dropdown-active': sort === 'sizeAsc'}" @click="onSelect('sizeAsc')">{{
+        $t("sort.sizeAsc")
+      }}
+    </b-dropdown-item>
+    <b-dropdown-item :class="{'dropdown-active': sort === 'sizeDesc'}" @click="onSelect('sizeDesc')">
+      {{ $t("sort.sizeDesc") }}
+    </b-dropdown-item>
+
+    <b-dropdown-item :class="{'dropdown-active': sort === 'random'}" @click="onSelect('random')">
+      {{ $t("sort.random") }}
+    </b-dropdown-item>
+
+    <template #button-content>
+      <svg aria-hidden="true" width="20px" height="20px" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 320 512">
+        <path
+            fill="currentColor"
+            d="M41 288h238c21.4 0 32.1 25.9 17 41L177 448c-9.4 9.4-24.6 9.4-33.9 0L24 329c-15.1-15.1-4.4-41 17-41zm255-105L177 64c-9.4-9.4-24.6-9.4-33.9 0L24 183c-15.1 15.1-4.4 41 17 41h238c21.4 0 32.1-25.9 17-41z"></path>
+      </svg>
+    </template>
+  </b-dropdown>
+</template>
+
+<script>
+export default {
+  name: "SortSelect",
+  computed: {
+    sort() {
+      return this.$store.state.sortMode;
+    }
+  },
+  methods: {
+    onSelect(sortMode) {
+      if (sortMode === "random") {
+        this.$store.commit("setSeed", Math.round(Math.random() * 100000));
+      }
+      this.$store.commit("setSortMode", sortMode);
+    }
+  },
+}
+</script>
+
+<style>
+.dropdown-active a {
+  font-weight: bold;
+}
+</style>

sist2-vue/src/components/TagContainer.vue

@@ -0,0 +1,337 @@
+<template>
+  <div @mouseenter="showAddButton = true" @mouseleave="showAddButton = false">
+
+    <b-modal v-model="showModal" :title="$t('saveTagModalTitle')" hide-footer no-fade centered size="lg" static lazy>
+      <b-row>
+        <b-col style="flex-grow: 2" sm>
+          <VueSimpleSuggest
+              ref="suggest"
+              :value="tagText"
+              @select="setTagText($event)"
+              @input="setTagText($event)"
+              class="form-control-fix-flex"
+              style="margin-top: 17px"
+              :list="suggestTag"
+              :max-suggestions="0"
+              :placeholder="$t('saveTagPlaceholder')"
+          >
+            <!-- Suggestion item template-->
+            <div slot="suggestion-item" slot-scope="{ suggestion, query}"
+            >
+              <div class="suggestion-line">
+                <span
+                    class="badge badge-suggestion"
+                    :style="{background: getBg(suggestion), color: getFg(suggestion)}"
+                >
+                <strong>{{ query }}</strong>{{ getSuggestionWithoutQueryPrefix(suggestion, query) }}
+                </span>
+              </div>
+            </div>
+          </VueSimpleSuggest>
+        </b-col>
+        <b-col class="mt-4">
+          <TwitterColorPicker v-model="color" triangle="hide" :width="252" class="mr-auto ml-auto"></TwitterColorPicker>
+        </b-col>
+      </b-row>
+
+      <b-button variant="primary" style="float: right" class="mt-2" @click="saveTag()">{{ $t("confirm") }}
+      </b-button>
+    </b-modal>
+
+
+    <template v-for="tag in hit._tags">
+      <div v-if="tag.userTag" :key="tag.rawText" style="display: inline-block">
+        <span
+            :id="hit._id+tag.rawText"
+            :title="tag.text"
+            tabindex="-1"
+            class="badge pointer"
+            :style="badgeStyle(tag)" :class="badgeClass(tag)"
+            @click.right="onTagRightClick(tag, $event)"
+        >{{ tag.text.split(".").pop() }}</span>
+
+        <b-popover :target="hit._id+tag.rawText" triggers="focus blur" placement="top">
+          <b-button variant="danger" @click="onTagDeleteClick(tag, $event)">Delete</b-button>
+        </b-popover>
+      </div>
+
+      <span
+          v-else :key="tag.text"
+          class="badge"
+          :style="badgeStyle(tag)" :class="badgeClass(tag)"
+      >{{ tag.text.split(".").pop() }}</span>
+    </template>
+
+    <!-- Add button -->
+    <small v-if="showAddButton" class="badge add-tag-button" @click="tagAdd()">Add</small>
+
+    <!-- Size tag-->
+    <small v-else class="text-muted badge-size">{{
+        humanFileSize(hit._source.size)
+      }}</small>
+  </div>
+</template>
+
+<script>
+import {humanFileSize, lum} from "@/util";
+import Vue from "vue";
+import {Twitter} from 'vue-color'
+import Sist2Api from "@/Sist2Api";
+import VueSimpleSuggest from 'vue-simple-suggest'
+
+export default Vue.extend({
+  components: {
+    "TwitterColorPicker": Twitter,
+    VueSimpleSuggest
+  },
+  props: ["hit"],
+  data() {
+    return {
+      showAddButton: false,
+      showModal: false,
+      tagText: null,
+      color: {
+        hex: "#e0e0e0",
+      },
+    }
+  },
+  computed: {
+    tagHover() {
+      return this.$store.getters["uiTagHover"];
+    }
+  },
+  methods: {
+    humanFileSize: humanFileSize,
+    getSuggestionWithoutQueryPrefix(suggestion, query) {
+      return suggestion.id.slice(query.length, -8)
+    },
+    getBg(suggestion) {
+      return suggestion.id.slice(-7);
+    },
+    getFg(suggestion) {
+      return lum(suggestion.id.slice(-7)) > 50 ? "#000" : "#fff";
+    },
+    setTagText(value) {
+      this.$refs.suggest.clearSuggestions();
+
+      if (typeof value === "string") {
+        this.tagText = {
+          id: value,
+          title: value
+        };
+        return;
+      }
+
+      this.color = {
+        hex: "#" + value.id.split("#")[1]
+      }
+
+      this.tagText = value;
+    },
+    badgeClass(tag) {
+      return `badge-${tag.style}`;
+    },
+    badgeStyle(tag) {
+      return {
+        background: tag.bg,
+        color: tag.fg,
+      };
+    },
+    onTagHover(tag) {
+      if (tag.userTag) {
+        this.$store.commit("setUiTagHover", tag);
+      }
+    },
+    onTagLeave() {
+      this.$store.commit("setUiTagHover", null);
+    },
+    onTagDeleteClick(tag, e) {
+      this.hit._tags = this.hit._tags.filter(t => t !== tag);
+
+      Sist2Api.deleteTag(tag.rawText, this.hit).then(() => {
+        //toast
+        this.$store.commit("busUpdateWallItems");
+        this.$store.commit("busUpdateTags");
+      });
+    },
+    tagAdd() {
+      this.showModal = true;
+    },
+    saveTag() {
+      if (this.tagText.id.includes("#")) {
+        this.$bvToast.toast(
+            this.$t("toast.invalidTag"),
+            {
+              title: this.$t("toast.invalidTagTitle"),
+              noAutoHide: true,
+              toaster: "b-toaster-bottom-right",
+              headerClass: "toast-header-error",
+              bodyClass: "toast-body-error",
+            });
+        return;
+      }
+
+      let tag = this.tagText.id + this.color.hex.replace("#", ".#");
+      const userTags = this.hit._tags.filter(t => t.userTag);
+
+      if (userTags.find(t => t.rawText === tag) != null) {
+        this.$bvToast.toast(
+            this.$t("toast.dupeTag"),
+            {
+              title: this.$t("toast.dupeTagTitle"),
+              noAutoHide: true,
+              toaster: "b-toaster-bottom-right",
+              headerClass: "toast-header-error",
+              bodyClass: "toast-body-error",
+            });
+        return;
+      }
+
+      this.hit._tags.push(Sist2Api.createUserTag(tag));
+
+      Sist2Api.saveTag(tag, this.hit).then(() => {
+        this.tagText = null;
+        this.showModal = false;
+        this.$store.commit("busUpdateWallItems");
+        this.$store.commit("busUpdateTags");
+        // TODO: toast
+      });
+    },
+    async suggestTag(term) {
+      term = term.toLowerCase();
+
+      const choices = await this.getTagChoices(term);
+
+      let matches = [];
+      for (let i = 0; i < choices.length; i++) {
+        if (~choices[i].toLowerCase().indexOf(term)) {
+          matches.push(choices[i]);
+        }
+      }
+
+      return matches.sort().map(match => {
+        return {
+          title: match.split(".").slice(0,-1).join("."),
+          id: match
+        }
+      });
+    },
+    getTagChoices(prefix) {
+      return new Promise(getPaths => {
+        Sist2Api.esQuery({
+          suggest: {
+            tag: {
+              prefix: prefix,
+              completion: {
+                field: "suggest-tag",
+                skip_duplicates: true,
+                size: 10000
+              }
+            }
+          }
+        }).then(resp => {
+          const result = [];
+          resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
+            tags.forEach(tag => {
+              const t = tag.slice(0, -8);
+              if (!result.find(x => x.slice(0, -8) === t)) {
+                result.push(tag);
+              }
+            });
+          });
+          getPaths(result);
+        });
+      });
+    }
+  },
+});
+</script>
+
+<style scoped>
+
+
+.badge-video {
+  color: #FFFFFF;
+  background-color: #F27761;
+}
+
+.badge-image {
+  color: #FFFFFF;
+  background-color: #AA99C9;
+}
+
+.badge-audio {
+  color: #FFFFFF;
+  background-color: #00ADEF;
+}
+
+.badge-user {
+  color: #212529;
+  background-color: #e0e0e0;
+}
+
+.badge-user:hover, .add-tag-button:hover {
+  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);
+}
+
+.badge-text {
+  color: #FFFFFF;
+  background-color: #FAAB3C;
+}
+
+.badge {
+  margin-right: 3px;
+}
+
+.badge-delete {
+  margin-right: -2px;
+  margin-left: 2px;
+  margin-top: -1px;
+  font-family: monospace;
+  font-size: 90%;
+  background: rgba(0, 0, 0, 0.2);
+  padding: 0.1em 0.4em;
+  color: white;
+  cursor: pointer;
+}
+
+.badge-size {
+  width: 50px;
+  display: inline-block;
+}
+
+.add-tag-button {
+  cursor: pointer;
+  color: #212529;
+  background-color: #e0e0e0;
+  width: 50px;
+}
+
+.badge {
+  user-select: none;
+}
+
+.badge-suggestion {
+  font-size: 90%;
+  font-weight: normal;
+}
+</style>
+
+<style>
+.vc-twitter-body {
+  padding: 0 !important;
+}
+
+.vc-twitter {
+  box-shadow: none !important;
+  background: none !important;
+}
+
+.tooltip {
+  user-select: none;
+}
+
+.toast {
+  border: none;
+}
+</style>

sist2-vue/src/components/TagPicker.vue

@@ -0,0 +1,185 @@
+<template>
+  <div id="tagTree"></div>
+</template>
+
+<script>
+import InspireTree from "inspire-tree";
+import InspireTreeDOM from "inspire-tree-dom";
+
+import "inspire-tree-dom/dist/inspire-tree-light.min.css";
+import {getSelectedTreeNodes} from "@/util";
+import Sist2Api from "@/Sist2Api";
+
+function resetState(node) {
+  node._tree.defaultState.forEach(function (val, prop) {
+    node.state(prop, val);
+  });
+
+  return node;
+}
+
+function baseStateChange(prop, value, verb, node, deep) {
+  if (node.state(prop) !== value) {
+    node._tree.batch();
+
+    if (node._tree.config.nodes.resetStateOnRestore && verb === 'restored') {
+      resetState(node);
+    }
+
+    node.state(prop, value);
+
+    node._tree.emit('node.' + verb, node, false);
+
+    if (deep && node.hasChildren()) {
+      node.children.recurseDown(function (child) {
+        baseStateChange(prop, value, verb, child);
+      });
+    }
+
+    node.markDirty();
+    node._tree.end();
+  }
+
+  return node;
+}
+
+function addTag(map, tag, id, count) {
+  const tags = tag.split(".");
+
+  const child = {
+    id: id,
+    count: count,
+    text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
+    name: tags[0],
+    children: [],
+    // Overwrite base functions
+    blur: function () {
+      // noop
+    },
+    select: function () {
+      this.state("selected", true);
+      return this.check()
+    },
+    deselect: function () {
+      this.state("selected", false);
+      return this.uncheck()
+    },
+    uncheck: function () {
+      baseStateChange('checked', false, 'unchecked', this, false);
+      this.state('indeterminate', false);
+
+      if (this.hasParent()) {
+        this.getParent().refreshIndeterminateState();
+      }
+
+      this._tree.end();
+      return this;
+    },
+    check: function () {
+      baseStateChange('checked', true, 'checked', this, false);
+
+      if (this.hasParent()) {
+        this.getParent().refreshIndeterminateState();
+      }
+
+      this._tree.end();
+      return this;
+    }
+  };
+
+  let found = false;
+  map.forEach(node => {
+    if (node.name === child.name) {
+      found = true;
+      if (tags.length !== 1) {
+        addTag(node.children, tags.slice(1).join("."), id, count);
+      } else {
+        // Same name, different color
+        console.error("FIXME: Duplicate tag?")
+        console.trace(node)
+      }
+    }
+  });
+  if (!found) {
+    if (tags.length !== 1) {
+      addTag(child.children, tags.slice(1).join("."), id, count);
+      map.push(child);
+    } else {
+      map.push(child);
+    }
+  }
+}
+
+export default {
+  name: "TagPicker",
+  data() {
+    return {
+      tagTree: null,
+      loadedFromArgs: false,
+    }
+  },
+  mounted() {
+    this.$store.subscribe((mutation) => {
+      if (mutation.type === "setUiMimeMap") {
+        this.initializeTree();
+        this.updateTree();
+      } else if (mutation.type === "busUpdateTags") {
+        window.setTimeout(this.updateTree, 2000);
+      }
+    });
+  },
+  methods: {
+    initializeTree() {
+      const tagMap = [];
+      this.tagTree = new InspireTree({
+        selection: {
+          mode: "checkbox",
+          autoDeselect: false,
+        },
+        checkbox: {
+          autoCheckChildren: false,
+        },
+        data: tagMap
+      });
+      new InspireTreeDOM(this.tagTree, {
+        target: '#tagTree'
+      });
+      this.tagTree.on("node.state.changed", this.handleTreeClick);
+    },
+    updateTree() {
+      const tagMap = [];
+      Sist2Api.getTags().then(tags => {
+        tags.forEach(tag => addTag(tagMap, tag.id, tag.id, tag.count));
+        this.tagTree.removeAll();
+        this.tagTree.addNodes(tagMap);
+
+        if (this.$store.state._onLoadSelectedTags.length > 0 && !this.loadedFromArgs) {
+          this.$store.state._onLoadSelectedTags.forEach(mime => {
+            this.tagTree.node(mime).select();
+            this.loadedFromArgs = true;
+          });
+        }
+      });
+    },
+    handleTreeClick(node, e) {
+      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused") {
+        return;
+      }
+
+      this.$store.commit("setSelectedTags", getSelectedTreeNodes(this.tagTree));
+    },
+  }
+}
+</script>
+
+<style scoped>
+#mimeTree {
+  max-height: 350px;
+  overflow: auto;
+}
+</style>
+<style>
+.inspire-tree .focused>.wholerow {
+  border: none;
+}
+</style>

sist2-vue/src/i18n/messages.ts

@@ -0,0 +1,298 @@
+export default {
+    en: {
+        searchBar: {
+            simple: "Search",
+            advanced: "Advanced search",
+            fuzzy: "Fuzzy"
+        },
+        download: "Download",
+        and: "and",
+        page: "page",
+        pages: "pages",
+        mimeTypes: "Media types",
+        tags: "Tags",
+        help: {
+            simpleSearch: "Simple search",
+            advancedSearch: "Advanced search",
+            help: "Help",
+            term: "<TERM>",
+            and: "AND operator",
+            or: "OR operator",
+            not: "negates a single term",
+            quotes: "will match the enclosed sequence of terms in that specific order",
+            prefix: "will match any term with a given prefix when used at the end of a word",
+            parens: "used to group expressions",
+            tildeTerm: "match a term with a given edit distance",
+            tildePhrase: "match a phrase with a given number of allowed intervening unmatched words",
+            example1:
+                "For example: <code>\"fried eggs\" +(eggplant | potato) -frittata</code> will match the " +
+                "phrase <i>fried eggs</i> and either <i>eggplant</i> or <i>potato</i>, but will ignore results " +
+                "containing <i>frittata</i>.",
+            defaultOperator:
+                "When neither <code>+</code> or <code>|</code> is specified, the default operator is " +
+                "<code>+</code> (and).",
+            fuzzy:
+                "When the <b>Fuzzy</b> option is checked, partial matches based on 3-grams are also returned.",
+            moreInfoSimple: "For more information, see <a target=\"_blank\" " +
+                "rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">Elasticsearch documentation</a>",
+            moreInfoAdvanced: "For documentation about the advanced search mode, see <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">Elasticsearch documentation</a>"
+        },
+        config: "Configuration",
+        configDescription: "Configuration is saved in real time for this browser.",
+        configReset: "Reset configuration",
+        searchOptions: "Search options",
+        treemapOptions: "Treemap options",
+        displayOptions: "Display options",
+        opt: {
+            lang: "Language",
+            highlight: "Enable highlighting",
+            fuzzy: "Set fuzzy search by default",
+            searchInPath: "Enable matching query against document path",
+            suggestPath: "Enable auto-complete in path filter bar",
+            fragmentSize: "Highlight context size in characters",
+            queryMode: "Search mode",
+            displayMode: "Display",
+            columns: "Column count",
+            treemapType: "Treemap type",
+            treemapTiling: "Treemap tiling",
+            treemapColorGroupingDepth: "Treemap color grouping depth (flat)",
+            treemapColor: "Treemap color (cascaded)",
+            treemapSize: "Treemap size",
+            theme: "Theme",
+            lightboxLoadOnlyCurrent: "Do not preload full-size images for adjacent slides in image viewer.",
+            slideDuration: "Slide duration",
+            resultSize: "Number of results per page",
+            tagOrOperator: "Use OR operator when specifying multiple tags.",
+            hideDuplicates: "Hide duplicate results based on checksum"
+        },
+        queryMode: {
+            simple: "Simple",
+            advanced: "Advanced",
+        },
+        lang: {
+            en: "English",
+            fr: "Français"
+        },
+        displayMode: {
+            grid: "Grid",
+            list: "List",
+        },
+        columns: {
+            auto: "Auto"
+        },
+        treemapType: {
+            cascaded: "Cascaded",
+            flat: "Flat (compact)"
+        },
+        treemapSize: {
+            small: "Small",
+            medium: "Medium",
+            large: "Large",
+            xLarge: "xLarge",
+            xxLarge: "xxLarge",
+            custom: "Custom",
+        },
+        treemapTiling: {
+            binary: "Binary",
+            squarify: "Squarify",
+            slice: "Slice",
+            dice: "Dice",
+            sliceDice: "Slice & Dice",
+        },
+        theme: {
+            light: "Light",
+            black: "Black"
+        },
+        hit: "hit",
+        hits: "hits",
+        details: "Details",
+        stats: "Stats",
+        queryTime: "Query time",
+        totalSize: "Total size",
+        pathBar: {
+            placeholder: "Filter path",
+            modalTitle: "Select path"
+        },
+        debug: "Debug information",
+        debugDescription: "Information useful for debugging. If you encounter bugs or have suggestions for" +
+            " new features, please submit a new issue <a href='https://github.com/simon987/sist2/issues/new/choose'>here</a>.",
+        tagline: "Tagline",
+        toast: {
+            esConnErrTitle: "Elasticsearch connection error",
+            esConnErr: "sist2 web module encountered an error while connecting to Elasticsearch." +
+                " See server logs for more information.",
+            esQueryErrTitle: "Query error",
+            esQueryErr: "Could not parse or execute query, please check the Advanced search documentation. " +
+                "See server logs for more information.",
+            dupeTagTitle: "Duplicate tag",
+            dupeTag: "This tag already exists for this document."
+        },
+        saveTagModalTitle: "Add tag",
+        saveTagPlaceholder: "Tag name",
+        confirm: "Confirm",
+        indexPickerPlaceholder: "Select indices",
+        sort: {
+            relevance: "Relevance",
+            dateAsc: "Date (Older first)",
+            dateDesc: "Date (Newer first)",
+            sizeAsc: "Size (Smaller first)",
+            sizeDesc: "Size (Larger first)",
+            random: "Random",
+        },
+        d3: {
+            mimeCount: "File count distribution by media type",
+            mimeSize: "Size distribution by media type",
+            dateHistogram: "File modification time distribution",
+            sizeHistogram: "File size distribution",
+        }
+    },
+    fr: {
+        searchBar: {
+            simple: "Recherche",
+            advanced: "Recherche avancée",
+            fuzzy: "Approximatif"
+        },
+        download: "Télécharger",
+        and: "et",
+        page: "page",
+        pages: "pages",
+        mimeTypes: "Types de médias",
+        tags: "Tags",
+        help: {
+            simpleSearch: "Recherche simple",
+            advancedSearch: "Recherche avancée",
+            help: "Aide",
+            term: "<TERME>",
+            and: "opérator ET",
+            or: "opérator OU",
+            not: "exclut un terme",
+            quotes: "recherche la séquence de termes dans cet ordre spécifique.",
+            prefix: "lorsqu'utilisé à la fin d'un mot, recherche tous les termes avec le préfixe donné.",
+            parens: "utilisé pour regrouper des expressions",
+            tildeTerm: "recherche un terme avec une distance d'édition donnée",
+            tildePhrase: "recherche une phrase avec un nombre donné de mots intermédiaires tolérés",
+            example1:
+                "Par exemple: <code>\"fried eggs\" +(eggplant | potato) -frittata</code> va rechercher la " +
+                "phrase <i>fried eggs</i> et soit <i>eggplant</i> ou <i>potato</i>, mais vas exlure les résultats " +
+                "qui contiennent <i>frittata</i>.",
+            defaultOperator:
+                "Lorsqu'aucun des opérateurs <code>+</code> ou <code>|</code> sont spécifiés, l'opérateur par défaut " +
+                "est <code>+</code> (ET).",
+            fuzzy:
+                "Lorsque l'option <b>Approximatif</b> est activée, les résultats partiels basés sur les trigrammes sont" +
+                " également inclus.",
+            moreInfoSimple: "Pour plus d'information, voir <a target=\"_blank\" " +
+                "rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html\">documentation Elasticsearch</a>",
+            moreInfoAdvanced: "Pour plus d'information sur la recherche avancée, voir <a target=\"_blank\" rel=\"noreferrer\" href=\"//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax\">documentation Elasticsearch</a>"
+        },
+        config: "Configuration",
+        configDescription: "La configuration est enregistrée en temps réel pour ce navigateur.",
+        configReset: "Réinitialiser la configuration",
+        searchOptions: "Options de recherche",
+        treemapOptions: "Options du Treemap",
+        displayOptions: "Options d'affichage",
+        opt: {
+            lang: "Langue",
+            highlight: "Activer le surlignage",
+            fuzzy: "Activer la recherche approximative par défaut",
+            searchInPath: "Activer la recherche dans le chemin des documents",
+            suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
+            fragmentSize: "Longueur du contexte de surlignage, en nombre de caractères",
+            queryMode: "Mode de recherche",
+            displayMode: "Affichage",
+            columns: "Nombre de colonnes",
+            treemapType: "Type de Treemap",
+            treemapTiling: "Treemap tiling",
+            treemapColorGroupingDepth: "Groupage de couleur du Treemap (plat)",
+            treemapColor: "Couleur du Treemap (en cascade)",
+            treemapSize: "Taille du Treemap",
+            theme: "Thème",
+            lightboxLoadOnlyCurrent: "Désactiver le chargement des diapositives adjacentes pour le visualiseur d'images",
+            slideDuration: "Durée des diapositives",
+            resultSize: "Nombre de résultats par page",
+            tagOrOperator: "Utiliser l'opérateur OU lors de la spécification de plusieurs tags",
+            hideDuplicates: "Masquer les résultats en double"
+        },
+        queryMode: {
+            simple: "Simple",
+            advanced: "Avancé",
+        },
+        lang: {
+            en: "English",
+            fr: "Français"
+        },
+        displayMode: {
+            grid: "Grille",
+            list: "Liste",
+        },
+        columns: {
+            auto: "Auto"
+        },
+        treemapType: {
+            cascaded: "En cascade",
+            flat: "Plat (compact)"
+        },
+        treemapSize: {
+            small: "Petit",
+            medium: "Moyen",
+            large: "Grand",
+            xLarge: "xGrand",
+            xxLarge: "xxGrand",
+            custom: "Personnalisé",
+        },
+        treemapTiling: {
+            binary: "Binary",
+            squarify: "Squarify",
+            slice: "Slice",
+            dice: "Dice",
+            sliceDice: "Slice & Dice",
+        },
+        theme: {
+            light: "Clair",
+            black: "Noir"
+        },
+        hit: "résultat",
+        hits: "résultats",
+        details: "Détails",
+        stats: "Stats",
+        queryTime: "Durée de la requête",
+        totalSize: "Taille totale",
+        pathBar: {
+            placeholder: "Filtrer le chemin",
+            modalTitle: "Sélectionner le chemin"
+        },
+        debug: "Information de débogage",
+        debugDescription: "Informations utiles pour le débogage\n" +
+            "Si vous rencontrez des bogues ou si vous avez des suggestions pour de nouvelles fonctionnalités," +
+            " veuillez soumettre un nouvel Issue <a href='https://github.com/simon987/sist2/issues/new/choose'>ici</a>.",
+        tagline: "Tagline",
+        toast: {
+            esConnErrTitle: "Erreur de connexion Elasticsearch",
+            esConnErr: "Le module web a rencontré une erreur lors de la connexion à Elasticsearch." +
+                " Consultez les journaux du serveur pour plus d'informations..",
+            esQueryErrTitle: "Erreur de requête",
+            esQueryErr: "Impossible d'analyser ou d'exécuter la requête, veuillez consulter la documentation sur la " +
+                "recherche avancée. Voir les journaux du serveur pour plus d'informations.",
+            dupeTagTitle: "Tag en double",
+            dupeTag: "Ce tag existe déjà pour ce document."
+        },
+        saveTagModalTitle: "Ajouter un tag",
+        saveTagPlaceholder: "Nom du tag",
+        confirm: "Confirmer",
+        indexPickerPlaceholder: "Sélectionner un index",
+        sort: {
+            relevance: "Pertinence",
+            dateAsc: "Date (Plus ancient)",
+            dateDesc: "Date (Plus récent)",
+            sizeAsc: "Taille (Plus petit)",
+            sizeDesc: "Taille (Plus grand)",
+            random: "Aléatoire",
+        },
+        d3: {
+            mimeCount: "Distribution du nombre de fichiers par type de média",
+            mimeSize: "Distribution des tailles de fichiers par type de média",
+            dateHistogram: "Distribution des dates de modification",
+            sizeHistogram: "Distribution des tailles de fichier",
+        }
+    }
+}

sist2-vue/src/main.js

@@ -0,0 +1,29 @@
+import '@babel/polyfill'
+import 'mutationobserver-shim'
+import Vue from 'vue'
+import './plugins/bootstrap-vue'
+import App from './App.vue'
+import router from './router'
+import store from './store'
+import VueI18n from "vue-i18n";
+import messages from "@/i18n/messages";
+
+
+import VueRouter from "vue-router";
+
+Vue.config.productionTip = false;
+
+Vue.use(VueI18n);
+Vue.use(VueRouter);
+
+const i18n = new VueI18n({
+    locale: "en",
+    messages: messages
+});
+
+new Vue({
+    router,
+    store,
+    i18n,
+    render: h => h(App)
+}).$mount("#app");

sist2-vue/src/plugins/bootstrap-vue.js

@@ -0,0 +1,7 @@
+import Vue from "vue"
+
+import BootstrapVue from "bootstrap-vue"
+import "bootstrap/dist/css/bootstrap.min.css"
+import "bootstrap-vue/dist/bootstrap-vue.css"
+
+Vue.use(BootstrapVue)

sist2-vue/src/router/index.ts

@@ -0,0 +1,36 @@
+import Vue from "vue"
+import VueRouter, {RouteConfig} from "vue-router"
+import StatsPage from "../views/StatsPage.vue"
+import Configuration from "../views/Configuration.vue"
+import SearchPage from "@/views/SearchPage.vue";
+
+Vue.use(VueRouter)
+
+const routes: Array<RouteConfig> = [
+    {
+        path: "/",
+        name: "SearchPage",
+        component: SearchPage
+    },
+    {
+        path: "/stats",
+        name: "Stats",
+        component: StatsPage
+    },
+    {
+        path: "/config",
+        name: "Configuration",
+        component: Configuration
+    }
+]
+
+const router = new VueRouter({
+    mode: "hash",
+    base: process.env.BASE_URL,
+    routes,
+    scrollBehavior (to, from, savedPosition) {
+        // return desired position
+    }
+})
+
+export default router

sist2-vue/src/shims-tsx.d.ts

@@ -0,0 +1,17 @@
+import Vue, {VNode} from 'vue'
+
+declare global {
+    namespace JSX {
+        // tslint:disable no-empty-interface
+        interface Element extends VNode {
+        }
+
+        // tslint:disable no-empty-interface
+        interface ElementClass extends Vue {
+        }
+
+        interface IntrinsicElements {
+            [elem: string]: any
+        }
+    }
+}

sist2-vue/src/shims-vue.d.ts

@@ -0,0 +1,4 @@
+declare module '*.vue' {
+  import Vue from 'vue'
+  export default Vue
+}

sist2-vue/src/store/index.ts

@@ -0,0 +1,343 @@
+import Vue from "vue"
+import Vuex from "vuex"
+import VueRouter, {Route} from "vue-router";
+import {EsHit, EsResult, EsTag, Index, Tag} from "@/Sist2Api";
+import {deserializeMimes, serializeMimes} from "@/util";
+
+Vue.use(Vuex)
+
+export default new Vuex.Store({
+    state: {
+        seed: 0,
+        indices: [] as Index[],
+        tags: [] as EsTag[],
+        sist2Info: null as any,
+
+        sizeMin: undefined,
+        sizeMax: undefined,
+        dateBoundsMin: null,
+        dateBoundsMax: null,
+        dateMin: undefined,
+        dateMax: undefined,
+        searchText: "",
+        pathText: "",
+        sortMode: "score",
+
+        fuzzy: false,
+        size: 60,
+
+        optLang: "en",
+        optHideDuplicates: true,
+        optTheme: "light",
+        optDisplay: "grid",
+
+        optHighlight: true,
+        optTagOrOperator: false,
+        optFuzzy: true,
+        optFragmentSize: 200,
+        optQueryMode: "simple",
+        optSearchInPath: false,
+        optColumns: "auto",
+        optSuggestPath: true,
+        optTreemapType: "cascaded",
+        optTreemapTiling: "squarify",
+        optTreemapColorGroupingDepth: 3,
+        optTreemapSize: "medium",
+        optTreemapColor: "PuBuGn",
+        optLightboxLoadOnlyCurrent: false,
+        optLightboxSlideDuration: 15,
+
+        _onLoadSelectedIndices: [] as string[],
+        _onLoadSelectedMimeTypes: [] as string[],
+        _onLoadSelectedTags: [] as string[],
+        selectedIndices: [] as Index[],
+        selectedMimeTypes: [] as string[],
+        selectedTags: [] as string[],
+
+        lastQueryResults: null,
+
+        keySequence: 0,
+        querySequence: 0,
+
+        uiTagHover: null as Tag | null,
+        uiLightboxIsOpen: false,
+        uiShowLightbox: false,
+        uiLightboxSources: [] as string[],
+        uiLightboxThumbs: [] as string[],
+        uiLightboxCaptions: [] as any[],
+        uiLightboxTypes: [] as string[],
+        uiLightboxKey: 0,
+        uiLightboxSlide: 0,
+        uiReachedScrollEnd: false,
+
+        uiMimeMap: [] as any[]
+    },
+    mutations: {
+        setUiReachedScrollEnd: (state, val) => state.uiReachedScrollEnd = val,
+        setTags: (state, val) => state.tags = val,
+        setPathText: (state, val) => state.pathText = val,
+        setSizeMin: (state, val) => state.sizeMin = val,
+        setSizeMax: (state, val) => state.sizeMax = val,
+        setSist2Info: (state, val) => state.sist2Info = val,
+        setSeed: (state, val) => state.seed = val,
+        setOptHideDuplicates: (state, val) => state.optHideDuplicates = val,
+        setOptLang: (state, val) => state.optLang = val,
+        setSortMode: (state, val) => state.sortMode = val,
+        setIndices: (state, val) => {
+            state.indices = val;
+
+            if (state._onLoadSelectedIndices.length > 0) {
+
+                state.selectedIndices = val.filter(
+                    (idx: Index) => state._onLoadSelectedIndices.some(prefix => idx.id.startsWith(prefix))
+                );
+            } else {
+                state.selectedIndices = val;
+            }
+        },
+        setDateMin: (state, val) => state.dateMin = val,
+        setDateMax: (state, val) => state.dateMax = val,
+        setDateBoundsMin: (state, val) => state.dateBoundsMin = val,
+        setDateBoundsMax: (state, val) => state.dateBoundsMax = val,
+        setSearchText: (state, val) => state.searchText = val,
+        setFuzzy: (state, val) => state.fuzzy = val,
+        setLastQueryResult: (state, val) => state.lastQueryResults = val,
+        _setOnLoadSelectedIndices: (state, val) => state._onLoadSelectedIndices = val,
+        _setOnLoadSelectedMimeTypes: (state, val) => state._onLoadSelectedMimeTypes = val,
+        _setOnLoadSelectedTags: (state, val) => state._onLoadSelectedTags = val,
+        setSelectedIndices: (state, val) => state.selectedIndices = val,
+        setSelectedMimeTypes: (state, val) => state.selectedMimeTypes = val,
+        setSelectedTags: (state, val) => state.selectedTags = val,
+        setUiTagHover: (state, val: Tag | null) => state.uiTagHover = val,
+        setUiLightboxIsOpen: (state, val: boolean) => state.uiLightboxIsOpen = val,
+        _setUiShowLightbox: (state, val: boolean) => state.uiShowLightbox = val,
+        setUiLightboxKey: (state, val: number) => state.uiLightboxKey = val,
+        _setKeySequence: (state, val: number) => state.keySequence = val,
+        _setQuerySequence: (state, val: number) => state.querySequence = val,
+        addLightboxSource: (state, {source, thumbnail, caption, type}) => {
+            state.uiLightboxSources.push(source);
+            state.uiLightboxThumbs.push(thumbnail);
+            state.uiLightboxCaptions.push(caption);
+            state.uiLightboxTypes.push(type);
+        },
+        setUiLightboxSlide: (state, val: number) => state.uiLightboxSlide = val,
+
+        setUiLightboxSources: (state, val) => state.uiLightboxSources = val,
+        setUiLightboxThumbs: (state, val) => state.uiLightboxThumbs = val,
+        setUiLightboxTypes: (state, val) => state.uiLightboxTypes = val,
+        setUiLightboxCaptions: (state, val) => state.uiLightboxCaptions = val,
+
+        setOptTheme: (state, val) => state.optTheme = val,
+        setOptDisplay: (state, val) => state.optDisplay = val,
+        setOptColumns: (state, val) => state.optColumns = val,
+        setOptHighlight: (state, val) => state.optHighlight = val,
+        setOptFuzzy: (state, val) => state.fuzzy = val,
+        setOptSearchInPath: (state, val) => state.optSearchInPath = val,
+        setOptSuggestPath: (state, val) => state.optSuggestPath = val,
+        setOptFragmentSize: (state, val) => state.optFragmentSize = val,
+        setOptQueryMode: (state, val) => state.optQueryMode = val,
+        setOptResultSize: (state, val) => state.size = val,
+        setOptTagOrOperator: (state, val) => state.optTagOrOperator = val,
+
+        setOptTreemapType: (state, val) => state.optTreemapType = val,
+        setOptTreemapTiling: (state, val) => state.optTreemapTiling = val,
+        setOptTreemapColorGroupingDepth: (state, val) => state.optTreemapColorGroupingDepth = val,
+        setOptTreemapSize: (state, val) => state.optTreemapSize = val,
+        setOptTreemapColor: (state, val) => state.optTreemapColor = val,
+
+        setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
+
+        setUiMimeMap: (state, val) => state.uiMimeMap = val,
+
+        busUpdateWallItems: () => {
+            // noop
+        },
+        busUpdateTags: () => {
+            // noop
+        },
+    },
+    actions: {
+        loadFromArgs({commit}, route: Route) {
+
+            if (route.query.q) {
+                commit("setSearchText", route.query.q);
+            }
+
+            if (route.query.fuzzy !== undefined) {
+                commit("setFuzzy", true);
+            }
+
+            if (route.query.i) {
+                commit("_setOnLoadSelectedIndices", Array.isArray(route.query.i) ? route.query.i : [route.query.i,]);
+            }
+
+            if (route.query.dMin) {
+                commit("setDateMin", Number(route.query.dMin))
+            }
+
+            if (route.query.dMax) {
+                commit("setDateMax", Number(route.query.dMax))
+            }
+
+            if (route.query.sMin) {
+                commit("setSizeMin", Number(route.query.sMin))
+            }
+
+            if (route.query.sMax) {
+                commit("setSizeMax", Number(route.query.sMax))
+            }
+
+            if (route.query.path) {
+                commit("setPathText", route.query.path)
+            }
+
+            if (route.query.m) {
+                commit("_setOnLoadSelectedMimeTypes", deserializeMimes(route.query.m as string));
+            }
+
+            if (route.query.t) {
+                commit("_setOnLoadSelectedTags", (route.query.t as string).split(","));
+            }
+
+            if (route.query.sort) {
+                commit("setSortMode", route.query.sort);
+                commit("setSeed", Number(route.query.seed));
+            }
+        },
+        async updateArgs({state}, router: VueRouter) {
+            await router.push({
+                query: {
+                    q: state.searchText.trim() ? state.searchText.trim().replace(/\s+/g, " ") : undefined,
+                    fuzzy: state.fuzzy ? null : undefined,
+                    i: state.selectedIndices ? state.selectedIndices.map((idx: Index) => idx.idPrefix) : undefined,
+                    dMin: state.dateMin,
+                    dMax: state.dateMax,
+                    sMin: state.sizeMin,
+                    sMax: state.sizeMax,
+                    path: state.pathText ? state.pathText : undefined,
+                    m: serializeMimes(state.selectedMimeTypes),
+                    t: state.selectedTags.length == 0 ? undefined : state.selectedTags.join(","),
+                    sort: state.sortMode === "score" ? undefined : state.sortMode,
+                    seed: state.sortMode === "random" ? state.seed.toString() : undefined
+                }
+            }).catch(() => {
+                // ignore
+            });
+        },
+        updateConfiguration({state}) {
+            const conf = {} as any;
+
+            Object.keys(state).forEach((key) => {
+                if (key.startsWith("opt")) {
+                    conf[key] = (state as any)[key];
+                }
+            });
+
+            localStorage.setItem("sist2_configuration", JSON.stringify(conf));
+        },
+        loadConfiguration({state}) {
+            const confString = localStorage.getItem("sist2_configuration");
+            if (confString) {
+                const conf = JSON.parse(confString);
+
+                Object.keys(state).forEach((key) => {
+                    if (key.startsWith("opt")) {
+                        (state as any)[key] = conf[key];
+                    }
+                });
+            }
+        },
+        setSelectedIndices: ({commit}, val) => commit("setSelectedIndices", val),
+        getKeySequence({commit, state}) {
+            const val = state.keySequence;
+            commit("_setKeySequence", val + 1);
+
+            return val
+        },
+        incrementQuerySequence({commit, state}) {
+            const val = state.querySequence;
+            commit("_setQuerySequence", val + 1);
+
+            return val
+        },
+        remountLightbox({commit, state}) {
+            // Change key to an arbitrary number to force the lightbox to remount
+            commit("setUiLightboxKey", state.uiLightboxKey + 1);
+        },
+        showLightbox({commit, state}) {
+            commit("_setUiShowLightbox", !state.uiShowLightbox);
+        },
+        clearResults({commit}) {
+            commit("setLastQueryResult", null);
+            commit("_setKeySequence", 0);
+            commit("_setUiShowLightbox", false);
+            commit("setUiLightboxSources", []);
+            commit("setUiLightboxThumbs", []);
+            commit("setUiLightboxTypes", []);
+            commit("setUiLightboxCaptions", []);
+            commit("setUiLightboxKey", 0);
+        }
+    },
+    modules: {},
+    getters: {
+        seed: (state) => state.seed,
+        getPathText: (state) => state.pathText,
+        indices: state => state.indices,
+        sist2Info: state => state.sist2Info,
+        indexMap: state => {
+            const map = {} as any;
+            state.indices.forEach(idx => map[idx.id] = idx);
+            return map;
+        },
+        selectedIndices: (state) => state.selectedIndices,
+        _onLoadSelectedIndices: (state) => state._onLoadSelectedIndices,
+        selectedMimeTypes: (state) => state.selectedMimeTypes,
+        selectedTags: (state) => state.selectedTags,
+        dateMin: state => state.dateMin,
+        dateMax: state => state.dateMax,
+        sizeMin: state => state.sizeMin,
+        sizeMax: state => state.sizeMax,
+        searchText: state => state.searchText,
+        pathText: state => state.pathText,
+        fuzzy: state => state.fuzzy,
+        size: state => state.size,
+        sortMode: state => state.sortMode,
+        lastQueryResult: state => state.lastQueryResults,
+        lastDoc: function (state): EsHit | null {
+            if (state.lastQueryResults == null) {
+                return null;
+            }
+
+            return (state.lastQueryResults as unknown as EsResult).hits.hits.slice(-1)[0];
+        },
+        uiTagHover: state => state.uiTagHover,
+        uiShowLightbox: state => state.uiShowLightbox,
+        uiLightboxSources: state => state.uiLightboxSources,
+        uiLightboxThumbs: state => state.uiLightboxThumbs,
+        uiLightboxCaptions: state => state.uiLightboxCaptions,
+        uiLightboxTypes: state => state.uiLightboxTypes,
+        uiLightboxKey: state => state.uiLightboxKey,
+        uiLightboxSlide: state => state.uiLightboxSlide,
+
+        optHideDuplicates: state => state.optHideDuplicates,
+        optLang: state => state.optLang,
+        optTheme: state => state.optTheme,
+        optDisplay: state => state.optDisplay,
+        optColumns: state => state.optColumns,
+        optHighlight: state => state.optHighlight,
+        optTagOrOperator: state => state.optTagOrOperator,
+        optFuzzy: state => state.optFuzzy,
+        optSearchInPath: state => state.optSearchInPath,
+        optSuggestPath: state => state.optSuggestPath,
+        optFragmentSize: state => state.optFragmentSize,
+        optQueryMode: state => state.optQueryMode,
+        optTreemapType: state => state.optTreemapType,
+        optTreemapTiling: state => state.optTreemapTiling,
+        optTreemapSize: state => state.optTreemapSize,
+        optTreemapColorGroupingDepth: state => state.optTreemapColorGroupingDepth,
+        optTreemapColor: state => state.optTreemapColor,
+        optLightboxLoadOnlyCurrent: state => state.optLightboxLoadOnlyCurrent,
+        optLightboxSlideDuration: state => state.optLightboxSlideDuration,
+        optResultSize: state => state.size,
+    }
+})

sist2-vue/src/util-js.js

@@ -0,0 +1,139 @@
+export function mergeTooltips(slider, threshold, separator, fixTooltips) {
+
+    const isMobile = window.innerWidth <= 650;
+    if (isMobile) {
+        threshold = 25;
+    }
+
+    const textIsRtl = getComputedStyle(slider).direction === 'rtl';
+    const isRtl = slider.noUiSlider.options.direction === 'rtl';
+    const isVertical = slider.noUiSlider.options.orientation === 'vertical';
+    const tooltips = slider.noUiSlider.getTooltips();
+    const origins = slider.noUiSlider.getOrigins();
+
+    // Move tooltips into the origin element. The default stylesheet handles this.
+    tooltips.forEach(function (tooltip, index) {
+        if (tooltip) {
+            origins[index].appendChild(tooltip);
+        }
+    });
+
+    slider.noUiSlider.on('update', function (values, handle, unencoded, tap, positions) {
+
+        const pools = [[]];
+        const poolPositions = [[]];
+        const poolValues = [[]];
+        let atPool = 0;
+
+        // Assign the first tooltip to the first pool, if the tooltip is configured
+        if (tooltips[0]) {
+            pools[0][0] = 0;
+            poolPositions[0][0] = positions[0];
+            poolValues[0][0] = values[0];
+        }
+
+        for (let i = 1; i < positions.length; i++) {
+            if (!tooltips[i] || (positions[i] - positions[i - 1]) > threshold) {
+                atPool++;
+                pools[atPool] = [];
+                poolValues[atPool] = [];
+                poolPositions[atPool] = [];
+            }
+
+            if (tooltips[i]) {
+                pools[atPool].push(i);
+                poolValues[atPool].push(values[i]);
+                poolPositions[atPool].push(positions[i]);
+            }
+        }
+
+        pools.forEach(function (pool, poolIndex) {
+            const handlesInPool = pool.length;
+
+            for (let j = 0; j < handlesInPool; j++) {
+                const handleNumber = pool[j];
+
+                if (j === handlesInPool - 1) {
+                    let offset = 0;
+
+                    poolPositions[poolIndex].forEach(function (value) {
+                        offset += 1000 - 10 * value;
+                    });
+
+                    const direction = isVertical ? 'bottom' : 'right';
+                    const last = isRtl ? 0 : handlesInPool - 1;
+                    const lastOffset = 1000 - 10 * poolPositions[poolIndex][last];
+                    offset = (textIsRtl && !isVertical ? 100 : 0) + (offset / handlesInPool) - lastOffset;
+
+                    // Center this tooltip over the affected handles
+                    tooltips[handleNumber].innerHTML = poolValues[poolIndex].join(separator);
+                    tooltips[handleNumber].style.display = 'block';
+
+                    tooltips[handleNumber].style[direction] = offset + '%';
+                } else {
+                    // Hide this tooltip
+                    tooltips[handleNumber].style.display = 'none';
+                }
+            }
+        });
+
+        if (fixTooltips) {
+            const isMobile = window.innerWidth <= 650;
+            const len = isMobile ? 20 : 5;
+
+            if (positions[0] < len) {
+                tooltips[0].style.right = `${(1 - ((positions[0]) / len)) * -35}px`
+            } else {
+                tooltips[0].style.right = "0"
+            }
+
+            if (positions[1] > (100 - len)) {
+                tooltips[1].style.right = `${((positions[1] - (100 - len)) / len) * 35}px`
+            } else {
+                tooltips[1].style.right = "0"
+            }
+        }
+    });
+}
+
+
+export function burrow(table, addSelfDir, rootName) {
+    const root = {};
+    table.forEach(row => {
+        let layer = root;
+
+        row.taxonomy.forEach(key => {
+            layer[key] = key in layer ? layer[key] : {};
+            layer = layer[key];
+        });
+        if (Object.keys(layer).length === 0) {
+            layer["$size$"] = row.size;
+        } else if (addSelfDir) {
+            layer["."] = {
+                "$size$": row.size,
+            };
+        }
+    });
+
+    const descend = function (obj, depth) {
+        return Object.keys(obj).filter(k => k !== "$size$").map(k => {
+            const child = {
+                name: k,
+                depth: depth,
+                value: 0,
+                children: descend(obj[k], depth + 1)
+            };
+            if ("$size$" in obj[k]) {
+                child.value = obj[k]["$size$"];
+            }
+            return child;
+        });
+    };
+
+    return {
+        name: rootName,
+        children: descend(root, 1),
+        value: 0,
+        depth: 0,
+    }
+}

sist2-vue/src/util.ts

@@ -0,0 +1,137 @@
+import {EsHit} from "@/Sist2Api";
+
+export function ext(hit: EsHit) {
+    return Object.prototype.hasOwnProperty.call(hit._source, "extension")
+    && hit["_source"]["extension"] !== "" ? "." + hit["_source"]["extension"] : "";
+}
+
+export function strUnescape(str: string): string {
+    let result = "";
+
+    for (let i = 0; i < str.length; i++) {
+        const c = str[i];
+        const next = str[i + 1];
+
+        if (c === "]") {
+            if (next === "]") {
+                result += c;
+                i += 1;
+            } else {
+                result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
+                i += 2;
+            }
+        } else {
+            result += c;
+        }
+    }
+    return result;
+}
+
+const thresh = 1000;
+const units = ["k", "M", "G", "T", "P", "E", "Z", "Y"];
+
+export function humanFileSize(bytes: number): string {
+    if (bytes === 0) {
+        return "0 B"
+    }
+
+    if (Math.abs(bytes) < thresh) {
+        return bytes + ' B';
+    }
+    let u = -1;
+    do {
+        bytes /= thresh;
+        ++u;
+    } while (Math.abs(bytes) >= thresh && u < units.length - 1);
+
+    return bytes.toFixed(1) + units[u];
+}
+
+export function humanTime(sec_num: number): string {
+    sec_num = Math.floor(sec_num);
+    const hours = Math.floor(sec_num / 3600);
+    const minutes = Math.floor((sec_num - (hours * 3600)) / 60);
+    const seconds = sec_num - (hours * 3600) - (minutes * 60);
+
+    return `${hours < 10 ? "0" : ""}${hours}:${minutes < 10 ? "0" : ""}${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
+}
+
+export function humanDate(numMilis: number): string {
+    const date = (new Date(numMilis * 1000));
+    return date.getUTCFullYear() + "-" + ("0" + (date.getUTCMonth() + 1)).slice(-2) + "-" + ("0" + date.getUTCDate()).slice(-2)
+
+}
+
+export function lum(c: string) {
+    c = c.substring(1);
+    const rgb = parseInt(c, 16);
+    const r = (rgb >> 16) & 0xff;
+    const g = (rgb >> 8) & 0xff;
+    const b = (rgb >> 0) & 0xff;
+
+    return 0.2126 * r + 0.7152 * g + 0.0722 * b;
+}
+
+
+export function getSelectedTreeNodes(tree: any) {
+    const selectedNodes = new Set();
+
+    const selected = tree.selected();
+
+    for (let i = 0; i < selected.length; i++) {
+
+        if (selected[i].id === "any") {
+            return ["any"]
+        }
+
+        //Only get children
+        if (selected[i].text.indexOf("(") !== -1) {
+            if (selected[i].values) {
+                selectedNodes.add(selected[i].values.slice(-1)[0]);
+            } else {
+                selectedNodes.add(selected[i].id);
+            }
+        }
+    }
+
+    return Array.from(selectedNodes);
+}
+
+export function serializeMimes(mimes: string[]): string | undefined {
+    if (mimes.length == 0) {
+        return undefined;
+    }
+    return mimes.map(mime => compressMime(mime)).join("");
+}
+
+export function deserializeMimes(mimeString: string): string[] {
+    return mimeString
+        .replaceAll(/([IVATUF])/g, "$$$&")
+        .split("$")
+        .map(mime => decompressMime(mime))
+        .slice(1) // Ignore the first (empty) token
+}
+
+export function compressMime(mime: string): string {
+    return mime
+        .replace("image/", "I")
+        .replace("video/", "V")
+        .replace("application/", "A")
+        .replace("text/", "T")
+        .replace("audio/", "U")
+        .replace("font/", "F")
+        .replace("+", ",")
+        .replace("x-", "X")
+}
+
+export function decompressMime(mime: string): string {
+    return mime
+        .replace("I", "image/")
+        .replace("V", "video/")
+        .replace("A", "application/")
+        .replace("T", "text/")
+        .replace("U", "audio/")
+        .replace("F", "font/")
+        .replace(",", "+")
+        .replace("X", "x-")
+}

sist2-vue/src/views/Configuration.vue

@@ -0,0 +1,271 @@
+<template>
+  <!--  <div :style="{width: `${$store.getters.optContainerWidth}px`}"-->
+  <div
+      v-if="!configLoading"
+      style="margin-left: auto; margin-right: auto;" class="container">
+
+    <b-card>
+      <b-card-title>
+        <GearIcon></GearIcon>
+        {{ $t("config") }}
+      </b-card-title>
+      <p>{{ $t("configDescription") }}</p>
+
+      <b-card-body>
+        <h4>{{ $t("displayOptions") }}</h4>
+
+        <b-card>
+          <b-form-checkbox :checked="optLightboxLoadOnlyCurrent" @input="setOptLightboxLoadOnlyCurrent">
+            {{ $t("opt.lightboxLoadOnlyCurrent") }}
+          </b-form-checkbox>
+
+          <label>{{ $t("opt.lang") }}</label>
+          <b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
+
+          <label>{{ $t("opt.theme") }}</label>
+          <b-form-select :options="themeOptions" :value="optTheme" @input="setOptTheme"></b-form-select>
+
+          <label>{{ $t("opt.displayMode") }}</label>
+          <b-form-select :options="displayModeOptions" :value="optDisplay" @input="setOptDisplay"></b-form-select>
+
+          <label>{{ $t("opt.columns") }}</label>
+          <b-form-select :options="columnsOptions" :value="optColumns" @input="setOptColumns"></b-form-select>
+        </b-card>
+
+        <br/>
+        <h4>{{ $t("searchOptions") }}</h4>
+        <b-card>
+          <b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
+              $t("opt.hideDuplicates")
+            }}
+          </b-form-checkbox>
+
+          <b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{ $t("opt.highlight") }}</b-form-checkbox>
+          <b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
+              $t("opt.tagOrOperator")
+            }}
+          </b-form-checkbox>
+          <b-form-checkbox :checked="optFuzzy" @input="setOptFuzzy">{{ $t("opt.fuzzy") }}</b-form-checkbox>
+          <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
+              $t("opt.searchInPath")
+            }}
+          </b-form-checkbox>
+          <b-form-checkbox :checked="optSuggestPath" @input="setOptSuggestPath">{{
+              $t("opt.suggestPath")
+            }}
+          </b-form-checkbox>
+
+          <br/>
+          <label>{{ $t("opt.fragmentSize") }}</label>
+          <b-form-input :value="optFragmentSize" step="10" type="number" min="0"
+                        @input="setOptFragmentSize"></b-form-input>
+
+          <label>{{ $t("opt.resultSize") }}</label>
+          <b-form-input :value="optResultSize" type="number" min="10"
+                        @input="setOptResultSize"></b-form-input>
+
+          <label>{{ $t("opt.queryMode") }}</label>
+          <b-form-select :options="queryModeOptions" :value="optQueryMode" @input="setOptQueryMode"></b-form-select>
+
+          <label>{{ $t("opt.slideDuration") }}</label>
+          <b-form-input :value="optLightboxSlideDuration" type="number" min="1"
+                        @input="setOptLightboxSlideDuration"></b-form-input>
+        </b-card>
+
+        <h4 class="mt-3">{{ $t("treemapOptions") }}</h4>
+        <b-card>
+          <label>{{ $t("opt.treemapType") }}</label>
+          <b-form-select :value="optTreemapType" :options="treemapTypeOptions"
+                         @input="setOptTreemapType"></b-form-select>
+
+          <label>{{ $t("opt.treemapTiling") }}</label>
+          <b-form-select :value="optTreemapTiling" :options="treemapTilingOptions"
+                         @input="setOptTreemapTiling"></b-form-select>
+
+          <label>{{ $t("opt.treemapColorGroupingDepth") }}</label>
+          <b-form-input :value="optTreemapColorGroupingDepth" type="number" min="1"
+                        @input="setOptTreemapColorGroupingDepth"></b-form-input>
+
+          <label>{{ $t("opt.treemapSize") }}</label>
+          <b-form-select :value="optTreemapSize" :options="treemapSizeOptions"
+                         @input="setOptTreemapSize"></b-form-select>
+
+          <template v-if="$store.getters.optTreemapSize === 'custom'">
+            <!-- TODO Width/Height input -->
+            <b-form-input type="number" min="0" step="10"></b-form-input>
+            <b-form-input type="number" min="0" step="10"></b-form-input>
+          </template>
+
+          <label>{{ $t("opt.treemapColor") }}</label>
+          <b-form-select :value="optTreemapColor" :options="treemapColorOptions"
+                         @input="setOptTreemapColor"></b-form-select>
+        </b-card>
+
+        <b-button variant="danger" class="mt-4" @click="onResetClick()">{{ $t("configReset") }}</b-button>
+      </b-card-body>
+    </b-card>
+
+    <b-card v-if="loading" class="mt-4">
+      <Preloader></Preloader>
+    </b-card>
+    <DebugInfo v-else></DebugInfo>
+  </div>
+</template>
+
+<script>
+import Vue from "vue";
+import {mapGetters, mapMutations} from "vuex";
+import DebugInfo from "@/components/DebugInfo.vue";
+import Preloader from "@/components/Preloader.vue";
+import sist2 from "@/Sist2Api";
+import GearIcon from "@/components/GearIcon.vue";
+
+export default {
+  components: {GearIcon, DebugInfo, Preloader},
+  data() {
+    return {
+      loading: true,
+      configLoading: false,
+      langOptions: [
+        {value: "en", text: this.$t("lang.en")},
+        {value: "fr", text: this.$t("lang.fr")},
+      ],
+      queryModeOptions: [
+        {value: "simple", text: this.$t("queryMode.simple")},
+        {value: "advanced", text: this.$t("queryMode.advanced")}
+      ],
+      displayModeOptions: [
+        {value: "grid", text: this.$t("displayMode.grid")},
+        {value: "list", text: this.$t("displayMode.list")}
+      ],
+      columnsOptions: [
+        {value: "auto", text: this.$t("columns.auto")},
+        {value: 1, text: "1"},
+        {value: 2, text: "2"},
+        {value: 3, text: "3"},
+        {value: 4, text: "4"},
+        {value: 5, text: "5"},
+        {value: 6, text: "6"},
+        {value: 7, text: "7"},
+        {value: 8, text: "8"},
+        {value: 9, text: "9"},
+        {value: 10, text: "10"},
+        {value: 11, text: "11"},
+        {value: 12, text: "12"},
+      ],
+      treemapTypeOptions: [
+        {value: "cascaded", text: this.$t("treemapType.cascaded")},
+        {value: "flat", text: this.$t("treemapType.flat")}
+      ],
+      treemapTilingOptions: [
+        {value: "binary", text: this.$t("treemapTiling.binary")},
+        {value: "squarify", text: this.$t("treemapTiling.squarify")},
+        {value: "slice", text: this.$t("treemapTiling.slice")},
+        {value: "dice", text: this.$t("treemapTiling.dice")},
+        {value: "sliceDice", text: this.$t("treemapTiling.sliceDice")},
+      ],
+      treemapSizeOptions: [
+        {value: "small", text: this.$t("treemapSize.small")},
+        {value: "medium", text: this.$t("treemapSize.medium")},
+        {value: "large", text: this.$t("treemapSize.large")},
+        {value: "x-large", text: this.$t("treemapSize.xLarge")},
+        {value: "xx-large", text: this.$t("treemapSize.xxLarge")},
+        // {value: "custom", text: this.$t("treemapSize.custom")},
+      ],
+      treemapColorOptions: [
+        {value: "PuBuGn", text: "Purple-Blue-Green"},
+        {value: "PuRd", text: "Purple-Red"},
+        {value: "PuBu", text: "Purple-Blue"},
+        {value: "YlOrBr", text: "Yellow-Orange-Brown"},
+        {value: "YlOrRd", text: "Yellow-Orange-Red"},
+        {value: "YlGn", text: "Yellow-Green"},
+        {value: "YlGnBu", text: "Yellow-Green-Blue"},
+        {value: "Plasma", text: "Plasma"},
+        {value: "Magma", text: "Magma"},
+        {value: "Inferno", text: "Inferno"},
+        {value: "Viridis", text: "Viridis"},
+        {value: "Turbo", text: "Turbo"},
+      ],
+      themeOptions: [
+        {value: "light", text: this.$t("theme.light")},
+        {value: "black", text: this.$t("theme.black")}
+      ]
+
+    }
+  },
+  computed: {
+    ...mapGetters([
+      "optTheme",
+      "optDisplay",
+      "optColumns",
+      "optHighlight",
+      "optFuzzy",
+      "optSearchInPath",
+      "optSuggestPath",
+      "optFragmentSize",
+      "optQueryMode",
+      "optTreemapType",
+      "optTreemapTiling",
+      "optTreemapColorGroupingDepth",
+      "optTreemapColor",
+      "optTreemapSize",
+      "optLightboxLoadOnlyCurrent",
+      "optLightboxSlideDuration",
+      "optResultSize",
+      "optTagOrOperator",
+      "optLang",
+      "optHideDuplicates",
+    ]),
+    clientWidth() {
+      return window.innerWidth;
+    }
+  },
+  mounted() {
+    sist2.getSist2Info().then(data => {
+      this.$store.commit("setSist2Info", data)
+      this.loading = false;
+    });
+
+    this.$store.subscribe((mutation) => {
+      if (mutation.type.startsWith("setOpt")) {
+        this.$store.dispatch("updateConfiguration");
+      }
+    });
+  },
+  methods: {
+    ...mapMutations([
+      "setOptTheme",
+      "setOptDisplay",
+      "setOptColumns",
+      "setOptHighlight",
+      "setOptFuzzy",
+      "setOptSearchInPath",
+      "setOptSuggestPath",
+      "setOptFragmentSize",
+      "setOptQueryMode",
+      "setOptTreemapType",
+      "setOptTreemapTiling",
+      "setOptTreemapColorGroupingDepth",
+      "setOptTreemapColor",
+      "setOptTreemapSize",
+      "setOptLightboxLoadOnlyCurrent",
+      "setOptLightboxSlideDuration",
+      "setOptContainerWidth",
+      "setOptResultSize",
+      "setOptTagOrOperator",
+      "setOptLang",
+      "setOptHideDuplicates"
+    ]),
+    onResetClick() {
+      localStorage.removeItem("sist2_configuration");
+      window.location.reload();
+    }
+  },
+}
+</script>
+
+<style>
+.shrink {
+  flex-grow: inherit;
+}
+</style>

sist2-vue/src/views/SearchPage.vue

@@ -0,0 +1,302 @@
+<template>
+  <div class="container">
+    <Lightbox></Lightbox>
+    <HelpDialog :show="showHelp" @close="showHelp = false"></HelpDialog>
+
+    <b-card v-if="uiLoading">
+      <Preloader></Preloader>
+    </b-card>
+
+    <b-card v-show="!uiLoading" id="search-panel">
+      <SearchBar @show-help="showHelp=true"></SearchBar>
+      <b-row>
+        <b-col style="height: 70px;" sm="6">
+          <SizeSlider></SizeSlider>
+        </b-col>
+        <b-col>
+          <PathTree @search="search(true)"></PathTree>
+        </b-col>
+      </b-row>
+      <b-row>
+        <b-col sm="6">
+          <b-row>
+            <b-col style="height: 70px;">
+              <DateSlider></DateSlider>
+            </b-col>
+          </b-row>
+          <b-row>
+            <b-col>
+              <IndexPicker></IndexPicker>
+            </b-col>
+          </b-row>
+        </b-col>
+        <b-col>
+          <b-tabs>
+            <b-tab :title="$t('mimeTypes')">
+              <MimePicker></MimePicker>
+            </b-tab>
+            <b-tab :title="$t('tags')">
+              <TagPicker></TagPicker>
+            </b-tab>
+          </b-tabs>
+        </b-col>
+      </b-row>
+    </b-card>
+
+    <Preloader v-if="searchBusy && docs.length === 0" class="mt-3"></Preloader>
+
+    <div v-else-if="docs.length > 0">
+      <ResultsCard></ResultsCard>
+
+      <DocCardWall v-if="optDisplay==='grid'" :docs="docs" :append="appendFunc"></DocCardWall>
+      <DocList v-else :docs="docs" :append="appendFunc"></DocList>
+    </div>
+  </div>
+</template>
+
+<script lang="ts">
+import Preloader from "@/components/Preloader.vue";
+import {mapGetters, mapMutations} from "vuex";
+import sist2 from "../Sist2Api";
+import Sist2Api, {EsHit, EsResult} from "../Sist2Api";
+import SearchBar from "@/components/SearchBar.vue";
+import IndexPicker from "@/components/IndexPicker.vue";
+import Vue from "vue";
+import Sist2Query from "@/Sist2Query";
+import _debounce from "lodash/debounce";
+import DocCardWall from "@/components/DocCardWall.vue";
+import Lightbox from "@/components/Lightbox.vue";
+import LightboxCaption from "@/components/LightboxCaption.vue";
+import MimePicker from "../components/MimePicker.vue";
+import ResultsCard from "@/components/ResultsCard.vue";
+import PathTree from "@/components/PathTree.vue";
+import SizeSlider from "@/components/SizeSlider.vue";
+import DateSlider from "@/components/DateSlider.vue";
+import TagPicker from "@/components/TagPicker.vue";
+import DocList from "@/components/DocList.vue";
+import HelpDialog from "@/components/HelpDialog.vue";
+
+
+export default Vue.extend({
+  components: {
+    HelpDialog,
+    DocList,
+    TagPicker,
+    DateSlider,
+    SizeSlider, PathTree, ResultsCard, MimePicker, Lightbox, DocCardWall, IndexPicker, SearchBar, Preloader
+  },
+  data: () => ({
+    loading: false,
+    uiLoading: true,
+    search: undefined as any,
+    docs: [] as EsHit[],
+    docIds: new Set(),
+    docChecksums: new Set(),
+    searchBusy: false,
+    Sist2Query: Sist2Query,
+    showHelp: false
+  }),
+  computed: {
+    ...mapGetters(["indices", "optDisplay"]),
+  },
+  mounted() {
+    this.search = _debounce(async (clear: boolean) => {
+      if (clear) {
+        await this.clearResults();
+      }
+
+      await this.searchNow(Sist2Query.searchQuery());
+
+    }, 350, {leading: false});
+
+    Sist2Api.getMimeTypes().then(mimeMap => {
+      this.$store.commit("setUiMimeMap", mimeMap);
+    });
+
+    this.$store.dispatch("loadFromArgs", this.$route).then(() => {
+      this.$store.subscribe(() => this.$store.dispatch("updateArgs", this.$router));
+      this.$store.subscribe((mutation) => {
+        if ([
+          "setSizeMin", "setSizeMax", "setDateMin", "setDateMax", "setSearchText", "setPathText",
+          "setSortMode", "setOptHighlight", "setOptFragmentSize", "setFuzzy", "setSize", "setSelectedIndices",
+          "setSelectedMimeTypes", "setSelectedTags", "setOptQueryMode", "setOptSearchInPath",
+        ].includes(mutation.type)) {
+          if (this.searchBusy) {
+            return;
+          }
+
+          this.search(true);
+        }
+      });
+    });
+
+
+    this.getDateRange().then((range: { min: number, max: number }) => {
+      this.setDateBoundsMin(range.min);
+      this.setDateBoundsMax(range.max);
+
+      sist2.getSist2Info().then(data => {
+        this.setSist2Info(data);
+        this.setIndices(data.indices);
+        this.uiLoading = false;
+
+        this.search(true);
+      }).catch(() => {
+        this.showErrorToast();
+      });
+    });
+  },
+  methods: {
+    ...mapMutations({
+      setSist2Info: "setSist2Info",
+      setIndices: "setIndices",
+      setDateBoundsMin: "setDateBoundsMin",
+      setDateBoundsMax: "setDateBoundsMax",
+      setTags: "setTags",
+    }),
+    showErrorToast() {
+      this.$bvToast.toast(
+          this.$t("toast.esConnErr"),
+          {
+            title: this.$t("toast.esConnErrTitle"),
+            noAutoHide: true,
+            toaster: "b-toaster-bottom-right",
+            headerClass: "toast-header-error",
+            bodyClass: "toast-body-error",
+          });
+    },
+    showSyntaxErrorToast: function (): void {
+      this.$bvToast.toast(
+          this.$t("toast.esQueryErr"),
+          {
+            title: this.$t("toast.esQueryErrTitle"),
+            noAutoHide: true,
+            toaster: "b-toaster-bottom-right",
+            headerClass: "toast-header-warning",
+            bodyClass: "toast-body-warning",
+          });
+    },
+    async searchNow(q: any) {
+      this.searchBusy = true;
+      await this.$store.dispatch("incrementQuerySequence");
+
+      Sist2Api.esQuery(q).then(async (resp: EsResult) => {
+        await this.handleSearch(resp);
+        this.searchBusy = false;
+      }).catch(err => {
+        if (err.response.status === 500 && this.$store.state.optQueryMode === "advanced") {
+          this.showSyntaxErrorToast();
+        } else {
+          this.showErrorToast();
+        }
+      });
+    },
+    async clearResults() {
+      this.docs = [];
+      this.docIds.clear();
+      this.docChecksums.clear();
+      await this.$store.dispatch("clearResults");
+      this.$store.commit("setUiReachedScrollEnd", false);
+    },
+    async handleSearch(resp: EsResult) {
+      if (resp.hits.hits.length == 0) {
+        this.$store.commit("setUiReachedScrollEnd", true);
+      }
+
+      resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
+
+      if (this.$store.state.optHideDuplicates) {
+        resp.hits.hits = resp.hits.hits.filter(hit => {
+
+          if (!("checksum" in hit._source)) {
+              return true;
+          }
+
+          const isDupe = !this.docChecksums.has(hit._source.checksum);
+          this.docChecksums.add(hit._source.checksum);
+          return isDupe;
+        });
+      }
+
+      for (const hit of resp.hits.hits) {
+        if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {
+          hit._seq = await this.$store.dispatch("getKeySequence");
+          this.$store.commit("addLightboxSource", {
+            source: `f/${hit._id}`,
+            thumbnail: hit._props.hasThumbnail
+                ? `t/${hit._source.index}/${hit._id}`
+                : null,
+            caption: {
+              component: LightboxCaption,
+              props: {hit: hit}
+            },
+            type: hit._props.isVideo ? "video" : "image"
+          });
+        }
+      }
+
+      await this.$store.dispatch("remountLightbox");
+      this.$store.commit("setLastQueryResult", resp);
+
+      this.docs.push(...resp.hits.hits);
+    },
+    getDateRange(): Promise<{ min: number, max: number }> {
+      return sist2.esQuery({
+        // TODO: filter current selected indices
+        aggs: {
+          dateMin: {min: {field: "mtime"}},
+          dateMax: {max: {field: "mtime"}},
+        },
+        size: 0
+      }).then(res => {
+        return {
+          min: res.aggregations.dateMin.value,
+          max: res.aggregations.dateMax.value,
+        }
+      })
+    },
+    appendFunc() {
+      if (!this.$store.state.uiReachedScrollEnd && this.search && !this.searchBusy) {
+        this.searchNow(Sist2Query.searchQuery());
+      }
+    }
+  },
+  beforeRouteUpdate(to, from, next) {
+    if (this.$store.state.uiLightboxIsOpen) {
+      this.$store.commit("_setUiShowLightbox", false);
+      next(false);
+    } else {
+      next();
+    }
+  },
+})
+</script>
+
+<style>
+
+#search-panel {
+  box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+  border-radius: 0;
+  border: none;
+}
+
+.toast-header-error, .toast-body-error {
+  background: #a94442;
+  color: #f2dede !important;
+}
+
+.toast-header-error {
+  color: #fff !important;
+  border-bottom: none;
+  margin-bottom: -1em;
+}
+
+.toast-header-error .close {
+  text-shadow: none;
+}
+
+.toast-header-warning, .toast-body-warning {
+  background: #FF8F00;
+  color: #FFF3E0 !important;
+}
+</style>

sist2-vue/src/views/StatsPage.vue

@@ -0,0 +1,79 @@
+<template>
+  <b-container>
+
+    <b-card v-if="loading">
+      <Preloader></Preloader>
+    </b-card>
+
+    <template v-else>
+      <b-card>
+        <b-card-body>
+          <b-select v-model="selectedIndex" :options="indexOptions">
+            <template #first>
+              <b-form-select-option :value="null" disabled>{{ $t("indexPickerPlaceholder") }}</b-form-select-option>
+            </template>
+          </b-select>
+        </b-card-body>
+      </b-card>
+
+      <b-card v-if="selectedIndex !== null" class="mt-3">
+        <b-card-body>
+          <D3Treemap :index-id="selectedIndex"></D3Treemap>
+
+
+        </b-card-body>
+      </b-card>
+
+      <b-card v-if="selectedIndex !== null" class="stats-card mt-3">
+        <D3MimeBarCount :index-id="selectedIndex"></D3MimeBarCount>
+        <D3MimeBarSize :index-id="selectedIndex"></D3MimeBarSize>
+        <D3DateHistogram :index-id="selectedIndex"></D3DateHistogram>
+        <D3SizeHistogram :index-id="selectedIndex"></D3SizeHistogram>
+      </b-card>
+    </template>
+  </b-container>
+</template>
+<script>
+import D3Treemap from "@/components/D3Treemap";
+import Sist2Api from "@/Sist2Api";
+import Preloader from "@/components/Preloader.vue";
+import D3MimeBarCount from "@/components/D3MimeBarCount";
+import D3MimeBarSize from "@/components/D3MimeBarSize";
+import D3DateHistogram from "@/components/D3DateHistogram";
+import D3SizeHistogram from "@/components/D3SizeHistogram";
+
+export default {
+  components: {D3SizeHistogram, D3DateHistogram, D3MimeBarSize, D3MimeBarCount, D3Treemap, Preloader},
+  data() {
+    return {
+      loading: true,
+      selectedIndex: null,
+      indices: []
+    }
+  },
+  computed: {
+    indexOptions() {
+      return this.indices.map(idx => {
+        return {
+          text: idx.name,
+          value: idx.id
+        };
+      })
+    }
+  },
+  mounted() {
+    Sist2Api.getSist2Info().then(data => {
+      this.indices = data.indices;
+      this.loading = false;
+    })
+  }
+}
+</script>
+
+<style>
+
+.stats-card {
+  text-align: center;
+  padding: 1em;
+}
+</style>

sist2-vue/tsconfig.json

@@ -0,0 +1,40 @@
+{
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "esnext",
+    "strict": false,
+    "jsx": "preserve",
+    "importHelpers": true,
+    "moduleResolution": "node",
+    "experimentalDecorators": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "sourceMap": false,
+    "baseUrl": ".",
+    "types": [
+      "webpack-env",
+    ],
+    "paths": {
+      "@/*": [
+        "src/*"
+      ]
+    },
+    "lib": [
+      "esnext",
+      "dom",
+      "dom.iterable",
+      "scripthost"
+    ]
+  },
+  "include": [
+    "src/**/*.ts",
+    "src/**/*.tsx",
+    "src/**/*.vue",
+    "tests/**/*.ts",
+    "tests/**/*.tsx"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}

sist2-vue/vue.config.js

@@ -0,0 +1,10 @@
+module.exports = {
+    filenameHashing: false,
+    productionSourceMap: false,
+    publicPath: "./",
+    pages: {
+        index: {
+            entry: "src/main.js"
+        }
+    }
+}

sist2-vue/watch.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+./node_modules/@vue/cli-service/bin/vue-cli-service.js build --watch

src/cli.c

@@ -4,16 +4,22 @@
 
 #define DEFAULT_OUTPUT "index.sist2/"
 #define DEFAULT_CONTENT_SIZE 32768
-#define DEFAULT_QUALITY 5
-#define DEFAULT_SIZE 500
+#define DEFAULT_QUALITY 1
+#define DEFAULT_SIZE 300
 #define DEFAULT_REWRITE_URL ""
 
 #define DEFAULT_ES_URL "http://localhost:9200"
+#define DEFAULT_ES_INDEX "sist2"
 #define DEFAULT_BATCH_SIZE 100
+#define DEFAULT_TAGLINE "Lightning-fast file system indexer and search tool"
+#define DEFAULT_LANG "en"
 
 #define DEFAULT_LISTEN_ADDRESS "localhost:4090"
+#define DEFAULT_TREEMAP_THRESHOLD 0.0005
 
-const char* TESS_DATAPATHS[] = {
+#define DEFAULT_MAX_MEM_BUFFER 2000
+
+const char *TESS_DATAPATHS[] = {
         "/usr/share/tessdata/",
         "/usr/share/tesseract-ocr/tessdata/",
         "./",
@@ -29,10 +35,18 @@ scan_args_t *scan_args_create() {
     return args;
 }
 
+exec_args_t *exec_args_create() {
+    exec_args_t *args = calloc(sizeof(exec_args_t), 1);
+    return args;
+}
+
 void scan_args_destroy(scan_args_t *args) {
     if (args->name != NULL) {
         free(args->name);
     }
+    if (args->incremental != NULL) {
+        free(args->incremental);
+    }
     if (args->path != NULL) {
         free(args->path);
     }
@@ -44,6 +58,12 @@ void scan_args_destroy(scan_args_t *args) {
 
 void index_args_destroy(index_args_t *args) {
     //todo
+    if (args->es_mappings_path) {
+        free(args->es_mappings);
+    }
+    if (args->es_settings_path) {
+        free(args->es_settings);
+    }
     free(args);
 }
 
@@ -52,6 +72,10 @@ void web_args_destroy(web_args_t *args) {
     free(args);
 }
 
+void exec_args_destroy(exec_args_t *args) {
+    free(args);
+}
+
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (argc < 2) {
         fprintf(stderr, "Required positional argument: PATH.\n");
@@ -67,9 +91,9 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     }
 
     if (args->incremental != NULL) {
-        abs_path = abspath(args->incremental);
+        args->incremental = abspath(args->incremental);
         if (abs_path == NULL) {
-            sist_log("main.c", SIST_WARNING, "Could not open original index! Disabled incremental scan feature.");
+            sist_log("main.c", LOG_SIST_WARNING, "Could not open original index! Disabled incremental scan feature.");
             args->incremental = NULL;
         }
     }
@@ -112,7 +136,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
-    if (args->depth < 0) {
+    if (args->depth <= 0) {
         args->depth = G_MAXINT32;
     } else {
         args->depth += 1;
@@ -120,6 +144,10 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     if (args->name == NULL) {
         args->name = g_path_get_basename(args->output);
+    } else {
+        char* tmp = malloc(strlen(args->name) + 1);
+        strcpy(tmp, args->name);
+        args->name = tmp;
     }
 
     if (args->rewrite_url == NULL) {
@@ -144,7 +172,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
         char filename[128];
         sprintf(filename, "%s.traineddata", args->tesseract_lang);
-        const char * path = find_file_in_paths(TESS_DATAPATHS, filename);
+        const char *path = find_file_in_paths(TESS_DATAPATHS, filename);
         if (path == NULL) {
             LOG_FATAL("cli.c", "Could not find tesseract language file!");
         }
@@ -180,6 +208,16 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         ScanCtx.exclude = NULL;
     }
 
+    if (args->treemap_threshold_str == 0) {
+        args->treemap_threshold = DEFAULT_TREEMAP_THRESHOLD;
+    } else {
+        args->treemap_threshold = atof(args->treemap_threshold_str);
+    }
+
+    if (args->max_memory_buffer == 0) {
+        args->max_memory_buffer = DEFAULT_MAX_MEM_BUFFER;
+    }
+
     LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
     LOG_DEBUGF("cli.c", "arg size=%d", args->size)
     LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
@@ -191,10 +229,42 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
     LOG_DEBUGF("cli.c", "arg path=%s", args->path)
     LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
+    LOG_DEBUGF("cli.c", "arg archive_passphrase=%s", args->archive_passphrase)
     LOG_DEBUGF("cli.c", "arg tesseract_lang=%s", args->tesseract_lang)
     LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path)
     LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex)
     LOG_DEBUGF("cli.c", "arg fast=%d", args->fast)
+    LOG_DEBUGF("cli.c", "arg fast_epub=%d", args->fast_epub)
+    LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold)
+    LOG_DEBUGF("cli.c", "arg max_memory_buffer=%d", args->max_memory_buffer)
+
+    return 0;
+}
+
+int load_external_file(const char *file_path, char **dst) {
+    struct stat info;
+    int res = stat(file_path, &info);
+
+    if (res == -1) {
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
+        return 1;
+    }
+
+    int fd = open(file_path, O_RDONLY);
+    if (fd == -1) {
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
+        return 1;
+    }
+
+    *dst = malloc(info.st_size + 1);
+    res = read(fd, *dst, info.st_size);
+    if (res < 0) {
+        LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno))
+        return 1;
+    }
+
+    *(*dst + info.st_size) = '\0';
+    close(fd);
 
     return 0;
 }
@@ -208,6 +278,13 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
+    if (args->threads == 0) {
+        args->threads = 1;
+    } else if (args->threads < 0) {
+        fprintf(stderr, "Invalid threads: %d\n", args->threads);
+        return 1;
+    }
+
     char *index_path = abspath(argv[1]);
     if (index_path == NULL) {
         fprintf(stderr, "File not found: %s\n", argv[1]);
@@ -221,30 +298,26 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
         args->es_url = DEFAULT_ES_URL;
     }
 
+    if (args->es_index == NULL) {
+        args->es_index = DEFAULT_ES_INDEX;
+    }
+
     if (args->script_path != NULL) {
-        struct stat info;
-        int res = stat(args->script_path, &info);
-
-        if (res == -1) {
-            fprintf(stderr, "Error opening script file '%s': %s\n", args->script_path, strerror(errno));
+        if (load_external_file(args->script_path, &args->script) != 0) {
             return 1;
         }
+    }
 
-        int fd = open(args->script_path, O_RDONLY);
-        if (fd == -1) {
-            fprintf(stderr, "Error opening script file '%s': %s\n", args->script_path, strerror(errno));
+    if (args->es_settings_path != NULL) {
+        if (load_external_file(args->es_settings_path, &args->es_settings) != 0) {
             return 1;
         }
+    }
 
-        args->script = malloc(info.st_size + 1);
-        res = read(fd, args->script, info.st_size);
-        if (res < 0) {
-            fprintf(stderr, "Error reading script file '%s': %s\n", args->script_path, strerror(errno));
+    if (args->es_mappings_path != NULL) {
+        if (load_external_file(args->es_mappings_path, &args->es_mappings) != 0) {
             return 1;
         }
-
-        *(args->script + info.st_size) = '\0';
-        close(fd);
     }
 
     if (args->batch_size == 0) {
@@ -252,10 +325,16 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
     }
 
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
+    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
     LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path)
     LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
+    LOG_DEBUGF("cli.c", "arg async_script=%s", args->async_script)
     LOG_DEBUGF("cli.c", "arg script=%s", args->script)
     LOG_DEBUGF("cli.c", "arg print=%d", args->print)
+    LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path)
+    LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings)
+    LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path)
+    LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings)
     LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size)
     LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset)
 
@@ -279,15 +358,32 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         args->listen_address = DEFAULT_LISTEN_ADDRESS;
     }
 
+    if (args->es_index == NULL) {
+        args->es_index = DEFAULT_ES_INDEX;
+    }
+
+    if (args->lang == NULL) {
+        args->lang = DEFAULT_LANG;
+    }
+
+    if (args->tagline == NULL) {
+        args->tagline = DEFAULT_TAGLINE;
+    }
+
+    if (strlen(args->lang) != 2) {
+        fprintf(stderr, "Invalid --lang value, see usage\n");
+        return 1;
+    }
+
     if (args->credentials != NULL) {
-        char * ptr = strstr(args->credentials, ":");
+        char *ptr = strstr(args->credentials, ":");
         if (ptr == NULL) {
             fprintf(stderr, "Invalid --auth format, see usage\n");
             return 1;
         }
 
         strncpy(args->auth_user, args->credentials, (ptr - args->credentials));
-        strncpy(args->auth_pass, ptr + 1, strlen(ptr + 1));
+        strcpy(args->auth_pass, ptr + 1);
 
         if (strlen(args->auth_user) == 0) {
             fprintf(stderr, "--auth username must be at least one character long");
@@ -299,6 +395,31 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         args->auth_enabled = FALSE;
     }
 
+    if (args->tag_credentials != NULL && args->credentials != NULL) {
+        fprintf(stderr, "--auth and --tag-auth are mutually exclusive");
+        return 1;
+    }
+
+    if (args->tag_credentials != NULL) {
+        char *ptr = strstr(args->tag_credentials, ":");
+        if (ptr == NULL) {
+            fprintf(stderr, "Invalid --tag-auth format, see usage\n");
+            return 1;
+        }
+
+        strncpy(args->auth_user, args->tag_credentials, (ptr - args->tag_credentials));
+        strcpy(args->auth_pass, ptr + 1);
+
+        if (strlen(args->auth_user) == 0) {
+            fprintf(stderr, "--tag-auth username must be at least one character long");
+            return 1;
+        }
+
+        args->tag_auth_enabled = TRUE;
+    } else {
+        args->tag_auth_enabled = FALSE;
+    }
+
     args->index_count = argc - 1;
     args->indices = argv + 1;
 
@@ -311,8 +432,12 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
     }
 
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
+    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
+    LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline)
+    LOG_DEBUGF("cli.c", "arg dev=%d", args->dev)
     LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address)
     LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials)
+    LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials)
     LOG_DEBUGF("cli.c", "arg auth_user=%s", args->auth_user)
     LOG_DEBUGF("cli.c", "arg auth_pass=%s", args->auth_pass)
     LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count)
@@ -333,3 +458,39 @@ web_args_t *web_args_create() {
     return args;
 }
 
+int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
+
+    if (argc < 2) {
+        fprintf(stderr, "Required positional argument: PATH.\n");
+        return 1;
+    }
+
+    char *index_path = abspath(argv[1]);
+    if (index_path == NULL) {
+        fprintf(stderr, "File not found: %s\n", argv[1]);
+        return 1;
+    } else {
+        args->index_path = argv[1];
+        free(index_path);
+    }
+
+    if (args->es_url == NULL) {
+        args->es_url = DEFAULT_ES_URL;
+    }
+
+    if (args->es_index == NULL) {
+        args->es_index = DEFAULT_ES_INDEX;
+    }
+
+    if (args->script_path == NULL) {
+        LOG_FATAL("cli.c", "--script-file argument is required");
+    }
+
+    if (load_external_file(args->script_path, &args->script) != 0) {
+        return 1;
+    }
+
+    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
+    LOG_DEBUGF("cli.c", "arg script=%s", args->script)
+    return 0;
+}

src/cli.h

@@ -18,10 +18,17 @@ typedef struct scan_args {
     char *path;
     char *archive;
     archive_mode_t archive_mode;
+    char *archive_passphrase;
     char *tesseract_lang;
     const char *tesseract_path;
     char *exclude_regex;
     int fast;
+    const char* treemap_threshold_str;
+    double treemap_threshold;
+    int max_memory_buffer;
+    int read_subtitles;
+    int fast_epub;
+    int calculate_checksums;
 } scan_args_t;
 
 scan_args_t *scan_args_create();
@@ -32,25 +39,47 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv);
 
 typedef struct index_args {
     char *es_url;
+    char *es_index;
     const char *index_path;
     const char *script_path;
     char *script;
+    const char *es_settings_path;
+    char *es_settings;
+    const char *es_mappings_path;
+    char *es_mappings;
     int print;
     int batch_size;
+    int async_script;
     int force_reset;
+    int threads;
 } index_args_t;
 
 typedef struct web_args {
     char *es_url;
+    char *es_index;
     char *listen_address;
     char *credentials;
+    char *tag_credentials;
+    char *tagline;
+    char *lang;
     char auth_user[256];
     char auth_pass[256];
     int auth_enabled;
+    int tag_auth_enabled;
     int index_count;
+    int dev;
     const char **indices;
 } web_args_t;
 
+typedef struct exec_args {
+    char *es_url;
+    char *es_index;
+    const char *index_path;
+    const char *script_path;
+    int async_script;
+    char *script;
+} exec_args_t;
+
 index_args_t *index_args_create();
 
 void index_args_destroy(index_args_t *args);
@@ -63,4 +92,10 @@ int index_args_validate(index_args_t *args, int argc, const char **argv);
 
 int web_args_validate(web_args_t *args, int argc, const char **argv);
 
+exec_args_t *exec_args_create();
+
+void exec_args_destroy(exec_args_t *args);
+
+int exec_args_validate(exec_args_t *args, int argc, const char **argv);
+
 #endif

src/ctx.c

@@ -0,0 +1,10 @@
+#include "ctx.h"
+
+ScanCtx_t ScanCtx = {
+        .stat_index_size = 0,
+        .dbg_current_files = NULL,
+        .pool = NULL
+};
+WebCtx_t WebCtx;
+IndexCtx_t IndexCtx;
+LogCtx_t LogCtx;

src/ctx.h

@@ -5,19 +5,23 @@
 #include "tpool.h"
 #include "libscan/scan.h"
 #include "libscan/arc/arc.h"
-#include "libscan/cbr/cbr.h"
+#include "libscan/comic/comic.h"
 #include "libscan/ebook/ebook.h"
 #include "libscan/font/font.h"
 #include "libscan/media/media.h"
 #include "libscan/ooxml/ooxml.h"
 #include "libscan/text/text.h"
 #include "libscan/mobi/scan_mobi.h"
+#include "libscan/raw/raw.h"
+#include "libscan/msdoc/msdoc.h"
+#include "libscan/wpd/wpd.h"
+#include "libscan/json/json.h"
+#include "src/io/store.h"
 
 #include <glib.h>
 #include <pcre.h>
 
-//TODO Move to individual scan ctx
-struct {
+typedef struct {
     struct index_t index;
 
     GHashTable *mime_table;
@@ -25,8 +29,11 @@ struct {
 
     tpool_t *pool;
 
+    tpool_t *writer_pool;
+
     int threads;
     int depth;
+    int calculate_checksums;
 
     size_t stat_tn_size;
     size_t stat_index_size;
@@ -38,35 +45,63 @@ struct {
     pcre_extra *exclude_extra;
     int fast;
 
+    GHashTable *dbg_current_files;
+    pthread_mutex_t dbg_current_files_mu;
+
+    int dbg_failed_files_count;
+    int dbg_skipped_files_count;
+    int dbg_excluded_files_count;
+    pthread_mutex_t dbg_file_counts_mu;
+
     scan_arc_ctx_t arc_ctx;
-    scan_cbr_ctx_t cbr_ctx;
+    scan_comic_ctx_t comic_ctx;
     scan_ebook_ctx_t ebook_ctx;
     scan_font_ctx_t font_ctx;
     scan_media_ctx_t media_ctx;
     scan_ooxml_ctx_t ooxml_ctx;
     scan_text_ctx_t text_ctx;
     scan_mobi_ctx_t mobi_ctx;
-} ScanCtx;
+    scan_raw_ctx_t raw_ctx;
+    scan_msdoc_ctx_t msdoc_ctx;
+    scan_wpd_ctx_t wpd_ctx;
+    scan_json_ctx_t json_ctx;
+} ScanCtx_t;
 
-struct {
+typedef struct {
     int verbose;
     int very_verbose;
     int no_color;
-} LogCtx;
+} LogCtx_t;
 
-struct {
+typedef struct {
     char *es_url;
+    char *es_index;
     int batch_size;
-} IndexCtx;
+    tpool_t *pool;
+    store_t *tag_store;
+    GHashTable *tags;
+    store_t *meta_store;
+    GHashTable *meta;
+} IndexCtx_t;
 
-struct {
+typedef struct {
     char *es_url;
+    char *es_index;
     int index_count;
     char *auth_user;
     char *auth_pass;
     int auth_enabled;
-    struct index_t indices[16];
-} WebCtx;
+    int tag_auth_enabled;
+    char *tagline;
+    struct index_t indices[256];
+    char lang[3];
+    int dev;
+} WebCtx_t;
+
+extern ScanCtx_t ScanCtx;
+extern WebCtx_t WebCtx;
+extern IndexCtx_t IndexCtx;
+extern LogCtx_t LogCtx;
 
 
 #endif

src/index/elastic.c

@@ -9,21 +9,33 @@
 typedef struct es_indexer {
     int queued;
     char *es_url;
+    char *es_index;
     es_bulk_line_t *line_head;
     es_bulk_line_t *line_tail;
 } es_indexer_t;
 
 
-static es_indexer_t *Indexer;
+static __thread es_indexer_t *Indexer;
 
 void delete_queue(int max);
 
-void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
+void elastic_flush();
+
+void elastic_cleanup() {
+    elastic_flush();
+    if (Indexer != NULL) {
+        free(Indexer->es_index);
+        free(Indexer->es_url);
+        free(Indexer);
+    }
+}
+
+void print_json(cJSON *document, const char id_str[MD5_STR_LENGTH]) {
 
     cJSON *line = cJSON_CreateObject();
 
-    cJSON_AddStringToObject(line, "_id", uuid_str);
-    cJSON_AddStringToObject(line, "_index", "sist2");
+    cJSON_AddStringToObject(line, "_id", id_str);
+    cJSON_AddStringToObject(line, "_index", IndexCtx.es_index);
     cJSON_AddStringToObject(line, "_type", "_doc");
     cJSON_AddItemReferenceToObject(line, "_source", document);
 
@@ -35,23 +47,31 @@ void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
     cJSON_Delete(line);
 }
 
-void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
+void index_json_func(void *arg) {
+    es_bulk_line_t *line = arg;
+    elastic_index_line(line);
+}
 
+void index_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]) {
     char *json = cJSON_PrintUnformatted(document);
 
     size_t json_len = strlen(json);
     es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t) + json_len + 2);
     memcpy(bulk_line->line, json, json_len);
-    memcpy(bulk_line->uuid_str, uuid_str, UUID_STR_LEN);
+    memcpy(bulk_line->path_md5_str, index_id_str, MD5_STR_LENGTH);
     *(bulk_line->line + json_len) = '\n';
     *(bulk_line->line + json_len + 1) = '\0';
     bulk_line->next = NULL;
 
     cJSON_free(json);
-    elastic_index_line(bulk_line);
+    tpool_add_work(IndexCtx.pool, index_json_func, bulk_line);
 }
 
-void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]) {
+void execute_update_script(const char *script, int async, const char index_id[MD5_STR_LENGTH]) {
+
+    if (Indexer == NULL) {
+        Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
+    }
 
     cJSON *body = cJSON_CreateObject();
     cJSON *script_obj = cJSON_AddObjectToObject(body, "script");
@@ -65,9 +85,16 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
     char *str = cJSON_Print(body);
 
     char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_update_by_query?pretty", Indexer->es_url);
+    if (async) {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query?wait_for_completion=false", Indexer->es_url,
+                 Indexer->es_index);
+    } else {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query", Indexer->es_url, Indexer->es_index);
+    }
     response_t *r = web_post(bulk_url, str);
-    LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
+    if (!async) {
+        LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
+    }
     cJSON *resp = cJSON_Parse(r->body);
 
     cJSON_free(str);
@@ -82,6 +109,11 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
         cJSON_free(error_str);
     }
 
+    if (async) {
+        cJSON *task = cJSON_GetObjectItem(resp, "task");
+        LOG_INFOF("elastic.c", "User script queued: %s/_tasks/%s", Indexer->es_url, task->valuestring);
+    }
+
     cJSON_Delete(resp);
 }
 
@@ -91,16 +123,25 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
 
     size_t buf_size = 0;
     size_t buf_cur = 0;
-    char *buf = malloc(1);
+    char *buf = malloc(8192);
+    size_t buf_capacity = 8192;
 
     while (line != NULL && *count < max) {
-        char action_str[512];
-        snprintf(action_str, 512,
-                 "{\"index\":{\"_id\":\"%s\", \"_type\":\"_doc\", \"_index\":\"sist2\"}}\n", line->uuid_str);
-        size_t action_str_len = strlen(action_str);
+        char action_str[256];
+        snprintf(
+                action_str, sizeof(action_str),
+                "{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n",
+                line->path_md5_str, Indexer->es_index
+        );
 
+        size_t action_str_len = strlen(action_str);
         size_t line_len = strlen(line->line);
-        buf = realloc(buf, buf_size + line_len + action_str_len);
+
+        while (buf_size + line_len + action_str_len > buf_capacity) {
+            buf_capacity *= 2;
+            buf = realloc(buf, buf_capacity);
+        }
+
         buf_size += line_len + action_str_len;
 
         memcpy(buf + buf_cur, action_str, action_str_len);
@@ -111,7 +152,11 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
         line = line->next;
         (*count)++;
     }
-    buf = realloc(buf, buf_size + 1);
+
+    if (buf_size + 1 > buf_capacity) {
+        buf = realloc(buf, buf_capacity + 1);
+    }
+
     *(buf + buf_cur) = '\0';
 
     *buf_len = buf_cur;
@@ -119,7 +164,7 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
 }
 
 void print_errors(response_t *r) {
-    char * tmp = malloc(r->size + 1);
+    char *tmp = malloc(r->size + 1);
     memcpy(tmp, r->body, r->size);
     *(tmp + r->size) = '\0';
 
@@ -138,6 +183,21 @@ void print_errors(response_t *r) {
     free(tmp);
 }
 
+void print_error(response_t *r) {
+    char *tmp = malloc(r->size + 1);
+    memcpy(tmp, r->body, r->size);
+    *(tmp + r->size) = '\0';
+
+    cJSON *ret_json = cJSON_Parse(tmp);
+    if (cJSON_GetObjectItem(ret_json, "error") != NULL) {
+        char *str = cJSON_Print(cJSON_GetObjectItem(ret_json, "error"));
+        LOG_ERRORF("elastic.c", "%s\n", str);
+        cJSON_free(str);
+    }
+    cJSON_Delete(ret_json);
+    free(tmp);
+}
+
 void _elastic_flush(int max) {
 
     if (max == 0) {
@@ -150,7 +210,7 @@ void _elastic_flush(int max) {
     void *buf = create_bulk_buffer(max, &count, &buf_len);
 
     char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_bulk?pipeline=tie", Indexer->es_url);
+    snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_bulk?pipeline=tie", Indexer->es_url, Indexer->es_index);
     response_t *r = web_post(bulk_url, buf);
 
     if (r->status_code == 0) {
@@ -160,7 +220,7 @@ void _elastic_flush(int max) {
     if (r->status_code == 413) {
 
         if (max <= 1) {
-            LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->uuid_str)
+            LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->path_md5_str)
             free_response(r);
             free(buf);
             delete_queue(1);
@@ -177,6 +237,15 @@ void _elastic_flush(int max) {
         _elastic_flush(max / 2);
         return;
 
+    } else if (r->status_code == 429) {
+
+        free_response(r);
+        free(buf);
+        LOG_WARNING("elastic.c", "Got 429 status, will retry after delay")
+        usleep(1000000 * 20);
+        _elastic_flush(max);
+        return;
+
     } else if (r->status_code != 200) {
         print_errors(r);
         delete_queue(Indexer->queued);
@@ -202,9 +271,8 @@ void delete_queue(int max) {
         Indexer->line_head = tmp->next;
         if (Indexer->line_head == NULL) {
             Indexer->line_tail = NULL;
-        } else {
-            free(tmp);
         }
+        free(tmp);
         Indexer->queued -= 1;
     }
 }
@@ -212,7 +280,7 @@ void delete_queue(int max) {
 void elastic_flush() {
 
     if (Indexer == NULL) {
-        Indexer = create_indexer(IndexCtx.es_url);
+        Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
     }
 
     _elastic_flush(Indexer->queued);
@@ -221,7 +289,7 @@ void elastic_flush() {
 void elastic_index_line(es_bulk_line_t *line) {
 
     if (Indexer == NULL) {
-        Indexer = create_indexer(IndexCtx.es_url);
+        Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
     }
 
     if (Indexer->line_head == NULL) {
@@ -239,14 +307,18 @@ void elastic_index_line(es_bulk_line_t *line) {
     }
 }
 
-es_indexer_t *create_indexer(const char *url) {
+es_indexer_t *create_indexer(const char *url, const char *index) {
 
     char *es_url = malloc(strlen(url) + 1);
     strcpy(es_url, url);
 
+    char *es_index = malloc(strlen(index) + 1);
+    strcpy(es_index, index);
+
     es_indexer_t *indexer = malloc(sizeof(es_indexer_t));
 
     indexer->es_url = es_url;
+    indexer->es_index = es_index;
     indexer->queued = 0;
     indexer->line_head = NULL;
     indexer->line_tail = NULL;
@@ -254,41 +326,42 @@ es_indexer_t *create_indexer(const char *url) {
     return indexer;
 }
 
-void destroy_indexer(char *script, char index_id[UUID_STR_LEN]) {
+void finish_indexer(char *script, int async_script, char *index_id) {
 
     char url[4096];
 
-    snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
+    snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index);
     response_t *r = web_post(url, "");
     LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
     free_response(r);
 
     if (script != NULL) {
-        execute_update_script(script, index_id);
+        execute_update_script(script, async_script, index_id);
+        free(script);
+
+        snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index);
+        r = web_post(url, "");
+        LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
+        free_response(r);
     }
 
-    snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
-    r = web_post(url, "");
-    LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
-    free_response(r);
-
-    snprintf(url, sizeof(url), "%s/sist2/_forcemerge", IndexCtx.es_url);
+    snprintf(url, sizeof(url), "%s/%s/_forcemerge", IndexCtx.es_url, IndexCtx.es_index);
     r = web_post(url, "");
     LOG_INFOF("elastic.c", "Merge index <%d>", r->status_code);
     free_response(r);
 
-    if (Indexer != NULL) {
-        free(Indexer->es_url);
-        free(Indexer);
-    }
+    snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
+    r = web_put(url, "{\"index\":{\"refresh_interval\":\"1s\"}}");
+    LOG_INFOF("elastic.c", "Set refresh interval <%d>", r->status_code);
+    free_response(r);
 }
 
-void elastic_init(int force_reset) {
+void elastic_init(int force_reset, const char* user_mappings, const char* user_settings) {
 
     // Check if index exists
     char url[4096];
-    snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
-    response_t *r = web_get(url);
+    snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index);
+    response_t *r = web_get(url, 30);
     int index_exists = r->status_code == 200;
     free_response(r);
 
@@ -297,46 +370,64 @@ void elastic_init(int force_reset) {
         LOG_INFOF("elastic.c", "Delete index <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index);
         r = web_put(url, "");
+
+        if (r->status_code != 200) {
+            print_error(r);
+            LOG_FATAL("elastic.c", "Could not create index")
+        }
+
         LOG_INFOF("elastic.c", "Create index <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2/_close", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_close", IndexCtx.es_url, IndexCtx.es_index);
         r = web_post(url, "");
         LOG_INFOF("elastic.c", "Close index <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/_ingest/pipeline/tie", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/_ingest/pipeline/tie", IndexCtx.es_url);
         r = web_put(url, pipeline_json);
         LOG_INFOF("elastic.c", "Create pipeline <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2/_settings", IndexCtx.es_url);
-        r = web_put(url, settings_json);
-        LOG_INFOF("elastic.c", "Update settings <%d>", r->status_code);
+        snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
+        r = web_put(url, user_settings ? user_settings : settings_json);
+        LOG_INFOF("elastic.c", "Update ES settings <%d>", r->status_code);
+        if (r->status_code != 200) {
+            print_error(r);
+            LOG_FATAL("elastic.c", "Could not update user settings")
+        }
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2/_mappings/_doc?include_type_name=true", IndexCtx.es_url);
-        r = web_put(url, mappings_json);
-        LOG_INFOF("elastic.c", "Update mappings <%d>", r->status_code);
+        snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index);
+        r = web_put(url, user_mappings ? user_mappings : mappings_json);
+        LOG_INFOF("elastic.c", "Update ES mappings <%d>", r->status_code);
+        if (r->status_code != 200) {
+            print_error(r);
+            LOG_FATAL("elastic.c", "Could not update user mappings")
+        }
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2/_open", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_open", IndexCtx.es_url, IndexCtx.es_index);
         r = web_post(url, "");
         LOG_INFOF("elastic.c", "Open index <%d>", r->status_code);
         free_response(r);
     }
 }
 
-cJSON *elastic_get_document(const char *uuid_str) {
+cJSON *elastic_get_document(const char *id_str) {
     char url[4096];
-    snprintf(url, 4096, "%s/sist2/_doc/%s", WebCtx.es_url, uuid_str);
+    snprintf(url, sizeof(url), "%s/%s/_doc/%s", WebCtx.es_url, WebCtx.es_index, id_str);
 
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 3);
     cJSON *json = NULL;
     if (r->status_code == 200) {
-        json = cJSON_Parse(r->body);
+        char *tmp = malloc(r->size + 1);
+        memcpy(tmp, r->body, r->size);
+        *(tmp + r->size) = '\0';
+        json = cJSON_Parse(tmp);
+        free(tmp);
     }
     free_response(r);
     return json;
@@ -344,21 +435,25 @@ cJSON *elastic_get_document(const char *uuid_str) {
 
 char *elastic_get_status() {
     char url[4096];
-    snprintf(url, 4096,
-             "%s/_cluster/state/metadata/sist2?filter_path=metadata.indices.*.state", WebCtx.es_url);
+    snprintf(url, sizeof(url),
+             "%s/_cluster/state/metadata/%s?filter_path=metadata.indices.*.state", WebCtx.es_url, WebCtx.es_index);
 
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
     cJSON *json = NULL;
     char *status = malloc(128 * sizeof(char));
     status[0] = '\0';
 
     if (r->status_code == 200) {
-        json = cJSON_Parse(r->body);
+        char *tmp = malloc(r->size + 1);
+        memcpy(tmp, r->body, r->size);
+        *(tmp + r->size) = '\0';
+        json = cJSON_Parse(tmp);
+        free(tmp);
         const cJSON *metadata = cJSON_GetObjectItem(json, "metadata");
         if (metadata != NULL) {
             const cJSON *indices = cJSON_GetObjectItem(metadata, "indices");
-            const cJSON *sist2 = cJSON_GetObjectItem(indices, "sist2");
-            const cJSON *state = cJSON_GetObjectItem(sist2, "state");
+            const cJSON *index = cJSON_GetObjectItem(indices, WebCtx.es_index);
+            const cJSON *state = cJSON_GetObjectItem(index, "state");
             strcpy(status, state->valuestring);
         }
     }

src/index/elastic.h

@@ -5,7 +5,7 @@
 
 typedef struct es_bulk_line {
     struct es_bulk_line *next;
-    char uuid_str[UUID_STR_LEN];
+    char path_md5_str[MD5_STR_LENGTH];
     char line[0];
 } es_bulk_line_t;
 
@@ -16,20 +16,21 @@ typedef struct es_indexer es_indexer_t;
 
 void elastic_index_line(es_bulk_line_t *line);
 
-void elastic_flush();
+void print_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]);
 
-void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);
+void index_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]);
 
-void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);
+es_indexer_t *create_indexer(const char *url, const char *index);
 
-es_indexer_t *create_indexer(const char* es_url);
+void elastic_cleanup();
+void finish_indexer(char *script, int async_script, char *index_id);
 
-void destroy_indexer(char *script, char index_id[UUID_STR_LEN]);
+void elastic_init(int force_reset, const char* user_mappings, const char* user_settings);
 
-void elastic_init(int force_reset);
-
-cJSON *elastic_get_document(const char *uuid_str);
+cJSON *elastic_get_document(const char *id_str);
 
 char *elastic_get_status();
 
+void execute_update_script(const char *script, int async, const char index_id[MD5_STR_LENGTH]);
+
 #endif