Fix #81

Update USAGE.md

.github/ISSUE_TEMPLATE/issue-template.md

@@ -9,7 +9,9 @@ assignees: ''
 
 sist2 version:
 
-Platform (please indicate if you're using Docker):
+Platform (Linux or Docker):
+
+Elasticsearch version:
 
 Command with arguments:  `ex: "scan ~/Documents -o ./i2 --threads 3 -q 1.0`
 

CMakeLists.txt

@@ -30,7 +30,7 @@ add_executable(
         third-party/argparse/argparse.h third-party/argparse/argparse.c
 
         src/cli.c src/cli.h
-)
+        src/stats.c src/stats.h src/ctx.c)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
 set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
@@ -40,6 +40,7 @@ find_package(cJSON CONFIG REQUIRED)
 find_package(unofficial-glib CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
 find_library(UUID_LIB NAMES uuid)
+find_package(CURL CONFIG REQUIRED)
 
 #find_package(OpenSSL REQUIRED)
 
@@ -56,7 +57,6 @@ target_compile_options(
         sist2
         PRIVATE
         -fPIC
-        -Werror
 )
 
 if (SIST_DEBUG)
@@ -67,6 +67,7 @@ if (SIST_DEBUG)
             -fstack-protector
             -fno-omit-frame-pointer
             -fsanitize=address
+            -O2
     )
     target_link_options(
             sist2
@@ -80,6 +81,7 @@ if (SIST_DEBUG)
             OUTPUT_NAME sist2_debug
     )
 else ()
+#    set(VCPKG_BUILD_TYPE release)
     target_compile_options(
             sist2
             PRIVATE
@@ -105,6 +107,7 @@ target_link_libraries(
         unofficial::glib::glib
         unofficial::mongoose::mongoose
 #        OpenSSL::SSL OpenSSL::Crypto
+        CURL::libcurl
 
         ${UUID_LIB}
         pthread

README.md

@@ -15,18 +15,22 @@ sist2 (Simple incremental search tool)
 * Fast, low memory usage, multi-threaded
 * Mobile-friendly Web interface
 * Portable (all its features are packaged in a single executable)
-* Extracts text from common file types \*
+* Extracts text and metadata from common file types \*
 * Generates thumbnails \*
 * Incremental scanning
-* Automatic tagging from file attributes via [user scripts](scripting/README.md)
+* Manual tagging from the UI and automatic tagging based on file attributes via [user scripts](docs/scripting.md)
 * Recursive scan inside archive files \*\*
 * OCR support with tesseract \*\*\*
+* Stats page & disk utilisation visualization
 
 
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
 \*\*\* See [OCR](#ocr)    
 
+![stats](docs/stats.png)
+
+
 ## Getting Started
 
 1. Have an Elasticsearch (>= 6.X.X) instance running
@@ -49,7 +53,7 @@ sist2 (Simple incremental search tool)
     1. *(or)* Download a [development snapshot](https://files.simon987.net/artifacts/Sist2/Build/) *(Not recommended!)*
     1. *(or)* `docker pull simon987/sist2:latest`
 
-1. See [Usage guide](DOCS/USAGE.md)
+1. See [Usage guide](docs/USAGE.md)
    
 
 \* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)    
@@ -57,7 +61,7 @@ sist2 (Simple incremental search tool)
 
 ## Example usage
 
-See [Usage guide](DOCS/USAGE.md) for more details
+See [Usage guide](docs/USAGE.md) for more details
 
 1. Scan a directory: `sist2 scan ~/Documents -o ./docs_idx`
 1. Push index to Elasticsearch: `sist2 index ./docs_idx`
@@ -68,14 +72,17 @@ See [Usage guide](DOCS/USAGE.md) for more details
 
 File type | Library | Content | Thumbnail | Metadata
 :---|:---|:---|:---|:---
-pdf,xps,cbz,cbr,fb2,epub | MuPDF | text+ocr | yes, `png` | title |
-`audio/*` | ffmpeg | - | yes, `jpeg` | ID3 tags |
-`video/*` | ffmpeg | - | yes, `jpeg` | title, comment, artist |
-`image/*` | ffmpeg | - | yes, `jpeg` | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
+pdf,xps,fb2,epub | MuPDF | text+ocr | yes | title |
+cbz,cbr | *(none)* | - | yes | - |
+`audio/*` | ffmpeg | - | yes | ID3 tags |
+`video/*` | ffmpeg | - | yes | title, comment, artist |
+`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
+raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags |
 ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
 `text/plain` | *(none)* | yes | no | - |
+html, xml | *(none)* | yes | no | - |
 tar, zip, rar, 7z, ar ...  | Libarchive | yes\* | - | no |
-docx, xlsx, pptx | *(none)* | yes | no | creator, modified_by, title |
+docx, xlsx, pptx | *(none)* | yes | if embedded | creator, modified_by, title |
 mobi, azw, azw3 | libmobi | yes | no | author, title |
 
 \* *See [Archive files](#archive-files)*
@@ -86,18 +93,16 @@ they were directly in the file system. Recursive (archives inside archives)
 scan is also supported.
 
 **Limitations**:
-* Parsing media files with formats that require
-*seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.) is not supported.
+* Support for parsing media files with formats that require *seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.) 
+  is limitted (see `--mem-buffer` option)
 * Archive files are scanned sequentially, by a single thread. On systems where
 **sist2** is not I/O bound, scans might be faster when larger archives are split
  into smaller parts.
-
-To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
  
  
 ### OCR
 
-You can enable OCR support for pdf,xps,cbz,cbr,fb2,epub file types with the
+You can enable OCR support for pdf,xps,fb2,epub file types with the
 `--ocr <lang>` option. Download the language data files with your
 package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
@@ -119,12 +124,12 @@ binaries (GCC 7+ required).
 1. Install compile-time dependencies
 
    ```bash
-   vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libuuid libmagic
+   vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libuuid libmagic libraw
    ```
 
 2. Build
     ```bash
     git clone --recursive https://github.com/simon987/sist2/
-    cmake -D <VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
+    cmake -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

ci/build.sh

@@ -1,16 +1,17 @@
 #!/usr/bin/env bash
 
+VCPKG_ROOT="/vcpkg"
 
 rm *.gz
 
 rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
-make
+cmake -DSIST_DEBUG=off -DVCPKG_BUILD_TYPE=release -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j 12
 strip sist2
 gzip -9 sist2
 
 rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_DEBUG=on -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
-make
+cmake -DSIST_DEBUG=on -DVCPKG_BUILD_TYPE=debug -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j 12
 cp /usr/lib/x86_64-linux-gnu/libasan.so.2.0.0 libasan.so.2
 tar -czf sist2_debug.tar.gz sist2_debug libasan.so.2

docs/USAGE.md

@@ -14,47 +14,57 @@
     * [examples](#web-examples)
     * [rewrite_url](#rewrite_url)
     * [link to specific indices](#link-to-specific-indices)
+* [exec-script](#exec-script)
+* [tagging](#tagging)
 
 ```
 Usage: sist2 scan [OPTION]... PATH
    or: sist2 index [OPTION]... INDEX
    or: sist2 web [OPTION]... INDEX...
+   or: sist2 exec-script [OPTION]... INDEX
 Lightning-fast file system indexer and search tool.
 
-    -h, --help            show this help message and exit
-    -v, --version         Show version and exit
-    --verbose             Turn on logging
-    --very-verbose        Turn on debug messages
+    -h, --help                    show this help message and exit
+    -v, --version                 Show version and exit
+    --verbose                     Turn on logging
+    --very-verbose                Turn on debug messages
 
 Scan options
-    -t, --threads=<int>   Number of threads. DEFAULT=1
-    -q, --quality=<flt>   Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=5
-    --size=<int>          Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
-    --content-size=<int>  Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
-    --incremental=<str>   Reuse an existing index and only scan modified files.
-    -o, --output=<str>    Output directory. DEFAULT=index.sist2/
-    --rewrite-url=<str>   Serve files from this url instead of from disk.
-    --name=<str>          Index display name. DEFAULT: (name of the directory)
-    --depth=<int>         Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --archive=<str>       Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
-    --ocr=<str>           Tesseract language (use tesseract --list-langs to see which are installed on your machine)
-    -e, --exclude=<str>   Files that match this regex will not be scanned
-    --fast                Only index file names & mime type
+    -t, --threads=<int>           Number of threads. DEFAULT=1
+    -q, --quality=<flt>           Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=5
+    --size=<int>                  Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
+    --content-size=<int>          Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
+    --incremental=<str>           Reuse an existing index and only scan modified files.
+    -o, --output=<str>            Output directory. DEFAULT=index.sist2/
+    --rewrite-url=<str>           Serve files from this url instead of from disk.
+    --name=<str>                  Index display name. DEFAULT: (name of the directory)
+    --depth=<int>                 Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
+    --archive=<str>               Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
+    --ocr=<str>                   Tesseract language (use tesseract --list-langs to see which are installed on your machine)
+    -e, --exclude=<str>           Files that match this regex will not be scanned
+    --fast                        Only index file names & mime type
+    --treemap-threshold=<str>     Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
+    --mem-buffer=<int>            Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000
 
 Index options
-    --es-url=<str>        Elasticsearch url with port. DEFAULT=http://localhost:9200
-    -p, --print           Just print JSON documents to stdout.
-    --script-file=<str>   Path to user script.
-    --batch-size=<int>    Index batch size. DEFAULT: 100
-    -f, --force-reset     Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
+    -t, --threads=<int>           Number of threads. DEFAULT=1
+    --es-url=<str>                Elasticsearch url with port. DEFAULT=http://localhost:9200
+    -p, --print                   Just print JSON documents to stdout.
+    --script-file=<str>           Path to user script.
+    --async-script                Execute user script asynchronously.
+    --batch-size=<int>            Index batch size. DEFAULT: 100
+    -f, --force-reset             Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
 
 Web options
-    --es-url=<str>        Elasticsearch url. DEFAULT=http://localhost:9200
-    --bind=<str>          Listen on this address. DEFAULT=localhost
-    --port=<str>          Listen on this port. DEFAULT=4090
-    --auth=<str>          Basic auth in user:password format
-Made by simon987 <me@simon987.net>. Released under GPL-3.0
+    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --bind=<str>                  Listen on this address. DEFAULT=localhost:4090
+    --auth=<str>                  Basic auth in user:password format
+    --tag-auth=<str>              Basic auth in user:password format for tagging
 
+Exec-script options
+    --script-file=<str>           Path to user script.
+    --async-script                Execute user script asynchronously.
+Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
 ## Scan
@@ -62,7 +72,7 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ### Scan options
 
 * `-t, --threads` 
-      Number of threads for file parsing. **Do not set a number higher than `$(nproc)`!**.
+      Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-WmiObject Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
 * `-q, --quality` 
     Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. *Does not affect PDF thumbnails quality*
 * `--size` 
@@ -94,6 +104,18 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
     * `-e "(^/usr/)|(^/var/)|(^/media/DRIVE-A/tmp/)|(^/media/DRIVE-B/Trash/)"` Exclude the
      `/usr`, `/var`, `/media/DRIVE-A/tmp`, `/media/DRIVE-B/Trash` directories
 * `--fast` Only index file names and mime type
+* `--treemap-threshold` Directories smaller than (`treemap-threshold` * `<total size of the index>`)
+    will not be considered for the disk utilisation visualization; their size will be added to
+    the parent directory. If the parent directory is still smaller than the threshold, it will also be "merged upwards"
+    and so on.
+    
+    In effect, smaller `treemap-threshold` values will yield a more detailed 
+    (but also a more cluttered and harder to read) visualization. 
+    
+* `--mem-buffer` Maximum memory buffer size in MB (per thread) for files inside archives. Media files 
+    larger than this number will be read sequentially and no *seek* operations will be supported.
+
+    To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
 
 ### Scan examples
 
@@ -122,7 +144,15 @@ documents.idx/
 ├── _index_139965425223424
 ├── _index_139965433616128
 ├── _index_139965442008832
-└── thumbs
+├── _index_139965442008832
+├── treemap.csv
+├── agg_mime.csv
+├── agg_date.csv
+├── add_size.csv
+├── thumbs
+|   ├── data.mdb
+|   └── lock.mdb
+└── tags
     ├── data.mdb
     └── lock.mdb
 ```
@@ -137,6 +167,8 @@ database containing the thumbnails.
 The `descriptor.json` file contains general information about the index. The 
 following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rewrite_url) and `timestamp`.
 
+The `.csv` are pre-computed aggregations necessary for the stats page.
+
 
 *Advanced usage*
 
@@ -212,7 +244,10 @@ it is currently unsupported and has no guaranties of back/forward compatibility.
  * `-p, --print` 
     Print index in JSON format to stdout.
  * `--script-file` 
-    Path to user script. See [Scripting](scripting/README.md).
+    Path to user script. See [Scripting](scripting.md).
+ * `--async-script` 
+    Use `wait_for_completion=false` elasticsearch option while executing user script.
+     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
  * `--batch-size=<int>` 
     Index batch size. Indexing is generally faster with larger batches, but payloads that
     are too large will fail and additional overhead for retrying with smaller sizes may slow
@@ -244,14 +279,15 @@ sist2 index --print ./my_index/ | jq | less
 ### Web options
  * `--es-url=<str>` Elasticsearch url.
  * `--bind=<str>` Listen on this address.
- * `--port=<str>` Listen on this port.
  * `--auth=<str>` Basic auth in user:password format
+ * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
+    `--auth` argument, but authentication is only applied the `/tag/` endpoint.
  
 ### Web examples
 
 **Single index**
 ```bash
-sist2 web --auth admin:hunter2 --bind 0.0.0.0 --port 8888 my_index
+sist2 web --auth admin:hunter2 --bind 0.0.0.0:8888 my_index
 ```
 
 **Multiple indices**
@@ -273,3 +309,35 @@ Both the `root` and `rewrite_url` fields are safe to manually modify from the
 To link to specific indices, you can add a list of comma-separated index name to 
 the URL: `?i=<name>,<name>`. By default, indices with `"(nsfw)"` in their name are
 not displayed.
+
+## exec-script
+
+The `exec-script` command is used to execute a user script for an index that has already been imported to Elasticsearch with the `index` command. Note that the documents will not be reset to their default state before each execution as the `index` command does: if you make undesired changes to the documents by accident, you will need to run `index` again to revert to the original state.
+
+
+# Tagging
+
+### Manual tagging
+
+You can modify tags of individual documents directly from the 
+ `web` interface. Note that you can setup authentication for this feature
+ with the `--tag-auth` option (See [web options](#web-options))
+
+![manual_tag](manual_tag.png)
+
+Tags that are manually added are saved both in the 
+ index folder (in `/tags/`) and in Elasticsearch*. When re-`index`ing, 
+ they are read from the index and automatically applied.
+ 
+You can safely copy the `/tags/` database to another index.
+
+See [Automatic tagging](#automatic-tagging) for information about tag 
+ hierarchies and tag colors.
+
+\* *It can take a few seconds to take effect in new search queries, and the page needs 
+    to be reloaded for the tag tab to update*
+
+
+### Automatic tagging
+
+See [scripting](docs/scripting.md) documentation.

docs/scripting.md

@@ -39,7 +39,7 @@ it adds the `genre.<genre>` tag.
 ArrayList tags = ctx._source.tag = new ArrayList();
 
 if (ctx._source?.genre != null) {
-    tags.add("genre." + ctx._source.genre.toLowerCase())
+    tags.add("genre." + ctx._source.genre.toLowerCase());
 }
 ```
 
@@ -67,7 +67,7 @@ ArrayList tags = ctx._source.tag = new ArrayList();
 
 Matcher m = /[\(\.+](20[0-9]{2})[\)\.+]/.matcher(ctx._source.name);
 if (m.find()) {
-    tags.add("year." + m.group(1))
+    tags.add("year." + m.group(1));
 }
 ```
 
@@ -111,16 +111,6 @@ if (ctx._source.path != "") {
 }
 ```
 
-Set the name of the last folder (`/path/to/<studio>/file.mp4`) to `studio.<studio>` tag
-```Java
-ArrayList tags = ctx._source.tag = new ArrayList();
-
-if (ctx._source.path != "") {
-    String[] names = ctx._source.path.splitOnToken('/');
-    tags.add("studio." + names[names.length-1]);
-}
-```
-
 Parse `EXIF:F Number` tag
 ```Java
 if (ctx._source?.exif_fnumber != null) {

schema/mappings.json

@@ -10,8 +10,22 @@
     "path": {
       "type": "text",
       "analyzer": "path_analyzer",
+      "copy_to": "suggest-path",
       "fielddata": true,
-      "index_prefixes": {}
+      "fields": {
+        "nGram": {
+          "type": "text",
+          "analyzer": "my_nGram"
+        },
+        "text": {
+          "type": "text",
+          "analyzer": "content_analyzer"
+        }
+      }
+    },
+    "suggest-path": {
+      "type": "completion",
+      "analyzer": "case_insensitive_kw_analyzer"
     },
     "mime": {
       "type": "keyword"
@@ -112,7 +126,12 @@
       }
     },
     "tag": {
-      "type": "keyword"
+      "type": "keyword",
+      "copy_to": "suggest-tag"
+    },
+    "suggest-tag": {
+      "type": "completion",
+      "analyzer": "case_insensitive_kw_analyzer"
     },
     "exif_make": {
       "type": "text"

schema/settings.json

@@ -1,7 +1,8 @@
 {
   "index": {
     "refresh_interval": "30s",
-    "codec": "best_compression"
+    "codec": "best_compression",
+    "number_of_replicas": 0
   },
   "analysis": {
     "tokenizer": {

scripts/before_build.sh

@@ -4,7 +4,7 @@ rm -rf index.sist2/
 
 rm src/static/js/bundle.js 2> /dev/null
 cat `ls src/static/js/*.min.js` > src/static/js/bundle.js
-cat src/static/js/{util,dom,search}.js >> src/static/js/bundle.js
+cat src/static/js/{util,dom}.js >> src/static/js/bundle.js
 
 rm src/static/css/bundle*.css 2> /dev/null
 cat src/static/css/*.min.css > src/static/css/bundle.css

scripts/mime.csv

@@ -13,7 +13,7 @@ application/epub+zip, epub
 application/freeloader, frl
 application/futuresplash, spl
 application/groupwise, vew
-application/gzip, gz
+application/gzip, gz|tgz
 application/hta, hta
 application/i-deas, unv
 application/iges, iges|igs
@@ -111,7 +111,7 @@ application/x-dbf, dbf
 application/x-dbt,
 application/x-debian-package, deb
 application/x-deepv, deepv
-application/x-director, dcr|dir|dxr
+application/x-director, dir|dxr
 application/x-dmp, dmp
 application/x-dosdriver,
 application/x-dosexec, dll
@@ -347,7 +347,8 @@ text/javascript, js
 text/mcf, mcf
 text/pascal, pas
 text/PGP,
-text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml
+text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml|make|log|markdown|yaml
+application/vnd.coffeescript, coffee
 text/richtext, rt|rtf|rtx
 text/rtf,
 text/scriplet, wsc
@@ -429,3 +430,22 @@ video/x-qtc, qtc
 video/x-sgi-movie, movie|mv
 x-epoc/x-sisx-app,
 application/x-zstd-dictionary,
+application/vnd.ms-outlook, msg
+image/x-olympus-orf, orf
+image/x-nikon-nef, nef
+image/x-fuji-raf, raf
+image/x-panasonic-raw, rw2|raw
+image/x-adobe-dng, dng
+image/x-canon-cr2, cr2
+image/x-canon-crw, crw
+image/x-dcraw,
+image/x-kodak-dcr, dcr
+image/x-kodak-k25, k25
+image/x-kodak-kdc, kdc
+image/x-minolta-mrw, mrw
+image/x-pentax-pef, pef
+image/x-sigma-x3f, xf3
+image/x-sony-arw, arw
+image/x-sony-sr2, sr2
+image/x-sony-srf, srf
+image/x-epson-erf, erf

scripts/mime.py

@@ -18,7 +18,6 @@ major_mime = {
 
 pdf = (
     "application/pdf",
-    "application/x-cbz",
     "application/epub+zip",
     "application/vnd.ms-xpsdocument",
 )
@@ -73,6 +72,29 @@ markup = (
     "text/x-sgml"
 )
 
+raw = (
+    "image/x-olympus-orf",
+    "image/x-nikon-nef",
+    "image/x-fuji-raf",
+    "image/x-panasonic-raw",
+    "image/x-adobe-dng",
+    "image/x-canon-cr2",
+    "image/x-canon-crw",
+    "image/x-dcraw",
+    "image/x-kodak-dcr",
+    "image/x-kodak-k25",
+    "image/x-kodak-kdc",
+    "image/x-minolta-mrw",
+    "image/x-pentax-pef",
+    "image/x-sigma-x3f",
+    "image/x-sony-arw",
+    "image/x-sony-sr2",
+    "image/x-sony-srf",
+    "image/x-minolta-mrw",
+    "image/x-pentax-pef",
+    "image/x-epson-erf",
+)
+
 cnt = 1
 
 
@@ -97,6 +119,8 @@ def mime_id(mime):
         mime_id += " | 0x02000000"
     elif mime in markup:
         mime_id += " | 0x01000000"
+    elif mime in raw:
+        mime_id += " | 0x00800000"
     elif mime == "application/x-empty":
         return "1"
     return mime_id

scripts/serve_static.py

@@ -2,9 +2,11 @@ files = [
     "src/static/css/bundle.css",
     "src/static/css/bundle_dark.css",
     "src/static/js/bundle.js",
+    "src/static/js/search.js",
     "src/static/img/sprite-skin-flat.png",
     "src/static/img/sprite-skin-flat-dark.png",
     "src/static/search.html",
+    "src/static/stats.html",
 ]
 
 

src/cli.c

@@ -12,8 +12,11 @@
 #define DEFAULT_BATCH_SIZE 100
 
 #define DEFAULT_LISTEN_ADDRESS "localhost:4090"
+#define DEFAULT_TREEMAP_THRESHOLD 0.0005
 
-const char* TESS_DATAPATHS[] = {
+#define DEFAULT_MAX_MEM_BUFFER 2000
+
+const char *TESS_DATAPATHS[] = {
         "/usr/share/tessdata/",
         "/usr/share/tesseract-ocr/tessdata/",
         "./",
@@ -29,10 +32,18 @@ scan_args_t *scan_args_create() {
     return args;
 }
 
+exec_args_t *exec_args_create() {
+    exec_args_t *args = calloc(sizeof(exec_args_t), 1);
+    return args;
+}
+
 void scan_args_destroy(scan_args_t *args) {
     if (args->name != NULL) {
         free(args->name);
     }
+    if (args->incremental != NULL) {
+        free(args->incremental);
+    }
     if (args->path != NULL) {
         free(args->path);
     }
@@ -52,6 +63,10 @@ void web_args_destroy(web_args_t *args) {
     free(args);
 }
 
+void exec_args_destroy(exec_args_t *args) {
+    free(args);
+}
+
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (argc < 2) {
         fprintf(stderr, "Required positional argument: PATH.\n");
@@ -67,7 +82,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     }
 
     if (args->incremental != NULL) {
-        abs_path = abspath(args->incremental);
+        args->incremental = abspath(args->incremental);
         if (abs_path == NULL) {
             sist_log("main.c", SIST_WARNING, "Could not open original index! Disabled incremental scan feature.");
             args->incremental = NULL;
@@ -112,7 +127,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
-    if (args->depth < 0) {
+    if (args->depth <= 0) {
         args->depth = G_MAXINT32;
     } else {
         args->depth += 1;
@@ -120,6 +135,10 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     if (args->name == NULL) {
         args->name = g_path_get_basename(args->output);
+    } else {
+        char* tmp = malloc(strlen(args->name) + 1);
+        strcpy(tmp, args->name);
+        args->name = tmp;
     }
 
     if (args->rewrite_url == NULL) {
@@ -144,7 +163,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
         char filename[128];
         sprintf(filename, "%s.traineddata", args->tesseract_lang);
-        const char * path = find_file_in_paths(TESS_DATAPATHS, filename);
+        const char *path = find_file_in_paths(TESS_DATAPATHS, filename);
         if (path == NULL) {
             LOG_FATAL("cli.c", "Could not find tesseract language file!");
         }
@@ -180,6 +199,16 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         ScanCtx.exclude = NULL;
     }
 
+    if (args->treemap_threshold_str == 0) {
+        args->treemap_threshold = DEFAULT_TREEMAP_THRESHOLD;
+    } else {
+        args->treemap_threshold = atof(args->treemap_threshold_str);
+    }
+
+    if (args->max_memory_buffer == 0) {
+        args->max_memory_buffer = DEFAULT_MAX_MEM_BUFFER;
+    }
+
     LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
     LOG_DEBUGF("cli.c", "arg size=%d", args->size)
     LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
@@ -195,6 +224,36 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path)
     LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex)
     LOG_DEBUGF("cli.c", "arg fast=%d", args->fast)
+    LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold)
+    LOG_DEBUGF("cli.c", "arg max_memory_buffer=%d", args->max_memory_buffer)
+
+    return 0;
+}
+
+int load_script(const char *script_path, char **dst) {
+    struct stat info;
+    int res = stat(script_path, &info);
+
+    if (res == -1) {
+        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        return 1;
+    }
+
+    int fd = open(script_path, O_RDONLY);
+    if (fd == -1) {
+        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        return 1;
+    }
+
+    *dst = malloc(info.st_size + 1);
+    res = read(fd, *dst, info.st_size);
+    if (res < 0) {
+        fprintf(stderr, "Error reading script file '%s': %s\n", script_path, strerror(errno));
+        return 1;
+    }
+
+    *(*dst + info.st_size) = '\0';
+    close(fd);
 
     return 0;
 }
@@ -208,6 +267,13 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
+    if (args->threads == 0) {
+        args->threads = 1;
+    } else if (args->threads < 0) {
+        fprintf(stderr, "Invalid threads: %d\n", args->threads);
+        return 1;
+    }
+
     char *index_path = abspath(argv[1]);
     if (index_path == NULL) {
         fprintf(stderr, "File not found: %s\n", argv[1]);
@@ -222,29 +288,9 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
     }
 
     if (args->script_path != NULL) {
-        struct stat info;
-        int res = stat(args->script_path, &info);
-
-        if (res == -1) {
-            fprintf(stderr, "Error opening script file '%s': %s\n", args->script_path, strerror(errno));
+        if (load_script(args->script_path, &args->script) != 0) {
             return 1;
         }
-
-        int fd = open(args->script_path, O_RDONLY);
-        if (fd == -1) {
-            fprintf(stderr, "Error opening script file '%s': %s\n", args->script_path, strerror(errno));
-            return 1;
-        }
-
-        args->script = malloc(info.st_size + 1);
-        res = read(fd, args->script, info.st_size);
-        if (res < 0) {
-            fprintf(stderr, "Error reading script file '%s': %s\n", args->script_path, strerror(errno));
-            return 1;
-        }
-
-        *(args->script + info.st_size) = '\0';
-        close(fd);
     }
 
     if (args->batch_size == 0) {
@@ -254,6 +300,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
     LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path)
     LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
+    LOG_DEBUGF("cli.c", "arg async_script=%s", args->async_script)
     LOG_DEBUGF("cli.c", "arg script=%s", args->script)
     LOG_DEBUGF("cli.c", "arg print=%d", args->print)
     LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size)
@@ -280,14 +327,14 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
     }
 
     if (args->credentials != NULL) {
-        char * ptr = strstr(args->credentials, ":");
+        char *ptr = strstr(args->credentials, ":");
         if (ptr == NULL) {
             fprintf(stderr, "Invalid --auth format, see usage\n");
             return 1;
         }
 
         strncpy(args->auth_user, args->credentials, (ptr - args->credentials));
-        strncpy(args->auth_pass, ptr + 1, strlen(ptr + 1));
+        strcpy(args->auth_pass, ptr + 1);
 
         if (strlen(args->auth_user) == 0) {
             fprintf(stderr, "--auth username must be at least one character long");
@@ -299,6 +346,31 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         args->auth_enabled = FALSE;
     }
 
+    if (args->tag_credentials != NULL && args->credentials != NULL) {
+        fprintf(stderr, "--auth and --tag-auth are mutually exclusive");
+        return 1;
+    }
+
+    if (args->tag_credentials != NULL) {
+        char *ptr = strstr(args->tag_credentials, ":");
+        if (ptr == NULL) {
+            fprintf(stderr, "Invalid --tag-auth format, see usage\n");
+            return 1;
+        }
+
+        strncpy(args->auth_user, args->tag_credentials, (ptr - args->tag_credentials));
+        strcpy(args->auth_pass, ptr + 1);
+
+        if (strlen(args->auth_user) == 0) {
+            fprintf(stderr, "--tag-auth username must be at least one character long");
+            return 1;
+        }
+
+        args->tag_auth_enabled = TRUE;
+    } else {
+        args->tag_auth_enabled = FALSE;
+    }
+
     args->index_count = argc - 1;
     args->indices = argv + 1;
 
@@ -313,6 +385,7 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
     LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address)
     LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials)
+    LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials)
     LOG_DEBUGF("cli.c", "arg auth_user=%s", args->auth_user)
     LOG_DEBUGF("cli.c", "arg auth_pass=%s", args->auth_pass)
     LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count)
@@ -333,3 +406,35 @@ web_args_t *web_args_create() {
     return args;
 }
 
+int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
+
+    if (argc < 2) {
+        fprintf(stderr, "Required positional argument: PATH.\n");
+        return 1;
+    }
+
+    char *index_path = abspath(argv[1]);
+    if (index_path == NULL) {
+        fprintf(stderr, "File not found: %s\n", argv[1]);
+        return 1;
+    } else {
+        args->index_path = argv[1];
+        free(index_path);
+    }
+
+    if (args->es_url == NULL) {
+        args->es_url = DEFAULT_ES_URL;
+    }
+
+    if (args->script_path == NULL) {
+        LOG_FATAL("cli.c", "--script-file argument is required");
+    }
+
+    if (load_script(args->script_path, &args->script) != 0) {
+        return 1;
+    }
+
+    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
+    LOG_DEBUGF("cli.c", "arg script=%s", args->script)
+    return 0;
+}

src/cli.h

@@ -22,6 +22,9 @@ typedef struct scan_args {
     const char *tesseract_path;
     char *exclude_regex;
     int fast;
+    const char* treemap_threshold_str;
+    double treemap_threshold;
+    int max_memory_buffer;
 } scan_args_t;
 
 scan_args_t *scan_args_create();
@@ -37,20 +40,32 @@ typedef struct index_args {
     char *script;
     int print;
     int batch_size;
+    int async_script;
     int force_reset;
+    int threads;
 } index_args_t;
 
 typedef struct web_args {
     char *es_url;
     char *listen_address;
     char *credentials;
+    char *tag_credentials;
     char auth_user[256];
     char auth_pass[256];
     int auth_enabled;
+    int tag_auth_enabled;
     int index_count;
     const char **indices;
 } web_args_t;
 
+typedef struct exec_args {
+    char *es_url;
+    const char *index_path;
+    const char *script_path;
+    int async_script;
+    char *script;
+} exec_args_t;
+
 index_args_t *index_args_create();
 
 void index_args_destroy(index_args_t *args);
@@ -63,4 +78,10 @@ int index_args_validate(index_args_t *args, int argc, const char **argv);
 
 int web_args_validate(web_args_t *args, int argc, const char **argv);
 
+exec_args_t *exec_args_create();
+
+void exec_args_destroy(exec_args_t *args);
+
+int exec_args_validate(exec_args_t *args, int argc, const char **argv);
+
 #endif

src/ctx.c

@@ -0,0 +1,6 @@
+#include "ctx.h"
+
+ScanCtx_t ScanCtx;
+WebCtx_t WebCtx;
+IndexCtx_t IndexCtx;
+LogCtx_t LogCtx;

src/ctx.h

@@ -5,19 +5,20 @@
 #include "tpool.h"
 #include "libscan/scan.h"
 #include "libscan/arc/arc.h"
-#include "libscan/cbr/cbr.h"
+#include "libscan/comic/comic.h"
 #include "libscan/ebook/ebook.h"
 #include "libscan/font/font.h"
 #include "libscan/media/media.h"
 #include "libscan/ooxml/ooxml.h"
 #include "libscan/text/text.h"
 #include "libscan/mobi/scan_mobi.h"
+#include "libscan/raw/raw.h"
+#include "src/io/store.h"
 
 #include <glib.h>
 #include <pcre.h>
 
-//TODO Move to individual scan ctx
-struct {
+typedef struct {
     struct index_t index;
 
     GHashTable *mime_table;
@@ -27,8 +28,6 @@ struct {
 
     int threads;
     int depth;
-    int verbose;
-    int very_verbose;
 
     size_t stat_tn_size;
     size_t stat_index_size;
@@ -41,34 +40,44 @@ struct {
     int fast;
 
     scan_arc_ctx_t arc_ctx;
-    scan_cbr_ctx_t cbr_ctx;
+    scan_comic_ctx_t comic_ctx;
     scan_ebook_ctx_t ebook_ctx;
     scan_font_ctx_t font_ctx;
     scan_media_ctx_t media_ctx;
     scan_ooxml_ctx_t ooxml_ctx;
     scan_text_ctx_t text_ctx;
     scan_mobi_ctx_t mobi_ctx;
-} ScanCtx;
+    scan_raw_ctx_t raw_ctx;
+} ScanCtx_t;
 
-struct {
+typedef struct {
     int verbose;
     int very_verbose;
     int no_color;
-} LogCtx;
+} LogCtx_t;
 
-struct {
+typedef struct {
     char *es_url;
     int batch_size;
-} IndexCtx;
+    tpool_t *pool;
+    store_t *tag_store;
+    GHashTable *tags;
+} IndexCtx_t;
 
-struct {
+typedef struct {
     char *es_url;
     int index_count;
     char *auth_user;
     char *auth_pass;
     int auth_enabled;
-    struct index_t indices[16];
-} WebCtx;
+    int tag_auth_enabled;
+    struct index_t indices[64];
+} WebCtx_t;
+
+extern ScanCtx_t ScanCtx;
+extern WebCtx_t WebCtx;
+extern IndexCtx_t IndexCtx;
+extern LogCtx_t LogCtx;
 
 
 #endif

src/index/elastic.c

@@ -14,9 +14,18 @@ typedef struct es_indexer {
 } es_indexer_t;
 
 
-static es_indexer_t *Indexer;
+static __thread es_indexer_t *Indexer;
 
 void delete_queue(int max);
+void elastic_flush();
+
+void elastic_cleanup() {
+    elastic_flush();
+    if (Indexer != NULL) {
+        free(Indexer->es_url);
+        free(Indexer);
+    }
+}
 
 void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
 
@@ -35,8 +44,12 @@ void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
     cJSON_Delete(line);
 }
 
-void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
+void index_json_func(void *arg) {
+    es_bulk_line_t *line = arg;
+    elastic_index_line(line);
+}
 
+void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
     char *json = cJSON_PrintUnformatted(document);
 
     size_t json_len = strlen(json);
@@ -48,10 +61,14 @@ void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
     bulk_line->next = NULL;
 
     cJSON_free(json);
-    elastic_index_line(bulk_line);
+    tpool_add_work(IndexCtx.pool, index_json_func, bulk_line);
 }
 
-void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]) {
+void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]) {
+
+    if (Indexer == NULL) {
+        Indexer = create_indexer(IndexCtx.es_url);
+    }
 
     cJSON *body = cJSON_CreateObject();
     cJSON *script_obj = cJSON_AddObjectToObject(body, "script");
@@ -65,9 +82,15 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
     char *str = cJSON_Print(body);
 
     char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_update_by_query?pretty", Indexer->es_url);
+    if (async) {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/sist2/_update_by_query?wait_for_completion=false", Indexer->es_url);
+    } else {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/sist2/_update_by_query", Indexer->es_url);
+    }
     response_t *r = web_post(bulk_url, str);
-    LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
+    if (!async) {
+        LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
+    }
     cJSON *resp = cJSON_Parse(r->body);
 
     cJSON_free(str);
@@ -82,36 +105,52 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
         cJSON_free(error_str);
     }
 
+    if (async) {
+        cJSON *task = cJSON_GetObjectItem(resp, "task");
+        LOG_INFOF("elastic.c", "User script queued: %s/_tasks/%s", Indexer->es_url, task->valuestring);
+    }
+
     cJSON_Delete(resp);
 }
 
+#define ACTION_STR_LEN 91
+
 void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
     es_bulk_line_t *line = Indexer->line_head;
     *count = 0;
 
     size_t buf_size = 0;
     size_t buf_cur = 0;
-    char *buf = malloc(1);
+    char *buf = malloc(8192);
+    size_t buf_capacity = 8192;
 
     while (line != NULL && *count < max) {
-        char action_str[512];
-        snprintf(action_str, 512,
+        char action_str[256];
+        snprintf(action_str, 256,
                  "{\"index\":{\"_id\":\"%s\", \"_type\":\"_doc\", \"_index\":\"sist2\"}}\n", line->uuid_str);
-        size_t action_str_len = strlen(action_str);
 
         size_t line_len = strlen(line->line);
-        buf = realloc(buf, buf_size + line_len + action_str_len);
-        buf_size += line_len + action_str_len;
 
-        memcpy(buf + buf_cur, action_str, action_str_len);
-        buf_cur += action_str_len;
+        while (buf_size + line_len + ACTION_STR_LEN > buf_capacity) {
+            buf_capacity *= 2;
+            buf = realloc(buf, buf_capacity);
+        }
+
+        buf_size += line_len + ACTION_STR_LEN;
+
+        memcpy(buf + buf_cur, action_str, ACTION_STR_LEN);
+        buf_cur += ACTION_STR_LEN;
         memcpy(buf + buf_cur, line->line, line_len);
         buf_cur += line_len;
 
         line = line->next;
         (*count)++;
     }
-    buf = realloc(buf, buf_size + 1);
+
+    if (buf_size + 1 > buf_capacity) {
+        buf = realloc(buf, buf_capacity + 1);
+    }
+
     *(buf + buf_cur) = '\0';
 
     *buf_len = buf_cur;
@@ -119,7 +158,7 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
 }
 
 void print_errors(response_t *r) {
-    char * tmp = malloc(r->size + 1);
+    char *tmp = malloc(r->size + 1);
     memcpy(tmp, r->body, r->size);
     *(tmp + r->size) = '\0';
 
@@ -177,6 +216,15 @@ void _elastic_flush(int max) {
         _elastic_flush(max / 2);
         return;
 
+    } else if (r->status_code == 429) {
+
+        free_response(r);
+        free(buf);
+        LOG_WARNING("elastic.c", "Got 429 status, will retry after delay")
+        usleep(1000000 * 20);
+        _elastic_flush(max);
+        return;
+
     } else if (r->status_code != 200) {
         print_errors(r);
         delete_queue(Indexer->queued);
@@ -202,9 +250,8 @@ void delete_queue(int max) {
         Indexer->line_head = tmp->next;
         if (Indexer->line_head == NULL) {
             Indexer->line_tail = NULL;
-        } else {
-            free(tmp);
         }
+        free(tmp);
         Indexer->queued -= 1;
     }
 }
@@ -254,7 +301,7 @@ es_indexer_t *create_indexer(const char *url) {
     return indexer;
 }
 
-void destroy_indexer(char *script, char index_id[UUID_STR_LEN]) {
+void finish_indexer(char *script, int async_script, char *index_id) {
 
     char url[4096];
 
@@ -264,23 +311,24 @@ void destroy_indexer(char *script, char index_id[UUID_STR_LEN]) {
     free_response(r);
 
     if (script != NULL) {
-        execute_update_script(script, index_id);
-    }
+        execute_update_script(script, async_script, index_id);
+        free(script);
 
-    snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
-    r = web_post(url, "");
-    LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
-    free_response(r);
+        snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
+        r = web_post(url, "");
+        LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
+        free_response(r);
+    }
 
     snprintf(url, sizeof(url), "%s/sist2/_forcemerge", IndexCtx.es_url);
     r = web_post(url, "");
     LOG_INFOF("elastic.c", "Merge index <%d>", r->status_code);
     free_response(r);
 
-    if (Indexer != NULL) {
-        free(Indexer->es_url);
-        free(Indexer);
-    }
+    snprintf(url, sizeof(url), "%s/sist2/_settings", IndexCtx.es_url);
+    r = web_put(url, "{\"index\":{\"refresh_interval\":\"1s\"}}");
+    LOG_INFOF("elastic.c", "Set refresh interval <%d>", r->status_code);
+    free_response(r);
 }
 
 void elastic_init(int force_reset) {
@@ -288,7 +336,7 @@ void elastic_init(int force_reset) {
     // Check if index exists
     char url[4096];
     snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
     int index_exists = r->status_code == 200;
     free_response(r);
 
@@ -333,7 +381,7 @@ cJSON *elastic_get_document(const char *uuid_str) {
     char url[4096];
     snprintf(url, 4096, "%s/sist2/_doc/%s", WebCtx.es_url, uuid_str);
 
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 3);
     cJSON *json = NULL;
     if (r->status_code == 200) {
         json = cJSON_Parse(r->body);
@@ -347,7 +395,7 @@ char *elastic_get_status() {
     snprintf(url, 4096,
              "%s/_cluster/state/metadata/sist2?filter_path=metadata.indices.*.state", WebCtx.es_url);
 
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
     cJSON *json = NULL;
     char *status = malloc(128 * sizeof(char));
     status[0] = '\0';

src/index/elastic.h

@@ -16,15 +16,14 @@ typedef struct es_indexer es_indexer_t;
 
 void elastic_index_line(es_bulk_line_t *line);
 
-void elastic_flush();
-
 void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);
 
 void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);
 
 es_indexer_t *create_indexer(const char* es_url);
 
-void destroy_indexer(char *script, char index_id[UUID_STR_LEN]);
+void elastic_cleanup();
+void finish_indexer(char *script, int async_script, char *index_id);
 
 void elastic_init(int force_reset);
 
@@ -32,4 +31,6 @@ cJSON *elastic_get_document(const char *uuid_str);
 
 char *elastic_get_status();
 
+void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]);
+
 #endif

src/index/web.c

@@ -1,11 +1,19 @@
 #include "web.h"
 #include "src/sist.h"
-#include "src/ctx.h"
 
 #include <mongoose.h>
 #include <pthread.h>
+#include <curl/curl.h>
 
 
+size_t write_cb(char *ptr, size_t size, size_t nmemb, void *user_data) {
+
+    size_t real_size = size * nmemb;
+    dyn_buffer_t *buf = user_data;
+    dyn_buffer_write(buf, ptr, real_size);
+    return real_size;
+}
+
 void free_response(response_t *resp) {
     if (resp->body != NULL) {
         free(resp->body);
@@ -75,7 +83,7 @@ subreq_ctx_t *http_req(const char *url, const char *extra_headers, const char *p
     subreq_ctx_t *ctx = malloc(sizeof(subreq_ctx_t));
     mg_mgr_init(&ctx->mgr, NULL);
 
-    char address[8196];
+    char address[8192];
     snprintf(address, sizeof(address), "tcp://%.*s:%u", (int) host.len, host.p, port);
     struct mg_connection *nc = mg_connect(&ctx->mgr, address, http_req_ev);
     nc->user_data = &ctx->ev_data;
@@ -100,55 +108,126 @@ subreq_ctx_t *http_req(const char *url, const char *extra_headers, const char *p
     return ctx;
 }
 
-response_t *web_get(const char *url) {
-    subreq_ctx_t *ctx = http_req(url, SIST2_HEADERS, NULL, "GET");
-    while (ctx->ev_data.done == FALSE) {
-        mg_mgr_poll(&ctx->mgr, 50);
-    }
-    mg_mgr_free(&ctx->mgr);
-
-    response_t *ret = ctx->ev_data.resp;
-    free(ctx);
-    return ret;
-}
-
 subreq_ctx_t *web_post_async(const char *url, const char *data) {
     return http_req(url, SIST2_HEADERS, data, "POST");
 }
 
-response_t *web_post(const char *url, const char *data) {
-    subreq_ctx_t *ctx = http_req(url, SIST2_HEADERS, data, "POST");
+response_t *web_get(const char *url, int timeout) {
+    response_t *resp = malloc(sizeof(response_t));
 
-    while (ctx->ev_data.done == FALSE) {
-        mg_mgr_poll(&ctx->mgr, 50);
-    }
-    mg_mgr_free(&ctx->mgr);
+    CURL *curl;
+    dyn_buffer_t buffer = dyn_buffer_create();
 
-    response_t *ret = ctx->ev_data.resp;
-    free(ctx);
-    return ret;
+    curl = curl_easy_init();
+    curl_easy_setopt(curl, CURLOPT_URL, url);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&buffer));
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
+    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+    curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
+
+    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+
+    curl_easy_perform(curl);
+    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);
+
+    curl_easy_cleanup(curl);
+    curl_slist_free_all(headers);
+
+    resp->body = buffer.buf;
+    resp->size = buffer.cur;
+    return resp;
 }
 
-response_t *web_put(const char *url, const char *data) {
-    subreq_ctx_t *ctx = http_req(url, SIST2_HEADERS, data, "PUT");
-    while (ctx->ev_data.done == FALSE) {
-        mg_mgr_poll(&ctx->mgr, 50);
-    }
-    mg_mgr_free(&ctx->mgr);
+response_t *web_post(const char *url, const char *data) {
 
-    response_t *ret = ctx->ev_data.resp;
-    free(ctx);
-    return ret;
+    response_t *resp = malloc(sizeof(response_t));
+
+    CURL *curl;
+    dyn_buffer_t buffer = dyn_buffer_create();
+
+    curl = curl_easy_init();
+    curl_easy_setopt(curl, CURLOPT_URL, url);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&buffer));
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
+    curl_easy_setopt(curl, CURLOPT_POST, 1);
+    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+
+    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);
+
+    curl_easy_perform(curl);
+    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);
+
+    curl_easy_cleanup(curl);
+    curl_slist_free_all(headers);
+
+    resp->body = buffer.buf;
+    resp->size = buffer.cur;
+
+    return resp;
+}
+
+
+response_t *web_put(const char *url, const char *data) {
+
+    response_t *resp = malloc(sizeof(response_t));
+
+    CURL *curl;
+    dyn_buffer_t buffer = dyn_buffer_create();
+
+    curl = curl_easy_init();
+    curl_easy_setopt(curl, CURLOPT_URL, url);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&buffer));
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
+    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
+    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+    curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
+    curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4 );
+
+    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);
+
+    curl_easy_perform(curl);
+    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);
+
+    curl_easy_cleanup(curl);
+    curl_slist_free_all(headers);
+
+    resp->body = buffer.buf;
+    resp->size = buffer.cur;
+    return resp;
 }
 
 response_t *web_delete(const char *url) {
-    subreq_ctx_t *ctx = http_req(url, SIST2_HEADERS, NULL, "DELETE");
-    while (ctx->ev_data.done == FALSE) {
-        mg_mgr_poll(&ctx->mgr, 50);
-    }
-    mg_mgr_free(&ctx->mgr);
 
-    response_t *ret = ctx->ev_data.resp;
-    free(ctx);
-    return ret;
-}
+    response_t *resp = malloc(sizeof(response_t));
+
+    CURL *curl;
+    dyn_buffer_t buffer = dyn_buffer_create();
+
+    curl = curl_easy_init();
+    curl_easy_setopt(curl, CURLOPT_URL, url);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&buffer));
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
+    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
+    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "");
+    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+
+    curl_easy_perform(curl);
+    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);
+
+    curl_easy_cleanup(curl);
+    curl_slist_free_all(headers);
+
+    resp->body = buffer.buf;
+    resp->size = buffer.cur;
+    return resp;
+}

src/index/web.h

@@ -20,7 +20,7 @@ typedef struct {
     struct mg_mgr mgr;
 } subreq_ctx_t;
 
-response_t *web_get(const char *url);
+response_t *web_get(const char *url, int timeout);
 response_t *web_post(const char * url, const char * data);
 subreq_ctx_t *web_post_async(const char *url, const char *data);
 response_t *web_put(const char *url, const char *data);

src/io/serialize.c

@@ -62,7 +62,7 @@ index_descriptor_t read_index_descriptor(char *path) {
     int fd = open(path, O_RDONLY);
 
     if (fd == -1) {
-        LOG_FATALF("serialize.c", "Invalid/corrupt index (Could not find descriptor): %s: %s\n", path ,strerror(errno))
+        LOG_FATALF("serialize.c", "Invalid/corrupt index (Could not find descriptor): %s: %s\n", path, strerror(errno))
     }
 
     char *buf = malloc(info.st_size + 1);
@@ -172,8 +172,8 @@ void write_document(document_t *doc) {
     dyn_buffer_t buf = dyn_buffer_create();
 
     // Ignore root directory in the file path
-    doc->ext = doc->ext - ScanCtx.index.desc.root_len;
-    doc->base = doc->base - ScanCtx.index.desc.root_len;
+    doc->ext = (short) (doc->ext - ScanCtx.index.desc.root_len);
+    doc->base = (short) (doc->base - ScanCtx.index.desc.root_len);
     doc->filepath += ScanCtx.index.desc.root_len;
 
     dyn_buffer_write(&buf, doc, sizeof(line_t));
@@ -230,7 +230,7 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
         char uuid_str[UUID_STR_LEN];
         uuid_unparse(line.uuid, uuid_str);
 
-        const char* mime_text = mime_get_mime_text(line.mime);
+        const char *mime_text = mime_get_mime_text(line.mime);
         if (mime_text == NULL) {
             cJSON_AddNullToObject(document, "mime");
         } else {
@@ -239,22 +239,37 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
         cJSON_AddNumberToObject(document, "size", (double) line.size);
         cJSON_AddNumberToObject(document, "mtime", line.mtime);
 
-        int c;
+        int c = 0;
         while ((c = getc(file)) != 0) {
             dyn_buffer_write_char(&buf, (char) c);
         }
         dyn_buffer_write_char(&buf, '\0');
 
+        if (IndexCtx.tags != NULL) {
+            const char *tags_string = g_hash_table_lookup(IndexCtx.tags, buf.buf);
+            if (tags_string != NULL) {
+                cJSON *tags_arr = cJSON_Parse(tags_string);
+                cJSON_AddItemToObject(document, "tag", tags_arr);
+            }
+        }
+
         cJSON_AddStringToObject(document, "extension", buf.buf + line.ext);
         if (*(buf.buf + line.ext - 1) == '.') {
             *(buf.buf + line.ext - 1) = '\0';
         } else {
             *(buf.buf + line.ext) = '\0';
         }
-        cJSON_AddStringToObject(document, "name", buf.buf + line.base);
+
+        char tmp[PATH_MAX * 3];
+
+        str_escape(tmp, buf.buf + line.base);
+        cJSON_AddStringToObject(document, "name", tmp);
+
         if (line.base > 0) {
             *(buf.buf + line.base - 1) = '\0';
-            cJSON_AddStringToObject(document, "path", buf.buf);
+
+            str_escape(tmp, buf.buf);
+            cJSON_AddStringToObject(document, "path", tmp);
         } else {
             cJSON_AddStringToObject(document, "path", "");
         }

src/io/store.c

@@ -1,9 +1,10 @@
 #include "store.h"
 #include "src/ctx.h"
 
-store_t *store_create(char *path) {
+store_t *store_create(char *path, size_t chunk_size) {
 
     store_t *store = malloc(sizeof(struct store_t));
+    store->chunk_size = chunk_size;
     pthread_rwlock_init(&store->lock, NULL);
 
     mdb_env_create(&store->env);
@@ -18,7 +19,7 @@ store_t *store_create(char *path) {
         LOG_FATALF("store.c", "Error while opening store: %s (%s)\n", mdb_strerror(open_ret), path)
     }
 
-    store->size = (size_t) 1024 * 1024 * 5;
+    store->size = (size_t) store->chunk_size;
     ScanCtx.stat_tn_size = 0;
     mdb_env_set_mapsize(store->env, store->size);
 
@@ -69,7 +70,7 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
         // Cannot resize when there is a opened transaction.
         //  Resize take effect on the next commit.
         pthread_rwlock_wrlock(&store->lock);
-        store->size += 1024 * 1024 * 50;
+        store->size += store->chunk_size;
         mdb_env_set_mapsize(store->env, store->size);
         mdb_txn_begin(store->env, NULL, 0, &txn);
         put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
@@ -110,3 +111,40 @@ char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen)
     return buf;
 }
 
+GHashTable *store_read_all(store_t *store) {
+
+    int count = 0;
+
+    GHashTable *table = g_hash_table_new_full(g_str_hash, g_str_equal, free, free);
+
+    MDB_txn *txn = NULL;
+    mdb_txn_begin(store->env, NULL, MDB_RDONLY, &txn);
+
+    MDB_cursor *cur = NULL;
+    mdb_cursor_open(txn, store->dbi, &cur);
+
+    MDB_val key;
+    MDB_val value;
+
+    while (mdb_cursor_get(cur, &key, &value, MDB_NEXT) == 0) {
+        char *key_str = malloc(key.mv_size);
+        memcpy(key_str, key.mv_data, key.mv_size);
+        char *val_str = malloc(value.mv_size);
+        memcpy(val_str, value.mv_data, value.mv_size);
+
+        g_hash_table_insert(table, key_str, val_str);
+        count += 1;
+    }
+
+    LOG_DEBUGF("store.c", "Read tags for %d documents", count);
+
+    mdb_cursor_close(cur);
+    mdb_txn_abort(txn);
+    return table;
+}
+
+
+void store_copy(store_t *store, const char *destination) {
+    mkdir(destination, S_IWUSR | S_IRUSR | S_IXUSR);
+    mdb_env_copy(store->env, destination);
+}

src/io/store.h

@@ -4,14 +4,20 @@
 #include <pthread.h>
 #include <lmdb.h>
 
+#include <glib.h>
+
+#define STORE_SIZE_TN 1024 * 1024 * 5
+#define STORE_SIZE_TAG 1024 * 16
+
 typedef struct store_t {
     MDB_dbi dbi;
     MDB_env *env;
     size_t size;
+    size_t chunk_size;
     pthread_rwlock_t lock;
 } store_t;
 
-store_t *store_create(char *path);
+store_t *store_create(char *path, size_t chunk_size);
 
 void store_destroy(store_t *store);
 
@@ -19,4 +25,8 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
 
 char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen);
 
+GHashTable *store_read_all(store_t *store);
+
+void store_copy(store_t *store, const char *destination);
+
 #endif

src/log.c

@@ -12,7 +12,7 @@ const char *log_levels[] = {
         "DEBUG", "INFO", "WARNING", "ERROR", "FATAL"
 };
 
-void sist_logf(const char *filepath, int level, char *format, ...) {
+void vsist_logf(const char *filepath, int level, char *format, va_list ap) {
 
     static int is_tty = -1;
     if (is_tty == -1) {
@@ -46,11 +46,8 @@ void sist_logf(const char *filepath, int level, char *format, ...) {
         );
     }
 
-    va_list ap;
-    va_start(ap, format);
     size_t maxsize = sizeof(log_str) - log_len;
     log_len += vsnprintf(log_str + log_len, maxsize, format, ap);
-    va_end(ap);
 
     if (is_tty) {
         log_len += sprintf(log_str + log_len, "\033[0m\n");
@@ -65,6 +62,13 @@ void sist_logf(const char *filepath, int level, char *format, ...) {
     }
 }
 
+void sist_logf(const char *filepath, int level, char *format, ...) {
+    va_list ap;
+    va_start(ap, format);
+    vsist_logf(filepath, level, format, ap);
+    va_end(ap);
+}
+
 void sist_log(const char *filepath, int level, char *str) {
 
     static int is_tty = -1;

src/log.h

@@ -40,6 +40,7 @@
 #include "sist.h"
 
 void sist_logf(const char *filepath, int level, char *format, ...);
+void vsist_logf(const char *filepath, int level, char *format, va_list ap);
 
 void sist_log(const char *filepath, int level, char *str);
 

src/main.c

@@ -2,7 +2,7 @@
 #include "ctx.h"
 
 #include <third-party/argparse/argparse.h>
-#include <glib.h>
+#include <locale.h>
 
 #include "cli.h"
 #include "io/serialize.h"
@@ -14,16 +14,19 @@
 #include "parsing/mime.h"
 #include "parsing/parse.h"
 
+#include "stats.h"
+
 #define DESCRIPTION "Lightning-fast file system indexer and search tool."
 
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "2.1.0";
+static const char *const Version = "2.8.0";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
         "sist2 web [OPTION]... INDEX...",
+        "sist2 exec-script [OPTION]... INDEX",
         NULL,
 };
 
@@ -55,9 +58,9 @@ void _log(const char *filepath, int level, char *str) {
         exit(-1);
     }
 
-    if (ScanCtx.verbose) {
+    if (LogCtx.verbose) {
         if (level == LEVEL_DEBUG) {
-            if (ScanCtx.very_verbose) {
+            if (LogCtx.very_verbose) {
                 sist_log(filepath, level, str);
             }
         } else {
@@ -72,17 +75,17 @@ void _logf(const char *filepath, int level, char *format, ...) {
 
     va_start(args, format);
     if (level == LEVEL_FATAL) {
-        sist_logf(filepath, level, format, args);
+        vsist_logf(filepath, level, format, args);
         exit(-1);
     }
 
-    if (ScanCtx.verbose) {
+    if (LogCtx.verbose) {
         if (level == LEVEL_DEBUG) {
-            if (ScanCtx.very_verbose) {
-                sist_logf(filepath, level, format, args);
+            if (LogCtx.very_verbose) {
+                vsist_logf(filepath, level, format, args);
             }
         } else {
-            sist_logf(filepath, level, format, args);
+            vsist_logf(filepath, level, format, args);
         }
     }
     va_end(args);
@@ -96,11 +99,14 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.arc_ctx.logf = _logf;
     ScanCtx.arc_ctx.parse = (parse_callback_t) parse;
 
-    // Cbr
-    ScanCtx.cbr_ctx.log = _log;
-    ScanCtx.cbr_ctx.logf = _logf;
-    ScanCtx.cbr_ctx.store = _store;
-    ScanCtx.cbr_ctx.cbr_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbr");
+    // Comic
+    ScanCtx.comic_ctx.log = _log;
+    ScanCtx.comic_ctx.logf = _logf;
+    ScanCtx.comic_ctx.store = _store;
+    ScanCtx.comic_ctx.tn_size = args->size;
+    ScanCtx.comic_ctx.tn_qscale = args->quality;
+    ScanCtx.comic_ctx.cbr_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbr");
+    ScanCtx.comic_ctx.cbz_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbz");
 
     // Ebook
     pthread_mutex_init(&ScanCtx.ebook_ctx.mupdf_mutex, NULL);
@@ -124,12 +130,14 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.media_ctx.log = _log;
     ScanCtx.media_ctx.logf = _logf;
     ScanCtx.media_ctx.store = _store;
+    ScanCtx.media_ctx.max_media_buffer = (long) args->max_memory_buffer * 1024 * 1024;
     init_media();
 
     // OOXML
     ScanCtx.ooxml_ctx.content_size = args->content_size;
     ScanCtx.ooxml_ctx.log = _log;
     ScanCtx.ooxml_ctx.logf = _logf;
+    ScanCtx.ooxml_ctx.store = _store;
 
     // MOBI
     ScanCtx.mobi_ctx.content_size = args->content_size;
@@ -150,6 +158,13 @@ void initialize_scan_context(scan_args_t *args) {
     strncpy(ScanCtx.index.desc.rewrite_url, args->rewrite_url, sizeof(ScanCtx.index.desc.rewrite_url));
     ScanCtx.index.desc.root_len = (short) strlen(ScanCtx.index.desc.root);
     ScanCtx.fast = args->fast;
+
+    // Raw
+    ScanCtx.raw_ctx.tn_qscale = args->quality;
+    ScanCtx.raw_ctx.tn_size = args->size;
+    ScanCtx.raw_ctx.log = _log;
+    ScanCtx.raw_ctx.logf = _logf;
+    ScanCtx.raw_ctx.store = _store;
 }
 
 
@@ -165,7 +180,7 @@ void sist2_scan(scan_args_t *args) {
     char store_path[PATH_MAX];
     snprintf(store_path, PATH_MAX, "%sthumbs", ScanCtx.index.path);
     mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
-    ScanCtx.index.store = store_create(store_path);
+    ScanCtx.index.store = store_create(store_path, STORE_SIZE_TN);
 
     scan_print_header();
 
@@ -200,17 +215,19 @@ void sist2_scan(scan_args_t *args) {
         LOG_INFOF("main.c", "Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table))
     }
 
-    ScanCtx.pool = tpool_create(args->threads, thread_cleanup);
+    ScanCtx.pool = tpool_create(args->threads, thread_cleanup, TRUE);
     tpool_start(ScanCtx.pool);
     walk_directory_tree(ScanCtx.index.desc.root);
     tpool_wait(ScanCtx.pool);
     tpool_destroy(ScanCtx.pool);
 
+    generate_stats(&ScanCtx.index, args->treemap_threshold, ScanCtx.index.path);
+
     if (args->incremental != NULL) {
         char dst_path[PATH_MAX];
         snprintf(store_path, PATH_MAX, "%sthumbs", args->incremental);
         snprintf(dst_path, PATH_MAX, "%s_index_original", ScanCtx.index.path);
-        store_t *source = store_create(store_path);
+        store_t *source = store_create(store_path, STORE_SIZE_TN);
 
         DIR *dir = opendir(args->incremental);
         if (dir == NULL) {
@@ -227,6 +244,13 @@ void sist2_scan(scan_args_t *args) {
         }
         closedir(dir);
         store_destroy(source);
+
+        snprintf(store_path, PATH_MAX, "%stags", args->incremental);
+        snprintf(dst_path, PATH_MAX, "%stags", ScanCtx.index.path);
+        mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
+        store_t *source_tags = store_create(store_path, STORE_SIZE_TAG);
+        store_copy(source_tags, dst_path);
+        store_destroy(source_tags);
     }
 
     store_destroy(ScanCtx.index.store);
@@ -258,6 +282,12 @@ void sist2_index(index_args_t *args) {
         LOG_FATALF("main.c", "Could not open index %s: %s", args->index_path, strerror(errno))
     }
 
+    char path_tmp[PATH_MAX];
+    snprintf(path_tmp, sizeof(path_tmp), "%s/tags", args->index_path);
+    mkdir(path_tmp, S_IWUSR | S_IRUSR | S_IXUSR);
+    IndexCtx.tag_store = store_create(path_tmp, STORE_SIZE_TAG);
+    IndexCtx.tags = store_read_all(IndexCtx.tag_store);
+
     index_func f;
     if (args->print) {
         f = print_json;
@@ -265,6 +295,16 @@ void sist2_index(index_args_t *args) {
         f = index_json;
     }
 
+    void (*cleanup)();
+    if (args->print) {
+        cleanup = NULL;
+    } else {
+        cleanup = elastic_cleanup;
+    }
+
+    IndexCtx.pool = tpool_create(args->threads, cleanup, FALSE);
+    tpool_start(IndexCtx.pool);
+
     struct dirent *de;
     while ((de = readdir(dir)) != NULL) {
         if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
@@ -275,10 +315,33 @@ void sist2_index(index_args_t *args) {
     }
     closedir(dir);
 
+    tpool_wait(IndexCtx.pool);
+
+    tpool_destroy(IndexCtx.pool);
+
     if (!args->print) {
-        elastic_flush();
-        destroy_indexer(args->script, desc.uuid);
+        finish_indexer(args->script, args->async_script, desc.uuid);
     }
+
+    store_destroy(IndexCtx.tag_store);
+    g_hash_table_remove_all(IndexCtx.tags);
+    g_hash_table_destroy(IndexCtx.tags);
+}
+
+void sist2_exec_script(exec_args_t *args) {
+
+    LogCtx.verbose = TRUE;
+
+    char descriptor_path[PATH_MAX];
+    snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->index_path);
+    index_descriptor_t desc = read_index_descriptor(descriptor_path);
+
+    IndexCtx.es_url = args->es_url;
+
+    LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type)
+
+    execute_update_script(args->script, args->async_script, desc.uuid);
+    free(args->script);
 }
 
 void sist2_web(web_args_t *args) {
@@ -288,6 +351,7 @@ void sist2_web(web_args_t *args) {
     WebCtx.auth_user = args->auth_user;
     WebCtx.auth_pass = args->auth_pass;
     WebCtx.auth_enabled = args->auth_enabled;
+    WebCtx.tag_auth_enabled = args->tag_auth_enabled;
 
     for (int i = 0; i < args->index_count; i++) {
         char *abs_path = abspath(args->indices[i]);
@@ -297,7 +361,11 @@ void sist2_web(web_args_t *args) {
         char path_tmp[PATH_MAX];
 
         snprintf(path_tmp, PATH_MAX, "%sthumbs", abs_path);
-        WebCtx.indices[i].store = store_create(path_tmp);
+        WebCtx.indices[i].store = store_create(path_tmp, STORE_SIZE_TN);
+
+        snprintf(path_tmp, PATH_MAX, "%stags", abs_path);
+        mkdir(path_tmp, S_IWUSR | S_IRUSR | S_IXUSR);
+        WebCtx.indices[i].tag_store = store_create(path_tmp, STORE_SIZE_TAG);
 
         snprintf(path_tmp, PATH_MAX, "%sdescriptor.json", abs_path);
         WebCtx.indices[i].desc = read_index_descriptor(path_tmp);
@@ -312,14 +380,19 @@ void sist2_web(web_args_t *args) {
 
 
 int main(int argc, const char *argv[]) {
+    setlocale(LC_ALL, "");
 
     scan_args_t *scan_args = scan_args_create();
     index_args_t *index_args = index_args_create();
     web_args_t *web_args = web_args_create();
+    exec_args_t *exec_args = exec_args_create();
 
     int arg_version = 0;
 
     char *common_es_url = NULL;
+    char *common_script_path = NULL;
+    int common_async_script = 0;
+    int common_threads = 0;
 
     struct argparse_option options[] = {
             OPT_HELP(),
@@ -329,7 +402,7 @@ int main(int argc, const char *argv[]) {
             OPT_BOOLEAN(0, "very-verbose", &LogCtx.very_verbose, "Turn on debug messages"),
 
             OPT_GROUP("Scan options"),
-            OPT_INTEGER('t', "threads", &scan_args->threads, "Number of threads. DEFAULT=1"),
+            OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),
             OPT_FLOAT('q', "quality", &scan_args->quality,
                       "Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=5"),
             OPT_INTEGER(0, "size", &scan_args->size,
@@ -350,11 +423,18 @@ int main(int argc, const char *argv[]) {
                                                              "which are installed on your machine)"),
             OPT_STRING('e', "exclude", &scan_args->exclude_regex, "Files that match this regex will not be scanned"),
             OPT_BOOLEAN(0, "fast", &scan_args->fast, "Only index file names & mime type"),
+            OPT_STRING(0, "treemap-threshold", &scan_args->treemap_threshold_str, "Relative size threshold for treemap "
+                                                                                  "(see USAGE.md). DEFAULT: 0.0005"),
+            OPT_INTEGER(0, "mem-buffer", &scan_args->max_memory_buffer,
+                        "Maximum memory buffer size per thread in MB for files inside archives "
+                        "(see USAGE.md). DEFAULT: 2000"),
 
             OPT_GROUP("Index options"),
+            OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"),
             OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
-            OPT_STRING(0, "script-file", &index_args->script_path, "Path to user script."),
+            OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
+            OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),
             OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 100"),
             OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. "
                                                                       "(You must use this option the first time you use the index command)"),
@@ -363,6 +443,11 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
             OPT_STRING(0, "bind", &web_args->listen_address, "Listen on this address. DEFAULT=localhost:4090"),
             OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"),
+            OPT_STRING(0, "tag-auth", &web_args->tag_credentials, "Basic auth in user:password format for tagging"),
+
+            OPT_GROUP("Exec-script options"),
+            OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
+            OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),
 
             OPT_END(),
     };
@@ -383,6 +468,13 @@ int main(int argc, const char *argv[]) {
 
     web_args->es_url = common_es_url;
     index_args->es_url = common_es_url;
+    exec_args->es_url = common_es_url;
+    index_args->script_path = common_script_path;
+    exec_args->script_path = common_script_path;
+    index_args->threads = common_threads;
+    scan_args->threads = common_threads;
+    exec_args->async_script = common_async_script;
+    index_args->async_script = common_async_script;
 
     if (argc == 0) {
         argparse_usage(&argparse);
@@ -411,6 +503,14 @@ int main(int argc, const char *argv[]) {
         }
         sist2_web(web_args);
 
+    }  else if (strcmp(argv[0], "exec-script") == 0) {
+
+        int err = exec_args_validate(exec_args, argc, argv);
+        if (err != 0) {
+            goto end;
+        }
+        sist2_exec_script(exec_args);
+
     } else {
         fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
         argparse_usage(&argparse);
@@ -422,6 +522,7 @@ int main(int argc, const char *argv[]) {
     scan_args_destroy(scan_args);
     index_args_destroy(index_args);
     web_args_destroy(web_args);
+    exec_args_destroy(exec_args);
 
     return 0;
 }

src/parsing/mime.h

@@ -3,7 +3,7 @@
 
 #include "../sist.h"
 
-#define MAJOR_MIME(mime_id) (mime_id & 0x00FF0000) >> 16
+#define MAJOR_MIME(mime_id) (mime_id & 0x000F0000) >> 16
 
 #define MIME_EMPTY 1
 
@@ -31,6 +31,9 @@
 #define MARKUP_MASK 0x01000000
 #define IS_MARKUP(mime_id) (mime_id & MARKUP_MASK) == MARKUP_MASK
 
+#define RAW_MASK 0x00800000
+#define IS_RAW(mime_id) (mime_id & RAW_MASK) == RAW_MASK
+
 enum major_mime {
     MimeInvalid = 0,
     MimeModel = 1,

src/parsing/mime_generated.c

@@ -54,387 +54,407 @@ enum mime {
     application_streamingmedia=655406,
     application_vda=655407,
     application_vnd_amazon_mobi8_ebook=655408 | 0x02000000,
-    application_vnd_fdf=655409,
-    application_vnd_font_fontforge_sfd=655410,
-    application_vnd_hp_hpgl=655411,
-    application_vnd_iccprofile=655412,
-    application_vnd_lotus_1_2_3=655413,
-    application_vnd_ms_cab_compressed=655414,
-    application_vnd_ms_excel=655415,
-    application_vnd_ms_fontobject=655416,
-    application_vnd_ms_opentype=655417 | 0x20000000,
-    application_vnd_ms_pki_certstore=655418,
-    application_vnd_ms_pki_pko=655419,
-    application_vnd_ms_pki_seccat=655420,
-    application_vnd_ms_powerpoint=655421,
-    application_vnd_ms_project=655422,
-    application_vnd_oasis_opendocument_base=655423,
-    application_vnd_oasis_opendocument_formula=655424,
-    application_vnd_oasis_opendocument_graphics=655425,
-    application_vnd_oasis_opendocument_presentation=655426,
-    application_vnd_oasis_opendocument_spreadsheet=655427,
-    application_vnd_oasis_opendocument_text=655428,
-    application_vnd_openxmlformats_officedocument_presentationml_presentation=655429 | 0x04000000,
-    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655430 | 0x04000000,
-    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655431 | 0x04000000,
-    application_vnd_symbian_install=655432,
-    application_vnd_tcpdump_pcap=655433,
-    application_vnd_wap_wmlc=655434,
-    application_vnd_wap_wmlscriptc=655435,
-    application_vnd_xara=655436,
-    application_vocaltec_media_desc=655437,
-    application_vocaltec_media_file=655438,
-    application_warc=655439,
-    application_winhelp=655440,
-    application_wordperfect=655441,
-    application_wordperfect6_0=655442,
-    application_wordperfect6_1=655443,
-    application_x_123=655444,
-    application_x_7z_compressed=655445 | 0x10000000,
-    application_x_aim=655446,
-    application_x_apple_diskimage=655447,
-    application_x_arc=655448 | 0x10000000,
-    application_x_archive=655449,
-    application_x_atari_7800_rom=655450,
-    application_x_authorware_bin=655451,
-    application_x_authorware_map=655452,
-    application_x_authorware_seg=655453,
-    application_x_avira_qua=655454,
-    application_x_bcpio=655455,
-    application_x_bittorrent=655456,
-    application_x_bsh=655457,
-    application_x_bytecode_python=655458,
-    application_x_bzip=655459,
-    application_x_bzip2=655460 | 0x08000000,
-    application_x_cbr=655461,
-    application_x_cbz=655462 | 0x40000000,
-    application_x_cdlink=655463,
-    application_x_chat=655464,
-    application_x_chrome_extension=655465,
-    application_x_cocoa=655466,
-    application_x_conference=655467,
-    application_x_coredump=655468,
-    application_x_cpio=655469,
-    application_x_dbf=655470,
-    application_x_dbt=655471,
-    application_x_debian_package=655472,
-    application_x_deepv=655473,
-    application_x_director=655474,
-    application_x_dmp=655475,
-    application_x_dosdriver=655476,
-    application_x_dosexec=655477,
-    application_x_dvi=655478,
-    application_x_elc=655479,
+    application_vnd_coffeescript=655409,
+    application_vnd_fdf=655410,
+    application_vnd_font_fontforge_sfd=655411,
+    application_vnd_hp_hpgl=655412,
+    application_vnd_iccprofile=655413,
+    application_vnd_lotus_1_2_3=655414,
+    application_vnd_ms_cab_compressed=655415,
+    application_vnd_ms_excel=655416,
+    application_vnd_ms_fontobject=655417,
+    application_vnd_ms_opentype=655418 | 0x20000000,
+    application_vnd_ms_outlook=655419,
+    application_vnd_ms_pki_certstore=655420,
+    application_vnd_ms_pki_pko=655421,
+    application_vnd_ms_pki_seccat=655422,
+    application_vnd_ms_powerpoint=655423,
+    application_vnd_ms_project=655424,
+    application_vnd_oasis_opendocument_base=655425,
+    application_vnd_oasis_opendocument_formula=655426,
+    application_vnd_oasis_opendocument_graphics=655427,
+    application_vnd_oasis_opendocument_presentation=655428,
+    application_vnd_oasis_opendocument_spreadsheet=655429,
+    application_vnd_oasis_opendocument_text=655430,
+    application_vnd_openxmlformats_officedocument_presentationml_presentation=655431 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655432 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655433 | 0x04000000,
+    application_vnd_symbian_install=655434,
+    application_vnd_tcpdump_pcap=655435,
+    application_vnd_wap_wmlc=655436,
+    application_vnd_wap_wmlscriptc=655437,
+    application_vnd_xara=655438,
+    application_vocaltec_media_desc=655439,
+    application_vocaltec_media_file=655440,
+    application_warc=655441,
+    application_winhelp=655442,
+    application_wordperfect=655443,
+    application_wordperfect6_0=655444,
+    application_wordperfect6_1=655445,
+    application_x_123=655446,
+    application_x_7z_compressed=655447 | 0x10000000,
+    application_x_aim=655448,
+    application_x_apple_diskimage=655449,
+    application_x_arc=655450 | 0x10000000,
+    application_x_archive=655451,
+    application_x_atari_7800_rom=655452,
+    application_x_authorware_bin=655453,
+    application_x_authorware_map=655454,
+    application_x_authorware_seg=655455,
+    application_x_avira_qua=655456,
+    application_x_bcpio=655457,
+    application_x_bittorrent=655458,
+    application_x_bsh=655459,
+    application_x_bytecode_python=655460,
+    application_x_bzip=655461,
+    application_x_bzip2=655462 | 0x08000000,
+    application_x_cbr=655463,
+    application_x_cbz=655464,
+    application_x_cdlink=655465,
+    application_x_chat=655466,
+    application_x_chrome_extension=655467,
+    application_x_cocoa=655468,
+    application_x_conference=655469,
+    application_x_coredump=655470,
+    application_x_cpio=655471,
+    application_x_dbf=655472,
+    application_x_dbt=655473,
+    application_x_debian_package=655474,
+    application_x_deepv=655475,
+    application_x_director=655476,
+    application_x_dmp=655477,
+    application_x_dosdriver=655478,
+    application_x_dosexec=655479,
+    application_x_dvi=655480,
+    application_x_elc=655481,
     application_x_empty=1,
-    application_x_envoy=655481,
-    application_x_esrehber=655482,
-    application_x_excel=655483,
-    application_x_executable=655484,
-    application_x_font_gdos=655485,
-    application_x_font_pf2=655486,
-    application_x_font_pfm=655487,
-    application_x_font_sfn=655488,
-    application_x_font_ttf=655489 | 0x20000000,
-    application_x_fptapplication_x_dbt=655490,
-    application_x_freelance=655491,
-    application_x_gamecube_rom=655492,
-    application_x_gdbm=655493,
-    application_x_gettext_translation=655494,
-    application_x_git=655495,
-    application_x_gsp=655496,
-    application_x_gss=655497,
-    application_x_gtar=655498,
-    application_x_gzip=655499,
-    application_x_hdf=655500,
-    application_x_helpfile=655501,
-    application_x_httpd_imap=655502,
-    application_x_ima=655503,
-    application_x_innosetup=655504,
-    application_x_internett_signup=655505,
-    application_x_inventor=655506,
-    application_x_ip2=655507,
-    application_x_java_applet=655508,
-    application_x_java_commerce=655509,
-    application_x_java_image=655510,
-    application_x_java_jmod=655511,
-    application_x_java_keystore=655512,
-    application_x_kdelnk=655513,
-    application_x_koan=655514,
-    application_x_latex=655515,
-    application_x_livescreen=655516,
-    application_x_lotus=655517,
-    application_x_lz4=655518 | 0x08000000,
-    application_x_lz4_json=655519,
-    application_x_lzh=655520,
-    application_x_lzh_compressed=655521,
-    application_x_lzip=655522 | 0x08000000,
-    application_x_lzma=655523 | 0x08000000,
-    application_x_lzop=655524 | 0x08000000,
-    application_x_lzx=655525,
-    application_x_mach_binary=655526,
-    application_x_mach_executable=655527,
-    application_x_magic_cap_package_1_0=655528,
-    application_x_mathcad=655529,
-    application_x_maxis_dbpf=655530,
-    application_x_meme=655531,
-    application_x_midi=655532,
-    application_x_mif=655533,
-    application_x_mix_transfer=655534,
-    application_x_mobipocket_ebook=655535 | 0x02000000,
-    application_x_ms_compress_szdd=655536,
-    application_x_ms_pdb=655537,
-    application_x_ms_reader=655538,
-    application_x_msaccess=655539,
-    application_x_n64_rom=655540,
-    application_x_navi_animation=655541,
-    application_x_navidoc=655542,
-    application_x_navimap=655543,
-    application_x_navistyle=655544,
-    application_x_nes_rom=655545,
-    application_x_netcdf=655546,
-    application_x_newton_compatible_pkg=655547,
-    application_x_nintendo_ds_rom=655548,
-    application_x_object=655549,
-    application_x_omc=655550,
-    application_x_omcdatamaker=655551,
-    application_x_omcregerator=655552,
-    application_x_pagemaker=655553,
-    application_x_pcl=655554,
-    application_x_pgp_keyring=655555,
-    application_x_pixclscript=655556,
-    application_x_pkcs7_certreqresp=655557,
-    application_x_pkcs7_signature=655558,
-    application_x_project=655559,
-    application_x_qpro=655560,
-    application_x_rar=655561 | 0x10000000,
-    application_x_rpm=655562,
-    application_x_sdp=655563,
-    application_x_sea=655564,
-    application_x_seelogo=655565,
-    application_x_setupscript=655566,
-    application_x_shar=655567,
-    application_x_sharedlib=655568,
-    application_x_shockwave_flash=655569,
-    application_x_snappy_framed=655570,
-    application_x_sprite=655571,
-    application_x_sqlite3=655572,
-    application_x_stargallery_thm=655573,
-    application_x_stuffit=655574,
-    application_x_sv4cpio=655575,
-    application_x_sv4crc=655576,
-    application_x_tar=655577 | 0x10000000,
-    application_x_tbook=655578,
-    application_x_terminfo=655579,
-    application_x_terminfo2=655580,
-    application_x_tex_tfm=655581,
-    application_x_texinfo=655582,
-    application_x_ustar=655583,
-    application_x_visio=655584,
-    application_x_vnd_audioexplosion_mzz=655585,
-    application_x_vnd_ls_xpix=655586,
-    application_x_vrml=655587,
-    application_x_wais_source=655588,
-    application_x_wine_extension_ini=655589,
-    application_x_wintalk=655590,
-    application_x_world=655591,
-    application_x_wri=655592,
-    application_x_x509_ca_cert=655593,
-    application_x_xz=655594 | 0x08000000,
-    application_x_zip=655595,
-    application_x_zstd=655596 | 0x08000000,
-    application_x_zstd_dictionary=655597,
-    application_xml=655598,
-    application_zip=655599 | 0x10000000,
-    application_zlib=655600,
-    audio_basic=458993 | 0x80000000,
-    audio_it=458994,
-    audio_make=458995,
-    audio_mid=458996,
-    audio_midi=458997,
-    audio_mp4=458998,
-    audio_mpeg=458999,
-    audio_ogg=459000,
-    audio_s3m=459001,
-    audio_tsp_audio=459002,
-    audio_tsplayer=459003,
-    audio_vnd_qcelp=459004,
-    audio_voxware=459005,
-    audio_x_aiff=459006,
-    audio_x_flac=459007,
-    audio_x_gsm=459008,
-    audio_x_hx_aac_adts=459009,
-    audio_x_jam=459010,
-    audio_x_liveaudio=459011,
-    audio_x_m4a=459012,
-    audio_x_midi=459013,
-    audio_x_mod=459014,
-    audio_x_mp4a_latm=459015,
-    audio_x_mpeg_3=459016,
-    audio_x_mpequrl=459017,
-    audio_x_nspaudio=459018,
-    audio_x_pn_realaudio=459019,
-    audio_x_psid=459020,
-    audio_x_realaudio=459021,
-    audio_x_s3m=459022,
-    audio_x_twinvq=459023,
-    audio_x_twinvq_plugin=459024,
-    audio_x_voc=459025,
-    audio_x_wav=459026,
-    audio_x_xbox_executable=459027 | 0x80000000,
-    audio_x_xbox360_executable=459028 | 0x80000000,
-    audio_xm=459029,
-    font_otf=327958 | 0x20000000,
-    font_sfnt=327959 | 0x20000000,
-    font_woff=327960 | 0x20000000,
-    font_woff2=327961 | 0x20000000,
-    image_bmp=524570,
-    image_cmu_raster=524571,
-    image_fif=524572,
-    image_florian=524573,
-    image_g3fax=524574,
-    image_gif=524575,
-    image_heic=524576,
-    image_ief=524577,
-    image_jpeg=524578,
-    image_jutvision=524579,
-    image_naplps=524580,
-    image_pict=524581,
-    image_png=524582,
-    image_svg=524583 | 0x80000000,
-    image_svg_xml=524584 | 0x80000000,
-    image_tiff=524585,
-    image_vnd_adobe_photoshop=524586 | 0x80000000,
-    image_vnd_djvu=524587 | 0x80000000,
-    image_vnd_fpx=524588,
-    image_vnd_microsoft_icon=524589,
-    image_vnd_rn_realflash=524590,
-    image_vnd_rn_realpix=524591,
-    image_vnd_wap_wbmp=524592,
-    image_vnd_xiff=524593,
-    image_webp=524594,
-    image_wmf=524595,
-    image_x_3ds=524596,
-    image_x_award_bioslogo=524597,
-    image_x_cmu_raster=524598,
-    image_x_cur=524599,
-    image_x_dwg=524600,
-    image_x_eps=524601,
-    image_x_exr=524602,
-    image_x_gem=524603,
-    image_x_icns=524604,
-    image_x_icon=524605 | 0x80000000,
-    image_x_jg=524606,
-    image_x_jps=524607,
-    image_x_ms_bmp=524608,
-    image_x_niff=524609,
-    image_x_pcx=524610,
-    image_x_pict=524611,
-    image_x_portable_bitmap=524612,
-    image_x_portable_graymap=524613,
-    image_x_portable_pixmap=524614,
-    image_x_quicktime=524615,
-    image_x_rgb=524616,
-    image_x_tga=524617,
-    image_x_tiff=524618,
-    image_x_win_bitmap=524619,
-    image_x_xcf=524620 | 0x80000000,
-    image_x_xpixmap=524621 | 0x80000000,
-    image_x_xwindowdump=524622,
-    message_news=196943,
-    message_rfc822=196944,
-    model_vnd_dwf=65873,
-    model_vnd_gdl=65874,
-    model_vnd_gs_gdl=65875,
-    model_vrml=65876,
-    model_x_pov=65877,
-    text_PGP=590166,
-    text_asp=590167,
-    text_css=590168,
-    text_html=590169 | 0x01000000,
-    text_javascript=590170,
-    text_mcf=590171,
-    text_pascal=590172,
-    text_plain=590173,
-    text_richtext=590174,
-    text_rtf=590175,
-    text_scriplet=590176,
-    text_tab_separated_values=590177,
-    text_troff=590178,
-    text_uri_list=590179,
-    text_vnd_abc=590180,
-    text_vnd_fmi_flexstor=590181,
-    text_vnd_wap_wml=590182,
-    text_vnd_wap_wmlscript=590183,
-    text_webviewhtml=590184,
-    text_x_Algol68=590185,
-    text_x_asm=590186,
-    text_x_audiosoft_intra=590187,
-    text_x_awk=590188,
-    text_x_bcpl=590189,
-    text_x_c=590190,
-    text_x_c__=590191,
-    text_x_component=590192,
-    text_x_diff=590193,
-    text_x_fortran=590194,
-    text_x_java=590195,
-    text_x_la_asf=590196,
-    text_x_lisp=590197,
-    text_x_m=590198,
-    text_x_m4=590199,
-    text_x_makefile=590200,
-    text_x_ms_regedit=590201,
-    text_x_msdos_batch=590202,
-    text_x_objective_c=590203,
-    text_x_pascal=590204,
-    text_x_perl=590205,
-    text_x_php=590206,
-    text_x_po=590207,
-    text_x_python=590208,
-    text_x_ruby=590209,
-    text_x_sass=590210,
-    text_x_scss=590211,
-    text_x_server_parsed_html=590212,
-    text_x_setext=590213,
-    text_x_sgml=590214 | 0x01000000,
-    text_x_shellscript=590215,
-    text_x_speech=590216,
-    text_x_tcl=590217,
-    text_x_tex=590218,
-    text_x_uil=590219,
-    text_x_uuencode=590220,
-    text_x_vcalendar=590221,
-    text_x_vcard=590222,
-    text_xml=590223 | 0x01000000,
-    video_MP2T=393616,
-    video_animaflex=393617,
-    video_avi=393618,
-    video_avs_video=393619,
-    video_mp4=393620,
-    video_mpeg=393621,
-    video_quicktime=393622,
-    video_vdo=393623,
-    video_vivo=393624,
-    video_vnd_rn_realvideo=393625,
-    video_vosaic=393626,
-    video_webm=393627,
-    video_x_amt_demorun=393628,
-    video_x_amt_showrun=393629,
-    video_x_atomic3d_feature=393630,
-    video_x_dl=393631,
-    video_x_dv=393632,
-    video_x_fli=393633,
-    video_x_flv=393634,
-    video_x_isvideo=393635,
-    video_x_jng=393636 | 0x80000000,
-    video_x_m4v=393637,
-    video_x_matroska=393638,
-    video_x_mng=393639,
-    video_x_motion_jpeg=393640,
-    video_x_ms_asf=393641,
-    video_x_msvideo=393642,
-    video_x_qtc=393643,
-    video_x_sgi_movie=393644,
-    x_epoc_x_sisx_app=721325,
+    application_x_envoy=655483,
+    application_x_esrehber=655484,
+    application_x_excel=655485,
+    application_x_executable=655486,
+    application_x_font_gdos=655487,
+    application_x_font_pf2=655488,
+    application_x_font_pfm=655489,
+    application_x_font_sfn=655490,
+    application_x_font_ttf=655491 | 0x20000000,
+    application_x_fptapplication_x_dbt=655492,
+    application_x_freelance=655493,
+    application_x_gamecube_rom=655494,
+    application_x_gdbm=655495,
+    application_x_gettext_translation=655496,
+    application_x_git=655497,
+    application_x_gsp=655498,
+    application_x_gss=655499,
+    application_x_gtar=655500,
+    application_x_gzip=655501,
+    application_x_hdf=655502,
+    application_x_helpfile=655503,
+    application_x_httpd_imap=655504,
+    application_x_ima=655505,
+    application_x_innosetup=655506,
+    application_x_internett_signup=655507,
+    application_x_inventor=655508,
+    application_x_ip2=655509,
+    application_x_java_applet=655510,
+    application_x_java_commerce=655511,
+    application_x_java_image=655512,
+    application_x_java_jmod=655513,
+    application_x_java_keystore=655514,
+    application_x_kdelnk=655515,
+    application_x_koan=655516,
+    application_x_latex=655517,
+    application_x_livescreen=655518,
+    application_x_lotus=655519,
+    application_x_lz4=655520 | 0x08000000,
+    application_x_lz4_json=655521,
+    application_x_lzh=655522,
+    application_x_lzh_compressed=655523,
+    application_x_lzip=655524 | 0x08000000,
+    application_x_lzma=655525 | 0x08000000,
+    application_x_lzop=655526 | 0x08000000,
+    application_x_lzx=655527,
+    application_x_mach_binary=655528,
+    application_x_mach_executable=655529,
+    application_x_magic_cap_package_1_0=655530,
+    application_x_mathcad=655531,
+    application_x_maxis_dbpf=655532,
+    application_x_meme=655533,
+    application_x_midi=655534,
+    application_x_mif=655535,
+    application_x_mix_transfer=655536,
+    application_x_mobipocket_ebook=655537 | 0x02000000,
+    application_x_ms_compress_szdd=655538,
+    application_x_ms_pdb=655539,
+    application_x_ms_reader=655540,
+    application_x_msaccess=655541,
+    application_x_n64_rom=655542,
+    application_x_navi_animation=655543,
+    application_x_navidoc=655544,
+    application_x_navimap=655545,
+    application_x_navistyle=655546,
+    application_x_nes_rom=655547,
+    application_x_netcdf=655548,
+    application_x_newton_compatible_pkg=655549,
+    application_x_nintendo_ds_rom=655550,
+    application_x_object=655551,
+    application_x_omc=655552,
+    application_x_omcdatamaker=655553,
+    application_x_omcregerator=655554,
+    application_x_pagemaker=655555,
+    application_x_pcl=655556,
+    application_x_pgp_keyring=655557,
+    application_x_pixclscript=655558,
+    application_x_pkcs7_certreqresp=655559,
+    application_x_pkcs7_signature=655560,
+    application_x_project=655561,
+    application_x_qpro=655562,
+    application_x_rar=655563 | 0x10000000,
+    application_x_rpm=655564,
+    application_x_sdp=655565,
+    application_x_sea=655566,
+    application_x_seelogo=655567,
+    application_x_setupscript=655568,
+    application_x_shar=655569,
+    application_x_sharedlib=655570,
+    application_x_shockwave_flash=655571,
+    application_x_snappy_framed=655572,
+    application_x_sprite=655573,
+    application_x_sqlite3=655574,
+    application_x_stargallery_thm=655575,
+    application_x_stuffit=655576,
+    application_x_sv4cpio=655577,
+    application_x_sv4crc=655578,
+    application_x_tar=655579 | 0x10000000,
+    application_x_tbook=655580,
+    application_x_terminfo=655581,
+    application_x_terminfo2=655582,
+    application_x_tex_tfm=655583,
+    application_x_texinfo=655584,
+    application_x_ustar=655585,
+    application_x_visio=655586,
+    application_x_vnd_audioexplosion_mzz=655587,
+    application_x_vnd_ls_xpix=655588,
+    application_x_vrml=655589,
+    application_x_wais_source=655590,
+    application_x_wine_extension_ini=655591,
+    application_x_wintalk=655592,
+    application_x_world=655593,
+    application_x_wri=655594,
+    application_x_x509_ca_cert=655595,
+    application_x_xz=655596 | 0x08000000,
+    application_x_zip=655597,
+    application_x_zstd=655598 | 0x08000000,
+    application_x_zstd_dictionary=655599,
+    application_xml=655600,
+    application_zip=655601 | 0x10000000,
+    application_zlib=655602,
+    audio_basic=458995 | 0x80000000,
+    audio_it=458996,
+    audio_make=458997,
+    audio_mid=458998,
+    audio_midi=458999,
+    audio_mp4=459000,
+    audio_mpeg=459001,
+    audio_ogg=459002,
+    audio_s3m=459003,
+    audio_tsp_audio=459004,
+    audio_tsplayer=459005,
+    audio_vnd_qcelp=459006,
+    audio_voxware=459007,
+    audio_x_aiff=459008,
+    audio_x_flac=459009,
+    audio_x_gsm=459010,
+    audio_x_hx_aac_adts=459011,
+    audio_x_jam=459012,
+    audio_x_liveaudio=459013,
+    audio_x_m4a=459014,
+    audio_x_midi=459015,
+    audio_x_mod=459016,
+    audio_x_mp4a_latm=459017,
+    audio_x_mpeg_3=459018,
+    audio_x_mpequrl=459019,
+    audio_x_nspaudio=459020,
+    audio_x_pn_realaudio=459021,
+    audio_x_psid=459022,
+    audio_x_realaudio=459023,
+    audio_x_s3m=459024,
+    audio_x_twinvq=459025,
+    audio_x_twinvq_plugin=459026,
+    audio_x_voc=459027,
+    audio_x_wav=459028,
+    audio_x_xbox_executable=459029 | 0x80000000,
+    audio_x_xbox360_executable=459030 | 0x80000000,
+    audio_xm=459031,
+    font_otf=327960 | 0x20000000,
+    font_sfnt=327961 | 0x20000000,
+    font_woff=327962 | 0x20000000,
+    font_woff2=327963 | 0x20000000,
+    image_bmp=524572,
+    image_cmu_raster=524573,
+    image_fif=524574,
+    image_florian=524575,
+    image_g3fax=524576,
+    image_gif=524577,
+    image_heic=524578,
+    image_ief=524579,
+    image_jpeg=524580,
+    image_jutvision=524581,
+    image_naplps=524582,
+    image_pict=524583,
+    image_png=524584,
+    image_svg=524585 | 0x80000000,
+    image_svg_xml=524586 | 0x80000000,
+    image_tiff=524587,
+    image_vnd_adobe_photoshop=524588 | 0x80000000,
+    image_vnd_djvu=524589 | 0x80000000,
+    image_vnd_fpx=524590,
+    image_vnd_microsoft_icon=524591,
+    image_vnd_rn_realflash=524592,
+    image_vnd_rn_realpix=524593,
+    image_vnd_wap_wbmp=524594,
+    image_vnd_xiff=524595,
+    image_webp=524596,
+    image_wmf=524597,
+    image_x_3ds=524598,
+    image_x_adobe_dng=524599 | 0x00800000,
+    image_x_award_bioslogo=524600,
+    image_x_canon_cr2=524601 | 0x00800000,
+    image_x_canon_crw=524602 | 0x00800000,
+    image_x_cmu_raster=524603,
+    image_x_cur=524604,
+    image_x_dcraw=524605 | 0x00800000,
+    image_x_dwg=524606,
+    image_x_eps=524607,
+    image_x_epson_erf=524608 | 0x00800000,
+    image_x_exr=524609,
+    image_x_fuji_raf=524610 | 0x00800000,
+    image_x_gem=524611,
+    image_x_icns=524612,
+    image_x_icon=524613 | 0x80000000,
+    image_x_jg=524614,
+    image_x_jps=524615,
+    image_x_kodak_dcr=524616 | 0x00800000,
+    image_x_kodak_k25=524617 | 0x00800000,
+    image_x_kodak_kdc=524618 | 0x00800000,
+    image_x_minolta_mrw=524619 | 0x00800000,
+    image_x_ms_bmp=524620,
+    image_x_niff=524621,
+    image_x_nikon_nef=524622 | 0x00800000,
+    image_x_olympus_orf=524623 | 0x00800000,
+    image_x_panasonic_raw=524624 | 0x00800000,
+    image_x_pcx=524625,
+    image_x_pentax_pef=524626 | 0x00800000,
+    image_x_pict=524627,
+    image_x_portable_bitmap=524628,
+    image_x_portable_graymap=524629,
+    image_x_portable_pixmap=524630,
+    image_x_quicktime=524631,
+    image_x_rgb=524632,
+    image_x_sigma_x3f=524633 | 0x00800000,
+    image_x_sony_arw=524634 | 0x00800000,
+    image_x_sony_sr2=524635 | 0x00800000,
+    image_x_sony_srf=524636 | 0x00800000,
+    image_x_tga=524637,
+    image_x_tiff=524638,
+    image_x_win_bitmap=524639,
+    image_x_xcf=524640 | 0x80000000,
+    image_x_xpixmap=524641 | 0x80000000,
+    image_x_xwindowdump=524642,
+    message_news=196963,
+    message_rfc822=196964,
+    model_vnd_dwf=65893,
+    model_vnd_gdl=65894,
+    model_vnd_gs_gdl=65895,
+    model_vrml=65896,
+    model_x_pov=65897,
+    text_PGP=590186,
+    text_asp=590187,
+    text_css=590188,
+    text_html=590189 | 0x01000000,
+    text_javascript=590190,
+    text_mcf=590191,
+    text_pascal=590192,
+    text_plain=590193,
+    text_richtext=590194,
+    text_rtf=590195,
+    text_scriplet=590196,
+    text_tab_separated_values=590197,
+    text_troff=590198,
+    text_uri_list=590199,
+    text_vnd_abc=590200,
+    text_vnd_fmi_flexstor=590201,
+    text_vnd_wap_wml=590202,
+    text_vnd_wap_wmlscript=590203,
+    text_webviewhtml=590204,
+    text_x_Algol68=590205,
+    text_x_asm=590206,
+    text_x_audiosoft_intra=590207,
+    text_x_awk=590208,
+    text_x_bcpl=590209,
+    text_x_c=590210,
+    text_x_c__=590211,
+    text_x_component=590212,
+    text_x_diff=590213,
+    text_x_fortran=590214,
+    text_x_java=590215,
+    text_x_la_asf=590216,
+    text_x_lisp=590217,
+    text_x_m=590218,
+    text_x_m4=590219,
+    text_x_makefile=590220,
+    text_x_ms_regedit=590221,
+    text_x_msdos_batch=590222,
+    text_x_objective_c=590223,
+    text_x_pascal=590224,
+    text_x_perl=590225,
+    text_x_php=590226,
+    text_x_po=590227,
+    text_x_python=590228,
+    text_x_ruby=590229,
+    text_x_sass=590230,
+    text_x_scss=590231,
+    text_x_server_parsed_html=590232,
+    text_x_setext=590233,
+    text_x_sgml=590234 | 0x01000000,
+    text_x_shellscript=590235,
+    text_x_speech=590236,
+    text_x_tcl=590237,
+    text_x_tex=590238,
+    text_x_uil=590239,
+    text_x_uuencode=590240,
+    text_x_vcalendar=590241,
+    text_x_vcard=590242,
+    text_xml=590243 | 0x01000000,
+    video_MP2T=393636,
+    video_animaflex=393637,
+    video_avi=393638,
+    video_avs_video=393639,
+    video_mp4=393640,
+    video_mpeg=393641,
+    video_quicktime=393642,
+    video_vdo=393643,
+    video_vivo=393644,
+    video_vnd_rn_realvideo=393645,
+    video_vosaic=393646,
+    video_webm=393647,
+    video_x_amt_demorun=393648,
+    video_x_amt_showrun=393649,
+    video_x_atomic3d_feature=393650,
+    video_x_dl=393651,
+    video_x_dv=393652,
+    video_x_fli=393653,
+    video_x_flv=393654,
+    video_x_isvideo=393655,
+    video_x_jng=393656 | 0x80000000,
+    video_x_m4v=393657,
+    video_x_matroska=393658,
+    video_x_mng=393659,
+    video_x_motion_jpeg=393660,
+    video_x_ms_asf=393661,
+    video_x_msvideo=393662,
+    video_x_qtc=393663,
+    video_x_sgi_movie=393664,
+    x_epoc_x_sisx_app=721345,
 };
 char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
 case application_arj: return "application/arj";
@@ -785,6 +805,7 @@ case text_mcf: return "text/mcf";
 case text_pascal: return "text/pascal";
 case text_PGP: return "text/PGP";
 case text_plain: return "text/plain";
+case application_vnd_coffeescript: return "application/vnd.coffeescript";
 case text_richtext: return "text/richtext";
 case text_rtf: return "text/rtf";
 case text_scriplet: return "text/scriplet";
@@ -866,6 +887,25 @@ case video_x_qtc: return "video/x-qtc";
 case video_x_sgi_movie: return "video/x-sgi-movie";
 case x_epoc_x_sisx_app: return "x-epoc/x-sisx-app";
 case application_x_zstd_dictionary: return "application/x-zstd-dictionary";
+case application_vnd_ms_outlook: return "application/vnd.ms-outlook";
+case image_x_olympus_orf: return "image/x-olympus-orf";
+case image_x_nikon_nef: return "image/x-nikon-nef";
+case image_x_fuji_raf: return "image/x-fuji-raf";
+case image_x_panasonic_raw: return "image/x-panasonic-raw";
+case image_x_adobe_dng: return "image/x-adobe-dng";
+case image_x_canon_cr2: return "image/x-canon-cr2";
+case image_x_canon_crw: return "image/x-canon-crw";
+case image_x_dcraw: return "image/x-dcraw";
+case image_x_kodak_dcr: return "image/x-kodak-dcr";
+case image_x_kodak_k25: return "image/x-kodak-k25";
+case image_x_kodak_kdc: return "image/x-kodak-kdc";
+case image_x_minolta_mrw: return "image/x-minolta-mrw";
+case image_x_pentax_pef: return "image/x-pentax-pef";
+case image_x_sigma_x3f: return "image/x-sigma-x3f";
+case image_x_sony_arw: return "image/x-sony-arw";
+case image_x_sony_sr2: return "image/x-sony-sr2";
+case image_x_sony_srf: return "image/x-sony-srf";
+case image_x_epson_erf: return "image/x-epson-erf";
 default: return NULL;}}
 GHashTable *mime_get_ext_table() {GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(ext_table, "arj", (gpointer)application_arj);
@@ -883,6 +923,7 @@ g_hash_table_insert(ext_table, "frl", (gpointer)application_freeloader);
 g_hash_table_insert(ext_table, "spl", (gpointer)application_futuresplash);
 g_hash_table_insert(ext_table, "vew", (gpointer)application_groupwise);
 g_hash_table_insert(ext_table, "gz", (gpointer)application_gzip);
+g_hash_table_insert(ext_table, "tgz", (gpointer)application_gzip);
 g_hash_table_insert(ext_table, "hta", (gpointer)application_hta);
 g_hash_table_insert(ext_table, "unv", (gpointer)application_i_deas);
 g_hash_table_insert(ext_table, "iges", (gpointer)application_iges);
@@ -997,7 +1038,6 @@ g_hash_table_insert(ext_table, "cpio", (gpointer)application_x_cpio);
 g_hash_table_insert(ext_table, "dbf", (gpointer)application_x_dbf);
 g_hash_table_insert(ext_table, "deb", (gpointer)application_x_debian_package);
 g_hash_table_insert(ext_table, "deepv", (gpointer)application_x_deepv);
-g_hash_table_insert(ext_table, "dcr", (gpointer)application_x_director);
 g_hash_table_insert(ext_table, "dir", (gpointer)application_x_director);
 g_hash_table_insert(ext_table, "dxr", (gpointer)application_x_director);
 g_hash_table_insert(ext_table, "dmp", (gpointer)application_x_dmp);
@@ -1277,6 +1317,11 @@ g_hash_table_insert(ext_table, "sfv", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "m3u", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "csv", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "eml", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "make", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "log", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "markdown", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "yaml", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "coffee", (gpointer)application_vnd_coffeescript);
 g_hash_table_insert(ext_table, "rt", (gpointer)text_richtext);
 g_hash_table_insert(ext_table, "rtf", (gpointer)text_richtext);
 g_hash_table_insert(ext_table, "rtx", (gpointer)text_richtext);
@@ -1385,6 +1430,25 @@ g_hash_table_insert(ext_table, "divx", (gpointer)video_x_msvideo);
 g_hash_table_insert(ext_table, "qtc", (gpointer)video_x_qtc);
 g_hash_table_insert(ext_table, "movie", (gpointer)video_x_sgi_movie);
 g_hash_table_insert(ext_table, "mv", (gpointer)video_x_sgi_movie);
+g_hash_table_insert(ext_table, "msg", (gpointer)application_vnd_ms_outlook);
+g_hash_table_insert(ext_table, "orf", (gpointer)image_x_olympus_orf);
+g_hash_table_insert(ext_table, "nef", (gpointer)image_x_nikon_nef);
+g_hash_table_insert(ext_table, "raf", (gpointer)image_x_fuji_raf);
+g_hash_table_insert(ext_table, "rw2", (gpointer)image_x_panasonic_raw);
+g_hash_table_insert(ext_table, "raw", (gpointer)image_x_panasonic_raw);
+g_hash_table_insert(ext_table, "dng", (gpointer)image_x_adobe_dng);
+g_hash_table_insert(ext_table, "cr2", (gpointer)image_x_canon_cr2);
+g_hash_table_insert(ext_table, "crw", (gpointer)image_x_canon_crw);
+g_hash_table_insert(ext_table, "dcr", (gpointer)image_x_kodak_dcr);
+g_hash_table_insert(ext_table, "k25", (gpointer)image_x_kodak_k25);
+g_hash_table_insert(ext_table, "kdc", (gpointer)image_x_kodak_kdc);
+g_hash_table_insert(ext_table, "mrw", (gpointer)image_x_minolta_mrw);
+g_hash_table_insert(ext_table, "pef", (gpointer)image_x_pentax_pef);
+g_hash_table_insert(ext_table, "xf3", (gpointer)image_x_sigma_x3f);
+g_hash_table_insert(ext_table, "arw", (gpointer)image_x_sony_arw);
+g_hash_table_insert(ext_table, "sr2", (gpointer)image_x_sony_sr2);
+g_hash_table_insert(ext_table, "srf", (gpointer)image_x_sony_srf);
+g_hash_table_insert(ext_table, "erf", (gpointer)image_x_epson_erf);
 return ext_table;}
 GHashTable *mime_get_mime_table() {GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(mime_table, "application/arj", (gpointer)application_arj);
@@ -1735,6 +1799,7 @@ g_hash_table_insert(mime_table, "text/mcf", (gpointer)text_mcf);
 g_hash_table_insert(mime_table, "text/pascal", (gpointer)text_pascal);
 g_hash_table_insert(mime_table, "text/PGP", (gpointer)text_PGP);
 g_hash_table_insert(mime_table, "text/plain", (gpointer)text_plain);
+g_hash_table_insert(mime_table, "application/vnd.coffeescript", (gpointer)application_vnd_coffeescript);
 g_hash_table_insert(mime_table, "text/richtext", (gpointer)text_richtext);
 g_hash_table_insert(mime_table, "text/rtf", (gpointer)text_rtf);
 g_hash_table_insert(mime_table, "text/scriplet", (gpointer)text_scriplet);
@@ -1816,5 +1881,24 @@ g_hash_table_insert(mime_table, "video/x-qtc", (gpointer)video_x_qtc);
 g_hash_table_insert(mime_table, "video/x-sgi-movie", (gpointer)video_x_sgi_movie);
 g_hash_table_insert(mime_table, "x-epoc/x-sisx-app", (gpointer)x_epoc_x_sisx_app);
 g_hash_table_insert(mime_table, "application/x-zstd-dictionary", (gpointer)application_x_zstd_dictionary);
+g_hash_table_insert(mime_table, "application/vnd.ms-outlook", (gpointer)application_vnd_ms_outlook);
+g_hash_table_insert(mime_table, "image/x-olympus-orf", (gpointer)image_x_olympus_orf);
+g_hash_table_insert(mime_table, "image/x-nikon-nef", (gpointer)image_x_nikon_nef);
+g_hash_table_insert(mime_table, "image/x-fuji-raf", (gpointer)image_x_fuji_raf);
+g_hash_table_insert(mime_table, "image/x-panasonic-raw", (gpointer)image_x_panasonic_raw);
+g_hash_table_insert(mime_table, "image/x-adobe-dng", (gpointer)image_x_adobe_dng);
+g_hash_table_insert(mime_table, "image/x-canon-cr2", (gpointer)image_x_canon_cr2);
+g_hash_table_insert(mime_table, "image/x-canon-crw", (gpointer)image_x_canon_crw);
+g_hash_table_insert(mime_table, "image/x-dcraw", (gpointer)image_x_dcraw);
+g_hash_table_insert(mime_table, "image/x-kodak-dcr", (gpointer)image_x_kodak_dcr);
+g_hash_table_insert(mime_table, "image/x-kodak-k25", (gpointer)image_x_kodak_k25);
+g_hash_table_insert(mime_table, "image/x-kodak-kdc", (gpointer)image_x_kodak_kdc);
+g_hash_table_insert(mime_table, "image/x-minolta-mrw", (gpointer)image_x_minolta_mrw);
+g_hash_table_insert(mime_table, "image/x-pentax-pef", (gpointer)image_x_pentax_pef);
+g_hash_table_insert(mime_table, "image/x-sigma-x3f", (gpointer)image_x_sigma_x3f);
+g_hash_table_insert(mime_table, "image/x-sony-arw", (gpointer)image_x_sony_arw);
+g_hash_table_insert(mime_table, "image/x-sony-sr2", (gpointer)image_x_sony_sr2);
+g_hash_table_insert(mime_table, "image/x-sony-srf", (gpointer)image_x_sony_srf);
+g_hash_table_insert(mime_table, "image/x-epson-erf", (gpointer)image_x_epson_erf);
 return mime_table;}
 #endif

src/parsing/parse.c

@@ -38,6 +38,8 @@ void fs_reset(struct vfile *f) {
     }
 }
 
+#define IS_GIT_OBJ (strlen(doc.filepath + doc.base) == 38 && (strstr(doc.filepath, "objects") != NULL))
+
 void parse(void *arg) {
 
     parse_job_t *job = arg;
@@ -49,7 +51,6 @@ void parse(void *arg) {
         return;
     }
 
-
     doc.filepath = job->filepath;
     doc.ext = (short) job->ext;
     doc.base = (short) job->base;
@@ -61,7 +62,7 @@ void parse(void *arg) {
     doc.mtime = job->vfile.info.st_mtim.tv_sec;
 
     uuid_generate(doc.uuid);
-    char *buf[PARSE_BUF_SIZE];
+    char *buf[MAGIC_BUF_SIZE];
 
     if (LogCtx.very_verbose) {
         char uuid_str[UUID_STR_LEN];
@@ -78,8 +79,17 @@ void parse(void *arg) {
     int bytes_read = 0;
 
     if (doc.mime == 0 && !ScanCtx.fast) {
+        if (IS_GIT_OBJ) {
+            goto abort;
+        }
+
         // Get mime type with libmagic
-        bytes_read = job->vfile.read(&job->vfile, buf, PARSE_BUF_SIZE);
+        if (!job->vfile.is_fs_file) {
+            LOG_WARNING(job->filepath, "Guessing mime type with libmagic inside archive files is not currently supported");
+            goto abort;
+        }
+
+        bytes_read = job->vfile.read(&job->vfile, buf, MAGIC_BUF_SIZE);
         if (bytes_read < 0) {
 
             if (job->vfile.is_fs_file) {
@@ -115,6 +125,8 @@ void parse(void *arg) {
 
     if (!(SHOULD_PARSE(doc.mime))) {
 
+    } else if (IS_RAW(doc.mime)) {
+        parse_raw(&ScanCtx.raw_ctx, &job->vfile, &doc);
     } else if ((mmime == MimeVideo && doc.size >= MIN_VIDEO_SIZE) ||
                (mmime == MimeImage && doc.size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) {
 
@@ -139,23 +151,21 @@ void parse(void *arg) {
                     (IS_ARC_FILTER(doc.mime) && should_parse_filtered_file(doc.filepath, doc.ext))
             )) {
         parse_archive(&ScanCtx.arc_ctx, &job->vfile, &doc);
-    } else if (ScanCtx.ooxml_ctx.content_size > 0 && IS_DOC(doc.mime)) {
+    } else if ((ScanCtx.ooxml_ctx.content_size > 0 || ScanCtx.media_ctx.tn_size > 0) && IS_DOC(doc.mime)) {
         parse_ooxml(&ScanCtx.ooxml_ctx, &job->vfile, &doc);
-
-    } else if (is_cbr(&ScanCtx.cbr_ctx, doc.mime)) {
-        parse_cbr(&ScanCtx.cbr_ctx, &job->vfile, &doc);
+    } else if (is_cbr(&ScanCtx.comic_ctx, doc.mime) || is_cbz(&ScanCtx.comic_ctx, doc.mime)) {
+        parse_comic(&ScanCtx.comic_ctx, &job->vfile, &doc);
     } else if (IS_MOBI(doc.mime)) {
         parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, &doc);
     }
 
+    abort:
+
     //Parent meta
     if (!uuid_is_null(job->parent)) {
-        char tmp[UUID_STR_LEN];
-        uuid_unparse(job->parent, tmp);
-
         meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + UUID_STR_LEN + 1);
         meta_parent->key = MetaParent;
-        strcpy(meta_parent->str_val, tmp);
+        uuid_unparse(job->parent, meta_parent->str_val);
         APPEND_META((&doc), meta_parent)
     }
 

src/parsing/parse.h

@@ -3,7 +3,7 @@
 
 #include "../sist.h"
 
-#define PARSE_BUF_SIZE 4096
+#define MAGIC_BUF_SIZE 4096 * 6
 
 int fs_read(struct vfile *f, void *buf, size_t size);
 void fs_close(struct vfile *f);

src/static/css/autocomplete.min.css

@@ -0,0 +1,4 @@
+.autocomplete-suggestions { text-align: left; cursor: default; border: 1px solid #ccc; border-top: 0; background: #fff; box-shadow: -1px 1px 3px rgba(0,0,0,.1); position: absolute; display: none; z-index: 9999; max-height: 254px; overflow: hidden; overflow-y: auto; box-sizing: border-box; }
+.autocomplete-suggestion { position: relative; padding: 0 .6em; line-height: 23px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; font-size: 1.02em; color: #333; }
+.autocomplete-suggestion b { font-weight: normal; color: #1f8dd6; }
+.autocomplete-suggestion.selected { background: #f0f0f0; }

src/static/css/dark.css

@@ -121,7 +121,7 @@ body {
     background: #546b7a;
 }
 
-.navbar a:hover {
+a:hover, .btn:hover {
     color: #fff;
 }
 
@@ -130,7 +130,11 @@ body {
 }
 
 .document {
-    padding: 0.5rem;
+    padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }
 
 .document p {
@@ -166,6 +170,12 @@ body {
     background-color: #FAAB3C;
 }
 
+.add-tag-button {
+    cursor: pointer;
+    color: #212529;
+    background-color: #e0e0e0;
+}
+
 .card-img-overlay {
     pointer-events: none;
     padding: 0.75rem;
@@ -191,22 +201,41 @@ body {
     margin-right: 3px;
 }
 
+.badge-delete {
+    margin-right: -2px;
+    margin-left: 2px;
+    margin-top: -1px;
+    font-family: monospace;
+    font-size: 90%;
+    background: rgba(0, 0, 0, 0.2);
+    padding: 0.1em 0.4em;
+    color: white;
+    cursor: pointer;
+}
+
 .badge-user {
     color: #212529;
     background-color: #e0e0e0;
 }
 
+.card-img-top {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+}
+
 .fit {
     display: block;
     min-width: 64px;
     max-width: 100%;
-    max-height: 175px;
+    max-height: 400px;
     margin: 0 auto 0;
-    padding: 3px 3px 0;
-    width: auto;
     height: auto;
 }
 
+.img-padding {
+    padding: 4px 4px 0 4px;
+}
+
 .fit-sm {
     display: block;
     max-width: 64px;
@@ -223,20 +252,6 @@ body {
     width: 100%;
 }
 
-@media screen and (min-width: 1500px) {
-    .container {
-        max-width: 1440px;
-    }
-
-    .bricklayer-column-sizer {
-        width: 20% !important;
-    }
-
-    .bricklayer-column {
-        max-width: 20%;
-    }
-}
-
 @media screen and (min-width: 1800px) {
     .container {
         max-width: 1550px;
@@ -266,6 +281,7 @@ mark {
     margin: 3px;
     white-space: normal;
     color: rgb(224, 224, 224);
+    overflow: hidden;
 }
 
 .irs-single, .irs-from, .irs-to {
@@ -432,6 +448,7 @@ option {
     .small-btn {
         display: none;
     }
+
     .large-btn {
         display: inherit;
     }
@@ -441,6 +458,7 @@ option {
     .small-btn {
         display: inherit;
     }
+
     .large-btn {
         display: none;
     }
@@ -480,3 +498,42 @@ svg {
     cursor: pointer;
 }
 
+.stats-card {
+    text-align: center;
+    margin-top: 1em;
+    padding: 1em;
+
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
+
+    background: #212121;
+}
+
+.graph {
+    display: inline-block;
+    width: 40%;
+}
+
+.full-screen {
+    position: absolute;
+    left: 0;
+    width: 100%;
+}
+
+.stats-btn {
+    float: right;
+    margin-bottom: 10px;
+}
+
+#graphs-card svg text {
+    fill: #eee;
+}
+
+.wholerow {
+    outline: none !important;
+}
+
+.stat > .card-body {
+    padding: 0.7em 1.25em;
+}

src/static/css/light.css

@@ -70,7 +70,11 @@ body {
 }
 
 .document {
-    padding: 0.5rem;
+   padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }
 
 .document p {
@@ -106,11 +110,33 @@ body {
     background-color: #e0e0e0;
 }
 
+.badge {
+    margin-right: 3px;
+}
+
+.badge-delete {
+    margin-right: -2px;
+    margin-left: 2px;
+    margin-top: -1px;
+    font-family: monospace;
+    font-size: 90%;
+    background: rgba(0,0,0,0.2);
+    padding: 0.1em 0.4em;
+    color: white;
+    cursor: pointer;
+}
+
 .badge-text {
     color: #FFFFFF;
     background-color: #FAAB3C;
 }
 
+.add-tag-button {
+    cursor: pointer;
+    color: #212529;
+    background-color: #e0e0e0;
+}
+
 .card-img-overlay {
     pointer-events: none;
     padding: 0.75rem;
@@ -131,21 +157,25 @@ body {
     overflow: hidden;
 }
 
-.badge {
-    margin-right: 3px;
+.card-img-top {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
 }
 
 .fit {
     display: block;
     min-width: 64px;
     max-width: 100%;
-    max-height: 175px;
+    max-height: 400px;
     margin: 0 auto 0;
-    padding: 3px 3px 0 3px;
     width: auto;
     height: auto;
 }
 
+.img-padding {
+    padding: 4px 4px 0 4px;
+}
+
 .fit-sm {
     display: block;
     max-width: 64px;
@@ -162,6 +192,10 @@ body {
     width: 100%;
 }
 
+.bricklayer {
+    /*max-width: 100%;*/
+}
+
 @media screen and (max-width: 1200px) {
     .bricklayer-column {
         max-width: 100%;
@@ -205,6 +239,7 @@ mark {
     margin: 3px;
     white-space: normal;
     color: #000;
+    overflow: hidden;
 }
 
 .irs-single, .irs-from, .irs-to {
@@ -347,3 +382,38 @@ mark {
     cursor: pointer;
 }
 
+.stats-card {
+    text-align: center;
+    margin-top: 1em;
+    padding: 1em;
+
+    box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
+    border-radius: 0;
+    border: none;
+
+    background: #fff;
+}
+
+.graph {
+    display: inline-block;
+    width: 40%;
+}
+
+.full-screen {
+    position: absolute;
+    left: 0;
+    width: 100%;
+}
+
+.stats-btn {
+    float: right;
+    margin-bottom: 10px;
+}
+
+.wholerow {
+    outline: none !important;
+}
+
+.stat > .card-body {
+    padding: 0.7em 1.25em;
+}

src/static/js/dom.js

@@ -27,18 +27,24 @@ function gifOver(thumbnail, hit) {
 }
 
 function getContentHighlight(hit) {
-    const re = RegExp(/<mark>/g);
-
-    const sortByMathCount = (a, b) => {
-        return b.match(re).length - a.match(re).length;
-    };
-
     if (hit.hasOwnProperty("highlight")) {
         if (hit["highlight"].hasOwnProperty("content")) {
-            return hit["highlight"]["content"].sort(sortByMathCount)[0];
+            return hit["highlight"]["content"][0];
 
         } else if (hit["highlight"].hasOwnProperty("content.nGram")) {
-            return hit["highlight"]["content.nGram"].sort(sortByMathCount)[0];
+            return hit["highlight"]["content.nGram"][0];
+        }
+    }
+
+    return undefined;
+}
+
+function getPathHighlight(hit) {
+    if (hit.hasOwnProperty("highlight")) {
+        if (hit["highlight"].hasOwnProperty("path.text")) {
+            return hit["highlight"]["path.text"][0];
+        } else if (hit["highlight"].hasOwnProperty("path.nGram")) {
+            return hit["highlight"]["path.nGram"][0];
         }
     }
 
@@ -65,6 +71,7 @@ function shouldPlayVideo(hit) {
 
     return mime &&
         mime.startsWith("video/") &&
+        !("parent" in hit["_source"]) &&
         hit["_source"]["extension"] !== "mkv" &&
         hit["_source"]["extension"] !== "avi" &&
         videoc !== "hevc" &&
@@ -80,6 +87,7 @@ function shouldDisplayRawImage(hit) {
         hit["_source"]["mime"] &&
         !hit["_source"]["parent"] &&
         hit["_source"]["videoc"] !== "tiff" &&
+        hit["_source"]["videoc"] !== "raw" &&
         hit["_source"]["videoc"] !== "ppm";
 }
 
@@ -145,26 +153,44 @@ function getTags(hit, mimeCategory) {
     // User tags
     if (hit["_source"].hasOwnProperty("tag")) {
         hit["_source"]["tag"].forEach(tag => {
-            const userTag = document.createElement("span");
-            userTag.setAttribute("class", "badge badge-pill badge-user");
-
-            const tokens = tag.split("#");
-
-            if (tokens.length > 1) {
-                const bg = "#" + tokens[1];
-                const fg = lum(tokens[1]) > 40 ? "#000" : "#fff";
-                userTag.setAttribute("style", `background-color: ${bg}; color: ${fg}`);
-            }
-
-            const name = tokens[0].split(".")[tokens[0].split(".").length - 1];
-            userTag.appendChild(document.createTextNode(name));
-            tags.push(userTag);
+            tags.push(makeUserTag(tag, hit));
         })
     }
 
     return tags
 }
 
+function makeUserTag(tag, hit) {
+    const userTag = document.createElement("span");
+    userTag.setAttribute("class", "badge badge-pill badge-user");
+    userTag.setAttribute("title", tag.split("#")[0])
+
+    const tokens = tag.split("#");
+
+    if (tokens.length > 1) {
+        const bg = "#" + tokens[1];
+        const fg = lum(tokens[1]) > 50 ? "#000" : "#fff";
+        userTag.setAttribute("style", `background-color: ${bg}; color: ${fg}`);
+    }
+
+    const deleteButton = document.createElement("span");
+    deleteButton.setAttribute("class", "badge badge-pill badge-delete")
+    deleteButton.setAttribute("title", "Delete tag")
+    deleteButton.appendChild(document.createTextNode("X"));
+    deleteButton.addEventListener("click", () => {
+        deleteTag(tag, hit).then(() => {
+            userTag.remove();
+        });
+    });
+    userTag.addEventListener("mouseenter", () => userTag.appendChild(deleteButton));
+    userTag.addEventListener("mouseleave", () => deleteButton.remove());
+
+    const name = tokens[0].split(".")[tokens[0].split(".").length - 1];
+    userTag.appendChild(document.createTextNode(name));
+
+    return userTag;
+}
+
 function infoButtonCb(hit) {
     return () => {
         getDocumentInfo(hit["_id"]).then(doc => {
@@ -184,7 +210,8 @@ function infoButtonCb(hit) {
 
             const displayFields = new Set([
                 "mime", "size", "mtime", "path", "title", "width", "height", "duration", "audioc", "videoc",
-                "bitrate", "artist", "album", "album_artist", "genre", "title", "font_name", "tag"
+                "bitrate", "artist", "album", "album_artist", "genre", "title", "font_name", "tag", "author",
+                "modified_by"
             ]);
             Object.keys(doc)
                 .filter(key => key.startsWith("_keyword.") || key.startsWith("_text.") || displayFields.has(key) || key.startsWith("exif_"))
@@ -330,9 +357,31 @@ function createDocCard(hit) {
 
     docCardBody.appendChild(tagContainer);
 
+    attachTagContainerEventListener(tagContainer, hit);
     return docCard;
 }
 
+function attachTagContainerEventListener(tagContainer, hit) {
+    const sizeTag = Array.from(tagContainer.children).find(child => child.tagName === "SMALL");
+
+    const addTagButton = document.createElement("span");
+    addTagButton.setAttribute("class", "badge badge-pill add-tag-button");
+    addTagButton.appendChild(document.createTextNode("+Add"));
+
+    tagContainer.addEventListener("mouseenter", () => tagContainer.insertBefore(addTagButton, sizeTag));
+    tagContainer.addEventListener("mouseleave", () => addTagButton.remove());
+
+    addTagButton.addEventListener("click", () => {
+        tagBar.value = "";
+        currentDocToTag = hit;
+        currentTagCallback = tag => {
+            tagContainer.insertBefore(makeUserTag(tag, hit), sizeTag);
+        }
+        $("#tagModal").modal("show");
+        tagBar.focus();
+    });
+}
+
 function makeThumbnail(mimeCategory, hit, imgWrapper, small) {
 
     if (!hit["_source"].hasOwnProperty("thumbnail")) {
@@ -343,7 +392,11 @@ function makeThumbnail(mimeCategory, hit, imgWrapper, small) {
     if (small) {
         thumbnail.setAttribute("class", "fit-sm");
     } else {
-        thumbnail.setAttribute("class", "card-img-top fit");
+        if (hit["_source"].hasOwnProperty("parent")) {
+            thumbnail.setAttribute("class", "card-img-top fit img-padding");
+        } else {
+            thumbnail.setAttribute("class", "card-img-top fit");
+        }
     }
     thumbnail.setAttribute("src", `t/${hit["_source"]["index"]}/${hit["_id"]}`);
 
@@ -405,7 +458,6 @@ function createDocLine(hit) {
 
     if (hit["_source"].hasOwnProperty("parent")) {
         line.classList.add("sub-document");
-        isSubDocument = true;
     }
 
     const infoButton = makeInfoButton(hit);
@@ -443,7 +495,7 @@ function createDocLine(hit) {
     if (contentHl !== undefined) {
         const contentDiv = document.createElement("div");
         contentDiv.setAttribute("class", "content-div");
-        contentDiv.insertAdjacentHTML('afterbegin', contentHl);
+        contentDiv.insertAdjacentHTML("afterbegin", contentHl);
         titleDiv.appendChild(contentDiv);
     }
 
@@ -453,7 +505,13 @@ function createDocLine(hit) {
     let path = document.createElement("div");
     path.setAttribute("class", "path-line");
     path.setAttribute("title", hit["_source"]["path"] + "/");
-    path.appendChild(document.createTextNode(hit["_source"]["path"] + "/"));
+
+    const pathHighlight = getPathHighlight(hit);
+    if (pathHighlight) {
+        path.insertAdjacentHTML("afterbegin", pathHighlight + "/");
+    } else {
+        path.appendChild(document.createTextNode(hit["_source"]["path"] + "/"));
+    }
 
     let tagContainer = document.createElement("div");
     tagContainer.setAttribute("class", "tag-container");
@@ -472,6 +530,8 @@ function createDocLine(hit) {
     pathLine.appendChild(path);
     pathLine.appendChild(tagContainer);
 
+    attachTagContainerEventListener(tagContainer, hit);
+
     return line;
 }
 

src/static/js/search.js

@@ -1,4 +1,4 @@
-const SIZE = 40;
+const SIZE = 60;
 let mimeMap = [];
 let tagMap = [];
 let mimeTree;
@@ -6,6 +6,9 @@ let tagTree;
 
 let searchBar = document.getElementById("searchBar");
 let pathBar = document.getElementById("pathBar");
+let tagBar = document.getElementById("tagBar");
+let currentDocToTag = null;
+let currentTagCallback = null;
 let lastDoc = null;
 let reachedEnd = false;
 let docCount = 0;
@@ -20,15 +23,6 @@ let size_max = 10000000000000;
 let date_min = null;
 let date_max = null;
 
-const CONF = new Settings();
-
-const _defaults = {
-    display: "grid",
-    fuzzy: true,
-    highlight: true,
-    sort: "score"
-};
-
 SORT_MODES = {
     score: {
         text: "Relevance",
@@ -66,32 +60,7 @@ SORT_MODES = {
         ],
         key: hit => hit["_source"]["size"]
     },
-}
-
-function Settings() {
-    this.options = {};
-
-    this._onUpdate = function () {
-        $("#fuzzyToggle").prop("checked", this.options.fuzzy);
-    }
-
-    this.load = function () {
-        const raw = window.localStorage.getItem("options");
-        if (raw === null) {
-            this.options = _defaults;
-        } else {
-            this.options = JSON.parse(raw);
-        }
-
-        this._onUpdate();
-    }
-
-    this.save = function () {
-        window.localStorage.setItem("options", JSON.stringify(this.options));
-        this._onUpdate();
-    }
-}
-
+};
 
 function showEsError() {
     $.toast({
@@ -106,35 +75,149 @@ function showEsError() {
     });
 }
 
-jQuery["jsonPost"] = function (url, data) {
-    return jQuery.ajax({
-        url: url,
-        type: "post",
-        data: JSON.stringify(data),
-        contentType: "application/json"
-    }).fail(err => {
-        showEsError();
-        console.log(err);
+window.onload = () => {
+    CONF.load();
+    new autoComplete({
+        selector: '#pathBar',
+        minChars: 1,
+        delay: 400,
+        renderItem: function (item) {
+            return '<div class="autocomplete-suggestion" data-val="' + item + '">' + item + '</div>';
+        },
+        source: async function (term, suggest) {
+
+            if (!CONF.options.suggestPath) {
+                return []
+            }
+
+            term = term.toLowerCase();
+
+            const choices = await getPathChoices();
+
+            let matches = [];
+            for (let i = 0; i < choices.length; i++) {
+                if (~choices[i].toLowerCase().indexOf(term)) {
+                    matches.push(choices[i]);
+                }
+            }
+            suggest(matches.sort());
+        },
+        onSelect: function () {
+            searchDebounced();
+        }
+    });
+    searchBar.addEventListener("keyup", searchDebounced);
+    pathBar.addEventListener("keyup", e => {
+        if (e.key === "Enter") {
+            searchDebounced();
+        }
+    });
+    new autoComplete({
+        selector: '#tagBar',
+        minChars: 1,
+        delay: 200,
+        renderItem: function (item) {
+            return '<div class="autocomplete-suggestion" data-val="' + item + '">' + item.split("#")[0] + '</div>';
+        },
+        source: async function (term, suggest) {
+            term = term.toLowerCase();
+
+            const choices = await getTagChoices();
+
+            let matches = [];
+            for (let i = 0; i < choices.length; i++) {
+                if (~choices[i].toLowerCase().indexOf(term)) {
+                    matches.push(choices[i]);
+                }
+            }
+            suggest(matches.sort());
+        },
+        onSelect: function (e, item) {
+            const name = item.split("#")[0];
+            const color = "#" + item.split("#")[1];
+            $("#tag-color").val(color);
+            $("#tag-color").trigger("keyup", color);
+            tagBar.value = name;
+            e.preventDefault();
+        }
+    });
+    [tagBar, document.getElementById("tag-color")].forEach(elem => {
+        elem.addEventListener("keyup", e => {
+            if (e.key === "Enter" && tagBar.value.length > 0) {
+                const tag = tagBar.value + document.getElementById("tag-color").value;
+                saveTag(tag, currentDocToTag).then(() => currentTagCallback(tag));
+            }
+        });
+    })
+    $("#tag-color").colorpicker({
+        format: "hex",
+        sliders: {
+            saturation: {
+                selector: '.colorpicker-saturation',
+                callLeft: 'setSaturationRatio',
+                callTop: 'setValueRatio'
+            },
+            hue: {
+                selector: '.colorpicker-hue',
+                maxLeft: 0,
+                callLeft: false,
+                callTop: 'setHueRatio'
+            }
+        }
     });
 };
 
-window.onload = () => {
-    $("#theme").on("click", () => {
-        if (!document.cookie.includes("sist")) {
-            document.cookie = "sist=dark";
-        } else {
-            document.cookie = "sist=; Max-Age=-99999999;";
-        }
-        window.location.reload();
+function saveTag(tag, hit) {
+    const relPath = hit["_source"]["path"] + "/" + hit["_source"]["name"] + ext(hit);
+
+    return $.jsonPost("/tag/" + hit["_source"]["index"], {
+        delete: false,
+        name: tag,
+        doc_id: hit["_id"],
+        relpath: relPath
+    }).then(() => {
+        tagBar.blur();
+        $("#tagModal").modal("hide");
+        $.toast({
+            heading: "Tag added",
+            text: "Tag saved to index storage and updated in ElasticSearch",
+            stack: 3,
+            bgColor: "#00a4bc",
+            textColor: "#fff",
+            position: 'bottom-right',
+            hideAfter: 3000,
+            loaderBg: "#08c7e8",
+        });
     })
-    CONF.load();
-};
+}
+
+function deleteTag(tag, hit) {
+    const relPath = hit["_source"]["path"] + "/" + hit["_source"]["name"] + ext(hit);
+
+    return $.jsonPost("/tag/" + hit["_source"]["index"], {
+        delete: true,
+        name: tag,
+        doc_id: hit["_id"],
+        relpath: relPath
+    }).then(() => {
+        $.toast({
+            heading: "Tag deleted",
+            text: "Tag deleted index storage and updated in ElasticSearch",
+            stack: 3,
+            bgColor: "#00a4bc",
+            textColor: "#fff",
+            position: 'bottom-right',
+            hideAfter: 3000,
+            loaderBg: "#08c7e8",
+        });
+    })
+}
 
 function toggleFuzzy() {
     searchDebounced();
 }
 
-$.jsonPost("i").then(resp => {
+$.get("i").then(resp => {
 
     const urlIndices = (new URLSearchParams(location.search)).get("i");
     resp["indices"].forEach(idx => {
@@ -159,10 +242,7 @@ $.jsonPost("i").then(resp => {
 });
 
 function getDocumentInfo(id) {
-    return $.getJSON("d/" + id).fail(e => {
-        console.log(e);
-        showEsError();
-    })
+    return $.getJSON("d/" + id).fail(showEsError)
 }
 
 function handleTreeClick(tree) {
@@ -173,7 +253,7 @@ function handleTreeClick(tree) {
 
         if (node.id === "any") {
             if (!node.itree.state.checked) {
-                tree.deselect();
+                tree.deselectDeep();
             }
         } else {
             tree.node("any").deselect();
@@ -256,6 +336,9 @@ $.jsonPost("es", {
         selection: {
             mode: 'checkbox'
         },
+        checkbox: {
+            autoCheckChildren: false
+        },
         data: tagMap
     });
     new InspireTreeDOM(tagTree, {
@@ -267,20 +350,70 @@ $.jsonPost("es", {
 });
 
 function addTag(map, tag, id, count) {
-    let tags = tag.split("#")[0].split(".");
+    // let tags = tag.split("#")[0].split(".");
+    let tags = tag.split(".");
 
     let child = {
         id: id,
-        text: tags.length !== 1 ? tags[0] : `${tags[0]} (${count})`,
-        children: []
+        values: [id],
+        count: count,
+        text: tags.length !== 1 ? tags[0] : `${tags[0].split("#")[0]} (${count})`,
+        name: tags[0],
+        children: [],
+        isLeaf: tags.length === 1,
+        //Overwrite base functions
+        blur: function () {
+        },
+        select: function () {
+            this.state("selected", true);
+            return this.check()
+        },
+        deselect: function () {
+            this.state("selected", false);
+            return this.uncheck()
+        },
+        uncheck: function () {
+            if (!this.isLeaf) {
+                return;
+            }
+
+            baseStateChange('checked', false, 'unchecked', this, false);
+            this.state('indeterminate', false);
+
+            if (this.hasParent()) {
+                this.getParent().refreshIndeterminateState();
+            }
+
+            this._tree.end();
+            return this;
+        },
+        check: function () {
+            if (!this.isLeaf) {
+                return;
+            }
+
+            baseStateChange('checked', true, 'checked', this, false);
+
+            if (this.hasParent()) {
+                this.getParent().refreshIndeterminateState();
+            }
+
+            this._tree.end();
+            return this;
+        }
     };
 
     let found = false;
     map.forEach(node => {
-        if (node.text === child.text) {
+        if (node.name.split("#")[0] === child.name.split("#")[0]) {
             found = true;
             if (tags.length !== 1) {
                 addTag(node.children, tags.slice(1).join("."), id, count);
+            } else {
+                // Same name, different color
+                node.count += count;
+                node.text = `${tags[0].split("#")[0]} (${node.count})`;
+                node.values.push(id);
             }
         }
     });
@@ -332,7 +465,11 @@ function getSelectedNodes(tree) {
 
         //Only get children
         if (selected[i].text.indexOf("(") !== -1) {
-            selectedNodes.push(selected[i].id);
+            if (selected[i].values) {
+                selectedNodes.push(selected[i].values);
+            } else {
+                selectedNodes.push(selected[i].id);
+            }
         }
     }
 
@@ -372,31 +509,40 @@ function search(after = null) {
         "font_name^6"
     ];
 
+    if (CONF.options.searchInPath) {
+        fields.push("path.text^5");
+    }
+
     if ($("#fuzzyToggle").prop("checked")) {
         fields.push("content.nGram");
+        if (CONF.options.searchInPath) {
+            fields.push("path.nGram");
+        }
         fields.push("name.nGram^3");
     }
 
     let path = pathBar.value.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
     if (path !== "") {
-        filters.push([{term: {path: path}}])
+        filters.push({term: {path: path}})
     }
     let mimeTypes = getSelectedNodes(mimeTree);
     if (!mimeTypes.includes("any")) {
-        filters.push([{terms: {"mime": mimeTypes}}]);
+        filters.push({terms: {"mime": mimeTypes}});
     }
 
     let tags = getSelectedNodes(tagTree);
     if (!tags.includes("any")) {
-        filters.push([{terms: {"tag": tags}}]);
+        tags.forEach(tagGroup => {
+            filters.push({terms: {"tag": tagGroup}})
+        })
     }
 
     if (date_min && date_max) {
-        filters.push([{range: {mtime: {gte: date_min, lte: date_max}}}])
+        filters.push({range: {mtime: {gte: date_min, lte: date_max}}})
     } else if (date_min) {
-        filters.push([{range: {mtime: {gte: date_min}}}])
+        filters.push({range: {mtime: {gte: date_min}}})
     } else if (date_max) {
-        filters.push([{range: {mtime: {lte: date_max}}}])
+        filters.push({range: {mtime: {lte: date_max}}})
     }
 
     let q = {
@@ -432,6 +578,9 @@ function search(after = null) {
         q.highlight = {
             pre_tags: ["<mark>"],
             post_tags: ["</mark>"],
+            fragment_size: CONF.options.fragmentSize,
+            number_of_fragments: 1,
+            order: "score",
             fields: {
                 content: {},
                 // "content.nGram": {},
@@ -440,6 +589,10 @@ function search(after = null) {
                 font_name: {},
             }
         };
+        if (CONF.options.searchInPath) {
+            q.highlight.fields["path.text"] = {};
+            q.highlight.fields["path.nGram"] = {};
+        }
     }
 
     $.jsonPost("es", q).then(searchResult => {
@@ -448,6 +601,11 @@ function search(after = null) {
             lastDoc = hits[hits.length - 1];
         }
 
+        hits.forEach(hit => {
+            hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
+            hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+        });
+
         if (!after) {
             preload.remove();
             searchResults.appendChild(makeStatsCard(searchResult));
@@ -479,8 +637,6 @@ let searchDebounced = _.debounce(function () {
     search()
 }, 500);
 
-searchBar.addEventListener("keyup", searchDebounced);
-pathBar.addEventListener("keyup", searchDebounced);
 
 //Size slider
 $("#sizeSlider").ionRangeSlider({
@@ -586,7 +742,7 @@ function getNextDepth(node) {
             }
         },
         size: 0
-    }
+    };
 
     if (node.depth > 0) {
         q.query.bool.must = {
@@ -622,6 +778,7 @@ function getNextDepth(node) {
                     text: `${name}/ (${bucket.doc_count})`,
                     depth: node.depth + 1,
                     index: node.index,
+                    values: [bucket.key],
                     children: true,
                 }
             }).filter(x => x !== null)
@@ -645,18 +802,20 @@ function createPathTree(target) {
     let pathTree = new InspireTree({
         data: function (node, resolve, reject) {
             return getNextDepth(node);
-        }
+        },
+        sort: "text"
     });
 
     selectedIndices.forEach(index => {
         pathTree.addNode({
             id: "/" + index,
+            values: ["/" + index],
             text: `/[${indexMap[index]}]`,
             index: index,
             depth: 0,
             children: true
         })
-    })
+    });
 
     new InspireTreeDOM(pathTree, {
         target: target
@@ -665,30 +824,48 @@ function createPathTree(target) {
     pathTree.on("node.click", handlePathTreeClick(pathTree));
 }
 
-function updateSettings() {
-    CONF.options.display = $("#settingDisplay").val();
-    CONF.options.fuzzy = $("#settingFuzzy").prop("checked");
-    CONF.options.highlight = $("#settingHighlight").prop("checked");
-    CONF.save();
-
-    searchDebounced();
-
-    $.toast({
-        heading: "Settings updated",
-        text: "Settings saved to browser storage",
-        stack: 3,
-        bgColor: "#00a4bc",
-        textColor: "#fff",
-        position: 'bottom-right',
-        hideAfter: 3000,
-        loaderBg: "#08c7e8",
+function getPathChoices() {
+    return new Promise(getPaths => {
+        $.jsonPost("es", {
+            suggest: {
+                path: {
+                    prefix: pathBar.value,
+                    completion: {
+                        field: "suggest-path",
+                        skip_duplicates: true,
+                        size: 10000
+                    }
+                }
+            }
+        }).then(resp => getPaths(resp["suggest"]["path"][0]["options"].map(opt => opt["_source"]["path"])));
     });
 }
 
-function loadSettings() {
-    CONF.load();
 
-    $("#settingDisplay").val(CONF.options.display);
-    $("#settingFuzzy").prop("checked", CONF.options.fuzzy);
-    $("#settingHighlight").prop("checked", CONF.options.highlight);
+function getTagChoices() {
+    return new Promise(getPaths => {
+        $.jsonPost("es", {
+            suggest: {
+                tag: {
+                    prefix: tagBar.value,
+                    completion: {
+                        field: "suggest-tag",
+                        skip_duplicates: true,
+                        size: 10000
+                    }
+                }
+            }
+        }).then(resp => {
+            const result = [];
+            resp["suggest"]["tag"][0]["options"].map(opt => opt["_source"]["tag"]).forEach(tags => {
+                tags.forEach(tag => {
+                    const t = tag.split("#")[0];
+                    if (!result.find(x => x.split("#")[0] === t)) {
+                        result.push(tag);
+                    }
+                });
+            });
+            getPaths(result);
+        });
+    });
 }

src/static/js/util.js

@@ -64,3 +64,169 @@ function lum(c) {
 
     return 0.2126 * r + 0.7152 * g + 0.0722 * b;
 }
+
+function strUnescape(str) {
+    let result = "";
+
+    for (let i = 0; i < str.length; i++) {
+        const c = str[i];
+        const next = str[i+1];
+
+        if (c === ']') {
+            if (next === ']') {
+                result += c;
+                i += 1;
+            } else {
+                result += String.fromCharCode(parseInt(str.slice(i, i + 2), 16));
+                i += 2;
+            }
+        } else {
+            result += c;
+        }
+    }
+    return result;
+}
+
+const CONF = new Settings();
+
+const _defaults = {
+    display: "grid",
+    fuzzy: true,
+    highlight: true,
+    sort: "score",
+    searchInPath: false,
+    treemapType: "cascaded",
+    treemapTiling: "squarify",
+    treemapGroupingDepth: 3,
+    treemapColor: "PuBuGn",
+    treemapSize: "large",
+    suggestPath: true,
+    fragmentSize: 100,
+    columns: 5
+};
+
+function loadSettings() {
+    CONF.load();
+
+    $("#settingDisplay").val(CONF.options.display);
+    $("#settingFuzzy").prop("checked", CONF.options.fuzzy);
+    $("#settingHighlight").prop("checked", CONF.options.highlight);
+    $("#settingSearchInPath").prop("checked", CONF.options.searchInPath);
+    $("#settingTreemapTiling").val(CONF.options.treemapTiling);
+    $("#settingTreemapGroupingDepth").val(CONF.options.treemapGroupingDepth);
+    $("#settingTreemapColor").val(CONF.options.treemapColor);
+    $("#settingTreemapSize").val(CONF.options.treemapSize);
+    $("#settingTreemapType").val(CONF.options.treemapType);
+    $("#settingSuggestPath").prop("checked", CONF.options.suggestPath);
+    $("#settingFragmentSize").val(CONF.options.fragmentSize);
+    $("#settingColumns").val(CONF.options.columns);
+}
+
+function Settings() {
+    this.options = {};
+
+    this._onUpdate = function () {
+        $("#fuzzyToggle").prop("checked", this.options.fuzzy);
+        updateColumnStyle();
+    };
+
+    this.load = function () {
+        const raw = window.localStorage.getItem("options");
+        if (raw === null) {
+            this.options = _defaults;
+        } else {
+            const j = JSON.parse(raw);
+            if (!j || Object.keys(_defaults).some(k => !j.hasOwnProperty(k))) {
+                this.options = _defaults;
+            } else {
+                this.options = j;
+            }
+        }
+
+        this._onUpdate();
+    };
+
+    this.save = function () {
+        window.localStorage.setItem("options", JSON.stringify(this.options));
+        this._onUpdate();
+    }
+}
+
+function updateSettings() {
+    CONF.options.display = $("#settingDisplay").val();
+    CONF.options.fuzzy = $("#settingFuzzy").prop("checked");
+    CONF.options.highlight = $("#settingHighlight").prop("checked");
+    CONF.options.searchInPath = $("#settingSearchInPath").prop("checked");
+    CONF.options.treemapTiling = $("#settingTreemapTiling").val();
+    CONF.options.treemapGroupingDepth = $("#settingTreemapGroupingDepth").val();
+    CONF.options.treemapColor = $("#settingTreemapColor").val();
+    CONF.options.treemapSize = $("#settingTreemapSize").val();
+    CONF.options.treemapType = $("#settingTreemapType").val();
+    CONF.options.suggestPath = $("#settingSuggestPath").prop("checked");
+    CONF.options.fragmentSize = $("#settingFragmentSize").val();
+    CONF.options.columns = $("#settingColumns").val();
+    CONF.save();
+
+    if (typeof searchDebounced !== "undefined") {
+        searchDebounced();
+    }
+
+    if (typeof updateStats !== "undefined") {
+        updateStats();
+    }
+
+    $.toast({
+        heading: "Settings updated",
+        text: "Settings saved to browser storage",
+        stack: 3,
+        bgColor: "#00a4bc",
+        textColor: "#fff",
+        position: 'bottom-right',
+        hideAfter: 3000,
+        loaderBg: "#08c7e8",
+    });
+}
+
+jQuery["jsonPost"] = function (url, data) {
+    return jQuery.ajax({
+        url: url,
+        type: "post",
+        data: JSON.stringify(data),
+        contentType: "application/json"
+    }).fail(err => {
+        showEsError();
+        console.log(err);
+    });
+};
+
+function toggleTheme() {
+    if (!document.cookie.includes("sist")) {
+        document.cookie = "sist=dark;SameSite=Strict";
+    } else {
+        document.cookie = "sist=; Max-Age=-99999999;";
+    }
+    window.location.reload();
+}
+
+function updateColumnStyle() {
+    const style = document.getElementById("style");
+    if (style) {
+        style.innerHTML =
+        `
+@media screen and (min-width: 1500px) {
+    .container {
+            max-width: 1440px;
+        }
+
+    .bricklayer-column-sizer {
+            width: ${100 / CONF.options.columns}% !important;
+        }
+
+    .bricklayer-column {
+            max-width: ${100 / CONF.options.columns}%;
+        }
+    }
+}
+        `
+    }
+}

src/static/search.html

@@ -6,15 +6,18 @@
     <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
 
     <link href="css" rel="stylesheet" type="text/css">
+    <style id="style"></style>
 </head>
 <body>
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.0.1</span>
+    <span class="badge badge-pill version">2.8.0</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
-    <button style="margin-left: auto" class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings</button>
-    <a id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
+    <a class="btn ml-auto" href="/stats">Stats</a>
+    <button class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings
+    </button>
+    <button class="btn" title="Toggle theme" onclick="toggleTheme()">Theme</button>
 </nav>
 
 <div class="container">
@@ -47,8 +50,11 @@
                 <div class="col">
                     <div class="input-group" style="margin-bottom: 0.5em; margin-top: 1em">
                         <div class="input-group-prepend">
-                            <button id="pathBarHelper" class="btn btn-outline-secondary" data-toggle="modal" data-target="#pathTreeModal">
-                                <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px"><path d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"/></svg>
+                            <button id="pathBarHelper" class="btn btn-outline-secondary" data-toggle="modal"
+                                    data-target="#pathTreeModal">
+                                <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px">
+                                    <path d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"/>
+                                </svg>
                             </button>
                         </div>
                         <input id="pathBar" type="search" class="form-control" placeholder="Filter path">
@@ -155,7 +161,8 @@
                         <i>fried eggs</i> and either <i>eggplant</i> or <i>potato</i>, but will ignore results
                         containing <i>frittata</i>.</p>
 
-                    <p>When neither <code>+</code> or <code>|</code> is specified, the default operator is <code>+</code> (and).</p>
+                    <p>When neither <code>+</code> or <code>|</code> is specified, the default operator is
+                        <code>+</code> (and).</p>
                     <p>When the <b>Fuzzy</b> option is checked, partial matches are also returned.</p>
                     <br>
                     <p>For more information, see <a target="_blank"
@@ -186,14 +193,101 @@
                         <label class="custom-control-label" for="settingFuzzy">Set fuzzy search by default</label>
                     </div>
 
+                    <div class="custom-control custom-checkbox">
+                        <input type="checkbox" class="custom-control-input" id="settingSearchInPath">
+                        <label class="custom-control-label" for="settingSearchInPath">Enable matching query against
+                            document path</label>
+                    </div>
+
+                    <div class="custom-control custom-checkbox">
+                        <input type="checkbox" class="custom-control-input" id="settingSuggestPath">
+                        <label class="custom-control-label" for="settingSuggestPath">Enable auto-complete in path filter
+                            bar</label>
+                    </div>
+
+                    <br/>
+                    <div class="form-group">
+                        <input type="number" class="form-control" id="settingFragmentSize">
+                        <label for="settingFragmentSize">Highlight context size in characters</label>
+                    </div>
+
                     <label for="settingDisplay">Display</label>
                     <select id="settingDisplay" class="form-control form-control-sm">
                         <option value="grid">Grid</option>
                         <option value="list">List</option>
                     </select>
 
+                    <div class="form-group">
+                        <label for="settingColumns">Maximum column count</label>
+                        <select id="settingColumns" class="form-control form-control-sm">
+                            <option value="3">3</option>
+                            <option value="4">4</option>
+                            <option value="5">5</option>
+                            <option value="6">6</option>
+                            <option value="7">7</option>
+                            <option value="8">8</option>
+                            <option value="9">9</option>
+                        </select>
+                    </div>
+
+                    <hr/>
+                    <h4>Stats</h4>
+
+                    <div class="form-group">
+                        <label for="settingTreemapType">Treemap type</label>
+                        <select id="settingTreemapType" class="form-control form-control-sm">
+                            <option value="cascaded">Cascaded</option>
+                            <option value="flat">Flat (compact)</option>
+                        </select>
+                    </div>
+
+                    <div class="form-group">
+                        <label for="settingTreemapTiling">Treemap tiling</label>
+                        <select id="settingTreemapTiling" class="form-control form-control-sm">
+                            <option value="binary">Binary</option>
+                            <option value="squarify">Squarify</option>
+                            <option value="slice">Slice</option>
+                            <option value="dice">Dice</option>
+                            <option value="sliceDice">Slide & Dice</option>
+                        </select>
+                    </div>
+
+                    <div class="form-group">
+                        <label for="settingTreemapGroupingDepth">Treemap color grouping depth (flat)</label>
+                        <input type="number" class="form-control" id="settingTreemapGroupingDepth" min="1" max="10">
+                    </div>
+
+                    <div class="form-group">
+                        <label for="settingTreemapColor">Treemap color (cascaded)</label>
+                        <select id="settingTreemapColor" class="form-control form-control-sm">
+                            <option value="PuBuGn">Purple-Blue-Green</option>
+                            <option value="PuRd">Purple-Red</option>
+                            <option value="PuBu">Purple-Blue</option>
+                            <option value="YlOrBr">Yellow-Orange-Brown</option>
+                            <option value="YlOrRd">Yellow-Orange-Red</option>
+                            <option value="YlGn">Yellow-Green</option>
+                            <option value="YlGnBu">Yellow-Green-Blue</option>
+                            <option value="Plasma">Plasma</option>
+                            <option value="Magma">Magma</option>
+                            <option value="Inferno">Inferno</option>
+                            <option value="Viridis">Viridis</option>
+                            <option value="Turbo">Turbo</option>
+                        </select>
+                    </div>
+
+                    <div class="form-group">
+                        <label for="settingTreemapSize">Treemap size</label>
+                        <select id="settingTreemapSize" class="form-control form-control-sm">
+                            <option value="small">Small</option>
+                            <option value="medium">Medium</option>
+                            <option value="large">Large</option>
+                            <option value="x-large">X-Large</option>
+                            <option value="xx-large">XX-Large</option>
+                        </select>
+                    </div>
+
                     <br>
-                    <button style="float: right" class="btn btn-primary" onclick="updateSettings()">Update settings</button>
+                    <button class="btn btn-primary ml-auto" onclick="updateSettings()">Update settings</button>
                 </div>
             </div>
         </div>
@@ -215,9 +309,36 @@
         </div>
     </div>
 
+    <div class="modal" id="tagModal" tabindex="-1" role="dialog" aria-labelledby="modal-title" aria-hidden="true">
+        <div class="modal-dialog modal-dialog-centered" role="document">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h5 class="modal-title">Add tag</h5>
+                    <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                        <span aria-hidden="true">&times;</span>
+                    </button>
+                </div>
+
+                <div class="modal-body">
+                    <div class="form-group">
+                        <div class="row">
+                            <div class="col col-8">
+                                <input type="text" id="tagBar" class="form-control">
+                            </div>
+                            <div class="col col-4">
+                                <input type="text" id="tag-color" value="" class="form-control"/>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <div id="searchResults"></div>
 </div>
 
-<script src="js" type="text/javascript"></script>
+<script src="jslib" type="text/javascript"></script>
+<script src="jssearch" type="text/javascript"></script>
 </body>
 </html>

src/static/stats.html

@@ -0,0 +1,813 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <title>sist2 - Stats</title>
+    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
+    <link href="css" rel="stylesheet" type="text/css">
+</head>
+<body>
+
+<nav class="navbar navbar-expand-lg">
+    <a class="navbar-brand" href="/">sist2</a>
+    <span class="badge badge-pill version">2.8.0</span>
+    <span class="tagline">Lightning-fast file system indexer and search tool </span>
+    <a style="margin-left: auto" class="btn" href="/">Back</a>
+    <button class="btn" type="button" data-toggle="modal" data-target="#settings"
+            onclick="loadSettings()">Settings
+    </button>
+    <button class="btn" title="Toggle theme" onclick="toggleTheme()">Theme</button>
+</nav>
+
+<div class="container pb-3">
+    <div class="card">
+        <div class="card-body">
+
+            <label for="indices">Index</label>
+            <select id="indices" onchange="updateStats()"></select>
+        </div>
+    </div>
+
+    <div id="treemap-card" class="stats-card">
+        <button class="btn stats-btn" onclick="fullScreen('treemap-card')">Enlarge</button>
+        <button class="btn stats-btn" onclick="exportTreemap()">Export</button>
+        <svg id="treemap"></svg>
+    </div>
+
+    <div id="graphs-card" class="stats-card">
+        <button class="btn stats-btn" onclick="fullScreen('graphs-card')">Enlarge</button>
+        <div class="graph">
+            <svg id="agg_mime_size"></svg>
+        </div>
+        <div class="graph">
+            <svg id="agg_mime_count"></svg>
+        </div>
+        <div class="graph">
+            <svg id="date_histogram"></svg>
+        </div>
+        <div class="graph">
+            <svg id="size_histogram"></svg>
+        </div>
+    </div>
+</div>
+
+<div class="modal" id="settings" tabindex="-1" role="dialog" aria-labelledby="modal-title" aria-hidden="true">
+    <div class="modal-dialog modal-dialog-centered" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h5 class="modal-title">Settings</h5>
+                <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                    <span aria-hidden="true">&times;</span>
+                </button>
+            </div>
+            <div class="modal-body">
+                <div class="custom-control custom-checkbox">
+                    <input type="checkbox" class="custom-control-input" id="settingHighlight">
+                    <label class="custom-control-label" for="settingHighlight">Enable highlighting</label>
+                </div>
+
+                <div class="custom-control custom-checkbox">
+                    <input type="checkbox" class="custom-control-input" id="settingFuzzy">
+                    <label class="custom-control-label" for="settingFuzzy">Set fuzzy search by default</label>
+                </div>
+
+                <div class="custom-control custom-checkbox">
+                    <input type="checkbox" class="custom-control-input" id="settingSearchInPath">
+                    <label class="custom-control-label" for="settingSearchInPath">Enable matching query against document
+                        path</label>
+                </div>
+
+                <div class="custom-control custom-checkbox">
+                    <input type="checkbox" class="custom-control-input" id="settingSuggestPath">
+                    <label class="custom-control-label" for="settingSuggestPath">Enable auto-complete in path filter bar</label>
+                </div>
+
+                <br/>
+                <div class="form-group">
+                    <input type="number" class="form-control" id="settingFragmentSize">
+                    <label for="settingFragmentSize">Highlight context size in characters</label>
+                </div>
+
+                <label for="settingDisplay">Display</label>
+                <select id="settingDisplay" class="form-control form-control-sm">
+                    <option value="grid">Grid</option>
+                    <option value="list">List</option>
+                </select>
+
+                <div class="form-group">
+                    <label for="settingColumns">Maximum column count</label>
+                    <select id="settingColumns" class="form-control form-control-sm">
+                        <option value="3">3</option>
+                        <option value="4">4</option>
+                        <option value="5">5</option>
+                        <option value="6">6</option>
+                        <option value="7">7</option>
+                        <option value="8">8</option>
+                        <option value="9">9</option>
+                    </select>
+                </div>
+
+                <hr/>
+                <h4>Stats</h4>
+
+                <div class="form-group">
+                    <label for="settingTreemapType">Treemap type</label>
+                    <select id="settingTreemapType" class="form-control form-control-sm">
+                        <option value="cascaded">Cascaded</option>
+                        <option value="flat">Flat (compact)</option>
+                    </select>
+                </div>
+
+                <div class="form-group">
+                    <label for="settingTreemapTiling">Treemap tiling</label>
+                    <select id="settingTreemapTiling" class="form-control form-control-sm">
+                        <option value="binary">Binary</option>
+                        <option value="squarify">Squarify</option>
+                        <option value="slice">Slice</option>
+                        <option value="dice">Dice</option>
+                        <option value="sliceDice">Slide & Dice</option>
+                    </select>
+                </div>
+
+                <div class="form-group">
+                    <label for="settingTreemapGroupingDepth">Treemap color grouping depth (flat)</label>
+                    <input type="number" class="form-control" id="settingTreemapGroupingDepth" min="1" max="10">
+                </div>
+
+                <div class="form-group">
+                    <label for="settingTreemapColor">Treemap color (cascaded)</label>
+                    <select id="settingTreemapColor" class="form-control form-control-sm">
+                        <option value="PuBuGn">Purple-Blue-Green</option>
+                        <option value="PuRd">Purple-Red</option>
+                        <option value="PuBu">Purple-Blue</option>
+                        <option value="YlOrBr">Yellow-Orange-Brown</option>
+                        <option value="YlOrRd">Yellow-Orange-Red</option>
+                        <option value="YlGn">Yellow-Green</option>
+                        <option value="YlGnBu">Yellow-Green-Blue</option>
+                        <option value="Plasma">Plasma</option>
+                        <option value="Magma">Magma</option>
+                        <option value="Inferno">Inferno</option>
+                        <option value="Viridis">Viridis</option>
+                        <option value="Turbo">Turbo</option>
+                    </select>
+                </div>
+
+                <div class="form-group">
+                    <label for="settingTreemapSize">Treemap size</label>
+                    <select id="settingTreemapSize" class="form-control form-control-sm">
+                        <option value="small">Small</option>
+                        <option value="medium">Medium</option>
+                        <option value="large">Large</option>
+                        <option value="x-large">X-Large</option>
+                        <option value="xx-large">XX-Large</option>
+                    </select>
+                </div>
+
+                <br>
+                <button class="btn btn-primary float-right" onclick="updateSettings()">Update settings</button>
+            </div>
+        </div>
+    </div>
+</div>
+
+<script src="jslib" type="text/javascript"></script>
+<script>
+let width;
+let height;
+let indexMap = {};
+
+const barHeight = 20;
+const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
+
+const formatSI = d3.format("~s");
+
+
+const TILING_MODES = {
+    "squarify": d3.treemapSquarify,
+    "binary": d3.treemapBinary,
+    "sliceDice": d3.treemapSliceDice,
+    "slice": d3.treemapSlice,
+    "dice": d3.treemapDice,
+};
+
+const COLORS = {
+    "PuBuGn": d3.interpolatePuBuGn,
+    "PuRd": d3.interpolatePuRd,
+    "PuBu": d3.interpolatePuBu,
+    "YlOrBr": d3.interpolateYlOrBr,
+    "YlOrRd": d3.interpolateYlOrRd,
+    "YlGn": d3.interpolateYlGn,
+    "YlGnBu": d3.interpolateYlGnBu,
+    "Plasma": d3.interpolatePlasma,
+    "Magma": d3.interpolateMagma,
+    "Inferno": d3.interpolateInferno,
+    "Viridis": d3.interpolateViridis,
+    "Turbo": d3.interpolateTurbo,
+};
+
+const SIZES = {
+    "small": [800, 600],
+    "medium": [1300, 750],
+    "large": [1900, 900],
+    "x-large": [2800, 1700],
+    "xx-large": [3600, 2000],
+};
+
+const fillOpacity = document.cookie.includes("sist") ? 0.9 : 0.6;
+
+const uids = {};
+
+function uid(name) {
+    let id = uids[name] || 0;
+    uids[name] = id + 1;
+    return name + id;
+}
+
+const burrow = function (table, addSelfDir) {
+    const root = {};
+    table.forEach(row => {
+        let layer = root;
+
+        row.taxonomy.forEach(key => {
+            layer[key] = key in layer ? layer[key] : {};
+            layer = layer[key];
+        });
+        if (Object.keys(layer).length === 0) {
+            layer["$size$"] = row.size;
+        } else if (addSelfDir) {
+            layer["."] = {
+                "$size$": row.size,
+            };
+        }
+    });
+
+    const descend = function (obj, depth) {
+        return Object.keys(obj).filter(k => k !== "$size$").map(k => {
+            const child = {
+                name: k,
+                depth: depth,
+                value: 0,
+                children: descend(obj[k], depth + 1)
+            };
+            if ("$size$" in obj[k]) {
+                child.value = obj[k]["$size$"];
+            }
+            return child;
+        });
+    };
+
+    return {
+        name: `[${indexMap[$("#indices").val()]}]`,
+        children: descend(root, 1),
+        value: 0,
+        depth: 0,
+    }
+};
+
+function flatTreemap(data, svg) {
+    const root = d3.treemap()
+        .tile(TILING_MODES[CONF.options.treemapTiling])
+        .size([width, height])
+        .padding(1)
+        .round(true)(
+            d3.hierarchy(data)
+                .sum(d => d.value)
+                .sort((a, b) => b.value - a.value)
+        );
+
+    const leaf = svg.selectAll("g")
+        .data(root.leaves())
+        .join("g")
+        .attr("transform", d => `translate(${d.x0},${d.y0})`);
+
+    leaf.append("title")
+        .text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+
+    leaf.append("rect")
+        .attr("id", d => (d.leafUid = uid("leaf")))
+        .attr("fill", d => {
+            while (d.depth > CONF.options.treemapGroupingDepth) d = d.parent;
+            return ordinalColor(d.data.name);
+        })
+        .attr("fill-opacity", fillOpacity)
+        .attr("width", d => d.x1 - d.x0)
+        .attr("height", d => d.y1 - d.y0);
+
+    leaf.append("clipPath")
+        .attr("id", d => (d.clipUid = uid("clip")))
+        .append("use")
+        .attr("href", d => `#${d.leafUid}`);
+
+    leaf.append("text")
+        .attr("clip-path", d => `url(#${d.clipUid})`)
+        .selectAll("tspan")
+        .data(d => {
+            if (d.data.name === ".") {
+                d = d.parent;
+            }
+            return [d.data.name, humanFileSize(d.value)]
+        })
+        .join("tspan")
+        .attr("x", 2)
+        .attr("y", (d, i, nodes) => `${i === 0 ? 1.1 : 2.3}em`)
+        .text(d => d);
+}
+
+function cascade(root, offset) {
+    const x = new Map;
+    const y = new Map;
+    return root.eachAfter(d => {
+        if (d.children && d.children.length !== 0) {
+            x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
+            y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
+        } else {
+            x.set(d, 0);
+            y.set(d, 0);
+        }
+    }).eachBefore(d => {
+        d.x1 -= 2 * offset * x.get(d);
+        d.y1 -= 2 * offset * y.get(d);
+    });
+}
+
+function cascadeTreemap(data, svg) {
+
+    const root = cascade(
+        d3.treemap()
+            .size([width, height])
+            .tile(TILING_MODES[CONF.options.treemapTiling])
+            .paddingOuter(3)
+            .paddingTop(16)
+            .paddingInner(1)
+            .round(true)(
+                d3.hierarchy(data)
+                    .sum(d => d.value)
+                    .sort((a, b) => b.value - a.value)
+            ),
+        3 // treemap.paddingOuter
+    );
+
+    const maxDepth = Math.max(...root.descendants().map(d => d.depth));
+    const color = d3.scaleSequential([maxDepth, -1], COLORS[CONF.options.treemapColor]);
+
+    svg.append("filter")
+        .attr("id", "shadow")
+        .append("feDropShadow")
+        .attr("flood-opacity", 0.3)
+        .attr("dx", 0)
+        .attr("stdDeviation", 3);
+
+    const node = svg.selectAll("g")
+        .data(
+            d3.nest()
+                .key(d => d.depth).sortKeys(d3.ascending)
+                .entries(root.descendants())
+        )
+        .join("g")
+        .attr("filter", "url(#shadow)")
+        .selectAll("g")
+        .data(d => d.values)
+        .join("g")
+        .attr("transform", d => `translate(${d.x0},${d.y0})`);
+
+    node.append("title")
+        .text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
+
+    node.append("rect")
+        .attr("id", d => (d.nodeUid = uid("node")))
+        .attr("fill", d => color(d.depth))
+        .attr("width", d => d.x1 - d.x0)
+        .attr("height", d => d.y1 - d.y0);
+
+    node.append("clipPath")
+        .attr("id", d => (d.clipUid = uid("clip")))
+        .append("use")
+        .attr("href", d => `#${d.nodeUid}`);
+
+    node.append("text")
+        .attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
+        .attr("clip-path", d => `url(#${d.clipUid})`)
+        .selectAll("tspan")
+        .data(d => [d.data.name, humanFileSize(d.value)])
+        .join("tspan")
+        .text(d => d);
+
+    node.filter(d => d.children).selectAll("tspan")
+        .attr("dx", 3)
+        .attr("y", 13);
+
+    node.filter(d => !d.children).selectAll("tspan")
+        .attr("x", 3)
+        .attr("y", (d, i, nodes) => `${i === 0 ? 1.1 : 2.3}em`);
+}
+
+
+function mimeBarSize(data, svg) {
+
+    const margin = {
+        top: 50,
+        right: 0,
+        bottom: 10,
+        left: Math.max(
+            d3.max(data.sort((a, b) => b.count - a.count).slice(0, 15), d => d.mime.length) * 6,
+            d3.max(data.sort((a, b) => b.size - a.size).slice(0, 15), d => d.mime.length) * 6,
+        )
+    };
+
+    data.forEach(d => {
+        d.name = d.mime;
+        d.value = Number(d.size);
+    });
+    data = data.sort((a, b) => b.value - a.value).slice(0, 15);
+
+    const width = 550;
+    const height = Math.ceil((data.length + 0.1) * barHeight) + margin.top + margin.bottom;
+
+    svg.selectAll("*").remove();
+    svg.attr("viewBox", [0, 0, width, height]);
+
+    const y = d3.scaleBand()
+        .domain(d3.range(data.length))
+        .rangeRound([margin.top, height - margin.bottom]);
+
+    const x = d3.scaleLinear()
+        .domain([0, d3.max(data, d => d.value)])
+        .range([margin.left, width - margin.right]);
+
+    svg.append("g")
+        .attr("fill-opacity", fillOpacity)
+        .selectAll("rect")
+        .data(data)
+        .join("rect")
+        .attr("fill", d => ordinalColor(d.name))
+        .attr("x", x(0))
+        .attr("y", (d, i) => y(i))
+        .attr("width", d => x(d.value) - x(0))
+        .attr("height", y.bandwidth())
+        .append("title")
+        .text(d => formatSI(d.value));
+
+    svg.append("g")
+        .attr("transform", `translate(0,${margin.top})`)
+        .call(d3.axisTop(x).ticks(width / 80, data.format).tickFormat(formatSI))
+        .call(g => g.select(".domain").remove());
+
+    svg.append("g")
+        .attr("transform", `translate(${margin.left},0)`)
+        .call(d3.axisLeft(y).tickFormat(i => data[i].name).tickSizeOuter(0));
+
+    svg.append("text")
+        .attr("x", (width / 2))
+        .attr("y", (margin.top / 2))
+        .attr("text-anchor", "middle")
+        .style("font-size", "16px")
+        .text("Size distribution by MIME type");
+}
+
+function mimeBarCount(data, svg) {
+
+    const margin = {
+        top: 50,
+        right: 0,
+        bottom: 10,
+        left: Math.max(
+            d3.max(data.sort((a, b) => b.count - a.count).slice(0, 15), d => d.mime.length) * 6,
+            d3.max(data.sort((a, b) => b.size - a.size).slice(0, 15), d => d.mime.length) * 6,
+        )
+    };
+
+    data.forEach(d => {
+        d.name = d.mime;
+        d.value = Number(d.count);
+    });
+
+    data = data.sort((a, b) => b.value - a.value).slice(0, 15);
+
+    const width = 550;
+    const height = Math.ceil((data.length + 0.1) * barHeight) + margin.top + margin.bottom;
+
+    svg.selectAll("*").remove();
+    svg.attr("viewBox", [0, 0, width, height]);
+
+    const y = d3.scaleBand()
+        .domain(d3.range(data.length))
+        .rangeRound([margin.top, height - margin.bottom]);
+
+    const x = d3.scaleLinear()
+        .domain([0, d3.max(data, d => d.value)])
+        .range([margin.left, width - margin.right]);
+
+    svg.append("g")
+        .attr("fill-opacity", fillOpacity)
+        .selectAll("rect")
+        .data(data)
+        .join("rect")
+        .attr("fill", d => ordinalColor(d.name))
+        .attr("x", x(0))
+        .attr("y", (d, i) => y(i))
+        .attr("width", d => x(d.value) - x(0))
+        .attr("height", y.bandwidth())
+        .append("title")
+        .text(d => d3.format(",")(d.value));
+
+    svg.append("g")
+        .attr("transform", `translate(0,${margin.top})`)
+        .call(d3.axisTop(x).ticks(width / 80, data.format).tickFormat(formatSI))
+        .call(g => g.select(".domain").remove());
+
+    svg.append("g")
+        .attr("transform", `translate(${margin.left},0)`)
+        .call(d3.axisLeft(y).tickFormat(i => data[i].name).tickSizeOuter(0));
+
+    svg.append("text")
+        .attr("x", (width / 2))
+        .attr("y", (margin.top / 2))
+        .attr("text-anchor", "middle")
+        .style("font-size", "16px")
+        .text("File count distribution by MIME type");
+}
+
+function dateHistogram(data, svg) {
+
+    let bins = data.map(d => {
+        return {
+            length: Number(d.count),
+            x0: Number(d.bucket),
+            x1: Number(d.bucket) + 2629800
+        }
+    });
+    bins.sort((a, b) => a.length - b.length);
+
+    const margin = {
+        top: 50,
+        right: 20,
+        bottom: 70,
+        left: 40
+    };
+
+    const thresh = d3.quantile(bins, 0.9, d => d.length);
+    bins = bins.filter(d => d.length > thresh);
+
+    const width = 550;
+    const height = 450;
+
+    svg.selectAll("*").remove();
+    svg.attr("viewBox", [0, 0, width, height]);
+
+    const y = d3.scaleLinear()
+        .domain([0, d3.max(bins, d => d.length)]).nice()
+        .range([height - margin.bottom, margin.top]);
+
+    const x = d3.scaleLinear()
+        .domain(d3.extent(bins, d => d.x0)).nice()
+        .range([margin.left, width - margin.right]);
+
+    svg.append("g")
+        .attr("fill", "steelblue")
+        .selectAll("rect")
+        .data(bins)
+        .join("rect")
+        .attr("x", d => x(d.x0) + 1)
+        .attr("width", d => Math.max(1, x(d.x1) - x(d.x0) - 1))
+        .attr("y", d => y(d.length))
+        .attr("height", d => y(0) - y(d.length))
+        .call(g => g
+            .append("title")
+            .text(d => d.length)
+        );
+
+    svg.append("g")
+        .attr("transform", `translate(0,${height - margin.bottom})`)
+        .call(
+            d3.axisBottom(x)
+                .ticks(width / 30)
+                .tickSizeOuter(0)
+                .tickFormat(t => d3.timeFormat("%Y-%m-%d")(d3.utcParse("%s")(t)))
+        )
+        .call(g => g
+            .selectAll("text")
+            .style("text-anchor", "end")
+            .attr("dx", "-.8em")
+            .attr("dy", ".15em")
+            .attr("transform", "rotate(-65)")
+        )
+        .call(g => g.append("text")
+            .attr("x", width - margin.right)
+            .attr("y", -4)
+            .attr("fill", "currentColor")
+            .attr("font-weight", "bold")
+            .attr("text-anchor", "end")
+            .text("mtime")
+        );
+
+    svg.append("g")
+        .attr("transform", `translate(${margin.left},0)`)
+        .call(
+            d3.axisLeft(y)
+                .ticks(height / 40)
+                .tickFormat(t => formatSI(t))
+        )
+        .call(g => g.select(".domain").remove())
+        .call(g => g.select(".tick:last-of-type text").clone()
+            .attr("x", 4)
+            .attr("text-anchor", "start")
+            .attr("font-weight", "bold")
+            .text("File count"));
+
+    svg.append("text")
+        .attr("x", (width / 2))
+        .attr("y", (margin.top / 2))
+        .attr("text-anchor", "middle")
+        .style("font-size", "16px")
+        .text("File modification time distribution");
+}
+
+function sizeHistogram(data, svg) {
+
+    let bins = data.map(d => {
+        return {
+            length: Number(d.count),
+            x0: Number(d.bucket),
+            x1: Number(d.bucket) + (5 * 1024 * 1024)
+        }
+    });
+    bins = bins.sort((a, b) => b.length - a.length).slice(0, 25);
+
+    const margin = {
+        top: 50,
+        right: 20,
+        bottom: 70,
+        left: 40
+    };
+
+    const width = 550;
+    const height = 450;
+
+    svg.selectAll("*").remove();
+    svg.attr("viewBox", [0, 0, width, height]);
+
+    const y = d3.scaleLinear()
+        .domain([0, d3.max(bins, d => d.length)])
+        .range([height - margin.bottom, margin.top]);
+
+    const x = d3.scaleLinear()
+        .domain(d3.extent(bins, d => d.x0)).nice()
+        .range([margin.left, width - margin.right]);
+
+    svg.append("g")
+        .attr("fill", "steelblue")
+        .selectAll("rect")
+        .data(bins)
+        .join("rect")
+        .attr("x", d => x(d.x0) + 1)
+        .attr("width", d => Math.max(1, x(d.x1) - x(d.x0) - 1))
+        .attr("y", d => y(d.length))
+        .attr("height", d => y(0) - y(d.length))
+        .call(g => g
+            .append("title")
+            .text(d => d.length)
+        );
+
+    svg.append("g")
+        .attr("transform", `translate(0,${height - margin.bottom})`)
+        .call(
+            d3.axisBottom(x)
+                .ticks(width / 30)
+                .tickSizeOuter(0)
+                .tickFormat(formatSI)
+        )
+        .call(g => g
+            .selectAll("text")
+            .style("text-anchor", "end")
+            .attr("dx", "-.8em")
+            .attr("dy", ".15em")
+            .attr("transform", "rotate(-65)")
+        )
+        .call(g => g.append("text")
+            .attr("x", width - margin.right)
+            .attr("y", -4)
+            .attr("fill", "currentColor")
+            .attr("font-weight", "bold")
+            .attr("text-anchor", "end")
+            .text("size (bytes)")
+        );
+
+    svg.append("g")
+        .attr("transform", `translate(${margin.left},0)`)
+        .call(
+            d3.axisLeft(y)
+                .ticks(height / 40)
+                .tickFormat(t => formatSI(t))
+        )
+        .call(g => g.select(".domain").remove())
+        .call(g => g.select(".tick:last-of-type text").clone()
+            .attr("x", 4)
+            .attr("text-anchor", "start")
+            .attr("font-weight", "bold")
+            .text("File count"));
+
+    svg.append("text")
+        .attr("x", (width / 2))
+        .attr("y", (margin.top / 2))
+        .attr("text-anchor", "middle")
+        .style("font-size", "16px")
+        .text("File size distribution");
+}
+
+
+function updateStats() {
+    width = SIZES[CONF.options.treemapSize][0];
+    height = SIZES[CONF.options.treemapSize][1];
+
+    const treemapSvg = d3.select("#treemap");
+    const mimeSvgSize = d3.select("#agg_mime_size");
+    const mimeSvgCount = d3.select("#agg_mime_count");
+    const dateHistogramSvg = d3.select("#date_histogram");
+    const sizeHistogramSvg = d3.select("#size_histogram");
+
+    const indexId = $("#indices").val();
+
+    d3.csv(`/s/${indexId}/1`).then(tabularData => {
+        tabularData.forEach(row => {
+            row.taxonomy = row.path.split("/");
+            row.size = Number(row.size);
+        });
+
+        if (CONF.options.treemapType === "cascaded") {
+            const data = burrow(tabularData, false);
+            cascadeTreemap(data, treemapSvg);
+        } else {
+            const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
+            flatTreemap(data, treemapSvg);
+        }
+    });
+
+    d3.csv(`/s/${indexId}/2`).then(tabularData => {
+        mimeBarSize(tabularData.slice(), mimeSvgSize);
+        mimeBarCount(tabularData.slice(), mimeSvgCount);
+    });
+
+    d3.csv(`/s/${indexId}/3`).then(tabularData => {
+        sizeHistogram(tabularData, sizeHistogramSvg);
+    });
+
+    d3.csv(`/s/${indexId}/4`).then(tabularData => {
+        dateHistogram(tabularData, dateHistogramSvg);
+    });
+
+    treemapSvg.selectAll("*").remove();
+    treemapSvg.attr("viewBox", [0, 0, width, height])
+        .attr("xmlns", "http://www.w3.org/2000/svg")
+        .attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
+        .attr("version", "1.1")
+        .style("overflow", "visible")
+        .style("font", "10px sans-serif");
+
+}
+
+window.onload = function () {
+    CONF.load();
+
+    $.jsonPost("i").then(resp => {
+        const select = $("#indices");
+
+        const urlIndices = (new URLSearchParams(location.search)).get("i");
+        resp["indices"].forEach(idx => {
+            indexMap[idx.id] = idx.name;
+            select.append($("<option>")
+                .attr("value", idx.id)
+                .append(idx.name));
+
+            if (urlIndices && urlIndices.split(",").indexOf(idx.name) !== -1) {
+                select.select(idx.name);
+            }
+        });
+
+        updateStats();
+    });
+};
+
+function fullScreen(selector) {
+    const card = document.getElementById(selector);
+    card.classList.toggle("full-screen");
+}
+
+function exportTreemap() {
+    domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
+        .then(function (blob) {
+            let a = document.createElement("a");
+            let url = URL.createObjectURL(blob);
+
+            a.href = url;
+            a.download = `${indexMap[$("#indices").val()]}_treemap.png`;
+            document.body.appendChild(a);
+            a.click();
+            setTimeout(function() {
+                document.body.removeChild(a);
+                window.URL.revokeObjectURL(url);
+            }, 0);
+        });
+}
+</script>
+</body>
+</html>

src/stats.c

@@ -0,0 +1,353 @@
+#include "sist.h"
+#include "io/serialize.h"
+#include "ctx.h"
+
+#include <glib.h>
+
+static GHashTable *FlatTree;
+static GHashTable *BufferTable;
+
+static GHashTable *AggMime;
+static GHashTable *AggSize;
+static GHashTable *AggDate;
+
+#define SIZE_BUCKET (long)(5 * 1024 * 1024)
+#define DATE_BUCKET (long)(2629800)
+
+static long TotalSize = 0;
+static long DocumentCount = 0;
+
+typedef struct {
+    long size;
+    long count;
+} agg_t;
+
+void fill_tables(cJSON *document, UNUSED(const char uuid_str[UUID_STR_LEN])) {
+
+    if (cJSON_GetObjectItem(document, "parent") != NULL) {
+        return;
+    }
+
+    const char *json_path = cJSON_GetObjectItem(document, "path")->valuestring;
+    char *path = malloc(strlen(json_path) + 1);
+    strcpy(path, json_path);
+
+    const char *json_mime = cJSON_GetObjectItem(document, "mime")->valuestring;
+    char *mime;
+    if (json_mime == NULL) {
+        mime = NULL;
+    } else {
+        mime = malloc(strlen(json_mime) + 1);
+        strcpy(mime, json_mime);
+    }
+
+    long size = (long) cJSON_GetObjectItem(document, "size")->valuedouble;
+    int mtime = cJSON_GetObjectItem(document, "mtime")->valueint;
+
+    // treemap
+    void *existing_path = g_hash_table_lookup(FlatTree, path);
+    if (existing_path == NULL) {
+        g_hash_table_insert(FlatTree, path, (gpointer) size);
+    } else {
+        g_hash_table_replace(FlatTree, path, (gpointer) ((long) existing_path + size));
+    }
+
+    // mime agg
+    if (mime != NULL) {
+        agg_t *orig_agg = g_hash_table_lookup(AggMime, mime);
+        if (orig_agg == NULL) {
+            agg_t *agg = malloc(sizeof(agg_t));
+            agg->size = size;
+            agg->count = 1;
+            g_hash_table_insert(AggMime, mime, agg);
+        } else {
+            orig_agg->size += size;
+            orig_agg->count += 1;
+            free(mime);
+        }
+    }
+
+    // size agg
+    long size_bucket = size - (size % SIZE_BUCKET);
+    agg_t *orig_agg = g_hash_table_lookup(AggSize, (gpointer) size_bucket);
+    if (orig_agg == NULL) {
+        agg_t *agg = malloc(sizeof(agg_t));
+        agg->size = size;
+        agg->count = 1;
+        g_hash_table_insert(AggSize, (gpointer) size_bucket, agg);
+    } else {
+        orig_agg->count += 1;
+        orig_agg->size += size;
+    }
+
+    // date agg
+    long date_bucket = mtime - (mtime % DATE_BUCKET);
+    orig_agg = g_hash_table_lookup(AggDate, (gpointer) date_bucket);
+    if (orig_agg == NULL) {
+        agg_t *agg = malloc(sizeof(agg_t));
+        agg->size = size;
+        agg->count = 1;
+        g_hash_table_insert(AggDate, (gpointer) date_bucket, agg);
+    } else {
+        orig_agg->count += 1;
+        orig_agg->size += size;
+    }
+
+    TotalSize += size;
+    DocumentCount += 1;
+}
+
+void read_index_into_tables(index_t *index) {
+    DIR *dir = opendir(index->path);
+    struct dirent *de;
+    while ((de = readdir(dir)) != NULL) {
+        if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
+            char file_path[PATH_MAX];
+            snprintf(file_path, PATH_MAX, "%s/%s", index->path, de->d_name);
+            read_index(file_path, index->desc.uuid, index->desc.type, fill_tables);
+        }
+    }
+    closedir(dir);
+}
+
+static size_t rfind(const char *str, int c) {
+    for (int i = (int)strlen(str); i >= 0; i--) {
+        if (str[i] == c) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+int merge_up(double thresh) {
+    long min_size = (long) (thresh * (double) TotalSize);
+
+    int count = 0;
+    GHashTableIter iter;
+    g_hash_table_iter_init(&iter, FlatTree);
+
+    void *key;
+    void *value;
+
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        long size = (long) value;
+
+        if (size < min_size) {
+            int stop = rfind(key, '/');
+            if (stop == -1) {
+                stop = 0;
+            }
+            char *parent = malloc(stop + 1);
+            strncpy(parent, key, stop);
+            *(parent + stop) = '\0';
+
+            void *existing_parent = g_hash_table_lookup(FlatTree, parent);
+            if (existing_parent == NULL) {
+                void *existing_parent2_key;
+                void *existing_parent2_val;
+                int found = g_hash_table_lookup_extended(BufferTable, parent, &existing_parent2_key,
+                                                         &existing_parent2_val);
+                if (!found) {
+                    g_hash_table_insert(BufferTable, parent, value);
+                } else {
+                    g_hash_table_replace(BufferTable, parent, (gpointer) ((long) existing_parent2_val + size));
+                    free(existing_parent2_key);
+                }
+            } else {
+                g_hash_table_replace(FlatTree, parent, (gpointer) ((long) existing_parent + size));
+            }
+
+            g_hash_table_iter_remove(&iter);
+
+            count += 1;
+        }
+    }
+
+    g_hash_table_iter_init(&iter, BufferTable);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        g_hash_table_insert(FlatTree, key, value);
+        g_hash_table_iter_remove(&iter);
+    }
+
+    int size = g_hash_table_size(FlatTree);
+
+    LOG_DEBUGF("stats.c", "Merge up iteration (%d merged, %d in tree)", count, size)
+    return count;
+}
+
+/**
+ * Assumes out is at at least PATH_MAX *4
+ */
+void csv_escape(char *dst, const char *str) {
+
+    const char *ptr = str;
+    char *out = dst;
+
+    if (rfind(str, ',') == -1 && rfind(str, '"') == -1) {
+        strcpy(dst, str);
+        return;
+    }
+
+    *out++ = '"';
+    char c;
+    while ((c = *ptr++) != 0) {
+        if (c == '"') {
+            *out++ = '"';
+            *out++ = '"';
+        } else {
+            *out++ = c;
+        }
+    }
+    *out++ = '"';
+    *out = '\0';
+}
+
+int open_or_exit(const char *path) {
+    int fd = open(path, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR);
+    if (fd < 0) {
+        LOG_FATALF("stats.c", "Error while creating file: %s [%d]\n", strerror(errno), errno)
+    }
+    return fd;
+}
+
+#define TREEMAP_CSV_HEADER "path,size"
+#define MIME_AGG_CSV_HEADER "mime,size,count"
+#define SIZE_AGG_CSV_HEADER "bucket,size,count"
+#define DATE_AGG_CSV_HEADER "bucket,size,count"
+
+void write_treemap_csv(double thresh, const char *out_path) {
+
+    void *key;
+    void *value;
+
+    long min_size = (long) (thresh * (double) TotalSize);
+
+    int fd = open_or_exit(out_path);
+    int ret = write(fd, TREEMAP_CSV_HEADER, sizeof(TREEMAP_CSV_HEADER) - 1);
+    if (ret == -1) {
+        LOG_FATALF("stats.c", "Write error: %s", strerror(errno))
+    }
+
+    GHashTableIter iter;
+    g_hash_table_iter_init(&iter, FlatTree);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        long size = (long) value;
+
+        if (size >= min_size) {
+            char path_buf[PATH_MAX * 4];
+            char buf[PATH_MAX * 4 + 16];
+
+            csv_escape(path_buf, key);
+            size_t written = sprintf(buf, "\n%s,%ld", path_buf, (long) value);
+            ret = write(fd, buf, written);
+            if (ret == -1) {
+                LOG_FATALF("stats.c", "Write error: %s", strerror(errno))
+            }
+        }
+    }
+    close(fd);
+}
+
+void write_agg_csv_str(const char *out_path, const char *header, GHashTable *table) {
+    void *key;
+    void *value;
+    char buf[4096];
+
+    int fd = open_or_exit(out_path);
+    int ret = write(fd, header, strlen(header));
+    if (ret == -1) {
+        LOG_FATALF("stats.c", "Write error: %s", strerror(errno))
+    }
+
+    GHashTableIter iter;
+    g_hash_table_iter_init(&iter, table);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        agg_t *agg = value;
+
+        size_t written = sprintf(buf, "\n%s,%ld,%ld", (const char*)key, agg->size, agg->count);
+        ret = write(fd, buf, written);
+        if (ret == -1) {
+            LOG_FATALF("stats.c", "Write error: %s", strerror(errno))
+        }
+    }
+
+    close(fd);
+}
+
+void write_agg_csv_long(const char *out_path, const char *header, GHashTable *table) {
+    void *key;
+    void *value;
+    char buf[4096];
+
+    int fd = open_or_exit(out_path);
+    int ret = write(fd, header, strlen(header));
+    if (ret == -1) {
+        LOG_FATALF("stats.c", "Write error: %s", strerror(errno))
+    }
+
+    GHashTableIter iter;
+    g_hash_table_iter_init(&iter, table);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        agg_t *agg = value;
+        size_t written = sprintf(buf, "\n%ld,%ld,%ld", (long)key, agg->size, agg->count);
+        ret = write(fd, buf, written);
+        if (ret == -1) {
+            LOG_FATALF("stats.c", "Write error: %s", strerror(errno))
+        }
+    }
+
+    close(fd);
+}
+
+int generate_stats(index_t *index, const double threshold, const char *out_prefix) {
+
+    FlatTree = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL);
+    BufferTable = g_hash_table_new(g_str_hash, g_str_equal);
+
+    AggMime = g_hash_table_new_full(g_str_hash, g_str_equal, free, free);
+    AggSize = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, free);
+    AggDate = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, free);
+
+    LOG_INFO("stats.c", "Generating stats...")
+
+    read_index_into_tables(index);
+
+    LOG_DEBUG("stats.c", "Read index into tables")
+    LOG_DEBUGF("stats.c", "Total size is %ld", TotalSize)
+    LOG_DEBUGF("stats.c", "Document count is %ld", DocumentCount)
+    LOG_DEBUGF("stats.c", "Merging small directories upwards with a threshold of %f%%", threshold * 100)
+
+    while (merge_up(threshold) > 100) {}
+
+    char tmp[PATH_MAX];
+
+    strncpy(tmp, out_prefix, sizeof(tmp));
+    strcat(tmp, "treemap.csv");
+    write_treemap_csv(threshold, tmp);
+
+    strncpy(tmp, out_prefix, sizeof(tmp));
+    strcat(tmp, "mime_agg.csv");
+    write_agg_csv_str(tmp, MIME_AGG_CSV_HEADER, AggMime);
+
+    strncpy(tmp, out_prefix, sizeof(tmp));
+    strcat(tmp, "size_agg.csv");
+    write_agg_csv_long(tmp, SIZE_AGG_CSV_HEADER, AggSize);
+
+    strncpy(tmp, out_prefix, sizeof(tmp));
+    strcat(tmp, "date_agg.csv");
+    write_agg_csv_long(tmp, DATE_AGG_CSV_HEADER, AggDate);
+
+    g_hash_table_remove_all(FlatTree);
+    g_hash_table_destroy(FlatTree);
+    g_hash_table_destroy(BufferTable);
+
+    g_hash_table_remove_all(AggMime);
+    g_hash_table_destroy(AggMime);
+    g_hash_table_remove_all(AggSize);
+    g_hash_table_destroy(AggSize);
+    g_hash_table_remove_all(AggDate);
+    g_hash_table_destroy(AggDate);
+
+    return 0;
+}
+

src/stats.h

@@ -0,0 +1,6 @@
+#ifndef SIST2_STATS_H
+#define SIST2_STATS_H
+
+int generate_stats(index_t *index, double threshold, const char* out_prefix);
+
+#endif

src/tpool.c

@@ -3,6 +3,8 @@
 #include "sist.h"
 #include <pthread.h>
 
+#define MAX_QUEUE_SIZE 10000
+
 typedef void (*thread_func_t)(void *arg);
 
 typedef struct tpool_work {
@@ -26,6 +28,7 @@ typedef struct tpool {
     int work_cnt;
     int done_cnt;
 
+    int free_arg;
     int stop;
 
     void (*cleanup_func)();
@@ -79,6 +82,10 @@ int tpool_add_work(tpool_t *pool, thread_func_t func, void *arg) {
         return 0;
     }
 
+    while ((pool->work_cnt - pool->done_cnt) >= MAX_QUEUE_SIZE) {
+        usleep(100000);
+    }
+
     pthread_mutex_lock(&(pool->work_mutex));
     if (pool->work_head == NULL) {
         pool->work_head = work;
@@ -121,7 +128,9 @@ static void *tpool_worker(void *arg) {
             }
 
             work->func(work->arg);
-            free(work->arg);
+            if (pool->free_arg) {
+                free(work->arg);
+            }
             free(work);
         }
 
@@ -138,8 +147,10 @@ static void *tpool_worker(void *arg) {
         pthread_mutex_unlock(&(pool->work_mutex));
     }
 
-    LOG_INFO("tpool.c", "Executing cleaup function")
-    pool->cleanup_func();
+    if (pool->cleanup_func != NULL) {
+        LOG_INFO("tpool.c", "Executing cleanup function")
+        pool->cleanup_func();
+    }
 
     pthread_cond_signal(&(pool->working_cond));
     pthread_mutex_unlock(&(pool->work_mutex));
@@ -207,13 +218,14 @@ void tpool_destroy(tpool_t *pool) {
  * Create a thread pool
  * @param thread_cnt Worker threads count
  */
-tpool_t *tpool_create(size_t thread_cnt, void cleanup_func()) {
+tpool_t *tpool_create(size_t thread_cnt, void cleanup_func(), int free_arg) {
 
     tpool_t *pool = malloc(sizeof(tpool_t));
     pool->thread_cnt = thread_cnt;
     pool->work_cnt = 0;
     pool->done_cnt = 0;
     pool->stop = 0;
+    pool->free_arg = free_arg;
     pool->cleanup_func = cleanup_func;
     pool->threads = calloc(sizeof(pthread_t), thread_cnt);
 

src/tpool.h

@@ -8,7 +8,7 @@ typedef struct tpool tpool_t;
 
 typedef void (*thread_func_t)(void *arg);
 
-tpool_t *tpool_create(size_t num, void (*cleanup_func)());
+tpool_t *tpool_create(size_t num, void (*cleanup_func)(), int free_arg);
 void tpool_start(tpool_t *pool);
 void tpool_destroy(tpool_t *tm);
 

src/types.h

@@ -10,7 +10,7 @@ typedef struct index_descriptor {
     char version[64];
     long timestamp;
     char root[PATH_MAX];
-    char rewrite_url[8196];
+    char rewrite_url[8192];
     short root_len;
     char name[1024];
     char type[64];
@@ -19,6 +19,7 @@ typedef struct index_descriptor {
 typedef struct index_t {
     struct index_descriptor desc;
     struct store_t *store;
+    struct store_t *tag_store;
     char path[PATH_MAX];
 } index_t;
 

src/util.c

@@ -26,10 +26,11 @@ dyn_buffer_t url_escape(char *str) {
 }
 
 char *abspath(const char *path) {
-    wordexp_t w;
-    wordexp(path, &w, 0);
 
-    char *abs = realpath(w.we_wordv[0], NULL);
+    char *expanded = expandpath(path);
+
+    char *abs = realpath(expanded, NULL);
+    free(expanded);
     if (abs == NULL) {
         return NULL;
     }
@@ -38,16 +39,46 @@ char *abspath(const char *path) {
         strcat(abs, "/");
     }
 
-    wordfree(&w);
     return abs;
 }
 
-char *expandpath(const char *path) {
-    wordexp_t w;
-    wordexp(path, &w, 0);
+void shell_escape(char *dst, const char *src) {
+    const char *ptr = src;
+    char *out = dst;
+    while ((*ptr)) {
+        char c = *ptr++;
 
-    char *expanded = malloc(strlen(w.we_wordv[0]) + 2);
-    strcpy(expanded, w.we_wordv[0]);
+        if (c == '&' || c == '\n' || c == '|' || c == ';' || c == '<' ||
+            c == '>' || c == '(' || c == ')' || c == '{' || c == '}') {
+            *out++ = '\\';
+        }
+        *out++ = c;
+    }
+    *out = 0;
+}
+
+char *expandpath(const char *path) {
+    char tmp[PATH_MAX * 2];
+
+    shell_escape(tmp, path);
+
+    wordexp_t w;
+    wordexp(tmp, &w, 0);
+
+    if (w.we_wordv == NULL) {
+        return NULL;
+    }
+
+    *tmp = '\0';
+    for (int i = 0; i < w.we_wordc; i++) {
+        strcat(tmp, w.we_wordv[i]);
+        if (i != w.we_wordc - 1) {
+            strcat(tmp, " ");
+        }
+    }
+
+    char *expanded = malloc(strlen(tmp) + 2);
+    strcpy(expanded, tmp);
     strcat(expanded, "/");
 
     wordfree(&w);
@@ -123,4 +154,92 @@ const char *find_file_in_paths(const char *paths[], const char *filename) {
     return NULL;
 }
 
+#define ESCAPE_CHAR ']'
 
+void str_escape(char *dst, const char *str) {
+    const size_t len = strlen(str);
+
+    char buf[16384];
+    memset(buf + len, 0, 8);
+    strcpy(buf, str);
+
+    char *cur = dst;
+    const char *ptr = buf;
+    const char *oldPtr = ptr;
+
+    utf8_int32_t c;
+    char tmp[16];
+
+    do {
+        ptr = (char *) utf8codepoint(ptr, &c);
+        *(int *) tmp = 0x00000000;
+        size_t code_len = (ptr - oldPtr);
+        memcpy(tmp, oldPtr, code_len);
+        oldPtr = ptr;
+
+        if (!utf8_validchr2(tmp)) {
+            for (int i = 0; i < code_len; i++) {
+                if (tmp[i] == 0) {
+                    break;
+                }
+
+                cur += sprintf(cur, "%c%02X", ESCAPE_CHAR, (unsigned char) tmp[i]);
+            }
+            continue;
+        }
+
+        if (c == ESCAPE_CHAR) {
+            *cur++ = ESCAPE_CHAR;
+            *cur++ = ESCAPE_CHAR;
+            continue;
+        }
+
+        if (((utf8_int32_t) 0xffffff80 & c) == 0) {
+            *(cur++) = (char) c;
+        } else if (((utf8_int32_t) 0xfffff800 & c) == 0) {
+            *(cur++) = 0xc0 | (char) (c >> 6);
+            *(cur++) = 0x80 | (char) (c & 0x3f);
+        } else if (((utf8_int32_t) 0xffff0000 & c) == 0) {
+            *(cur++) = 0xe0 | (char) (c >> 12);
+            *(cur++) = 0x80 | (char) ((c >> 6) & 0x3f);
+            *(cur++) = 0x80 | (char) (c & 0x3f);
+        } else {
+            *(cur++) = 0xf0 | (char) (c >> 18);
+            *(cur++) = 0x80 | (char) ((c >> 12) & 0x3f);
+            *(cur++) = 0x80 | (char) ((c >> 6) & 0x3f);
+            *(cur++) = 0x80 | (char) (c & 0x3f);
+        }
+
+    } while (*ptr != '\0');
+
+    *cur = '\0';
+}
+
+void str_unescape(char *dst, const char *str) {
+    char *cur = dst;
+    const char *ptr = str;
+
+    char tmp[3];
+    tmp[2] = '\0';
+
+    while (*ptr != 0) {
+        char c = *ptr++;
+
+        if (c == ESCAPE_CHAR) {
+            char next = *ptr;
+
+            if (next == ESCAPE_CHAR) {
+                *cur++ = (char) c;
+                ptr += 1;
+            } else {
+                tmp[0] = *(ptr);
+                tmp[1] = *(ptr + 1);
+                *cur++ = (char) strtol(tmp, NULL, 16);
+                ptr += 2;
+            }
+        } else {
+            *cur++ = c;
+        }
+    }
+    *cur = '\0';
+}

src/util.h

@@ -43,4 +43,9 @@ static int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode
 
 const char *find_file_in_paths(const char **paths, const char *filename);
 
+
+void str_escape(char *dst, const char *str);
+
+void str_unescape(char *dst, const char *str);
+
 #endif

src/web/serve.c

@@ -53,18 +53,87 @@ store_t *get_store(const char *index_id) {
     return NULL;
 }
 
+store_t *get_tag_store(const char *index_id) {
+    index_t *idx = get_index_by_id(index_id);
+    if (idx != NULL) {
+        return idx->tag_store;
+    }
+    return NULL;
+}
+
 void search_index(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(search_html), "Content-Type: text/html");
     mg_send(nc, search_html, sizeof(search_html));
     nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void javascript(struct mg_connection *nc) {
+void stats(struct mg_connection *nc) {
+    send_response_line(nc, 200, sizeof(stats_html), "Content-Type: text/html");
+    mg_send(nc, stats_html, sizeof(stats_html));
+    nc->flags |= MG_F_SEND_AND_CLOSE;
+}
+
+void stats_files(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
+
+    if (path->len != UUID_STR_LEN + 4) {
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return;
+    }
+
+    char arg_uuid[UUID_STR_LEN];
+    memcpy(arg_uuid, hm->uri.p + 3, UUID_STR_LEN);
+    *(arg_uuid + UUID_STR_LEN - 1) = '\0';
+
+    index_t *index = get_index_by_id(arg_uuid);
+    if (index == NULL) {
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return;
+    }
+
+    const char *file;
+    switch (atoi(hm->uri.p + 3 + UUID_STR_LEN)) {
+        case 1:
+            file = "treemap.csv";
+            break;
+        case 2:
+            file = "mime_agg.csv";
+            break;
+        case 3:
+            file = "size_agg.csv";
+            break;
+        case 4:
+            file = "date_agg.csv";
+            break;
+        default:
+            nc->flags |= MG_F_SEND_AND_CLOSE;
+            return;
+    }
+
+    char disposition[8192];
+    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s\"", file);
+
+    char full_path[PATH_MAX];
+    strcpy(full_path, index->path);
+    strcat(full_path, file);
+
+    mg_http_serve_file(nc, hm, full_path, mg_mk_str("text/csv"), mg_mk_str(disposition));
+    nc->flags |= MG_F_SEND_AND_CLOSE;
+}
+
+void javascript_lib(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(bundle_js), "Content-Type: application/javascript");
     mg_send(nc, bundle_js, sizeof(bundle_js));
     nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
+void javascript_search(struct mg_connection *nc) {
+    send_response_line(nc, 200, sizeof(search_js), "Content-Type: application/javascript");
+    mg_send(nc, search_js, sizeof(search_js));
+    nc->flags |= MG_F_SEND_AND_CLOSE;
+}
+
 int client_requested_dark_theme(struct http_message *hm) {
     struct mg_str *cookie_header = mg_get_http_header(hm, "cookie");
     if (cookie_header == NULL) {
@@ -114,6 +183,7 @@ void thumbnail(struct mg_connection *nc, struct http_message *hm, struct mg_str
 
     if (path->len != UUID_STR_LEN * 2 + 2) {
         LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) path->len, path->p)
+        mg_http_send_error(nc, 404, NULL);
         nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
@@ -130,6 +200,7 @@ void thumbnail(struct mg_connection *nc, struct http_message *hm, struct mg_str
     int ret = uuid_parse(arg_uuid, uuid);
     if (ret != 0) {
         LOG_DEBUGF("serve.c", "Invalid thumbnail UUID: %s", arg_uuid)
+        mg_http_send_error(nc, 404, NULL);
         nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
@@ -137,6 +208,7 @@ void thumbnail(struct mg_connection *nc, struct http_message *hm, struct mg_str
     store_t *store = get_store(arg_index);
     if (store == NULL) {
         LOG_DEBUGF("serve.c", "Could not get store for index: %s", arg_index)
+        mg_http_send_error(nc, 404, NULL);
         nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
@@ -155,6 +227,7 @@ void search(struct mg_connection *nc, struct http_message *hm) {
 
     if (hm->body.len == 0) {
         LOG_DEBUG("serve.c", "Client sent empty body, ignoring request")
+        mg_http_send_error(nc, 500, NULL);
         nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
@@ -174,12 +247,19 @@ int serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
 
     const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
     const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
+
+    char name_unescaped[PATH_MAX * 3];
+    str_unescape(name_unescaped, name);
+
+    char path_unescaped[PATH_MAX * 3];
+    str_unescape(path_unescaped, path);
+
     const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
 
-    char url[8196];
+    char url[8192];
     snprintf(url, sizeof(url),
              "%s%s/%s%s%s",
-             idx->desc.rewrite_url, path, name, strlen(ext) == 0 ? "" : ".", ext);
+             idx->desc.rewrite_url, path_unescaped, name_unescaped, strlen(ext) == 0 ? "" : ".", ext);
 
     dyn_buffer_t encoded = url_escape(url);
     mg_http_send_redirect(
@@ -198,14 +278,20 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
     const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
     const char *mime = cJSON_GetObjectItem(json, "mime")->valuestring;
 
+    char name_unescaped[PATH_MAX * 3];
+    str_unescape(name_unescaped, name);
+
+    char path_unescaped[PATH_MAX * 3];
+    str_unescape(path_unescaped, path);
+
     char full_path[PATH_MAX];
     snprintf(full_path, PATH_MAX, "%s%s%s%s%s%s",
-             idx->desc.root, path, strlen(path) == 0 ? "" : "/",
-             name, strlen(ext) == 0 ? "" : ".", ext);
+             idx->desc.root, path_unescaped, strlen(path_unescaped) == 0 ? "" : "/",
+             name_unescaped, strlen(ext) == 0 ? "" : ".", ext);
 
     LOG_DEBUGF("serve.c", "Serving file from disk: %s", full_path)
 
-    char disposition[8196];
+    char disposition[8192];
     snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s%s%s\"",
              name, strlen(ext) == 0 ? "" : ".", ext);
 
@@ -242,6 +328,7 @@ void document_info(struct mg_connection *nc, struct http_message *hm, struct mg_
 
     if (path->len != UUID_STR_LEN + 2) {
         LOG_DEBUGF("serve.c", "Invalid document_info path: %.*s", (int) path->len, path->p)
+        mg_http_send_error(nc, 404, NULL);
         nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
@@ -256,6 +343,7 @@ void document_info(struct mg_connection *nc, struct http_message *hm, struct mg_
     cJSON *index_id = cJSON_GetObjectItem(source, "index");
     if (index_id == NULL) {
         cJSON_Delete(doc);
+        mg_http_send_error(nc, 404, NULL);
         nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
@@ -263,6 +351,7 @@ void document_info(struct mg_connection *nc, struct http_message *hm, struct mg_
     index_t *idx = get_index_by_id(index_id->valuestring);
     if (idx == NULL) {
         cJSON_Delete(doc);
+        mg_http_send_error(nc, 404, NULL);
         nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
@@ -280,6 +369,7 @@ void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path
 
     if (path->len != UUID_STR_LEN + 2) {
         LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) path->len, path->p)
+        mg_http_send_error(nc, 404, NULL);
         nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
@@ -299,6 +389,7 @@ void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path
         index_id = cJSON_GetObjectItem(source, "index");
         if (index_id == NULL) {
             cJSON_Delete(doc);
+            mg_http_send_error(nc, 404, NULL);
             nc->flags |= MG_F_SEND_AND_CLOSE;
             return;
         }
@@ -314,6 +405,7 @@ void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path
     if (idx == NULL) {
         cJSON_Delete(doc);
         nc->flags |= MG_F_SEND_AND_CLOSE;
+        mg_http_send_error(nc, 404, NULL);
         return;
     }
 
@@ -338,6 +430,177 @@ void status(struct mg_connection *nc) {
     nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
+typedef struct {
+    char *name;
+    int delete;
+    char *relpath;
+    char *doc_id;
+} tag_req_t;
+
+tag_req_t *parse_tag_request(cJSON *json) {
+
+    if (!cJSON_IsObject(json)) {
+        return NULL;
+    }
+
+    cJSON *arg_name = cJSON_GetObjectItem(json, "name");
+    if (arg_name == NULL || !cJSON_IsString(arg_name)) {
+        return NULL;
+    }
+
+    cJSON *arg_delete = cJSON_GetObjectItem(json, "delete");
+    if (arg_delete == NULL || !cJSON_IsBool(arg_delete)) {
+        return NULL;
+    }
+
+    cJSON *arg_relpath = cJSON_GetObjectItem(json, "relpath");
+    if (arg_relpath == NULL || !cJSON_IsString(arg_relpath)) {
+        return NULL;
+    }
+
+    cJSON *arg_doc_id = cJSON_GetObjectItem(json, "doc_id");
+    if (arg_doc_id == NULL || !cJSON_IsString(arg_doc_id)) {
+        return NULL;
+    }
+
+    tag_req_t *req = malloc(sizeof(tag_req_t));
+    req->delete = arg_delete->valueint;
+    req->name = arg_name->valuestring;
+    req->relpath = arg_relpath->valuestring;
+    req->doc_id = arg_doc_id->valuestring;
+
+    return req;
+}
+
+void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
+    if (path->len != UUID_STR_LEN + 4) {
+        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) path->len, path->p)
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return;
+    }
+
+    char arg_index[UUID_STR_LEN];
+    memcpy(arg_index, hm->uri.p + 5, UUID_STR_LEN);
+    *(arg_index + UUID_STR_LEN - 1) = '\0';
+
+    if (hm->body.len < 2 || hm->method.len != 4 || memcmp(&hm->method, "POST", 4) == 0) {
+        LOG_DEBUG("serve.c", "Invalid tag request")
+        mg_http_send_error(nc, 400, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return;
+    }
+
+    store_t *store = get_tag_store(arg_index);
+    if (store == NULL) {
+        LOG_DEBUGF("serve.c", "Could not get tag store for index: %s", arg_index)
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return;
+    }
+
+    char *body = malloc(hm->body.len + 1);
+    memcpy(body, hm->body.p, hm->body.len);
+    *(body + hm->body.len) = '\0';
+    cJSON *json = cJSON_Parse(body);
+
+    tag_req_t *arg_req = parse_tag_request(json);
+    if (arg_req == NULL) {
+        LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index)
+        cJSON_Delete(json);
+        free(body);
+        mg_http_send_error(nc, 400, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return;
+    }
+
+    cJSON *arr = NULL;
+
+    size_t data_len = 0;
+    const char *data = store_read(store, arg_req->relpath, strlen(arg_req->relpath), &data_len);
+    if (data_len == 0) {
+        arr = cJSON_CreateArray();
+    } else {
+        arr = cJSON_Parse(data);
+    }
+
+    if (arg_req->delete) {
+
+        if (data_len > 0) {
+            cJSON *element = NULL;
+            int i = 0;
+            cJSON_ArrayForEach(element, arr) {
+                if (strcmp(element->valuestring, arg_req->name) == 0) {
+                    cJSON_DeleteItemFromArray(arr, i);
+                    break;
+                }
+                i++;
+            }
+        }
+
+        char buf[8192];
+        snprintf(buf, sizeof(buf),
+                 "{"
+                 "    \"script\" : {"
+                 "        \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\","
+                 "        \"lang\": \"painless\","
+                 "        \"params\" : {"
+                 "            \"tag\" : \"%s\""
+                 "        }"
+                 "    }"
+                 "}", arg_req->name
+        );
+
+        char url[4096];
+        snprintf(url, sizeof(url), "%s/sist2/_update/%s", WebCtx.es_url, arg_req->doc_id);
+        nc->user_data = web_post_async(url, buf);
+
+    } else {
+        cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name));
+
+        char buf[8192];
+        snprintf(buf, sizeof(buf),
+                 "{"
+                 "    \"script\" : {"
+                 "        \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\","
+                 "        \"lang\": \"painless\","
+                 "        \"params\" : {"
+                 "            \"tag\" : \"%s\""
+                 "        }"
+                 "    }"
+                 "}", arg_req->name
+        );
+
+        char url[4096];
+        snprintf(url, sizeof(url), "%s/sist2/_update/%s", WebCtx.es_url, arg_req->doc_id);
+        nc->user_data = web_post_async(url, buf);
+    }
+
+    char *json_str = cJSON_PrintUnformatted(arr);
+    store_write(store, arg_req->relpath, strlen(arg_req->relpath) + 1, json_str, strlen(json_str) + 1);
+
+    free(arg_req);
+    free(json_str);
+    cJSON_Delete(json);
+    cJSON_Delete(arr);
+    free(body);
+}
+
+int validate_auth(struct mg_connection *nc, struct http_message *hm) {
+    char user[256] = {0,};
+    char pass[256] = {0,};
+
+    int ret = mg_get_http_basic_auth(hm, user, sizeof(user), pass, sizeof(pass));
+    if (ret == -1 || strcmp(user, WebCtx.auth_user) != 0 || strcmp(pass, WebCtx.auth_pass) != 0) {
+        mg_printf(nc, "HTTP/1.1 401 Unauthorized\r\n"
+                      "WWW-Authenticate: Basic realm=\"sist2\"\r\n"
+                      "Content-Length: 0\r\n\r\n");
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        return FALSE;
+    }
+    return TRUE;
+}
+
 static void ev_router(struct mg_connection *nc, int ev, void *p) {
     struct mg_str scheme;
     struct mg_str user_info;
@@ -351,21 +614,14 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
         struct http_message *hm = (struct http_message *) p;
 
         if (mg_parse_uri(hm->uri, &scheme, &user_info, &host, &port, &path, &query, &fragment) != 0) {
+            mg_http_send_error(nc, 400, NULL);
             nc->flags |= MG_F_SEND_AND_CLOSE;
             return;
         }
 
 
         if (WebCtx.auth_enabled == TRUE) {
-            char user[256] = {0,};
-            char pass[256] = {0,};
-
-            int ret = mg_get_http_basic_auth(hm, user, sizeof(user), pass, sizeof(pass));
-            if (ret == -1 || strcmp(user, WebCtx.auth_user) != 0 || strcmp(pass, WebCtx.auth_pass) != 0) {
-                mg_printf(nc, "HTTP/1.1 401 Unauthorized\r\n"
-                              "WWW-Authenticate: Basic realm=\"sist2\"\r\n"
-                              "Content-Length: 0\r\n\r\n");
-                nc->flags |= MG_F_SEND_AND_CLOSE;
+            if (!validate_auth(nc, hm)) {
                 return;
             }
         }
@@ -374,8 +630,12 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
             search_index(nc);
         } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/css")))) {
             style(nc, hm);
-        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/js")))) {
-            javascript(nc);
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/stats")))) {
+            stats(nc);
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/jslib")))) {
+            javascript_lib(nc);
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/jssearch")))) {
+            javascript_search(nc);
         } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/img/sprite-skin-flat.png")))) {
             img_sprite_skin_flat(nc, hm);
         } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/es")))) {
@@ -388,9 +648,19 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
             file(nc, hm, &path);
         } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/t/")))) {
             thumbnail(nc, hm, &path);
+        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/s/")))) {
+            stats_files(nc, hm, &path);
+        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/tag/")))) {
+            if (WebCtx.tag_auth_enabled == TRUE) {
+                if (!validate_auth(nc, hm)) {
+                    return;
+                }
+            }
+            tag(nc, hm, &path);
         } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/d/")))) {
             document_info(nc, hm, &path);
         } else {
+            mg_http_send_error(nc, 404, NULL);
             nc->flags |= MG_F_SEND_AND_CLOSE;
         }
 
@@ -421,7 +691,7 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
                         free(json_str);
                         free(tmp);
                     }
-                    //todo return error code
+                    mg_http_send_error(nc, 500, NULL);
                 }
 
                 free_response(r);