Update libmupdf

Optionally ES schema from file #117
sidecar files #114 , version bump
2025-12-12 15:08:53 +00:00 · 2020-10-25 09:44:30 -04:00 · 2020-10-25 09:44:30 -04:00 · 2020-10-25 09:44:30 -04:00 · 2020-10-25 09:44:30 -04:00 · 2020-10-25 09:44:30 -04:00
39 changed files with 767 additions and 311 deletions
--- a/.github/ISSUE_TEMPLATE/issue-template.md
+++ b/.github/ISSUE_TEMPLATE/issue-template.md
@@ -9,7 +9,7 @@ assignees: ''

 sist2 version:

-Platform (Linux or Docker):
+Platform (Linux or Docker, x86-64 or arm64):

 Elasticsearch version:

--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -9,8 +9,11 @@ add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)

-add_executable(
-        sist2
+add_executable(sist2
+
+        # argparse
+        third-party/argparse/argparse.h third-party/argparse/argparse.c
+
        src/main.c
        src/sist.h
        src/io/walk.h src/io/walk.c
@@ -25,12 +28,9 @@ add_executable(
        src/util.c src/util.h
        src/ctx.h src/types.h
        src/log.c src/log.h
-
-        # argparse
-        third-party/argparse/argparse.h third-party/argparse/argparse.c
-
        src/cli.c src/cli.h
-        src/stats.c src/stats.h src/ctx.c)
+        src/stats.c src/stats.h src/ctx.c
+        src/parsing/sidecar.c src/parsing/sidecar.h)

 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
 set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
@@ -73,7 +73,6 @@ if (SIST_DEBUG)
            sist2
            PRIVATE
            -fsanitize=address
-#            -static
    )
    set_target_properties(
            sist2
@@ -81,7 +80,7 @@ if (SIST_DEBUG)
            OUTPUT_NAME sist2_debug
    )
 else ()
-#    set(VCPKG_BUILD_TYPE release)
+    #    set(VCPKG_BUILD_TYPE release)
    target_compile_options(
            sist2
            PRIVATE
@@ -106,7 +105,6 @@ target_link_libraries(
        argparse
        unofficial::glib::glib
        unofficial::mongoose::mongoose
-#        OpenSSL::SSL OpenSSL::Crypto
        CURL::libcurl

        ${UUID_LIB}
--- a/DockerArm64/Dockerfile
+++ b/DockerArm64/Dockerfile
@@ -0,0 +1,22 @@
+FROM ubuntu:19.10
+MAINTAINER simon987 <me@simon987.net>
+
+RUN apt update
+RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 \
+ curl libtiff5 libpng16-16 libpcre3
+
+RUN mkdir -p /usr/share/tessdata && \
+    cd /usr/share/tessdata/ && \
+    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
+
+ADD sist2_arm64 /root/sist2
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["/root/sist2"]
--- a/DockerArm64/build.sh
+++ b/DockerArm64/build.sh
@@ -0,0 +1,13 @@
+rm ./sist2_arm64
+cp ../sist2_arm64.gz .
+gzip -d sist2_arm64.gz
+
+version=$(./sist2_arm64 --version)
+
+echo "Version ${version}"
+docker build . -t simon987/sist2-arm64:"${version}" -t simon987/sist2-arm64:latest
+
+docker push simon987/sist2-arm64:"${version}"
+docker push simon987/sist2-arm64:latest
+
+docker run --rm simon987/sist2-arm64 -v
--- a/README.md
+++ b/README.md
@@ -2,6 +2,8 @@
 [![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
 [![Development snapshots](https://ci.simon987.net/app/rest/builds/buildType(Sist2_Build)/statusIcon)](https://files.simon987.net/artifacts/Sist2/Build/)

+**Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
+
 # sist2

 sist2 (Simple incremental search tool)
@@ -102,7 +104,7 @@ scan is also supported.
 
 ### OCR

-You can enable OCR support for pdf,xps,cbz,cbr,fb2,epub file types with the
+You can enable OCR support for pdf,xps,fb2,epub file types with the
 `--ocr <lang>` option. Download the language data files with your
 package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).

@@ -124,7 +126,7 @@ binaries (GCC 7+ required).
 1. Install compile-time dependencies

   ```bash
-   vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libuuid libmagic libraw
+   vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libuuid libmagic libraw curl[core,ssl] jbig2dec brotli libmupdf
   ```

 2. Build
--- a/ci/build_arm64.sh
+++ b/ci/build_arm64.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+VCPKG_ROOT="/vcpkg"
+
+rm *.gz
+
+rm -rf CMakeFiles CMakeCache.txt
+cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j 4
+strip sist2
+mv sist2 sist2_arm64
+gzip -9 sist2_arm64
--- a/docs/USAGE.md
+++ b/docs/USAGE.md
@@ -16,6 +16,7 @@
    * [link to specific indices](#link-to-specific-indices)
 * [exec-script](#exec-script)
 * [tagging](#tagging)
+* [sidecar files](#sidecar-files)

 ```
 Usage: sist2 scan [OPTION]... PATH
@@ -49,19 +50,27 @@ Scan options
 Index options
    -t, --threads=<int>           Number of threads. DEFAULT=1
    --es-url=<str>                Elasticsearch url with port. DEFAULT=http://localhost:9200
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
    -p, --print                   Just print JSON documents to stdout.
    --script-file=<str>           Path to user script.
+    --mappings-file=<str>         Path to Elasticsearch mappings.
+    --settings-file=<str>         Path to Elasticsearch settings.
+    --async-script                Execute user script asynchronously.
    --batch-size=<int>            Index batch size. DEFAULT: 100
    -f, --force-reset             Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)

 Web options
    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
    --bind=<str>                  Listen on this address. DEFAULT=localhost:4090
    --auth=<str>                  Basic auth in user:password format
    --tag-auth=<str>              Basic auth in user:password format for tagging

 Exec-script options
+    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
    --script-file=<str>           Path to user script.
+    --async-script                Execute user script asynchronously.
 Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```

@@ -147,10 +156,13 @@ documents.idx/
 ├── agg_mime.csv
 ├── agg_date.csv
 ├── add_size.csv
-├── thumbs
+├── thumbs/
 |   ├── data.mdb
 |   └── lock.mdb
-└── tags
+├── tags/
+|   ├── data.mdb
+|   └── lock.mdb
+└── meta/
    ├── data.mdb
    └── lock.mdb
 ```
@@ -177,9 +189,11 @@ by a third party application. The 'external' index must have the following forma
 my_index/
 ├── descriptor.json
 ├── _index_0
-└── thumbs
-    ├── data.mdb
-    └── lock.mdb
+└── thumbs/
+|   ├── data.mdb
+|   └── lock.mdb
+└── meta/
+    └── <empty>
 ```

 *descriptor.json*:
@@ -239,17 +253,25 @@ it is currently unsupported and has no guaranties of back/forward compatibility.
 * `--es-url` 
 Elasticsearch url and port. If you are using docker, make sure that both containers are on the
 same network.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
 * `-p, --print` 
    Print index in JSON format to stdout.
 * `--script-file` 
    Path to user script. See [Scripting](scripting.md).
+ * `--mappings-file`
+    Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used.
+ * `--settings-file`
+    Path to custom Elasticsearch settings. *(See above)*
+ * `--async-script` 
+    Use `wait_for_completion=false` elasticsearch option while executing user script.
+     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
 * `--batch-size=<int>` 
    Index batch size. Indexing is generally faster with larger batches, but payloads that
    are too large will fail and additional overhead for retrying with smaller sizes may slow
    down the process.
 * `-f, --force-reset` 
    Reset Elasticsearch mappings and settings.
-    **(You must use this option the first time you use the index command)**.
    
 ### Index examples

@@ -273,6 +295,8 @@ sist2 index --print ./my_index/ | jq | less

 ### Web options
 * `--es-url=<str>` Elasticsearch url.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
 * `--bind=<str>` Listen on this address.
 * `--auth=<str>` Basic auth in user:password format
 * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
@@ -330,9 +354,48 @@ See [Automatic tagging](#automatic-tagging) for information about tag
 hierarchies and tag colors.

 \* *It can take a few seconds to take effect in new search queries, and the page needs 
-    to be reloaded for the tag tab to update*
+    to be reloaded for the tags tab to update*


 ### Automatic tagging

-See [scripting](docs/scripting.md) documentation.
+See [scripting](scripting.md) documentation.
+
+# Sidecar files
+
+When scanning, sist2 will read metadata from `.s2meta` JSON files and overwrite the 
+original document's metadata. Sidecar metadata files will also work inside archives.
+Sidecar files themselves are not saved in the index.
+
+This feature is useful to leverage third-party applications such as speech-to-text or
+OCR to add additional metadata to a file.
+
+**Example**
+
+```
+~/Documents/
+├── Video.mp4
+└── Video.mp4.s2meta
+```
+
+The sidecar file must have exactly the same file path and the `.s2meta` suffix.
+
+`Video.mp4.s2meta`:
+```json
+{
+  "content": "This sidecar file will overwrite some metadata fields of Video.mp4",
+  "author": "Some author",
+  "duration": 12345,
+  "bitrate": 67890,
+  "some_arbitrary_field": [1,2,3]
+}
+```
+
+```
+sist2 scan ~/Documents -o ./docs.idx
+sist2 index ./docs.idx
+```
+
+*NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,
+it is not currently possible to restore both manual tags and sidecar tags without user scripts
+while reindexing.
--- a/schema/mappings.json
+++ b/schema/mappings.json
@@ -54,6 +54,10 @@
      "type": "integer",
      "index": false
    },
+    "pages": {
+      "type": "integer",
+      "index": false
+    },
    "mtime": {
      "type": "integer"
    },
--- a/scripts/mime.csv
+++ b/scripts/mime.csv
@@ -347,7 +347,8 @@ text/javascript, js
 text/mcf, mcf
 text/pascal, pas
 text/PGP,
-text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml
+text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml|make|log|markdown|yaml
+application/vnd.coffeescript, coffee
 text/richtext, rt|rtf|rtx
 text/rtf,
 text/scriplet, wsc
@@ -448,3 +449,4 @@ image/x-sony-arw, arw
 image/x-sony-sr2, sr2
 image/x-sony-srf, srf
 image/x-epson-erf, erf
+sist2/sidecar, s2meta
--- a/scripts/mime.py
+++ b/scripts/mime.py
@@ -3,6 +3,7 @@ noparse = set()
 ext_in_hash = set()

 major_mime = {
+    "sist2": 0,
    "model": 1,
    "example": 2,
    "message": 3,
@@ -122,7 +123,11 @@ def mime_id(mime):
    elif mime in raw:
        mime_id += " | 0x00800000"
    elif mime == "application/x-empty":
+        cnt -= 1
        return "1"
+    elif mime == "sist2/sidecar":
+        cnt -= 1
+        return "2"
    return mime_id


--- a/src/cli.c
+++ b/src/cli.c
@@ -9,6 +9,7 @@
 #define DEFAULT_REWRITE_URL ""

 #define DEFAULT_ES_URL "http://localhost:9200"
+#define DEFAULT_ES_INDEX "sist2"
 #define DEFAULT_BATCH_SIZE 100

 #define DEFAULT_LISTEN_ADDRESS "localhost:4090"
@@ -55,6 +56,12 @@ void scan_args_destroy(scan_args_t *args) {

 void index_args_destroy(index_args_t *args) {
    //todo
+    if (args->es_mappings_path) {
+        free(args->es_mappings);
+    }
+    if (args->es_settings_path) {
+        free(args->es_settings);
+    }
    free(args);
 }

@@ -135,6 +142,10 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {

    if (args->name == NULL) {
        args->name = g_path_get_basename(args->output);
+    } else {
+        char* tmp = malloc(strlen(args->name) + 1);
+        strcpy(tmp, args->name);
+        args->name = tmp;
    }

    if (args->rewrite_url == NULL) {
@@ -226,25 +237,25 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
    return 0;
 }

-int load_script(const char *script_path, char **dst) {
+int load_external_file(const char *file_path, char **dst) {
    struct stat info;
-    int res = stat(script_path, &info);
+    int res = stat(file_path, &info);

    if (res == -1) {
-        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
        return 1;
    }

-    int fd = open(script_path, O_RDONLY);
+    int fd = open(file_path, O_RDONLY);
    if (fd == -1) {
-        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
        return 1;
    }

    *dst = malloc(info.st_size + 1);
    res = read(fd, *dst, info.st_size);
    if (res < 0) {
-        fprintf(stderr, "Error reading script file '%s': %s\n", script_path, strerror(errno));
+        LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno))
        return 1;
    }

@@ -283,8 +294,24 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
        args->es_url = DEFAULT_ES_URL;
    }

+    if (args->es_index == NULL) {
+        args->es_index = DEFAULT_ES_INDEX;
+    }
+
    if (args->script_path != NULL) {
-        if (load_script(args->script_path, &args->script) != 0) {
+        if (load_external_file(args->script_path, &args->script) != 0) {
+            return 1;
+        }
+    }
+
+    if (args->es_settings_path != NULL) {
+        if (load_external_file(args->es_settings_path, &args->es_settings) != 0) {
+            return 1;
+        }
+    }
+
+    if (args->es_mappings_path != NULL) {
+        if (load_external_file(args->es_mappings_path, &args->es_mappings) != 0) {
            return 1;
        }
    }
@@ -294,10 +321,16 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
    }

    LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
+    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
    LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path)
    LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
+    LOG_DEBUGF("cli.c", "arg async_script=%s", args->async_script)
    LOG_DEBUGF("cli.c", "arg script=%s", args->script)
    LOG_DEBUGF("cli.c", "arg print=%d", args->print)
+    LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path)
+    LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings)
+    LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path)
+    LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings)
    LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size)
    LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset)

@@ -321,6 +354,10 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
        args->listen_address = DEFAULT_LISTEN_ADDRESS;
    }

+    if (args->es_index == NULL) {
+        args->es_index = DEFAULT_ES_INDEX;
+    }
+
    if (args->credentials != NULL) {
        char *ptr = strstr(args->credentials, ":");
        if (ptr == NULL) {
@@ -378,6 +415,7 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
    }

    LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
+    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
    LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address)
    LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials)
    LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials)
@@ -421,11 +459,15 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
        args->es_url = DEFAULT_ES_URL;
    }

+    if (args->es_index == NULL) {
+        args->es_index = DEFAULT_ES_INDEX;
+    }
+
    if (args->script_path == NULL) {
        LOG_FATAL("cli.c", "--script-file argument is required");
    }

-    if (load_script(args->script_path, &args->script) != 0) {
+    if (load_external_file(args->script_path, &args->script) != 0) {
        return 1;
    }

--- a/src/cli.h
+++ b/src/cli.h
@@ -35,17 +35,24 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv);

 typedef struct index_args {
    char *es_url;
+    char *es_index;
    const char *index_path;
    const char *script_path;
    char *script;
+    const char *es_settings_path;
+    char *es_settings;
+    const char *es_mappings_path;
+    char *es_mappings;
    int print;
    int batch_size;
+    int async_script;
    int force_reset;
    int threads;
 } index_args_t;

 typedef struct web_args {
    char *es_url;
+    char *es_index;
    char *listen_address;
    char *credentials;
    char *tag_credentials;
@@ -59,8 +66,10 @@ typedef struct web_args {

 typedef struct exec_args {
    char *es_url;
+    char *es_index;
    const char *index_path;
    const char *script_path;
+    int async_script;
    char *script;
 } exec_args_t;

--- a/src/ctx.h
+++ b/src/ctx.h
@@ -58,14 +58,18 @@ typedef struct {

 typedef struct {
    char *es_url;
+    char *es_index;
    int batch_size;
    tpool_t *pool;
    store_t *tag_store;
    GHashTable *tags;
+    store_t *meta_store;
+    GHashTable *meta;
 } IndexCtx_t;

 typedef struct {
    char *es_url;
+    char *es_index;
    int index_count;
    char *auth_user;
    char *auth_pass;
--- a/src/index/elastic.c
+++ b/src/index/elastic.c
@@ -9,6 +9,7 @@
 typedef struct es_indexer {
    int queued;
    char *es_url;
+    char *es_index;
    es_bulk_line_t *line_head;
    es_bulk_line_t *line_tail;
 } es_indexer_t;
@@ -17,11 +18,13 @@ typedef struct es_indexer {
 static __thread es_indexer_t *Indexer;

 void delete_queue(int max);
+
 void elastic_flush();

 void elastic_cleanup() {
    elastic_flush();
    if (Indexer != NULL) {
+        free(Indexer->es_index);
        free(Indexer->es_url);
        free(Indexer);
    }
@@ -32,7 +35,7 @@ void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
    cJSON *line = cJSON_CreateObject();

    cJSON_AddStringToObject(line, "_id", uuid_str);
-    cJSON_AddStringToObject(line, "_index", "sist2");
+    cJSON_AddStringToObject(line, "_index", IndexCtx.es_index);
    cJSON_AddStringToObject(line, "_type", "_doc");
    cJSON_AddItemReferenceToObject(line, "_source", document);

@@ -64,10 +67,10 @@ void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
    tpool_add_work(IndexCtx.pool, index_json_func, bulk_line);
 }

-void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]) {
+void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]) {

    if (Indexer == NULL) {
-        Indexer = create_indexer(IndexCtx.es_url);
+        Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
    }

    cJSON *body = cJSON_CreateObject();
@@ -82,9 +85,16 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
    char *str = cJSON_Print(body);

    char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_update_by_query?wait_for_completion=false", Indexer->es_url);
+    if (async) {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query?wait_for_completion=false", Indexer->es_url,
+                 Indexer->es_index);
+    } else {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query", Indexer->es_url, Indexer->es_index);
+    }
    response_t *r = web_post(bulk_url, str);
-    LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
+    if (!async) {
+        LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
+    }
    cJSON *resp = cJSON_Parse(r->body);

    cJSON_free(str);
@@ -99,36 +109,43 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
        cJSON_free(error_str);
    }

+    if (async) {
+        cJSON *task = cJSON_GetObjectItem(resp, "task");
+        LOG_INFOF("elastic.c", "User script queued: %s/_tasks/%s", Indexer->es_url, task->valuestring);
+    }
+
    cJSON_Delete(resp);
 }

-#define ACTION_STR_LEN 91
-
 void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
    es_bulk_line_t *line = Indexer->line_head;
    *count = 0;

    size_t buf_size = 0;
    size_t buf_cur = 0;
-    char *buf = malloc(8196);
-    size_t buf_capacity = 8196;
+    char *buf = malloc(8192);
+    size_t buf_capacity = 8192;

    while (line != NULL && *count < max) {
        char action_str[256];
-        snprintf(action_str, 256,
-                 "{\"index\":{\"_id\":\"%s\", \"_type\":\"_doc\", \"_index\":\"sist2\"}}\n", line->uuid_str);
+        snprintf(
+                action_str, 256,
+                "{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n",
+                line->uuid_str, Indexer->es_index
+        );

+        size_t action_str_len = strlen(action_str);
        size_t line_len = strlen(line->line);

-        while (buf_size + line_len + ACTION_STR_LEN > buf_capacity) {
+        while (buf_size + line_len + action_str_len > buf_capacity) {
            buf_capacity *= 2;
            buf = realloc(buf, buf_capacity);
        }

-        buf_size += line_len + ACTION_STR_LEN;
+        buf_size += line_len + action_str_len;

-        memcpy(buf + buf_cur, action_str, ACTION_STR_LEN);
-        buf_cur += ACTION_STR_LEN;
+        memcpy(buf + buf_cur, action_str, action_str_len);
+        buf_cur += action_str_len;
        memcpy(buf + buf_cur, line->line, line_len);
        buf_cur += line_len;

@@ -166,6 +183,21 @@ void print_errors(response_t *r) {
    free(tmp);
 }

+void print_error(response_t *r) {
+    char *tmp = malloc(r->size + 1);
+    memcpy(tmp, r->body, r->size);
+    *(tmp + r->size) = '\0';
+
+    cJSON *ret_json = cJSON_Parse(tmp);
+    if (cJSON_GetObjectItem(ret_json, "error") != NULL) {
+        char *str = cJSON_Print(cJSON_GetObjectItem(ret_json, "error"));
+        LOG_ERRORF("elastic.c", "%s\n", str);
+        cJSON_free(str);
+    }
+    cJSON_Delete(ret_json);
+    free(tmp);
+}
+
 void _elastic_flush(int max) {

    if (max == 0) {
@@ -178,7 +210,7 @@ void _elastic_flush(int max) {
    void *buf = create_bulk_buffer(max, &count, &buf_len);

    char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_bulk?pipeline=tie", Indexer->es_url);
+    snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_bulk?pipeline=tie", Indexer->es_url, Indexer->es_index);
    response_t *r = web_post(bulk_url, buf);

    if (r->status_code == 0) {
@@ -248,7 +280,7 @@ void delete_queue(int max) {
 void elastic_flush() {

    if (Indexer == NULL) {
-        Indexer = create_indexer(IndexCtx.es_url);
+        Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
    }

    _elastic_flush(Indexer->queued);
@@ -257,7 +289,7 @@ void elastic_flush() {
 void elastic_index_line(es_bulk_line_t *line) {

    if (Indexer == NULL) {
-        Indexer = create_indexer(IndexCtx.es_url);
+        Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
    }

    if (Indexer->line_head == NULL) {
@@ -275,14 +307,18 @@ void elastic_index_line(es_bulk_line_t *line) {
    }
 }

-es_indexer_t *create_indexer(const char *url) {
+es_indexer_t *create_indexer(const char *url, const char *index) {

    char *es_url = malloc(strlen(url) + 1);
    strcpy(es_url, url);

+    char *es_index = malloc(strlen(index) + 1);
+    strcpy(es_index, index);
+
    es_indexer_t *indexer = malloc(sizeof(es_indexer_t));

    indexer->es_url = es_url;
+    indexer->es_index = es_index;
    indexer->queued = 0;
    indexer->line_head = NULL;
    indexer->line_tail = NULL;
@@ -290,42 +326,42 @@ es_indexer_t *create_indexer(const char *url) {
    return indexer;
 }

-void finish_indexer(char *script, char *index_id) {
+void finish_indexer(char *script, int async_script, char *index_id) {

    char url[4096];

-    snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
+    snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index);
    response_t *r = web_post(url, "");
    LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
    free_response(r);

    if (script != NULL) {
-        execute_update_script(script, index_id);
+        execute_update_script(script, async_script, index_id);
        free(script);

-        snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index);
        r = web_post(url, "");
        LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
        free_response(r);
    }

-    snprintf(url, sizeof(url), "%s/sist2/_forcemerge", IndexCtx.es_url);
+    snprintf(url, sizeof(url), "%s/%s/_forcemerge", IndexCtx.es_url, IndexCtx.es_index);
    r = web_post(url, "");
    LOG_INFOF("elastic.c", "Merge index <%d>", r->status_code);
    free_response(r);

-    snprintf(url, sizeof(url), "%s/sist2/_settings", IndexCtx.es_url);
+    snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
    r = web_put(url, "{\"index\":{\"refresh_interval\":\"1s\"}}");
    LOG_INFOF("elastic.c", "Set refresh interval <%d>", r->status_code);
    free_response(r);
 }

-void elastic_init(int force_reset) {
+void elastic_init(int force_reset, const char* user_mappings, const char* user_settings) {

    // Check if index exists
    char url[4096];
-    snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
-    response_t *r = web_get(url);
+    snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index);
+    response_t *r = web_get(url, 30);
    int index_exists = r->status_code == 200;
    free_response(r);

@@ -334,32 +370,38 @@ void elastic_init(int force_reset) {
        LOG_INFOF("elastic.c", "Delete index <%d>", r->status_code);
        free_response(r);

-        snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index);
        r = web_put(url, "");
+
+        if (r->status_code != 200) {
+            print_error(r);
+            LOG_FATAL("elastic.c", "Could not create index")
+        }
+
        LOG_INFOF("elastic.c", "Create index <%d>", r->status_code);
        free_response(r);

-        snprintf(url, 4096, "%s/sist2/_close", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_close", IndexCtx.es_url, IndexCtx.es_index);
        r = web_post(url, "");
        LOG_INFOF("elastic.c", "Close index <%d>", r->status_code);
        free_response(r);

-        snprintf(url, 4096, "%s/_ingest/pipeline/tie", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/_ingest/pipeline/tie", IndexCtx.es_url);
        r = web_put(url, pipeline_json);
        LOG_INFOF("elastic.c", "Create pipeline <%d>", r->status_code);
        free_response(r);

-        snprintf(url, 4096, "%s/sist2/_settings", IndexCtx.es_url);
-        r = web_put(url, settings_json);
-        LOG_INFOF("elastic.c", "Update settings <%d>", r->status_code);
+        snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
+        r = web_put(url, user_settings ? user_settings : settings_json);
+        LOG_INFOF("elastic.c", "Update user_settings <%d>", r->status_code);
        free_response(r);

-        snprintf(url, 4096, "%s/sist2/_mappings/_doc?include_type_name=true", IndexCtx.es_url);
-        r = web_put(url, mappings_json);
-        LOG_INFOF("elastic.c", "Update mappings <%d>", r->status_code);
+        snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index);
+        r = web_put(url, user_mappings ? user_mappings : mappings_json);
+        LOG_INFOF("elastic.c", "Update user_mappings <%d>", r->status_code);
        free_response(r);

-        snprintf(url, 4096, "%s/sist2/_open", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_open", IndexCtx.es_url, IndexCtx.es_index);
        r = web_post(url, "");
        LOG_INFOF("elastic.c", "Open index <%d>", r->status_code);
        free_response(r);
@@ -368,12 +410,16 @@ void elastic_init(int force_reset) {

 cJSON *elastic_get_document(const char *uuid_str) {
    char url[4096];
-    snprintf(url, 4096, "%s/sist2/_doc/%s", WebCtx.es_url, uuid_str);
+    snprintf(url, sizeof(url), "%s/%s/_doc/%s", WebCtx.es_url, WebCtx.es_index, uuid_str);

-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 3);
    cJSON *json = NULL;
    if (r->status_code == 200) {
-        json = cJSON_Parse(r->body);
+        char *tmp = malloc(r->size + 1);
+        memcpy(tmp, r->body, r->size);
+        *(tmp + r->size) = '\0';
+        json = cJSON_Parse(tmp);
+        free(tmp);
    }
    free_response(r);
    return json;
@@ -381,21 +427,25 @@ cJSON *elastic_get_document(const char *uuid_str) {

 char *elastic_get_status() {
    char url[4096];
-    snprintf(url, 4096,
-             "%s/_cluster/state/metadata/sist2?filter_path=metadata.indices.*.state", WebCtx.es_url);
+    snprintf(url, sizeof(url),
+             "%s/_cluster/state/metadata/%s?filter_path=metadata.indices.*.state", WebCtx.es_url, WebCtx.es_index);

-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
    cJSON *json = NULL;
    char *status = malloc(128 * sizeof(char));
    status[0] = '\0';

    if (r->status_code == 200) {
-        json = cJSON_Parse(r->body);
+        char *tmp = malloc(r->size + 1);
+        memcpy(tmp, r->body, r->size);
+        *(tmp + r->size) = '\0';
+        json = cJSON_Parse(tmp);
+        free(tmp);
        const cJSON *metadata = cJSON_GetObjectItem(json, "metadata");
        if (metadata != NULL) {
            const cJSON *indices = cJSON_GetObjectItem(metadata, "indices");
-            const cJSON *sist2 = cJSON_GetObjectItem(indices, "sist2");
-            const cJSON *state = cJSON_GetObjectItem(sist2, "state");
+            const cJSON *index = cJSON_GetObjectItem(indices, WebCtx.es_index);
+            const cJSON *state = cJSON_GetObjectItem(index, "state");
            strcpy(status, state->valuestring);
        }
    }
--- a/src/index/elastic.h
+++ b/src/index/elastic.h
@@ -20,17 +20,17 @@ void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);

 void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);

-es_indexer_t *create_indexer(const char* es_url);
+es_indexer_t *create_indexer(const char *url, const char *index);

 void elastic_cleanup();
-void finish_indexer(char *script, char *index_id);
+void finish_indexer(char *script, int async_script, char *index_id);

-void elastic_init(int force_reset);
+void elastic_init(int force_reset, const char* user_mappings, const char* user_settings);

 cJSON *elastic_get_document(const char *uuid_str);

 char *elastic_get_status();

-void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]);
+void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]);

 #endif
--- a/src/index/static_generated.c
+++ b/src/index/static_generated.c
--- a/src/index/web.c
+++ b/src/index/web.c
@@ -1,5 +1,6 @@
 #include "web.h"
 #include "src/sist.h"
+#include "src/ctx.h"

 #include <mongoose.h>
 #include <pthread.h>
@@ -21,98 +22,85 @@ void free_response(response_t *resp) {
    free(resp);
 }

-#define SIST2_HEADERS "User-Agent: sist2\r\nContent-Type: application/json\r\n"
+void web_post_async_poll(subreq_ctx_t* req) {
+    fd_set fdread;
+    fd_set fdwrite;
+    fd_set fdexcep;
+    int maxfd = -1;

+    FD_ZERO(&fdread);
+    FD_ZERO(&fdwrite);
+    FD_ZERO(&fdexcep);

-void http_req_ev(struct mg_connection *nc, int ev, void *ptr) {
+    CURLMcode mc = curl_multi_fdset(req->multi, &fdread, &fdwrite, &fdexcep, &maxfd);

-    http_ev_data_t *ev_data = (http_ev_data_t *) nc->user_data;
+    if(mc != CURLM_OK) {
+        req->done = TRUE;
+        return;
+    }

-    switch (ev) {
-        case MG_EV_CONNECT: {
-            int connect_status = *(int *) ptr;
-            if (connect_status != 0) {
-                ev_data->done = TRUE;
-                ev_data->resp->status_code = 0;
-            }
+    if (maxfd == -1) {
+        // no fds ready yet
+        return;
+    }
+
+    struct timeval timeout = {1, 0};
+    int rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
+
+    switch(rc) {
+        case -1:
+            req->done = TRUE;
            break;
-        }
-        case MG_EV_HTTP_REPLY: {
-            struct http_message *hm = (struct http_message *) ptr;
-
-            //TODO: Check errors?
-
-            ev_data->resp->size = hm->body.len;
-            ev_data->resp->status_code = hm->resp_code;
-            ev_data->resp->body = malloc(hm->body.len + 1);
-            memcpy(ev_data->resp->body, hm->body.p, hm->body.len);
-            *(ev_data->resp->body + hm->body.len) = '\0';
-
-            ev_data->done = TRUE;
+        case 0:
            break;
-        }
-        case MG_EV_CLOSE: {
-            ev_data->done = TRUE;
-            break;
-        }
        default:
+            curl_multi_perform(req->multi, &req->running_handles);
            break;
    }
+
+    if (req->running_handles == 0) {
+        req->done = TRUE;
+        req->response->body = req->response_buf.buf;
+        req->response->size = req->response_buf.cur;
+        curl_easy_getinfo(req->handle, CURLINFO_RESPONSE_CODE, &req->response->status_code);
+
+        curl_multi_cleanup(req->multi);
+        curl_easy_cleanup(req->handle);
+        curl_slist_free_all(req->headers);
+        return;
+    }
 }

-subreq_ctx_t *http_req(const char *url, const char *extra_headers, const char *post_data, const char *method) {
+subreq_ctx_t *web_post_async(const char *url, char *data) {
+    subreq_ctx_t *req = calloc(1, sizeof(subreq_ctx_t));
+    req->response = calloc(1, sizeof(response_t));
+    req->data = data;
+    req->response_buf = dyn_buffer_create();

-    struct mg_str scheme;
-    struct mg_str user_info;
-    struct mg_str host;
-    unsigned int port;
-    struct mg_str path;
-    struct mg_str query;
-    struct mg_str fragment;
+    req->handle = curl_easy_init();
+    CURL *curl = req->handle;
+    curl_easy_setopt(curl, CURLOPT_URL, url);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&req->response_buf));
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
+    curl_easy_setopt(curl, CURLOPT_POST, 1);
+    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");

-    if (post_data == NULL) post_data = "";
-    if (extra_headers == NULL) extra_headers = "";
-    if (path.len == 0) path = mg_mk_str("/");
-    if (host.len == 0) host = mg_mk_str("");
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Content-Type: application/json");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

-    // [scheme://[user_info@]]host[:port][/path][?query][#fragment]
-    mg_parse_uri(mg_mk_str(url), &scheme, &user_info, &host, &port, &path, &query, &fragment);
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);

-    if (query.len > 0) path.len += query.len + 1;
+    req->multi = curl_multi_init();
+    curl_multi_add_handle(req->multi, curl);
+    curl_multi_perform(req->multi, &req->running_handles);

-    subreq_ctx_t *ctx = malloc(sizeof(subreq_ctx_t));
-    mg_mgr_init(&ctx->mgr, NULL);
+    LOG_DEBUGF("web.c", "async request POST %s", url)

-    char address[8196];
-    snprintf(address, sizeof(address), "tcp://%.*s:%u", (int) host.len, host.p, port);
-    struct mg_connection *nc = mg_connect(&ctx->mgr, address, http_req_ev);
-    nc->user_data = &ctx->ev_data;
-    mg_set_protocol_http_websocket(nc);
-
-    ctx->ev_data.resp = calloc(1, sizeof(response_t));
-    ctx->ev_data.done = FALSE;
-
-    mg_printf(
-            nc, "%s %.*s HTTP/1.1\r\n"
-                  "Host: %.*s\r\n"
-                  "Content-Length: %zu\r\n"
-                  "%s\r\n"
-                  "%s",
-            method, (int) path.len, path.p,
-            (int) (path.p - host.p), host.p,
-            strlen(post_data),
-            extra_headers,
-            post_data
-    );
-
-    return ctx;
+    return req;
 }

-subreq_ctx_t *web_post_async(const char *url, const char *data) {
-    return http_req(url, SIST2_HEADERS, data, "POST");
-}
-
-response_t *web_get(const char *url) {
+response_t *web_get(const char *url, int timeout) {
    response_t *resp = malloc(sizeof(response_t));

    CURL *curl;
@@ -123,8 +111,10 @@ response_t *web_get(const char *url) {
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&buffer));
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+    curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);

-    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Content-Type: application/json");
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

    curl_easy_perform(curl);
@@ -152,7 +142,8 @@ response_t *web_post(const char *url, const char *data) {
    curl_easy_setopt(curl, CURLOPT_POST, 1);
    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");

-    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Content-Type: application/json");
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);
@@ -186,7 +177,8 @@ response_t *web_put(const char *url, const char *data) {
    curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
    curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4 );

-    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Content-Type: application/json");
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);
@@ -217,7 +209,8 @@ response_t *web_delete(const char *url) {
    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");

    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "");
-    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Content-Type: application/json");
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

    curl_easy_perform(curl);
--- a/src/index/web.h
+++ b/src/index/web.h
@@ -3,6 +3,7 @@

 #include "src/sist.h"
 #include <mongoose.h>
+#include <curl/curl.h>

 typedef struct response {
    char *body;
@@ -16,13 +17,20 @@ typedef struct {
 } http_ev_data_t;

 typedef struct {
-    http_ev_data_t ev_data;
-    struct mg_mgr mgr;
+    char* data;
+    dyn_buffer_t response_buf;
+    struct curl_slist *headers;
+    CURL *handle;
+    CURLM *multi;
+    response_t *response;
+    int running_handles;
+    int done;
 } subreq_ctx_t;

-response_t *web_get(const char *url);
+response_t *web_get(const char *url, int timeout);
 response_t *web_post(const char * url, const char * data);
-subreq_ctx_t *web_post_async(const char *url, const char *data);
+void web_post_async_poll(subreq_ctx_t* req);
+subreq_ctx_t *web_post_async(const char *url, char *data);
 response_t *web_put(const char *url, const char *data);
 response_t *web_delete(const char *url);

--- a/src/io/serialize.c
+++ b/src/io/serialize.c
@@ -150,6 +150,8 @@ char *get_meta_key_text(enum metakey meta_key) {
            return "modified_by";
        case MetaThumbnail:
            return "thumbnail";
+        case MetaPages:
+            return "pages";
        default:
            return NULL;
    }
@@ -245,13 +247,8 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
        }
        dyn_buffer_write_char(&buf, '\0');

-        if (IndexCtx.tags != NULL) {
-            const char *tags_string = g_hash_table_lookup(IndexCtx.tags, buf.buf);
-            if (tags_string != NULL) {
-                cJSON *tags_arr = cJSON_Parse(tags_string);
-                cJSON_AddItemToObject(document, "tag", tags_arr);
-            }
-        }
+        char full_filename[PATH_MAX];
+        strcpy(full_filename, buf.buf);

        cJSON_AddStringToObject(document, "extension", buf.buf + line.ext);
        if (*(buf.buf + line.ext - 1) == '.') {
@@ -278,6 +275,7 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
        size_t ret = 0;
        while (key != '\n') {
            switch (key) {
+                case MetaPages:
                case MetaWidth:
                case MetaHeight: {
                    int value;
@@ -331,8 +329,36 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
            key = getc(file);
        }

+        cJSON *meta_obj = NULL;
+        if (IndexCtx.meta != NULL) {
+            const char *meta_string = g_hash_table_lookup(IndexCtx.meta, full_filename);
+            if (meta_string != NULL) {
+                meta_obj = cJSON_Parse(meta_string);
+
+                cJSON *child;
+                for (child = meta_obj->child; child != NULL; child = child->next) {
+                    char meta_key[4096];
+                    strcpy(meta_key, child->string);
+                    cJSON_DeleteItemFromObject(document, meta_key);
+                    cJSON_AddItemReferenceToObject(document, meta_key, child);
+                }
+            }
+        }
+
+        if (IndexCtx.tags != NULL) {
+            const char *tags_string = g_hash_table_lookup(IndexCtx.tags, full_filename);
+            if (tags_string != NULL) {
+                cJSON *tags_arr = cJSON_Parse(tags_string);
+                cJSON_DeleteItemFromObject(document, "tag");
+                cJSON_AddItemToObject(document, "tag", tags_arr);
+            }
+        }
+
        func(document, uuid_str);
        cJSON_Delete(document);
+        if (meta_obj) {
+            cJSON_Delete(meta_obj);
+        }
    }
    dyn_buffer_destroy(&buf);
    fclose(file);
--- a/src/io/store.c
+++ b/src/io/store.c
@@ -43,9 +43,13 @@ void store_destroy(store_t *store) {
 void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len) {

    if (LogCtx.very_verbose) {
-        char uuid_str[UUID_STR_LEN];
-        uuid_unparse((unsigned char *) key, uuid_str);
-        LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", uuid_str, buf_len)
+        if (key_len == 16) {
+            char uuid_str[UUID_STR_LEN] = {0, };
+            uuid_unparse((unsigned char *) key, uuid_str);
+            LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", uuid_str, buf_len)
+        } else {
+            LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", key, buf_len)
+        }
    }

    MDB_val mdb_key;
@@ -136,7 +140,9 @@ GHashTable *store_read_all(store_t *store) {
        count += 1;
    }

-    LOG_DEBUGF("store.c", "Read tags for %d documents", count);
+    const char *path;
+    mdb_env_get_path(store->env, &path);
+    LOG_DEBUGF("store.c", "Read %d entries from %s", count, path);

    mdb_cursor_close(cur);
    mdb_txn_abort(txn);
--- a/src/io/store.h
+++ b/src/io/store.h
@@ -8,6 +8,7 @@

 #define STORE_SIZE_TN 1024 * 1024 * 5
 #define STORE_SIZE_TAG 1024 * 16
+#define STORE_SIZE_META STORE_SIZE_TAG

 typedef struct store_t {
    MDB_dbi dbi;
--- a/src/log.c
+++ b/src/log.c
@@ -4,8 +4,7 @@
 #include <stdarg.h>

 const char *log_colors[] = {
-        "\033[34m", "\033[01;34m", "\033[0m",
-        "\033[01;33m", "\033[31m", "\033[01;31m"
+        "\033[34m", "\033[01;34m", "\033[01;33m", "\033[0m", "\033[31m", "\033[01;31m"
 };

 const char *log_levels[] = {
--- a/src/main.c
+++ b/src/main.c
@@ -21,7 +21,7 @@
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"


-static const char *const Version = "2.7.3";
+static const char *const Version = "2.8.5";
 static const char *const usage[] = {
        "sist2 scan [OPTION]... PATH",
        "sist2 index [OPTION]... INDEX",
@@ -182,6 +182,10 @@ void sist2_scan(scan_args_t *args) {
    mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
    ScanCtx.index.store = store_create(store_path, STORE_SIZE_TN);

+    snprintf(store_path, PATH_MAX, "%smeta", ScanCtx.index.path);
+    mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
+    ScanCtx.index.meta_store = store_create(store_path, STORE_SIZE_META);
+
    scan_print_header();

    if (args->incremental != NULL) {
@@ -259,10 +263,11 @@ void sist2_scan(scan_args_t *args) {
 void sist2_index(index_args_t *args) {

    IndexCtx.es_url = args->es_url;
+    IndexCtx.es_index = args->es_index;
    IndexCtx.batch_size = args->batch_size;

    if (!args->print) {
-        elastic_init(args->force_reset);
+        elastic_init(args->force_reset, args->es_mappings, args->es_settings);
    }

    char descriptor_path[PATH_MAX];
@@ -288,6 +293,10 @@ void sist2_index(index_args_t *args) {
    IndexCtx.tag_store = store_create(path_tmp, STORE_SIZE_TAG);
    IndexCtx.tags = store_read_all(IndexCtx.tag_store);

+    snprintf(path_tmp, sizeof(path_tmp), "%s/meta", args->index_path);
+    IndexCtx.meta_store = store_create(path_tmp, STORE_SIZE_META);
+    IndexCtx.meta = store_read_all(IndexCtx.meta_store);
+
    index_func f;
    if (args->print) {
        f = print_json;
@@ -317,11 +326,12 @@ void sist2_index(index_args_t *args) {

    tpool_wait(IndexCtx.pool);

-    if (!args->print) {
-        finish_indexer(args->script, desc.uuid);
-    }
    tpool_destroy(IndexCtx.pool);

+    if (!args->print) {
+        finish_indexer(args->script, args->async_script, desc.uuid);
+    }
+
    store_destroy(IndexCtx.tag_store);
    g_hash_table_remove_all(IndexCtx.tags);
    g_hash_table_destroy(IndexCtx.tags);
@@ -339,13 +349,14 @@ void sist2_exec_script(exec_args_t *args) {

    LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type)

-    execute_update_script(args->script, desc.uuid);
+    execute_update_script(args->script, args->async_script, desc.uuid);
    free(args->script);
 }

 void sist2_web(web_args_t *args) {

    WebCtx.es_url = args->es_url;
+    WebCtx.es_index = args->es_index;
    WebCtx.index_count = args->index_count;
    WebCtx.auth_user = args->auth_user;
    WebCtx.auth_pass = args->auth_pass;
@@ -389,7 +400,9 @@ int main(int argc, const char *argv[]) {
    int arg_version = 0;

    char *common_es_url = NULL;
+    char *common_es_index = NULL;
    char *common_script_path = NULL;
+    int common_async_script = 0;
    int common_threads = 0;

    struct argparse_option options[] = {
@@ -430,20 +443,28 @@ int main(int argc, const char *argv[]) {
            OPT_GROUP("Index options"),
            OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),
            OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"),
+            OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
            OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
            OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
+            OPT_STRING(0, "mappings-file", &index_args->es_mappings_path, "Path to Elasticsearch mappings."),
+            OPT_STRING(0, "settings-file", &index_args->es_settings_path, "Path to Elasticsearch settings."),
+            OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),
            OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 100"),
            OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. "
                                                                      "(You must use this option the first time you use the index command)"),

            OPT_GROUP("Web options"),
            OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
+            OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
            OPT_STRING(0, "bind", &web_args->listen_address, "Listen on this address. DEFAULT=localhost:4090"),
            OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"),
            OPT_STRING(0, "tag-auth", &web_args->tag_credentials, "Basic auth in user:password format for tagging"),

            OPT_GROUP("Exec-script options"),
+            OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
+            OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
            OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
+            OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),

            OPT_END(),
    };
@@ -465,10 +486,17 @@ int main(int argc, const char *argv[]) {
    web_args->es_url = common_es_url;
    index_args->es_url = common_es_url;
    exec_args->es_url = common_es_url;
+
+    web_args->es_index = common_es_index;
+    index_args->es_index = common_es_index;
+    exec_args->es_index = common_es_index;
+
    index_args->script_path = common_script_path;
    exec_args->script_path = common_script_path;
    index_args->threads = common_threads;
    scan_args->threads = common_threads;
+    exec_args->async_script = common_async_script;
+    index_args->async_script = common_async_script;

    if (argc == 0) {
        argparse_usage(&argparse);
--- a/src/parsing/mime.h
+++ b/src/parsing/mime.h
@@ -6,6 +6,7 @@
 #define MAJOR_MIME(mime_id) (mime_id & 0x000F0000) >> 16

 #define MIME_EMPTY 1
+#define MIME_SIST2_SIDECAR 2

 #define DONT_PARSE 0x80000000
 #define SHOULD_PARSE(mime_id) (ScanCtx.fast == 0 && (mime_id & DONT_PARSE) != DONT_PARSE && mime_id != 0)
--- a/src/parsing/mime_generated.c
+++ b/src/parsing/mime_generated.c
@@ -54,78 +54,79 @@ enum mime {
    application_streamingmedia=655406,
    application_vda=655407,
    application_vnd_amazon_mobi8_ebook=655408 | 0x02000000,
-    application_vnd_fdf=655409,
-    application_vnd_font_fontforge_sfd=655410,
-    application_vnd_hp_hpgl=655411,
-    application_vnd_iccprofile=655412,
-    application_vnd_lotus_1_2_3=655413,
-    application_vnd_ms_cab_compressed=655414,
-    application_vnd_ms_excel=655415,
-    application_vnd_ms_fontobject=655416,
-    application_vnd_ms_opentype=655417 | 0x20000000,
-    application_vnd_ms_outlook=655418,
-    application_vnd_ms_pki_certstore=655419,
-    application_vnd_ms_pki_pko=655420,
-    application_vnd_ms_pki_seccat=655421,
-    application_vnd_ms_powerpoint=655422,
-    application_vnd_ms_project=655423,
-    application_vnd_oasis_opendocument_base=655424,
-    application_vnd_oasis_opendocument_formula=655425,
-    application_vnd_oasis_opendocument_graphics=655426,
-    application_vnd_oasis_opendocument_presentation=655427,
-    application_vnd_oasis_opendocument_spreadsheet=655428,
-    application_vnd_oasis_opendocument_text=655429,
-    application_vnd_openxmlformats_officedocument_presentationml_presentation=655430 | 0x04000000,
-    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655431 | 0x04000000,
-    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655432 | 0x04000000,
-    application_vnd_symbian_install=655433,
-    application_vnd_tcpdump_pcap=655434,
-    application_vnd_wap_wmlc=655435,
-    application_vnd_wap_wmlscriptc=655436,
-    application_vnd_xara=655437,
-    application_vocaltec_media_desc=655438,
-    application_vocaltec_media_file=655439,
-    application_warc=655440,
-    application_winhelp=655441,
-    application_wordperfect=655442,
-    application_wordperfect6_0=655443,
-    application_wordperfect6_1=655444,
-    application_x_123=655445,
-    application_x_7z_compressed=655446 | 0x10000000,
-    application_x_aim=655447,
-    application_x_apple_diskimage=655448,
-    application_x_arc=655449 | 0x10000000,
-    application_x_archive=655450,
-    application_x_atari_7800_rom=655451,
-    application_x_authorware_bin=655452,
-    application_x_authorware_map=655453,
-    application_x_authorware_seg=655454,
-    application_x_avira_qua=655455,
-    application_x_bcpio=655456,
-    application_x_bittorrent=655457,
-    application_x_bsh=655458,
-    application_x_bytecode_python=655459,
-    application_x_bzip=655460,
-    application_x_bzip2=655461 | 0x08000000,
-    application_x_cbr=655462,
-    application_x_cbz=655463,
-    application_x_cdlink=655464,
-    application_x_chat=655465,
-    application_x_chrome_extension=655466,
-    application_x_cocoa=655467,
-    application_x_conference=655468,
-    application_x_coredump=655469,
-    application_x_cpio=655470,
-    application_x_dbf=655471,
-    application_x_dbt=655472,
-    application_x_debian_package=655473,
-    application_x_deepv=655474,
-    application_x_director=655475,
-    application_x_dmp=655476,
-    application_x_dosdriver=655477,
-    application_x_dosexec=655478,
-    application_x_dvi=655479,
-    application_x_elc=655480,
+    application_vnd_coffeescript=655409,
+    application_vnd_fdf=655410,
+    application_vnd_font_fontforge_sfd=655411,
+    application_vnd_hp_hpgl=655412,
+    application_vnd_iccprofile=655413,
+    application_vnd_lotus_1_2_3=655414,
+    application_vnd_ms_cab_compressed=655415,
+    application_vnd_ms_excel=655416,
+    application_vnd_ms_fontobject=655417,
+    application_vnd_ms_opentype=655418 | 0x20000000,
+    application_vnd_ms_outlook=655419,
+    application_vnd_ms_pki_certstore=655420,
+    application_vnd_ms_pki_pko=655421,
+    application_vnd_ms_pki_seccat=655422,
+    application_vnd_ms_powerpoint=655423,
+    application_vnd_ms_project=655424,
+    application_vnd_oasis_opendocument_base=655425,
+    application_vnd_oasis_opendocument_formula=655426,
+    application_vnd_oasis_opendocument_graphics=655427,
+    application_vnd_oasis_opendocument_presentation=655428,
+    application_vnd_oasis_opendocument_spreadsheet=655429,
+    application_vnd_oasis_opendocument_text=655430,
+    application_vnd_openxmlformats_officedocument_presentationml_presentation=655431 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655432 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655433 | 0x04000000,
+    application_vnd_symbian_install=655434,
+    application_vnd_tcpdump_pcap=655435,
+    application_vnd_wap_wmlc=655436,
+    application_vnd_wap_wmlscriptc=655437,
+    application_vnd_xara=655438,
+    application_vocaltec_media_desc=655439,
+    application_vocaltec_media_file=655440,
+    application_warc=655441,
+    application_winhelp=655442,
+    application_wordperfect=655443,
+    application_wordperfect6_0=655444,
+    application_wordperfect6_1=655445,
+    application_x_123=655446,
+    application_x_7z_compressed=655447 | 0x10000000,
+    application_x_aim=655448,
+    application_x_apple_diskimage=655449,
+    application_x_arc=655450 | 0x10000000,
+    application_x_archive=655451,
+    application_x_atari_7800_rom=655452,
+    application_x_authorware_bin=655453,
+    application_x_authorware_map=655454,
+    application_x_authorware_seg=655455,
+    application_x_avira_qua=655456,
+    application_x_bcpio=655457,
+    application_x_bittorrent=655458,
+    application_x_bsh=655459,
+    application_x_bytecode_python=655460,
+    application_x_bzip=655461,
+    application_x_bzip2=655462 | 0x08000000,
+    application_x_cbr=655463,
+    application_x_cbz=655464,
+    application_x_cdlink=655465,
+    application_x_chat=655466,
+    application_x_chrome_extension=655467,
+    application_x_cocoa=655468,
+    application_x_conference=655469,
+    application_x_coredump=655470,
+    application_x_cpio=655471,
+    application_x_dbf=655472,
+    application_x_dbt=655473,
+    application_x_debian_package=655474,
+    application_x_deepv=655475,
+    application_x_director=655476,
+    application_x_dmp=655477,
+    application_x_dosdriver=655478,
+    application_x_dosexec=655479,
+    application_x_dvi=655480,
+    application_x_elc=655481,
    application_x_empty=1,
    application_x_envoy=655482,
    application_x_esrehber=655483,
@@ -366,6 +367,7 @@ enum mime {
    model_vnd_gs_gdl=65894,
    model_vrml=65895,
    model_x_pov=65896,
+    sist2_sidecar=2,
    text_PGP=590185,
    text_asp=590186,
    text_css=590187,
@@ -804,6 +806,7 @@ case text_mcf: return "text/mcf";
 case text_pascal: return "text/pascal";
 case text_PGP: return "text/PGP";
 case text_plain: return "text/plain";
+case application_vnd_coffeescript: return "application/vnd.coffeescript";
 case text_richtext: return "text/richtext";
 case text_rtf: return "text/rtf";
 case text_scriplet: return "text/scriplet";
@@ -904,6 +907,7 @@ case image_x_sony_arw: return "image/x-sony-arw";
 case image_x_sony_sr2: return "image/x-sony-sr2";
 case image_x_sony_srf: return "image/x-sony-srf";
 case image_x_epson_erf: return "image/x-epson-erf";
+case sist2_sidecar: return "sist2/sidecar";
 default: return NULL;}}
 GHashTable *mime_get_ext_table() {GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(ext_table, "arj", (gpointer)application_arj);
@@ -1315,6 +1319,11 @@ g_hash_table_insert(ext_table, "sfv", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "m3u", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "csv", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "eml", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "make", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "log", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "markdown", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "yaml", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "coffee", (gpointer)application_vnd_coffeescript);
 g_hash_table_insert(ext_table, "rt", (gpointer)text_richtext);
 g_hash_table_insert(ext_table, "rtf", (gpointer)text_richtext);
 g_hash_table_insert(ext_table, "rtx", (gpointer)text_richtext);
@@ -1442,6 +1451,7 @@ g_hash_table_insert(ext_table, "arw", (gpointer)image_x_sony_arw);
 g_hash_table_insert(ext_table, "sr2", (gpointer)image_x_sony_sr2);
 g_hash_table_insert(ext_table, "srf", (gpointer)image_x_sony_srf);
 g_hash_table_insert(ext_table, "erf", (gpointer)image_x_epson_erf);
+g_hash_table_insert(ext_table, "s2meta", (gpointer)sist2_sidecar);
 return ext_table;}
 GHashTable *mime_get_mime_table() {GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(mime_table, "application/arj", (gpointer)application_arj);
@@ -1792,6 +1802,7 @@ g_hash_table_insert(mime_table, "text/mcf", (gpointer)text_mcf);
 g_hash_table_insert(mime_table, "text/pascal", (gpointer)text_pascal);
 g_hash_table_insert(mime_table, "text/PGP", (gpointer)text_PGP);
 g_hash_table_insert(mime_table, "text/plain", (gpointer)text_plain);
+g_hash_table_insert(mime_table, "application/vnd.coffeescript", (gpointer)application_vnd_coffeescript);
 g_hash_table_insert(mime_table, "text/richtext", (gpointer)text_richtext);
 g_hash_table_insert(mime_table, "text/rtf", (gpointer)text_rtf);
 g_hash_table_insert(mime_table, "text/scriplet", (gpointer)text_scriplet);
@@ -1892,5 +1903,6 @@ g_hash_table_insert(mime_table, "image/x-sony-arw", (gpointer)image_x_sony_arw);
 g_hash_table_insert(mime_table, "image/x-sony-sr2", (gpointer)image_x_sony_sr2);
 g_hash_table_insert(mime_table, "image/x-sony-srf", (gpointer)image_x_sony_srf);
 g_hash_table_insert(mime_table, "image/x-epson-erf", (gpointer)image_x_epson_erf);
+g_hash_table_insert(mime_table, "sist2/sidecar", (gpointer)sist2_sidecar);
 return mime_table;}
 #endif
--- a/src/parsing/parse.c
+++ b/src/parsing/parse.c
@@ -4,6 +4,7 @@
 #include "src/ctx.h"
 #include "mime.h"
 #include "src/io/serialize.h"
+#include "src/parsing/sidecar.h"

 #include <magic.h>

@@ -38,6 +39,8 @@ void fs_reset(struct vfile *f) {
    }
 }

+#define IS_GIT_OBJ (strlen(doc.filepath + doc.base) == 38 && (strstr(doc.filepath, "objects") != NULL))
+
 void parse(void *arg) {

    parse_job_t *job = arg;
@@ -77,6 +80,10 @@ void parse(void *arg) {
    int bytes_read = 0;

    if (doc.mime == 0 && !ScanCtx.fast) {
+        if (IS_GIT_OBJ) {
+            goto abort;
+        }
+
        // Get mime type with libmagic
        if (!job->vfile.is_fs_file) {
            LOG_WARNING(job->filepath, "Guessing mime type with libmagic inside archive files is not currently supported");
@@ -151,6 +158,10 @@ void parse(void *arg) {
        parse_comic(&ScanCtx.comic_ctx, &job->vfile, &doc);
    } else if (IS_MOBI(doc.mime)) {
        parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, &doc);
+    } else if (doc.mime == MIME_SIST2_SIDECAR) {
+        parse_sidecar(&job->vfile, &doc);
+        CLOSE_FILE(job->vfile)
+        return;
    }

    abort:
--- a/src/parsing/sidecar.c
+++ b/src/parsing/sidecar.c
@@ -0,0 +1,35 @@
+#include "sidecar.h"
+
+#include "src/ctx.h"
+
+void parse_sidecar(vfile_t *vfile, document_t *doc) {
+
+    LOG_DEBUGF("sidecar.c", "Parsing sidecar file %s", vfile->filepath)
+
+    size_t size;
+    char* buf = read_all(vfile, &size);
+    if (buf == NULL) {
+        LOG_ERRORF("sidecar.c", "Read error for %s", vfile->filepath)
+        return;
+    }
+
+    buf = realloc(buf, size + 1);
+    *(buf + size) = '\0';
+
+    cJSON *json = cJSON_Parse(buf);
+    if (json == NULL) {
+        LOG_ERRORF("sidecar.c", "Could not parse JSON sidecar %s", vfile->filepath)
+        return;
+    }
+    char *json_str = cJSON_PrintUnformatted(json);
+
+    char filepath[PATH_MAX];
+    memcpy(filepath, vfile->filepath + ScanCtx.index.desc.root_len, doc->ext - 1 - ScanCtx.index.desc.root_len);
+    *(filepath + doc->ext - 1) = '\0';
+
+    store_write(ScanCtx.index.meta_store, filepath, doc->ext, json_str, strlen(json_str) + 1);
+
+    cJSON_Delete(json);
+    free(json_str);
+    free(buf);
+}
--- a/src/parsing/sidecar.h
+++ b/src/parsing/sidecar.h
@@ -0,0 +1,8 @@
+#ifndef SIST2_SIDECAR_H
+#define SIST2_SIDECAR_H
+
+#include "src/sist.h"
+
+void parse_sidecar(vfile_t *vfile, document_t *doc);
+
+#endif
--- a/src/static/css/dark.css
+++ b/src/static/css/dark.css
@@ -121,7 +121,7 @@ body {
    background: #546b7a;
 }

-a:hover,.btn:hover {
+a:hover, .btn:hover {
    color: #fff;
 }

@@ -130,7 +130,11 @@ a:hover,.btn:hover {
 }

 .document {
-    padding: 0.5rem;
+    padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }

 .document p {
@@ -203,7 +207,7 @@ a:hover,.btn:hover {
    margin-top: -1px;
    font-family: monospace;
    font-size: 90%;
-    background: rgba(0,0,0,0.2);
+    background: rgba(0, 0, 0, 0.2);
    padding: 0.1em 0.4em;
    color: white;
    cursor: pointer;
@@ -214,17 +218,25 @@ a:hover,.btn:hover {
    background-color: #e0e0e0;
 }

+.card-img-top {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+}
+
 .fit {
    display: block;
    min-width: 64px;
    max-width: 100%;
-    max-height: 175px;
+    max-height: 400px;
    margin: 0 auto 0;
-    padding: 3px 3px 0;
    width: auto;
    height: auto;
 }

+.img-padding {
+    padding: 4px 4px 0 4px;
+}
+
 .fit-sm {
    display: block;
    max-width: 64px;
@@ -241,20 +253,6 @@ a:hover,.btn:hover {
    width: 100%;
 }

-@media screen and (min-width: 1500px) {
-    .container {
-        max-width: 1440px;
-    }
-
-    .bricklayer-column-sizer {
-        width: 20% !important;
-    }
-
-    .bricklayer-column {
-        max-width: 20%;
-    }
-}
-
@media screen and (min-width: 1800px) {
    .container {
        max-width: 1550px;
@@ -451,6 +449,7 @@ option {
    .small-btn {
        display: none;
    }
+
    .large-btn {
        display: inherit;
    }
@@ -460,6 +459,7 @@ option {
    .small-btn {
        display: inherit;
    }
+
    .large-btn {
        display: none;
    }
@@ -538,3 +538,7 @@ svg {
 .stat > .card-body {
    padding: 0.7em 1.25em;
 }
+
+#modal-body > .img-wrapper {
+    margin-bottom: 1em;
+}
--- a/src/static/css/light.css
+++ b/src/static/css/light.css
@@ -70,7 +70,11 @@ body {
 }

 .document {
-    padding: 0.5rem;
+   padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }

 .document p {
@@ -153,18 +157,25 @@ body {
    overflow: hidden;
 }

+.card-img-top {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+}

 .fit {
    display: block;
    min-width: 64px;
    max-width: 100%;
-    max-height: 175px;
+    max-height: 400px;
    margin: 0 auto 0;
-    padding: 3px 3px 0 3px;
    width: auto;
    height: auto;
 }

+.img-padding {
+    padding: 4px 4px 0 4px;
+}
+
 .fit-sm {
    display: block;
    max-width: 64px;
@@ -181,6 +192,10 @@ body {
    width: 100%;
 }

+.bricklayer {
+    /*max-width: 100%;*/
+}
+
@media screen and (max-width: 1200px) {
    .bricklayer-column {
        max-width: 100%;
@@ -401,4 +416,8 @@ mark {

 .stat > .card-body {
    padding: 0.7em 1.25em;
+}
+
+#modal-body > .img-wrapper {
+    margin-bottom: 1em;
 }
--- a/src/static/js/dom.js
+++ b/src/static/js/dom.js
@@ -75,6 +75,7 @@ function shouldPlayVideo(hit) {
        hit["_source"]["extension"] !== "mkv" &&
        hit["_source"]["extension"] !== "avi" &&
        videoc !== "hevc" &&
+        videoc !== "mpeg1video" &&
        videoc !== "mpeg2video" &&
        videoc !== "wmv3";
 }
@@ -194,10 +195,21 @@ function makeUserTag(tag, hit) {
 function infoButtonCb(hit) {
    return () => {
        getDocumentInfo(hit["_id"]).then(doc => {
+            $("#modal-body").empty()
+
            $("#modal-title").text(doc["name"] + ext(hit));

+            if (doc["mime"]) {
+                const mimeCategory = doc["mime"].split("/")[0];
+                const imgWrapper = document.createElement("div");
+                imgWrapper.setAttribute("style", "position: relative");
+                imgWrapper.setAttribute("class", "img-wrapper");
+                makeThumbnail(mimeCategory, hit, imgWrapper, false);
+                $("#modal-body").append(imgWrapper);
+            }
+
            const tbody = $("<tbody>");
-            $("#modal-body").empty()
+            $("#modal-body")
                .append($("<table class='table table-sm'>")
                    .append($("<thead>")
                        .append($("<tr>")
@@ -208,10 +220,19 @@ function infoButtonCb(hit) {
                    .append(tbody)
                );

+            tbody.append($("<tr>")
+                .append($("<td>").text("index"))
+                .append($("<td>").text(`[${indexMap[doc["index"]]}]`))
+            ).append($("<tr>")
+                .append($("<td>").text("mtime"))
+                .append($("<td>")
+                    .text(new Date(doc["mtime"] * 1000).toISOString().split(".")[0].replace("T", " "))
+                    .attr("title", doc["mtime"]))
+            );
            const displayFields = new Set([
-                "mime", "size", "mtime", "path", "title", "width", "height", "duration", "audioc", "videoc",
+                "mime", "size", "path", "title", "width", "height", "duration", "audioc", "videoc",
                "bitrate", "artist", "album", "album_artist", "genre", "title", "font_name", "tag", "author",
-                "modified_by"
+                "modified_by", "pages"
            ]);
            Object.keys(doc)
                .filter(key => key.startsWith("_keyword.") || key.startsWith("_text.") || displayFields.has(key) || key.startsWith("exif_"))
@@ -392,7 +413,11 @@ function makeThumbnail(mimeCategory, hit, imgWrapper, small) {
    if (small) {
        thumbnail.setAttribute("class", "fit-sm");
    } else {
-        thumbnail.setAttribute("class", "card-img-top fit");
+        if (hit["_source"].hasOwnProperty("parent")) {
+            thumbnail.setAttribute("class", "card-img-top fit img-padding");
+        } else {
+            thumbnail.setAttribute("class", "card-img-top fit");
+        }
    }
    thumbnail.setAttribute("src", `t/${hit["_source"]["index"]}/${hit["_id"]}`);

--- a/src/static/js/search.js
+++ b/src/static/js/search.js
@@ -1,4 +1,4 @@
-const SIZE = 40;
+const SIZE = 60;
 let mimeMap = [];
 let tagMap = [];
 let mimeTree;
@@ -168,7 +168,7 @@ window.onload = () => {
 };

 function saveTag(tag, hit) {
-    const relPath = hit["_source"]["path"] + "/" + hit["_source"]["name"] + ext(hit);
+    const relPath = hit["_source"]["path"] + (hit["_source"]["path"] ? "/" : "") + hit["_source"]["name"] + ext(hit);

    return $.jsonPost("/tag/" + hit["_source"]["index"], {
        delete: false,
@@ -253,7 +253,7 @@ function handleTreeClick(tree) {

        if (node.id === "any") {
            if (!node.itree.state.checked) {
-                tree.deselect();
+                tree.deselectDeep();
            }
        } else {
            tree.node("any").deselect();
--- a/src/static/js/util.js
+++ b/src/static/js/util.js
@@ -101,7 +101,8 @@ const _defaults = {
    treemapColor: "PuBuGn",
    treemapSize: "large",
    suggestPath: true,
-    fragmentSize: 100
+    fragmentSize: 100,
+    columns: 5
 };

 function loadSettings() {
@@ -118,6 +119,7 @@ function loadSettings() {
    $("#settingTreemapType").val(CONF.options.treemapType);
    $("#settingSuggestPath").prop("checked", CONF.options.suggestPath);
    $("#settingFragmentSize").val(CONF.options.fragmentSize);
+    $("#settingColumns").val(CONF.options.columns);
 }

 function Settings() {
@@ -125,6 +127,7 @@ function Settings() {

    this._onUpdate = function () {
        $("#fuzzyToggle").prop("checked", this.options.fuzzy);
+        updateColumnStyle();
    };

    this.load = function () {
@@ -161,6 +164,7 @@ function updateSettings() {
    CONF.options.treemapType = $("#settingTreemapType").val();
    CONF.options.suggestPath = $("#settingSuggestPath").prop("checked");
    CONF.options.fragmentSize = $("#settingFragmentSize").val();
+    CONF.options.columns = $("#settingColumns").val();
    CONF.save();

    if (typeof searchDebounced !== "undefined") {
@@ -203,3 +207,26 @@ function toggleTheme() {
    }
    window.location.reload();
 }
+
+function updateColumnStyle() {
+    const style = document.getElementById("style");
+    if (style) {
+        style.innerHTML =
+        `
+@media screen and (min-width: 1500px) {
+    .container {
+            max-width: 1440px;
+        }
+
+    .bricklayer-column-sizer {
+            width: ${100 / CONF.options.columns}% !important;
+        }
+
+    .bricklayer-column {
+            max-width: ${100 / CONF.options.columns}%;
+        }
+    }
+}
+        `
+    }
+}
--- a/src/static/search.html
+++ b/src/static/search.html
@@ -6,12 +6,13 @@
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>

    <link href="css" rel="stylesheet" type="text/css">
+    <style id="style"></style>
 </head>
 <body>

 <nav class="navbar navbar-expand-lg">
    <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.7.3</span>
+    <span class="badge badge-pill version">2.8.5</span>
    <span class="tagline">Lightning-fast file system indexer and search tool </span>
    <a class="btn ml-auto" href="/stats">Stats</a>
    <button class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings
@@ -216,6 +217,19 @@
                        <option value="list">List</option>
                    </select>

+                    <div class="form-group">
+                        <label for="settingColumns">Maximum column count</label>
+                        <select id="settingColumns" class="form-control form-control-sm">
+                            <option value="3">3</option>
+                            <option value="4">4</option>
+                            <option value="5">5</option>
+                            <option value="6">6</option>
+                            <option value="7">7</option>
+                            <option value="8">8</option>
+                            <option value="9">9</option>
+                        </select>
+                    </div>
+
                    <hr/>
                    <h4>Stats</h4>

--- a/src/static/stats.html
+++ b/src/static/stats.html
@@ -10,7 +10,7 @@

 <nav class="navbar navbar-expand-lg">
    <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.7.3</span>
+    <span class="badge badge-pill version">2.8.5</span>
    <span class="tagline">Lightning-fast file system indexer and search tool </span>
    <a style="margin-left: auto" class="btn" href="/">Back</a>
    <button class="btn" type="button" data-toggle="modal" data-target="#settings"
@@ -94,6 +94,19 @@
                    <option value="list">List</option>
                </select>

+                <div class="form-group">
+                    <label for="settingColumns">Maximum column count</label>
+                    <select id="settingColumns" class="form-control form-control-sm">
+                        <option value="3">3</option>
+                        <option value="4">4</option>
+                        <option value="5">5</option>
+                        <option value="6">6</option>
+                        <option value="7">7</option>
+                        <option value="8">8</option>
+                        <option value="9">9</option>
+                    </select>
+                </div>
+
                <hr/>
                <h4>Stats</h4>

--- a/src/types.h
+++ b/src/types.h
@@ -10,7 +10,7 @@ typedef struct index_descriptor {
    char version[64];
    long timestamp;
    char root[PATH_MAX];
-    char rewrite_url[8196];
+    char rewrite_url[8192];
    short root_len;
    char name[1024];
    char type[64];
@@ -20,6 +20,7 @@ typedef struct index_t {
    struct index_descriptor desc;
    struct store_t *store;
    struct store_t *tag_store;
+    struct store_t *meta_store;
    char path[PATH_MAX];
 } index_t;

--- a/src/web/serve.c
+++ b/src/web/serve.c
@@ -10,8 +10,6 @@

 #include <mongoose.h>

-#define CHUNK_SIZE 1024 * 1024 * 10
-

 static int has_prefix(const struct mg_str *str, const struct mg_str *prefix) {
    return str->len > prefix->len && memcmp(str->p, prefix->p, prefix->len) == 0;
@@ -111,7 +109,7 @@ void stats_files(struct mg_connection *nc, struct http_message *hm, struct mg_st
            return;
    }

-    char disposition[8196];
+    char disposition[8192];
    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s\"", file);

    char full_path[PATH_MAX];
@@ -237,13 +235,12 @@ void search(struct mg_connection *nc, struct http_message *hm) {
    *(body + hm->body.len) = '\0';

    char url[4096];
-    snprintf(url, 4096, "%s/sist2/_search", WebCtx.es_url);
+    snprintf(url, 4096, "%s/%s/_search", WebCtx.es_url, WebCtx.es_index);

    nc->user_data = web_post_async(url, body);
-    free(body);
 }

-int serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
+void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {

    const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
    const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
@@ -256,7 +253,7 @@ int serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {

    const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;

-    char url[8196];
+    char url[8192];
    snprintf(url, sizeof(url),
             "%s%s/%s%s%s",
             idx->desc.rewrite_url, path_unescaped, name_unescaped, strlen(ext) == 0 ? "" : ".", ext);
@@ -291,7 +288,7 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s

    LOG_DEBUGF("serve.c", "Serving file from disk: %s", full_path)

-    char disposition[8196];
+    char disposition[8192];
    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s%s%s\"",
             name, strlen(ext) == 0 ? "" : ".", ext);

@@ -538,8 +535,8 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
            }
        }

-        char buf[8196];
-        snprintf(buf, sizeof(buf),
+        char *buf = malloc(sizeof(char) * 8192);
+        snprintf(buf, 8192,
                 "{"
                 "    \"script\" : {"
                 "        \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\","
@@ -552,14 +549,14 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
        );

        char url[4096];
-        snprintf(url, sizeof(url), "%s/sist2/_update/%s", WebCtx.es_url, arg_req->doc_id);
+        snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
        nc->user_data = web_post_async(url, buf);

    } else {
        cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name));

-        char buf[8196];
-        snprintf(buf, sizeof(buf),
+        char *buf = malloc(sizeof(char) * 8192);
+        snprintf(buf, 8192,
                 "{"
                 "    \"script\" : {"
                 "        \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\","
@@ -572,7 +569,7 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
        );

        char url[4096];
-        snprintf(url, sizeof(url), "%s/sist2/_update/%s", WebCtx.es_url, arg_req->doc_id);
+        snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
        nc->user_data = web_post_async(url, buf);
    }

@@ -668,11 +665,11 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
        if (nc->user_data != NULL) {
            //Waiting for ES reply
            subreq_ctx_t *ctx = (subreq_ctx_t *) nc->user_data;
-            mg_mgr_poll(&ctx->mgr, 0);
+            web_post_async_poll(ctx);

-            if (ctx->ev_data.done == TRUE) {
+            if (ctx->done == TRUE) {

-                response_t *r = ctx->ev_data.resp;
+                response_t *r = ctx->response;

                if (r->status_code == 200) {
                    send_response_line(nc, 200, r->size, "Content-Type: application/json");
@@ -695,6 +692,8 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
                }

                free_response(r);
+                free(ctx->data);
+                free(ctx);
                nc->flags |= MG_F_SEND_AND_CLOSE;
                nc->user_data = NULL;
            }
--- a/src/web/static_generated.c
+++ b/src/web/static_generated.c
--- a/third-party/libscan
+++ b/third-party/libscan
Author	SHA1	Message	Date
simon987	95bbe39afc	Update libmupdf	2020-10-25 09:44:30 -04:00
simon987	72ce217f9c	Optionally ES schema from file #117	2020-10-25 09:44:30 -04:00
simon987	641a8ec90c	sidecar files #114 , version bump	2020-10-25 09:44:30 -04:00
simon987	7a505c2287	Fix typo	2020-10-25 09:44:30 -04:00
simon987	12f162d760	Fix #110	2020-10-25 09:44:30 -04:00
simon987	4b4ab12fac	Version bump	2020-09-22 21:08:24 -04:00
simon987	ae283f77ad	Fix #112	2020-09-22 21:08:24 -04:00
simon987	d3bd53a5ea	Fix arm Dockerfile	2020-09-13 16:16:26 -04:00
simon987	f7887f24d1	sync libscan	2020-09-13 16:16:26 -04:00
simon987	5c8de19188	Update build instructions	2020-09-13 16:16:26 -04:00
simon987	d861d278a4	version bump	2020-09-13 16:16:26 -04:00
simon987	b6ddeee0e0	Use async curl for ES requests #108	2020-09-13 16:16:26 -04:00
simon987	0cd2523b05	arm64 build	2020-09-13 16:16:26 -04:00
simon987	5e798f9367	Update issue-template.md	2020-09-13 10:29:42 -04:00
simon987	5da6c1488b	Handle null mime in document info dialog	2020-08-29 10:34:58 -04:00
simon987	9568e25f84	Fix #99	2020-08-29 10:17:28 -04:00
simon987	6a8027789a	Limited support for UTF16	2020-08-29 10:17:28 -04:00
simon987	b1d16d8abf	Fix #100	2020-08-29 10:17:28 -04:00
simon987	b2a157e24d	Update docs	2020-08-25 10:38:38 -04:00
simon987	9aead9389a	Fix typo in elastic.c	2020-08-25 10:38:38 -04:00
simon987	a32c68cba8	Build fixes	2020-08-25 10:38:38 -04:00
simon987	d116cf9d91	Default index for web & exec	2020-08-25 10:38:38 -04:00
Andrew	a020a8b32c	Update USAGE.md Fix link to scripting.	2020-08-25 10:38:38 -04:00
simon987	5d5d9c3092	Fix heap buffer overflow warning	2020-08-25 10:38:38 -04:00
simon987	3379d5ce71	Fix #97	2020-08-25 10:38:38 -04:00
simon987	a0ff4a1f01	Fix heap buffer overflow warning	2020-08-25 10:38:38 -04:00
simon987	4589f3bde7	Fix #94	2020-08-25 10:38:38 -04:00
simon987	1c898640cf	Fix #88	2020-08-25 10:38:38 -04:00
simon987	a0739d5177	Fix #92	2020-08-25 10:38:38 -04:00
simon987	8f9d29dbc6	Fix #91	2020-08-25 10:38:38 -04:00
simon987	3ff4b70223	Update README.md	2020-08-25 10:38:38 -04:00
simon987	02ad035b09	Workaround when first ebook page is blank	2020-08-25 10:38:38 -04:00
simon987	c11feb213d	Gracefully handle archive errors in comic.c	2020-08-25 10:38:38 -04:00
simon987	72902947cd	Fix for #90	2020-08-25 10:38:38 -04:00
simon987	a18bb81222	remove warning	2020-08-25 10:38:38 -04:00
simon987	1520288f19	Fix #89	2020-08-25 10:38:38 -04:00
simon987	e507de194b	Fix log colors	2020-08-25 10:38:38 -04:00
simon987	0e517d5e2b	Fix #81	2020-08-03 20:09:07 -04:00
simon987	8223ef3860	Update USAGE.md	2020-08-03 19:48:49 -04:00
simon987	995a196690	Log user script task, add async arg	2020-08-03 19:44:43 -04:00
simon987	465d017e18	CSS tweaks, fix #87	2020-08-03 19:15:12 -04:00
simon987	ca994d3914	Fix bug with media files, don't encode thumbnail when not necessary	2020-07-26 11:52:48 -04:00
simon987	db2285973f	Configurable column count	2020-07-26 11:50:21 -04:00
simon987	61de9e9f14	Set timeout for HTTP get request	2020-07-25 19:55:27 -04:00
simon987	3015ef0ff4	Increase file preview file	2020-07-25 17:26:17 -04:00
simon987	b55d432841	Fix #65	2020-07-25 09:37:37 -04:00
simon987	ed90a140ce	Update README.md	2020-07-19 14:53:03 -04:00