Update libmupdf

Fix link to scripting.

.github/ISSUE_TEMPLATE/issue-template.md

@@ -9,7 +9,7 @@ assignees: ''
 
 sist2 version:
 
-Platform (Linux or Docker):
+Platform (Linux or Docker, x86-64 or arm64):
 
 Elasticsearch version:
 

CMakeLists.txt

@@ -9,8 +9,11 @@ add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
-add_executable(
+add_executable(sist2
-        sist2
+
+        # argparse
+        third-party/argparse/argparse.h third-party/argparse/argparse.c
+
         src/main.c
         src/sist.h
         src/io/walk.h src/io/walk.c
@@ -25,12 +28,9 @@ add_executable(
         src/util.c src/util.h
         src/ctx.h src/types.h
         src/log.c src/log.h
-
-        # argparse
-        third-party/argparse/argparse.h third-party/argparse/argparse.c
-
         src/cli.c src/cli.h
-        src/stats.c src/stats.h src/ctx.c)
+        src/stats.c src/stats.h src/ctx.c
+        src/parsing/sidecar.c src/parsing/sidecar.h)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
 set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
@@ -73,7 +73,6 @@ if (SIST_DEBUG)
             sist2
             PRIVATE
             -fsanitize=address
-#            -static
     )
     set_target_properties(
             sist2
@@ -81,7 +80,7 @@ if (SIST_DEBUG)
             OUTPUT_NAME sist2_debug
     )
 else ()
-#    set(VCPKG_BUILD_TYPE release)
+    #    set(VCPKG_BUILD_TYPE release)
     target_compile_options(
             sist2
             PRIVATE
@@ -106,7 +105,6 @@ target_link_libraries(
         argparse
         unofficial::glib::glib
         unofficial::mongoose::mongoose
-#        OpenSSL::SSL OpenSSL::Crypto
         CURL::libcurl
 
         ${UUID_LIB}

DockerArm64/Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:19.10
+MAINTAINER simon987 <me@simon987.net>
+
+RUN apt update
+RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 \
+ curl libtiff5 libpng16-16 libpcre3
+
+RUN mkdir -p /usr/share/tessdata && \
+    cd /usr/share/tessdata/ && \
+    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
+
+ADD sist2_arm64 /root/sist2
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["/root/sist2"]

DockerArm64/build.sh

@@ -0,0 +1,13 @@
+rm ./sist2_arm64
+cp ../sist2_arm64.gz .
+gzip -d sist2_arm64.gz
+
+version=$(./sist2_arm64 --version)
+
+echo "Version ${version}"
+docker build . -t simon987/sist2-arm64:"${version}" -t simon987/sist2-arm64:latest
+
+docker push simon987/sist2-arm64:"${version}"
+docker push simon987/sist2-arm64:latest
+
+docker run --rm simon987/sist2-arm64 -v

README.md

@@ -2,6 +2,8 @@
 [![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
 [![Development snapshots](https://ci.simon987.net/app/rest/builds/buildType(Sist2_Build)/statusIcon)](https://files.simon987.net/artifacts/Sist2/Build/)
 
+**Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
+
 # sist2
 
 sist2 (Simple incremental search tool)
@@ -102,7 +104,7 @@ scan is also supported.
  
 ### OCR
 
-You can enable OCR support for pdf,xps,cbz,cbr,fb2,epub file types with the
+You can enable OCR support for pdf,xps,fb2,epub file types with the
 `--ocr <lang>` option. Download the language data files with your
 package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
@@ -124,7 +126,7 @@ binaries (GCC 7+ required).
 1. Install compile-time dependencies
 
    ```bash
-   vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libuuid libmagic libraw
+   vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libuuid libmagic libraw curl[core,ssl] jbig2dec brotli libmupdf
    ```
 
 2. Build

ci/build_arm64.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+VCPKG_ROOT="/vcpkg"
+
+rm *.gz
+
+rm -rf CMakeFiles CMakeCache.txt
+cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+make -j 4
+strip sist2
+mv sist2 sist2_arm64
+gzip -9 sist2_arm64

docs/USAGE.md

@@ -16,6 +16,7 @@
     * [link to specific indices](#link-to-specific-indices)
 * [exec-script](#exec-script)
 * [tagging](#tagging)
+* [sidecar files](#sidecar-files)
 
 ```
 Usage: sist2 scan [OPTION]... PATH
@@ -49,19 +50,27 @@ Scan options
 Index options
     -t, --threads=<int>           Number of threads. DEFAULT=1
     --es-url=<str>                Elasticsearch url with port. DEFAULT=http://localhost:9200
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
     -p, --print                   Just print JSON documents to stdout.
     --script-file=<str>           Path to user script.
+    --mappings-file=<str>         Path to Elasticsearch mappings.
+    --settings-file=<str>         Path to Elasticsearch settings.
+    --async-script                Execute user script asynchronously.
     --batch-size=<int>            Index batch size. DEFAULT: 100
     -f, --force-reset             Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
 
 Web options
     --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
     --bind=<str>                  Listen on this address. DEFAULT=localhost:4090
     --auth=<str>                  Basic auth in user:password format
     --tag-auth=<str>              Basic auth in user:password format for tagging
 
 Exec-script options
+    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
     --script-file=<str>           Path to user script.
+    --async-script                Execute user script asynchronously.
 Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
@@ -147,10 +156,13 @@ documents.idx/
 ├── agg_mime.csv
 ├── agg_date.csv
 ├── add_size.csv
-├── thumbs
+├── thumbs/
 |   ├── data.mdb
 |   └── lock.mdb
-└── tags
+├── tags/
+|   ├── data.mdb
+|   └── lock.mdb
+└── meta/
     ├── data.mdb
     └── lock.mdb
 ```
@@ -177,9 +189,11 @@ by a third party application. The 'external' index must have the following forma
 my_index/
 ├── descriptor.json
 ├── _index_0
-└── thumbs
+└── thumbs/
-    ├── data.mdb
+|   ├── data.mdb
-    └── lock.mdb
+|   └── lock.mdb
+└── meta/
+    └── <empty>
 ```
 
 *descriptor.json*:
@@ -239,17 +253,25 @@ it is currently unsupported and has no guaranties of back/forward compatibility.
  * `--es-url` 
  Elasticsearch url and port. If you are using docker, make sure that both containers are on the
  same network.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
  * `-p, --print` 
     Print index in JSON format to stdout.
  * `--script-file` 
     Path to user script. See [Scripting](scripting.md).
+ * `--mappings-file`
+    Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used.
+ * `--settings-file`
+    Path to custom Elasticsearch settings. *(See above)*
+ * `--async-script` 
+    Use `wait_for_completion=false` elasticsearch option while executing user script.
+     (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
  * `--batch-size=<int>` 
     Index batch size. Indexing is generally faster with larger batches, but payloads that
     are too large will fail and additional overhead for retrying with smaller sizes may slow
     down the process.
  * `-f, --force-reset` 
     Reset Elasticsearch mappings and settings.
-    **(You must use this option the first time you use the index command)**.
     
 ### Index examples
 
@@ -273,6 +295,8 @@ sist2 index --print ./my_index/ | jq | less
 
 ### Web options
  * `--es-url=<str>` Elasticsearch url.
+ * `--es-index` 
+    Elasticsearch index name. DEFAULT=sist2
  * `--bind=<str>` Listen on this address.
  * `--auth=<str>` Basic auth in user:password format
  * `--tag-auth=<str>` Basic auth in user:password format. Works the same way as the 
@@ -330,9 +354,48 @@ See [Automatic tagging](#automatic-tagging) for information about tag
  hierarchies and tag colors.
 
 \* *It can take a few seconds to take effect in new search queries, and the page needs 
-    to be reloaded for the tag tab to update*
+    to be reloaded for the tags tab to update*
 
 
 ### Automatic tagging
 
-See [scripting](docs/scripting.md) documentation.
+See [scripting](scripting.md) documentation.
+
+# Sidecar files
+
+When scanning, sist2 will read metadata from `.s2meta` JSON files and overwrite the 
+original document's metadata. Sidecar metadata files will also work inside archives.
+Sidecar files themselves are not saved in the index.
+
+This feature is useful to leverage third-party applications such as speech-to-text or
+OCR to add additional metadata to a file.
+
+**Example**
+
+```
+~/Documents/
+├── Video.mp4
+└── Video.mp4.s2meta
+```
+
+The sidecar file must have exactly the same file path and the `.s2meta` suffix.
+
+`Video.mp4.s2meta`:
+```json
+{
+  "content": "This sidecar file will overwrite some metadata fields of Video.mp4",
+  "author": "Some author",
+  "duration": 12345,
+  "bitrate": 67890,
+  "some_arbitrary_field": [1,2,3]
+}
+```
+
+```
+sist2 scan ~/Documents -o ./docs.idx
+sist2 index ./docs.idx
+```
+
+*NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,
+it is not currently possible to restore both manual tags and sidecar tags without user scripts
+while reindexing.

schema/mappings.json

@@ -54,6 +54,10 @@
       "type": "integer",
       "index": false
     },
+    "pages": {
+      "type": "integer",
+      "index": false
+    },
     "mtime": {
       "type": "integer"
     },

scripts/mime.csv

@@ -347,7 +347,8 @@ text/javascript, js
 text/mcf, mcf
 text/pascal, pas
 text/PGP,
-text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml
+text/plain, com|cmd|conf|def|g|idc|list|lst|mar|sdml|text|txt|md|groovy|license|properties|desktop|ini|rst|cmake|ipynb|readme|less|lo|go|yml|d|cs|hpp|srt|nfo|sfv|m3u|csv|eml|make|log|markdown|yaml
+application/vnd.coffeescript, coffee
 text/richtext, rt|rtf|rtx
 text/rtf,
 text/scriplet, wsc
@@ -448,3 +449,4 @@ image/x-sony-arw, arw
 image/x-sony-sr2, sr2
 image/x-sony-srf, srf
 image/x-epson-erf, erf
+sist2/sidecar, s2meta

scripts/mime.py

@@ -3,6 +3,7 @@ noparse = set()
 ext_in_hash = set()
 
 major_mime = {
+    "sist2": 0,
     "model": 1,
     "example": 2,
     "message": 3,
@@ -122,7 +123,11 @@ def mime_id(mime):
     elif mime in raw:
         mime_id += " | 0x00800000"
     elif mime == "application/x-empty":
+        cnt -= 1
         return "1"
+    elif mime == "sist2/sidecar":
+        cnt -= 1
+        return "2"
     return mime_id
 
 

src/cli.c

@@ -9,6 +9,7 @@
 #define DEFAULT_REWRITE_URL ""
 
 #define DEFAULT_ES_URL "http://localhost:9200"
+#define DEFAULT_ES_INDEX "sist2"
 #define DEFAULT_BATCH_SIZE 100
 
 #define DEFAULT_LISTEN_ADDRESS "localhost:4090"
@@ -55,6 +56,12 @@ void scan_args_destroy(scan_args_t *args) {
 
 void index_args_destroy(index_args_t *args) {
     //todo
+    if (args->es_mappings_path) {
+        free(args->es_mappings);
+    }
+    if (args->es_settings_path) {
+        free(args->es_settings);
+    }
     free(args);
 }
 
@@ -135,6 +142,10 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     if (args->name == NULL) {
         args->name = g_path_get_basename(args->output);
+    } else {
+        char* tmp = malloc(strlen(args->name) + 1);
+        strcpy(tmp, args->name);
+        args->name = tmp;
     }
 
     if (args->rewrite_url == NULL) {
@@ -226,25 +237,25 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     return 0;
 }
 
-int load_script(const char *script_path, char **dst) {
+int load_external_file(const char *file_path, char **dst) {
     struct stat info;
-    int res = stat(script_path, &info);
+    int res = stat(file_path, &info);
 
     if (res == -1) {
-        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
         return 1;
     }
 
-    int fd = open(script_path, O_RDONLY);
+    int fd = open(file_path, O_RDONLY);
     if (fd == -1) {
-        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
         return 1;
     }
 
     *dst = malloc(info.st_size + 1);
     res = read(fd, *dst, info.st_size);
     if (res < 0) {
-        fprintf(stderr, "Error reading script file '%s': %s\n", script_path, strerror(errno));
+        LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno))
         return 1;
     }
 
@@ -283,8 +294,24 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
         args->es_url = DEFAULT_ES_URL;
     }
 
+    if (args->es_index == NULL) {
+        args->es_index = DEFAULT_ES_INDEX;
+    }
+
     if (args->script_path != NULL) {
-        if (load_script(args->script_path, &args->script) != 0) {
+        if (load_external_file(args->script_path, &args->script) != 0) {
+            return 1;
+        }
+    }
+
+    if (args->es_settings_path != NULL) {
+        if (load_external_file(args->es_settings_path, &args->es_settings) != 0) {
+            return 1;
+        }
+    }
+
+    if (args->es_mappings_path != NULL) {
+        if (load_external_file(args->es_mappings_path, &args->es_mappings) != 0) {
             return 1;
         }
     }
@@ -294,10 +321,16 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
     }
 
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
+    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
     LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path)
     LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
+    LOG_DEBUGF("cli.c", "arg async_script=%s", args->async_script)
     LOG_DEBUGF("cli.c", "arg script=%s", args->script)
     LOG_DEBUGF("cli.c", "arg print=%d", args->print)
+    LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path)
+    LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings)
+    LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path)
+    LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings)
     LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size)
     LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset)
 
@@ -321,6 +354,10 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
         args->listen_address = DEFAULT_LISTEN_ADDRESS;
     }
 
+    if (args->es_index == NULL) {
+        args->es_index = DEFAULT_ES_INDEX;
+    }
+
     if (args->credentials != NULL) {
         char *ptr = strstr(args->credentials, ":");
         if (ptr == NULL) {
@@ -378,6 +415,7 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
     }
 
     LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
+    LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index)
     LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address)
     LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials)
     LOG_DEBUGF("cli.c", "arg tag_credentials=%s", args->tag_credentials)
@@ -421,11 +459,15 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
         args->es_url = DEFAULT_ES_URL;
     }
 
+    if (args->es_index == NULL) {
+        args->es_index = DEFAULT_ES_INDEX;
+    }
+
     if (args->script_path == NULL) {
         LOG_FATAL("cli.c", "--script-file argument is required");
     }
 
-    if (load_script(args->script_path, &args->script) != 0) {
+    if (load_external_file(args->script_path, &args->script) != 0) {
         return 1;
     }
 

src/cli.h

@@ -35,17 +35,24 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv);
 
 typedef struct index_args {
     char *es_url;
+    char *es_index;
     const char *index_path;
     const char *script_path;
     char *script;
+    const char *es_settings_path;
+    char *es_settings;
+    const char *es_mappings_path;
+    char *es_mappings;
     int print;
     int batch_size;
+    int async_script;
     int force_reset;
     int threads;
 } index_args_t;
 
 typedef struct web_args {
     char *es_url;
+    char *es_index;
     char *listen_address;
     char *credentials;
     char *tag_credentials;
@@ -59,8 +66,10 @@ typedef struct web_args {
 
 typedef struct exec_args {
     char *es_url;
+    char *es_index;
     const char *index_path;
     const char *script_path;
+    int async_script;
     char *script;
 } exec_args_t;
 

src/ctx.h

@@ -58,14 +58,18 @@ typedef struct {
 
 typedef struct {
     char *es_url;
+    char *es_index;
     int batch_size;
     tpool_t *pool;
     store_t *tag_store;
     GHashTable *tags;
+    store_t *meta_store;
+    GHashTable *meta;
 } IndexCtx_t;
 
 typedef struct {
     char *es_url;
+    char *es_index;
     int index_count;
     char *auth_user;
     char *auth_pass;

src/index/elastic.c

@@ -9,6 +9,7 @@
 typedef struct es_indexer {
     int queued;
     char *es_url;
+    char *es_index;
     es_bulk_line_t *line_head;
     es_bulk_line_t *line_tail;
 } es_indexer_t;
@@ -17,11 +18,13 @@ typedef struct es_indexer {
 static __thread es_indexer_t *Indexer;
 
 void delete_queue(int max);
+
 void elastic_flush();
 
 void elastic_cleanup() {
     elastic_flush();
     if (Indexer != NULL) {
+        free(Indexer->es_index);
         free(Indexer->es_url);
         free(Indexer);
     }
@@ -32,7 +35,7 @@ void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
     cJSON *line = cJSON_CreateObject();
 
     cJSON_AddStringToObject(line, "_id", uuid_str);
-    cJSON_AddStringToObject(line, "_index", "sist2");
+    cJSON_AddStringToObject(line, "_index", IndexCtx.es_index);
     cJSON_AddStringToObject(line, "_type", "_doc");
     cJSON_AddItemReferenceToObject(line, "_source", document);
 
@@ -64,10 +67,10 @@ void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
     tpool_add_work(IndexCtx.pool, index_json_func, bulk_line);
 }
 
-void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]) {
+void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]) {
 
     if (Indexer == NULL) {
-        Indexer = create_indexer(IndexCtx.es_url);
+        Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
     }
 
     cJSON *body = cJSON_CreateObject();
@@ -82,9 +85,16 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
     char *str = cJSON_Print(body);
 
     char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_update_by_query?wait_for_completion=false", Indexer->es_url);
+    if (async) {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query?wait_for_completion=false", Indexer->es_url,
+                 Indexer->es_index);
+    } else {
+        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query", Indexer->es_url, Indexer->es_index);
+    }
     response_t *r = web_post(bulk_url, str);
+    if (!async) {
         LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
+    }
     cJSON *resp = cJSON_Parse(r->body);
 
     cJSON_free(str);
@@ -99,36 +109,43 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
         cJSON_free(error_str);
     }
 
+    if (async) {
+        cJSON *task = cJSON_GetObjectItem(resp, "task");
+        LOG_INFOF("elastic.c", "User script queued: %s/_tasks/%s", Indexer->es_url, task->valuestring);
+    }
+
     cJSON_Delete(resp);
 }
 
-#define ACTION_STR_LEN 91
-
 void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
     es_bulk_line_t *line = Indexer->line_head;
     *count = 0;
 
     size_t buf_size = 0;
     size_t buf_cur = 0;
-    char *buf = malloc(8196);
+    char *buf = malloc(8192);
-    size_t buf_capacity = 8196;
+    size_t buf_capacity = 8192;
 
     while (line != NULL && *count < max) {
         char action_str[256];
-        snprintf(action_str, 256,
+        snprintf(
-                 "{\"index\":{\"_id\":\"%s\", \"_type\":\"_doc\", \"_index\":\"sist2\"}}\n", line->uuid_str);
+                action_str, 256,
+                "{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n",
+                line->uuid_str, Indexer->es_index
+        );
 
+        size_t action_str_len = strlen(action_str);
         size_t line_len = strlen(line->line);
 
-        while (buf_size + line_len + ACTION_STR_LEN > buf_capacity) {
+        while (buf_size + line_len + action_str_len > buf_capacity) {
             buf_capacity *= 2;
             buf = realloc(buf, buf_capacity);
         }
 
-        buf_size += line_len + ACTION_STR_LEN;
+        buf_size += line_len + action_str_len;
 
-        memcpy(buf + buf_cur, action_str, ACTION_STR_LEN);
+        memcpy(buf + buf_cur, action_str, action_str_len);
-        buf_cur += ACTION_STR_LEN;
+        buf_cur += action_str_len;
         memcpy(buf + buf_cur, line->line, line_len);
         buf_cur += line_len;
 
@@ -166,6 +183,21 @@ void print_errors(response_t *r) {
     free(tmp);
 }
 
+void print_error(response_t *r) {
+    char *tmp = malloc(r->size + 1);
+    memcpy(tmp, r->body, r->size);
+    *(tmp + r->size) = '\0';
+
+    cJSON *ret_json = cJSON_Parse(tmp);
+    if (cJSON_GetObjectItem(ret_json, "error") != NULL) {
+        char *str = cJSON_Print(cJSON_GetObjectItem(ret_json, "error"));
+        LOG_ERRORF("elastic.c", "%s\n", str);
+        cJSON_free(str);
+    }
+    cJSON_Delete(ret_json);
+    free(tmp);
+}
+
 void _elastic_flush(int max) {
 
     if (max == 0) {
@@ -178,7 +210,7 @@ void _elastic_flush(int max) {
     void *buf = create_bulk_buffer(max, &count, &buf_len);
 
     char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_bulk?pipeline=tie", Indexer->es_url);
+    snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_bulk?pipeline=tie", Indexer->es_url, Indexer->es_index);
     response_t *r = web_post(bulk_url, buf);
 
     if (r->status_code == 0) {
@@ -248,7 +280,7 @@ void delete_queue(int max) {
 void elastic_flush() {
 
     if (Indexer == NULL) {
-        Indexer = create_indexer(IndexCtx.es_url);
+        Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
     }
 
     _elastic_flush(Indexer->queued);
@@ -257,7 +289,7 @@ void elastic_flush() {
 void elastic_index_line(es_bulk_line_t *line) {
 
     if (Indexer == NULL) {
-        Indexer = create_indexer(IndexCtx.es_url);
+        Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
     }
 
     if (Indexer->line_head == NULL) {
@@ -275,14 +307,18 @@ void elastic_index_line(es_bulk_line_t *line) {
     }
 }
 
-es_indexer_t *create_indexer(const char *url) {
+es_indexer_t *create_indexer(const char *url, const char *index) {
 
     char *es_url = malloc(strlen(url) + 1);
     strcpy(es_url, url);
 
+    char *es_index = malloc(strlen(index) + 1);
+    strcpy(es_index, index);
+
     es_indexer_t *indexer = malloc(sizeof(es_indexer_t));
 
     indexer->es_url = es_url;
+    indexer->es_index = es_index;
     indexer->queued = 0;
     indexer->line_head = NULL;
     indexer->line_tail = NULL;
@@ -290,42 +326,42 @@ es_indexer_t *create_indexer(const char *url) {
     return indexer;
 }
 
-void finish_indexer(char *script, char *index_id) {
+void finish_indexer(char *script, int async_script, char *index_id) {
 
     char url[4096];
 
-    snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
+    snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index);
     response_t *r = web_post(url, "");
     LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
     free_response(r);
 
     if (script != NULL) {
-        execute_update_script(script, index_id);
+        execute_update_script(script, async_script, index_id);
         free(script);
 
-        snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index);
         r = web_post(url, "");
         LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
         free_response(r);
     }
 
-    snprintf(url, sizeof(url), "%s/sist2/_forcemerge", IndexCtx.es_url);
+    snprintf(url, sizeof(url), "%s/%s/_forcemerge", IndexCtx.es_url, IndexCtx.es_index);
     r = web_post(url, "");
     LOG_INFOF("elastic.c", "Merge index <%d>", r->status_code);
     free_response(r);
 
-    snprintf(url, sizeof(url), "%s/sist2/_settings", IndexCtx.es_url);
+    snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
     r = web_put(url, "{\"index\":{\"refresh_interval\":\"1s\"}}");
     LOG_INFOF("elastic.c", "Set refresh interval <%d>", r->status_code);
     free_response(r);
 }
 
-void elastic_init(int force_reset) {
+void elastic_init(int force_reset, const char* user_mappings, const char* user_settings) {
 
     // Check if index exists
     char url[4096];
-    snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
+    snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index);
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
     int index_exists = r->status_code == 200;
     free_response(r);
 
@@ -334,32 +370,38 @@ void elastic_init(int force_reset) {
         LOG_INFOF("elastic.c", "Delete index <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index);
         r = web_put(url, "");
+
+        if (r->status_code != 200) {
+            print_error(r);
+            LOG_FATAL("elastic.c", "Could not create index")
+        }
+
         LOG_INFOF("elastic.c", "Create index <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2/_close", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_close", IndexCtx.es_url, IndexCtx.es_index);
         r = web_post(url, "");
         LOG_INFOF("elastic.c", "Close index <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/_ingest/pipeline/tie", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/_ingest/pipeline/tie", IndexCtx.es_url);
         r = web_put(url, pipeline_json);
         LOG_INFOF("elastic.c", "Create pipeline <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2/_settings", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_put(url, settings_json);
+        r = web_put(url, user_settings ? user_settings : settings_json);
-        LOG_INFOF("elastic.c", "Update settings <%d>", r->status_code);
+        LOG_INFOF("elastic.c", "Update user_settings <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2/_mappings/_doc?include_type_name=true", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_put(url, mappings_json);
+        r = web_put(url, user_mappings ? user_mappings : mappings_json);
-        LOG_INFOF("elastic.c", "Update mappings <%d>", r->status_code);
+        LOG_INFOF("elastic.c", "Update user_mappings <%d>", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2/_open", IndexCtx.es_url);
+        snprintf(url, sizeof(url), "%s/%s/_open", IndexCtx.es_url, IndexCtx.es_index);
         r = web_post(url, "");
         LOG_INFOF("elastic.c", "Open index <%d>", r->status_code);
         free_response(r);
@@ -368,12 +410,16 @@ void elastic_init(int force_reset) {
 
 cJSON *elastic_get_document(const char *uuid_str) {
     char url[4096];
-    snprintf(url, 4096, "%s/sist2/_doc/%s", WebCtx.es_url, uuid_str);
+    snprintf(url, sizeof(url), "%s/%s/_doc/%s", WebCtx.es_url, WebCtx.es_index, uuid_str);
 
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 3);
     cJSON *json = NULL;
     if (r->status_code == 200) {
-        json = cJSON_Parse(r->body);
+        char *tmp = malloc(r->size + 1);
+        memcpy(tmp, r->body, r->size);
+        *(tmp + r->size) = '\0';
+        json = cJSON_Parse(tmp);
+        free(tmp);
     }
     free_response(r);
     return json;
@@ -381,21 +427,25 @@ cJSON *elastic_get_document(const char *uuid_str) {
 
 char *elastic_get_status() {
     char url[4096];
-    snprintf(url, 4096,
+    snprintf(url, sizeof(url),
-             "%s/_cluster/state/metadata/sist2?filter_path=metadata.indices.*.state", WebCtx.es_url);
+             "%s/_cluster/state/metadata/%s?filter_path=metadata.indices.*.state", WebCtx.es_url, WebCtx.es_index);
 
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
     cJSON *json = NULL;
     char *status = malloc(128 * sizeof(char));
     status[0] = '\0';
 
     if (r->status_code == 200) {
-        json = cJSON_Parse(r->body);
+        char *tmp = malloc(r->size + 1);
+        memcpy(tmp, r->body, r->size);
+        *(tmp + r->size) = '\0';
+        json = cJSON_Parse(tmp);
+        free(tmp);
         const cJSON *metadata = cJSON_GetObjectItem(json, "metadata");
         if (metadata != NULL) {
             const cJSON *indices = cJSON_GetObjectItem(metadata, "indices");
-            const cJSON *sist2 = cJSON_GetObjectItem(indices, "sist2");
+            const cJSON *index = cJSON_GetObjectItem(indices, WebCtx.es_index);
-            const cJSON *state = cJSON_GetObjectItem(sist2, "state");
+            const cJSON *state = cJSON_GetObjectItem(index, "state");
             strcpy(status, state->valuestring);
         }
     }

src/index/elastic.h

@@ -20,17 +20,17 @@ void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);
 
 void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);
 
-es_indexer_t *create_indexer(const char* es_url);
+es_indexer_t *create_indexer(const char *url, const char *index);
 
 void elastic_cleanup();
-void finish_indexer(char *script, char *index_id);
+void finish_indexer(char *script, int async_script, char *index_id);
 
-void elastic_init(int force_reset);
+void elastic_init(int force_reset, const char* user_mappings, const char* user_settings);
 
 cJSON *elastic_get_document(const char *uuid_str);
 
 char *elastic_get_status();
 
-void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]);
+void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]);
 
 #endif

src/index/web.c

@@ -1,5 +1,6 @@
 #include "web.h"
 #include "src/sist.h"
+#include "src/ctx.h"
 
 #include <mongoose.h>
 #include <pthread.h>
@@ -21,98 +22,85 @@ void free_response(response_t *resp) {
     free(resp);
 }
 
-#define SIST2_HEADERS "User-Agent: sist2\r\nContent-Type: application/json\r\n"
+void web_post_async_poll(subreq_ctx_t* req) {
+    fd_set fdread;
+    fd_set fdwrite;
+    fd_set fdexcep;
+    int maxfd = -1;
 
+    FD_ZERO(&fdread);
+    FD_ZERO(&fdwrite);
+    FD_ZERO(&fdexcep);
 
-void http_req_ev(struct mg_connection *nc, int ev, void *ptr) {
+    CURLMcode mc = curl_multi_fdset(req->multi, &fdread, &fdwrite, &fdexcep, &maxfd);
 
-    http_ev_data_t *ev_data = (http_ev_data_t *) nc->user_data;
+    if(mc != CURLM_OK) {
-
+        req->done = TRUE;
-    switch (ev) {
+        return;
-        case MG_EV_CONNECT: {
-            int connect_status = *(int *) ptr;
-            if (connect_status != 0) {
-                ev_data->done = TRUE;
-                ev_data->resp->status_code = 0;
     }
+
+    if (maxfd == -1) {
+        // no fds ready yet
+        return;
+    }
+
+    struct timeval timeout = {1, 0};
+    int rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
+
+    switch(rc) {
+        case -1:
+            req->done = TRUE;
             break;
-        }
+        case 0:
-        case MG_EV_HTTP_REPLY: {
-            struct http_message *hm = (struct http_message *) ptr;
-
-            //TODO: Check errors?
-
-            ev_data->resp->size = hm->body.len;
-            ev_data->resp->status_code = hm->resp_code;
-            ev_data->resp->body = malloc(hm->body.len + 1);
-            memcpy(ev_data->resp->body, hm->body.p, hm->body.len);
-            *(ev_data->resp->body + hm->body.len) = '\0';
-
-            ev_data->done = TRUE;
             break;
-        }
-        case MG_EV_CLOSE: {
-            ev_data->done = TRUE;
-            break;
-        }
         default:
+            curl_multi_perform(req->multi, &req->running_handles);
             break;
     }
+
+    if (req->running_handles == 0) {
+        req->done = TRUE;
+        req->response->body = req->response_buf.buf;
+        req->response->size = req->response_buf.cur;
+        curl_easy_getinfo(req->handle, CURLINFO_RESPONSE_CODE, &req->response->status_code);
+
+        curl_multi_cleanup(req->multi);
+        curl_easy_cleanup(req->handle);
+        curl_slist_free_all(req->headers);
+        return;
+    }
 }
 
-subreq_ctx_t *http_req(const char *url, const char *extra_headers, const char *post_data, const char *method) {
+subreq_ctx_t *web_post_async(const char *url, char *data) {
+    subreq_ctx_t *req = calloc(1, sizeof(subreq_ctx_t));
+    req->response = calloc(1, sizeof(response_t));
+    req->data = data;
+    req->response_buf = dyn_buffer_create();
 
-    struct mg_str scheme;
+    req->handle = curl_easy_init();
-    struct mg_str user_info;
+    CURL *curl = req->handle;
-    struct mg_str host;
+    curl_easy_setopt(curl, CURLOPT_URL, url);
-    unsigned int port;
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&req->response_buf));
-    struct mg_str path;
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
-    struct mg_str query;
+    curl_easy_setopt(curl, CURLOPT_POST, 1);
-    struct mg_str fragment;
+    curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
 
-    if (post_data == NULL) post_data = "";
+    struct curl_slist *headers = NULL;
-    if (extra_headers == NULL) extra_headers = "";
+    headers = curl_slist_append(headers, "Content-Type: application/json");
-    if (path.len == 0) path = mg_mk_str("/");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
-    if (host.len == 0) host = mg_mk_str("");
 
-    // [scheme://[user_info@]]host[:port][/path][?query][#fragment]
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);
-    mg_parse_uri(mg_mk_str(url), &scheme, &user_info, &host, &port, &path, &query, &fragment);
 
-    if (query.len > 0) path.len += query.len + 1;
+    req->multi = curl_multi_init();
+    curl_multi_add_handle(req->multi, curl);
+    curl_multi_perform(req->multi, &req->running_handles);
 
-    subreq_ctx_t *ctx = malloc(sizeof(subreq_ctx_t));
+    LOG_DEBUGF("web.c", "async request POST %s", url)
-    mg_mgr_init(&ctx->mgr, NULL);
 
-    char address[8196];
+    return req;
-    snprintf(address, sizeof(address), "tcp://%.*s:%u", (int) host.len, host.p, port);
-    struct mg_connection *nc = mg_connect(&ctx->mgr, address, http_req_ev);
-    nc->user_data = &ctx->ev_data;
-    mg_set_protocol_http_websocket(nc);
-
-    ctx->ev_data.resp = calloc(1, sizeof(response_t));
-    ctx->ev_data.done = FALSE;
-
-    mg_printf(
-            nc, "%s %.*s HTTP/1.1\r\n"
-                  "Host: %.*s\r\n"
-                  "Content-Length: %zu\r\n"
-                  "%s\r\n"
-                  "%s",
-            method, (int) path.len, path.p,
-            (int) (path.p - host.p), host.p,
-            strlen(post_data),
-            extra_headers,
-            post_data
-    );
-
-    return ctx;
 }
 
-subreq_ctx_t *web_post_async(const char *url, const char *data) {
+response_t *web_get(const char *url, int timeout) {
-    return http_req(url, SIST2_HEADERS, data, "POST");
-}
-
-response_t *web_get(const char *url) {
     response_t *resp = malloc(sizeof(response_t));
 
     CURL *curl;
@@ -123,8 +111,10 @@ response_t *web_get(const char *url) {
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&buffer));
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+    curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
 
-    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Content-Type: application/json");
     curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
 
     curl_easy_perform(curl);
@@ -152,7 +142,8 @@ response_t *web_post(const char *url, const char *data) {
     curl_easy_setopt(curl, CURLOPT_POST, 1);
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
 
-    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Content-Type: application/json");
     curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
 
     curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);
@@ -186,7 +177,8 @@ response_t *web_put(const char *url, const char *data) {
     curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
     curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4 );
 
-    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Content-Type: application/json");
     curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
 
     curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);
@@ -217,7 +209,8 @@ response_t *web_delete(const char *url) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
 
     curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "");
-    struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
+    struct curl_slist *headers = NULL;
+    headers = curl_slist_append(headers, "Content-Type: application/json");
     curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
 
     curl_easy_perform(curl);

src/index/web.h

@@ -3,6 +3,7 @@
 
 #include "src/sist.h"
 #include <mongoose.h>
+#include <curl/curl.h>
 
 typedef struct response {
     char *body;
@@ -16,13 +17,20 @@ typedef struct {
 } http_ev_data_t;
 
 typedef struct {
-    http_ev_data_t ev_data;
+    char* data;
-    struct mg_mgr mgr;
+    dyn_buffer_t response_buf;
+    struct curl_slist *headers;
+    CURL *handle;
+    CURLM *multi;
+    response_t *response;
+    int running_handles;
+    int done;
 } subreq_ctx_t;
 
-response_t *web_get(const char *url);
+response_t *web_get(const char *url, int timeout);
 response_t *web_post(const char * url, const char * data);
-subreq_ctx_t *web_post_async(const char *url, const char *data);
+void web_post_async_poll(subreq_ctx_t* req);
+subreq_ctx_t *web_post_async(const char *url, char *data);
 response_t *web_put(const char *url, const char *data);
 response_t *web_delete(const char *url);
 

src/io/serialize.c

@@ -150,6 +150,8 @@ char *get_meta_key_text(enum metakey meta_key) {
             return "modified_by";
         case MetaThumbnail:
             return "thumbnail";
+        case MetaPages:
+            return "pages";
         default:
             return NULL;
     }
@@ -245,13 +247,8 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
         }
         dyn_buffer_write_char(&buf, '\0');
 
-        if (IndexCtx.tags != NULL) {
+        char full_filename[PATH_MAX];
-            const char *tags_string = g_hash_table_lookup(IndexCtx.tags, buf.buf);
+        strcpy(full_filename, buf.buf);
-            if (tags_string != NULL) {
-                cJSON *tags_arr = cJSON_Parse(tags_string);
-                cJSON_AddItemToObject(document, "tag", tags_arr);
-            }
-        }
 
         cJSON_AddStringToObject(document, "extension", buf.buf + line.ext);
         if (*(buf.buf + line.ext - 1) == '.') {
@@ -278,6 +275,7 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
         size_t ret = 0;
         while (key != '\n') {
             switch (key) {
+                case MetaPages:
                 case MetaWidth:
                 case MetaHeight: {
                     int value;
@@ -331,8 +329,36 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
             key = getc(file);
         }
 
+        cJSON *meta_obj = NULL;
+        if (IndexCtx.meta != NULL) {
+            const char *meta_string = g_hash_table_lookup(IndexCtx.meta, full_filename);
+            if (meta_string != NULL) {
+                meta_obj = cJSON_Parse(meta_string);
+
+                cJSON *child;
+                for (child = meta_obj->child; child != NULL; child = child->next) {
+                    char meta_key[4096];
+                    strcpy(meta_key, child->string);
+                    cJSON_DeleteItemFromObject(document, meta_key);
+                    cJSON_AddItemReferenceToObject(document, meta_key, child);
+                }
+            }
+        }
+
+        if (IndexCtx.tags != NULL) {
+            const char *tags_string = g_hash_table_lookup(IndexCtx.tags, full_filename);
+            if (tags_string != NULL) {
+                cJSON *tags_arr = cJSON_Parse(tags_string);
+                cJSON_DeleteItemFromObject(document, "tag");
+                cJSON_AddItemToObject(document, "tag", tags_arr);
+            }
+        }
+
         func(document, uuid_str);
         cJSON_Delete(document);
+        if (meta_obj) {
+            cJSON_Delete(meta_obj);
+        }
     }
     dyn_buffer_destroy(&buf);
     fclose(file);

src/io/store.c

@@ -43,9 +43,13 @@ void store_destroy(store_t *store) {
 void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len) {
 
     if (LogCtx.very_verbose) {
-        char uuid_str[UUID_STR_LEN];
+        if (key_len == 16) {
+            char uuid_str[UUID_STR_LEN] = {0, };
             uuid_unparse((unsigned char *) key, uuid_str);
             LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", uuid_str, buf_len)
+        } else {
+            LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", key, buf_len)
+        }
     }
 
     MDB_val mdb_key;
@@ -136,7 +140,9 @@ GHashTable *store_read_all(store_t *store) {
         count += 1;
     }
 
-    LOG_DEBUGF("store.c", "Read tags for %d documents", count);
+    const char *path;
+    mdb_env_get_path(store->env, &path);
+    LOG_DEBUGF("store.c", "Read %d entries from %s", count, path);
 
     mdb_cursor_close(cur);
     mdb_txn_abort(txn);

src/io/store.h

@@ -8,6 +8,7 @@
 
 #define STORE_SIZE_TN 1024 * 1024 * 5
 #define STORE_SIZE_TAG 1024 * 16
+#define STORE_SIZE_META STORE_SIZE_TAG
 
 typedef struct store_t {
     MDB_dbi dbi;

src/log.c

@@ -4,8 +4,7 @@
 #include <stdarg.h>
 
 const char *log_colors[] = {
-        "\033[34m", "\033[01;34m", "\033[0m",
+        "\033[34m", "\033[01;34m", "\033[01;33m", "\033[0m", "\033[31m", "\033[01;31m"
-        "\033[01;33m", "\033[31m", "\033[01;31m"
 };
 
 const char *log_levels[] = {

src/main.c

@@ -21,7 +21,7 @@
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "2.7.3";
+static const char *const Version = "2.8.5";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
@@ -182,6 +182,10 @@ void sist2_scan(scan_args_t *args) {
     mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
     ScanCtx.index.store = store_create(store_path, STORE_SIZE_TN);
 
+    snprintf(store_path, PATH_MAX, "%smeta", ScanCtx.index.path);
+    mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
+    ScanCtx.index.meta_store = store_create(store_path, STORE_SIZE_META);
+
     scan_print_header();
 
     if (args->incremental != NULL) {
@@ -259,10 +263,11 @@ void sist2_scan(scan_args_t *args) {
 void sist2_index(index_args_t *args) {
 
     IndexCtx.es_url = args->es_url;
+    IndexCtx.es_index = args->es_index;
     IndexCtx.batch_size = args->batch_size;
 
     if (!args->print) {
-        elastic_init(args->force_reset);
+        elastic_init(args->force_reset, args->es_mappings, args->es_settings);
     }
 
     char descriptor_path[PATH_MAX];
@@ -288,6 +293,10 @@ void sist2_index(index_args_t *args) {
     IndexCtx.tag_store = store_create(path_tmp, STORE_SIZE_TAG);
     IndexCtx.tags = store_read_all(IndexCtx.tag_store);
 
+    snprintf(path_tmp, sizeof(path_tmp), "%s/meta", args->index_path);
+    IndexCtx.meta_store = store_create(path_tmp, STORE_SIZE_META);
+    IndexCtx.meta = store_read_all(IndexCtx.meta_store);
+
     index_func f;
     if (args->print) {
         f = print_json;
@@ -317,11 +326,12 @@ void sist2_index(index_args_t *args) {
 
     tpool_wait(IndexCtx.pool);
 
-    if (!args->print) {
-        finish_indexer(args->script, desc.uuid);
-    }
     tpool_destroy(IndexCtx.pool);
 
+    if (!args->print) {
+        finish_indexer(args->script, args->async_script, desc.uuid);
+    }
+
     store_destroy(IndexCtx.tag_store);
     g_hash_table_remove_all(IndexCtx.tags);
     g_hash_table_destroy(IndexCtx.tags);
@@ -339,13 +349,14 @@ void sist2_exec_script(exec_args_t *args) {
 
     LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type)
 
-    execute_update_script(args->script, desc.uuid);
+    execute_update_script(args->script, args->async_script, desc.uuid);
     free(args->script);
 }
 
 void sist2_web(web_args_t *args) {
 
     WebCtx.es_url = args->es_url;
+    WebCtx.es_index = args->es_index;
     WebCtx.index_count = args->index_count;
     WebCtx.auth_user = args->auth_user;
     WebCtx.auth_pass = args->auth_pass;
@@ -389,7 +400,9 @@ int main(int argc, const char *argv[]) {
     int arg_version = 0;
 
     char *common_es_url = NULL;
+    char *common_es_index = NULL;
     char *common_script_path = NULL;
+    int common_async_script = 0;
     int common_threads = 0;
 
     struct argparse_option options[] = {
@@ -430,20 +443,28 @@ int main(int argc, const char *argv[]) {
             OPT_GROUP("Index options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"),
+            OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
             OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
             OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
+            OPT_STRING(0, "mappings-file", &index_args->es_mappings_path, "Path to Elasticsearch mappings."),
+            OPT_STRING(0, "settings-file", &index_args->es_settings_path, "Path to Elasticsearch settings."),
+            OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),
             OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 100"),
             OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. "
                                                                       "(You must use this option the first time you use the index command)"),
 
             OPT_GROUP("Web options"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
+            OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
             OPT_STRING(0, "bind", &web_args->listen_address, "Listen on this address. DEFAULT=localhost:4090"),
             OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"),
             OPT_STRING(0, "tag-auth", &web_args->tag_credentials, "Basic auth in user:password format for tagging"),
 
             OPT_GROUP("Exec-script options"),
+            OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
+            OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
             OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
+            OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),
 
             OPT_END(),
     };
@@ -465,10 +486,17 @@ int main(int argc, const char *argv[]) {
     web_args->es_url = common_es_url;
     index_args->es_url = common_es_url;
     exec_args->es_url = common_es_url;
+
+    web_args->es_index = common_es_index;
+    index_args->es_index = common_es_index;
+    exec_args->es_index = common_es_index;
+
     index_args->script_path = common_script_path;
     exec_args->script_path = common_script_path;
     index_args->threads = common_threads;
     scan_args->threads = common_threads;
+    exec_args->async_script = common_async_script;
+    index_args->async_script = common_async_script;
 
     if (argc == 0) {
         argparse_usage(&argparse);

src/parsing/mime.h

@@ -6,6 +6,7 @@
 #define MAJOR_MIME(mime_id) (mime_id & 0x000F0000) >> 16
 
 #define MIME_EMPTY 1
+#define MIME_SIST2_SIDECAR 2
 
 #define DONT_PARSE 0x80000000
 #define SHOULD_PARSE(mime_id) (ScanCtx.fast == 0 && (mime_id & DONT_PARSE) != DONT_PARSE && mime_id != 0)

src/parsing/mime_generated.c

@@ -54,78 +54,79 @@ enum mime {
     application_streamingmedia=655406,
     application_vda=655407,
     application_vnd_amazon_mobi8_ebook=655408 | 0x02000000,
-    application_vnd_fdf=655409,
+    application_vnd_coffeescript=655409,
-    application_vnd_font_fontforge_sfd=655410,
+    application_vnd_fdf=655410,
-    application_vnd_hp_hpgl=655411,
+    application_vnd_font_fontforge_sfd=655411,
-    application_vnd_iccprofile=655412,
+    application_vnd_hp_hpgl=655412,
-    application_vnd_lotus_1_2_3=655413,
+    application_vnd_iccprofile=655413,
-    application_vnd_ms_cab_compressed=655414,
+    application_vnd_lotus_1_2_3=655414,
-    application_vnd_ms_excel=655415,
+    application_vnd_ms_cab_compressed=655415,
-    application_vnd_ms_fontobject=655416,
+    application_vnd_ms_excel=655416,
-    application_vnd_ms_opentype=655417 | 0x20000000,
+    application_vnd_ms_fontobject=655417,
-    application_vnd_ms_outlook=655418,
+    application_vnd_ms_opentype=655418 | 0x20000000,
-    application_vnd_ms_pki_certstore=655419,
+    application_vnd_ms_outlook=655419,
-    application_vnd_ms_pki_pko=655420,
+    application_vnd_ms_pki_certstore=655420,
-    application_vnd_ms_pki_seccat=655421,
+    application_vnd_ms_pki_pko=655421,
-    application_vnd_ms_powerpoint=655422,
+    application_vnd_ms_pki_seccat=655422,
-    application_vnd_ms_project=655423,
+    application_vnd_ms_powerpoint=655423,
-    application_vnd_oasis_opendocument_base=655424,
+    application_vnd_ms_project=655424,
-    application_vnd_oasis_opendocument_formula=655425,
+    application_vnd_oasis_opendocument_base=655425,
-    application_vnd_oasis_opendocument_graphics=655426,
+    application_vnd_oasis_opendocument_formula=655426,
-    application_vnd_oasis_opendocument_presentation=655427,
+    application_vnd_oasis_opendocument_graphics=655427,
-    application_vnd_oasis_opendocument_spreadsheet=655428,
+    application_vnd_oasis_opendocument_presentation=655428,
-    application_vnd_oasis_opendocument_text=655429,
+    application_vnd_oasis_opendocument_spreadsheet=655429,
-    application_vnd_openxmlformats_officedocument_presentationml_presentation=655430 | 0x04000000,
+    application_vnd_oasis_opendocument_text=655430,
-    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655431 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_presentationml_presentation=655431 | 0x04000000,
-    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655432 | 0x04000000,
+    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655432 | 0x04000000,
-    application_vnd_symbian_install=655433,
+    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655433 | 0x04000000,
-    application_vnd_tcpdump_pcap=655434,
+    application_vnd_symbian_install=655434,
-    application_vnd_wap_wmlc=655435,
+    application_vnd_tcpdump_pcap=655435,
-    application_vnd_wap_wmlscriptc=655436,
+    application_vnd_wap_wmlc=655436,
-    application_vnd_xara=655437,
+    application_vnd_wap_wmlscriptc=655437,
-    application_vocaltec_media_desc=655438,
+    application_vnd_xara=655438,
-    application_vocaltec_media_file=655439,
+    application_vocaltec_media_desc=655439,
-    application_warc=655440,
+    application_vocaltec_media_file=655440,
-    application_winhelp=655441,
+    application_warc=655441,
-    application_wordperfect=655442,
+    application_winhelp=655442,
-    application_wordperfect6_0=655443,
+    application_wordperfect=655443,
-    application_wordperfect6_1=655444,
+    application_wordperfect6_0=655444,
-    application_x_123=655445,
+    application_wordperfect6_1=655445,
-    application_x_7z_compressed=655446 | 0x10000000,
+    application_x_123=655446,
-    application_x_aim=655447,
+    application_x_7z_compressed=655447 | 0x10000000,
-    application_x_apple_diskimage=655448,
+    application_x_aim=655448,
-    application_x_arc=655449 | 0x10000000,
+    application_x_apple_diskimage=655449,
-    application_x_archive=655450,
+    application_x_arc=655450 | 0x10000000,
-    application_x_atari_7800_rom=655451,
+    application_x_archive=655451,
-    application_x_authorware_bin=655452,
+    application_x_atari_7800_rom=655452,
-    application_x_authorware_map=655453,
+    application_x_authorware_bin=655453,
-    application_x_authorware_seg=655454,
+    application_x_authorware_map=655454,
-    application_x_avira_qua=655455,
+    application_x_authorware_seg=655455,
-    application_x_bcpio=655456,
+    application_x_avira_qua=655456,
-    application_x_bittorrent=655457,
+    application_x_bcpio=655457,
-    application_x_bsh=655458,
+    application_x_bittorrent=655458,
-    application_x_bytecode_python=655459,
+    application_x_bsh=655459,
-    application_x_bzip=655460,
+    application_x_bytecode_python=655460,
-    application_x_bzip2=655461 | 0x08000000,
+    application_x_bzip=655461,
-    application_x_cbr=655462,
+    application_x_bzip2=655462 | 0x08000000,
-    application_x_cbz=655463,
+    application_x_cbr=655463,
-    application_x_cdlink=655464,
+    application_x_cbz=655464,
-    application_x_chat=655465,
+    application_x_cdlink=655465,
-    application_x_chrome_extension=655466,
+    application_x_chat=655466,
-    application_x_cocoa=655467,
+    application_x_chrome_extension=655467,
-    application_x_conference=655468,
+    application_x_cocoa=655468,
-    application_x_coredump=655469,
+    application_x_conference=655469,
-    application_x_cpio=655470,
+    application_x_coredump=655470,
-    application_x_dbf=655471,
+    application_x_cpio=655471,
-    application_x_dbt=655472,
+    application_x_dbf=655472,
-    application_x_debian_package=655473,
+    application_x_dbt=655473,
-    application_x_deepv=655474,
+    application_x_debian_package=655474,
-    application_x_director=655475,
+    application_x_deepv=655475,
-    application_x_dmp=655476,
+    application_x_director=655476,
-    application_x_dosdriver=655477,
+    application_x_dmp=655477,
-    application_x_dosexec=655478,
+    application_x_dosdriver=655478,
-    application_x_dvi=655479,
+    application_x_dosexec=655479,
-    application_x_elc=655480,
+    application_x_dvi=655480,
+    application_x_elc=655481,
     application_x_empty=1,
     application_x_envoy=655482,
     application_x_esrehber=655483,
@@ -366,6 +367,7 @@ enum mime {
     model_vnd_gs_gdl=65894,
     model_vrml=65895,
     model_x_pov=65896,
+    sist2_sidecar=2,
     text_PGP=590185,
     text_asp=590186,
     text_css=590187,
@@ -804,6 +806,7 @@ case text_mcf: return "text/mcf";
 case text_pascal: return "text/pascal";
 case text_PGP: return "text/PGP";
 case text_plain: return "text/plain";
+case application_vnd_coffeescript: return "application/vnd.coffeescript";
 case text_richtext: return "text/richtext";
 case text_rtf: return "text/rtf";
 case text_scriplet: return "text/scriplet";
@@ -904,6 +907,7 @@ case image_x_sony_arw: return "image/x-sony-arw";
 case image_x_sony_sr2: return "image/x-sony-sr2";
 case image_x_sony_srf: return "image/x-sony-srf";
 case image_x_epson_erf: return "image/x-epson-erf";
+case sist2_sidecar: return "sist2/sidecar";
 default: return NULL;}}
 GHashTable *mime_get_ext_table() {GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(ext_table, "arj", (gpointer)application_arj);
@@ -1315,6 +1319,11 @@ g_hash_table_insert(ext_table, "sfv", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "m3u", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "csv", (gpointer)text_plain);
 g_hash_table_insert(ext_table, "eml", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "make", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "log", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "markdown", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "yaml", (gpointer)text_plain);
+g_hash_table_insert(ext_table, "coffee", (gpointer)application_vnd_coffeescript);
 g_hash_table_insert(ext_table, "rt", (gpointer)text_richtext);
 g_hash_table_insert(ext_table, "rtf", (gpointer)text_richtext);
 g_hash_table_insert(ext_table, "rtx", (gpointer)text_richtext);
@@ -1442,6 +1451,7 @@ g_hash_table_insert(ext_table, "arw", (gpointer)image_x_sony_arw);
 g_hash_table_insert(ext_table, "sr2", (gpointer)image_x_sony_sr2);
 g_hash_table_insert(ext_table, "srf", (gpointer)image_x_sony_srf);
 g_hash_table_insert(ext_table, "erf", (gpointer)image_x_epson_erf);
+g_hash_table_insert(ext_table, "s2meta", (gpointer)sist2_sidecar);
 return ext_table;}
 GHashTable *mime_get_mime_table() {GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(mime_table, "application/arj", (gpointer)application_arj);
@@ -1792,6 +1802,7 @@ g_hash_table_insert(mime_table, "text/mcf", (gpointer)text_mcf);
 g_hash_table_insert(mime_table, "text/pascal", (gpointer)text_pascal);
 g_hash_table_insert(mime_table, "text/PGP", (gpointer)text_PGP);
 g_hash_table_insert(mime_table, "text/plain", (gpointer)text_plain);
+g_hash_table_insert(mime_table, "application/vnd.coffeescript", (gpointer)application_vnd_coffeescript);
 g_hash_table_insert(mime_table, "text/richtext", (gpointer)text_richtext);
 g_hash_table_insert(mime_table, "text/rtf", (gpointer)text_rtf);
 g_hash_table_insert(mime_table, "text/scriplet", (gpointer)text_scriplet);
@@ -1892,5 +1903,6 @@ g_hash_table_insert(mime_table, "image/x-sony-arw", (gpointer)image_x_sony_arw);
 g_hash_table_insert(mime_table, "image/x-sony-sr2", (gpointer)image_x_sony_sr2);
 g_hash_table_insert(mime_table, "image/x-sony-srf", (gpointer)image_x_sony_srf);
 g_hash_table_insert(mime_table, "image/x-epson-erf", (gpointer)image_x_epson_erf);
+g_hash_table_insert(mime_table, "sist2/sidecar", (gpointer)sist2_sidecar);
 return mime_table;}
 #endif

src/parsing/parse.c

@@ -4,6 +4,7 @@
 #include "src/ctx.h"
 #include "mime.h"
 #include "src/io/serialize.h"
+#include "src/parsing/sidecar.h"
 
 #include <magic.h>
 
@@ -38,6 +39,8 @@ void fs_reset(struct vfile *f) {
     }
 }
 
+#define IS_GIT_OBJ (strlen(doc.filepath + doc.base) == 38 && (strstr(doc.filepath, "objects") != NULL))
+
 void parse(void *arg) {
 
     parse_job_t *job = arg;
@@ -77,6 +80,10 @@ void parse(void *arg) {
     int bytes_read = 0;
 
     if (doc.mime == 0 && !ScanCtx.fast) {
+        if (IS_GIT_OBJ) {
+            goto abort;
+        }
+
         // Get mime type with libmagic
         if (!job->vfile.is_fs_file) {
             LOG_WARNING(job->filepath, "Guessing mime type with libmagic inside archive files is not currently supported");
@@ -151,6 +158,10 @@ void parse(void *arg) {
         parse_comic(&ScanCtx.comic_ctx, &job->vfile, &doc);
     } else if (IS_MOBI(doc.mime)) {
         parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, &doc);
+    } else if (doc.mime == MIME_SIST2_SIDECAR) {
+        parse_sidecar(&job->vfile, &doc);
+        CLOSE_FILE(job->vfile)
+        return;
     }
 
     abort:

src/parsing/sidecar.c

@@ -0,0 +1,35 @@
+#include "sidecar.h"
+
+#include "src/ctx.h"
+
+void parse_sidecar(vfile_t *vfile, document_t *doc) {
+
+    LOG_DEBUGF("sidecar.c", "Parsing sidecar file %s", vfile->filepath)
+
+    size_t size;
+    char* buf = read_all(vfile, &size);
+    if (buf == NULL) {
+        LOG_ERRORF("sidecar.c", "Read error for %s", vfile->filepath)
+        return;
+    }
+
+    buf = realloc(buf, size + 1);
+    *(buf + size) = '\0';
+
+    cJSON *json = cJSON_Parse(buf);
+    if (json == NULL) {
+        LOG_ERRORF("sidecar.c", "Could not parse JSON sidecar %s", vfile->filepath)
+        return;
+    }
+    char *json_str = cJSON_PrintUnformatted(json);
+
+    char filepath[PATH_MAX];
+    memcpy(filepath, vfile->filepath + ScanCtx.index.desc.root_len, doc->ext - 1 - ScanCtx.index.desc.root_len);
+    *(filepath + doc->ext - 1) = '\0';
+
+    store_write(ScanCtx.index.meta_store, filepath, doc->ext, json_str, strlen(json_str) + 1);
+
+    cJSON_Delete(json);
+    free(json_str);
+    free(buf);
+}

src/parsing/sidecar.h

@@ -0,0 +1,8 @@
+#ifndef SIST2_SIDECAR_H
+#define SIST2_SIDECAR_H
+
+#include "src/sist.h"
+
+void parse_sidecar(vfile_t *vfile, document_t *doc);
+
+#endif

src/static/css/dark.css

@@ -121,7 +121,7 @@ body {
     background: #546b7a;
 }
 
-a:hover,.btn:hover {
+a:hover, .btn:hover {
     color: #fff;
 }
 
@@ -130,7 +130,11 @@ a:hover,.btn:hover {
 }
 
 .document {
-    padding: 0.5rem;
+    padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }
 
 .document p {
@@ -203,7 +207,7 @@ a:hover,.btn:hover {
     margin-top: -1px;
     font-family: monospace;
     font-size: 90%;
-    background: rgba(0,0,0,0.2);
+    background: rgba(0, 0, 0, 0.2);
     padding: 0.1em 0.4em;
     color: white;
     cursor: pointer;
@@ -214,17 +218,25 @@ a:hover,.btn:hover {
     background-color: #e0e0e0;
 }
 
+.card-img-top {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+}
+
 .fit {
     display: block;
     min-width: 64px;
     max-width: 100%;
-    max-height: 175px;
+    max-height: 400px;
     margin: 0 auto 0;
-    padding: 3px 3px 0;
     width: auto;
     height: auto;
 }
 
+.img-padding {
+    padding: 4px 4px 0 4px;
+}
+
 .fit-sm {
     display: block;
     max-width: 64px;
@@ -241,20 +253,6 @@ a:hover,.btn:hover {
     width: 100%;
 }
 
-@media screen and (min-width: 1500px) {
-    .container {
-        max-width: 1440px;
-    }
-
-    .bricklayer-column-sizer {
-        width: 20% !important;
-    }
-
-    .bricklayer-column {
-        max-width: 20%;
-    }
-}
-
 @media screen and (min-width: 1800px) {
     .container {
         max-width: 1550px;
@@ -451,6 +449,7 @@ option {
     .small-btn {
         display: none;
     }
+
     .large-btn {
         display: inherit;
     }
@@ -460,6 +459,7 @@ option {
     .small-btn {
         display: inherit;
     }
+
     .large-btn {
         display: none;
     }
@@ -538,3 +538,7 @@ svg {
 .stat > .card-body {
     padding: 0.7em 1.25em;
 }
+
+#modal-body > .img-wrapper {
+    margin-bottom: 1em;
+}

src/static/css/light.css

@@ -70,7 +70,11 @@ body {
 }
 
 .document {
-    padding: 0.5rem;
+   padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }
 
 .document p {
@@ -153,18 +157,25 @@ body {
     overflow: hidden;
 }
 
+.card-img-top {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+}
 
 .fit {
     display: block;
     min-width: 64px;
     max-width: 100%;
-    max-height: 175px;
+    max-height: 400px;
     margin: 0 auto 0;
-    padding: 3px 3px 0 3px;
     width: auto;
     height: auto;
 }
 
+.img-padding {
+    padding: 4px 4px 0 4px;
+}
+
 .fit-sm {
     display: block;
     max-width: 64px;
@@ -181,6 +192,10 @@ body {
     width: 100%;
 }
 
+.bricklayer {
+    /*max-width: 100%;*/
+}
+
 @media screen and (max-width: 1200px) {
     .bricklayer-column {
         max-width: 100%;
@@ -402,3 +417,7 @@ mark {
 .stat > .card-body {
     padding: 0.7em 1.25em;
 }
+
+#modal-body > .img-wrapper {
+    margin-bottom: 1em;
+}

src/static/js/dom.js

@@ -75,6 +75,7 @@ function shouldPlayVideo(hit) {
         hit["_source"]["extension"] !== "mkv" &&
         hit["_source"]["extension"] !== "avi" &&
         videoc !== "hevc" &&
+        videoc !== "mpeg1video" &&
         videoc !== "mpeg2video" &&
         videoc !== "wmv3";
 }
@@ -194,10 +195,21 @@ function makeUserTag(tag, hit) {
 function infoButtonCb(hit) {
     return () => {
         getDocumentInfo(hit["_id"]).then(doc => {
+            $("#modal-body").empty()
+
             $("#modal-title").text(doc["name"] + ext(hit));
 
+            if (doc["mime"]) {
+                const mimeCategory = doc["mime"].split("/")[0];
+                const imgWrapper = document.createElement("div");
+                imgWrapper.setAttribute("style", "position: relative");
+                imgWrapper.setAttribute("class", "img-wrapper");
+                makeThumbnail(mimeCategory, hit, imgWrapper, false);
+                $("#modal-body").append(imgWrapper);
+            }
+
             const tbody = $("<tbody>");
-            $("#modal-body").empty()
+            $("#modal-body")
                 .append($("<table class='table table-sm'>")
                     .append($("<thead>")
                         .append($("<tr>")
@@ -208,10 +220,19 @@ function infoButtonCb(hit) {
                     .append(tbody)
                 );
 
+            tbody.append($("<tr>")
+                .append($("<td>").text("index"))
+                .append($("<td>").text(`[${indexMap[doc["index"]]}]`))
+            ).append($("<tr>")
+                .append($("<td>").text("mtime"))
+                .append($("<td>")
+                    .text(new Date(doc["mtime"] * 1000).toISOString().split(".")[0].replace("T", " "))
+                    .attr("title", doc["mtime"]))
+            );
             const displayFields = new Set([
-                "mime", "size", "mtime", "path", "title", "width", "height", "duration", "audioc", "videoc",
+                "mime", "size", "path", "title", "width", "height", "duration", "audioc", "videoc",
                 "bitrate", "artist", "album", "album_artist", "genre", "title", "font_name", "tag", "author",
-                "modified_by"
+                "modified_by", "pages"
             ]);
             Object.keys(doc)
                 .filter(key => key.startsWith("_keyword.") || key.startsWith("_text.") || displayFields.has(key) || key.startsWith("exif_"))
@@ -391,9 +412,13 @@ function makeThumbnail(mimeCategory, hit, imgWrapper, small) {
     let thumbnail = document.createElement("img");
     if (small) {
         thumbnail.setAttribute("class", "fit-sm");
+    } else {
+        if (hit["_source"].hasOwnProperty("parent")) {
+            thumbnail.setAttribute("class", "card-img-top fit img-padding");
         } else {
             thumbnail.setAttribute("class", "card-img-top fit");
         }
+    }
     thumbnail.setAttribute("src", `t/${hit["_source"]["index"]}/${hit["_id"]}`);
 
     if (shouldDisplayRawImage(hit)) {

src/static/js/search.js

@@ -1,4 +1,4 @@
-const SIZE = 40;
+const SIZE = 60;
 let mimeMap = [];
 let tagMap = [];
 let mimeTree;
@@ -168,7 +168,7 @@ window.onload = () => {
 };
 
 function saveTag(tag, hit) {
-    const relPath = hit["_source"]["path"] + "/" + hit["_source"]["name"] + ext(hit);
+    const relPath = hit["_source"]["path"] + (hit["_source"]["path"] ? "/" : "") + hit["_source"]["name"] + ext(hit);
 
     return $.jsonPost("/tag/" + hit["_source"]["index"], {
         delete: false,
@@ -253,7 +253,7 @@ function handleTreeClick(tree) {
 
         if (node.id === "any") {
             if (!node.itree.state.checked) {
-                tree.deselect();
+                tree.deselectDeep();
             }
         } else {
             tree.node("any").deselect();

src/static/js/util.js

@@ -101,7 +101,8 @@ const _defaults = {
     treemapColor: "PuBuGn",
     treemapSize: "large",
     suggestPath: true,
-    fragmentSize: 100
+    fragmentSize: 100,
+    columns: 5
 };
 
 function loadSettings() {
@@ -118,6 +119,7 @@ function loadSettings() {
     $("#settingTreemapType").val(CONF.options.treemapType);
     $("#settingSuggestPath").prop("checked", CONF.options.suggestPath);
     $("#settingFragmentSize").val(CONF.options.fragmentSize);
+    $("#settingColumns").val(CONF.options.columns);
 }
 
 function Settings() {
@@ -125,6 +127,7 @@ function Settings() {
 
     this._onUpdate = function () {
         $("#fuzzyToggle").prop("checked", this.options.fuzzy);
+        updateColumnStyle();
     };
 
     this.load = function () {
@@ -161,6 +164,7 @@ function updateSettings() {
     CONF.options.treemapType = $("#settingTreemapType").val();
     CONF.options.suggestPath = $("#settingSuggestPath").prop("checked");
     CONF.options.fragmentSize = $("#settingFragmentSize").val();
+    CONF.options.columns = $("#settingColumns").val();
     CONF.save();
 
     if (typeof searchDebounced !== "undefined") {
@@ -203,3 +207,26 @@ function toggleTheme() {
     }
     window.location.reload();
 }
+
+function updateColumnStyle() {
+    const style = document.getElementById("style");
+    if (style) {
+        style.innerHTML =
+        `
+@media screen and (min-width: 1500px) {
+    .container {
+            max-width: 1440px;
+        }
+
+    .bricklayer-column-sizer {
+            width: ${100 / CONF.options.columns}% !important;
+        }
+
+    .bricklayer-column {
+            max-width: ${100 / CONF.options.columns}%;
+        }
+    }
+}
+        `
+    }
+}

src/static/search.html

@@ -6,12 +6,13 @@
     <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
 
     <link href="css" rel="stylesheet" type="text/css">
+    <style id="style"></style>
 </head>
 <body>
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.7.3</span>
+    <span class="badge badge-pill version">2.8.5</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <a class="btn ml-auto" href="/stats">Stats</a>
     <button class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings
@@ -216,6 +217,19 @@
                         <option value="list">List</option>
                     </select>
 
+                    <div class="form-group">
+                        <label for="settingColumns">Maximum column count</label>
+                        <select id="settingColumns" class="form-control form-control-sm">
+                            <option value="3">3</option>
+                            <option value="4">4</option>
+                            <option value="5">5</option>
+                            <option value="6">6</option>
+                            <option value="7">7</option>
+                            <option value="8">8</option>
+                            <option value="9">9</option>
+                        </select>
+                    </div>
+
                     <hr/>
                     <h4>Stats</h4>
 

src/static/stats.html

@@ -10,7 +10,7 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.7.3</span>
+    <span class="badge badge-pill version">2.8.5</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <a style="margin-left: auto" class="btn" href="/">Back</a>
     <button class="btn" type="button" data-toggle="modal" data-target="#settings"
@@ -94,6 +94,19 @@
                     <option value="list">List</option>
                 </select>
 
+                <div class="form-group">
+                    <label for="settingColumns">Maximum column count</label>
+                    <select id="settingColumns" class="form-control form-control-sm">
+                        <option value="3">3</option>
+                        <option value="4">4</option>
+                        <option value="5">5</option>
+                        <option value="6">6</option>
+                        <option value="7">7</option>
+                        <option value="8">8</option>
+                        <option value="9">9</option>
+                    </select>
+                </div>
+
                 <hr/>
                 <h4>Stats</h4>
 

src/types.h

@@ -10,7 +10,7 @@ typedef struct index_descriptor {
     char version[64];
     long timestamp;
     char root[PATH_MAX];
-    char rewrite_url[8196];
+    char rewrite_url[8192];
     short root_len;
     char name[1024];
     char type[64];
@@ -20,6 +20,7 @@ typedef struct index_t {
     struct index_descriptor desc;
     struct store_t *store;
     struct store_t *tag_store;
+    struct store_t *meta_store;
     char path[PATH_MAX];
 } index_t;
 

src/web/serve.c

@@ -10,8 +10,6 @@
 
 #include <mongoose.h>
 
-#define CHUNK_SIZE 1024 * 1024 * 10
-
 
 static int has_prefix(const struct mg_str *str, const struct mg_str *prefix) {
     return str->len > prefix->len && memcmp(str->p, prefix->p, prefix->len) == 0;
@@ -111,7 +109,7 @@ void stats_files(struct mg_connection *nc, struct http_message *hm, struct mg_st
             return;
     }
 
-    char disposition[8196];
+    char disposition[8192];
     snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s\"", file);
 
     char full_path[PATH_MAX];
@@ -237,13 +235,12 @@ void search(struct mg_connection *nc, struct http_message *hm) {
     *(body + hm->body.len) = '\0';
 
     char url[4096];
-    snprintf(url, 4096, "%s/sist2/_search", WebCtx.es_url);
+    snprintf(url, 4096, "%s/%s/_search", WebCtx.es_url, WebCtx.es_index);
 
     nc->user_data = web_post_async(url, body);
-    free(body);
 }
 
-int serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
+void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
 
     const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
     const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
@@ -256,7 +253,7 @@ int serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
 
     const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
 
-    char url[8196];
+    char url[8192];
     snprintf(url, sizeof(url),
              "%s%s/%s%s%s",
              idx->desc.rewrite_url, path_unescaped, name_unescaped, strlen(ext) == 0 ? "" : ".", ext);
@@ -291,7 +288,7 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
 
     LOG_DEBUGF("serve.c", "Serving file from disk: %s", full_path)
 
-    char disposition[8196];
+    char disposition[8192];
     snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s%s%s\"",
              name, strlen(ext) == 0 ? "" : ".", ext);
 
@@ -538,8 +535,8 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
             }
         }
 
-        char buf[8196];
+        char *buf = malloc(sizeof(char) * 8192);
-        snprintf(buf, sizeof(buf),
+        snprintf(buf, 8192,
                  "{"
                  "    \"script\" : {"
                  "        \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\","
@@ -552,14 +549,14 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
         );
 
         char url[4096];
-        snprintf(url, sizeof(url), "%s/sist2/_update/%s", WebCtx.es_url, arg_req->doc_id);
+        snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
         nc->user_data = web_post_async(url, buf);
 
     } else {
         cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name));
 
-        char buf[8196];
+        char *buf = malloc(sizeof(char) * 8192);
-        snprintf(buf, sizeof(buf),
+        snprintf(buf, 8192,
                  "{"
                  "    \"script\" : {"
                  "        \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\","
@@ -572,7 +569,7 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
         );
 
         char url[4096];
-        snprintf(url, sizeof(url), "%s/sist2/_update/%s", WebCtx.es_url, arg_req->doc_id);
+        snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
         nc->user_data = web_post_async(url, buf);
     }
 
@@ -668,11 +665,11 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
         if (nc->user_data != NULL) {
             //Waiting for ES reply
             subreq_ctx_t *ctx = (subreq_ctx_t *) nc->user_data;
-            mg_mgr_poll(&ctx->mgr, 0);
+            web_post_async_poll(ctx);
 
-            if (ctx->ev_data.done == TRUE) {
+            if (ctx->done == TRUE) {
 
-                response_t *r = ctx->ev_data.resp;
+                response_t *r = ctx->response;
 
                 if (r->status_code == 200) {
                     send_response_line(nc, 200, r->size, "Content-Type: application/json");
@@ -695,6 +692,8 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
                 }
 
                 free_response(r);
+                free(ctx->data);
+                free(ctx);
                 nc->flags |= MG_F_SEND_AND_CLOSE;
                 nc->user_data = NULL;
             }