epub fix

CMakeLists.txt

@@ -1,38 +1,74 @@
 cmake_minimum_required(VERSION 3.7)
 set(CMAKE_C_STANDARD 11)
 
+option(WITH_SIST2 "Build main executable" ON)
+option(WITH_SIST2_SCAN "Build scan executable" ON)
+
 project(sist2 C)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMakeModules")
 
-add_executable(
-        sist2
-        src/main.c
-        src/sist.h
-        src/io/walk.h src/io/walk.c
-        src/parsing/media.h src/parsing/media.c
-        src/parsing/pdf.h src/parsing/pdf.c
-        src/io/store.h src/io/store.c
-        src/tpool.h src/tpool.c
-        src/parsing/parse.h src/parsing/parse.c
-        src/io/serialize.h src/io/serialize.c
-        src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
-        src/parsing/text.h src/parsing/text.c
-        src/index/web.c src/index/web.h
-        src/web/serve.c src/web/serve.h
-        src/index/elastic.c src/index/elastic.h
-        src/util.c src/util.h
-        src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
+if (WITH_SIST2)
+    add_executable(
+            sist2
+            src/main.c
+            src/sist.h
+            src/io/walk.h src/io/walk.c
+            src/parsing/media.h src/parsing/media.c
+            src/parsing/pdf.h src/parsing/pdf.c
+            src/io/store.h src/io/store.c
+            src/tpool.h src/tpool.c
+            src/parsing/parse.h src/parsing/parse.c
+            src/io/serialize.h src/io/serialize.c
+            src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
+            src/parsing/text.h src/parsing/text.c
+            src/index/web.c src/index/web.h
+            src/web/serve.c src/web/serve.h
+            src/index/elastic.c src/index/elastic.h
+            src/util.c src/util.h
+            src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
 
-        # argparse
-        argparse/argparse.h argparse/argparse.c
+            # argparse
+            argparse/argparse.h argparse/argparse.c
 
-        # cJSON
-        cJSON/cJSON.h cJSON/cJSON.c
+            # cJSON
+            cJSON/cJSON.h cJSON/cJSON.c
 
-        # LMDB
-        lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
-        lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
-        src/cli.c src/cli.h)
+            # LMDB
+            lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
+            lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
+            src/cli.c src/cli.h
+    )
+endif ()
+
+if (WITH_SIST2_SCAN)
+    add_executable(
+            sist2_scan
+            src/main.c
+            src/sist.h
+            src/io/walk.h src/io/walk.c
+            src/parsing/media.h src/parsing/media.c
+            src/parsing/pdf.h src/parsing/pdf.c
+            src/io/store.h src/io/store.c
+            src/tpool.h src/tpool.c
+            src/parsing/parse.h src/parsing/parse.c
+            src/io/serialize.h src/io/serialize.c
+            src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
+            src/parsing/text.h src/parsing/text.c
+            src/util.c src/util.h
+            src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
+
+            # argparse
+            argparse/argparse.h argparse/argparse.c
+
+            # cJSON
+            cJSON/cJSON.h cJSON/cJSON.c
+
+            # LMDB
+            lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
+            lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
+            src/cli.c src/cli.h
+    )
+endif ()
 
 find_package(PkgConfig REQUIRED)
 set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig/")
@@ -46,82 +82,149 @@ pkg_check_modules(GLIB REQUIRED glib-2.0)
 pkg_check_modules(GOBJECT REQUIRED gobject-2.0)
 pkg_check_modules(UUID REQUIRED uuid)
 
-include_directories(${LIBMAGIC_INCLUDE_DIRS})
-link_directories(${LIBMAGIC_LIBRARY_DIRS})
 add_definitions(${LIBMAGIC_CFLAGS_OTHER})
-
-link_directories(${UUID_LIBRARY_DIRS})
-include_directories(${UUID_INCLUDE_DIRS})
 add_definitions(${UUID_CFLAGS_OTHER})
-
-include_directories(${GLIB_INCLUDE_DIRS})
-link_directories(${GLIB_LIBRARY_DIRS})
 add_definitions(${GLIB_CFLAGS_OTHER})
-
-include_directories(${GOBJECT_INCLUDE_DIRS})
-link_directories(${GOBJECT_LIBRARY_DIRS})
 add_definitions(${GOBJECT_CFLAGS_OTHER})
-
-link_directories(${FFMPEG_LIBRARY_DIRS})
-include_directories(${FFMPEG_INCLUDE_DIRS})
-
-include_directories(${OPENSSL_INCLUDE_DIR})
-link_directories(${OPENSSL_CRYPTO_LIBRARY})
+add_definitions(${FREETYPE_CFLAGS_OTHER})
 
 list(REMOVE_ITEM GLIB_LIBRARIES pcre)
 list(REMOVE_ITEM GOBJECT_LIBRARIES pcre)
 list(REMOVE_ITEM UUID_LIBRARIES pcre)
 
-include_directories(${FREETYPE_INCLUDE_DIRS})
-add_definitions(${FREETYPE_CFLAGS_OTHER})
+if (WITH_SIST2)
+    target_include_directories(
+            sist2 PUBLIC
+            ${LIBMAGIC_INCLUDE_DIRS}
+            ${GOBJECT_INCLUDE_DIRS}
+            ${OPENSSL_INCLUDE_DIR}
+            ${FFMPEG_INCLUDE_DIRS}
+            ${GLIB_INCLUDE_DIRS}
+            ${FREETYPE_INCLUDE_DIRS}
+            ${UUID_INCLUDE_DIRS}
+            ${PROJECT_SOURCE_DIR}/
+            ${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
+            ${PROJECT_SOURCE_DIR}/lib/onion/src/
+            ${PROJECT_SOURCE_DIR}/lib/mupdf/include/
+    )
+    target_link_directories(
+            sist2 PUBLIC
+            ${UUID_LIBRARY_DIRS}
+            ${FFMPEG_LIBRARY_DIRS}
+    )
 
-include_directories(
-        ${PROJECT_SOURCE_DIR}/
-        ${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
-        ${PROJECT_SOURCE_DIR}/lib/onion/src/
-        ${PROJECT_SOURCE_DIR}/lib/mupdf/include/
-)
 
-target_compile_options(sist2
-        PRIVATE
-        -O3
-#        -march=native
-        -fno-stack-protector
-        -fomit-frame-pointer
-        )
+    target_compile_options(sist2
+            PRIVATE
+            -O3
+            #        -march=native
+            -fno-stack-protector
+            -fomit-frame-pointer
+            )
 
-TARGET_LINK_LIBRARIES(
-        sist2
+    TARGET_LINK_LIBRARIES(
+            sist2
 
-        ${GLIB_LIBRARIES}
-        ${GOBJECT_LIBRARIES}
-        ${UUID_LIBRARIES}
+            ${GLIB_LIBRARIES}
+            ${GOBJECT_LIBRARIES}
+            ${UUID_LIBRARIES}
 
-        # ffmpeg
-        ${PROJECT_SOURCE_DIR}/lib/libavcodec.a
-        ${PROJECT_SOURCE_DIR}/lib/libavformat.a
-        ${PROJECT_SOURCE_DIR}/lib/libavutil.a
-        ${PROJECT_SOURCE_DIR}/lib/libswscale.a
-        ${PROJECT_SOURCE_DIR}/lib/libswresample.a
-#        ${FFMPEG_LIBRARIES}
-#        swscale
+            # ffmpeg
+            ${PROJECT_SOURCE_DIR}/lib/libavcodec.a
+            ${PROJECT_SOURCE_DIR}/lib/libavformat.a
+            ${PROJECT_SOURCE_DIR}/lib/libavutil.a
+            ${PROJECT_SOURCE_DIR}/lib/libswscale.a
+            ${PROJECT_SOURCE_DIR}/lib/libswresample.a
+            #        ${FFMPEG_LIBRARIES}
+            #        swscale
 
-        # mupdf
-        ${PROJECT_SOURCE_DIR}/lib/libmupdf.a
-        ${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
+            # mupdf
+            ${PROJECT_SOURCE_DIR}/lib/libmupdf.a
+            ${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
 
-        # onion
-        ${PROJECT_SOURCE_DIR}/lib/libonion_static.a
+            # onion
+            ${PROJECT_SOURCE_DIR}/lib/libonion_static.a
 
-        pthread
-        curl
-        m
-        bz2
-        magic
-)
+            pthread
+            curl
+            m
+            bz2
+            magic
+    )
+
+endif ()
+
+if (WITH_SIST2_SCAN)
+
+    set_target_properties(
+            sist2_scan
+            PROPERTIES COMPILE_DEFINITIONS SIST_SCAN_ONLY
+    )
+    set_target_properties(
+            sist2_scan
+            PROPERTIES
+            COMPILE_DEFINITIONS SIST_SCAN_ONLY
+            LINK_FLAGS -static
+    )
+    target_include_directories(
+            sist2_scan PUBLIC
+            ${LIBMAGIC_INCLUDE_DIRS}
+            ${GOBJECT_INCLUDE_DIRS}
+            ${OPENSSL_INCLUDE_DIR}
+            ${FFMPEG_INCLUDE_DIRS}
+            ${GLIB_INCLUDE_DIRS}
+            ${UUID_INCLUDE_DIRS}
+            ${FREETYPE_INCLUDE_DIRS}
+            ${PROJECT_SOURCE_DIR}/
+            ${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
+            ${PROJECT_SOURCE_DIR}/lib/onion/src/
+            ${PROJECT_SOURCE_DIR}/lib/mupdf/include/
+    )
+    target_link_directories(
+            sist2_scan PUBLIC
+            ${UUID_LIBRARY_DIRS}
+            ${FFMPEG_LIBRARY_DIRS}
+    )
+    target_compile_options(sist2_scan
+            PRIVATE
+            -O3
+            #        -march=native
+            -fno-stack-protector
+            -fomit-frame-pointer
+            )
+    TARGET_LINK_LIBRARIES(
+            sist2_scan
+
+            ${GLIB_LIBRARIES}
+            ${GOBJECT_LIBRARIES}
+            ${UUID_LIBRARIES}
+
+            # ffmpeg
+            ${PROJECT_SOURCE_DIR}/lib/libavcodec.a
+            ${PROJECT_SOURCE_DIR}/lib/libavformat.a
+            ${PROJECT_SOURCE_DIR}/lib/libavutil.a
+            ${PROJECT_SOURCE_DIR}/lib/libswscale.a
+            ${PROJECT_SOURCE_DIR}/lib/libswresample.a
+
+            # mupdf
+            ${PROJECT_SOURCE_DIR}/lib/libmupdf.a
+            ${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
+
+            ${PROJECT_SOURCE_DIR}/lib/libbz2.a
+            ${PROJECT_SOURCE_DIR}/lib/libmagic.a
+
+            pthread
+            m
+    )
+endif ()
 
 add_custom_target(
         before_sist2
         COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/before_build.sh
 )
-add_dependencies(sist2 before_sist2)
+
+IF (WITH_SIST2)
+    add_dependencies(sist2 before_sist2)
+else ()
+    add_dependencies(sist2_scan before_sist2)
+endif ()

README.md

@@ -80,7 +80,12 @@ binaries.
         libssl-dev uuid-dev libavformat-dev libswscale-dev \
         python3 libmagic-dev libfreetype6-dev libcurl-dev \
         libbz2-dev yasm
-   
+   ```
+    *(FreeBSD)*
+    ```bash
+   pkg install cmake gcc yasm gmake bash ffmpeg e2fsprogs-uuid
+   ```
+  __ 
 2. Build
     ```bash
     git clone --recurse-submodules https://github.com/simon987/sist2

mime.csv

@@ -356,4 +356,6 @@ text/x-vcard, vcf
 application/x-innosetup,
 application/winhelp, hlp
 image/x-tga,
-application/x-wine-extension-ini,
+application/x-wine-extension-ini,
+application/x-cbz, cbz
+application/x-cbr, cbr

schema/settings.json

@@ -1,6 +1,6 @@
 {
   "index": {
-    "refresh_interval": "-1",
+    "refresh_interval": "30s",
     "codec": "best_compression"
   },
   "analysis": {
@@ -24,13 +24,15 @@
       "my_nGram": {
         "tokenizer": "my_nGram_tokenizer",
         "filter": [
-          "lowercase"
+          "lowercase",
+          "asciifolding"
         ]
       },
       "content_analyzer": {
         "tokenizer": "standard",
         "filter": [
-          "lowercase"
+          "lowercase",
+          "asciifolding"
         ]
       }
     }

scripts/before_build.sh

@@ -1,9 +1,9 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 rm -rf index.sist2/
 
 rm web/js/bundle.js 2> /dev/null
-cat `ls -v web/js/*.min.js` > web/js/bundle.js
+cat `ls web/js/*.min.js` > web/js/bundle.js
 cat web/js/{util,dom,search}.js >> web/js/bundle.js
 
 rm web/css/bundle.css 2> /dev/null

scripts/get_static_libs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 cd lib
 
 cd mupdf
@@ -36,4 +36,22 @@ make -j 4
 cd ../..
 
 mv onion/build/src/onion/libonion_static.a .
+
+#bzip2
+git clone https://github.com/enthought/bzip2-1.0.6
+cd bzip2-1.0.6
+make -j 4
+cd ..
+mv bzip2-1.0.6/libbz2.a .
+
+# magic
+git clone https://github.com/threatstack/libmagic
+cd libmagic
+./autogen.sh
+./configure --enable-static --disable-shared
+make -j 4
+cd ..
+mv libmagic/src/.libs/libmagic.a .
+
+
 cd ..

scripts/get_static_libs_freebsd.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+cd lib
+
+# mupdf
+cd mupdf
+HAVE_X11=no HAVE_GLUT=no gmake -j 4
+cd ..
+
+mv mupdf/build/release/libmupdf.a .
+mv mupdf/build/release/libmupdf-third.a .
+
+# ffmpeg
+cd ffmpeg
+./configure --disable-shared --enable-static --disable-ffmpeg --disable-ffplay \
+ --disable-ffprobe --disable-doc\
+ --disable-manpages --disable-postproc --disable-avfilter \
+ --disable-alsa --disable-lzma --disable-xlib --disable-debug\
+ --disable-vdpau --disable-vaapi --disable-sdl2 --disable-network
+gmake -j 4
+cd ..
+
+mv ffmpeg/libavcodec/libavcodec.a .
+mv ffmpeg/libavformat/libavformat.a .
+mv ffmpeg/libavutil/libavutil.a .
+mv ffmpeg/libswresample/libswresample.a .
+mv ffmpeg/libswscale/libswscale.a .
+
+#bzip2
+git clone https://github.com/enthought/bzip2-1.0.6
+cd bzip2-1.0.6
+make -j 4
+cd ..
+mv bzip2-1.0.6/libbz2.a .
+
+# magic
+git clone https://github.com/threatstack/libmagic
+cd libmagic
+./autogen.sh
+./configure --enable-static --disable-shared
+make -j 4
+cd ..
+mv libmagic/src/.libs/libmagic.a .
+
+cd ..

scripts/mime.py

@@ -17,8 +17,8 @@ major_mime = {
 
 pdf = (
     "application/pdf",
-    "application/x-cbr",
     "application/x-cbz",
+    "application/epub+zip",
     "application/vnd.ms-xpsdocument",
 )
 
@@ -71,7 +71,7 @@ with open("mime.csv") as f:
     print("#include <stdlib.h>\n")
     # Enum
     print("enum mime {")
-    for mime, ext in mimes.items():
+    for mime, ext in sorted(mimes.items()):
         print("    " + clean(mime) + "=" + mime_id(mime) + ",")
     print("};")
 

src/cli.c

@@ -17,16 +17,6 @@ scan_args_t *scan_args_create() {
     return args;
 }
 
-index_args_t *index_args_create() {
-    index_args_t *args = calloc(sizeof(index_args_t), 1);
-    return args;
-}
-
-web_args_t *web_args_create() {
-    web_args_t *args = calloc(sizeof(web_args_t), 1);
-    return args;
-}
-
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (argc < 2) {
         fprintf(stderr, "Required positional argument: PATH.\n");
@@ -100,6 +90,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     return 0;
 }
 
+#ifndef SIST_SCAN_ONLY
 int index_args_validate(index_args_t *args, int argc, const char **argv) {
 
     if (argc < 2) {
@@ -153,3 +144,14 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
     return 0;
 }
 
+index_args_t *index_args_create() {
+    index_args_t *args = calloc(sizeof(index_args_t), 1);
+    return args;
+}
+
+web_args_t *web_args_create() {
+    web_args_t *args = calloc(sizeof(web_args_t), 1);
+    return args;
+}
+#endif
+

src/cli.h

@@ -15,6 +15,10 @@ typedef struct scan_args {
     char *path;
 } scan_args_t;
 
+scan_args_t *scan_args_create();
+int scan_args_validate(scan_args_t *args, int argc, const char **argv);
+
+#ifndef SIST_SCAN_ONLY
 typedef struct index_args {
     char *es_url;
     const char *index_path;
@@ -30,12 +34,11 @@ typedef struct web_args {
     const char **indices;
 } web_args_t;
 
-scan_args_t *scan_args_create();
 index_args_t *index_args_create();
 web_args_t *web_args_create();
 
-int scan_args_validate(scan_args_t *args, int argc, const char **argv);
 int index_args_validate(index_args_t *args, int argc, const char **argv);
 int web_args_validate(web_args_t *args, int argc, const char **argv);
+#endif
 
 #endif

src/ctx.h

@@ -26,6 +26,7 @@ struct {
 } ScanCtx;
 
 
+#ifndef SIST_SCAN_ONLY
 struct {
     char *es_url;
 } IndexCtx;
@@ -35,6 +36,7 @@ struct {
     int index_count;
     struct index_t indices[16];
 } WebCtx;
+#endif
 
 
 #endif

src/index/elastic.c

@@ -70,7 +70,8 @@ void elastic_flush() {
 
     while (line != NULL) {
         char action_str[512];
-        snprintf(action_str, 512, "{\"index\":{\"_id\": \"%s\", \"_type\":\"_doc\"}}", line->uuid_str);
+        snprintf(action_str, 512,
+                "{\"index\":{\"_id\":\"%s\", \"_type\":\"_doc\", \"_index\":\"sist2\"}}\n", line->uuid_str);
         size_t action_str_len = strlen(action_str);
 
         size_t line_len = strlen(line->line);
@@ -98,6 +99,14 @@ void elastic_flush() {
     snprintf(bulk_url, 4096, "%s/sist2/_bulk", Indexer->es_url);
     response_t *r = web_post(bulk_url, buf, "Content-Type: application/x-ndjson");
     printf("Indexed %3d documents (%zukB) <%d>\n", count, buf_cur / 1024, r->status_code);
+
+    cJSON *ret_json = cJSON_Parse(r->body);
+    if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) {
+        fprintf(stderr, "%s\n", r->body);
+    }
+
+    cJSON_Delete(ret_json);
+
     free_response(r);
 }
 
@@ -187,7 +196,7 @@ void elastic_init(int force_reset) {
         printf("Update settings <%d>\n", r->status_code);
         free_response(r);
 
-        snprintf(url, 4096, "%s/sist2/_mappings/_doc", IndexCtx.es_url);
+        snprintf(url, 4096, "%s/sist2/_mappings/_doc?include_type_name=true", IndexCtx.es_url);
         r = web_put(url, mappings_json, "Content-Type: application/json");
         printf("Update mappings <%d>\n", r->status_code);
         free_response(r);
@@ -201,8 +210,11 @@ void elastic_init(int force_reset) {
 
 cJSON *elastic_get_document(const char *uuid_str) {
     char url[4096];
-    snprintf(url, 4096, "%s/sist2/_source/%s", WebCtx.es_url, uuid_str);
+    snprintf(url, 4096, "%s/sist2/_doc/%s", WebCtx.es_url, uuid_str);
 
     response_t *r = web_get(url);
-    return cJSON_Parse(r->body);
+    if (r->status_code == 200) {
+        return cJSON_Parse(r->body);
+    }
+    return NULL;
 }

src/io/serialize.c

@@ -115,8 +115,8 @@ void write_document(document_t *doc) {
 
     if (IndexFd == -1) {
         char dstfile[PATH_MAX];
-        pid_t tid = syscall(SYS_gettid);
-        snprintf(dstfile, PATH_MAX, "%s_index_%d", ScanCtx.index.path, tid);
+        pthread_t self = pthread_self();
+        snprintf(dstfile, PATH_MAX, "%s_index_%lu", ScanCtx.index.path, self);
         IndexFd = open(dstfile, O_CREAT | O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR);
 
         if (IndexFd == -1) {
@@ -232,7 +232,9 @@ void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func
                 case MetaTitle: {
                     buf.cur = 0;
                     while ((c = getc(file)) != 0) {
-                        dyn_buffer_write_char(&buf, (char) c);
+                        if (!(SHOULD_IGNORE_CHAR(c)) || c == ' ') {
+                            dyn_buffer_write_char(&buf, (char) c);
+                        }
                     }
                     dyn_buffer_write_char(&buf, '\0');
                     cJSON_AddStringToObject(document, get_meta_key_text(key), buf.buf);

src/main.c

@@ -1,11 +1,16 @@
 #include "sist.h"
 #include "ctx.h"
 
+#ifndef SIST_SCAN_ONLY
 #define DESCRIPTION "Lightning-fast file system indexer and search tool."
+#else
+#define DESCRIPTION "Lightning-fast file system indexer and search tool. (SCAN ONLY)"
+#endif
+
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "1.0.1";
+static const char *const Version = "1.0.9";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
@@ -14,7 +19,9 @@ static const char *const usage[] = {
 };
 
 void global_init() {
+    #ifndef SIST_SCAN_ONLY
     curl_global_init(CURL_GLOBAL_NOTHING);
+    #endif
     av_log_set_level(AV_LOG_QUIET);
 }
 
@@ -23,7 +30,7 @@ void init_dir(const char *dirpath) {
     snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
 
     uuid_t uuid;
-    uuid_generate_time_safe(uuid);
+    uuid_generate(uuid);
     uuid_unparse(uuid, ScanCtx.index.desc.uuid);
     time(&ScanCtx.index.desc.timestamp);
     strcpy(ScanCtx.index.desc.version, Version);
@@ -116,6 +123,7 @@ void sist2_scan(scan_args_t *args) {
     store_destroy(ScanCtx.index.store);
 }
 
+#ifndef SIST_SCAN_ONLY
 void sist2_index(index_args_t *args) {
 
     IndexCtx.es_url = args->es_url;
@@ -186,6 +194,7 @@ void sist2_web(web_args_t *args) {
 
     serve(args->bind, args->port);
 }
+#endif
 
 
 int main(int argc, const char *argv[]) {
@@ -193,8 +202,10 @@ int main(int argc, const char *argv[]) {
     global_init();
 
     scan_args_t *scan_args = scan_args_create();
+    #ifndef SIST_SCAN_ONLY
     index_args_t *index_args = index_args_create();
     web_args_t *web_args = web_args_create();
+    #endif
 
     char * common_es_url = NULL;
 
@@ -214,6 +225,7 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."),
             OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: (name of the directory)"),
 
+            #ifndef SIST_SCAN_ONLY
             OPT_GROUP("Index options"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
             OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
@@ -224,6 +236,7 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
             OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"),
             OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"),
+            #endif
 
             OPT_END(),
     };
@@ -233,8 +246,10 @@ int main(int argc, const char *argv[]) {
     argparse_describe(&argparse, DESCRIPTION, EPILOG);
     argc = argparse_parse(&argparse, argc, argv);
 
+    #ifndef SIST_SCAN_ONLY
     web_args->es_url = common_es_url;
     index_args->es_url = common_es_url;
+    #endif
 
     if (argc == 0) {
         argparse_usage(&argparse);
@@ -247,7 +262,10 @@ int main(int argc, const char *argv[]) {
         }
         sist2_scan(scan_args);
 
-    } else if (strcmp(argv[0], "index") == 0) {
+    }
+
+    #ifndef SIST_SCAN_ONLY
+    else if (strcmp(argv[0], "index") == 0) {
 
         int err = index_args_validate(index_args, argc, argv);
         if (err != 0) {
@@ -263,7 +281,9 @@ int main(int argc, const char *argv[]) {
         }
         sist2_web(web_args);
 
-    } else {
+    }
+    #endif
+    else {
         fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
         argparse_usage(&argparse);
         return 1;

src/parsing/media.c

@@ -151,6 +151,7 @@ void append_audio_meta(AVFormatContext *pFormatCtx, document_t *doc) {
 void parse_media(const char *filepath, document_t *doc) {
 
     int video_stream = -1;
+    int audio_stream = -1;
 
     AVFormatContext *pFormatCtx = avformat_alloc_context();
     if (pFormatCtx == NULL) {
@@ -169,31 +170,35 @@ void parse_media(const char *filepath, document_t *doc) {
         AVStream *stream = pFormatCtx->streams[i];
 
         if (stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
-            meta_line_t *meta_audio = malloc(sizeof(meta_line_t));
-            meta_audio->key = MetaMediaAudioCodec;
-            meta_audio->intval = stream->codecpar->codec_id;
-            APPEND_META(doc, meta_audio)
-
-            append_audio_meta(pFormatCtx, doc);
+            if (audio_stream == -1) {
+                meta_line_t *meta_audio = malloc(sizeof(meta_line_t));
+                meta_audio->key = MetaMediaAudioCodec;
+                meta_audio->intval = stream->codecpar->codec_id;
+                APPEND_META(doc, meta_audio)
 
+                append_audio_meta(pFormatCtx, doc);
+                audio_stream = i;
+            }
         } else if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
 
-            meta_line_t *meta_vid = malloc(sizeof(meta_line_t));
-            meta_vid->key = MetaMediaVideoCodec;
-            meta_vid->intval = stream->codecpar->codec_id;
-            APPEND_META(doc, meta_vid)
+            if (video_stream == -1) {
+                meta_line_t *meta_vid = malloc(sizeof(meta_line_t));
+                meta_vid->key = MetaMediaVideoCodec;
+                meta_vid->intval = stream->codecpar->codec_id;
+                APPEND_META(doc, meta_vid)
 
-            meta_line_t *meta_w = malloc(sizeof(meta_line_t));
-            meta_w->key = MetaWidth;
-            meta_w->intval = stream->codecpar->width;
-            APPEND_META(doc, meta_w)
+                meta_line_t *meta_w = malloc(sizeof(meta_line_t));
+                meta_w->key = MetaWidth;
+                meta_w->intval = stream->codecpar->width;
+                APPEND_META(doc, meta_w)
 
-            meta_line_t *meta_h = malloc(sizeof(meta_line_t));
-            meta_h->key = MetaHeight;
-            meta_h->intval = stream->codecpar->height;
-            APPEND_META(doc, meta_h)
+                meta_line_t *meta_h = malloc(sizeof(meta_line_t));
+                meta_h->key = MetaHeight;
+                meta_h->intval = stream->codecpar->height;
+                APPEND_META(doc, meta_h)
 
-            video_stream = i;
+                video_stream = i;
+            }
         }
     }
 

src/parsing/mime_generated.c

@@ -6,364 +6,366 @@
 #include <stdlib.h>
 
 enum mime {
-    application_arj=655361,
-    application_base64=655362,
-    application_binhex=655363,
-    application_book=655364,
-    application_CDFV2=655365,
-    application_clariscad=655366,
-    application_commonground=655367,
-    application_drafting=655368,
-    application_freeloader=655369,
-    application_futuresplash=655370,
-    application_groupwise=655371,
-    application_gzip=655372,
-    application_hta=655373,
-    application_i_deas=655374,
-    application_iges=655375,
-    application_inf=655376,
-    application_java_archive=655377,
-    application_java=655378,
-    application_javascript=655379,
-    application_x_archive=655380,
-    application_json=655381,
-    application_marc=655382,
-    application_mbedlet=655383,
-    application_mime=655384,
-    application_mspowerpoint=655385,
-    application_msword=655386,
-    application_netmc=655387,
-    application_octet_stream=655388,
-    application_oda=655389,
-    application_pdf=655390 | 0x40000000,
-    application_pgp_signature=655391,
-    application_pkcs7_signature=655392,
-    application_pkix_cert=655393,
-    application_postscript=655394,
-    application_pro_eng=655395,
-    application_ringing_tones=655396,
-    application_smil=655397,
-    application_solids=655398,
-    application_sounder=655399,
-    application_step=655400,
-    application_streamingmedia=655401,
-    application_vda=655402,
-    application_vnd_fdf=655403,
-    application_vnd_font_fontforge_sfd=655404,
-    application_vnd_hp_hpgl=655405,
-    application_vnd_ms_excel=655406,
-    application_vnd_ms_fontobject=655407,
-    application_vnd_ms_opentype=655408 | 0x20000000,
-    application_vnd_ms_pki_certstore=655409,
-    application_vnd_ms_pki_pko=655410,
-    application_vnd_ms_pki_seccat=655411,
-    application_vnd_ms_powerpoint=655412,
-    application_vnd_ms_project=655413,
-    application_vnd_oasis_opendocument_base=655414,
-    application_vnd_oasis_opendocument_formula=655415,
-    application_vnd_oasis_opendocument_graphics=655416,
-    application_vnd_oasis_opendocument_text=655417,
-    application_vnd_wap_wmlc=655418,
-    application_vnd_wap_wmlscriptc=655419,
-    application_vnd_xara=655420,
-    application_vocaltec_media_desc=655421,
-    application_vocaltec_media_file=655422,
-    application_wordperfect6_0=655423,
-    application_wordperfect6_1=655424,
-    application_wordperfect=655425,
-    application_x_123=655426,
-    application_x_aim=655427,
-    application_x_authorware_bin=655428,
-    application_x_authorware_map=655429,
-    application_x_authorware_seg=655430,
-    application_x_bcpio=655431,
-    application_x_bittorrent=655432,
-    application_x_bsh=655433,
-    application_x_bytecode_python=655434,
-    application_x_bzip2=655435,
-    application_x_bzip=655436,
-    application_x_cdlink=655437,
-    application_x_chat=655438,
-    application_x_cocoa=655439,
-    application_x_conference=655440,
-    application_x_cpio=655441,
-    application_x_dbf=655442,
-    application_x_dbt=655443,
-    application_x_deepv=655444,
-    application_x_director=655445,
-    application_x_dosexec=655446,
-    application_x_dvi=655447,
-    application_x_elc=655448,
-    application_x_envoy=655449,
-    application_x_esrehber=655450,
-    application_x_excel=655451,
-    application_x_executable=655452,
-    application_x_font_sfn=655453 | 0x20000000,
-    application_x_font_ttf=655454 | 0x20000000,
-    application_x_freelance=655455,
-    application_x_git=655456,
-    application_x_gsp=655457,
-    application_x_gss=655458,
-    application_x_gtar=655459,
-    application_x_gzip=655460,
-    application_x_hdf=655461,
-    application_x_helpfile=655462,
-    application_x_httpd_imap=655463,
-    application_x_ima=655464,
-    application_x_internett_signup=655465,
-    application_x_inventor=655466,
-    application_x_ip2=655467,
-    application_x_java_applet=655468,
-    application_x_java_commerce=655469,
-    application_x_java_image=655470,
-    application_x_java_keystore=655471,
-    application_x_koan=655472,
-    application_x_latex=655473,
-    application_x_livescreen=655474,
-    application_x_lotus=655475,
-    application_x_lzh=655476,
-    application_x_lzx=655477,
-    application_x_mach_binary=655478,
-    application_x_mach_executable=655479,
-    application_x_magic_cap_package_1_0=655480,
-    application_x_mathcad=655481,
-    application_x_meme=655482,
-    application_x_midi=655483,
-    application_x_mif=655484,
-    application_x_mix_transfer=655485,
-    application_xml=655486,
-    application_x_ms_pdb=655487,
-    application_x_navi_animation=655488,
-    application_x_navidoc=655489,
-    application_x_navimap=655490,
-    application_x_navistyle=655491,
-    application_x_netcdf=655492,
-    application_x_newton_compatible_pkg=655493,
-    application_x_object=655494,
-    application_x_omcdatamaker=655495,
-    application_x_omc=655496,
-    application_x_omcregerator=655497,
-    application_x_pagemaker=655498,
-    application_x_pcl=655499,
-    application_x_pixclscript=655500,
-    application_x_pkcs7_certreqresp=655501,
-    application_x_pkcs7_signature=655502,
-    application_x_project=655503,
-    application_x_qpro=655504,
-    application_x_sdp=655505,
-    application_x_sea=655506,
-    application_x_seelogo=655507,
-    application_x_setupscript=655508,
-    application_x_sharedlib=655509,
-    application_x_shar=655510,
-    application_x_shockwave_flash=655511,
-    application_x_sprite=655512,
-    application_x_sqlite3=655513,
-    application_x_sv4cpio=655514,
-    application_x_sv4crc=655515,
-    application_x_tar=655516,
-    application_x_tbook=655517,
-    application_x_texinfo=655518,
-    application_x_tex_tfm=655519,
-    application_x_ustar=655520,
-    application_x_visio=655521,
-    application_x_vnd_audioexplosion_mzz=655522,
-    application_x_vnd_ls_xpix=655523,
-    application_x_vrml=655524,
-    application_x_wais_source=655525,
-    application_x_wintalk=655526,
-    application_x_world=655527,
-    application_x_wri=655528,
-    application_x_x509_ca_cert=655529,
-    application_x_xz=655530,
-    application_zip=655531,
-    audio_it=458924,
-    audio_make=458925,
-    audio_midi=458926,
-    audio_mid=458927,
-    audio_mpeg=458928,
-    audio_ogg=458929,
-    audio_s3m=458930,
-    audio_tsp_audio=458931,
-    audio_tsplayer=458932,
-    audio_vnd_qcelp=458933,
-    audio_voxware=458934,
-    audio_x_gsm=458935,
-    audio_x_jam=458936,
-    audio_x_liveaudio=458937,
-    audio_x_m4a=458938,
-    audio_x_midi=458939,
-    audio_x_mod=458940,
-    audio_x_mp4a_latm=458941,
-    audio_x_mpeg_3=458942,
-    audio_x_mpequrl=458943,
-    audio_xm=458944,
-    audio_x_nspaudio=458945,
-    audio_x_pn_realaudio=458946,
-    audio_x_psid=458947,
-    audio_x_realaudio=458948,
-    audio_x_twinvq_plugin=458949,
-    audio_x_twinvq=458950,
-    audio_x_voc=458951,
-    audio_x_wav=458952,
-    font_otf=327881 | 0x20000000,
-    font_sfnt=327882 | 0x20000000,
-    image_cmu_raster=524491,
-    image_fif=524492,
-    image_florian=524493,
-    image_g3fax=524494,
-    image_gif=524495,
-    image_ief=524496,
-    image_jpeg=524497,
-    image_jutvision=524498,
-    image_naplps=524499,
-    image_pict=524500,
-    image_png=524501,
-    image_svg=524502 | 0x80000000,
-    image_svg_xml=524503 | 0x80000000,
-    image_vnd_fpx=524504,
-    image_vnd_microsoft_icon=524505,
-    image_vnd_rn_realflash=524506,
-    image_vnd_rn_realpix=524507,
-    image_vnd_wap_wbmp=524508,
-    image_vnd_xiff=524509,
-    image_webp=524510,
-    image_x_cmu_raster=524511,
-    image_x_dwg=524512,
-    image_x_eps=524513,
-    image_x_icns=524514,
-    image_x_icon=524515 | 0x80000000,
-    image_x_jg=524516,
-    image_x_jps=524517,
-    image_x_ms_bmp=524518,
-    image_x_niff=524519,
-    image_x_pcx=524520,
-    image_x_pict=524521,
-    image_x_portable_bitmap=524522,
-    image_x_portable_graymap=524523,
-    image_x_portable_pixmap=524524,
-    image_x_quicktime=524525,
-    image_x_rgb=524526,
-    image_x_tiff=524527,
-    image_tiff=524528,
-    image_x_xcf=524529 | 0x80000000,
-    image_x_xpixmap=524530 | 0x80000000,
-    message_rfc822=196851,
-    model_vnd_dwf=65780,
-    model_vrml=65781,
-    model_x_pov=65782,
-    text_asp=590071,
-    text_css=590072,
-    text_x_sass=590073,
-    text_x_scss=590074,
-    text_html=590075,
-    text_javascript=590076,
-    text_mcf=590077,
-    text_pascal=590078,
-    text_plain=590079,
-    text_richtext=590080,
-    text_scriplet=590081,
-    text_x_awk=590082,
-    video_x_jng=393475 | 0x80000000,
-    video_x_mng=393476,
-    image_x_cur=524549,
-    image_x_xwindowdump=524550,
-    image_vnd_adobe_photoshop=524551 | 0x80000000,
-    text_tab_separated_values=590088,
-    text_troff=590089,
-    text_uri_list=590090,
-    text_vnd_abc=590091,
-    text_vnd_fmi_flexstor=590092,
-    text_vnd_wap_wmlscript=590093,
-    text_vnd_wap_wml=590094,
-    text_webviewhtml=590095,
-    text_x_Algol68=590096,
-    text_x_asm=590097,
-    text_x_audiosoft_intra=590098,
-    text_x_bcpl=590099,
-    text_x_c=590100,
-    text_x_c__=590101,
-    text_x_component=590102,
-    text_x_diff=590103,
-    text_x_fortran=590104,
-    text_x_java=590105,
-    text_x_la_asf=590106,
-    text_x_lisp=590107,
-    text_x_m4=590108,
-    text_x_makefile=590109,
-    text_xml=590110,
-    text_x_m=590111,
-    text_x_msdos_batch=590112,
-    text_x_pascal=590113,
-    text_x_perl=590114,
-    text_x_php=590115,
-    text_x_python=590116,
-    text_x_ruby=590117,
-    text_x_server_parsed_html=590118,
-    text_x_setext=590119,
-    text_x_sgml=590120,
-    text_x_shellscript=590121,
-    text_x_speech=590122,
-    text_x_tex=590123,
-    text_x_uil=590124,
-    text_x_uuencode=590125,
-    text_x_vcalendar=590126,
-    video_animaflex=393519,
-    video_avi=393520,
-    video_avs_video=393521,
-    video_mp4=393522,
-    video_mpeg=393523,
-    video_quicktime=393524,
-    video_vdo=393525,
-    video_vivo=393526,
-    video_vnd_rn_realvideo=393527,
-    video_vosaic=393528,
-    video_webm=393529,
-    video_x_amt_demorun=393530,
-    video_x_amt_showrun=393531,
-    video_x_atomic3d_feature=393532,
-    video_x_dl=393533,
-    video_x_dv=393534,
-    video_x_fli=393535,
-    video_x_isvideo=393536,
-    video_x_motion_jpeg=393537,
-    video_x_ms_asf=393538,
-    video_x_qtc=393539,
-    video_x_sgi_movie=393540,
-    application_x_7z_compressed=655685,
-    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655686,
-    text_x_po=590151,
-    application_x_rpm=655688,
-    application_x_debian_package=655689,
-    application_vnd_iccprofile=655690,
-    application_dicom=655691,
-    image_x_exr=524620,
-    video_x_matroska=393549,
+    application_CDFV2=655361,
+    application_CDFV2_corrupt=655362,
+    application_arj=655363,
+    application_base64=655364,
+    application_binhex=655365,
+    application_book=655366,
+    application_clariscad=655367,
+    application_commonground=655368,
+    application_dicom=655369,
+    application_drafting=655370,
+    application_epub_zip=655371 | 0x40000000,
+    application_freeloader=655372,
+    application_futuresplash=655373,
+    application_groupwise=655374,
+    application_gzip=655375,
+    application_hta=655376,
+    application_i_deas=655377,
+    application_iges=655378,
+    application_inf=655379,
+    application_java=655380,
+    application_java_archive=655381,
+    application_javascript=655382,
+    application_json=655383,
+    application_marc=655384,
+    application_mbedlet=655385,
+    application_mime=655386,
+    application_mspowerpoint=655387,
+    application_msword=655388,
+    application_netmc=655389,
+    application_octet_stream=655390,
+    application_oda=655391,
+    application_ogg=655392,
+    application_pdf=655393 | 0x40000000,
+    application_pgp_signature=655394,
+    application_pkcs7_signature=655395,
+    application_pkix_cert=655396,
+    application_postscript=655397,
+    application_pro_eng=655398,
+    application_ringing_tones=655399,
+    application_smil=655400,
+    application_solids=655401,
+    application_sounder=655402,
+    application_step=655403,
+    application_streamingmedia=655404,
+    application_vda=655405,
+    application_vnd_fdf=655406,
+    application_vnd_font_fontforge_sfd=655407,
+    application_vnd_hp_hpgl=655408,
+    application_vnd_iccprofile=655409,
+    application_vnd_ms_cab_compressed=655410,
+    application_vnd_ms_excel=655411,
+    application_vnd_ms_fontobject=655412,
+    application_vnd_ms_opentype=655413 | 0x20000000,
+    application_vnd_ms_pki_certstore=655414,
+    application_vnd_ms_pki_pko=655415,
+    application_vnd_ms_pki_seccat=655416,
+    application_vnd_ms_powerpoint=655417,
+    application_vnd_ms_project=655418,
+    application_vnd_oasis_opendocument_base=655419,
+    application_vnd_oasis_opendocument_formula=655420,
+    application_vnd_oasis_opendocument_graphics=655421,
+    application_vnd_oasis_opendocument_text=655422,
+    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655423,
+    application_vnd_openxmlformats_officedocument_wordprocessingml_document=655424,
+    application_vnd_wap_wmlc=655425,
+    application_vnd_wap_wmlscriptc=655426,
+    application_vnd_xara=655427,
+    application_vocaltec_media_desc=655428,
+    application_vocaltec_media_file=655429,
+    application_winhelp=655430,
+    application_wordperfect=655431,
+    application_wordperfect6_0=655432,
+    application_wordperfect6_1=655433,
+    application_x_123=655434,
+    application_x_7z_compressed=655435,
+    application_x_aim=655436,
+    application_x_archive=655437,
+    application_x_authorware_bin=655438,
+    application_x_authorware_map=655439,
+    application_x_authorware_seg=655440,
+    application_x_bcpio=655441,
+    application_x_bittorrent=655442,
+    application_x_bsh=655443,
+    application_x_bytecode_python=655444,
+    application_x_bzip=655445,
+    application_x_bzip2=655446,
+    application_x_cbr=655447,
+    application_x_cbz=655448 | 0x40000000,
+    application_x_cdlink=655449,
+    application_x_chat=655450,
+    application_x_cocoa=655451,
+    application_x_conference=655452,
+    application_x_cpio=655453,
+    application_x_dbf=655454,
+    application_x_dbt=655455,
+    application_x_debian_package=655456,
+    application_x_deepv=655457,
+    application_x_director=655458,
+    application_x_dosexec=655459,
+    application_x_dvi=655460,
+    application_x_elc=655461,
     application_x_empty=1,
-    model_vnd_gdl=65871,
-    model_vnd_gs_gdl=65872,
-    font_woff=328017 | 0x20000000,
-    font_woff2=328018 | 0x20000000,
-    application_epub_zip=655699,
-    application_x_mobipocket_ebook=655700,
-    audio_x_flac=459093,
-    application_x_rar=655702,
-    video_x_msvideo=393559,
-    video_x_flv=393560,
-    application_x_kdelnk=655705,
-    text_x_tcl=590170,
-    application_ogg=655707,
-    application_vnd_openxmlformats_officedocument_spreadsheetml_sheet=655708,
-    application_vnd_ms_cab_compressed=655709,
-    audio_mp4=459102,
-    image_vnd_djvu=524639 | 0x80000000,
-    application_x_ms_reader=655712,
-    application_CDFV2_corrupt=655713,
-    text_x_vcard=590178,
-    application_x_innosetup=655715,
-    application_winhelp=655716,
-    image_x_tga=524645,
-    application_x_wine_extension_ini=655718,
+    application_x_envoy=655463,
+    application_x_esrehber=655464,
+    application_x_excel=655465,
+    application_x_executable=655466,
+    application_x_font_sfn=655467 | 0x20000000,
+    application_x_font_ttf=655468 | 0x20000000,
+    application_x_freelance=655469,
+    application_x_git=655470,
+    application_x_gsp=655471,
+    application_x_gss=655472,
+    application_x_gtar=655473,
+    application_x_gzip=655474,
+    application_x_hdf=655475,
+    application_x_helpfile=655476,
+    application_x_httpd_imap=655477,
+    application_x_ima=655478,
+    application_x_innosetup=655479,
+    application_x_internett_signup=655480,
+    application_x_inventor=655481,
+    application_x_ip2=655482,
+    application_x_java_applet=655483,
+    application_x_java_commerce=655484,
+    application_x_java_image=655485,
+    application_x_java_keystore=655486,
+    application_x_kdelnk=655487,
+    application_x_koan=655488,
+    application_x_latex=655489,
+    application_x_livescreen=655490,
+    application_x_lotus=655491,
+    application_x_lzh=655492,
+    application_x_lzx=655493,
+    application_x_mach_binary=655494,
+    application_x_mach_executable=655495,
+    application_x_magic_cap_package_1_0=655496,
+    application_x_mathcad=655497,
+    application_x_meme=655498,
+    application_x_midi=655499,
+    application_x_mif=655500,
+    application_x_mix_transfer=655501,
+    application_x_mobipocket_ebook=655502,
+    application_x_ms_pdb=655503,
+    application_x_ms_reader=655504,
+    application_x_navi_animation=655505,
+    application_x_navidoc=655506,
+    application_x_navimap=655507,
+    application_x_navistyle=655508,
+    application_x_netcdf=655509,
+    application_x_newton_compatible_pkg=655510,
+    application_x_object=655511,
+    application_x_omc=655512,
+    application_x_omcdatamaker=655513,
+    application_x_omcregerator=655514,
+    application_x_pagemaker=655515,
+    application_x_pcl=655516,
+    application_x_pixclscript=655517,
+    application_x_pkcs7_certreqresp=655518,
+    application_x_pkcs7_signature=655519,
+    application_x_project=655520,
+    application_x_qpro=655521,
+    application_x_rar=655522,
+    application_x_rpm=655523,
+    application_x_sdp=655524,
+    application_x_sea=655525,
+    application_x_seelogo=655526,
+    application_x_setupscript=655527,
+    application_x_shar=655528,
+    application_x_sharedlib=655529,
+    application_x_shockwave_flash=655530,
+    application_x_sprite=655531,
+    application_x_sqlite3=655532,
+    application_x_sv4cpio=655533,
+    application_x_sv4crc=655534,
+    application_x_tar=655535,
+    application_x_tbook=655536,
+    application_x_tex_tfm=655537,
+    application_x_texinfo=655538,
+    application_x_ustar=655539,
+    application_x_visio=655540,
+    application_x_vnd_audioexplosion_mzz=655541,
+    application_x_vnd_ls_xpix=655542,
+    application_x_vrml=655543,
+    application_x_wais_source=655544,
+    application_x_wine_extension_ini=655545,
+    application_x_wintalk=655546,
+    application_x_world=655547,
+    application_x_wri=655548,
+    application_x_x509_ca_cert=655549,
+    application_x_xz=655550,
+    application_xml=655551,
+    application_zip=655552,
+    audio_it=458945,
+    audio_make=458946,
+    audio_mid=458947,
+    audio_midi=458948,
+    audio_mp4=458949,
+    audio_mpeg=458950,
+    audio_ogg=458951,
+    audio_s3m=458952,
+    audio_tsp_audio=458953,
+    audio_tsplayer=458954,
+    audio_vnd_qcelp=458955,
+    audio_voxware=458956,
+    audio_x_flac=458957,
+    audio_x_gsm=458958,
+    audio_x_jam=458959,
+    audio_x_liveaudio=458960,
+    audio_x_m4a=458961,
+    audio_x_midi=458962,
+    audio_x_mod=458963,
+    audio_x_mp4a_latm=458964,
+    audio_x_mpeg_3=458965,
+    audio_x_mpequrl=458966,
+    audio_x_nspaudio=458967,
+    audio_x_pn_realaudio=458968,
+    audio_x_psid=458969,
+    audio_x_realaudio=458970,
+    audio_x_twinvq=458971,
+    audio_x_twinvq_plugin=458972,
+    audio_x_voc=458973,
+    audio_x_wav=458974,
+    audio_xm=458975,
+    font_otf=327904 | 0x20000000,
+    font_sfnt=327905 | 0x20000000,
+    font_woff=327906 | 0x20000000,
+    font_woff2=327907 | 0x20000000,
+    image_cmu_raster=524516,
+    image_fif=524517,
+    image_florian=524518,
+    image_g3fax=524519,
+    image_gif=524520,
+    image_ief=524521,
+    image_jpeg=524522,
+    image_jutvision=524523,
+    image_naplps=524524,
+    image_pict=524525,
+    image_png=524526,
+    image_svg=524527 | 0x80000000,
+    image_svg_xml=524528 | 0x80000000,
+    image_tiff=524529,
+    image_vnd_adobe_photoshop=524530 | 0x80000000,
+    image_vnd_djvu=524531 | 0x80000000,
+    image_vnd_fpx=524532,
+    image_vnd_microsoft_icon=524533,
+    image_vnd_rn_realflash=524534,
+    image_vnd_rn_realpix=524535,
+    image_vnd_wap_wbmp=524536,
+    image_vnd_xiff=524537,
+    image_webp=524538,
+    image_x_cmu_raster=524539,
+    image_x_cur=524540,
+    image_x_dwg=524541,
+    image_x_eps=524542,
+    image_x_exr=524543,
+    image_x_icns=524544,
+    image_x_icon=524545 | 0x80000000,
+    image_x_jg=524546,
+    image_x_jps=524547,
+    image_x_ms_bmp=524548,
+    image_x_niff=524549,
+    image_x_pcx=524550,
+    image_x_pict=524551,
+    image_x_portable_bitmap=524552,
+    image_x_portable_graymap=524553,
+    image_x_portable_pixmap=524554,
+    image_x_quicktime=524555,
+    image_x_rgb=524556,
+    image_x_tga=524557,
+    image_x_tiff=524558,
+    image_x_xcf=524559 | 0x80000000,
+    image_x_xpixmap=524560 | 0x80000000,
+    image_x_xwindowdump=524561,
+    message_rfc822=196882,
+    model_vnd_dwf=65811,
+    model_vnd_gdl=65812,
+    model_vnd_gs_gdl=65813,
+    model_vrml=65814,
+    model_x_pov=65815,
+    text_asp=590104,
+    text_css=590105,
+    text_html=590106,
+    text_javascript=590107,
+    text_mcf=590108,
+    text_pascal=590109,
+    text_plain=590110,
+    text_richtext=590111,
+    text_scriplet=590112,
+    text_tab_separated_values=590113,
+    text_troff=590114,
+    text_uri_list=590115,
+    text_vnd_abc=590116,
+    text_vnd_fmi_flexstor=590117,
+    text_vnd_wap_wml=590118,
+    text_vnd_wap_wmlscript=590119,
+    text_webviewhtml=590120,
+    text_x_Algol68=590121,
+    text_x_asm=590122,
+    text_x_audiosoft_intra=590123,
+    text_x_awk=590124,
+    text_x_bcpl=590125,
+    text_x_c=590126,
+    text_x_c__=590127,
+    text_x_component=590128,
+    text_x_diff=590129,
+    text_x_fortran=590130,
+    text_x_java=590131,
+    text_x_la_asf=590132,
+    text_x_lisp=590133,
+    text_x_m=590134,
+    text_x_m4=590135,
+    text_x_makefile=590136,
+    text_x_msdos_batch=590137,
+    text_x_pascal=590138,
+    text_x_perl=590139,
+    text_x_php=590140,
+    text_x_po=590141,
+    text_x_python=590142,
+    text_x_ruby=590143,
+    text_x_sass=590144,
+    text_x_scss=590145,
+    text_x_server_parsed_html=590146,
+    text_x_setext=590147,
+    text_x_sgml=590148,
+    text_x_shellscript=590149,
+    text_x_speech=590150,
+    text_x_tcl=590151,
+    text_x_tex=590152,
+    text_x_uil=590153,
+    text_x_uuencode=590154,
+    text_x_vcalendar=590155,
+    text_x_vcard=590156,
+    text_xml=590157,
+    video_animaflex=393550,
+    video_avi=393551,
+    video_avs_video=393552,
+    video_mp4=393553,
+    video_mpeg=393554,
+    video_quicktime=393555,
+    video_vdo=393556,
+    video_vivo=393557,
+    video_vnd_rn_realvideo=393558,
+    video_vosaic=393559,
+    video_webm=393560,
+    video_x_amt_demorun=393561,
+    video_x_amt_showrun=393562,
+    video_x_atomic3d_feature=393563,
+    video_x_dl=393564,
+    video_x_dv=393565,
+    video_x_fli=393566,
+    video_x_flv=393567,
+    video_x_isvideo=393568,
+    video_x_jng=393569 | 0x80000000,
+    video_x_matroska=393570,
+    video_x_mng=393571,
+    video_x_motion_jpeg=393572,
+    video_x_ms_asf=393573,
+    video_x_msvideo=393574,
+    video_x_qtc=393575,
+    video_x_sgi_movie=393576,
 };
 char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
 case application_arj: return "application/arj";
@@ -724,6 +726,8 @@ case application_x_innosetup: return "application/x-innosetup";
 case application_winhelp: return "application/winhelp";
 case image_x_tga: return "image/x-tga";
 case application_x_wine_extension_ini: return "application/x-wine-extension-ini";
+case application_x_cbz: return "application/x-cbz";
+case application_x_cbr: return "application/x-cbr";
 default: return NULL;}}
 GHashTable *mime_get_ext_table() {GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(ext_table, "arj", (gpointer)application_arj);
@@ -1201,6 +1205,8 @@ g_hash_table_insert(ext_table, "djvu", (gpointer)image_vnd_djvu);
 g_hash_table_insert(ext_table, "lit", (gpointer)application_x_ms_reader);
 g_hash_table_insert(ext_table, "vcf", (gpointer)text_x_vcard);
 g_hash_table_insert(ext_table, "hlp", (gpointer)application_winhelp);
+g_hash_table_insert(ext_table, "cbz", (gpointer)application_x_cbz);
+g_hash_table_insert(ext_table, "cbr", (gpointer)application_x_cbr);
 return ext_table;}
 GHashTable *mime_get_mime_table() {GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(mime_table, "application/arj", (gpointer)application_arj);
@@ -1561,5 +1567,7 @@ g_hash_table_insert(mime_table, "application/x-innosetup", (gpointer)application
 g_hash_table_insert(mime_table, "application/winhelp", (gpointer)application_winhelp);
 g_hash_table_insert(mime_table, "image/x-tga", (gpointer)image_x_tga);
 g_hash_table_insert(mime_table, "application/x-wine-extension-ini", (gpointer)application_x_wine_extension_ini);
+g_hash_table_insert(mime_table, "application/x-cbz", (gpointer)application_x_cbz);
+g_hash_table_insert(mime_table, "application/x-cbr", (gpointer)application_x_cbr);
 return mime_table;}
 #endif

src/parsing/parse.c

@@ -57,7 +57,7 @@ void parse(void *arg) {
     doc.ino = job->info.st_ino;
     doc.mtime = job->info.st_mtim.tv_sec;
 
-    uuid_generate_time_safe(doc.uuid);
+    uuid_generate(doc.uuid);
     char *buf[PARSE_BUF_SIZE];
 
     if (job->info.st_size == 0) {

src/sist.h

@@ -25,19 +25,20 @@
 #include <pthread.h>
 #include <sys/stat.h>
 #include <wordexp.h>
+
+#ifndef SIST_SCAN_ONLY
 #include <onion/onion.h>
 #include <onion/handler.h>
 #include <onion/block.h>
 #include <onion/shortcuts.h>
 #include <curl/curl.h>
-
+#endif
 
 #include "cJSON/cJSON.h"
 
 #include "types.h"
 #include "tpool.h"
 #include "util.h"
-#include "src/index/elastic.h"
 #include "io/store.h"
 #include "io/serialize.h"
 #include "io/walk.h"
@@ -47,9 +48,13 @@
 #include "parsing/pdf.h"
 #include "parsing/media.h"
 #include "parsing/font.h"
+#include "cli.h"
+
+#ifndef SIST_SCAN_ONLY
+#include "src/index/elastic.h"
 #include "index/web.h"
 #include "web/serve.h"
-#include "cli.h"
+#endif
 
 ;
 

src/util.c

@@ -1,15 +1,160 @@
-#define _GNU_SOURCE
 #include "util.h"
 
+dyn_buffer_t dyn_buffer_create() {
+    dyn_buffer_t buf;
+
+    buf.size = INITIAL_BUF_SIZE;
+    buf.cur = 0;
+    buf.buf = malloc(INITIAL_BUF_SIZE);
+
+    return buf;
+}
+
+void grow_buffer(dyn_buffer_t *buf, size_t size) {
+    if (buf->cur + size > buf->size) {
+        do {
+            buf->size *= 2;
+        } while (buf->cur + size > buf->size);
+
+        buf->buf = realloc(buf->buf, buf->size);
+    }
+}
+
+void grow_buffer_small(dyn_buffer_t *buf) {
+    if (buf->cur + sizeof(long) > buf->size) {
+        buf->size *= 2;
+        buf->buf = realloc(buf->buf, buf->size);
+    }
+}
+
+void dyn_buffer_write(dyn_buffer_t *buf, void *data, size_t size) {
+    grow_buffer(buf, size);
+
+    memcpy(buf->buf + buf->cur, data, size);
+    buf->cur += size;
+}
+
+void dyn_buffer_write_char(dyn_buffer_t *buf, char c) {
+    grow_buffer_small(buf);
+
+    *(buf->buf + buf->cur) = c;
+    buf->cur += sizeof(c);
+}
+
+void dyn_buffer_write_str(dyn_buffer_t *buf, char *str) {
+    dyn_buffer_write(buf, str, strlen(str));
+    dyn_buffer_write_char(buf, '\0');
+}
+
+void dyn_buffer_write_int(dyn_buffer_t *buf, int d) {
+    grow_buffer_small(buf);
+
+    *(int *) (buf->buf + buf->cur) = d;
+    buf->cur += sizeof(int);
+}
+
+void dyn_buffer_write_short(dyn_buffer_t *buf, short s) {
+    grow_buffer_small(buf);
+
+    *(short *) (buf->buf + buf->cur) = s;
+    buf->cur += sizeof(short);
+}
+
+void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l) {
+    grow_buffer_small(buf);
+
+    *(unsigned long *) (buf->buf + buf->cur) = l;
+    buf->cur += sizeof(unsigned long);
+}
+
+void dyn_buffer_destroy(dyn_buffer_t *buf) {
+    free(buf->buf);
+}
+
+void text_buffer_destroy(text_buffer_t *buf) {
+    dyn_buffer_destroy(&buf->dyn_buffer);
+}
+
+text_buffer_t text_buffer_create(int max_size) {
+    text_buffer_t text_buf;
+
+    text_buf.dyn_buffer = dyn_buffer_create();
+    text_buf.max_size = max_size;
+    text_buf.last_char_was_whitespace = FALSE;
+
+    return text_buf;
+}
+
+void text_buffer_terminate_string(text_buffer_t *buf) {
+    dyn_buffer_write_char(&buf->dyn_buffer, '\0');
+}
+
+int text_buffer_append_char(text_buffer_t *buf, int c) {
+
+    if (SHOULD_IGNORE_CHAR(c)) {
+        if (!buf->last_char_was_whitespace) {
+            dyn_buffer_write_char(&buf->dyn_buffer, ' ');
+            buf->last_char_was_whitespace = TRUE;
+
+            if (buf->dyn_buffer.cur >= buf->max_size) {
+                return TEXT_BUF_FULL;
+            }
+        }
+    } else {
+        buf->last_char_was_whitespace = FALSE;
+        dyn_buffer_write_char(&buf->dyn_buffer, (char) c);
+
+        if (buf->dyn_buffer.cur >= buf->max_size) {
+            return TEXT_BUF_FULL;
+        }
+    }
+
+    return 0;
+}
+
+void incremental_put(GHashTable *table, unsigned long inode_no, int mtime) {
+    g_hash_table_insert(table, (gpointer) inode_no, GINT_TO_POINTER(mtime));
+}
+
+int incremental_get(GHashTable *table, unsigned long inode_no) {
+    if (table != NULL) {
+        return GPOINTER_TO_INT(g_hash_table_lookup(table, (gpointer) inode_no));
+    } else {
+        return 0;
+    }
+}
+
+int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no) {
+    g_hash_table_insert(table, GINT_TO_POINTER(inode_no), GINT_TO_POINTER(1));
+}
+
 
 #define PBSTR "========================================"
 #define PBWIDTH 40
 
+dyn_buffer_t url_escape(char *str) {
+
+    dyn_buffer_t text = dyn_buffer_create();
+
+    char * ptr = str;
+    while (*ptr) {
+        if (*ptr == '#') {
+            dyn_buffer_write(&text, "%23", 3);
+            ptr++;
+        }
+
+        dyn_buffer_write_char(&text, *ptr++);
+    }
+    dyn_buffer_write_char(&text, '\0');
+
+    return text;
+}
+
 char *abspath(const char *path) {
     wordexp_t w;
     wordexp(path, &w, 0);
 
-    char *abs = canonicalize_file_name(w.we_wordv[0]);
+    char *abs = realpath(w.we_wordv[0], NULL);
     if (abs == NULL) {
         return NULL;
     }

src/util.h

@@ -21,159 +21,48 @@ typedef struct text_buffer {
     dyn_buffer_t dyn_buffer;
 } text_buffer_t;
 
-__always_inline
-dyn_buffer_t dyn_buffer_create() {
-    dyn_buffer_t buf;
-
-    buf.size = INITIAL_BUF_SIZE;
-    buf.cur = 0;
-    buf.buf = malloc(INITIAL_BUF_SIZE);
-
-    return buf;
-}
-
-__always_inline
-void grow_buffer(dyn_buffer_t *buf, size_t size) {
-    if (buf->cur + size > buf->size) {
-        do {
-            buf->size *= 2;
-        } while (buf->cur + size > buf->size);
-
-        buf->buf = realloc(buf->buf, buf->size);
-    }
-}
-
-__always_inline
-void grow_buffer_small(dyn_buffer_t *buf) {
-    if (buf->cur + sizeof(long) > buf->size) {
-        buf->size *= 2;
-        buf->buf = realloc(buf->buf, buf->size);
-    }
-}
-
-__always_inline
-void dyn_buffer_write(dyn_buffer_t *buf, void *data, size_t size) {
-    grow_buffer(buf, size);
-
-    memcpy(buf->buf + buf->cur, data, size);
-    buf->cur += size;
-}
-
-__always_inline
-void dyn_buffer_write_char(dyn_buffer_t *buf, char c) {
-    grow_buffer_small(buf);
-
-    *(buf->buf + buf->cur) = c;
-    buf->cur += sizeof(c);
-}
-
-__always_inline
-void dyn_buffer_write_str(dyn_buffer_t *buf, char *str) {
-    dyn_buffer_write(buf, str, strlen(str));
-    dyn_buffer_write_char(buf, '\0');
-}
-
-__always_inline
-void dyn_buffer_write_int(dyn_buffer_t *buf, int d) {
-    grow_buffer_small(buf);
-
-    *(int *) (buf->buf + buf->cur) = d;
-    buf->cur += sizeof(int);
-}
-
-__always_inline
-void dyn_buffer_write_short(dyn_buffer_t *buf, short s) {
-    grow_buffer_small(buf);
-
-    *(short *) (buf->buf + buf->cur) = s;
-    buf->cur += sizeof(short);
-}
-
-__always_inline
-void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l) {
-    grow_buffer_small(buf);
-
-    *(unsigned long *) (buf->buf + buf->cur) = l;
-    buf->cur += sizeof(unsigned long);
-}
-
-__always_inline
-void dyn_buffer_destroy(dyn_buffer_t *buf) {
-    free(buf->buf);
-}
-
-__always_inline
-void text_buffer_destroy(text_buffer_t *buf) {
-    dyn_buffer_destroy(&buf->dyn_buffer);
-}
-
-__always_inline
-text_buffer_t text_buffer_create(int max_size) {
-    text_buffer_t text_buf;
-
-    text_buf.dyn_buffer = dyn_buffer_create();
-    text_buf.max_size = max_size;
-    text_buf.last_char_was_whitespace = FALSE;
-
-    return text_buf;
-}
-
-__always_inline
-void text_buffer_terminate_string(text_buffer_t *buf) {
-    dyn_buffer_write_char(&buf->dyn_buffer, '\0');
-}
-
-__always_inline
-int text_buffer_append_char(text_buffer_t *buf, int c) {
-
-    if (SHOULD_IGNORE_CHAR(c)) {
-        if (!buf->last_char_was_whitespace) {
-            dyn_buffer_write_char(&buf->dyn_buffer, ' ');
-            buf->last_char_was_whitespace = TRUE;
-
-            if (buf->dyn_buffer.cur >= buf->max_size) {
-                return TEXT_BUF_FULL;
-            }
-        }
-    } else {
-        buf->last_char_was_whitespace = FALSE;
-        dyn_buffer_write_char(&buf->dyn_buffer, (char) c);
-
-        if (buf->dyn_buffer.cur >= buf->max_size) {
-            return TEXT_BUF_FULL;
-        }
-    }
-
-    return 0;
-}
-
 char *abspath(const char * path);
 char *expandpath(const char *path);
+dyn_buffer_t url_escape(char *str);
 
 void progress_bar_print(double percentage, size_t tn_size, size_t index_size);
 
 
-__always_inline
-void incremental_put(GHashTable *table, unsigned long inode_no, int mtime) {
-    g_hash_table_insert(table, (gpointer) inode_no, GINT_TO_POINTER(mtime));
-}
-
-__always_inline
-int incremental_get(GHashTable *table, unsigned long inode_no) {
-    if (table != NULL) {
-        return GPOINTER_TO_INT(g_hash_table_lookup(table, (gpointer) inode_no));
-    } else {
-        return 0;
-    }
-}
-
-__always_inline
-int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no) {
-    g_hash_table_insert(table, GINT_TO_POINTER(inode_no), GINT_TO_POINTER(1));
-}
-
 GHashTable *incremental_get_table();
 
+dyn_buffer_t dyn_buffer_create();
+
+void grow_buffer(dyn_buffer_t *buf, size_t size);
+
+void grow_buffer_small(dyn_buffer_t *buf);
+
+void dyn_buffer_write(dyn_buffer_t *buf, void *data, size_t size);
+
+void dyn_buffer_write_char(dyn_buffer_t *buf, char c);
+
+void dyn_buffer_write_str(dyn_buffer_t *buf, char *str);
+
+void dyn_buffer_write_int(dyn_buffer_t *buf, int d);
+
+void dyn_buffer_write_short(dyn_buffer_t *buf, short s);
+
+void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l);
+
+void dyn_buffer_destroy(dyn_buffer_t *buf);
+
+void text_buffer_destroy(text_buffer_t *buf);
+
+text_buffer_t text_buffer_create(int max_size);
+
+void text_buffer_terminate_string(text_buffer_t *buf);
+
+int text_buffer_append_char(text_buffer_t *buf, int c);
+
+void incremental_put(GHashTable *table, unsigned long inode_no, int mtime);
+
+int incremental_get(GHashTable *table, unsigned long inode_no);
+int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no);
+
 
 
 #endif

src/web/serve.c

@@ -287,7 +287,10 @@ int serve_file_from_url(cJSON *json, index_t *idx, onion_request *req, onion_res
              "%s%s/%s%s%s",
              idx->desc.rewrite_url, path, name, strlen(ext) == 0 ? "" : ".", ext);
 
-    return onion_shortcut_redirect(url, req, res);
+    dyn_buffer_t encoded = url_escape(url);
+    int ret = onion_shortcut_redirect(encoded.buf, req, res);
+    dyn_buffer_destroy(&encoded);
+    return ret;
 }
 
 int serve_file_from_disk(cJSON *json, index_t *idx, onion_request *req, onion_response *res) {
@@ -301,6 +304,11 @@ int serve_file_from_disk(cJSON *json, index_t *idx, onion_request *req, onion_re
     snprintf(full_path, PATH_MAX, "%s%s/%s%s%s",
              idx->desc.root, path, name, strlen(ext) == 0 ? "" : ".", ext);
 
+    char disposition[8196];
+    snprintf(disposition, sizeof(disposition), "inline; filename=\"%s%s%s\"",
+             name, strlen(ext) == 0 ? "" : ".", ext);
+    onion_response_set_header(res, "Content-Disposition", disposition);
+
     return chunked_response_file(full_path, mime, 1, req, res);
 }
 
@@ -337,23 +345,19 @@ int file(void *p, onion_request *req, onion_response *res) {
         return OCS_PROCESSED;
     }
 
-    cJSON *source = elastic_get_document(arg_uuid);
+    cJSON *doc = elastic_get_document(arg_uuid);
+    cJSON *source = cJSON_GetObjectItem(doc, "_source");
+    cJSON *index_id = cJSON_GetObjectItem(source, "index");
+    if (index_id == NULL) {
+        return OCS_NOT_PROCESSED;
+    }
 
-    const char *index_id = cJSON_GetObjectItem(source, "index")->valuestring;
-
-    index_t *idx = get_index_by_id(index_id);
+    index_t *idx = get_index_by_id(index_id->valuestring);
 
     if (idx == NULL) {
         return OCS_NOT_PROCESSED;
     }
 
-    const char *name = cJSON_GetObjectItem(source, "name")->valuestring;
-    const char *ext = cJSON_GetObjectItem(source, "extension")->valuestring;
-    char disposition[8196];
-    snprintf(disposition, sizeof(disposition), "inline; filename=\"%s%s%s\"",
-             name, strlen(ext) == 0 ? "" : ".", ext);
-    onion_response_set_header(res, "Content-Disposition", disposition);
-
     if (strlen(idx->desc.rewrite_url) == 0) {
         return serve_file_from_disk(source, idx, req, res);
     } else {

web/css/main.css

@@ -1,4 +1,9 @@
 body {overflow-y:scroll;}
+
+.progress {
+    margin-top: 1em;
+}
+
 .card {
     margin-top: 1em;
 }

web/js/dom.js

@@ -129,6 +129,8 @@ function createDocCard(hit) {
             });
         } else if ((hit["_source"].hasOwnProperty("width") && hit["_source"]["width"] > 20 && hit["_source"]["height"] > 20)
             || hit["_source"]["mime"] === "application/pdf"
+            || hit["_source"]["mime"] === "application/epub+zip"
+            || hit["_source"]["mime"] === "application/x-cbz"
             || hit["_source"].hasOwnProperty("font_name")
         ) {
             thumbnail = document.createElement("img");
@@ -170,11 +172,10 @@ function createDocCard(hit) {
         }
 
         //Tags
-        //todo: handle new tags
         switch (mimeCategory) {
             case "video":
             case "image":
-                if (hit.hasOwnProperty("videoc")) {
+                if (hit["_source"].hasOwnProperty("videoc")) {
                     let formatTag = document.createElement("span");
                     formatTag.setAttribute("class", "badge badge-pill badge-video");
                     formatTag.appendChild(document.createTextNode(hit["_source"]["videoc"].replace(" ", "")));
@@ -182,7 +183,7 @@ function createDocCard(hit) {
                 }
                 break;
             case "audio": {
-                if (hit.hasOwnProperty("audioc")) {
+                if (hit["_source"].hasOwnProperty("audioc")) {
                     let formatTag = document.createElement("span");
                     formatTag.setAttribute("class", "badge badge-pill badge-audio");
                     formatTag.appendChild(document.createTextNode(hit["_source"]["audioc"]));
@@ -243,6 +244,17 @@ function createDocCard(hit) {
     return docCard;
 }
 
+function makePreloader() {
+    const elem = document.createElement("div");
+    elem.setAttribute("class", "progress");
+    const bar = document.createElement("div");
+    bar.setAttribute("class", "progress-bar progress-bar-striped progress-bar-animated");
+    bar.setAttribute("style", "width: 100%");
+    elem.appendChild(bar);
+
+    return elem;
+}
+
 function makePageIndicator(searchResult) {
     let pageIndicator = document.createElement("div");
     pageIndicator.setAttribute("class", "page-indicator shadow-sm bg-light font-weight-light");

web/js/search.js

@@ -1,4 +1,4 @@
-const SIZE = 20;
+const SIZE = 40;
 let mimeMap = [];
 let tree;
 
@@ -29,7 +29,7 @@ $.jsonPost("i").then(resp => {
     resp["indices"].forEach(idx => {
         $("#indices").append($("<option>")
             .attr("value", idx.id)
-            .attr("selected", true)
+            .attr("selected", !idx.name.includes("(nsfw)"))
             .append(idx.name)
         );
         selectedIndices.push(idx.id);
@@ -193,14 +193,19 @@ function search() {
         return;
     }
     searchBusy = true;
+
     //Clear old search results
     let searchResults = document.getElementById("searchResults");
     while (searchResults.firstChild) {
         searchResults.removeChild(searchResults.firstChild);
     }
 
+    const preload = makePreloader();
+    searchResults.appendChild(preload);
+
     let query = searchBar.value;
-    let condition = $("#barToggle").prop("checked") ? "must" : "should";
+    let empty = query === "";
+    let condition = $("#barToggle").prop("checked") && !empty ? "must" : "should";
     let filters = [
         {range: {size: {gte: size_min, lte: size_max}}},
         {terms: {index: selectedIndices}}
@@ -257,6 +262,7 @@ function search() {
     }).then(searchResult => {
         scroll_id = searchResult["_scroll_id"];
 
+        preload.remove();
         //Search stats
         searchResults.appendChild(makeStatsCard(searchResult));