.dockerignore

@@ -1,24 +0,0 @@
-.idea
-*/thumbs
-*.cbp
-CMakeCache.txt
-CMakeFiles
-cmake-build-debug
-cmake_install.cmake
-Makefile
-*.out
-LOG
-sist2*
-index.sist2/
-bundle*.css
-bundle.js
-**/*.a
-**/vgcore.*
-build/
-.git/
-third-party/libscan/libscan-test-files/
-**/ext_ffmpeg
-**/ext_libmobi
-**/scan_a_test
-Dockerfile
-*.idx/

.drone.yml

@@ -8,7 +8,7 @@ platform:
 
 steps:
   - name: build
-    image: simon987/sist2-build
+    image: simon987/ubuntu_ci
     commands:
       - ./ci/build.sh
   - name: docker
@@ -19,8 +19,8 @@ steps:
       password:
         from_secret: DOCKER_PASSWORD
       repo: simon987/sist2
-      context: ./
+      context: ./Docker/
-      dockerfile: ./Dockerfile
+      dockerfile: ./Docker/Dockerfile
       auto_tag: true
       auto_tag_suffix: x64-linux
       when:
@@ -41,32 +41,32 @@ steps:
       source:
         - ./VERSION
         - ./sist2-x64-linux
-        - ./sist2-x64-linux-debug
+        - ./sist2-x64-linux-debug.tar.gz
 
-#---
+---
-#kind: pipeline
+kind: pipeline
-#type: docker
+type: docker
-#name: arm64
+name: arm64
-#
+
-#platform:
+platform:
-#  arch: arm64
+  arch: arm64
-#
+
-#steps:
+steps:
-#  - name: build
+  - name: build
-#    image: simon987/ubuntu_ci_arm
+    image: simon987/ubuntu_ci_arm
-#    commands:
+    commands:
-#      - ./ci/build_arm64.sh
+      - ./ci/build_arm64.sh
-#  - name: scp files
+  - name: scp files
-#    image: appleboy/drone-scp
+    image: appleboy/drone-scp
-#    settings:
+    settings:
-#      host:
+      host:
-#        from_secret: SSH_HOST
+        from_secret: SSH_HOST
-#      port:
+      port:
-#        from_secret: SSH_PORT
+        from_secret: SSH_PORT
-#      user:
+      user:
-#        from_secret: SSH_USER
+        from_secret: SSH_USER
-#      key:
+      key:
-#        from_secret: SSH_KEY
+        from_secret: SSH_KEY
-#      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
-#      source:
+      source:
-#        - ./sist2-arm64-linux
+        - ./sist2-arm64-linux

.gitignore

@@ -16,4 +16,3 @@ bundle.js
 vgcore.*
 build/
 third-party/
-*.idx/

CMakeLists.txt

@@ -5,7 +5,7 @@ project(sist2 C)
 
 option(SIST_DEBUG "Build a debug executable" on)
 
-set(BUILD_TESTS on)
+set(BUILD_TESTS off)
 add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
@@ -36,15 +36,14 @@ add_executable(sist2
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
 set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
 
-find_package(PkgConfig REQUIRED)
-
-pkg_search_module(GLIB REQUIRED glib-2.0)
-
 find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
+find_package(unofficial-glib CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
 
+#find_package(OpenSSL REQUIRED)
+
 
 target_include_directories(
         sist2 PUBLIC
@@ -52,7 +51,6 @@ target_include_directories(
         ${CMAKE_SOURCE_DIR}/third-party/utf8.h/
         ${CMAKE_SOURCE_DIR}/third-party/libscan/
         ${CMAKE_SOURCE_DIR}/
-        ${GLIB_INCLUDE_DIRS}
 )
 
 target_compile_options(
@@ -105,7 +103,7 @@ target_link_libraries(
         lmdb
         cjson
         argparse
-        ${GLIB_LDFLAGS}
+        unofficial::glib::glib
         unofficial::mongoose::mongoose
         CURL::libcurl
 

Docker/Dockerfile

@@ -1,14 +1,6 @@
-FROM simon987/sist2-build as build
+FROM ubuntu:20.04
 MAINTAINER simon987 <me@simon987.net>
 
-WORKDIR /build/
-ADD . /build/
-RUN cmake -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
-RUN make -j$(nproc)
-RUN strip sist2
-
-FROM ubuntu:20.10
-
 RUN apt update
 RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 \
  curl libtiff5 libpng16-16 libpcre3
@@ -20,9 +12,9 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
     curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
 
-COPY --from=build /build/sist2 /root/sist2
+ADD sist2 /root/sist2
 
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8

DockerArm64/Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:19.10
+MAINTAINER simon987 <me@simon987.net>
+
+RUN apt update
+RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 \
+ curl libtiff5 libpng16-16 libpcre3
+
+RUN mkdir -p /usr/share/tessdata && \
+    cd /usr/share/tessdata/ && \
+    curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
+    curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
+    curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
+    curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
+    curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
+
+ADD sist2_arm64 /root/sist2
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENTRYPOINT ["/root/sist2"]

DockerArm64/build.sh

@@ -0,0 +1,13 @@
+rm ./sist2_arm64
+cp ../sist2_arm64.gz .
+gzip -d sist2_arm64.gz
+
+version=$(./sist2_arm64 --version)
+
+echo "Version ${version}"
+docker build . -t simon987/sist2-arm64:"${version}" -t simon987/sist2-arm64:latest
+
+docker push simon987/sist2-arm64:"${version}"
+docker push simon987/sist2-arm64:latest
+
+docker run --rm simon987/sist2-arm64 -v

Dockerfile.arm64

@@ -1 +0,0 @@
-# TODO

README.md

@@ -1,6 +1,6 @@
 ![GitHub](https://img.shields.io/github/license/simon987/sist2.svg)
 [![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
-[![Development snapshots](https://ci.simon987.net/api/badges/simon987/sist2/status.svg)](https://files.simon987.net/.gate/sist2/simon987_sist2/)
+[![Development snapshots](https://ci.simon987.net/api/badges/simon987/sist2/status.svg)](https://files.simon987.net/sist2/simon987_sist2/)
 
 **Demo**: [sist2.simon987.net](https://sist2.simon987.net/?i=Demo%20files)
 
@@ -25,12 +25,14 @@ sist2 (Simple incremental search tool)
 * OCR support with tesseract \*\*\*
 * Stats page & disk utilisation visualization
 
+
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
 \*\*\* See [OCR](#ocr)    
 
 ![stats](docs/stats.png)
 
+
 ## Getting Started
 
 1. Have an Elasticsearch (>= 6.X.X) instance running
@@ -55,8 +57,10 @@ sist2 (Simple incremental search tool)
 
 1. See [Usage guide](docs/USAGE.md)
    
+
 \* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)    
 
+
 ## Example usage
 
 See [Usage guide](docs/USAGE.md) for more details
@@ -65,6 +69,7 @@ See [Usage guide](docs/USAGE.md) for more details
 1. Push index to Elasticsearch: `sist2 index ./docs_idx`
 1. Start web interface: `sist2 web ./docs_idx`
 
+
 ## Format support
 
 File type | Library | Content | Thumbnail | Metadata
@@ -73,8 +78,8 @@ pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title |
 cbz,cbr | *(none)* | - | yes | - |
 `audio/*` | ffmpeg | - | yes | ID3 tags |
 `video/*` | ffmpeg | - | yes | title, comment, artist |
-`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
+`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
-raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags, GPS tags |
+raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags |
 ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
 `text/plain` | *(none)* | yes | no | - |
 html, xml | *(none)* | yes | no | - |
@@ -86,37 +91,38 @@ mobi, azw, azw3 | libmobi | yes | no | author, title |
 \* *See [Archive files](#archive-files)*
  
 ### Archive files
-
+**sist2** will scan files stored into archive files (zip, tar, 7z...) as if
-**sist2** will scan files stored into archive files (zip, tar, 7z...) as if they were directly in the file system.
+they were directly in the file system. Recursive (archives inside archives)
-Recursive (archives inside archives)
 scan is also supported.
 
 **Limitations**:
-
 * Support for parsing media files with formats that require *seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.) 
   is limitted (see `--mem-buffer` option)
 * Archive files are scanned sequentially, by a single thread. On systems where
-  **sist2** is not I/O bound, scans might be faster when larger archives are split into smaller parts.
+**sist2** is not I/O bound, scans might be faster when larger archives are split
+ into smaller parts.
+ 
  
 ### OCR
 
 You can enable OCR support for pdf,xps,fb2,epub file types with the
-`--ocr <lang>` option. Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
+`--ocr <lang>` option. Download the language data files with your
-directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
+package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
 The `simon987/sist2` image comes with common languages 
 (hin, jpn, eng, fra, rus, spa) pre-installed.
 
 Examples
-
 ```bash
 sist2 scan --ocr jpn ~/Books/Manga/
 sist2 scan --ocr eng ~/Books/Textbooks/
 ```
 
+
 ## Build from source
 
-You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries (GCC 7+ required).
+You can compile **sist2** by yourself if you don't want to use the pre-compiled
+binaries (GCC 7+ required).
 
 1. Install compile-time dependencies
 

ci/build.sh

@@ -8,7 +8,7 @@ git submodule update --init --recursive
 
 rm -rf CMakeFiles CMakeCache.txt
 cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
-make -j $(nproc)
+make -j 33
 strip sist2
 ./sist2 -v > VERSION
 cp sist2 Docker/
@@ -16,5 +16,7 @@ mv sist2 sist2-x64-linux
 
 rm -rf CMakeFiles CMakeCache.txt
 cmake -DSIST_DEBUG=on -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
-make -j  $(nproc)
+make -j 33
+cp /usr/lib/x86_64-linux-gnu/libasan.so.2.0.0 libasan.so.2
 mv sist2_debug sist2-x64-linux-debug
+tar -czf sist2-x64-linux-debug.tar.gz sist2-x64-linux-debug libasan.so.2

docs/USAGE.md

@@ -46,7 +46,6 @@ Scan options
     --fast                        Only index file names & mime type
     --treemap-threshold=<str>     Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
     --mem-buffer=<int>            Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000
-    --read-subtitles              Read subtitles from media files
 
 Index options
     -t, --threads=<int>           Number of threads. DEFAULT=1
@@ -92,7 +91,7 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
     Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
     will be copied to the new index and will not be parsed again.
 * `-o, --output` Output directory. 
-* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
+* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
 * `--name` Set the `name` option for the web module
 * `--depth` Maximum scan dept. Set to 0 only scan files directly in the root directory, set to -1 for infinite depth
 * `--archive` Archive file mode.
@@ -124,7 +123,6 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
     larger than this number will be read sequentially and no *seek* operations will be supported.
 
     To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
-* `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
 
 ### Scan examples
 

schema/mappings.json

@@ -165,30 +165,6 @@
     "exif_user_comment": {
       "type": "text"
     },
-    "exif_gps_longitude_ref": {
-      "type": "keyword",
-      "index": false
-    },
-    "exif_gps_longitude_dms": {
-      "type": "keyword",
-      "index": false
-    },
-    "exif_gps_longitude_dec": {
-      "type": "keyword",
-      "index": false
-    },
-    "exif_gps_latitude_ref": {
-      "type": "keyword",
-      "index": false
-    },
-    "exif_gps_latitude_dms": {
-      "type": "keyword",
-      "index": false
-    },
-    "exif_gps_latitude_dec": {
-      "type": "keyword",
-      "index": false
-    },
     "author": {
       "type": "text"
     },

scripts/reset.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-make clean
-rm -rf CMakeFiles/ CMakeCache.txt Makefile \
-  third-party/libscan/CMakeFiles third-party/libscan/CMakeCache.txt third-party/libscan/third-party/ext_ffmpeg \
-  third-party/libscan/third-party/ext_libmobi third-party/libscan/Makefile

src/cli.c

@@ -227,7 +227,6 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
     LOG_DEBUGF("cli.c", "arg path=%s", args->path)
     LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
-    LOG_DEBUGF("cli.c", "arg archive_passphrase=%s", args->archive_passphrase)
     LOG_DEBUGF("cli.c", "arg tesseract_lang=%s", args->tesseract_lang)
     LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path)
     LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex)

src/cli.h

@@ -18,7 +18,6 @@ typedef struct scan_args {
     char *path;
     char *archive;
     archive_mode_t archive_mode;
-    char *archive_passphrase;
     char *tesseract_lang;
     const char *tesseract_path;
     char *exclude_regex;
@@ -26,7 +25,6 @@ typedef struct scan_args {
     const char* treemap_threshold_str;
     double treemap_threshold;
     int max_memory_buffer;
-    int read_subtitles;
 } scan_args_t;
 
 scan_args_t *scan_args_create();

src/ctx.h

@@ -40,8 +40,6 @@ typedef struct {
     pcre_extra *exclude_extra;
     int fast;
 
-    GHashTable *dbg_current_files;
-
     scan_arc_ctx_t arc_ctx;
     scan_comic_ctx_t comic_ctx;
     scan_ebook_ctx_t ebook_ctx;

src/io/serialize.c

@@ -15,13 +15,9 @@ typedef struct {
     char has_parent;
 } line_t;
 
-#define META_NEXT 0xFFFF
-
 void skip_meta(FILE *file) {
-    enum metakey key = 0;
+    enum metakey key = getc(file);
-    fread(&key, sizeof(uint16_t), 1, file);
+    while (key != '\n') {
-
-    while (key != META_NEXT) {
         if (IS_META_INT(key)) {
             fseek(file, sizeof(int), SEEK_CUR);
         } else if (IS_META_LONG(key)) {
@@ -30,7 +26,7 @@ void skip_meta(FILE *file) {
             while ((getc(file))) {}
         }
 
-        fread(&key, sizeof(uint16_t), 1, file);
+        key = getc(file);
     }
 }
 
@@ -70,7 +66,7 @@ index_descriptor_t read_index_descriptor(char *path) {
     }
 
     char *buf = malloc(info.st_size + 1);
-    size_t ret = read(fd, buf, info.st_size);
+    int ret = read(fd, buf, info.st_size);
     if (ret == -1) {
         LOG_FATALF("serialize.c", "Could not read index descriptor: %s", strerror(errno));
     }
@@ -156,20 +152,8 @@ char *get_meta_key_text(enum metakey meta_key) {
             return "thumbnail";
         case MetaPages:
             return "pages";
-        case MetaExifGpsLongitudeRef:
-            return "exif_gps_longitude_ref";
-        case MetaExifGpsLongitudeDMS:
-            return "exif_gps_longitude_dms";
-        case MetaExifGpsLongitudeDec:
-            return "exif_gps_longitude_dec";
-        case MetaExifGpsLatitudeRef:
-            return "exif_gps_latitude_ref";
-        case MetaExifGpsLatitudeDMS:
-            return "exif_gps_latitude_dms";
-        case MetaExifGpsLatitudeDec:
-            return "exif_gps_latitude_dec";
         default:
-        LOG_FATALF("serialize.c", "FIXME: Unknown meta key: %d", meta_key)
+            return NULL;
     }
 }
 
@@ -199,7 +183,7 @@ void write_document(document_t *doc) {
 
     meta_line_t *meta = doc->meta_head;
     while (meta != NULL) {
-        dyn_buffer_write_short(&buf, (uint16_t) meta->key);
+        dyn_buffer_write_char(&buf, meta->key);
 
         if (IS_META_INT(meta->key)) {
             dyn_buffer_write_int(&buf, meta->int_val);
@@ -213,7 +197,7 @@ void write_document(document_t *doc) {
         meta = meta->next;
         free(tmp);
     }
-    dyn_buffer_write_short(&buf, META_NEXT);
+    dyn_buffer_write_char(&buf, '\n');
 
     int res = write(index_fd, buf.buf, buf.cur);
     if (res == -1) {
@@ -237,7 +221,7 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
     FILE *file = fopen(path, "rb");
     while (TRUE) {
         buf.cur = 0;
-        size_t _ = fread((void *) &line, sizeof(line_t), 1, file);
+        size_t _ = fread((void *) &line, 1, sizeof(line_t), file);
         if (feof(file)) {
             break;
         }
@@ -284,10 +268,9 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
             cJSON_AddStringToObject(document, "path", "");
         }
 
-        enum metakey key = 0;
+        enum metakey key = getc(file);
-        fread(&key, sizeof(uint16_t), 1, file);
+        size_t ret = 0;
-        size_t ret;
+        while (key != '\n') {
-        while (key != META_NEXT) {
             switch (key) {
                 case MetaPages:
                 case MetaWidth:
@@ -325,12 +308,6 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
                 case MetaAuthor:
                 case MetaModifiedBy:
                 case MetaThumbnail:
-                case MetaExifGpsLongitudeDMS:
-                case MetaExifGpsLongitudeDec:
-                case MetaExifGpsLongitudeRef:
-                case MetaExifGpsLatitudeDMS:
-                case MetaExifGpsLatitudeDec:
-                case MetaExifGpsLatitudeRef:
                 case MetaTitle: {
                     buf.cur = 0;
                     while ((c = getc(file)) != 0) {
@@ -346,7 +323,7 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
                 LOG_FATALF("serialize.c", "Invalid meta key (corrupt index): %x", key)
             }
 
-            fread(&key, sizeof(uint16_t), 1, file);
+            key = getc(file);
         }
 
         cJSON *meta_obj = NULL;
@@ -481,7 +458,7 @@ void incremental_read(GHashTable *table, const char *filepath) {
 
         incremental_put(table, line.path_md5, line.mtime);
 
-        while ((getc(file)) != 0) {}
+        while ((getc(file))) {}
         skip_meta(file);
     }
     fclose(file);
@@ -531,11 +508,11 @@ void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
                 free(buf);
             }
 
-            enum metakey key = 0;
+            enum metakey key;
             while (1) {
-                fread(&key, sizeof(uint16_t), 1, file);
+                key = getc(file);
-                fwrite(&key, sizeof(uint16_t), 1, dst_file);
+                fwrite(&key, sizeof(char), 1, dst_file);
-                if (key == META_NEXT) {
+                if (key == '\n') {
                     break;
                 }
 

src/io/store.c

@@ -4,7 +4,6 @@
 store_t *store_create(char *path, size_t chunk_size) {
 
     store_t *store = malloc(sizeof(struct store_t));
-#if (SIST_FAKE_STORE != 1)
     store->chunk_size = chunk_size;
     pthread_rwlock_init(&store->lock, NULL);
 
@@ -29,18 +28,15 @@ store_t *store_create(char *path, size_t chunk_size) {
     mdb_txn_begin(store->env, NULL, 0, &txn);
     mdb_dbi_open(txn, NULL, 0, &store->dbi);
     mdb_txn_commit(txn);
-#endif
 
     return store;
 }
 
 void store_destroy(store_t *store) {
 
-#if (SIST_FAKE_STORE != 1)
     pthread_rwlock_destroy(&store->lock);
     mdb_close(store->env, store->dbi);
     mdb_env_close(store->env);
-#endif
     free(store);
 }
 
@@ -60,8 +56,6 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
         }
     }
 
-#if (SIST_FAKE_STORE != 1)
-
     MDB_val mdb_key;
     mdb_key.mv_data = key;
     mdb_key.mv_size = key_len;
@@ -98,13 +92,10 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
     if (put_ret != 0) {
         LOG_ERROR("store.c", mdb_strerror(put_ret))
     }
-#endif
 }
 
 char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen) {
     char *buf = NULL;
-
-#if (SIST_FAKE_STORE != 1)
     MDB_val mdb_key;
     mdb_key.mv_data = key;
     mdb_key.mv_size = key_len;
@@ -125,7 +116,6 @@ char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen)
     }
 
     mdb_txn_abort(txn);
-#endif
     return buf;
 }
 

src/main.c

@@ -21,7 +21,7 @@
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "2.10.1";
+static const char *const Version = "2.9.0";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
@@ -30,69 +30,6 @@ static const char *const usage[] = {
         NULL,
 };
 
-#include<signal.h>
-#include<unistd.h>
-
-static __sighandler_t sigsegv_handler = NULL;
-static __sighandler_t sigabrt_handler = NULL;
-
-void sig_handler(int signum) {
-
-    LogCtx.verbose = 1;
-    LogCtx.very_verbose = 1;
-
-    LOG_ERROR("*SIGNAL HANDLER*", "=============================================\n\n");
-    LOG_ERRORF("*SIGNAL HANDLER*", "Uh oh! Caught fatal signal: %s", strsignal(signum));
-
-    GHashTableIter iter;
-    g_hash_table_iter_init(&iter, ScanCtx.dbg_current_files);
-
-    void *key;
-    void *value;
-    while (g_hash_table_iter_next(&iter, &key, &value)) {
-        parse_job_t *job = value;
-
-        if (isatty(STDERR_FILENO)) {
-            LOG_DEBUGF(
-                    "*SIGNAL HANDLER*",
-                    "Thread \033[%dm[%04llX]\033[0m was working on job '%s'",
-                    31 + ((unsigned int) key) % 7, key, job->filepath
-            );
-        } else {
-            LOG_DEBUGF(
-                    "*SIGNAL HANDLER*",
-                    "THREAD [%04llX] was working on job %s",
-                    key, job->filepath
-            );
-        }
-    }
-
-    tpool_dump_debug_info(ScanCtx.pool);
-
-    LOG_INFO(
-            "*SIGNAL HANDLER*",
-            "Please consider creating a bug report at https://github.com/simon987/sist2/issues !"
-    )
-    LOG_INFO(
-            "*SIGNAL HANDLER*",
-            "sist2 is an open source project and relies on the collaboration of its users to diagnose and fix bugs"
-    )
-
-#ifndef SIST_DEBUG
-    LOG_WARNING(
-            "*SIGNAL HANDLER*",
-            "You are running sist2 in release mode! Please consider downloading the debug binary from the Github "
-            "releases page to provide additionnal information when submitting a bug report."
-    )
-#endif
-
-    if (signum == SIGSEGV && sigsegv_handler != NULL) {
-        sigsegv_handler(signum);
-    } else if (signum == SIGABRT && sigabrt_handler != NULL) {
-        sigabrt_handler(signum);
-    }
-}
-
 void init_dir(const char *dirpath) {
     char path[PATH_MAX];
     snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
@@ -162,13 +99,6 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.arc_ctx.log = _log;
     ScanCtx.arc_ctx.logf = _logf;
     ScanCtx.arc_ctx.parse = (parse_callback_t) parse;
-    if (args->archive_passphrase != NULL) {
-        strcpy(ScanCtx.arc_ctx.passphrase, args->archive_passphrase);
-    } else {
-        ScanCtx.arc_ctx.passphrase[0] = 0;
-    }
-
-    ScanCtx.dbg_current_files = g_hash_table_new(g_int64_hash, g_int64_equal);
 
     // Comic
     ScanCtx.comic_ctx.log = _log;
@@ -202,7 +132,6 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.media_ctx.logf = _logf;
     ScanCtx.media_ctx.store = _store;
     ScanCtx.media_ctx.max_media_buffer = (long) args->max_memory_buffer * 1024 * 1024;
-    ScanCtx.media_ctx.read_subtitles = args->read_subtitles;
     init_media();
 
     // OOXML
@@ -470,9 +399,6 @@ void sist2_web(web_args_t *args) {
 
 
 int main(int argc, const char *argv[]) {
-    sigsegv_handler = signal(SIGSEGV, sig_handler);
-    sigabrt_handler = signal(SIGABRT, sig_handler);
-
     setlocale(LC_ALL, "");
 
     scan_args_t *scan_args = scan_args_create();
@@ -513,9 +439,6 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "archive", &scan_args->archive, "Archive file mode (skip|list|shallow|recurse). "
                                                           "skip: Don't parse, list: only get file names as text, "
                                                           "shallow: Don't parse archives inside archives. DEFAULT: recurse"),
-            OPT_STRING(0, "archive-passphrase", &scan_args->archive_passphrase,
-                       "Passphrase for encrypted archive files"),
-
             OPT_STRING(0, "ocr", &scan_args->tesseract_lang, "Tesseract language (use tesseract --list-langs to see "
                                                              "which are installed on your machine)"),
             OPT_STRING('e', "exclude", &scan_args->exclude_regex, "Files that match this regex will not be scanned"),
@@ -525,7 +448,6 @@ int main(int argc, const char *argv[]) {
             OPT_INTEGER(0, "mem-buffer", &scan_args->max_memory_buffer,
                         "Maximum memory buffer size per thread in MB for files inside archives "
                         "(see USAGE.md). DEFAULT: 2000"),
-            OPT_BOOLEAN(0, "read-subtitles", &scan_args->read_subtitles, "Read subtitles from media files."),
 
             OPT_GROUP("Index options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),

src/parsing/parse.c

@@ -41,18 +41,11 @@ void fs_reset(struct vfile *f) {
 
 #define IS_GIT_OBJ (strlen(doc.filepath + doc.base) == 38 && (strstr(doc.filepath, "objects") != NULL))
 
-void set_dbg_current_file(parse_job_t *job) {
-    unsigned long long pid = (unsigned long long) pthread_self();
-    g_hash_table_replace(ScanCtx.dbg_current_files, GINT_TO_POINTER(pid), job);
-}
-
 void parse(void *arg) {
 
     parse_job_t *job = arg;
     document_t doc;
 
-    set_dbg_current_file(job);
-
     doc.filepath = job->filepath;
     doc.ext = (short) job->ext;
     doc.base = (short) job->base;

src/static/js/dom.js

@@ -192,19 +192,6 @@ function makeUserTag(tag, hit) {
     return userTag;
 }
 
-function makeGpsMetaRow(tbody, latitude, longitude) {
-    tbody.append($("<tr>")
-        .append($("<td>").text("Exif GPS"))
-        .append($("<td>")
-            .append($("<a>")
-                .text(`${latitude}, ${longitude}`)
-                .attr("href", `https://maps.google.com/?q=${latitude},${longitude}&ll=${latitude},${longitude}&t=k&z=17`)
-                .attr("target", "_blank")
-            )
-        )
-    );
-}
-
 function infoButtonCb(hit) {
     return () => {
         getDocumentInfo(hit["_id"]).then(doc => {
@@ -242,25 +229,13 @@ function infoButtonCb(hit) {
                     .text(new Date(doc["mtime"] * 1000).toISOString().split(".")[0].replace("T", " "))
                     .attr("title", doc["mtime"]))
             );
-
-            // Exif GPS
-            if ("exif_gps_longitude_dec" in doc) {
-                makeGpsMetaRow(tbody, doc["exif_gps_latitude_dec"], doc["exif_gps_longitude_dec"])
-            } else if ("exif_gps_longitude_dms" in doc) {
-                makeGpsMetaRow(
-                    tbody,
-                    dmsToDecimal(doc["exif_gps_latitude_dms"], doc["exif_gps_latitude_ref"]),
-                    dmsToDecimal(doc["exif_gps_longitude_dms"], doc["exif_gps_longitude_ref"]),
-                )
-            }
-
             const displayFields = new Set([
                 "mime", "size", "path", "title", "width", "height", "duration", "audioc", "videoc",
                 "bitrate", "artist", "album", "album_artist", "genre", "title", "font_name", "tag", "author",
                 "modified_by", "pages"
             ]);
             Object.keys(doc)
-                .filter(key => key.startsWith("_keyword.") || key.startsWith("_text.") || displayFields.has(key) || (key.startsWith("exif_") && !key.includes("gps")))
+                .filter(key => key.startsWith("_keyword.") || key.startsWith("_text.") || displayFields.has(key) || key.startsWith("exif_"))
                 .forEach(key => {
                     tbody.append($("<tr>")
                         .append($("<td>").text(key))
@@ -375,14 +350,6 @@ function createDocCard(hit) {
             audio.setAttribute("controls", "");
             audio.setAttribute("type", hit["_source"]["mime"]);
             audio.setAttribute("src", "f/" + hit["_id"]);
-            audio.addEventListener("play", () => {
-                // Pause all currently playing audio tags
-                $("audio").each(function () {
-                    if (this !== audio) {
-                        this.pause();
-                    }
-                });
-            });
 
             docCard.appendChild(audio)
         }

src/static/js/search.js

@@ -511,8 +511,8 @@ function search(after = null) {
         searchResults.appendChild(preload);
     }
 
-    let searchBarValue = searchBar.value;
+    let query = searchBar.value;
-    let empty = searchBarValue === "";
+    let empty = query === "";
     let condition = empty ? "should" : "must";
     let filters = [
         {range: {size: {gte: size_min, lte: size_max}}},
@@ -561,32 +561,19 @@ function search(after = null) {
         filters.push({range: {mtime: {lte: date_max}}})
     }
 
-    let query;
-    if (CONF.options.queryMode === "simple") {
-        query = {
-            simple_query_string: {
-                query: searchBarValue,
-                fields: fields,
-                default_operator: "and"
-            }
-        }
-    } else {
-        query = {
-            query_string: {
-                query: searchBarValue,
-                default_field: "name",
-                default_operator: "and"
-            }
-        }
-    }
-
     let q = {
         "_source": {
             excludes: ["content", "_tie"]
         },
         query: {
             bool: {
-                [condition]: query,
+                [condition]: {
+                    simple_query_string: {
+                        query: query,
+                        fields: fields,
+                        default_operator: "and"
+                    }
+                },
                 filter: filters
             }
         },
@@ -624,9 +611,7 @@ function search(after = null) {
         }
     }
 
-    const showError = CONF.options.queryMode === "advanced";
+    $.jsonPost("es", q).then(searchResult => {
-
-    $.jsonPost("es", q, showError).then(searchResult => {
         let hits = searchResult["hits"]["hits"];
         if (hits) {
             lastDoc = hits[hits.length - 1];
@@ -660,25 +645,7 @@ function search(after = null) {
         reachedEnd = hits.length !== SIZE;
         insertHits(resultContainer, hits);
         searchBusy = false;
-    }).fail(() => {
-        searchBusy = false;
-        if (!after) {
-            preload.remove();
-        }
-
-        console.log("QUERY:")
-        console.log(q)
-        $.toast({
-            heading: "Query error",
-            text: "Could not parse or execute query, please check the Advanced search documentation. " +
-                "See server logs for more information.",
-            stack: false,
-            bgColor: "#FF8F00",
-            textColor: "#FFF3E0",
-            position: 'bottom-right',
-            hideAfter: false
     });
-    })
 }
 
 

src/static/js/util.js

@@ -102,8 +102,7 @@ const _defaults = {
     treemapSize: "large",
     suggestPath: true,
     fragmentSize: 100,
-    columns: 5,
+    columns: 5
-    queryMode: "simple"
 };
 
 function loadSettings() {
@@ -121,7 +120,6 @@ function loadSettings() {
     $("#settingSuggestPath").prop("checked", CONF.options.suggestPath);
     $("#settingFragmentSize").val(CONF.options.fragmentSize);
     $("#settingColumns").val(CONF.options.columns);
-    $("#settingQueryMode").val(CONF.options.queryMode);
 }
 
 function Settings() {
@@ -129,7 +127,6 @@ function Settings() {
 
     this._onUpdate = function () {
         $("#fuzzyToggle").prop("checked", this.options.fuzzy);
-        $("#searchBar").attr("placeholder", this.options.queryMode === "simple" ? "Search" : "Advanced search");
         updateColumnStyle();
     };
 
@@ -168,7 +165,6 @@ function updateSettings() {
     CONF.options.suggestPath = $("#settingSuggestPath").prop("checked");
     CONF.options.fragmentSize = $("#settingFragmentSize").val();
     CONF.options.columns = $("#settingColumns").val();
-    CONF.options.queryMode = $("#settingQueryMode").val();
     CONF.save();
 
     if (typeof searchDebounced !== "undefined") {
@@ -191,16 +187,14 @@ function updateSettings() {
     });
 }
 
-jQuery["jsonPost"] = function (url, data, showError = true) {
+jQuery["jsonPost"] = function (url, data) {
     return jQuery.ajax({
         url: url,
         type: "post",
         data: JSON.stringify(data),
         contentType: "application/json"
     }).fail(err => {
-        if (showError) {
         showEsError();
-        }
         console.log(err);
     });
 };
@@ -236,13 +230,3 @@ function updateColumnStyle() {
         `
     }
 }
-
-function dmsToDecimal(dms, ref) {
-    const tokens = dms.split(",")
-
-    const d = Number(tokens[0].trim().split(":")[0]) / Number(tokens[0].trim().split(":")[1])
-    const m = Number(tokens[1].trim().split(":")[0]) / Number(tokens[1].trim().split(":")[1])
-    const s = Number(tokens[2].trim().split(":")[0]) / Number(tokens[2].trim().split(":")[1])
-
-    return (d + (m / 60) + (s / 3600)) * (ref === "S" || ref === "W" ? -1 : 1)
-}

src/static/search.html

@@ -12,7 +12,7 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.10.1</span>
+    <span class="badge badge-pill version">2.9.0</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <a class="btn ml-auto" href="stats">Stats</a>
     <button class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings
@@ -120,8 +120,6 @@
                 </div>
                 <div class="modal-body">
 
-                    <h2>Simple search</h2>
-
                     <table class="table">
                         <tbody>
                         <tr>
@@ -170,12 +168,6 @@
                     <p>For more information, see <a target="_blank"
                                                     href="//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html">Elasticsearch
                         documentation</a></p>
-
-                    <h2>Advanced search</h2>
-                    <p>For documentation about the advanced search mode, see <a target="_blank"
-                                                                                href="//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax">Elasticsearch
-                        documentation</a></p>
-
                 </div>
             </div>
         </div>
@@ -215,16 +207,10 @@
 
                     <br/>
                     <div class="form-group">
-                        <label for="settingFragmentSize">Highlight context size in characters</label>
                         <input type="number" class="form-control" id="settingFragmentSize">
+                        <label for="settingFragmentSize">Highlight context size in characters</label>
                     </div>
 
-                    <label for="settingQueryMode">Search mode</label>
-                    <select id="settingQueryMode" class="form-control form-control-sm">
-                        <option value="simple">Simple</option>
-                        <option value="advanced">Advanced</option>
-                    </select>
-
                     <label for="settingDisplay">Display</label>
                     <select id="settingDisplay" class="form-control form-control-sm">
                         <option value="grid">Grid</option>

src/static/stats.html

@@ -10,7 +10,7 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.10.1</span>
+    <span class="badge badge-pill version">2.9.0</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <a style="margin-left: auto" class="btn" href="/">Back</a>
     <button class="btn" type="button" data-toggle="modal" data-target="#settings"
@@ -84,16 +84,10 @@
 
                 <br/>
                 <div class="form-group">
-                    <label for="settingFragmentSize">Highlight context size in characters</label>
                     <input type="number" class="form-control" id="settingFragmentSize">
+                    <label for="settingFragmentSize">Highlight context size in characters</label>
                 </div>
 
-                <label for="settingQueryMode">Search mode</label>
-                <select id="settingQueryMode" class="form-control form-control-sm">
-                    <option value="simple">Simple</option>
-                    <option value="advanced">Advanced</option>
-                </select>
-
                 <label for="settingDisplay">Display</label>
                 <select id="settingDisplay" class="form-control form-control-sm">
                     <option value="grid">Grid</option>

src/tpool.c

@@ -3,7 +3,7 @@
 #include "sist.h"
 #include <pthread.h>
 
-#define MAX_QUEUE_SIZE 1000000
+#define MAX_QUEUE_SIZE 10000
 
 typedef void (*thread_func_t)(void *arg);
 
@@ -52,13 +52,6 @@ static tpool_work_t *tpool_work_create(thread_func_t func, void *arg) {
     return work;
 }
 
-void tpool_dump_debug_info(tpool_t *pool) {
-    LOG_DEBUGF("tpool.c", "pool->thread_cnt = %d", pool->thread_cnt)
-    LOG_DEBUGF("tpool.c", "pool->work_cnt = %d", pool->work_cnt)
-    LOG_DEBUGF("tpool.c", "pool->done_cnt = %d", pool->done_cnt)
-    LOG_DEBUGF("tpool.c", "pool->stop = %d", pool->stop)
-}
-
 /**
  * Pop work object from thread pool
  */
@@ -90,7 +83,7 @@ int tpool_add_work(tpool_t *pool, thread_func_t func, void *arg) {
     }
 
     while ((pool->work_cnt - pool->done_cnt) >= MAX_QUEUE_SIZE) {
-        usleep(10000);
+        usleep(100000);
     }
 
     pthread_mutex_lock(&(pool->work_mutex));
@@ -157,7 +150,6 @@ static void *tpool_worker(void *arg) {
     if (pool->cleanup_func != NULL) {
         LOG_INFO("tpool.c", "Executing cleanup function")
         pool->cleanup_func();
-        LOG_DEBUG("tpool.c", "Done executing cleanup function")
     }
 
     pthread_cond_signal(&(pool->working_cond));

src/tpool.h

@@ -10,12 +10,10 @@ typedef void (*thread_func_t)(void *arg);
 
 tpool_t *tpool_create(size_t num, void (*cleanup_func)(), int free_arg);
 void tpool_start(tpool_t *pool);
-void tpool_destroy(tpool_t *pool);
+void tpool_destroy(tpool_t *tm);
 
 int tpool_add_work(tpool_t *pool, thread_func_t func, void *arg);
-void tpool_wait(tpool_t *pool);
+void tpool_wait(tpool_t *tm);
-
-void tpool_dump_debug_info(tpool_t *pool);
 
 #endif
 

src/web/serve.c

@@ -8,8 +8,18 @@
 
 #include <src/ctx.h>
 
+#include <mongoose.h>
 
-static void send_response_line(struct mg_connection *nc, int status_code, size_t length, char *extra_headers) {
+
+static int has_prefix(const struct mg_str *str, const struct mg_str *prefix) {
+    return str->len > prefix->len && memcmp(str->p, prefix->p, prefix->len) == 0;
+}
+
+static int is_equal(const struct mg_str *s1, const struct mg_str *s2) {
+    return s1->len == s2->len && memcmp(s1->p, s2->p, s2->len) == 0;
+}
+
+static void send_response_line(struct mg_connection *nc, int status_code, int length, char *extra_headers) {
     mg_printf(
             nc,
             "HTTP/1.1 %d %s\r\n"
@@ -52,32 +62,36 @@ store_t *get_tag_store(const char *index_id) {
 void search_index(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(search_html), "Content-Type: text/html");
     mg_send(nc, search_html, sizeof(search_html));
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 void stats(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(stats_html), "Content-Type: text/html");
     mg_send(nc, stats_html, sizeof(stats_html));
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
+void stats_files(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
 
-    if (hm->uri.len != MD5_STR_LENGTH + 4) {
+    if (path->len != MD5_STR_LENGTH + 4) {
-        mg_http_reply(nc, 404, "", "");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     char arg_md5[MD5_STR_LENGTH];
-    memcpy(arg_md5, hm->uri.ptr + 3, MD5_STR_LENGTH);
+    memcpy(arg_md5, hm->uri.p + 3, MD5_STR_LENGTH);
     *(arg_md5 + MD5_STR_LENGTH - 1) = '\0';
 
     index_t *index = get_index_by_id(arg_md5);
     if (index == NULL) {
-        mg_http_reply(nc, 404, "", "");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     const char *file;
-    switch (atoi(hm->uri.ptr + 3 + MD5_STR_LENGTH)) {
+    switch (atoi(hm->uri.p + 3 + MD5_STR_LENGTH)) {
         case 1:
             file = "treemap.csv";
             break;
@@ -91,41 +105,54 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
             file = "date_agg.csv";
             break;
         default:
+            nc->flags |= MG_F_SEND_AND_CLOSE;
             return;
     }
 
     char disposition[8192];
-    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s\"\r\n", file);
+    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s\"", file);
 
     char full_path[PATH_MAX];
     strcpy(full_path, index->path);
     strcat(full_path, file);
 
-    mg_http_serve_file(nc, hm, full_path, "text/csv", disposition);
+    mg_http_serve_file(nc, hm, full_path, mg_mk_str("text/csv"), mg_mk_str(disposition));
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 void javascript_lib(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(bundle_js), "Content-Type: application/javascript");
     mg_send(nc, bundle_js, sizeof(bundle_js));
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 void javascript_search(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(search_js), "Content-Type: application/javascript");
     mg_send(nc, search_js, sizeof(search_js));
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-int client_requested_dark_theme(struct mg_http_message *hm) {
+int client_requested_dark_theme(struct http_message *hm) {
-    struct mg_str *cookie_header = mg_http_get_header(hm, "cookie");
+    struct mg_str *cookie_header = mg_get_http_header(hm, "cookie");
     if (cookie_header == NULL) {
         return FALSE;
     }
 
-    struct mg_str sist_cookie = mg_http_get_header_var(*cookie_header, mg_str_n("sist", 4));
+    char buf[4096];
-
+    char *sist_cookie = buf;
-    return mg_strcmp(sist_cookie, mg_str_n("dark", 4)) == 0;
+    if (mg_http_parse_header2(cookie_header, "sist", &sist_cookie, sizeof(buf)) == 0) {
+        return FALSE;
     }
 
-void style(struct mg_connection *nc, struct mg_http_message *hm) {
+    int ret = strcmp(sist_cookie, "dark") == 0;
+    if (sist_cookie != buf) {
+        free(sist_cookie);
+    }
+
+    return ret;
+}
+
+void style(struct mg_connection *nc, struct http_message *hm) {
 
     if (client_requested_dark_theme(hm)) {
         send_response_line(nc, 200, sizeof(bundle_dark_css), "Content-Type: text/css");
@@ -134,9 +161,11 @@ void style(struct mg_connection *nc, struct mg_http_message *hm) {
         send_response_line(nc, 200, sizeof(bundle_css), "Content-Type: text/css");
         mg_send(nc, bundle_css, sizeof(bundle_css));
     }
+
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void img_sprite_skin_flat(struct mg_connection *nc, struct mg_http_message *hm) {
+void img_sprite_skin_flat(struct mg_connection *nc, struct http_message *hm) {
     if (client_requested_dark_theme(hm)) {
         send_response_line(nc, 200, sizeof(sprite_skin_flat_dark_png), "Content-Type: image/png");
         mg_send(nc, sprite_skin_flat_dark_png, sizeof(sprite_skin_flat_dark_png));
@@ -144,22 +173,25 @@ void img_sprite_skin_flat(struct mg_connection *nc, struct mg_http_message *hm)
         send_response_line(nc, 200, sizeof(sprite_skin_flat_png), "Content-Type: image/png");
         mg_send(nc, sprite_skin_flat_png, sizeof(sprite_skin_flat_png));
     }
+
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) {
+void thumbnail(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
 
-    if (hm->uri.len != 68) {
+    if (path->len != 68) {
-        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr)
+        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) path->len, path->p)
-        mg_http_reply(nc, 404, "", "Not found");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     char arg_file_md5[MD5_STR_LENGTH];
     char arg_index[MD5_STR_LENGTH];
 
-    memcpy(arg_index, hm->uri.ptr + 3, MD5_STR_LENGTH);
+    memcpy(arg_index, hm->uri.p + 3, MD5_STR_LENGTH);
     *(arg_index + MD5_STR_LENGTH - 1) = '\0';
-    memcpy(arg_file_md5, hm->uri.ptr + 3 + MD5_STR_LENGTH, MD5_STR_LENGTH);
+    memcpy(arg_file_md5, hm->uri.p + 3 + MD5_STR_LENGTH, MD5_STR_LENGTH);
     *(arg_file_md5 + MD5_STR_LENGTH - 1) = '\0';
 
     unsigned char md5_buf[MD5_DIGEST_LENGTH];
@@ -168,7 +200,8 @@ void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) {
     store_t *store = get_store(arg_index);
     if (store == NULL) {
         LOG_DEBUGF("serve.c", "Could not get store for index: %s", arg_index)
-        mg_http_reply(nc, 404, "", "Not found");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
@@ -179,24 +212,26 @@ void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) {
         mg_send(nc, data, data_len);
         free(data);
     }
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void search(struct mg_connection *nc, struct mg_http_message *hm) {
+void search(struct mg_connection *nc, struct http_message *hm) {
 
     if (hm->body.len == 0) {
         LOG_DEBUG("serve.c", "Client sent empty body, ignoring request")
-        mg_http_reply(nc, 500, "", "Invalid request");
+        mg_http_send_error(nc, 500, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.ptr, hm->body.len);
+    memcpy(body, hm->body.p, hm->body.len);
     *(body + hm->body.len) = '\0';
 
     char url[4096];
     snprintf(url, 4096, "%s/%s/_search", WebCtx.es_url, WebCtx.es_index);
 
-    nc->fn_data = web_post_async(url, body);
+    nc->user_data = web_post_async(url, body);
 }
 
 void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
@@ -218,13 +253,16 @@ void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
              idx->desc.rewrite_url, path_unescaped, name_unescaped, strlen(ext) == 0 ? "" : ".", ext);
 
     dyn_buffer_t encoded = url_escape(url);
-    dyn_buffer_write_char(&encoded, '\0');
+    mg_http_send_redirect(
-
+            nc, 308,
-    mg_http_reply(nc, 308, "Location: %s", encoded.buf);
+            (struct mg_str) MG_MK_STR_N(encoded.buf, encoded.cur),
+            (struct mg_str) MG_NULL_STR
+    );
     dyn_buffer_destroy(&encoded);
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, struct mg_http_message *hm) {
+void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, struct http_message *hm) {
 
     const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
     const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
@@ -245,10 +283,10 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
     LOG_DEBUGF("serve.c", "Serving file from disk: %s", full_path)
 
     char disposition[8192];
-    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s%s%s\"\r\n",
+    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s%s%s\"",
              name, strlen(ext) == 0 ? "" : ".", ext);
 
-    mg_http_serve_file(nc, hm, full_path, mime, disposition);
+    mg_http_serve_file(nc, hm, full_path, mg_mk_str(mime), mg_mk_str(disposition));
 }
 
 void index_info(struct mg_connection *nc) {
@@ -272,19 +310,22 @@ void index_info(struct mg_connection *nc) {
     mg_send(nc, json_str, strlen(json_str));
     free(json_str);
     cJSON_Delete(json);
+
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 
-void document_info(struct mg_connection *nc, struct mg_http_message *hm) {
+void document_info(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
 
-    if (hm->uri.len != MD5_STR_LENGTH + 2) {
+    if (path->len != MD5_STR_LENGTH + 2) {
-        LOG_DEBUGF("serve.c", "Invalid document_info path: %.*s", (int) hm->uri.len, hm->uri.ptr)
+        LOG_DEBUGF("serve.c", "Invalid document_info path: %.*s", (int) path->len, path->p)
-        mg_http_reply(nc, 404, "", "Not found");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     char arg_md5[MD5_STR_LENGTH];
-    memcpy(arg_md5, hm->uri.ptr + 3, MD5_STR_LENGTH);
+    memcpy(arg_md5, hm->uri.p + 3, MD5_STR_LENGTH);
     *(arg_md5 + MD5_STR_LENGTH - 1) = '\0';
 
     cJSON *doc = elastic_get_document(arg_md5);
@@ -293,14 +334,16 @@ void document_info(struct mg_connection *nc, struct mg_http_message *hm) {
     cJSON *index_id = cJSON_GetObjectItem(source, "index");
     if (index_id == NULL) {
         cJSON_Delete(doc);
-        mg_http_reply(nc, 404, "", "Not found");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     index_t *idx = get_index_by_id(index_id->valuestring);
     if (idx == NULL) {
         cJSON_Delete(doc);
-        mg_http_reply(nc, 404, "", "Not found");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
@@ -309,18 +352,21 @@ void document_info(struct mg_connection *nc, struct mg_http_message *hm) {
     mg_send(nc, json_str, (int) strlen(json_str));
     free(json_str);
     cJSON_Delete(doc);
+
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void file(struct mg_connection *nc, struct mg_http_message *hm) {
+void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
 
-    if (hm->uri.len != MD5_STR_LENGTH + 2) {
+    if (path->len != MD5_STR_LENGTH + 2) {
-        LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr)
+        LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) path->len, path->p)
-        mg_http_reply(nc, 404, "", "Not found");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     char arg_md5[MD5_STR_LENGTH];
-    memcpy(arg_md5, hm->uri.ptr + 3, MD5_STR_LENGTH);
+    memcpy(arg_md5, hm->uri.p + 3, MD5_STR_LENGTH);
     *(arg_md5 + MD5_STR_LENGTH - 1) = '\0';
 
     const char *next = arg_md5;
@@ -334,7 +380,8 @@ void file(struct mg_connection *nc, struct mg_http_message *hm) {
         index_id = cJSON_GetObjectItem(source, "index");
         if (index_id == NULL) {
             cJSON_Delete(doc);
-            mg_http_reply(nc, 404, "", "Not found");
+            mg_http_send_error(nc, 404, NULL);
+            nc->flags |= MG_F_SEND_AND_CLOSE;
             return;
         }
         cJSON *parent = cJSON_GetObjectItem(source, "parent");
@@ -348,7 +395,8 @@ void file(struct mg_connection *nc, struct mg_http_message *hm) {
 
     if (idx == NULL) {
         cJSON_Delete(doc);
-        mg_http_reply(nc, 404, "", "Not found");
+        nc->flags |= MG_F_SEND_AND_CLOSE;
+        mg_http_send_error(nc, 404, NULL);
         return;
     }
 
@@ -369,6 +417,8 @@ void status(struct mg_connection *nc) {
     }
 
     free(status);
+
+    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 typedef struct {
@@ -414,32 +464,35 @@ tag_req_t *parse_tag_request(cJSON *json) {
     return req;
 }
 
-void tag(struct mg_connection *nc, struct mg_http_message *hm) {
+void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
-    if (hm->uri.len != MD5_STR_LENGTH + 4) {
+    if (path->len != MD5_STR_LENGTH + 4) {
-        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr)
+        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) path->len, path->p)
-        mg_http_reply(nc, 404, "", "Not found");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     char arg_index[MD5_STR_LENGTH];
-    memcpy(arg_index, hm->uri.ptr + 5, MD5_STR_LENGTH);
+    memcpy(arg_index, hm->uri.p + 5, MD5_STR_LENGTH);
     *(arg_index + MD5_STR_LENGTH - 1) = '\0';
 
     if (hm->body.len < 2 || hm->method.len != 4 || memcmp(&hm->method, "POST", 4) == 0) {
         LOG_DEBUG("serve.c", "Invalid tag request")
-        mg_http_reply(nc, 404, "", "Not found");
+        mg_http_send_error(nc, 400, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     store_t *store = get_tag_store(arg_index);
     if (store == NULL) {
         LOG_DEBUGF("serve.c", "Could not get tag store for index: %s", arg_index)
-        mg_http_reply(nc, 404, "", "Not found");
+        mg_http_send_error(nc, 404, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.ptr, hm->body.len);
+    memcpy(body, hm->body.p, hm->body.len);
     *(body + hm->body.len) = '\0';
     cJSON *json = cJSON_Parse(body);
 
@@ -448,7 +501,8 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) {
         LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index)
         cJSON_Delete(json);
         free(body);
-        mg_http_reply(nc, 400, "", "Invalid request");
+        mg_http_send_error(nc, 400, NULL);
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
@@ -491,7 +545,7 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) {
 
         char url[4096];
         snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
-        nc->fn_data = web_post_async(url, buf);
+        nc->user_data = web_post_async(url, buf);
 
     } else {
         cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name));
@@ -511,7 +565,7 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) {
 
         char url[4096];
         snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
-        nc->fn_data = web_post_async(url, buf);
+        nc->user_data = web_post_async(url, buf);
     }
 
     char *json_str = cJSON_PrintUnformatted(arr);
@@ -525,73 +579,92 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) {
     free(body);
 }
 
-int validate_auth(struct mg_connection *nc, struct mg_http_message *hm) {
+int validate_auth(struct mg_connection *nc, struct http_message *hm) {
     char user[256] = {0,};
     char pass[256] = {0,};
 
-    mg_http_creds(hm, user, sizeof(user), pass, sizeof(pass));
+    int ret = mg_get_http_basic_auth(hm, user, sizeof(user), pass, sizeof(pass));
-    if (strcmp(user, WebCtx.auth_user) != 0 || strcmp(pass, WebCtx.auth_pass) != 0) {
+    if (ret == -1 || strcmp(user, WebCtx.auth_user) != 0 || strcmp(pass, WebCtx.auth_pass) != 0) {
-        mg_http_reply(nc, 401, "WWW-Authenticate: Basic realm=\"sist2\"", "");
+        mg_printf(nc, "HTTP/1.1 401 Unauthorized\r\n"
+                      "WWW-Authenticate: Basic realm=\"sist2\"\r\n"
+                      "Content-Length: 0\r\n\r\n");
+        nc->flags |= MG_F_SEND_AND_CLOSE;
         return FALSE;
     }
     return TRUE;
 }
 
-static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(void *fn_data)) {
+static void ev_router(struct mg_connection *nc, int ev, void *p) {
+    struct mg_str scheme;
+    struct mg_str user_info;
+    struct mg_str host;
+    unsigned int port;
+    struct mg_str path;
+    struct mg_str query;
+    struct mg_str fragment;
+
+    if (ev == MG_EV_HTTP_REQUEST) {
+        struct http_message *hm = (struct http_message *) p;
+
+        if (mg_parse_uri(hm->uri, &scheme, &user_info, &host, &port, &path, &query, &fragment) != 0) {
+            mg_http_send_error(nc, 400, NULL);
+            nc->flags |= MG_F_SEND_AND_CLOSE;
+            return;
+        }
 
-    if (ev == MG_EV_HTTP_MSG) {
-        struct mg_http_message *hm = (struct mg_http_message *) ev_data;
 
         if (WebCtx.auth_enabled == TRUE) {
             if (!validate_auth(nc, hm)) {
-                nc->is_closing = 1;
                 return;
             }
         }
 
-        if (mg_http_match_uri(hm, "/")) {
+        if (is_equal(&path, &((struct mg_str) MG_MK_STR("/")))) {
             search_index(nc);
-        } else if (mg_http_match_uri(hm, "/css")) {
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/css")))) {
             style(nc, hm);
-        } else if (mg_http_match_uri(hm, "/stats")) {
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/stats")))) {
             stats(nc);
-        } else if (mg_http_match_uri(hm, "/jslib")) {
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/jslib")))) {
             javascript_lib(nc);
-        } else if (mg_http_match_uri(hm, "/jssearch")) {
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/jssearch")))) {
             javascript_search(nc);
-        } else if (mg_http_match_uri(hm, "/img/sprite-skin-flat.png")) {
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/img/sprite-skin-flat.png")))) {
             img_sprite_skin_flat(nc, hm);
-        } else if (mg_http_match_uri(hm, "/es")) {
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/es")))) {
             search(nc, hm);
-        } else if (mg_http_match_uri(hm, "/i")) {
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/i")))) {
             index_info(nc);
-        } else if (mg_http_match_uri(hm, "/status")) {
+        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/status")))) {
             status(nc);
-        } else if (mg_http_match_uri(hm, "/f/*")) {
+        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/f/")))) {
-            file(nc, hm);
+            file(nc, hm, &path);
-        } else if (mg_http_match_uri(hm, "/t/*/*")) {
+        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/t/")))) {
-            thumbnail(nc, hm);
+            thumbnail(nc, hm, &path);
-        } else if (mg_http_match_uri(hm, "/s/*/*")) {
+        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/s/")))) {
-            stats_files(nc, hm);
+            stats_files(nc, hm, &path);
-        } else if (mg_http_match_uri(hm, "/tag/*")) {
+        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/tag/")))) {
-            if (WebCtx.tag_auth_enabled == TRUE && !validate_auth(nc, hm)) {
+            if (WebCtx.tag_auth_enabled == TRUE) {
-                nc->is_closing = 1;
+                if (!validate_auth(nc, hm)) {
                     return;
                 }
-            tag(nc, hm);
+            }
-        } else if (mg_http_match_uri(hm, "/d/*")) {
+            tag(nc, hm, &path);
-            document_info(nc, hm);
+        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/d/")))) {
+            document_info(nc, hm, &path);
         } else {
-            mg_http_reply(nc, 404, "", "Page not found");
+            mg_http_send_error(nc, 404, NULL);
+            nc->flags |= MG_F_SEND_AND_CLOSE;
         }
 
     } else if (ev == MG_EV_POLL) {
-        if (nc->fn_data != NULL) {
+        if (nc->user_data != NULL) {
             //Waiting for ES reply
-            subreq_ctx_t *ctx = (subreq_ctx_t *) nc->fn_data;
+            subreq_ctx_t *ctx = (subreq_ctx_t *) nc->user_data;
             web_post_async_poll(ctx);
 
             if (ctx->done == TRUE) {
+
                 response_t *r = ctx->response;
 
                 if (r->status_code == 200) {
@@ -611,14 +684,14 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
                         free(json_str);
                         free(tmp);
                     }
-
+                    mg_http_send_error(nc, 500, NULL);
-                    mg_http_reply(nc, 500, "", "");
                 }
 
                 free_response(r);
                 free(ctx->data);
                 free(ctx);
-                nc->fn_data = NULL;
+                nc->flags |= MG_F_SEND_AND_CLOSE;
+                nc->user_data = NULL;
             }
         }
     }
@@ -629,18 +702,15 @@ void serve(const char *listen_address) {
     printf("Starting web server @ http://%s\n", listen_address);
 
     struct mg_mgr mgr;
-    mg_mgr_init(&mgr);
+    mg_mgr_init(&mgr, NULL);
 
-    int ok = 1;
+    struct mg_connection *nc = mg_bind(&mgr, listen_address, ev_router);
-
-    struct mg_connection *nc = mg_http_listen(&mgr, listen_address, ev_router, NULL);
     if (nc == NULL) {
         LOG_FATALF("serve.c", "Couldn't bind web server on address %s", listen_address)
     }
+    mg_set_protocol_http_websocket(nc);
 
-    while (ok) {
+    for (;;) {
         mg_mgr_poll(&mgr, 10);
     }
-    mg_mgr_free(&mgr);
-    LOG_INFO("serve.c", "Finished web event loop")
 }

tests/test_scan.py

@@ -17,19 +17,17 @@ def copy_files(files):
 
 
 def sist2(*args):
-    print("./sist2 " + " ".join(args))
-
     return subprocess.check_output(
-        args=["./sist2", *args],
+        args=["./sist2_debug", *args],
     )
 
 
 def sist2_index(files, *args):
     path = copy_files(files)
 
-    shutil.rmtree("test_i", ignore_errors=True)
+    shutil.rmtree("i", ignore_errors=True)
-    sist2("scan", path, "-o", "test_i", *args)
+    sist2("scan", path, "-o", "i", *args)
-    return iter(sist2_index_to_dict("test_i"))
+    return iter(sist2_index_to_dict("i"))
 
 
 def sist2_incremental_index(files, func=None, *args):
@@ -38,14 +36,14 @@ def sist2_incremental_index(files, func=None, *args):
     if func:
         func(path)
 
-    shutil.rmtree("test_i_inc", ignore_errors=True)
+    shutil.rmtree("i_inc", ignore_errors=True)
-    sist2("scan", path, "-o", "test_i_inc", "--incremental", "test_i", *args)
+    sist2("scan", path, "-o", "i_inc", "--incremental", "i", *args)
-    return iter(sist2_index_to_dict("test_i_inc"))
+    return iter(sist2_index_to_dict("i_inc"))
 
 
 def sist2_index_to_dict(index):
     res = subprocess.check_output(
-        args=["./sist2", "index", "--print", index],
+        args=["./sist2_debug", "index", "--print", index],
     )
 
     for line in res.splitlines():

third-party/argparse

@@ -1 +1 @@
-Subproject commit ffd9c23427d0cb105e27f27f0cf97b463b6a8bf8
+Subproject commit 3f4e3594a6891b942d5a711781d5425111aa13bf

third-party/libscan

@@ -1 +1 @@
-Subproject commit 9be4f02851107edac65894a1fdde16a80cad43ac
+Subproject commit ae9fadec473e6e4ade05259fe359c5366c3f3af6