update docker image version in readme

v2.10.1

.dockerignore

@@ -0,0 +1,25 @@
+.idea
+*/thumbs
+*.cbp
+CMakeCache.txt
+CMakeFiles
+cmake-build-debug
+cmake_install.cmake
+Makefile
+*.out
+LOG
+sist2*
+index.sist2/
+bundle*.css
+bundle.js
+**/*.a
+**/vgcore.*
+build/
+.git/
+third-party/libscan/libscan-test-files/
+**/ext_ffmpeg
+**/ext_libmobi
+**/scan_a_test
+Dockerfile
+*.idx/
+VERSION

.drone.yml

@@ -0,0 +1,72 @@
+kind: pipeline
+type: docker
+name: amd64
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+  - name: build
+    image: simon987/sist2-build
+    commands:
+      - ./ci/build.sh
+  - name: docker
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: simon987/sist2
+      context: ./
+      dockerfile: ./Dockerfile
+      auto_tag: true
+      auto_tag_suffix: x64-linux
+      when:
+        event:
+          - tag
+  - name: scp files
+    image: appleboy/drone-scp
+    settings:
+      host:
+        from_secret: SSH_HOST
+      port:
+        from_secret: SSH_PORT
+      user:
+        from_secret: SSH_USER
+      key:
+        from_secret: SSH_KEY
+      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      source:
+        - ./VERSION
+        - ./sist2-x64-linux
+        - ./sist2-x64-linux-debug
+
+---
+kind: pipeline
+type: docker
+name: arm64
+
+platform:
+  arch: arm64
+
+steps:
+  - name: build
+    image: simon987/sist2-build-arm64
+    commands:
+      - ./ci/build_arm64.sh
+  - name: scp files
+    image: appleboy/drone-scp
+    settings:
+      host:
+        from_secret: SSH_HOST
+      port:
+        from_secret: SSH_PORT
+      user:
+        from_secret: SSH_USER
+      key:
+        from_secret: SSH_KEY
+      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      source:
+        - ./sist2-arm64-linux

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,40 @@
+---
+name: "🐞 Bug Report"
+about: Submit a bug report 
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Device Information (please complete the following information):**
+ - OS: `[e.g., Ubuntu 20.04, WSL2]`
+ - Deployment: `[Linux, Linux ARM64 or Docker]`
+ - Browser *(if relevant)*: `[e.g., chrome, safari]`
+ - SIST2 Version: `[e.g., v2.9.0]`
+ - Elasticsearch Version *(if relevant)* : ``
+
+**Command with arguments** 
+<!-- `ex: "scan ~/Documents -o ./i2 --threads 3 -q 1.0` -->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**Steps To Reproduce**
+Please be specific!
+1. Go to '...'
+2. Click on '....'
+3. etc.
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Actual Behavior**
+<!-- A clear and concise description of what actually happens. -->
+
+**Screenshots**
+<!-- If applicable, add screenshots to help explain your problem. -->
+
+**Additional context**
+<!-- Add any other context about the problem here. If applicable, please include why you think the bug is occurring and/or troubleshooting you have already performed. -->
+<!-- If the issue is related to the `scan` module, please attach the files necessary to reproduce the error or email them to me[at]simon987.net. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: SIST2 Documentation
+    url: https://github.com/simon987/sist2/blob/master/docs/USAGE.md
+    about: Check out the SIST2 documentation for answers to common questions

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: "🚀 Feature Request"
+about: Suggest an idea for SIST2
+title: ''
+assignees: ''
+
+---
+**Which SIST2 component is your Feature Request related to?**
+<!-- e.g., Scan, Index, or Web? -->
+
+**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what the problem is. e.g., "I'm always frustrated when [...]" -->
+
+**What would you like to see happen?**
+<!-- A clear and concise description of what you want to happen. -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.gitignore

@@ -1,6 +1,5 @@
 .idea
 thumbs
-test
 *.cbp
 CMakeCache.txt
 CMakeFiles
@@ -17,3 +16,5 @@ bundle.js
 vgcore.*
 build/
 third-party/
+*.idx/
+VERSION

.teamcity/settings.kts

@@ -1,69 +0,0 @@
-import jetbrains.buildServer.configs.kotlin.v2019_2.*
-import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.ExecBuildStep
-import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.exec
-import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs
-import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot
-
-/*
-The settings script is an entry point for defining a TeamCity
-project hierarchy. The script should contain a single call to the
-project() function with a Project instance or an init function as
-an argument.
-
-VcsRoots, BuildTypes, Templates, and subprojects can be
-registered inside the project using the vcsRoot(), buildType(),
-template(), and subProject() methods respectively.
-
-To debug settings scripts in command-line, run the
-
-    mvnDebug org.jetbrains.teamcity:teamcity-configs-maven-plugin:generate
-
-command and attach your debugger to the port 8000.
-
-To debug in IntelliJ Idea, open the 'Maven Projects' tool window (View
--> Tool Windows -> Maven Projects), find the generate task node
-(Plugins -> teamcity-configs -> teamcity-configs:generate), the
-'Debug' option is available in the context menu for the task.
-*/
-
-version = "2019.2"
-
-project {
-
-    vcsRoot(HttpsGithubComSimon987sist2refsHeadsMaster)
-
-    buildType(Build)
-}
-
-object Build : BuildType({
-    name = "Build"
-
-    artifactRules = """
-        sist2
-        sist2_scan
-    """.trimIndent()
-
-    vcs {
-        root(HttpsGithubComSimon987sist2refsHeadsMaster)
-    }
-
-    steps {
-        exec {
-            name = "Build"
-            path = "./ci/build.sh"
-            dockerImage = "simon987/general_ci"
-            dockerImagePlatform = ExecBuildStep.ImagePlatform.Linux
-            dockerPull = true
-        }
-    }
-
-    triggers {
-        vcs {
-        }
-    }
-})
-
-object HttpsGithubComSimon987sist2refsHeadsMaster : GitVcsRoot({
-    name = "https://github.com/simon987/sist2#refs/heads/master"
-    url = "https://github.com/simon987/sist2"
-})

CMakeLists.txt

@@ -5,12 +5,16 @@ project(sist2 C)
 
 option(SIST_DEBUG "Build a debug executable" on)
 
+set(BUILD_TESTS on)
 add_subdirectory(third-party/libscan)
 set(ARGPARSE_SHARED off)
 add_subdirectory(third-party/argparse)
 
-add_executable(
+add_executable(sist2
-        sist2
+
+        # argparse
+        third-party/argparse/argparse.h third-party/argparse/argparse.c
+
         src/main.c
         src/sist.h
         src/io/walk.h src/io/walk.c
@@ -25,25 +29,22 @@ add_executable(
         src/util.c src/util.h
         src/ctx.h src/types.h
         src/log.c src/log.h
-
-        # argparse
-        third-party/argparse/argparse.h third-party/argparse/argparse.c
-
         src/cli.c src/cli.h
-        src/stats.c src/stats.h src/ctx.c)
+        src/stats.c src/stats.h src/ctx.c
+        src/parsing/sidecar.c src/parsing/sidecar.h)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
 set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
 
+find_package(PkgConfig REQUIRED)
+
+pkg_search_module(GLIB REQUIRED glib-2.0)
+
 find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
-find_package(unofficial-glib CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
-find_library(UUID_LIB NAMES uuid)
 find_package(CURL CONFIG REQUIRED)
 
-#find_package(OpenSSL REQUIRED)
-
 
 target_include_directories(
         sist2 PUBLIC
@@ -51,6 +52,7 @@ target_include_directories(
         ${CMAKE_SOURCE_DIR}/third-party/utf8.h/
         ${CMAKE_SOURCE_DIR}/third-party/libscan/
         ${CMAKE_SOURCE_DIR}/
+        ${GLIB_INCLUDE_DIRS}
 )
 
 target_compile_options(
@@ -67,13 +69,13 @@ if (SIST_DEBUG)
             -fstack-protector
             -fno-omit-frame-pointer
             -fsanitize=address
-            -O2
+            -fno-inline
+#            -O2
     )
     target_link_options(
             sist2
             PRIVATE
             -fsanitize=address
-#            -static
     )
     set_target_properties(
             sist2
@@ -81,7 +83,6 @@ if (SIST_DEBUG)
             OUTPUT_NAME sist2_debug
     )
 else ()
-#    set(VCPKG_BUILD_TYPE release)
     target_compile_options(
             sist2
             PRIVATE
@@ -104,15 +105,15 @@ target_link_libraries(
         lmdb
         cjson
         argparse
-        unofficial::glib::glib
+        ${GLIB_LDFLAGS}
         unofficial::mongoose::mongoose
-#        OpenSSL::SSL OpenSSL::Crypto
         CURL::libcurl
 
-        ${UUID_LIB}
         pthread
         magic
 
+        c
+
         scan
 )
 

Docker/build.sh

@@ -1,14 +0,0 @@
-rm ./sist2 sist2_debug
-cp ../sist2.gz .
-gzip -d sist2.gz
-strip sist2
-
-version=$(./sist2 --version)
-
-echo "Version ${version}"
-docker build . -t simon987/sist2:${version} -t simon987/sist2:latest
-
-docker push simon987/sist2:${version}
-docker push simon987/sist2:latest
-
-docker run --rm simon987/sist2 -v

DockerArm64/build.sh

@@ -1,13 +0,0 @@
-rm ./sist2_arm64
-cp ../sist2_arm64.gz .
-gzip -d sist2_arm64.gz
-
-version=$(./sist2_arm64 --version)
-
-echo "Version ${version}"
-docker build . -t simon987/sist2-arm64:"${version}" -t simon987/sist2-arm64:latest
-
-docker push simon987/sist2-arm64:"${version}"
-docker push simon987/sist2-arm64:latest
-
-docker run --rm simon987/sist2-arm64 -v

Dockerfile

@@ -1,9 +1,15 @@
-FROM ubuntu:19.10
+FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
-RUN apt update
+WORKDIR /build/
-RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 \
+ADD . /build/
- curl libtiff5 libpng16-16 libpcre3
+RUN cmake -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
+RUN make -j$(nproc)
+RUN strip sist2
+
+FROM ubuntu:20.10
+
+RUN apt update && apt install -y curl
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -12,9 +18,9 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
     curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
 
-ADD sist2 /root/sist2
+COPY --from=build /build/sist2 /root/sist2
 
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8

Dockerfile.arm64

@@ -1,9 +1,15 @@
-FROM ubuntu:19.10
+FROM simon987/sist2-build-arm64 as build
 MAINTAINER simon987 <me@simon987.net>
 
-RUN apt update
+WORKDIR /build/
-RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 \
+ADD . /build/
- curl libtiff5 libpng16-16 libpcre3
+RUN cmake -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
+RUN make -j$(nproc)
+RUN strip sist2
+
+FROM ubuntu:20.10
+
+RUN apt update && apt install -y curl
 
 RUN mkdir -p /usr/share/tessdata && \
     cd /usr/share/tessdata/ && \
@@ -12,11 +18,11 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
     curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
     curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
-    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
+    curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata
 
-ADD sist2_arm64 /root/sist2
+COPY --from=build /build/sist2 /root/sist2
 
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 
-ENTRYPOINT ["/root/sist2"]
+ENTRYPOINT ["/root/sist2"]

README.md

@@ -1,8 +1,8 @@
 ![GitHub](https://img.shields.io/github/license/simon987/sist2.svg)
 [![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
-[![Development snapshots](https://ci.simon987.net/app/rest/builds/buildType(Sist2_Build)/statusIcon)](https://files.simon987.net/artifacts/Sist2/Build/)
+[![Development snapshots](https://ci.simon987.net/api/badges/simon987/sist2/status.svg)](https://files.simon987.net/.gate/sist2/simon987_sist2/)
 
-**Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
+**Demo**: [sist2.simon987.net](https://sist2.simon987.net/?i=Demo%20files)
 
 # sist2
 
@@ -25,14 +25,12 @@ sist2 (Simple incremental search tool)
 * OCR support with tesseract \*\*\*
 * Stats page & disk utilisation visualization
 
-
 \* See [format support](#format-support)    
 \*\* See [Archive files](#archive-files)    
-\*\*\* See [OCR](#ocr)    
+\*\*\* See [OCR](#ocr)
 
 ![stats](docs/stats.png)
 
-
 ## Getting Started
 
 1. Have an Elasticsearch (>= 6.X.X) instance running
@@ -52,14 +50,12 @@ sist2 (Simple incremental search tool)
         ```
 1. Download sist2 executable
     1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) *
-    1. *(or)* Download a [development snapshot](https://files.simon987.net/artifacts/Sist2/Build/) *(Not recommended!)*
+    1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not recommended!)*
-    1. *(or)* `docker pull simon987/sist2:latest`
+    1. *(or)* `docker pull simon987/sist2:2.10.3-x64-linux`
 
 1. See [Usage guide](docs/USAGE.md)
-   
-
-\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)    
 
+\* *Windows users*: **sist2** runs under [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
 
 ## Example usage
 
@@ -69,69 +65,88 @@ See [Usage guide](docs/USAGE.md) for more details
 1. Push index to Elasticsearch: `sist2 index ./docs_idx`
 1. Start web interface: `sist2 web ./docs_idx`
 
-
 ## Format support
 
-File type | Library | Content | Thumbnail | Metadata
+File type | Library | Content | Thumbnail | Metadata
 :---|:---|:---|:---|:---
-pdf,xps,fb2,epub | MuPDF | text+ocr | yes | title |
+pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title |
 cbz,cbr | *(none)* | - | yes | - |
 `audio/*` | ffmpeg | - | yes | ID3 tags |
 `video/*` | ffmpeg | - | yes | title, comment, artist |
-`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
+`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
-raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags |
+raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags, GPS tags |
 ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
 `text/plain` | *(none)* | yes | no | - |
 html, xml | *(none)* | yes | no | - |
 tar, zip, rar, 7z, ar ...  | Libarchive | yes\* | - | no |
 docx, xlsx, pptx | *(none)* | yes | if embedded | creator, modified_by, title |
+doc (MS Word 97-2003) | antiword | yes | yes | author, title |
 mobi, azw, azw3 | libmobi | yes | no | author, title |
 
 \* *See [Archive files](#archive-files)*
- 
+
 ### Archive files
-**sist2** will scan files stored into archive files (zip, tar, 7z...) as if
+
-they were directly in the file system. Recursive (archives inside archives)
+**sist2** will scan files stored into archive files (zip, tar, 7z...) as if they were directly in the file system.
+Recursive (archives inside archives)
 scan is also supported.
 
 **Limitations**:
-* Support for parsing media files with formats that require *seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.) 
+
+* Support for parsing media files with formats that require *seek* (e.g. `.gif`, `.mp4` w/ fragmented metadata etc.)
   is limitted (see `--mem-buffer` option)
 * Archive files are scanned sequentially, by a single thread. On systems where
-**sist2** is not I/O bound, scans might be faster when larger archives are split
+  **sist2** is not I/O bound, scans might be faster when larger archives are split into smaller parts.
- into smaller parts.
+
- 
- 
 ### OCR
 
 You can enable OCR support for pdf,xps,fb2,epub file types with the
-`--ocr <lang>` option. Download the language data files with your
+`--ocr <lang>` option. Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
-package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
+directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
-The `simon987/sist2` image comes with common languages 
+The `simon987/sist2` image comes with common languages
 (hin, jpn, eng, fra, rus, spa) pre-installed.
 
 Examples
+
 ```bash
 sist2 scan --ocr jpn ~/Books/Manga/
 sist2 scan --ocr eng ~/Books/Textbooks/
 ```
 
-
 ## Build from source
 
-You can compile **sist2** by yourself if you don't want to use the pre-compiled
+You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries
-binaries (GCC 7+ required).
+
+### With docker (recommended)
+
+```bash
+git clone --recursive https://github.com/simon987/sist2/
+cd sist2
+docker build . -f ./Dockerfile -t my-sist2-image
+docker run --rm my-sist2-image cat /root/sist2 > sist2-x64-linux
+```
+
+### On a linux computer
 
 1. Install compile-time dependencies
 
    ```bash
-   vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libuuid libmagic libraw curl[core,ssl] jbig2dec brotli libmupdf
+   apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git
    ```
+   
+1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
 
-2. Build
+1. Install vcpkg dependencies
+
+    ```bash
+    vcpkg install curl[core,openssl]
+    vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libuuid libmagic libraw jasper lcms gumbo
+    ```
+
+1. Build
     ```bash
     git clone --recursive https://github.com/simon987/sist2/
-    cmake -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
+    cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

ci/build.sh

@@ -2,16 +2,18 @@
 
 VCPKG_ROOT="/vcpkg"
 
-rm *.gz
+rm *.gz &>/dev/null
+
+git submodule update --init --recursive
 
 rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_DEBUG=off -DVCPKG_BUILD_TYPE=release -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
-make -j 12
+make -j $(nproc)
 strip sist2
-gzip -9 sist2
+./sist2 -v > VERSION
+mv sist2 sist2-x64-linux
 
 rm -rf CMakeFiles CMakeCache.txt
-cmake -DSIST_DEBUG=on -DVCPKG_BUILD_TYPE=debug -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
+cmake -DSIST_DEBUG=on -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
-make -j 12
+make -j  $(nproc)
-cp /usr/lib/x86_64-linux-gnu/libasan.so.2.0.0 libasan.so.2
+mv sist2_debug sist2-x64-linux-debug
-tar -czf sist2_debug.tar.gz sist2_debug libasan.so.2

ci/build_arm64.sh

@@ -2,11 +2,12 @@
 
 VCPKG_ROOT="/vcpkg"
 
-rm *.gz
+rm *.gz &>/dev/null
+
+git submodule update --init --recursive
 
 rm -rf CMakeFiles CMakeCache.txt
 cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" .
-make -j 4
+make -j $(nproc)
 strip sist2
-mv sist2 sist2_arm64
+mv sist2 sist2-arm64-linux
-gzip -9 sist2_arm64

docs/USAGE.md

@@ -16,6 +16,7 @@
     * [link to specific indices](#link-to-specific-indices)
 * [exec-script](#exec-script)
 * [tagging](#tagging)
+* [sidecar files](#sidecar-files)
 
 ```
 Usage: sist2 scan [OPTION]... PATH
@@ -45,6 +46,7 @@ Scan options
     --fast                        Only index file names & mime type
     --treemap-threshold=<str>     Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
     --mem-buffer=<int>            Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000
+    --read-subtitles              Read subtitles from media files
 
 Index options
     -t, --threads=<int>           Number of threads. DEFAULT=1
@@ -52,6 +54,8 @@ Index options
     --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
     -p, --print                   Just print JSON documents to stdout.
     --script-file=<str>           Path to user script.
+    --mappings-file=<str>         Path to Elasticsearch mappings.
+    --settings-file=<str>         Path to Elasticsearch settings.
     --async-script                Execute user script asynchronously.
     --batch-size=<int>            Index batch size. DEFAULT: 100
     -f, --force-reset             Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
@@ -88,7 +92,7 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
     Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
     will be copied to the new index and will not be parsed again.
 * `-o, --output` Output directory. 
-* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
+* `--rewrite-url` Set the `rewrite_url` option for the web module (See [rewrite_url](#rewrite_url)) 
 * `--name` Set the `name` option for the web module
 * `--depth` Maximum scan dept. Set to 0 only scan files directly in the root directory, set to -1 for infinite depth
 * `--archive` Archive file mode.
@@ -120,6 +124,7 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
     larger than this number will be read sequentially and no *seek* operations will be supported.
 
     To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
+* `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
 
 ### Scan examples
 
@@ -153,10 +158,13 @@ documents.idx/
 ├── agg_mime.csv
 ├── agg_date.csv
 ├── add_size.csv
-├── thumbs
+├── thumbs/
 |   ├── data.mdb
 |   └── lock.mdb
-└── tags
+├── tags/
+|   ├── data.mdb
+|   └── lock.mdb
+└── meta/
     ├── data.mdb
     └── lock.mdb
 ```
@@ -183,9 +191,11 @@ by a third party application. The 'external' index must have the following forma
 my_index/
 ├── descriptor.json
 ├── _index_0
-└── thumbs
+└── thumbs/
-    ├── data.mdb
+|   ├── data.mdb
-    └── lock.mdb
+|   └── lock.mdb
+└── meta/
+    └── <empty>
 ```
 
 *descriptor.json*:
@@ -233,9 +243,11 @@ The `_text.*` items will be indexed and searchable as **text** fields (fuzzy sea
 
 *thumbs/*:
 
-LMDB key-value store. Keys are **binary** 128-bit UUID4s (`_id` field)
+LMDB key-value store. Keys are **binary** 16-byte md5 hash* (`_id` field)
 and values are raw image bytes.
 
+*\* Hash is calculated from the full path of the file, including the extension, relative to the index root*
+
 Importing an external `binary` type index is technically possible but
 it is currently unsupported and has no guaranties of back/forward compatibility.
 
@@ -251,6 +263,10 @@ it is currently unsupported and has no guaranties of back/forward compatibility.
     Print index in JSON format to stdout.
  * `--script-file` 
     Path to user script. See [Scripting](scripting.md).
+ * `--mappings-file`
+    Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used.
+ * `--settings-file`
+    Path to custom Elasticsearch settings. *(See above)*
  * `--async-script` 
     Use `wait_for_completion=false` elasticsearch option while executing user script.
      (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html))
@@ -260,7 +276,6 @@ it is currently unsupported and has no guaranties of back/forward compatibility.
     down the process.
  * `-f, --force-reset` 
     Reset Elasticsearch mappings and settings.
-    **(You must use this option the first time you use the index command)**.
     
 ### Index examples
 
@@ -342,10 +357,48 @@ You can safely copy the `/tags/` database to another index.
 See [Automatic tagging](#automatic-tagging) for information about tag 
  hierarchies and tag colors.
 
-\* *It can take a few seconds to take effect in new search queries, and the page needs 
+\* *It can take a few seconds to take effect in new search queries.*
-    to be reloaded for the tag tab to update*
 
 
 ### Automatic tagging
 
 See [scripting](scripting.md) documentation.
+
+# Sidecar files
+
+When scanning, sist2 will read metadata from `.s2meta` JSON files and overwrite the 
+original document's metadata. Sidecar metadata files will also work inside archives.
+Sidecar files themselves are not saved in the index.
+
+This feature is useful to leverage third-party applications such as speech-to-text or
+OCR to add additional metadata to a file.
+
+**Example**
+
+```
+~/Documents/
+├── Video.mp4
+└── Video.mp4.s2meta
+```
+
+The sidecar file must have exactly the same file path and the `.s2meta` suffix.
+
+`Video.mp4.s2meta`:
+```json
+{
+  "content": "This sidecar file will overwrite some metadata fields of Video.mp4",
+  "author": "Some author",
+  "duration": 12345,
+  "bitrate": 67890,
+  "some_arbitrary_field": [1,2,3]
+}
+```
+
+```
+sist2 scan ~/Documents -o ./docs.idx
+sist2 index ./docs.idx
+```
+
+*NOTE*: It is technically possible to overwrite the `tag` value using sidecar files, however,
+it is not currently possible to restore both manual tags and sidecar tags without user scripts
+while reindexing.

schema/mappings.json

@@ -30,6 +30,10 @@
     "mime": {
       "type": "keyword"
     },
+    "parent": {
+      "type": "keyword",
+      "index": false
+    },
     "thumbnail": {
       "type": "keyword",
       "index": false
@@ -101,10 +105,10 @@
       "analyzer": "my_nGram",
       "type": "text"
     },
-    "_keyword.*":  {
+    "_keyword.*": {
       "type": "keyword"
     },
-    "_text.*":  {
+    "_text.*": {
       "analyzer": "content_analyzer",
       "type": "text",
       "fields": {
@@ -161,6 +165,30 @@
     "exif_user_comment": {
       "type": "text"
     },
+    "exif_gps_longitude_ref": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_longitude_dms": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_longitude_dec": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_latitude_ref": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_latitude_dms": {
+      "type": "keyword",
+      "index": false
+    },
+    "exif_gps_latitude_dec": {
+      "type": "keyword",
+      "index": false
+    },
     "author": {
       "type": "text"
     },

scripts/mime.csv

@@ -449,3 +449,4 @@ image/x-sony-arw, arw
 image/x-sony-sr2, sr2
 image/x-sony-srf, srf
 image/x-epson-erf, erf
+sist2/sidecar, s2meta

scripts/mime.py

@@ -3,6 +3,7 @@ noparse = set()
 ext_in_hash = set()
 
 major_mime = {
+    "sist2": 0,
     "model": 1,
     "example": 2,
     "message": 3,
@@ -122,7 +123,11 @@ def mime_id(mime):
     elif mime in raw:
         mime_id += " | 0x00800000"
     elif mime == "application/x-empty":
+        cnt -= 1
         return "1"
+    elif mime == "sist2/sidecar":
+        cnt -= 1
+        return "2"
     return mime_id
 
 

scripts/reset.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+make clean
+rm -rf CMakeFiles/ CMakeCache.txt Makefile \
+  third-party/libscan/CMakeFiles third-party/libscan/CMakeCache.txt third-party/libscan/third-party/ext_ffmpeg \
+  third-party/libscan/third-party/ext_libmobi third-party/libscan/Makefile

src/cli.c

@@ -56,6 +56,12 @@ void scan_args_destroy(scan_args_t *args) {
 
 void index_args_destroy(index_args_t *args) {
     //todo
+    if (args->es_mappings_path) {
+        free(args->es_mappings);
+    }
+    if (args->es_settings_path) {
+        free(args->es_settings);
+    }
     free(args);
 }
 
@@ -221,35 +227,37 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
     LOG_DEBUGF("cli.c", "arg path=%s", args->path)
     LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
+    LOG_DEBUGF("cli.c", "arg archive_passphrase=%s", args->archive_passphrase)
     LOG_DEBUGF("cli.c", "arg tesseract_lang=%s", args->tesseract_lang)
     LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path)
     LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex)
     LOG_DEBUGF("cli.c", "arg fast=%d", args->fast)
+    LOG_DEBUGF("cli.c", "arg fast_epub=%d", args->fast_epub)
     LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold)
     LOG_DEBUGF("cli.c", "arg max_memory_buffer=%d", args->max_memory_buffer)
 
     return 0;
 }
 
-int load_script(const char *script_path, char **dst) {
+int load_external_file(const char *file_path, char **dst) {
     struct stat info;
-    int res = stat(script_path, &info);
+    int res = stat(file_path, &info);
 
     if (res == -1) {
-        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
         return 1;
     }
 
-    int fd = open(script_path, O_RDONLY);
+    int fd = open(file_path, O_RDONLY);
     if (fd == -1) {
-        fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno));
+        LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno))
         return 1;
     }
 
     *dst = malloc(info.st_size + 1);
     res = read(fd, *dst, info.st_size);
     if (res < 0) {
-        fprintf(stderr, "Error reading script file '%s': %s\n", script_path, strerror(errno));
+        LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno))
         return 1;
     }
 
@@ -293,7 +301,19 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
     }
 
     if (args->script_path != NULL) {
-        if (load_script(args->script_path, &args->script) != 0) {
+        if (load_external_file(args->script_path, &args->script) != 0) {
+            return 1;
+        }
+    }
+
+    if (args->es_settings_path != NULL) {
+        if (load_external_file(args->es_settings_path, &args->es_settings) != 0) {
+            return 1;
+        }
+    }
+
+    if (args->es_mappings_path != NULL) {
+        if (load_external_file(args->es_mappings_path, &args->es_mappings) != 0) {
             return 1;
         }
     }
@@ -309,6 +329,10 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
     LOG_DEBUGF("cli.c", "arg async_script=%s", args->async_script)
     LOG_DEBUGF("cli.c", "arg script=%s", args->script)
     LOG_DEBUGF("cli.c", "arg print=%d", args->print)
+    LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path)
+    LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings)
+    LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path)
+    LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings)
     LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size)
     LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset)
 
@@ -445,7 +469,7 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) {
         LOG_FATAL("cli.c", "--script-file argument is required");
     }
 
-    if (load_script(args->script_path, &args->script) != 0) {
+    if (load_external_file(args->script_path, &args->script) != 0) {
         return 1;
     }
 

src/cli.h

@@ -18,6 +18,7 @@ typedef struct scan_args {
     char *path;
     char *archive;
     archive_mode_t archive_mode;
+    char *archive_passphrase;
     char *tesseract_lang;
     const char *tesseract_path;
     char *exclude_regex;
@@ -25,6 +26,8 @@ typedef struct scan_args {
     const char* treemap_threshold_str;
     double treemap_threshold;
     int max_memory_buffer;
+    int read_subtitles;
+    int fast_epub;
 } scan_args_t;
 
 scan_args_t *scan_args_create();
@@ -39,6 +42,10 @@ typedef struct index_args {
     const char *index_path;
     const char *script_path;
     char *script;
+    const char *es_settings_path;
+    char *es_settings;
+    const char *es_mappings_path;
+    char *es_mappings;
     int print;
     int batch_size;
     int async_script;

src/ctx.h

@@ -13,6 +13,7 @@
 #include "libscan/text/text.h"
 #include "libscan/mobi/scan_mobi.h"
 #include "libscan/raw/raw.h"
+#include "libscan/msdoc/msdoc.h"
 #include "src/io/store.h"
 
 #include <glib.h>
@@ -39,6 +40,14 @@ typedef struct {
     pcre_extra *exclude_extra;
     int fast;
 
+    GHashTable *dbg_current_files;
+    pthread_mutex_t dbg_current_files_mu;
+
+    int dbg_failed_files_count;
+    int dbg_skipped_files_count;
+    int dbg_excluded_files_count;
+    pthread_mutex_t dbg_file_counts_mu;
+
     scan_arc_ctx_t arc_ctx;
     scan_comic_ctx_t comic_ctx;
     scan_ebook_ctx_t ebook_ctx;
@@ -48,6 +57,7 @@ typedef struct {
     scan_text_ctx_t text_ctx;
     scan_mobi_ctx_t mobi_ctx;
     scan_raw_ctx_t raw_ctx;
+    scan_msdoc_ctx_t msdoc_ctx;
 } ScanCtx_t;
 
 typedef struct {
@@ -63,6 +73,8 @@ typedef struct {
     tpool_t *pool;
     store_t *tag_store;
     GHashTable *tags;
+    store_t *meta_store;
+    GHashTable *meta;
 } IndexCtx_t;
 
 typedef struct {

src/index/elastic.c

@@ -30,11 +30,11 @@ void elastic_cleanup() {
     }
 }
 
-void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
+void print_json(cJSON *document, const char id_str[MD5_STR_LENGTH]) {
 
     cJSON *line = cJSON_CreateObject();
 
-    cJSON_AddStringToObject(line, "_id", uuid_str);
+    cJSON_AddStringToObject(line, "_id", id_str);
     cJSON_AddStringToObject(line, "_index", IndexCtx.es_index);
     cJSON_AddStringToObject(line, "_type", "_doc");
     cJSON_AddItemReferenceToObject(line, "_source", document);
@@ -52,13 +52,13 @@ void index_json_func(void *arg) {
     elastic_index_line(line);
 }
 
-void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
+void index_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]) {
     char *json = cJSON_PrintUnformatted(document);
 
     size_t json_len = strlen(json);
     es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t) + json_len + 2);
     memcpy(bulk_line->line, json, json_len);
-    memcpy(bulk_line->uuid_str, uuid_str, UUID_STR_LEN);
+    memcpy(bulk_line->path_md5_str, index_id_str, MD5_STR_LENGTH);
     *(bulk_line->line + json_len) = '\n';
     *(bulk_line->line + json_len + 1) = '\0';
     bulk_line->next = NULL;
@@ -67,7 +67,7 @@ void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
     tpool_add_work(IndexCtx.pool, index_json_func, bulk_line);
 }
 
-void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]) {
+void execute_update_script(const char *script, int async, const char index_id[MD5_STR_LENGTH]) {
 
     if (Indexer == NULL) {
         Indexer = create_indexer(IndexCtx.es_url, IndexCtx.es_index);
@@ -129,9 +129,9 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
     while (line != NULL && *count < max) {
         char action_str[256];
         snprintf(
-                action_str, 256,
+                action_str, sizeof(action_str),
                 "{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n",
-                line->uuid_str, Indexer->es_index
+                line->path_md5_str, Indexer->es_index
         );
 
         size_t action_str_len = strlen(action_str);
@@ -220,7 +220,7 @@ void _elastic_flush(int max) {
     if (r->status_code == 413) {
 
         if (max <= 1) {
-            LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->uuid_str)
+            LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->path_md5_str)
             free_response(r);
             free(buf);
             delete_queue(1);
@@ -356,7 +356,7 @@ void finish_indexer(char *script, int async_script, char *index_id) {
     free_response(r);
 }
 
-void elastic_init(int force_reset) {
+void elastic_init(int force_reset, const char* user_mappings, const char* user_settings) {
 
     // Check if index exists
     char url[4096];
@@ -392,13 +392,13 @@ void elastic_init(int force_reset) {
         free_response(r);
 
         snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_put(url, settings_json);
+        r = web_put(url, user_settings ? user_settings : settings_json);
-        LOG_INFOF("elastic.c", "Update settings <%d>", r->status_code);
+        LOG_INFOF("elastic.c", "Update user_settings <%d>", r->status_code);
         free_response(r);
 
         snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_put(url, mappings_json);
+        r = web_put(url, user_mappings ? user_mappings : mappings_json);
-        LOG_INFOF("elastic.c", "Update mappings <%d>", r->status_code);
+        LOG_INFOF("elastic.c", "Update user_mappings <%d>", r->status_code);
         free_response(r);
 
         snprintf(url, sizeof(url), "%s/%s/_open", IndexCtx.es_url, IndexCtx.es_index);
@@ -408,9 +408,9 @@ void elastic_init(int force_reset) {
     }
 }
 
-cJSON *elastic_get_document(const char *uuid_str) {
+cJSON *elastic_get_document(const char *id_str) {
     char url[4096];
-    snprintf(url, sizeof(url), "%s/%s/_doc/%s", WebCtx.es_url, WebCtx.es_index, uuid_str);
+    snprintf(url, sizeof(url), "%s/%s/_doc/%s", WebCtx.es_url, WebCtx.es_index, id_str);
 
     response_t *r = web_get(url, 3);
     cJSON *json = NULL;

src/index/elastic.h

@@ -5,7 +5,7 @@
 
 typedef struct es_bulk_line {
     struct es_bulk_line *next;
-    char uuid_str[UUID_STR_LEN];
+    char path_md5_str[MD5_STR_LENGTH];
     char line[0];
 } es_bulk_line_t;
 
@@ -16,21 +16,21 @@ typedef struct es_indexer es_indexer_t;
 
 void elastic_index_line(es_bulk_line_t *line);
 
-void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);
+void print_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]);
 
-void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]);
+void index_json(cJSON *document, const char index_id_str[MD5_STR_LENGTH]);
 
 es_indexer_t *create_indexer(const char *url, const char *index);
 
 void elastic_cleanup();
 void finish_indexer(char *script, int async_script, char *index_id);
 
-void elastic_init(int force_reset);
+void elastic_init(int force_reset, const char* user_mappings, const char* user_settings);
 
-cJSON *elastic_get_document(const char *uuid_str);
+cJSON *elastic_get_document(const char *id_str);
 
 char *elastic_get_status();
 
-void execute_update_script(const char *script, int async, const char index_id[UUID_STR_LEN]);
+void execute_update_script(const char *script, int async, const char index_id[MD5_STR_LENGTH]);
 
 #endif

src/io/serialize.c

@@ -6,18 +6,22 @@
 static __thread int index_fd = -1;
 
 typedef struct {
-    unsigned char uuid[16];
+    unsigned char path_md5[MD5_DIGEST_LENGTH];
-    unsigned long ino;
     unsigned long size;
     unsigned int mime;
     int mtime;
     short base;
     short ext;
+    char has_parent;
 } line_t;
 
+#define META_NEXT 0xFFFF
+
 void skip_meta(FILE *file) {
-    enum metakey key = getc(file);
+    enum metakey key = 0;
-    while (key != '\n') {
+    fread(&key, sizeof(uint16_t), 1, file);
+
+    while (key != META_NEXT) {
         if (IS_META_INT(key)) {
             fseek(file, sizeof(int), SEEK_CUR);
         } else if (IS_META_LONG(key)) {
@@ -26,13 +30,13 @@ void skip_meta(FILE *file) {
             while ((getc(file))) {}
         }
 
-        key = getc(file);
+        fread(&key, sizeof(uint16_t), 1, file);
     }
 }
 
 void write_index_descriptor(char *path, index_descriptor_t *desc) {
     cJSON *json = cJSON_CreateObject();
-    cJSON_AddStringToObject(json, "uuid", desc->uuid);
+    cJSON_AddStringToObject(json, "id", desc->id);
     cJSON_AddStringToObject(json, "version", desc->version);
     cJSON_AddStringToObject(json, "root", desc->root);
     cJSON_AddStringToObject(json, "name", desc->name);
@@ -66,7 +70,7 @@ index_descriptor_t read_index_descriptor(char *path) {
     }
 
     char *buf = malloc(info.st_size + 1);
-    int ret = read(fd, buf, info.st_size);
+    size_t ret = read(fd, buf, info.st_size);
     if (ret == -1) {
         LOG_FATALF("serialize.c", "Could not read index descriptor: %s", strerror(errno));
     }
@@ -82,7 +86,7 @@ index_descriptor_t read_index_descriptor(char *path) {
     strcpy(descriptor.rewrite_url, cJSON_GetObjectItem(json, "rewrite_url")->valuestring);
     descriptor.root_len = (short) strlen(descriptor.root);
     strcpy(descriptor.version, cJSON_GetObjectItem(json, "version")->valuestring);
-    strcpy(descriptor.uuid, cJSON_GetObjectItem(json, "uuid")->valuestring);
+    strcpy(descriptor.id, cJSON_GetObjectItem(json, "id")->valuestring);
     if (cJSON_GetObjectItem(json, "type") == NULL) {
         strcpy(descriptor.type, INDEX_TYPE_BIN);
     } else {
@@ -152,8 +156,20 @@ char *get_meta_key_text(enum metakey meta_key) {
             return "thumbnail";
         case MetaPages:
             return "pages";
+        case MetaExifGpsLongitudeRef:
+            return "exif_gps_longitude_ref";
+        case MetaExifGpsLongitudeDMS:
+            return "exif_gps_longitude_dms";
+        case MetaExifGpsLongitudeDec:
+            return "exif_gps_longitude_dec";
+        case MetaExifGpsLatitudeRef:
+            return "exif_gps_latitude_ref";
+        case MetaExifGpsLatitudeDMS:
+            return "exif_gps_latitude_dms";
+        case MetaExifGpsLatitudeDec:
+            return "exif_gps_latitude_dec";
         default:
-            return NULL;
+        LOG_FATALF("serialize.c", "FIXME: Unknown meta key: %d", meta_key)
     }
 }
 
@@ -183,7 +199,7 @@ void write_document(document_t *doc) {
 
     meta_line_t *meta = doc->meta_head;
     while (meta != NULL) {
-        dyn_buffer_write_char(&buf, meta->key);
+        dyn_buffer_write_short(&buf, (uint16_t) meta->key);
 
         if (IS_META_INT(meta->key)) {
             dyn_buffer_write_int(&buf, meta->int_val);
@@ -197,7 +213,7 @@ void write_document(document_t *doc) {
         meta = meta->next;
         free(tmp);
     }
-    dyn_buffer_write_char(&buf, '\n');
+    dyn_buffer_write_short(&buf, META_NEXT);
 
     int res = write(index_fd, buf.buf, buf.cur);
     if (res == -1) {
@@ -219,9 +235,9 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
     dyn_buffer_t buf = dyn_buffer_create();
 
     FILE *file = fopen(path, "rb");
-    while (1) {
+    while (TRUE) {
         buf.cur = 0;
-        size_t _ = fread((void *) &line, 1, sizeof(line_t), file);
+        size_t _ = fread((void *) &line, sizeof(line_t), 1, file);
         if (feof(file)) {
             break;
         }
@@ -229,8 +245,8 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
         cJSON *document = cJSON_CreateObject();
         cJSON_AddStringToObject(document, "index", index_id);
 
-        char uuid_str[UUID_STR_LEN];
+        char path_md5_str[MD5_STR_LENGTH];
-        uuid_unparse(line.uuid, uuid_str);
+        buf2hex(line.path_md5, sizeof(line.path_md5), path_md5_str);
 
         const char *mime_text = mime_get_mime_text(line.mime);
         if (mime_text == NULL) {
@@ -247,14 +263,6 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
         }
         dyn_buffer_write_char(&buf, '\0');
 
-        if (IndexCtx.tags != NULL) {
-            const char *tags_string = g_hash_table_lookup(IndexCtx.tags, buf.buf);
-            if (tags_string != NULL) {
-                cJSON *tags_arr = cJSON_Parse(tags_string);
-                cJSON_AddItemToObject(document, "tag", tags_arr);
-            }
-        }
-
         cJSON_AddStringToObject(document, "extension", buf.buf + line.ext);
         if (*(buf.buf + line.ext - 1) == '.') {
             *(buf.buf + line.ext - 1) = '\0';
@@ -276,9 +284,10 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
             cJSON_AddStringToObject(document, "path", "");
         }
 
-        enum metakey key = getc(file);
+        enum metakey key = 0;
-        size_t ret = 0;
+        fread(&key, sizeof(uint16_t), 1, file);
-        while (key != '\n') {
+        size_t ret;
+        while (key != META_NEXT) {
             switch (key) {
                 case MetaPages:
                 case MetaWidth:
@@ -316,6 +325,12 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
                 case MetaAuthor:
                 case MetaModifiedBy:
                 case MetaThumbnail:
+                case MetaExifGpsLongitudeDMS:
+                case MetaExifGpsLongitudeDec:
+                case MetaExifGpsLongitudeRef:
+                case MetaExifGpsLatitudeDMS:
+                case MetaExifGpsLatitudeDec:
+                case MetaExifGpsLatitudeRef:
                 case MetaTitle: {
                     buf.cur = 0;
                     while ((c = getc(file)) != 0) {
@@ -331,11 +346,39 @@ void read_index_bin(const char *path, const char *index_id, index_func func) {
                 LOG_FATALF("serialize.c", "Invalid meta key (corrupt index): %x", key)
             }
 
-            key = getc(file);
+            fread(&key, sizeof(uint16_t), 1, file);
         }
 
-        func(document, uuid_str);
+        cJSON *meta_obj = NULL;
+        if (IndexCtx.meta != NULL) {
+            const char *meta_string = g_hash_table_lookup(IndexCtx.meta, path_md5_str);
+            if (meta_string != NULL) {
+                meta_obj = cJSON_Parse(meta_string);
+
+                cJSON *child;
+                for (child = meta_obj->child; child != NULL; child = child->next) {
+                    char meta_key[4096];
+                    strcpy(meta_key, child->string);
+                    cJSON_DeleteItemFromObject(document, meta_key);
+                    cJSON_AddItemReferenceToObject(document, meta_key, child);
+                }
+            }
+        }
+
+        if (IndexCtx.tags != NULL) {
+            const char *tags_string = g_hash_table_lookup(IndexCtx.tags, path_md5_str);
+            if (tags_string != NULL) {
+                cJSON *tags_arr = cJSON_Parse(tags_string);
+                cJSON_DeleteItemFromObject(document, "tag");
+                cJSON_AddItemToObject(document, "tag", tags_arr);
+            }
+        }
+
+        func(document, path_md5_str);
         cJSON_Delete(document);
+        if (meta_obj) {
+            cJSON_Delete(meta_obj);
+        }
     }
     dyn_buffer_destroy(&buf);
     fclose(file);
@@ -359,7 +402,7 @@ const char *json_type_array_fields[] = {
 void read_index_json(const char *path, UNUSED(const char *index_id), index_func func) {
 
     FILE *file = fopen(path, "r");
-    while (1) {
+    while (TRUE) {
         char *line = NULL;
         size_t len;
         size_t read = getline(&line, &len, file);
@@ -379,7 +422,7 @@ void read_index_json(const char *path, UNUSED(const char *index_id), index_func
         }
 
         cJSON *document = cJSON_CreateObject();
-        const char *uuid_str = cJSON_GetObjectItem(input, "_id")->valuestring;
+        const char *id_str = cJSON_GetObjectItem(input, "_id")->valuestring;
 
         for (int i = 0; i < (sizeof(json_type_copy_fields) / sizeof(json_type_copy_fields[0])); i++) {
             cJSON *value = cJSON_GetObjectItem(input, json_type_copy_fields[i]);
@@ -407,7 +450,7 @@ void read_index_json(const char *path, UNUSED(const char *index_id), index_func
             }
         }
 
-        func(document, uuid_str);
+        func(document, id_str);
         cJSON_Delete(document);
         cJSON_Delete(input);
 
@@ -415,7 +458,7 @@ void read_index_json(const char *path, UNUSED(const char *index_id), index_func
     fclose(file);
 }
 
-void read_index(const char *path, const char index_id[UUID_STR_LEN], const char *type, index_func func) {
+void read_index(const char *path, const char index_id[MD5_STR_LENGTH], const char *type, index_func func) {
 
     if (strcmp(type, INDEX_TYPE_BIN) == 0) {
         read_index_bin(path, index_id, func);
@@ -428,15 +471,17 @@ void incremental_read(GHashTable *table, const char *filepath) {
     FILE *file = fopen(filepath, "rb");
     line_t line;
 
+    LOG_DEBUGF("serialize.c", "Incremental read %s", filepath)
+
     while (1) {
-        size_t ret = fread((void *) &line, 1, sizeof(line_t), file);
+        size_t ret = fread((void *) &line, sizeof(line_t), 1, file);
         if (ret != 1 || feof(file)) {
             break;
         }
 
-        incremental_put(table, line.ino, line.mtime);
+        incremental_put(table, line.path_md5, line.mtime);
 
-        while ((getc(file))) {}
+        while ((getc(file)) != 0) {}
         skip_meta(file);
     }
     fclose(file);
@@ -452,33 +497,47 @@ void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
     FILE *dst_file = fopen(dst_filepath, "ab");
     line_t line;
 
-    while (1) {
+    LOG_DEBUGF("serialize.c", "Incremental copy %s", filepath)
-        size_t ret = fread((void *) &line, 1, sizeof(line_t), file);
+
+    while (TRUE) {
+        size_t ret = fread((void *) &line, sizeof(line_t), 1, file);
         if (ret != 1 || feof(file)) {
             break;
         }
 
-        if (incremental_get(copy_table, line.ino)) {
+        // Assume that files with parents still exist.
+        //  One way to "fix" this would be to check if the parent is marked for copy but it would consistently
+        //  delete files with grandparents, which is a side-effect worse than having orphaned files
+        if (line.has_parent || incremental_get(copy_table, line.path_md5)) {
             fwrite(&line, sizeof(line), 1, dst_file);
 
-            size_t buf_len;
+            // Copy filepath
-            char *buf = store_read(store, (char *) line.uuid, 16, &buf_len);
+            char filepath_buf[PATH_MAX];
-            store_write(dst_store, (char *) line.uuid, 16, buf, buf_len);
-            free(buf);
-
             char c;
+            char *ptr = filepath_buf;
             while ((c = (char) getc(file))) {
-                fwrite(&c, sizeof(c), 1, dst_file);
+                *ptr++ = c;
             }
-            fwrite("\0", sizeof(c), 1, dst_file);
+            *ptr = '\0';
+            fwrite(filepath_buf, (ptr - filepath_buf) + 1, 1, dst_file);
 
-            enum metakey key;
+            // Copy tn store contents
+            size_t buf_len;
+            char path_md5[MD5_DIGEST_LENGTH];
+            MD5((unsigned char *) filepath_buf, (ptr - filepath_buf), (unsigned char *) path_md5);
+            char *buf = store_read(store, path_md5, sizeof(path_md5), &buf_len);
+            if (buf_len != 0) {
+                store_write(dst_store, path_md5, sizeof(path_md5), buf, buf_len);
+                free(buf);
+            }
+
+            enum metakey key = 0;
             while (1) {
-                key = getc(file);
+                fread(&key, sizeof(uint16_t), 1, file);
-                if (key == '\n') {
+                fwrite(&key, sizeof(uint16_t), 1, dst_file);
+                if (key == META_NEXT) {
                     break;
                 }
-                fwrite(&key, sizeof(char), 1, dst_file);
 
                 if (IS_META_INT(key)) {
                     int val;
@@ -494,14 +553,12 @@ void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
                     }
                     fwrite("\0", sizeof(c), 1, dst_file);
                 }
-
-                if (ret != 1) {
-                    break;
-                }
             }
         } else {
+            while ((getc(file))) {}
             skip_meta(file);
         }
     }
     fclose(file);
+    fclose(dst_file);
 }

src/io/serialize.h

@@ -7,14 +7,14 @@
 #include <sys/syscall.h>
 #include <glib.h>
 
-typedef void(*index_func)(cJSON *, const char[UUID_STR_LEN]);
+typedef void(*index_func)(cJSON *, const char[MD5_STR_LENGTH]);
 
 void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
                       const char *dst_filepath, GHashTable *copy_table);
 
 void write_document(document_t *doc);
 
-void read_index(const char *path, const char[UUID_STR_LEN], const char *type, index_func);
+void read_index(const char *path, const char[MD5_STR_LENGTH], const char *type, index_func);
 
 void incremental_read(GHashTable *table, const char *filepath);
 

src/io/store.c

@@ -4,6 +4,7 @@
 store_t *store_create(char *path, size_t chunk_size) {
 
     store_t *store = malloc(sizeof(struct store_t));
+#if (SIST_FAKE_STORE != 1)
     store->chunk_size = chunk_size;
     pthread_rwlock_init(&store->lock, NULL);
 
@@ -28,26 +29,39 @@ store_t *store_create(char *path, size_t chunk_size) {
     mdb_txn_begin(store->env, NULL, 0, &txn);
     mdb_dbi_open(txn, NULL, 0, &store->dbi);
     mdb_txn_commit(txn);
+#endif
 
     return store;
 }
 
 void store_destroy(store_t *store) {
 
+#if (SIST_FAKE_STORE != 1)
     pthread_rwlock_destroy(&store->lock);
     mdb_close(store->env, store->dbi);
     mdb_env_close(store->env);
+#endif
     free(store);
 }
 
+void store_flush(store_t *store) {
+    mdb_env_sync(store->env, TRUE);
+}
+
 void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len) {
 
     if (LogCtx.very_verbose) {
-        char uuid_str[UUID_STR_LEN];
+        if (key_len == MD5_DIGEST_LENGTH) {
-        uuid_unparse((unsigned char *) key, uuid_str);
+            char path_md5_str[MD5_STR_LENGTH];
-        LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", uuid_str, buf_len)
+            buf2hex((unsigned char *) key, MD5_DIGEST_LENGTH, path_md5_str);
+            LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", path_md5_str, buf_len)
+        } else {
+            LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", key, buf_len)
+        }
     }
 
+#if (SIST_FAKE_STORE != 1)
+
     MDB_val mdb_key;
     mdb_key.mv_data = key;
     mdb_key.mv_size = key_len;
@@ -84,10 +98,13 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
     if (put_ret != 0) {
         LOG_ERROR("store.c", mdb_strerror(put_ret))
     }
+#endif
 }
 
 char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen) {
     char *buf = NULL;
+
+#if (SIST_FAKE_STORE != 1)
     MDB_val mdb_key;
     mdb_key.mv_data = key;
     mdb_key.mv_size = key_len;
@@ -108,6 +125,7 @@ char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen)
     }
 
     mdb_txn_abort(txn);
+#endif
     return buf;
 }
 
@@ -136,7 +154,9 @@ GHashTable *store_read_all(store_t *store) {
         count += 1;
     }
 
-    LOG_DEBUGF("store.c", "Read tags for %d documents", count);
+    const char *path;
+    mdb_env_get_path(store->env, &path);
+    LOG_DEBUGF("store.c", "Read %d entries from %s", count, path);
 
     mdb_cursor_close(cur);
     mdb_txn_abort(txn);

src/io/store.h

@@ -8,6 +8,7 @@
 
 #define STORE_SIZE_TN 1024 * 1024 * 5
 #define STORE_SIZE_TAG 1024 * 16
+#define STORE_SIZE_META STORE_SIZE_TAG
 
 typedef struct store_t {
     MDB_dbi dbi;
@@ -23,6 +24,8 @@ void store_destroy(store_t *store);
 
 void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len);
 
+void store_flush(store_t *store);
+
 char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen);
 
 GHashTable *store_read_all(store_t *store);

src/io/walk.c

@@ -20,7 +20,7 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info,
 
     job->vfile.info = *info;
 
-    memset(job->parent, 0, 16);
+    memset(job->parent, 0, MD5_DIGEST_LENGTH);
 
     job->vfile.filepath = job->filepath;
     job->vfile.read = fs_read;
@@ -41,6 +41,10 @@ int handle_entry(const char *filepath, const struct stat *info, int typeflag, st
 
         if (ScanCtx.exclude != NULL && EXCLUDED(filepath)) {
             LOG_DEBUGF("walk.c", "Excluded: %s", filepath)
+
+            pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu);
+            ScanCtx.dbg_excluded_files_count += 1;
+            pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu);
             return 0;
         }
 
@@ -51,6 +55,8 @@ int handle_entry(const char *filepath, const struct stat *info, int typeflag, st
     return 0;
 }
 
+#define MAX_FILE_DESCRIPTORS 64
+
 int walk_directory_tree(const char *dirpath) {
-    return nftw(dirpath, handle_entry, 15, FTW_PHYS);
+    return nftw(dirpath, handle_entry, MAX_FILE_DESCRIPTORS, FTW_PHYS | FTW_DEPTH);
 }

src/main.c

@@ -21,7 +21,7 @@
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "2.8.3";
+static const char *const Version = "2.10.3";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
@@ -30,13 +30,77 @@ static const char *const usage[] = {
         NULL,
 };
 
+#include<signal.h>
+#include<unistd.h>
+
+static __sighandler_t sigsegv_handler = NULL;
+static __sighandler_t sigabrt_handler = NULL;
+
+void sig_handler(int signum) {
+
+    LogCtx.verbose = 1;
+    LogCtx.very_verbose = 1;
+
+    LOG_ERROR("*SIGNAL HANDLER*", "=============================================\n\n");
+    LOG_ERRORF("*SIGNAL HANDLER*", "Uh oh! Caught fatal signal: %s", strsignal(signum));
+
+    GHashTableIter iter;
+    g_hash_table_iter_init(&iter, ScanCtx.dbg_current_files);
+
+    void *key;
+    void *value;
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        parse_job_t *job = value;
+
+        if (isatty(STDERR_FILENO)) {
+            LOG_DEBUGF(
+                    "*SIGNAL HANDLER*",
+                    "Thread \033[%dm[%04llX]\033[0m was working on job '%s'",
+                    31 + ((unsigned int) key) % 7, key, job->filepath
+            );
+        } else {
+            LOG_DEBUGF(
+                    "*SIGNAL HANDLER*",
+                    "THREAD [%04llX] was working on job %s",
+                    key, job->filepath
+            );
+        }
+    }
+
+    tpool_dump_debug_info(ScanCtx.pool);
+
+    LOG_INFO(
+            "*SIGNAL HANDLER*",
+            "Please consider creating a bug report at https://github.com/simon987/sist2/issues !"
+    )
+    LOG_INFO(
+            "*SIGNAL HANDLER*",
+            "sist2 is an open source project and relies on the collaboration of its users to diagnose and fix bugs"
+    )
+
+#ifndef SIST_DEBUG
+    LOG_WARNING(
+            "*SIGNAL HANDLER*",
+            "You are running sist2 in release mode! Please consider downloading the debug binary from the Github "
+            "releases page to provide additionnal information when submitting a bug report."
+    )
+#endif
+
+    if (signum == SIGSEGV && sigsegv_handler != NULL) {
+        sigsegv_handler(signum);
+    } else if (signum == SIGABRT && sigabrt_handler != NULL) {
+        sigabrt_handler(signum);
+    }
+}
+
 void init_dir(const char *dirpath) {
     char path[PATH_MAX];
     snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
 
-    uuid_t uuid;
+    unsigned char index_md5[MD5_DIGEST_LENGTH];
-    uuid_generate(uuid);
+    MD5((unsigned char *) ScanCtx.index.desc.name, strlen(ScanCtx.index.desc.name), index_md5);
-    uuid_unparse(uuid, ScanCtx.index.desc.uuid);
+    buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
+
     time(&ScanCtx.index.desc.timestamp);
     strcpy(ScanCtx.index.desc.version, Version);
     strcpy(ScanCtx.index.desc.type, INDEX_TYPE_BIN);
@@ -98,6 +162,16 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.arc_ctx.log = _log;
     ScanCtx.arc_ctx.logf = _logf;
     ScanCtx.arc_ctx.parse = (parse_callback_t) parse;
+    if (args->archive_passphrase != NULL) {
+        strcpy(ScanCtx.arc_ctx.passphrase, args->archive_passphrase);
+    } else {
+        ScanCtx.arc_ctx.passphrase[0] = 0;
+    }
+
+    ScanCtx.dbg_current_files = g_hash_table_new_full(g_int64_hash, g_int64_equal, NULL, NULL);
+    pthread_mutex_init(&ScanCtx.dbg_current_files_mu, NULL);
+
+    pthread_mutex_init(&ScanCtx.dbg_file_counts_mu, NULL);
 
     // Comic
     ScanCtx.comic_ctx.log = _log;
@@ -117,6 +191,7 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.ebook_ctx.log = _log;
     ScanCtx.ebook_ctx.logf = _logf;
     ScanCtx.ebook_ctx.store = _store;
+    ScanCtx.ebook_ctx.fast_epub_parse = args->fast_epub;
 
     // Font
     ScanCtx.font_ctx.enable_tn = args->size > 0;
@@ -131,6 +206,7 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.media_ctx.logf = _logf;
     ScanCtx.media_ctx.store = _store;
     ScanCtx.media_ctx.max_media_buffer = (long) args->max_memory_buffer * 1024 * 1024;
+    ScanCtx.media_ctx.read_subtitles = args->read_subtitles;
     init_media();
 
     // OOXML
@@ -149,6 +225,14 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.text_ctx.log = _log;
     ScanCtx.text_ctx.logf = _logf;
 
+    // MSDOC
+    ScanCtx.msdoc_ctx.tn_size = args->size;
+    ScanCtx.msdoc_ctx.content_size = args->content_size;
+    ScanCtx.msdoc_ctx.log = _log;
+    ScanCtx.msdoc_ctx.logf = _logf;
+    ScanCtx.msdoc_ctx.store = _store;
+    ScanCtx.msdoc_ctx.msdoc_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/msword");
+
     ScanCtx.threads = args->threads;
     ScanCtx.depth = args->depth;
 
@@ -168,6 +252,37 @@ void initialize_scan_context(scan_args_t *args) {
 }
 
 
+void load_incremental_index(const scan_args_t *args) {
+    ScanCtx.original_table = incremental_get_table();
+    ScanCtx.copy_table = incremental_get_table();
+
+    DIR *dir = opendir(args->incremental);
+    if (dir == NULL) {
+        LOG_FATALF("main.c", "Could not open original index for incremental scan: %s", strerror(errno))
+    }
+
+    char descriptor_path[PATH_MAX];
+    snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->incremental);
+    index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
+
+    if (strcmp(original_desc.version, Version) != 0) {
+        LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s/%s", original_desc.version,
+                   Version, INDEX_VERSION_EXTERNAL)
+    }
+
+    struct dirent *de;
+    while ((de = readdir(dir)) != NULL) {
+        if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
+            char file_path[PATH_MAX];
+            snprintf(file_path, PATH_MAX, "%s%s", args->incremental, de->d_name);
+            incremental_read(ScanCtx.original_table, file_path);
+        }
+    }
+    closedir(dir);
+
+    LOG_INFOF("main.c", "Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table))
+}
+
 void sist2_scan(scan_args_t *args) {
 
     ScanCtx.mime_table = mime_get_mime_table();
@@ -182,46 +297,28 @@ void sist2_scan(scan_args_t *args) {
     mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
     ScanCtx.index.store = store_create(store_path, STORE_SIZE_TN);
 
+    snprintf(store_path, PATH_MAX, "%smeta", ScanCtx.index.path);
+    mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
+    ScanCtx.index.meta_store = store_create(store_path, STORE_SIZE_META);
+
     scan_print_header();
 
     if (args->incremental != NULL) {
-        ScanCtx.original_table = incremental_get_table();
+        load_incremental_index(args);
-        ScanCtx.copy_table = incremental_get_table();
-
-        DIR *dir = opendir(args->incremental);
-        if (dir == NULL) {
-            LOG_FATALF("main.c", "Could not open original index for incremental scan: %s", strerror(errno))
-        }
-
-        char descriptor_path[PATH_MAX];
-        snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->incremental);
-        index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
-
-        if (strcmp(original_desc.version, Version) != 0) {
-            LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s/%s", original_desc.version,
-                       Version, INDEX_VERSION_EXTERNAL)
-        }
-
-        struct dirent *de;
-        while ((de = readdir(dir)) != NULL) {
-            if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
-                char file_path[PATH_MAX];
-                snprintf(file_path, PATH_MAX, "%s/%s", args->incremental, de->d_name);
-                incremental_read(ScanCtx.original_table, file_path);
-            }
-        }
-        closedir(dir);
-
-        LOG_INFOF("main.c", "Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table))
     }
 
     ScanCtx.pool = tpool_create(args->threads, thread_cleanup, TRUE);
     tpool_start(ScanCtx.pool);
-    walk_directory_tree(ScanCtx.index.desc.root);
+    int walk_ret = walk_directory_tree(ScanCtx.index.desc.root);
+    if (walk_ret == -1) {
+        LOG_FATALF("main.c", "walk_directory_tree() failed! %s (%d)", strerror(errno), errno)
+    }
     tpool_wait(ScanCtx.pool);
     tpool_destroy(ScanCtx.pool);
 
-    generate_stats(&ScanCtx.index, args->treemap_threshold, ScanCtx.index.path);
+    LOG_DEBUGF("main.c", "Skipped files: %d", ScanCtx.dbg_skipped_files_count)
+    LOG_DEBUGF("main.c", "Excluded files: %d", ScanCtx.dbg_excluded_files_count)
+    LOG_DEBUGF("main.c", "Failed files: %d", ScanCtx.dbg_failed_files_count)
 
     if (args->incremental != NULL) {
         char dst_path[PATH_MAX];
@@ -238,7 +335,7 @@ void sist2_scan(scan_args_t *args) {
         while ((de = readdir(dir)) != NULL) {
             if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
                 char file_path[PATH_MAX];
-                snprintf(file_path, PATH_MAX, "%s/%s", args->incremental, de->d_name);
+                snprintf(file_path, PATH_MAX, "%s%s", args->incremental, de->d_name);
                 incremental_copy(source, ScanCtx.index.store, file_path, dst_path, ScanCtx.copy_table);
             }
         }
@@ -253,6 +350,8 @@ void sist2_scan(scan_args_t *args) {
         store_destroy(source_tags);
     }
 
+    generate_stats(&ScanCtx.index, args->treemap_threshold, ScanCtx.index.path);
+
     store_destroy(ScanCtx.index.store);
 }
 
@@ -263,7 +362,7 @@ void sist2_index(index_args_t *args) {
     IndexCtx.batch_size = args->batch_size;
 
     if (!args->print) {
-        elastic_init(args->force_reset);
+        elastic_init(args->force_reset, args->es_mappings, args->es_settings);
     }
 
     char descriptor_path[PATH_MAX];
@@ -289,6 +388,10 @@ void sist2_index(index_args_t *args) {
     IndexCtx.tag_store = store_create(path_tmp, STORE_SIZE_TAG);
     IndexCtx.tags = store_read_all(IndexCtx.tag_store);
 
+    snprintf(path_tmp, sizeof(path_tmp), "%s/meta", args->index_path);
+    IndexCtx.meta_store = store_create(path_tmp, STORE_SIZE_META);
+    IndexCtx.meta = store_read_all(IndexCtx.meta_store);
+
     index_func f;
     if (args->print) {
         f = print_json;
@@ -311,7 +414,7 @@ void sist2_index(index_args_t *args) {
         if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
             char file_path[PATH_MAX];
             snprintf(file_path, PATH_MAX, "%s/%s", args->index_path, de->d_name);
-            read_index(file_path, desc.uuid, desc.type, f);
+            read_index(file_path, desc.id, desc.type, f);
         }
     }
     closedir(dir);
@@ -321,7 +424,7 @@ void sist2_index(index_args_t *args) {
     tpool_destroy(IndexCtx.pool);
 
     if (!args->print) {
-        finish_indexer(args->script, args->async_script, desc.uuid);
+        finish_indexer(args->script, args->async_script, desc.id);
     }
 
     store_destroy(IndexCtx.tag_store);
@@ -338,10 +441,11 @@ void sist2_exec_script(exec_args_t *args) {
     index_descriptor_t desc = read_index_descriptor(descriptor_path);
 
     IndexCtx.es_url = args->es_url;
+    IndexCtx.es_index = args->es_index;
 
     LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type)
 
-    execute_update_script(args->script, args->async_script, desc.uuid);
+    execute_update_script(args->script, args->async_script, desc.id);
     free(args->script);
 }
 
@@ -382,6 +486,9 @@ void sist2_web(web_args_t *args) {
 
 
 int main(int argc, const char *argv[]) {
+    sigsegv_handler = signal(SIGSEGV, sig_handler);
+    sigabrt_handler = signal(SIGABRT, sig_handler);
+
     setlocale(LC_ALL, "");
 
     scan_args_t *scan_args = scan_args_create();
@@ -422,6 +529,9 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "archive", &scan_args->archive, "Archive file mode (skip|list|shallow|recurse). "
                                                           "skip: Don't parse, list: only get file names as text, "
                                                           "shallow: Don't parse archives inside archives. DEFAULT: recurse"),
+            OPT_STRING(0, "archive-passphrase", &scan_args->archive_passphrase,
+                       "Passphrase for encrypted archive files"),
+
             OPT_STRING(0, "ocr", &scan_args->tesseract_lang, "Tesseract language (use tesseract --list-langs to see "
                                                              "which are installed on your machine)"),
             OPT_STRING('e', "exclude", &scan_args->exclude_regex, "Files that match this regex will not be scanned"),
@@ -431,6 +541,8 @@ int main(int argc, const char *argv[]) {
             OPT_INTEGER(0, "mem-buffer", &scan_args->max_memory_buffer,
                         "Maximum memory buffer size per thread in MB for files inside archives "
                         "(see USAGE.md). DEFAULT: 2000"),
+            OPT_BOOLEAN(0, "read-subtitles", &scan_args->read_subtitles, "Read subtitles from media files."),
+            OPT_BOOLEAN(0, "fast-epub", &scan_args->fast_epub, "Faster but less accurate EPUB parsing (no thumbnails, metadata)"),
 
             OPT_GROUP("Index options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),
@@ -438,6 +550,8 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
             OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
             OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
+            OPT_STRING(0, "mappings-file", &index_args->es_mappings_path, "Path to Elasticsearch mappings."),
+            OPT_STRING(0, "settings-file", &index_args->es_settings_path, "Path to Elasticsearch settings."),
             OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."),
             OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 100"),
             OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. "
@@ -515,7 +629,7 @@ int main(int argc, const char *argv[]) {
         }
         sist2_web(web_args);
 
-    }  else if (strcmp(argv[0], "exec-script") == 0) {
+    } else if (strcmp(argv[0], "exec-script") == 0) {
 
         int err = exec_args_validate(exec_args, argc, argv);
         if (err != 0) {

src/parsing/mime.h

@@ -6,6 +6,7 @@
 #define MAJOR_MIME(mime_id) (mime_id & 0x000F0000) >> 16
 
 #define MIME_EMPTY 1
+#define MIME_SIST2_SIDECAR 2
 
 #define DONT_PARSE 0x80000000
 #define SHOULD_PARSE(mime_id) (ScanCtx.fast == 0 && (mime_id & DONT_PARSE) != DONT_PARSE && mime_id != 0)

src/parsing/mime_generated.c

@@ -128,333 +128,334 @@ enum mime {
     application_x_dvi=655480,
     application_x_elc=655481,
     application_x_empty=1,
-    application_x_envoy=655483,
+    application_x_envoy=655482,
-    application_x_esrehber=655484,
+    application_x_esrehber=655483,
-    application_x_excel=655485,
+    application_x_excel=655484,
-    application_x_executable=655486,
+    application_x_executable=655485,
-    application_x_font_gdos=655487,
+    application_x_font_gdos=655486,
-    application_x_font_pf2=655488,
+    application_x_font_pf2=655487,
-    application_x_font_pfm=655489,
+    application_x_font_pfm=655488,
-    application_x_font_sfn=655490,
+    application_x_font_sfn=655489,
-    application_x_font_ttf=655491 | 0x20000000,
+    application_x_font_ttf=655490 | 0x20000000,
-    application_x_fptapplication_x_dbt=655492,
+    application_x_fptapplication_x_dbt=655491,
-    application_x_freelance=655493,
+    application_x_freelance=655492,
-    application_x_gamecube_rom=655494,
+    application_x_gamecube_rom=655493,
-    application_x_gdbm=655495,
+    application_x_gdbm=655494,
-    application_x_gettext_translation=655496,
+    application_x_gettext_translation=655495,
-    application_x_git=655497,
+    application_x_git=655496,
-    application_x_gsp=655498,
+    application_x_gsp=655497,
-    application_x_gss=655499,
+    application_x_gss=655498,
-    application_x_gtar=655500,
+    application_x_gtar=655499,
-    application_x_gzip=655501,
+    application_x_gzip=655500,
-    application_x_hdf=655502,
+    application_x_hdf=655501,
-    application_x_helpfile=655503,
+    application_x_helpfile=655502,
-    application_x_httpd_imap=655504,
+    application_x_httpd_imap=655503,
-    application_x_ima=655505,
+    application_x_ima=655504,
-    application_x_innosetup=655506,
+    application_x_innosetup=655505,
-    application_x_internett_signup=655507,
+    application_x_internett_signup=655506,
-    application_x_inventor=655508,
+    application_x_inventor=655507,
-    application_x_ip2=655509,
+    application_x_ip2=655508,
-    application_x_java_applet=655510,
+    application_x_java_applet=655509,
-    application_x_java_commerce=655511,
+    application_x_java_commerce=655510,
-    application_x_java_image=655512,
+    application_x_java_image=655511,
-    application_x_java_jmod=655513,
+    application_x_java_jmod=655512,
-    application_x_java_keystore=655514,
+    application_x_java_keystore=655513,
-    application_x_kdelnk=655515,
+    application_x_kdelnk=655514,
-    application_x_koan=655516,
+    application_x_koan=655515,
-    application_x_latex=655517,
+    application_x_latex=655516,
-    application_x_livescreen=655518,
+    application_x_livescreen=655517,
-    application_x_lotus=655519,
+    application_x_lotus=655518,
-    application_x_lz4=655520 | 0x08000000,
+    application_x_lz4=655519 | 0x08000000,
-    application_x_lz4_json=655521,
+    application_x_lz4_json=655520,
-    application_x_lzh=655522,
+    application_x_lzh=655521,
-    application_x_lzh_compressed=655523,
+    application_x_lzh_compressed=655522,
-    application_x_lzip=655524 | 0x08000000,
+    application_x_lzip=655523 | 0x08000000,
-    application_x_lzma=655525 | 0x08000000,
+    application_x_lzma=655524 | 0x08000000,
-    application_x_lzop=655526 | 0x08000000,
+    application_x_lzop=655525 | 0x08000000,
-    application_x_lzx=655527,
+    application_x_lzx=655526,
-    application_x_mach_binary=655528,
+    application_x_mach_binary=655527,
-    application_x_mach_executable=655529,
+    application_x_mach_executable=655528,
-    application_x_magic_cap_package_1_0=655530,
+    application_x_magic_cap_package_1_0=655529,
-    application_x_mathcad=655531,
+    application_x_mathcad=655530,
-    application_x_maxis_dbpf=655532,
+    application_x_maxis_dbpf=655531,
-    application_x_meme=655533,
+    application_x_meme=655532,
-    application_x_midi=655534,
+    application_x_midi=655533,
-    application_x_mif=655535,
+    application_x_mif=655534,
-    application_x_mix_transfer=655536,
+    application_x_mix_transfer=655535,
-    application_x_mobipocket_ebook=655537 | 0x02000000,
+    application_x_mobipocket_ebook=655536 | 0x02000000,
-    application_x_ms_compress_szdd=655538,
+    application_x_ms_compress_szdd=655537,
-    application_x_ms_pdb=655539,
+    application_x_ms_pdb=655538,
-    application_x_ms_reader=655540,
+    application_x_ms_reader=655539,
-    application_x_msaccess=655541,
+    application_x_msaccess=655540,
-    application_x_n64_rom=655542,
+    application_x_n64_rom=655541,
-    application_x_navi_animation=655543,
+    application_x_navi_animation=655542,
-    application_x_navidoc=655544,
+    application_x_navidoc=655543,
-    application_x_navimap=655545,
+    application_x_navimap=655544,
-    application_x_navistyle=655546,
+    application_x_navistyle=655545,
-    application_x_nes_rom=655547,
+    application_x_nes_rom=655546,
-    application_x_netcdf=655548,
+    application_x_netcdf=655547,
-    application_x_newton_compatible_pkg=655549,
+    application_x_newton_compatible_pkg=655548,
-    application_x_nintendo_ds_rom=655550,
+    application_x_nintendo_ds_rom=655549,
-    application_x_object=655551,
+    application_x_object=655550,
-    application_x_omc=655552,
+    application_x_omc=655551,
-    application_x_omcdatamaker=655553,
+    application_x_omcdatamaker=655552,
-    application_x_omcregerator=655554,
+    application_x_omcregerator=655553,
-    application_x_pagemaker=655555,
+    application_x_pagemaker=655554,
-    application_x_pcl=655556,
+    application_x_pcl=655555,
-    application_x_pgp_keyring=655557,
+    application_x_pgp_keyring=655556,
-    application_x_pixclscript=655558,
+    application_x_pixclscript=655557,
-    application_x_pkcs7_certreqresp=655559,
+    application_x_pkcs7_certreqresp=655558,
-    application_x_pkcs7_signature=655560,
+    application_x_pkcs7_signature=655559,
-    application_x_project=655561,
+    application_x_project=655560,
-    application_x_qpro=655562,
+    application_x_qpro=655561,
-    application_x_rar=655563 | 0x10000000,
+    application_x_rar=655562 | 0x10000000,
-    application_x_rpm=655564,
+    application_x_rpm=655563,
-    application_x_sdp=655565,
+    application_x_sdp=655564,
-    application_x_sea=655566,
+    application_x_sea=655565,
-    application_x_seelogo=655567,
+    application_x_seelogo=655566,
-    application_x_setupscript=655568,
+    application_x_setupscript=655567,
-    application_x_shar=655569,
+    application_x_shar=655568,
-    application_x_sharedlib=655570,
+    application_x_sharedlib=655569,
-    application_x_shockwave_flash=655571,
+    application_x_shockwave_flash=655570,
-    application_x_snappy_framed=655572,
+    application_x_snappy_framed=655571,
-    application_x_sprite=655573,
+    application_x_sprite=655572,
-    application_x_sqlite3=655574,
+    application_x_sqlite3=655573,
-    application_x_stargallery_thm=655575,
+    application_x_stargallery_thm=655574,
-    application_x_stuffit=655576,
+    application_x_stuffit=655575,
-    application_x_sv4cpio=655577,
+    application_x_sv4cpio=655576,
-    application_x_sv4crc=655578,
+    application_x_sv4crc=655577,
-    application_x_tar=655579 | 0x10000000,
+    application_x_tar=655578 | 0x10000000,
-    application_x_tbook=655580,
+    application_x_tbook=655579,
-    application_x_terminfo=655581,
+    application_x_terminfo=655580,
-    application_x_terminfo2=655582,
+    application_x_terminfo2=655581,
-    application_x_tex_tfm=655583,
+    application_x_tex_tfm=655582,
-    application_x_texinfo=655584,
+    application_x_texinfo=655583,
-    application_x_ustar=655585,
+    application_x_ustar=655584,
-    application_x_visio=655586,
+    application_x_visio=655585,
-    application_x_vnd_audioexplosion_mzz=655587,
+    application_x_vnd_audioexplosion_mzz=655586,
-    application_x_vnd_ls_xpix=655588,
+    application_x_vnd_ls_xpix=655587,
-    application_x_vrml=655589,
+    application_x_vrml=655588,
-    application_x_wais_source=655590,
+    application_x_wais_source=655589,
-    application_x_wine_extension_ini=655591,
+    application_x_wine_extension_ini=655590,
-    application_x_wintalk=655592,
+    application_x_wintalk=655591,
-    application_x_world=655593,
+    application_x_world=655592,
-    application_x_wri=655594,
+    application_x_wri=655593,
-    application_x_x509_ca_cert=655595,
+    application_x_x509_ca_cert=655594,
-    application_x_xz=655596 | 0x08000000,
+    application_x_xz=655595 | 0x08000000,
-    application_x_zip=655597,
+    application_x_zip=655596,
-    application_x_zstd=655598 | 0x08000000,
+    application_x_zstd=655597 | 0x08000000,
-    application_x_zstd_dictionary=655599,
+    application_x_zstd_dictionary=655598,
-    application_xml=655600,
+    application_xml=655599,
-    application_zip=655601 | 0x10000000,
+    application_zip=655600 | 0x10000000,
-    application_zlib=655602,
+    application_zlib=655601,
-    audio_basic=458995 | 0x80000000,
+    audio_basic=458994 | 0x80000000,
-    audio_it=458996,
+    audio_it=458995,
-    audio_make=458997,
+    audio_make=458996,
-    audio_mid=458998,
+    audio_mid=458997,
-    audio_midi=458999,
+    audio_midi=458998,
-    audio_mp4=459000,
+    audio_mp4=458999,
-    audio_mpeg=459001,
+    audio_mpeg=459000,
-    audio_ogg=459002,
+    audio_ogg=459001,
-    audio_s3m=459003,
+    audio_s3m=459002,
-    audio_tsp_audio=459004,
+    audio_tsp_audio=459003,
-    audio_tsplayer=459005,
+    audio_tsplayer=459004,
-    audio_vnd_qcelp=459006,
+    audio_vnd_qcelp=459005,
-    audio_voxware=459007,
+    audio_voxware=459006,
-    audio_x_aiff=459008,
+    audio_x_aiff=459007,
-    audio_x_flac=459009,
+    audio_x_flac=459008,
-    audio_x_gsm=459010,
+    audio_x_gsm=459009,
-    audio_x_hx_aac_adts=459011,
+    audio_x_hx_aac_adts=459010,
-    audio_x_jam=459012,
+    audio_x_jam=459011,
-    audio_x_liveaudio=459013,
+    audio_x_liveaudio=459012,
-    audio_x_m4a=459014,
+    audio_x_m4a=459013,
-    audio_x_midi=459015,
+    audio_x_midi=459014,
-    audio_x_mod=459016,
+    audio_x_mod=459015,
-    audio_x_mp4a_latm=459017,
+    audio_x_mp4a_latm=459016,
-    audio_x_mpeg_3=459018,
+    audio_x_mpeg_3=459017,
-    audio_x_mpequrl=459019,
+    audio_x_mpequrl=459018,
-    audio_x_nspaudio=459020,
+    audio_x_nspaudio=459019,
-    audio_x_pn_realaudio=459021,
+    audio_x_pn_realaudio=459020,
-    audio_x_psid=459022,
+    audio_x_psid=459021,
-    audio_x_realaudio=459023,
+    audio_x_realaudio=459022,
-    audio_x_s3m=459024,
+    audio_x_s3m=459023,
-    audio_x_twinvq=459025,
+    audio_x_twinvq=459024,
-    audio_x_twinvq_plugin=459026,
+    audio_x_twinvq_plugin=459025,
-    audio_x_voc=459027,
+    audio_x_voc=459026,
-    audio_x_wav=459028,
+    audio_x_wav=459027,
-    audio_x_xbox_executable=459029 | 0x80000000,
+    audio_x_xbox_executable=459028 | 0x80000000,
-    audio_x_xbox360_executable=459030 | 0x80000000,
+    audio_x_xbox360_executable=459029 | 0x80000000,
-    audio_xm=459031,
+    audio_xm=459030,
-    font_otf=327960 | 0x20000000,
+    font_otf=327959 | 0x20000000,
-    font_sfnt=327961 | 0x20000000,
+    font_sfnt=327960 | 0x20000000,
-    font_woff=327962 | 0x20000000,
+    font_woff=327961 | 0x20000000,
-    font_woff2=327963 | 0x20000000,
+    font_woff2=327962 | 0x20000000,
-    image_bmp=524572,
+    image_bmp=524571,
-    image_cmu_raster=524573,
+    image_cmu_raster=524572,
-    image_fif=524574,
+    image_fif=524573,
-    image_florian=524575,
+    image_florian=524574,
-    image_g3fax=524576,
+    image_g3fax=524575,
-    image_gif=524577,
+    image_gif=524576,
-    image_heic=524578,
+    image_heic=524577,
-    image_ief=524579,
+    image_ief=524578,
-    image_jpeg=524580,
+    image_jpeg=524579,
-    image_jutvision=524581,
+    image_jutvision=524580,
-    image_naplps=524582,
+    image_naplps=524581,
-    image_pict=524583,
+    image_pict=524582,
-    image_png=524584,
+    image_png=524583,
-    image_svg=524585 | 0x80000000,
+    image_svg=524584 | 0x80000000,
-    image_svg_xml=524586 | 0x80000000,
+    image_svg_xml=524585 | 0x80000000,
-    image_tiff=524587,
+    image_tiff=524586,
-    image_vnd_adobe_photoshop=524588 | 0x80000000,
+    image_vnd_adobe_photoshop=524587 | 0x80000000,
-    image_vnd_djvu=524589 | 0x80000000,
+    image_vnd_djvu=524588 | 0x80000000,
-    image_vnd_fpx=524590,
+    image_vnd_fpx=524589,
-    image_vnd_microsoft_icon=524591,
+    image_vnd_microsoft_icon=524590,
-    image_vnd_rn_realflash=524592,
+    image_vnd_rn_realflash=524591,
-    image_vnd_rn_realpix=524593,
+    image_vnd_rn_realpix=524592,
-    image_vnd_wap_wbmp=524594,
+    image_vnd_wap_wbmp=524593,
-    image_vnd_xiff=524595,
+    image_vnd_xiff=524594,
-    image_webp=524596,
+    image_webp=524595,
-    image_wmf=524597,
+    image_wmf=524596,
-    image_x_3ds=524598,
+    image_x_3ds=524597,
-    image_x_adobe_dng=524599 | 0x00800000,
+    image_x_adobe_dng=524598 | 0x00800000,
-    image_x_award_bioslogo=524600,
+    image_x_award_bioslogo=524599,
-    image_x_canon_cr2=524601 | 0x00800000,
+    image_x_canon_cr2=524600 | 0x00800000,
-    image_x_canon_crw=524602 | 0x00800000,
+    image_x_canon_crw=524601 | 0x00800000,
-    image_x_cmu_raster=524603,
+    image_x_cmu_raster=524602,
-    image_x_cur=524604,
+    image_x_cur=524603,
-    image_x_dcraw=524605 | 0x00800000,
+    image_x_dcraw=524604 | 0x00800000,
-    image_x_dwg=524606,
+    image_x_dwg=524605,
-    image_x_eps=524607,
+    image_x_eps=524606,
-    image_x_epson_erf=524608 | 0x00800000,
+    image_x_epson_erf=524607 | 0x00800000,
-    image_x_exr=524609,
+    image_x_exr=524608,
-    image_x_fuji_raf=524610 | 0x00800000,
+    image_x_fuji_raf=524609 | 0x00800000,
-    image_x_gem=524611,
+    image_x_gem=524610,
-    image_x_icns=524612,
+    image_x_icns=524611,
-    image_x_icon=524613 | 0x80000000,
+    image_x_icon=524612 | 0x80000000,
-    image_x_jg=524614,
+    image_x_jg=524613,
-    image_x_jps=524615,
+    image_x_jps=524614,
-    image_x_kodak_dcr=524616 | 0x00800000,
+    image_x_kodak_dcr=524615 | 0x00800000,
-    image_x_kodak_k25=524617 | 0x00800000,
+    image_x_kodak_k25=524616 | 0x00800000,
-    image_x_kodak_kdc=524618 | 0x00800000,
+    image_x_kodak_kdc=524617 | 0x00800000,
-    image_x_minolta_mrw=524619 | 0x00800000,
+    image_x_minolta_mrw=524618 | 0x00800000,
-    image_x_ms_bmp=524620,
+    image_x_ms_bmp=524619,
-    image_x_niff=524621,
+    image_x_niff=524620,
-    image_x_nikon_nef=524622 | 0x00800000,
+    image_x_nikon_nef=524621 | 0x00800000,
-    image_x_olympus_orf=524623 | 0x00800000,
+    image_x_olympus_orf=524622 | 0x00800000,
-    image_x_panasonic_raw=524624 | 0x00800000,
+    image_x_panasonic_raw=524623 | 0x00800000,
-    image_x_pcx=524625,
+    image_x_pcx=524624,
-    image_x_pentax_pef=524626 | 0x00800000,
+    image_x_pentax_pef=524625 | 0x00800000,
-    image_x_pict=524627,
+    image_x_pict=524626,
-    image_x_portable_bitmap=524628,
+    image_x_portable_bitmap=524627,
-    image_x_portable_graymap=524629,
+    image_x_portable_graymap=524628,
-    image_x_portable_pixmap=524630,
+    image_x_portable_pixmap=524629,
-    image_x_quicktime=524631,
+    image_x_quicktime=524630,
-    image_x_rgb=524632,
+    image_x_rgb=524631,
-    image_x_sigma_x3f=524633 | 0x00800000,
+    image_x_sigma_x3f=524632 | 0x00800000,
-    image_x_sony_arw=524634 | 0x00800000,
+    image_x_sony_arw=524633 | 0x00800000,
-    image_x_sony_sr2=524635 | 0x00800000,
+    image_x_sony_sr2=524634 | 0x00800000,
-    image_x_sony_srf=524636 | 0x00800000,
+    image_x_sony_srf=524635 | 0x00800000,
-    image_x_tga=524637,
+    image_x_tga=524636,
-    image_x_tiff=524638,
+    image_x_tiff=524637,
-    image_x_win_bitmap=524639,
+    image_x_win_bitmap=524638,
-    image_x_xcf=524640 | 0x80000000,
+    image_x_xcf=524639 | 0x80000000,
-    image_x_xpixmap=524641 | 0x80000000,
+    image_x_xpixmap=524640 | 0x80000000,
-    image_x_xwindowdump=524642,
+    image_x_xwindowdump=524641,
-    message_news=196963,
+    message_news=196962,
-    message_rfc822=196964,
+    message_rfc822=196963,
-    model_vnd_dwf=65893,
+    model_vnd_dwf=65892,
-    model_vnd_gdl=65894,
+    model_vnd_gdl=65893,
-    model_vnd_gs_gdl=65895,
+    model_vnd_gs_gdl=65894,
-    model_vrml=65896,
+    model_vrml=65895,
-    model_x_pov=65897,
+    model_x_pov=65896,
-    text_PGP=590186,
+    sist2_sidecar=2,
-    text_asp=590187,
+    text_PGP=590185,
-    text_css=590188,
+    text_asp=590186,
-    text_html=590189 | 0x01000000,
+    text_css=590187,
-    text_javascript=590190,
+    text_html=590188 | 0x01000000,
-    text_mcf=590191,
+    text_javascript=590189,
-    text_pascal=590192,
+    text_mcf=590190,
-    text_plain=590193,
+    text_pascal=590191,
-    text_richtext=590194,
+    text_plain=590192,
-    text_rtf=590195,
+    text_richtext=590193,
-    text_scriplet=590196,
+    text_rtf=590194,
-    text_tab_separated_values=590197,
+    text_scriplet=590195,
-    text_troff=590198,
+    text_tab_separated_values=590196,
-    text_uri_list=590199,
+    text_troff=590197,
-    text_vnd_abc=590200,
+    text_uri_list=590198,
-    text_vnd_fmi_flexstor=590201,
+    text_vnd_abc=590199,
-    text_vnd_wap_wml=590202,
+    text_vnd_fmi_flexstor=590200,
-    text_vnd_wap_wmlscript=590203,
+    text_vnd_wap_wml=590201,
-    text_webviewhtml=590204,
+    text_vnd_wap_wmlscript=590202,
-    text_x_Algol68=590205,
+    text_webviewhtml=590203,
-    text_x_asm=590206,
+    text_x_Algol68=590204,
-    text_x_audiosoft_intra=590207,
+    text_x_asm=590205,
-    text_x_awk=590208,
+    text_x_audiosoft_intra=590206,
-    text_x_bcpl=590209,
+    text_x_awk=590207,
-    text_x_c=590210,
+    text_x_bcpl=590208,
-    text_x_c__=590211,
+    text_x_c=590209,
-    text_x_component=590212,
+    text_x_c__=590210,
-    text_x_diff=590213,
+    text_x_component=590211,
-    text_x_fortran=590214,
+    text_x_diff=590212,
-    text_x_java=590215,
+    text_x_fortran=590213,
-    text_x_la_asf=590216,
+    text_x_java=590214,
-    text_x_lisp=590217,
+    text_x_la_asf=590215,
-    text_x_m=590218,
+    text_x_lisp=590216,
-    text_x_m4=590219,
+    text_x_m=590217,
-    text_x_makefile=590220,
+    text_x_m4=590218,
-    text_x_ms_regedit=590221,
+    text_x_makefile=590219,
-    text_x_msdos_batch=590222,
+    text_x_ms_regedit=590220,
-    text_x_objective_c=590223,
+    text_x_msdos_batch=590221,
-    text_x_pascal=590224,
+    text_x_objective_c=590222,
-    text_x_perl=590225,
+    text_x_pascal=590223,
-    text_x_php=590226,
+    text_x_perl=590224,
-    text_x_po=590227,
+    text_x_php=590225,
-    text_x_python=590228,
+    text_x_po=590226,
-    text_x_ruby=590229,
+    text_x_python=590227,
-    text_x_sass=590230,
+    text_x_ruby=590228,
-    text_x_scss=590231,
+    text_x_sass=590229,
-    text_x_server_parsed_html=590232,
+    text_x_scss=590230,
-    text_x_setext=590233,
+    text_x_server_parsed_html=590231,
-    text_x_sgml=590234 | 0x01000000,
+    text_x_setext=590232,
-    text_x_shellscript=590235,
+    text_x_sgml=590233 | 0x01000000,
-    text_x_speech=590236,
+    text_x_shellscript=590234,
-    text_x_tcl=590237,
+    text_x_speech=590235,
-    text_x_tex=590238,
+    text_x_tcl=590236,
-    text_x_uil=590239,
+    text_x_tex=590237,
-    text_x_uuencode=590240,
+    text_x_uil=590238,
-    text_x_vcalendar=590241,
+    text_x_uuencode=590239,
-    text_x_vcard=590242,
+    text_x_vcalendar=590240,
-    text_xml=590243 | 0x01000000,
+    text_x_vcard=590241,
-    video_MP2T=393636,
+    text_xml=590242 | 0x01000000,
-    video_animaflex=393637,
+    video_MP2T=393635,
-    video_avi=393638,
+    video_animaflex=393636,
-    video_avs_video=393639,
+    video_avi=393637,
-    video_mp4=393640,
+    video_avs_video=393638,
-    video_mpeg=393641,
+    video_mp4=393639,
-    video_quicktime=393642,
+    video_mpeg=393640,
-    video_vdo=393643,
+    video_quicktime=393641,
-    video_vivo=393644,
+    video_vdo=393642,
-    video_vnd_rn_realvideo=393645,
+    video_vivo=393643,
-    video_vosaic=393646,
+    video_vnd_rn_realvideo=393644,
-    video_webm=393647,
+    video_vosaic=393645,
-    video_x_amt_demorun=393648,
+    video_webm=393646,
-    video_x_amt_showrun=393649,
+    video_x_amt_demorun=393647,
-    video_x_atomic3d_feature=393650,
+    video_x_amt_showrun=393648,
-    video_x_dl=393651,
+    video_x_atomic3d_feature=393649,
-    video_x_dv=393652,
+    video_x_dl=393650,
-    video_x_fli=393653,
+    video_x_dv=393651,
-    video_x_flv=393654,
+    video_x_fli=393652,
-    video_x_isvideo=393655,
+    video_x_flv=393653,
-    video_x_jng=393656 | 0x80000000,
+    video_x_isvideo=393654,
-    video_x_m4v=393657,
+    video_x_jng=393655 | 0x80000000,
-    video_x_matroska=393658,
+    video_x_m4v=393656,
-    video_x_mng=393659,
+    video_x_matroska=393657,
-    video_x_motion_jpeg=393660,
+    video_x_mng=393658,
-    video_x_ms_asf=393661,
+    video_x_motion_jpeg=393659,
-    video_x_msvideo=393662,
+    video_x_ms_asf=393660,
-    video_x_qtc=393663,
+    video_x_msvideo=393661,
-    video_x_sgi_movie=393664,
+    video_x_qtc=393662,
-    x_epoc_x_sisx_app=721345,
+    video_x_sgi_movie=393663,
+    x_epoc_x_sisx_app=721344,
 };
 char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
 case application_arj: return "application/arj";
@@ -906,6 +907,7 @@ case image_x_sony_arw: return "image/x-sony-arw";
 case image_x_sony_sr2: return "image/x-sony-sr2";
 case image_x_sony_srf: return "image/x-sony-srf";
 case image_x_epson_erf: return "image/x-epson-erf";
+case sist2_sidecar: return "sist2/sidecar";
 default: return NULL;}}
 GHashTable *mime_get_ext_table() {GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(ext_table, "arj", (gpointer)application_arj);
@@ -1449,6 +1451,7 @@ g_hash_table_insert(ext_table, "arw", (gpointer)image_x_sony_arw);
 g_hash_table_insert(ext_table, "sr2", (gpointer)image_x_sony_sr2);
 g_hash_table_insert(ext_table, "srf", (gpointer)image_x_sony_srf);
 g_hash_table_insert(ext_table, "erf", (gpointer)image_x_epson_erf);
+g_hash_table_insert(ext_table, "s2meta", (gpointer)sist2_sidecar);
 return ext_table;}
 GHashTable *mime_get_mime_table() {GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);
 g_hash_table_insert(mime_table, "application/arj", (gpointer)application_arj);
@@ -1900,5 +1903,6 @@ g_hash_table_insert(mime_table, "image/x-sony-arw", (gpointer)image_x_sony_arw);
 g_hash_table_insert(mime_table, "image/x-sony-sr2", (gpointer)image_x_sony_sr2);
 g_hash_table_insert(mime_table, "image/x-sony-srf", (gpointer)image_x_sony_srf);
 g_hash_table_insert(mime_table, "image/x-epson-erf", (gpointer)image_x_epson_erf);
+g_hash_table_insert(mime_table, "sist2/sidecar", (gpointer)sist2_sidecar);
 return mime_table;}
 #endif

src/parsing/parse.c

@@ -4,6 +4,7 @@
 #include "src/ctx.h"
 #include "mime.h"
 #include "src/io/serialize.h"
+#include "src/parsing/sidecar.h"
 
 #include <magic.h>
 
@@ -40,34 +41,50 @@ void fs_reset(struct vfile *f) {
 
 #define IS_GIT_OBJ (strlen(doc.filepath + doc.base) == 38 && (strstr(doc.filepath, "objects") != NULL))
 
+void set_dbg_current_file(parse_job_t *job) {
+    unsigned long long pid = (unsigned long long) pthread_self();
+    pthread_mutex_lock(&ScanCtx.dbg_current_files_mu);
+    g_hash_table_replace(ScanCtx.dbg_current_files, GINT_TO_POINTER(pid), job);
+    pthread_mutex_unlock(&ScanCtx.dbg_current_files_mu);
+}
+
 void parse(void *arg) {
 
     parse_job_t *job = arg;
     document_t doc;
 
-    int inc_ts = incremental_get(ScanCtx.original_table, job->vfile.info.st_ino);
+    set_dbg_current_file(job);
-    if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) {
-        incremental_mark_file_for_copy(ScanCtx.copy_table, job->vfile.info.st_ino);
-        return;
-    }
 
     doc.filepath = job->filepath;
     doc.ext = (short) job->ext;
     doc.base = (short) job->base;
+
+    char *rel_path = doc.filepath + ScanCtx.index.desc.root_len;
+    MD5((unsigned char *) rel_path, strlen(rel_path), doc.path_md5);
+
     doc.meta_head = NULL;
     doc.meta_tail = NULL;
     doc.mime = 0;
     doc.size = job->vfile.info.st_size;
-    doc.ino = job->vfile.info.st_ino;
     doc.mtime = job->vfile.info.st_mtim.tv_sec;
 
-    uuid_generate(doc.uuid);
+    int inc_ts = incremental_get(ScanCtx.original_table, doc.path_md5);
+    if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) {
+        incremental_mark_file_for_copy(ScanCtx.copy_table, doc.path_md5);
+
+        pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu);
+        ScanCtx.dbg_skipped_files_count += 1;
+        pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu);
+
+        return;
+    }
+
     char *buf[MAGIC_BUF_SIZE];
 
     if (LogCtx.very_verbose) {
-        char uuid_str[UUID_STR_LEN];
+        char path_md5_str[MD5_STR_LENGTH];
-        uuid_unparse(doc.uuid, uuid_str);
+        buf2hex(doc.path_md5, MD5_DIGEST_LENGTH, path_md5_str);
-        LOG_DEBUGF(job->filepath, "Starting parse job {%s}", uuid_str)
+        LOG_DEBUGF(job->filepath, "Starting parse job {%s}", path_md5_str)
     }
 
     if (job->vfile.info.st_size == 0) {
@@ -85,7 +102,8 @@ void parse(void *arg) {
 
         // Get mime type with libmagic
         if (!job->vfile.is_fs_file) {
-            LOG_WARNING(job->filepath, "Guessing mime type with libmagic inside archive files is not currently supported");
+            LOG_WARNING(job->filepath,
+                        "Guessing mime type with libmagic inside archive files is not currently supported");
             goto abort;
         }
 
@@ -99,6 +117,10 @@ void parse(void *arg) {
             }
 
             CLOSE_FILE(job->vfile)
+
+            pthread_mutex_lock(&ScanCtx.dbg_file_counts_mu);
+            ScanCtx.dbg_failed_files_count += 1;
+            pthread_mutex_unlock(&ScanCtx.dbg_file_counts_mu);
             return;
         }
 
@@ -157,16 +179,26 @@ void parse(void *arg) {
         parse_comic(&ScanCtx.comic_ctx, &job->vfile, &doc);
     } else if (IS_MOBI(doc.mime)) {
         parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, &doc);
+    } else if (doc.mime == MIME_SIST2_SIDECAR) {
+        parse_sidecar(&job->vfile, &doc);
+        CLOSE_FILE(job->vfile)
+        return;
+    } else if (is_msdoc(&ScanCtx.msdoc_ctx, doc.mime)) {
+        parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, &doc);
     }
 
     abort:
 
     //Parent meta
-    if (!uuid_is_null(job->parent)) {
+    if (!md5_digest_is_null(job->parent)) {
-        meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + UUID_STR_LEN + 1);
+        meta_line_t *meta_parent = malloc(sizeof(meta_line_t) + MD5_STR_LENGTH);
         meta_parent->key = MetaParent;
-        uuid_unparse(job->parent, meta_parent->str_val);
+        buf2hex(job->parent, MD5_DIGEST_LENGTH, meta_parent->str_val);
         APPEND_META((&doc), meta_parent)
+
+        doc.has_parent = TRUE;
+    } else {
+        doc.has_parent = FALSE;
     }
 
     write_document(&doc);

src/parsing/sidecar.c

@@ -0,0 +1,35 @@
+#include "sidecar.h"
+
+#include "src/ctx.h"
+
+void parse_sidecar(vfile_t *vfile, document_t *doc) {
+
+    LOG_DEBUGF("sidecar.c", "Parsing sidecar file %s", vfile->filepath)
+
+    size_t size;
+    char *buf = read_all(vfile, &size);
+    if (buf == NULL) {
+        LOG_ERRORF("sidecar.c", "Read error for %s", vfile->filepath)
+        return;
+    }
+
+    buf = realloc(buf, size + 1);
+    *(buf + size) = '\0';
+
+    cJSON *json = cJSON_Parse(buf);
+    if (json == NULL) {
+        LOG_ERRORF("sidecar.c", "Could not parse JSON sidecar %s", vfile->filepath)
+        return;
+    }
+    char *json_str = cJSON_PrintUnformatted(json);
+
+    unsigned char path_md5[MD5_DIGEST_LENGTH];
+    MD5((unsigned char *) vfile->filepath + ScanCtx.index.desc.root_len, doc->ext - 1 - ScanCtx.index.desc.root_len,
+        path_md5);
+
+    store_write(ScanCtx.index.meta_store, (char *) path_md5, sizeof(path_md5), json_str, strlen(json_str) + 1);
+
+    cJSON_Delete(json);
+    free(json_str);
+    free(buf);
+}

src/parsing/sidecar.h

@@ -0,0 +1,8 @@
+#ifndef SIST2_SIDECAR_H
+#define SIST2_SIDECAR_H
+
+#include "src/sist.h"
+
+void parse_sidecar(vfile_t *vfile, document_t *doc);
+
+#endif

src/sist.h

@@ -23,9 +23,10 @@
 #undef ABS
 #define ABS(a)	   (((a) < 0) ? -(a) : (a))
 
-#define UUID_STR_LEN 37
 #define UNUSED(x) __attribute__((__unused__))  x
 
+#define MD5_STR_LENGTH 33
+
 #include "util.h"
 #include "log.h"
 #include "types.h"
@@ -47,5 +48,4 @@
 #include <errno.h>
 #include <ctype.h>
 
-
 #endif

src/static/css/ion.rangeSlider.skinFlat.min.css

@@ -1 +1 @@
-.irs-bar,.irs-bar-edge,.irs-line-left,.irs-line-mid,.irs-line-right,.irs-slider{background:url("../img/sprite-skin-flat.png") repeat-x}.irs{height:40px}.irs-with-grid{height:60px}.irs-line{height:12px;top:25px}.irs-line-left{height:12px;background-position:0 -30px}.irs-line-mid{height:12px;background-position:0 0}.irs-line-right{height:12px;background-position:100% -30px}.irs-bar{height:12px;top:25px;background-position:0 -60px}.irs-bar-edge{top:25px;height:12px;width:9px;background-position:0 -90px}.irs-shadow{height:3px;top:34px;background:#000;opacity:0.25}.lt-ie9 .irs-shadow{filter: alpha(opacity=25)}.irs-slider{width:16px;height:18px;top:22px;background-position:0 -120px}.irs-slider.state_hover,.irs-slider:hover{background-position:0 -150px}.irs-max,.irs-min{color:#999;font-size:10px;line-height:1.333;text-shadow:none;top:0;padding:1px 3px;background:#e1e4e9;-moz-border-radius:4px;border-radius:4px}.irs-from,.irs-single,.irs-to{color:#fff;font-size:10px;line-height:1.333;text-shadow:none;padding:1px 5px;background:#2196F3;-moz-border-radius:4px;border-radius:4px}.irs-from:after,.irs-single:after,.irs-to:after{position:absolute;display:block;content:"";bottom:-6px;left:50%;width:0;height:0;margin-left:-3px;overflow:hidden;border:3px solid transparent;border-top-color:#2196F3}.irs-grid-pol{background:#e1e4e9}.irs-grid-text{color:#999}.irs-disabled{}
+.irs-bar,.irs-bar-edge,.irs-line-left,.irs-line-mid,.irs-line-right,.irs-slider{background:url("./img/sprite-skin-flat.png") repeat-x}.irs{height:40px}.irs-with-grid{height:60px}.irs-line{height:12px;top:25px}.irs-line-left{height:12px;background-position:0 -30px}.irs-line-mid{height:12px;background-position:0 0}.irs-line-right{height:12px;background-position:100% -30px}.irs-bar{height:12px;top:25px;background-position:0 -60px}.irs-bar-edge{top:25px;height:12px;width:9px;background-position:0 -90px}.irs-shadow{height:3px;top:34px;background:#000;opacity:0.25}.lt-ie9 .irs-shadow{filter: alpha(opacity=25)}.irs-slider{width:16px;height:18px;top:22px;background-position:0 -120px}.irs-slider.state_hover,.irs-slider:hover{background-position:0 -150px}.irs-max,.irs-min{color:#999;font-size:10px;line-height:1.333;text-shadow:none;top:0;padding:1px 3px;background:#e1e4e9;-moz-border-radius:4px;border-radius:4px}.irs-from,.irs-single,.irs-to{color:#fff;font-size:10px;line-height:1.333;text-shadow:none;padding:1px 5px;background:#2196F3;-moz-border-radius:4px;border-radius:4px}.irs-from:after,.irs-single:after,.irs-to:after{position:absolute;display:block;content:"";bottom:-6px;left:50%;width:0;height:0;margin-left:-3px;overflow:hidden;border:3px solid transparent;border-top-color:#2196F3}.irs-grid-pol{background:#e1e4e9}.irs-grid-text{color:#999}.irs-disabled{}

src/static/js/8_md5.min.js

@@ -0,0 +1 @@
+!function(n){"use strict";function d(n,t){var r=(65535&n)+(65535&t);return(n>>16)+(t>>16)+(r>>16)<<16|65535&r}function f(n,t,r,e,o,u){return d((c=d(d(t,n),d(e,u)))<<(f=o)|c>>>32-f,r);var c,f}function l(n,t,r,e,o,u,c){return f(t&r|~t&e,n,t,o,u,c)}function v(n,t,r,e,o,u,c){return f(t&e|r&~e,n,t,o,u,c)}function g(n,t,r,e,o,u,c){return f(t^r^e,n,t,o,u,c)}function m(n,t,r,e,o,u,c){return f(r^(t|~e),n,t,o,u,c)}function i(n,t){var r,e,o,u;n[t>>5]|=128<<t%32,n[14+(t+64>>>9<<4)]=t;for(var c=1732584193,f=-271733879,i=-1732584194,a=271733878,h=0;h<n.length;h+=16)c=l(r=c,e=f,o=i,u=a,n[h],7,-680876936),a=l(a,c,f,i,n[h+1],12,-389564586),i=l(i,a,c,f,n[h+2],17,606105819),f=l(f,i,a,c,n[h+3],22,-1044525330),c=l(c,f,i,a,n[h+4],7,-176418897),a=l(a,c,f,i,n[h+5],12,1200080426),i=l(i,a,c,f,n[h+6],17,-1473231341),f=l(f,i,a,c,n[h+7],22,-45705983),c=l(c,f,i,a,n[h+8],7,1770035416),a=l(a,c,f,i,n[h+9],12,-1958414417),i=l(i,a,c,f,n[h+10],17,-42063),f=l(f,i,a,c,n[h+11],22,-1990404162),c=l(c,f,i,a,n[h+12],7,1804603682),a=l(a,c,f,i,n[h+13],12,-40341101),i=l(i,a,c,f,n[h+14],17,-1502002290),c=v(c,f=l(f,i,a,c,n[h+15],22,1236535329),i,a,n[h+1],5,-165796510),a=v(a,c,f,i,n[h+6],9,-1069501632),i=v(i,a,c,f,n[h+11],14,643717713),f=v(f,i,a,c,n[h],20,-373897302),c=v(c,f,i,a,n[h+5],5,-701558691),a=v(a,c,f,i,n[h+10],9,38016083),i=v(i,a,c,f,n[h+15],14,-660478335),f=v(f,i,a,c,n[h+4],20,-405537848),c=v(c,f,i,a,n[h+9],5,568446438),a=v(a,c,f,i,n[h+14],9,-1019803690),i=v(i,a,c,f,n[h+3],14,-187363961),f=v(f,i,a,c,n[h+8],20,1163531501),c=v(c,f,i,a,n[h+13],5,-1444681467),a=v(a,c,f,i,n[h+2],9,-51403784),i=v(i,a,c,f,n[h+7],14,1735328473),c=g(c,f=v(f,i,a,c,n[h+12],20,-1926607734),i,a,n[h+5],4,-378558),a=g(a,c,f,i,n[h+8],11,-2022574463),i=g(i,a,c,f,n[h+11],16,1839030562),f=g(f,i,a,c,n[h+14],23,-35309556),c=g(c,f,i,a,n[h+1],4,-1530992060),a=g(a,c,f,i,n[h+4],11,1272893353),i=g(i,a,c,f,n[h+7],16,-155497632),f=g(f,i,a,c,n[h+10],23,-1094730640),c=g(c,f,i,a,n[h+13],4,681279174),a=g(a,c,f,i,n[h],11,-358537222),i=g(i,a,c,f,n[h+3],16,-722521979),f=g(f,i,a,c,n[h+6],23,76029189),c=g(c,f,i,a,n[h+9],4,-640364487),a=g(a,c,f,i,n[h+12],11,-421815835),i=g(i,a,c,f,n[h+15],16,530742520),c=m(c,f=g(f,i,a,c,n[h+2],23,-995338651),i,a,n[h],6,-198630844),a=m(a,c,f,i,n[h+7],10,1126891415),i=m(i,a,c,f,n[h+14],15,-1416354905),f=m(f,i,a,c,n[h+5],21,-57434055),c=m(c,f,i,a,n[h+12],6,1700485571),a=m(a,c,f,i,n[h+3],10,-1894986606),i=m(i,a,c,f,n[h+10],15,-1051523),f=m(f,i,a,c,n[h+1],21,-2054922799),c=m(c,f,i,a,n[h+8],6,1873313359),a=m(a,c,f,i,n[h+15],10,-30611744),i=m(i,a,c,f,n[h+6],15,-1560198380),f=m(f,i,a,c,n[h+13],21,1309151649),c=m(c,f,i,a,n[h+4],6,-145523070),a=m(a,c,f,i,n[h+11],10,-1120210379),i=m(i,a,c,f,n[h+2],15,718787259),f=m(f,i,a,c,n[h+9],21,-343485551),c=d(c,r),f=d(f,e),i=d(i,o),a=d(a,u);return[c,f,i,a]}function a(n){for(var t="",r=32*n.length,e=0;e<r;e+=8)t+=String.fromCharCode(n[e>>5]>>>e%32&255);return t}function h(n){var t=[];for(t[(n.length>>2)-1]=void 0,e=0;e<t.length;e+=1)t[e]=0;for(var r=8*n.length,e=0;e<r;e+=8)t[e>>5]|=(255&n.charCodeAt(e/8))<<e%32;return t}function e(n){for(var t,r="0123456789abcdef",e="",o=0;o<n.length;o+=1)t=n.charCodeAt(o),e+=r.charAt(t>>>4&15)+r.charAt(15&t);return e}function r(n){return unescape(encodeURIComponent(n))}function o(n){return a(i(h(t=r(n)),8*t.length));var t}function u(n,t){return function(n,t){var r,e,o=h(n),u=[],c=[];for(u[15]=c[15]=void 0,16<o.length&&(o=i(o,8*n.length)),r=0;r<16;r+=1)u[r]=909522486^o[r],c[r]=1549556828^o[r];return e=i(u.concat(h(t)),512+8*t.length),a(i(c.concat(e),640))}(r(n),r(t))}function t(n,t,r){return t?r?u(t,n):e(u(t,n)):r?o(n):e(o(n))}"function"==typeof define&&define.amd?define(function(){return t}):"object"==typeof module&&module.exports?module.exports=t:n.md5=t}(this);

src/static/js/dom.js

@@ -22,7 +22,7 @@ function gifOver(thumbnail, hit) {
     thumbnail.addEventListener("mouseout", function () {
         //Reset timer
         thumbnail.mouseStayedOver = false;
-        thumbnail.setAttribute("src", `t/${hit["_source"]["index"]}/${hit["_id"]}`);
+        thumbnail.setAttribute("src", `t/${hit["_source"]["index"]}/${hit["_path_md5"]}`);
     })
 }
 
@@ -192,6 +192,19 @@ function makeUserTag(tag, hit) {
     return userTag;
 }
 
+function makeGpsMetaRow(tbody, latitude, longitude) {
+    tbody.append($("<tr>")
+        .append($("<td>").text("Exif GPS"))
+        .append($("<td>")
+            .append($("<a>")
+                .text(`${latitude}, ${longitude}`)
+                .attr("href", `https://maps.google.com/?q=${latitude},${longitude}&ll=${latitude},${longitude}&t=k&z=17`)
+                .attr("target", "_blank")
+            )
+        )
+    );
+}
+
 function infoButtonCb(hit) {
     return () => {
         getDocumentInfo(hit["_id"]).then(doc => {
@@ -229,13 +242,25 @@ function infoButtonCb(hit) {
                     .text(new Date(doc["mtime"] * 1000).toISOString().split(".")[0].replace("T", " "))
                     .attr("title", doc["mtime"]))
             );
+
+            // Exif GPS
+            if ("exif_gps_longitude_dec" in doc) {
+                makeGpsMetaRow(tbody, doc["exif_gps_latitude_dec"], doc["exif_gps_longitude_dec"])
+            } else if ("exif_gps_longitude_dms" in doc) {
+                makeGpsMetaRow(
+                    tbody,
+                    dmsToDecimal(doc["exif_gps_latitude_dms"], doc["exif_gps_latitude_ref"]),
+                    dmsToDecimal(doc["exif_gps_longitude_dms"], doc["exif_gps_longitude_ref"]),
+                )
+            }
+
             const displayFields = new Set([
                 "mime", "size", "path", "title", "width", "height", "duration", "audioc", "videoc",
                 "bitrate", "artist", "album", "album_artist", "genre", "title", "font_name", "tag", "author",
                 "modified_by", "pages"
             ]);
             Object.keys(doc)
-                .filter(key => key.startsWith("_keyword.") || key.startsWith("_text.") || displayFields.has(key) || key.startsWith("exif_"))
+                .filter(key => key.startsWith("_keyword.") || key.startsWith("_text.") || displayFields.has(key) || (key.startsWith("exif_") && !key.includes("gps")))
                 .forEach(key => {
                     tbody.append($("<tr>")
                         .append($("<td>").text(key))
@@ -350,6 +375,14 @@ function createDocCard(hit) {
             audio.setAttribute("controls", "");
             audio.setAttribute("type", hit["_source"]["mime"]);
             audio.setAttribute("src", "f/" + hit["_id"]);
+            audio.addEventListener("play", () => {
+                // Pause all currently playing audio tags
+                $("audio").each(function () {
+                    if (this !== audio) {
+                        this.pause();
+                    }
+                });
+            });
 
             docCard.appendChild(audio)
         }
@@ -419,7 +452,7 @@ function makeThumbnail(mimeCategory, hit, imgWrapper, small) {
             thumbnail.setAttribute("class", "card-img-top fit");
         }
     }
-    thumbnail.setAttribute("src", `t/${hit["_source"]["index"]}/${hit["_id"]}`);
+    thumbnail.setAttribute("src", `t/${hit["_source"]["index"]}/${hit["_path_md5"]}`);
 
     if (shouldDisplayRawImage(hit)) {
         thumbnail.addEventListener("click", () => {

src/static/js/search.js

@@ -165,16 +165,19 @@ window.onload = () => {
             }
         }
     });
+
+    initTagTree();
+    updateTagTree();
 };
 
 function saveTag(tag, hit) {
-    const relPath = hit["_source"]["path"] + "/" + hit["_source"]["name"] + ext(hit);
+    const relPath = hit["_source"]["path"] + (hit["_source"]["path"] ? "/" : "") + hit["_source"]["name"] + ext(hit);
 
     return $.jsonPost("/tag/" + hit["_source"]["index"], {
         delete: false,
         name: tag,
         doc_id: hit["_id"],
-        relpath: relPath
+        path_md5: md5(relPath)
     }).then(() => {
         tagBar.blur();
         $("#tagModal").modal("hide");
@@ -188,6 +191,8 @@ function saveTag(tag, hit) {
             hideAfter: 3000,
             loaderBg: "#08c7e8",
         });
+
+        window.setTimeout(updateTagTree, 2000);
     })
 }
 
@@ -198,7 +203,7 @@ function deleteTag(tag, hit) {
         delete: true,
         name: tag,
         doc_id: hit["_id"],
-        relpath: relPath
+        path_md5: md5(relPath)
     }).then(() => {
         $.toast({
             heading: "Tag deleted",
@@ -210,6 +215,8 @@ function deleteTag(tag, hit) {
             hideAfter: 3000,
             loaderBg: "#08c7e8",
         });
+
+        window.setTimeout(updateTagTree, 2000);
     })
 }
 
@@ -313,25 +320,8 @@ $.jsonPost("es", {
     mimeTree.node("any").select();
 });
 
-// Tags tree
+function initTagTree() {
-$.jsonPost("es", {
+    tagMap = [{text: "All", id: "any"}];
-    aggs: {
-        tags: {
-            terms: {
-                field: "tag",
-                size: 10000
-            }
-        }
-    },
-    size: 0,
-}).then(resp => {
-    resp["aggregations"]["tags"]["buckets"]
-        .sort((a, b) => a["key"].localeCompare(b["key"]))
-        .forEach(bucket => {
-            addTag(tagMap, bucket["key"], bucket["key"], bucket["doc_count"])
-        });
-
-    tagMap.push({"text": "All", "id": "any"});
     tagTree = new InspireTree({
         selection: {
             mode: 'checkbox'
@@ -346,8 +336,34 @@ $.jsonPost("es", {
     });
     tagTree.on("node.state.changed", handleTreeClick(tagTree));
     tagTree.node("any").select();
-    searchBusy = false;
+}
-});
+
+function updateTagTree() {
+    $.jsonPost("es", {
+        aggs: {
+            tags: {
+                terms: {
+                    field: "tag",
+                    size: 10000
+                }
+            }
+        },
+        size: 0,
+    }).then(resp => {
+        tagMap = [];
+        resp["aggregations"]["tags"]["buckets"]
+            .sort((a, b) => a["key"].localeCompare(b["key"]))
+            .forEach(bucket => {
+                addTag(tagMap, bucket["key"], bucket["key"], bucket["doc_count"])
+            });
+
+        tagTree.removeAll();
+        tagMap.push({text: "All", id: "any"})
+        tagTree.addNodes(tagMap);
+        searchBusy = false;
+    });
+}
+
 
 function addTag(map, tag, id, count) {
     // let tags = tag.split("#")[0].split(".");
@@ -495,8 +511,8 @@ function search(after = null) {
         searchResults.appendChild(preload);
     }
 
-    let query = searchBar.value;
+    let searchBarValue = searchBar.value;
-    let empty = query === "";
+    let empty = searchBarValue === "";
     let condition = empty ? "should" : "must";
     let filters = [
         {range: {size: {gte: size_min, lte: size_max}}},
@@ -545,19 +561,32 @@ function search(after = null) {
         filters.push({range: {mtime: {lte: date_max}}})
     }
 
+    let query;
+    if (CONF.options.queryMode === "simple") {
+        query = {
+            simple_query_string: {
+                query: searchBarValue,
+                fields: fields,
+                default_operator: "and"
+            }
+        }
+    } else {
+        query = {
+            query_string: {
+                query: searchBarValue,
+                default_field: "name",
+                default_operator: "and"
+            }
+        }
+    }
+
     let q = {
         "_source": {
             excludes: ["content", "_tie"]
         },
         query: {
             bool: {
-                [condition]: {
+                [condition]: query,
-                    simple_query_string: {
-                        query: query,
-                        fields: fields,
-                        default_operator: "and"
-                    }
-                },
                 filter: filters
             }
         },
@@ -595,7 +624,9 @@ function search(after = null) {
         }
     }
 
-    $.jsonPost("es", q).then(searchResult => {
+    const showError = CONF.options.queryMode === "advanced";
+
+    $.jsonPost("es", q, showError).then(searchResult => {
         let hits = searchResult["hits"]["hits"];
         if (hits) {
             lastDoc = hits[hits.length - 1];
@@ -604,6 +635,7 @@ function search(after = null) {
         hits.forEach(hit => {
             hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
             hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+            hit["_path_md5"] = md5(hit["_source"]["path"] + (hit["_source"]["path"] ? "/" : "") + hit["_source"]["name"] + ext(hit));
         });
 
         if (!after) {
@@ -628,7 +660,25 @@ function search(after = null) {
         reachedEnd = hits.length !== SIZE;
         insertHits(resultContainer, hits);
         searchBusy = false;
-    });
+    }).fail(() => {
+        searchBusy = false;
+        if (!after) {
+            preload.remove();
+        }
+
+        console.log("QUERY:")
+        console.log(q)
+        $.toast({
+            heading: "Query error",
+            text: "Could not parse or execute query, please check the Advanced search documentation. " +
+                "See server logs for more information.",
+            stack: false,
+            bgColor: "#FF8F00",
+            textColor: "#FFF3E0",
+            position: 'bottom-right',
+            hideAfter: false
+        });
+    })
 }
 
 

src/static/js/util.js

@@ -70,7 +70,7 @@ function strUnescape(str) {
 
     for (let i = 0; i < str.length; i++) {
         const c = str[i];
-        const next = str[i+1];
+        const next = str[i + 1];
 
         if (c === ']') {
             if (next === ']') {
@@ -102,7 +102,8 @@ const _defaults = {
     treemapSize: "large",
     suggestPath: true,
     fragmentSize: 100,
-    columns: 5
+    columns: 5,
+    queryMode: "simple"
 };
 
 function loadSettings() {
@@ -120,6 +121,7 @@ function loadSettings() {
     $("#settingSuggestPath").prop("checked", CONF.options.suggestPath);
     $("#settingFragmentSize").val(CONF.options.fragmentSize);
     $("#settingColumns").val(CONF.options.columns);
+    $("#settingQueryMode").val(CONF.options.queryMode);
 }
 
 function Settings() {
@@ -127,6 +129,7 @@ function Settings() {
 
     this._onUpdate = function () {
         $("#fuzzyToggle").prop("checked", this.options.fuzzy);
+        $("#searchBar").attr("placeholder", this.options.queryMode === "simple" ? "Search" : "Advanced search");
         updateColumnStyle();
     };
 
@@ -165,6 +168,7 @@ function updateSettings() {
     CONF.options.suggestPath = $("#settingSuggestPath").prop("checked");
     CONF.options.fragmentSize = $("#settingFragmentSize").val();
     CONF.options.columns = $("#settingColumns").val();
+    CONF.options.queryMode = $("#settingQueryMode").val();
     CONF.save();
 
     if (typeof searchDebounced !== "undefined") {
@@ -187,14 +191,16 @@ function updateSettings() {
     });
 }
 
-jQuery["jsonPost"] = function (url, data) {
+jQuery["jsonPost"] = function (url, data, showError = true) {
     return jQuery.ajax({
         url: url,
         type: "post",
         data: JSON.stringify(data),
         contentType: "application/json"
     }).fail(err => {
-        showEsError();
+        if (showError) {
+            showEsError();
+        }
         console.log(err);
     });
 };
@@ -212,7 +218,7 @@ function updateColumnStyle() {
     const style = document.getElementById("style");
     if (style) {
         style.innerHTML =
-        `
+            `
 @media screen and (min-width: 1500px) {
     .container {
             max-width: 1440px;
@@ -229,4 +235,14 @@ function updateColumnStyle() {
 }
         `
     }
+}
+
+function dmsToDecimal(dms, ref) {
+    const tokens = dms.split(",")
+
+    const d = Number(tokens[0].trim().split(":")[0]) / Number(tokens[0].trim().split(":")[1])
+    const m = Number(tokens[1].trim().split(":")[0]) / Number(tokens[1].trim().split(":")[1])
+    const s = Number(tokens[2].trim().split(":")[0]) / Number(tokens[2].trim().split(":")[1])
+
+    return (d + (m / 60) + (s / 3600)) * (ref === "S" || ref === "W" ? -1 : 1)
 }

src/static/search.html

@@ -12,9 +12,9 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.8.3</span>
+    <span class="badge badge-pill version">2.10.3</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
-    <a class="btn ml-auto" href="/stats">Stats</a>
+    <a class="btn ml-auto" href="stats">Stats</a>
     <button class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings
     </button>
     <button class="btn" title="Toggle theme" onclick="toggleTheme()">Theme</button>
@@ -120,6 +120,8 @@
                 </div>
                 <div class="modal-body">
 
+                    <h2>Simple search</h2>
+
                     <table class="table">
                         <tbody>
                         <tr>
@@ -168,6 +170,12 @@
                     <p>For more information, see <a target="_blank"
                                                     href="//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html">Elasticsearch
                         documentation</a></p>
+
+                    <h2>Advanced search</h2>
+                    <p>For documentation about the advanced search mode, see <a target="_blank"
+                                                                                href="//www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax">Elasticsearch
+                        documentation</a></p>
+
                 </div>
             </div>
         </div>
@@ -207,10 +215,16 @@
 
                     <br/>
                     <div class="form-group">
-                        <input type="number" class="form-control" id="settingFragmentSize">
                         <label for="settingFragmentSize">Highlight context size in characters</label>
+                        <input type="number" class="form-control" id="settingFragmentSize">
                     </div>
 
+                    <label for="settingQueryMode">Search mode</label>
+                    <select id="settingQueryMode" class="form-control form-control-sm">
+                        <option value="simple">Simple</option>
+                        <option value="advanced">Advanced</option>
+                    </select>
+
                     <label for="settingDisplay">Display</label>
                     <select id="settingDisplay" class="form-control form-control-sm">
                         <option value="grid">Grid</option>

src/static/stats.html

@@ -10,7 +10,7 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.8.3</span>
+    <span class="badge badge-pill version">2.10.3</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <a style="margin-left: auto" class="btn" href="/">Back</a>
     <button class="btn" type="button" data-toggle="modal" data-target="#settings"
@@ -29,13 +29,13 @@
     </div>
 
     <div id="treemap-card" class="stats-card">
-        <button class="btn stats-btn" onclick="fullScreen('treemap-card')">Enlarge</button>
+        <button class="btn stats-btn" onclick="fullScreen('treemap-card')" id="treemap-card-enlarge">Enlarge</button>
         <button class="btn stats-btn" onclick="exportTreemap()">Export</button>
         <svg id="treemap"></svg>
     </div>
 
     <div id="graphs-card" class="stats-card">
-        <button class="btn stats-btn" onclick="fullScreen('graphs-card')">Enlarge</button>
+        <button class="btn stats-btn" onclick="fullScreen('graphs-card')" id="graphs-card-enlarge">Enlarge</button>
         <div class="graph">
             <svg id="agg_mime_size"></svg>
         </div>
@@ -84,10 +84,16 @@
 
                 <br/>
                 <div class="form-group">
-                    <input type="number" class="form-control" id="settingFragmentSize">
                     <label for="settingFragmentSize">Highlight context size in characters</label>
+                    <input type="number" class="form-control" id="settingFragmentSize">
                 </div>
 
+                <label for="settingQueryMode">Search mode</label>
+                <select id="settingQueryMode" class="form-control form-control-sm">
+                    <option value="simple">Simple</option>
+                    <option value="advanced">Advanced</option>
+                </select>
+
                 <label for="settingDisplay">Display</label>
                 <select id="settingDisplay" class="form-control form-control-sm">
                     <option value="grid">Grid</option>
@@ -727,7 +733,7 @@ function updateStats() {
 
     const indexId = $("#indices").val();
 
-    d3.csv(`/s/${indexId}/1`).then(tabularData => {
+    d3.csv(`./s/${indexId}/1`).then(tabularData => {
         tabularData.forEach(row => {
             row.taxonomy = row.path.split("/");
             row.size = Number(row.size);
@@ -742,16 +748,16 @@ function updateStats() {
         }
     });
 
-    d3.csv(`/s/${indexId}/2`).then(tabularData => {
+    d3.csv(`./s/${indexId}/2`).then(tabularData => {
         mimeBarSize(tabularData.slice(), mimeSvgSize);
         mimeBarCount(tabularData.slice(), mimeSvgCount);
     });
 
-    d3.csv(`/s/${indexId}/3`).then(tabularData => {
+    d3.csv(`./s/${indexId}/3`).then(tabularData => {
         sizeHistogram(tabularData, sizeHistogramSvg);
     });
 
-    d3.csv(`/s/${indexId}/4`).then(tabularData => {
+    d3.csv(`./s/${indexId}/4`).then(tabularData => {
         dateHistogram(tabularData, dateHistogramSvg);
     });
 
@@ -789,7 +795,15 @@ window.onload = function () {
 
 function fullScreen(selector) {
     const card = document.getElementById(selector);
+    const btn = document.getElementById(selector + "-enlarge");
+
     card.classList.toggle("full-screen");
+
+    if (card.classList.contains("full-screen")) {
+        btn.innerText = "Shrink";
+    } else {
+        btn.innerText = "Enlarge";
+    }
 }
 
 function exportTreemap() {

src/stats.c

@@ -2,8 +2,6 @@
 #include "io/serialize.h"
 #include "ctx.h"
 
-#include <glib.h>
-
 static GHashTable *FlatTree;
 static GHashTable *BufferTable;
 
@@ -22,7 +20,7 @@ typedef struct {
     long count;
 } agg_t;
 
-void fill_tables(cJSON *document, UNUSED(const char uuid_str[UUID_STR_LEN])) {
+void fill_tables(cJSON *document, UNUSED(const char index_id[MD5_STR_LENGTH])) {
 
     if (cJSON_GetObjectItem(document, "parent") != NULL) {
         return;
@@ -103,8 +101,8 @@ void read_index_into_tables(index_t *index) {
     while ((de = readdir(dir)) != NULL) {
         if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
             char file_path[PATH_MAX];
-            snprintf(file_path, PATH_MAX, "%s/%s", index->path, de->d_name);
+            snprintf(file_path, PATH_MAX, "%s%s", index->path, de->d_name);
-            read_index(file_path, index->desc.uuid, index->desc.type, fill_tables);
+            read_index(file_path, index->desc.id, index->desc.type, fill_tables);
         }
     }
     closedir(dir);

src/tpool.c

@@ -3,7 +3,7 @@
 #include "sist.h"
 #include <pthread.h>
 
-#define MAX_QUEUE_SIZE 10000
+#define MAX_QUEUE_SIZE 1000000
 
 typedef void (*thread_func_t)(void *arg);
 
@@ -27,6 +27,7 @@ typedef struct tpool {
     int thread_cnt;
     int work_cnt;
     int done_cnt;
+    int busy_cnt;
 
     int free_arg;
     int stop;
@@ -52,6 +53,14 @@ static tpool_work_t *tpool_work_create(thread_func_t func, void *arg) {
     return work;
 }
 
+void tpool_dump_debug_info(tpool_t *pool) {
+    LOG_DEBUGF("tpool.c", "pool->thread_cnt = %d", pool->thread_cnt)
+    LOG_DEBUGF("tpool.c", "pool->work_cnt = %d", pool->work_cnt)
+    LOG_DEBUGF("tpool.c", "pool->done_cnt = %d", pool->done_cnt)
+    LOG_DEBUGF("tpool.c", "pool->busy_cnt = %d", pool->busy_cnt)
+    LOG_DEBUGF("tpool.c", "pool->stop = %d", pool->stop)
+}
+
 /**
  * Pop work object from thread pool
  */
@@ -83,7 +92,7 @@ int tpool_add_work(tpool_t *pool, thread_func_t func, void *arg) {
     }
 
     while ((pool->work_cnt - pool->done_cnt) >= MAX_QUEUE_SIZE) {
-        usleep(100000);
+        usleep(10000);
     }
 
     pthread_mutex_lock(&(pool->work_mutex));
@@ -120,6 +129,10 @@ static void *tpool_worker(void *arg) {
         }
 
         tpool_work_t *work = tpool_work_get(pool);
+        if (work != NULL) {
+            pool->busy_cnt += 1;
+        }
+
         pthread_mutex_unlock(&(pool->work_mutex));
 
         if (work != NULL) {
@@ -136,6 +149,7 @@ static void *tpool_worker(void *arg) {
 
         pthread_mutex_lock(&(pool->work_mutex));
         if (work != NULL) {
+            pool->busy_cnt -= 1;
             pool->done_cnt++;
         }
 
@@ -150,6 +164,7 @@ static void *tpool_worker(void *arg) {
     if (pool->cleanup_func != NULL) {
         LOG_INFO("tpool.c", "Executing cleanup function")
         pool->cleanup_func();
+        LOG_DEBUG("tpool.c", "Done executing cleanup function")
     }
 
     pthread_cond_signal(&(pool->working_cond));
@@ -160,14 +175,14 @@ static void *tpool_worker(void *arg) {
 void tpool_wait(tpool_t *pool) {
     LOG_INFO("tpool.c", "Waiting for worker threads to finish")
     pthread_mutex_lock(&(pool->work_mutex));
-    while (1) {
+    while (TRUE) {
         if (pool->done_cnt < pool->work_cnt) {
             pthread_cond_wait(&(pool->working_cond), &(pool->work_mutex));
         } else {
-            usleep(500000);
+            LOG_INFOF("tpool.c", "Received head=NULL signal, busy_cnt=%d", pool->busy_cnt);
-            if (pool->done_cnt == pool->work_cnt) {
+
-                pool->stop = 1;
+            if (pool->done_cnt == pool->work_cnt && pool->busy_cnt == 0) {
-                usleep(1000000);
+                pool->stop = TRUE;
                 break;
             }
         }
@@ -187,12 +202,16 @@ void tpool_destroy(tpool_t *pool) {
 
     pthread_mutex_lock(&(pool->work_mutex));
     tpool_work_t *work = pool->work_head;
+    int count = 0;
     while (work != NULL) {
         tpool_work_t *tmp = work->next;
         free(work);
         work = tmp;
+        count += 1;
     }
 
+    LOG_DEBUGF("tpool.c", "Destroyed %d jobs", count);
+
     pthread_cond_broadcast(&(pool->has_work_cond));
     pthread_mutex_unlock(&(pool->work_mutex));
 
@@ -218,13 +237,14 @@ void tpool_destroy(tpool_t *pool) {
  * Create a thread pool
  * @param thread_cnt Worker threads count
  */
-tpool_t *tpool_create(size_t thread_cnt, void cleanup_func(), int free_arg) {
+tpool_t *tpool_create(int thread_cnt, void cleanup_func(), int free_arg) {
 
     tpool_t *pool = malloc(sizeof(tpool_t));
     pool->thread_cnt = thread_cnt;
     pool->work_cnt = 0;
     pool->done_cnt = 0;
-    pool->stop = 0;
+    pool->busy_cnt = 0;
+    pool->stop = FALSE;
     pool->free_arg = free_arg;
     pool->cleanup_func = cleanup_func;
     pool->threads = calloc(sizeof(pthread_t), thread_cnt);

src/tpool.h

@@ -8,12 +8,14 @@ typedef struct tpool tpool_t;
 
 typedef void (*thread_func_t)(void *arg);
 
-tpool_t *tpool_create(size_t num, void (*cleanup_func)(), int free_arg);
+tpool_t *tpool_create(int num, void (*cleanup_func)(), int free_arg);
 void tpool_start(tpool_t *pool);
-void tpool_destroy(tpool_t *tm);
+void tpool_destroy(tpool_t *pool);
 
 int tpool_add_work(tpool_t *pool, thread_func_t func, void *arg);
-void tpool_wait(tpool_t *tm);
+void tpool_wait(tpool_t *pool);
+
+void tpool_dump_debug_info(tpool_t *pool);
 
 #endif
 

src/types.h

@@ -6,7 +6,7 @@
 #define INDEX_VERSION_EXTERNAL "_external_v1"
 
 typedef struct index_descriptor {
-    char uuid[UUID_STR_LEN];
+    char id[MD5_STR_LENGTH];
     char version[64];
     long timestamp;
     char root[PATH_MAX];
@@ -20,6 +20,7 @@ typedef struct index_t {
     struct index_descriptor desc;
     struct store_t *store;
     struct store_t *tag_store;
+    struct store_t *meta_store;
     char path[PATH_MAX];
 } index_t;
 

src/util.c

@@ -2,7 +2,6 @@
 #include "src/ctx.h"
 
 #include <wordexp.h>
-#include <glib.h>
 
 #define PBSTR "========================================"
 #define PBWIDTH 40
@@ -125,7 +124,7 @@ void progress_bar_print(double percentage, size_t tn_size, size_t index_size) {
 }
 
 GHashTable *incremental_get_table() {
-    GHashTable *file_table = g_hash_table_new(g_direct_hash, g_direct_equal);
+    GHashTable *file_table = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL);
     return file_table;
 }
 

src/util.h

@@ -10,6 +10,8 @@
 #include "third-party/utf8.h/utf8.h"
 #include "libscan/scan.h"
 
+#define MD5_STR_LENGTH 33
+
 
 char *abspath(const char *path);
 
@@ -21,25 +23,6 @@ void progress_bar_print(double percentage, size_t tn_size, size_t index_size);
 
 GHashTable *incremental_get_table();
 
-__always_inline
-static void incremental_put(GHashTable *table, unsigned long inode_no, int mtime) {
-    g_hash_table_insert(table, (gpointer) inode_no, GINT_TO_POINTER(mtime));
-}
-
-__always_inline
-static int incremental_get(GHashTable *table, unsigned long inode_no) {
-    if (table != NULL) {
-        return GPOINTER_TO_INT(g_hash_table_lookup(table, (gpointer) inode_no));
-    } else {
-        return 0;
-    }
-}
-
-__always_inline
-static int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no) {
-    return g_hash_table_insert(table, GINT_TO_POINTER(inode_no), GINT_TO_POINTER(1));
-}
-
 
 const char *find_file_in_paths(const char **paths, const char *filename);
 
@@ -48,4 +31,95 @@ void str_escape(char *dst, const char *str);
 
 void str_unescape(char *dst, const char *str);
 
+static int hex2buf(const char *str, int len, unsigned char *bytes) {
+    static const uint8_t hashmap[] = {
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+            0x08, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+    };
+
+    for (int pos = 0; pos < len; pos += 2) {
+        int idx0 = (uint8_t) str[pos + 0];
+        int idx1 = (uint8_t) str[pos + 1];
+        bytes[pos / 2] = (uint8_t) (hashmap[idx0] << 4) | hashmap[idx1];
+    }
+    return TRUE;
+}
+
+__always_inline
+static void buf2hex(const unsigned char *buf, size_t buflen, char *hex_string) {
+    static const char hexdig[] = "0123456789abcdef";
+
+    const unsigned char *p;
+    size_t i;
+
+    char *s = hex_string;
+    for (i = 0, p = buf; i < buflen; i++, p++) {
+        *s++ = hexdig[(*p >> 4) & 0x0f];
+        *s++ = hexdig[*p & 0x0f];
+    }
+    *s = '\0';
+}
+
+
+__always_inline
+static int md5_digest_is_null(const unsigned char digest[MD5_DIGEST_LENGTH]) {
+    return (*(int64_t *) digest) == 0 && (*((int64_t *) digest + 1)) == 0;
+}
+
+
+__always_inline
+static void incremental_put(GHashTable *table, unsigned char path_md5[MD5_DIGEST_LENGTH], int mtime) {
+    char *ptr = malloc(MD5_STR_LENGTH);
+    buf2hex(path_md5, MD5_DIGEST_LENGTH, ptr);
+    g_hash_table_insert(table, ptr, GINT_TO_POINTER(mtime));
+}
+
+__always_inline
+static int incremental_get(GHashTable *table, unsigned char path_md5[MD5_DIGEST_LENGTH]) {
+    if (table != NULL) {
+        char md5_str[MD5_STR_LENGTH];
+        buf2hex(path_md5, MD5_DIGEST_LENGTH, md5_str);
+        return GPOINTER_TO_INT(g_hash_table_lookup(table, md5_str));
+    } else {
+        return 0;
+    }
+}
+
+__always_inline
+static int incremental_mark_file_for_copy(GHashTable *table, unsigned char path_md5[MD5_DIGEST_LENGTH]) {
+    char *ptr = malloc(MD5_STR_LENGTH);
+    buf2hex(path_md5, MD5_DIGEST_LENGTH, ptr);
+    return g_hash_table_insert(table, ptr, GINT_TO_POINTER(1));
+}
+
 #endif

src/web/serve.c

@@ -8,18 +8,8 @@
 
 #include <src/ctx.h>
 
-#include <mongoose.h>
 
-
+static void send_response_line(struct mg_connection *nc, int status_code, size_t length, char *extra_headers) {
-static int has_prefix(const struct mg_str *str, const struct mg_str *prefix) {
-    return str->len > prefix->len && memcmp(str->p, prefix->p, prefix->len) == 0;
-}
-
-static int is_equal(const struct mg_str *s1, const struct mg_str *s2) {
-    return s1->len == s2->len && memcmp(s1->p, s2->p, s2->len) == 0;
-}
-
-static void send_response_line(struct mg_connection *nc, int status_code, int length, char *extra_headers) {
     mg_printf(
             nc,
             "HTTP/1.1 %d %s\r\n"
@@ -36,7 +26,7 @@ static void send_response_line(struct mg_connection *nc, int status_code, int le
 
 index_t *get_index_by_id(const char *index_id) {
     for (int i = WebCtx.index_count; i >= 0; i--) {
-        if (strcmp(index_id, WebCtx.indices[i].desc.uuid) == 0) {
+        if (strncmp(index_id, WebCtx.indices[i].desc.id, MD5_STR_LENGTH) == 0) {
             return &WebCtx.indices[i];
         }
     }
@@ -62,36 +52,32 @@ store_t *get_tag_store(const char *index_id) {
 void search_index(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(search_html), "Content-Type: text/html");
     mg_send(nc, search_html, sizeof(search_html));
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 void stats(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(stats_html), "Content-Type: text/html");
     mg_send(nc, stats_html, sizeof(stats_html));
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void stats_files(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
+void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
 
-    if (path->len != UUID_STR_LEN + 4) {
+    if (hm->uri.len != MD5_STR_LENGTH + 4) {
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
-    char arg_uuid[UUID_STR_LEN];
+    char arg_md5[MD5_STR_LENGTH];
-    memcpy(arg_uuid, hm->uri.p + 3, UUID_STR_LEN);
+    memcpy(arg_md5, hm->uri.ptr + 3, MD5_STR_LENGTH);
-    *(arg_uuid + UUID_STR_LEN - 1) = '\0';
+    *(arg_md5 + MD5_STR_LENGTH - 1) = '\0';
 
-    index_t *index = get_index_by_id(arg_uuid);
+    index_t *index = get_index_by_id(arg_md5);
     if (index == NULL) {
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     const char *file;
-    switch (atoi(hm->uri.p + 3 + UUID_STR_LEN)) {
+    switch (atoi(hm->uri.ptr + 3 + MD5_STR_LENGTH)) {
         case 1:
             file = "treemap.csv";
             break;
@@ -105,54 +91,41 @@ void stats_files(struct mg_connection *nc, struct http_message *hm, struct mg_st
             file = "date_agg.csv";
             break;
         default:
-            nc->flags |= MG_F_SEND_AND_CLOSE;
             return;
     }
 
     char disposition[8192];
-    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s\"", file);
+    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s\"\r\n", file);
 
     char full_path[PATH_MAX];
     strcpy(full_path, index->path);
     strcat(full_path, file);
 
-    mg_http_serve_file(nc, hm, full_path, mg_mk_str("text/csv"), mg_mk_str(disposition));
+    mg_http_serve_file(nc, hm, full_path, "text/csv", disposition);
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 void javascript_lib(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(bundle_js), "Content-Type: application/javascript");
     mg_send(nc, bundle_js, sizeof(bundle_js));
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 void javascript_search(struct mg_connection *nc) {
     send_response_line(nc, 200, sizeof(search_js), "Content-Type: application/javascript");
     mg_send(nc, search_js, sizeof(search_js));
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-int client_requested_dark_theme(struct http_message *hm) {
+int client_requested_dark_theme(struct mg_http_message *hm) {
-    struct mg_str *cookie_header = mg_get_http_header(hm, "cookie");
+    struct mg_str *cookie_header = mg_http_get_header(hm, "cookie");
     if (cookie_header == NULL) {
         return FALSE;
     }
 
-    char buf[4096];
+    struct mg_str sist_cookie = mg_http_get_header_var(*cookie_header, mg_str_n("sist", 4));
-    char *sist_cookie = buf;
-    if (mg_http_parse_header2(cookie_header, "sist", &sist_cookie, sizeof(buf)) == 0) {
-        return FALSE;
-    }
 
-    int ret = strcmp(sist_cookie, "dark") == 0;
+    return mg_strcmp(sist_cookie, mg_str_n("dark", 4)) == 0;
-    if (sist_cookie != buf) {
-        free(sist_cookie);
-    }
-
-    return ret;
 }
 
-void style(struct mg_connection *nc, struct http_message *hm) {
+void style(struct mg_connection *nc, struct mg_http_message *hm) {
 
     if (client_requested_dark_theme(hm)) {
         send_response_line(nc, 200, sizeof(bundle_dark_css), "Content-Type: text/css");
@@ -161,11 +134,9 @@ void style(struct mg_connection *nc, struct http_message *hm) {
         send_response_line(nc, 200, sizeof(bundle_css), "Content-Type: text/css");
         mg_send(nc, bundle_css, sizeof(bundle_css));
     }
-
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void img_sprite_skin_flat(struct mg_connection *nc, struct http_message *hm) {
+void img_sprite_skin_flat(struct mg_connection *nc, struct mg_http_message *hm) {
     if (client_requested_dark_theme(hm)) {
         send_response_line(nc, 200, sizeof(sprite_skin_flat_dark_png), "Content-Type: image/png");
         mg_send(nc, sprite_skin_flat_dark_png, sizeof(sprite_skin_flat_dark_png));
@@ -173,71 +144,62 @@ void img_sprite_skin_flat(struct mg_connection *nc, struct http_message *hm) {
         send_response_line(nc, 200, sizeof(sprite_skin_flat_png), "Content-Type: image/png");
         mg_send(nc, sprite_skin_flat_png, sizeof(sprite_skin_flat_png));
     }
-
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void thumbnail(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
+void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) {
 
-    if (path->len != UUID_STR_LEN * 2 + 2) {
+    if (hm->uri.len != 68) {
-        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) path->len, path->p)
+        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr)
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "Not found");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
-    char arg_uuid[UUID_STR_LEN];
+    char arg_file_md5[MD5_STR_LENGTH];
-    char arg_index[UUID_STR_LEN];
+    char arg_index[MD5_STR_LENGTH];
 
-    memcpy(arg_index, hm->uri.p + 3, UUID_STR_LEN);
+    memcpy(arg_index, hm->uri.ptr + 3, MD5_STR_LENGTH);
-    *(arg_index + UUID_STR_LEN - 1) = '\0';
+    *(arg_index + MD5_STR_LENGTH - 1) = '\0';
-    memcpy(arg_uuid, hm->uri.p + 3 + UUID_STR_LEN, UUID_STR_LEN);
+    memcpy(arg_file_md5, hm->uri.ptr + 3 + MD5_STR_LENGTH, MD5_STR_LENGTH);
-    *(arg_uuid + UUID_STR_LEN - 1) = '\0';
+    *(arg_file_md5 + MD5_STR_LENGTH - 1) = '\0';
 
-    uuid_t uuid;
+    unsigned char md5_buf[MD5_DIGEST_LENGTH];
-    int ret = uuid_parse(arg_uuid, uuid);
+    hex2buf(arg_file_md5, MD5_STR_LENGTH - 1, md5_buf);
-    if (ret != 0) {
-        LOG_DEBUGF("serve.c", "Invalid thumbnail UUID: %s", arg_uuid)
-        mg_http_send_error(nc, 404, NULL);
-        nc->flags |= MG_F_SEND_AND_CLOSE;
-        return;
-    }
 
     store_t *store = get_store(arg_index);
     if (store == NULL) {
         LOG_DEBUGF("serve.c", "Could not get store for index: %s", arg_index)
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "Not found");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     size_t data_len = 0;
-    char *data = store_read(store, (char *) uuid, sizeof(uuid_t), &data_len);
+    char *data = store_read(store, (char *) md5_buf, sizeof(md5_buf), &data_len);
     if (data_len != 0) {
         send_response_line(nc, 200, data_len, "Content-Type: image/jpeg");
         mg_send(nc, data, data_len);
         free(data);
+    } else {
+        mg_http_reply(nc, 404, "Content-Type: text/plain;charset=utf-8\r\n", "Not found");
+        return;
     }
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void search(struct mg_connection *nc, struct http_message *hm) {
+void search(struct mg_connection *nc, struct mg_http_message *hm) {
 
     if (hm->body.len == 0) {
         LOG_DEBUG("serve.c", "Client sent empty body, ignoring request")
-        mg_http_send_error(nc, 500, NULL);
+        mg_http_reply(nc, 500, "", "Invalid request");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.p, hm->body.len);
+    memcpy(body, hm->body.ptr, hm->body.len);
     *(body + hm->body.len) = '\0';
 
     char url[4096];
     snprintf(url, 4096, "%s/%s/_search", WebCtx.es_url, WebCtx.es_index);
 
-    nc->user_data = web_post_async(url, body);
+    nc->fn_data = web_post_async(url, body);
 }
 
 void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
@@ -259,16 +221,16 @@ void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
              idx->desc.rewrite_url, path_unescaped, name_unescaped, strlen(ext) == 0 ? "" : ".", ext);
 
     dyn_buffer_t encoded = url_escape(url);
-    mg_http_send_redirect(
+    dyn_buffer_write_char(&encoded, '\0');
-            nc, 308,
+
-            (struct mg_str) MG_MK_STR_N(encoded.buf, encoded.cur),
+    char location_header[8192];
-            (struct mg_str) MG_NULL_STR
+    snprintf(location_header, sizeof(location_header), "Location: %s\r\n", encoded.buf);
-    );
+
+    mg_http_reply(nc, 308, location_header, "");
     dyn_buffer_destroy(&encoded);
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, struct http_message *hm) {
+void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, struct mg_http_message *hm) {
 
     const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
     const char *name = cJSON_GetObjectItem(json, "name")->valuestring;
@@ -289,10 +251,10 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
     LOG_DEBUGF("serve.c", "Serving file from disk: %s", full_path)
 
     char disposition[8192];
-    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s%s%s\"",
+    snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s%s%s\"\r\n",
              name, strlen(ext) == 0 ? "" : ".", ext);
 
-    mg_http_serve_file(nc, hm, full_path, mg_mk_str(mime), mg_mk_str(disposition));
+    mg_http_serve_file(nc, hm, full_path, mime, disposition);
 }
 
 void index_info(struct mg_connection *nc) {
@@ -305,7 +267,7 @@ void index_info(struct mg_connection *nc) {
         cJSON *idx_json = cJSON_CreateObject();
         cJSON_AddStringToObject(idx_json, "name", idx->desc.name);
         cJSON_AddStringToObject(idx_json, "version", idx->desc.version);
-        cJSON_AddStringToObject(idx_json, "id", idx->desc.uuid);
+        cJSON_AddStringToObject(idx_json, "id", idx->desc.id);
         cJSON_AddNumberToObject(idx_json, "timestamp", (double) idx->desc.timestamp);
         cJSON_AddItemToArray(arr, idx_json);
     }
@@ -316,40 +278,35 @@ void index_info(struct mg_connection *nc) {
     mg_send(nc, json_str, strlen(json_str));
     free(json_str);
     cJSON_Delete(json);
-
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 
-void document_info(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
+void document_info(struct mg_connection *nc, struct mg_http_message *hm) {
 
-    if (path->len != UUID_STR_LEN + 2) {
+    if (hm->uri.len != MD5_STR_LENGTH + 2) {
-        LOG_DEBUGF("serve.c", "Invalid document_info path: %.*s", (int) path->len, path->p)
+        LOG_DEBUGF("serve.c", "Invalid document_info path: %.*s", (int) hm->uri.len, hm->uri.ptr)
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "Not found");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
-    char arg_uuid[UUID_STR_LEN];
+    char arg_md5[MD5_STR_LENGTH];
-    memcpy(arg_uuid, hm->uri.p + 3, UUID_STR_LEN);
+    memcpy(arg_md5, hm->uri.ptr + 3, MD5_STR_LENGTH);
-    *(arg_uuid + UUID_STR_LEN - 1) = '\0';
+    *(arg_md5 + MD5_STR_LENGTH - 1) = '\0';
 
-    cJSON *doc = elastic_get_document(arg_uuid);
+    cJSON *doc = elastic_get_document(arg_md5);
     cJSON *source = cJSON_GetObjectItem(doc, "_source");
 
     cJSON *index_id = cJSON_GetObjectItem(source, "index");
     if (index_id == NULL) {
         cJSON_Delete(doc);
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "Not found");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     index_t *idx = get_index_by_id(index_id->valuestring);
     if (idx == NULL) {
         cJSON_Delete(doc);
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "Not found");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
@@ -358,24 +315,21 @@ void document_info(struct mg_connection *nc, struct http_message *hm, struct mg_
     mg_send(nc, json_str, (int) strlen(json_str));
     free(json_str);
     cJSON_Delete(doc);
-
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
-void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
+void file(struct mg_connection *nc, struct mg_http_message *hm) {
 
-    if (path->len != UUID_STR_LEN + 2) {
+    if (hm->uri.len != MD5_STR_LENGTH + 2) {
-        LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) path->len, path->p)
+        LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr)
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "Not found");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
-    char arg_uuid[UUID_STR_LEN];
+    char arg_md5[MD5_STR_LENGTH];
-    memcpy(arg_uuid, hm->uri.p + 3, UUID_STR_LEN);
+    memcpy(arg_md5, hm->uri.ptr + 3, MD5_STR_LENGTH);
-    *(arg_uuid + UUID_STR_LEN - 1) = '\0';
+    *(arg_md5 + MD5_STR_LENGTH - 1) = '\0';
 
-    const char *next = arg_uuid;
+    const char *next = arg_md5;
     cJSON *doc = NULL;
     cJSON *index_id = NULL;
     cJSON *source = NULL;
@@ -386,8 +340,7 @@ void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path
         index_id = cJSON_GetObjectItem(source, "index");
         if (index_id == NULL) {
             cJSON_Delete(doc);
-            mg_http_send_error(nc, 404, NULL);
+            mg_http_reply(nc, 404, "", "Not found");
-            nc->flags |= MG_F_SEND_AND_CLOSE;
             return;
         }
         cJSON *parent = cJSON_GetObjectItem(source, "parent");
@@ -401,8 +354,7 @@ void file(struct mg_connection *nc, struct http_message *hm, struct mg_str *path
 
     if (idx == NULL) {
         cJSON_Delete(doc);
-        nc->flags |= MG_F_SEND_AND_CLOSE;
+        mg_http_reply(nc, 404, "", "Not found");
-        mg_http_send_error(nc, 404, NULL);
         return;
     }
 
@@ -423,14 +375,12 @@ void status(struct mg_connection *nc) {
     }
 
     free(status);
-
-    nc->flags |= MG_F_SEND_AND_CLOSE;
 }
 
 typedef struct {
     char *name;
     int delete;
-    char *relpath;
+    char *path_md5_str;
     char *doc_id;
 } tag_req_t;
 
@@ -450,8 +400,9 @@ tag_req_t *parse_tag_request(cJSON *json) {
         return NULL;
     }
 
-    cJSON *arg_relpath = cJSON_GetObjectItem(json, "relpath");
+    cJSON *arg_path_md5 = cJSON_GetObjectItem(json, "path_md5");
-    if (arg_relpath == NULL || !cJSON_IsString(arg_relpath)) {
+    if (arg_path_md5 == NULL || !cJSON_IsString(arg_path_md5) ||
+        strlen(arg_path_md5->valuestring) != MD5_STR_LENGTH - 1) {
         return NULL;
     }
 
@@ -463,41 +414,38 @@ tag_req_t *parse_tag_request(cJSON *json) {
     tag_req_t *req = malloc(sizeof(tag_req_t));
     req->delete = arg_delete->valueint;
     req->name = arg_name->valuestring;
-    req->relpath = arg_relpath->valuestring;
+    req->path_md5_str = arg_path_md5->valuestring;
     req->doc_id = arg_doc_id->valuestring;
 
     return req;
 }
 
-void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
+void tag(struct mg_connection *nc, struct mg_http_message *hm) {
-    if (path->len != UUID_STR_LEN + 4) {
+    if (hm->uri.len != MD5_STR_LENGTH + 4) {
-        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) path->len, path->p)
+        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr)
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "Not found");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
-    char arg_index[UUID_STR_LEN];
+    char arg_index[MD5_STR_LENGTH];
-    memcpy(arg_index, hm->uri.p + 5, UUID_STR_LEN);
+    memcpy(arg_index, hm->uri.ptr + 5, MD5_STR_LENGTH);
-    *(arg_index + UUID_STR_LEN - 1) = '\0';
+    *(arg_index + MD5_STR_LENGTH - 1) = '\0';
 
     if (hm->body.len < 2 || hm->method.len != 4 || memcmp(&hm->method, "POST", 4) == 0) {
         LOG_DEBUG("serve.c", "Invalid tag request")
-        mg_http_send_error(nc, 400, NULL);
+        mg_http_reply(nc, 404, "", "Not found");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     store_t *store = get_tag_store(arg_index);
     if (store == NULL) {
         LOG_DEBUGF("serve.c", "Could not get tag store for index: %s", arg_index)
-        mg_http_send_error(nc, 404, NULL);
+        mg_http_reply(nc, 404, "", "Not found");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.p, hm->body.len);
+    memcpy(body, hm->body.ptr, hm->body.len);
     *(body + hm->body.len) = '\0';
     cJSON *json = cJSON_Parse(body);
 
@@ -506,15 +454,14 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
         LOG_DEBUGF("serve.c", "Could not parse tag request", arg_index)
         cJSON_Delete(json);
         free(body);
-        mg_http_send_error(nc, 400, NULL);
+        mg_http_reply(nc, 400, "", "Invalid request");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return;
     }
 
     cJSON *arr = NULL;
 
     size_t data_len = 0;
-    const char *data = store_read(store, arg_req->relpath, strlen(arg_req->relpath), &data_len);
+    const char *data = store_read(store, arg_req->path_md5_str, MD5_STR_LENGTH, &data_len);
     if (data_len == 0) {
         arr = cJSON_CreateArray();
     } else {
@@ -535,8 +482,8 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
             }
         }
 
-        char buf[8192];
+        char *buf = malloc(sizeof(char) * 8192);
-        snprintf(buf, sizeof(buf),
+        snprintf(buf, 8192,
                  "{"
                  "    \"script\" : {"
                  "        \"source\": \"if (ctx._source.tag.contains(params.tag)) { ctx._source.tag.remove(ctx._source.tag.indexOf(params.tag)) }\","
@@ -550,13 +497,13 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
 
         char url[4096];
         snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
-        nc->user_data = web_post_async(url, buf);
+        nc->fn_data = web_post_async(url, buf);
 
     } else {
         cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name));
 
-        char buf[8192];
+        char *buf = malloc(sizeof(char) * 8192);
-        snprintf(buf, sizeof(buf),
+        snprintf(buf, 8192,
                  "{"
                  "    \"script\" : {"
                  "        \"source\": \"if(ctx._source.tag == null) {ctx._source.tag = new ArrayList()} ctx._source.tag.add(params.tag)\","
@@ -570,11 +517,12 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
 
         char url[4096];
         snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id);
-        nc->user_data = web_post_async(url, buf);
+        nc->fn_data = web_post_async(url, buf);
     }
 
     char *json_str = cJSON_PrintUnformatted(arr);
-    store_write(store, arg_req->relpath, strlen(arg_req->relpath) + 1, json_str, strlen(json_str) + 1);
+    store_write(store, arg_req->path_md5_str, MD5_STR_LENGTH, json_str, strlen(json_str) + 1);
+    store_flush(store);
 
     free(arg_req);
     free(json_str);
@@ -583,39 +531,22 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
     free(body);
 }
 
-int validate_auth(struct mg_connection *nc, struct http_message *hm) {
+int validate_auth(struct mg_connection *nc, struct mg_http_message *hm) {
     char user[256] = {0,};
     char pass[256] = {0,};
 
-    int ret = mg_get_http_basic_auth(hm, user, sizeof(user), pass, sizeof(pass));
+    mg_http_creds(hm, user, sizeof(user), pass, sizeof(pass));
-    if (ret == -1 || strcmp(user, WebCtx.auth_user) != 0 || strcmp(pass, WebCtx.auth_pass) != 0) {
+    if (strcmp(user, WebCtx.auth_user) != 0 || strcmp(pass, WebCtx.auth_pass) != 0) {
-        mg_printf(nc, "HTTP/1.1 401 Unauthorized\r\n"
+        mg_http_reply(nc, 401, "WWW-Authenticate: Basic realm=\"sist2\"\r\n", "");
-                      "WWW-Authenticate: Basic realm=\"sist2\"\r\n"
-                      "Content-Length: 0\r\n\r\n");
-        nc->flags |= MG_F_SEND_AND_CLOSE;
         return FALSE;
     }
     return TRUE;
 }
 
-static void ev_router(struct mg_connection *nc, int ev, void *p) {
+static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(void *fn_data)) {
-    struct mg_str scheme;
-    struct mg_str user_info;
-    struct mg_str host;
-    unsigned int port;
-    struct mg_str path;
-    struct mg_str query;
-    struct mg_str fragment;
-
-    if (ev == MG_EV_HTTP_REQUEST) {
-        struct http_message *hm = (struct http_message *) p;
-
-        if (mg_parse_uri(hm->uri, &scheme, &user_info, &host, &port, &path, &query, &fragment) != 0) {
-            mg_http_send_error(nc, 400, NULL);
-            nc->flags |= MG_F_SEND_AND_CLOSE;
-            return;
-        }
 
+    if (ev == MG_EV_HTTP_MSG) {
+        struct mg_http_message *hm = (struct mg_http_message *) ev_data;
 
         if (WebCtx.auth_enabled == TRUE) {
             if (!validate_auth(nc, hm)) {
@@ -623,52 +554,48 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
             }
         }
 
-        if (is_equal(&path, &((struct mg_str) MG_MK_STR("/")))) {
+        if (mg_http_match_uri(hm, "/")) {
             search_index(nc);
-        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/css")))) {
+        } else if (mg_http_match_uri(hm, "/css")) {
             style(nc, hm);
-        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/stats")))) {
+        } else if (mg_http_match_uri(hm, "/stats")) {
             stats(nc);
-        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/jslib")))) {
+        } else if (mg_http_match_uri(hm, "/jslib")) {
             javascript_lib(nc);
-        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/jssearch")))) {
+        } else if (mg_http_match_uri(hm, "/jssearch")) {
             javascript_search(nc);
-        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/img/sprite-skin-flat.png")))) {
+        } else if (mg_http_match_uri(hm, "/img/sprite-skin-flat.png")) {
             img_sprite_skin_flat(nc, hm);
-        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/es")))) {
+        } else if (mg_http_match_uri(hm, "/es")) {
             search(nc, hm);
-        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/i")))) {
+        } else if (mg_http_match_uri(hm, "/i")) {
             index_info(nc);
-        } else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/status")))) {
+        } else if (mg_http_match_uri(hm, "/status")) {
             status(nc);
-        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/f/")))) {
+        } else if (mg_http_match_uri(hm, "/f/*")) {
-            file(nc, hm, &path);
+            file(nc, hm);
-        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/t/")))) {
+        } else if (mg_http_match_uri(hm, "/t/*/*")) {
-            thumbnail(nc, hm, &path);
+            thumbnail(nc, hm);
-        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/s/")))) {
+        } else if (mg_http_match_uri(hm, "/s/*/*")) {
-            stats_files(nc, hm, &path);
+            stats_files(nc, hm);
-        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/tag/")))) {
+        } else if (mg_http_match_uri(hm, "/tag/*")) {
-            if (WebCtx.tag_auth_enabled == TRUE) {
+            if (WebCtx.tag_auth_enabled == TRUE && !validate_auth(nc, hm)) {
-                if (!validate_auth(nc, hm)) {
+                return;
-                    return;
-                }
             }
-            tag(nc, hm, &path);
+            tag(nc, hm);
-        } else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/d/")))) {
+        } else if (mg_http_match_uri(hm, "/d/*")) {
-            document_info(nc, hm, &path);
+            document_info(nc, hm);
         } else {
-            mg_http_send_error(nc, 404, NULL);
+            mg_http_reply(nc, 404, "", "Page not found");
-            nc->flags |= MG_F_SEND_AND_CLOSE;
         }
 
     } else if (ev == MG_EV_POLL) {
-        if (nc->user_data != NULL) {
+        if (nc->fn_data != NULL) {
             //Waiting for ES reply
-            subreq_ctx_t *ctx = (subreq_ctx_t *) nc->user_data;
+            subreq_ctx_t *ctx = (subreq_ctx_t *) nc->fn_data;
             web_post_async_poll(ctx);
 
             if (ctx->done == TRUE) {
-
                 response_t *r = ctx->response;
 
                 if (r->status_code == 200) {
@@ -688,14 +615,14 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
                         free(json_str);
                         free(tmp);
                     }
-                    mg_http_send_error(nc, 500, NULL);
+
+                    mg_http_reply(nc, 500, "", "");
                 }
 
                 free_response(r);
                 free(ctx->data);
                 free(ctx);
-                nc->flags |= MG_F_SEND_AND_CLOSE;
+                nc->fn_data = NULL;
-                nc->user_data = NULL;
             }
         }
     }
@@ -706,15 +633,18 @@ void serve(const char *listen_address) {
     printf("Starting web server @ http://%s\n", listen_address);
 
     struct mg_mgr mgr;
-    mg_mgr_init(&mgr, NULL);
+    mg_mgr_init(&mgr);
 
-    struct mg_connection *nc = mg_bind(&mgr, listen_address, ev_router);
+    int ok = 1;
+
+    struct mg_connection *nc = mg_http_listen(&mgr, listen_address, ev_router, NULL);
     if (nc == NULL) {
         LOG_FATALF("serve.c", "Couldn't bind web server on address %s", listen_address)
     }
-    mg_set_protocol_http_websocket(nc);
 
-    for (;;) {
+    while (ok) {
         mg_mgr_poll(&mgr, 10);
     }
+    mg_mgr_free(&mgr);
+    LOG_INFO("serve.c", "Finished web event loop")
 }

tests/test_scan.py

@@ -0,0 +1,77 @@
+import unittest
+import subprocess
+import shutil
+import json
+import os
+
+TEST_FILES = "third-party/libscan/libscan-test-files/test_files"
+
+
+def copy_files(files):
+    base = os.path.basename(files)
+    new_path = os.path.join("/tmp/sist2_test/", base)
+
+    shutil.rmtree(new_path, ignore_errors=True)
+    shutil.copytree(files, new_path)
+    return new_path
+
+
+def sist2(*args):
+    print("./sist2 " + " ".join(args))
+
+    return subprocess.check_output(
+        args=["./sist2", *args],
+    )
+
+
+def sist2_index(files, *args):
+    path = copy_files(files)
+
+    shutil.rmtree("test_i", ignore_errors=True)
+    sist2("scan", path, "-o", "test_i", *args)
+    return iter(sist2_index_to_dict("test_i"))
+
+
+def sist2_incremental_index(files, func=None, *args):
+    path = copy_files(files)
+
+    if func:
+        func(path)
+
+    shutil.rmtree("test_i_inc", ignore_errors=True)
+    sist2("scan", path, "-o", "test_i_inc", "--incremental", "test_i", *args)
+    return iter(sist2_index_to_dict("test_i_inc"))
+
+
+def sist2_index_to_dict(index):
+    res = subprocess.check_output(
+        args=["./sist2", "index", "--print", index],
+    )
+
+    for line in res.splitlines():
+        if line:
+            yield json.loads(line)
+
+
+class ScanTest(unittest.TestCase):
+
+    def test_incremental1(self):
+        def remove_files(path):
+            os.remove(os.path.join(path, "msdoc/test1.doc"))
+            os.remove(os.path.join(path, "msdoc/test2.doc"))
+
+        def add_files(path):
+            with open(os.path.join(path, "newfile1"), "w"):
+                pass
+            with open(os.path.join(path, "newfile2"), "w"):
+                pass
+            with open(os.path.join(path, "newfile3"), "w"):
+                pass
+
+        file_count = sum(1 for _ in sist2_index(TEST_FILES))
+        self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, remove_files)), file_count - 2)
+        self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, add_files)), file_count + 3)
+
+
+if __name__ == "__main__":
+    unittest.main()