.drone.yml

@@ -7,36 +7,11 @@ platform:
   arch: amd64
 
 steps:
-  - name: submodules
-    image: alpine/git
-    commands:
-      - git submodule update --init --recursive
-  - name: docker
-    image: plugins/docker
-    depends_on:
-      - submodules
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: sist2app/sist2
-      context: ./
-      dockerfile: ./Dockerfile
-      auto_tag: true
-      auto_tag_suffix: x64-linux
-      when:
-        event:
-          - tag
   - name: build
-    image: sist2app/sist2-build
+    image: simon987/sist2-build
-    depends_on:
-    - submodules
     commands:
       - ./scripts/build.sh
   - name: scp files
-    depends_on:
-      - build
     image: appleboy/drone-scp
     settings:
       host:
@@ -47,11 +22,26 @@ steps:
         from_secret: SSH_USER
       key:
         from_secret: SSH_KEY
-      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
       source:
         - ./VERSION
         - ./sist2-x64-linux
         - ./sist2-x64-linux-debug
+  - name: docker
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: simon987/sist2
+      context: ./
+      dockerfile: ./Dockerfile
+      auto_tag: true
+      auto_tag_suffix: x64-linux
+      when:
+        event:
+          - tag
 
 ---
 kind: pipeline
@@ -62,36 +52,11 @@ platform:
   arch: arm64
 
 steps:
-  - name: submodules
-    image: alpine/git
-    commands:
-      - git submodule update --init --recursive
-  - name: docker
-    image: plugins/docker
-    depends_on:
-      - submodules
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: sist2app/sist2
-      context: ./
-      dockerfile: ./Dockerfile.arm64
-      auto_tag: true
-      auto_tag_suffix: arm64-linux
-      when:
-        event:
-          - tag
   - name: build
-    image: sist2app/sist2-build-arm64
+    image: simon987/sist2-build-arm64
-    depends_on:
-      - submodules
     commands:
       - ./scripts/build_arm64.sh
   - name: scp files
-    depends_on:
-      - build
     image: appleboy/drone-scp
     settings:
       host:
@@ -102,7 +67,22 @@ steps:
         from_secret: SSH_USER
       key:
         from_secret: SSH_KEY
-      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
       source:
         - ./sist2-arm64-linux
         - ./sist2-arm64-linux-debug
+  - name: docker
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: simon987/sist2
+      context: ./
+      dockerfile: ./Dockerfile.arm64
+      auto_tag: true
+      auto_tag_suffix: arm64-linux
+      when:
+        event:
+          - tag

CMakeLists.txt

@@ -62,9 +62,7 @@ add_executable(
         src/database/database_schema.c
         src/database/database_fts.c
         src/web/web_fts.c
-        src/database/database_embeddings.c
+        src/database/database_embeddings.c)
-        src/ignorelist.c
-        src/ignorelist.h)
 set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
 
 target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
@@ -78,7 +76,6 @@ find_package(CURL CONFIG REQUIRED)
 find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
 find_package(unofficial-sqlite3 CONFIG REQUIRED)
 find_package(OpenBLAS CONFIG REQUIRED)
-find_package(libgit2 CONFIG REQUIRED)
 
 
 target_include_directories(
@@ -92,7 +89,7 @@ target_include_directories(
 target_compile_options(
         sist2
         PRIVATE
-#        -fPIC
+        -fPIC
 )
 
 if (SIST_DEBUG)
@@ -150,9 +147,7 @@ add_dependencies(
 target_link_libraries(
         sist2
 
-#        m
         z
-        libgit2::libgit2package
         argparse
         unofficial::mongoose::mongoose
         CURL::libcurl

Dockerfile

@@ -1,4 +1,5 @@
-FROM sist2app/sist2-build as build
+FROM simon987/sist2-build as build
+MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 

Dockerfile.arm64

@@ -1,4 +1,5 @@
-FROM sist2app/sist2-build-arm64 as build
+FROM simon987/sist2-build-arm64 as build
+MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 

README.md

@@ -1,11 +1,9 @@
-![GitHub](https://img.shields.io/github/license/sist2app/sist2.svg)
+![GitHub](https://img.shields.io/github/license/simon987/sist2.svg)
-[![CodeFactor](https://www.codefactor.io/repository/github/sist2app/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/sist2app/sist2)
+[![CodeFactor](https://www.codefactor.io/repository/github/simon987/sist2/badge?s=05daa325188aac4eae32c786f3d9cf4e0593f822)](https://www.codefactor.io/repository/github/simon987/sist2)
 [![Development snapshots](https://ci.simon987.net/api/badges/simon987/sist2/status.svg)](https://files.simon987.net/.gate/sist2/simon987_sist2/)
 
 **Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
 
-**Community URL:** [Discord](https://discord.gg/2PEjDy3Rfs)
-
 # sist2
 
 sist2 (Simple incremental search tool)
@@ -38,32 +36,26 @@ sist2 (Simple incremental search tool)
 ### Using Docker Compose *(Windows/Linux/Mac)*
 
 ```yaml
+version: "3"
+
 services:
   elasticsearch:
     image: elasticsearch:7.17.9
     restart: unless-stopped
-    volumes:
-      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
-      - /data/sist2-es-data/:/usr/share/elasticsearch/data
     environment:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
-      - "PUID=1000"
-      - "PGID=1000"
   sist2-admin:
-    image: sist2app/sist2:x64-linux
+    image: simon987/sist2:3.3.3-x64-linux
     restart: unless-stopped
     volumes:
-      - /data/sist2-admin-data/:/sist2-admin/
+      - ./sist2-admin-data/:/sist2-admin/
-      - /<path to index>/:/host
+      - /:/host
     ports:
-      - 4090:4090
+      - 4090:4090 # sist2
-      # NOTE: Don't expose this port publicly!
+      - 8080:8080 # sist2-admin
-      - 8080:8080
     working_dir: /root/sist2-admin/
-    entrypoint: python3
+    entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
-    command:
-      - /root/sist2-admin/sist2_admin/app.py
 ```
 
 Navigate to http://localhost:8080/ to configure sist2-admin.
@@ -79,7 +71,7 @@ Navigate to http://localhost:8080/ to configure sist2-admin.
             ```
     * **SQLite**: No installation required
 
-2. Download the [latest sist2 release](https://github.com/sist2app/sist2/releases).
+2. Download the [latest sist2 release](https://github.com/simon987/sist2/releases).
    Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x`.
 3. See [usage guide](docs/USAGE.md) for command line usage.
 
@@ -88,30 +80,28 @@ Example usage:
 1. Scan a directory: `sist2 scan ~/Documents --output ./documents.sist2`
 2. Prepare search index:
     * **Elasticsearch**: `sist2 index --es-url http://localhost:9200 ./documents.sist2`
-    * **SQLite**: `sist2 sqlite-index --search-index ./search.sist2 ./documents.sist2`
+    * **SQLite**: `sist2 index --search-index ./search.sist2 ./documents.sist2`
-3. Start web interface: 
+3. Start web interface: `sist2 web ./documents.sist2`
-   * **Elasticsearch**: `sist2 web ./documents.sist2`
-   * **SQLite**: `sist2 web --search-index ./search.sist2 ./documents.sist2`
 
 ## Format support
 
 | File type                                                                 | Library                                                                      | Content  | Thumbnail   | Metadata                                                                                                                               |
 |:--------------------------------------------------------------------------|:-----------------------------------------------------------------------------|:---------|:------------|:---------------------------------------------------------------------------------------------------------------------------------------|
 | pdf,xps,fb2,epub                                                          | MuPDF                                                                        | text+ocr | yes         | author, title                                                                                                                          |
-| cbz,cbr                                                                   | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | -        | yes         | -                                                                                                                                      |
+| cbz,cbr                                                                   | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | -        | yes         | -                                                                                                                                      |
 | `audio/*`                                                                 | ffmpeg                                                                       | -        | yes         | ID3 tags                                                                                                                               |
 | `video/*`                                                                 | ffmpeg                                                                       | -        | yes         | title, comment, artist                                                                                                                 |
-| `image/*`                                                                 | ffmpeg                                                                       | ocr      | yes         | [Common EXIF tags](https://github.com/sist2app/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
+| `image/*`                                                                 | ffmpeg                                                                       | ocr      | yes         | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
 | raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf | LibRaw                                                                       | no       | yes         | Common EXIF tags, GPS tags                                                                                                             |
 | ttf,ttc,cff,woff,fnt,otf                                                  | Freetype2                                                                    | -        | yes, `bmp`  | Name & style                                                                                                                           |
-| `text/plain`                                                              | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| `text/plain`                                                              | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
-| html, xml                                                                 | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
+| html, xml                                                                 | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
 | tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
-| docx, xlsx, pptx                                                          | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
+| docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
 | doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
 | mobi, azw, azw3                                                           | libmobi                                                                      | yes      | yes         | author, title                                                                                                                          |
 | wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
-| json, jsonl, ndjson                                                       | [libscan](https://github.com/sist2app/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
+| json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
 
 \* *See [Archive files](#archive-files)*
 
@@ -135,7 +125,7 @@ You can enable OCR support for ebook (pdf,xps,fb2,epub) or image file types with
 Download the language data files with your package manager (`apt install tesseract-ocr-eng`) or
 directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 
-The `sist2app/sist2` image comes with common languages
+The `simon987/sist2` image comes with common languages
 (hin, jpn, eng, fra, rus, spa, chi_sim, deu, pol) pre-installed.
 
 You can use the `+` separator to specify multiple languages. The language
@@ -163,6 +153,7 @@ indices, but it uses much less memory and is easier to set up.
 | Query syntax                                 |      [fts5](https://www.sqlite.org/fts5.html)       | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
 | Fuzzy search                                 |                                                     |                                                                   ✓                                                                   |
 | Media Types tree real-time updating          |                                                     |                                                                   ✓                                                                   |
+| Search in file `path`                        | [WIP](https://github.com/simon987/sist2/issues/402) |                                                                   ✓                                                                   |
 | Manual tagging                               |                          ✓                          |                                                                   ✓                                                                   |
 | User scripts                                 |                          ✓                          |                                                                   ✓                                                                   |
 | Media Type breakdown for search results      |                                                     |                                                                   ✓                                                                   |
@@ -175,13 +166,13 @@ sist2 v3.0.4+ supports named-entity recognition (NER). Simply add a supported re
 to enable it.
 
 The text processing is done in your browser, no data is sent to any third-party services.
-See [sist2app/sist2-ner-models](https://github.com/sist2app/sist2-ner-models) for more details.
+See [simon987/sist2-ner-models](https://github.com/simon987/sist2-ner-models) for more details.
 
 #### List of available repositories:
 
 | URL                                                                                                     | Maintainer                              | Purpose |
 |---------------------------------------------------------------------------------------------------------|-----------------------------------------|---------|
-| [sist2app/sist2-ner-models](https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json) | [sist2app](https://github.com/sist2app) | General |
+| [simon987/sist2-ner-models](https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json) | [simon987](https://github.com/simon987) | General |
 
 <details>
   <summary>Screenshot</summary>
@@ -197,7 +188,7 @@ You can compile **sist2** by yourself if you don't want to use the pre-compiled
 ### Using docker
 
 ```bash
-git clone --recursive https://github.com/sist2app/sist2/
+git clone --recursive https://github.com/simon987/sist2/
 cd sist2
 docker build . -t my-sist2-image
 # Copy sist2 executable from docker image
@@ -212,18 +203,18 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
    apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
    ```
 
-2. Install vcpkg using my fork: https://github.com/sist2app/vcpkg
+2. Install vcpkg using my fork: https://github.com/simon987/vcpkg
 3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5,json1] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib] libgit2[core,pcre]
+    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib]
     ```
 
 4. Build
     ```bash
-    git clone --recursive https://github.com/sist2app/sist2/
+    git clone --recursive https://github.com/simon987/sist2/
     (cd sist2-vue; npm install; npm run build)
     (cd sist2-admin/frontend; npm install; npm run build)
-    cmake -DSIST_DEBUG=off -G "Unix Makefiles" -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
+    cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

docker-compose.yml

@@ -4,21 +4,16 @@ services:
   elasticsearch:
     image: elasticsearch:7.17.9
     container_name: sist2-es
-    volumes:
-      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
-      - /data/sist2-es-data/:/usr/share/elasticsearch/data
     environment:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
-      - "PUID=1000"
-      - "PGID=1000"
   sist2-admin:
     build:
       context: .
     container_name: sist2-admin
     volumes:
-      - /data/sist2-admin-data/:/sist2-admin/
+      - /mnt/array/sist2-admin-data/:/sist2-admin/
-      - /<path to index>/:/host
+      - /:/host
     ports:
       - 4090:4090
       # NOTE: Don't export this port publicly!

docs/USAGE.md

@@ -108,27 +108,6 @@ sist scan ~/Documents -o ./documents.sist2 --incremental
 sist scan ~/Documents -o ./documents.sist2 --incremental
 ```
 
-### Excluding files
-
-You can use the `--exclude` option to specify exclude patterns. For more complex setups, you can create a 
-`.sist2ignore` file at the root of the scan path (For example, `~/Documents/.sist2ignore` for the example above). 
-
-The syntax for sist2ignore is the same as .gitignore for Git (reference [here](https://git-scm.com/docs/gitignore)).
-
-Example:
-
-**.sist2ignore**
-```gitignore
-# Ignore all PDF files
-*.pdf
-
-# But don't ignore them for the /important_files/ directory
-!/important_files/*.pdf
-
-# Ignore all files in _staging/ directories
-_staging/
-```
-
 ### Index documents to Elasticsearch search backend
 
 ```bash
@@ -193,10 +172,6 @@ Using a version >=7.14.0 is recommended to enable the following features:
 
 - Bug fix for large documents (See #198)
 
-Using a version >=8.0.0 is recommended to enable the following features:
-
-- Approximate KNN search for Embeddings search (faster queries).
-
 When using a legacy version of ES, a notice will be displayed next to the sist2 version in the web UI.
 If you don't care about the features above, you can ignore it or disable it in the configuration page.
 
@@ -210,7 +185,7 @@ Since v3.2.0, User scripts can be used to generate _embeddings_ (vector of float
 
 In theory, embeddings can be created for any type of documents (image, text, audio etc.).
 
-For example, the [clip](https://github.com/sist2app/sist2-script-clip) User Script, generates 512-d embeddings of images
+For example, the [clip](https://github.com/simon987/sist2-script-clip) User Script, generates 512-d embeddings of images
 (videos are also supported using the thumbnails generated by sist2). When the user enters a query in the "Embeddings Search"
 textbox, the query's embedding is generated in their browser, leveraging the ONNX web runtime.
 

scripts/build.sh

@@ -2,6 +2,8 @@
 
 VCPKG_ROOT="/vcpkg"
 
+git submodule update --init --recursive
+
 (
   cd sist2-vue/
   npm install

scripts/magic_static.py

@@ -1,16 +1,8 @@
 
-MAGIC_PATHS = [
+try:
-    "/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
+    with open("/usr/lib/file/magic.mgc", "rb") as f:
-    "/work/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
+        data = f.read()
-    "/usr/lib/file/magic.mgc"
+except:
-]
+    data = bytes([])
-
-for path in MAGIC_PATHS:
-    try:
-        with open(path, "rb") as f:
-            data = f.read()
-        break
-    except:
-        continue
 
 print("char magic_database_buffer[%d] = {%s};" % (len(data), ",".join(str(int(b)) for b in data)))

scripts/mime.csv

@@ -450,4 +450,3 @@ image/x-sony-arw, arw
 image/x-sony-sr2, sr2
 image/x-sony-srf, srf
 image/x-epson-erf, erf
-image/jp2, jp2

sist2-admin/frontend/package.json

@@ -8,7 +8,7 @@
     "watch": "vue-cli-service build --watch"
   },
   "dependencies": {
-    "axios": "^1.6.0",
+    "axios": "^0.27.2",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
     "moment": "^2.29.3",

sist2-admin/frontend/src/App.vue

@@ -4,7 +4,7 @@
         <b-container class="pt-4">
             <b-alert show dismissible variant="info">
                 This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
-                to the <a href="https://github.com/sist2app/sist2/issues/new/choose" target="_blank">issue tracker on
+                to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
                 Github</a>. Thank you!
             </b-alert>
             <router-view v-if="$store.state.sist2AdminInfo"/>

sist2-admin/frontend/src/Sist2AdminApi.js

@@ -89,12 +89,9 @@ class Sist2AdminApi {
 
     /**
      * @param {string} name
-     * @param {bool} full
      */
-    runJob(name, full) {
+    runJob(name) {
-        return axios.get(`${this.baseUrl}/api/job/${name}/run`, {
+        return axios.get(`${this.baseUrl}/api/job/${name}/run`);
-            params: {full}
-        });
     }
 
     /**

sist2-admin/frontend/src/components/ScanOptions.vue

@@ -95,7 +95,6 @@ export default {
   methods: {
     onOcrLangChange() {
       this.options.ocr_lang = this.selectedOcrLangs.join("+");
-      this.update();
     },
     update() {
       this.disableOcrLang = this.options.ocr_images === false && this.options.ocr_ebooks === false;

sist2-admin/frontend/src/components/WebOptions.vue

@@ -1,70 +1,59 @@
 <template>
-  <div>
+    <div>
-    <h4>{{ $t("webOptions.title") }}</h4>
+        <h4>{{ $t("webOptions.title") }}</h4>
-    <b-card>
+        <b-card>
-      <label>{{ $t("webOptions.lang") }}</label>
+            <label>{{ $t("webOptions.lang") }}</label>
-      <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
+            <b-form-select v-model="options.lang" :options="['en', 'fr', 'zh-CN', 'pl', 'de']"
-                     @change="update()"></b-form-select>
+                           @change="update()"></b-form-select>
 
-      <label>{{ $t("webOptions.bind") }}</label>
+            <label>{{ $t("webOptions.bind") }}</label>
-      <b-form-input v-model="options.bind" @change="update()"></b-form-input>
+            <b-form-input v-model="options.bind" @change="update()"></b-form-input>
 
-      <label>{{ $t("webOptions.tagline") }}</label>
+            <label>{{ $t("webOptions.tagline") }}</label>
-      <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
+            <b-form-textarea v-model="options.tagline" @change="update()"></b-form-textarea>
 
-      <label>{{ $t("webOptions.auth") }}</label>
+            <label>{{ $t("webOptions.auth") }}</label>
-      <b-form-input v-model="options.auth" @change="update()"></b-form-input>
+            <b-form-input v-model="options.auth" @change="update()"></b-form-input>
 
-      <label>{{ $t("webOptions.tagAuth") }}</label>
+            <label>{{ $t("webOptions.tagAuth") }}</label>
-      <b-form-input v-model="options.tag_auth" @change="update()" :disabled="Boolean(options.auth)"></b-form-input>
+            <b-form-input v-model="options.tag_auth" @change="update()"></b-form-input>
+        </b-card>
 
-      <b-form-checkbox v-model="options.verbose" @change="update()">
+        <br>
-        {{$t("webOptions.verbose")}}
+        <h4>Auth0 options</h4>
-      </b-form-checkbox>
+        <b-card>
-    </b-card>
+            <label>{{ $t("webOptions.auth0Audience") }}</label>
+            <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
-    <br>
+            <label>{{ $t("webOptions.auth0Domain") }}</label>
-    <h4>Auth0 options</h4>
+            <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
-    <b-card>
-      <label>{{ $t("webOptions.auth0Audience") }}</label>
-      <b-form-input v-model="options.auth0_audience" @change="update()"></b-form-input>
 
-      <label>{{ $t("webOptions.auth0Domain") }}</label>
+            <label>{{ $t("webOptions.auth0ClientId") }}</label>
-      <b-form-input v-model="options.auth0_domain" @change="update()"></b-form-input>
+            <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
 
-      <label>{{ $t("webOptions.auth0ClientId") }}</label>
+            <label>{{ $t("webOptions.auth0PublicKey") }}</label>
-      <b-form-input v-model="options.auth0_client_id" @change="update()"></b-form-input>
+            <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
-
+        </b-card>
-      <label>{{ $t("webOptions.auth0PublicKey") }}</label>
+    </div>
-      <b-textarea rows="10" v-model="options.auth0_public_key" @change="update()"></b-textarea>
-    </b-card>
-  </div>
 </template>
 
 <script>
 
 export default {
-  name: "WebOptions",
+    name: "WebOptions",
-  props: ["options", "frontendName"],
+    props: ["options", "frontendName"],
-  data() {
+    data() {
-    return {
+        return {
-      showEsTestAlert: false,
+            showEsTestAlert: false,
-      esTestOk: false,
+            esTestOk: false,
-      esTestMessage: ""
+            esTestMessage: "",
-    }
+        }
-  },
-  methods: {
-    update() {
-
-      console.log(this.options)
-      if (this.options.auth && this.options.tag_auth) {
-        // If both are set, remove tagAuth
-        this.options.tag_auth = "";
-      }
-
-      this.$emit("change", this.options);
     },
-  }
+    methods: {
+        update() {
+            this.$emit("change", this.options);
+        },
+    }
 }
 </script>
 

sist2-admin/frontend/src/i18n/messages.js

@@ -8,7 +8,6 @@ export default {
         view: "View",
         delete: "Delete",
         runNow: "Index now",
-        runNowFull: "Full re-index",
         create: "Create",
         cancel: "Cancel",
         test: "Test",
@@ -65,9 +64,6 @@ export default {
         gitRepository: "Git repository URL",
         extraArgs: "Extra command line arguments",
 
-        couldNotStartFrontend: "Could not start frontend",
-        couldNotStartFrontendBody: "Unable to start the frontend, check server logs for more details.",
-
         selectJobs: "Available jobs",
         selectJob: "Select a job",
         webOptions: {
@@ -81,7 +77,6 @@ export default {
             auth0Domain: "Auth0 domain",
             auth0ClientId: "Auth0 client ID",
             auth0PublicKey: "Auth0 public key",
-            verbose: "Verbose logs"
         },
         backendOptions: {
             title: "Search backend options",
@@ -93,9 +88,7 @@ export default {
             threads: "Number of threads",
             batchSize: "Index batch size",
             script: "User script",
-            searchIndex: "Search index file location",
+            searchIndex: "Search index file location"
-            esMappings: "Elasticsearch mappings file override",
-            esSettings: "Elasticsearch settings file override"
         },
         scanOptions: {
             title: "Scanning options",

sist2-admin/frontend/src/views/Frontend.vue

@@ -1,63 +1,63 @@
 <template>
-  <b-card>
+    <b-card>
-    <b-card-title>
+        <b-card-title>
-      {{ name }}
+            {{ name }}
-      <small style="vertical-align: top">
+            <small style="vertical-align: top">
-        <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
+                <b-badge v-if="!loading && frontend.running" variant="success">{{ $t("online") }}</b-badge>
-        <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
+                <b-badge v-else-if="!loading" variant="secondary">{{ $t("offline") }}</b-badge>
-      </small>
+            </small>
-    </b-card-title>
+        </b-card-title>
 
-    <!-- Action buttons-->
+        <!-- Action buttons-->
-    <div class="mb-3" v-if="!loading">
+        <div class="mb-3" v-if="!loading">
-      <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
+            <b-button class="mr-1" :disabled="frontend.running || !valid" variant="success" @click="start()">{{
-          $t("start")
+                $t("start")
-        }}
+                }}
-      </b-button>
+            </b-button>
-      <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
+            <b-button class="mr-1" :disabled="!frontend.running" variant="danger" @click="stop()">{{
-          $t("stop")
+                $t("stop")
-        }}
+                }}
-      </b-button>
+            </b-button>
-      <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
+            <b-button class="mr-1" :disabled="!frontend.running" variant="primary" :href="frontendUrl" target="_blank">
-        {{ $t("go") }}
+                {{ $t("go") }}
-      </b-button>
+            </b-button>
-      <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
+            <b-button variant="danger" @click="deleteFrontend()">{{ $t("delete") }}</b-button>
-    </div>
+        </div>
 
-    <b-progress v-if="loading" striped animated value="100"></b-progress>
+        <b-progress v-if="loading" striped animated value="100"></b-progress>
-    <b-card-body v-else>
+        <b-card-body v-else>
 
-      <h4>{{ $t("backendOptions.title") }}</h4>
+            <h4>{{ $t("backendOptions.title") }}</h4>
-      <b-card>
+            <b-card>
-        <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
+                <b-alert v-if="!valid" variant="warning" show>{{ $t("frontendOptions.noJobSelectedWarning") }}</b-alert>
 
-        <SearchBackendSelect :value="frontend.web_options.search_backend"
+                <SearchBackendSelect :value="frontend.web_options.search_backend"
-                             @change="onBackendSelect($event)"></SearchBackendSelect>
+                                     @change="onBackendSelect($event)"></SearchBackendSelect>
 
-        <br>
+                <br>
-        <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
+                <JobCheckboxGroup :frontend="frontend" @input="update()"></JobCheckboxGroup>
-      </b-card>
+            </b-card>
 
-      <br/>
+            <br/>
 
-      <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
+            <WebOptions :options="frontend.web_options" :frontend-name="$route.params.name"
-                  @change="update()"></WebOptions>
+                        @change="update()"></WebOptions>
-      <br/>
+            <br/>
 
-      <h4>{{ $t("frontendOptions.title") }}</h4>
+            <h4>{{ $t("frontendOptions.title") }}</h4>
-      <b-card>
+            <b-card>
-        <b-form-checkbox v-model="frontend.auto_start" @change="update()">
+                <b-form-checkbox v-model="frontend.auto_start" @change="update()">
-          {{ $t("autoStart") }}
+                    {{ $t("autoStart") }}
-        </b-form-checkbox>
+                </b-form-checkbox>
 
-        <label>{{ $t("extraQueryArgs") }}</label>
+                <label>{{ $t("extraQueryArgs") }}</label>
-        <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
+                <b-form-input v-model="frontend.extra_query_args" @change="update()"></b-form-input>
 
-        <label>{{ $t("customUrl") }}</label>
+                <label>{{ $t("customUrl") }}</label>
-        <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
+                <b-form-input v-model="frontend.custom_url" @change="update()" placeholder="http://"></b-form-input>
-      </b-card>
+            </b-card>
-    </b-card-body>
+        </b-card-body>
-  </b-card>
+    </b-card>
 </template>
 
 <script>
@@ -68,78 +68,71 @@ import WebOptions from "@/components/WebOptions";
 import SearchBackendSelect from "@/components/SearchBackendSelect.vue";
 
 export default {
-  name: 'Frontend',
+    name: 'Frontend',
-  components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
+    components: {SearchBackendSelect, JobCheckboxGroup, WebOptions},
-  data() {
+    data() {
-    return {
+        return {
-      loading: true,
+            loading: true,
-      frontend: null,
+            frontend: null,
-    }
+        }
-  },
-  computed: {
-    valid() {
-      return !this.loading && this.frontend.jobs.length > 0;
     },
-    frontendUrl() {
+    computed: {
-      if (this.frontend.custom_url) {
+        valid() {
-        return this.frontend.custom_url + this.args;
+            return !this.loading && this.frontend.jobs.length > 0;
-      }
+        },
+        frontendUrl() {
+            if (this.frontend.custom_url) {
+                return this.frontend.custom_url + this.args;
+            }
 
-      if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
+            if (this.frontend.web_options.bind.startsWith("0.0.0.0")) {
-        return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
+                return window.location.protocol + "//" + window.location.hostname + ":" + this.port + this.args;
-      }
+            }
 
-      return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+            return window.location.protocol + "//" + this.frontend.web_options.bind + this.args;
+        },
+        name() {
+            return this.$route.params.name;
+        },
+        port() {
+            return this.frontend.web_options.bind.split(":")[1]
+        },
+        args() {
+            const args = this.frontend.extra_query_args;
+            if (args !== "") {
+                return "#" + (args.startsWith("?") ? (args) : ("?" + args));
+            }
+            return "";
+        }
     },
-    name() {
+    mounted() {
-      return this.$route.params.name;
+        Sist2AdminApi.getFrontend(this.name).then(resp => {
-    },
+            this.frontend = resp.data;
-    port() {
+            this.loading = false;
-      return this.frontend.web_options.bind.split(":")[1]
-    },
-    args() {
-      const args = this.frontend.extra_query_args;
-      if (args !== "") {
-        return "#" + (args.startsWith("?") ? (args) : ("?" + args));
-      }
-      return "";
-    }
-  },
-  mounted() {
-    Sist2AdminApi.getFrontend(this.name).then(resp => {
-      this.frontend = resp.data;
-      this.loading = false;
-    });
-  },
-  methods: {
-    start() {
-      Sist2AdminApi.startFrontend(this.name).then(() => {
-        this.frontend.running = true;
-      }).catch(() => {
-        this.$bvToast.toast(this.$t("couldNotStartFrontendBody"), {
-          title: this.$t("couldNotStartFrontend"),
-          variant: "danger",
-          toaster: "b-toaster-bottom-right"
         });
-      });
     },
-    stop() {
+    methods: {
-      this.frontend.running = false;
+        start() {
-      Sist2AdminApi.stopFrontend(this.name)
+            this.frontend.running = true;
-    },
+            Sist2AdminApi.startFrontend(this.name)
-    deleteFrontend() {
+        },
-      Sist2AdminApi.deleteFrontend(this.name).then(() => {
+        stop() {
-        this.$router.push("/");
+            this.frontend.running = false;
-      });
+            Sist2AdminApi.stopFrontend(this.name)
-    },
+        },
-    update() {
+        deleteFrontend() {
-      Sist2AdminApi.updateFrontend(this.name, this.frontend);
+            Sist2AdminApi.deleteFrontend(this.name).then(() => {
-    },
+                this.$router.push("/");
-    onBackendSelect(backend) {
+            });
-      this.frontend.web_options.search_backend = backend;
+        },
-      this.frontend.jobs = [];
+        update() {
-      this.update();
+            Sist2AdminApi.updateFrontend(this.name, this.frontend);
+        },
+        onBackendSelect(backend) {
+            this.frontend.web_options.search_backend = backend;
+            this.frontend.jobs = [];
+            this.update();
+        }
     }
-  }
 }
 </script>

sist2-admin/frontend/src/views/Job.vue

@@ -6,19 +6,7 @@
         </b-card-title>
 
         <div class="mb-3">
-
+            <b-button class="mr-1" variant="primary" @click="runJob()" :disabled="!valid">{{ $t("runNow") }}</b-button>
-            <b-dropdown
-                    split
-                    split-variant="primary"
-                    variant="primary"
-                    :text="$t('runNow')"
-                    class="mr-1"
-                    :disabled="!valid"
-                    @click="runJob()"
-            >
-                <b-dropdown-item href="#" @click="runJob(true)">{{ $t("runNowFull") }}</b-dropdown-item>
-            </b-dropdown>
-
             <b-button variant="danger" @click="deleteJob()">{{ $t("delete") }}</b-button>
         </div>
 
@@ -81,7 +69,6 @@ export default {
         return {
             loading: true,
             job: null,
-            console: console
         }
     },
     methods: {
@@ -91,8 +78,8 @@ export default {
         update() {
             Sist2AdminApi.updateJob(this.getName(), this.job);
         },
-        runJob(full = false) {
+        runJob() {
-            Sist2AdminApi.runJob(this.getName(), full).then(() => {
+            Sist2AdminApi.runJob(this.getName()).then(() => {
                 this.$bvToast.toast(this.$t("runJobConfirmation"), {
                     title: this.$t("runJobConfirmationTitle"),
                     variant: "success",

sist2-admin/frontend/src/views/SearchBackend.vue

@@ -44,12 +44,6 @@
 
                 <label>{{ $t("backendOptions.batchSize") }}</label>
                 <b-form-input v-model="backend.batch_size" type="number" min="1" @change="update()"></b-form-input>
-
-                <label>{{ $t("backendOptions.esMappings") }}</label>
-                <b-form-textarea v-model="backend.es_mappings" rows="4" @change="update()"></b-form-textarea>
-
-                <label>{{ $t("backendOptions.esSettings") }}</label>
-                <b-form-textarea v-model="backend.es_settings" rows="4" @change="update()"></b-form-textarea>
             </template>
             <template v-else>
                 <label>{{ $t("backendOptions.searchIndex") }}</label>

sist2-admin/frontend/src/views/Tail.vue

@@ -170,6 +170,6 @@ span.ADMIN {
     margin: 3px;
     white-space: pre;
     color: #000;
-    overflow-y: hidden;
+    overflow: hidden;
 }
 </style>

sist2-admin/requirements.txt

@@ -4,4 +4,4 @@ uvicorn
 websockets
 pycron
 GitPython
-git+https://github.com/sist2app/sist2-python.git@2.1
+git+https://github.com/simon987/sist2-python.git

sist2-admin/sist2_admin/app.py

@@ -2,7 +2,6 @@ import asyncio
 import os
 import signal
 from datetime import datetime
-from time import sleep
 from urllib.parse import urlparse
 
 import requests
@@ -26,7 +25,6 @@ from state import migrate_v1_to_v2, RUNNING_FRONTENDS, TESSERACT_LANGS, DB_SCHEM
     get_log_files_to_remove, delete_log_file, create_default_search_backends
 from web import Sist2Frontend
 from script import UserScript, SCRIPT_TEMPLATES
-from util import tail_sync, pid_is_running
 
 sist2 = Sist2(SIST2_BINARY, DATA_FOLDER)
 db = PersistentState(dbfile=os.path.join(DATA_FOLDER, "state.db"))
@@ -171,14 +169,11 @@ def _run_job(job: Sist2Job):
 
 
 @app.get("/api/job/{name:str}/run")
-async def run_job(name: str, full: bool = False):
+async def run_job(name: str):
-    job: Sist2Job = db["jobs"][name]
+    job = db["jobs"][name]
     if not job:
         raise HTTPException(status_code=404)
 
-    if full:
-        job.do_full_scan = True
-
     _run_job(job)
 
     return "ok"
@@ -326,18 +321,7 @@ def start_frontend_(frontend: Sist2Frontend):
     logger.debug(f"Fetched search backend options for {backend_name}")
 
     pid = sist2.web(frontend.web_options, search_backend, frontend.name)
-
-    sleep(0.2)
-    if not pid_is_running(pid):
-        frontend_log = frontend.get_log_path(LOG_FOLDER)
-        logger.error(f"Frontend exited too quickly, check {frontend_log} for more details:")
-        for line in tail_sync(frontend.get_log_path(LOG_FOLDER), 3):
-            logger.error(line.strip())
-
-        return False
-
     RUNNING_FRONTENDS[frontend.name] = pid
-    return True
 
 
 @app.post("/api/frontend/{name:str}/start")
@@ -346,12 +330,7 @@ async def start_frontend(name: str):
     if not frontend:
         raise HTTPException(status_code=404)
 
-    ok = start_frontend_(frontend)
+    start_frontend_(frontend)
-
-    if not ok:
-        raise HTTPException(status_code=500)
-
-    return "ok"
 
 
 @app.post("/api/frontend/{name:str}/stop")

sist2-admin/sist2_admin/jobs.py

@@ -204,7 +204,7 @@ class Sist2IndexTask(Sist2Task):
         self.job.previous_index_path = self.job.index_path
         db["jobs"][self.job.name] = self.job
 
-        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}, {ok=}"}))
+        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
 
         logger.info(f"Completed {self.display_name} ({return_code=})")
 
@@ -220,7 +220,7 @@ class Sist2IndexTask(Sist2Task):
             except ProcessLookupError:
                 pass
             try:
-                os.waitpid(pid, 0)
+                os.wait()
             except ChildProcessError:
                 pass
 

sist2-admin/sist2_admin/script.py

@@ -96,7 +96,7 @@ SCRIPT_TEMPLATES = {
     "CLIP - Generate embeddings to predict the most relevant image based on the text prompt": lambda name: UserScript(
         name=name,
         type=ScriptType.GIT,
-        git_repository="https://github.com/sist2app/sist2-script-clip",
+        git_repository="https://github.com/simon987/sist2-script-clip",
         extra_args="--num-tags=1 --tags-file=general.txt --color=#dcd7ff"
     ),
     "Whisper - Speech to text with OpenAI Whisper": lambda name: UserScript(

sist2-admin/sist2_admin/sist2.py

@@ -2,15 +2,14 @@ import datetime
 import json
 import logging
 import os.path
-import sys
 from datetime import datetime
 from enum import Enum
 from io import TextIOWrapper
-from logging import FileHandler, StreamHandler
+from logging import FileHandler
 from subprocess import Popen, PIPE
 from tempfile import NamedTemporaryFile
 from threading import Thread
-from typing import List, Optional
+from typing import List
 
 from pydantic import BaseModel
 
@@ -40,8 +39,6 @@ class Sist2SearchBackend(BaseModel):
 
     es_url: str = "http://elasticsearch:9200"
     es_insecure_ssl: bool = False
-    es_mappings: Optional[str] = None
-    es_settings: Optional[str] = None
     es_index: str = "sist2"
     threads: int = 1
     batch_size: int = 70
@@ -59,8 +56,6 @@ class IndexOptions(BaseModel):
     path: str = None
     incremental_index: bool = True
     search_backend: str = None
-    es_mappings_file: Optional[str] = None
-    es_settings_file: Optional[str] = None
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -79,12 +74,6 @@ class IndexOptions(BaseModel):
 
             if search_backend.es_insecure_ssl:
                 args.append(f"--es-insecure-ssl")
-
-            if self.es_mappings_file:
-                args.append(f"--mappings-file={self.es_mappings_file}")
-            if self.es_settings_file:
-                args.append(f"--settings-file={self.es_settings_file}")
-
             if self.incremental_index:
                 args.append(f"--incremental-index")
 
@@ -211,7 +200,6 @@ class WebOptions(BaseModel):
     auth0_client_id: str = None
     auth0_public_key: str = None
     auth0_public_key_file: str = None
-    verbose: bool = False
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -243,8 +231,6 @@ class WebOptions(BaseModel):
             args.append(f"--tag-auth={self.tag_auth}")
         if self.dev:
             args.append(f"--dev")
-        if self.verbose:
-            args.append(f"--very-verbose")
 
         args.extend(self.indices)
 
@@ -259,20 +245,6 @@ class Sist2:
 
     def index(self, options: IndexOptions, search_backend: Sist2SearchBackend, logs_cb, set_pid_cb):
 
-        if search_backend.es_mappings:
-            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
-                f.write(search_backend.es_mappings)
-            options.es_mappings_file = f.name
-        else:
-            options.es_mappings_file = None
-
-        if search_backend.es_settings:
-            with NamedTemporaryFile("w", prefix="sist2-admin", suffix=".txt", delete=False) as f:
-                f.write(search_backend.es_settings)
-            options.es_settings_file = f.name
-        else:
-            options.es_settings_file = None
-
         args = [
             self.bin_path,
             *options.args(search_backend),
@@ -285,7 +257,7 @@ class Sist2:
 
         set_pid_cb(proc.pid)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
         t_stderr.start()
 
         self._consume_logs_stdout(logs_cb, proc)
@@ -312,7 +284,7 @@ class Sist2:
 
         set_pid_cb(proc.pid)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, None, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
         t_stderr.start()
 
         self._consume_logs_stdout(logs_cb, proc)
@@ -322,7 +294,7 @@ class Sist2:
         return proc.returncode
 
     @staticmethod
-    def _consume_logs_stderr(logs_cb, exit_cb, proc):
+    def _consume_logs_stderr(logs_cb, proc):
         pipe_wrapper = TextIOWrapper(proc.stderr, encoding="utf8", errors="ignore")
         try:
             for line in pipe_wrapper:
@@ -330,9 +302,7 @@ class Sist2:
                     continue
                 logs_cb({"stderr": line})
         finally:
-            return_code = proc.wait()
+            proc.wait()
-            if exit_cb:
-                exit_cb(return_code)
             pipe_wrapper.close()
 
     @staticmethod
@@ -366,19 +336,15 @@ class Sist2:
 
         web_logger = logging.Logger(name=f"sist2-frontend-{name}")
         web_logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"frontend-{name}.log")))
-        web_logger.addHandler(StreamHandler())
 
         def logs_cb(message):
             web_logger.info(json.dumps(message))
 
-        def exit_cb(return_code):
-            logger.info(f"Web frontend exited with return code {return_code}")
-
         logger.info(f"Starting frontend {' '.join(args)}")
 
         proc = Popen(args, stdout=PIPE, stderr=PIPE)
 
-        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, exit_cb, proc))
+        t_stderr = Thread(target=self._consume_logs_stderr, args=(logs_cb, proc))
         t_stderr.start()
 
         t_stdout = Thread(target=self._consume_logs_stdout, args=(logs_cb, proc))

sist2-admin/sist2_admin/util.py

@@ -1,41 +0,0 @@
-from glob import glob
-import os
-from config import DATA_FOLDER
-
-
-def get_old_index_files(name):
-    files = glob(os.path.join(DATA_FOLDER, f"scan-{name.replace('/', '_')}-*.sist2"))
-    files = list(sorted(files, key=lambda f: os.stat(f).st_mtime))
-    files = files[-1:]
-
-    return files
-
-
-def tail_sync(filename, lines=1, _buffer=4098):
-    with open(filename) as f:
-        lines_found = []
-
-        block_counter = -1
-
-        while len(lines_found) < lines:
-            try:
-                f.seek(block_counter * _buffer, os.SEEK_END)
-            except IOError:
-                f.seek(0)
-                lines_found = f.readlines()
-                break
-
-            lines_found = f.readlines()
-
-            block_counter -= 1
-
-        return lines_found[-lines:]
-
-
-def pid_is_running(pid):
-    try:
-        os.kill(pid, 0)
-    except OSError:
-        return False
-
-    return True

sist2-vue/package.json

@@ -9,7 +9,7 @@
   "dependencies": {
     "@auth0/auth0-spa-js": "^2.0.2",
     "@egjs/vue-infinitegrid": "3.3.0",
-    "axios": "^1.6.0",
+    "axios": "^0.25.0",
     "bootstrap-vue": "^2.21.2",
     "core-js": "^3.6.5",
     "d3": "^5.6.1",
@@ -17,7 +17,7 @@
     "dom-to-image": "^2.6.0",
     "fslightbox-vue": "fslightbox-vue.tgz",
     "nouislider": "^15.2.0",
-    "onnxruntime-web": "1.15.1",
+    "onnxruntime-web": "^1.15.1",
     "underscore": "^1.13.1",
     "vue": "^2.6.12",
     "vue-color": "^2.8.1",

sist2-vue/src/Sist2Api.js

@@ -33,6 +33,18 @@ class Sist2Api {
 
     getSist2Info() {
         return axios.get(`${this.baseUrl}i`).then(resp => {
+            const indices = resp.data.indices;
+
+            resp.data.indices = indices.map(idx => {
+                return {
+                    id: idx.id,
+                    name: idx.name,
+                    timestamp: idx.timestamp,
+                    version: idx.version,
+                    models: idx.models,
+                };
+            });
+
             this.sist2Info = resp.data;
 
             return resp.data;
@@ -69,8 +81,7 @@ class Sist2Api {
                     hit._props.isImage = true;
                 }
                 if ("width" in hit._source && !hit._props.isSubDocument && hit._source.videoc !== "tiff"
-                    && hit._source.videoc !== "raw" && hit._source.videoc !== "ppm"
+                    && hit._source.videoc !== "raw" && hit._source.videoc !== "ppm") {
-                    && hit._source.mime !== "image/jp2") {
                     hit._props.isPlayableImage = true;
                 }
                 if ("width" in hit._source && "height" in hit._source) {
@@ -144,10 +155,6 @@ class Sist2Api {
         }
     }
 
-    _getIndexRoot(indexId) {
-        return this.sist2Info.indices.find(idx => idx.id === indexId).root;
-    }
-
     esQuery(query) {
         return axios.post(`${this.baseUrl}es`, query).then(resp => {
             const res = resp.data;
@@ -156,7 +163,6 @@ class Sist2Api {
                 res.hits.hits.forEach((hit) => {
                     hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
                     hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
-                    hit["_source"]["indexRoot"] = this._getIndexRoot(hit["_source"]["index"]);
 
                     this.setHitProps(hit);
                     this.setHitTags(hit);
@@ -310,7 +316,7 @@ class Sist2Api {
     }
 
     getTagsSqlite() {
-        return axios.get(`${this.baseUrl}fts/tags`)
+        return axios.get(`${this.baseUrl}/fts/tags`)
             .then(resp => {
                 return resp.data.map(tag => this._createEsTag(tag.tag, tag.count))
             });
@@ -567,7 +573,7 @@ class Sist2Api {
     }
 
     getDocumentSqlite(sid) {
-        return axios.get(`${this.baseUrl}fts/d/${sid}`)
+        return axios.get(`${this.baseUrl}/fts/d/${sid}`)
             .then(resp => ({
                 _source: resp.data
             }));
@@ -590,7 +596,7 @@ class Sist2Api {
     }
 
     getTagSuggestionsSqlite(prefix) {
-        return axios.post(`${this.baseUrl}fts/suggestTags`, prefix)
+        return axios.post(`${this.baseUrl}/fts/suggestTags`, prefix)
             .then(resp => (resp.data));
     }
 
@@ -621,7 +627,7 @@ class Sist2Api {
     }
 
     getEmbeddings(sid, modelId) {
-        return axios.post(`${this.baseUrl}e/${sid}/${modelId.toString().padStart(3, '0')}`)
+        return axios.post(`${this.baseUrl}/e/${sid}/${modelId.toString().padStart(3, '0')}`)
             .then(resp => (resp.data));
     }
 }

sist2-vue/src/Sist2ElasticsearchQuery.js

@@ -117,11 +117,11 @@ class Sist2ElasticsearchQuery {
             }
 
             if (dateMin && dateMax) {
-                filters.push({range: {mtime: {gte: dateMin, lte: dateMax, format: "epoch_second"}}})
+                filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
             } else if (dateMin) {
-                filters.push({range: {mtime: {gte: dateMin, format: "epoch_second"}}})
+                filters.push({range: {mtime: {gte: dateMin}}})
             } else if (dateMax) {
-                filters.push({range: {mtime: {lte: dateMax, format: "epoch_second"}}})
+                filters.push({range: {mtime: {lte: dateMax}}})
             }
 
             const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
@@ -238,7 +238,7 @@ class Sist2ElasticsearchQuery {
                 pre_tags: ["<mark>"],
                 post_tags: ["</mark>"],
                 fragment_size: getters.optFragmentSize,
-                number_of_fragments: getters.optFragmentCount,
+                number_of_fragments: 1,
                 order: "score",
                 fields: {
                     content: {},

sist2-vue/src/Sist2SqliteQuery.js

@@ -106,8 +106,6 @@ class Sist2ElasticsearchQuery {
             q["sortAsc"] = true;
         }
 
-        q["searchInPath"] = getters.optSearchInPath;
-
         return q;
     }
 }

sist2-vue/src/components/ContentDiv.vue

@@ -3,8 +3,6 @@
 </template>
 
 <script>
-const FRAGMENT_SEPARATOR = "<br /><i style='line-height: 2.4'>[…]</i><br/>";
-
 export default {
   name: "ContentDiv",
   props: ["doc"],
@@ -15,10 +13,10 @@ export default {
       }
 
       if (this.doc.highlight["content.nGram"]) {
-        return this.doc.highlight["content.nGram"].join(FRAGMENT_SEPARATOR);
+        return this.doc.highlight["content.nGram"][0];
       }
       if (this.doc.highlight.content) {
-        return this.doc.highlight.content.join(FRAGMENT_SEPARATOR);
+        return this.doc.highlight.content[0];
       }
     }
   }

sist2-vue/src/components/InfoTable.vue

@@ -59,7 +59,7 @@ export default {
 
       const fields = [
         "title", "duration", "audioc", "videoc",
-        "bitrate", "artist", "album", "album_artist", "genre", "font_name", "author", "media_comment",
+        "bitrate", "artist", "album", "album_artist", "genre", "font_name", "author",
         "modified_by", "pages", "tag",
         "exif_make", "exif_software", "exif_exposure_time", "exif_fnumber", "exif_focal_length",
         "exif_user_comment", "exif_iso_speed_ratings", "exif_model", "exif_datetime",

sist2-vue/src/components/Lightbox.vue

@@ -77,7 +77,6 @@ export default {
         return listener(e);
       }
     };
-
   },
   methods: {
     keyDownListener(e) {

sist2-vue/src/components/NavBar.vue

@@ -9,7 +9,7 @@
 
     <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
       v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
-        href="https://github.com/sist2app/sist2/blob/master/docs/USAGE.md#elasticsearch"
+        href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
         target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
     </span>
 

sist2-vue/src/i18n/messages.js

@@ -59,7 +59,6 @@ export default {
             searchInPath: "Enable matching query against document path",
             suggestPath: "Enable auto-complete in path filter bar",
             fragmentSize: "Highlight context size",
-            fragmentCount: "Number of highlight snippets",
             queryMode: "Search mode",
             displayMode: "Display",
             columns: "Column count",
@@ -139,7 +138,7 @@ export default {
         },
         debug: "Debug information",
         debugDescription: "Information useful for debugging. If you encounter bugs or have suggestions for" +
-            " new features, please submit a new issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>here</a>.",
+            " new features, please submit a new issue <a href='https://github.com/simon987/sist2/issues/new/choose'>here</a>.",
         tagline: "Tagline",
         toast: {
             esConnErrTitle: "Elasticsearch connection error",
@@ -243,7 +242,6 @@ export default {
             searchInPath: "Abgleich der Abfrage mit dem Dokumentpfad aktivieren",
             suggestPath: "Aktiviere Auto-Vervollständigung in Pfadfilter-Leiste",
             fragmentSize: "Kontextgröße",
-            fragmentCount: "Anzahl der hervorgehobenen Snippets",
             queryMode: "Such-Modus",
             displayMode: "Ansicht",
             columns: "Anzahl Spalten",
@@ -320,7 +318,7 @@ export default {
         },
         debug: "Debug Informationen",
         debugDescription: "Informationen für das Debugging. Wenn du Bugs gefunden oder Anregungen für " +
-            "neue Features hast, poste sie bitte <a href='https://github.com/sist2app/sist2/issues/new/choose'>hier</a>.",
+            "neue Features hast, poste sie bitte <a href='https://github.com/simon987/sist2/issues/new/choose'>hier</a>.",
         tagline: "Tagline",
         toast: {
             esConnErrTitle: "Elasticsearch Verbindungsfehler",
@@ -419,7 +417,6 @@ export default {
             searchInPath: "Activer la recherche dans le chemin des documents",
             suggestPath: "Activer l'autocomplétion dans la barre de filtre de chemin",
             fragmentSize: "Longueur du contexte de surlignage",
-            fragmentCount: "Nombre d'extraits surlignés",
             queryMode: "Mode de recherche",
             displayMode: "Affichage",
             columns: "Nombre de colonnes",
@@ -497,7 +494,7 @@ export default {
         debug: "Information de débogage",
         debugDescription: "Informations utiles pour le débogage\n" +
             "Si vous rencontrez des bogues ou si vous avez des suggestions pour de nouvelles fonctionnalités," +
-            " veuillez soumettre un nouvel Issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>ici</a>.",
+            " veuillez soumettre un nouvel Issue <a href='https://github.com/simon987/sist2/issues/new/choose'>ici</a>.",
         tagline: "Tagline",
         toast: {
             esConnErrTitle: "Erreur de connexion Elasticsearch",
@@ -595,7 +592,6 @@ export default {
             searchInPath: "匹配文档路径",
             suggestPath: "搜索框启用自动补全",
             fragmentSize: "高亮上下文大小",
-            fragmentCount: "突出显示的项目数",
             queryMode: "搜索模式",
             displayMode: "显示",
             columns: "列数",
@@ -672,7 +668,7 @@ export default {
         },
         debug: "调试信息",
         debugDescription: "对调试除错有用的信息。 若您遇到bug或者想建议新功能，请提交新Issue到" +
-            "<a href='https://github.com/sist2app/sist2/issues/new/choose'>这里</a>.",
+            "<a href='https://github.com/simon987/sist2/issues/new/choose'>这里</a>.",
         tagline: "标签栏",
         toast: {
             esConnErrTitle: "Elasticsearch连接错误",
@@ -771,7 +767,6 @@ export default {
             searchInPath: "Włącz szukanie również w ścieżce dokumentu",
             suggestPath: "Włącz auto-uzupełnianie w filtrze ścieżek",
             fragmentSize: "Podświetl wielkość kontekstu w znakach",
-            fragmentCount: "Liczba wyróżnionych fragmentów",
             queryMode: "Tryb szukania",
             displayMode: "Wyświetlanie",
             columns: "Liczba kolumn",
@@ -851,7 +846,7 @@ export default {
         },
         debug: "Informacje dla programistów",
         debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
-            " propozycje zmian, zgłoś to proszę <a href='https://github.com/sist2app/sist2/issues/new/choose'>tutaj</a>.",
+            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
         tagline: "Slogan",
         toast: {
             esConnErrTitle: "Problem z połączeniem z Elasticsearch",

sist2-vue/src/ml/CLIPTransformerModel.js

@@ -22,9 +22,7 @@ export class CLIPTransformerModel {
 
     async loadModel(onProgress) {
         ort.env.wasm.wasmPaths = ORT_WASM_PATHS;
-        if (window.crossOriginIsolated) {
+        ort.env.wasm.numThreads = 2;
-            ort.env.wasm.numThreads = 2;
-        }
 
         let buf = await ModelStore.get(this._modelUrl);
         if (!buf) {

sist2-vue/src/store/index.js

@@ -3,7 +3,7 @@ import Vuex from "vuex"
 import {deserializeMimes, randomSeed, serializeMimes} from "@/util";
 import {getInstance} from "@/plugins/auth0.js";
 
-const CONF_VERSION = 4;
+const CONF_VERSION = 3;
 
 Vue.use(Vuex);
 
@@ -41,7 +41,6 @@ export default new Vuex.Store({
         optTagOrOperator: false,
         optFuzzy: true,
         optFragmentSize: 200,
-        optFragmentCount: 1,
         optQueryMode: "simple",
         optSearchInPath: false,
         optColumns: "auto",
@@ -59,7 +58,7 @@ export default new Vuex.Store({
         optVidPreviewInterval: 700,
         optSimpleLightbox: true,
         optShowTagPickerFilter: true,
-        optMlRepositories: "https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json",
+        optMlRepositories: "https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json",
         optAutoAnalyze: false,
         optMlDefaultModel: null,
 
@@ -171,7 +170,6 @@ export default new Vuex.Store({
         setOptSearchInPath: (state, val) => state.optSearchInPath = val,
         setOptSuggestPath: (state, val) => state.optSuggestPath = val,
         setOptFragmentSize: (state, val) => state.optFragmentSize = val,
-        setOptFragmentCount: (state, val) => state.optFragmentCount = val,
         setOptQueryMode: (state, val) => state.optQueryMode = val,
         setOptResultSize: (state, val) => state.optSize = val,
         setOptTagOrOperator: (state, val) => state.optTagOrOperator = val,
@@ -432,7 +430,6 @@ export default new Vuex.Store({
         optSearchInPath: state => state.optSearchInPath,
         optSuggestPath: state => state.optSuggestPath,
         optFragmentSize: state => state.optFragmentSize,
-        optFragmentCount: state => state.optFragmentCount,
         optQueryMode: state => state.optQueryMode,
         optTreemapType: state => state.optTreemapType,
         optTreemapTiling: state => state.optTreemapTiling,

sist2-vue/src/views/Configuration.vue

@@ -81,7 +81,6 @@
                             <li><code>doc.artist</code></li>
                             <li><code>doc.title</code></li>
                             <li><code>doc.genre</code></li>
-                            <li><code>doc.media_comment</code></li>
                             <li><code>doc.album_artist</code></li>
                             <li><code>doc.exif_make</code></li>
                             <li><code>doc.exif_model</code></li>
@@ -137,7 +136,7 @@
                         {{ $t("opt.fuzzy") }}
                     </b-form-checkbox>
 
-                    <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
+                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optSearchInPath" @input="setOptSearchInPath">{{
                         $t("opt.searchInPath")
                         }}
                     </b-form-checkbox>
@@ -151,10 +150,6 @@
                     <b-form-input :value="optFragmentSize" step="10" type="number" min="0"
                                   @input="setOptFragmentSize"></b-form-input>
 
-                  <label :class="{'text-muted': uiSqliteMode}">{{ $t("opt.fragmentCount") }}</label>
-                  <b-form-input :value="optFragmentCount" :disabled="uiSqliteMode" step="1" type="number" min="1"
-                                @input="setOptFragmentCount"></b-form-input>
-
                     <label>{{ $t("opt.resultSize") }}</label>
                     <b-form-input :value="optResultSize" type="number" min="10"
                                   @input="setOptResultSize"></b-form-input>
@@ -318,7 +313,6 @@ export default {
             "optSearchInPath",
             "optSuggestPath",
             "optFragmentSize",
-            "optFragmentCount",
             "optQueryMode",
             "optTreemapType",
             "optTreemapTiling",
@@ -365,7 +359,6 @@ export default {
             "setOptSearchInPath",
             "setOptSuggestPath",
             "setOptFragmentSize",
-            "setOptFragmentCount",
             "setOptQueryMode",
             "setOptTreemapType",
             "setOptTreemapTiling",

sist2-vue/watch.sh

@@ -1,5 +1,3 @@
 #!/usr/bin/env bash
 
-export NODE_OPTIONS=--openssl-legacy-provider
-
 ./node_modules/@vue/cli-service/bin/vue-cli-service.js build --watch

src/cli.c

@@ -25,7 +25,6 @@ const char *TESS_DATAPATHS[] = {
         "/usr/share/tessdata/",
         "/usr/share/tesseract-ocr/tessdata/",
         "/usr/share/tesseract-ocr/4.00/tessdata/",
-        "/usr/share/tesseract-ocr/5/tessdata/",
         "./",
         NULL
 };

src/ctx.h

@@ -19,7 +19,6 @@
 #include "src/database/database.h"
 #include "src/index/elastic.h"
 #include "sqlite3.h"
-#include "ignorelist.h"
 
 #include <pcre.h>
 
@@ -35,7 +34,6 @@ typedef struct {
     pcre *exclude;
     pcre_extra *exclude_extra;
     int fast;
-    ignorelist_t *ignorelist;
 
     scan_arc_ctx_t arc_ctx;
     scan_comic_ctx_t comic_ctx;

src/database/database.c

@@ -114,7 +114,7 @@ void save_current_job_info(sqlite3_context *ctx, int argc, sqlite3_value **argv)
     char buf[PATH_MAX];
     strcpy(buf, current_job);
 
-    SET_CURRENT_JOB(ipc_ctx, current_job);
+    strcpy(ipc_ctx->current_job[ProcData.thread_id], current_job);
 
     sqlite3_result_text(ctx, "ok", -1, SQLITE_STATIC);
 }
@@ -478,7 +478,8 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) {
 database_iterator_t *database_create_delete_list_iterator(database_t *db) {
 
     sqlite3_stmt *stmt;
-    sqlite3_prepare_v2(db->db, "SELECT id FROM delete_list", -1, &stmt, NULL);
+    sqlite3_prepare_v2(db->db, "SELECT doc.id FROM delete_list "
+                               "INNER JOIN document doc ON doc.ROWID = delete_list.id;", -1, &stmt, NULL);
 
     database_iterator_t *iter = malloc(sizeof(database_iterator_t));
 

src/database/database.h

@@ -64,8 +64,6 @@ typedef struct {
     char current_job[MAX_THREADS][PATH_MAX * 2];
 } database_ipc_ctx_t;
 
-#define SET_CURRENT_JOB(ctx, job) (strcpy((ctx)->current_job[ProcData.thread_id], job))
-
 typedef struct {
     double date_min;
     double date_max;

src/database/database_fts.c

@@ -102,9 +102,7 @@ void database_fts_index(database_t *db) {
             db->db, "DELETE FROM fts.mime_index;", NULL, NULL, NULL));
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db, "INSERT INTO fts.mime_index (index_id, mime, count) "
-                    "SELECT index_id, mime, count(*) FROM fts.document_index "
+                    "SELECT index_id, mime, count(*) FROM fts.document_index GROUP BY index_id, mime",
-                    "WHERE mime IS NOT NULL "
-                    "GROUP BY index_id, mime",
             NULL, NULL, NULL));
 
     LOG_DEBUG("database_fts.c", "Generating path index");
@@ -162,8 +160,7 @@ void database_fts_index(database_t *db) {
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "INSERT INTO search(rowid, name, content, title, path) "
+            "INSERT INTO search(rowid, name, content, title) SELECT id, name, content, title from document_view",
-            "SELECT id, name, content, title, path from document_view",
             NULL, NULL, NULL));
 }
 

src/database/database_schema.c

@@ -75,25 +75,23 @@ const char *FtsDatabaseSchema =
         "  WHERE id = OLD.id;"
         " END;"
         ""
-        "CREATE VIEW IF NOT EXISTS document_view (id, name, content, title, path)"
+        "CREATE VIEW IF NOT EXISTS document_view (id, name, content, title)"
         " AS"
         " SELECT id,"
         "  json_data->>'name',"
         "  json_data->>'content',"
-        "  json_data->>'title',"
+        "  json_data->>'title'"
-        "  json_data->>'path'"
         " FROM document_index;"
         ""
         "CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5 ("
         "   name,"
         "   content,"
         "   title,"
-        "   path,"
         "   content='document_view',"
         "   content_rowid='id'"
         ");"
-        // name^8, content^3, title^8, path^5
+        // name^8, content^3, title^8
-        "INSERT INTO search(search, rank) VALUES('rank', 'bm25(8, 3, 8, 5)');"
+        "INSERT INTO search(search, rank) VALUES('rank', 'bm25(8, 3, 8)');"
         "";
 
 const char *IpcDatabaseSchema =

src/ignorelist.c

@@ -1,106 +0,0 @@
-#include "ignorelist.h"
-#include "ctx.h"
-#include <git2.h>
-
-typedef struct ignorelist {
-    git_repository *repo;
-    char repo_path[PATH_MAX];
-    int has_rules;
-} ignorelist_t;
-
-char *get_tempdir() {
-    char *tempdir_env = getenv("TMPDIR");
-
-    if (tempdir_env != NULL) {
-        return tempdir_env;
-    }
-
-    return "/tmp/";
-}
-
-void ignorelist_destroy(ignorelist_t* ignorelist) {
-    git_libgit2_shutdown();
-
-    if (ignorelist->repo != NULL) {
-        git_repository_free(ignorelist->repo);
-    }
-
-    free(ignorelist);
-}
-
-ignorelist_t *ignorelist_create() {
-    git_libgit2_init();
-
-    ignorelist_t *ignorelist = malloc(sizeof(ignorelist_t));
-
-    ignorelist->repo = NULL;
-    ignorelist->has_rules = FALSE;
-
-    char *tempdir = get_tempdir();
-
-    if (tempdir[strlen(tempdir) - 1] == '/') {
-        sprintf(ignorelist->repo_path, "%ssist2-ignorelist-%d", tempdir, getpid());
-    } else {
-        sprintf(ignorelist->repo_path, "%s/sist2-ignorelist-%d", tempdir, getpid());
-    }
-
-    return ignorelist;
-}
-
-void ignorelist_load_ignore_file(ignorelist_t *ignorelist, const char *filepath) {
-
-    FILE *file;
-    char line[PATH_MAX * 2];
-
-    file = fopen(filepath, "r");
-
-    if(file == NULL) {
-        // No ignore list
-        return;
-    }
-
-    LOG_DEBUGF("ignorelist.c", "Opening temporary git repository %s", ignorelist->repo_path);
-    int init_result = git_repository_init(&ignorelist->repo, ignorelist->repo_path, TRUE);
-
-    if (init_result != 0) {
-        LOG_FATALF("ignorelist.c", "Got error code from git_repository_init(): %d", init_result);
-    }
-
-    git_ignore_clear_internal_rules(ignorelist->repo);
-
-    while(fgets(line, PATH_MAX * 2, file)){
-
-        line[strlen(line) - 1] = '\0'; // Strip trailing newline
-        char *rules = {line,};
-
-        int result = git_ignore_add_rule(ignorelist->repo, rules);
-
-        if (result == 0) {
-            LOG_DEBUGF("ignorelist.c", "Load ignore rule: %s", line);
-            ignorelist->has_rules = TRUE;
-        } else {
-            LOG_FATALF("ignorelist.c", "Invalid ignore rule: %s", line);
-        }
-    }
-
-    fclose(file);
-}
-
-int ignorelist_is_ignored(ignorelist_t *ignorelist, const char *filepath) {
-
-    if (!ignorelist->has_rules) {
-        return FALSE;
-    }
-
-    const char *rel_path = filepath + ScanCtx.index.desc.root_len;
-
-    int ignored = -1;
-
-    int result = git_ignore_path_is_ignored(&ignored, ignorelist->repo, rel_path);
-
-    if (result != 0) {
-        LOG_FATALF("ignorelist.c", "git_ignore_path_is_ignored returned error code: %d", result);
-    }
-
-    return ignored;
-}

src/ignorelist.h

@@ -1,16 +0,0 @@
-#ifndef SIST2_IGNORELIST_H
-#define SIST2_IGNORELIST_H
-
-#include "src/sist.h"
-
-typedef struct ignorelist ignorelist_t;
-
-ignorelist_t *ignorelist_create();
-
-void ignorelist_destroy(ignorelist_t* ignorelist);
-
-void ignorelist_load_ignore_file(ignorelist_t* ignorelist, const char* filepath);
-
-int ignorelist_is_ignored(ignorelist_t* ignorelist, const char* filepath);
-
-#endif //SIST2_IGNORELIST_H

src/index/web.c

@@ -90,7 +90,6 @@ subreq_ctx_t *web_post_async(const char *url, char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
-        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, req->curl_err_buffer);
@@ -124,7 +123,6 @@ response_t *web_get(const char *url, int timeout, int insecure) {
     curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
-        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     struct curl_slist *headers = NULL;
@@ -164,7 +162,6 @@ response_t *web_post(const char *url, const char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
-        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     char err_buffer[CURL_ERROR_SIZE + 1] = {};
@@ -206,11 +203,10 @@ response_t *web_put(const char *url, const char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
     curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
-    curl_easy_setopt(curl, CURLOPT_SHARE, 0);
+    curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
     curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4);
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
-        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     struct curl_slist *headers = NULL;
@@ -245,7 +241,6 @@ response_t *web_delete(const char *url, int insecure) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
-        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "");

src/io/serialize.c

@@ -30,8 +30,6 @@ char *get_meta_key_text(enum metakey meta_key) {
             return "genre";
         case MetaTitle:
             return "title";
-        case MetaMediaComment:
-            return "media_comment";
         case MetaFontName:
             return "font_name";
         case MetaExifMake:
@@ -161,7 +159,6 @@ void write_document(document_t *doc) {
             case MetaExifGpsLatitudeDec:
             case MetaExifGpsLatitudeRef:
             case MetaChecksum:
-            case MetaMediaComment:
             case MetaTitle: {
                 cJSON_AddStringToObject(json, get_meta_key_text(meta->key), meta->str_val);
                 buffer_size_guess += (int) strlen(meta->str_val);

src/io/walk.c

@@ -23,17 +23,8 @@ int handle_entry(const char *filepath, const struct stat *info, int typeflag, st
     if (ScanCtx.exclude != NULL && EXCLUDED(filepath)) {
         LOG_DEBUGF("walk.c", "Excluded: %s", filepath);
 
-        if (typeflag == FTW_D) {
+        if (typeflag == FTW_F && S_ISREG(info->st_mode)) {
-            return FTW_SKIP_SUBTREE;
+        } else if (typeflag == FTW_D) {
-        }
-
-        return FTW_CONTINUE;
-    }
-
-    if (ignorelist_is_ignored(ScanCtx.ignorelist, filepath)) {
-        LOG_DEBUGF("walk.c", "Ignored: %s", filepath);
-
-        if (typeflag == FTW_D) {
             return FTW_SKIP_SUBTREE;
         }
 

src/main.c

@@ -11,7 +11,7 @@
 #include "web/serve.h"
 #include "parsing/mime.h"
 #include "parsing/parse.h"
-#include "ignorelist.h"
+#include "auth0/auth0_c_api.h"
 
 #include <signal.h>
 #include <pthread.h>
@@ -240,13 +240,6 @@ void sist2_scan(scan_args_t *args) {
 
     LOG_INFOF("main.c", "sist2 v%s", Version);
 
-    ScanCtx.ignorelist = ignorelist_create();
-
-    char ignore_filepath[PATH_MAX];
-    sprintf(ignore_filepath, "%s.sist2ignore", args->path);
-
-    ignorelist_load_ignore_file(ScanCtx.ignorelist, ignore_filepath);
-
     ScanCtx.pool = tpool_create(ScanCtx.threads, TRUE);
     tpool_start(ScanCtx.pool);
 
@@ -276,7 +269,6 @@ void sist2_scan(scan_args_t *args) {
 
     database_generate_stats(db, args->treemap_threshold);
     database_close(db, args->optimize_database);
-    ignorelist_destroy(ScanCtx.ignorelist);
 }
 
 void sist2_index(index_args_t *args) {
@@ -433,8 +425,6 @@ int set_to_negative_if_value_is_zero(UNUSED(struct argparse *self), const struct
         fprintf(stderr, "error: option `--%s` Value must be >= 0\n", option->long_name);
         exit(1);
     }
-
-    return 0;
 }
 
 int main(int argc, const char *argv[]) {
@@ -554,7 +544,7 @@ int main(int argc, const char *argv[]) {
             OPT_END(),
     };
 
-    struct argparse argparse = {};
+    struct argparse argparse;
     argparse_init(&argparse, options, usage, 0);
     argparse_describe(
             &argparse,

src/parsing/mime_generated.c

@@ -295,167 +295,166 @@ image_g3fax=524575,
 image_gif=524576,
 image_heic=524577,
 image_ief=524578,
-image_jp2=524579,
+image_jpeg=524579,
-image_jpeg=524580,
+image_jutvision=524580,
-image_jutvision=524581,
+image_naplps=524581,
-image_naplps=524582,
+image_pict=524582,
-image_pict=524583,
+image_png=524583,
-image_png=524584,
+image_svg=524584 | 0x80000000,
-image_svg=524585 | 0x80000000,
+image_svg_xml=524585 | 0x80000000,
-image_svg_xml=524586 | 0x80000000,
+image_tiff=524586,
-image_tiff=524587,
+image_vnd_adobe_photoshop=524587 | 0x80000000,
-image_vnd_adobe_photoshop=524588 | 0x80000000,
+image_vnd_djvu=524588 | 0x80000000,
-image_vnd_djvu=524589 | 0x80000000,
+image_vnd_fpx=524589,
-image_vnd_fpx=524590,
+image_vnd_microsoft_icon=524590,
-image_vnd_microsoft_icon=524591,
+image_vnd_rn_realflash=524591,
-image_vnd_rn_realflash=524592,
+image_vnd_rn_realpix=524592,
-image_vnd_rn_realpix=524593,
+image_vnd_wap_wbmp=524593,
-image_vnd_wap_wbmp=524594,
+image_vnd_xiff=524594,
-image_vnd_xiff=524595,
+image_webp=524595,
-image_webp=524596,
+image_wmf=524596,
-image_wmf=524597,
+image_x_3ds=524597,
-image_x_3ds=524598,
+image_x_adobe_dng=524598 | 0x00800000,
-image_x_adobe_dng=524599 | 0x00800000,
+image_x_award_bioslogo=524599,
-image_x_award_bioslogo=524600,
+image_x_canon_cr2=524600 | 0x00800000,
-image_x_canon_cr2=524601 | 0x00800000,
+image_x_canon_crw=524601 | 0x00800000,
-image_x_canon_crw=524602 | 0x00800000,
+image_x_cmu_raster=524602,
-image_x_cmu_raster=524603,
+image_x_cur=524603,
-image_x_cur=524604,
+image_x_dcraw=524604 | 0x00800000,
-image_x_dcraw=524605 | 0x00800000,
+image_x_dwg=524605,
-image_x_dwg=524606,
+image_x_eps=524606,
-image_x_eps=524607,
+image_x_epson_erf=524607 | 0x00800000,
-image_x_epson_erf=524608 | 0x00800000,
+image_x_exr=524608,
-image_x_exr=524609,
+image_x_fuji_raf=524609 | 0x00800000,
-image_x_fuji_raf=524610 | 0x00800000,
+image_x_gem=524610,
-image_x_gem=524611,
+image_x_icns=524611,
-image_x_icns=524612,
+image_x_icon=524612 | 0x80000000,
-image_x_icon=524613 | 0x80000000,
+image_x_jg=524613,
-image_x_jg=524614,
+image_x_jps=524614,
-image_x_jps=524615,
+image_x_kodak_dcr=524615 | 0x00800000,
-image_x_kodak_dcr=524616 | 0x00800000,
+image_x_kodak_k25=524616 | 0x00800000,
-image_x_kodak_k25=524617 | 0x00800000,
+image_x_kodak_kdc=524617 | 0x00800000,
-image_x_kodak_kdc=524618 | 0x00800000,
+image_x_minolta_mrw=524618 | 0x00800000,
-image_x_minolta_mrw=524619 | 0x00800000,
+image_x_ms_bmp=524619,
-image_x_ms_bmp=524620,
+image_x_niff=524620,
-image_x_niff=524621,
+image_x_nikon_nef=524621 | 0x00800000,
-image_x_nikon_nef=524622 | 0x00800000,
+image_x_olympus_orf=524622 | 0x00800000,
-image_x_olympus_orf=524623 | 0x00800000,
+image_x_panasonic_raw=524623 | 0x00800000,
-image_x_panasonic_raw=524624 | 0x00800000,
+image_x_pcx=524624,
-image_x_pcx=524625,
+image_x_pentax_pef=524625 | 0x00800000,
-image_x_pentax_pef=524626 | 0x00800000,
+image_x_pict=524626,
-image_x_pict=524627,
+image_x_portable_bitmap=524627,
-image_x_portable_bitmap=524628,
+image_x_portable_graymap=524628,
-image_x_portable_graymap=524629,
+image_x_portable_pixmap=524629,
-image_x_portable_pixmap=524630,
+image_x_quicktime=524630,
-image_x_quicktime=524631,
+image_x_rgb=524631,
-image_x_rgb=524632,
+image_x_sigma_x3f=524632 | 0x00800000,
-image_x_sigma_x3f=524633 | 0x00800000,
+image_x_sony_arw=524633 | 0x00800000,
-image_x_sony_arw=524634 | 0x00800000,
+image_x_sony_sr2=524634 | 0x00800000,
-image_x_sony_sr2=524635 | 0x00800000,
+image_x_sony_srf=524635 | 0x00800000,
-image_x_sony_srf=524636 | 0x00800000,
+image_x_tga=524636,
-image_x_tga=524637,
+image_x_tiff=524637,
-image_x_tiff=524638,
+image_x_win_bitmap=524638,
-image_x_win_bitmap=524639,
+image_x_xcf=524639 | 0x80000000,
-image_x_xcf=524640 | 0x80000000,
+image_x_xpixmap=524640 | 0x80000000,
-image_x_xpixmap=524641 | 0x80000000,
+image_x_xwindowdump=524641,
-image_x_xwindowdump=524642,
+message_news=196962,
-message_news=196963,
+message_rfc822=196963,
-message_rfc822=196964,
+model_vnd_dwf=65892,
-model_vnd_dwf=65893,
+model_vnd_gdl=65893,
-model_vnd_gdl=65894,
+model_vnd_gs_gdl=65894,
-model_vnd_gs_gdl=65895,
+model_vrml=65895,
-model_vrml=65896,
+model_x_pov=65896,
-model_x_pov=65897,
+text_PGP=590185,
-text_PGP=590186,
+text_asp=590186,
-text_asp=590187,
+text_css=590187,
-text_css=590188,
+text_csv=590188,
-text_csv=590189,
+text_html=590189 | 0x01000000,
-text_html=590190 | 0x01000000,
+text_javascript=590190,
-text_javascript=590191,
+text_mcf=590191,
-text_mcf=590192,
+text_pascal=590192,
-text_pascal=590193,
+text_plain=590193,
-text_plain=590194,
+text_richtext=590194,
-text_richtext=590195,
+text_rtf=590195,
-text_rtf=590196,
+text_scriplet=590196,
-text_scriplet=590197,
+text_tab_separated_values=590197,
-text_tab_separated_values=590198,
+text_troff=590198,
-text_troff=590199,
+text_uri_list=590199,
-text_uri_list=590200,
+text_vnd_abc=590200,
-text_vnd_abc=590201,
+text_vnd_fmi_flexstor=590201,
-text_vnd_fmi_flexstor=590202,
+text_vnd_wap_wml=590202,
-text_vnd_wap_wml=590203,
+text_vnd_wap_wmlscript=590203,
-text_vnd_wap_wmlscript=590204,
+text_webviewhtml=590204,
-text_webviewhtml=590205,
+text_x_Algol68=590205,
-text_x_Algol68=590206,
+text_x_asm=590206,
-text_x_asm=590207,
+text_x_audiosoft_intra=590207,
-text_x_audiosoft_intra=590208,
+text_x_awk=590208,
-text_x_awk=590209,
+text_x_bcpl=590209,
-text_x_bcpl=590210,
+text_x_c=590210,
-text_x_c=590211,
+text_x_c__=590211,
-text_x_c__=590212,
+text_x_component=590212,
-text_x_component=590213,
+text_x_diff=590213,
-text_x_diff=590214,
+text_x_fortran=590214,
-text_x_fortran=590215,
+text_x_java=590215,
-text_x_java=590216,
+text_x_la_asf=590216,
-text_x_la_asf=590217,
+text_x_lisp=590217,
-text_x_lisp=590218,
+text_x_m=590218,
-text_x_m=590219,
+text_x_m4=590219,
-text_x_m4=590220,
+text_x_makefile=590220,
-text_x_makefile=590221,
+text_x_ms_regedit=590221,
-text_x_ms_regedit=590222,
+text_x_msdos_batch=590222,
-text_x_msdos_batch=590223,
+text_x_objective_c=590223,
-text_x_objective_c=590224,
+text_x_pascal=590224,
-text_x_pascal=590225,
+text_x_perl=590225,
-text_x_perl=590226,
+text_x_php=590226,
-text_x_php=590227,
+text_x_po=590227,
-text_x_po=590228,
+text_x_python=590228,
-text_x_python=590229,
+text_x_ruby=590229,
-text_x_ruby=590230,
+text_x_sass=590230,
-text_x_sass=590231,
+text_x_script_python=590231,
-text_x_script_python=590232,
+text_x_scss=590232,
-text_x_scss=590233,
+text_x_server_parsed_html=590233,
-text_x_server_parsed_html=590234,
+text_x_setext=590234,
-text_x_setext=590235,
+text_x_sgml=590235 | 0x01000000,
-text_x_sgml=590236 | 0x01000000,
+text_x_shellscript=590236,
-text_x_shellscript=590237,
+text_x_speech=590237,
-text_x_speech=590238,
+text_x_tcl=590238,
-text_x_tcl=590239,
+text_x_tex=590239,
-text_x_tex=590240,
+text_x_uil=590240,
-text_x_uil=590241,
+text_x_uuencode=590241,
-text_x_uuencode=590242,
+text_x_vcalendar=590242,
-text_x_vcalendar=590243,
+text_x_vcard=590243,
-text_x_vcard=590244,
+text_xml=590244 | 0x01000000,
-text_xml=590245 | 0x01000000,
+video_MP2T=393637,
-video_MP2T=393638,
+video_animaflex=393638,
-video_animaflex=393639,
+video_avi=393639,
-video_avi=393640,
+video_avs_video=393640,
-video_avs_video=393641,
+video_mp4=393641,
-video_mp4=393642,
+video_mpeg=393642,
-video_mpeg=393643,
+video_quicktime=393643,
-video_quicktime=393644,
+video_vdo=393644,
-video_vdo=393645,
+video_vivo=393645,
-video_vivo=393646,
+video_vnd_rn_realvideo=393646,
-video_vnd_rn_realvideo=393647,
+video_vosaic=393647,
-video_vosaic=393648,
+video_webm=393648,
-video_webm=393649,
+video_x_amt_demorun=393649,
-video_x_amt_demorun=393650,
+video_x_amt_showrun=393650,
-video_x_amt_showrun=393651,
+video_x_atomic3d_feature=393651,
-video_x_atomic3d_feature=393652,
+video_x_dl=393652,
-video_x_dl=393653,
+video_x_dv=393653,
-video_x_dv=393654,
+video_x_fli=393654,
-video_x_fli=393655,
+video_x_flv=393655,
-video_x_flv=393656,
+video_x_isvideo=393656,
-video_x_isvideo=393657,
+video_x_jng=393657 | 0x80000000,
-video_x_jng=393658 | 0x80000000,
+video_x_m4v=393658,
-video_x_m4v=393659,
+video_x_matroska=393659,
-video_x_matroska=393660,
+video_x_mng=393660,
-video_x_mng=393661,
+video_x_motion_jpeg=393661,
-video_x_motion_jpeg=393662,
+video_x_ms_asf=393662,
-video_x_ms_asf=393663,
+video_x_msvideo=393663,
-video_x_msvideo=393664,
+video_x_qtc=393664,
-video_x_qtc=393665,
+video_x_sgi_movie=393665,
-video_x_sgi_movie=393666,
+x_epoc_x_sisx_app=721346,
-x_epoc_x_sisx_app=721347,
 };
 char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
 case application_x_matlab_data: return "application/x-matlab-data";
@@ -909,7 +908,6 @@ case image_x_sony_arw: return "image/x-sony-arw";
 case image_x_sony_sr2: return "image/x-sony-sr2";
 case image_x_sony_srf: return "image/x-sony-srf";
 case image_x_epson_erf: return "image/x-epson-erf";
-case image_jp2: return "image/jp2";
 default: return NULL;}}
 unsigned int mime_extension_lookup(unsigned long extension_crc32) {switch (extension_crc32) {
 case 2495639202:return application_x_matlab_data;
@@ -1293,7 +1291,6 @@ case 1698465774:return image_x_sony_arw;
 case 2083014127:return image_x_sony_sr2;
 case 271503362:return image_x_sony_srf;
 case 142938048:return image_x_epson_erf;
-case 1575600018:return image_jp2;
 default: return 0;}}
 unsigned int mime_name_lookup(unsigned long mime_crc32) {switch (mime_crc32) {
 case 3272851765: return application_x_matlab_data;
@@ -1747,8 +1744,7 @@ case 3060720351: return image_x_sony_arw;
 case 2944016606: return image_x_sony_sr2;
 case 3279729971: return image_x_sony_srf;
 case 1665206815: return image_x_epson_erf;
-case 1849479005: return image_jp2;
 default: return 0;}}
-unsigned int mime_ids[] = {655530,655363,655364,655365,655366,655362,655361,655367,655368,655369,655370,655371,655372 | 0x40000000,655373,655374,655375,655376 | 0x08000000,655377,655378,655379,655380,655382,655381,655383,655384,655390,655385,655386,655387,655388,655389,655391,655392,655393,655394,655395 | 0x40000000,655396,655397,655398,655399,655400,655401,655402,655403,655404,655405,655406,655407,655408,655411,655412,655413,655414,655415,655416,655417,655418,655419 | 0x20000000,655421,655422,655423,655424,655425,655426,655427,655428,655429,655430,655431,655432 | 0x04000000,655433 | 0x04000000,655434 | 0x04000000,655435,655436,655437,655438,655439,655440,655441,655442,655443,655444,655445,655446 | 0x10000000,655447,655448,655449 | 0x10000000,655450,655451,655452,655453,655454,655455,655456,655457,655458,655459,655461 | 0x08000000,655460,655462,655463,655464,655465,655466,655467,655468,655469,655470,655471,655472,655473,655474,655475,655476,655477,655478,655479,655480,1,655481,655482,655483,655484,655485,655486,655487,655488,655489 | 0x20000000,655490,655491,655492,655493,655494,655495,655496,655497,655498,655499,655500,655501,655502,655503,655504,655505,655506,655507,655508,655509,655510,655511,655512,655513,655514,655515,655516,655517,655519,655518 | 0x08000000,655521,655520,655522 | 0x08000000,655523 | 0x08000000,655524 | 0x08000000,655525,655526,655527,655528,655529,655531,655532,655533,655534,655535,655599,655536 | 0x02000000,655409 | 0x02000000,655540,655537,655538,655539,655541,655542,655543,655544,655545,655546,655547,655548,655549,655550,655552,655551,655553,655554,655555,655556,655557,655558,655559,655560,655561,655562 | 0x10000000,655563,655564,655565,655566,655567,655569,655568,655570,655571,655572,655573,655574,655575,655576,655577,655578 | 0x10000000,655579,655580,655581,655583,655582,655584,655585,655586,655587,655588,655589,655590,655591,655592,655593,655594,655595 | 0x08000000,655596,655597 | 0x08000000,655600 | 0x10000000,655601,458994 | 0x80000000,458995,458996,458998,458997,458999,459000,459001,459002,459003,459004,459005,459006,459007,459008,459009,459010,459011,459012,459013,459014,459015,459016,459017,459018,459030,459019,459020,459021,459022,459023,459025,459024,459026,459027,459029 | 0x80000000,459028 | 0x80000000,327959 | 0x20000000,327960 | 0x20000000,327962 | 0x20000000,327961 | 0x20000000,524571,524572,524573,524574,524575,524576,524577,524578,524580,524581,524582,524583,524584,524585 | 0x80000000,524586 | 0x80000000,524587,524588 | 0x80000000,524589 | 0x80000000,524590,524591,524592,524593,524594,524595,524596,524597,524598,524600,524603,524604,524606,524607,524609,524611,524612,524613 | 0x80000000,524614,524615,524620,524621,524625,524627,524628,524629,524630,524631,524632,524637,524638,524639,524640 | 0x80000000,524641 | 0x80000000,524642,196963,196964,65893,65894,65895,65896,65897,590187,590188,590190 | 0x01000000,590191,590192,590193,590186,590194,590232,590189,655410,590195,590196,590197,590198,590199,590200,590201,590202,590204,590203,590205,590206,590207,590208,590209,590210,590211,590212,590213,590214,590215,590216,590217,590218,590220,590221,590245 | 0x01000000,590219,590223,590222,590224,590225,590226,590227,590228,590229,590230,590231,590233,590234,590235,590236 | 0x01000000,590237,590238,590239,590240,590241,590242,590243,590244,393639,393640,393641,393638,393642,393643,393644,393645,393646,393647,393648,393649,393650,393651,393652,393653,393654,393655,393656,393657,393658 | 0x80000000,393659,393660,393661,393662,393663,393664,393665,393666,721347,655598,655420,524623 | 0x00800000,524622 | 0x00800000,524610 | 0x00800000,524624 | 0x00800000,524599 | 0x00800000,524601 | 0x00800000,524602 | 0x00800000,524605 | 0x00800000,524616 | 0x00800000,524617 | 0x00800000,524618 | 0x00800000,524619 | 0x00800000,524626 | 0x00800000,524633 | 0x00800000,524634 | 0x00800000,524635 | 0x00800000,524636 | 0x00800000,524608 | 0x00800000,524579,0};
+unsigned int mime_ids[] = {655530,655363,655364,655365,655366,655362,655361,655367,655368,655369,655370,655371,655372 | 0x40000000,655373,655374,655375,655376 | 0x08000000,655377,655378,655379,655380,655382,655381,655383,655384,655390,655385,655386,655387,655388,655389,655391,655392,655393,655394,655395 | 0x40000000,655396,655397,655398,655399,655400,655401,655402,655403,655404,655405,655406,655407,655408,655411,655412,655413,655414,655415,655416,655417,655418,655419 | 0x20000000,655421,655422,655423,655424,655425,655426,655427,655428,655429,655430,655431,655432 | 0x04000000,655433 | 0x04000000,655434 | 0x04000000,655435,655436,655437,655438,655439,655440,655441,655442,655443,655444,655445,655446 | 0x10000000,655447,655448,655449 | 0x10000000,655450,655451,655452,655453,655454,655455,655456,655457,655458,655459,655461 | 0x08000000,655460,655462,655463,655464,655465,655466,655467,655468,655469,655470,655471,655472,655473,655474,655475,655476,655477,655478,655479,655480,1,655481,655482,655483,655484,655485,655486,655487,655488,655489 | 0x20000000,655490,655491,655492,655493,655494,655495,655496,655497,655498,655499,655500,655501,655502,655503,655504,655505,655506,655507,655508,655509,655510,655511,655512,655513,655514,655515,655516,655517,655519,655518 | 0x08000000,655521,655520,655522 | 0x08000000,655523 | 0x08000000,655524 | 0x08000000,655525,655526,655527,655528,655529,655531,655532,655533,655534,655535,655599,655536 | 0x02000000,655409 | 0x02000000,655540,655537,655538,655539,655541,655542,655543,655544,655545,655546,655547,655548,655549,655550,655552,655551,655553,655554,655555,655556,655557,655558,655559,655560,655561,655562 | 0x10000000,655563,655564,655565,655566,655567,655569,655568,655570,655571,655572,655573,655574,655575,655576,655577,655578 | 0x10000000,655579,655580,655581,655583,655582,655584,655585,655586,655587,655588,655589,655590,655591,655592,655593,655594,655595 | 0x08000000,655596,655597 | 0x08000000,655600 | 0x10000000,655601,458994 | 0x80000000,458995,458996,458998,458997,458999,459000,459001,459002,459003,459004,459005,459006,459007,459008,459009,459010,459011,459012,459013,459014,459015,459016,459017,459018,459030,459019,459020,459021,459022,459023,459025,459024,459026,459027,459029 | 0x80000000,459028 | 0x80000000,327959 | 0x20000000,327960 | 0x20000000,327962 | 0x20000000,327961 | 0x20000000,524571,524572,524573,524574,524575,524576,524577,524578,524579,524580,524581,524582,524583,524584 | 0x80000000,524585 | 0x80000000,524586,524587 | 0x80000000,524588 | 0x80000000,524589,524590,524591,524592,524593,524594,524595,524596,524597,524599,524602,524603,524605,524606,524608,524610,524611,524612 | 0x80000000,524613,524614,524619,524620,524624,524626,524627,524628,524629,524630,524631,524636,524637,524638,524639 | 0x80000000,524640 | 0x80000000,524641,196962,196963,65892,65893,65894,65895,65896,590186,590187,590189 | 0x01000000,590190,590191,590192,590185,590193,590231,590188,655410,590194,590195,590196,590197,590198,590199,590200,590201,590203,590202,590204,590205,590206,590207,590208,590209,590210,590211,590212,590213,590214,590215,590216,590217,590219,590220,590244 | 0x01000000,590218,590222,590221,590223,590224,590225,590226,590227,590228,590229,590230,590232,590233,590234,590235 | 0x01000000,590236,590237,590238,590239,590240,590241,590242,590243,393638,393639,393640,393637,393641,393642,393643,393644,393645,393646,393647,393648,393649,393650,393651,393652,393653,393654,393655,393656,393657 | 0x80000000,393658,393659,393660,393661,393662,393663,393664,393665,721346,655598,655420,524622 | 0x00800000,524621 | 0x00800000,524609 | 0x00800000,524623 | 0x00800000,524598 | 0x00800000,524600 | 0x00800000,524601 | 0x00800000,524604 | 0x00800000,524615 | 0x00800000,524616 | 0x00800000,524617 | 0x00800000,524618 | 0x00800000,524625 | 0x00800000,524632 | 0x00800000,524633 | 0x00800000,524634 | 0x00800000,524635 | 0x00800000,524607 | 0x00800000,0};
 unsigned int* get_mime_ids() { return mime_ids; }
 #endif

src/parsing/parse.c

@@ -142,10 +142,6 @@ void parse(parse_job_t *job) {
         job->vfile.calculate_checksum = ScanCtx.calculate_checksums;
     }
 
-    if (IS_SUB_JOB(job)) {
-        SET_CURRENT_JOB(ProcData.ipc_db->ipc_ctx, job->filepath);
-    }
-
     document_t *doc = malloc(sizeof(document_t));
 
     strcpy(doc->filepath, job->filepath);
@@ -165,8 +161,7 @@ void parse(parse_job_t *job) {
         return;
     }
 
-    int document_exists = database_mark_document(ProcData.index_db, doc->filepath + ScanCtx.index.desc.root_len, doc->mtime);
+    if (database_mark_document(ProcData.index_db, doc->filepath + ScanCtx.index.desc.root_len, doc->mtime)) {
-    if (document_exists) {
         CLOSE_FILE(job->vfile)
         free(doc);
         return;

src/sist.h

@@ -51,17 +51,17 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.5.0"
+#define VERSION "3.3.3"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
-static const int VersionMinor = 5;
+static const int VersionMinor = 3;
-static const int VersionPatch = 0;
+static const int VersionPatch = 3;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown
 #endif
 
-#define EXPECTED_MONGOOSE_VERSION "7.16"
+#define EXPECTED_MONGOOSE_VERSION "7.7"
 
 #define Q(x) #x
 #define QUOTE(x) Q(x)

src/web/serve.c

@@ -50,13 +50,13 @@ void get_embedding(struct mg_connection *nc, struct mg_http_message *hm) {
 
     sist_id_t sid;
 
-    if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.buf + 3)) {
+    if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.ptr + 3)) {
-        LOG_DEBUGF("serve.c", "Invalid embedding path: %.*s", (int) hm->uri.len, hm->uri.buf);
+        LOG_DEBUGF("serve.c", "Invalid embedding path: %.*s", (int) hm->uri.len, hm->uri.ptr);
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
-    int model_id = (int) strtol(hm->uri.buf + SIST_SID_LEN + 3, NULL, 10);
+    int model_id = (int) strtol(hm->uri.ptr + SIST_SID_LEN + 3, NULL, 10);
 
     database_t *db = web_get_database(sid.index_id);
     if (db == NULL) {
@@ -86,11 +86,11 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
     char index_id_str[9];
     char arg_stat_type[5];
 
-    memcpy(index_id_str, hm->uri.buf + 3, 8);
+    memcpy(index_id_str, hm->uri.ptr + 3, 8);
     *(index_id_str + 8) = '\0';
-    int index_id = (int) strtol(index_id_str, NULL, 16);
+    int index_id = (int)strtol(index_id_str, NULL, 16);
 
-    memcpy(arg_stat_type, hm->uri.buf + 3 + 9, 4);
+    memcpy(arg_stat_type, hm->uri.ptr + 3 + 9, 4);
     *(arg_stat_type + sizeof(arg_stat_type) - 1) = '\0';
 
     database_stat_type_d stat_type = database_get_stat_type_by_mnemonic(arg_stat_type);
@@ -108,6 +108,7 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
 
     cJSON *json = database_get_stats(db, stat_type);
     mg_send_json(nc, json);
+
     cJSON_Delete(json);
 }
 
@@ -135,19 +136,19 @@ void serve_chunk_vendors_js(struct mg_connection *nc, struct mg_http_message *hm
     }
 }
 
-void serve_favicon_ico(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
+void serve_favicon_ico(struct mg_connection *nc, struct mg_http_message *hm) {
     web_serve_asset_favicon_ico(nc);
 }
 
-void serve_style_css(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
+void serve_style_css(struct mg_connection *nc, struct mg_http_message *hm) {
     web_serve_asset_style_css(nc);
 }
 
-void serve_chunk_vendors_css(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
+void serve_chunk_vendors_css(struct mg_connection *nc, struct mg_http_message *hm) {
     web_serve_asset_chunk_vendors_css(nc);
 }
 
-void serve_thumbnail(struct mg_connection *nc, UNUSED(struct mg_http_message *hm), int index_id,
+void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, int index_id,
                      int doc_id, int arg_num) {
 
     database_t *db = web_get_database(index_id);
@@ -168,7 +169,6 @@ void serve_thumbnail(struct mg_connection *nc, UNUSED(struct mg_http_message *hm
                 "Cache-Control: max-age=31536000"
         );
         mg_send(nc, data, data_len);
-        nc->is_resp = 0;
         free(data);
     } else {
         HTTP_REPLY_NOT_FOUND
@@ -179,13 +179,13 @@ void serve_thumbnail(struct mg_connection *nc, UNUSED(struct mg_http_message *hm
 void thumbnail_with_num(struct mg_connection *nc, struct mg_http_message *hm) {
     sist_id_t sid;
 
-    if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.buf + 3)) {
+    if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.ptr + 3)) {
-        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.buf);
+        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr);
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
-    int num = (int) strtol(hm->uri.buf + SIST_SID_LEN + 3, NULL, 10);
+    int num = (int) strtol(hm->uri.ptr + SIST_SID_LEN + 3, NULL, 10);
 
     serve_thumbnail(nc, hm, sid.index_id, sid.doc_id, num);
 }
@@ -193,8 +193,8 @@ void thumbnail_with_num(struct mg_connection *nc, struct mg_http_message *hm) {
 void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) {
     sist_id_t sid;
 
-    if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.buf + 3)) {
+    if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.ptr + 3)) {
-        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.buf);
+        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr);
         HTTP_REPLY_NOT_FOUND
         return;
     }
@@ -210,14 +210,13 @@ void search(struct mg_connection *nc, struct mg_http_message *hm) {
     }
 
     char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.buf, hm->body.len);
+    memcpy(body, hm->body.ptr, hm->body.len);
     *(body + hm->body.len) = '\0';
 
     char url[4096];
     snprintf(url, 4096, "%s/%s/_search", WebCtx.es_url, WebCtx.es_index);
 
     nc->fn_data = web_post_async(url, body, WebCtx.es_insecure_ssl);
-    nc->is_resp = 1;
 }
 
 void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
@@ -369,10 +368,6 @@ void index_info(struct mg_connection *nc) {
         cJSON_AddNumberToObject(idx_json, "timestamp", (double) idx->desc.timestamp);
         cJSON_AddItemToArray(arr, idx_json);
 
-#ifdef SIST_DEBUG_INFO
-        cJSON_AddStringToObject(idx_json, "root", idx->desc.root);
-#endif
-
         cJSON *models = database_get_models(idx->db);
         cJSON_AddItemToObject(idx_json, "models", models);
     }
@@ -383,7 +378,11 @@ void index_info(struct mg_connection *nc) {
         cJSON_AddStringToObject(json, "searchBackend", "elasticsearch");
     }
 
-    mg_send_json(nc, json);
+    char *json_str = cJSON_PrintUnformatted(json);
+
+    web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
+    mg_send(nc, json_str, strlen(json_str));
+    free(json_str);
     cJSON_Delete(json);
 }
 
@@ -416,8 +415,8 @@ cJSON *get_root_document_by_id(int index_id, int doc_id) {
 void file(struct mg_connection *nc, struct mg_http_message *hm) {
     sist_id_t sid;
 
-    if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.buf + 3)) {
+    if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.ptr + 3)) {
-        LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.buf);
+        LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr);
         HTTP_REPLY_NOT_FOUND
         return;
     }
@@ -430,11 +429,6 @@ void file(struct mg_connection *nc, struct mg_http_message *hm) {
 
     cJSON *source = get_root_document_by_id(sid.index_id, sid.doc_id);
 
-    if (source == NULL) {
-        HTTP_REPLY_NOT_FOUND
-        return;
-    }
-
     if (strlen(idx->desc.rewrite_url) == 0) {
         serve_file_from_disk(source, idx, nc, hm);
     } else {
@@ -452,7 +446,6 @@ void status(struct mg_connection *nc) {
     }
 
     free(status);
-    nc->is_resp = 0;
 }
 
 typedef struct {
@@ -487,7 +480,7 @@ tag_req_t *parse_tag_request(cJSON *json) {
     return req;
 }
 
-subreq_ctx_t *elastic_delete_tag(const char *sid, const tag_req_t *req) {
+subreq_ctx_t *elastic_delete_tag(const char* sid, const tag_req_t *req) {
     char *buf = malloc(sizeof(char) * 8192);
     snprintf(buf, 8192,
              "{"
@@ -507,7 +500,7 @@ subreq_ctx_t *elastic_delete_tag(const char *sid, const tag_req_t *req) {
     return web_post_async(url, buf, WebCtx.es_insecure_ssl);
 }
 
-subreq_ctx_t *elastic_write_tag(const char *sid, const tag_req_t *req) {
+subreq_ctx_t *elastic_write_tag(const char* sid, const tag_req_t *req) {
     char *buf = malloc(sizeof(char) * 8192);
     snprintf(buf, 8192,
              "{"
@@ -528,14 +521,14 @@ subreq_ctx_t *elastic_write_tag(const char *sid, const tag_req_t *req) {
 
 void tag(struct mg_connection *nc, struct mg_http_message *hm) {
     sist_id_t sid;
-    if (hm->uri.len != 22 || !parse_sid(&sid, hm->uri.buf + 5)) {
+    if (hm->uri.len != 22 || !parse_sid(&sid, hm->uri.ptr + 5)) {
-        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.buf);
+        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr);
         HTTP_REPLY_NOT_FOUND
         return;
     }
 
     char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.buf, hm->body.len);
+    memcpy(body, hm->body.ptr, hm->body.len);
     *(body + hm->body.len) = '\0';
     cJSON *json = cJSON_Parse(body);
     free(body);
@@ -612,7 +605,7 @@ int check_auth0(struct mg_http_message *hm) {
     }
 
     token_str = malloc(token.len + 1);
-    strncpy(token_str, token.buf, token.len);
+    strncpy(token_str, token.ptr, token.len);
     *(token_str + token.len) = '\0';
 
     int res = auth0_verify_jwt(
@@ -630,7 +623,7 @@ int check_auth0(struct mg_http_message *hm) {
     return TRUE;
 }
 
-static void ev_router(struct mg_connection *nc, int ev, void *ev_data) {
+static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(void *fn_data)) {
 
     if (ev == MG_EV_HTTP_MSG) {
         struct mg_http_message *hm = (struct mg_http_message *) ev_data;
@@ -642,15 +635,13 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data) {
         }
 
         char uri[256];
-        memcpy(uri, hm->uri.buf, hm->uri.len);
+        memcpy(uri, hm->uri.ptr, hm->uri.len);
         *(uri + hm->uri.len) = '\0';
         LOG_DEBUGF("serve.c", "<%s> GET %s",
                    web_address_to_string(&(nc->rem)),
                    uri
         );
 
-#define mg_http_match_uri(hm, pattern) mg_match((hm)->uri, mg_str(pattern), NULL)
-
         if (mg_http_match_uri(hm, "/")) {
             serve_index_html(nc, hm);
             return;
@@ -743,7 +734,6 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data) {
                 if (r->status_code == 200) {
                     web_send_headers(nc, 200, r->size, "Content-Type: application/json");
                     mg_send(nc, r->body, r->size);
-                    nc->is_resp = 0;
                 } else if (r->status_code == 0) {
                     sist_log("serve.c", LOG_SIST_ERROR, "Could not connect to elasticsearch!");
 

src/web/web_fts.c

@@ -179,8 +179,7 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
 
     json_value req_query, req_path, req_size_min, req_size_max, req_date_min, req_date_max, req_page_size,
             req_index_ids, req_mime_types, req_tags, req_sort_asc, req_sort, req_seed, req_after,
-            req_fetch_aggregations, req_highlight, req_highlight_context_size, req_embedding, req_model,
+            req_fetch_aggregations, req_highlight, req_highlight_context_size, req_embedding, req_model;
-            req_search_in_path;
 
     if (!cJSON_IsObject(json) ||
         (req_query = get_json_string(json, "query")).invalid ||
@@ -198,7 +197,6 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
         (req_index_ids = get_json_number_array(json, "indexIds")).invalid ||
         (req_mime_types = get_json_array(json, "mimeTypes")).invalid ||
         (req_highlight = get_json_bool(json, "highlight")).invalid ||
-        (req_search_in_path = get_json_bool(json, "searchInPath")).invalid ||
         (req_highlight_context_size = get_json_number(json, "highlightContextSize")).invalid ||
         (req_embedding = get_json_number_array(json, "embedding")).invalid ||
         (req_model = get_json_number(json, "model")).invalid ||
@@ -254,6 +252,7 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
     fts_search_req_t *req = malloc(sizeof(fts_search_req_t));
 
     req->sort = sort;
+    req->query = req_query.val ? strdup(req_query.val->valuestring) : NULL;
     req->path = req_path.val ? strdup(req_path.val->valuestring) : NULL;
     req->size_min = req_size_min.val ? req_size_min.val->valuedouble : 0;
     req->size_max = req_size_max.val ? req_size_max.val->valuedouble : 0;
@@ -272,16 +271,6 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
                                   ? req_highlight_context_size.val->valueint
                                   : DEFAULT_HIGHLIGHT_CONTEXT_SIZE;
     req->model = req_model.val ? req_model.val->valueint : 0;
-
-    if (req_search_in_path.val->valueint == FALSE && req_query.val) {
-        if (asprintf(&req->query, "- path : %s", req_query.val->valuestring) == -1) {
-            cJSON_Delete(json);
-            return NULL;
-        }
-    } else {
-        req->query = req_query.val ? strdup(req_query.val->valuestring) : NULL;
-    }
-
     req->embedding = req_model.val
                      ? get_float_buffer(req_embedding.val, &req->embedding_size)
                      : NULL;
@@ -420,8 +409,8 @@ void fts_get_document(struct mg_connection *nc, struct mg_http_message *hm) {
 
     sist_id_t sid;
 
-    if (hm->uri.len != 24 || !parse_sid(&sid, hm->uri.buf + 7)) {
+    if (hm->uri.len != 24 || !parse_sid(&sid, hm->uri.ptr + 7)) {
-        LOG_DEBUGF("serve.c", "Invalid /fts/d/ path: %.*s", (int) hm->uri.len, hm->uri.buf);
+        LOG_DEBUGF("serve.c", "Invalid /fts/d/ path: %.*s", (int) hm->uri.len, hm->uri.ptr);
         HTTP_REPLY_NOT_FOUND
         return;
     }

src/web/web_util.c

@@ -5,37 +5,31 @@
 void web_serve_asset_index_html(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(index_html), HTTP_CROSS_ORIGIN_HEADERS "Content-Type: text/html");
     mg_send(nc, index_html, sizeof(index_html));
-    nc->is_resp = 0;
 }
 
 void web_serve_asset_index_js(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(index_js), "Content-Type: application/javascript");
     mg_send(nc, index_js, sizeof(index_js));
-    nc->is_resp = 0;
 }
 
 void web_serve_asset_chunk_vendors_js(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(chunk_vendors_js), "Content-Type: application/javascript");
     mg_send(nc, chunk_vendors_js, sizeof(chunk_vendors_js));
-    nc->is_resp = 0;
 }
 
 void web_serve_asset_favicon_ico(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(favicon_ico), "Content-Type: image/x-icon");
     mg_send(nc, favicon_ico, sizeof(favicon_ico));
-    nc->is_resp = 0;
 }
 
 void web_serve_asset_style_css(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(index_css), "Content-Type: text/css");
     mg_send(nc, index_css, sizeof(index_css));
-    nc->is_resp = 0;
 }
 
 void web_serve_asset_chunk_vendors_css(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(chunk_vendors_css), "Content-Type: text/css");
     mg_send(nc, chunk_vendors_css, sizeof(chunk_vendors_css));
-    nc->is_resp = 0;
 }
 
 index_t *web_get_index_by_id(int index_id) {
@@ -73,7 +67,7 @@ cJSON *web_get_json_body(struct mg_http_message *hm) {
     }
 
     char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.buf, hm->body.len);
+    memcpy(body, hm->body.ptr, hm->body.len);
     *(body + hm->body.len) = '\0';
     cJSON *json = cJSON_Parse(body);
     free(body);
@@ -87,7 +81,7 @@ char *web_get_string_body(struct mg_http_message *hm) {
     }
 
     char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.buf, hm->body.len);
+    memcpy(body, hm->body.ptr, hm->body.len);
     *(body + hm->body.len) = '\0';
 
     return body;
@@ -98,7 +92,6 @@ void mg_send_json(struct mg_connection *nc, const cJSON *json) {
 
     web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
     mg_send(nc, json_str, strlen(json_str));
-    nc->is_resp = 0;
 
     free(json_str);
 }

src/web/web_util.h

@@ -16,26 +16,9 @@ database_t *web_get_database(int index_id);
 
 __always_inline
 static char *web_address_to_string(struct mg_addr *addr) {
-    static char address_to_string_buf[64];
+    static char address_to_string_buf[INET6_ADDRSTRLEN];
 
-    if (addr->is_ip6) {
+    return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf));
-        snprintf(address_to_string_buf, sizeof(address_to_string_buf),
-            "%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x",
-                addr->ip[0], addr->ip[1],
-                addr->ip[2], addr->ip[3],
-                addr->ip[4], addr->ip[5],
-                addr->ip[6], addr->ip[7],
-                addr->ip[8], addr->ip[9],
-                addr->ip[10], addr->ip[11],
-                addr->ip[12], addr->ip[13],
-                addr->ip[14], addr->ip[15]);
-    } else {
-        snprintf(address_to_string_buf, sizeof(address_to_string_buf),
-                 "%d.%d.%d.%d",
-                 addr->ip[0], addr->ip[1], addr->ip[2], addr->ip[3]);
-    }
-
-    return address_to_string_buf;
 }
 
 void web_send_headers(struct mg_connection *nc, int status_code, size_t length, char *extra_headers);

third-party/libscan/CMakeLists.txt

@@ -106,33 +106,12 @@ find_library(MUPDF_LIB NAMES liblibmupdf.a)
 find_library(CMS_LIB NAMES lcms2)
 find_library(JAS_LIB NAMES jasper)
 find_library(GUMBO_LIB NAMES gumbo)
-find_library(GOMP_LIB NAMES libgomp.a gomp
+find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/8/ /usr/lib/gcc/aarch64-linux-gnu/8/)
-        PATHS
-        /usr/lib/gcc/x86_64-linux-gnu/5/
-        /usr/lib/gcc/x86_64-linux-gnu/6/
-        /usr/lib/gcc/x86_64-linux-gnu/7/
-        /usr/lib/gcc/x86_64-linux-gnu/8/
-        /usr/lib/gcc/x86_64-linux-gnu/9/
-        /usr/lib/gcc/x86_64-linux-gnu/10/
-        /usr/lib/gcc/x86_64-linux-gnu/11/
-        /usr/lib/gcc/x86_64-linux-gnu/12/
-
-        /usr/lib/gcc/aarch64-linux-gnu/5/
-        /usr/lib/gcc/aarch64-linux-gnu/6/
-        /usr/lib/gcc/aarch64-linux-gnu/7/
-        /usr/lib/gcc/aarch64-linux-gnu/8/
-        /usr/lib/gcc/aarch64-linux-gnu/9/
-        /usr/lib/gcc/aarch64-linux-gnu/10/
-        /usr/lib/gcc/aarch64-linux-gnu/11/
-        /usr/lib/gcc/aarch64-linux-gnu/12/
-)
 find_package(Leptonica CONFIG REQUIRED)
+find_package(FFMPEG REQUIRED)
 find_package(libraw CONFIG REQUIRED)
 find_package(Freetype REQUIRED)
 
-find_package(FFMPEG REQUIRED)
-list(REMOVE_ITEM FFMPEG_LIBRARIES /usr/lib/x86_64-linux-gnu/libm.a)
-list(REMOVE_ITEM FFMPEG_LIBRARIES /usr/lib/aarch64-linux-gnu/libm.a)
 
 target_compile_options(
         scan
@@ -187,6 +166,7 @@ target_link_libraries(
         ${WPD_LIB_DIR}/libwpd-0.9.a
         ${WPD_LIB_DIR}/libwpd-stream-0.9.a
 
+        ${FREETYPE_LIB}
         ${HARFBUZZ_LIB}
         ${JBIG2DEC_LIB}
 

third-party/libscan/libscan/ebook/ebook.c

@@ -175,19 +175,9 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
     return TRUE;
 }
 
-#define IS_IGNORED_MESSAGE(message)           \
-    (                                          \
-        strstr(message, "invalid glyph index") \
-        || strstr(message, "... repeated")     \
-    )                                          \
-
 void fz_err_callback(void *user, const char *message) {
     document_t *doc = (document_t *) user;
 
-    if (IS_IGNORED_MESSAGE(message)) {
-        return;
-    }
-
     const scan_ebook_ctx_t *ctx = &thread_ctx;
     CTX_LOG_WARNINGF(doc->filepath, "FZ: %s", message);
 }
@@ -195,10 +185,6 @@ void fz_err_callback(void *user, const char *message) {
 void fz_warn_callback(void *user, const char *message) {
     document_t *doc = (document_t *) user;
 
-    if (IS_IGNORED_MESSAGE(message)) {
-        return;
-    }
-
     const scan_ebook_ctx_t *ctx = &thread_ctx;
     CTX_LOG_DEBUGF(doc->filepath, "FZ: %s", message);
 }

third-party/libscan/libscan/media/media.c

@@ -223,10 +223,14 @@ read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *d
 
 void append_tag_meta_if_not_exists(scan_media_ctx_t *ctx, document_t *doc, AVDictionaryEntry *tag, enum metakey key) {
 
-    if (meta_contains_key(doc->meta_head, key)) {
+    meta_line_t *meta = doc->meta_head;
-        CTX_LOG_DEBUGF(doc->filepath, "Ignoring duplicate tag: '%02x=%s'",
+    while (meta != NULL) {
-                       key, tag->value);
+        if (meta->key == key) {
-        return;
+            CTX_LOG_DEBUGF(doc->filepath, "Ignoring duplicate tag: '%02x=%s' and '%02x=%s'",
+                           key, meta->str_val, key, tag->value);
+            return;
+        }
+        meta = meta->next;
     }
 
     text_buffer_t tex = text_buffer_create(-1);
@@ -268,7 +272,6 @@ static void append_audio_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx
             APPEND_TAG_META(MetaAlbum);
         } else if (strcmp(key, "comment") == 0) {
             append_tag_meta_if_not_exists(ctx, doc, tag, MetaContent);
-            APPEND_TAG_META(MetaMediaComment);
         }
     }
 }
@@ -441,7 +444,7 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
         return SAVE_THUMBNAIL_FAILED;
     }
 
-    if (ctx->tesseract_lang != NULL && thumbnail_index == 0 && !meta_contains_key(doc->meta_head, MetaContent)) {
+    if (ctx->tesseract_lang != NULL && thumbnail_index == 0) {
         ocr_image(ctx, doc, decoder, frame_and_packet->frame);
     }
 
@@ -565,9 +568,6 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
         AVStream *stream = pFormatCtx->streams[video_stream];
 
         if (stream->codecpar->width <= MIN_SIZE || stream->codecpar->height <= MIN_SIZE) {
-            CTX_LOG_DEBUGF(doc->filepath,
-                           "Will not generate thumbnail because image is too small: %dx%d",
-                           stream->codecpar->width, stream->codecpar->width);
             avformat_close_input(&pFormatCtx);
             avformat_free_context(pFormatCtx);
             return;

third-party/libscan/libscan/ocr/ocr.h

@@ -5,8 +5,8 @@
 #include <tesseract/capi.h>
 
 #define MIN_OCR_WIDTH 350
-#define MIN_OCR_HEIGHT 33
+#define MIN_OCR_HEIGHT 100
-#define MIN_OCR_LEN 3
+#define MIN_OCR_LEN 10
 
 #define OCR_IS_VALID_BPP(d)                                                    \
   ((d) == 1 || (d) == 2 || (d) == 4 || (d) == 8 || (d) == 16 || (d) == 24 ||   \
@@ -28,13 +28,7 @@ ocr_extract_text(const char *tesseract_path, const char *tesseract_lang,
     TessBaseAPI *api = TessBaseAPICreate();
     TessBaseAPIInit3(api, tesseract_path, tesseract_lang);
 
-    // https://github.com/simon987/sist2/issues/443
+    TessBaseAPISetPageSegMode(api, PSM_AUTO_OSD);
-    if (strstr(tesseract_lang, "chi") != NULL) {
-        TessBaseAPISetVariable(api, "preserve_interword_spaces", "1");
-    }
-
-    // TODO: add this as param?
-//    TessBaseAPISetPageSegMode(api, PSM_AUTO_OSD);
 
     TessBaseAPISetImage(api, img_buf, img_w, img_h, img_bpp, img_stride);
     TessBaseAPISetSourceResolution(api, img_xres);

third-party/libscan/libscan/scan.h

@@ -63,7 +63,6 @@ enum metakey {
     MetaAlbumArtist,
     MetaGenre,
     MetaTitle,
-    MetaMediaComment,
     MetaFontName,
     MetaExifMake,
     MetaExifDescription,
@@ -172,8 +171,6 @@ typedef struct {
     char filepath[PATH_MAX * 2 + 1];
 } parse_job_t;
 
-#define IS_SUB_JOB(job) ((job)->parent[0] != '\0')
-
 
 #include "util.h"
 

third-party/libscan/libscan/util.h

@@ -392,18 +392,4 @@ static parse_job_t *create_parse_job(const char *filepath, int mtime, size_t st_
     return job;
 }
 
-
-static int meta_contains_key (meta_line_t *meta_head, enum metakey key) {
-
-    meta_line_t *meta = meta_head;
-    while (meta != NULL) {
-        if (meta->key == key) {
-            return TRUE;
-        }
-        meta = meta->next;
-    }
-
-    return FALSE;
-}
-
 #endif