Update compose file to avoid confusion. Fixes #490

Fix #492
Fix magic database problem
2025-12-12 15:08:53 +00:00 · 2025-01-23 21:45:01 -05:00 · 2025-01-23 21:40:37 -05:00 · 2025-01-23 21:40:27 -05:00 · 2025-01-23 19:10:17 -05:00 · 2025-01-23 19:10:02 -05:00
27 changed files with 168 additions and 116 deletions
--- a/.drone.yml
+++ b/.drone.yml
@@ -7,11 +7,36 @@ platform:
  arch: amd64

 steps:
+  - name: submodules
+    image: alpine/git
+    commands:
+      - git submodule update --init --recursive
+  - name: docker
+    image: plugins/docker
+    depends_on:
+      - submodules
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: sist2app/sist2
+      context: ./
+      dockerfile: ./Dockerfile
+      auto_tag: true
+      auto_tag_suffix: x64-linux
+      when:
+        event:
+          - tag
  - name: build
-    image: simon987/sist2-build
+    image: sist2app/sist2-build
+    depends_on:
+    - submodules
    commands:
      - ./scripts/build.sh
  - name: scp files
+    depends_on:
+      - build
    image: appleboy/drone-scp
    settings:
      host:
@@ -22,26 +47,11 @@ steps:
        from_secret: SSH_USER
      key:
        from_secret: SSH_KEY
-      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
      source:
        - ./VERSION
        - ./sist2-x64-linux
        - ./sist2-x64-linux-debug
-  - name: docker
-    image: plugins/docker
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: simon987/sist2
-      context: ./
-      dockerfile: ./Dockerfile
-      auto_tag: true
-      auto_tag_suffix: x64-linux
-      when:
-        event:
-          - tag

 ---
 kind: pipeline
@@ -52,11 +62,36 @@ platform:
  arch: arm64

 steps:
+  - name: submodules
+    image: alpine/git
+    commands:
+      - git submodule update --init --recursive
+  - name: docker
+    image: plugins/docker
+    depends_on:
+      - submodules
+    settings:
+      username:
+        from_secret: DOCKER_USER
+      password:
+        from_secret: DOCKER_PASSWORD
+      repo: sist2app/sist2
+      context: ./
+      dockerfile: ./Dockerfile.arm64
+      auto_tag: true
+      auto_tag_suffix: arm64-linux
+      when:
+        event:
+          - tag
  - name: build
-    image: simon987/sist2-build-arm64
+    image: sist2app/sist2-build-arm64
+    depends_on:
+      - submodules
    commands:
      - ./scripts/build_arm64.sh
  - name: scp files
+    depends_on:
+      - build
    image: appleboy/drone-scp
    settings:
      host:
@@ -67,22 +102,7 @@ steps:
        from_secret: SSH_USER
      key:
        from_secret: SSH_KEY
-      target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
+      target: ~/files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/
      source:
        - ./sist2-arm64-linux
        - ./sist2-arm64-linux-debug
-  - name: docker
-    image: plugins/docker
-    settings:
-      username:
-        from_secret: DOCKER_USER
-      password:
-        from_secret: DOCKER_PASSWORD
-      repo: simon987/sist2
-      context: ./
-      dockerfile: ./Dockerfile.arm64
-      auto_tag: true
-      auto_tag_suffix: arm64-linux
-      when:
-        event:
-          - tag
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -147,6 +147,7 @@ add_dependencies(
 target_link_libraries(
        sist2

+        m
        z
        argparse
        unofficial::mongoose::mongoose
--- a/3
+++ b/3
@@ -1,5 +1,4 @@
-FROM simon987/sist2-build as build
-MAINTAINER simon987 <me@simon987.net>
+FROM sist2app/sist2-build as build

 WORKDIR /build/

--- a/Dockerfile.arm64
+++ b/Dockerfile.arm64
@@ -1,5 +1,4 @@
-FROM simon987/sist2-build-arm64 as build
-MAINTAINER simon987 <me@simon987.net>
+FROM sist2app/sist2-build-arm64 as build

 WORKDIR /build/

--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@ services:
    restart: unless-stopped
    volumes:
      - /data/sist2-admin-data/:/sist2-admin/
-      - /:/host
+      - /<path to index>/:/host
    ports:
      - 4090:4090
      # NOTE: Don't expose this port publicly!
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -18,7 +18,7 @@ services:
    container_name: sist2-admin
    volumes:
      - /data/sist2-admin-data/:/sist2-admin/
-      - /:/host
+      - /<path to index>/:/host
    ports:
      - 4090:4090
      # NOTE: Don't export this port publicly!
--- a/docs/USAGE.md
+++ b/docs/USAGE.md
@@ -189,7 +189,7 @@ Since v3.2.0, User scripts can be used to generate _embeddings_ (vector of float

 In theory, embeddings can be created for any type of documents (image, text, audio etc.).

-For example, the [clip](https://github.com/simon987/sist2-script-clip) User Script, generates 512-d embeddings of images
+For example, the [clip](https://github.com/sist2app/sist2-script-clip) User Script, generates 512-d embeddings of images
 (videos are also supported using the thumbnails generated by sist2). When the user enters a query in the "Embeddings Search"
 textbox, the query's embedding is generated in their browser, leveraging the ONNX web runtime.

--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -2,8 +2,6 @@

 VCPKG_ROOT="/vcpkg"

-git submodule update --init --recursive
-
 (
  cd sist2-vue/
  npm install
--- a/scripts/magic_static.py
+++ b/scripts/magic_static.py
@@ -1,8 +1,16 @@

-try:
-    with open("/usr/lib/file/magic.mgc", "rb") as f:
-        data = f.read()
-except:
-    data = bytes([])
+MAGIC_PATHS = [
+    "/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
+    "/work/vcpkg/installed/x64-linux/share/libmagic/misc/magic.mgc",
+    "/usr/lib/file/magic.mgc"
+]
+
+for path in MAGIC_PATHS:
+    try:
+        with open(path, "rb") as f:
+            data = f.read()
+        break
+    except:
+        continue

 print("char magic_database_buffer[%d] = {%s};" % (len(data), ",".join(str(int(b)) for b in data)))
--- a/sist2-admin/frontend/package-lock.json
+++ b/sist2-admin/frontend/package-lock.json
@@ -10807,16 +10807,16 @@
      }
    },
    "node_modules/webpack-dev-server/node_modules/ws": {
-      "version": "8.17.1",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
-      "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
+      "version": "8.11.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
+      "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
      "dev": true,
      "engines": {
        "node": ">=10.0.0"
      },
      "peerDependencies": {
        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
+        "utf-8-validate": "^5.0.2"
      },
      "peerDependenciesMeta": {
        "bufferutil": {
@@ -10946,9 +10946,9 @@
      "license": "ISC"
    },
    "node_modules/ws": {
-      "version": "7.5.10",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz",
-      "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==",
+      "version": "7.5.9",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
+      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
      "dev": true,
      "engines": {
        "node": ">=8.3.0"
@@ -18927,9 +18927,9 @@
          }
        },
        "ws": {
-          "version": "8.17.1",
-          "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
-          "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
+          "version": "8.11.0",
+          "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
+          "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
          "dev": true,
          "requires": {}
        }
@@ -19022,9 +19022,9 @@
      "dev": true
    },
    "ws": {
-      "version": "7.5.10",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz",
-      "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==",
+      "version": "7.5.9",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
+      "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
      "dev": true,
      "requires": {}
    },
--- a/sist2-admin/frontend/src/App.vue
+++ b/sist2-admin/frontend/src/App.vue
@@ -4,7 +4,7 @@
        <b-container class="pt-4">
            <b-alert show dismissible variant="info">
                This is a beta version of sist2-admin. Please submit bug reports, usability issues and feature requests
-                to the <a href="https://github.com/simon987/sist2/issues/new/choose" target="_blank">issue tracker on
+                to the <a href="https://github.com/sist2app/sist2/issues/new/choose" target="_blank">issue tracker on
                Github</a>. Thank you!
            </b-alert>
            <router-view v-if="$store.state.sist2AdminInfo"/>
--- a/sist2-admin/frontend/yarn.lock
+++ b/sist2-admin/frontend/yarn.lock
@@ -4924,16 +4924,16 @@ wrappy@1:
  resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz"

 ws@^7.3.1:
-  version "7.5.10"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9"
+  version "7.5.9"
+  resolved "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz"

 ws@^8.4.2:
-  version "8.17.1"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b"
+  version "8.11.0"
+  resolved "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz"

 ws@~8.2.3:
  version "8.2.3"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-8.2.3.tgz#63a56456db1b04367d0b721a0b80cae6d8becbba"
+  resolved "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz"

 xmlhttprequest-ssl@~2.0.0:
  version "2.0.0"
--- a/sist2-admin/requirements.txt
+++ b/sist2-admin/requirements.txt
@@ -4,4 +4,4 @@ uvicorn
 websockets
 pycron
 GitPython
-git+https://github.com/simon987/sist2-python.git
+git+https://github.com/sist2app/sist2-python.git
--- a/sist2-admin/sist2_admin/script.py
+++ b/sist2-admin/sist2_admin/script.py
@@ -96,7 +96,7 @@ SCRIPT_TEMPLATES = {
    "CLIP - Generate embeddings to predict the most relevant image based on the text prompt": lambda name: UserScript(
        name=name,
        type=ScriptType.GIT,
-        git_repository="https://github.com/simon987/sist2-script-clip",
+        git_repository="https://github.com/sist2app/sist2-script-clip",
        extra_args="--num-tags=1 --tags-file=general.txt --color=#dcd7ff"
    ),
    "Whisper - Speech to text with OpenAI Whisper": lambda name: UserScript(
--- a/sist2-vue/src/components/NavBar.vue
+++ b/sist2-vue/src/components/NavBar.vue
@@ -9,7 +9,7 @@

    <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
      v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
-        href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch"
+        href="https://github.com/sist2app/sist2/blob/master/docs/USAGE.md#elasticsearch"
        target="_blank">legacyES</a></span><span v-if="$store.state.uiSqliteMode">-SQLite</span>
    </span>

--- a/sist2-vue/src/i18n/messages.js
+++ b/sist2-vue/src/i18n/messages.js
@@ -138,7 +138,7 @@ export default {
        },
        debug: "Debug information",
        debugDescription: "Information useful for debugging. If you encounter bugs or have suggestions for" +
-            " new features, please submit a new issue <a href='https://github.com/simon987/sist2/issues/new/choose'>here</a>.",
+            " new features, please submit a new issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>here</a>.",
        tagline: "Tagline",
        toast: {
            esConnErrTitle: "Elasticsearch connection error",
@@ -318,7 +318,7 @@ export default {
        },
        debug: "Debug Informationen",
        debugDescription: "Informationen für das Debugging. Wenn du Bugs gefunden oder Anregungen für " +
-            "neue Features hast, poste sie bitte <a href='https://github.com/simon987/sist2/issues/new/choose'>hier</a>.",
+            "neue Features hast, poste sie bitte <a href='https://github.com/sist2app/sist2/issues/new/choose'>hier</a>.",
        tagline: "Tagline",
        toast: {
            esConnErrTitle: "Elasticsearch Verbindungsfehler",
@@ -494,7 +494,7 @@ export default {
        debug: "Information de débogage",
        debugDescription: "Informations utiles pour le débogage\n" +
            "Si vous rencontrez des bogues ou si vous avez des suggestions pour de nouvelles fonctionnalités," +
-            " veuillez soumettre un nouvel Issue <a href='https://github.com/simon987/sist2/issues/new/choose'>ici</a>.",
+            " veuillez soumettre un nouvel Issue <a href='https://github.com/sist2app/sist2/issues/new/choose'>ici</a>.",
        tagline: "Tagline",
        toast: {
            esConnErrTitle: "Erreur de connexion Elasticsearch",
@@ -668,7 +668,7 @@ export default {
        },
        debug: "调试信息",
        debugDescription: "对调试除错有用的信息。 若您遇到bug或者想建议新功能，请提交新Issue到" +
-            "<a href='https://github.com/simon987/sist2/issues/new/choose'>这里</a>.",
+            "<a href='https://github.com/sist2app/sist2/issues/new/choose'>这里</a>.",
        tagline: "标签栏",
        toast: {
            esConnErrTitle: "Elasticsearch连接错误",
@@ -846,7 +846,7 @@ export default {
        },
        debug: "Informacje dla programistów",
        debugDescription: "Informacje przydatne do znajdowania błędów w oprogramowaniu. Jeśli napotkasz błąd lub masz" +
-            " propozycje zmian, zgłoś to proszę <a href='https://github.com/simon987/sist2/issues/new/choose'>tutaj</a>.",
+            " propozycje zmian, zgłoś to proszę <a href='https://github.com/sist2app/sist2/issues/new/choose'>tutaj</a>.",
        tagline: "Slogan",
        toast: {
            esConnErrTitle: "Problem z połączeniem z Elasticsearch",
--- a/sist2-vue/src/store/index.js
+++ b/sist2-vue/src/store/index.js
@@ -58,7 +58,7 @@ export default new Vuex.Store({
        optVidPreviewInterval: 700,
        optSimpleLightbox: true,
        optShowTagPickerFilter: true,
-        optMlRepositories: "https://raw.githubusercontent.com/simon987/sist2-ner-models/main/repo.json",
+        optMlRepositories: "https://raw.githubusercontent.com/sist2app/sist2-ner-models/main/repo.json",
        optAutoAnalyze: false,
        optMlDefaultModel: null,

--- a/src/cli.c
+++ b/src/cli.c
@@ -25,6 +25,7 @@ const char *TESS_DATAPATHS[] = {
        "/usr/share/tessdata/",
        "/usr/share/tesseract-ocr/tessdata/",
        "/usr/share/tesseract-ocr/4.00/tessdata/",
+        "/usr/share/tesseract-ocr/5/tessdata/",
        "./",
        NULL
 };
--- a/src/database/database.c
+++ b/src/database/database.c
@@ -478,8 +478,7 @@ index_descriptor_t *database_read_index_descriptor(database_t *db) {
 database_iterator_t *database_create_delete_list_iterator(database_t *db) {

    sqlite3_stmt *stmt;
-    sqlite3_prepare_v2(db->db, "SELECT doc.id FROM delete_list "
-                               "INNER JOIN document doc ON doc.ROWID = delete_list.id;", -1, &stmt, NULL);
+    sqlite3_prepare_v2(db->db, "SELECT id FROM delete_list", -1, &stmt, NULL);

    database_iterator_t *iter = malloc(sizeof(database_iterator_t));

--- a/src/parsing/parse.c
+++ b/src/parsing/parse.c
@@ -161,7 +161,8 @@ void parse(parse_job_t *job) {
        return;
    }

-    if (database_mark_document(ProcData.index_db, doc->filepath + ScanCtx.index.desc.root_len, doc->mtime)) {
+    int document_exists = database_mark_document(ProcData.index_db, doc->filepath + ScanCtx.index.desc.root_len, doc->mtime);
+    if (document_exists) {
        CLOSE_FILE(job->vfile)
        free(doc);
        return;
--- a/src/sist.h
+++ b/src/sist.h
@@ -55,13 +55,13 @@
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
 static const int VersionMinor = 4;
-static const int VersionPatch = 2;
+static const int VersionPatch = 3;

 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown
 #endif

-#define EXPECTED_MONGOOSE_VERSION "7.13"
+#define EXPECTED_MONGOOSE_VERSION "7.16"

 #define Q(x) #x
 #define QUOTE(x) Q(x)
--- a/src/web/serve.c
+++ b/src/web/serve.c
@@ -50,13 +50,13 @@ void get_embedding(struct mg_connection *nc, struct mg_http_message *hm) {

    sist_id_t sid;

-    if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.ptr + 3)) {
-        LOG_DEBUGF("serve.c", "Invalid embedding path: %.*s", (int) hm->uri.len, hm->uri.ptr);
+    if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.buf + 3)) {
+        LOG_DEBUGF("serve.c", "Invalid embedding path: %.*s", (int) hm->uri.len, hm->uri.buf);
        HTTP_REPLY_NOT_FOUND
        return;
    }

-    int model_id = (int) strtol(hm->uri.ptr + SIST_SID_LEN + 3, NULL, 10);
+    int model_id = (int) strtol(hm->uri.buf + SIST_SID_LEN + 3, NULL, 10);

    database_t *db = web_get_database(sid.index_id);
    if (db == NULL) {
@@ -86,11 +86,11 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
    char index_id_str[9];
    char arg_stat_type[5];

-    memcpy(index_id_str, hm->uri.ptr + 3, 8);
+    memcpy(index_id_str, hm->uri.buf + 3, 8);
    *(index_id_str + 8) = '\0';
    int index_id = (int) strtol(index_id_str, NULL, 16);

-    memcpy(arg_stat_type, hm->uri.ptr + 3 + 9, 4);
+    memcpy(arg_stat_type, hm->uri.buf + 3 + 9, 4);
    *(arg_stat_type + sizeof(arg_stat_type) - 1) = '\0';

    database_stat_type_d stat_type = database_get_stat_type_by_mnemonic(arg_stat_type);
@@ -135,19 +135,19 @@ void serve_chunk_vendors_js(struct mg_connection *nc, struct mg_http_message *hm
    }
 }

-void serve_favicon_ico(struct mg_connection *nc, struct mg_http_message *hm) {
+void serve_favicon_ico(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
    web_serve_asset_favicon_ico(nc);
 }

-void serve_style_css(struct mg_connection *nc, struct mg_http_message *hm) {
+void serve_style_css(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
    web_serve_asset_style_css(nc);
 }

-void serve_chunk_vendors_css(struct mg_connection *nc, struct mg_http_message *hm) {
+void serve_chunk_vendors_css(struct mg_connection *nc, UNUSED(struct mg_http_message *hm)) {
    web_serve_asset_chunk_vendors_css(nc);
 }

-void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, int index_id,
+void serve_thumbnail(struct mg_connection *nc, UNUSED(struct mg_http_message *hm), int index_id,
                     int doc_id, int arg_num) {

    database_t *db = web_get_database(index_id);
@@ -179,13 +179,13 @@ void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, int i
 void thumbnail_with_num(struct mg_connection *nc, struct mg_http_message *hm) {
    sist_id_t sid;

-    if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.ptr + 3)) {
-        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr);
+    if (hm->uri.len != SIST_SID_LEN + 2 + 4 || !parse_sid(&sid, hm->uri.buf + 3)) {
+        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.buf);
        HTTP_REPLY_NOT_FOUND
        return;
    }

-    int num = (int) strtol(hm->uri.ptr + SIST_SID_LEN + 3, NULL, 10);
+    int num = (int) strtol(hm->uri.buf + SIST_SID_LEN + 3, NULL, 10);

    serve_thumbnail(nc, hm, sid.index_id, sid.doc_id, num);
 }
@@ -193,8 +193,8 @@ void thumbnail_with_num(struct mg_connection *nc, struct mg_http_message *hm) {
 void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) {
    sist_id_t sid;

-    if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.ptr + 3)) {
-        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.ptr);
+    if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.buf + 3)) {
+        LOG_DEBUGF("serve.c", "Invalid thumbnail path: %.*s", (int) hm->uri.len, hm->uri.buf);
        HTTP_REPLY_NOT_FOUND
        return;
    }
@@ -210,7 +210,7 @@ void search(struct mg_connection *nc, struct mg_http_message *hm) {
    }

    char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.ptr, hm->body.len);
+    memcpy(body, hm->body.buf, hm->body.len);
    *(body + hm->body.len) = '\0';

    char url[4096];
@@ -416,8 +416,8 @@ cJSON *get_root_document_by_id(int index_id, int doc_id) {
 void file(struct mg_connection *nc, struct mg_http_message *hm) {
    sist_id_t sid;

-    if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.ptr + 3)) {
-        LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.ptr);
+    if (hm->uri.len != 20 || !parse_sid(&sid, hm->uri.buf + 3)) {
+        LOG_DEBUGF("serve.c", "Invalid file path: %.*s", (int) hm->uri.len, hm->uri.buf);
        HTTP_REPLY_NOT_FOUND
        return;
    }
@@ -528,14 +528,14 @@ subreq_ctx_t *elastic_write_tag(const char *sid, const tag_req_t *req) {

 void tag(struct mg_connection *nc, struct mg_http_message *hm) {
    sist_id_t sid;
-    if (hm->uri.len != 22 || !parse_sid(&sid, hm->uri.ptr + 5)) {
-        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.ptr);
+    if (hm->uri.len != 22 || !parse_sid(&sid, hm->uri.buf + 5)) {
+        LOG_DEBUGF("serve.c", "Invalid tag path: %.*s", (int) hm->uri.len, hm->uri.buf);
        HTTP_REPLY_NOT_FOUND
        return;
    }

    char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.ptr, hm->body.len);
+    memcpy(body, hm->body.buf, hm->body.len);
    *(body + hm->body.len) = '\0';
    cJSON *json = cJSON_Parse(body);
    free(body);
@@ -612,7 +612,7 @@ int check_auth0(struct mg_http_message *hm) {
    }

    token_str = malloc(token.len + 1);
-    strncpy(token_str, token.ptr, token.len);
+    strncpy(token_str, token.buf, token.len);
    *(token_str + token.len) = '\0';

    int res = auth0_verify_jwt(
@@ -642,13 +642,15 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data) {
        }

        char uri[256];
-        memcpy(uri, hm->uri.ptr, hm->uri.len);
+        memcpy(uri, hm->uri.buf, hm->uri.len);
        *(uri + hm->uri.len) = '\0';
        LOG_DEBUGF("serve.c", "<%s> GET %s",
                   web_address_to_string(&(nc->rem)),
                   uri
        );

+#define mg_http_match_uri(hm, pattern) mg_match((hm)->uri, mg_str(pattern), NULL)
+
        if (mg_http_match_uri(hm, "/")) {
            serve_index_html(nc, hm);
            return;
--- a/src/web/web_fts.c
+++ b/src/web/web_fts.c
@@ -420,8 +420,8 @@ void fts_get_document(struct mg_connection *nc, struct mg_http_message *hm) {

    sist_id_t sid;

-    if (hm->uri.len != 24 || !parse_sid(&sid, hm->uri.ptr + 7)) {
-        LOG_DEBUGF("serve.c", "Invalid /fts/d/ path: %.*s", (int) hm->uri.len, hm->uri.ptr);
+    if (hm->uri.len != 24 || !parse_sid(&sid, hm->uri.buf + 7)) {
+        LOG_DEBUGF("serve.c", "Invalid /fts/d/ path: %.*s", (int) hm->uri.len, hm->uri.buf);
        HTTP_REPLY_NOT_FOUND
        return;
    }
--- a/src/web/web_util.c
+++ b/src/web/web_util.c
@@ -73,7 +73,7 @@ cJSON *web_get_json_body(struct mg_http_message *hm) {
    }

    char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.ptr, hm->body.len);
+    memcpy(body, hm->body.buf, hm->body.len);
    *(body + hm->body.len) = '\0';
    cJSON *json = cJSON_Parse(body);
    free(body);
@@ -87,7 +87,7 @@ char *web_get_string_body(struct mg_http_message *hm) {
    }

    char *body = malloc(hm->body.len + 1);
-    memcpy(body, hm->body.ptr, hm->body.len);
+    memcpy(body, hm->body.buf, hm->body.len);
    *(body + hm->body.len) = '\0';

    return body;
--- a/third-party/libscan/libscan/ebook/ebook.c
+++ b/third-party/libscan/libscan/ebook/ebook.c
@@ -175,9 +175,19 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
    return TRUE;
 }

+#define IS_IGNORED_MESSAGE(message)           \
+    (                                          \
+        strstr(message, "invalid glyph index") \
+        || strstr(message, "... repeated")     \
+    )                                          \
+
 void fz_err_callback(void *user, const char *message) {
    document_t *doc = (document_t *) user;

+    if (IS_IGNORED_MESSAGE(message)) {
+        return;
+    }
+
    const scan_ebook_ctx_t *ctx = &thread_ctx;
    CTX_LOG_WARNINGF(doc->filepath, "FZ: %s", message);
 }
@@ -185,6 +195,10 @@ void fz_err_callback(void *user, const char *message) {
 void fz_warn_callback(void *user, const char *message) {
    document_t *doc = (document_t *) user;

+    if (IS_IGNORED_MESSAGE(message)) {
+        return;
+    }
+
    const scan_ebook_ctx_t *ctx = &thread_ctx;
    CTX_LOG_DEBUGF(doc->filepath, "FZ: %s", message);
 }
--- a/third-party/libscan/libscan/media/media.c
+++ b/third-party/libscan/libscan/media/media.c
@@ -223,14 +223,10 @@ read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *d

 void append_tag_meta_if_not_exists(scan_media_ctx_t *ctx, document_t *doc, AVDictionaryEntry *tag, enum metakey key) {

-    meta_line_t *meta = doc->meta_head;
-    while (meta != NULL) {
-        if (meta->key == key) {
-            CTX_LOG_DEBUGF(doc->filepath, "Ignoring duplicate tag: '%02x=%s' and '%02x=%s'",
-                           key, meta->str_val, key, tag->value);
-            return;
-        }
-        meta = meta->next;
+    if (meta_contains_key(doc->meta_head, key)) {
+        CTX_LOG_DEBUGF(doc->filepath, "Ignoring duplicate tag: '%02x=%s'",
+                       key, tag->value);
+        return;
    }

    text_buffer_t tex = text_buffer_create(-1);
@@ -445,7 +441,7 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
        return SAVE_THUMBNAIL_FAILED;
    }

-    if (ctx->tesseract_lang != NULL && thumbnail_index == 0) {
+    if (ctx->tesseract_lang != NULL && thumbnail_index == 0 && !meta_contains_key(doc->meta_head, MetaContent)) {
        ocr_image(ctx, doc, decoder, frame_and_packet->frame);
    }

--- a/third-party/libscan/libscan/util.h
+++ b/third-party/libscan/libscan/util.h
@@ -392,4 +392,18 @@ static parse_job_t *create_parse_job(const char *filepath, int mtime, size_t st_
    return job;
 }

+
+static int meta_contains_key (meta_line_t *meta_head, enum metakey key) {
+
+    meta_line_t *meta = meta_head;
+    while (meta != NULL) {
+        if (meta->key == key) {
+            return TRUE;
+        }
+        meta = meta->next;
+    }
+
+    return FALSE;
+}
+
 #endif
Author	SHA1	Message	Date
Shy	d44679131b	Update compose file to avoid confusion. Fixes #490	2025-01-23 21:45:01 -05:00
Shy	4dd5e70406	Fix #492	2025-01-23 21:40:37 -05:00
Shy	5a82581992	Fix magic database problem	2025-01-23 21:40:27 -05:00
Shy	0dc18a56c0	Fix #509	2025-01-23 19:10:17 -05:00
Shy	258b2e31e6	Version bump	2025-01-23 19:10:02 -05:00
Shy	c726074029	Update tessdata paths	2025-01-23 19:09:54 -05:00
Shy	7873ef003d	Fix CI build attempt 6	2025-01-22 22:16:42 -05:00
Shy	d41266e136	Fix CI build attempt 5	2025-01-22 22:15:37 -05:00
Shy	0e946092eb	Fix CI build attempt 4	2025-01-22 21:58:55 -05:00
Shy	95b19e2e67	Fix CI build attempt 3	2025-01-22 21:55:09 -05:00
Shy	bd98eb2522	Fix CI build attempt 2	2025-01-22 21:51:59 -05:00
Shy	3d99add79e	Fix CI build	2025-01-22 21:43:23 -05:00
Shy	2d6553d5d2	Update magic gen script	2025-01-22 21:39:23 -05:00
Shy	7d67354b96	Update CI build config	2025-01-22 21:32:54 -05:00
Shy	1b77daef16	Update repository URLs	2025-01-22 21:27:27 -05:00
Shy	d7038be35b	Fix #506	2025-01-16 18:32:33 -05:00
Shy	c1573a803e	Update third-party dependencies	2025-01-12 11:55:14 -05:00