Bump express from 4.18.2 to 4.19.2 in /sist2-vue

Bumps [express](https://github.com/expressjs/express) from 4.18.2 to 4.19.2.
- [Release notes](https://github.com/expressjs/express/releases)
- [Changelog](https://github.com/expressjs/express/blob/master/History.md)
- [Commits](https://github.com/expressjs/express/compare/4.18.2...4.19.2)

---
updated-dependencies:
- dependency-name: express
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

README.md

@@ -44,20 +44,28 @@ services:
   elasticsearch:
     image: elasticsearch:7.17.9
     restart: unless-stopped
+    volumes:
+      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
+      - /data/sist2-es-data/:/usr/share/elasticsearch/data
     environment:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "PUID=1000"
+      - "PGID=1000"
   sist2-admin:
-    image: simon987/sist2:3.3.4-x64-linux
+    image: simon987/sist2:3.4.0-x64-linux
     restart: unless-stopped
     volumes:
-      - ./sist2-admin-data/:/sist2-admin/
+      - /data/sist2-admin-data/:/sist2-admin/
       - /:/host
     ports:
-      - 4090:4090 # sist2
-      - 8080:8080 # sist2-admin
+      - 4090:4090
+      # NOTE: Don't expose this port publicly!
+      - 8080:8080
     working_dir: /root/sist2-admin/
-    entrypoint: python3 /root/sist2-admin/sist2_admin/app.py
+    entrypoint: python3
+    command:
+      - /root/sist2-admin/sist2_admin/app.py
 ```
 
 Navigate to http://localhost:8080/ to configure sist2-admin.
@@ -208,7 +216,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 3. Install vcpkg dependencies
 
     ```bash
-    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib]
+    vcpkg install openblas curl[core,openssl] sqlite3[core,fts5,json1] cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf[ocr] gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample,webp,opus,mp3lame,vpx,zlib]
     ```
 
 4. Build

docker-compose.yml

@@ -4,15 +4,20 @@ services:
   elasticsearch:
     image: elasticsearch:7.17.9
     container_name: sist2-es
+    volumes:
+      # This directory must have 1000:1000 permissions (or update PUID & PGID below)
+      - /data/sist2-es-data/:/usr/share/elasticsearch/data
     environment:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "PUID=1000"
+      - "PGID=1000"
   sist2-admin:
     build:
       context: .
     container_name: sist2-admin
     volumes:
-      - /mnt/array/sist2-admin-data/:/sist2-admin/
+      - /data/sist2-admin-data/:/sist2-admin/
       - /:/host
     ports:
       - 4090:4090

sist2-admin/frontend/src/components/WebOptions.vue

@@ -17,6 +17,10 @@
 
       <label>{{ $t("webOptions.tagAuth") }}</label>
       <b-form-input v-model="options.tag_auth" @change="update()" :disabled="Boolean(options.auth)"></b-form-input>
+
+      <b-form-checkbox v-model="options.verbose" @change="update()">
+        {{$t("webOptions.verbose")}}
+      </b-form-checkbox>
     </b-card>
 
     <br>

sist2-admin/frontend/src/i18n/messages.js

@@ -81,6 +81,7 @@ export default {
             auth0Domain: "Auth0 domain",
             auth0ClientId: "Auth0 client ID",
             auth0PublicKey: "Auth0 public key",
+            verbose: "Verbose logs"
         },
         backendOptions: {
             title: "Search backend options",

sist2-admin/sist2_admin/jobs.py

@@ -204,7 +204,7 @@ class Sist2IndexTask(Sist2Task):
         self.job.previous_index_path = self.job.index_path
         db["jobs"][self.job.name] = self.job
 
-        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}"}))
+        self._logger.info(json.dumps({"sist2-admin": f"Sist2Scan task finished {return_code=}, {duration=}, {ok=}"}))
 
         logger.info(f"Completed {self.display_name} ({return_code=})")
 

sist2-admin/sist2_admin/sist2.py

@@ -2,10 +2,11 @@ import datetime
 import json
 import logging
 import os.path
+import sys
 from datetime import datetime
 from enum import Enum
 from io import TextIOWrapper
-from logging import FileHandler
+from logging import FileHandler, StreamHandler
 from subprocess import Popen, PIPE
 from tempfile import NamedTemporaryFile
 from threading import Thread
@@ -200,6 +201,7 @@ class WebOptions(BaseModel):
     auth0_client_id: str = None
     auth0_public_key: str = None
     auth0_public_key_file: str = None
+    verbose: bool = False
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -231,6 +233,8 @@ class WebOptions(BaseModel):
             args.append(f"--tag-auth={self.tag_auth}")
         if self.dev:
             args.append(f"--dev")
+        if self.verbose:
+            args.append(f"--very-verbose")
 
         args.extend(self.indices)
 
@@ -338,6 +342,7 @@ class Sist2:
 
         web_logger = logging.Logger(name=f"sist2-frontend-{name}")
         web_logger.addHandler(FileHandler(os.path.join(LOG_FOLDER, f"frontend-{name}.log")))
+        web_logger.addHandler(StreamHandler())
 
         def logs_cb(message):
             web_logger.info(json.dumps(message))

sist2-vue/src/Sist2Api.js

@@ -144,8 +144,6 @@ class Sist2Api {
     }
 
     _getIndexRoot(indexId) {
-        console.log(indexId)
-        console.log(this.sist2Info.indices.find(idx => idx.id === indexId))
         return this.sist2Info.indices.find(idx => idx.id === indexId).root;
     }
 

sist2-vue/src/components/Lightbox.vue

@@ -77,6 +77,17 @@ export default {
         return listener(e);
       }
     };
+
+    document.onpointerdown = (e) => {
+      if (e.button === 2) {
+
+        const isLightboxOpen = this.$refs.lightbox === undefined || this.$refs.lightbox.$el.tagName === undefined;
+        if (isLightboxOpen) {
+          e.preventDefault();
+          return false;
+        }
+      }
+    }
   },
   methods: {
     keyDownListener(e) {

sist2-vue/watch.sh

@@ -1,3 +1,5 @@
 #!/usr/bin/env bash
 
+export NODE_OPTIONS=--openssl-legacy-provider
+
 ./node_modules/@vue/cli-service/bin/vue-cli-service.js build --watch

src/database/database_fts.c

@@ -102,7 +102,9 @@ void database_fts_index(database_t *db) {
             db->db, "DELETE FROM fts.mime_index;", NULL, NULL, NULL));
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db, "INSERT INTO fts.mime_index (index_id, mime, count) "
-                    "SELECT index_id, mime, count(*) FROM fts.document_index GROUP BY index_id, mime",
+                    "SELECT index_id, mime, count(*) FROM fts.document_index "
+                    "WHERE mime IS NOT NULL "
+                    "GROUP BY index_id, mime",
             NULL, NULL, NULL));
 
     LOG_DEBUG("database_fts.c", "Generating path index");

src/index/web.c

@@ -206,7 +206,7 @@ response_t *web_put(const char *url, const char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
     curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
-    curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
+    curl_easy_setopt(curl, CURLOPT_SHARE, 0);
     curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4);
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);

src/sist.h

@@ -51,17 +51,17 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.4.1"
+#define VERSION "3.4.2"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
 static const int VersionMinor = 4;
-static const int VersionPatch = 1;
+static const int VersionPatch = 2;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown
 #endif
 
-#define EXPECTED_MONGOOSE_VERSION "7.7"
+#define EXPECTED_MONGOOSE_VERSION "7.13"
 
 #define Q(x) #x
 #define QUOTE(x) Q(x)

src/web/serve.c

@@ -108,7 +108,6 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
 
     cJSON *json = database_get_stats(db, stat_type);
     mg_send_json(nc, json);
-
     cJSON_Delete(json);
 }
 
@@ -169,6 +168,7 @@ void serve_thumbnail(struct mg_connection *nc, struct mg_http_message *hm, int i
                 "Cache-Control: max-age=31536000"
         );
         mg_send(nc, data, data_len);
+        nc->is_resp = 0;
         free(data);
     } else {
         HTTP_REPLY_NOT_FOUND
@@ -217,6 +217,7 @@ void search(struct mg_connection *nc, struct mg_http_message *hm) {
     snprintf(url, 4096, "%s/%s/_search", WebCtx.es_url, WebCtx.es_index);
 
     nc->fn_data = web_post_async(url, body, WebCtx.es_insecure_ssl);
+    nc->is_resp = 1;
 }
 
 void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
@@ -382,11 +383,7 @@ void index_info(struct mg_connection *nc) {
         cJSON_AddStringToObject(json, "searchBackend", "elasticsearch");
     }
 
-    char *json_str = cJSON_PrintUnformatted(json);
-
-    web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
-    mg_send(nc, json_str, strlen(json_str));
-    free(json_str);
+    mg_send_json(nc, json);
     cJSON_Delete(json);
 }
 
@@ -450,6 +447,7 @@ void status(struct mg_connection *nc) {
     }
 
     free(status);
+    nc->is_resp = 0;
 }
 
 typedef struct {
@@ -627,7 +625,7 @@ int check_auth0(struct mg_http_message *hm) {
     return TRUE;
 }
 
-static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(void *fn_data)) {
+static void ev_router(struct mg_connection *nc, int ev, void *ev_data) {
 
     if (ev == MG_EV_HTTP_MSG) {
         struct mg_http_message *hm = (struct mg_http_message *) ev_data;
@@ -738,6 +736,7 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
                 if (r->status_code == 200) {
                     web_send_headers(nc, 200, r->size, "Content-Type: application/json");
                     mg_send(nc, r->body, r->size);
+                    nc->is_resp = 0;
                 } else if (r->status_code == 0) {
                     sist_log("serve.c", LOG_SIST_ERROR, "Could not connect to elasticsearch!");
 

src/web/web_util.c

@@ -5,31 +5,37 @@
 void web_serve_asset_index_html(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(index_html), HTTP_CROSS_ORIGIN_HEADERS "Content-Type: text/html");
     mg_send(nc, index_html, sizeof(index_html));
+    nc->is_resp = 0;
 }
 
 void web_serve_asset_index_js(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(index_js), "Content-Type: application/javascript");
     mg_send(nc, index_js, sizeof(index_js));
+    nc->is_resp = 0;
 }
 
 void web_serve_asset_chunk_vendors_js(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(chunk_vendors_js), "Content-Type: application/javascript");
     mg_send(nc, chunk_vendors_js, sizeof(chunk_vendors_js));
+    nc->is_resp = 0;
 }
 
 void web_serve_asset_favicon_ico(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(favicon_ico), "Content-Type: image/x-icon");
     mg_send(nc, favicon_ico, sizeof(favicon_ico));
+    nc->is_resp = 0;
 }
 
 void web_serve_asset_style_css(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(index_css), "Content-Type: text/css");
     mg_send(nc, index_css, sizeof(index_css));
+    nc->is_resp = 0;
 }
 
 void web_serve_asset_chunk_vendors_css(struct mg_connection *nc) {
     web_send_headers(nc, 200, sizeof(chunk_vendors_css), "Content-Type: text/css");
     mg_send(nc, chunk_vendors_css, sizeof(chunk_vendors_css));
+    nc->is_resp = 0;
 }
 
 index_t *web_get_index_by_id(int index_id) {
@@ -92,6 +98,7 @@ void mg_send_json(struct mg_connection *nc, const cJSON *json) {
 
     web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
     mg_send(nc, json_str, strlen(json_str));
+    nc->is_resp = 0;
 
     free(json_str);
 }

src/web/web_util.h

@@ -16,9 +16,26 @@ database_t *web_get_database(int index_id);
 
 __always_inline
 static char *web_address_to_string(struct mg_addr *addr) {
-    static char address_to_string_buf[INET6_ADDRSTRLEN];
+    static char address_to_string_buf[64];
 
-    return mg_ntoa(addr, address_to_string_buf, sizeof(address_to_string_buf));
+    if (addr->is_ip6) {
+        snprintf(address_to_string_buf, sizeof(address_to_string_buf),
+            "%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x",
+                addr->ip[0], addr->ip[1],
+                addr->ip[2], addr->ip[3],
+                addr->ip[4], addr->ip[5],
+                addr->ip[6], addr->ip[7],
+                addr->ip[8], addr->ip[9],
+                addr->ip[10], addr->ip[11],
+                addr->ip[12], addr->ip[13],
+                addr->ip[14], addr->ip[15]);
+    } else {
+        snprintf(address_to_string_buf, sizeof(address_to_string_buf),
+                 "%d.%d.%d.%d",
+                 addr->ip[0], addr->ip[1], addr->ip[2], addr->ip[3]);
+    }
+
+    return address_to_string_buf;
 }
 
 void web_send_headers(struct mg_connection *nc, int status_code, size_t length, char *extra_headers);

third-party/libscan/CMakeLists.txt

@@ -106,12 +106,33 @@ find_library(MUPDF_LIB NAMES liblibmupdf.a)
 find_library(CMS_LIB NAMES lcms2)
 find_library(JAS_LIB NAMES jasper)
 find_library(GUMBO_LIB NAMES gumbo)
-find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/8/ /usr/lib/gcc/aarch64-linux-gnu/8/)
+find_library(GOMP_LIB NAMES libgomp.a gomp
+        PATHS
+        /usr/lib/gcc/x86_64-linux-gnu/5/
+        /usr/lib/gcc/x86_64-linux-gnu/6/
+        /usr/lib/gcc/x86_64-linux-gnu/7/
+        /usr/lib/gcc/x86_64-linux-gnu/8/
+        /usr/lib/gcc/x86_64-linux-gnu/9/
+        /usr/lib/gcc/x86_64-linux-gnu/10/
+        /usr/lib/gcc/x86_64-linux-gnu/11/
+        /usr/lib/gcc/x86_64-linux-gnu/12/
+
+        /usr/lib/gcc/aarch64-linux-gnu/5/
+        /usr/lib/gcc/aarch64-linux-gnu/6/
+        /usr/lib/gcc/aarch64-linux-gnu/7/
+        /usr/lib/gcc/aarch64-linux-gnu/8/
+        /usr/lib/gcc/aarch64-linux-gnu/9/
+        /usr/lib/gcc/aarch64-linux-gnu/10/
+        /usr/lib/gcc/aarch64-linux-gnu/11/
+        /usr/lib/gcc/aarch64-linux-gnu/12/
+)
 find_package(Leptonica CONFIG REQUIRED)
-find_package(FFMPEG REQUIRED)
 find_package(libraw CONFIG REQUIRED)
 find_package(Freetype REQUIRED)
 
+find_package(FFMPEG REQUIRED)
+list(REMOVE_ITEM FFMPEG_LIBRARIES /usr/lib/x86_64-linux-gnu/libm.a)
+list(REMOVE_ITEM FFMPEG_LIBRARIES /usr/lib/aarch64-linux-gnu/libm.a)
 
 target_compile_options(
         scan
@@ -166,7 +187,6 @@ target_link_libraries(
         ${WPD_LIB_DIR}/libwpd-0.9.a
         ${WPD_LIB_DIR}/libwpd-stream-0.9.a
 
-        ${FREETYPE_LIB}
         ${HARFBUZZ_LIB}
         ${JBIG2DEC_LIB}
 

third-party/libscan/libscan/media/media.c

@@ -569,6 +569,9 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
         AVStream *stream = pFormatCtx->streams[video_stream];
 
         if (stream->codecpar->width <= MIN_SIZE || stream->codecpar->height <= MIN_SIZE) {
+            CTX_LOG_DEBUGF(doc->filepath,
+                           "Will not generate thumbnail because image is too small: %dx%d",
+                           stream->codecpar->width, stream->codecpar->width);
             avformat_close_input(&pFormatCtx);
             avformat_free_context(pFormatCtx);
             return;