Dockerfile

@@ -3,7 +3,7 @@ MAINTAINER simon987 <me@simon987.net>
 
 WORKDIR /build/
 COPY . .
-RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
+RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=on -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
 RUN make -j$(nproc)
 RUN strip sist2 || mv sist2_debug sist2
 

README.md

@@ -52,7 +52,7 @@ sist2 (Simple incremental search tool)
 Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x` *
     2. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
        recommended!)*
-    3. *(or)* `docker pull simon987/sist2:2.11.7-x64-linux`
+    3. *(or)* `docker pull simon987/sist2:2.11.6-x64-linux`
 
 1. See [Usage guide](docs/USAGE.md)
 

docs/USAGE.md

@@ -13,6 +13,7 @@
     * [options](#web-options)
     * [examples](#web-examples)
     * [rewrite_url](#rewrite_url)
+    * [link to specific indices](#link-to-specific-indices)
 * [elasticsearch](#elasticsearch)
 * [exec-script](#exec-script)
 * [tagging](#tagging)
@@ -25,66 +26,62 @@ Usage: sist2 scan [OPTION]... PATH
    or: sist2 exec-script [OPTION]... INDEX
 Lightning-fast file system indexer and search tool.
 
-    -h, --help                        show this help message and exit
+    -h, --help                    show this help message and exit
-    -v, --version                     Show version and exit
+    -v, --version                 Show version and exit
-    --verbose                         Turn on logging
+    --verbose                     Turn on logging
-    --very-verbose                    Turn on debug messages
+    --very-verbose                Turn on debug messages
 
 Scan options
-    -t, --threads=<int>               Number of threads. DEFAULT=1
+    -t, --threads=<int>           Number of threads. DEFAULT=1
-    --mem-throttle=<int>              Total memory threshold in MiB for scan throttling. DEFAULT=0
+    -q, --quality=<flt>           Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=3
-    -q, --thumbnail-quality=<flt>     Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best. DEFAULT=1
+    --size=<int>                  Thumbnail size, in pixels. Use negative value to disable. DEFAULT=500
-    --thumbnail-size=<int>            Thumbnail size, in pixels. DEFAULT=500
+    --content-size=<int>          Number of bytes to be extracted from text documents. Use negative value to disable. DEFAULT=32768
-    --thumbnail-count=<int>           Number of thumbnails to generate. Set a value > 1 to create video previews, set to 0 to disable thumbnails. DEFAULT=1
+    --incremental=<str>           Reuse an existing index and only scan modified files.
-    --content-size=<int>              Number of bytes to be extracted from text documents. Set to 0 to disable. DEFAULT=32768
+    -o, --output=<str>            Output directory. DEFAULT=index.sist2/
-    --incremental=<str>               Reuse an existing index and only scan modified files.
+    --rewrite-url=<str>           Serve files from this url instead of from disk.
-    -o, --output=<str>                Output directory. DEFAULT=index.sist2/
+    --name=<str>                  Index display name. DEFAULT: (name of the directory)
-    --rewrite-url=<str>               Serve files from this url instead of from disk.
+    --depth=<int>                 Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
-    --name=<str>                      Index display name. DEFAULT: (name of the directory)
+    --archive=<str>               Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
-    --depth=<int>                     Scan up to DEPTH subdirectories deep. Use 0 to only scan files in PATH. DEFAULT: -1
+    --archive-passphrase=<str>    Passphrase for encrypted archive files
-    --archive=<str>                   Archive file mode (skip|list|shallow|recurse). skip: Don't parse, list: only get file names as text, shallow: Don't parse archives inside archives. DEFAULT: recurse
+    --ocr-lang=<str>              Tesseract language (use 'tesseract --list-langs' to see which are installed on your machine)
-    --archive-passphrase=<str>        Passphrase for encrypted archive files
+    --ocr-images                  Enable OCR'ing of image files.
-    --ocr-lang=<str>                  Tesseract language (use 'tesseract --list-langs' to see which are installed on your machine)
+    --ocr-ebooks                  Enable OCR'ing of ebook files.
-    --ocr-images                      Enable OCR'ing of image files.
+    -e, --exclude=<str>           Files that match this regex will not be scanned
-    --ocr-ebooks                      Enable OCR'ing of ebook files.
+    --fast                        Only index file names & mime type
-    -e, --exclude=<str>               Files that match this regex will not be scanned
+    --treemap-threshold=<str>     Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
-    --fast                            Only index file names & mime type
+    --mem-buffer=<int>            Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000
-    --treemap-threshold=<str>         Relative size threshold for treemap (see USAGE.md). DEFAULT: 0.0005
+    --read-subtitles              Read subtitles from media files.
-    --mem-buffer=<int>                Maximum memory buffer size per thread in MiB for files inside archives (see USAGE.md). DEFAULT: 2000
+    --fast-epub                   Faster but less accurate EPUB parsing (no thumbnails, metadata)
-    --read-subtitles                  Read subtitles from media files.
+    --checksums                   Calculate file checksums when scanning.
-    --fast-epub                       Faster but less accurate EPUB parsing (no thumbnails, metadata)
+    --list-file=<str>             Specify a list of newline-delimited paths to be scanned instead of normal directory traversal. Use '-' to read from stdin.
-    --checksums                       Calculate file checksums when scanning.
-    --list-file=<str>                 Specify a list of newline-delimited paths to be scanned instead of normal directory traversal. Use '-' to read from stdin.
 
 Index options
-    -t, --threads=<int>               Number of threads. DEFAULT=1
+    -t, --threads=<int>           Number of threads. DEFAULT=1
-    --es-url=<str>                    Elasticsearch url with port. DEFAULT=http://localhost:9200
+    --es-url=<str>                Elasticsearch url with port. DEFAULT=http://localhost:9200
-    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
-    -p, --print                       Just print JSON documents to stdout.
+    -p, --print                   Just print JSON documents to stdout.
-    --incremental-index               Conduct incremental indexing, assumes that the old index is already digested by Elasticsearch.
+    --script-file=<str>           Path to user script.
-    --script-file=<str>               Path to user script.
+    --mappings-file=<str>         Path to Elasticsearch mappings.
-    --mappings-file=<str>             Path to Elasticsearch mappings.
+    --settings-file=<str>         Path to Elasticsearch settings.
-    --settings-file=<str>             Path to Elasticsearch settings.
+    --async-script                Execute user script asynchronously.
-    --async-script                    Execute user script asynchronously.
+    --batch-size=<int>            Index batch size. DEFAULT: 100
-    --batch-size=<int>                Index batch size. DEFAULT: 100
+    -f, --force-reset             Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
-    -f, --force-reset                 Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command)
 
 Web options
-    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
-    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
-    --bind=<str>                      Listen on this address. DEFAULT=localhost:4090
+    --bind=<str>                  Listen on this address. DEFAULT=localhost:4090
-    --auth=<str>                      Basic auth in user:password format
+    --auth=<str>                  Basic auth in user:password format
-    --tag-auth=<str>                  Basic auth in user:password format for tagging
+    --tag-auth=<str>              Basic auth in user:password format for tagging
-    --tagline=<str>                   Tagline in navbar
+    --tagline=<str>               Tagline in navbar
-    --dev                             Serve html & js files from disk (for development)
+    --dev                         Serve html & js files from disk (for development)
-    --lang=<str>                      Default UI language. Can be changed by the user
+    --lang=<str>                  Default UI language. Can be changed by the user
 
 Exec-script options
-    --es-url=<str>                    Elasticsearch url. DEFAULT=http://localhost:9200
+    --es-url=<str>                Elasticsearch url. DEFAULT=http://localhost:9200
-    --es-index=<str>                  Elasticsearch index name. DEFAULT=sist2
+    --es-index=<str>              Elasticsearch index name. DEFAULT=sist2
-    --script-file=<str>               Path to user script.
+    --script-file=<str>           Path to user script.
-    --async-script                    Execute user script asynchronously.
+    --async-script                Execute user script asynchronously.
-Made by simon987 <me@simon987.net>. Released under GPL-3.0
 ```
 
 ## Scan
@@ -93,21 +90,13 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
 
 * `-t, --threads` 
       Number of threads for file parsing. **Do not set a number higher than `$(nproc)` or `$(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors` in Windows!**
-* `--mem-throttle`
+* `-q, --quality` 
-    Total memory threshold in MiB for scan throttling. Worker threads will not start a new parse job
-    until the total memory usage of sist2 is below this threshold. Set to 0 to disable. DEFAULT=0
-* `-q, --thumbnail-quality` 
     Thumbnail quality, on a scale of 1.0 to 31.0, 1.0 being the best.
-* `--thumbnail-size` 
+* `--size` 
     Thumbnail size in pixels.
-* `--thumbnail-count`
-    Maximum number of thumbnails to generate. When set to a value >= 2, thumbnails for video previews
-    will be generated. The actual number of thumbnails generated depends on the length of the video (maximum 1 image 
-    every ~5s). Set to 0 to completely disable thumbnails.
 * `--content-size` 
-    Number of bytes of text to be extracted from the content of files (plain text, PDFs etc.).
+    Number of bytes of text to be extracted from the content of files (plain text and PDFs).
     Repeated whitespace and special characters do not count toward this limit.
-    Set to 0 to completely disable content parsing.
 * `--incremental`
     Specify an existing index. Information about files in this index that were not modified (based on *mtime* attribute)
     will be copied to the new index and will not be parsed again.
@@ -140,13 +129,13 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
     In effect, smaller `treemap-threshold` values will yield a more detailed 
     (but also a more cluttered and harder to read) visualization. 
     
-* `--mem-buffer` Maximum memory buffer size in MiB (per thread) for files inside archives. Media files 
+* `--mem-buffer` Maximum memory buffer size in MB (per thread) for files inside archives. Media files 
     larger than this number will be read sequentially and no *seek* operations will be supported.
 
     To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
 * `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
 * `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
-* `--checksums` Calculate file checksums (SHA1) when scanning files. This option does not cause any additional read 
+* `--checksums` Calculate file checksums (sha1) when scanning files. This option does not cause any additional read 
   operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
   files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
 
@@ -216,9 +205,6 @@ and values are raw image bytes.
     Elasticsearch index name. DEFAULT=sist2
  * `-p, --print` 
     Print index in JSON format to stdout.
- * `--incremental-index`
-   Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch.
-   Only the new changes since the last scan will be sent.
  * `--script-file` 
     Path to user script. See [Scripting](scripting.md).
  * `--mappings-file`

sist2-vue/dist/index.html

@@ -1,3 +0,0 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>sist2</title><link href="css/chunk-vendors.css" rel="preload" as="style"><link href="css/index.css" rel="preload" as="style"><link href="js/chunk-vendors.js" rel="preload" as="script"><link href="js/index.js" rel="preload" as="script"><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/index.css" rel="stylesheet"></head><body><noscript><style>body {
-            height: initial;
-        }</style><div style="text-align: center; margin-top: 100px"><strong>We're sorry but sist2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong><br><strong>Nous sommes désolés mais sist2 ne fonctionne pas correctement si JavaScript est activé. Veuillez l'activer pour continuer.</strong></div></noscript><div id="app"></div><script src="js/chunk-vendors.js"></script><script src="js/index.js"></script></body></html>

src/main.c

@@ -103,7 +103,7 @@ void sig_handler(int signum) {
     exit(-1);
 }
 
-void init_dir(const char *dirpath, scan_args_t* args) {
+void init_dir(const char *dirpath) {
     char path[PATH_MAX];
     snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
 
@@ -111,18 +111,9 @@ void init_dir(const char *dirpath, scan_args_t* args) {
     strcpy(ScanCtx.index.desc.version, Version);
     strcpy(ScanCtx.index.desc.type, INDEX_TYPE_NDJSON);
 
-    if (args->incremental != NULL) {
+    unsigned char index_md5[MD5_DIGEST_LENGTH];
-      // copy old index id
+    MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
-      char descriptor_path[PATH_MAX];
+    buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
-      snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental);
-      index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
-      memcpy(ScanCtx.index.desc.id, original_desc.id, sizeof(original_desc.id));
-    } else {
-      // genreate new index id based on timestamp
-      unsigned char index_md5[MD5_DIGEST_LENGTH];
-      MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
-      buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
-    }
 
     write_index_descriptor(path, &ScanCtx.index.desc);
 }
@@ -387,7 +378,7 @@ void sist2_scan(scan_args_t *args) {
 
     initialize_scan_context(args);
 
-    init_dir(ScanCtx.index.path, args);
+    init_dir(ScanCtx.index.path);
 
     char store_path[PATH_MAX];
     snprintf(store_path, PATH_MAX, "%sthumbs", ScanCtx.index.path);
@@ -683,7 +674,7 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"),
             OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
             OPT_BOOLEAN(0, "incremental-index", &index_args->incremental,
-                        "Conduct incremental indexing. Assumes that the old index is already ingested in Elasticsearch."),
+                        "Conduct incremental indexing, assumes that the old index is already digested by Elasticsearch."),
             OPT_STRING(0, "script-file", &common_script_path, "Path to user script."),
             OPT_STRING(0, "mappings-file", &index_args->es_mappings_path, "Path to Elasticsearch mappings."),
             OPT_STRING(0, "settings-file", &index_args->es_settings_path, "Path to Elasticsearch settings."),

third-party/libscan/CMakeLists.txt

@@ -11,6 +11,11 @@ if (SIST_DEBUG)
             antiword
             DEBUG
     )
+else()
+    add_compile_definitions(
+            antiword
+            NDEBUG
+    )
     target_compile_options(
             antiword
             PRIVATE
@@ -20,11 +25,6 @@ if (SIST_DEBUG)
             -fsanitize=address
             -fno-inline
     )
-else()
-    add_compile_definitions(
-            antiword
-            NDEBUG
-    )
 endif()
 
 add_library(

third-party/libscan/libscan/media/media.c

@@ -251,7 +251,7 @@ void append_tag_meta_if_not_exists(scan_media_ctx_t *ctx, document_t *doc, AVDic
     for (; *ptr; ++ptr) *ptr = (char) tolower(*ptr);
 
 __always_inline
-static void append_audio_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, document_t *doc) {
+static void append_audio_meta(AVFormatContext *pFormatCtx, document_t *doc) {
 
     AVDictionaryEntry *tag = NULL;
     while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
@@ -269,7 +269,7 @@ static void append_audio_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx
         } else if (strcmp(key, "album") == 0) {
             APPEND_TAG_META(MetaAlbum)
         } else if (strcmp(key, "comment") == 0) {
-            append_tag_meta_if_not_exists(ctx, doc, tag, MetaContent);
+            APPEND_TAG_META(MetaContent)
         }
     }
 }
@@ -437,7 +437,7 @@ int decode_frame_and_save_thumbnail(scan_media_ctx_t *ctx, AVFormatContext *pFor
         return SAVE_THUMBNAIL_FAILED;
     }
 
-    if (ctx->tesseract_lang != NULL && IS_VIDEO(pFormatCtx) && thumbnail_index == 0) {
+    if (ctx->tesseract_lang != NULL && IS_VIDEO(pFormatCtx)) {
         ocr_image(ctx, doc, decoder, frame_and_packet->frame);
     }
 
@@ -558,7 +558,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
     }
 
     if (audio_stream != -1) {
-        append_audio_meta(ctx, pFormatCtx, doc);
+        append_audio_meta(pFormatCtx, doc);
     }
 
     if (video_stream != -1 && ctx->tn_count > 0) {

third-party/libscan/libscan/raw/raw.c

@@ -7,22 +7,8 @@
 
 #define MIN_SIZE 32
 
-int store_thumbnail_jpeg(scan_raw_ctx_t *ctx, libraw_thumbnail_t img, document_t *doc) {
+int store_thumbnail_jpeg(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, document_t *doc) {
-
+    return store_image_thumbnail((scan_media_ctx_t *) ctx, img->data, img->data_size, doc, "x.jpeg");
-    scan_media_ctx_t media_ctx = {
-            .read_subtitles = FALSE,
-            .tn_count = 1,
-            .max_media_buffer = 0,
-            .store = ctx->store,
-            .log = ctx->log,
-            .logf = ctx->logf,
-            .tn_size = ctx->tn_size,
-            .tn_qscale = ctx->tn_qscale,
-            .tesseract_lang = NULL,
-            .tesseract_path = NULL
-    };
-
-    return store_image_thumbnail(&media_ctx, img.thumb, img.tlength, doc, "x.jpeg");
 }
 
 int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, document_t *doc) {
@@ -185,25 +171,25 @@ void parse_raw(scan_raw_ctx_t *ctx, vfile_t *f, document_t *doc) {
         return;
     }
 
-    int tn_ok = 0;
+    int errc = 0;
+    libraw_processed_image_t *thumb = libraw_dcraw_make_mem_thumb(libraw_lib, &errc);
+    if (errc != 0) {
+        free(buf);
+        libraw_dcraw_clear_mem(thumb);
+        libraw_close(libraw_lib);
+        return;
+    }
 
+    int tn_ok = 0;
     if (libraw_lib->thumbnail.tformat == LIBRAW_THUMBNAIL_JPEG) {
-        tn_ok = store_thumbnail_jpeg(ctx, libraw_lib->thumbnail, doc);
+        tn_ok = store_thumbnail_jpeg(ctx, thumb, doc);
     } else if (libraw_lib->thumbnail.tformat == LIBRAW_THUMBNAIL_BITMAP) {
         // TODO: technically this should work but is currently untested
-
-        int errc = 0;
-        libraw_processed_image_t *thumb = libraw_dcraw_make_mem_thumb(libraw_lib, &errc);
-        if (errc != 0) {
-            free(buf);
-            libraw_dcraw_clear_mem(thumb);
-            libraw_close(libraw_lib);
-            return;
-        }
-
         tn_ok = store_thumbnail_rgb24(ctx, thumb, doc);
     }
 
+    libraw_dcraw_clear_mem(thumb);
+
     if (tn_ok == TRUE) {
         free(buf);
         libraw_close(libraw_lib);
@@ -220,7 +206,7 @@ void parse_raw(scan_raw_ctx_t *ctx, vfile_t *f, document_t *doc) {
 
     libraw_dcraw_process(libraw_lib);
 
-    int errc = 0;
+    errc = 0;
     libraw_processed_image_t *img = libraw_dcraw_make_mem_image(libraw_lib, &errc);
     if (errc != 0) {
         free(buf);