Fix bug with media files, don't encode thumbnail when not necessary

README.md

@@ -72,11 +72,12 @@ See [Usage guide](docs/USAGE.md) for more details
 
 File type | Library | Content | Thumbnail | Metadata
 :---|:---|:---|:---|:---
-pdf,xps,cbz,cbr,fb2,epub | MuPDF | text+ocr | yes, `png` | title |
-`audio/*` | ffmpeg | - | yes, `jpeg` | ID3 tags |
-`video/*` | ffmpeg | - | yes, `jpeg` | title, comment, artist |
-`image/*` | ffmpeg | - | yes, `jpeg` | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
-raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes, `jpeg` | Common EXIF tags |
+pdf,xps,fb2,epub | MuPDF | text+ocr | yes | title |
+cbz,cbr | *(none)* | - | yes | - |
+`audio/*` | ffmpeg | - | yes | ID3 tags |
+`video/*` | ffmpeg | - | yes | title, comment, artist |
+`image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190) |
+raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags |
 ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
 `text/plain` | *(none)* | yes | no | - |
 html, xml | *(none)* | yes | no | - |
@@ -101,7 +102,7 @@ scan is also supported.
  
 ### OCR
 
-You can enable OCR support for pdf,xps,cbz,cbr,fb2,epub file types with the
+You can enable OCR support for pdf,xps,fb2,epub file types with the
 `--ocr <lang>` option. Download the language data files with your
 package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
 

scripts/mime.py

@@ -18,7 +18,6 @@ major_mime = {
 
 pdf = (
     "application/pdf",
-    "application/x-cbz",
     "application/epub+zip",
     "application/vnd.ms-xpsdocument",
 )

src/cli.c

@@ -41,6 +41,9 @@ void scan_args_destroy(scan_args_t *args) {
     if (args->name != NULL) {
         free(args->name);
     }
+    if (args->incremental != NULL) {
+        free(args->incremental);
+    }
     if (args->path != NULL) {
         free(args->path);
     }
@@ -79,7 +82,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     }
 
     if (args->incremental != NULL) {
-        abs_path = abspath(args->incremental);
+        args->incremental = abspath(args->incremental);
         if (abs_path == NULL) {
             sist_log("main.c", SIST_WARNING, "Could not open original index! Disabled incremental scan feature.");
             args->incremental = NULL;

src/ctx.h

@@ -5,7 +5,7 @@
 #include "tpool.h"
 #include "libscan/scan.h"
 #include "libscan/arc/arc.h"
-#include "libscan/cbr/cbr.h"
+#include "libscan/comic/comic.h"
 #include "libscan/ebook/ebook.h"
 #include "libscan/font/font.h"
 #include "libscan/media/media.h"
@@ -40,7 +40,7 @@ typedef struct {
     int fast;
 
     scan_arc_ctx_t arc_ctx;
-    scan_cbr_ctx_t cbr_ctx;
+    scan_comic_ctx_t comic_ctx;
     scan_ebook_ctx_t ebook_ctx;
     scan_font_ctx_t font_ctx;
     scan_media_ctx_t media_ctx;

src/index/elastic.c

@@ -82,7 +82,7 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
     char *str = cJSON_Print(body);
 
     char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_update_by_query?pretty", Indexer->es_url);
+    snprintf(bulk_url, 4096, "%s/sist2/_update_by_query?wait_for_completion=false", Indexer->es_url);
     response_t *r = web_post(bulk_url, str);
     LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
     cJSON *resp = cJSON_Parse(r->body);
@@ -110,8 +110,8 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
 
     size_t buf_size = 0;
     size_t buf_cur = 0;
-    char *buf = malloc(8196);
-    size_t buf_capacity = 8196;
+    char *buf = malloc(8192);
+    size_t buf_capacity = 8192;
 
     while (line != NULL && *count < max) {
         char action_str[256];
@@ -325,7 +325,7 @@ void elastic_init(int force_reset) {
     // Check if index exists
     char url[4096];
     snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
     int index_exists = r->status_code == 200;
     free_response(r);
 
@@ -370,7 +370,7 @@ cJSON *elastic_get_document(const char *uuid_str) {
     char url[4096];
     snprintf(url, 4096, "%s/sist2/_doc/%s", WebCtx.es_url, uuid_str);
 
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 3);
     cJSON *json = NULL;
     if (r->status_code == 200) {
         json = cJSON_Parse(r->body);
@@ -384,7 +384,7 @@ char *elastic_get_status() {
     snprintf(url, 4096,
              "%s/_cluster/state/metadata/sist2?filter_path=metadata.indices.*.state", WebCtx.es_url);
 
-    response_t *r = web_get(url);
+    response_t *r = web_get(url, 30);
     cJSON *json = NULL;
     char *status = malloc(128 * sizeof(char));
     status[0] = '\0';

src/index/web.c

@@ -83,7 +83,7 @@ subreq_ctx_t *http_req(const char *url, const char *extra_headers, const char *p
     subreq_ctx_t *ctx = malloc(sizeof(subreq_ctx_t));
     mg_mgr_init(&ctx->mgr, NULL);
 
-    char address[8196];
+    char address[8192];
     snprintf(address, sizeof(address), "tcp://%.*s:%u", (int) host.len, host.p, port);
     struct mg_connection *nc = mg_connect(&ctx->mgr, address, http_req_ev);
     nc->user_data = &ctx->ev_data;
@@ -112,7 +112,7 @@ subreq_ctx_t *web_post_async(const char *url, const char *data) {
     return http_req(url, SIST2_HEADERS, data, "POST");
 }
 
-response_t *web_get(const char *url) {
+response_t *web_get(const char *url, int timeout) {
     response_t *resp = malloc(sizeof(response_t));
 
     CURL *curl;
@@ -123,6 +123,7 @@ response_t *web_get(const char *url) {
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) (&buffer));
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
+    curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
 
     struct curl_slist *headers = curl_slist_append(headers, "Content-Type: application/json");
     curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

src/index/web.h

@@ -20,7 +20,7 @@ typedef struct {
     struct mg_mgr mgr;
 } subreq_ctx_t;
 
-response_t *web_get(const char *url);
+response_t *web_get(const char *url, int timeout);
 response_t *web_post(const char * url, const char * data);
 subreq_ctx_t *web_post_async(const char *url, const char *data);
 response_t *web_put(const char *url, const char *data);

src/io/store.c

@@ -143,3 +143,8 @@ GHashTable *store_read_all(store_t *store) {
     return table;
 }
 
+
+void store_copy(store_t *store, const char *destination) {
+    mkdir(destination, S_IWUSR | S_IRUSR | S_IXUSR);
+    mdb_env_copy(store->env, destination);
+}

src/io/store.h

@@ -27,4 +27,6 @@ char *store_read(store_t *store, char *key, size_t key_len, size_t *ret_vallen);
 
 GHashTable *store_read_all(store_t *store);
 
+void store_copy(store_t *store, const char *destination);
+
 #endif

src/main.c

@@ -21,7 +21,7 @@
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "2.6.1";
+static const char *const Version = "2.7.4";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
@@ -75,7 +75,7 @@ void _logf(const char *filepath, int level, char *format, ...) {
 
     va_start(args, format);
     if (level == LEVEL_FATAL) {
-        sist_logf(filepath, level, format, args);
+        vsist_logf(filepath, level, format, args);
         exit(-1);
     }
 
@@ -85,7 +85,7 @@ void _logf(const char *filepath, int level, char *format, ...) {
                 vsist_logf(filepath, level, format, args);
             }
         } else {
-            sist_logf(filepath, level, format, args);
+            vsist_logf(filepath, level, format, args);
         }
     }
     va_end(args);
@@ -99,11 +99,14 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.arc_ctx.logf = _logf;
     ScanCtx.arc_ctx.parse = (parse_callback_t) parse;
 
-    // Cbr
-    ScanCtx.cbr_ctx.log = _log;
-    ScanCtx.cbr_ctx.logf = _logf;
-    ScanCtx.cbr_ctx.store = _store;
-    ScanCtx.cbr_ctx.cbr_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbr");
+    // Comic
+    ScanCtx.comic_ctx.log = _log;
+    ScanCtx.comic_ctx.logf = _logf;
+    ScanCtx.comic_ctx.store = _store;
+    ScanCtx.comic_ctx.tn_size = args->size;
+    ScanCtx.comic_ctx.tn_qscale = args->quality;
+    ScanCtx.comic_ctx.cbr_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbr");
+    ScanCtx.comic_ctx.cbz_mime = mime_get_mime_by_string(ScanCtx.mime_table, "application/x-cbz");
 
     // Ebook
     pthread_mutex_init(&ScanCtx.ebook_ctx.mupdf_mutex, NULL);
@@ -241,6 +244,13 @@ void sist2_scan(scan_args_t *args) {
         }
         closedir(dir);
         store_destroy(source);
+
+        snprintf(store_path, PATH_MAX, "%stags", args->incremental);
+        snprintf(dst_path, PATH_MAX, "%stags", ScanCtx.index.path);
+        mkdir(store_path, S_IWUSR | S_IRUSR | S_IXUSR);
+        store_t *source_tags = store_create(store_path, STORE_SIZE_TAG);
+        store_copy(source_tags, dst_path);
+        store_destroy(source_tags);
     }
 
     store_destroy(ScanCtx.index.store);

src/parsing/mime_generated.c

@@ -108,7 +108,7 @@ enum mime {
     application_x_bzip=655460,
     application_x_bzip2=655461 | 0x08000000,
     application_x_cbr=655462,
-    application_x_cbz=655463 | 0x40000000,
+    application_x_cbz=655463,
     application_x_cdlink=655464,
     application_x_chat=655465,
     application_x_chrome_extension=655466,

src/parsing/parse.c

@@ -145,11 +145,10 @@ void parse(void *arg) {
                     (IS_ARC_FILTER(doc.mime) && should_parse_filtered_file(doc.filepath, doc.ext))
             )) {
         parse_archive(&ScanCtx.arc_ctx, &job->vfile, &doc);
-    } else if (ScanCtx.ooxml_ctx.content_size > 0 && IS_DOC(doc.mime)) {
+    } else if ((ScanCtx.ooxml_ctx.content_size > 0 || ScanCtx.media_ctx.tn_size > 0) && IS_DOC(doc.mime)) {
         parse_ooxml(&ScanCtx.ooxml_ctx, &job->vfile, &doc);
-
-    } else if (is_cbr(&ScanCtx.cbr_ctx, doc.mime)) {
-        parse_cbr(&ScanCtx.cbr_ctx, &job->vfile, &doc);
+    } else if (is_cbr(&ScanCtx.comic_ctx, doc.mime) || is_cbz(&ScanCtx.comic_ctx, doc.mime)) {
+        parse_comic(&ScanCtx.comic_ctx, &job->vfile, &doc);
     } else if (IS_MOBI(doc.mime)) {
         parse_mobi(&ScanCtx.mobi_ctx, &job->vfile, &doc);
     }

src/static/css/dark.css

@@ -121,7 +121,7 @@ body {
     background: #546b7a;
 }
 
-a:hover,.btn:hover {
+a:hover, .btn:hover {
     color: #fff;
 }
 
@@ -130,7 +130,11 @@ a:hover,.btn:hover {
 }
 
 .document {
-    padding: 0.5rem;
+    padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }
 
 .document p {
@@ -203,7 +207,7 @@ a:hover,.btn:hover {
     margin-top: -1px;
     font-family: monospace;
     font-size: 90%;
-    background: rgba(0,0,0,0.2);
+    background: rgba(0, 0, 0, 0.2);
     padding: 0.1em 0.4em;
     color: white;
     cursor: pointer;
@@ -218,7 +222,7 @@ a:hover,.btn:hover {
     display: block;
     min-width: 64px;
     max-width: 100%;
-    max-height: 175px;
+    max-height: 240px;
     margin: 0 auto 0;
     padding: 3px 3px 0;
     width: auto;
@@ -241,20 +245,6 @@ a:hover,.btn:hover {
     width: 100%;
 }
 
-@media screen and (min-width: 1500px) {
-    .container {
-        max-width: 1440px;
-    }
-
-    .bricklayer-column-sizer {
-        width: 20% !important;
-    }
-
-    .bricklayer-column {
-        max-width: 20%;
-    }
-}
-
 @media screen and (min-width: 1800px) {
     .container {
         max-width: 1550px;
@@ -451,6 +441,7 @@ option {
     .small-btn {
         display: none;
     }
+
     .large-btn {
         display: inherit;
     }
@@ -460,6 +451,7 @@ option {
     .small-btn {
         display: inherit;
     }
+
     .large-btn {
         display: none;
     }

src/static/css/light.css

@@ -70,7 +70,11 @@ body {
 }
 
 .document {
-    padding: 0.5rem;
+   padding: 0.3rem;
+}
+
+.card-text:last-child {
+    margin-top: -1px;
 }
 
 .document p {
@@ -158,7 +162,7 @@ body {
     display: block;
     min-width: 64px;
     max-width: 100%;
-    max-height: 175px;
+    max-height: 240px;
     margin: 0 auto 0;
     padding: 3px 3px 0 3px;
     width: auto;
@@ -181,6 +185,10 @@ body {
     width: 100%;
 }
 
+.bricklayer {
+    /*max-width: 100%;*/
+}
+
 @media screen and (max-width: 1200px) {
     .bricklayer-column {
         max-width: 100%;

src/static/js/dom.js

@@ -163,6 +163,7 @@ function getTags(hit, mimeCategory) {
 function makeUserTag(tag, hit) {
     const userTag = document.createElement("span");
     userTag.setAttribute("class", "badge badge-pill badge-user");
+    userTag.setAttribute("title", tag.split("#")[0])
 
     const tokens = tag.split("#");
 

src/static/js/search.js

@@ -1,4 +1,4 @@
-const SIZE = 40;
+const SIZE = 60;
 let mimeMap = [];
 let tagMap = [];
 let mimeTree;
@@ -253,7 +253,7 @@ function handleTreeClick(tree) {
 
         if (node.id === "any") {
             if (!node.itree.state.checked) {
-                tree.deselect();
+                tree.deselectDeep();
             }
         } else {
             tree.node("any").deselect();

src/static/js/util.js

@@ -101,7 +101,8 @@ const _defaults = {
     treemapColor: "PuBuGn",
     treemapSize: "large",
     suggestPath: true,
-    fragmentSize: 100
+    fragmentSize: 100,
+    columns: 5
 };
 
 function loadSettings() {
@@ -118,6 +119,7 @@ function loadSettings() {
     $("#settingTreemapType").val(CONF.options.treemapType);
     $("#settingSuggestPath").prop("checked", CONF.options.suggestPath);
     $("#settingFragmentSize").val(CONF.options.fragmentSize);
+    $("#settingColumns").val(CONF.options.columns);
 }
 
 function Settings() {
@@ -125,6 +127,7 @@ function Settings() {
 
     this._onUpdate = function () {
         $("#fuzzyToggle").prop("checked", this.options.fuzzy);
+        updateColumnStyle();
     };
 
     this.load = function () {
@@ -161,6 +164,7 @@ function updateSettings() {
     CONF.options.treemapType = $("#settingTreemapType").val();
     CONF.options.suggestPath = $("#settingSuggestPath").prop("checked");
     CONF.options.fragmentSize = $("#settingFragmentSize").val();
+    CONF.options.columns = $("#settingColumns").val();
     CONF.save();
 
     if (typeof searchDebounced !== "undefined") {
@@ -203,3 +207,26 @@ function toggleTheme() {
     }
     window.location.reload();
 }
+
+function updateColumnStyle() {
+    const style = document.getElementById("style");
+    if (style) {
+        style.innerHTML =
+        `
+@media screen and (min-width: 1500px) {
+    .container {
+            max-width: 1440px;
+        }
+
+    .bricklayer-column-sizer {
+            width: ${100 / CONF.options.columns}% !important;
+        }
+
+    .bricklayer-column {
+            max-width: ${100 / CONF.options.columns}%;
+        }
+    }
+}
+        `
+    }
+}

src/static/search.html

@@ -6,12 +6,13 @@
     <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
 
     <link href="css" rel="stylesheet" type="text/css">
+    <style id="style"></style>
 </head>
 <body>
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.6.1</span>
+    <span class="badge badge-pill version">2.7.4</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <a class="btn ml-auto" href="/stats">Stats</a>
     <button class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings
@@ -216,6 +217,20 @@
                         <option value="list">List</option>
                     </select>
 
+                    <div class="form-group">
+                        <label for="settingColumns">Maximum column count</label>
+                        <select id="settingColumns" class="form-control form-control-sm">
+                            <option value="3">3</option>
+                            <option value="4">4</option>
+                            <option value="5">5</option>
+                            <option value="6">6</option>
+                            <option value="7">7</option>
+                            <option value="8">8</option>
+                            <option value="9">9</option>
+                            <option value="13">13</option>
+                        </select>
+                    </div>
+
                     <hr/>
                     <h4>Stats</h4>
 

src/static/stats.html

@@ -10,7 +10,7 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">2.6.1</span>
+    <span class="badge badge-pill version">2.7.4</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <a style="margin-left: auto" class="btn" href="/">Back</a>
     <button class="btn" type="button" data-toggle="modal" data-target="#settings"
@@ -94,6 +94,19 @@
                     <option value="list">List</option>
                 </select>
 
+                <div class="form-group">
+                    <label for="settingColumns">Maximum column count</label>
+                    <select id="settingColumns" class="form-control form-control-sm">
+                        <option value="3">3</option>
+                        <option value="4">4</option>
+                        <option value="5">5</option>
+                        <option value="6">6</option>
+                        <option value="7">7</option>
+                        <option value="8">8</option>
+                        <option value="9">9</option>
+                    </select>
+                </div>
+
                 <hr/>
                 <h4>Stats</h4>
 

src/types.h

@@ -10,7 +10,7 @@ typedef struct index_descriptor {
     char version[64];
     long timestamp;
     char root[PATH_MAX];
-    char rewrite_url[8196];
+    char rewrite_url[8192];
     short root_len;
     char name[1024];
     char type[64];

src/web/serve.c

@@ -111,7 +111,7 @@ void stats_files(struct mg_connection *nc, struct http_message *hm, struct mg_st
             return;
     }
 
-    char disposition[8196];
+    char disposition[8192];
     snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s\"", file);
 
     char full_path[PATH_MAX];
@@ -256,7 +256,7 @@ int serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) {
 
     const char *ext = cJSON_GetObjectItem(json, "extension")->valuestring;
 
-    char url[8196];
+    char url[8192];
     snprintf(url, sizeof(url),
              "%s%s/%s%s%s",
              idx->desc.rewrite_url, path_unescaped, name_unescaped, strlen(ext) == 0 ? "" : ".", ext);
@@ -291,7 +291,7 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
 
     LOG_DEBUGF("serve.c", "Serving file from disk: %s", full_path)
 
-    char disposition[8196];
+    char disposition[8192];
     snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s%s%s\"",
              name, strlen(ext) == 0 ? "" : ".", ext);
 
@@ -538,7 +538,7 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
             }
         }
 
-        char buf[8196];
+        char buf[8192];
         snprintf(buf, sizeof(buf),
                  "{"
                  "    \"script\" : {"
@@ -558,7 +558,7 @@ void tag(struct mg_connection *nc, struct http_message *hm, struct mg_str *path)
     } else {
         cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name));
 
-        char buf[8196];
+        char buf[8192];
         snprintf(buf, sizeof(buf),
                  "{"
                  "    \"script\" : {"