Improve scroll feature, UI fix

schema/mappings.json

@@ -1,5 +1,9 @@
 {
   "properties": {
+    "_tie": {
+      "type": "keyword",
+      "doc_values": true
+    },
     "path": {
       "type": "text",
       "analyzer": "path_analyzer",
@@ -105,6 +109,30 @@
     },
     "tag": {
       "type": "keyword"
+    },
+    "exif_make": {
+      "type": "text"
+    },
+    "exif_model": {
+      "type": "text"
+    },
+    "exif:software": {
+      "type": "text"
+    },
+    "exif_exposure_time": {
+      "type": "keyword"
+    },
+    "exif_fnumber": {
+      "type": "keyword"
+    },
+    "exif_iso_speed_ratings": {
+      "type": "keyword"
+    },
+    "exif_focal_length": {
+      "type": "keyword"
+    },
+    "exif_user_comment": {
+      "type": "text"
     }
   }
 }

schema/pipeline.json

@@ -0,0 +1,10 @@
+{
+  "description": "Copy _id to _tie",
+  "processors": [
+    {
+      "script": {
+        "source": "ctx._tie = ctx._id;"
+      }
+    }
+  ]
+}

scripts/index_static.py

@@ -1,6 +1,9 @@
+import json
+
 files = [
     "schema/mappings.json",
     "schema/settings.json",
+    "schema/pipeline.json",
 ]
 
 
@@ -9,6 +12,6 @@ def clean(filepath):
 
 
 for file in files:
-    with open(file, "rb") as f:
+    with open(file, "r") as f:
-        data = f.read()
+        data = json.dumps(json.load(f), separators=(",", ":")).encode()
     print("char %s[%d] = {%s};" % (clean(file), len(data), ",".join(str(int(b)) for b in data)))

src/index/elastic.c

@@ -129,7 +129,7 @@ void elastic_flush() {
     Indexer->queued = 0;
 
     char bulk_url[4096];
-    snprintf(bulk_url, 4096, "%s/sist2/_bulk", Indexer->es_url);
+    snprintf(bulk_url, 4096, "%s/sist2/_bulk?pipeline=tie", Indexer->es_url);
     response_t *r = web_post(bulk_url, buf, "Content-Type: application/x-ndjson");
 
     if (r->status_code == 0) {
@@ -245,6 +245,11 @@ void elastic_init(int force_reset) {
         LOG_INFOF("elastic.c", "Close index <%d>", r->status_code);
         free_response(r);
 
+        snprintf(url, 4096, "%s/_ingest/pipeline/tie", IndexCtx.es_url);
+        r = web_put(url, pipeline_json, "Content-Type: application/json");
+        LOG_INFOF("elastic.c", "Create pipeline <%d>", r->status_code);
+        free_response(r);
+
         snprintf(url, 4096, "%s/sist2/_settings", IndexCtx.es_url);
         r = web_put(url, settings_json, "Content-Type: application/json");
         LOG_INFOF("elastic.c", "Update settings <%d>", r->status_code);

src/main.c

@@ -6,7 +6,7 @@
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "1.2.5";
+static const char *const Version = "1.2.6";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",

src/web/serve.c

@@ -221,14 +221,6 @@ int search(UNUSED(void *p), onion_request *req, onion_response *res) {
         return OCS_NOT_PROCESSED;
     }
 
-    char *scroll_param;
-    const char *scroll = onion_request_get_query(req, "scroll");
-    if (scroll != NULL) {
-        scroll_param = "?scroll=3m";
-    } else {
-        scroll_param = "";
-    }
-
     const struct onion_block_t *block = onion_request_get_data(req);
 
     if (block == NULL) {
@@ -236,7 +228,7 @@ int search(UNUSED(void *p), onion_request *req, onion_response *res) {
     }
 
     char url[4096];
-    snprintf(url, 4096, "%s/sist2/_search%s", WebCtx.es_url, scroll_param);
+    snprintf(url, 4096, "%s/sist2/_search", WebCtx.es_url);
     response_t *r = web_post(url, onion_block_data(block), "Content-Type: application/json");
 
     set_default_headers(res);
@@ -254,43 +246,6 @@ int search(UNUSED(void *p), onion_request *req, onion_response *res) {
     return OCS_PROCESSED;
 }
 
-int scroll(UNUSED(void *p), onion_request *req, onion_response *res) {
-
-    int flags = onion_request_get_flags(req);
-    if ((flags & OR_METHODS) != OR_GET) {
-        return OCS_NOT_PROCESSED;
-    }
-
-    char url[4096];
-    snprintf(url, 4096, "%s/_search/scroll", WebCtx.es_url);
-
-    const char *scroll_id = onion_request_get_query(req, "scroll_id");
-
-    cJSON *json = cJSON_CreateObject();
-    cJSON_AddStringToObject(json, "scroll_id", scroll_id);
-    cJSON_AddStringToObject(json, "scroll", "3m");
-
-    char *json_str = cJSON_PrintUnformatted(json);
-    response_t *r = web_post(url, json_str, "Content-Type: application/json");
-
-    cJSON_Delete(json);
-    cJSON_free(json_str);
-
-    if (r->status_code != 200) {
-        free_response(r);
-        return OCS_NOT_PROCESSED;
-    }
-
-    set_default_headers(res);
-    onion_response_set_header(res, "Content-Type", "application/json");
-    onion_response_set_header(res, "Content-Disposition", "application/json");
-    onion_response_set_length(res, r->size);
-    onion_response_write(res, r->body, r->size);
-    free_response(r);
-
-    return OCS_PROCESSED;
-}
-
 int serve_file_from_url(cJSON *json, index_t *idx, onion_request *req, onion_response *res) {
 
     const char *path = cJSON_GetObjectItem(json, "path")->valuestring;
@@ -466,7 +421,6 @@ void serve(const char *hostname, const char *port) {
     onion_url_add(urls, "img/sprite-skin-flat.png", img_sprite_skin_flag);
 
     onion_url_add(urls, "es", search);
-    onion_url_add(urls, "scroll", scroll);
     onion_url_add(urls, "status", status);
     onion_url_add(
             urls,

web/js/dom.js

@@ -113,7 +113,7 @@ function getTags(hit, mimeCategory) {
     switch (mimeCategory) {
         case "video":
         case "image":
-            if (hit["_source"].hasOwnProperty("videoc")) {
+            if (hit["_source"].hasOwnProperty("videoc") && hit["_source"]["videoc"]) {
                 const formatTag = document.createElement("span");
                 formatTag.setAttribute("class", "badge badge-pill badge-video");
                 formatTag.appendChild(document.createTextNode(hit["_source"]["videoc"].replace(" ", "")));
@@ -121,7 +121,7 @@ function getTags(hit, mimeCategory) {
             }
             break;
         case "audio": {
-            if (hit["_source"].hasOwnProperty("audioc")) {
+            if (hit["_source"].hasOwnProperty("audioc") && hit["_source"]["audioc"]) {
                 let formatTag = document.createElement("span");
                 formatTag.setAttribute("class", "badge badge-pill badge-audio");
                 formatTag.appendChild(document.createTextNode(hit["_source"]["audioc"]));

web/js/search.js

@@ -6,7 +6,8 @@ let tagTree;
 
 let searchBar = document.getElementById("searchBar");
 let pathBar = document.getElementById("pathBar");
-let scroll_id = null;
+let lastDoc = null;
+let reachedEnd = false;
 let docCount = 0;
 let coolingDown = false;
 let searchBusy = true;
@@ -259,41 +260,18 @@ function insertHits(resultContainer, hits) {
 }
 
 window.addEventListener("scroll", function () {
-    if (!coolingDown && !searchBusy) {
+    if (!searchBusy) {
         let threshold = 400;
 
         if ((window.innerHeight + window.scrollY) >= document.body.offsetHeight - threshold) {
+            if (!reachedEnd) {
                 coolingDown = true;
-            doScroll();
+                search(lastDoc);
+            }
         }
     }
 });
 
-function doScroll() {
-    $.get("scroll", {scroll_id: scroll_id})
-        .then(searchResult => {
-            let searchResults = document.getElementById("searchResults");
-            let hits = searchResult["hits"]["hits"];
-
-            //Page indicator
-            let pageIndicator = makePageIndicator(searchResult);
-            searchResults.appendChild(pageIndicator);
-
-            //Result container
-            let resultContainer = makeResultContainer();
-            searchResults.appendChild(resultContainer);
-
-            insertHits(resultContainer, hits);
-
-            if (hits.length === SIZE) {
-                coolingDown = false;
-            }
-        })
-        .fail(() => {
-            window.location.reload();
-        })
-}
-
 function getSelectedNodes(tree) {
     let selectedNodes = [];
 
@@ -314,20 +292,24 @@ function getSelectedNodes(tree) {
     return selectedNodes
 }
 
-function search() {
+function search(after = null) {
+    lastDoc = null;
+
     if (searchBusy) {
         return;
     }
     searchBusy = true;
 
-    //Clear old search results
     let searchResults = document.getElementById("searchResults");
+    //Clear old search results
+    let preload;
+    if (!after) {
         while (searchResults.firstChild) {
             searchResults.removeChild(searchResults.firstChild);
         }
-
+        preload = makePreloader();
-    const preload = makePreloader();
         searchResults.appendChild(preload);
+    }
 
     let query = searchBar.value;
     let empty = query === "";
@@ -362,9 +344,9 @@ function search() {
         filters.push([{terms: {"tag": tags}}]);
     }
 
-    $.jsonPost("es?scroll=1", {
+    let q = {
         "_source": {
-            excludes: ["content"]
+            excludes: ["content", "_tie"]
         },
         query: {
             bool: {
@@ -379,8 +361,9 @@ function search() {
                 filter: filters
             }
         },
-        sort: [
+        "sort": [
-            "_score"
+            {"_score": {"order": "desc"}},
+            {"_tie": {"order":"asc"}}
         ],
         highlight: {
             pre_tags: ["<mark>"],
@@ -397,20 +380,35 @@ function search() {
             total_size: {"sum": {"field": "size"}}
         },
         size: SIZE,
-    }).then(searchResult => {
+    }
-        scroll_id = searchResult["_scroll_id"];
 
+    if (after) {
+        q.search_after = [after["_score"], after["_id"]];
+    }
+
+    $.jsonPost("es", q).then(searchResult => {
+        let hits = searchResult["hits"]["hits"];
+        if (hits) {
+            lastDoc = hits[hits.length - 1];
+        }
+
+        if (!after) {
             preload.remove();
-        //Search stats
             searchResults.appendChild(makeStatsCard(searchResult));
+        } else {
+            let pageIndicator = makePageIndicator(searchResult);
+            searchResults.appendChild(pageIndicator);
+        }
 
         //Setup page
         let resultContainer = makeResultContainer();
         searchResults.appendChild(resultContainer);
 
+        if (!after) {
             docCount = 0;
-        insertHits(resultContainer, searchResult["hits"]["hits"]);
+        }
-
+        reachedEnd = hits.length !== SIZE;
+        insertHits(resultContainer, hits);
         searchBusy = false;
     });
 }

web/search.html

@@ -11,7 +11,7 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">v1.2.5</span>
+    <span class="badge badge-pill version">v1.2.6</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <a style="margin-left: auto" id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
 </nav>