(breaking) Upgrade path filter bar

schema/mappings.json

@@ -4,14 +4,14 @@
       "type": "keyword",
       "doc_values": true
     },
+    "_depth": {
+      "type": "integer"
+    },
     "path": {
       "type": "text",
       "analyzer": "path_analyzer",
-      "copy_to": "suggest_path"
-    },
-    "suggest_path": {
-      "type": "completion",
-      "analyzer": "case_insensitive_kw_analyzer"
+      "fielddata": true,
+      "index_prefixes": {}
     },
     "mime": {
       "type": "keyword"

schema/pipeline.json

@@ -3,7 +3,7 @@
   "processors": [
     {
       "script": {
-        "source": "ctx._tie = ctx._id; ctx._depth = ctx.path.length() - ctx.path.replace(\"/\", \"\").length();"
+        "source": "ctx._tie = ctx._id; ctx._depth = ctx.path.length() == 0 ? 0 : 1 + ctx.path.length() - ctx.path.replace(\"/\", \"\").length();"
       }
     }
   ]

src/index/elastic.c

@@ -122,6 +122,21 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) {
     return buf;
 }
 
+void *print_errors(response_t *r) {
+    cJSON *ret_json = cJSON_Parse(r->body);
+    if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) {
+        cJSON *err;
+        cJSON_ArrayForEach(err, cJSON_GetObjectItem(ret_json, "items")) {
+            if (cJSON_GetObjectItem(cJSON_GetObjectItem(err, "index"), "status")->valueint != 201) {
+                char *str = cJSON_Print(err);
+                LOG_ERRORF("elastic.c", "%s\n", str);
+                cJSON_free(str);
+            }
+        }
+    }
+    cJSON_Delete(ret_json);
+}
+
 void _elastic_flush(int max) {
     size_t buf_len;
     int count;
@@ -156,24 +171,13 @@ void _elastic_flush(int max) {
         return;
 
     } else if (r->status_code != 200) {
-        cJSON *ret_json = cJSON_Parse(r->body);
-        if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) {
-            cJSON *err;
-            cJSON_ArrayForEach(err, cJSON_GetObjectItem(ret_json, "items")) {
-                if (cJSON_GetObjectItem(cJSON_GetObjectItem(err, "index"), "status")->valueint != 201) {
-                    char *str = cJSON_Print(err);
-                    LOG_ERRORF("elastic.c", "%s\n", str);
-                    cJSON_free(str);
-                }
-            }
-        }
-
-        cJSON_Delete(ret_json);
+        print_errors(r);
         delete_queue(Indexer->queued);
 
     } else {
-        LOG_INFOF("elastic.c", "Indexed %d documents (%zukB) <%d>", count, buf_len / 1024, r->status_code);
 
+        print_errors(r);
+        LOG_INFOF("elastic.c", "Indexed %d documents (%zukB) <%d>", count, buf_len / 1024, r->status_code);
         delete_queue(max);
 
         if (Indexer->queued != 0) {

src/web/serve.c

@@ -243,6 +243,7 @@ int search(UNUSED(void *p), onion_request *req, onion_response *res) {
     if (r->status_code == 200) {
         onion_response_write(res, r->body, r->size);
     } else {
+        sist_log("serve.c", SIST_WARNING, "ElasticSearch error during query");
         onion_response_set_code(res, HTTP_INTERNAL_ERROR);
     }
 

web/css/auto-complete.min.css

@@ -1,9 +0,0 @@
-.autocomplete-suggestions {
-    text-align: left; cursor: default; border: 1px solid #ccc; border-top: 0; background: #fff; box-shadow: -1px 1px 3px rgba(0,0,0,.1);
-
-    /* core styles should not be changed */
-    position: absolute; display: none; z-index: 9999; max-height: 254px; overflow: hidden; overflow-y: auto; box-sizing: border-box;
-}
-.autocomplete-suggestion { position: relative; padding: 0 .6em; line-height: 23px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; font-size: 1.02em; color: #333; }
-.autocomplete-suggestion b { font-weight: normal; color: #1f8dd6; }
-.autocomplete-suggestion.selected { background: #f0f0f0; }

web/css/dark.css

@@ -449,3 +449,11 @@ option {
 .input-group > .custom-select:not(:first-child), .input-group > .form-control:not(:first-child) {
     border-right: none;
 }
+
+#pathTree .title {
+    cursor: pointer;
+}
+
+svg {
+    fill: white;
+}

web/css/light.css

@@ -310,3 +310,7 @@ mark {
 .input-group > .custom-select:not(:first-child), .input-group > .form-control:not(:first-child) {
     border-right: none;
 }
+
+#pathTree .title {
+    cursor: pointer;
+}

web/js/search.js

@@ -12,6 +12,7 @@ let docCount = 0;
 let coolingDown = false;
 let searchBusy = true;
 let selectedIndices = [];
+let indexMap = {};
 
 const CONF = new Settings();
 
@@ -24,11 +25,11 @@ const _defaults = {
 function Settings() {
     this.options = {};
 
-    this._onUpdate = function() {
+    this._onUpdate = function () {
         $("#fuzzyToggle").prop("checked", this.options.fuzzy);
     }
 
-    this.load = function() {
+    this.load = function () {
         const raw = window.localStorage.getItem("options");
         if (raw === null) {
             this.options = _defaults;
@@ -39,7 +40,7 @@ function Settings() {
         this._onUpdate();
     }
 
-    this.save = function() {
+    this.save = function () {
         window.localStorage.setItem("options", JSON.stringify(this.options));
         this._onUpdate();
     }
@@ -90,8 +91,8 @@ function toggleFuzzy() {
 $.jsonPost("i").then(resp => {
 
     const urlIndices = (new URLSearchParams(location.search)).get("i");
-
     resp["indices"].forEach(idx => {
+        indexMap[idx.id] = idx.name;
         const opt = $("<option>")
             .attr("value", idx.id)
             .append(idx.name);
@@ -107,6 +108,8 @@ $.jsonPost("i").then(resp => {
         }
         $("#indices").append(opt);
     });
+
+    createPathTree("#pathTree");
 });
 
 function getDocumentInfo(id) {
@@ -133,6 +136,7 @@ function handleTreeClick(tree) {
     }
 }
 
+//TODO: filter based on selected indexes, sort mime types
 $.jsonPost("es", {
     aggs: {
         mimeTypes: {
@@ -183,11 +187,6 @@ $.jsonPost("es", {
     mimeTree.node("any").select();
 });
 
-function leafTag(tag) {
-    const tokens = tag.split(".");
-    return tokens[tokens.length - 1]
-}
-
 // Tags tree
 $.jsonPost("es", {
     aggs: {
@@ -249,31 +248,6 @@ function addTag(map, tag, id, count) {
     }
 }
 
-new autoComplete({
-    selector: '#pathBar',
-    minChars: 1,
-    delay: 400,
-    renderItem: function (item) {
-        return '<div class="autocomplete-suggestion" data-val="' + item + '">' + item + '</div>';
-    },
-    source: async function (term, suggest) {
-        term = term.toLowerCase();
-
-        const choices = await getPathChoices();
-
-        let matches = [];
-        for (let i = 0; i < choices.length; i++) {
-            if (~choices[i].toLowerCase().indexOf(term)) {
-                matches.push(choices[i]);
-            }
-        }
-        suggest(matches);
-    },
-    onSelect: function () {
-        searchDebounced();
-    }
-});
-
 function insertHits(resultContainer, hits) {
     for (let i = 0; i < hits.length; i++) {
 
@@ -406,8 +380,8 @@ function search(after = null) {
     if (CONF.options.highlight) {
         q.highlight = {
             pre_tags: ["<mark>"],
-                post_tags: ["</mark>"],
-                fields: {
+            post_tags: ["</mark>"],
+            fields: {
                 content: {},
                 // "content.nGram": {},
                 name: {},
@@ -504,7 +478,7 @@ function updateIndices() {
 document.getElementById("indices").addEventListener("change", updateIndices);
 updateIndices();
 
-window.onkeyup = function(e) {
+window.onkeyup = function (e) {
     if (e.key === "/" || e.key === "Escape") {
         const bar = document.getElementById("searchBar");
         bar.scrollIntoView();
@@ -512,24 +486,104 @@ window.onkeyup = function(e) {
     }
 };
 
-//Suggest
-function getPathChoices() {
-    return new Promise(getPaths => {
-        $.jsonPost("es", {
-            suggest: {
-                path: {
-                    prefix: pathBar.value,
-                    completion: {
-                        field: "suggest_path",
-                        skip_duplicates: true,
-                        size: 10000
-                    }
+function getNextDepth(node) {
+    let q = {
+        query: {
+            bool: {
+                filter: [
+                    {term: {index: node.index}},
+                    {term: {_depth: node.depth + 1}}
+                ]
+            }
+        },
+        aggs: {
+            paths: {
+                terms: {
+                    field: "path",
+                    size: 10000
                 }
             }
-        }).then(resp => getPaths(resp["suggest"]["path"][0]["options"].map(opt => opt["_source"]["path"])));
+        },
+        size: 0
+    }
+
+    if (node.depth > 0) {
+        q.query.bool.must = {
+            prefix: {
+                path: node.id,
+            }
+        };
+    }
+
+    return $.jsonPost("es", q).then(resp => {
+        const buckets = resp["aggregations"]["paths"]["buckets"];
+        if (!buckets) {
+            return false;
+        }
+        return buckets
+            .filter(bucket => bucket.key.length > node.id.length || node.id.startsWith("/"))
+            .sort((a, b) => a.key > b.key)
+            .map(bucket => {
+                const i = bucket.key.lastIndexOf("/");
+                const name = (i === -1 || i === 1) ? bucket.key : bucket.key.slice(i + 1);
+
+                return {
+                    id: bucket.key,
+                    text: `${name}/ (${bucket.doc_count})`,
+                    depth: node.depth + 1,
+                    index: node.index,
+                    children: true,
+                }
+            })
     })
 }
 
+function handlePathTreeClick(tree) {
+    return (event, node, handler) => {
+
+        if (node.depth !== 0) {
+            $("#pathBar").val(node.id)
+            $("#pathTreeModal").modal("hide")
+            searchDebounced();
+        }
+
+        handler();
+    }
+}
+
+function createPathTree(target) {
+    let pathTree = new InspireTree({
+        data: function (node, resolve, reject) {
+            return getNextDepth(node);
+        }
+    });
+
+    selectedIndices.forEach(index => {
+        pathTree.addNode({
+            id: "/" + index,
+            text: `/[${indexMap[index]}]`,
+            index: index,
+            depth: 0,
+            children: true
+        })
+    })
+
+    new InspireTreeDOM(pathTree, {
+        target: target
+    });
+
+    pathTree.on("node.click", handlePathTreeClick(pathTree));
+
+    const button = document.querySelector("#pathBarHelper")
+    const tooltip = document.querySelector("#pathTreeTooltip")
+    console.log(button)
+    console.log(tooltip)
+    Popper.createPopper(button, tooltip ,{
+        trigger: "click",
+        placement: "right",
+    });
+}
+
 function updateSettings() {
     CONF.options.display = $("#settingDisplay").val();
     CONF.options.fuzzy = $("#settingFuzzy").prop("checked");

web/search.html

@@ -20,9 +20,18 @@
 <div class="container">
     <div class="card">
         <div class="card-body">
+
             <div class="form-group">
-                <input id="pathBar" type="search" class="form-control" placeholder="Filter path">
+                <div class="input-group">
+                    <div class="input-group-prepend">
+                        <button id="pathBarHelper" class="btn btn-outline-secondary" data-toggle="modal" data-target="#pathTreeModal">
+                            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" width="20px"><path d="M288 224h224a32 32 0 0 0 32-32V64a32 32 0 0 0-32-32H400L368 0h-80a32 32 0 0 0-32 32v64H64V8a8 8 0 0 0-8-8H40a8 8 0 0 0-8 8v392a16 16 0 0 0 16 16h208v64a32 32 0 0 0 32 32h224a32 32 0 0 0 32-32V352a32 32 0 0 0-32-32H400l-32-32h-80a32 32 0 0 0-32 32v64H64V128h192v64a32 32 0 0 0 32 32zm0 96h66.74l32 32H512v128H288zm0-288h66.74l32 32H512v128H288z"/></svg>
+                        </button>
+                    </div>
+                    <input id="pathBar" type="search" class="form-control" placeholder="Filter path">
+                </div>
             </div>
+
             <div class="input-group">
                 <div class="input-group-prepend">
                     <div class="input-group-text">
@@ -185,6 +194,22 @@
         </div>
     </div>
 
+    <div class="modal" id="pathTreeModal" tabindex="-1" role="dialog" aria-labelledby="modal-title" aria-hidden="true">
+        <div class="modal-dialog modal-lg" role="document">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h5 class="modal-title">Select path</h5>
+                    <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                        <span aria-hidden="true">&times;</span>
+                    </button>
+                </div>
+                <div class="modal-body">
+                    <div id="pathTree" class="tree"></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <div id="searchResults"></div>
 </div>