Add option to search in path for sqlite #402

README.md

@@ -4,6 +4,8 @@
 
 **Demo**: [sist2.simon987.net](https://sist2.simon987.net/)
 
+**Community URL:** [Discord](https://discord.gg/2PEjDy3Rfs)
+
 # sist2
 
 sist2 (Simple incremental search tool)
@@ -46,7 +48,7 @@ services:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
   sist2-admin:
-    image: simon987/sist2:3.3.3-x64-linux
+    image: simon987/sist2:3.3.4-x64-linux
     restart: unless-stopped
     volumes:
       - ./sist2-admin-data/:/sist2-admin/
@@ -153,7 +155,6 @@ indices, but it uses much less memory and is easier to set up.
 | Query syntax                                 |      [fts5](https://www.sqlite.org/fts5.html)       | [query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) |
 | Fuzzy search                                 |                                                     |                                                                   ✓                                                                   |
 | Media Types tree real-time updating          |                                                     |                                                                   ✓                                                                   |
-| Search in file `path`                        | [WIP](https://github.com/simon987/sist2/issues/402) |                                                                   ✓                                                                   |
 | Manual tagging                               |                          ✓                          |                                                                   ✓                                                                   |
 | User scripts                                 |                          ✓                          |                                                                   ✓                                                                   |
 | Media Type breakdown for search results      |                                                     |                                                                   ✓                                                                   |

sist2-vue/src/Sist2Api.js

@@ -33,18 +33,6 @@ class Sist2Api {
 
     getSist2Info() {
         return axios.get(`${this.baseUrl}i`).then(resp => {
-            const indices = resp.data.indices;
-
-            resp.data.indices = indices.map(idx => {
-                return {
-                    id: idx.id,
-                    name: idx.name,
-                    timestamp: idx.timestamp,
-                    version: idx.version,
-                    models: idx.models,
-                };
-            });
-
             this.sist2Info = resp.data;
 
             return resp.data;
@@ -155,6 +143,12 @@ class Sist2Api {
         }
     }
 
+    _getIndexRoot(indexId) {
+        console.log(indexId)
+        console.log(this.sist2Info.indices.find(idx => idx.id === indexId))
+        return this.sist2Info.indices.find(idx => idx.id === indexId).root;
+    }
+
     esQuery(query) {
         return axios.post(`${this.baseUrl}es`, query).then(resp => {
             const res = resp.data;
@@ -163,6 +157,7 @@ class Sist2Api {
                 res.hits.hits.forEach((hit) => {
                     hit["_source"]["name"] = strUnescape(hit["_source"]["name"]);
                     hit["_source"]["path"] = strUnescape(hit["_source"]["path"]);
+                    hit["_source"]["indexRoot"] = this._getIndexRoot(hit["_source"]["index"]);
 
                     this.setHitProps(hit);
                     this.setHitTags(hit);

sist2-vue/src/Sist2SqliteQuery.js

@@ -106,6 +106,8 @@ class Sist2ElasticsearchQuery {
             q["sortAsc"] = true;
         }
 
+        q["searchInPath"] = getters.optSearchInPath;
+
         return q;
     }
 }

sist2-vue/src/views/Configuration.vue

@@ -136,7 +136,7 @@
                         {{ $t("opt.fuzzy") }}
                     </b-form-checkbox>
 
-                    <b-form-checkbox :disabled="uiSqliteMode" :checked="optSearchInPath" @input="setOptSearchInPath">{{
+                    <b-form-checkbox :checked="optSearchInPath" @input="setOptSearchInPath">{{
                         $t("opt.searchInPath")
                         }}
                     </b-form-checkbox>

src/database/database_fts.c

@@ -160,7 +160,8 @@ void database_fts_index(database_t *db) {
 
     CRASH_IF_NOT_SQLITE_OK(sqlite3_exec(
             db->db,
-            "INSERT INTO search(rowid, name, content, title) SELECT id, name, content, title from document_view",
+            "INSERT INTO search(rowid, name, content, title, path) "
+            "SELECT id, name, content, title, path from document_view",
             NULL, NULL, NULL));
 }
 

src/database/database_schema.c

@@ -75,23 +75,25 @@ const char *FtsDatabaseSchema =
         "  WHERE id = OLD.id;"
         " END;"
         ""
-        "CREATE VIEW IF NOT EXISTS document_view (id, name, content, title)"
+        "CREATE VIEW IF NOT EXISTS document_view (id, name, content, title, path)"
         " AS"
         " SELECT id,"
         "  json_data->>'name',"
         "  json_data->>'content',"
-        "  json_data->>'title'"
+        "  json_data->>'title',"
+        "  json_data->>'path'"
         " FROM document_index;"
         ""
         "CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5 ("
         "   name,"
         "   content,"
         "   title,"
+        "   path,"
         "   content='document_view',"
         "   content_rowid='id'"
         ");"
-        // name^8, content^3, title^8
-        "INSERT INTO search(search, rank) VALUES('rank', 'bm25(8, 3, 8)');"
+        // name^8, content^3, title^8, path^5
+        "INSERT INTO search(search, rank) VALUES('rank', 'bm25(8, 3, 8, 5)');"
         "";
 
 const char *IpcDatabaseSchema =

src/index/web.c

@@ -90,6 +90,7 @@ subreq_ctx_t *web_post_async(const char *url, char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, req->curl_err_buffer);
@@ -123,6 +124,7 @@ response_t *web_get(const char *url, int timeout, int insecure) {
     curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     struct curl_slist *headers = NULL;
@@ -162,6 +164,7 @@ response_t *web_post(const char *url, const char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     char err_buffer[CURL_ERROR_SIZE + 1] = {};
@@ -207,6 +210,7 @@ response_t *web_put(const char *url, const char *data, int insecure) {
     curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4);
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     struct curl_slist *headers = NULL;
@@ -241,6 +245,7 @@ response_t *web_delete(const char *url, int insecure) {
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     if (insecure) {
         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
     }
 
     curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "");

src/sist.h

@@ -51,11 +51,11 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "3.3.3"
+#define VERSION "3.4.0"
 static const char *const Version = VERSION;
 static const int VersionMajor = 3;
-static const int VersionMinor = 3;
-static const int VersionPatch = 3;
+static const int VersionMinor = 4;
+static const int VersionPatch = 0;
 
 #ifndef SIST_PLATFORM
 #define SIST_PLATFORM unknown

src/web/serve.c

@@ -88,7 +88,7 @@ void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
 
     memcpy(index_id_str, hm->uri.ptr + 3, 8);
     *(index_id_str + 8) = '\0';
-    int index_id = (int)strtol(index_id_str, NULL, 16);
+    int index_id = (int) strtol(index_id_str, NULL, 16);
 
     memcpy(arg_stat_type, hm->uri.ptr + 3 + 9, 4);
     *(arg_stat_type + sizeof(arg_stat_type) - 1) = '\0';
@@ -368,6 +368,10 @@ void index_info(struct mg_connection *nc) {
         cJSON_AddNumberToObject(idx_json, "timestamp", (double) idx->desc.timestamp);
         cJSON_AddItemToArray(arr, idx_json);
 
+#ifdef SIST_DEBUG_INFO
+        cJSON_AddStringToObject(idx_json, "root", idx->desc.root);
+#endif
+
         cJSON *models = database_get_models(idx->db);
         cJSON_AddItemToObject(idx_json, "models", models);
     }
@@ -480,7 +484,7 @@ tag_req_t *parse_tag_request(cJSON *json) {
     return req;
 }
 
-subreq_ctx_t *elastic_delete_tag(const char* sid, const tag_req_t *req) {
+subreq_ctx_t *elastic_delete_tag(const char *sid, const tag_req_t *req) {
     char *buf = malloc(sizeof(char) * 8192);
     snprintf(buf, 8192,
              "{"
@@ -500,7 +504,7 @@ subreq_ctx_t *elastic_delete_tag(const char* sid, const tag_req_t *req) {
     return web_post_async(url, buf, WebCtx.es_insecure_ssl);
 }
 
-subreq_ctx_t *elastic_write_tag(const char* sid, const tag_req_t *req) {
+subreq_ctx_t *elastic_write_tag(const char *sid, const tag_req_t *req) {
     char *buf = malloc(sizeof(char) * 8192);
     snprintf(buf, 8192,
              "{"

src/web/web_fts.c

@@ -179,7 +179,8 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
 
     json_value req_query, req_path, req_size_min, req_size_max, req_date_min, req_date_max, req_page_size,
             req_index_ids, req_mime_types, req_tags, req_sort_asc, req_sort, req_seed, req_after,
-            req_fetch_aggregations, req_highlight, req_highlight_context_size, req_embedding, req_model;
+            req_fetch_aggregations, req_highlight, req_highlight_context_size, req_embedding, req_model,
+            req_search_in_path;
 
     if (!cJSON_IsObject(json) ||
         (req_query = get_json_string(json, "query")).invalid ||
@@ -197,6 +198,7 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
         (req_index_ids = get_json_number_array(json, "indexIds")).invalid ||
         (req_mime_types = get_json_array(json, "mimeTypes")).invalid ||
         (req_highlight = get_json_bool(json, "highlight")).invalid ||
+        (req_search_in_path = get_json_bool(json, "searchInPath")).invalid ||
         (req_highlight_context_size = get_json_number(json, "highlightContextSize")).invalid ||
         (req_embedding = get_json_number_array(json, "embedding")).invalid ||
         (req_model = get_json_number(json, "model")).invalid ||
@@ -252,7 +254,6 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
     fts_search_req_t *req = malloc(sizeof(fts_search_req_t));
 
     req->sort = sort;
-    req->query = req_query.val ? strdup(req_query.val->valuestring) : NULL;
     req->path = req_path.val ? strdup(req_path.val->valuestring) : NULL;
     req->size_min = req_size_min.val ? req_size_min.val->valuedouble : 0;
     req->size_max = req_size_max.val ? req_size_max.val->valuedouble : 0;
@@ -271,6 +272,16 @@ fts_search_req_t *get_search_req(struct mg_http_message *hm) {
                                   ? req_highlight_context_size.val->valueint
                                   : DEFAULT_HIGHLIGHT_CONTEXT_SIZE;
     req->model = req_model.val ? req_model.val->valueint : 0;
+
+    if (req_search_in_path.val->valueint == FALSE && req_query.val) {
+        if (asprintf(&req->query, "- path : %s", req_query.val->valuestring) == -1) {
+            cJSON_Delete(json);
+            return NULL;
+        }
+    } else {
+        req->query = req_query.val ? strdup(req_query.val->valuestring) : NULL;
+    }
+
     req->embedding = req_model.val
                      ? get_float_buffer(req_embedding.val, &req->embedding_size)
                      : NULL;