Read ES version, handle legacy versions, add notice & debug info

README.md

@@ -33,12 +33,11 @@ sist2 (Simple incremental search tool)
 
 ## Getting Started
 
-1. Have an Elasticsearch (>= 6.X.X) instance running
+1. Have an Elasticsearch (>= 6.8.X, ideally >=7.14.0) instance running
     1. Download [from official website](https://www.elastic.co/downloads/elasticsearch)
     1. *(or)* Run using docker:
         ```bash
-       docker run -d --name es1 --net sist2_net -p 9200:9200 \
+        docker run -d --name es1 -p 9201:9200 -e "discovery.type=single-node" elasticsearch:7.14.0
-            -e "discovery.type=single-node" elasticsearch:7.14.0
         ```
     1. *(or)* Run using docker-compose:
         ```yaml
@@ -52,7 +51,7 @@ sist2 (Simple incremental search tool)
     1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) *
     1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
        recommended!)*
-    1. *(or)* `docker pull simon987/sist2:2.11.2-x64-linux`
+    1. *(or)* `docker pull simon987/sist2:2.11.4-x64-linux`
 
 1. See [Usage guide](docs/USAGE.md)
 

docs/USAGE.md

@@ -14,6 +14,7 @@
     * [examples](#web-examples)
     * [rewrite_url](#rewrite_url)
     * [link to specific indices](#link-to-specific-indices)
+* [elasticsearch](#elasticsearch)
 * [exec-script](#exec-script)
 * [tagging](#tagging)
 * [sidecar files](#sidecar-files)
@@ -266,9 +267,20 @@ sist2 web index1 index2 index3 index4
 When the `rewrite_url` field is not empty, the web module ignores the `root`
 field and will return a HTTP redirect to `<rewrite_url><path>/<name><extension>`
 instead of serving the file from disk. 
-Both the `root` and `rewrite_url` fields are safe to manually modify from the 
+Both the `root` and `rewrite_url` fields are safe to manually modify from the 
 `descriptor.json` file.
 
+# Elasticsearch
+
+Elasticsearch versions >=6.8.0, <8.0.0 are supported by sist2. 
+
+Using a version >=7.14.0 is recommended to enable the following features:
+
+- Bug fix for large documents (See #198)
+
+When using a legacy version of ES, a notice will be displayed next to the sist2 version in the web UI.
+If you don't care about the features above, you can ignore it or disable it in the configuration page.
+
 ## exec-script
 
 The `exec-script` command is used to execute a user script for an index that has already been imported to Elasticsearch with the `index` command. Note that the documents will not be reset to their default state before each execution as the `index` command does: if you make undesired changes to the documents by accident, you will need to run `index` again to revert to the original state.

scripts/index_static.py

@@ -3,6 +3,7 @@ import json
 files = [
     "schema/mappings.json",
     "schema/settings.json",
+    "schema/settings_legacy.json",
     "schema/pipeline.json",
 ]
 

sist2-vue/src/Sist2Query.ts

@@ -73,6 +73,8 @@ class Sist2Query {
         const selectedMimeTypes = getters.selectedMimeTypes;
         const selectedTags = getters.selectedTags;
 
+        const legacyES = store.state.sist2Info.esVersionLegacy;
+
         const filters = [
             {terms: {index: selectedIndexIds}}
         ] as any[];
@@ -187,9 +189,13 @@ class Sist2Query {
                     "name.nGram": {},
                     "content.nGram": {},
                     font_name: {},
-                },
+                }
-                max_analyzed_offset: 9_999_999
             };
+
+            if (!legacyES) {
+                q.highlight.max_analyzed_offset = 9_999_999;
+            }
+
             if (getters.optSearchInPath) {
                 q.highlight.fields["path.text"] = {};
                 q.highlight.fields["path.nGram"] = {};

sist2-vue/src/components/DebugInfo.vue

@@ -5,7 +5,6 @@
 
     <b-card-body>
 
-      <!-- TODO: ES connectivity, Link to GH page -->
       <b-table :items="tableItems" small borderless responsive="md" thead-class="hidden" class="mb-0"></b-table>
 
       <hr />
@@ -32,6 +31,9 @@ export default {
         {key: "esIndex", value: this.$store.state.sist2Info.esIndex},
         {key: "tagline", value: this.$store.state.sist2Info.tagline},
         {key: "dev", value: this.$store.state.sist2Info.dev},
+        {key: "esVersion", value: this.$store.state.sist2Info.esVersion},
+        {key: "esVersionSupported", value: this.$store.state.sist2Info.esVersionSupported},
+        {key: "esVersionLegacy", value: this.$store.state.sist2Info.esVersionLegacy},
       ]
     }
   }

sist2-vue/src/components/NavBar.vue

@@ -8,7 +8,8 @@
     </b-navbar-brand>
 
     <span class="badge badge-pill version" v-if="$store && $store.state.sist2Info">
-      v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span>
+      v{{ sist2Version() }}<span v-if="isDebug()">-dbg</span><span v-if="isLegacy() && !hideLegacy()">-<a
+        href="https://github.com/simon987/sist2/blob/master/docs/USAGE.md#elasticsearch" target="_blank">legacyES</a></span>
     </span>
 
     <span v-if="$store && $store.state.sist2Info" class="tagline" v-html="tagline()"></span>
@@ -20,6 +21,7 @@
 
 <script>
 import Sist2Icon from "@/components/Sist2Icon";
+
 export default {
   name: "NavBar",
   components: {Sist2Icon},
@@ -32,6 +34,12 @@ export default {
     },
     isDebug() {
       return this.$store.state.sist2Info.debug;
+    },
+    isLegacy() {
+      return this.$store.state.sist2Info.esVersionLegacy;
+    },
+    hideLegacy() {
+      return this.$store.state.optHideLegacy;
     }
   }
 }

sist2-vue/src/i18n/messages.ts

@@ -63,7 +63,8 @@ export default {
             slideDuration: "Slide duration",
             resultSize: "Number of results per page",
             tagOrOperator: "Use OR operator when specifying multiple tags.",
-            hideDuplicates: "Hide duplicate results based on checksum"
+            hideDuplicates: "Hide duplicate results based on checksum",
+            hideLegacy: "Hide the 'legacyES' Elasticsearch notice"
         },
         queryMode: {
             simple: "Simple",
@@ -211,7 +212,8 @@ export default {
             slideDuration: "Durée des diapositives",
             resultSize: "Nombre de résultats par page",
             tagOrOperator: "Utiliser l'opérateur OU lors de la spécification de plusieurs tags",
-            hideDuplicates: "Masquer les résultats en double"
+            hideDuplicates: "Masquer les résultats en double",
+            hideLegacy: "Masquer la notice 'legacyES' Elasticsearch"
         },
         queryMode: {
             simple: "Simple",

sist2-vue/src/store/index.ts

@@ -46,6 +46,7 @@ export default new Vuex.Store({
         optTreemapColor: "PuBuGn",
         optLightboxLoadOnlyCurrent: false,
         optLightboxSlideDuration: 15,
+        optHideLegacy: false,
 
         _onLoadSelectedIndices: [] as string[],
         _onLoadSelectedMimeTypes: [] as string[],
@@ -144,6 +145,7 @@ export default new Vuex.Store({
         setOptTreemapColorGroupingDepth: (state, val) => state.optTreemapColorGroupingDepth = val,
         setOptTreemapSize: (state, val) => state.optTreemapSize = val,
         setOptTreemapColor: (state, val) => state.optTreemapColor = val,
+        setOptHideLegacy: (state, val) => state.optHideLegacy = val,
 
         setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
 
@@ -339,5 +341,6 @@ export default new Vuex.Store({
         optLightboxLoadOnlyCurrent: state => state.optLightboxLoadOnlyCurrent,
         optLightboxSlideDuration: state => state.optLightboxSlideDuration,
         optResultSize: state => state.size,
+        optHideLegacy: state => state.optHideLegacy,
     }
 })

sist2-vue/src/views/Configuration.vue

@@ -19,6 +19,10 @@
             {{ $t("opt.lightboxLoadOnlyCurrent") }}
           </b-form-checkbox>
 
+          <b-form-checkbox :checked="optHideLegacy" @input="setOptHideLegacy">
+            {{ $t("opt.hideLegacy") }}
+          </b-form-checkbox>
+
           <label>{{ $t("opt.lang") }}</label>
           <b-form-select :options="langOptions" :value="optLang" @input="setOptLang"></b-form-select>
 
@@ -215,6 +219,7 @@ export default {
       "optTagOrOperator",
       "optLang",
       "optHideDuplicates",
+      "optHideLegacy",
     ]),
     clientWidth() {
       return window.innerWidth;
@@ -254,7 +259,8 @@ export default {
       "setOptResultSize",
       "setOptTagOrOperator",
       "setOptLang",
-      "setOptHideDuplicates"
+      "setOptHideDuplicates",
+      "setOptHideLegacy"
     ]),
     onResetClick() {
       localStorage.removeItem("sist2_configuration");

src/ctx.h

@@ -17,6 +17,7 @@
 #include "libscan/wpd/wpd.h"
 #include "libscan/json/json.h"
 #include "src/io/store.h"
+#include "src/index/elastic.h"
 
 #include <glib.h>
 #include <pcre.h>
@@ -75,6 +76,7 @@ typedef struct {
 
 typedef struct {
     char *es_url;
+    es_version_t *es_version;
     char *es_index;
     int batch_size;
     tpool_t *pool;
@@ -86,6 +88,7 @@ typedef struct {
 
 typedef struct {
     char *es_url;
+    es_version_t *es_version;
     char *es_index;
     int index_count;
     char *auth_user;

src/index/elastic.c

@@ -356,8 +356,66 @@ void finish_indexer(char *script, int async_script, char *index_id) {
     free_response(r);
 }
 
+es_version_t *elastic_get_version(const char *es_url) {
+    response_t *r = web_get(es_url, 30);
+
+    char *tmp = malloc(r->size + 1);
+    memcpy(tmp, r->body, r->size);
+    *(tmp + r->size) = '\0';
+    cJSON *response = cJSON_Parse(tmp);
+    free(tmp);
+    free_response(r);
+
+    if (response == NULL) {
+        return NULL;
+    }
+
+    if (cJSON_GetObjectItem(response, "version") == NULL ||
+        cJSON_GetObjectItem(cJSON_GetObjectItem(response, "version"), "number") == NULL) {
+        cJSON_Delete(response);
+        return NULL;
+    }
+
+    char *version_str = cJSON_GetObjectItem(cJSON_GetObjectItem(response, "version"), "number")->valuestring;
+
+    es_version_t *version = malloc(sizeof(es_version_t));
+
+    const char *tok = strtok(version_str, ".");
+    version->major = atoi(tok);
+    tok = strtok(NULL, ".");
+    version->minor = atoi(tok);
+    tok = strtok(NULL, ".");
+    version->patch = atoi(tok);
+
+    cJSON_Delete(response);
+
+    return version;
+}
+
 void elastic_init(int force_reset, const char *user_mappings, const char *user_settings) {
 
+    es_version_t *es_version = elastic_get_version(IndexCtx.es_url);
+    IndexCtx.es_version = es_version;
+
+    if (es_version == NULL) {
+        LOG_FATAL("elastic.c", "Could not get ES version")
+    }
+
+    LOG_INFOF("elastic.c",
+              "Elasticsearch version is %s (supported=%d, legacy=%d)",
+              format_es_version(es_version), IS_SUPPORTED_ES_VERSION(es_version), USE_LEGACY_ES_SETTINGS(es_version));
+
+    if (!IS_SUPPORTED_ES_VERSION(es_version)) {
+        LOG_FATAL("elastic.c", "sist2 only supports Elasticsearch v6.8 or newer")
+    }
+
+    char *settings = NULL;
+    if (USE_LEGACY_ES_SETTINGS(es_version)) {
+        settings = settings_json;
+    } else {
+        settings = settings_legacy_json;
+    }
+
     // Check if index exists
     char url[4096];
     snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index);
@@ -392,7 +450,7 @@ void elastic_init(int force_reset, const char* user_mappings, const char* user_s
         free_response(r);
 
         snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index);
-        r = web_put(url, user_settings ? user_settings : settings_json);
+        r = web_put(url, user_settings ? user_settings : settings);
         LOG_INFOF("elastic.c", "Update ES settings <%d>", r->status_code);
         if (r->status_code != 200) {
             print_error(r);

src/index/elastic.h

@@ -9,6 +9,26 @@ typedef struct es_bulk_line {
     char line[0];
 } es_bulk_line_t;
 
+typedef struct {
+    int major;
+    int minor;
+    int patch;
+} es_version_t;
+
+#define VERSION_GE(version, maj, min) ((version)->major > (maj) || ((version)->major == (maj) && (version)->minor >= (min)))
+#define IS_SUPPORTED_ES_VERSION(es_version) VERSION_GE((es_version), 6, 8)
+#define USE_LEGACY_ES_SETTINGS(es_version) (!VERSION_GE((es_version), 7, 14))
+
+__always_inline
+static const char *format_es_version(es_version_t *version) {
+    static char buf[64];
+
+    snprintf(buf, sizeof(buf), "%d.%d.%d", version->major, version->minor, version->patch);
+
+    return buf;
+}
+
+
 /**
  * Note: indexer is *not* thread safe
  */
@@ -31,6 +51,8 @@ cJSON *elastic_get_document(const char *id_str);
 
 char *elastic_get_status();
 
+es_version_t *elastic_get_version(const char *es_url);
+
 void execute_update_script(const char *script, int async, const char index_id[MD5_STR_LENGTH]);
 
 #endif

src/sist.h

@@ -51,7 +51,7 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "2.11.3"
+#define VERSION "2.11.4"
 static const char *const Version = VERSION;
 
 #ifndef SIST_PLATFORM

src/web/serve.c

@@ -252,12 +252,32 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
     mg_http_serve_file(nc, hm, full_path, mime, disposition);
 }
 
+void cache_es_version() {
+    static int is_cached = FALSE;
+
+    if (is_cached == TRUE) {
+        return;
+    }
+
+    es_version_t *es_version = elastic_get_version(WebCtx.es_url);
+    if (es_version != NULL) {
+        WebCtx.es_version = es_version;
+        is_cached = TRUE;
+    }
+}
+
 void index_info(struct mg_connection *nc) {
+
+    cache_es_version();
+
     cJSON *json = cJSON_CreateObject();
     cJSON *arr = cJSON_AddArrayToObject(json, "indices");
 
     cJSON_AddStringToObject(json, "esIndex", WebCtx.es_index);
     cJSON_AddStringToObject(json, "version", Version);
+    cJSON_AddStringToObject(json, "esVersion", format_es_version(WebCtx.es_version));
+    cJSON_AddBoolToObject(json, "esVersionSupported", IS_SUPPORTED_ES_VERSION(WebCtx.es_version));
+    cJSON_AddBoolToObject(json, "esVersionLegacy", USE_LEGACY_ES_SETTINGS(WebCtx.es_version));
     cJSON_AddStringToObject(json, "platform", QUOTE(SIST_PLATFORM));
     cJSON_AddStringToObject(json, "sist2Hash", Sist2CommitHash);
     cJSON_AddStringToObject(json, "libscanHash", LibScanCommitHash);