Rework document IDs

.gitmodules

@@ -7,6 +7,3 @@
 [submodule "third-party/libscan/third-party/antiword"]
 	path = third-party/libscan/third-party/antiword
 	url = https://github.com/simon987/antiword
-[submodule "third-party/libscan/third-party/libmobi"]
-	path = third-party/libscan/third-party/libmobi
-	url = https://github.com/bfabiszewski/libmobi

CMakeLists.txt

@@ -4,7 +4,6 @@ set(CMAKE_C_STANDARD 11)
 project(sist2 C)
 
 option(SIST_DEBUG "Build a debug executable" on)
-option(SIST_FAST "Enable more optimisation flags" off)
 option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0)
 
 add_compile_definitions(
@@ -55,10 +54,6 @@ find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
-find_library(MAGIC_LIB
-        NAMES libmagic.so.1 magic
-        PATHS /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/
-)
 
 
 target_include_directories(
@@ -98,25 +93,16 @@ if (SIST_DEBUG)
             PROPERTIES
             OUTPUT_NAME sist2_debug
     )
-elseif (SIST_FAST)
-    target_compile_options(
-            sist2
-            PRIVATE
-
-            -Ofast
-            -march=native
-            -fno-stack-protector
-            -fomit-frame-pointer
-            -freciprocal-math
-    )
 else ()
     target_compile_options(
             sist2
             PRIVATE
 
             -Ofast
+            #-march=native
             -fno-stack-protector
             -fomit-frame-pointer
+            #-freciprocal-math
     )
 endif ()
 
@@ -138,12 +124,13 @@ target_link_libraries(
         CURL::libcurl
 
         pthread
+        #magic
 
         c
 
         scan
 
-        ${MAGIC_LIB}
+        /usr/lib/x86_64-linux-gnu/libmagic.so.1
 )
 
 add_custom_target(

README.md

@@ -52,7 +52,7 @@ sist2 (Simple incremental search tool)
 Select the file corresponding to your CPU architecture and mark the binary as executable with `chmod +x` *
     2. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
        recommended!)*
-    3. *(or)* `docker pull simon987/sist2:2.12.0-x64-linux`
+    3. *(or)* `docker pull simon987/sist2:2.11.7-x64-linux`
 
 1. See [Usage guide](docs/USAGE.md)
 

schema/settings.json

@@ -3,7 +3,7 @@
     "refresh_interval": "30s",
     "codec": "best_compression",
     "number_of_replicas": 0,
-    "highlight.max_analyzed_offset": 1000000
+    "highlight.max_analyzed_offset": 10000000
   },
   "analysis": {
     "tokenizer": {

sist2-vue/package-lock.json

@@ -3288,9 +3288,9 @@
       }
     },
     "node_modules/async": {
-      "version": "2.6.4",
-      "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz",
-      "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==",
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
+      "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
       "dev": true,
       "dependencies": {
         "lodash": "^4.17.14"
@@ -9736,9 +9736,9 @@
       }
     },
     "node_modules/minimist": {
-      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
-      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
       "dev": true
     },
     "node_modules/minipass": {
@@ -14092,9 +14092,9 @@
       }
     },
     "node_modules/url-parse": {
-      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.4.tgz",
+      "integrity": "sha512-ITeAByWWoqutFClc/lRZnFplgXgEZr3WJ6XngMM/N9DMIm4K8zXPCZ1Jdu0rERwO84w1WC5wkle2ubwTA4NTBg==",
       "dev": true,
       "dependencies": {
         "querystringify": "^2.1.1",
@@ -17937,9 +17937,9 @@
       "dev": true
     },
     "async": {
-      "version": "2.6.4",
-      "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz",
-      "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==",
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
+      "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
       "dev": true,
       "requires": {
         "lodash": "^4.17.14"
@@ -23324,9 +23324,9 @@
       }
     },
     "minimist": {
-      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
-      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
       "dev": true
     },
     "minipass": {
@@ -27008,9 +27008,9 @@
       }
     },
     "url-parse": {
-      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.4.tgz",
+      "integrity": "sha512-ITeAByWWoqutFClc/lRZnFplgXgEZr3WJ6XngMM/N9DMIm4K8zXPCZ1Jdu0rERwO84w1WC5wkle2ubwTA4NTBg==",
       "dev": true,
       "requires": {
         "querystringify": "^2.1.1",

sist2-vue/src/Sist2Api.ts

@@ -336,6 +336,10 @@ class Sist2Api {
         };
     }
 
+    getDocInfo(docId: string) {
+        return axios.get(`${this.baseUrl}d/${docId}`);
+    }
+
     getTags() {
         return this.esQuery({
             aggs: {

sist2-vue/src/Sist2Query.ts

@@ -69,7 +69,7 @@ interface SortMode {
 
 class Sist2Query {
 
-    searchQuery(blankSearch: boolean = false): any {
+    searchQuery(): any {
 
         const getters = store.getters;
 
@@ -93,6 +93,22 @@ class Sist2Query {
             {terms: {index: selectedIndexIds}}
         ] as any[];
 
+        if (sizeMin && sizeMax) {
+            filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
+        } else if (sizeMin) {
+            filters.push({range: {size: {gte: sizeMin}}})
+        } else if (sizeMax) {
+            filters.push({range: {size: {lte: sizeMax}}})
+        }
+
+        if (dateMin && dateMax) {
+            filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
+        } else if (dateMin) {
+            filters.push({range: {mtime: {gte: dateMin}}})
+        } else if (dateMax) {
+            filters.push({range: {mtime: {lte: dateMax}}})
+        }
+
         const fields = [
             "name^8",
             "content^3",
@@ -112,39 +128,20 @@ class Sist2Query {
             fields.push("name.nGram^3");
         }
 
-        if (!blankSearch) {
-            if (sizeMin && sizeMax) {
-                filters.push({range: {size: {gte: sizeMin, lte: sizeMax}}})
-            } else if (sizeMin) {
-                filters.push({range: {size: {gte: sizeMin}}})
-            } else if (sizeMax) {
-                filters.push({range: {size: {lte: sizeMax}}})
-            }
+        const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
+        if (path !== "") {
+            filters.push({term: {path: path}})
+        }
 
-            if (dateMin && dateMax) {
-                filters.push({range: {mtime: {gte: dateMin, lte: dateMax}}})
-            } else if (dateMin) {
-                filters.push({range: {mtime: {gte: dateMin}}})
-            } else if (dateMax) {
-                filters.push({range: {mtime: {lte: dateMax}}})
-            }
+        if (selectedMimeTypes.length > 0) {
+            filters.push({terms: {"mime": selectedMimeTypes}});
+        }
 
-            const path = pathText.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
-
-            if (path !== "") {
-                filters.push({term: {path: path}})
-            }
-
-            if (selectedMimeTypes.length > 0) {
-                filters.push({terms: {"mime": selectedMimeTypes}});
-            }
-
-            if (selectedTags.length > 0) {
-                if (getters.optTagOrOperator) {
-                    filters.push({terms: {"tag": selectedTags}});
-                } else {
-                    selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
-                }
+        if (selectedTags.length > 0) {
+            if (getters.optTagOrOperator) {
+                filters.push({terms: {"tag": selectedTags}});
+            } else {
+                selectedTags.forEach((tag: string) => filters.push({term: {"tag": tag}}));
             }
         }
 
@@ -185,7 +182,7 @@ class Sist2Query {
             size: size,
         } as any;
 
-        if (!empty && !blankSearch) {
+        if (!empty) {
             q.query.bool.must = query;
         }
 
@@ -210,7 +207,7 @@ class Sist2Query {
             };
 
             if (!legacyES) {
-                q.highlight.max_analyzed_offset = 999_999;
+                q.highlight.max_analyzed_offset = 9_999_999;
             }
 
             if (getters.optSearchInPath) {
@@ -240,7 +237,7 @@ class Sist2Query {
                 }
             }
 
-            if (!empty && !blankSearch) {
+            if (!empty) {
                 q.query.function_score.query.bool.must.push(query);
             }
         }

sist2-vue/src/components/InfoTable.vue

@@ -72,12 +72,6 @@ export default {
         }
       });
 
-      Object.keys(src).forEach(key => {
-        if (key.startsWith("mt_") || key.startsWith("int_")) {
-          items.push({key: key, value: src[key]});
-        }
-      });
-
       // Exif GPS
       if ("exif_gps_longitude_dec" in src) {
         items.push({

sist2-vue/src/components/LazyContentDiv.vue

@@ -1,13 +1,11 @@
 <template>
   <Preloader v-if="loading"></Preloader>
-  <div v-else-if="content" class="content-div" v-html="content"></div>
+  <div v-else-if="content" class="content-div">{{ content }}</div>
 </template>
 
 <script>
 import Sist2Api from "@/Sist2Api";
 import Preloader from "@/components/Preloader";
-import Sist2Query from "@/Sist2Query";
-import store from "@/store";
 
 export default {
   name: "LazyContentDiv",
@@ -20,72 +18,10 @@ export default {
     }
   },
   mounted() {
-    const query = Sist2Query.searchQuery();
-
-    if (this.$store.state.optHighlight) {
-
-      const fields = this.$store.state.fuzzy
-          ? {"content.nGram": {}}
-          : {content: {}};
-
-      query.highlight = {
-        pre_tags: ["<mark>"],
-        post_tags: ["</mark>"],
-        number_of_fragments: 0,
-        fields,
-      };
-
-      if (!store.state.sist2Info.esVersionLegacy) {
-        query.highlight.max_analyzed_offset = 999_999;
-      }
-    }
-
-    if ("function_score" in query.query) {
-      query.query = query.query.function_score.query;
-    }
-
-    if (!("must" in query.query.bool)) {
-      query.query.bool.must = [];
-    } else if (!Array.isArray(query.query.bool.must)) {
-      query.query.bool.must = [query.query.bool.must];
-    }
-
-    query.query.bool.must.push({match: {_id: this.docId}});
-
-    delete query["sort"];
-    delete query["aggs"];
-    delete query["search_after"];
-    delete query.query["function_score"];
-
-    query._source = {
-      includes: ["content", "name", "path", "extension"]
-    }
-
-    query.size = 1;
-
-    Sist2Api.esQuery(query).then(resp => {
+    Sist2Api.getDocInfo(this.docId).then(src => {
+      this.content = src.data.content;
       this.loading = false;
-      if (resp.hits.hits.length === 1) {
-        this.content = this.getContent(resp.hits.hits[0]);
-      } else {
-        console.log("FIXME: could not get content")
-        console.log(resp)
-      }
-    });
-  },
-  methods: {
-    getContent(doc) {
-      if (!doc.highlight) {
-        return doc._source.content;
-      }
-
-      if (doc.highlight["content.nGram"]) {
-        return doc.highlight["content.nGram"][0];
-      }
-      if (doc.highlight.content) {
-        return doc.highlight.content[0];
-      }
-    }
+    })
   }
 }
 </script>

sist2-vue/src/components/Lightbox.vue

@@ -81,9 +81,7 @@ export default {
   methods: {
     keyDownListener(e) {
 
-      const isLightboxOpen = this.$refs.lightbox === undefined || this.$refs.lightbox.$el.tagName === undefined;
-
-      if (isLightboxOpen) {
+      if (this.$refs.lightbox === undefined) {
         return true;
       }
 
@@ -91,6 +89,7 @@ export default {
 
       switch (e.key) {
         case " ": {
+          console.log("SPACE")
           e.preventDefault();
           e.stopPropagation();
           e.stopImmediatePropagation();
@@ -99,12 +98,16 @@ export default {
           [...document.getElementsByClassName("fslightbox-absoluted")].forEach(elem => {
             if (elem.style.transform === "translate(0px)" || elem.style.transform === "translate(0px, 0px)") {
               const vid = elem.getElementsByTagName("video")[0];
+              console.log(elem)
+              console.log(vid)
 
               if (vid) {
                 if (vid.paused) {
                   vid.play();
+                  console.log("PLAY")
                 } else {
                   vid.pause()
+                  console.log("PAUSE")
                 }
               }
             }
@@ -116,28 +119,24 @@ export default {
         }
         case "ArrowUp":
         case "k": {
-          if (!lightboxStore.data.isThumbing && lightboxStore.core.thumbsToggler) {
+          if (!lightboxStore.data.isThumbing) {
             lightboxStore.core.thumbsToggler.toggleThumbs();
           }
           return false;
         }
         case "ArrowDown":
         case "j": {
-          if (lightboxStore.data.isThumbing && lightboxStore.core.thumbsToggler) {
+          if (lightboxStore.data.isThumbing) {
             lightboxStore.core.thumbsToggler.toggleThumbs();
           }
           return false;
         }
         case "h": {
-          if (lightboxStore.core.stageManager.getPreviousSlideIndex) {
-            lightboxStore.core.slideIndexChanger.jumpTo(lightboxStore.core.stageManager.getPreviousSlideIndex());
-          }
-          return false;
+          lightboxStore.core.slideIndexChanger.jumpTo(lightboxStore.core.stageManager.getPreviousSlideIndex());
+          break;
         }
         case "l": {
-          if (lightboxStore.core.stageManager.getNextSlideIndex) {
-            lightboxStore.core.slideIndexChanger.jumpTo(lightboxStore.core.stageManager.getNextSlideIndex());
-          }
+          lightboxStore.core.slideIndexChanger.jumpTo(lightboxStore.core.stageManager.getNextSlideIndex());
           return false;
         }
       }

sist2-vue/src/components/TagPicker.vue

@@ -1,13 +1,5 @@
 <template>
-  <div>
-    <b-input-group v-if="showSearchBar" id="tag-picker-filter-bar">
-      <b-form-input :value="filter"
-                    :placeholder="$t('tagFilter')"
-                    @input="onFilter($event)"></b-form-input>
-    </b-input-group>
-
-    <div id="tagTree"></div>
-  </div>
+  <div id="tagTree"></div>
 </template>
 
 <script>
@@ -120,12 +112,10 @@ function addTag(map, tag, id, count) {
 
 export default {
   name: "TagPicker",
-  props: ["showSearchBar"],
   data() {
     return {
       tagTree: null,
       loadedFromArgs: false,
-      filter: ""
     }
   },
   mounted() {
@@ -139,10 +129,6 @@ export default {
     });
   },
   methods: {
-    onFilter(value) {
-      this.filter = value;
-      this.tagTree.search(value);
-    },
     initializeTree() {
       const tagMap = [];
       this.tagTree = new InspireTree({
@@ -177,8 +163,7 @@ export default {
       });
     },
     handleTreeClick(node, e) {
-      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused"
-          || e === "matched" || e === "hidden") {
+      if (e === "indeterminate" || e === "collapsed" || e === 'rendered' || e === "focused") {
         return;
       }
 
@@ -195,15 +180,7 @@ export default {
 }
 </style>
 <style>
-.inspire-tree .focused > .wholerow {
+.inspire-tree .focused>.wholerow {
   border: none;
 }
-
-#tag-picker-filter-bar {
-  padding: 10px 4px 4px;
-}
-
-.theme-black .inspire-tree .matched > .wholerow {
-  background: rgba(251, 191, 41, 0.25);
-}
 </style>

sist2-vue/src/i18n/messages.ts

@@ -16,7 +16,6 @@ export default {
         pages: "pages",
         mimeTypes: "Media types",
         tags: "Tags",
-        tagFilter: "Filter tags",
         help: {
             simpleSearch: "Simple search",
             advancedSearch: "Advanced search",
@@ -75,7 +74,6 @@ export default {
             useDatePicker: "Use a Date Picker component rather than a slider",
             vidPreviewInterval: "Video preview frame duration in ms",
             simpleLightbox: "Disable animations in image viewer",
-            showTagPickerFilter: "Display the tag filter bar"
         },
         queryMode: {
             simple: "Simple",
@@ -185,7 +183,6 @@ export default {
         pages: "pages",
         mimeTypes: "Types de médias",
         tags: "Tags",
-        tagFilter: "Filtrer les tags",
         help: {
             simpleSearch: "Recherche simple",
             advancedSearch: "Recherche avancée",
@@ -245,7 +242,6 @@ export default {
             useDatePicker: "Afficher un composant « Date Picker » plutôt qu'un slider",
             vidPreviewInterval: "Durée des images d'aperçu video en millisecondes",
             simpleLightbox: "Désactiver les animations du visualiseur d'images",
-            showTagPickerFilter: "Afficher le filtre dans l'onglet Tags"
         },
         queryMode: {
             simple: "Simple",
@@ -356,7 +352,6 @@ export default {
         pages: "页",
         mimeTypes: "文件类型",
         tags: "标签",
-        tagFilter: "筛选标签",
         help: {
             simpleSearch: "简易搜索",
             advancedSearch: "高级搜索",
@@ -415,7 +410,6 @@ export default {
             useDatePicker: "使用日期选择器组件而不是滑块",
             vidPreviewInterval: "视频预览帧的持续时间，以毫秒为单位",
             simpleLightbox: "在图片查看器中，禁用动画",
-            showTagPickerFilter: "显示标签过滤栏"
         },
         queryMode: {
             simple: "简单",

sist2-vue/src/store/index.ts

@@ -4,8 +4,6 @@ import VueRouter, {Route} from "vue-router";
 import {EsHit, EsResult, EsTag, Index, Tag} from "@/Sist2Api";
 import {deserializeMimes, serializeMimes} from "@/util";
 
-const CONF_VERSION = 2;
-
 Vue.use(Vuex)
 
 export default new Vuex.Store({
@@ -26,6 +24,7 @@ export default new Vuex.Store({
         sortMode: "score",
 
         fuzzy: false,
+        size: 60,
 
         optLang: "en",
         optLangIsDefault: true,
@@ -33,7 +32,6 @@ export default new Vuex.Store({
         optTheme: "light",
         optDisplay: "grid",
 
-        optSize: 60,
         optHighlight: true,
         optTagOrOperator: false,
         optFuzzy: true,
@@ -54,7 +52,6 @@ export default new Vuex.Store({
         optUseDatePicker: false,
         optVidPreviewInterval: 700,
         optSimpleLightbox: true,
-        optShowTagPickerFilter: true,
 
         _onLoadSelectedIndices: [] as string[],
         _onLoadSelectedMimeTypes: [] as string[],
@@ -153,7 +150,7 @@ export default new Vuex.Store({
         setOptSuggestPath: (state, val) => state.optSuggestPath = val,
         setOptFragmentSize: (state, val) => state.optFragmentSize = val,
         setOptQueryMode: (state, val) => state.optQueryMode = val,
-        setOptResultSize: (state, val) => state.optSize = val,
+        setOptResultSize: (state, val) => state.size = val,
         setOptTagOrOperator: (state, val) => state.optTagOrOperator = val,
 
         setOptTreemapType: (state, val) => state.optTreemapType = val,
@@ -166,7 +163,6 @@ export default new Vuex.Store({
         setOptUseDatePicker: (state, val) => state.optUseDatePicker = val,
         setOptVidPreviewInterval: (state, val) => state.optVidPreviewInterval = val,
         setOptSimpleLightbox: (state, val) => state.optSimpleLightbox = val,
-        setOptShowTagPickerFilter: (state, val) => state.optShowTagPickerFilter = val,
 
         setOptLightboxLoadOnlyCurrent: (state, val) => state.optLightboxLoadOnlyCurrent = val,
         setOptLightboxSlideDuration: (state, val) => state.optLightboxSlideDuration = val,
@@ -245,11 +241,6 @@ export default new Vuex.Store({
             }
         },
         async updateArgs({state}, router: VueRouter) {
-
-            if (router.currentRoute.path !== "/") {
-                return;
-            }
-
             await router.push({
                 query: {
                     q: state.searchText.trim() ? state.searchText.trim().replace(/\s+/g, " ") : undefined,
@@ -278,8 +269,6 @@ export default new Vuex.Store({
                 }
             });
 
-            conf["version"] = CONF_VERSION;
-
             localStorage.setItem("sist2_configuration", JSON.stringify(conf));
         },
         loadConfiguration({state}) {
@@ -287,11 +276,6 @@ export default new Vuex.Store({
             if (confString) {
                 const conf = JSON.parse(confString);
 
-                if (!("version" in conf) || conf["version"] != CONF_VERSION) {
-                    localStorage.removeItem("sist2_configuration");
-                    window.location.reload();
-                }
-
                 Object.keys(state).forEach((key) => {
                     if (key.startsWith("opt")) {
                         (state as any)[key] = conf[key];
@@ -353,7 +337,7 @@ export default new Vuex.Store({
         searchText: state => state.searchText,
         pathText: state => state.pathText,
         fuzzy: state => state.fuzzy,
-        size: state => state.optSize,
+        size: state => state.size,
         sortMode: state => state.sortMode,
         lastQueryResult: state => state.lastQueryResults,
         lastDoc: function (state): EsHit | null {
@@ -391,12 +375,11 @@ export default new Vuex.Store({
         optTreemapColor: state => state.optTreemapColor,
         optLightboxLoadOnlyCurrent: state => state.optLightboxLoadOnlyCurrent,
         optLightboxSlideDuration: state => state.optLightboxSlideDuration,
-        optResultSize: state => state.optSize,
+        optResultSize: state => state.size,
         optHideLegacy: state => state.optHideLegacy,
         optUpdateMimeMap: state => state.optUpdateMimeMap,
         optUseDatePicker: state => state.optUseDatePicker,
         optVidPreviewInterval: state => state.optVidPreviewInterval,
         optSimpleLightbox: state => state.optSimpleLightbox,
-        optShowTagPickerFilter: state => state.optShowTagPickerFilter,
     }
 })

sist2-vue/src/views/Configuration.vue

@@ -50,11 +50,6 @@
               $t("opt.simpleLightbox")
             }}
           </b-form-checkbox>
-
-          <b-form-checkbox :checked="optShowTagPickerFilter" @input="setOptShowTagPickerFilter">{{
-              $t("opt.showTagPickerFilter")
-            }}
-          </b-form-checkbox>
         </b-card>
 
         <br/>
@@ -250,7 +245,6 @@ export default {
       "optUseDatePicker",
       "optVidPreviewInterval",
       "optSimpleLightbox",
-      "optShowTagPickerFilter",
     ]),
     clientWidth() {
       return window.innerWidth;
@@ -298,7 +292,6 @@ export default {
       "setOptUseDatePicker",
       "setOptVidPreviewInterval",
       "setOptSimpleLightbox",
-      "setOptShowTagPickerFilter",
     ]),
     onResetClick() {
       localStorage.removeItem("sist2_configuration");

sist2-vue/src/views/SearchPage.vue

@@ -32,7 +32,7 @@
               <MimePicker></MimePicker>
             </b-tab>
             <b-tab :title="$t('tags')">
-              <TagPicker :show-search-bar="$store.state.optShowTagPickerFilter"></TagPicker>
+              <TagPicker></TagPicker>
             </b-tab>
           </b-tabs>
         </b-col>
@@ -139,9 +139,7 @@ export default Vue.extend({
         this.setSist2Info(data);
         this.setIndices(data.indices);
 
-        const doBlankSearch = !this.$store.state.optUpdateMimeMap;
-
-        Sist2Api.getMimeTypes(Sist2Query.searchQuery(doBlankSearch)).then(({mimeMap}) => {
+        Sist2Api.getMimeTypes(Sist2Query.searchQuery()).then(({mimeMap}) => {
           this.$store.commit("setUiMimeMap", mimeMap);
           this.uiLoading = false;
           this.search(true);
@@ -208,7 +206,7 @@ export default Vue.extend({
       this.$store.commit("setUiReachedScrollEnd", false);
     },
     async handleSearch(resp: EsResult) {
-      if (resp.hits.hits.length == 0 || resp.hits.hits.length < this.$store.state.optSize) {
+      if (resp.hits.hits.length == 0) {
         this.$store.commit("setUiReachedScrollEnd", true);
       }
 
@@ -248,8 +246,6 @@ export default Vue.extend({
       this.$store.commit("setLastQueryResult", resp);
 
       this.docs.push(...resp.hits.hits);
-
-      resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
     },
     getDateRange(): Promise<{ min: number, max: number }> {
       return sist2.esQuery({

src/cli.c

@@ -81,11 +81,6 @@ void web_args_destroy(web_args_t *args) {
 }
 
 void exec_args_destroy(exec_args_t *args) {
-
-    if (args->index_path != NULL) {
-        free(args->index_path);
-    }
-
     free(args);
 }
 

src/cli.h

@@ -85,7 +85,7 @@ typedef struct web_args {
 typedef struct exec_args {
     char *es_url;
     char *es_index;
-    char *index_path;
+    const char *index_path;
     const char *script_path;
     int async_script;
     char *script;

src/index/elastic.c

@@ -21,8 +21,6 @@ void free_queue(int max);
 
 void elastic_flush();
 
-void print_error(response_t *r);
-
 void destroy_indexer(es_indexer_t *indexer) {
 
     if (indexer == NULL) {
@@ -110,16 +108,16 @@ void execute_update_script(const char *script, int async, const char index_id[SI
     cJSON *term_obj = cJSON_AddObjectToObject(query, "term");
     cJSON_AddStringToObject(term_obj, "index", index_id);
 
-    char *str = cJSON_PrintUnformatted(body);
+    char *str = cJSON_Print(body);
 
-    char url[4096];
+    char bulk_url[4096];
     if (async) {
-        snprintf(url, sizeof(url), "%s/%s/_update_by_query?wait_for_completion=false", Indexer->es_url,
+        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query?wait_for_completion=false", Indexer->es_url,
                  Indexer->es_index);
     } else {
-        snprintf(url, sizeof(url), "%s/%s/_update_by_query", Indexer->es_url, Indexer->es_index);
+        snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_update_by_query", Indexer->es_url, Indexer->es_index);
     }
-    response_t *r = web_post(url, str);
+    response_t *r = web_post(bulk_url, str);
     if (!async) {
         LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
     }
@@ -139,11 +137,6 @@ void execute_update_script(const char *script, int async, const char index_id[SI
 
     if (async) {
         cJSON *task = cJSON_GetObjectItem(resp, "task");
-
-        if (task == NULL) {
-            LOG_FATALF("elastic.c", "FIXME: Could not get task id: %s", r->body);
-        }
-
         LOG_INFOF("elastic.c", "User script queued: %s/_tasks/%s", Indexer->es_url, task->valuestring);
     }
 
@@ -219,13 +212,7 @@ void print_errors(response_t *r) {
     *(tmp + r->size) = '\0';
 
     cJSON *ret_json = cJSON_Parse(tmp);
-    cJSON *errors = cJSON_GetObjectItem(ret_json, "errors");
-
-    if (errors == NULL) {
-        char *str = cJSON_Print(ret_json);
-        LOG_ERRORF("elastic.c", "%s\n", str);
-        cJSON_free(str);
-    } else if (errors->valueint != 0) {
+    if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) {
         cJSON *err;
         cJSON_ArrayForEach(err, cJSON_GetObjectItem(ret_json, "items")) {
             if (cJSON_GetObjectItem(cJSON_GetObjectItem(err, "index"), "status")->valueint != 201) {
@@ -426,20 +413,12 @@ es_version_t *elastic_get_version(const char *es_url) {
     *(tmp + r->size) = '\0';
     cJSON *response = cJSON_Parse(tmp);
     free(tmp);
+    free_response(r);
 
     if (response == NULL) {
         return NULL;
     }
 
-    if (cJSON_GetObjectItem(response, "error") != NULL) {
-        LOG_WARNING("elastic.c", "Could not get Elasticsearch version")
-        print_error(r);
-        free_response(r);
-        return NULL;
-    }
-
-    free_response(r);
-
     if (cJSON_GetObjectItem(response, "version") == NULL ||
         cJSON_GetObjectItem(cJSON_GetObjectItem(response, "version"), "number") == NULL) {
         cJSON_Delete(response);

src/index/elastic.h

@@ -20,8 +20,8 @@ typedef struct {
 } es_version_t;
 
 #define VERSION_GE(version, maj, min) ((version)->major > (maj) || ((version)->major == (maj) && (version)->minor >= (min)))
-#define IS_SUPPORTED_ES_VERSION(es_version) ((es_version) != NULL && VERSION_GE((es_version), 6, 8))
-#define USE_LEGACY_ES_SETTINGS(es_version) ((es_version) != NULL && (!VERSION_GE((es_version), 7, 14)))
+#define IS_SUPPORTED_ES_VERSION(es_version) VERSION_GE((es_version), 6, 8)
+#define USE_LEGACY_ES_SETTINGS(es_version) (!VERSION_GE((es_version), 7, 14))
 
 __always_inline
 static const char *format_es_version(es_version_t *version) {

src/index/web.c

@@ -22,7 +22,7 @@ void free_response(response_t *resp) {
     free(resp);
 }
 
-void web_post_async_poll(subreq_ctx_t *req) {
+void web_post_async_poll(subreq_ctx_t* req) {
     fd_set fdread;
     fd_set fdwrite;
     fd_set fdexcep;
@@ -34,7 +34,7 @@ void web_post_async_poll(subreq_ctx_t *req) {
 
     CURLMcode mc = curl_multi_fdset(req->multi, &fdread, &fdwrite, &fdexcep, &maxfd);
 
-    if (mc != CURLM_OK) {
+    if(mc != CURLM_OK) {
         req->done = TRUE;
         return;
     }
@@ -47,7 +47,7 @@ void web_post_async_poll(subreq_ctx_t *req) {
     struct timeval timeout = {1, 0};
     int rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
 
-    switch (rc) {
+    switch(rc) {
         case -1:
             req->done = TRUE;
             break;
@@ -142,9 +142,6 @@ response_t *web_post(const char *url, const char *data) {
     curl_easy_setopt(curl, CURLOPT_POST, 1);
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
 
-    char err_buffer[CURL_ERROR_SIZE + 1] = {};
-    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer);
-
     struct curl_slist *headers = NULL;
     headers = curl_slist_append(headers, "Content-Type: application/json");
     curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
@@ -154,16 +151,12 @@ response_t *web_post(const char *url, const char *data) {
     curl_easy_perform(curl);
     curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);
 
-    resp->body = buffer.buf;
-    resp->size = buffer.cur;
-
-    if (resp->status_code == 0) {
-        LOG_ERRORF("web.c", "CURL Error: %s", err_buffer)
-    }
-
     curl_easy_cleanup(curl);
     curl_slist_free_all(headers);
 
+    resp->body = buffer.buf;
+    resp->size = buffer.cur;
+
     return resp;
 }
 
@@ -182,7 +175,7 @@ response_t *web_put(const char *url, const char *data) {
     curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
     curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2");
     curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0);
-    curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4);
+    curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4 );
 
     struct curl_slist *headers = NULL;
     headers = curl_slist_append(headers, "Content-Type: application/json");

src/io/serialize.c

@@ -505,9 +505,9 @@ void incremental_copy_handle_doc(cJSON *document, UNUSED(const char id_str[SIST_
 
         // Copy tn store contents
         size_t buf_len;
-        char *buf = store_read(IncrementalCopySourceStore, (char *) doc_id, SIST_DOC_ID_LEN, &buf_len);
+        char *buf = store_read(IncrementalCopySourceStore, (char *) doc_id, sizeof(doc_id), &buf_len);
         if (buf_len != 0) {
-            store_write(IncrementalCopyDestinationStore, (char *) doc_id, SIST_DOC_ID_LEN, buf, buf_len);
+            store_write(IncrementalCopyDestinationStore, (char *) doc_id, sizeof(doc_id), buf, buf_len);
             free(buf);
         }
     }

src/io/serialize.h

@@ -42,13 +42,13 @@ index_descriptor_t read_index_descriptor(char *path);
 // caller ensures char file_path[PATH_MAX]
 #define READ_INDICES(file_path, index_path, action_ok, action_main_fail, cond_original) \
     snprintf(file_path, PATH_MAX, "%s_index_main.ndjson.zst", index_path);              \
-    if (access(file_path, R_OK) == 0) {                                                 \
+    if (0 == access(file_path, R_OK)) {                                                 \
         action_ok;                                                                      \
     } else {                                                                            \
         action_main_fail;                                                               \
     }                                                                                   \
     snprintf(file_path, PATH_MAX, "%s_index_original.ndjson.zst", index_path);          \
-    if ((cond_original) && access(file_path, R_OK) == 0) {                              \
+    if ((cond_original) && (0 == access(file_path, R_OK))) {                            \
         action_ok;                                                                      \
     }                                                                                   \
 

src/main.c

@@ -38,8 +38,8 @@ static __sighandler_t sigabrt_handler = NULL;
 
 void sig_handler(int signum) {
 
-    LogCtx.verbose = TRUE;
-    LogCtx.very_verbose = TRUE;
+    LogCtx.verbose = 1;
+    LogCtx.very_verbose = 1;
 
     LOG_ERROR("*SIGNAL HANDLER*", "=============================================\n\n");
     LOG_ERRORF("*SIGNAL HANDLER*", "Uh oh! Caught fatal signal: %s", strsignal(signum));
@@ -103,7 +103,7 @@ void sig_handler(int signum) {
     exit(-1);
 }
 
-void init_dir(const char *dirpath, scan_args_t *args) {
+void init_dir(const char *dirpath, scan_args_t* args) {
     char path[PATH_MAX];
     snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
 
@@ -112,16 +112,16 @@ void init_dir(const char *dirpath, scan_args_t *args) {
     strcpy(ScanCtx.index.desc.type, INDEX_TYPE_NDJSON);
 
     if (args->incremental != NULL) {
-        // copy old index id
-        char descriptor_path[PATH_MAX];
-        snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental);
-        index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
-        memcpy(ScanCtx.index.desc.id, original_desc.id, sizeof(original_desc.id));
+      // copy old index id
+      char descriptor_path[PATH_MAX];
+      snprintf(descriptor_path, PATH_MAX, "%sdescriptor.json", args->incremental);
+      index_descriptor_t original_desc = read_index_descriptor(descriptor_path);
+      memcpy(ScanCtx.index.desc.id, original_desc.id, sizeof(original_desc.id));
     } else {
-        // generate new index id based on timestamp
-        unsigned char index_md5[MD5_DIGEST_LENGTH];
-        MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
-        buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
+      // generate new index id based on timestamp
+      unsigned char index_md5[MD5_DIGEST_LENGTH];
+      MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
+      buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
     }
 
     write_index_descriptor(path, &ScanCtx.index.desc);
@@ -324,13 +324,9 @@ void load_incremental_index(const scan_args_t *args) {
         LOG_FATALF("main.c", "Version mismatch! Index is %s but executable is %s", original_desc.version, Version)
     }
 
-    READ_INDICES(
-            file_path,
-            args->incremental,
-            incremental_read(ScanCtx.original_table, file_path, &original_desc),
-            LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)),
-            TRUE
-    );
+    READ_INDICES(file_path, args->incremental, incremental_read(ScanCtx.original_table, file_path, &original_desc),
+                 LOG_FATALF("main.c", "Could not open original main index for incremental scan: %s", strerror(errno)),
+                 1);
 
     LOG_INFOF("main.c", "Loaded %d items in to mtime table.", g_hash_table_size(ScanCtx.original_table))
 }
@@ -538,7 +534,6 @@ void sist2_exec_script(exec_args_t *args) {
 
     IndexCtx.es_url = args->es_url;
     IndexCtx.es_index = args->es_index;
-    IndexCtx.needs_es_connection = TRUE;
 
     LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type)
 
@@ -781,8 +776,9 @@ int main(int argc, const char *argv[]) {
         sist2_exec_script(exec_args);
 
     } else {
+        fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
         argparse_usage(&argparse);
-        LOG_FATALF("main.c", "Invalid command: '%s'\n", argv[0])
+        goto end;
     }
     printf("\n");
 

src/sist.h

@@ -27,6 +27,10 @@
 
 #define UNUSED(x) __attribute__((__unused__))  x
 
+#define MD5_STR_LENGTH 33
+#define SHA1_STR_LENGTH 41
+#define SHA1_DIGEST_LENGTH 20
+
 #include "util.h"
 #include "log.h"
 #include "types.h"

src/web/serve.c

@@ -187,7 +187,6 @@ void thumbnail(struct mg_connection *nc, struct mg_http_message *hm) {
 
         memcpy(tn_key, arg_doc_id, sizeof(arg_doc_id));
         memcpy(tn_key + sizeof(arg_doc_id) - 1, tn_index, sizeof(char) * 4);
-        *(tn_key + sizeof(tn_key) - 1) = '\0';
 
         data = store_read(store, (char *) tn_key, sizeof(tn_key), &data_len);
     } else {
@@ -282,7 +281,7 @@ void serve_file_from_disk(cJSON *json, index_t *idx, struct mg_connection *nc, s
     char disposition[8192];
     snprintf(disposition, sizeof(disposition),
              HTTP_SERVER_HEADER "Content-Disposition: inline; filename=\"%s%s%s\"\r\n"
-             "Accept-Ranges: bytes\r\nCache-Control: no-store\r\n",
+                                "Accept-Ranges: bytes\r\nCache-Control: no-store\r\n",
              name, strlen(ext) == 0 ? "" : ".", ext);
 
     char mime_mapping[1024];
@@ -313,18 +312,13 @@ void index_info(struct mg_connection *nc) {
 
     cache_es_version();
 
-    const char *es_version = "0.0.0";
-    if (WebCtx.es_version != NULL) {
-        es_version = format_es_version(WebCtx.es_version);
-    }
-
     cJSON *json = cJSON_CreateObject();
     cJSON *arr = cJSON_AddArrayToObject(json, "indices");
 
     cJSON_AddStringToObject(json, "mongooseVersion", MG_VERSION);
     cJSON_AddStringToObject(json, "esIndex", WebCtx.es_index);
     cJSON_AddStringToObject(json, "version", Version);
-    cJSON_AddStringToObject(json, "esVersion", es_version);
+    cJSON_AddStringToObject(json, "esVersion", format_es_version(WebCtx.es_version));
     cJSON_AddBoolToObject(json, "esVersionSupported", IS_SUPPORTED_ES_VERSION(WebCtx.es_version));
     cJSON_AddBoolToObject(json, "esVersionLegacy", USE_LEGACY_ES_SETTINGS(WebCtx.es_version));
     cJSON_AddStringToObject(json, "platform", QUOTE(SIST_PLATFORM));
@@ -359,6 +353,42 @@ void index_info(struct mg_connection *nc) {
 }
 
 
+void document_info(struct mg_connection *nc, struct mg_http_message *hm) {
+
+    if (hm->uri.len != SIST_DOC_ID_LEN + 2) {
+        LOG_DEBUGF("serve.c", "Invalid document_info path: %.*s", (int) hm->uri.len, hm->uri.ptr)
+        HTTP_REPLY_NOT_FOUND
+        return;
+    }
+
+    char arg_doc_id[SIST_DOC_ID_LEN];
+    memcpy(arg_doc_id, hm->uri.ptr + 3, SIST_DOC_ID_LEN);
+    *(arg_doc_id + SIST_DOC_ID_LEN - 1) = '\0';
+
+    cJSON *doc = elastic_get_document(arg_doc_id);
+    cJSON *source = cJSON_GetObjectItem(doc, "_source");
+
+    cJSON *index_id = cJSON_GetObjectItem(source, "index");
+    if (index_id == NULL) {
+        cJSON_Delete(doc);
+        HTTP_REPLY_NOT_FOUND
+        return;
+    }
+
+    index_t *idx = get_index_by_id(index_id->valuestring);
+    if (idx == NULL) {
+        cJSON_Delete(doc);
+        HTTP_REPLY_NOT_FOUND
+        return;
+    }
+
+    char *json_str = cJSON_PrintUnformatted(source);
+    send_response_line(nc, 200, (int) strlen(json_str), "Content-Type: application/json");
+    mg_send(nc, json_str, (int) strlen(json_str));
+    free(json_str);
+    cJSON_Delete(doc);
+}
+
 void file(struct mg_connection *nc, struct mg_http_message *hm) {
 
     if (hm->uri.len != SIST_DOC_ID_LEN + 2) {
@@ -617,6 +647,8 @@ static void ev_router(struct mg_connection *nc, int ev, void *ev_data, UNUSED(vo
                 return;
             }
             tag(nc, hm);
+        } else if (mg_http_match_uri(hm, "/d/*")) {
+            document_info(nc, hm);
         } else {
             HTTP_REPLY_NOT_FOUND
         }

tests/test_scan.py

@@ -35,20 +35,10 @@ def sist2_index(files, *args):
     path = copy_files(files)
 
     shutil.rmtree("test_i", ignore_errors=True)
-    sist2("scan", path, "-o", "test_i", "-t12", *args)
+    sist2("scan", path, "-o", "test_i", *args)
     return iter(sist2_index_to_dict("test_i"))
 
 
-def get_lmdb_contents(path):
-    import lmdb
-
-    env = lmdb.open(path)
-
-    txn = env.begin(write=False)
-
-    return dict((k, v) for k, v in txn.cursor())
-
-
 def sist2_incremental_index(files, func=None, incremental_index=False, *args):
     path = copy_files(files)
 
@@ -56,7 +46,7 @@ def sist2_incremental_index(files, func=None, incremental_index=False, *args):
         func(path)
 
     shutil.rmtree("test_i_inc", ignore_errors=True)
-    sist2("scan", path, "-o", "test_i_inc", "--incremental", "test_i", "-t12", *args)
+    sist2("scan", path, "-o", "test_i_inc", "--incremental", "test_i", *args)
     return iter(sist2_index_to_dict("test_i_inc", incremental_index))
 
 
@@ -86,31 +76,9 @@ class ScanTest(unittest.TestCase):
                 pass
 
         file_count = sum(1 for _ in sist2_index(TEST_FILES))
-        lmdb_full = get_lmdb_contents("test_i/thumbs")
-
-        # Remove files
-        num_files_rm1 = len(list(sist2_incremental_index(TEST_FILES, remove_files)))
-        lmdb_rm1 = get_lmdb_contents("test_i_inc/thumbs")
-        self.assertEqual(num_files_rm1, file_count - 2)
-        self.assertEqual(len(set(lmdb_full.keys() - set(lmdb_rm1.keys()))), 2)
-
-        # add files (incremental_index=True)
-        num_files_add_inc = len(list(sist2_incremental_index(TEST_FILES, add_files, incremental_index=True)))
-        lmdb_add_inc = get_lmdb_contents("test_i_inc/thumbs")
-        self.assertEqual(num_files_add_inc, 3)
-        self.assertEqual(set(lmdb_full.keys()), set(lmdb_add_inc.keys()))
-
-        # add files
-        num_files_add = len(list(sist2_incremental_index(TEST_FILES, add_files)))
-        lmdb_add = get_lmdb_contents("test_i_inc/thumbs")
-        self.assertEqual(num_files_add, file_count + 3)
-        self.assertEqual(set(lmdb_full.keys()), set(lmdb_add.keys()))
-
-        # (No action)
-        sist2_incremental_index(TEST_FILES)
-        lmdb_inc = get_lmdb_contents("test_i_inc/thumbs")
-
-        self.assertEqual(set(lmdb_full.keys()), set(lmdb_inc.keys()))
+        self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, remove_files)), file_count - 2)
+        self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, add_files, incremental_index=True)), 3)
+        self.assertEqual(sum(1 for _ in sist2_incremental_index(TEST_FILES, add_files)), file_count + 3)
 
 
 if __name__ == "__main__":

third-party/libscan/CMakeLists.txt

@@ -7,11 +7,6 @@ option(BUILD_TESTS "Build tests" on)
 
 add_subdirectory(third-party/antiword)
 
-set(USE_LIBXML2 OFF CACHE BOOL "" FORCE)
-set(USE_XMLWRITER OFF CACHE BOOL "" FORCE)
-set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
-add_subdirectory(third-party/libmobi)
-
 add_library(
         scan
         libscan/util.c libscan/util.h
@@ -47,22 +42,6 @@ if (SIST_DEBUG)
             -fsanitize=address
             -fno-inline
     )
-elseif (SIST_FAST)
-    add_compile_definitions(
-            antiword
-            NDEBUG
-    )
-
-    target_compile_options(
-            scan
-            PRIVATE
-
-            -Ofast
-            -march=native
-            -fno-stack-protector
-            -fomit-frame-pointer
-            -freciprocal-math
-    )
 else()
     add_compile_definitions(
             antiword
@@ -118,15 +97,35 @@ target_compile_options(
         -g
 )
 
+include(ExternalProject)
+find_program(MAKE_EXE NAMES gmake nmake make)
+ExternalProject_Add(
+        libmobi
+        GIT_REPOSITORY https://github.com/simon987/libmobi.git
+        GIT_TAG "public"
+
+        UPDATE_COMMAND ""
+        PATCH_COMMAND ""
+        TEST_COMMAND ""
+        CONFIGURE_COMMAND ./autogen.sh && ./configure
+        INSTALL_COMMAND ""
+
+        PREFIX "third-party/ext_libmobi"
+        SOURCE_DIR "third-party/ext_libmobi/src/libmobi"
+        BINARY_DIR "third-party/ext_libmobi/src/libmobi"
+
+        BUILD_COMMAND ${MAKE_EXE} -j 8 --silent
+)
+
+SET(MOBI_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libmobi/src/libmobi/src/.libs/)
+SET(MOBI_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libmobi/src/libmobi/src/)
+
 if (SIST_DEBUG)
     SET(FFMPEG_DEBUG "--enable-debug=3" "--disable-optimizations")
 else()
     SET(FFMPEG_DEBUG "")
 endif()
 
-include(ExternalProject)
-find_program(MAKE_EXE NAMES gmake nmake make)
-
 ExternalProject_Add(
         ffmpeg
         GIT_REPOSITORY https://git.ffmpeg.org/ffmpeg.git
@@ -172,10 +171,10 @@ SET(WPD_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwp
 
 add_dependencies(
         scan
+        libmobi
         ffmpeg
         antiword
         libwpd
-        mobi
 )
 
 target_link_libraries(
@@ -193,6 +192,8 @@ target_link_libraries(
         ${MUPDF_LIB}
         openjp2
 
+        ${MOBI_LIB_DIR}/libmobi.a
+
         ${WPD_LIB_DIR}/libwpd-0.9.a
         ${WPD_LIB_DIR}/libwpd-stream-0.9.a
 
@@ -229,7 +230,6 @@ target_link_libraries(
         ${GUMBO_LIB}
         dl
         antiword
-        mobi
         unofficial::pcre::pcre unofficial::pcre::pcre16 unofficial::pcre::pcre32 unofficial::pcre::pcrecpp
 )
 

third-party/libscan/libscan/macros.h

@@ -20,10 +20,8 @@
 #undef ABS
 #define ABS(a) (((a) < 0) ? -(a) : (a))
 
-#define SHA1_DIGEST_LENGTH SHA_DIGEST_LENGTH
-
-#define SHA1_STR_LENGTH (SHA1_DIGEST_LENGTH * 2 + 1)
-#define MD5_STR_LENGTH (MD5_DIGEST_LENGTH * 2 + 1)
+#define SHA1_STR_LENGTH 41
+#define SHA1_DIGEST_LENGTH 20
 
 #define APPEND_STR_META(doc, keyname, value) \
     {meta_line_t *meta_str = malloc(sizeof(meta_line_t) + strlen(value)); \

third-party/libscan/libscan/media/media.c

@@ -4,12 +4,7 @@
 
 #define MIN_SIZE 32
 #define AVIO_BUF_SIZE 8192
-#define IS_VIDEO(fmt) ( \
-    (fmt)->iformat->name && strcmp((fmt)->iformat->name, "image2") != 0 \
-    && strcmp((fmt)->iformat->name, "jpeg_pipe") != 0 \
-    && strcmp((fmt)->iformat->name, "webp_pipe") != 0 \
-    && strcmp((fmt)->iformat->name, "png_pipe") != 0 \
-    )
+#define IS_VIDEO(fmt) ((fmt)->iformat->name && strcmp((fmt)->iformat->name, "image2") != 0)
 
 
 #define STORE_AS_IS ((void*)-1)
@@ -284,22 +279,18 @@ static void
 append_video_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int is_video) {
 
     if (is_video) {
-        if (pFormatCtx->duration / AV_TIME_BASE != 0) {
-            meta_line_t *meta_duration = malloc(sizeof(meta_line_t));
-            meta_duration->key = MetaMediaDuration;
-            meta_duration->long_val = pFormatCtx->duration / AV_TIME_BASE;
-            if (meta_duration->long_val > INT32_MAX) {
-                meta_duration->long_val = 0;
-            }
-            APPEND_META(doc, meta_duration)
+        meta_line_t *meta_duration = malloc(sizeof(meta_line_t));
+        meta_duration->key = MetaMediaDuration;
+        meta_duration->long_val = pFormatCtx->duration / AV_TIME_BASE;
+        if (meta_duration->long_val > INT32_MAX) {
+            meta_duration->long_val = 0;
         }
+        APPEND_META(doc, meta_duration)
 
-        if (pFormatCtx->bit_rate != 0) {
-            meta_line_t *meta_bitrate = malloc(sizeof(meta_line_t));
-            meta_bitrate->key = MetaMediaBitrate;
-            meta_bitrate->long_val = pFormatCtx->bit_rate;
-            APPEND_META(doc, meta_bitrate)
-        }
+        meta_line_t *meta_bitrate = malloc(sizeof(meta_line_t));
+        meta_bitrate->key = MetaMediaBitrate;
+        meta_bitrate->long_val = pFormatCtx->bit_rate;
+        APPEND_META(doc, meta_bitrate)
     }
 
     AVDictionaryEntry *tag = NULL;
@@ -586,8 +577,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
         int video_duration_in_seconds = (int) (pFormatCtx->duration / AV_TIME_BASE);
 
-        int thumbnails_to_generate = (IS_VIDEO(pFormatCtx) && stream->codecpar->codec_id != AV_CODEC_ID_GIF &&
-                                      video_duration_in_seconds >= 15)
+        int thumbnails_to_generate = (IS_VIDEO(pFormatCtx) && stream->codecpar->codec_id != AV_CODEC_ID_GIF && video_duration_in_seconds >= 15)
                                      // Limit to ~1 thumbnail every 7s
                                      ? MAX(MIN(ctx->tn_count, video_duration_in_seconds / 7 + 1), 1) + 1
                                      : 1;

third-party/libscan/libscan/mobi/scan_mobi.c

@@ -1,6 +1,6 @@
 #include "scan_mobi.h"
 
-#include "../../third-party/libmobi/src/mobi.h"
+#include <mobi.h>
 #include <errno.h>
 #include "stdlib.h"
 

third-party/libscan/libscan/scan.h

@@ -48,6 +48,7 @@ typedef int scan_code_t;
 #define CTX_LOG_FATALF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_FATAL, fmt, __VA_ARGS__); exit(-1);
 #define CTX_LOG_FATAL(filepath, str) ctx->log(filepath, LEVEL_FATAL, str); exit(-1);
 
+#define MD5_STR_LENGTH 33
 #define SIST_DOC_ID_LEN MD5_STR_LENGTH
 #define SIST_INDEX_ID_LEN MD5_STR_LENGTH
 

third-party/libscan/test/main.cpp

@@ -923,6 +923,7 @@ TEST(Msdoc, Test1Pdf) {
     ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "October 2000") != nullptr);
     ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "INTERNATIONAL ORGANIZATION FOR STANDARDIZATION");
     ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Oliver Morgan");
+    ASSERT_EQ(get_meta(&doc, MetaPages)->long_val, 57);
     ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
     ASSERT_NE(size_before, store_size);
 
@@ -1029,23 +1030,6 @@ TEST(Msdoc, TestUtf8Text) {
     cleanup(&doc, &f);
 }
 
-TEST(Msdoc, Test5Pdf) {
-    vfile_t f;
-    document_t doc;
-    load_doc_file("libscan-test-files/test_files/msdoc/test5.doc", &f, &doc);
-
-    size_t size_before = store_size;
-
-    parse_msdoc(&msdoc_ctx, &f, &doc);
-
-    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "орган Федеральной") != nullptr);
-    ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "uswo");
-    ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
-    ASSERT_NE(size_before, store_size);
-
-    cleanup(&doc, &f);
-}
-
 TEST(Msdoc, TestFuzz1) {
     vfile_t f;
     document_t doc;
@@ -1205,7 +1189,4 @@ int main(int argc, char **argv) {
     av_log_set_level(AV_LOG_QUIET);
     ::testing::InitGoogleTest(&argc, argv);
     return RUN_ALL_TESTS();
-}
-
-// 0x6130000d2580
-// "/mnt/Hatchery/m ain/downloads/qbittorrent/downloads/Roskomnadzor/УПРАВЛЕНИЕ РОСКОМНАДЗОРА по РБ.zip#/УПРАВЛЕНИЕ РОСКОМНАДЗОРА по РБ/Лопатин Ю.М/Секнин/2015 год/Обучение по ", <incomplete sequence \320>...
+}