Merge pull request #200 from simon987/dev

v2.11.3

README.md

@@ -50,7 +50,8 @@ sist2 (Simple incremental search tool)
         ```
 1. Download sist2 executable
     1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) *
-    1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not recommended!)*
+    1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not
+       recommended!)*
     1. *(or)* `docker pull simon987/sist2:2.11.2-x64-linux`
 
 1. See [Usage guide](docs/USAGE.md)
@@ -70,19 +71,20 @@ See [Usage guide](docs/USAGE.md) for more details
 File type | Library | Content | Thumbnail | Metadata
 :---|:---|:---|:---|:---
 pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title |
-cbz,cbr | *(none)* | - | yes | - |
+cbz,cbr | [libscan](https://github.com/simon987/libscan) | - | yes | - |
 `audio/*` | ffmpeg | - | yes | ID3 tags |
 `video/*` | ffmpeg | - | yes | title, comment, artist |
 `image/*` | ffmpeg | - | yes | [Common EXIF tags](https://github.com/simon987/sist2/blob/efdde2734eca9b14a54f84568863b7ffd59bdba3/src/parsing/media.c#L190), GPS tags |
 raw, rw2, dng, cr2, crw, dcr, k25, kdc, mrw, pef, xf3, arw, sr2, srf, erf  | LibRaw | - | yes | Common EXIF tags, GPS tags |
 ttf,ttc,cff,woff,fnt,otf | Freetype2 | - | yes, `bmp` | Name & style |
-`text/plain` | *(none)* | yes | no | - |
+`text/plain` | [libscan](https://github.com/simon987/libscan) | yes | no | - |
-html, xml | *(none)* | yes | no | - |
+html, xml | [libscan](https://github.com/simon987/libscan) | yes | no | - |
 tar, zip, rar, 7z, ar ...  | Libarchive | yes\* | - | no |
-docx, xlsx, pptx | *(none)* | yes | if embedded | creator, modified_by, title |
+docx, xlsx, pptx | [libscan](https://github.com/simon987/libscan) | yes | if embedded | creator, modified_by, title |
 doc (MS Word 97-2003) | antiword | yes | yes | author, title |
 mobi, azw, azw3 | libmobi | yes | no | author, title |
 wpd (WordPerfect) | libwpd | yes | no | *planned* |
+json, jsonl, ndjson | [libscan](https://github.com/simon987/libscan) | yes | - | - |
 
 \* *See [Archive files](#archive-files)*
 
@@ -135,7 +137,7 @@ docker run --rm my-sist2-image cat /root/sist2 > sist2-x64-linux
    ```bash
    apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git
    ```
-   
+
 1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
 
 1. Install vcpkg dependencies

docs/USAGE.md

@@ -49,6 +49,7 @@ Scan options
     --mem-buffer=<int>            Maximum memory buffer size per thread in MB for files inside archives (see USAGE.md). DEFAULT: 2000
     --read-subtitles              Read subtitles from media files.
     --fast-epub                   Faster but less accurate EPUB parsing (no thumbnails, metadata)
+    --checksums                   Calculate file checksums when scanning.
 
 Index options
     -t, --threads=<int>           Number of threads. DEFAULT=1
@@ -129,6 +130,9 @@ Exec-script options
     To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
 * `--read-subtitles` When enabled, will attempt to read the subtitles stream from media files.
 * `--fast-epub` Much faster but less accurate EPUB parsing. When enabled, sist2 will use a simple HTML parser to read epub files instead of the MuPDF library. No thumbnails are generated and author/title metadata are not parsed.
+* `--checksums` Calculate file checksums (sha1) when scanning files. This option does not cause any additional read 
+  operations. Checksums are not calculated for all file types, unless the file is inside an archive. When enabled, duplicate
+  files are hidden in the web UI (this behaviour can be toggled in the Configuration page).
 
 ### Scan examples
 

schema/mappings.json

@@ -4,6 +4,10 @@
       "type": "keyword",
       "doc_values": true
     },
+    "checksum": {
+      "type": "keyword",
+      "index": false
+    },
     "_depth": {
       "type": "integer"
     },

schema/settings.json

@@ -2,7 +2,8 @@
   "index": {
     "refresh_interval": "30s",
     "codec": "best_compression",
-    "number_of_replicas": 0
+    "number_of_replicas": 0,
+    "highlight.max_analyzed_offset": 10000000
   },
   "analysis": {
     "tokenizer": {

sist2-vue/package-lock.json

@@ -23,7 +23,6 @@
         "vue-color": "^2.8.1",
         "vue-i18n": "^8.24.4",
         "vue-masonry-wall": "^0.3.2",
-        "vue-multiselect": "^2.1.6",
         "vue-router": "^3.2.0",
         "vue-simple-suggest": "^1.11.1",
         "vuex": "^3.4.0"
@@ -13604,15 +13603,6 @@
         "node": ">=10"
       }
     },
-    "node_modules/vue-multiselect": {
-      "version": "2.1.6",
-      "resolved": "https://registry.npmjs.org/vue-multiselect/-/vue-multiselect-2.1.6.tgz",
-      "integrity": "sha512-s7jmZPlm9FeueJg1RwJtnE9KNPtME/7C8uRWSfp9/yEN4M8XcS/d+bddoyVwVnvFyRh9msFo0HWeW0vTL8Qv+w==",
-      "engines": {
-        "node": ">= 4.0.0",
-        "npm": ">= 3.0.0"
-      }
-    },
     "node_modules/vue-observe-visibility": {
       "version": "0.4.6",
       "resolved": "https://registry.npmjs.org/vue-observe-visibility/-/vue-observe-visibility-0.4.6.tgz",
@@ -26376,11 +26366,6 @@
         "vue-observe-visibility": "^0.4.6"
       }
     },
-    "vue-multiselect": {
-      "version": "2.1.6",
-      "resolved": "https://registry.npmjs.org/vue-multiselect/-/vue-multiselect-2.1.6.tgz",
-      "integrity": "sha512-s7jmZPlm9FeueJg1RwJtnE9KNPtME/7C8uRWSfp9/yEN4M8XcS/d+bddoyVwVnvFyRh9msFo0HWeW0vTL8Qv+w=="
-    },
     "vue-observe-visibility": {
       "version": "0.4.6",
       "resolved": "https://registry.npmjs.org/vue-observe-visibility/-/vue-observe-visibility-0.4.6.tgz",

sist2-vue/package.json

@@ -22,7 +22,6 @@
     "vue-color": "^2.8.1",
     "vue-i18n": "^8.24.4",
     "vue-masonry-wall": "^0.3.2",
-    "vue-multiselect": "^2.1.6",
     "vue-router": "^3.2.0",
     "vue-simple-suggest": "^1.11.1",
     "vuex": "^3.4.0"

sist2-vue/src/Sist2Api.ts

@@ -50,6 +50,7 @@ export interface EsHit {
         height: number
         duration: number
         tag: string[]
+        checksum: string
     }
     _props: {
         isSubDocument: boolean

sist2-vue/src/Sist2Query.ts

@@ -187,7 +187,8 @@ class Sist2Query {
                     "name.nGram": {},
                     "content.nGram": {},
                     font_name: {},
-                }
+                },
+                max_analyzed_offset: 9_999_999
             };
             if (getters.optSearchInPath) {
                 q.highlight.fields["path.text"] = {};

sist2-vue/src/components/DocInfoModal.vue

@@ -4,7 +4,8 @@
     <template #modal-title>
       <h5 class="modal-title" :title="doc._source.name + ext(doc)">{{ doc._source.name + ext(doc) }}</h5>
     </template>
-    <img :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">
+
+    <img v-if="doc._props.hasThumbnail" :src="`t/${doc._source.index}/${doc._id}`" alt="" class="fit card-img-top">
 
     <InfoTable :doc="doc"></InfoTable>
 

sist2-vue/src/components/DocListItem.vue

@@ -1,5 +1,5 @@
 <template>
-  <b-list-group-item class="flex-column align-items-start mb-2">
+  <b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}">
 
     <!-- Info modal-->
     <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
@@ -40,9 +40,11 @@
         </div>
 
         <div v-if="doc._source.pages || doc._source.author" class="path-row text-muted">
-          <span v-if="doc._source.pages">{{ doc._source.pages }} {{ doc._source.pages > 1 ? $t("pages") : $t("page") }}</span>
+          <span v-if="doc._source.pages">{{ doc._source.pages }} {{
+              doc._source.pages > 1 ? $t("pages") : $t("page")
+            }}</span>
           <span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
-          <span v-if="doc._source.author">{{doc._source.author}}</span>
+          <span v-if="doc._source.author">{{ doc._source.author }}</span>
         </div>
       </div>
     </div>
@@ -89,6 +91,14 @@ export default {
 </script>
 
 <style scoped>
+.sub-document {
+  background: #AB47BC1F !important;
+}
+
+.theme-black .sub-document {
+  background: #37474F !important;
+}
+
 .list-group {
   margin-top: 1em;
 }

sist2-vue/src/components/IndexPicker.vue

@@ -1,93 +1,95 @@
 <template>
-  <VueMultiselect
+  <div v-if="isMobile">
-      multiple
+    <b-form-select
-      label="name"
+        :value="selectedIndicesIds"
-      :value="selectedIndices"
+        @change="onSelect($event)"
-      :options="indices"
+        :options="indices" multiple :select-size="6" text-field="name"
-      :close-on-select="indices.length <= 1"
+        value-field="id"></b-form-select>
-      :placeholder="$t('indexPickerPlaceholder')"
+  </div>
-      @select="addItem"
+  <div v-else>
-      @remove="removeItem">
+    <b-list-group id="index-picker-desktop">
-
+      <b-list-group-item
-    <template slot="option" slot-scope="idx">
+          v-for="idx in indices"
-      <b-row>
+          @click="toggleIndex(idx)"
-        <b-col>
+          class="d-flex justify-content-between align-items-center list-group-item-action pointer">
-          <span class="mr-1">{{ idx.option.name }}</span>
+        <div class="d-flex">
-          <SmallBadge pill :text="idx.option.version"></SmallBadge>
+          <b-checkbox @change="toggleIndex(idx)" :checked="isSelected(idx)"></b-checkbox>
-        </b-col>
+          {{ idx.name }}
-      </b-row>
+          <span class="text-muted timestamp-text ml-2">{{ formatIdxDate(idx.timestamp) }}</span>
-      <b-row class="mt-1">
+        </div>
-        <b-col>
+        <b-badge class="version-badge">v{{ idx.version }}</b-badge>
-          <span>{{ formatIdxDate(idx.option.timestamp) }}</span>
+      </b-list-group-item>
-        </b-col>
+    </b-list-group>
-      </b-row>
+  </div>
-    </template>
-
-  </VueMultiselect>
 </template>
 
 <script lang="ts">
-import VueMultiselect from "vue-multiselect"
 import SmallBadge from "./SmallBadge.vue"
 import {mapActions, mapGetters} from "vuex";
-import {Index} from "@/Sist2Api";
 import Vue from "vue";
 import {format} from "date-fns";
 
 export default Vue.extend({
   components: {
-    VueMultiselect,
     SmallBadge
   },
   data() {
     return {
-      loading: true
+      loading: true,
     }
   },
   computed: {
     ...mapGetters([
       "indices", "selectedIndices"
     ]),
+    selectedIndicesIds() {
+      return this.selectedIndices.map(idx => idx.id)
+    },
+    isMobile() {
+      return window.innerWidth <= 650;
+    }
   },
   methods: {
     ...mapActions({
       setSelectedIndices: "setSelectedIndices"
     }),
-    removeItem(val: Index): void {
+    onSelect(value) {
-      this.setSelectedIndices(this.selectedIndices.filter((item: Index) => item !== val))
+      this.setSelectedIndices(this.indices.filter(idx => value.includes(idx.id)));
-    },
-    addItem(val: Index): void {
-      this.setSelectedIndices([...this.selectedIndices, val])
     },
     formatIdxDate(timestamp: number): string {
       return format(new Date(timestamp * 1000), "yyyy-MM-dd");
+    },
+    toggleIndex(index) {
+      if (this.isSelected(index)) {
+        this.setSelectedIndices(this.selectedIndices.filter(idx => idx.id != index.id));
+      } else {
+        this.setSelectedIndices([index, ...this.selectedIndices]);
+      }
+    },
+    isSelected(index) {
+      return this.selectedIndices.find(idx => idx.id == index.id) != null;
     }
   },
 })
 </script>
 
-<style src="vue-multiselect/dist/vue-multiselect.min.css"></style>
+<style scoped>
-
+.timestamp-text {
-<style>
+  line-height: 24px;
-.multiselect__option {
+  font-size: 80%;
-  padding: 5px 10px;
 }
 
-.multiselect__content-wrapper {
+.version-badge {
-  overflow: hidden;
+  color: #222 !important;
+  background: none;
 }
 
-.theme-black .multiselect__tags {
+.list-group-item {
-  background: #37474F;
+  padding: 0.2em 0.4em;
-  border: 1px solid #616161 !important
 }
 
-.theme-black .multiselect__input {
+#index-picker-desktop {
-  color: #dbdbdb;
+  overflow-y: auto;
-  background: #37474F;
+  max-height: 132px;
-}
-
-.theme-black .multiselect__content-wrapper {
-  border: none
 }
 </style>

sist2-vue/src/components/InfoTable.vue

@@ -3,7 +3,7 @@
 
     <template #cell(value)="data">
       <span v-if="'html' in data.item" v-html="data.item.html"></span>
-      <span v-else>{{data.value}}</span>
+      <span v-else>{{ data.value }}</span>
     </template>
   </b-table>
 </template>
@@ -57,7 +57,8 @@ export default {
         "bitrate", "artist", "album", "album_artist", "genre", "font_name", "author",
         "modified_by", "pages", "tag",
         "exif_make", "exif_software", "exif_exposure_time", "exif_fnumber", "exif_focal_length",
-          "exif_user_comment", "exif_iso_speed_ratings", "exif_model", "exif_datetime",
+        "exif_user_comment", "exif_iso_speed_ratings", "exif_model", "exif_datetime",
+        "checksum"
       ];
 
       fields.forEach(field => {
@@ -76,9 +77,9 @@ export default {
         items.push({
           key: "Exif GPS",
           html: makeGpsLink(
-                  dmsToDecimal(src["exif_gps_latitude_dms"], src["exif_gps_latitude_ref"]),
+              dmsToDecimal(src["exif_gps_latitude_dms"], src["exif_gps_latitude_ref"]),
-                  dmsToDecimal(src["exif_gps_longitude_dms"], src["exif_gps_longitude_ref"]),
+              dmsToDecimal(src["exif_gps_longitude_dms"], src["exif_gps_longitude_ref"]),
-              ),
+          ),
         });
       }
 

sist2-vue/src/i18n/messages.ts

@@ -62,7 +62,8 @@ export default {
             lightboxLoadOnlyCurrent: "Do not preload full-size images for adjacent slides in image viewer.",
             slideDuration: "Slide duration",
             resultSize: "Number of results per page",
-            tagOrOperator: "Use OR operator when specifying multiple tags."
+            tagOrOperator: "Use OR operator when specifying multiple tags.",
+            hideDuplicates: "Hide duplicate results based on checksum"
         },
         queryMode: {
             simple: "Simple",
@@ -209,7 +210,8 @@ export default {
             lightboxLoadOnlyCurrent: "Désactiver le chargement des diapositives adjacentes pour le visualiseur d'images",
             slideDuration: "Durée des diapositives",
             resultSize: "Nombre de résultats par page",
-            tagOrOperator: "Utiliser l'opérateur OU lors de la spécification de plusieurs tags"
+            tagOrOperator: "Utiliser l'opérateur OU lors de la spécification de plusieurs tags",
+            hideDuplicates: "Masquer les résultats en double"
         },
         queryMode: {
             simple: "Simple",

sist2-vue/src/store/index.ts

@@ -27,6 +27,7 @@ export default new Vuex.Store({
         size: 60,
 
         optLang: "en",
+        optHideDuplicates: true,
         optTheme: "light",
         optDisplay: "grid",
 
@@ -79,6 +80,7 @@ export default new Vuex.Store({
         setSizeMax: (state, val) => state.sizeMax = val,
         setSist2Info: (state, val) => state.sist2Info = val,
         setSeed: (state, val) => state.seed = val,
+        setOptHideDuplicates: (state, val) => state.optHideDuplicates = val,
         setOptLang: (state, val) => state.optLang = val,
         setSortMode: (state, val) => state.sortMode = val,
         setIndices: (state, val) => {
@@ -317,6 +319,7 @@ export default new Vuex.Store({
         uiLightboxKey: state => state.uiLightboxKey,
         uiLightboxSlide: state => state.uiLightboxSlide,
 
+        optHideDuplicates: state => state.optHideDuplicates,
         optLang: state => state.optLang,
         optTheme: state => state.optTheme,
         optDisplay: state => state.optDisplay,

sist2-vue/src/views/Configuration.vue

@@ -35,6 +35,11 @@
         <br/>
         <h4>{{ $t("searchOptions") }}</h4>
         <b-card>
+          <b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
+              $t("opt.hideDuplicates")
+            }}
+          </b-form-checkbox>
+
           <b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{ $t("opt.highlight") }}</b-form-checkbox>
           <b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
               $t("opt.tagOrOperator")
@@ -206,10 +211,10 @@ export default {
       "optTreemapSize",
       "optLightboxLoadOnlyCurrent",
       "optLightboxSlideDuration",
-      "optContainerWidth",
       "optResultSize",
       "optTagOrOperator",
-      "optLang"
+      "optLang",
+      "optHideDuplicates",
     ]),
     clientWidth() {
       return window.innerWidth;
@@ -248,7 +253,8 @@ export default {
       "setOptContainerWidth",
       "setOptResultSize",
       "setOptTagOrOperator",
-      "setOptLang"
+      "setOptLang",
+      "setOptHideDuplicates"
     ]),
     onResetClick() {
       localStorage.removeItem("sist2_configuration");

sist2-vue/src/views/SearchPage.vue

@@ -91,6 +91,7 @@ export default Vue.extend({
     search: undefined as any,
     docs: [] as EsHit[],
     docIds: new Set(),
+    docChecksums: new Set(),
     searchBusy: false,
     Sist2Query: Sist2Query,
     showHelp: false
@@ -193,6 +194,7 @@ export default Vue.extend({
     async clearResults() {
       this.docs = [];
       this.docIds.clear();
+      this.docChecksums.clear();
       await this.$store.dispatch("clearResults");
       this.$store.commit("setUiReachedScrollEnd", false);
     },
@@ -202,7 +204,19 @@ export default Vue.extend({
       }
 
       resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
-      resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
+
+      if (this.$store.state.optHideDuplicates) {
+        resp.hits.hits = resp.hits.hits.filter(hit => {
+
+          if (!("checksum" in hit._source)) {
+              return true;
+          }
+
+          const isDupe = !this.docChecksums.has(hit._source.checksum);
+          this.docChecksums.add(hit._source.checksum);
+          return isDupe;
+        });
+      }
 
       for (const hit of resp.hits.hits) {
         if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {

src/cli.h

@@ -28,6 +28,7 @@ typedef struct scan_args {
     int max_memory_buffer;
     int read_subtitles;
     int fast_epub;
+    int calculate_checksums;
 } scan_args_t;
 
 scan_args_t *scan_args_create();

src/ctx.h

@@ -33,6 +33,7 @@ typedef struct {
 
     int threads;
     int depth;
+    int calculate_checksums;
 
     size_t stat_tn_size;
     size_t stat_index_size;

src/io/serialize.c

@@ -74,6 +74,8 @@ char *get_meta_key_text(enum metakey meta_key) {
             return "exif_gps_latitude_dms";
         case MetaExifGpsLatitudeDec:
             return "exif_gps_latitude_dec";
+        case MetaChecksum:
+            return "checksum";
         default:
         LOG_FATALF("serialize.c", "FIXME: Unknown meta key: %d", meta_key)
     }
@@ -165,6 +167,7 @@ char *build_json_string(document_t *doc) {
             case MetaExifGpsLatitudeDMS:
             case MetaExifGpsLatitudeDec:
             case MetaExifGpsLatitudeRef:
+            case MetaChecksum:
             case MetaTitle: {
                 cJSON_AddStringToObject(json, get_meta_key_text(meta->key), meta->str_val);
                 buffer_size_guess += (int) strlen(meta->str_val);

src/io/store.c

@@ -4,6 +4,7 @@
 store_t *store_create(const char *path, size_t chunk_size) {
     store_t *store = malloc(sizeof(struct store_t));
     mkdir(path, S_IWUSR | S_IRUSR | S_IXUSR);
+    strcpy(store->path, path);
 
 #if (SIST_FAKE_STORE != 1)
     store->chunk_size = chunk_size;
@@ -78,27 +79,57 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
     int put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
     ScanCtx.stat_tn_size += buf_len;
 
+    int db_full = FALSE;
+    int should_abort_transaction = FALSE;
+
     if (put_ret == MDB_MAP_FULL) {
-        mdb_txn_abort(txn);
+        db_full = TRUE;
+        should_abort_transaction = TRUE;
+    } else {
+        int commit_ret = mdb_txn_commit(txn);
+
+        if (commit_ret == MDB_MAP_FULL) {
+            db_full = TRUE;
+        }
+    }
+
+    if (db_full) {
+        LOG_INFOF("store.c", "Updating mdb mapsize to %lu bytes", store->size)
+
+        if (should_abort_transaction) {
+            mdb_txn_abort(txn);
+        }
+
         pthread_rwlock_unlock(&store->lock);
 
         // Cannot resize when there is a opened transaction.
         //  Resize take effect on the next commit.
         pthread_rwlock_wrlock(&store->lock);
         store->size += store->chunk_size;
-        mdb_env_set_mapsize(store->env, store->size);
+        int resize_ret = mdb_env_set_mapsize(store->env, store->size);
+        if (resize_ret != 0) {
+            LOG_ERROR("store.c", mdb_strerror(put_ret))
+        }
         mdb_txn_begin(store->env, NULL, 0, &txn);
-        put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
+        int put_ret_retry = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
 
+        if (put_ret_retry != 0) {
+            LOG_ERROR("store.c", mdb_strerror(put_ret))
+        }
+
+        int ret = mdb_txn_commit(txn);
+        if (ret != 0) {
+            LOG_FATALF("store.c", "FIXME: Could not commit to store %s: %s (%d), %d, %d %d",
+                       store->path, mdb_strerror(ret), ret,
+                       put_ret, put_ret_retry);
+        }
         LOG_INFOF("store.c", "Updated mdb mapsize to %lu bytes", store->size)
-    }
+    } else if (put_ret != 0) {
-
-    mdb_txn_commit(txn);
-    pthread_rwlock_unlock(&store->lock);
-
-    if (put_ret != 0) {
         LOG_ERROR("store.c", mdb_strerror(put_ret))
     }
+
+    pthread_rwlock_unlock(&store->lock);
+
 #endif
 }
 

src/io/store.h

@@ -6,12 +6,12 @@
 
 #include <glib.h>
 
-#define STORE_SIZE_TN 1024 * 1024 * 5
+#define STORE_SIZE_TN (1024 * 1024 * 5)
-#define STORE_SIZE_TAG 1024 * 16
+#define STORE_SIZE_TAG (1024 * 1024)
 #define STORE_SIZE_META STORE_SIZE_TAG
 
 typedef struct store_t {
-    char *path;
+    char path[PATH_MAX];
     char *tmp_path;
     MDB_dbi dbi;
     MDB_env *env;

src/io/walk.c

@@ -24,16 +24,22 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info,
 
     job->vfile.filepath = job->filepath;
     job->vfile.read = fs_read;
+    // Filesystem reads are always rewindable
+    job->vfile.read_rewindable = fs_read;
     job->vfile.reset = fs_reset;
     job->vfile.close = fs_close;
     job->vfile.fd = -1;
     job->vfile.is_fs_file = TRUE;
+    job->vfile.has_checksum = FALSE;
+    job->vfile.rewind_buffer_size = 0;
+    job->vfile.rewind_buffer = NULL;
+    job->vfile.calculate_checksum = ScanCtx.calculate_checksums;
 
     return job;
 }
 
 int sub_strings[30];
-#define EXCLUDED(str) (pcre_exec(ScanCtx.exclude, ScanCtx.exclude_extra, filepath, strlen(filepath), 0, 0, sub_strings, sizeof(sub_strings)) >= 0)
+#define EXCLUDED(str) (pcre_exec(ScanCtx.exclude, ScanCtx.exclude_extra, str, strlen(str), 0, 0, sub_strings, sizeof(sub_strings)) >= 0)
 
 int handle_entry(const char *filepath, const struct stat *info, int typeflag, struct FTW *ftw) {
 

src/main.c

@@ -170,6 +170,8 @@ void initialize_scan_context(scan_args_t *args) {
     pthread_mutex_init(&ScanCtx.dbg_current_files_mu, NULL);
     pthread_mutex_init(&ScanCtx.dbg_file_counts_mu, NULL);
 
+    ScanCtx.calculate_checksums = args->calculate_checksums;
+
     // Archive
     ScanCtx.arc_ctx.mode = args->archive_mode;
     ScanCtx.arc_ctx.log = _log;
@@ -516,8 +518,8 @@ void sist2_web(web_args_t *args) {
 
 
 int main(int argc, const char *argv[]) {
-    sigsegv_handler = signal(SIGSEGV, sig_handler);
+//    sigsegv_handler = signal(SIGSEGV, sig_handler);
-    sigabrt_handler = signal(SIGABRT, sig_handler);
+//    sigabrt_handler = signal(SIGABRT, sig_handler);
 
     setlocale(LC_ALL, "");
 
@@ -574,6 +576,7 @@ int main(int argc, const char *argv[]) {
             OPT_BOOLEAN(0, "read-subtitles", &scan_args->read_subtitles, "Read subtitles from media files."),
             OPT_BOOLEAN(0, "fast-epub", &scan_args->fast_epub,
                         "Faster but less accurate EPUB parsing (no thumbnails, metadata)"),
+            OPT_BOOLEAN(0, "checksums", &scan_args->calculate_checksums, "Calculate file checksums when scanning."),
 
             OPT_GROUP("Index options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),

src/parsing/parse.c

@@ -10,25 +10,34 @@
 
 
 #define MIN_VIDEO_SIZE (1024 * 64)
-#define MIN_IMAGE_SIZE (1024 * 2)
+#define MIN_IMAGE_SIZE (512)
 
 int fs_read(struct vfile *f, void *buf, size_t size) {
 
     if (f->fd == -1) {
+        SHA1_Init(&f->sha1_ctx);
+
         f->fd = open(f->filepath, O_RDONLY);
         if (f->fd == -1) {
-            LOG_ERRORF(f->filepath, "open(): [%d] %s", errno, strerror(errno))
             return -1;
         }
     }
 
-    return read(f->fd, buf, size);
+    int ret = (int) read(f->fd, buf, size);
+
+    if (ret != 0 && f->calculate_checksum) {
+        f->has_checksum = TRUE;
+        safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret);
+    }
+
+    return ret;
 }
 
 #define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));};
 
 void fs_close(struct vfile *f) {
     if (f->fd != -1) {
+        SHA1_Final(f->sha1_digest, &f->sha1_ctx);
         close(f->fd);
     }
 }
@@ -66,7 +75,7 @@ void parse(void *arg) {
     doc->meta_tail = NULL;
     doc->mime = 0;
     doc->size = job->vfile.info.st_size;
-    doc->mtime = job->vfile.info.st_mtim.tv_sec;
+    doc->mtime = (int) job->vfile.info.st_mtim.tv_sec;
 
     int inc_ts = incremental_get(ScanCtx.original_table, doc->path_md5);
     if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) {
@@ -93,18 +102,17 @@ void parse(void *arg) {
         doc->mime = mime_get_mime_by_ext(ScanCtx.ext_table, job->filepath + job->ext);
     }
 
-    int bytes_read = 0;
 
     if (doc->mime == 0 && !ScanCtx.fast) {
 
         // Get mime type with libmagic
-        if (!job->vfile.is_fs_file) {
+        if (job->vfile.read_rewindable == NULL) {
             LOG_WARNING(job->filepath,
-                        "Guessing mime type with libmagic inside archive files is not currently supported");
+                        "File does not support rewindable reads, cannot guess Media type");
             goto abort;
         }
 
-        bytes_read = job->vfile.read(&job->vfile, buf, MAGIC_BUF_SIZE);
+        int bytes_read = job->vfile.read_rewindable(&job->vfile, buf, MAGIC_BUF_SIZE);
         if (bytes_read < 0) {
 
             if (job->vfile.is_fs_file) {
@@ -135,7 +143,9 @@ void parse(void *arg) {
             }
         }
 
-        job->vfile.reset(&job->vfile);
+        if (job->vfile.reset != NULL) {
+            job->vfile.reset(&job->vfile);
+        }
 
         magic_close(magic);
     }
@@ -149,7 +159,7 @@ void parse(void *arg) {
     } else if ((mmime == MimeVideo && doc->size >= MIN_VIDEO_SIZE) ||
                (mmime == MimeImage && doc->size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) {
 
-        parse_media(&ScanCtx.media_ctx, &job->vfile, doc);
+        parse_media(&ScanCtx.media_ctx, &job->vfile, doc, mime_get_mime_text(doc->mime));
 
     } else if (IS_PDF(doc->mime)) {
         parse_ebook(&ScanCtx.ebook_ctx, &job->vfile, mime_get_mime_text(doc->mime), doc);
@@ -169,7 +179,7 @@ void parse(void *arg) {
                     IS_ARC(doc->mime) ||
                     (IS_ARC_FILTER(doc->mime) && should_parse_filtered_file(doc->filepath, doc->ext))
             )) {
-        parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc);
+        parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc, ScanCtx.exclude, ScanCtx.exclude_extra);
     } else if ((ScanCtx.ooxml_ctx.content_size > 0 || ScanCtx.media_ctx.tn_size > 0) && IS_DOC(doc->mime)) {
         parse_ooxml(&ScanCtx.ooxml_ctx, &job->vfile, doc);
     } else if (is_cbr(&ScanCtx.comic_ctx, doc->mime) || is_cbz(&ScanCtx.comic_ctx, doc->mime)) {
@@ -179,6 +189,8 @@ void parse(void *arg) {
     } else if (doc->mime == MIME_SIST2_SIDECAR) {
         parse_sidecar(&job->vfile, doc);
         CLOSE_FILE(job->vfile)
+        free(doc->filepath);
+        free(doc);
         return;
     } else if (is_msdoc(&ScanCtx.msdoc_ctx, doc->mime)) {
         parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, doc);
@@ -202,9 +214,15 @@ void parse(void *arg) {
         doc->has_parent = FALSE;
     }
 
-    write_document(doc);
-
     CLOSE_FILE(job->vfile)
+
+    if (job->vfile.has_checksum) {
+        char sha1_digest_str[SHA1_STR_LENGTH];
+        buf2hex((unsigned char *) job->vfile.sha1_digest, SHA1_DIGEST_LENGTH, (char *) sha1_digest_str);
+        APPEND_STR_META(doc, MetaChecksum, (const char *) sha1_digest_str);
+    }
+
+    write_document(doc);
 }
 
 void cleanup_parse() {

src/parsing/sidecar.c

@@ -27,7 +27,10 @@ void parse_sidecar(vfile_t *vfile, document_t *doc) {
     MD5((unsigned char *) vfile->filepath + ScanCtx.index.desc.root_len, doc->ext - 1 - ScanCtx.index.desc.root_len,
         path_md5);
 
-    store_write(ScanCtx.index.meta_store, (char *) path_md5, sizeof(path_md5), json_str, strlen(json_str) + 1);
+    char path_md5_str[MD5_STR_LENGTH];
+    buf2hex(path_md5, MD5_DIGEST_LENGTH, path_md5_str);
+
+    store_write(ScanCtx.index.meta_store, path_md5_str, MD5_STR_LENGTH, json_str, strlen(json_str) + 1);
 
     cJSON_Delete(json);
     free(json_str);

src/sist.h

@@ -26,6 +26,8 @@
 #define UNUSED(x) __attribute__((__unused__))  x
 
 #define MD5_STR_LENGTH 33
+#define SHA1_STR_LENGTH 41
+#define SHA1_DIGEST_LENGTH 20
 
 #include "util.h"
 #include "log.h"
@@ -49,7 +51,7 @@
 #include <ctype.h>
 #include "git_hash.h"
 
-#define VERSION "2.11.2"
+#define VERSION "2.11.3"
 static const char *const Version = VERSION;
 
 #ifndef SIST_PLATFORM