schema/mappings.json

@@ -4,10 +4,6 @@
       "type": "keyword",
       "doc_values": true
     },
-    "checksum": {
-      "type": "keyword",
-      "index": false
-    },
     "_depth": {
       "type": "integer"
     },

sist2-vue/src/Sist2Api.ts

@@ -50,7 +50,6 @@ export interface EsHit {
         height: number
         duration: number
         tag: string[]
-        checksum: string
     }
     _props: {
         isSubDocument: boolean

sist2-vue/src/components/InfoTable.vue

@@ -58,7 +58,6 @@ export default {
         "modified_by", "pages", "tag",
         "exif_make", "exif_software", "exif_exposure_time", "exif_fnumber", "exif_focal_length",
           "exif_user_comment", "exif_iso_speed_ratings", "exif_model", "exif_datetime",
-        "checksum"
       ];
 
       fields.forEach(field => {

sist2-vue/src/i18n/messages.ts

@@ -62,8 +62,7 @@ export default {
             lightboxLoadOnlyCurrent: "Do not preload full-size images for adjacent slides in image viewer.",
             slideDuration: "Slide duration",
             resultSize: "Number of results per page",
-            tagOrOperator: "Use OR operator when specifying multiple tags.",
+            tagOrOperator: "Use OR operator when specifying multiple tags."
-            hideDuplicates: "Hide duplicate results based on checksum"
         },
         queryMode: {
             simple: "Simple",
@@ -210,8 +209,7 @@ export default {
             lightboxLoadOnlyCurrent: "Désactiver le chargement des diapositives adjacentes pour le visualiseur d'images",
             slideDuration: "Durée des diapositives",
             resultSize: "Nombre de résultats par page",
-            tagOrOperator: "Utiliser l'opérateur OU lors de la spécification de plusieurs tags",
+            tagOrOperator: "Utiliser l'opérateur OU lors de la spécification de plusieurs tags"
-            hideDuplicates: "Masquer les résultats en double"
         },
         queryMode: {
             simple: "Simple",

sist2-vue/src/store/index.ts

@@ -27,7 +27,6 @@ export default new Vuex.Store({
         size: 60,
 
         optLang: "en",
-        optHideDuplicates: true,
         optTheme: "light",
         optDisplay: "grid",
 
@@ -80,7 +79,6 @@ export default new Vuex.Store({
         setSizeMax: (state, val) => state.sizeMax = val,
         setSist2Info: (state, val) => state.sist2Info = val,
         setSeed: (state, val) => state.seed = val,
-        setOptHideDuplicates: (state, val) => state.optHideDuplicates = val,
         setOptLang: (state, val) => state.optLang = val,
         setSortMode: (state, val) => state.sortMode = val,
         setIndices: (state, val) => {
@@ -319,7 +317,6 @@ export default new Vuex.Store({
         uiLightboxKey: state => state.uiLightboxKey,
         uiLightboxSlide: state => state.uiLightboxSlide,
 
-        optHideDuplicates: state => state.optHideDuplicates,
         optLang: state => state.optLang,
         optTheme: state => state.optTheme,
         optDisplay: state => state.optDisplay,

sist2-vue/src/views/Configuration.vue

@@ -35,11 +35,6 @@
         <br/>
         <h4>{{ $t("searchOptions") }}</h4>
         <b-card>
-          <b-form-checkbox :checked="optHideDuplicates" @input="setOptHideDuplicates">{{
-              $t("opt.hideDuplicates")
-            }}
-          </b-form-checkbox>
-
           <b-form-checkbox :checked="optHighlight" @input="setOptHighlight">{{ $t("opt.highlight") }}</b-form-checkbox>
           <b-form-checkbox :checked="optTagOrOperator" @input="setOptTagOrOperator">{{
               $t("opt.tagOrOperator")
@@ -211,10 +206,10 @@ export default {
       "optTreemapSize",
       "optLightboxLoadOnlyCurrent",
       "optLightboxSlideDuration",
+      "optContainerWidth",
       "optResultSize",
       "optTagOrOperator",
-      "optLang",
+      "optLang"
-      "optHideDuplicates",
     ]),
     clientWidth() {
       return window.innerWidth;
@@ -253,8 +248,7 @@ export default {
       "setOptContainerWidth",
       "setOptResultSize",
       "setOptTagOrOperator",
-      "setOptLang",
+      "setOptLang"
-      "setOptHideDuplicates"
     ]),
     onResetClick() {
       localStorage.removeItem("sist2_configuration");

sist2-vue/src/views/SearchPage.vue

@@ -91,7 +91,6 @@ export default Vue.extend({
     search: undefined as any,
     docs: [] as EsHit[],
     docIds: new Set(),
-    docChecksums: new Set(),
     searchBusy: false,
     Sist2Query: Sist2Query,
     showHelp: false
@@ -194,7 +193,6 @@ export default Vue.extend({
     async clearResults() {
       this.docs = [];
       this.docIds.clear();
-      this.docChecksums.clear();
       await this.$store.dispatch("clearResults");
       this.$store.commit("setUiReachedScrollEnd", false);
     },
@@ -204,19 +202,7 @@ export default Vue.extend({
       }
 
       resp.hits.hits = resp.hits.hits.filter(hit => !this.docIds.has(hit._id));
-
+      resp.hits.hits.forEach(hit => this.docIds.add(hit._id));
-      if (this.$store.state.optHideDuplicates) {
-        resp.hits.hits = resp.hits.hits.filter(hit => {
-
-          if (!("checksum" in hit._source)) {
-              return true;
-          }
-
-          const isDupe = !this.docChecksums.has(hit._source.checksum);
-          this.docChecksums.add(hit._source.checksum);
-          return isDupe;
-        });
-      }
 
       for (const hit of resp.hits.hits) {
         if (hit._props.isPlayableImage || hit._props.isPlayableVideo) {

src/cli.h

@@ -28,7 +28,6 @@ typedef struct scan_args {
     int max_memory_buffer;
     int read_subtitles;
     int fast_epub;
-    int calculate_checksums;
 } scan_args_t;
 
 scan_args_t *scan_args_create();

src/ctx.h

@@ -33,7 +33,6 @@ typedef struct {
 
     int threads;
     int depth;
-    int calculate_checksums;
 
     size_t stat_tn_size;
     size_t stat_index_size;

src/io/serialize.c

@@ -74,8 +74,6 @@ char *get_meta_key_text(enum metakey meta_key) {
             return "exif_gps_latitude_dms";
         case MetaExifGpsLatitudeDec:
             return "exif_gps_latitude_dec";
-        case MetaChecksum:
-            return "checksum";
         default:
         LOG_FATALF("serialize.c", "FIXME: Unknown meta key: %d", meta_key)
     }
@@ -167,7 +165,6 @@ char *build_json_string(document_t *doc) {
             case MetaExifGpsLatitudeDMS:
             case MetaExifGpsLatitudeDec:
             case MetaExifGpsLatitudeRef:
-            case MetaChecksum:
             case MetaTitle: {
                 cJSON_AddStringToObject(json, get_meta_key_text(meta->key), meta->str_val);
                 buffer_size_guess += (int) strlen(meta->str_val);

src/io/walk.c

@@ -24,16 +24,10 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info,
 
     job->vfile.filepath = job->filepath;
     job->vfile.read = fs_read;
-    // Filesystem reads are always rewindable
-    job->vfile.read_rewindable = fs_read;
     job->vfile.reset = fs_reset;
     job->vfile.close = fs_close;
     job->vfile.fd = -1;
     job->vfile.is_fs_file = TRUE;
-    job->vfile.has_checksum = FALSE;
-    job->vfile.rewind_buffer_size = 0;
-    job->vfile.rewind_buffer = NULL;
-    job->vfile.calculate_checksum = ScanCtx.calculate_checksums;
 
     return job;
 }

src/main.c

@@ -170,8 +170,6 @@ void initialize_scan_context(scan_args_t *args) {
     pthread_mutex_init(&ScanCtx.dbg_current_files_mu, NULL);
     pthread_mutex_init(&ScanCtx.dbg_file_counts_mu, NULL);
 
-    ScanCtx.calculate_checksums = args->calculate_checksums;
-
     // Archive
     ScanCtx.arc_ctx.mode = args->archive_mode;
     ScanCtx.arc_ctx.log = _log;
@@ -518,8 +516,8 @@ void sist2_web(web_args_t *args) {
 
 
 int main(int argc, const char *argv[]) {
-//    sigsegv_handler = signal(SIGSEGV, sig_handler);
+    sigsegv_handler = signal(SIGSEGV, sig_handler);
-//    sigabrt_handler = signal(SIGABRT, sig_handler);
+    sigabrt_handler = signal(SIGABRT, sig_handler);
 
     setlocale(LC_ALL, "");
 
@@ -576,7 +574,6 @@ int main(int argc, const char *argv[]) {
             OPT_BOOLEAN(0, "read-subtitles", &scan_args->read_subtitles, "Read subtitles from media files."),
             OPT_BOOLEAN(0, "fast-epub", &scan_args->fast_epub,
                         "Faster but less accurate EPUB parsing (no thumbnails, metadata)"),
-            OPT_BOOLEAN(0, "checksums", &scan_args->calculate_checksums, "Calculate file checksums when scanning."),
 
             OPT_GROUP("Index options"),
             OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"),

src/parsing/parse.c

@@ -10,34 +10,25 @@
 
 
 #define MIN_VIDEO_SIZE (1024 * 64)
-#define MIN_IMAGE_SIZE (512)
+#define MIN_IMAGE_SIZE (1024 * 2)
 
 int fs_read(struct vfile *f, void *buf, size_t size) {
 
     if (f->fd == -1) {
-        SHA1_Init(&f->sha1_ctx);
-
         f->fd = open(f->filepath, O_RDONLY);
         if (f->fd == -1) {
+            LOG_ERRORF(f->filepath, "open(): [%d] %s", errno, strerror(errno))
             return -1;
         }
     }
 
-    int ret = (int) read(f->fd, buf, size);
+    return read(f->fd, buf, size);
-
-    if (ret != 0 && f->calculate_checksum) {
-        f->has_checksum = TRUE;
-        safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret);
-    }
-
-    return ret;
 }
 
 #define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));};
 
 void fs_close(struct vfile *f) {
     if (f->fd != -1) {
-        SHA1_Final(f->sha1_digest, &f->sha1_ctx);
         close(f->fd);
     }
 }
@@ -75,7 +66,7 @@ void parse(void *arg) {
     doc->meta_tail = NULL;
     doc->mime = 0;
     doc->size = job->vfile.info.st_size;
-    doc->mtime = (int) job->vfile.info.st_mtim.tv_sec;
+    doc->mtime = job->vfile.info.st_mtim.tv_sec;
 
     int inc_ts = incremental_get(ScanCtx.original_table, doc->path_md5);
     if (inc_ts != 0 && inc_ts == job->vfile.info.st_mtim.tv_sec) {
@@ -102,17 +93,18 @@ void parse(void *arg) {
         doc->mime = mime_get_mime_by_ext(ScanCtx.ext_table, job->filepath + job->ext);
     }
 
+    int bytes_read = 0;
 
     if (doc->mime == 0 && !ScanCtx.fast) {
 
         // Get mime type with libmagic
-        if (job->vfile.read_rewindable == NULL) {
+        if (!job->vfile.is_fs_file) {
             LOG_WARNING(job->filepath,
-                        "File does not support rewindable reads, cannot guess Media type");
+                        "Guessing mime type with libmagic inside archive files is not currently supported");
             goto abort;
         }
 
-        int bytes_read = job->vfile.read_rewindable(&job->vfile, buf, MAGIC_BUF_SIZE);
+        bytes_read = job->vfile.read(&job->vfile, buf, MAGIC_BUF_SIZE);
         if (bytes_read < 0) {
 
             if (job->vfile.is_fs_file) {
@@ -143,9 +135,7 @@ void parse(void *arg) {
             }
         }
 
-        if (job->vfile.reset != NULL) {
         job->vfile.reset(&job->vfile);
-        }
 
         magic_close(magic);
     }
@@ -159,7 +149,7 @@ void parse(void *arg) {
     } else if ((mmime == MimeVideo && doc->size >= MIN_VIDEO_SIZE) ||
                (mmime == MimeImage && doc->size >= MIN_IMAGE_SIZE) || mmime == MimeAudio) {
 
-        parse_media(&ScanCtx.media_ctx, &job->vfile, doc, mime_get_mime_text(doc->mime));
+        parse_media(&ScanCtx.media_ctx, &job->vfile, doc);
 
     } else if (IS_PDF(doc->mime)) {
         parse_ebook(&ScanCtx.ebook_ctx, &job->vfile, mime_get_mime_text(doc->mime), doc);
@@ -212,15 +202,9 @@ void parse(void *arg) {
         doc->has_parent = FALSE;
     }
 
-    CLOSE_FILE(job->vfile)
-
-    if (job->vfile.has_checksum) {
-        char sha1_digest_str[SHA1_STR_LENGTH];
-        buf2hex((unsigned char *) job->vfile.sha1_digest, SHA1_DIGEST_LENGTH, (char *) sha1_digest_str);
-        APPEND_STR_META(doc, MetaChecksum, (const char *) sha1_digest_str);
-    }
-
     write_document(doc);
+
+    CLOSE_FILE(job->vfile)
 }
 
 void cleanup_parse() {

src/sist.h

@@ -26,8 +26,6 @@
 #define UNUSED(x) __attribute__((__unused__))  x
 
 #define MD5_STR_LENGTH 33
-#define SHA1_STR_LENGTH 41
-#define SHA1_DIGEST_LENGTH 20
 
 #include "util.h"
 #include "log.h"

third-party/libscan

@@ -1 +1 @@
-Subproject commit da172823745b67662846cf1970a47ebcea8fe50e
+Subproject commit fe53e1a219246d829439bb26093713a415a58924