Fix max_analyzed_offset error

schema/settings.json

@@ -2,7 +2,8 @@
   "index": {
     "refresh_interval": "30s",
     "codec": "best_compression",
-    "number_of_replicas": 0
+    "number_of_replicas": 0,
+    "highlight.max_analyzed_offset": 10000000
   },
   "analysis": {
     "tokenizer": {

sist2-vue/src/Sist2Query.ts

@@ -187,7 +187,8 @@ class Sist2Query {
                     "name.nGram": {},
                     "content.nGram": {},
                     font_name: {},
-                }
+                },
+                max_analyzed_offset: 9_999_999
             };
             if (getters.optSearchInPath) {
                 q.highlight.fields["path.text"] = {};

sist2-vue/src/components/DocListItem.vue

@@ -1,5 +1,5 @@
 <template>
-  <b-list-group-item class="flex-column align-items-start mb-2">
+  <b-list-group-item class="flex-column align-items-start mb-2" :class="{'sub-document': doc._props.isSubDocument}">
 
     <!-- Info modal-->
     <DocInfoModal :show="showInfo" :doc="doc" @close="showInfo = false"></DocInfoModal>
@@ -40,9 +40,11 @@
         </div>
 
         <div v-if="doc._source.pages || doc._source.author" class="path-row text-muted">
-          <span v-if="doc._source.pages">{{ doc._source.pages }} {{ doc._source.pages > 1 ? $t("pages") : $t("page") }}</span>
+          <span v-if="doc._source.pages">{{ doc._source.pages }} {{
+              doc._source.pages > 1 ? $t("pages") : $t("page")
+            }}</span>
           <span v-if="doc._source.author && doc._source.pages" class="mx-1">-</span>
-          <span v-if="doc._source.author">{{doc._source.author}}</span>
+          <span v-if="doc._source.author">{{ doc._source.author }}</span>
         </div>
       </div>
     </div>
@@ -89,6 +91,14 @@ export default {
 </script>
 
 <style scoped>
+.sub-document {
+  background: #AB47BC1F !important;
+}
+
+.theme-black .sub-document {
+  background: #37474F !important;
+}
+
 .list-group {
   margin-top: 1em;
 }

src/io/store.c

@@ -4,6 +4,7 @@
 store_t *store_create(const char *path, size_t chunk_size) {
     store_t *store = malloc(sizeof(struct store_t));
     mkdir(path, S_IWUSR | S_IRUSR | S_IXUSR);
+    strcpy(store->path, path);
 
 #if (SIST_FAKE_STORE != 1)
     store->chunk_size = chunk_size;
@@ -78,27 +79,57 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
     int put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
     ScanCtx.stat_tn_size += buf_len;
 
+    int db_full = FALSE;
+    int should_abort_transaction = FALSE;
+
     if (put_ret == MDB_MAP_FULL) {
+        db_full = TRUE;
+        should_abort_transaction = TRUE;
+    } else {
+        int commit_ret = mdb_txn_commit(txn);
+
+        if (commit_ret == MDB_MAP_FULL) {
+            db_full = TRUE;
+        }
+    }
+
+    if (db_full) {
+        LOG_INFOF("store.c", "Updating mdb mapsize to %lu bytes", store->size)
+
+        if (should_abort_transaction) {
             mdb_txn_abort(txn);
+        }
+
         pthread_rwlock_unlock(&store->lock);
 
         // Cannot resize when there is a opened transaction.
         //  Resize take effect on the next commit.
         pthread_rwlock_wrlock(&store->lock);
         store->size += store->chunk_size;
-        mdb_env_set_mapsize(store->env, store->size);
-        mdb_txn_begin(store->env, NULL, 0, &txn);
-        put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
-
-        LOG_INFOF("store.c", "Updated mdb mapsize to %lu bytes", store->size)
-    }
-
-    mdb_txn_commit(txn);
-    pthread_rwlock_unlock(&store->lock);
-
-    if (put_ret != 0) {
+        int resize_ret = mdb_env_set_mapsize(store->env, store->size);
+        if (resize_ret != 0) {
             LOG_ERROR("store.c", mdb_strerror(put_ret))
         }
+        mdb_txn_begin(store->env, NULL, 0, &txn);
+        int put_ret_retry = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
+
+        if (put_ret_retry != 0) {
+            LOG_ERROR("store.c", mdb_strerror(put_ret))
+        }
+
+        int ret = mdb_txn_commit(txn);
+        if (ret != 0) {
+            LOG_FATALF("store.c", "FIXME: Could not commit to store %s: %s (%d), %d, %d %d",
+                       store->path, mdb_strerror(ret), ret,
+                       put_ret, put_ret_retry);
+        }
+        LOG_INFOF("store.c", "Updated mdb mapsize to %lu bytes", store->size)
+    } else if (put_ret != 0) {
+        LOG_ERROR("store.c", mdb_strerror(put_ret))
+    }
+
+    pthread_rwlock_unlock(&store->lock);
+
 #endif
 }
 

src/io/store.h

@@ -6,12 +6,12 @@
 
 #include <glib.h>
 
-#define STORE_SIZE_TN 1024 * 1024 * 5
-#define STORE_SIZE_TAG 1024 * 16
+#define STORE_SIZE_TN (1024 * 1024 * 5)
+#define STORE_SIZE_TAG (1024 * 1024)
 #define STORE_SIZE_META STORE_SIZE_TAG
 
 typedef struct store_t {
-    char *path;
+    char path[PATH_MAX];
     char *tmp_path;
     MDB_dbi dbi;
     MDB_env *env;

src/io/walk.c

@@ -39,7 +39,7 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info,
 }
 
 int sub_strings[30];
-#define EXCLUDED(str) (pcre_exec(ScanCtx.exclude, ScanCtx.exclude_extra, filepath, strlen(filepath), 0, 0, sub_strings, sizeof(sub_strings)) >= 0)
+#define EXCLUDED(str) (pcre_exec(ScanCtx.exclude, ScanCtx.exclude_extra, str, strlen(str), 0, 0, sub_strings, sizeof(sub_strings)) >= 0)
 
 int handle_entry(const char *filepath, const struct stat *info, int typeflag, struct FTW *ftw) {
 

src/parsing/parse.c

@@ -179,7 +179,7 @@ void parse(void *arg) {
                     IS_ARC(doc->mime) ||
                     (IS_ARC_FILTER(doc->mime) && should_parse_filtered_file(doc->filepath, doc->ext))
             )) {
-        parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc);
+        parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc, ScanCtx.exclude, ScanCtx.exclude_extra);
     } else if ((ScanCtx.ooxml_ctx.content_size > 0 || ScanCtx.media_ctx.tn_size > 0) && IS_DOC(doc->mime)) {
         parse_ooxml(&ScanCtx.ooxml_ctx, &job->vfile, doc);
     } else if (is_cbr(&ScanCtx.comic_ctx, doc->mime) || is_cbz(&ScanCtx.comic_ctx, doc->mime)) {
@@ -189,6 +189,8 @@ void parse(void *arg) {
     } else if (doc->mime == MIME_SIST2_SIDECAR) {
         parse_sidecar(&job->vfile, doc);
         CLOSE_FILE(job->vfile)
+        free(doc->filepath);
+        free(doc);
         return;
     } else if (is_msdoc(&ScanCtx.msdoc_ctx, doc->mime)) {
         parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, doc);

src/parsing/sidecar.c

@@ -27,7 +27,10 @@ void parse_sidecar(vfile_t *vfile, document_t *doc) {
     MD5((unsigned char *) vfile->filepath + ScanCtx.index.desc.root_len, doc->ext - 1 - ScanCtx.index.desc.root_len,
         path_md5);
 
-    store_write(ScanCtx.index.meta_store, (char *) path_md5, sizeof(path_md5), json_str, strlen(json_str) + 1);
+    char path_md5_str[MD5_STR_LENGTH];
+    buf2hex(path_md5, MD5_DIGEST_LENGTH, path_md5_str);
+
+    store_write(ScanCtx.index.meta_store, path_md5_str, MD5_STR_LENGTH, json_str, strlen(json_str) + 1);
 
     cJSON_Delete(json);
     free(json_str);

third-party/libscan

@@ -1 +1 @@
-Subproject commit da172823745b67662846cf1970a47ebcea8fe50e
+Subproject commit 3787475ecba7453a2a97ab470103606c2cecabb2