diff --git a/src/io/store.c b/src/io/store.c index 47bc750..c5054bc 100644 --- a/src/io/store.c +++ b/src/io/store.c @@ -4,6 +4,7 @@ store_t *store_create(const char *path, size_t chunk_size) { store_t *store = malloc(sizeof(struct store_t)); mkdir(path, S_IWUSR | S_IRUSR | S_IXUSR); + strcpy(store->path, path); #if (SIST_FAKE_STORE != 1) store->chunk_size = chunk_size; @@ -78,27 +79,57 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu int put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0); ScanCtx.stat_tn_size += buf_len; + int db_full = FALSE; + int should_abort_transaction = FALSE; + if (put_ret == MDB_MAP_FULL) { - mdb_txn_abort(txn); + db_full = TRUE; + should_abort_transaction = TRUE; + } else { + int commit_ret = mdb_txn_commit(txn); + + if (commit_ret == MDB_MAP_FULL) { + db_full = TRUE; + } + } + + if (db_full) { + LOG_INFOF("store.c", "Updating mdb mapsize to %lu bytes", store->size) + + if (should_abort_transaction) { + mdb_txn_abort(txn); + } + pthread_rwlock_unlock(&store->lock); // Cannot resize when there is a opened transaction. // Resize take effect on the next commit. pthread_rwlock_wrlock(&store->lock); store->size += store->chunk_size; - mdb_env_set_mapsize(store->env, store->size); + int resize_ret = mdb_env_set_mapsize(store->env, store->size); + if (resize_ret != 0) { + LOG_ERROR("store.c", mdb_strerror(put_ret)) + } mdb_txn_begin(store->env, NULL, 0, &txn); - put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0); + int put_ret_retry = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0); + if (put_ret_retry != 0) { + LOG_ERROR("store.c", mdb_strerror(put_ret)) + } + + int ret = mdb_txn_commit(txn); + if (ret != 0) { + LOG_FATALF("store.c", "FIXME: Could not commit to store %s: %s (%d), %d, %d %d", + store->path, mdb_strerror(ret), ret, + put_ret, put_ret_retry); + } LOG_INFOF("store.c", "Updated mdb mapsize to %lu bytes", store->size) - } - - mdb_txn_commit(txn); - pthread_rwlock_unlock(&store->lock); - - if (put_ret != 0) { + } else if (put_ret != 0) { LOG_ERROR("store.c", mdb_strerror(put_ret)) } + + pthread_rwlock_unlock(&store->lock); + #endif } diff --git a/src/io/store.h b/src/io/store.h index 4d6632c..a1f7217 100644 --- a/src/io/store.h +++ b/src/io/store.h @@ -6,12 +6,12 @@ #include -#define STORE_SIZE_TN 1024 * 1024 * 5 -#define STORE_SIZE_TAG 1024 * 16 +#define STORE_SIZE_TN (1024 * 1024 * 5) +#define STORE_SIZE_TAG (1024 * 1024) #define STORE_SIZE_META STORE_SIZE_TAG typedef struct store_t { - char *path; + char path[PATH_MAX]; char *tmp_path; MDB_dbi dbi; MDB_env *env; diff --git a/src/io/walk.c b/src/io/walk.c index 2aec195..96a59da 100644 --- a/src/io/walk.c +++ b/src/io/walk.c @@ -39,7 +39,7 @@ parse_job_t *create_fs_parse_job(const char *filepath, const struct stat *info, } int sub_strings[30]; -#define EXCLUDED(str) (pcre_exec(ScanCtx.exclude, ScanCtx.exclude_extra, filepath, strlen(filepath), 0, 0, sub_strings, sizeof(sub_strings)) >= 0) +#define EXCLUDED(str) (pcre_exec(ScanCtx.exclude, ScanCtx.exclude_extra, str, strlen(str), 0, 0, sub_strings, sizeof(sub_strings)) >= 0) int handle_entry(const char *filepath, const struct stat *info, int typeflag, struct FTW *ftw) { diff --git a/src/parsing/parse.c b/src/parsing/parse.c index 844ecaf..f266f00 100644 --- a/src/parsing/parse.c +++ b/src/parsing/parse.c @@ -179,7 +179,7 @@ void parse(void *arg) { IS_ARC(doc->mime) || (IS_ARC_FILTER(doc->mime) && should_parse_filtered_file(doc->filepath, doc->ext)) )) { - parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc); + parse_archive(&ScanCtx.arc_ctx, &job->vfile, doc, ScanCtx.exclude, ScanCtx.exclude_extra); } else if ((ScanCtx.ooxml_ctx.content_size > 0 || ScanCtx.media_ctx.tn_size > 0) && IS_DOC(doc->mime)) { parse_ooxml(&ScanCtx.ooxml_ctx, &job->vfile, doc); } else if (is_cbr(&ScanCtx.comic_ctx, doc->mime) || is_cbz(&ScanCtx.comic_ctx, doc->mime)) { @@ -189,6 +189,8 @@ void parse(void *arg) { } else if (doc->mime == MIME_SIST2_SIDECAR) { parse_sidecar(&job->vfile, doc); CLOSE_FILE(job->vfile) + free(doc->filepath); + free(doc); return; } else if (is_msdoc(&ScanCtx.msdoc_ctx, doc->mime)) { parse_msdoc(&ScanCtx.msdoc_ctx, &job->vfile, doc); diff --git a/src/parsing/sidecar.c b/src/parsing/sidecar.c index 4043b5b..4a713b7 100644 --- a/src/parsing/sidecar.c +++ b/src/parsing/sidecar.c @@ -27,7 +27,10 @@ void parse_sidecar(vfile_t *vfile, document_t *doc) { MD5((unsigned char *) vfile->filepath + ScanCtx.index.desc.root_len, doc->ext - 1 - ScanCtx.index.desc.root_len, path_md5); - store_write(ScanCtx.index.meta_store, (char *) path_md5, sizeof(path_md5), json_str, strlen(json_str) + 1); + char path_md5_str[MD5_STR_LENGTH]; + buf2hex(path_md5, MD5_DIGEST_LENGTH, path_md5_str); + + store_write(ScanCtx.index.meta_store, path_md5_str, MD5_STR_LENGTH, json_str, strlen(json_str) + 1); cJSON_Delete(json); free(json_str); diff --git a/third-party/libscan b/third-party/libscan index da17282..3787475 160000 --- a/third-party/libscan +++ b/third-party/libscan @@ -1 +1 @@ -Subproject commit da172823745b67662846cf1970a47ebcea8fe50e +Subproject commit 3787475ecba7453a2a97ab470103606c2cecabb2