Update CMakeLists.txt

update

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "third-party/utf8.h"]
 	path = third-party/utf8.h
 	url = https://github.com/sheredom/utf8.h
+[submodule "third-party/antiword"]
+	path = third-party/antiword
+	url = https://github.com/simon987/antiword

CMakeLists.txt

@@ -3,7 +3,13 @@ cmake_minimum_required(VERSION 3.15)
 project(scan)
 set(CMAKE_C_STANDARD 11)
 
-option(BUILD_TESTS "Build tests" off)
+option(BUILD_TESTS "Build tests" on)
+
+add_subdirectory(third-party/antiword)
+add_compile_definitions(
+        antiword
+        NDEBUG
+)
 
 add_library(
         scan
@@ -18,13 +24,13 @@ add_library(
         libscan/ooxml/ooxml.c libscan/ooxml/ooxml.h
         libscan/media/media.c libscan/media/media.h
         libscan/font/font.c libscan/font/font.h
+        libscan/msdoc/msdoc.c libscan/msdoc/msdoc.h
 
         third-party/utf8.h
         libscan/mobi/scan_mobi.c libscan/mobi/scan_mobi.h libscan/raw/raw.c libscan/raw/raw.h)
 set_target_properties(scan PROPERTIES LINKER_LANGUAGE C)
 
 set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib .so)
-target_link_directories(scan PUBLIC /usr/share/vcpkg/installed/x64-linux/lib/)
 
 find_package(LibArchive REQUIRED)
 find_package(BZip2 REQUIRED)
@@ -35,8 +41,6 @@ find_package(Tesseract CONFIG REQUIRED)
 find_package(OpenJPEG CONFIG REQUIRED)
 find_package(JPEG REQUIRED)
 find_package(LibXml2 REQUIRED)
-find_package(FFMPEG REQUIRED)
-#find_package(OpenSSL REQUIRED)
 find_package(LibLZMA REQUIRED)
 find_package(ZLIB REQUIRED)
 
@@ -50,7 +54,8 @@ find_library(RAW_LIB NAMES libraw.a)
 find_library(MUPDF_LIB NAMES liblibmupdf.a)
 find_library(CMS_LIB NAMES lcms)
 find_library(JAS_LIB NAMES jasper)
-find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/10/)
+find_library(GUMBO_LIB NAMES gumbo)
+find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/)
 
 
 target_compile_options(
@@ -82,33 +87,61 @@ ExternalProject_Add(
 SET(MOBI_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libmobi/src/libmobi/src/.libs/)
 SET(MOBI_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libmobi/src/libmobi/src/)
 
+if (SIST_DEBUG)
+    SET(FFMPEG_DEBUG "--enable-debug=3" "--disable-optimizations")
+else()
+    SET(FFMPEG_DEBUG "")
+endif()
+
 ExternalProject_Add(
         ffmpeg
         GIT_REPOSITORY https://git.ffmpeg.org/ffmpeg.git
-        GIT_TAG "master"
+        GIT_TAG "n4.4"
 
         UPDATE_COMMAND ""
         PATCH_COMMAND ""
         TEST_COMMAND ""
         CONFIGURE_COMMAND ./configure --disable-shared --enable-static --disable-ffmpeg --disable-ffplay
         --disable-ffprobe --disable-doc --disable-manpages --disable-postproc --disable-avfilter --disable-alsa
-        --disable-lzma --disable-xlib --disable-debug --disable-vdpau --disable-vaapi --disable-sdl2
-        --disable-network --extra-cflags=-fPIC
+        --disable-lzma --disable-xlib --disable-vdpau --disable-vaapi --disable-sdl2
+        --disable-network  ${FFMPEG_DEBUG}
         INSTALL_COMMAND ""
 
         PREFIX "third-party/ext_ffmpeg"
         SOURCE_DIR "third-party/ext_ffmpeg/src/ffmpeg"
         BINARY_DIR "third-party/ext_ffmpeg/src/ffmpeg"
 
-        BUILD_COMMAND ${MAKE_EXE} -j 8 --silent
+        BUILD_COMMAND ${MAKE_EXE} -j33 --silent
 )
 
 SET(FFMPEG_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_ffmpeg/src/ffmpeg)
+SET(FFMPEG_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_ffmpeg/src/ffmpeg)
+
+#ExternalProject_Add(
+#        libwpd
+#        URL http://prdownloads.sourceforge.net/libwpd/libwpd-0.9.9.tar.gz
+#
+#        UPDATE_COMMAND ""
+#        PATCH_COMMAND ""
+#        TEST_COMMAND ""
+#        CONFIGURE_COMMAND ./configure --without-docs --enable-static --disable-shared
+#        INSTALL_COMMAND ""
+#
+#        PREFIX "third-party/ext_libwpd"
+#        SOURCE_DIR "third-party/ext_libwpd/src/libwpd"
+#        BINARY_DIR "third-party/ext_libwpd/src/libwpd"
+#
+#        BUILD_COMMAND ${MAKE_EXE} -j33
+#)
+#SET(WPD_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwpd/src/lib/.libs/)
+#SET(WPD_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwpd/inc/)
 
 add_dependencies(
         scan
         libmobi
         ffmpeg
+        antiword
+#        libwpd
 )
 
 target_link_libraries(
@@ -127,10 +160,11 @@ target_link_libraries(
 
         ${MOBI_LIB_DIR}/libmobi.a
 
+#        ${WPD_LIB_DIR}/libwpd-0.9.a
+
         ${FREETYPE_LIB}
         ${HARFBUZZ_LIB}
         ${JBIG2DEC_LIB}
-        #        OpenSSL::SSL OpenSSL::Crypto
 
         stdc++
 
@@ -158,7 +192,9 @@ target_link_libraries(
         ${GOMP_LIB}
         ${CMS_LIB}
         ${JAS_LIB}
+        ${GUMBO_LIB}
         dl
+        antiword
 )
 
 target_include_directories(
@@ -169,6 +205,7 @@ target_include_directories(
         ${LIBXML2_INCLUDE_DIR}
         ${FFMPEG_INCLUDE_DIR}
         ${MOBI_INCLUDE_DIR}
+#        ${WPD_INCLUDE_DIR}
 )
 
 if (BUILD_TESTS)
@@ -181,4 +218,8 @@ if (BUILD_TESTS)
     add_executable(scan_a_test test/main.cpp test/test_util.cpp test/test_util.h)
     target_compile_options(scan_a_test PRIVATE -g -fsanitize=address -fno-omit-frame-pointer)
     target_link_libraries(scan_a_test PRIVATE GTest::gtest GTest::gtest_main -fsanitize=address scan)
+
+    add_executable(scan_test test/main.cpp test/test_util.cpp test/test_util.h)
+    target_compile_options(scan_test PRIVATE -g -fno-omit-frame-pointer)
+    target_link_libraries(scan_test PRIVATE GTest::gtest GTest::gtest_main scan)
 endif()

README.md

@@ -1,8 +1,7 @@
-*(wip)*
+Please use [sist2](https://github.com/simon987/sist2) tracker for issues
 
+
+### Run fuzz tests:
 ```bash
-vcpkg install libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest
-
-cmake -DCMAKE_TOOLCHAIN_FILE=/usr/share/vcpkg/scripts/buildsystems/vcpkg.cmake .
-make -j 4
+./scan_a_test --gtest_filter=*Fuzz* --gtest_repeat=100
 ```

libscan/arc/arc.c

@@ -5,8 +5,6 @@
 #include <string.h>
 #include <fcntl.h>
 
-#define MAX_SUBFILE_SIZE (long)(10000000000)
-
 
 int should_parse_filtered_file(const char *filepath, int ext) {
     char tmp[PATH_MAX * 2];
@@ -49,19 +47,25 @@ int arc_read(struct vfile *f, void *buf, size_t size) {
     return read;
 }
 
-int arc_open(vfile_t *f, struct archive **a, arc_data_t *arc_data, int allow_recurse) {
+int arc_open(scan_arc_ctx_t *ctx, vfile_t *f, struct archive **a, arc_data_t *arc_data, int allow_recurse) {
     arc_data->f = f;
 
     if (f->is_fs_file) {
         *a = archive_read_new();
         archive_read_support_filter_all(*a);
         archive_read_support_format_all(*a);
+        if (ctx->passphrase[0] != 0) {
+            archive_read_add_passphrase(*a, ctx->passphrase);
+        }
 
        return archive_read_open_filename(*a, f->filepath, ARC_BUF_SIZE);
     } else if (allow_recurse) {
         *a = archive_read_new();
         archive_read_support_filter_all(*a);
         archive_read_support_format_all(*a);
+        if (ctx->passphrase[0] != 0) {
+            archive_read_add_passphrase(*a, ctx->passphrase);
+        }
 
         return archive_read_open(
                 *a, arc_data,
@@ -82,7 +86,7 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc) {
     arc_data_t arc_data;
     arc_data.f = f;
 
-    int ret = arc_open(f, &a, &arc_data, ctx->mode == ARC_MODE_RECURSE);
+    int ret = arc_open(ctx, f, &a, &arc_data, ctx->mode == ARC_MODE_RECURSE);
     if (ret == ARC_SKIPPED) {
         return SCAN_OK;
     }
@@ -110,7 +114,7 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc) {
         meta_line_t *meta_list = malloc(sizeof(meta_line_t) + buf.cur);
         meta_list->key = MetaContent;
         strcpy(meta_list->str_val, buf.buf);
-        APPEND_META(doc, meta_list);
+        APPEND_META(doc, meta_list)
         dyn_buffer_destroy(&buf);
 
     } else {
@@ -125,7 +129,7 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc) {
         sub_job->vfile.is_fs_file = FALSE;
         sub_job->vfile.log = ctx->log;
         sub_job->vfile.logf = ctx->logf;
-        memcpy(sub_job->parent, doc->uuid, sizeof(uuid_t));
+        memcpy(sub_job->parent, doc->path_md5, MD5_DIGEST_LENGTH);
 
         while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
             sub_job->vfile.info = *archive_entry_stat(entry);

libscan/arc/arc.h

@@ -20,6 +20,7 @@ typedef struct {
     log_callback_t log;
     logf_callback_t logf;
     store_callback_t store;
+    char passphrase[4096];
 } scan_arc_ctx_t;
 
 #define ARC_BUF_SIZE 8192
@@ -56,7 +57,7 @@ static int vfile_close_callback(struct archive *a, void *user_data) {
     return ARCHIVE_OK;
 }
 
-int arc_open(vfile_t *f, struct archive **a, arc_data_t *arc_data, int allow_recurse);
+int arc_open(scan_arc_ctx_t *ctx, vfile_t *f, struct archive **a, arc_data_t *arc_data, int allow_recurse);
 
 int should_parse_filtered_file(const char *filepath, int ext);
 

libscan/comic/comic.c

@@ -5,13 +5,18 @@
 #include <stdlib.h>
 #include <archive.h>
 
+static scan_arc_ctx_t arc_ctx = (scan_arc_ctx_t) {.passphrase = {0,}};
 
 void parse_comic(scan_comic_ctx_t *ctx, vfile_t *f, document_t *doc) {
     struct archive *a = NULL;
     struct archive_entry *entry = NULL;
     arc_data_t arc_data;
 
-    int ret = arc_open(f, &a, &arc_data, TRUE);
+    if (ctx->tn_size <= 0) {
+        return;
+    }
+
+    int ret = arc_open(&arc_ctx, f, &a, &arc_data, TRUE);
     if (ret != ARCHIVE_OK) {
         CTX_LOG_ERRORF(f->filepath, "(cbr.c) [%d] %s", ret, archive_error_string(a))
         archive_read_free(a);
@@ -21,17 +26,17 @@ void parse_comic(scan_comic_ctx_t *ctx, vfile_t *f, document_t *doc) {
     while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
         struct stat info = *archive_entry_stat(entry);
         if (S_ISREG(info.st_mode)) {
-            const char* utf8_name = archive_entry_pathname_utf8(entry);
-            const char* file_path = utf8_name == NULL ? archive_entry_pathname(entry) : utf8_name;
+            const char *utf8_name = archive_entry_pathname_utf8(entry);
+            const char *file_path = utf8_name == NULL ? archive_entry_pathname(entry) : utf8_name;
 
             char *p = strrchr(file_path, '.');
-            if (p != NULL && strcmp(p, ".png") == 0 || strcmp(p, ".jpg") == 0 || strcmp(p, ".jpeg") == 0) {
+            if (p != NULL && (strcmp(p, ".png") == 0 || strcmp(p, ".jpg") == 0 || strcmp(p, ".jpeg") == 0)) {
                 size_t entry_size = archive_entry_size(entry);
-                void* buf = malloc(entry_size);
-                int read = archive_read_data(a, buf, entry_size);
+                void *buf = malloc(entry_size);
+                size_t read = archive_read_data(a, buf, entry_size);
 
                 if (read != entry_size) {
-                    const char* err_str = archive_error_string(a);
+                    const char *err_str = archive_error_string(a);
                     if (err_str) {
                         CTX_LOG_ERRORF("comic.c", "Error while reading entry: %s", err_str)
                     }
@@ -39,7 +44,7 @@ void parse_comic(scan_comic_ctx_t *ctx, vfile_t *f, document_t *doc) {
                     break;
                 }
 
-                ret = store_image_thumbnail((scan_media_ctx_t*)ctx, buf, entry_size, doc, file_path);
+                ret = store_image_thumbnail((scan_media_ctx_t *) ctx, buf, entry_size, doc, file_path);
                 free(buf);
 
                 if (ret == TRUE) {

libscan/ebook/ebook.c

@@ -4,6 +4,7 @@
 #include <tesseract/capi.h>
 
 #include "../media/media.h"
+#include "../arc/arc.h"
 
 #define MIN_OCR_SIZE 350
 #define MIN_OCR_LEN 10
@@ -15,13 +16,13 @@ __thread scan_ebook_ctx_t thread_ctx;
 pthread_mutex_t Mutex;
 
 static void my_fz_lock(UNUSED(void *user), int lock) {
-    if (lock == FZ_LOCK_JPX) {
+    if (lock == FZ_LOCK_FREETYPE) {
         pthread_mutex_lock(&Mutex);
     }
 }
 
 static void my_fz_unlock(UNUSED(void *user), int lock) {
-    if (lock == FZ_LOCK_JPX) {
+    if (lock == FZ_LOCK_FREETYPE) {
         pthread_mutex_unlock(&Mutex);
     }
 }
@@ -38,19 +39,17 @@ int pixmap_is_blank(const fz_pixmap *pixmap) {
     return TRUE;
 }
 
-fz_pixmap *load_pixmap(scan_ebook_ctx_t *ctx, int page, fz_context *fzctx, fz_document *fzdoc, document_t *doc, fz_page **cover) {
+fz_pixmap *
+load_pixmap(scan_ebook_ctx_t *ctx, int page, fz_context *fzctx, fz_document *fzdoc, document_t *doc, fz_page **cover) {
 
     int err = 0;
 
     fz_var(cover);
     fz_var(err);
-    fz_try(fzctx)
-        *cover = fz_load_page(fzctx, fzdoc, page);
-    fz_catch(fzctx)
-        err = 1;
+    fz_try(fzctx)*cover = fz_load_page(fzctx, fzdoc, page);
+    fz_catch(fzctx)err = 1;
 
     if (err != 0) {
-        fz_drop_page(fzctx, *cover);
         CTX_LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, fzctx->error.message)
         return NULL;
     }
@@ -76,14 +75,11 @@ fz_pixmap *load_pixmap(scan_ebook_ctx_t *ctx, int page, fz_context *fzctx, fz_do
 
     fz_var(err);
     fz_try(fzctx) {
-        fz_run_page(fzctx, *cover, dev, fz_identity, NULL);
-    }
-    fz_always(fzctx) {
-        fz_close_device(fzctx, dev);
-        fz_drop_device(fzctx, dev);
-    }
-    fz_catch(fzctx)
-        err = fzctx->error.errcode;
+                fz_run_page(fzctx, *cover, dev, fz_identity, NULL);
+            } fz_always(fzctx) {
+            fz_close_device(fzctx, dev);
+            fz_drop_device(fzctx, dev);
+        } fz_catch(fzctx)err = fzctx->error.errcode;
 
     if (err != 0) {
         CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message)
@@ -132,10 +128,14 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
     int dst_buf_len = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, pixmap->w, pixmap->h, 1);
     uint8_t *dst_buf = (uint8_t *) av_malloc(dst_buf_len);
 
-    av_image_fill_arrays(scaled_frame->data, scaled_frame->linesize, dst_buf, AV_PIX_FMT_YUV420P, pixmap->w, pixmap->h, 1);
+    av_image_fill_arrays(scaled_frame->data, scaled_frame->linesize, dst_buf, AV_PIX_FMT_YUV420P, pixmap->w, pixmap->h,
+                         1);
 
-    const uint8_t *in_data[1] = {pixmap->samples};
-    int in_line_size[1] = {pixmap->stride};
+    unsigned char *samples = calloc(1, 1024 * 1024 * 1024);
+    memcpy(samples, pixmap->samples, pixmap->stride * pixmap->h);
+
+    const uint8_t *in_data[1] = {samples,};
+    int in_line_size[1] = {(int) pixmap->stride};
 
     sws_scale(sws_ctx,
               in_data, in_line_size,
@@ -150,7 +150,7 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
     sws_freeContext(sws_ctx);
 
     // YUV420p -> JPEG
-    AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(pixmap->w, pixmap->h, 1.0f);
+    AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(pixmap->w, pixmap->h, ctx->tn_qscale);
     avcodec_send_frame(jpeg_encoder, scaled_frame);
 
     AVPacket jpeg_packet;
@@ -158,8 +158,9 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
     avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
 
     APPEND_TN_META(doc, pixmap->w, pixmap->h)
-    ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) jpeg_packet.data, jpeg_packet.size);
 
+    free(samples);
     av_packet_unref(&jpeg_packet);
     av_free(*scaled_frame->data);
     av_frame_free(&scaled_frame);
@@ -175,24 +176,23 @@ void fz_err_callback(void *user, const char *message) {
     document_t *doc = (document_t *) user;
 
     const scan_ebook_ctx_t *ctx = &thread_ctx;
-    CTX_LOG_WARNINGF(doc->filepath, "FZ: %s", message);
+    CTX_LOG_WARNINGF(doc->filepath, "FZ: %s", message)
 }
 
 void fz_warn_callback(void *user, const char *message) {
     document_t *doc = (document_t *) user;
 
     const scan_ebook_ctx_t *ctx = &thread_ctx;
-    CTX_LOG_DEBUGF(doc->filepath, "FZ: %s", message);
+    CTX_LOG_DEBUGF(doc->filepath, "FZ: %s", message)
 }
 
 static void init_fzctx(fz_context *fzctx, document_t *doc) {
-    fz_disable_icc(fzctx);
     fz_register_document_handlers(fzctx);
 
-    static int mu_is_initialized = 0;
+    static int mu_is_initialized = FALSE;
     if (!mu_is_initialized) {
         pthread_mutex_init(&Mutex, NULL);
-        mu_is_initialized = 1;
+        mu_is_initialized = TRUE;
     }
 
     fzctx->warn.print_user = doc;
@@ -225,7 +225,7 @@ static int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
     return 0;
 }
 
-#define IS_VALID_BPP(d) (d==1 || d==2 || d==4 || d==8 || d==16 || d==24 || d==32)
+#define IS_VALID_BPP(d) ((d)==1 || (d)==2 || (d)==4 || (d)==8 || (d)==16 || (d)==24 || (d)==32)
 
 void fill_image(fz_context *fzctx, UNUSED(fz_device *dev),
                 fz_image *img, UNUSED(fz_matrix ctm), UNUSED(float alpha),
@@ -257,9 +257,10 @@ void fill_image(fz_context *fzctx, UNUSED(fz_device *dev),
     }
 }
 
-void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mime_str, document_t *doc) {
+void
+parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mime_str, document_t *doc, int tn_only) {
 
-    fz_context *fzctx = fz_new_context(NULL, NULL, FZ_STORE_UNLIMITED);
+    fz_context *fzctx = fz_new_context(NULL, NULL, FZ_STORE_DEFAULT);
     thread_ctx = *ctx;
 
     init_fzctx(fzctx, doc);
@@ -272,13 +273,10 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha
     fz_var(stream);
     fz_var(err);
 
-    fz_try(fzctx)
-    {
-        stream = fz_open_memory(fzctx, buf, buf_len);
-        fzdoc = fz_open_document_with_stream(fzctx, mime_str, stream);
-    }
-    fz_catch(fzctx)
-        err = fzctx->error.errcode;
+    fz_try(fzctx) {
+                stream = fz_open_memory(fzctx, buf, buf_len);
+                fzdoc = fz_open_document_with_stream(fzctx, mime_str, stream);
+            } fz_catch(fzctx)err = fzctx->error.errcode;
 
     if (err != 0) {
         fz_drop_stream(fzctx, stream);
@@ -287,38 +285,10 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha
         return;
     }
 
-    char title[8192] = {'\0',};
-    fz_try(fzctx)
-        fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_TITLE, title, sizeof(title));
-    fz_catch(fzctx)
-        ;
-
-    if (strlen(title) > 0) {
-        meta_line_t *meta_title = malloc(sizeof(meta_line_t) + strlen(title));
-        meta_title->key = MetaTitle;
-        strcpy(meta_title->str_val, title);
-        APPEND_META(doc, meta_title)
-    }
-
-    char author[4096] = {'\0',};
-    fz_try(fzctx)
-        fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_AUTHOR, author, sizeof(author));
-    fz_catch(fzctx)
-        ;
-
-    if (strlen(author) > 0) {
-        meta_line_t *meta_author = malloc(sizeof(meta_line_t) + strlen(author));
-        meta_author->key = MetaAuthor;
-        strcpy(meta_author->str_val, author);
-        APPEND_META(doc, meta_author)
-    }
-
     int page_count = -1;
     fz_var(err);
-    fz_try(fzctx)
-        page_count = fz_count_pages(fzctx, fzdoc);
-    fz_catch(fzctx)
-        err = fzctx->error.errcode;
+    fz_try(fzctx)page_count = fz_count_pages(fzctx, fzdoc);
+    fz_catch(fzctx)err = fzctx->error.errcode;
 
     if (err) {
         CTX_LOG_WARNINGF(doc->filepath, "fz_count_pages() returned error code [%d] %s", err, fzctx->error.message)
@@ -328,7 +298,7 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha
         return;
     }
 
-    APPEND_INT_META(doc, MetaPages, page_count)
+    APPEND_LONG_META(doc, MetaPages, page_count)
 
     if (ctx->tn_size > 0) {
         if (render_cover(ctx, fzctx, doc, fzdoc) == FALSE) {
@@ -339,6 +309,29 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha
         }
     }
 
+    if (tn_only) {
+        fz_drop_stream(fzctx, stream);
+        fz_drop_document(fzctx, fzdoc);
+        fz_drop_context(fzctx);
+        return;
+    }
+
+    char title[8192] = {'\0',};
+    fz_try(fzctx)fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_TITLE, title, sizeof(title));
+    fz_catch(fzctx);
+
+    if (strlen(title) > 0) {
+        APPEND_UTF8_META(doc, MetaTitle, title)
+    }
+
+    char author[4096] = {'\0',};
+    fz_try(fzctx)fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_AUTHOR, author, sizeof(author));
+    fz_catch(fzctx);
+
+    if (strlen(author) > 0) {
+        APPEND_UTF8_META(doc, MetaAuthor, author)
+    }
+
 
     if (ctx->content_size > 0) {
         fz_stext_options opts = {0};
@@ -347,10 +340,8 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha
         for (int current_page = 0; current_page < page_count; current_page++) {
             fz_page *page = NULL;
             fz_var(err);
-            fz_try(fzctx)
-                page = fz_load_page(fzctx, fzdoc, current_page);
-            fz_catch(fzctx)
-                err = fzctx->error.errcode;
+            fz_try(fzctx)page = fz_load_page(fzctx, fzdoc, current_page);
+            fz_catch(fzctx)err = fzctx->error.errcode;
             if (err != 0) {
                 CTX_LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, fzctx->error.message)
                 text_buffer_destroy(&thread_buffer);
@@ -374,15 +365,11 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha
             }
 
             fz_var(err);
-            fz_try(fzctx)
-                fz_run_page(fzctx, page, dev, fz_identity, NULL);
-            fz_always(fzctx)
-            {
-                fz_close_device(fzctx, dev);
-                fz_drop_device(fzctx, dev);
-            }
-            fz_catch(fzctx)
-                err = fzctx->error.errcode;
+            fz_try(fzctx)fz_run_page(fzctx, page, dev, fz_identity, NULL);
+            fz_always(fzctx) {
+                    fz_close_device(fzctx, dev);
+                    fz_drop_device(fzctx, dev);
+                } fz_catch(fzctx)err = fzctx->error.errcode;
 
             if (err != 0) {
                 CTX_LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, fzctx->error.message)
@@ -425,7 +412,77 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha
     fz_drop_context(fzctx);
 }
 
+static scan_arc_ctx_t arc_ctx = (scan_arc_ctx_t) {.passphrase = {0,}};
+
+void parse_epub_fast(scan_ebook_ctx_t *ctx, vfile_t *f, document_t *doc) {
+    struct archive *a = NULL;
+    struct archive_entry *entry = NULL;
+    arc_data_t arc_data;
+
+    text_buffer_t content_buffer = text_buffer_create(ctx->content_size);
+
+    if (ctx->tn_size <= 0) {
+        return;
+    }
+
+    int ret = arc_open(&arc_ctx, f, &a, &arc_data, TRUE);
+    if (ret != ARCHIVE_OK) {
+        CTX_LOG_ERRORF(f->filepath, "(ebook.c) [%d] %s", ret, archive_error_string(a))
+        archive_read_free(a);
+        return;
+    }
+
+    while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
+        struct stat info = *archive_entry_stat(entry);
+        if (S_ISREG(info.st_mode)) {
+            const char *utf8_name = archive_entry_pathname_utf8(entry);
+            const char *file_path = utf8_name == NULL ? archive_entry_pathname(entry) : utf8_name;
+
+            char *p = strrchr(file_path, '.');
+            if (p != NULL && (strcmp(p, ".html") == 0 || (strcmp(p, ".xhtml") == 0))) {
+                size_t entry_size = archive_entry_size(entry);
+                void *buf = malloc(entry_size + 1);
+                size_t read = archive_read_data(a, buf, entry_size);
+                *(char *) (buf + entry_size) = '\0';
+
+                if (read != entry_size) {
+                    const char *err_str = archive_error_string(a);
+                    if (err_str) {
+                        CTX_LOG_ERRORF("ebook.c", "Error while reading entry: %s", err_str)
+                    }
+                    free(buf);
+                    break;
+                }
+
+                ret = text_buffer_append_markup(&content_buffer, buf);
+                free(buf);
+
+                if (ret == TEXT_BUF_FULL) {
+                    break;
+                }
+            }
+        }
+    }
+
+    text_buffer_terminate_string(&content_buffer);
+
+    meta_line_t *meta_content = malloc(sizeof(meta_line_t) + content_buffer.dyn_buffer.cur);
+    meta_content->key = MetaContent;
+    memcpy(meta_content->str_val, content_buffer.dyn_buffer.buf, content_buffer.dyn_buffer.cur);
+    APPEND_META(doc, meta_content)
+
+    text_buffer_destroy(&content_buffer);
+
+    archive_read_free(a);
+}
+
 void parse_ebook(scan_ebook_ctx_t *ctx, vfile_t *f, const char *mime_str, document_t *doc) {
+
+    if (ctx->fast_epub_parse && is_epub(mime_str)) {
+        parse_epub_fast(ctx, f, doc);
+        return;
+    }
+
     size_t buf_len;
     void *buf = read_all(f, &buf_len);
     if (buf == NULL) {
@@ -433,6 +490,6 @@ void parse_ebook(scan_ebook_ctx_t *ctx, vfile_t *f, const char *mime_str, docume
         return;
     }
 
-    parse_ebook_mem(ctx, buf, buf_len, mime_str, doc);
+    parse_ebook_mem(ctx, buf, buf_len, mime_str, doc, FALSE);
     free(buf);
 }

libscan/ebook/ebook.h

@@ -13,9 +13,18 @@ typedef struct {
     log_callback_t log;
     logf_callback_t logf;
     store_callback_t store;
+    int fast_epub_parse;
+    float tn_qscale;
 } scan_ebook_ctx_t;
 
-void parse_ebook(scan_ebook_ctx_t *ctx, vfile_t *f, const char* mime_str,  document_t *doc);
-void parse_ebook_mem(scan_ebook_ctx_t *ctx, void* buf, size_t buf_len, const char* mime_str,  document_t *doc);
+void parse_ebook(scan_ebook_ctx_t *ctx, vfile_t *f, const char *mime_str, document_t *doc);
+
+void
+parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mime_str, document_t *doc, int tn_only);
+
+__always_inline
+static int is_epub(const char *mime_string) {
+    return strcmp(mime_string, "application/epub+zip") == 0;
+}
 
 #endif

libscan/font/font.c

@@ -144,27 +144,28 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) {
     }
 
     size_t buf_len = 0;
-    void * buf = read_all(f, &buf_len);
+    void *buf = read_all(f, &buf_len);
     if (buf == NULL) {
         CTX_LOG_ERROR(f->filepath, "read_all() failed")
         return;
     }
 
     FT_Face face;
-    FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, buf_len, 0, &face);
+    FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, (int) buf_len, 0, &face);
     if (err != 0) {
-        CTX_LOG_ERRORF(doc->filepath, "(font.c) FT_New_Memory_Face() returned error code [%d] %s", err, FT_Error_String(err))
+        CTX_LOG_ERRORF(doc->filepath, "(font.c) FT_New_Memory_Face() returned error code [%d] %s", err,
+                       FT_Error_String(err))
         free(buf);
         return;
     }
 
-    char font_name[1024];
+    char font_name[4096];
 
-    if (face->style_name == NULL || *(face->style_name) == '?') {
+    if (face->style_name == NULL || (strcmp(face->style_name, "?") == 0)) {
         if (face->family_name == NULL) {
             strcpy(font_name, "(null)");
         } else {
-            strcpy(font_name, face->family_name);
+            strncpy(font_name, face->family_name, sizeof(font_name));
         }
     } else {
         snprintf(font_name, sizeof(font_name), "%s %s", face->family_name, face->style_name);
@@ -186,7 +187,8 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) {
 
     err = FT_Set_Pixel_Sizes(face, 0, pixel);
     if (err != 0) {
-        CTX_LOG_WARNINGF(doc->filepath, "(font.c) FT_Set_Pixel_Sizes() returned error code [%d] %s", err, FT_Error_String(err))
+        CTX_LOG_WARNINGF(doc->filepath, "(font.c) FT_Set_Pixel_Sizes() returned error code [%d] %s", err,
+                         FT_Error_String(err))
         FT_Done_Face(face);
         free(buf);
         return;
@@ -207,7 +209,8 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) {
             c = c >= 'a' && c <= 'z' ? c - 32 : c + 32;
             err = FT_Load_Char(face, c, FT_LOAD_NO_HINTING | FT_LOAD_RENDER);
             if (err != 0) {
-                CTX_LOG_WARNINGF(doc->filepath, "(font.c) FT_Load_Char() returned error code [%d] %s", err, FT_Error_String(err))
+                CTX_LOG_WARNINGF(doc->filepath, "(font.c) FT_Load_Char() returned error code [%d] %s", err,
+                                 FT_Error_String(err))
                 continue;
             }
         }
@@ -229,7 +232,7 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) {
     bmp_format(&bmp_data, dimensions, bitmap);
 
     APPEND_TN_META(doc, dimensions.width, dimensions.height)
-    ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) bmp_data.buf, bmp_data.cur);
+    ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) bmp_data.buf, bmp_data.cur);
 
     dyn_buffer_destroy(&bmp_data);
     free(bitmap);

libscan/macros.h

@@ -1,16 +1,16 @@
-#ifndef	FALSE
-#define	FALSE	(0)
+#ifndef FALSE
+#define FALSE (0)
 #define BOOL int
 #endif
 
-#ifndef	TRUE
-#define	TRUE	(!FALSE)
+#ifndef TRUE
+#define TRUE (!FALSE)
 #endif
 
-#undef	MAX
+#undef MAX
 #define MAX(a, b)  (((a) > (b)) ? (a) : (b))
 
-#undef	MIN
+#undef MIN
 #define MIN(a, b)  (((a) < (b)) ? (a) : (b))
 
 #ifndef PATH_MAX
@@ -18,7 +18,7 @@
 #endif
 
 #undef ABS
-#define ABS(a)	   (((a) < 0) ? -(a) : (a))
+#define ABS(a) (((a) < 0) ? -(a) : (a))
 
 #define APPEND_STR_META(doc, keyname, value) \
     {meta_line_t *meta_str = malloc(sizeof(meta_line_t) + strlen(value)); \
@@ -26,14 +26,34 @@
     strcpy(meta_str->str_val, value); \
     APPEND_META(doc, meta_str)}
 
-#define APPEND_INT_META(doc, keyname, value) \
-    {meta_line_t *meta_int = malloc(sizeof(meta_line_t)); \
-    meta_int->key = keyname; \
-    meta_int->int_val = value; \
-    APPEND_META(doc, meta_int)}
+#define APPEND_LONG_META(doc, keyname, value) \
+    {meta_line_t *meta_long = malloc(sizeof(meta_line_t)); \
+    meta_long->key = keyname; \
+    meta_long->long_val = value; \
+    APPEND_META(doc, meta_long)}
 
 #define APPEND_TN_META(doc, width, height) \
     {meta_line_t *meta_str = malloc(sizeof(meta_line_t) + 4 + 1 + 4); \
     meta_str->key = MetaThumbnail; \
     sprintf(meta_str->str_val, "%04d,%04d", width, height); \
     APPEND_META(doc, meta_str)}
+
+#define APPEND_META(doc, meta) \
+    meta->next = NULL;\
+    if (doc->meta_head == NULL) {\
+        doc->meta_head = meta;\
+        doc->meta_tail = doc->meta_head;\
+    } else {\
+        doc->meta_tail->next = meta;\
+        doc->meta_tail = meta;\
+    }
+
+#define APPEND_UTF8_META(doc, keyname, str) \
+    text_buffer_t tex = text_buffer_create(-1); \
+    text_buffer_append_string0(&tex, str); \
+    text_buffer_terminate_string(&tex); \
+    meta_line_t *meta_tag = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); \
+    meta_tag->key = keyname; \
+    strcpy(meta_tag->str_val, tex.dyn_buffer.buf); \
+    APPEND_META(doc, meta_tag) \
+    text_buffer_destroy(&tex);

libscan/media/media.c

@@ -1,8 +1,4 @@
 #include "media.h"
-
-#include "../util.h"
-
-
 #include <ctype.h>
 
 #define MIN_SIZE 32
@@ -89,8 +85,67 @@ static void frame_and_packet_free(frame_and_packet_t *frame_and_packet) {
 }
 
 __always_inline
-static frame_and_packet_t *read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx,
-                           document_t *doc) {
+static void read_subtitles(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, int stream_idx, document_t *doc) {
+
+    text_buffer_t tex = text_buffer_create(-1);
+
+    AVPacket packet;
+    AVSubtitle subtitle;
+
+    AVCodec *subtitle_codec = avcodec_find_decoder(pFormatCtx->streams[stream_idx]->codecpar->codec_id);
+    AVCodecContext *decoder = avcodec_alloc_context3(subtitle_codec);
+    avcodec_parameters_to_context(decoder, pFormatCtx->streams[stream_idx]->codecpar);
+    avcodec_open2(decoder, subtitle_codec, NULL);
+
+    decoder->sub_text_format = FF_SUB_TEXT_FMT_ASS;
+
+    int got_sub;
+
+    while (1) {
+        int read_frame_ret = av_read_frame(pFormatCtx, &packet);
+
+        if (read_frame_ret != 0) {
+            break;
+        }
+
+        if (packet.stream_index != stream_idx) {
+            av_packet_unref(&packet);
+            continue;
+        }
+
+        avcodec_decode_subtitle2(decoder, &subtitle, &got_sub, &packet);
+
+        if (got_sub) {
+            for (int i = 0; i < subtitle.num_rects; i++) {
+                const char *text = subtitle.rects[i]->ass;
+
+                if (text == NULL) {
+                    continue;
+                }
+
+                char *idx = strstr(text, "\\N");
+                if (idx != NULL && strlen(idx + 2) > 1) {
+                    text_buffer_append_string0(&tex, idx + 2);
+                    text_buffer_append_char(&tex, ' ');
+                }
+            }
+            avsubtitle_free(&subtitle);
+        }
+
+        av_packet_unref(&packet);
+    }
+
+    text_buffer_terminate_string(&tex);
+
+    APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf)
+    text_buffer_destroy(&tex);
+    avcodec_free_context(&decoder);
+}
+
+__always_inline
+static frame_and_packet_t *
+read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx,
+           document_t *doc) {
 
     frame_and_packet_t *result = calloc(1, sizeof(frame_and_packet_t));
     result->packet = av_packet_alloc();
@@ -166,15 +221,8 @@ void append_tag_meta_if_not_exists(scan_media_ctx_t *ctx, document_t *doc, AVDic
     text_buffer_destroy(&tex);
 }
 
-#define APPEND_TAG_META(doc, tag_, keyname) \
-    text_buffer_t tex = text_buffer_create(-1); \
-    text_buffer_append_string0(&tex, tag_->value); \
-    text_buffer_terminate_string(&tex); \
-    meta_line_t *meta_tag = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); \
-    meta_tag->key = keyname; \
-    strcpy(meta_tag->str_val, tex.dyn_buffer.buf); \
-    APPEND_META(doc, meta_tag) \
-    text_buffer_destroy(&tex);
+#define APPEND_TAG_META(keyname) \
+    APPEND_UTF8_META(doc, keyname, tag->value)
 
 #define STRCPY_TOLOWER(dst, str) \
     strncpy(dst, str, sizeof(dst)); \
@@ -190,17 +238,17 @@ static void append_audio_meta(AVFormatContext *pFormatCtx, document_t *doc) {
         STRCPY_TOLOWER(key, tag->key)
 
         if (strcmp(key, "artist") == 0) {
-            APPEND_TAG_META(doc, tag, MetaArtist)
+            APPEND_TAG_META(MetaArtist)
         } else if (strcmp(key, "genre") == 0) {
-            APPEND_TAG_META(doc, tag, MetaGenre)
+            APPEND_TAG_META(MetaGenre)
         } else if (strcmp(key, "title") == 0) {
-            APPEND_TAG_META(doc, tag, MetaTitle)
+            APPEND_TAG_META(MetaTitle)
         } else if (strcmp(key, "album_artist") == 0) {
-            APPEND_TAG_META(doc, tag, MetaAlbumArtist)
+            APPEND_TAG_META(MetaAlbumArtist)
         } else if (strcmp(key, "album") == 0) {
-            APPEND_TAG_META(doc, tag, MetaAlbum)
+            APPEND_TAG_META(MetaAlbum)
         } else if (strcmp(key, "comment") == 0) {
-            APPEND_TAG_META(doc, tag, MetaContent)
+            APPEND_TAG_META(MetaContent)
         }
     }
 }
@@ -213,6 +261,9 @@ append_video_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVFrame *f
         meta_line_t *meta_duration = malloc(sizeof(meta_line_t));
         meta_duration->key = MetaMediaDuration;
         meta_duration->long_val = pFormatCtx->duration / AV_TIME_BASE;
+        if (meta_duration->long_val > INT32_MAX) {
+            meta_duration->long_val = 0;
+        }
         APPEND_META(doc, meta_duration)
 
         meta_line_t *meta_bitrate = malloc(sizeof(meta_line_t));
@@ -243,26 +294,34 @@ append_video_meta(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVFrame *f
 
             if (strcmp(key, "artist") == 0) {
                 append_tag_meta_if_not_exists(ctx, doc, tag, MetaArtist);
-            } else if (strcmp(tag->key, "ImageDescription") == 0) {
-                APPEND_TAG_META(doc, tag, MetaContent)
-            } else if (strcmp(tag->key, "Make") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifMake)
-            } else if (strcmp(tag->key, "Model") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifModel)
-            } else if (strcmp(tag->key, "Software") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifSoftware)
-            } else if (strcmp(tag->key, "FNumber") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifFNumber)
-            } else if (strcmp(tag->key, "FocalLength") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifFocalLength)
-            } else if (strcmp(tag->key, "UserComment") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifUserComment)
-            } else if (strcmp(tag->key, "ISOSpeedRatings") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifIsoSpeedRatings)
-            } else if (strcmp(tag->key, "ExposureTime") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifExposureTime)
-            } else if (strcmp(tag->key, "DateTime") == 0) {
-                APPEND_TAG_META(doc, tag, MetaExifDateTime)
+            } else if (strcmp(key, "imagedescription") == 0) {
+                APPEND_TAG_META(MetaContent)
+            } else if (strcmp(key, "make") == 0) {
+                APPEND_TAG_META(MetaExifMake)
+            } else if (strcmp(key, "model") == 0) {
+                APPEND_TAG_META(MetaExifModel)
+            } else if (strcmp(key, "software") == 0) {
+                APPEND_TAG_META(MetaExifSoftware)
+            } else if (strcmp(key, "fnumber") == 0) {
+                APPEND_TAG_META(MetaExifFNumber)
+            } else if (strcmp(key, "focallength") == 0) {
+                APPEND_TAG_META(MetaExifFocalLength)
+            } else if (strcmp(key, "usercomment") == 0) {
+                APPEND_TAG_META(MetaExifUserComment)
+            } else if (strcmp(key, "isospeedratings") == 0) {
+                APPEND_TAG_META(MetaExifIsoSpeedRatings)
+            } else if (strcmp(key, "exposuretime") == 0) {
+                APPEND_TAG_META(MetaExifExposureTime)
+            } else if (strcmp(key, "datetime") == 0) {
+                APPEND_TAG_META(MetaExifDateTime)
+            } else if (strcmp(key, "gpslatitude") == 0) {
+                APPEND_TAG_META(MetaExifGpsLatitudeDMS)
+            } else if (strcmp(key, "gpslatituderef") == 0) {
+                APPEND_TAG_META(MetaExifGpsLatitudeRef)
+            } else if (strcmp(key, "gpslongitude") == 0) {
+                APPEND_TAG_META(MetaExifGpsLongitudeDMS)
+            } else if (strcmp(key, "gpslongituderef") == 0) {
+                APPEND_TAG_META(MetaExifGpsLongitudeRef)
             }
         }
     }
@@ -272,6 +331,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
     int video_stream = -1;
     int audio_stream = -1;
+    int subtitle_stream = -1;
 
     avformat_find_stream_info(pFormatCtx, NULL);
 
@@ -286,7 +346,6 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
                     APPEND_STR_META(doc, MetaMediaAudioCodec, desc->name)
                 }
 
-                append_audio_meta(pFormatCtx, doc);
                 audio_stream = i;
             }
         } else if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
@@ -300,19 +359,34 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
                 meta_line_t *meta_w = malloc(sizeof(meta_line_t));
                 meta_w->key = MetaWidth;
-                meta_w->int_val = stream->codecpar->width;
+                meta_w->long_val = stream->codecpar->width;
                 APPEND_META(doc, meta_w)
 
                 meta_line_t *meta_h = malloc(sizeof(meta_line_t));
                 meta_h->key = MetaHeight;
-                meta_h->int_val = stream->codecpar->height;
+                meta_h->long_val = stream->codecpar->height;
                 APPEND_META(doc, meta_h)
 
                 video_stream = i;
             }
+        } else if (stream->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) {
+            subtitle_stream = i;
         }
     }
 
+    if (subtitle_stream != -1 && ctx->read_subtitles) {
+        read_subtitles(ctx, pFormatCtx, subtitle_stream, doc);
+
+        // Reset stream
+        if (video_stream != -1) {
+            av_seek_frame(pFormatCtx, video_stream, 0, 0);
+        }
+    }
+
+    if (audio_stream != -1) {
+        append_audio_meta(pFormatCtx, doc);
+    }
+
     if (video_stream != -1 && ctx->tn_size > 0) {
         AVStream *stream = pFormatCtx->streams[video_stream];
 
@@ -330,7 +404,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
         //Seek
         if (stream->nb_frames > 1 && stream->codecpar->codec_id != AV_CODEC_ID_GIF) {
-            int seek_ret = 0;
+            int seek_ret;
             for (int i = 20; i >= 0; i--) {
                 seek_ret = av_seek_frame(pFormatCtx, video_stream,
                                          stream->duration * 0.10, 0);
@@ -363,7 +437,8 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
         if (scaled_frame == STORE_AS_IS) {
             APPEND_TN_META(doc, frame_and_packet->frame->width, frame_and_packet->frame->height)
-            ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) frame_and_packet->packet->data, frame_and_packet->packet->size);
+            ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) frame_and_packet->packet->data,
+                       frame_and_packet->packet->size);
         } else {
             // Encode frame to jpeg
             AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
@@ -376,7 +451,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
             // Save thumbnail
             APPEND_TN_META(doc, scaled_frame->width, scaled_frame->height)
-            ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) jpeg_packet.data, jpeg_packet.size);
+            ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) jpeg_packet.data, jpeg_packet.size);
 
             avcodec_free_context(&jpeg_encoder);
             av_packet_unref(&jpeg_packet);
@@ -543,7 +618,7 @@ void init_media() {
     av_log_set_level(AV_LOG_QUIET);
 }
 
-int store_image_thumbnail(scan_media_ctx_t *ctx, void* buf, size_t buf_len, document_t *doc, const char *url) {
+int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, document_t *doc, const char *url) {
     memfile_t memfile;
     AVIOContext *io_ctx = NULL;
 
@@ -615,7 +690,8 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void* buf, size_t buf_len, docu
 
     if (scaled_frame == STORE_AS_IS) {
         APPEND_TN_META(doc, frame_and_packet->frame->width, frame_and_packet->frame->height)
-        ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) frame_and_packet->packet->data, frame_and_packet->packet->size);
+        ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) frame_and_packet->packet->data,
+                   frame_and_packet->packet->size);
     } else {
         // Encode frame to jpeg
         AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
@@ -628,7 +704,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void* buf, size_t buf_len, docu
 
         // Save thumbnail
         APPEND_TN_META(doc, scaled_frame->width, scaled_frame->height)
-        ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) jpeg_packet.data, jpeg_packet.size);
+        ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) jpeg_packet.data, jpeg_packet.size);
 
         av_packet_unref(&jpeg_packet);
         avcodec_free_context(&jpeg_encoder);

libscan/media/media.h

@@ -18,12 +18,13 @@ typedef struct {
     int tn_size;
     float tn_qscale;
     long max_media_buffer;
+    int read_subtitles;
 } scan_media_ctx_t;
 
 __always_inline
 static AVCodecContext *alloc_jpeg_encoder(int w, int h, float qscale) {
 
-    AVCodec *jpeg_codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
+    const AVCodec *jpeg_codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
     AVCodecContext *jpeg = avcodec_alloc_context3(jpeg_codec);
     jpeg->width = w;
     jpeg->height = h;
@@ -43,8 +44,9 @@ static AVCodecContext *alloc_jpeg_encoder(int w, int h, float qscale) {
 
 
 void parse_media(scan_media_ctx_t *ctx, vfile_t *f, document_t *doc);
+
 void init_media();
 
-int store_image_thumbnail(scan_media_ctx_t *ctx, void* buf, size_t buf_len, document_t *doc, const char *url);
+int store_image_thumbnail(scan_media_ctx_t *ctx, void *buf, size_t buf_len, document_t *doc, const char *url);
 
 #endif

libscan/msdoc/msdoc.c

@@ -0,0 +1,147 @@
+#include "msdoc.h"
+#include <errno.h>
+
+#include <sys/mman.h>
+#include "../../third-party/antiword/src/antiword.h"
+
+#include "../ebook/ebook.h"
+
+void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, void* buf, size_t buf_len) {
+
+    // Open word doc
+    options_type *opts = direct_vGetOptions();
+    opts->iParagraphBreak = 74;
+    opts->eConversionType = conversion_text;
+    opts->bHideHiddenText = 1;
+    opts->bRemoveRemovedText = 1;
+    opts->bUseLandscape = 0;
+    opts->eEncoding = encoding_utf_8;
+    opts->iPageHeight = 842; // A4
+    opts->iPageWidth = 595;
+    opts->eImageLevel = level_ps_3;
+
+    int doc_word_version = iGuessVersionNumber(file_in, buf_len);
+    if (doc_word_version < 0 || doc_word_version == 3) {
+        free(buf);
+        return;
+    }
+    rewind(file_in);
+
+    size_t out_len;
+    char *out_buf;
+
+    FILE *file_out = open_memstream(&out_buf, &out_len);
+
+    diagram_type *diag = pCreateDiagram("antiword", NULL, file_out);
+    if (diag == NULL) {
+        fclose(file_in);
+        return;
+    }
+
+    iInitDocument(file_in, buf_len);
+    const char* author = szGetAuthor();
+    if (author != NULL) {
+        APPEND_UTF8_META(doc, MetaAuthor, author)
+    }
+
+    const char* title = szGetTitle();
+    if (title != NULL) {
+        APPEND_UTF8_META(doc, MetaTitle, title)
+    }
+    vFreeDocument();
+
+    bWordDecryptor(file_in, buf_len, diag);
+    vDestroyDiagram(diag);
+    fclose(file_out);
+
+    if (buf_len > 0) {
+        text_buffer_t tex = text_buffer_create(ctx->content_size);
+        text_buffer_append_string(&tex, out_buf, out_len);
+        text_buffer_terminate_string(&tex);
+
+        meta_line_t *meta_content = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur);
+        meta_content->key = MetaContent;
+        memcpy(meta_content->str_val, tex.dyn_buffer.buf, tex.dyn_buffer.cur);
+        APPEND_META(doc, meta_content)
+
+        text_buffer_destroy(&tex);
+    }
+
+    free(buf);
+    free(out_buf);
+}
+
+void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file, void* buf, size_t buf_len) {
+
+    scan_ebook_ctx_t ebook_ctx = {
+            .content_size = ctx->content_size,
+            .tn_size = ctx->tn_size,
+            .log = ctx->log,
+            .logf = ctx->logf,
+            .store = ctx->store,
+    };
+
+    // Open word doc
+    options_type *opts = direct_vGetOptions();
+    opts->iParagraphBreak = 74;
+    opts->eConversionType = conversion_pdf;
+    opts->bHideHiddenText = 1;
+    opts->bRemoveRemovedText = 1;
+    opts->bUseLandscape = 0;
+    opts->eEncoding = encoding_latin_1;
+    opts->iPageHeight = 842; // A4
+    opts->iPageWidth = 595;
+    opts->eImageLevel = level_ps_3;
+
+    int doc_word_version = iGuessVersionNumber(file, buf_len);
+    if (doc_word_version < 0 || doc_word_version == 3) {
+        free(buf);
+        return;
+    }
+    rewind(file);
+
+    size_t out_len;
+    char *out_buf;
+
+    FILE *file_out = open_memstream(&out_buf, &out_len);
+
+    diagram_type *diag = pCreateDiagram("antiword", NULL, file_out);
+    if (diag == NULL) {
+        return;
+    }
+
+    bWordDecryptor(file, buf_len, diag);
+    vDestroyDiagram(diag);
+
+    fclose(file_out);
+
+    parse_ebook_mem(&ebook_ctx, out_buf, out_len, "application/pdf", doc, TRUE);
+
+    free(buf);
+    free(out_buf);
+}
+
+void parse_msdoc(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc) {
+
+    size_t buf_len;
+    char *buf = read_all(f, &buf_len);
+    if (buf == NULL) {
+        CTX_LOG_ERROR(f->filepath, "read_all() failed")
+        return;
+    }
+
+    FILE *file = fmemopen(buf, buf_len, "rb");
+    if (file == NULL) {
+        free(buf);
+        CTX_LOG_ERRORF(f->filepath, "fmemopen() failed (%d)", errno)
+        return;
+    }
+
+    if (ctx->tn_size > 0) {
+        char *buf_pdf = malloc(buf_len);
+        memcpy(buf_pdf, buf, buf_len);
+        parse_msdoc_pdf(ctx, doc, file, buf_pdf, buf_len);
+    }
+    parse_msdoc_text(ctx, doc, file, buf, buf_len);
+    fclose(file);
+}

libscan/msdoc/msdoc.h

@@ -0,0 +1,24 @@
+#ifndef SCAN_SCAN_MSDOC_H
+#define SCAN_SCAN_MSDOC_H
+
+#include "../scan.h"
+
+typedef struct {
+    long content_size;
+    int tn_size;
+    log_callback_t log;
+    logf_callback_t logf;
+    store_callback_t store;
+    unsigned int msdoc_mime;
+} scan_msdoc_ctx_t;
+
+__always_inline
+static int is_msdoc(scan_msdoc_ctx_t *ctx, unsigned int mime) {
+    return mime == ctx->msdoc_mime;
+}
+
+void parse_msdoc(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc);
+
+void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, void* buf, size_t buf_len);
+
+#endif

libscan/ooxml/ooxml.c

@@ -1,6 +1,5 @@
 #include "ooxml.h"
 
-#include "../util.h"
 #include <archive.h>
 #include <archive_entry.h>
 #include <libxml/xmlstring.h>
@@ -48,7 +47,7 @@ int extract_text(scan_ooxml_ctx_t *ctx, xmlDoc *xml, xmlNode *node, text_buffer_
     }
 
     for (xmlNode *child = node; child; child = child->next) {
-        if (*child->name == 't' && *(child->name + 1) == '\0') {
+        if (child->name != NULL && *child->name == 't' && *(child->name + 1) == '\0') {
             xmlChar *text = xmlNodeListGetString(xml, child->xmlChildrenNode, 1);
 
             if (text) {
@@ -106,7 +105,7 @@ static int read_part(scan_ooxml_ctx_t *ctx, struct archive *a, text_buffer_t *bu
 }
 
 __always_inline
-static int read_doc_props(scan_ooxml_ctx_t *ctx, struct archive *a, text_buffer_t *buf, document_t *doc) {
+static int read_doc_props(scan_ooxml_ctx_t *ctx, struct archive *a, document_t *doc) {
     xmlDoc *xml = xmlReadIO(xml_io_read, xml_io_close, a, "/", NULL,
                             XML_PARSE_RECOVER | XML_PARSE_NOWARNING | XML_PARSE_NOERROR | XML_PARSE_NONET);
 
@@ -158,7 +157,7 @@ void read_thumbnail(scan_ooxml_ctx_t *ctx, document_t *doc, struct archive *a, s
     archive_read_data(a, buf, entry_size);
 
     APPEND_TN_META(doc, 1, 1) // Size unknown
-    ctx->store((char *) doc->uuid, sizeof(doc->uuid), buf, entry_size);
+    ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), buf, entry_size);
     free(buf);
 }
 
@@ -198,7 +197,7 @@ void parse_ooxml(scan_ooxml_ctx_t *ctx, vfile_t *f, document_t *doc) {
                     buffer_full = TRUE;
                 }
             } else if (strcmp(path, "docProps/core.xml") == 0) {
-                if (read_doc_props(ctx, a, &tex, doc) != 0) {
+                if (read_doc_props(ctx, a, doc) != 0) {
                     break;
                 }
             } else if (strcmp(path, "docProps/thumbnail.jpeg") == 0) {

libscan/raw/raw.c

@@ -2,14 +2,13 @@
 #include <libraw/libraw.h>
 
 #include "../media/media.h"
-#include <fcntl.h>
 #include <unistd.h>
 
 
 #define MIN_SIZE 32
 
 int store_thumbnail_jpeg(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, document_t *doc) {
-    return store_image_thumbnail((scan_media_ctx_t*)ctx, img->data, img->data_size, doc, "x.jpeg");
+    return store_image_thumbnail((scan_media_ctx_t *) ctx, img->data, img->data_size, doc, "x.jpeg");
 }
 
 int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, document_t *doc) {
@@ -37,7 +36,7 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
 
     AVFrame *scaled_frame = av_frame_alloc();
 
-    struct SwsContext *sws_ctx= sws_getContext(
+    struct SwsContext *sws_ctx = sws_getContext(
             img->width, img->height, AV_PIX_FMT_RGB24,
             dstW, dstH, AV_PIX_FMT_YUVJ420P,
             SIST_SWS_ALGO, 0, 0, 0
@@ -71,7 +70,7 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
     avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
 
     APPEND_TN_META(doc, scaled_frame->width, scaled_frame->height)
-    ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) jpeg_packet.data, jpeg_packet.size);
 
     av_packet_unref(&jpeg_packet);
     av_free(*scaled_frame->data);
@@ -81,6 +80,8 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
     return TRUE;
 }
 
+#define DMS_REF(ref) (((ref) == 'S' || (ref) == 'W') ? -1 : 1)
+
 void parse_raw(scan_raw_ctx_t *ctx, vfile_t *f, document_t *doc) {
     libraw_data_t *libraw_lib = libraw_init(0);
 
@@ -100,6 +101,7 @@ void parse_raw(scan_raw_ctx_t *ctx, vfile_t *f, document_t *doc) {
     if (ret != 0) {
         CTX_LOG_ERROR(f->filepath, "Could not open raw file")
         free(buf);
+        libraw_close(libraw_lib);
         return;
     }
 
@@ -112,8 +114,8 @@ void parse_raw(scan_raw_ctx_t *ctx, vfile_t *f, document_t *doc) {
     if (*libraw_lib->idata.software != '\0') {
         APPEND_STR_META(doc, MetaExifSoftware, libraw_lib->idata.software)
     }
-    APPEND_INT_META(doc, MetaWidth, libraw_lib->sizes.width)
-    APPEND_INT_META(doc, MetaHeight, libraw_lib->sizes.height)
+    APPEND_LONG_META(doc, MetaWidth, libraw_lib->sizes.width)
+    APPEND_LONG_META(doc, MetaHeight, libraw_lib->sizes.height)
     char tmp[1024];
     snprintf(tmp, sizeof(tmp), "%g", libraw_lib->other.iso_speed);
     APPEND_STR_META(doc, MetaExifIsoSpeedRatings, tmp)
@@ -135,10 +137,24 @@ void parse_raw(scan_raw_ctx_t *ctx, vfile_t *f, document_t *doc) {
     snprintf(tmp, sizeof(tmp), "%.1f", libraw_lib->other.aperture);
     APPEND_STR_META(doc, MetaExifFNumber, tmp)
 
-    int denominator = (int)roundf(1 / libraw_lib->other.shutter);
+    int denominator = (int) roundf(1 / libraw_lib->other.shutter);
     snprintf(tmp, sizeof(tmp), "1/%d", denominator);
     APPEND_STR_META(doc, MetaExifExposureTime, tmp)
 
+    libraw_gps_info_t gps = libraw_lib->other.parsed_gps;
+    double gps_longitude_dec =
+            (gps.longtitude[0] + gps.longtitude[1] / 60 + gps.longtitude[2] / 3600) * DMS_REF(gps.longref);
+    snprintf(tmp, sizeof(tmp), "%.15f", gps_longitude_dec);
+    if (gps_longitude_dec != 0.0) {
+        APPEND_STR_META(doc, MetaExifGpsLongitudeDec, tmp)
+    }
+
+    double gps_latitude_dec = (gps.latitude[0] + gps.latitude[1] / 60 + gps.latitude[2] / 3600) * DMS_REF(gps.latref);
+    snprintf(tmp, sizeof(tmp), "%.15f", gps_latitude_dec);
+    if (gps_latitude_dec != 0.0) {
+        APPEND_STR_META(doc, MetaExifGpsLatitudeDec, tmp)
+    }
+
     APPEND_STR_META(doc, MetaMediaVideoCodec, "raw")
 
     if (ctx->tn_size <= 0) {

libscan/scan.h

@@ -1,35 +1,27 @@
 #ifndef SCAN_SCAN_H
 #define SCAN_SCAN_H
 
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
 #include <stdio.h>
 #include <sys/stat.h>
-#include <uuid/uuid.h>
+#include <openssl/md5.h>
 
 #include "macros.h"
 
 #define SIST_SWS_ALGO SWS_LANCZOS
 
-#define META_INT_MASK 0x80
-#define META_STR_MASK 0x40
-#define META_LONG_MASK 0x20
-
 #define UNUSED(x) __attribute__((__unused__))  x
 
-#define META_STR(id) ((unsigned) id) | ((unsigned) META_STR_MASK)
-#define META_INT(id) ((unsigned) id) | ((unsigned) META_INT_MASK)
-#define META_LONG(id) ((unsigned) id) | ((unsigned) META_LONG_MASK)
-
-#define IS_META_INT(key) (key & META_INT_MASK) == META_INT_MASK
-#define IS_META_LONG(key) (key & META_LONG_MASK) == META_LONG_MASK
-#define IS_META_STR(meta) (meta->key & META_STR_MASK) == META_STR_MASK
-
 typedef void (*store_callback_t)(char *key, size_t key_len, char *buf, size_t buf_len);
 typedef void (*logf_callback_t)(const char *filepath, int level, char *format, ...);
 typedef void (*log_callback_t)(const char *filepath, int level, char *str);
 
 typedef int scan_code_t;
 #define SCAN_OK (scan_code_t) 0
-#define SCAN_ERR_READ (scan_code_t) -1
+#define SCAN_ERR_READ (scan_code_t) (-1)
 
 #define LEVEL_DEBUG 0
 #define LEVEL_INFO 1
@@ -52,35 +44,45 @@ typedef int scan_code_t;
 #define CTX_LOG_FATALF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_FATAL, fmt, __VA_ARGS__); exit(-1);
 #define CTX_LOG_FATAL(filepath, str) ctx->log(filepath, LEVEL_FATAL, str); exit(-1);
 
-// This is written to file as a 16-bit int!
 enum metakey {
-    MetaContent = META_STR(1),
-    MetaWidth = META_INT(2),
-    MetaHeight = META_INT(3),
-    MetaMediaDuration = META_LONG(4),
-    MetaMediaAudioCodec = META_STR(5),
-    MetaMediaVideoCodec = META_STR(6),
-    MetaMediaBitrate = META_LONG(7),
-    MetaArtist = META_STR(8),
-    MetaAlbum = META_STR(9),
-    MetaAlbumArtist = META_STR(10),
-    MetaGenre = META_STR(11),
-    MetaTitle = META_STR(12),
-    MetaFontName = META_STR(13),
-    MetaParent = META_STR(14),
-    MetaExifMake = META_STR(15),
-    MetaExifSoftware = META_STR(16),
-    MetaExifExposureTime = META_STR(17),
-    MetaExifFNumber = META_STR(18),
-    MetaExifFocalLength = META_STR(19),
-    MetaExifUserComment = META_STR(20),
-    MetaExifModel = META_STR(21),
-    MetaExifIsoSpeedRatings = META_STR(22),
-    MetaExifDateTime = META_STR(23),
-    MetaAuthor = META_STR(24),
-    MetaModifiedBy = META_STR(25),
-    MetaThumbnail = META_STR(26),
-    MetaPages = META_INT(27),
+    // String
+    MetaContent = 1,
+    MetaMediaAudioCodec,
+    MetaMediaVideoCodec,
+    MetaArtist,
+    MetaAlbum,
+    MetaAlbumArtist,
+    MetaGenre,
+    MetaTitle,
+    MetaFontName,
+    MetaParent,
+    MetaExifMake,
+    MetaExifSoftware,
+    MetaExifExposureTime,
+    MetaExifFNumber,
+    MetaExifFocalLength,
+    MetaExifUserComment,
+    MetaExifModel,
+    MetaExifIsoSpeedRatings,
+    MetaExifDateTime,
+    MetaAuthor,
+    MetaModifiedBy,
+    MetaThumbnail,
+
+    // Number
+    MetaWidth,
+    MetaHeight,
+    MetaMediaDuration,
+    MetaMediaBitrate,
+    MetaPages,
+
+    // ??
+    MetaExifGpsLongitudeDMS,
+    MetaExifGpsLongitudeRef,
+    MetaExifGpsLatitudeDMS,
+    MetaExifGpsLatitudeRef,
+    MetaExifGpsLatitudeDec,
+    MetaExifGpsLongitudeDec,
 };
 
 typedef struct meta_line {
@@ -88,20 +90,20 @@ typedef struct meta_line {
     enum metakey key;
     union {
         char str_val[0];
-        int int_val;
         unsigned long long_val;
+        double double_val;
     };
 } meta_line_t;
 
 
 typedef struct document {
-    unsigned char uuid[16];
-    unsigned long ino;
+    unsigned char path_md5[MD5_DIGEST_LENGTH];
     unsigned long size;
     unsigned int mime;
     int mtime;
     short base;
     short ext;
+    char has_parent;
     meta_line_t *meta_head;
     meta_line_t *meta_tail;
     char *filepath;
@@ -142,21 +144,11 @@ typedef struct parse_job_t {
     int base;
     int ext;
     struct vfile vfile;
-    uuid_t parent;
+    unsigned char parent[MD5_DIGEST_LENGTH];
     char filepath[1];
 } parse_job_t;
 
 
-#define APPEND_META(doc, meta) \
-    meta->next = NULL;\
-    if (doc->meta_head == NULL) {\
-        doc->meta_head = meta;\
-        doc->meta_tail = doc->meta_head;\
-    } else {\
-        doc->meta_tail->next = meta;\
-        doc->meta_tail = meta;\
-    }
-
 #include "util.h"
 
 typedef void (*parse_callback_t)(parse_job_t *job);

libscan/util.h

@@ -133,11 +133,11 @@ static void dyn_buffer_write_int(dyn_buffer_t *buf, int d) {
     buf->cur += sizeof(int);
 }
 
-static void dyn_buffer_write_short(dyn_buffer_t *buf, short s) {
+static void dyn_buffer_write_short(dyn_buffer_t *buf, uint16_t s) {
     grow_buffer_small(buf);
 
-    *(short *) (buf->buf + buf->cur) = s;
-    buf->cur += sizeof(short);
+    *(uint16_t *) (buf->buf + buf->cur) = s;
+    buf->cur += sizeof(uint16_t);
 }
 
 static void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l) {
@@ -215,15 +215,19 @@ static void text_buffer_terminate_string(text_buffer_t *buf) {
 
 // Naive UTF16 -> ascii conversion
 static int text_buffer_append_string16_le(text_buffer_t *buf, const char *str, size_t len) {
+    int ret = 0;
     for (int i = 1; i < len; i += 2) {
-        text_buffer_append_char(buf, str[i]);
+        ret = text_buffer_append_char(buf, str[i]);
     }
+    return ret;
 }
 
 static int text_buffer_append_string16_be(text_buffer_t *buf, const char *str, size_t len) {
+    int ret = 0;
     for (int i = 0; i < len; i += 2) {
-        text_buffer_append_char(buf, str[i]);
+        ret = text_buffer_append_char(buf, str[i]);
     }
+    return ret;
 }
 
 #define UTF8_END_OF_STRING \
@@ -251,7 +255,7 @@ static int text_buffer_append_string(text_buffer_t *buf, const char *str, size_t
     }
 
     utf8_int32_t c;
-    char tmp[16];
+    char tmp[16] = {0};
 
     do {
         ptr = (char *) utf8codepoint(ptr, &c);
@@ -273,7 +277,7 @@ static int text_buffer_append_string(text_buffer_t *buf, const char *str, size_t
     return 0;
 }
 
-static int text_buffer_append_string0(text_buffer_t *buf, char *str) {
+static int text_buffer_append_string0(text_buffer_t *buf, const char *str) {
     return text_buffer_append_string(buf, str, strlen(str));
 }
 

test/main.cpp

@@ -10,6 +10,7 @@ extern "C" {
 #include "../libscan/ooxml/ooxml.h"
 #include "../libscan/mobi/scan_mobi.h"
 #include "../libscan/raw/raw.h"
+#include "../libscan/msdoc/msdoc.h"
 #include <libavutil/avutil.h>
 }
 
@@ -20,6 +21,7 @@ static scan_text_ctx_t text_500_ctx;
 
 static scan_ebook_ctx_t ebook_ctx;
 static scan_ebook_ctx_t ebook_500_ctx;
+static scan_ebook_ctx_t ebook_fast_ctx;
 
 static scan_comic_ctx_t comic_ctx;
 
@@ -33,6 +35,10 @@ static scan_mobi_ctx_t mobi_500_ctx;
 
 static scan_raw_ctx_t raw_ctx;
 
+static scan_msdoc_ctx_t msdoc_ctx;
+
+static scan_msdoc_ctx_t msdoc_text_ctx;
+
 
 document_t LastSubDoc;
 
@@ -197,7 +203,7 @@ TEST(Ebook, CandlePdf) {
     ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), 500, 4);
     ASSERT_NE(get_meta(&doc, MetaContent)->str_val[0], ' ');
     ASSERT_NE(size_before, store_size);
-    ASSERT_EQ(get_meta(&doc, MetaPages)->int_val, 16);
+    ASSERT_EQ(get_meta(&doc, MetaPages)->long_val, 16);
 
     cleanup(&doc, &f);
 }
@@ -245,6 +251,28 @@ TEST(Ebook, Epub1) {
     cleanup(&doc, &f);
 }
 
+TEST(Ebook, EpubFastMupdfError) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/ebook/mupdf-issue-129.epub", &f, &doc);
+
+    parse_ebook(&ebook_fast_ctx, &f, "application/epub+zip", &doc);
+
+    ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), 500, 4);
+    cleanup(&doc, &f);
+}
+
+TEST(Ebook, Epub1Fast) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/ebook/epub1.epub", &f, &doc);
+
+    parse_ebook(&ebook_fast_ctx, &f, "application/epub+zip", &doc);
+
+    ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), 500, 4);
+    cleanup(&doc, &f);
+}
+
 TEST(Ebook, EpubBlankFirstPage) {
     vfile_t f;
     document_t doc;
@@ -286,6 +314,19 @@ TEST(Comic, ComicCbr) {
     cleanup(&doc, &f);
 }
 
+TEST(Comic, ComicIssue160) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/ebook/comic-segfault-issue-160.cbr", &f, &doc);
+
+    int tn_size_saved = comic_ctx.tn_size;
+    comic_ctx.tn_size = 0;
+    parse_comic(&comic_ctx, &f, &doc);
+    comic_ctx.tn_size = tn_size_saved;
+
+    cleanup(&doc, &f);
+}
+
 TEST(Comic, ComicCbrAsIs) {
     vfile_t f;
     document_t doc;
@@ -314,6 +355,22 @@ TEST(Comic, ComicCbrFilters) {
 
 /* Media (image) */
 
+TEST(MediaImage, ExifGps1) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/media/exif_GPS.jpg", &f, &doc);
+
+    parse_media(&media_ctx, &f, &doc);
+
+    ASSERT_STREQ(get_meta(&doc, MetaExifGpsLatitudeRef)->str_val, "N");
+    ASSERT_STREQ(get_meta(&doc, MetaExifGpsLatitudeDMS)->str_val, "48:1 , 56585399:1000000, 0:1");
+
+    ASSERT_STREQ(get_meta(&doc, MetaExifGpsLongitudeRef)->str_val, "E");
+    ASSERT_STREQ(get_meta(&doc, MetaExifGpsLongitudeDMS)->str_val, "9:1 , 28046900:1000000, 0:1");
+
+    cleanup(&doc, &f);
+}
+
 TEST(MediaImage, Exif1) {
     vfile_t f;
     document_t doc;
@@ -369,6 +426,36 @@ TEST(MediaImage, Mem2AsIs) {
     cleanup(&doc, &f);
 }
 
+TEST(MediaVideo, VidMkvSubDisabled) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/media/berd.mkv", &f, &doc);
+
+    size_t size_before = store_size;
+    parse_media(&media_ctx, &f, &doc);
+
+    ASSERT_NE(size_before, store_size);
+    ASSERT_EQ(get_meta(&doc, MetaContent), nullptr);
+
+    cleanup(&doc, &f);
+}
+
+TEST(MediaVideo, VidMkvSubEnabled) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/media/berd.mkv", &f, &doc);
+
+    size_t size_before = store_size;
+    media_ctx.read_subtitles = TRUE;
+    parse_media(&media_ctx, &f, &doc);
+    media_ctx.read_subtitles = FALSE;
+
+    ASSERT_NE(size_before, store_size);
+    ASSERT_NE(get_meta(&doc, MetaContent), nullptr);
+
+    cleanup(&doc, &f);
+}
+
 TEST(MediaVideo, Vid3Mp4) {
     vfile_t f;
     document_t doc;
@@ -590,6 +677,23 @@ TEST(Arc, Utf8) {
     cleanup(&doc, &f);
 }
 
+TEST(Arc, EncryptedZip) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/arc/encrypted.zip", &f, &doc);
+
+    size_t size_before = store_size;
+
+    strcpy(arc_recurse_media_ctx.passphrase, "sist2");
+    parse_archive(&arc_recurse_media_ctx, &f, &doc);
+
+    arc_recurse_media_ctx.passphrase[0] = '\0';
+
+    ASSERT_NE(size_before, store_size);
+
+    cleanup(&doc, &f);
+}
+
 /* RAW */
 TEST(RAW, Panasonic) {
     vfile_t f;
@@ -607,13 +711,30 @@ TEST(RAW, Panasonic) {
     ASSERT_STREQ(get_meta(&doc, MetaExifDateTime)->str_val, "2020:07:20 10:00:34");
     ASSERT_STREQ(get_meta(&doc, MetaExifFocalLength)->str_val, "20.0");
     ASSERT_STREQ(get_meta(&doc, MetaExifFNumber)->str_val, "2.0");
-    ASSERT_EQ(get_meta(&doc, MetaWidth)->int_val, 5200);
-    ASSERT_EQ(get_meta(&doc, MetaHeight)->int_val, 3904);
+    ASSERT_EQ(get_meta(&doc, MetaWidth)->long_val, 5200);
+    ASSERT_EQ(get_meta(&doc, MetaHeight)->long_val, 3904);
     ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
 }
 
+TEST(RAW, ExifGps1) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/raw/exif_gps.DNG", &f, &doc);
+
+    size_t size_before = store_size;
+
+    parse_raw(&raw_ctx, &f, &doc);
+
+    ASSERT_NE(size_before, store_size);
+
+    ASSERT_STREQ(get_meta(&doc, MetaExifGpsLatitudeDec)->str_val, "48.943088531494141");
+    ASSERT_STREQ(get_meta(&doc, MetaExifGpsLongitudeDec)->str_val, "9.467448234558105");
+
+    cleanup(&doc, &f);
+}
+
 TEST(RAW, Nikon) {
     vfile_t f;
     document_t doc;
@@ -626,8 +747,8 @@ TEST(RAW, Nikon) {
     ASSERT_STREQ(get_meta(&doc, MetaMediaVideoCodec)->str_val, "raw");
     ASSERT_STREQ(get_meta(&doc, MetaExifModel)->str_val, "D750");
     ASSERT_STREQ(get_meta(&doc, MetaExifMake)->str_val, "Nikon");
-    ASSERT_EQ(get_meta(&doc, MetaWidth)->int_val, 6032);
-    ASSERT_EQ(get_meta(&doc, MetaHeight)->int_val, 4032);
+    ASSERT_EQ(get_meta(&doc, MetaWidth)->long_val, 6032);
+    ASSERT_EQ(get_meta(&doc, MetaHeight)->long_val, 4032);
     ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
@@ -645,8 +766,8 @@ TEST(RAW, Sony) {
     ASSERT_STREQ(get_meta(&doc, MetaMediaVideoCodec)->str_val, "raw");
     ASSERT_STREQ(get_meta(&doc, MetaExifModel)->str_val, "ILCE-7RM3");
     ASSERT_STREQ(get_meta(&doc, MetaExifMake)->str_val, "Sony");
-    ASSERT_EQ(get_meta(&doc, MetaWidth)->int_val, 7968);
-    ASSERT_EQ(get_meta(&doc, MetaHeight)->int_val, 5320);
+    ASSERT_EQ(get_meta(&doc, MetaWidth)->long_val, 7968);
+    ASSERT_EQ(get_meta(&doc, MetaHeight)->long_val, 5320);
     ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
@@ -664,8 +785,8 @@ TEST(RAW, Olympus) {
     ASSERT_STREQ(get_meta(&doc, MetaMediaVideoCodec)->str_val, "raw");
     ASSERT_STREQ(get_meta(&doc, MetaExifModel)->str_val, "E-M5MarkII");
     ASSERT_STREQ(get_meta(&doc, MetaExifMake)->str_val, "Olympus");
-    ASSERT_EQ(get_meta(&doc, MetaWidth)->int_val, 4640);
-    ASSERT_EQ(get_meta(&doc, MetaHeight)->int_val, 3472);
+    ASSERT_EQ(get_meta(&doc, MetaWidth)->long_val, 4640);
+    ASSERT_EQ(get_meta(&doc, MetaHeight)->long_val, 3472);
     ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
@@ -682,13 +803,154 @@ TEST(RAW, Fuji) {
     ASSERT_STREQ(get_meta(&doc, MetaMediaVideoCodec)->str_val, "raw");
     ASSERT_STREQ(get_meta(&doc, MetaExifModel)->str_val, "X-T2");
     ASSERT_STREQ(get_meta(&doc, MetaExifMake)->str_val, "Fujifilm");
-    ASSERT_EQ(get_meta(&doc, MetaWidth)->int_val, 6032);
-    ASSERT_EQ(get_meta(&doc, MetaHeight)->int_val, 4028);
+    ASSERT_EQ(get_meta(&doc, MetaWidth)->long_val, 6032);
+    ASSERT_EQ(get_meta(&doc, MetaHeight)->long_val, 4028);
     ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
 }
 
+/* msdoc */
+TEST(Msdoc, Test1Pdf) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/test1.doc", &f, &doc);
+
+    size_t size_before = store_size;
+
+    parse_msdoc(&msdoc_ctx, &f, &doc);
+
+    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "October 2000") != nullptr);
+    ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "INTERNATIONAL ORGANIZATION FOR STANDARDIZATION");
+    ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Oliver Morgan");
+    ASSERT_EQ(get_meta(&doc, MetaPages)->long_val, 57);
+    ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
+    ASSERT_NE(size_before, store_size);
+
+    cleanup(&doc, &f);
+}
+
+TEST(Msdoc, Test1Text) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/test1.doc", &f, &doc);
+
+    size_t size_before = store_size;
+
+    parse_msdoc(&msdoc_text_ctx, &f, &doc);
+
+    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "October 2000") != nullptr);
+    ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "INTERNATIONAL ORGANIZATION FOR STANDARDIZATION");
+    ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Oliver Morgan");
+    ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
+    ASSERT_EQ(size_before, store_size);
+
+    cleanup(&doc, &f);
+}
+
+TEST(Msdoc, Test2Pdf) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/test2.doc", &f, &doc);
+
+    size_t size_before = store_size;
+
+    parse_msdoc(&msdoc_ctx, &f, &doc);
+
+    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "GNU Free Documentation License") != nullptr);
+    ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "DWARF Debugging Information Format");
+    ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Ron Brender");
+    ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
+    ASSERT_NE(size_before, store_size);
+
+    cleanup(&doc, &f);
+}
+
+TEST(Msdoc, Test3Pdf) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/test3.doc", &f, &doc);
+
+    size_t size_before = store_size;
+
+    parse_msdoc(&msdoc_ctx, &f, &doc);
+
+    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "INTERNATIONAL PATENT CLASSIFICATION") != nullptr);
+    ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "IPC Fixed Texts Specification");
+    ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Fievet");
+    ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
+    ASSERT_NE(size_before, store_size);
+
+    cleanup(&doc, &f);
+}
+
+TEST(Msdoc, Test4Pdf) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/test4.doc", &f, &doc);
+
+    size_t size_before = store_size;
+
+    parse_msdoc(&msdoc_ctx, &f, &doc);
+
+    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "SQL Server international data types") != nullptr);
+    ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "MSDN Authoring Template");
+    ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Brenda Yen");
+    ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
+    ASSERT_NE(size_before, store_size);
+
+    cleanup(&doc, &f);
+}
+
+TEST(Msdoc, TestUtf8Pdf) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/japanese.doc", &f, &doc);
+
+    size_t size_before = store_size;
+
+    parse_msdoc(&msdoc_ctx, &f, &doc);
+
+    ASSERT_NE(get_meta(&doc, MetaContent), nullptr);
+    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "调查项目 A questionnaire") != nullptr);
+    ASSERT_NE(size_before, store_size);
+
+    cleanup(&doc, &f);
+}
+
+TEST(Msdoc, TestUtf8Text) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/japanese.doc", &f, &doc);
+
+    parse_msdoc(&msdoc_text_ctx, &f, &doc);
+
+    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "调查项目 A questionnaire") != nullptr);
+
+    cleanup(&doc, &f);
+}
+
+TEST(Msdoc, TestFuzz1) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/fuzz_ole.doc", &f, &doc);
+
+    size_t buf_len;
+    char *buf = (char *) read_all(&f, &buf_len);
+
+    for (int i = 0; i < 1000; i++) {
+        size_t buf_len_copy = buf_len;
+        char *buf_copy = (char*)malloc(buf_len);
+        memcpy(buf_copy, buf, buf_len);
+
+        fuzz_buffer(buf_copy, &buf_len_copy, 3, 8, 5);
+        FILE *file = fmemopen(buf_copy, buf_len_copy, "rb");
+        parse_msdoc_text(&msdoc_text_ctx, &doc, file, buf_copy, buf_len_copy);
+    }
+    free(buf);
+    cleanup(&doc, &f);
+}
+
 
 int main(int argc, char **argv) {
     setlocale(LC_ALL, "");
@@ -715,10 +977,15 @@ int main(int argc, char **argv) {
     ebook_ctx.tn_size = 500;
     ebook_ctx.log = noop_log;
     ebook_ctx.logf = noop_logf;
+    ebook_ctx.fast_epub_parse = 0;
+    ebook_ctx.tn_qscale = 1.0;
 
     ebook_500_ctx = ebook_ctx;
     ebook_500_ctx.content_size = 500;
 
+    ebook_fast_ctx = ebook_500_ctx;
+    ebook_fast_ctx.fast_epub_parse = 1;
+
     comic_ctx.tn_qscale = 1.0;
     comic_ctx.tn_size = 500;
     comic_ctx.log = noop_log;
@@ -736,7 +1003,7 @@ int main(int argc, char **argv) {
     media_ctx.store = counter_store;
     media_ctx.tn_size = 500;
     media_ctx.tn_qscale = 1.0;
-    media_ctx.max_media_buffer = (long)2000 * (long)1024 * (long)1024;
+    media_ctx.max_media_buffer = (long) 2000 * (long) 1024 * (long) 1024;
 
     ooxml_500_ctx.content_size = 500;
     ooxml_500_ctx.log = noop_log;
@@ -753,6 +1020,18 @@ int main(int argc, char **argv) {
     raw_ctx.tn_size = 500;
     raw_ctx.tn_qscale = 5.0;
 
+    msdoc_ctx.log = noop_log;
+    msdoc_ctx.logf = noop_logf;
+    msdoc_ctx.store = counter_store;
+    msdoc_ctx.content_size = 500;
+    msdoc_ctx.tn_size = 500;
+
+    msdoc_text_ctx.log = noop_log;
+    msdoc_text_ctx.logf = noop_logf;
+    msdoc_text_ctx.store = counter_store;
+    msdoc_text_ctx.content_size = 500;
+    msdoc_text_ctx.tn_size = 0;
+
     av_log_set_level(AV_LOG_QUIET);
     ::testing::InitGoogleTest(&argc, argv);
     return RUN_ALL_TESTS();

test/test_util.cpp

@@ -94,3 +94,19 @@ void destroy_doc(document_t *doc) {
         free(tmp);
     }
 }
+
+void fuzz_buffer(char *buf, size_t *buf_len, int width, int n, int trunc_p) {
+    for (int i = 0; i < n; i++) {
+
+        size_t offset = rand() % (*buf_len - width - 1);
+
+        if (rand() % 100 < trunc_p) {
+            *buf_len = MAX(offset, 1000);
+            continue;
+        }
+
+        for (int disp = 0; disp < width; disp++) {
+            buf[offset + disp] = (int8_t)rand();
+        }
+    }
+}

test/test_util.h

@@ -4,7 +4,6 @@
 #include "../libscan/scan.h"
 #include <fcntl.h>
 #include <unistd.h>
-#include <uuid/uuid.h>
 
 void load_file(const char *filepath, vfile_t *f);
 void load_mem(void *mem, size_t size, vfile_t *f);
@@ -42,4 +41,6 @@ meta_line_t *get_meta_from(meta_line_t *meta, metakey key);
 
 void destroy_doc(document_t *doc);
 
+void fuzz_buffer(char *buf, size_t *buf_len, int width, int n, int trunc_p);
+
 #endif