Read subtitles from media files, fix bug in text_buffer

libscan/arc/arc.c

@@ -5,8 +5,6 @@
 #include <string.h>
 #include <fcntl.h>
 
-#define MAX_SUBFILE_SIZE (long)(10000000000)
-
 
 int should_parse_filtered_file(const char *filepath, int ext) {
     char tmp[PATH_MAX * 2];
@@ -49,19 +47,25 @@ int arc_read(struct vfile *f, void *buf, size_t size) {
     return read;
 }
 
-int arc_open(vfile_t *f, struct archive **a, arc_data_t *arc_data, int allow_recurse) {
+int arc_open(scan_arc_ctx_t *ctx, vfile_t *f, struct archive **a, arc_data_t *arc_data, int allow_recurse) {
     arc_data->f = f;
 
     if (f->is_fs_file) {
         *a = archive_read_new();
         archive_read_support_filter_all(*a);
         archive_read_support_format_all(*a);
+        if (ctx->passphrase[0] != 0) {
+            archive_read_add_passphrase(*a, ctx->passphrase);
+        }
 
        return archive_read_open_filename(*a, f->filepath, ARC_BUF_SIZE);
     } else if (allow_recurse) {
         *a = archive_read_new();
         archive_read_support_filter_all(*a);
         archive_read_support_format_all(*a);
+        if (ctx->passphrase[0] != 0) {
+            archive_read_add_passphrase(*a, ctx->passphrase);
+        }
 
         return archive_read_open(
                 *a, arc_data,
@@ -82,7 +86,7 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc) {
     arc_data_t arc_data;
     arc_data.f = f;
 
-    int ret = arc_open(f, &a, &arc_data, ctx->mode == ARC_MODE_RECURSE);
+    int ret = arc_open(ctx, f, &a, &arc_data, ctx->mode == ARC_MODE_RECURSE);
     if (ret == ARC_SKIPPED) {
         return SCAN_OK;
     }
@@ -110,7 +114,7 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc) {
         meta_line_t *meta_list = malloc(sizeof(meta_line_t) + buf.cur);
         meta_list->key = MetaContent;
         strcpy(meta_list->str_val, buf.buf);
-        APPEND_META(doc, meta_list);
+        APPEND_META(doc, meta_list)
         dyn_buffer_destroy(&buf);
 
     } else {
@@ -125,7 +129,7 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc) {
         sub_job->vfile.is_fs_file = FALSE;
         sub_job->vfile.log = ctx->log;
         sub_job->vfile.logf = ctx->logf;
-        memcpy(sub_job->parent, doc->uuid, sizeof(uuid_t));
+        memcpy(sub_job->parent, doc->path_md5, MD5_DIGEST_LENGTH);
 
         while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
             sub_job->vfile.info = *archive_entry_stat(entry);

libscan/arc/arc.h

@@ -20,6 +20,7 @@ typedef struct {
     log_callback_t log;
     logf_callback_t logf;
     store_callback_t store;
+    char passphrase[4096];
 } scan_arc_ctx_t;
 
 #define ARC_BUF_SIZE 8192
@@ -56,7 +57,7 @@ static int vfile_close_callback(struct archive *a, void *user_data) {
     return ARCHIVE_OK;
 }
 
-int arc_open(vfile_t *f, struct archive **a, arc_data_t *arc_data, int allow_recurse);
+int arc_open(scan_arc_ctx_t *ctx, vfile_t *f, struct archive **a, arc_data_t *arc_data, int allow_recurse);
 
 int should_parse_filtered_file(const char *filepath, int ext);
 

libscan/comic/comic.c

@@ -5,13 +5,14 @@
 #include <stdlib.h>
 #include <archive.h>
 
+static scan_arc_ctx_t arc_ctx = (scan_arc_ctx_t) {.passphrase = {0,}};
 
 void parse_comic(scan_comic_ctx_t *ctx, vfile_t *f, document_t *doc) {
     struct archive *a = NULL;
     struct archive_entry *entry = NULL;
     arc_data_t arc_data;
 
-    int ret = arc_open(f, &a, &arc_data, TRUE);
+    int ret = arc_open(&arc_ctx, f, &a, &arc_data, TRUE);
     if (ret != ARCHIVE_OK) {
         CTX_LOG_ERRORF(f->filepath, "(cbr.c) [%d] %s", ret, archive_error_string(a))
         archive_read_free(a);

libscan/ebook/ebook.c

@@ -157,7 +157,7 @@ int render_cover(scan_ebook_ctx_t *ctx, fz_context *fzctx, document_t *doc, fz_d
     avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
 
     APPEND_TN_META(doc, pixmap->w, pixmap->h)
-    ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) jpeg_packet.data, jpeg_packet.size);
 
     av_packet_unref(&jpeg_packet);
     av_free(*scaled_frame->data);
@@ -174,14 +174,14 @@ void fz_err_callback(void *user, const char *message) {
     document_t *doc = (document_t *) user;
 
     const scan_ebook_ctx_t *ctx = &thread_ctx;
-    CTX_LOG_WARNINGF(doc->filepath, "FZ: %s", message);
+    CTX_LOG_WARNINGF(doc->filepath, "FZ: %s", message)
 }
 
 void fz_warn_callback(void *user, const char *message) {
     document_t *doc = (document_t *) user;
 
     const scan_ebook_ctx_t *ctx = &thread_ctx;
-    CTX_LOG_DEBUGF(doc->filepath, "FZ: %s", message);
+    CTX_LOG_DEBUGF(doc->filepath, "FZ: %s", message)
 }
 
 static void init_fzctx(fz_context *fzctx, document_t *doc) {
@@ -255,7 +255,7 @@ void fill_image(fz_context *fzctx, UNUSED(fz_device *dev),
     }
 }
 
-void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mime_str, document_t *doc) {
+void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mime_str, document_t *doc, int tn_only) {
 
     fz_context *fzctx = fz_new_context(NULL, NULL, FZ_STORE_UNLIMITED);
     thread_ctx = *ctx;
@@ -285,26 +285,6 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha
         return;
     }
 
-    char title[8192] = {'\0',};
-    fz_try(fzctx)
-        fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_TITLE, title, sizeof(title));
-    fz_catch(fzctx)
-        ;
-
-    if (strlen(title) > 0) {
-        APPEND_UTF8_META(doc, MetaTitle, title)
-    }
-
-    char author[4096] = {'\0',};
-    fz_try(fzctx)
-        fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_AUTHOR, author, sizeof(author));
-    fz_catch(fzctx)
-        ;
-
-    if (strlen(author) > 0) {
-        APPEND_UTF8_META(doc, MetaAuthor, author)
-    }
-
     int page_count = -1;
     fz_var(err);
     fz_try(fzctx)
@@ -331,6 +311,33 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha
         }
     }
 
+    if (tn_only) {
+        fz_drop_stream(fzctx, stream);
+        fz_drop_document(fzctx, fzdoc);
+        fz_drop_context(fzctx);
+        return;
+    }
+
+    char title[8192] = {'\0',};
+    fz_try(fzctx)
+                fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_TITLE, title, sizeof(title));
+    fz_catch(fzctx)
+        ;
+
+    if (strlen(title) > 0) {
+        APPEND_UTF8_META(doc, MetaTitle, title)
+    }
+
+    char author[4096] = {'\0',};
+    fz_try(fzctx)
+                fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_AUTHOR, author, sizeof(author));
+    fz_catch(fzctx)
+        ;
+
+    if (strlen(author) > 0) {
+        APPEND_UTF8_META(doc, MetaAuthor, author)
+    }
+
 
     if (ctx->content_size > 0) {
         fz_stext_options opts = {0};
@@ -425,6 +432,6 @@ void parse_ebook(scan_ebook_ctx_t *ctx, vfile_t *f, const char *mime_str, docume
         return;
     }
 
-    parse_ebook_mem(ctx, buf, buf_len, mime_str, doc);
+    parse_ebook_mem(ctx, buf, buf_len, mime_str, doc, FALSE);
     free(buf);
 }

libscan/ebook/ebook.h

@@ -16,6 +16,6 @@ typedef struct {
 } scan_ebook_ctx_t;
 
 void parse_ebook(scan_ebook_ctx_t *ctx, vfile_t *f, const char* mime_str,  document_t *doc);
-void parse_ebook_mem(scan_ebook_ctx_t *ctx, void* buf, size_t buf_len, const char* mime_str,  document_t *doc);
+void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mime_str, document_t *doc, int tn_only);
 
 #endif

libscan/font/font.c

@@ -229,7 +229,7 @@ void parse_font(scan_font_ctx_t *ctx, vfile_t *f, document_t *doc) {
     bmp_format(&bmp_data, dimensions, bitmap);
 
     APPEND_TN_META(doc, dimensions.width, dimensions.height)
-    ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) bmp_data.buf, bmp_data.cur);
+    ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) bmp_data.buf, bmp_data.cur);
 
     dyn_buffer_destroy(&bmp_data);
     free(bitmap);

libscan/media/media.c

@@ -1,8 +1,4 @@
 #include "media.h"
-
-#include "../util.h"
-
-
 #include <ctype.h>
 
 #define MIN_SIZE 32
@@ -89,7 +85,59 @@ static void frame_and_packet_free(frame_and_packet_t *frame_and_packet) {
 }
 
 __always_inline
-static frame_and_packet_t *read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx,
+static void read_subtitles(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, int stream_idx, document_t *doc) {
+
+    text_buffer_t tex = text_buffer_create(-1);
+
+    AVPacket packet;
+    AVSubtitle subtitle;
+
+    AVCodec *subtitle_codec = avcodec_find_decoder(pFormatCtx->streams[stream_idx]->codecpar->codec_id);
+    AVCodecContext *decoder = avcodec_alloc_context3(subtitle_codec);
+    avcodec_parameters_to_context(decoder, pFormatCtx->streams[stream_idx]->codecpar);
+    avcodec_open2(decoder, subtitle_codec, NULL);
+
+    decoder->sub_text_format = FF_SUB_TEXT_FMT_ASS;
+
+    int got_sub;
+
+    while (1) {
+        int read_frame_ret = av_read_frame(pFormatCtx, &packet);
+
+        if (read_frame_ret != 0) {
+            break;
+        }
+
+        if (packet.stream_index != stream_idx) {
+            av_packet_unref(&packet);
+            continue;
+        }
+
+        avcodec_decode_subtitle2(decoder, &subtitle, &got_sub, &packet);
+
+        if (got_sub) {
+            for (int i = 0; i < subtitle.num_rects; i++) {
+                const char *text = subtitle.rects[i]->ass;
+
+                char *idx = strstr(text, "\\N");
+                if (idx != NULL && strlen(idx + 2) > 1) {
+                    text_buffer_append_string0(&tex, idx + 2);
+                    text_buffer_append_char(&tex, ' ');
+                }
+            }
+            avsubtitle_free(&subtitle);
+        }
+    }
+
+    text_buffer_terminate_string(&tex);
+
+    APPEND_STR_META(doc, MetaContent, tex.dyn_buffer.buf)
+    text_buffer_destroy(&tex);
+}
+
+__always_inline
+static frame_and_packet_t *
+read_frame(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx,
            document_t *doc) {
 
     frame_and_packet_t *result = calloc(1, sizeof(frame_and_packet_t));
@@ -265,6 +313,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
     int video_stream = -1;
     int audio_stream = -1;
+    int subtitle_stream = -1;
 
     avformat_find_stream_info(pFormatCtx, NULL);
 
@@ -303,6 +352,17 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
                 video_stream = i;
             }
+        } else if (stream->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) {
+            subtitle_stream = i;
+        }
+    }
+
+    if (subtitle_stream != -1 && ctx->read_subtitles) {
+        read_subtitles(ctx, pFormatCtx, subtitle_stream, doc);
+
+        // Reset stream
+        if (video_stream != -1) {
+            av_seek_frame(pFormatCtx, video_stream, 0, 0);
         }
     }
 
@@ -323,7 +383,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
         //Seek
         if (stream->nb_frames > 1 && stream->codecpar->codec_id != AV_CODEC_ID_GIF) {
-            int seek_ret = 0;
+            int seek_ret;
             for (int i = 20; i >= 0; i--) {
                 seek_ret = av_seek_frame(pFormatCtx, video_stream,
                                          stream->duration * 0.10, 0);
@@ -356,7 +416,8 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
         if (scaled_frame == STORE_AS_IS) {
             APPEND_TN_META(doc, frame_and_packet->frame->width, frame_and_packet->frame->height)
-            ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) frame_and_packet->packet->data, frame_and_packet->packet->size);
+            ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) frame_and_packet->packet->data,
+                       frame_and_packet->packet->size);
         } else {
             // Encode frame to jpeg
             AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
@@ -369,7 +430,7 @@ void parse_media_format_ctx(scan_media_ctx_t *ctx, AVFormatContext *pFormatCtx,
 
             // Save thumbnail
             APPEND_TN_META(doc, scaled_frame->width, scaled_frame->height)
-            ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) jpeg_packet.data, jpeg_packet.size);
+            ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) jpeg_packet.data, jpeg_packet.size);
 
             avcodec_free_context(&jpeg_encoder);
             av_packet_unref(&jpeg_packet);
@@ -608,7 +669,8 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void* buf, size_t buf_len, docu
 
     if (scaled_frame == STORE_AS_IS) {
         APPEND_TN_META(doc, frame_and_packet->frame->width, frame_and_packet->frame->height)
-        ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) frame_and_packet->packet->data, frame_and_packet->packet->size);
+        ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) frame_and_packet->packet->data,
+                   frame_and_packet->packet->size);
     } else {
         // Encode frame to jpeg
         AVCodecContext *jpeg_encoder = alloc_jpeg_encoder(scaled_frame->width, scaled_frame->height,
@@ -621,7 +683,7 @@ int store_image_thumbnail(scan_media_ctx_t *ctx, void* buf, size_t buf_len, docu
 
         // Save thumbnail
         APPEND_TN_META(doc, scaled_frame->width, scaled_frame->height)
-        ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) jpeg_packet.data, jpeg_packet.size);
+        ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) jpeg_packet.data, jpeg_packet.size);
 
         av_packet_unref(&jpeg_packet);
         avcodec_free_context(&jpeg_encoder);

libscan/msdoc/msdoc.c

@@ -6,7 +6,7 @@
 
 #include "../ebook/ebook.h"
 
-void parse_msdoc_text(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc, FILE *file_in, void* buf, size_t buf_len) {
+void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, void* buf, size_t buf_len) {
 
     // Open word doc
     options_type *opts = direct_vGetOptions();
@@ -22,7 +22,6 @@ void parse_msdoc_text(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc, FILE *
 
     int doc_word_version = iGuessVersionNumber(file_in, buf_len);
     if (doc_word_version < 0 || doc_word_version == 3) {
-        fclose(file_in);
         free(buf);
         return;
     }
@@ -68,12 +67,11 @@ void parse_msdoc_text(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc, FILE *
         text_buffer_destroy(&tex);
     }
 
-    fclose(file_in);
     free(buf);
     free(out_buf);
 }
 
-void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc, FILE *file, void* buf, size_t buf_len) {
+void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file, void* buf, size_t buf_len) {
 
     scan_ebook_ctx_t ebook_ctx = {
             .content_size = ctx->content_size,
@@ -84,21 +82,19 @@ void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc, FILE *f
     };
 
     // Open word doc
-
     options_type *opts = direct_vGetOptions();
     opts->iParagraphBreak = 74;
     opts->eConversionType = conversion_pdf;
     opts->bHideHiddenText = 1;
     opts->bRemoveRemovedText = 1;
     opts->bUseLandscape = 0;
-    opts->eEncoding = encoding_latin_2;
+    opts->eEncoding = encoding_latin_1;
     opts->iPageHeight = 842; // A4
     opts->iPageWidth = 595;
     opts->eImageLevel = level_ps_3;
 
     int doc_word_version = iGuessVersionNumber(file, buf_len);
     if (doc_word_version < 0 || doc_word_version == 3) {
-        fclose(file);
         free(buf);
         return;
     }
@@ -111,18 +107,16 @@ void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc, FILE *f
 
     diagram_type *diag = pCreateDiagram("antiword", NULL, file_out);
     if (diag == NULL) {
-        fclose(file);
         return;
     }
 
-    int ret = bWordDecryptor(file, buf_len, diag);
+    bWordDecryptor(file, buf_len, diag);
     vDestroyDiagram(diag);
 
     fclose(file_out);
 
-    parse_ebook_mem(&ebook_ctx, out_buf, out_len, "application/pdf", doc);
+    parse_ebook_mem(&ebook_ctx, out_buf, out_len, "application/pdf", doc, TRUE);
 
-    fclose(file);
     free(buf);
     free(out_buf);
 }
@@ -144,8 +138,10 @@ void parse_msdoc(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc) {
     }
 
     if (ctx->tn_size > 0) {
-        parse_msdoc_pdf(ctx, f, doc, file, buf, buf_len);
+        char *buf_pdf = malloc(buf_len);
-    } else {
+        memcpy(buf_pdf, buf, buf_len);
-        parse_msdoc_text(ctx, f, doc, file, buf, buf_len);
+        parse_msdoc_pdf(ctx, doc, file, buf_pdf, buf_len);
     }
+    parse_msdoc_text(ctx, doc, file, buf, buf_len);
+    fclose(file);
 }

libscan/msdoc/msdoc.h

@@ -19,6 +19,6 @@ static int is_msdoc(scan_msdoc_ctx_t *ctx, unsigned int mime) {
 
 void parse_msdoc(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc);
 
-void parse_msdoc_text(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc, FILE *file_in, void* buf, size_t buf_len);
+void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, void* buf, size_t buf_len);
 
 #endif

libscan/ooxml/ooxml.c

@@ -1,6 +1,5 @@
 #include "ooxml.h"
 
-#include "../util.h"
 #include <archive.h>
 #include <archive_entry.h>
 #include <libxml/xmlstring.h>
@@ -106,7 +105,7 @@ static int read_part(scan_ooxml_ctx_t *ctx, struct archive *a, text_buffer_t *bu
 }
 
 __always_inline
-static int read_doc_props(scan_ooxml_ctx_t *ctx, struct archive *a, text_buffer_t *buf, document_t *doc) {
+static int read_doc_props(scan_ooxml_ctx_t *ctx, struct archive *a, document_t *doc) {
     xmlDoc *xml = xmlReadIO(xml_io_read, xml_io_close, a, "/", NULL,
                             XML_PARSE_RECOVER | XML_PARSE_NOWARNING | XML_PARSE_NOERROR | XML_PARSE_NONET);
 
@@ -158,7 +157,7 @@ void read_thumbnail(scan_ooxml_ctx_t *ctx, document_t *doc, struct archive *a, s
     archive_read_data(a, buf, entry_size);
 
     APPEND_TN_META(doc, 1, 1) // Size unknown
-    ctx->store((char *) doc->uuid, sizeof(doc->uuid), buf, entry_size);
+    ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), buf, entry_size);
     free(buf);
 }
 
@@ -198,7 +197,7 @@ void parse_ooxml(scan_ooxml_ctx_t *ctx, vfile_t *f, document_t *doc) {
                     buffer_full = TRUE;
                 }
             } else if (strcmp(path, "docProps/core.xml") == 0) {
-                if (read_doc_props(ctx, a, &tex, doc) != 0) {
+                if (read_doc_props(ctx, a, doc) != 0) {
                     break;
                 }
             } else if (strcmp(path, "docProps/thumbnail.jpeg") == 0) {

libscan/raw/raw.c

@@ -2,7 +2,6 @@
 #include <libraw/libraw.h>
 
 #include "../media/media.h"
-#include <fcntl.h>
 #include <unistd.h>
 
 
@@ -71,7 +70,7 @@ int store_thumbnail_rgb24(scan_raw_ctx_t *ctx, libraw_processed_image_t *img, do
     avcodec_receive_packet(jpeg_encoder, &jpeg_packet);
 
     APPEND_TN_META(doc, scaled_frame->width, scaled_frame->height)
-    ctx->store((char *) doc->uuid, sizeof(doc->uuid), (char *) jpeg_packet.data, jpeg_packet.size);
+    ctx->store((char *) doc->path_md5, sizeof(doc->path_md5), (char *) jpeg_packet.data, jpeg_packet.size);
 
     av_packet_unref(&jpeg_packet);
     av_free(*scaled_frame->data);

libscan/scan.h

@@ -7,7 +7,7 @@
 
 #include <stdio.h>
 #include <sys/stat.h>
-#include <uuid/uuid.h>
+#include <openssl/md5.h>
 
 #include "macros.h"
 
@@ -25,7 +25,7 @@
 
 #define IS_META_INT(key) (key & META_INT_MASK) == META_INT_MASK
 #define IS_META_LONG(key) (key & META_LONG_MASK) == META_LONG_MASK
-#define IS_META_STR(meta) (meta->key & META_STR_MASK) == META_STR_MASK
+#define IS_META_STR(key) (key & META_STR_MASK) == META_STR_MASK
 
 typedef void (*store_callback_t)(char *key, size_t key_len, char *buf, size_t buf_len);
 typedef void (*logf_callback_t)(const char *filepath, int level, char *format, ...);
@@ -99,13 +99,13 @@ typedef struct meta_line {
 
 
 typedef struct document {
-    unsigned char uuid[16];
+    unsigned char path_md5[MD5_DIGEST_LENGTH];
-    unsigned long ino;
     unsigned long size;
     unsigned int mime;
     int mtime;
     short base;
     short ext;
+    char has_parent;
     meta_line_t *meta_head;
     meta_line_t *meta_tail;
     char *filepath;
@@ -146,7 +146,7 @@ typedef struct parse_job_t {
     int base;
     int ext;
     struct vfile vfile;
-    uuid_t parent;
+    unsigned char parent[MD5_DIGEST_LENGTH];
     char filepath[1];
 } parse_job_t;
 

test/main.cpp

@@ -595,6 +595,23 @@ TEST(Arc, Utf8) {
     cleanup(&doc, &f);
 }
 
+TEST(Arc, EncryptedZip) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/arc/encrypted.zip", &f, &doc);
+
+    size_t size_before = store_size;
+
+    strcpy(arc_recurse_media_ctx.passphrase, "sist2");
+    parse_archive(&arc_recurse_media_ctx, &f, &doc);
+
+    arc_recurse_media_ctx.passphrase[0] = '\0';
+
+    ASSERT_NE(size_before, store_size);
+
+    cleanup(&doc, &f);
+}
+
 /* RAW */
 TEST(RAW, Panasonic) {
     vfile_t f;
@@ -786,6 +803,34 @@ TEST(Msdoc, Test4Pdf) {
     cleanup(&doc, &f);
 }
 
+TEST(Msdoc, TestUtf8Pdf) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/japanese.doc", &f, &doc);
+
+    size_t size_before = store_size;
+
+    parse_msdoc(&msdoc_ctx, &f, &doc);
+
+    ASSERT_NE(get_meta(&doc, MetaContent), nullptr);
+    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "调查项目 A questionnaire") != nullptr);
+    ASSERT_NE(size_before, store_size);
+
+    cleanup(&doc, &f);
+}
+
+TEST(Msdoc, TestUtf8Text) {
+    vfile_t f;
+    document_t doc;
+    load_doc_file("libscan-test-files/test_files/msdoc/japanese.doc", &f, &doc);
+
+    parse_msdoc(&msdoc_text_ctx, &f, &doc);
+
+    ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "调查项目 A questionnaire") != nullptr);
+
+    cleanup(&doc, &f);
+}
+
 TEST(Msdoc, TestFuzz1) {
     vfile_t f;
     document_t doc;
@@ -801,7 +846,7 @@ TEST(Msdoc, TestFuzz1) {
 
         fuzz_buffer(buf_copy, &buf_len_copy, 3, 8, 5);
         FILE *file = fmemopen(buf_copy, buf_len_copy, "rb");
-        parse_msdoc_text(&msdoc_text_ctx, &f, &doc, file, buf_copy, buf_len_copy);
+        parse_msdoc_text(&msdoc_text_ctx, &doc, file, buf_copy, buf_len_copy);
     }
     free(buf);
     cleanup(&doc, &f);

test/test_util.h

@@ -4,7 +4,6 @@
 #include "../libscan/scan.h"
 #include <fcntl.h>
 #include <unistd.h>
-#include <uuid/uuid.h>
 
 void load_file(const char *filepath, vfile_t *f);
 void load_mem(void *mem, size_t size, vfile_t *f);