From 0a9742b6865da7b60b77790b8288ca6d8b17471c Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 16 Jan 2021 12:13:56 -0500 Subject: [PATCH] Workaround for UTF8 .doc files --- libscan/ebook/ebook.c | 51 ++++++++++++++++++++++++------------------- libscan/ebook/ebook.h | 2 +- libscan/msdoc/msdoc.c | 16 +++++--------- test/main.cpp | 28 ++++++++++++++++++++++++ third-party/antiword | 2 +- 5 files changed, 65 insertions(+), 34 deletions(-) diff --git a/libscan/ebook/ebook.c b/libscan/ebook/ebook.c index 63f28a4..098ad7a 100644 --- a/libscan/ebook/ebook.c +++ b/libscan/ebook/ebook.c @@ -255,7 +255,7 @@ void fill_image(fz_context *fzctx, UNUSED(fz_device *dev), } } -void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mime_str, document_t *doc) { +void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mime_str, document_t *doc, int tn_only) { fz_context *fzctx = fz_new_context(NULL, NULL, FZ_STORE_UNLIMITED); thread_ctx = *ctx; @@ -285,26 +285,6 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha return; } - char title[8192] = {'\0',}; - fz_try(fzctx) - fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_TITLE, title, sizeof(title)); - fz_catch(fzctx) - ; - - if (strlen(title) > 0) { - APPEND_UTF8_META(doc, MetaTitle, title) - } - - char author[4096] = {'\0',}; - fz_try(fzctx) - fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_AUTHOR, author, sizeof(author)); - fz_catch(fzctx) - ; - - if (strlen(author) > 0) { - APPEND_UTF8_META(doc, MetaAuthor, author) - } - int page_count = -1; fz_var(err); fz_try(fzctx) @@ -331,6 +311,33 @@ void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const cha } } + if (tn_only) { + fz_drop_stream(fzctx, stream); + fz_drop_document(fzctx, fzdoc); + fz_drop_context(fzctx); + return; + } + + char title[8192] = {'\0',}; + fz_try(fzctx) + fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_TITLE, title, sizeof(title)); + fz_catch(fzctx) + ; + + if (strlen(title) > 0) { + APPEND_UTF8_META(doc, MetaTitle, title) + } + + char author[4096] = {'\0',}; + fz_try(fzctx) + fz_lookup_metadata(fzctx, fzdoc, FZ_META_INFO_AUTHOR, author, sizeof(author)); + fz_catch(fzctx) + ; + + if (strlen(author) > 0) { + APPEND_UTF8_META(doc, MetaAuthor, author) + } + if (ctx->content_size > 0) { fz_stext_options opts = {0}; @@ -425,6 +432,6 @@ void parse_ebook(scan_ebook_ctx_t *ctx, vfile_t *f, const char *mime_str, docume return; } - parse_ebook_mem(ctx, buf, buf_len, mime_str, doc); + parse_ebook_mem(ctx, buf, buf_len, mime_str, doc, FALSE); free(buf); } diff --git a/libscan/ebook/ebook.h b/libscan/ebook/ebook.h index b72e1be..65754c7 100644 --- a/libscan/ebook/ebook.h +++ b/libscan/ebook/ebook.h @@ -16,6 +16,6 @@ typedef struct { } scan_ebook_ctx_t; void parse_ebook(scan_ebook_ctx_t *ctx, vfile_t *f, const char* mime_str, document_t *doc); -void parse_ebook_mem(scan_ebook_ctx_t *ctx, void* buf, size_t buf_len, const char* mime_str, document_t *doc); +void parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mime_str, document_t *doc, int tn_only); #endif diff --git a/libscan/msdoc/msdoc.c b/libscan/msdoc/msdoc.c index e9a2728..e68a5ea 100644 --- a/libscan/msdoc/msdoc.c +++ b/libscan/msdoc/msdoc.c @@ -22,7 +22,6 @@ void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, voi int doc_word_version = iGuessVersionNumber(file_in, buf_len); if (doc_word_version < 0 || doc_word_version == 3) { - fclose(file_in); free(buf); return; } @@ -68,7 +67,6 @@ void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, voi text_buffer_destroy(&tex); } - fclose(file_in); free(buf); free(out_buf); } @@ -84,7 +82,6 @@ void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file, void* b }; // Open word doc - options_type *opts = direct_vGetOptions(); opts->iParagraphBreak = 74; opts->eConversionType = conversion_pdf; @@ -98,7 +95,6 @@ void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file, void* b int doc_word_version = iGuessVersionNumber(file, buf_len); if (doc_word_version < 0 || doc_word_version == 3) { - fclose(file); free(buf); return; } @@ -111,7 +107,6 @@ void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file, void* b diagram_type *diag = pCreateDiagram("antiword", NULL, file_out); if (diag == NULL) { - fclose(file); return; } @@ -120,9 +115,8 @@ void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file, void* b fclose(file_out); - parse_ebook_mem(&ebook_ctx, out_buf, out_len, "application/pdf", doc); + parse_ebook_mem(&ebook_ctx, out_buf, out_len, "application/pdf", doc, TRUE); - fclose(file); free(buf); free(out_buf); } @@ -144,8 +138,10 @@ void parse_msdoc(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc) { } if (ctx->tn_size > 0) { - parse_msdoc_pdf(ctx, doc, file, buf, buf_len); - } else { - parse_msdoc_text(ctx, doc, file, buf, buf_len); + char *buf_pdf = malloc(buf_len); + memcpy(buf_pdf, buf, buf_len); + parse_msdoc_pdf(ctx, doc, file, buf_pdf, buf_len); } + parse_msdoc_text(ctx, doc, file, buf, buf_len); + fclose(file); } diff --git a/test/main.cpp b/test/main.cpp index ed87e8f..8ad0c33 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -786,6 +786,34 @@ TEST(Msdoc, Test4Pdf) { cleanup(&doc, &f); } +TEST(Msdoc, TestUtf8Pdf) { + vfile_t f; + document_t doc; + load_doc_file("libscan-test-files/test_files/msdoc/japanese.doc", &f, &doc); + + size_t size_before = store_size; + + parse_msdoc(&msdoc_ctx, &f, &doc); + + ASSERT_NE(get_meta(&doc, MetaContent), nullptr); + ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "调查项目 A questionnaire") != nullptr); + ASSERT_NE(size_before, store_size); + + cleanup(&doc, &f); +} + +TEST(Msdoc, TestUtf8Text) { + vfile_t f; + document_t doc; + load_doc_file("libscan-test-files/test_files/msdoc/japanese.doc", &f, &doc); + + parse_msdoc(&msdoc_text_ctx, &f, &doc); + + ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "调查项目 A questionnaire") != nullptr); + + cleanup(&doc, &f); +} + TEST(Msdoc, TestFuzz1) { vfile_t f; document_t doc; diff --git a/third-party/antiword b/third-party/antiword index eb8d737..62ae66d 160000 --- a/third-party/antiword +++ b/third-party/antiword @@ -1 +1 @@ -Subproject commit eb8d737eea2866bfb45e50423a1fd6c51454c2f6 +Subproject commit 62ae66db99e9dd88dfa31999f516f71bb8bdc8b2