Merge branch 'simon987:master' into UI-localization-german

.gitignore

@@ -33,3 +33,11 @@ state.db
 *.pyc
 !sist2-admin/frontend/dist
 *.js.map
+sist2-vue/dist
+sist2-admin/frontend/dist
+.ninja_deps
+.ninja_log
+build.ninja
+src/web/static_generated.c
+src/magic_generated.c
+src/index/static_generated.c

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "third-party/libscan/third-party/libmobi"]
 	path = third-party/libscan/third-party/libmobi
 	url = https://github.com/bfabiszewski/libmobi
+[submodule "third-party/libscan/libscan-test-files"]
+	path = third-party/libscan/libscan-test-files
+	url = https://github.com/simon987/libscan-test-files

Dockerfile

@@ -1,6 +1,11 @@
 FROM simon987/sist2-build as build
 MAINTAINER simon987 <me@simon987.net>
 
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash
+RUN apt update -y; apt install -y nodejs && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /build/
 
 COPY scripts scripts
@@ -9,6 +14,10 @@ COPY CMakeLists.txt .
 COPY third-party third-party
 COPY src src
 COPY sist2-vue sist2-vue
+COPY sist2-admin sist2-admin
+
+RUN cd sist2-vue/ && npm install && npm run build
+RUN cd sist2-admin/frontend/ && npm install && npm run build
 
 RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
 RUN make -j$(nproc)
@@ -45,4 +54,4 @@ COPY --from=build /build/sist2 /root/sist2
 # sist2-admin
 COPY sist2-admin/requirements.txt sist2-admin/
 RUN python3 -m pip install --no-cache -r sist2-admin/requirements.txt
-COPY sist2-admin/ sist2-admin/
+COPY --from=build /build/sist2-admin/ sist2-admin/

README.md

@@ -81,7 +81,7 @@ See [Usage guide](docs/USAGE.md) for more details
 | html, xml                                                                 | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | no          | -                                                                                                                                      |
 | tar, zip, rar, 7z, ar ...                                                 | Libarchive                                                                   | yes\*    | -           | no                                                                                                                                     |
 | docx, xlsx, pptx                                                          | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | if embedded | creator, modified_by, title                                                                                                            |
-| doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | yes         | author, title                                                                                                                          |
+| doc (MS Word 97-2003)                                                     | antiword                                                                     | yes      | no          | author, title                                                                                                                          |
 | mobi, azw, azw3                                                           | libmobi                                                                      | yes      | no          | author, title                                                                                                                          |
 | wpd (WordPerfect)                                                         | libwpd                                                                       | yes      | no          | *planned*                                                                                                                              |
 | json, jsonl, ndjson                                                       | [libscan](https://github.com/simon987/sist2/tree/master/third-party/libscan) | yes      | -           | -                                                                                                                                      |
@@ -141,7 +141,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 1. Install compile-time dependencies
 
    ```bash
-   apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git
+   apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git nodejs
    ```
 
 1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile
@@ -156,6 +156,8 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 1. Build
     ```bash
     git clone --recursive https://github.com/simon987/sist2/
+    (cd sist2-vue; npm install; npm run build)
+    (cd sist2-admin/frontend; npm install; npm run build)
     cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=<VCPKG_ROOT>/scripts/buildsystems/vcpkg.cmake .
     make
     ```

sist2-admin/frontend/dist/css/app.css

@@ -1 +0,0 @@
-.navbar[data-v-27bc1d68]{box-shadow:0 .125rem .25rem rgba(0,0,0,.08)!important;border-radius:0}.theme-black .navbar[data-v-27bc1d68]{background:rgba(84,107,122,.18823529411764706);border-bottom:none}.navbar-brand[data-v-27bc1d68]{color:#222!important;font-size:1.75rem;padding:0}.navbar-brand[data-v-27bc1d68]:hover{color:#000!important}.version[data-v-27bc1d68]{color:#222!important;margin-left:-18px;margin-top:-14px;font-size:11px;font-family:monospace}.btn-link[data-v-27bc1d68]{color:#222}body,html{height:100%}#app{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;color:#2c3e50;padding-bottom:1em;min-height:100%}.info-icon{width:1rem;margin-right:.2rem;cursor:pointer;line-height:1rem;height:1rem;background-image:url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjYuNjY3IDQyNi42NjciIGZpbGw9IiNmZmYiPjxwYXRoIGQ9Ik0xOTIgMTkyaDQyLjY2N3YxMjhIMTkyeiIvPjxwYXRoIGQ9Ik0yMTMuMzMzIDBDOTUuNDY3IDAgMCA5NS40NjcgMCAyMTMuMzMzczk1LjQ2NyAyMTMuMzMzIDIxMy4zMzMgMjEzLjMzM1M0MjYuNjY3IDMzMS4yIDQyNi42NjcgMjEzLjMzMyAzMzEuMiAwIDIxMy4zMzMgMHptMCAzODRjLTk0LjA4IDAtMTcwLjY2Ny03Ni41ODctMTcwLjY2Ny0xNzAuNjY3UzExOS4yNTMgNDIuNjY3IDIxMy4zMzMgNDIuNjY3IDM4NCAxMTkuMjUzIDM4NCAyMTMuMzMzIDMwNy40MTMgMzg0IDIxMy4zMzMgMzg0eiIvPjxwYXRoIGQ9Ik0xOTIgMTA2LjY2N2g0Mi42Njd2NDIuNjY3SDE5MnoiLz48L3N2Zz4=);filter:brightness(45%);display:block}.tabs{margin-top:10px}.modal-title{text-overflow:ellipsis;overflow:hidden;white-space:nowrap}@media screen and (min-width:1500px){.container{max-width:1440px}}label{margin-top:.5rem;margin-bottom:0}.shrink[data-v-9b017c42]{flex-grow:inherit}#task-history[data-v-46960281]{font-family:monospace;font-size:12px}#log-tail-output span{display:block}span.DEBUG{color:#9e9e9e}span.WARNING{color:#ffb300}span.INFO{color:#039be5}span.ERROR,span.FATAL{color:#f4511e}span.ADMIN{color:#ee05ff}#log-tail-output{font-size:13px;font-family:monospace;padding:6px;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px;margin:3px;white-space:pre;color:#000;overflow:hidden}

sist2-admin/frontend/dist/index.html

@@ -1 +0,0 @@
-<!DOCTYPE html><html lang=""><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="favicon.ico"><title>sist2-admin</title><link href="css/app.css" rel="preload" as="style"><link href="css/chunk-vendors.css" rel="preload" as="style"><link href="js/app.js" rel="preload" as="script"><link href="js/chunk-vendors.js" rel="preload" as="script"><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/app.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but sist2-admin-vue doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="js/chunk-vendors.js"></script><script src="js/app.js"></script></body></html>

sist2-admin/frontend/package.json

@@ -20,14 +20,11 @@
   },
   "devDependencies": {
     "@vue/cli-plugin-babel": "~5.0.8",
-    "@vue/cli-plugin-eslint": "~5.0.8",
     "@vue/cli-plugin-router": "~5.0.8",
     "@vue/cli-plugin-vuex": "~5.0.8",
     "@vue/cli-service": "~5.0.8",
     "babel-eslint": "^10.1.0",
     "bootstrap": "^4.5.2",
-    "eslint": "^6.7.2",
-    "eslint-plugin-vue": "^6.2.2",
     "vue-template-compiler": "^2.6.11"
   },
   "eslintConfig": {

sist2-vue/dist/index.html

@@ -1,3 +0,0 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"/><title>sist2</title><script defer="defer" src="js/chunk-vendors.js"></script><script defer="defer" src="js/index.js"></script><link href="css/chunk-vendors.css" rel="stylesheet"><link href="css/index.css" rel="stylesheet"></head><body><noscript><style>body {
-            height: initial;
-        }</style><div style="text-align: center; margin-top: 100px"><strong>We're sorry but sist2 doesn't work properly without JavaScript enabled. Please enable it to continue.</strong><br/><strong>Nous sommes désolés mais sist2 ne fonctionne pas correctement si JavaScript est activé. Veuillez l'activer pour continuer.</strong></div></noscript><div id="app"></div></body></html>

sist2-vue/package-lock.json

@@ -16,7 +16,7 @@
         "d3": "^5.6.1",
         "date-fns": "^2.21.3",
         "dom-to-image": "^2.6.0",
-        "fslightbox-vue": "file:../../../../mnt/Hatchery/projects/sist2/fslightbox-vue-pro-1.3.1.tgz",
+        "fslightbox-vue": "fslightbox-vue.tgz",
         "nouislider": "^15.2.0",
         "underscore": "^1.13.1",
         "vue": "^2.6.12",
@@ -6581,7 +6581,7 @@
     },
     "node_modules/fslightbox-vue": {
       "version": "1.3.1",
-      "resolved": "file:../../../Hatchery/projects/sist2/fslightbox-vue-pro-1.3.1.tgz",
+      "resolved": "file:fslightbox-vue.tgz",
       "integrity": "sha512-dK+X5hH5hpohfqcmBvGmqGvkkoQhjNqWGb7CZAS1Dz6aQ6y7SgdepVf6xiUPveemzpzoeAGO4KoJ+UBAZDRLwQ==",
       "license": "MIT",
       "peerDependencies": {
@@ -16602,7 +16602,7 @@
       "optional": true
     },
     "fslightbox-vue": {
-      "version": "file:../../../Hatchery/projects/sist2/fslightbox-vue-pro-1.3.1.tgz",
+      "version": "file:fslightbox-vue.tgz",
       "integrity": "sha512-dK+X5hH5hpohfqcmBvGmqGvkkoQhjNqWGb7CZAS1Dz6aQ6y7SgdepVf6xiUPveemzpzoeAGO4KoJ+UBAZDRLwQ==",
       "requires": {}
     },

sist2-vue/package.json

@@ -15,7 +15,7 @@
     "d3": "^5.6.1",
     "date-fns": "^2.21.3",
     "dom-to-image": "^2.6.0",
-    "fslightbox-vue": "file:../../../../mnt/Hatchery/projects/sist2/fslightbox-vue-pro-1.3.1.tgz",
+    "fslightbox-vue": "fslightbox-vue.tgz",
     "nouislider": "^15.2.0",
     "underscore": "^1.13.1",
     "vue": "^2.6.12",

src/main.c

@@ -261,8 +261,6 @@ void initialize_scan_context(scan_args_t *args) {
     ScanCtx.text_ctx.logf = _logf;
 
     // MSDOC
-    ScanCtx.msdoc_ctx.enable_tn = args->tn_count > 0;
-    ScanCtx.msdoc_ctx.tn_size = args->tn_size;
     ScanCtx.msdoc_ctx.content_size = args->content_size;
     ScanCtx.msdoc_ctx.log = _log;
     ScanCtx.msdoc_ctx.logf = _logf;

third-party/libscan/libscan-test-files

@@ -0,0 +1 @@
+Subproject commit cdf1f89423424b2091520bfe8c580d682fe01f7d

third-party/libscan/libscan/msdoc/msdoc.c

@@ -4,8 +4,6 @@
 #include <sys/mman.h>
 #include "../../third-party/antiword/src/antiword.h"
 
-#include "../ebook/ebook.h"
-
 void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, void *buf, size_t buf_len) {
 
     // Open word doc
@@ -71,57 +69,6 @@ void parse_msdoc_text(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file_in, voi
     free(out_buf);
 }
 
-void parse_msdoc_pdf(scan_msdoc_ctx_t *ctx, document_t *doc, FILE *file, void *buf, size_t buf_len) {
-
-    scan_ebook_ctx_t ebook_ctx = {
-            .content_size = ctx->content_size,
-            .tn_size = ctx->tn_size,
-            .enable_tn = TRUE,
-            .log = ctx->log,
-            .logf = ctx->logf,
-            .store = ctx->store,
-    };
-
-    // Open word doc
-    options_type *opts = direct_vGetOptions();
-    opts->iParagraphBreak = 74;
-    opts->eConversionType = conversion_pdf;
-    opts->bHideHiddenText = 1;
-    opts->bRemoveRemovedText = 1;
-    opts->bUseLandscape = 0;
-    opts->eEncoding = encoding_latin_1;
-    opts->iPageHeight = 842; // A4
-    opts->iPageWidth = 595;
-    opts->eImageLevel = level_ps_3;
-
-    int doc_word_version = iGuessVersionNumber(file, (int) buf_len);
-    if (doc_word_version < 0 || doc_word_version == 3) {
-        free(buf);
-        return;
-    }
-    rewind(file);
-
-    size_t out_len;
-    char *out_buf;
-
-    FILE *file_out = open_memstream(&out_buf, &out_len);
-
-    diagram_type *diag = pCreateDiagram("antiword", NULL, file_out);
-    if (diag == NULL) {
-        return;
-    }
-
-    bWordDecryptor(file, (int) buf_len, diag);
-    vDestroyDiagram(diag);
-
-    fclose(file_out);
-
-    parse_ebook_mem(&ebook_ctx, out_buf, out_len, "application/pdf", doc, TRUE);
-
-    free(buf);
-    free(out_buf);
-}
-
 void parse_msdoc(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc) {
 
     size_t buf_len;
@@ -138,11 +85,6 @@ void parse_msdoc(scan_msdoc_ctx_t *ctx, vfile_t *f, document_t *doc) {
         return;
     }
 
-    if (ctx->enable_tn) {
-        char *buf_pdf = malloc(buf_len);
-        memcpy(buf_pdf, buf, buf_len);
-        parse_msdoc_pdf(ctx, doc, file, buf_pdf, buf_len);
-    }
     parse_msdoc_text(ctx, doc, file, buf, buf_len);
     fclose(file);
 }

third-party/libscan/libscan/msdoc/msdoc.h

@@ -5,8 +5,6 @@
 
 typedef struct {
     long content_size;
-    int enable_tn;
-    int tn_size;
     log_callback_t log;
     logf_callback_t logf;
     store_callback_t store;

third-party/libscan/test/main.cpp

@@ -916,15 +916,12 @@ TEST(Msdoc, Test1Pdf) {
     document_t doc;
     load_doc_file("libscan-test-files/test_files/msdoc/test1.doc", &f, &doc);
 
-    size_t size_before = store_size;
-
     parse_msdoc(&msdoc_ctx, &f, &doc);
 
     ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "October 2000") != nullptr);
     ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "INTERNATIONAL ORGANIZATION FOR STANDARDIZATION");
     ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Oliver Morgan");
     ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
-    ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
 }
@@ -934,15 +931,12 @@ TEST(Msdoc, Test1Text) {
     document_t doc;
     load_doc_file("libscan-test-files/test_files/msdoc/test1.doc", &f, &doc);
 
-    size_t size_before = store_size;
-
     parse_msdoc(&msdoc_text_ctx, &f, &doc);
 
     ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "October 2000") != nullptr);
     ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "INTERNATIONAL ORGANIZATION FOR STANDARDIZATION");
     ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Oliver Morgan");
     ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
-    ASSERT_EQ(size_before, store_size);
 
     cleanup(&doc, &f);
 }
@@ -952,15 +946,12 @@ TEST(Msdoc, Test2Pdf) {
     document_t doc;
     load_doc_file("libscan-test-files/test_files/msdoc/test2.doc", &f, &doc);
 
-    size_t size_before = store_size;
-
     parse_msdoc(&msdoc_ctx, &f, &doc);
 
     ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "GNU Free Documentation License") != nullptr);
     ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "DWARF Debugging Information Format");
     ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Ron Brender");
     ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
-    ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
 }
@@ -970,15 +961,12 @@ TEST(Msdoc, Test3Pdf) {
     document_t doc;
     load_doc_file("libscan-test-files/test_files/msdoc/test3.doc", &f, &doc);
 
-    size_t size_before = store_size;
-
     parse_msdoc(&msdoc_ctx, &f, &doc);
 
     ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "INTERNATIONAL PATENT CLASSIFICATION") != nullptr);
     ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "IPC Fixed Texts Specification");
     ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Fievet");
     ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
-    ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
 }
@@ -988,15 +976,12 @@ TEST(Msdoc, Test4Pdf) {
     document_t doc;
     load_doc_file("libscan-test-files/test_files/msdoc/test4.doc", &f, &doc);
 
-    size_t size_before = store_size;
-
     parse_msdoc(&msdoc_ctx, &f, &doc);
 
     ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "SQL Server international data types") != nullptr);
     ASSERT_STREQ(get_meta(&doc, MetaTitle)->str_val, "MSDN Authoring Template");
     ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "Brenda Yen");
     ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
-    ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
 }
@@ -1012,7 +997,6 @@ TEST(Msdoc, TestUtf8Pdf) {
 
     ASSERT_NE(get_meta(&doc, MetaContent), nullptr);
     ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "调查项目 A questionnaire") != nullptr);
-    ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
 }
@@ -1034,14 +1018,11 @@ TEST(Msdoc, Test5Pdf) {
     document_t doc;
     load_doc_file("libscan-test-files/test_files/msdoc/test5.doc", &f, &doc);
 
-    size_t size_before = store_size;
-
     parse_msdoc(&msdoc_ctx, &f, &doc);
 
     ASSERT_TRUE(strstr(get_meta(&doc, MetaContent)->str_val, "орган Федеральной") != nullptr);
     ASSERT_STREQ(get_meta(&doc, MetaAuthor)->str_val, "uswo");
     ASSERT_NEAR(strlen(get_meta(&doc, MetaContent)->str_val), msdoc_ctx.content_size, 4);
-    ASSERT_NE(size_before, store_size);
 
     cleanup(&doc, &f);
 }
@@ -1184,15 +1165,11 @@ int main(int argc, char **argv) {
     msdoc_ctx.logf = noop_logf;
     msdoc_ctx.store = counter_store;
     msdoc_ctx.content_size = 500;
-    msdoc_ctx.tn_size = 500;
-    msdoc_ctx.enable_tn = TRUE;
 
     msdoc_text_ctx.log = noop_log;
     msdoc_text_ctx.logf = noop_logf;
     msdoc_text_ctx.store = counter_store;
     msdoc_text_ctx.content_size = 500;
-    msdoc_text_ctx.tn_size = 0;
-    msdoc_text_ctx.enable_tn = FALSE;
 
     wpd_ctx.log = noop_log;
     wpd_ctx.logf = noop_logf;