OCR support, remove static build

src/cli.c

@@ -1,6 +1,8 @@
 #include "cli.h"
 #include "ctx.h"
 
+#include <tesseract/capi.h>
+
 #define DEFAULT_OUTPUT "index.sist2/"
 #define DEFAULT_CONTENT_SIZE 32768
 #define DEFAULT_QUALITY 5
@@ -35,8 +37,6 @@ void scan_args_destroy(scan_args_t *args) {
     free(args);
 }
 
-#ifndef SIST_SCAN_ONLY
-
 void index_args_destroy(index_args_t *args) {
     //todo
     free(args);
@@ -47,8 +47,6 @@ void web_args_destroy(web_args_t *args) {
     free(args);
 }
 
-#endif
-
 int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     if (argc < 2) {
         fprintf(stderr, "Required positional argument: PATH.\n");
@@ -136,6 +134,17 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
         return 1;
     }
 
+    if (args->tesseract_lang != NULL) {
+        TessBaseAPI *api = TessBaseAPICreate();
+        ret = TessBaseAPIInit3(api, TESS_DATAPATH, args->tesseract_lang);
+        if (ret != 0) {
+            fprintf(stderr, "Could not initialize tesseract with lang '%s'\n", args->tesseract_lang);
+            return 1;
+        }
+        TessBaseAPIEnd(api);
+        TessBaseAPIDelete(api);
+    }
+
     LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
     LOG_DEBUGF("cli.c", "arg size=%d", args->size)
     LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
@@ -147,12 +156,11 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
     LOG_DEBUGF("cli.c", "arg path=%s", args->path)
     LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
+    LOG_DEBUGF("cli.c", "arg ocr=%s", args->tesseract_lang)
 
     return 0;
 }
 
-#ifndef SIST_SCAN_ONLY
-
 int index_args_validate(index_args_t *args, int argc, const char **argv) {
 
     if (argc < 2) {
@@ -273,5 +281,3 @@ web_args_t *web_args_create() {
     return args;
 }
 
-#endif
-

src/cli.h

@@ -16,13 +16,13 @@ typedef struct scan_args {
     char *path;
     char *archive;
     archive_mode_t archive_mode;
+    char *tesseract_lang;
 } scan_args_t;
 
 scan_args_t *scan_args_create();
 void scan_args_destroy(scan_args_t *args);
 int scan_args_validate(scan_args_t *args, int argc, const char **argv);
 
-#ifndef SIST_SCAN_ONLY
 typedef struct index_args {
     char *es_url;
     const char *index_path;
@@ -51,6 +51,5 @@ void web_args_destroy(web_args_t *args);
 
 int index_args_validate(index_args_t *args, int argc, const char **argv);
 int web_args_validate(web_args_t *args, int argc, const char **argv);
-#endif
 
 #endif

src/ctx.h

@@ -27,6 +27,7 @@ struct {
     GHashTable *copy_table;
 
     pthread_mutex_t mupdf_mu;
+    char * tesseract_lang;
 } ScanCtx;
 
 struct {
@@ -35,7 +36,6 @@ struct {
     int no_color;
 } LogCtx;
 
-#ifndef SIST_SCAN_ONLY
 struct {
     char *es_url;
     int batch_size;
@@ -47,7 +47,6 @@ struct {
     char *b64credentials;
     struct index_t indices[16];
 } WebCtx;
-#endif
 
 
 #endif

src/main.c

@@ -1,16 +1,12 @@
 #include "sist.h"
 #include "ctx.h"
 
-#ifndef SIST_SCAN_ONLY
 #define DESCRIPTION "Lightning-fast file system indexer and search tool."
-#else
-#define DESCRIPTION "Lightning-fast file system indexer and search tool. (SCAN ONLY)"
-#endif
 
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "1.1.15";
+static const char *const Version = "1.2.0";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
@@ -19,9 +15,7 @@ static const char *const usage[] = {
 };
 
 void global_init() {
-#ifndef SIST_SCAN_ONLY
     curl_global_init(CURL_GLOBAL_NOTHING);
-#endif
     av_log_set_level(AV_LOG_QUIET);
     opcInitLibrary();
 }
@@ -55,6 +49,7 @@ void sist2_scan(scan_args_t *args) {
     strncpy(ScanCtx.index.desc.name, args->name, sizeof(ScanCtx.index.desc.name));
     strncpy(ScanCtx.index.desc.root, args->path, sizeof(ScanCtx.index.desc.root));
     ScanCtx.index.desc.root_len = (short) strlen(ScanCtx.index.desc.root);
+    ScanCtx.tesseract_lang = args->tesseract_lang;
 
     init_dir(ScanCtx.index.path);
 
@@ -122,8 +117,6 @@ void sist2_scan(scan_args_t *args) {
     store_destroy(ScanCtx.index.store);
 }
 
-#ifndef SIST_SCAN_ONLY
-
 void sist2_index(index_args_t *args) {
 
     IndexCtx.es_url = args->es_url;
@@ -198,18 +191,14 @@ void sist2_web(web_args_t *args) {
     serve(args->bind, args->port);
 }
 
-#endif
-
 
 int main(int argc, const char *argv[]) {
 
     global_init();
 
     scan_args_t *scan_args = scan_args_create();
-#ifndef SIST_SCAN_ONLY
     index_args_t *index_args = index_args_create();
     web_args_t *web_args = web_args_create();
-#endif
 
     int arg_version = 0;
 
@@ -240,8 +229,9 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "archive", &scan_args->archive, "Archive file mode (skip|list|shallow|recurse). "
                                                           "skip: Don't parse, list: only get file names as text, "
                                                           "shallow: Don't parse archives inside archives. DEFAULT: recurse"),
+            OPT_STRING(0, "ocr", &scan_args->tesseract_lang, "Tesseract language (use tesseract --list-langs to see "
+                                                             "which are installed on your machine)"),
 
-#ifndef SIST_SCAN_ONLY
             OPT_GROUP("Index options"),
             OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"),
             OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
@@ -255,7 +245,6 @@ int main(int argc, const char *argv[]) {
             OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"),
             OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"),
             OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"),
-#endif
 
             OPT_END(),
     };
@@ -274,10 +263,8 @@ int main(int argc, const char *argv[]) {
         LogCtx.verbose = 1;
     }
 
-#ifndef SIST_SCAN_ONLY
     web_args->es_url = common_es_url;
     index_args->es_url = common_es_url;
-#endif
 
     if (argc == 0) {
         argparse_usage(&argparse);
@@ -292,7 +279,6 @@ int main(int argc, const char *argv[]) {
 
     }
 
-#ifndef SIST_SCAN_ONLY
     else if (strcmp(argv[0], "index") == 0) {
 
         int err = index_args_validate(index_args, argc, argv);
@@ -310,7 +296,6 @@ int main(int argc, const char *argv[]) {
         sist2_web(web_args);
 
     }
-#endif
     else {
         fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
         argparse_usage(&argparse);
@@ -320,10 +305,8 @@ int main(int argc, const char *argv[]) {
 
     scan_args_destroy(scan_args);
 
-#ifndef SIST_SCAN_ONLY
     index_args_destroy(index_args);
     web_args_destroy(web_args);
-#endif
 
     return 0;
 }

src/parsing/media.c

@@ -127,6 +127,7 @@ AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int st
 #define APPEND_TAG_META(doc, tag_, keyname) \
     text_buffer_t tex = text_buffer_create(-1); \
     text_buffer_append_string0(&tex, tag_->value); \
+    text_buffer_terminate_string(&tex); \
     meta_line_t *meta_tag = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); \
     meta_tag->key = keyname; \
     strcpy(meta_tag->strval, tex.dyn_buffer.buf); \

src/parsing/pdf.c

@@ -1,6 +1,10 @@
 #include "pdf.h"
 #include "src/ctx.h"
 
+#define MIN_OCR_SIZE 128
+__thread text_buffer_t thread_buffer;
+
+
 fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
 
     int err = 0;
@@ -86,14 +90,13 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
     return cover;
 }
 
-void fz_err_callback(void *user, __attribute__((unused)) const char *message) {
-   if (LogCtx.verbose) {
-       document_t *doc = (document_t*) user;
-       LOG_WARNINGF(doc->filepath, "FZ: %s", message)
-   }
+void fz_err_callback(void *user, UNUSED(const char *message)) {
+    if (LogCtx.verbose) {
+        document_t *doc = (document_t *) user;
+        LOG_WARNINGF(doc->filepath, "FZ: %s", message)
+    }
 }
 
-
 __always_inline
 void init_ctx(fz_context *ctx, document_t *doc) {
     fz_disable_icc(ctx);
@@ -124,6 +127,40 @@ int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
     return 0;
 }
 
+
+void fill_image(fz_context *ctx, UNUSED(fz_device *dev),
+        fz_image *img, UNUSED(fz_matrix ctm), UNUSED(float alpha),
+        UNUSED(fz_color_params color_params)) {
+
+    int l2factor = 0;
+
+    if (img->w > MIN_OCR_SIZE && img->h > MIN_OCR_SIZE) {
+
+        fz_pixmap *pix = img->get_pixmap(ctx, img, NULL, img->w, img->h, &l2factor);
+
+        if (pix->h > MIN_OCR_SIZE && img->h > MIN_OCR_SIZE && img->xres != 0) {
+            TessBaseAPI *api = TessBaseAPICreate();
+            TessBaseAPIInit3(api, TESS_DATAPATH, ScanCtx.tesseract_lang);
+
+            TessBaseAPISetImage(api, pix->samples, pix->w, pix->h, pix->n, pix->stride);
+            TessBaseAPISetSourceResolution(api, pix->xres);
+
+            char *text = TessBaseAPIGetUTF8Text(api);
+            size_t len = strlen(text);
+            text_buffer_append_string(&thread_buffer, text, len - 1);
+            LOG_DEBUGF(
+                    "pdf.c",
+                    "(OCR) %dx%d got %dB from tesseract (%s), buffer:%dB",
+                    pix->w, pix->h, len, ScanCtx.tesseract_lang, thread_buffer.dyn_buffer.cur
+            )
+
+            TessBaseAPIEnd(api);
+            TessBaseAPIDelete(api);
+            fz_drop_pixmap(ctx, pix);
+        }
+    }
+}
+
 void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
 
     if (buf == NULL) {
@@ -210,7 +247,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
 
     if (ScanCtx.content_size > 0) {
         fz_stext_options opts = {0};
-        text_buffer_t text_buf = text_buffer_create(ScanCtx.content_size);
+        thread_buffer = text_buffer_create(ScanCtx.content_size);
 
         for (int current_page = 0; current_page < page_count; current_page++) {
             fz_page *page = NULL;
@@ -224,7 +261,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
                     err = ctx->error.errcode;
                 if (err != 0) {
                     LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
-                    text_buffer_destroy(&text_buf);
+                    text_buffer_destroy(&thread_buffer);
                     fz_drop_page(ctx, page);
                     fz_drop_stream(ctx, stream);
                     fz_drop_document(ctx, fzdoc);
@@ -235,6 +272,15 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
 
             fz_stext_page *stext = fz_new_stext_page(ctx, fz_bound_page(ctx, page));
             fz_device *dev = fz_new_stext_device(ctx, stext, &opts);
+            dev->stroke_path = NULL;
+            dev->stroke_text = NULL;
+            dev->clip_text = NULL;
+            dev->clip_stroke_path = NULL;
+            dev->clip_stroke_text = NULL;
+
+            if (ScanCtx.tesseract_lang != NULL) {
+                dev->fill_image = fill_image;
+            }
 
             fz_var(err);
             fz_try(ctx)
@@ -249,7 +295,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
 
             if (err != 0) {
                 LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
-                text_buffer_destroy(&text_buf);
+                text_buffer_destroy(&thread_buffer);
                 fz_drop_page(ctx, page);
                 fz_drop_stext_page(ctx, stext);
                 fz_drop_stream(ctx, stream);
@@ -260,7 +306,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
 
             fz_stext_block *block = stext->first_block;
             while (block != NULL) {
-                int ret = read_stext_block(block, &text_buf);
+                int ret = read_stext_block(block, &thread_buffer);
                 if (ret == TEXT_BUF_FULL) {
                     break;
                 }
@@ -269,18 +315,18 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
             fz_drop_stext_page(ctx, stext);
             fz_drop_page(ctx, page);
 
-            if (text_buf.dyn_buffer.cur >= text_buf.dyn_buffer.size) {
+            if (thread_buffer.dyn_buffer.cur >= thread_buffer.dyn_buffer.size) {
                 break;
             }
         }
-        text_buffer_terminate_string(&text_buf);
+        text_buffer_terminate_string(&thread_buffer);
 
-        meta_line_t *meta_content = malloc(sizeof(meta_line_t) + text_buf.dyn_buffer.cur);
+        meta_line_t *meta_content = malloc(sizeof(meta_line_t) + thread_buffer.dyn_buffer.cur);
         meta_content->key = MetaContent;
-        memcpy(meta_content->strval, text_buf.dyn_buffer.buf, text_buf.dyn_buffer.cur);
+        memcpy(meta_content->strval, thread_buffer.dyn_buffer.buf, thread_buffer.dyn_buffer.cur);
         APPEND_META(doc, meta_content)
 
-        text_buffer_destroy(&text_buf);
+        text_buffer_destroy(&thread_buffer);
     }
 
     fz_drop_stream(ctx, stream);

src/parsing/pdf.h

@@ -2,6 +2,7 @@
 #define SIST2_PDF_H
 
 #include "src/sist.h"
+#include <tesseract/capi.h>
 
 
 void parse_pdf(void *buf, size_t buf_len, document_t *doc);

src/parsing/text.c

@@ -25,6 +25,7 @@ void parse_text(int bytes_read, struct vfile *f, char *buf, document_t *doc) {
     }
     text_buffer_t tex = text_buffer_create(ScanCtx.content_size);
     text_buffer_append_string(&tex, intermediate_buf, intermediate_buf_len);
+    text_buffer_terminate_string(&tex);
 
     meta_line_t *meta = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur);
     meta->key = MetaContent;

src/sist.h

@@ -2,6 +2,8 @@
 #define SIST_H
 
 #define UUID_STR_LEN 37
+#define UNUSED(x) __attribute__((__unused__))  x
+#define TESS_DATAPATH "/usr/share/tessdata/"
 
 #include <glib-2.0/glib.h>
 #include <unistd.h>
@@ -33,14 +35,12 @@
 #include <opc/opc.h>
 #include <libxml/xmlstring.h>
 
-#ifndef SIST_SCAN_ONLY
 #include <onion/onion.h>
 #include <onion/handler.h>
 #include <onion/block.h>
 #include <onion/shortcuts.h>
 #include <onion/codecs.h>
 #include <curl/curl.h>
-#endif
 
 #include "cJSON/cJSON.h"
 
@@ -62,12 +62,10 @@
 #include "log.h"
 #include "utf8.h/utf8.h"
 
-#ifndef SIST_SCAN_ONLY
 #include "src/index/elastic.h"
 #include "index/web.h"
 #include "web/serve.h"
 #include "web/auth_basic.h"
-#endif
 
 ;
 

src/types.h

@@ -70,8 +70,6 @@ typedef struct document {
     short ext;
     meta_line_t *meta_head;
     meta_line_t *meta_tail;
-    struct document *child_head;
-    struct document *child_tail;
     char *filepath;
 } document_t;
 
@@ -79,8 +77,6 @@ typedef struct vfile vfile_t;
 
 typedef int (*read_func_t)(struct vfile *, void *buf, size_t size);
 
-typedef int (*seek_func_t)(struct vfile *, size_t size, int whence);
-
 typedef void (*close_func_t)(struct vfile *);
 
 typedef struct vfile {

src/util.c

@@ -149,7 +149,6 @@ int text_buffer_append_string(text_buffer_t *buf, char *str, size_t len) {
         (0xe0 == (0xf0 & str[0]) && len < 3) ||
         (0xc0 == (0xe0 & str[0]) && len == 1) ||
         *(str) == 0) {
-        text_buffer_terminate_string(buf);
         return 0;
     }
 
@@ -158,7 +157,6 @@ int text_buffer_append_string(text_buffer_t *buf, char *str, size_t len) {
             text_buffer_append_char(buf, c);
         }
     }
-    text_buffer_terminate_string(buf);
     return 0;
 }
 
@@ -169,7 +167,6 @@ int text_buffer_append_string0(text_buffer_t *buf, char *str) {
             text_buffer_append_char(buf, c);
         }
     }
-    text_buffer_terminate_string(buf);
 }
 
 int text_buffer_append_char(text_buffer_t *buf, int c) {