Version bump

Fixes #20

.gitmodules

@@ -40,3 +40,6 @@
 [submodule "lib/libtiff"]
 	path = lib/libtiff
 	url = https://gitlab.com/libtiff/libtiff
+[submodule "lib/libpng"]
+	path = lib/libpng
+	url = https://github.com/glennrp/libpng

CMakeLists.txt

@@ -45,7 +45,6 @@ add_executable(
 find_package(PkgConfig REQUIRED)
 set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig/")
 
-#find_package(OpenSSL REQUIRED)
 find_package(Freetype REQUIRED)
 
 pkg_check_modules(GLIB REQUIRED glib-2.0)
@@ -84,7 +83,8 @@ target_link_directories(
 target_compile_options(sist2
         PRIVATE
         -Ofast
-        # -march=native
+#        -march=native
+        -fPIC
         -fno-stack-protector
         -fomit-frame-pointer
         )
@@ -111,7 +111,7 @@ TARGET_LINK_LIBRARIES(
         ${PROJECT_SOURCE_DIR}/lib/libonion_static.a
 
         pthread
-        curl
+
         m
         bz2
         ${PROJECT_SOURCE_DIR}/lib/libmagic.a
@@ -128,8 +128,14 @@ TARGET_LINK_LIBRARIES(
         ${PROJECT_SOURCE_DIR}/lib/libtesseract.a
         ${PROJECT_SOURCE_DIR}/lib/liblept.a
         ${PROJECT_SOURCE_DIR}/lib/libtiff.a
-        png
+        ${PROJECT_SOURCE_DIR}/lib/libpng16.a
         stdc++
+
+        # curl
+        ${PROJECT_SOURCE_DIR}/lib/libcurl.a
+        ${PROJECT_SOURCE_DIR}/lib/libcrypto.a
+        ${PROJECT_SOURCE_DIR}/lib/libssl.a
+        dl
 )
 
 add_custom_target(

Docker/Dockerfile

@@ -16,4 +16,7 @@ RUN mkdir -p /usr/share/tessdata && \
 
 ADD sist2 /root/sist2
 
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
 ENTRYPOINT ["/root/sist2"]

Docker/build.sh

@@ -8,3 +8,5 @@ echo "Version ${version}"
 docker build . -t simon987/sist2:${version} -t simon987/sist2:latest
 docker push simon987/sist2:${version}
 docker push simon987/sist2:latest
+
+docker run --rm -it simon987/sist2 -v

README.md

@@ -29,7 +29,7 @@ sist2 (Simple incremental search tool)
 1. Have an [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) instance running
 1. 
     1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) *
-    1. *(or)* Download an [development snapshot](https://files.simon987.net/artifacts/Sist2/Build/) *(Not recommended!)*
+    1. *(or)* Download a [development snapshot](https://files.simon987.net/artifacts/Sist2/Build/) *(Not recommended!)*
     1. *(or)* `docker pull simon987/sist2:latest`
    
 
@@ -39,6 +39,9 @@ sist2 (Simple incremental search tool)
 
 ## Example usage
 
+
+![demo](demo.gif)
+
 See help page `sist2 --help` for more details.
 
 **Scan a directory**

ci/build.sh

@@ -5,4 +5,3 @@
 cmake .
 make
 strip sist2
-strip sist2_scan

schema/mappings.json

@@ -7,7 +7,7 @@
     },
     "suggest-path": {
       "type": "completion",
-      "analyzer": "keyword"
+      "analyzer": "case_insensitive_kw_analyzer"
     },
     "mime": {
       "type": "keyword"

schema/settings.json

@@ -21,6 +21,12 @@
           "lowercase"
         ]
       },
+      "case_insensitive_kw_analyzer": {
+        "tokenizer": "keyword",
+        "filter": [
+          "lowercase"
+        ]
+      },
       "my_nGram": {
         "tokenizer": "my_nGram_tokenizer",
         "filter": [

scripts/get_static_libs.sh

@@ -13,8 +13,7 @@ mv mupdf/build/release/libmupdf-third.a .
 
 # openjp2
 cd openjpeg
-#cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-O3 -march=native -DNDEBUG"
-cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-O3"
+cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-O3 -march=native -DNDEBUG -fPIC"
 make -j $THREADS
 cd ..
 mv openjpeg/bin/libopenjp2.a .
@@ -22,7 +21,7 @@ mv openjpeg/bin/libopenjp2.a .
 # harfbuzz
 cd harfbuzz
 ./autogen.sh
-./configure --disable-shared --enable-static
+CFLAGS=-fPIC ./configure --disable-shared --enable-static
 make -j $THREADS
 cd ..
 mv harfbuzz/src/.libs/libharfbuzz.a .
@@ -75,7 +74,8 @@ mv libmagic/src/.libs/libmagic.a .
 cd tesseract
 mkdir build
 cd build
-cmake -DSTATIC=on -DBUILD_TRAINING_TOOLS=off ..
+cmake -DSTATIC=on -DBUILD_TRAINING_TOOLS=off -DBUILD_TESTS=off -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CXX_FLAGS="-fPIC"  ..
 make -j $THREADS
 cd ../..
 mv tesseract/build/libtesseract.a .
@@ -83,7 +83,7 @@ mv tesseract/build/libtesseract.a .
 # leptonica
 cd leptonica
 ./autogen.sh
-./configure --without-zlib --without-jpeg --without-giflib \
+CFLAGS="-fPIC" ./configure --without-zlib --without-jpeg --without-giflib \
   --without-giflib --without-libwebp --without-libwebpmux --without-libopenjpeg \
   --enable-static --disable-shared
 make -j $THREADS
@@ -93,8 +93,36 @@ mv leptonica/src/.libs/liblept.a .
 # tiff
 cd libtiff
 ./autogen.sh
-./configure --enable-static --disable-shared --disable-lzw --disable-jpeg --disable-webp \
+CFLAGS="-fPIC" CXXFLAGS="-fPIC" CXX_FLAGS="-fPIC" ./configure --enable-static --disable-shared --disable-lzw --disable-jpeg --disable-webp \
   --disable-lzma --disable-zstd --disable-jbig
 make -j $THREADS
 cd ..
-mv libtiff/libtiff/.libs/libtiff.a .
+mv libtiff/libtiff/.libs/libtiff.a .
+
+# png
+cd libpng
+CFLAGS="-fPIC" ./configure --enable-static --disable-shared
+make -j $THREADS
+cd ..
+mv libpng/.libs/libpng16.a .
+
+# openssl...
+git clone --depth 1 -b OpenSSL_1_1_0-stable https://github.com/openssl/openssl
+cd openssl
+./config --prefix=$(pwd)/../ssl
+make depend
+make -j $THREADS
+make install
+cd ..
+mv ./openssl/libcrypto.a ./openssl/libssl.a .
+
+# curl
+wget -nc https://curl.haxx.se/download/curl-7.68.0.tar.gz
+tar -xzf curl-7.68.0.tar.gz
+cd curl-7.68.0
+./configure --disable-ldap --disable-ldaps --without-librtmp --disable-rtsp --disable-crypto-auth \
+  --disable-smtp --without-libidn2 --without-nghttp2 --without-brotli --enable-static --disable-shared \
+  --without-libpsl --with-ssl=$(pwd)/../ssl
+make -j $THREADS
+cd ..
+mv curl-7.68.0/lib/.libs/libcurl.a .

src/cli.c

@@ -15,6 +15,13 @@
 #define DEFAULT_BIND_ADDR "localhost"
 #define DEFAULT_PORT "4090"
 
+const char* TESS_DATAPATHS[] = {
+        "/usr/share/tessdata/",
+        "/usr/share/tesseract-ocr/tessdata/",
+        "./",
+        NULL
+};
+
 
 scan_args_t *scan_args_create() {
     scan_args_t *args = calloc(sizeof(scan_args_t), 1);
@@ -136,13 +143,23 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
 
     if (args->tesseract_lang != NULL) {
         TessBaseAPI *api = TessBaseAPICreate();
-        ret = TessBaseAPIInit3(api, TESS_DATAPATH, args->tesseract_lang);
+
+        char filename[128];
+        sprintf(filename, "%s.traineddata", args->tesseract_lang);
+        const char * path = find_file_in_paths(TESS_DATAPATHS, filename);
+        if (path == NULL) {
+            LOG_FATAL("cli.c", "Could not find tesseract language file!");
+        }
+
+        ret = TessBaseAPIInit3(api, path, args->tesseract_lang);
         if (ret != 0) {
             fprintf(stderr, "Could not initialize tesseract with lang '%s'\n", args->tesseract_lang);
             return 1;
         }
         TessBaseAPIEnd(api);
         TessBaseAPIDelete(api);
+
+        args->tesseract_path = path;
     }
 
     LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
@@ -156,7 +173,8 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
     LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
     LOG_DEBUGF("cli.c", "arg path=%s", args->path)
     LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
-    LOG_DEBUGF("cli.c", "arg ocr=%s", args->tesseract_lang)
+    LOG_DEBUGF("cli.c", "arg tesseract_lang=%s", args->tesseract_lang)
+    LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path)
 
     return 0;
 }

src/cli.h

@@ -17,6 +17,7 @@ typedef struct scan_args {
     char *archive;
     archive_mode_t archive_mode;
     char *tesseract_lang;
+    const char *tesseract_path;
 } scan_args_t;
 
 scan_args_t *scan_args_create();

src/ctx.h

@@ -28,6 +28,7 @@ struct {
 
     pthread_mutex_t mupdf_mu;
     char * tesseract_lang;
+    char * tesseract_path;
 } ScanCtx;
 
 struct {

src/index/elastic.c

@@ -274,3 +274,28 @@ cJSON *elastic_get_document(const char *uuid_str) {
     free_response(r);
     return json;
 }
+
+char *elastic_get_status() {
+    char url[4096];
+    snprintf(url, 4096,
+            "%s/_cluster/state/metadata/sist2?filter_path=metadata.indices.*.state", WebCtx.es_url);
+
+    response_t *r = web_get(url);
+    cJSON *json = NULL;
+    char *status = malloc(128 * sizeof(char));
+    status[0] = '\0';
+
+    if (r->status_code == 200) {
+        json = cJSON_Parse(r->body);
+        const cJSON *metadata = cJSON_GetObjectItem(json, "metadata");
+        if (metadata != NULL) {
+            const cJSON *indices = cJSON_GetObjectItem(metadata, "indices");
+            const cJSON *sist2 = cJSON_GetObjectItem(indices, "sist2");
+            const cJSON *state = cJSON_GetObjectItem(sist2, "state");
+            strcpy(status, state->valuestring);
+        }
+    }
+    free_response(r);
+    cJSON_Delete(json);
+    return status;
+}

src/index/elastic.h

@@ -30,4 +30,6 @@ void elastic_init(int force_reset);
 
 cJSON *elastic_get_document(const char *uuid_str);
 
+char *elastic_get_status();
+
 #endif

src/main.c

@@ -6,7 +6,7 @@
 #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
 
 
-static const char *const Version = "1.2.2";
+static const char *const Version = "1.2.4";
 static const char *const usage[] = {
         "sist2 scan [OPTION]... PATH",
         "sist2 index [OPTION]... INDEX",
@@ -51,6 +51,7 @@ void sist2_scan(scan_args_t *args) {
     strncpy(ScanCtx.index.desc.root, args->path, sizeof(ScanCtx.index.desc.root));
     ScanCtx.index.desc.root_len = (short) strlen(ScanCtx.index.desc.root);
     ScanCtx.tesseract_lang = args->tesseract_lang;
+    ScanCtx.tesseract_path = args->tesseract_path;
 
     init_dir(ScanCtx.index.path);
 

src/parsing/doc.c

@@ -55,7 +55,14 @@ __always_inline
 void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part, document_t *doc) {
 
     mceTextReader_t reader;
-    int ret = opcXmlReaderOpen(c, &reader, part, NULL, "UTF-8", 0);
+    int options;
+    if (LogCtx.very_verbose) {
+        options = XML_PARSE_NONET;
+    } else {
+        options = XML_PARSE_NOWARNING | XML_PARSE_NOERROR | XML_PARSE_NONET;
+    }
+
+    int ret = opcXmlReaderOpen(c, &reader, part, NULL, "UTF-8", options);
 
     if (ret != OPC_ERROR_NONE) {
         LOG_ERRORF(doc->filepath, "(doc.c) opcXmlReaderOpen() returned error code %d", ret);

src/parsing/pdf.c

@@ -141,7 +141,7 @@ void fill_image(fz_context *ctx, UNUSED(fz_device *dev),
 
         if (pix->h > MIN_OCR_SIZE && img->h > MIN_OCR_SIZE && img->xres != 0) {
             TessBaseAPI *api = TessBaseAPICreate();
-            TessBaseAPIInit3(api, TESS_DATAPATH, ScanCtx.tesseract_lang);
+            TessBaseAPIInit3(api, ScanCtx.tesseract_path, ScanCtx.tesseract_lang);
 
             TessBaseAPISetImage(api, pix->samples, pix->w, pix->h, pix->n, pix->stride);
             TessBaseAPISetSourceResolution(api, pix->xres);
@@ -157,8 +157,8 @@ void fill_image(fz_context *ctx, UNUSED(fz_device *dev),
 
             TessBaseAPIEnd(api);
             TessBaseAPIDelete(api);
-            fz_drop_pixmap(ctx, pix);
         }
+        fz_drop_pixmap(ctx, pix);
     }
 }
 

src/sist.h

@@ -3,7 +3,6 @@
 
 #define UUID_STR_LEN 37
 #define UNUSED(x) __attribute__((__unused__))  x
-#define TESS_DATAPATH "/usr/share/tessdata/"
 
 #include <glib-2.0/glib.h>
 #include <unistd.h>

src/util.c

@@ -1,4 +1,5 @@
 #include "util.h"
+#include "src/ctx.h"
 
 dyn_buffer_t dyn_buffer_create() {
     dyn_buffer_t buf;
@@ -317,4 +318,29 @@ GHashTable *incremental_get_table() {
     return file_table;
 }
 
+const char *find_file_in_paths(const char *paths[], const char *filename) {
+
+    for (int i = 0; paths[i] != NULL; i++) {
+
+        char *apath = abspath(paths[i]);
+        if (apath == NULL) {
+            continue;
+        }
+
+        char path[PATH_MAX];
+        snprintf(path, sizeof(path), "%s%s", apath, filename);
+
+        LOG_DEBUGF("util.c", "Looking for '%s' in folder '%s'", filename, apath)
+        free(apath);
+
+        struct stat info;
+        int ret = stat(path, &info);
+        if (ret != -1) {
+            return paths[i];
+        }
+    }
+
+    return NULL;
+}
+
 

src/util.h

@@ -74,5 +74,6 @@ int incremental_get(GHashTable *table, unsigned long inode_no);
 
 int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no);
 
+const char *find_file_in_paths(const char **paths, const char *filename);
 
 #endif

src/web/serve.c

@@ -431,6 +431,23 @@ int file(UNUSED(void *p), onion_request *req, onion_response *res) {
     return ret;
 }
 
+int status(UNUSED(void *p), UNUSED(onion_request *req), onion_response *res) {
+    set_default_headers(res);
+
+    onion_response_set_header(res, "Content-Type", "application/x-empty");
+
+    char *status = elastic_get_status();
+    if (strcmp(status, "open") == 0) {
+        onion_response_set_code(res, 204);
+    } else {
+        onion_response_set_code(res, 500);
+    }
+
+    free(status);
+
+    return OCS_PROCESSED;
+}
+
 void serve(const char *hostname, const char *port) {
     onion *o = onion_new(O_POOL);
     onion_set_timeout(o, 3500);
@@ -450,6 +467,7 @@ void serve(const char *hostname, const char *port) {
 
     onion_url_add(urls, "es", search);
     onion_url_add(urls, "scroll", scroll);
+    onion_url_add(urls, "status", status);
     onion_url_add(
             urls,
             "^t/([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})/"

web/js/search.js

@@ -82,7 +82,10 @@ $.jsonPost("i").then(resp => {
 });
 
 function getDocumentInfo(id) {
-    return $.getJSON("d/" + id)
+    return $.getJSON("d/" + id).fail(e => {
+        console.log(e);
+        showEsError();
+    })
 }
 
 function handleTreeClick(tree) {

web/search.html

@@ -11,7 +11,7 @@
 
 <nav class="navbar navbar-expand-lg">
     <a class="navbar-brand" href="/">sist2</a>
-    <span class="badge badge-pill version">v1.2.2</span>
+    <span class="badge badge-pill version">v1.2.4</span>
     <span class="tagline">Lightning-fast file system indexer and search tool </span>
     <a style="margin-left: auto" id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
 </nav>