.gitmodules

@@ -7,6 +7,3 @@
 [submodule "third-party/libscan/third-party/antiword"]
 	path = third-party/libscan/third-party/antiword
 	url = https://github.com/simon987/antiword
-[submodule "third-party/libscan/third-party/libmobi"]
-	path = third-party/libscan/third-party/libmobi
-	url = https://github.com/bfabiszewski/libmobi

CMakeLists.txt

@@ -4,7 +4,6 @@ set(CMAKE_C_STANDARD 11)
 project(sist2 C)
 
 option(SIST_DEBUG "Build a debug executable" on)
-option(SIST_FAST "Enable more optimisation flags" off)
 option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0)
 
 add_compile_definitions(
@@ -55,10 +54,6 @@ find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
-find_library(MAGIC_LIB
-        NAMES libmagic.so.1 magic
-        PATHS /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/
-)
 
 
 target_include_directories(
@@ -98,25 +93,16 @@ if (SIST_DEBUG)
             PROPERTIES
             OUTPUT_NAME sist2_debug
     )
-elseif (SIST_FAST)
-    target_compile_options(
-            sist2
-            PRIVATE
-
-            -Ofast
-            -march=native
-            -fno-stack-protector
-            -fomit-frame-pointer
-            -freciprocal-math
-    )
 else ()
     target_compile_options(
             sist2
             PRIVATE
 
             -Ofast
+            #-march=native
             -fno-stack-protector
             -fomit-frame-pointer
+            #-freciprocal-math
     )
 endif ()
 
@@ -138,12 +124,13 @@ target_link_libraries(
         CURL::libcurl
 
         pthread
+        #magic
 
         c
 
         scan
 
-        ${MAGIC_LIB}
+        /usr/lib/x86_64-linux-gnu/libmagic.so.1
 )
 
 add_custom_target(

schema/settings.json

@@ -3,7 +3,7 @@
     "refresh_interval": "30s",
     "codec": "best_compression",
     "number_of_replicas": 0,
-    "highlight.max_analyzed_offset": 1000000
+    "highlight.max_analyzed_offset": 10000000
   },
   "analysis": {
     "tokenizer": {

sist2-vue/src/Sist2Query.ts

@@ -210,7 +210,7 @@ class Sist2Query {
             };
 
             if (!legacyES) {
-                q.highlight.max_analyzed_offset = 999_999;
+                q.highlight.max_analyzed_offset = 9_999_999;
             }
 
             if (getters.optSearchInPath) {

src/sist.h

@@ -27,6 +27,10 @@
 
 #define UNUSED(x) __attribute__((__unused__))  x
 
+#define MD5_STR_LENGTH 33
+#define SHA1_STR_LENGTH 41
+#define SHA1_DIGEST_LENGTH 20
+
 #include "util.h"
 #include "log.h"
 #include "types.h"

third-party/libscan/CMakeLists.txt

@@ -7,11 +7,6 @@ option(BUILD_TESTS "Build tests" on)
 
 add_subdirectory(third-party/antiword)
 
-set(USE_LIBXML2 OFF)
-set(USE_XMLWRITER OFF)
-set(BUILD_SHARED_LIBS OFF)
-add_subdirectory(third-party/libmobi)
-
 add_library(
         scan
         libscan/util.c libscan/util.h
@@ -47,22 +42,6 @@ if (SIST_DEBUG)
             -fsanitize=address
             -fno-inline
     )
-elseif (SIST_FAST)
-    add_compile_definitions(
-            antiword
-            NDEBUG
-    )
-
-    target_compile_options(
-            scan
-            PRIVATE
-
-            -Ofast
-            -march=native
-            -fno-stack-protector
-            -fomit-frame-pointer
-            -freciprocal-math
-    )
 else()
     add_compile_definitions(
             antiword
@@ -118,15 +97,35 @@ target_compile_options(
         -g
 )
 
+include(ExternalProject)
+find_program(MAKE_EXE NAMES gmake nmake make)
+ExternalProject_Add(
+        libmobi
+        GIT_REPOSITORY https://github.com/simon987/libmobi.git
+        GIT_TAG "public"
+
+        UPDATE_COMMAND ""
+        PATCH_COMMAND ""
+        TEST_COMMAND ""
+        CONFIGURE_COMMAND ./autogen.sh && ./configure
+        INSTALL_COMMAND ""
+
+        PREFIX "third-party/ext_libmobi"
+        SOURCE_DIR "third-party/ext_libmobi/src/libmobi"
+        BINARY_DIR "third-party/ext_libmobi/src/libmobi"
+
+        BUILD_COMMAND ${MAKE_EXE} -j 8 --silent
+)
+
+SET(MOBI_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libmobi/src/libmobi/src/.libs/)
+SET(MOBI_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libmobi/src/libmobi/src/)
+
 if (SIST_DEBUG)
     SET(FFMPEG_DEBUG "--enable-debug=3" "--disable-optimizations")
 else()
     SET(FFMPEG_DEBUG "")
 endif()
 
-include(ExternalProject)
-find_program(MAKE_EXE NAMES gmake nmake make)
-
 ExternalProject_Add(
         ffmpeg
         GIT_REPOSITORY https://git.ffmpeg.org/ffmpeg.git
@@ -172,10 +171,10 @@ SET(WPD_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwp
 
 add_dependencies(
         scan
+        libmobi
         ffmpeg
         antiword
         libwpd
-        mobi
 )
 
 target_link_libraries(
@@ -193,6 +192,8 @@ target_link_libraries(
         ${MUPDF_LIB}
         openjp2
 
+        ${MOBI_LIB_DIR}/libmobi.a
+
         ${WPD_LIB_DIR}/libwpd-0.9.a
         ${WPD_LIB_DIR}/libwpd-stream-0.9.a
 
@@ -229,7 +230,6 @@ target_link_libraries(
         ${GUMBO_LIB}
         dl
         antiword
-        mobi
         unofficial::pcre::pcre unofficial::pcre::pcre16 unofficial::pcre::pcre32 unofficial::pcre::pcrecpp
 )
 

third-party/libscan/libscan/mobi/scan_mobi.c

@@ -1,6 +1,6 @@
 #include "scan_mobi.h"
 
-#include "../../third-party/libmobi/src/mobi.h"
+#include <mobi.h>
 #include <errno.h>
 #include "stdlib.h"
 

third-party/libscan/libscan/scan.h

@@ -48,6 +48,7 @@ typedef int scan_code_t;
 #define CTX_LOG_FATALF(filepath, fmt, ...) ctx->logf(filepath, LEVEL_FATAL, fmt, __VA_ARGS__); exit(-1);
 #define CTX_LOG_FATAL(filepath, str) ctx->log(filepath, LEVEL_FATAL, str); exit(-1);
 
+#define MD5_STR_LENGTH 33
 #define SIST_DOC_ID_LEN MD5_STR_LENGTH
 #define SIST_INDEX_ID_LEN MD5_STR_LENGTH
 

third-party/libscan/third-party/libmobi

@@ -1 +0,0 @@
-Subproject commit 395dbde361a80353a9ed8b65d01d6066554142b3