Update dependencies, fix some build issues

.dockerignore

@@ -15,7 +15,6 @@ Makefile
 **/*.cbp
 VERSION
 **/node_modules/
-.git/
 sist2-*-linux-debug
 sist2-*-linux
 sist2_debug
@@ -33,4 +32,9 @@ tmp_scan/
 Dockerfile
 Dockerfile.arm64
 docker-compose.yml
-state.db
+state.db
+*-journal
+build/
+__pycache__/
+sist2-vue/dist
+sist2-admin/frontend/dist

.gitignore

@@ -42,4 +42,5 @@ src/web/static_generated.c
 src/magic_generated.c
 src/index/static_generated.c
 *.sist2
-*-shm
+*-shm
+*-journal

CMakeLists.txt

@@ -5,7 +5,6 @@ set(CMAKE_C_STANDARD 11)
 
 option(SIST_DEBUG "Build a debug executable" on)
 option(SIST_FAST "Enable more optimisation flags" off)
-option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0)
 
 add_compile_definitions(
         "SIST_PLATFORM=${SIST_PLATFORM}"
@@ -56,14 +55,10 @@ set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
 
 find_package(PkgConfig REQUIRED)
 
-find_package(lmdb CONFIG REQUIRED)
 find_package(cJSON CONFIG REQUIRED)
 find_package(unofficial-mongoose CONFIG REQUIRED)
 find_package(CURL CONFIG REQUIRED)
-find_library(MAGIC_LIB
+find_library(MAGIC_LIB NAMES libmagic.a REQUIRED)
-        NAMES libmagic.so.1 magic
-        PATHS /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/
-)
 find_package(unofficial-sqlite3 CONFIG REQUIRED)
 
 
@@ -91,7 +86,6 @@ if (SIST_DEBUG)
             -fsanitize=address
             -fno-inline
 #            -O2
-            -w
     )
     target_link_options(
             sist2
@@ -137,8 +131,6 @@ target_link_libraries(
         sist2
 
         z
-        lmdb
-        cjson
         argparse
         unofficial::mongoose::mongoose
         CURL::libcurl

Dockerfile

@@ -19,9 +19,9 @@ COPY sist2-admin sist2-admin
 RUN cd sist2-vue/ && npm install && npm run build
 RUN cd sist2-admin/frontend/ && npm install && npm run build
 
-RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .
+RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake ..
-RUN make -j$(nproc)
+RUN cd build && make -j$(nproc)
-RUN strip sist2 || mv sist2_debug sist2
+RUN strip build/sist2 || mv sist2_debug build/sist2
 
 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea
 
@@ -49,7 +49,7 @@ RUN mkdir -p /usr/share/tessdata && \
     curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata
 
 # sist2
-COPY --from=build /build/sist2 /root/sist2
+COPY --from=build /build/build/sist2 /root/sist2
 
 # sist2-admin
 COPY sist2-admin/requirements.txt sist2-admin/

README.md

@@ -149,8 +149,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux
 1. Install vcpkg dependencies
 
     ```bash
-    vcpkg install curl[core,openssl]
+    vcpkg install curl[core,openssl] sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample]
-    vcpkg install lmdb sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw jasper lcms gumbo
     ```
 
 1. Build

third-party/libscan/CMakeLists.txt

@@ -78,6 +78,7 @@ else()
             -fno-stack-protector
             -fomit-frame-pointer
             #-freciprocal-math
+            -w
     )
 endif()
 
@@ -99,17 +100,17 @@ find_package(unofficial-pcre CONFIG REQUIRED)
 
 find_library(JBIG2DEC_LIB NAMES jbig2decd jbig2dec)
 find_library(HARFBUZZ_LIB NAMES harfbuzz harfbuzzd)
-find_library(FREETYPE_LIB NAMES freetype freetyped)
-find_package(unofficial-brotli CONFIG REQUIRED)
 find_library(LZO2_LIB NAMES lzo2)
 
-find_library(RAW_LIB NAMES libraw.a)
 find_library(MUPDF_LIB NAMES liblibmupdf.a)
 find_library(CMS_LIB NAMES lcms2)
 find_library(JAS_LIB NAMES jasper)
 find_library(GUMBO_LIB NAMES gumbo)
 find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/)
 find_package(Leptonica CONFIG REQUIRED)
+find_package(FFMPEG REQUIRED)
+find_package(libraw CONFIG REQUIRED)
+find_package(Freetype REQUIRED)
 
 
 target_compile_options(
@@ -118,39 +119,7 @@ target_compile_options(
         -g
 )
 
-if (SIST_DEBUG)
-    SET(FFMPEG_DEBUG "--enable-debug=3" "--disable-optimizations")
-else()
-    SET(FFMPEG_DEBUG "")
-endif()
-
 include(ExternalProject)
-find_program(MAKE_EXE NAMES gmake nmake make)
-
-ExternalProject_Add(
-        ffmpeg
-        GIT_REPOSITORY https://git.ffmpeg.org/ffmpeg.git
-        GIT_TAG "n4.4"
-
-        UPDATE_COMMAND ""
-        PATCH_COMMAND ""
-        TEST_COMMAND ""
-        CONFIGURE_COMMAND ./configure --disable-shared --enable-static --disable-ffmpeg --disable-ffplay
-        --disable-ffprobe --disable-doc --disable-manpages --disable-postproc --disable-avfilter --disable-alsa
-        --disable-lzma --disable-xlib --disable-vdpau --disable-vaapi --disable-sdl2
-        --disable-network  ${FFMPEG_DEBUG}
-        INSTALL_COMMAND ""
-
-        PREFIX "third-party/ext_ffmpeg"
-        SOURCE_DIR "third-party/ext_ffmpeg/src/ffmpeg"
-        BINARY_DIR "third-party/ext_ffmpeg/src/ffmpeg"
-
-        BUILD_COMMAND ${MAKE_EXE} -j33 --silent
-)
-
-SET(FFMPEG_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_ffmpeg/src/ffmpeg)
-SET(FFMPEG_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_ffmpeg/src/ffmpeg)
-
 ExternalProject_Add(
         libwpd
         URL http://prdownloads.sourceforge.net/libwpd/libwpd-0.9.9.tar.gz
@@ -165,19 +134,20 @@ ExternalProject_Add(
         SOURCE_DIR "third-party/ext_libwpd/src/libwpd"
         BINARY_DIR "third-party/ext_libwpd/src/libwpd"
 
-        BUILD_COMMAND ${MAKE_EXE} -j33
+        BUILD_COMMAND make -j33
 )
 SET(WPD_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwpd/src/lib/.libs/)
 SET(WPD_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwpd/inc/)
 
 add_dependencies(
         scan
-        ffmpeg
         antiword
         libwpd
         mobi
 )
 
+
+target_link_directories(scan PUBLIC ${FFMPEG_LIBRARY_DIRS})
 target_link_libraries(
         scan
         PUBLIC
@@ -202,36 +172,26 @@ target_link_libraries(
 
         stdc++
 
-        -Wl,--whole-archive
-        m
-        -Wl,--no-whole-archive
-
         ${JPEG_LIBRARIES}
         ${Tesseract_LIBRARIES}
         ${LIBXML2_LIBRARIES}
-        ${FREETYPE_LIB}
+        Freetype::Freetype
-        unofficial::brotli::brotlidec-static
-
-        ${FFMPEG_LIB_DIR}/libavformat/libavformat.a
-        ${FFMPEG_LIB_DIR}/libavcodec/libavcodec.a
-        ${FFMPEG_LIB_DIR}/libavutil/libavutil.a
-        ${FFMPEG_LIB_DIR}/libswresample/libswresample.a
-        ${FFMPEG_LIB_DIR}/libswscale/libswscale.a
 
         z
 
         ${CMAKE_THREAD_LIBS_INIT}
 
-        ${RAW_LIB}
         ${GOMP_LIB}
         ${CMS_LIB}
         ${JAS_LIB}
         ${GUMBO_LIB}
-        dl
+
         antiword
         mobi
         unofficial::pcre::pcre unofficial::pcre::pcre16 unofficial::pcre::pcre32 unofficial::pcre::pcrecpp
         leptonica
+        libraw::raw
+        ${FFMPEG_LIBRARIES}
 )
 
 target_include_directories(
@@ -243,6 +203,7 @@ target_include_directories(
         ${FFMPEG_INCLUDE_DIR}
         ${MOBI_INCLUDE_DIR}
         ${WPD_INCLUDE_DIR}
+        ${FFMPEG_INCLUDE_DIRS}
 )
 
 if (BUILD_TESTS)