diff --git a/.dockerignore b/.dockerignore index 4020560..6806de8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,7 +15,6 @@ Makefile **/*.cbp VERSION **/node_modules/ -.git/ sist2-*-linux-debug sist2-*-linux sist2_debug @@ -33,4 +32,9 @@ tmp_scan/ Dockerfile Dockerfile.arm64 docker-compose.yml -state.db \ No newline at end of file +state.db +*-journal +build/ +__pycache__/ +sist2-vue/dist +sist2-admin/frontend/dist \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8f9f8a3..0ca141d 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,5 @@ src/web/static_generated.c src/magic_generated.c src/index/static_generated.c *.sist2 -*-shm \ No newline at end of file +*-shm +*-journal \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index d3869f0..cb92ba3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,6 @@ set(CMAKE_C_STANDARD 11) option(SIST_DEBUG "Build a debug executable" on) option(SIST_FAST "Enable more optimisation flags" off) -option(SIST_FAKE_STORE "Disable IO operations of LMDB stores for debugging purposes" 0) add_compile_definitions( "SIST_PLATFORM=${SIST_PLATFORM}" @@ -56,14 +55,10 @@ set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib) find_package(PkgConfig REQUIRED) -find_package(lmdb CONFIG REQUIRED) find_package(cJSON CONFIG REQUIRED) find_package(unofficial-mongoose CONFIG REQUIRED) find_package(CURL CONFIG REQUIRED) -find_library(MAGIC_LIB - NAMES libmagic.so.1 magic - PATHS /usr/lib/x86_64-linux-gnu/ /usr/lib/aarch64-linux-gnu/ -) +find_library(MAGIC_LIB NAMES libmagic.a REQUIRED) find_package(unofficial-sqlite3 CONFIG REQUIRED) @@ -91,7 +86,6 @@ if (SIST_DEBUG) -fsanitize=address -fno-inline # -O2 - -w ) target_link_options( sist2 @@ -137,8 +131,6 @@ target_link_libraries( sist2 z - lmdb - cjson argparse unofficial::mongoose::mongoose CURL::libcurl diff --git a/Dockerfile b/Dockerfile index 1ca0e01..ec69796 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,9 @@ COPY sist2-admin sist2-admin RUN cd sist2-vue/ && npm install && npm run build RUN cd sist2-admin/frontend/ && npm install && npm run build -RUN cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake . -RUN make -j$(nproc) -RUN strip sist2 || mv sist2_debug sist2 +RUN mkdir build && cd build && cmake -DSIST_PLATFORM=x64_linux -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake .. +RUN cd build && make -j$(nproc) +RUN strip build/sist2 || mv sist2_debug build/sist2 FROM --platform="linux/amd64" ubuntu@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea @@ -49,7 +49,7 @@ RUN mkdir -p /usr/share/tessdata && \ curl -o /usr/share/tessdata/chi_sim.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/chi_sim.traineddata # sist2 -COPY --from=build /build/sist2 /root/sist2 +COPY --from=build /build/build/sist2 /root/sist2 # sist2-admin COPY sist2-admin/requirements.txt sist2-admin/ diff --git a/README.md b/README.md index 39d0783..fa89016 100644 --- a/README.md +++ b/README.md @@ -149,8 +149,7 @@ docker run --rm --entrypoint cat my-sist2-image /root/sist2 > sist2-x64-linux 1. Install vcpkg dependencies ```bash - vcpkg install curl[core,openssl] - vcpkg install lmdb sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw jasper lcms gumbo + vcpkg install curl[core,openssl] sqlite3 cpp-jwt pcre cjson brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libmagic libraw gumbo ffmpeg[core,avcodec,avformat,swscale,swresample] ``` 1. Build diff --git a/third-party/libscan/CMakeLists.txt b/third-party/libscan/CMakeLists.txt index 3cc065d..5bb23af 100644 --- a/third-party/libscan/CMakeLists.txt +++ b/third-party/libscan/CMakeLists.txt @@ -78,6 +78,7 @@ else() -fno-stack-protector -fomit-frame-pointer #-freciprocal-math + -w ) endif() @@ -99,17 +100,17 @@ find_package(unofficial-pcre CONFIG REQUIRED) find_library(JBIG2DEC_LIB NAMES jbig2decd jbig2dec) find_library(HARFBUZZ_LIB NAMES harfbuzz harfbuzzd) -find_library(FREETYPE_LIB NAMES freetype freetyped) -find_package(unofficial-brotli CONFIG REQUIRED) find_library(LZO2_LIB NAMES lzo2) -find_library(RAW_LIB NAMES libraw.a) find_library(MUPDF_LIB NAMES liblibmupdf.a) find_library(CMS_LIB NAMES lcms2) find_library(JAS_LIB NAMES jasper) find_library(GUMBO_LIB NAMES gumbo) find_library(GOMP_LIB NAMES libgomp.a gomp PATHS /usr/lib/gcc/x86_64-linux-gnu/11/ /usr/lib/gcc/x86_64-linux-gnu/5/ /usr/lib/gcc/x86_64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/10/ /usr/lib/gcc/aarch64-linux-gnu/7/ /usr/lib/gcc/aarch64-linux-gnu/9/ /usr/lib/gcc/x86_64-linux-gnu/7/) find_package(Leptonica CONFIG REQUIRED) +find_package(FFMPEG REQUIRED) +find_package(libraw CONFIG REQUIRED) +find_package(Freetype REQUIRED) target_compile_options( @@ -118,39 +119,7 @@ target_compile_options( -g ) -if (SIST_DEBUG) - SET(FFMPEG_DEBUG "--enable-debug=3" "--disable-optimizations") -else() - SET(FFMPEG_DEBUG "") -endif() - include(ExternalProject) -find_program(MAKE_EXE NAMES gmake nmake make) - -ExternalProject_Add( - ffmpeg - GIT_REPOSITORY https://git.ffmpeg.org/ffmpeg.git - GIT_TAG "n4.4" - - UPDATE_COMMAND "" - PATCH_COMMAND "" - TEST_COMMAND "" - CONFIGURE_COMMAND ./configure --disable-shared --enable-static --disable-ffmpeg --disable-ffplay - --disable-ffprobe --disable-doc --disable-manpages --disable-postproc --disable-avfilter --disable-alsa - --disable-lzma --disable-xlib --disable-vdpau --disable-vaapi --disable-sdl2 - --disable-network ${FFMPEG_DEBUG} - INSTALL_COMMAND "" - - PREFIX "third-party/ext_ffmpeg" - SOURCE_DIR "third-party/ext_ffmpeg/src/ffmpeg" - BINARY_DIR "third-party/ext_ffmpeg/src/ffmpeg" - - BUILD_COMMAND ${MAKE_EXE} -j33 --silent -) - -SET(FFMPEG_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_ffmpeg/src/ffmpeg) -SET(FFMPEG_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_ffmpeg/src/ffmpeg) - ExternalProject_Add( libwpd URL http://prdownloads.sourceforge.net/libwpd/libwpd-0.9.9.tar.gz @@ -165,19 +134,20 @@ ExternalProject_Add( SOURCE_DIR "third-party/ext_libwpd/src/libwpd" BINARY_DIR "third-party/ext_libwpd/src/libwpd" - BUILD_COMMAND ${MAKE_EXE} -j33 + BUILD_COMMAND make -j33 ) SET(WPD_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwpd/src/lib/.libs/) SET(WPD_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/ext_libwpd/src/libwpd/inc/) add_dependencies( scan - ffmpeg antiword libwpd mobi ) + +target_link_directories(scan PUBLIC ${FFMPEG_LIBRARY_DIRS}) target_link_libraries( scan PUBLIC @@ -202,36 +172,26 @@ target_link_libraries( stdc++ - -Wl,--whole-archive - m - -Wl,--no-whole-archive - ${JPEG_LIBRARIES} ${Tesseract_LIBRARIES} ${LIBXML2_LIBRARIES} - ${FREETYPE_LIB} - unofficial::brotli::brotlidec-static - - ${FFMPEG_LIB_DIR}/libavformat/libavformat.a - ${FFMPEG_LIB_DIR}/libavcodec/libavcodec.a - ${FFMPEG_LIB_DIR}/libavutil/libavutil.a - ${FFMPEG_LIB_DIR}/libswresample/libswresample.a - ${FFMPEG_LIB_DIR}/libswscale/libswscale.a + Freetype::Freetype z ${CMAKE_THREAD_LIBS_INIT} - ${RAW_LIB} ${GOMP_LIB} ${CMS_LIB} ${JAS_LIB} ${GUMBO_LIB} - dl + antiword mobi unofficial::pcre::pcre unofficial::pcre::pcre16 unofficial::pcre::pcre32 unofficial::pcre::pcrecpp leptonica + libraw::raw + ${FFMPEG_LIBRARIES} ) target_include_directories( @@ -243,6 +203,7 @@ target_include_directories( ${FFMPEG_INCLUDE_DIR} ${MOBI_INCLUDE_DIR} ${WPD_INCLUDE_DIR} + ${FFMPEG_INCLUDE_DIRS} ) if (BUILD_TESTS)