From 3da2c8cae3e53bd70cc7511d0ab52ede56208b9b Mon Sep 17 00:00:00 2001 From: simon987 Date: Mon, 14 Jun 2021 14:02:16 -0400 Subject: [PATCH] Update CI scripts, Dockerfiles, enable arm64 build again --- .dockerignore | 1 + .drone.yml | 54 ++++++++++++++++++++++----------------------- .gitignore | 3 ++- Dockerfile | 4 +--- Dockerfile.arm64 | 29 +++++++++++++++++++++++- README.md | 32 ++++++++++++++++++++++----- ci/build.sh | 3 +-- ci/build_arm64.sh | 4 ++-- docs/USAGE.md | 3 +-- third-party/libscan | 2 +- 10 files changed, 90 insertions(+), 45 deletions(-) diff --git a/.dockerignore b/.dockerignore index 431b3ae..d7a0f76 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,3 +22,4 @@ third-party/libscan/libscan-test-files/ **/scan_a_test Dockerfile *.idx/ +VERSION diff --git a/.drone.yml b/.drone.yml index 1b2ed2d..bc2c429 100644 --- a/.drone.yml +++ b/.drone.yml @@ -43,30 +43,30 @@ steps: - ./sist2-x64-linux - ./sist2-x64-linux-debug -#--- -#kind: pipeline -#type: docker -#name: arm64 -# -#platform: -# arch: arm64 -# -#steps: -# - name: build -# image: simon987/ubuntu_ci_arm -# commands: -# - ./ci/build_arm64.sh -# - name: scp files -# image: appleboy/drone-scp -# settings: -# host: -# from_secret: SSH_HOST -# port: -# from_secret: SSH_PORT -# user: -# from_secret: SSH_USER -# key: -# from_secret: SSH_KEY -# target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/ -# source: -# - ./sist2-arm64-linux +--- +kind: pipeline +type: docker +name: arm64 + +platform: + arch: arm64 + +steps: + - name: build + image: simon987/sist2-build-arm64 + commands: + - ./ci/build_arm64.sh + - name: scp files + image: appleboy/drone-scp + settings: + host: + from_secret: SSH_HOST + port: + from_secret: SSH_PORT + user: + from_secret: SSH_USER + key: + from_secret: SSH_KEY + target: /files/sist2/${DRONE_REPO_OWNER}_${DRONE_REPO_NAME}/arm_${DRONE_BRANCH}_${DRONE_BUILD_NUMBER}_${DRONE_COMMIT}/ + source: + - ./sist2-arm64-linux diff --git a/.gitignore b/.gitignore index 471fda9..a1a6f24 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ bundle.js vgcore.* build/ third-party/ -*.idx/ \ No newline at end of file +*.idx/ +VERSION \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4b525cd..94f3603 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,7 @@ RUN strip sist2 FROM ubuntu:20.10 -RUN apt update -RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 \ - curl libtiff5 libpng16-16 libpcre3 +RUN apt update && apt install -y curl RUN mkdir -p /usr/share/tessdata && \ cd /usr/share/tessdata/ && \ diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index f87f5c1..4a03f60 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -1 +1,28 @@ -# TODO \ No newline at end of file +FROM simon987/sist2-build-arm64 as build +MAINTAINER simon987 + +WORKDIR /build/ +ADD . /build/ +RUN cmake -DSIST_DEBUG=off -DBUILD_TESTS=off -DCMAKE_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake . +RUN make -j$(nproc) +RUN strip sist2 + +FROM ubuntu:20.10 + +RUN apt update && apt install -y curl + +RUN mkdir -p /usr/share/tessdata && \ + cd /usr/share/tessdata/ && \ + curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\ + curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\ + curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\ + curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\ + curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\ + curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata + +COPY --from=build /build/sist2 /root/sist2 + +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +ENTRYPOINT ["/root/sist2"] \ No newline at end of file diff --git a/README.md b/README.md index 2c40135..886da33 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,8 @@ sist2 (Simple incremental search tool) ``` 1. Download sist2 executable 1. Download the [latest sist2 release](https://github.com/simon987/sist2/releases) * - 1. *(or)* Download a [development snapshot](https://files.simon987.net/sist2/simon987_sist2/) *(Not recommended!)* - 1. *(or)* `docker pull simon987/sist2:latest` + 1. *(or)* Download a [development snapshot](https://files.simon987.net/.gate/sist2/simon987_sist2/) *(Not recommended!)* + 1. *(or)* `docker pull simon987/sist2:2.10.1-x64-linux` 1. See [Usage guide](docs/USAGE.md) @@ -67,7 +67,7 @@ See [Usage guide](docs/USAGE.md) for more details ## Format support -File type | Library | Content | Thumbnail | Metadata +File type | Library | Content | Thumbnail | Metadata :---|:---|:---|:---|:--- pdf,xps,fb2,epub | MuPDF | text+ocr | yes | author, title | cbz,cbr | *(none)* | - | yes | - | @@ -116,15 +116,35 @@ sist2 scan --ocr eng ~/Books/Textbooks/ ## Build from source -You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries (GCC 7+ required). +You can compile **sist2** by yourself if you don't want to use the pre-compiled binaries + +### With docker (recommended) + +```bash +git clone --recursive https://github.com/simon987/sist2/ +cd sist2 +docker build . -f ./Dockerfile -t my-sist2-image +docker run --rm my-sist2-image cat /root/sist2 > sist2-x64-linux +``` + +### On a linux computer 1. Install compile-time dependencies ```bash - vcpkg install lmdb cjson glib libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 ffmpeg zstd gtest mongoose libmagic libraw curl[core,ssl] jbig2dec brotli libmupdf + apt install gcc g++ python3 yasm ragel automake autotools-dev wget libtool libssl-dev curl zip unzip tar xorg-dev libglu1-mesa-dev libxcursor-dev libxml2-dev libxinerama-dev gettext nasm git ``` + +1. Apply vcpkg patches, as per [sist2-build](https://github.com/simon987/sist2-build) Dockerfile -2. Build +1. Install vcpkg dependencies + + ```bash + vcpkg install curl[core,openssl] + vcpkg install lmdb cjson glib brotli libarchive[core,bzip2,libxml2,lz4,lzma,lzo] pthread tesseract libxml2 libmupdf gtest mongoose libuuid libmagic libraw jasper lcms gumbo + ``` + +1. Build ```bash git clone --recursive https://github.com/simon987/sist2/ cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake . diff --git a/ci/build.sh b/ci/build.sh index 0333686..b690de2 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -2,7 +2,7 @@ VCPKG_ROOT="/vcpkg" -rm *.gz +rm *.gz &>/dev/null git submodule update --init --recursive @@ -11,7 +11,6 @@ cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystem make -j $(nproc) strip sist2 ./sist2 -v > VERSION -cp sist2 Docker/ mv sist2 sist2-x64-linux rm -rf CMakeFiles CMakeCache.txt diff --git a/ci/build_arm64.sh b/ci/build_arm64.sh index ebea055..0a48f2d 100755 --- a/ci/build_arm64.sh +++ b/ci/build_arm64.sh @@ -2,12 +2,12 @@ VCPKG_ROOT="/vcpkg" -rm *.gz +rm *.gz &>/dev/null git submodule update --init --recursive rm -rf CMakeFiles CMakeCache.txt cmake -DSIST_DEBUG=off -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" . -make -j 4 +make -j $(nproc) strip sist2 mv sist2 sist2-arm64-linux \ No newline at end of file diff --git a/docs/USAGE.md b/docs/USAGE.md index a1cb6de..2fbde88 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -357,8 +357,7 @@ You can safely copy the `/tags/` database to another index. See [Automatic tagging](#automatic-tagging) for information about tag hierarchies and tag colors. -\* *It can take a few seconds to take effect in new search queries, and the page needs - to be reloaded for the tags tab to update* +\* *It can take a few seconds to take effect in new search queries.* ### Automatic tagging diff --git a/third-party/libscan b/third-party/libscan index 9be4f02..a12ec1c 160000 --- a/third-party/libscan +++ b/third-party/libscan @@ -1 +1 @@ -Subproject commit 9be4f02851107edac65894a1fdde16a80cad43ac +Subproject commit a12ec1cb06bfdecce783c791b205eb6db7ea0fd5