mirror of
https://github.com/simon987/sist2.git
synced 2025-04-16 08:56:45 +00:00
OCR support, remove static build
This commit is contained in:
parent
f5db78a69f
commit
573f94f24e
1
.gitignore
vendored
1
.gitignore
vendored
@ -15,3 +15,4 @@ bundle*.css
|
||||
bundle.js
|
||||
*.a
|
||||
vgcore.*
|
||||
build/
|
||||
|
||||
18
.gitmodules
vendored
18
.gitmodules
vendored
@ -31,15 +31,9 @@
|
||||
[submodule "lib/mupdf"]
|
||||
path = lib/mupdf
|
||||
url = git://git.ghostscript.com/mupdf.git
|
||||
[submodule "lib/zstd"]
|
||||
path = lib/zstd
|
||||
url = https://github.com/facebook/zstd
|
||||
[submodule "lib/lz4"]
|
||||
path = lib/lz4
|
||||
url = https://github.com/lz4/lz4
|
||||
[submodule "lib/libarchive"]
|
||||
path = lib/libarchive
|
||||
url = https://github.com/libarchive/libarchive
|
||||
[submodule "lib/libxml2"]
|
||||
path = lib/libxml2
|
||||
url = https://github.com/GNOME/libxml2
|
||||
[submodule "lib/tesseract"]
|
||||
path = lib/tesseract
|
||||
url = https://github.com/tesseract-ocr/tesseract
|
||||
[submodule "lib/leptonica"]
|
||||
path = lib/leptonica
|
||||
url = https://github.com/danbloomberg/leptonica
|
||||
|
||||
155
CMakeLists.txt
155
CMakeLists.txt
@ -1,14 +1,10 @@
|
||||
cmake_minimum_required(VERSION 3.7)
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
|
||||
option(WITH_SIST2 "Build main executable" ON)
|
||||
option(WITH_SIST2_SCAN "Build scan executable" ON)
|
||||
|
||||
project(sist2 C)
|
||||
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMakeModules")
|
||||
|
||||
if (WITH_SIST2)
|
||||
add_executable(
|
||||
add_executable(
|
||||
sist2
|
||||
src/main.c
|
||||
src/sist.h
|
||||
@ -44,44 +40,7 @@ if (WITH_SIST2)
|
||||
|
||||
# utf8.h
|
||||
utf8.h/utf8.h
|
||||
)
|
||||
endif ()
|
||||
|
||||
if (WITH_SIST2_SCAN)
|
||||
add_executable(
|
||||
sist2_scan
|
||||
src/main.c
|
||||
src/sist.h
|
||||
src/io/walk.h src/io/walk.c
|
||||
src/parsing/media.h src/parsing/media.c
|
||||
src/parsing/pdf.h src/parsing/pdf.c
|
||||
src/io/store.h src/io/store.c
|
||||
src/tpool.h src/tpool.c
|
||||
src/parsing/parse.h src/parsing/parse.c
|
||||
src/io/serialize.h src/io/serialize.c
|
||||
src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
|
||||
src/parsing/text.h src/parsing/text.c
|
||||
src/util.c src/util.h
|
||||
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
|
||||
src/parsing/arc.h src/parsing/arc.c
|
||||
src/parsing/doc.h src/parsing/doc.c
|
||||
src/log.h src/log.c
|
||||
|
||||
# argparse
|
||||
argparse/argparse.h argparse/argparse.c
|
||||
|
||||
# cJSON
|
||||
cJSON/cJSON.h cJSON/cJSON.c
|
||||
|
||||
# LMDB
|
||||
lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
|
||||
lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
|
||||
src/cli.c src/cli.h
|
||||
|
||||
# utf8.h
|
||||
utf8.h/utf8.h
|
||||
)
|
||||
endif ()
|
||||
)
|
||||
|
||||
find_package(PkgConfig REQUIRED)
|
||||
set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig/")
|
||||
@ -102,8 +61,7 @@ list(REMOVE_ITEM GLIB_LIBRARIES pcre)
|
||||
list(REMOVE_ITEM GOBJECT_LIBRARIES pcre)
|
||||
list(REMOVE_ITEM UUID_LIBRARIES pcre)
|
||||
|
||||
if (WITH_SIST2)
|
||||
target_include_directories(
|
||||
target_include_directories(
|
||||
sist2 PUBLIC
|
||||
${GOBJECT_INCLUDE_DIRS}
|
||||
${GLIB_INCLUDE_DIRS}
|
||||
@ -116,22 +74,22 @@ if (WITH_SIST2)
|
||||
${PROJECT_SOURCE_DIR}/lib/mupdf/include/
|
||||
${PROJECT_SOURCE_DIR}/include/
|
||||
/usr/include/libxml2/
|
||||
)
|
||||
target_link_directories(
|
||||
${PROJECT_SOURCE_DIR}/lib/tesseract_build/include/
|
||||
)
|
||||
target_link_directories(
|
||||
sist2 PUBLIC
|
||||
${UUID_LIBRARY_DIRS}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
target_compile_options(sist2
|
||||
target_compile_options(sist2
|
||||
PRIVATE
|
||||
-Ofast
|
||||
# -march=native
|
||||
# -march=native
|
||||
-fno-stack-protector
|
||||
-fomit-frame-pointer
|
||||
)
|
||||
|
||||
TARGET_LINK_LIBRARIES(
|
||||
TARGET_LINK_LIBRARIES(
|
||||
sist2
|
||||
|
||||
${GLIB_LIBRARIES}
|
||||
@ -162,100 +120,21 @@ if (WITH_SIST2)
|
||||
freetype
|
||||
archive
|
||||
|
||||
# ${PROJECT_SOURCE_DIR}/lib/libxml2.a
|
||||
xml2
|
||||
${PROJECT_SOURCE_DIR}/lib/libopc/libmce.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libopc/libopc.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libopc/libplib.a
|
||||
)
|
||||
|
||||
endif ()
|
||||
|
||||
if (WITH_SIST2_SCAN)
|
||||
|
||||
set_target_properties(
|
||||
sist2_scan
|
||||
PROPERTIES COMPILE_DEFINITIONS SIST_SCAN_ONLY
|
||||
)
|
||||
set_target_properties(
|
||||
sist2_scan
|
||||
PROPERTIES
|
||||
COMPILE_DEFINITIONS SIST_SCAN_ONLY
|
||||
LINK_FLAGS -static
|
||||
)
|
||||
target_include_directories(
|
||||
sist2_scan PUBLIC
|
||||
${LIBMAGIC_INCLUDE_DIRS}
|
||||
${GOBJECT_INCLUDE_DIRS}
|
||||
${OPENSSL_INCLUDE_DIR}
|
||||
${PROJECT_SOURCE_DIR}/lib/ffmpeg/
|
||||
${GLIB_INCLUDE_DIRS}
|
||||
${UUID_INCLUDE_DIRS}
|
||||
${FREETYPE_INCLUDE_DIRS}
|
||||
${PROJECT_SOURCE_DIR}/
|
||||
${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
|
||||
${PROJECT_SOURCE_DIR}/lib/onion/src/
|
||||
${PROJECT_SOURCE_DIR}/lib/mupdf/include/
|
||||
${PROJECT_SOURCE_DIR}/include/
|
||||
/usr/include/libxml2/
|
||||
)
|
||||
target_link_directories(
|
||||
sist2_scan PUBLIC
|
||||
${UUID_LIBRARY_DIRS}
|
||||
)
|
||||
target_compile_options(sist2_scan
|
||||
PRIVATE
|
||||
-Ofast
|
||||
# -march=native
|
||||
-fno-stack-protector
|
||||
-fomit-frame-pointer
|
||||
)
|
||||
TARGET_LINK_LIBRARIES(
|
||||
sist2_scan
|
||||
|
||||
${GLIB_LIBRARIES}
|
||||
${GOBJECT_LIBRARIES}
|
||||
${UUID_LIBRARIES}
|
||||
|
||||
# ffmpeg
|
||||
${PROJECT_SOURCE_DIR}/lib/libavcodec.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libavformat.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libavutil.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libswscale.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libswresample.a
|
||||
|
||||
# mupdf
|
||||
${PROJECT_SOURCE_DIR}/lib/libmupdf.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
|
||||
|
||||
${PROJECT_SOURCE_DIR}/lib/libbz2.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libmagic.a
|
||||
|
||||
pthread
|
||||
m
|
||||
${PROJECT_SOURCE_DIR}/lib/libharfbuzz.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libopenjp2.a
|
||||
freetype
|
||||
|
||||
${PROJECT_SOURCE_DIR}/lib/libarchive.a
|
||||
${PROJECT_SOURCE_DIR}/lib/liblz4.a
|
||||
${PROJECT_SOURCE_DIR}/lib/liblzma.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libzstd.a
|
||||
|
||||
${PROJECT_SOURCE_DIR}/lib/libxml2.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libopc/libmce.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libopc/libopc.a
|
||||
${PROJECT_SOURCE_DIR}/lib/libopc/libplib.a
|
||||
)
|
||||
endif ()
|
||||
${PROJECT_SOURCE_DIR}/lib/libtesseract.a
|
||||
${PROJECT_SOURCE_DIR}/lib/liblept.a
|
||||
png
|
||||
tiff
|
||||
stdc++
|
||||
)
|
||||
|
||||
add_custom_target(
|
||||
before_sist2
|
||||
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/before_build.sh
|
||||
)
|
||||
|
||||
IF (WITH_SIST2)
|
||||
add_dependencies(sist2 before_sist2)
|
||||
else ()
|
||||
add_dependencies(sist2_scan before_sist2)
|
||||
endif ()
|
||||
add_dependencies(sist2 before_sist2)
|
||||
|
||||
@ -2,7 +2,16 @@ FROM ubuntu:19.10
|
||||
MAINTAINER simon987 <me@simon987.net>
|
||||
|
||||
RUN apt update
|
||||
RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1
|
||||
RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 curl
|
||||
|
||||
RUN mkdir -p /usr/share/tessdata && \
|
||||
cd /usr/share/tessdata/ && \
|
||||
curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
|
||||
curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
|
||||
curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
|
||||
curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
|
||||
curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
|
||||
curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
|
||||
|
||||
ADD sist2 /root/sist2
|
||||
|
||||
|
||||
25
README.md
25
README.md
@ -11,15 +11,17 @@ sist2 (Simple incremental search tool)
|
||||
|
||||
* Fast, low memory usage, multi-threaded
|
||||
* Portable (all its features are packaged in a single executable)
|
||||
* Extracts text from common file types\*
|
||||
* Generates thumbnails\*
|
||||
* Extracts text from common file types \*
|
||||
* Generates thumbnails \*
|
||||
* Incremental scanning
|
||||
* Automatic tagging from file attributes via [user scripts](scripting/README.md)
|
||||
* Recursive scan inside archive files \*\*
|
||||
* OCR support with tesseract \*\*\*
|
||||
|
||||
|
||||
\* See [format support](#format-support)
|
||||
\** See [Archive files](#archive-files)
|
||||
\*\* See [Archive files](#archive-files)
|
||||
\*\*\* See [OCR](#ocr)
|
||||
|
||||
## Getting Started
|
||||
|
||||
@ -85,7 +87,7 @@ docker stop sist2
|
||||
|
||||
File type | Library | Content | Thumbnail | Metadata
|
||||
:---|:---|:---|:---|:---
|
||||
pdf,xps,cbz,fb2,epub | MuPDF | yes | yes, `png` | title |
|
||||
pdf,xps,cbz,fb2,epub | MuPDF | text+ocr | yes, `png` | title |
|
||||
`audio/*` | ffmpeg | - | yes, `jpeg` | ID3 tags |
|
||||
`video/*` | ffmpeg | - | yes, `jpeg` | title, comment, artist |
|
||||
`image/*` | ffmpeg | - | yes, `jpeg` | `EXIF:Artist`, `EXIF:ImageDescription` |
|
||||
@ -111,6 +113,21 @@ scan is also supported.
|
||||
To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
|
||||
|
||||
|
||||
### OCR
|
||||
|
||||
You can enable OCR support for pdf,xps,cbz,fb2,epub file types with the
|
||||
`--ocr <lang>` option. Download the language data files with your
|
||||
package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
|
||||
|
||||
The `simon987/sist2` github image comes with common languages
|
||||
(hin, jpn, eng, fra, rus, spa) pre-installed.
|
||||
|
||||
Examples
|
||||
```bash
|
||||
sist2 scan --ocr jpn ~/Books/Manga/
|
||||
sist2 scan --ocr eng ~/Books/Textbooks/
|
||||
```
|
||||
|
||||
|
||||
## Build from source
|
||||
|
||||
|
||||
1
lib/leptonica
Submodule
1
lib/leptonica
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit cc03be70fded1ad6a8cedad656456386a1bd08e8
|
||||
@ -1 +0,0 @@
|
||||
Subproject commit 6f4fceb714868f2ddbccef4871acc1670e45fc03
|
||||
@ -1 +0,0 @@
|
||||
Subproject commit 0815302dee2b78139832c2080348086a0564836b
|
||||
1
lib/lz4
1
lib/lz4
@ -1 +0,0 @@
|
||||
Subproject commit 0f749838bf29bc0d1df428e23cf3dbb76ec4e9fc
|
||||
1
lib/tesseract
Submodule
1
lib/tesseract
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit f268e6615e619a7140de78d9db3f8240fa8c68bd
|
||||
1
lib/zstd
1
lib/zstd
@ -1 +0,0 @@
|
||||
Subproject commit d73e2fb465c9289f5a6871cef1fe48de917a5cc2
|
||||
@ -70,43 +70,21 @@ make -j $THREADS
|
||||
cd ..
|
||||
mv libmagic/src/.libs/libmagic.a .
|
||||
|
||||
# libarchive
|
||||
cd libarchive/build
|
||||
# tesseract
|
||||
cd tesseract
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DSTATIC=on -DBUILD_TRAINING_TOOLS=off ..
|
||||
make -j $THREADS
|
||||
cd ../..
|
||||
mv tesseract/build/libtesseract.a .
|
||||
|
||||
# leptonica
|
||||
cd leptonica
|
||||
./autogen.sh
|
||||
cd ..
|
||||
./configure --without-nettle --without-expat --without-xml2 --without-openssl
|
||||
./configure --without-zlib --without-jpeg --without-giflib \
|
||||
--without-giflib --without-libwebp --without-libwebpmux --without-libopenjpeg \
|
||||
--enable-static --disable-shared
|
||||
make -j $THREADS
|
||||
cd ..
|
||||
mv libarchive/.libs/libarchive.a .
|
||||
|
||||
# lz4
|
||||
cd lz4
|
||||
make -j $THREADS
|
||||
cd ..
|
||||
mv lz4/lib/liblz4.a .
|
||||
|
||||
# lzma
|
||||
wget https://newcontinuum.dl.sourceforge.net/project/lzmautils/xz-5.2.3.tar.gz
|
||||
tar -xzf xz-5.2.3.tar.gz
|
||||
rm xz-5.2.3.tar.gz
|
||||
cd xz-5.2.3
|
||||
./autogen.sh
|
||||
./configure
|
||||
make -j $THREADS
|
||||
cd ..
|
||||
mv xz-5.2.3/src/liblzma/.libs/liblzma.a .
|
||||
|
||||
# zstd
|
||||
cd zstd
|
||||
make -j $THREADS
|
||||
cd ..
|
||||
mv zstd/lib/libzstd.a .
|
||||
|
||||
# xml2
|
||||
cd libxml2
|
||||
./autogen.sh --without-zlib --without-lzma
|
||||
make
|
||||
cd ..
|
||||
mv libxml2/.libs/libxml2.a .
|
||||
|
||||
cd ..
|
||||
mv leptonica/src/.libs/liblept.a .
|
||||
|
||||
22
src/cli.c
22
src/cli.c
@ -1,6 +1,8 @@
|
||||
#include "cli.h"
|
||||
#include "ctx.h"
|
||||
|
||||
#include <tesseract/capi.h>
|
||||
|
||||
#define DEFAULT_OUTPUT "index.sist2/"
|
||||
#define DEFAULT_CONTENT_SIZE 32768
|
||||
#define DEFAULT_QUALITY 5
|
||||
@ -35,8 +37,6 @@ void scan_args_destroy(scan_args_t *args) {
|
||||
free(args);
|
||||
}
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
|
||||
void index_args_destroy(index_args_t *args) {
|
||||
//todo
|
||||
free(args);
|
||||
@ -47,8 +47,6 @@ void web_args_destroy(web_args_t *args) {
|
||||
free(args);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "Required positional argument: PATH.\n");
|
||||
@ -136,6 +134,17 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (args->tesseract_lang != NULL) {
|
||||
TessBaseAPI *api = TessBaseAPICreate();
|
||||
ret = TessBaseAPIInit3(api, TESS_DATAPATH, args->tesseract_lang);
|
||||
if (ret != 0) {
|
||||
fprintf(stderr, "Could not initialize tesseract with lang '%s'\n", args->tesseract_lang);
|
||||
return 1;
|
||||
}
|
||||
TessBaseAPIEnd(api);
|
||||
TessBaseAPIDelete(api);
|
||||
}
|
||||
|
||||
LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
|
||||
LOG_DEBUGF("cli.c", "arg size=%d", args->size)
|
||||
LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
|
||||
@ -147,12 +156,11 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
|
||||
LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
|
||||
LOG_DEBUGF("cli.c", "arg path=%s", args->path)
|
||||
LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
|
||||
LOG_DEBUGF("cli.c", "arg ocr=%s", args->tesseract_lang)
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
|
||||
int index_args_validate(index_args_t *args, int argc, const char **argv) {
|
||||
|
||||
if (argc < 2) {
|
||||
@ -273,5 +281,3 @@ web_args_t *web_args_create() {
|
||||
return args;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
@ -16,13 +16,13 @@ typedef struct scan_args {
|
||||
char *path;
|
||||
char *archive;
|
||||
archive_mode_t archive_mode;
|
||||
char *tesseract_lang;
|
||||
} scan_args_t;
|
||||
|
||||
scan_args_t *scan_args_create();
|
||||
void scan_args_destroy(scan_args_t *args);
|
||||
int scan_args_validate(scan_args_t *args, int argc, const char **argv);
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
typedef struct index_args {
|
||||
char *es_url;
|
||||
const char *index_path;
|
||||
@ -51,6 +51,5 @@ void web_args_destroy(web_args_t *args);
|
||||
|
||||
int index_args_validate(index_args_t *args, int argc, const char **argv);
|
||||
int web_args_validate(web_args_t *args, int argc, const char **argv);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@ -27,6 +27,7 @@ struct {
|
||||
GHashTable *copy_table;
|
||||
|
||||
pthread_mutex_t mupdf_mu;
|
||||
char * tesseract_lang;
|
||||
} ScanCtx;
|
||||
|
||||
struct {
|
||||
@ -35,7 +36,6 @@ struct {
|
||||
int no_color;
|
||||
} LogCtx;
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
struct {
|
||||
char *es_url;
|
||||
int batch_size;
|
||||
@ -47,7 +47,6 @@ struct {
|
||||
char *b64credentials;
|
||||
struct index_t indices[16];
|
||||
} WebCtx;
|
||||
#endif
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
25
src/main.c
25
src/main.c
@ -1,16 +1,12 @@
|
||||
#include "sist.h"
|
||||
#include "ctx.h"
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
#define DESCRIPTION "Lightning-fast file system indexer and search tool."
|
||||
#else
|
||||
#define DESCRIPTION "Lightning-fast file system indexer and search tool. (SCAN ONLY)"
|
||||
#endif
|
||||
|
||||
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
|
||||
|
||||
|
||||
static const char *const Version = "1.1.15";
|
||||
static const char *const Version = "1.2.0";
|
||||
static const char *const usage[] = {
|
||||
"sist2 scan [OPTION]... PATH",
|
||||
"sist2 index [OPTION]... INDEX",
|
||||
@ -19,9 +15,7 @@ static const char *const usage[] = {
|
||||
};
|
||||
|
||||
void global_init() {
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
curl_global_init(CURL_GLOBAL_NOTHING);
|
||||
#endif
|
||||
av_log_set_level(AV_LOG_QUIET);
|
||||
opcInitLibrary();
|
||||
}
|
||||
@ -55,6 +49,7 @@ void sist2_scan(scan_args_t *args) {
|
||||
strncpy(ScanCtx.index.desc.name, args->name, sizeof(ScanCtx.index.desc.name));
|
||||
strncpy(ScanCtx.index.desc.root, args->path, sizeof(ScanCtx.index.desc.root));
|
||||
ScanCtx.index.desc.root_len = (short) strlen(ScanCtx.index.desc.root);
|
||||
ScanCtx.tesseract_lang = args->tesseract_lang;
|
||||
|
||||
init_dir(ScanCtx.index.path);
|
||||
|
||||
@ -122,8 +117,6 @@ void sist2_scan(scan_args_t *args) {
|
||||
store_destroy(ScanCtx.index.store);
|
||||
}
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
|
||||
void sist2_index(index_args_t *args) {
|
||||
|
||||
IndexCtx.es_url = args->es_url;
|
||||
@ -198,18 +191,14 @@ void sist2_web(web_args_t *args) {
|
||||
serve(args->bind, args->port);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
int main(int argc, const char *argv[]) {
|
||||
|
||||
global_init();
|
||||
|
||||
scan_args_t *scan_args = scan_args_create();
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
index_args_t *index_args = index_args_create();
|
||||
web_args_t *web_args = web_args_create();
|
||||
#endif
|
||||
|
||||
int arg_version = 0;
|
||||
|
||||
@ -240,8 +229,9 @@ int main(int argc, const char *argv[]) {
|
||||
OPT_STRING(0, "archive", &scan_args->archive, "Archive file mode (skip|list|shallow|recurse). "
|
||||
"skip: Don't parse, list: only get file names as text, "
|
||||
"shallow: Don't parse archives inside archives. DEFAULT: recurse"),
|
||||
OPT_STRING(0, "ocr", &scan_args->tesseract_lang, "Tesseract language (use tesseract --list-langs to see "
|
||||
"which are installed on your machine)"),
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
OPT_GROUP("Index options"),
|
||||
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"),
|
||||
OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
|
||||
@ -255,7 +245,6 @@ int main(int argc, const char *argv[]) {
|
||||
OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"),
|
||||
OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"),
|
||||
OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"),
|
||||
#endif
|
||||
|
||||
OPT_END(),
|
||||
};
|
||||
@ -274,10 +263,8 @@ int main(int argc, const char *argv[]) {
|
||||
LogCtx.verbose = 1;
|
||||
}
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
web_args->es_url = common_es_url;
|
||||
index_args->es_url = common_es_url;
|
||||
#endif
|
||||
|
||||
if (argc == 0) {
|
||||
argparse_usage(&argparse);
|
||||
@ -292,7 +279,6 @@ int main(int argc, const char *argv[]) {
|
||||
|
||||
}
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
else if (strcmp(argv[0], "index") == 0) {
|
||||
|
||||
int err = index_args_validate(index_args, argc, argv);
|
||||
@ -310,7 +296,6 @@ int main(int argc, const char *argv[]) {
|
||||
sist2_web(web_args);
|
||||
|
||||
}
|
||||
#endif
|
||||
else {
|
||||
fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
|
||||
argparse_usage(&argparse);
|
||||
@ -320,10 +305,8 @@ int main(int argc, const char *argv[]) {
|
||||
|
||||
scan_args_destroy(scan_args);
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
index_args_destroy(index_args);
|
||||
web_args_destroy(web_args);
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -127,6 +127,7 @@ AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int st
|
||||
#define APPEND_TAG_META(doc, tag_, keyname) \
|
||||
text_buffer_t tex = text_buffer_create(-1); \
|
||||
text_buffer_append_string0(&tex, tag_->value); \
|
||||
text_buffer_terminate_string(&tex); \
|
||||
meta_line_t *meta_tag = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); \
|
||||
meta_tag->key = keyname; \
|
||||
strcpy(meta_tag->strval, tex.dyn_buffer.buf); \
|
||||
|
||||
@ -1,6 +1,10 @@
|
||||
#include "pdf.h"
|
||||
#include "src/ctx.h"
|
||||
|
||||
#define MIN_OCR_SIZE 128
|
||||
__thread text_buffer_t thread_buffer;
|
||||
|
||||
|
||||
fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
|
||||
|
||||
int err = 0;
|
||||
@ -86,14 +90,13 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
|
||||
return cover;
|
||||
}
|
||||
|
||||
void fz_err_callback(void *user, __attribute__((unused)) const char *message) {
|
||||
void fz_err_callback(void *user, UNUSED(const char *message)) {
|
||||
if (LogCtx.verbose) {
|
||||
document_t *doc = (document_t*) user;
|
||||
document_t *doc = (document_t *) user;
|
||||
LOG_WARNINGF(doc->filepath, "FZ: %s", message)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
__always_inline
|
||||
void init_ctx(fz_context *ctx, document_t *doc) {
|
||||
fz_disable_icc(ctx);
|
||||
@ -124,6 +127,40 @@ int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
void fill_image(fz_context *ctx, UNUSED(fz_device *dev),
|
||||
fz_image *img, UNUSED(fz_matrix ctm), UNUSED(float alpha),
|
||||
UNUSED(fz_color_params color_params)) {
|
||||
|
||||
int l2factor = 0;
|
||||
|
||||
if (img->w > MIN_OCR_SIZE && img->h > MIN_OCR_SIZE) {
|
||||
|
||||
fz_pixmap *pix = img->get_pixmap(ctx, img, NULL, img->w, img->h, &l2factor);
|
||||
|
||||
if (pix->h > MIN_OCR_SIZE && img->h > MIN_OCR_SIZE && img->xres != 0) {
|
||||
TessBaseAPI *api = TessBaseAPICreate();
|
||||
TessBaseAPIInit3(api, TESS_DATAPATH, ScanCtx.tesseract_lang);
|
||||
|
||||
TessBaseAPISetImage(api, pix->samples, pix->w, pix->h, pix->n, pix->stride);
|
||||
TessBaseAPISetSourceResolution(api, pix->xres);
|
||||
|
||||
char *text = TessBaseAPIGetUTF8Text(api);
|
||||
size_t len = strlen(text);
|
||||
text_buffer_append_string(&thread_buffer, text, len - 1);
|
||||
LOG_DEBUGF(
|
||||
"pdf.c",
|
||||
"(OCR) %dx%d got %dB from tesseract (%s), buffer:%dB",
|
||||
pix->w, pix->h, len, ScanCtx.tesseract_lang, thread_buffer.dyn_buffer.cur
|
||||
)
|
||||
|
||||
TessBaseAPIEnd(api);
|
||||
TessBaseAPIDelete(api);
|
||||
fz_drop_pixmap(ctx, pix);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
||||
|
||||
if (buf == NULL) {
|
||||
@ -210,7 +247,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
||||
|
||||
if (ScanCtx.content_size > 0) {
|
||||
fz_stext_options opts = {0};
|
||||
text_buffer_t text_buf = text_buffer_create(ScanCtx.content_size);
|
||||
thread_buffer = text_buffer_create(ScanCtx.content_size);
|
||||
|
||||
for (int current_page = 0; current_page < page_count; current_page++) {
|
||||
fz_page *page = NULL;
|
||||
@ -224,7 +261,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
||||
err = ctx->error.errcode;
|
||||
if (err != 0) {
|
||||
LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
|
||||
text_buffer_destroy(&text_buf);
|
||||
text_buffer_destroy(&thread_buffer);
|
||||
fz_drop_page(ctx, page);
|
||||
fz_drop_stream(ctx, stream);
|
||||
fz_drop_document(ctx, fzdoc);
|
||||
@ -235,6 +272,15 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
||||
|
||||
fz_stext_page *stext = fz_new_stext_page(ctx, fz_bound_page(ctx, page));
|
||||
fz_device *dev = fz_new_stext_device(ctx, stext, &opts);
|
||||
dev->stroke_path = NULL;
|
||||
dev->stroke_text = NULL;
|
||||
dev->clip_text = NULL;
|
||||
dev->clip_stroke_path = NULL;
|
||||
dev->clip_stroke_text = NULL;
|
||||
|
||||
if (ScanCtx.tesseract_lang != NULL) {
|
||||
dev->fill_image = fill_image;
|
||||
}
|
||||
|
||||
fz_var(err);
|
||||
fz_try(ctx)
|
||||
@ -249,7 +295,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
||||
|
||||
if (err != 0) {
|
||||
LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
|
||||
text_buffer_destroy(&text_buf);
|
||||
text_buffer_destroy(&thread_buffer);
|
||||
fz_drop_page(ctx, page);
|
||||
fz_drop_stext_page(ctx, stext);
|
||||
fz_drop_stream(ctx, stream);
|
||||
@ -260,7 +306,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
||||
|
||||
fz_stext_block *block = stext->first_block;
|
||||
while (block != NULL) {
|
||||
int ret = read_stext_block(block, &text_buf);
|
||||
int ret = read_stext_block(block, &thread_buffer);
|
||||
if (ret == TEXT_BUF_FULL) {
|
||||
break;
|
||||
}
|
||||
@ -269,18 +315,18 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
||||
fz_drop_stext_page(ctx, stext);
|
||||
fz_drop_page(ctx, page);
|
||||
|
||||
if (text_buf.dyn_buffer.cur >= text_buf.dyn_buffer.size) {
|
||||
if (thread_buffer.dyn_buffer.cur >= thread_buffer.dyn_buffer.size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
text_buffer_terminate_string(&text_buf);
|
||||
text_buffer_terminate_string(&thread_buffer);
|
||||
|
||||
meta_line_t *meta_content = malloc(sizeof(meta_line_t) + text_buf.dyn_buffer.cur);
|
||||
meta_line_t *meta_content = malloc(sizeof(meta_line_t) + thread_buffer.dyn_buffer.cur);
|
||||
meta_content->key = MetaContent;
|
||||
memcpy(meta_content->strval, text_buf.dyn_buffer.buf, text_buf.dyn_buffer.cur);
|
||||
memcpy(meta_content->strval, thread_buffer.dyn_buffer.buf, thread_buffer.dyn_buffer.cur);
|
||||
APPEND_META(doc, meta_content)
|
||||
|
||||
text_buffer_destroy(&text_buf);
|
||||
text_buffer_destroy(&thread_buffer);
|
||||
}
|
||||
|
||||
fz_drop_stream(ctx, stream);
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#define SIST2_PDF_H
|
||||
|
||||
#include "src/sist.h"
|
||||
#include <tesseract/capi.h>
|
||||
|
||||
|
||||
void parse_pdf(void *buf, size_t buf_len, document_t *doc);
|
||||
|
||||
@ -25,6 +25,7 @@ void parse_text(int bytes_read, struct vfile *f, char *buf, document_t *doc) {
|
||||
}
|
||||
text_buffer_t tex = text_buffer_create(ScanCtx.content_size);
|
||||
text_buffer_append_string(&tex, intermediate_buf, intermediate_buf_len);
|
||||
text_buffer_terminate_string(&tex);
|
||||
|
||||
meta_line_t *meta = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur);
|
||||
meta->key = MetaContent;
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
#define SIST_H
|
||||
|
||||
#define UUID_STR_LEN 37
|
||||
#define UNUSED(x) __attribute__((__unused__)) x
|
||||
#define TESS_DATAPATH "/usr/share/tessdata/"
|
||||
|
||||
#include <glib-2.0/glib.h>
|
||||
#include <unistd.h>
|
||||
@ -33,14 +35,12 @@
|
||||
#include <opc/opc.h>
|
||||
#include <libxml/xmlstring.h>
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
#include <onion/onion.h>
|
||||
#include <onion/handler.h>
|
||||
#include <onion/block.h>
|
||||
#include <onion/shortcuts.h>
|
||||
#include <onion/codecs.h>
|
||||
#include <curl/curl.h>
|
||||
#endif
|
||||
|
||||
#include "cJSON/cJSON.h"
|
||||
|
||||
@ -62,12 +62,10 @@
|
||||
#include "log.h"
|
||||
#include "utf8.h/utf8.h"
|
||||
|
||||
#ifndef SIST_SCAN_ONLY
|
||||
#include "src/index/elastic.h"
|
||||
#include "index/web.h"
|
||||
#include "web/serve.h"
|
||||
#include "web/auth_basic.h"
|
||||
#endif
|
||||
|
||||
;
|
||||
|
||||
|
||||
@ -70,8 +70,6 @@ typedef struct document {
|
||||
short ext;
|
||||
meta_line_t *meta_head;
|
||||
meta_line_t *meta_tail;
|
||||
struct document *child_head;
|
||||
struct document *child_tail;
|
||||
char *filepath;
|
||||
} document_t;
|
||||
|
||||
@ -79,8 +77,6 @@ typedef struct vfile vfile_t;
|
||||
|
||||
typedef int (*read_func_t)(struct vfile *, void *buf, size_t size);
|
||||
|
||||
typedef int (*seek_func_t)(struct vfile *, size_t size, int whence);
|
||||
|
||||
typedef void (*close_func_t)(struct vfile *);
|
||||
|
||||
typedef struct vfile {
|
||||
|
||||
@ -149,7 +149,6 @@ int text_buffer_append_string(text_buffer_t *buf, char *str, size_t len) {
|
||||
(0xe0 == (0xf0 & str[0]) && len < 3) ||
|
||||
(0xc0 == (0xe0 & str[0]) && len == 1) ||
|
||||
*(str) == 0) {
|
||||
text_buffer_terminate_string(buf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -158,7 +157,6 @@ int text_buffer_append_string(text_buffer_t *buf, char *str, size_t len) {
|
||||
text_buffer_append_char(buf, c);
|
||||
}
|
||||
}
|
||||
text_buffer_terminate_string(buf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -169,7 +167,6 @@ int text_buffer_append_string0(text_buffer_t *buf, char *str) {
|
||||
text_buffer_append_char(buf, c);
|
||||
}
|
||||
}
|
||||
text_buffer_terminate_string(buf);
|
||||
}
|
||||
|
||||
int text_buffer_append_char(text_buffer_t *buf, int c) {
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -11,7 +11,7 @@
|
||||
|
||||
<nav class="navbar navbar-expand-lg">
|
||||
<a class="navbar-brand" href="/">sist2</a>
|
||||
<span class="badge badge-pill version">v1.1.15</span>
|
||||
<span class="badge badge-pill version">v1.2.0</span>
|
||||
<span class="tagline">Lightning-fast file system indexer and search tool </span>
|
||||
<a style="margin-left: auto" id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
|
||||
</nav>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user