simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-16 08:56:45 +00:00
Code Issues Projects Releases Wiki Activity

OCR support, remove static build

Browse Source
This commit is contained in:
simon 2020-01-14 10:26:40 -05:00
parent f5db78a69f
commit 573f94f24e
25 changed files with 239 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -15,3 +15,4 @@ bundle*.css
bundle.js bundle.js
*.a *.a
vgcore.* vgcore.*
build/

18
.gitmodules vendored
View File

@ -31,15 +31,9 @@
[submodule "lib/mupdf"] [submodule "lib/mupdf"]
path = lib/mupdf path = lib/mupdf
url = git://git.ghostscript.com/mupdf.git url = git://git.ghostscript.com/mupdf.git
[submodule "lib/zstd"] [submodule "lib/tesseract"]
path = lib/zstd path = lib/tesseract
url = https://github.com/facebook/zstd url = https://github.com/tesseract-ocr/tesseract
[submodule "lib/lz4"] [submodule "lib/leptonica"]
path = lib/lz4 path = lib/leptonica
url = https://github.com/lz4/lz4 url = https://github.com/danbloomberg/leptonica
[submodule "lib/libarchive"]
path = lib/libarchive
url = https://github.com/libarchive/libarchive
[submodule "lib/libxml2"]
path = lib/libxml2
url = https://github.com/GNOME/libxml2

311
CMakeLists.txt
View File

@ -1,87 +1,46 @@
cmake_minimum_required(VERSION 3.7) cmake_minimum_required(VERSION 3.7)
set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD 11)
option(WITH_SIST2 "Build main executable" ON)
option(WITH_SIST2_SCAN "Build scan executable" ON)
project(sist2 C) project(sist2 C)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMakeModules") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMakeModules")
if (WITH_SIST2) add_executable(
add_executable( sist2
sist2 src/main.c
src/main.c src/sist.h
src/sist.h src/io/walk.h src/io/walk.c
src/io/walk.h src/io/walk.c src/parsing/media.h src/parsing/media.c
src/parsing/media.h src/parsing/media.c src/parsing/pdf.h src/parsing/pdf.c
src/parsing/pdf.h src/parsing/pdf.c src/io/store.h src/io/store.c
src/io/store.h src/io/store.c src/tpool.h src/tpool.c
src/tpool.h src/tpool.c src/parsing/parse.h src/parsing/parse.c
src/parsing/parse.h src/parsing/parse.c src/io/serialize.h src/io/serialize.c
src/io/serialize.h src/io/serialize.c src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c src/parsing/text.h src/parsing/text.c
src/parsing/text.h src/parsing/text.c src/index/web.c src/index/web.h
src/index/web.c src/index/web.h src/web/serve.c src/web/serve.h
src/web/serve.c src/web/serve.h src/web/auth_basic.h src/web/auth_basic.c
src/web/auth_basic.h src/web/auth_basic.c src/index/elastic.c src/index/elastic.h
src/index/elastic.c src/index/elastic.h src/util.c src/util.h
src/util.c src/util.h src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h src/parsing/arc.c src/parsing/arc.h
src/parsing/arc.c src/parsing/arc.h src/parsing/doc.c src/parsing/doc.h
src/parsing/doc.c src/parsing/doc.h src/log.c src/log.h
src/log.c src/log.h
# argparse # argparse
argparse/argparse.h argparse/argparse.c argparse/argparse.h argparse/argparse.c
# cJSON # cJSON
cJSON/cJSON.h cJSON/cJSON.c cJSON/cJSON.h cJSON/cJSON.c
# LMDB # LMDB
lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
src/cli.c src/cli.h src/cli.c src/cli.h
# utf8.h # utf8.h
utf8.h/utf8.h utf8.h/utf8.h
) )
endif ()
if (WITH_SIST2_SCAN)
add_executable(
sist2_scan
src/main.c
src/sist.h
src/io/walk.h src/io/walk.c
src/parsing/media.h src/parsing/media.c
src/parsing/pdf.h src/parsing/pdf.c
src/io/store.h src/io/store.c
src/tpool.h src/tpool.c
src/parsing/parse.h src/parsing/parse.c
src/io/serialize.h src/io/serialize.c
src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
src/parsing/text.h src/parsing/text.c
src/util.c src/util.h
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
src/parsing/arc.h src/parsing/arc.c
src/parsing/doc.h src/parsing/doc.c
src/log.h src/log.c
# argparse
argparse/argparse.h argparse/argparse.c
# cJSON
cJSON/cJSON.h cJSON/cJSON.c
# LMDB
lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
src/cli.c src/cli.h
# utf8.h
utf8.h/utf8.h
)
endif ()
find_package(PkgConfig REQUIRED) find_package(PkgConfig REQUIRED)
set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig/") set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig/")
@ -102,160 +61,80 @@ list(REMOVE_ITEM GLIB_LIBRARIES pcre)
list(REMOVE_ITEM GOBJECT_LIBRARIES pcre) list(REMOVE_ITEM GOBJECT_LIBRARIES pcre)
list(REMOVE_ITEM UUID_LIBRARIES pcre) list(REMOVE_ITEM UUID_LIBRARIES pcre)
if (WITH_SIST2) target_include_directories(
target_include_directories( sist2 PUBLIC
sist2 PUBLIC ${GOBJECT_INCLUDE_DIRS}
${GOBJECT_INCLUDE_DIRS} ${GLIB_INCLUDE_DIRS}
${GLIB_INCLUDE_DIRS} ${PROJECT_SOURCE_DIR}/lib/ffmpeg/
${PROJECT_SOURCE_DIR}/lib/ffmpeg/ ${FREETYPE_INCLUDE_DIRS}
${FREETYPE_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS}
${UUID_INCLUDE_DIRS} ${PROJECT_SOURCE_DIR}/
${PROJECT_SOURCE_DIR}/ ${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/ ${PROJECT_SOURCE_DIR}/lib/onion/src/
${PROJECT_SOURCE_DIR}/lib/onion/src/ ${PROJECT_SOURCE_DIR}/lib/mupdf/include/
${PROJECT_SOURCE_DIR}/lib/mupdf/include/ ${PROJECT_SOURCE_DIR}/include/
${PROJECT_SOURCE_DIR}/include/ /usr/include/libxml2/
/usr/include/libxml2/ ${PROJECT_SOURCE_DIR}/lib/tesseract_build/include/
) )
target_link_directories( target_link_directories(
sist2 PUBLIC sist2 PUBLIC
${UUID_LIBRARY_DIRS} ${UUID_LIBRARY_DIRS}
) )
target_compile_options(sist2
PRIVATE
-Ofast
# -march=native
-fno-stack-protector
-fomit-frame-pointer
)
target_compile_options(sist2 TARGET_LINK_LIBRARIES(
PRIVATE sist2
-Ofast
# -march=native
-fno-stack-protector
-fomit-frame-pointer
)
TARGET_LINK_LIBRARIES( ${GLIB_LIBRARIES}
sist2 ${GOBJECT_LIBRARIES}
${UUID_LIBRARIES}
${GLIB_LIBRARIES} # ffmpeg
${GOBJECT_LIBRARIES} ${PROJECT_SOURCE_DIR}/lib/libavcodec.a
${UUID_LIBRARIES} ${PROJECT_SOURCE_DIR}/lib/libavformat.a
${PROJECT_SOURCE_DIR}/lib/libavutil.a
${PROJECT_SOURCE_DIR}/lib/libswscale.a
${PROJECT_SOURCE_DIR}/lib/libswresample.a
# ffmpeg # mupdf
${PROJECT_SOURCE_DIR}/lib/libavcodec.a ${PROJECT_SOURCE_DIR}/lib/libmupdf.a
${PROJECT_SOURCE_DIR}/lib/libavformat.a ${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
${PROJECT_SOURCE_DIR}/lib/libavutil.a
${PROJECT_SOURCE_DIR}/lib/libswscale.a
${PROJECT_SOURCE_DIR}/lib/libswresample.a
# mupdf # onion
${PROJECT_SOURCE_DIR}/lib/libmupdf.a ${PROJECT_SOURCE_DIR}/lib/libonion_static.a
${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
# onion pthread
${PROJECT_SOURCE_DIR}/lib/libonion_static.a curl
m
bz2
${PROJECT_SOURCE_DIR}/lib/libmagic.a
${PROJECT_SOURCE_DIR}/lib/libharfbuzz.a
${PROJECT_SOURCE_DIR}/lib/libopenjp2.a
freetype
archive
pthread xml2
curl ${PROJECT_SOURCE_DIR}/lib/libopc/libmce.a
m ${PROJECT_SOURCE_DIR}/lib/libopc/libopc.a
bz2 ${PROJECT_SOURCE_DIR}/lib/libopc/libplib.a
${PROJECT_SOURCE_DIR}/lib/libmagic.a
${PROJECT_SOURCE_DIR}/lib/libharfbuzz.a
${PROJECT_SOURCE_DIR}/lib/libopenjp2.a
freetype
archive
# ${PROJECT_SOURCE_DIR}/lib/libxml2.a ${PROJECT_SOURCE_DIR}/lib/libtesseract.a
xml2 ${PROJECT_SOURCE_DIR}/lib/liblept.a
${PROJECT_SOURCE_DIR}/lib/libopc/libmce.a png
${PROJECT_SOURCE_DIR}/lib/libopc/libopc.a tiff
${PROJECT_SOURCE_DIR}/lib/libopc/libplib.a stdc++
) )
endif ()
if (WITH_SIST2_SCAN)
set_target_properties(
sist2_scan
PROPERTIES COMPILE_DEFINITIONS SIST_SCAN_ONLY
)
set_target_properties(
sist2_scan
PROPERTIES
COMPILE_DEFINITIONS SIST_SCAN_ONLY
LINK_FLAGS -static
)
target_include_directories(
sist2_scan PUBLIC
${LIBMAGIC_INCLUDE_DIRS}
${GOBJECT_INCLUDE_DIRS}
${OPENSSL_INCLUDE_DIR}
${PROJECT_SOURCE_DIR}/lib/ffmpeg/
${GLIB_INCLUDE_DIRS}
${UUID_INCLUDE_DIRS}
${FREETYPE_INCLUDE_DIRS}
${PROJECT_SOURCE_DIR}/
${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
${PROJECT_SOURCE_DIR}/lib/onion/src/
${PROJECT_SOURCE_DIR}/lib/mupdf/include/
${PROJECT_SOURCE_DIR}/include/
/usr/include/libxml2/
)
target_link_directories(
sist2_scan PUBLIC
${UUID_LIBRARY_DIRS}
)
target_compile_options(sist2_scan
PRIVATE
-Ofast
# -march=native
-fno-stack-protector
-fomit-frame-pointer
)
TARGET_LINK_LIBRARIES(
sist2_scan
${GLIB_LIBRARIES}
${GOBJECT_LIBRARIES}
${UUID_LIBRARIES}
# ffmpeg
${PROJECT_SOURCE_DIR}/lib/libavcodec.a
${PROJECT_SOURCE_DIR}/lib/libavformat.a
${PROJECT_SOURCE_DIR}/lib/libavutil.a
${PROJECT_SOURCE_DIR}/lib/libswscale.a
${PROJECT_SOURCE_DIR}/lib/libswresample.a
# mupdf
${PROJECT_SOURCE_DIR}/lib/libmupdf.a
${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
${PROJECT_SOURCE_DIR}/lib/libbz2.a
${PROJECT_SOURCE_DIR}/lib/libmagic.a
pthread
m
${PROJECT_SOURCE_DIR}/lib/libharfbuzz.a
${PROJECT_SOURCE_DIR}/lib/libopenjp2.a
freetype
${PROJECT_SOURCE_DIR}/lib/libarchive.a
${PROJECT_SOURCE_DIR}/lib/liblz4.a
${PROJECT_SOURCE_DIR}/lib/liblzma.a
${PROJECT_SOURCE_DIR}/lib/libzstd.a
${PROJECT_SOURCE_DIR}/lib/libxml2.a
${PROJECT_SOURCE_DIR}/lib/libopc/libmce.a
${PROJECT_SOURCE_DIR}/lib/libopc/libopc.a
${PROJECT_SOURCE_DIR}/lib/libopc/libplib.a
)
endif ()
add_custom_target( add_custom_target(
before_sist2 before_sist2
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/before_build.sh COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/before_build.sh
) )
IF (WITH_SIST2) add_dependencies(sist2 before_sist2)
add_dependencies(sist2 before_sist2)
else ()
add_dependencies(sist2_scan before_sist2)
endif ()

13
Docker/Dockerfile
View File

@ -2,8 +2,17 @@ FROM ubuntu:19.10
MAINTAINER simon987 <me@simon987.net> MAINTAINER simon987 <me@simon987.net>
RUN apt update RUN apt update
RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 RUN apt install -y libglib2.0-0 libcurl4 libmagic1 libharfbuzz-bin libopenjp2-7 libarchive13 liblzma5 libzstd1 liblz4-1 curl
RUN mkdir -p /usr/share/tessdata && \
cd /usr/share/tessdata/ && \
curl -o /usr/share/tessdata/hin.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/hin.traineddata &&\
curl -o /usr/share/tessdata/jpn.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/jpn.traineddata &&\
curl -o /usr/share/tessdata/eng.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/eng.traineddata &&\
curl -o /usr/share/tessdata/fra.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/fra.traineddata &&\
curl -o /usr/share/tessdata/rus.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/rus.traineddata &&\
curl -o /usr/share/tessdata/spa.traineddata https://raw.githubusercontent.com/tesseract-ocr/tessdata/master/spa.traineddata && ls -lh
ADD sist2 /root/sist2 ADD sist2 /root/sist2
ENTRYPOINT ["/root/sist2"] ENTRYPOINT ["/root/sist2"]

25
README.md
View File

@ -11,15 +11,17 @@ sist2 (Simple incremental search tool)
* Fast, low memory usage, multi-threaded * Fast, low memory usage, multi-threaded
* Portable (all its features are packaged in a single executable) * Portable (all its features are packaged in a single executable)
* Extracts text from common file types\* * Extracts text from common file types \*
* Generates thumbnails\* * Generates thumbnails \*
* Incremental scanning * Incremental scanning
* Automatic tagging from file attributes via [user scripts](scripting/README.md) * Automatic tagging from file attributes via [user scripts](scripting/README.md)
* Recursive scan inside archive files \*\* * Recursive scan inside archive files \*\*
* OCR support with tesseract \*\*\*
\* See [format support](#format-support) \* See [format support](#format-support)
\** See [Archive files](#archive-files) \*\* See [Archive files](#archive-files)
\*\*\* See [OCR](#ocr)
## Getting Started ## Getting Started
@ -85,7 +87,7 @@ docker stop sist2
File type | Library | Content | Thumbnail | Metadata File type | Library | Content | Thumbnail | Metadata
:---|:---|:---|:---|:--- :---|:---|:---|:---|:---
pdf,xps,cbz,fb2,epub | MuPDF | yes | yes, `png` | title | pdf,xps,cbz,fb2,epub | MuPDF | text+ocr | yes, `png` | title |
`audio/*` | ffmpeg | - | yes, `jpeg` | ID3 tags | `audio/*` | ffmpeg | - | yes, `jpeg` | ID3 tags |
`video/*` | ffmpeg | - | yes, `jpeg` | title, comment, artist | `video/*` | ffmpeg | - | yes, `jpeg` | title, comment, artist |
`image/*` | ffmpeg | - | yes, `jpeg` | `EXIF:Artist`, `EXIF:ImageDescription` | `image/*` | ffmpeg | - | yes, `jpeg` | `EXIF:Artist`, `EXIF:ImageDescription` |
@ -110,6 +112,21 @@ scan is also supported.
To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -` To check if a media file can be parsed without *seek*, execute `cat file.mp4 | ffprobe -`
### OCR
You can enable OCR support for pdf,xps,cbz,fb2,epub file types with the
`--ocr <lang>` option. Download the language data files with your
package manager (`apt install tesseract-ocr-eng`) or directly [from Github](https://github.com/tesseract-ocr/tesseract/wiki/Data-Files).
The `simon987/sist2` github image comes with common languages
(hin, jpn, eng, fra, rus, spa) pre-installed.
Examples
```bash
sist2 scan --ocr jpn ~/Books/Manga/
sist2 scan --ocr eng ~/Books/Textbooks/
```
## Build from source ## Build from source

1
lib/leptonica Submodule

@ -0,0 +1 @@
Subproject commit cc03be70fded1ad6a8cedad656456386a1bd08e8

1
lib/libarchive

@ -1 +0,0 @@
Subproject commit 6f4fceb714868f2ddbccef4871acc1670e45fc03

1
lib/libxml2

@ -1 +0,0 @@
Subproject commit 0815302dee2b78139832c2080348086a0564836b

1
lib/lz4

@ -1 +0,0 @@
Subproject commit 0f749838bf29bc0d1df428e23cf3dbb76ec4e9fc

1
lib/tesseract Submodule

@ -0,0 +1 @@
Subproject commit f268e6615e619a7140de78d9db3f8240fa8c68bd

1
lib/zstd

@ -1 +0,0 @@
Subproject commit d73e2fb465c9289f5a6871cef1fe48de917a5cc2

52
scripts/get_static_libs.sh
View File

@ -70,43 +70,21 @@ make -j $THREADS
cd .. cd ..
mv libmagic/src/.libs/libmagic.a . mv libmagic/src/.libs/libmagic.a .
# libarchive # tesseract
cd libarchive/build cd tesseract
mkdir build
cd build
cmake -DSTATIC=on -DBUILD_TRAINING_TOOLS=off ..
make -j $THREADS
cd ../..
mv tesseract/build/libtesseract.a .
# leptonica
cd leptonica
./autogen.sh ./autogen.sh
cd .. ./configure --without-zlib --without-jpeg --without-giflib \
./configure --without-nettle --without-expat --without-xml2 --without-openssl --without-giflib --without-libwebp --without-libwebpmux --without-libopenjpeg \
--enable-static --disable-shared
make -j $THREADS make -j $THREADS
cd .. cd ..
mv libarchive/.libs/libarchive.a . mv leptonica/src/.libs/liblept.a .
# lz4
cd lz4
make -j $THREADS
cd ..
mv lz4/lib/liblz4.a .
# lzma
wget https://newcontinuum.dl.sourceforge.net/project/lzmautils/xz-5.2.3.tar.gz
tar -xzf xz-5.2.3.tar.gz
rm xz-5.2.3.tar.gz
cd xz-5.2.3
./autogen.sh
./configure
make -j $THREADS
cd ..
mv xz-5.2.3/src/liblzma/.libs/liblzma.a .
# zstd
cd zstd
make -j $THREADS
cd ..
mv zstd/lib/libzstd.a .
# xml2
cd libxml2
./autogen.sh --without-zlib --without-lzma
make
cd ..
mv libxml2/.libs/libxml2.a .
cd ..

22
src/cli.c
View File

@ -1,6 +1,8 @@
#include "cli.h" #include "cli.h"
#include "ctx.h" #include "ctx.h"
#include <tesseract/capi.h>
#define DEFAULT_OUTPUT "index.sist2/" #define DEFAULT_OUTPUT "index.sist2/"
#define DEFAULT_CONTENT_SIZE 32768 #define DEFAULT_CONTENT_SIZE 32768
#define DEFAULT_QUALITY 5 #define DEFAULT_QUALITY 5
@ -35,8 +37,6 @@ void scan_args_destroy(scan_args_t *args) {
free(args); free(args);
} }
#ifndef SIST_SCAN_ONLY
void index_args_destroy(index_args_t *args) { void index_args_destroy(index_args_t *args) {
//todo //todo
free(args); free(args);
@ -47,8 +47,6 @@ void web_args_destroy(web_args_t *args) {
free(args); free(args);
} }
#endif
int scan_args_validate(scan_args_t *args, int argc, const char **argv) { int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
if (argc < 2) { if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n"); fprintf(stderr, "Required positional argument: PATH.\n");
@ -136,6 +134,17 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
return 1; return 1;
} }
if (args->tesseract_lang != NULL) {
TessBaseAPI *api = TessBaseAPICreate();
ret = TessBaseAPIInit3(api, TESS_DATAPATH, args->tesseract_lang);
if (ret != 0) {
fprintf(stderr, "Could not initialize tesseract with lang '%s'\n", args->tesseract_lang);
return 1;
}
TessBaseAPIEnd(api);
TessBaseAPIDelete(api);
}
LOG_DEBUGF("cli.c", "arg quality=%f", args->quality) LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
LOG_DEBUGF("cli.c", "arg size=%d", args->size) LOG_DEBUGF("cli.c", "arg size=%d", args->size)
LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size) LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
@ -147,12 +156,11 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
LOG_DEBUGF("cli.c", "arg depth=%d", args->depth) LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
LOG_DEBUGF("cli.c", "arg path=%s", args->path) LOG_DEBUGF("cli.c", "arg path=%s", args->path)
LOG_DEBUGF("cli.c", "arg archive=%s", args->archive) LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
LOG_DEBUGF("cli.c", "arg ocr=%s", args->tesseract_lang)
return 0; return 0;
} }
#ifndef SIST_SCAN_ONLY
int index_args_validate(index_args_t *args, int argc, const char **argv) { int index_args_validate(index_args_t *args, int argc, const char **argv) {
if (argc < 2) { if (argc < 2) {
@ -273,5 +281,3 @@ web_args_t *web_args_create() {
return args; return args;
} }
#endif

3
src/cli.h
View File

@ -16,13 +16,13 @@ typedef struct scan_args {
char *path; char *path;
char *archive; char *archive;
archive_mode_t archive_mode; archive_mode_t archive_mode;
char *tesseract_lang;
} scan_args_t; } scan_args_t;
scan_args_t *scan_args_create(); scan_args_t *scan_args_create();
void scan_args_destroy(scan_args_t *args); void scan_args_destroy(scan_args_t *args);
int scan_args_validate(scan_args_t *args, int argc, const char **argv); int scan_args_validate(scan_args_t *args, int argc, const char **argv);
#ifndef SIST_SCAN_ONLY
typedef struct index_args { typedef struct index_args {
char *es_url; char *es_url;
const char *index_path; const char *index_path;
@ -51,6 +51,5 @@ void web_args_destroy(web_args_t *args);
int index_args_validate(index_args_t *args, int argc, const char **argv); int index_args_validate(index_args_t *args, int argc, const char **argv);
int web_args_validate(web_args_t *args, int argc, const char **argv); int web_args_validate(web_args_t *args, int argc, const char **argv);
#endif
#endif #endif

3
src/ctx.h
View File

@ -27,6 +27,7 @@ struct {
GHashTable *copy_table; GHashTable *copy_table;
pthread_mutex_t mupdf_mu; pthread_mutex_t mupdf_mu;
char * tesseract_lang;
} ScanCtx; } ScanCtx;
struct { struct {
@ -35,7 +36,6 @@ struct {
int no_color; int no_color;
} LogCtx; } LogCtx;
#ifndef SIST_SCAN_ONLY
struct { struct {
char *es_url; char *es_url;
int batch_size; int batch_size;
@ -47,7 +47,6 @@ struct {
char *b64credentials; char *b64credentials;
struct index_t indices[16]; struct index_t indices[16];
} WebCtx; } WebCtx;
#endif
#endif #endif

25
src/main.c
View File

@ -1,16 +1,12 @@
#include "sist.h" #include "sist.h"
#include "ctx.h" #include "ctx.h"
#ifndef SIST_SCAN_ONLY
#define DESCRIPTION "Lightning-fast file system indexer and search tool." #define DESCRIPTION "Lightning-fast file system indexer and search tool."
#else
#define DESCRIPTION "Lightning-fast file system indexer and search tool. (SCAN ONLY)"
#endif
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0" #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
static const char *const Version = "1.1.15"; static const char *const Version = "1.2.0";
static const char *const usage[] = { static const char *const usage[] = {
"sist2 scan [OPTION]... PATH", "sist2 scan [OPTION]... PATH",
"sist2 index [OPTION]... INDEX", "sist2 index [OPTION]... INDEX",
@ -19,9 +15,7 @@ static const char *const usage[] = {
}; };
void global_init() { void global_init() {
#ifndef SIST_SCAN_ONLY
curl_global_init(CURL_GLOBAL_NOTHING); curl_global_init(CURL_GLOBAL_NOTHING);
#endif
av_log_set_level(AV_LOG_QUIET); av_log_set_level(AV_LOG_QUIET);
opcInitLibrary(); opcInitLibrary();
} }
@ -55,6 +49,7 @@ void sist2_scan(scan_args_t *args) {
strncpy(ScanCtx.index.desc.name, args->name, sizeof(ScanCtx.index.desc.name)); strncpy(ScanCtx.index.desc.name, args->name, sizeof(ScanCtx.index.desc.name));
strncpy(ScanCtx.index.desc.root, args->path, sizeof(ScanCtx.index.desc.root)); strncpy(ScanCtx.index.desc.root, args->path, sizeof(ScanCtx.index.desc.root));
ScanCtx.index.desc.root_len = (short) strlen(ScanCtx.index.desc.root); ScanCtx.index.desc.root_len = (short) strlen(ScanCtx.index.desc.root);
ScanCtx.tesseract_lang = args->tesseract_lang;
init_dir(ScanCtx.index.path); init_dir(ScanCtx.index.path);
@ -122,8 +117,6 @@ void sist2_scan(scan_args_t *args) {
store_destroy(ScanCtx.index.store); store_destroy(ScanCtx.index.store);
} }
#ifndef SIST_SCAN_ONLY
void sist2_index(index_args_t *args) { void sist2_index(index_args_t *args) {
IndexCtx.es_url = args->es_url; IndexCtx.es_url = args->es_url;
@ -198,18 +191,14 @@ void sist2_web(web_args_t *args) {
serve(args->bind, args->port); serve(args->bind, args->port);
} }
#endif
int main(int argc, const char *argv[]) { int main(int argc, const char *argv[]) {
global_init(); global_init();
scan_args_t *scan_args = scan_args_create(); scan_args_t *scan_args = scan_args_create();
#ifndef SIST_SCAN_ONLY
index_args_t *index_args = index_args_create(); index_args_t *index_args = index_args_create();
web_args_t *web_args = web_args_create(); web_args_t *web_args = web_args_create();
#endif
int arg_version = 0; int arg_version = 0;
@ -240,8 +229,9 @@ int main(int argc, const char *argv[]) {
OPT_STRING(0, "archive", &scan_args->archive, "Archive file mode (skip|list|shallow|recurse). " OPT_STRING(0, "archive", &scan_args->archive, "Archive file mode (skip|list|shallow|recurse). "
"skip: Don't parse, list: only get file names as text, " "skip: Don't parse, list: only get file names as text, "
"shallow: Don't parse archives inside archives. DEFAULT: recurse"), "shallow: Don't parse archives inside archives. DEFAULT: recurse"),
OPT_STRING(0, "ocr", &scan_args->tesseract_lang, "Tesseract language (use tesseract --list-langs to see "
"which are installed on your machine)"),
#ifndef SIST_SCAN_ONLY
OPT_GROUP("Index options"), OPT_GROUP("Index options"),
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"), OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"),
OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."), OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
@ -255,7 +245,6 @@ int main(int argc, const char *argv[]) {
OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"), OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"),
OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"), OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"),
OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"), OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"),
#endif
OPT_END(), OPT_END(),
}; };
@ -274,10 +263,8 @@ int main(int argc, const char *argv[]) {
LogCtx.verbose = 1; LogCtx.verbose = 1;
} }
#ifndef SIST_SCAN_ONLY
web_args->es_url = common_es_url; web_args->es_url = common_es_url;
index_args->es_url = common_es_url; index_args->es_url = common_es_url;
#endif
if (argc == 0) { if (argc == 0) {
argparse_usage(&argparse); argparse_usage(&argparse);
@ -292,7 +279,6 @@ int main(int argc, const char *argv[]) {
} }
#ifndef SIST_SCAN_ONLY
else if (strcmp(argv[0], "index") == 0) { else if (strcmp(argv[0], "index") == 0) {
int err = index_args_validate(index_args, argc, argv); int err = index_args_validate(index_args, argc, argv);
@ -310,7 +296,6 @@ int main(int argc, const char *argv[]) {
sist2_web(web_args); sist2_web(web_args);
} }
#endif
else { else {
fprintf(stderr, "Invalid command: '%s'\n", argv[0]); fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
argparse_usage(&argparse); argparse_usage(&argparse);
@ -320,10 +305,8 @@ int main(int argc, const char *argv[]) {
scan_args_destroy(scan_args); scan_args_destroy(scan_args);
#ifndef SIST_SCAN_ONLY
index_args_destroy(index_args); index_args_destroy(index_args);
web_args_destroy(web_args); web_args_destroy(web_args);
#endif
return 0; return 0;
} }

1
src/parsing/media.c
View File

@ -127,6 +127,7 @@ AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int st
#define APPEND_TAG_META(doc, tag_, keyname) \ #define APPEND_TAG_META(doc, tag_, keyname) \
text_buffer_t tex = text_buffer_create(-1); \ text_buffer_t tex = text_buffer_create(-1); \
text_buffer_append_string0(&tex, tag_->value); \ text_buffer_append_string0(&tex, tag_->value); \
text_buffer_terminate_string(&tex); \
meta_line_t *meta_tag = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); \ meta_line_t *meta_tag = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); \
meta_tag->key = keyname; \ meta_tag->key = keyname; \
strcpy(meta_tag->strval, tex.dyn_buffer.buf); \ strcpy(meta_tag->strval, tex.dyn_buffer.buf); \

76
src/parsing/pdf.c
View File

@ -1,6 +1,10 @@
#include "pdf.h" #include "pdf.h"
#include "src/ctx.h" #include "src/ctx.h"
#define MIN_OCR_SIZE 128
__thread text_buffer_t thread_buffer;
fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) { fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
int err = 0; int err = 0;
@ -86,14 +90,13 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
return cover; return cover;
} }
void fz_err_callback(void *user, __attribute__((unused)) const char *message) { void fz_err_callback(void *user, UNUSED(const char *message)) {
if (LogCtx.verbose) { if (LogCtx.verbose) {
document_t *doc = (document_t*) user; document_t *doc = (document_t *) user;
LOG_WARNINGF(doc->filepath, "FZ: %s", message) LOG_WARNINGF(doc->filepath, "FZ: %s", message)
} }
} }
__always_inline __always_inline
void init_ctx(fz_context *ctx, document_t *doc) { void init_ctx(fz_context *ctx, document_t *doc) {
fz_disable_icc(ctx); fz_disable_icc(ctx);
@ -124,6 +127,40 @@ int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
return 0; return 0;
} }
void fill_image(fz_context *ctx, UNUSED(fz_device *dev),
fz_image *img, UNUSED(fz_matrix ctm), UNUSED(float alpha),
UNUSED(fz_color_params color_params)) {
int l2factor = 0;
if (img->w > MIN_OCR_SIZE && img->h > MIN_OCR_SIZE) {
fz_pixmap *pix = img->get_pixmap(ctx, img, NULL, img->w, img->h, &l2factor);
if (pix->h > MIN_OCR_SIZE && img->h > MIN_OCR_SIZE && img->xres != 0) {
TessBaseAPI *api = TessBaseAPICreate();
TessBaseAPIInit3(api, TESS_DATAPATH, ScanCtx.tesseract_lang);
TessBaseAPISetImage(api, pix->samples, pix->w, pix->h, pix->n, pix->stride);
TessBaseAPISetSourceResolution(api, pix->xres);
char *text = TessBaseAPIGetUTF8Text(api);
size_t len = strlen(text);
text_buffer_append_string(&thread_buffer, text, len - 1);
LOG_DEBUGF(
"pdf.c",
"(OCR) %dx%d got %dB from tesseract (%s), buffer:%dB",
pix->w, pix->h, len, ScanCtx.tesseract_lang, thread_buffer.dyn_buffer.cur
)
TessBaseAPIEnd(api);
TessBaseAPIDelete(api);
fz_drop_pixmap(ctx, pix);
}
}
}
void parse_pdf(void *buf, size_t buf_len, document_t *doc) { void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
if (buf == NULL) { if (buf == NULL) {
@ -210,7 +247,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
if (ScanCtx.content_size > 0) { if (ScanCtx.content_size > 0) {
fz_stext_options opts = {0}; fz_stext_options opts = {0};
text_buffer_t text_buf = text_buffer_create(ScanCtx.content_size); thread_buffer = text_buffer_create(ScanCtx.content_size);
for (int current_page = 0; current_page < page_count; current_page++) { for (int current_page = 0; current_page < page_count; current_page++) {
fz_page *page = NULL; fz_page *page = NULL;
@ -224,7 +261,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
err = ctx->error.errcode; err = ctx->error.errcode;
if (err != 0) { if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message) LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
text_buffer_destroy(&text_buf); text_buffer_destroy(&thread_buffer);
fz_drop_page(ctx, page); fz_drop_page(ctx, page);
fz_drop_stream(ctx, stream); fz_drop_stream(ctx, stream);
fz_drop_document(ctx, fzdoc); fz_drop_document(ctx, fzdoc);
@ -235,6 +272,15 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
fz_stext_page *stext = fz_new_stext_page(ctx, fz_bound_page(ctx, page)); fz_stext_page *stext = fz_new_stext_page(ctx, fz_bound_page(ctx, page));
fz_device *dev = fz_new_stext_device(ctx, stext, &opts); fz_device *dev = fz_new_stext_device(ctx, stext, &opts);
dev->stroke_path = NULL;
dev->stroke_text = NULL;
dev->clip_text = NULL;
dev->clip_stroke_path = NULL;
dev->clip_stroke_text = NULL;
if (ScanCtx.tesseract_lang != NULL) {
dev->fill_image = fill_image;
}
fz_var(err); fz_var(err);
fz_try(ctx) fz_try(ctx)
@ -249,7 +295,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
if (err != 0) { if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message) LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
text_buffer_destroy(&text_buf); text_buffer_destroy(&thread_buffer);
fz_drop_page(ctx, page); fz_drop_page(ctx, page);
fz_drop_stext_page(ctx, stext); fz_drop_stext_page(ctx, stext);
fz_drop_stream(ctx, stream); fz_drop_stream(ctx, stream);
@ -260,7 +306,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
fz_stext_block *block = stext->first_block; fz_stext_block *block = stext->first_block;
while (block != NULL) { while (block != NULL) {
int ret = read_stext_block(block, &text_buf); int ret = read_stext_block(block, &thread_buffer);
if (ret == TEXT_BUF_FULL) { if (ret == TEXT_BUF_FULL) {
break; break;
} }
@ -269,18 +315,18 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
fz_drop_stext_page(ctx, stext); fz_drop_stext_page(ctx, stext);
fz_drop_page(ctx, page); fz_drop_page(ctx, page);
if (text_buf.dyn_buffer.cur >= text_buf.dyn_buffer.size) { if (thread_buffer.dyn_buffer.cur >= thread_buffer.dyn_buffer.size) {
break; break;
} }
} }
text_buffer_terminate_string(&text_buf); text_buffer_terminate_string(&thread_buffer);
meta_line_t *meta_content = malloc(sizeof(meta_line_t) + text_buf.dyn_buffer.cur); meta_line_t *meta_content = malloc(sizeof(meta_line_t) + thread_buffer.dyn_buffer.cur);
meta_content->key = MetaContent; meta_content->key = MetaContent;
memcpy(meta_content->strval, text_buf.dyn_buffer.buf, text_buf.dyn_buffer.cur); memcpy(meta_content->strval, thread_buffer.dyn_buffer.buf, thread_buffer.dyn_buffer.cur);
APPEND_META(doc, meta_content) APPEND_META(doc, meta_content)
text_buffer_destroy(&text_buf); text_buffer_destroy(&thread_buffer);
} }
fz_drop_stream(ctx, stream); fz_drop_stream(ctx, stream);

1
src/parsing/pdf.h
View File

@ -2,6 +2,7 @@
#define SIST2_PDF_H #define SIST2_PDF_H
#include "src/sist.h" #include "src/sist.h"
#include <tesseract/capi.h>
void parse_pdf(void *buf, size_t buf_len, document_t *doc); void parse_pdf(void *buf, size_t buf_len, document_t *doc);

1
src/parsing/text.c
View File

@ -25,6 +25,7 @@ void parse_text(int bytes_read, struct vfile *f, char *buf, document_t *doc) {
} }
text_buffer_t tex = text_buffer_create(ScanCtx.content_size); text_buffer_t tex = text_buffer_create(ScanCtx.content_size);
text_buffer_append_string(&tex, intermediate_buf, intermediate_buf_len); text_buffer_append_string(&tex, intermediate_buf, intermediate_buf_len);
text_buffer_terminate_string(&tex);
meta_line_t *meta = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur); meta_line_t *meta = malloc(sizeof(meta_line_t) + tex.dyn_buffer.cur);
meta->key = MetaContent; meta->key = MetaContent;

6
src/sist.h
View File

@ -2,6 +2,8 @@
#define SIST_H #define SIST_H
#define UUID_STR_LEN 37 #define UUID_STR_LEN 37
#define UNUSED(x) __attribute__((__unused__)) x
#define TESS_DATAPATH "/usr/share/tessdata/"
#include <glib-2.0/glib.h> #include <glib-2.0/glib.h>
#include <unistd.h> #include <unistd.h>
@ -33,14 +35,12 @@
#include <opc/opc.h> #include <opc/opc.h>
#include <libxml/xmlstring.h> #include <libxml/xmlstring.h>
#ifndef SIST_SCAN_ONLY
#include <onion/onion.h> #include <onion/onion.h>
#include <onion/handler.h> #include <onion/handler.h>
#include <onion/block.h> #include <onion/block.h>
#include <onion/shortcuts.h> #include <onion/shortcuts.h>
#include <onion/codecs.h> #include <onion/codecs.h>
#include <curl/curl.h> #include <curl/curl.h>
#endif
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
@ -62,12 +62,10 @@
#include "log.h" #include "log.h"
#include "utf8.h/utf8.h" #include "utf8.h/utf8.h"
#ifndef SIST_SCAN_ONLY
#include "src/index/elastic.h" #include "src/index/elastic.h"
#include "index/web.h" #include "index/web.h"
#include "web/serve.h" #include "web/serve.h"
#include "web/auth_basic.h" #include "web/auth_basic.h"
#endif
; ;

4
src/types.h
View File

@ -70,8 +70,6 @@ typedef struct document {
short ext; short ext;
meta_line_t *meta_head; meta_line_t *meta_head;
meta_line_t *meta_tail; meta_line_t *meta_tail;
struct document *child_head;
struct document *child_tail;
char *filepath; char *filepath;
} document_t; } document_t;
@ -79,8 +77,6 @@ typedef struct vfile vfile_t;
typedef int (*read_func_t)(struct vfile *, void *buf, size_t size); typedef int (*read_func_t)(struct vfile *, void *buf, size_t size);
typedef int (*seek_func_t)(struct vfile *, size_t size, int whence);
typedef void (*close_func_t)(struct vfile *); typedef void (*close_func_t)(struct vfile *);
typedef struct vfile { typedef struct vfile {

3
src/util.c
View File

@ -149,7 +149,6 @@ int text_buffer_append_string(text_buffer_t *buf, char *str, size_t len) {
(0xe0 == (0xf0 & str[0]) && len < 3) || (0xe0 == (0xf0 & str[0]) && len < 3) ||
(0xc0 == (0xe0 & str[0]) && len == 1) || (0xc0 == (0xe0 & str[0]) && len == 1) ||
*(str) == 0) { *(str) == 0) {
text_buffer_terminate_string(buf);
return 0; return 0;
} }
@ -158,7 +157,6 @@ int text_buffer_append_string(text_buffer_t *buf, char *str, size_t len) {
text_buffer_append_char(buf, c); text_buffer_append_char(buf, c);
} }
} }
text_buffer_terminate_string(buf);
return 0; return 0;
} }
@ -169,7 +167,6 @@ int text_buffer_append_string0(text_buffer_t *buf, char *str) {
text_buffer_append_char(buf, c); text_buffer_append_char(buf, c);
} }
} }
text_buffer_terminate_string(buf);
} }
int text_buffer_append_char(text_buffer_t *buf, int c) { int text_buffer_append_char(text_buffer_t *buf, int c) {

2
src/web/static_generated.c
View File

File diff suppressed because one or more lines are too long

2
web/search.html
View File

@ -11,7 +11,7 @@
<nav class="navbar navbar-expand-lg"> <nav class="navbar navbar-expand-lg">
<a class="navbar-brand" href="/">sist2</a> <a class="navbar-brand" href="/">sist2</a>
<span class="badge badge-pill version">v1.1.15</span> <span class="badge badge-pill version">v1.2.0</span>
<span class="tagline">Lightning-fast file system indexer and search tool </span> <span class="tagline">Lightning-fast file system indexer and search tool </span>
<a style="margin-left: auto" id="theme" class="btn" title="Toggle theme" href="/">Theme</a> <a style="margin-left: auto" id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
</nav> </nav>