simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-16 00:46:43 +00:00
Code Issues Projects Releases Wiki Activity

Static build (scan only)

Browse Source
This commit is contained in:
simon 2019-10-27 11:17:11 -04:00
parent 4109ba6d34
commit 9b7c56a608
15 changed files with 473 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

279
CMakeLists.txt
View File

@ -1,38 +1,74 @@
cmake_minimum_required(VERSION 3.7)
set(CMAKE_C_STANDARD 11)
option(WITH_SIST2 "Build main executable" ON)
option(WITH_SIST2_SCAN "Build scan executable" ON)
project(sist2 C)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMakeModules")
add_executable(
sist2
src/main.c
src/sist.h
src/io/walk.h src/io/walk.c
src/parsing/media.h src/parsing/media.c
src/parsing/pdf.h src/parsing/pdf.c
src/io/store.h src/io/store.c
src/tpool.h src/tpool.c
src/parsing/parse.h src/parsing/parse.c
src/io/serialize.h src/io/serialize.c
src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
src/parsing/text.h src/parsing/text.c
src/index/web.c src/index/web.h
src/web/serve.c src/web/serve.h
src/index/elastic.c src/index/elastic.h
src/util.c src/util.h
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
if (WITH_SIST2)
add_executable(
sist2
src/main.c
src/sist.h
src/io/walk.h src/io/walk.c
src/parsing/media.h src/parsing/media.c
src/parsing/pdf.h src/parsing/pdf.c
src/io/store.h src/io/store.c
src/tpool.h src/tpool.c
src/parsing/parse.h src/parsing/parse.c
src/io/serialize.h src/io/serialize.c
src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
src/parsing/text.h src/parsing/text.c
src/index/web.c src/index/web.h
src/web/serve.c src/web/serve.h
src/index/elastic.c src/index/elastic.h
src/util.c src/util.h
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
# argparse
argparse/argparse.h argparse/argparse.c
# argparse
argparse/argparse.h argparse/argparse.c
# cJSON
cJSON/cJSON.h cJSON/cJSON.c
# cJSON
cJSON/cJSON.h cJSON/cJSON.c
# LMDB
lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
src/cli.c src/cli.h)
# LMDB
lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
src/cli.c src/cli.h
)
endif ()
if (WITH_SIST2_SCAN)
add_executable(
sist2_scan
src/main.c
src/sist.h
src/io/walk.h src/io/walk.c
src/parsing/media.h src/parsing/media.c
src/parsing/pdf.h src/parsing/pdf.c
src/io/store.h src/io/store.c
src/tpool.h src/tpool.c
src/parsing/parse.h src/parsing/parse.c
src/io/serialize.h src/io/serialize.c
src/parsing/mime.h src/parsing/mime.c src/parsing/mime_generated.c
src/parsing/text.h src/parsing/text.c
src/util.c src/util.h
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
# argparse
argparse/argparse.h argparse/argparse.c
# cJSON
cJSON/cJSON.h cJSON/cJSON.c
# LMDB
lmdb/libraries/liblmdb/lmdb.h lmdb/libraries/liblmdb/mdb.c
lmdb/libraries/liblmdb/midl.h lmdb/libraries/liblmdb/midl.c
src/cli.c src/cli.h
)
endif ()
find_package(PkgConfig REQUIRED)
set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig/")
@ -46,82 +82,151 @@ pkg_check_modules(GLIB REQUIRED glib-2.0)
pkg_check_modules(GOBJECT REQUIRED gobject-2.0)
pkg_check_modules(UUID REQUIRED uuid)
include_directories(${LIBMAGIC_INCLUDE_DIRS})
link_directories(${LIBMAGIC_LIBRARY_DIRS})
add_definitions(${LIBMAGIC_CFLAGS_OTHER})
link_directories(${UUID_LIBRARY_DIRS})
include_directories(${UUID_INCLUDE_DIRS})
add_definitions(${UUID_CFLAGS_OTHER})
include_directories(${GLIB_INCLUDE_DIRS})
link_directories(${GLIB_LIBRARY_DIRS})
add_definitions(${GLIB_CFLAGS_OTHER})
include_directories(${GOBJECT_INCLUDE_DIRS})
link_directories(${GOBJECT_LIBRARY_DIRS})
add_definitions(${GOBJECT_CFLAGS_OTHER})
link_directories(${FFMPEG_LIBRARY_DIRS})
include_directories(${FFMPEG_INCLUDE_DIRS})
include_directories(${OPENSSL_INCLUDE_DIR})
link_directories(${OPENSSL_CRYPTO_LIBRARY})
list(REMOVE_ITEM GLIB_LIBRARIES pcre)
list(REMOVE_ITEM GOBJECT_LIBRARIES pcre)
list(REMOVE_ITEM UUID_LIBRARIES pcre)
include_directories(${FREETYPE_INCLUDE_DIRS})
add_definitions(${FREETYPE_CFLAGS_OTHER})
if (WITH_SIST2)
target_include_directories(
sist2 PUBLIC
${LIBMAGIC_INCLUDE_DIRS}
${GOBJECT_INCLUDE_DIRS}
${OPENSSL_INCLUDE_DIR}
${FFMPEG_INCLUDE_DIRS}
${GLIB_INCLUDE_DIRS}
${OPENSSL_CRYPTO_LIBRARY}
${FREETYPE_INCLUDE_DIRS}
${UUID_INCLUDE_DIRS}
${PROJECT_SOURCE_DIR}/
${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
${PROJECT_SOURCE_DIR}/lib/onion/src/
${PROJECT_SOURCE_DIR}/lib/mupdf/include/
)
target_link_directories(
sist2 PUBLIC
${UUID_LIBRARY_DIRS}
${FFMPEG_LIBRARY_DIRS}
)
include_directories(
${PROJECT_SOURCE_DIR}/
${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
${PROJECT_SOURCE_DIR}/lib/onion/src/
${PROJECT_SOURCE_DIR}/lib/mupdf/include/
)
add_definitions(${LIBMAGIC_CFLAGS_OTHER})
add_definitions(${UUID_CFLAGS_OTHER})
add_definitions(${GLIB_CFLAGS_OTHER})
add_definitions(${GOBJECT_CFLAGS_OTHER})
add_definitions(${FREETYPE_CFLAGS_OTHER})
target_compile_options(sist2
PRIVATE
-O3
# -march=native
-fno-stack-protector
-fomit-frame-pointer
)
target_compile_options(sist2
PRIVATE
-O3
# -march=native
-fno-stack-protector
-fomit-frame-pointer
)
TARGET_LINK_LIBRARIES(
sist2
TARGET_LINK_LIBRARIES(
sist2
${GLIB_LIBRARIES}
${GOBJECT_LIBRARIES}
${UUID_LIBRARIES}
${GLIB_LIBRARIES}
${GOBJECT_LIBRARIES}
${UUID_LIBRARIES}
# ffmpeg
${PROJECT_SOURCE_DIR}/lib/libavcodec.a
${PROJECT_SOURCE_DIR}/lib/libavformat.a
${PROJECT_SOURCE_DIR}/lib/libavutil.a
${PROJECT_SOURCE_DIR}/lib/libswscale.a
${PROJECT_SOURCE_DIR}/lib/libswresample.a
# ${FFMPEG_LIBRARIES}
# swscale
# ffmpeg
${PROJECT_SOURCE_DIR}/lib/libavcodec.a
${PROJECT_SOURCE_DIR}/lib/libavformat.a
${PROJECT_SOURCE_DIR}/lib/libavutil.a
${PROJECT_SOURCE_DIR}/lib/libswscale.a
${PROJECT_SOURCE_DIR}/lib/libswresample.a
# ${FFMPEG_LIBRARIES}
# swscale
# mupdf
${PROJECT_SOURCE_DIR}/lib/libmupdf.a
${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
# mupdf
${PROJECT_SOURCE_DIR}/lib/libmupdf.a
${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
# onion
${PROJECT_SOURCE_DIR}/lib/libonion_static.a
# onion
${PROJECT_SOURCE_DIR}/lib/libonion_static.a
pthread
curl
m
bz2
magic
)
pthread
curl
m
bz2
magic
)
endif ()
if (WITH_SIST2_SCAN)
set_target_properties(
sist2_scan
PROPERTIES COMPILE_DEFINITIONS SIST_SCAN_ONLY
)
set_target_properties(
sist2_scan
PROPERTIES
COMPILE_DEFINITIONS SIST_SCAN_ONLY
LINK_FLAGS -static
)
target_include_directories(
sist2_scan PUBLIC
${LIBMAGIC_INCLUDE_DIRS}
${GOBJECT_INCLUDE_DIRS}
${OPENSSL_INCLUDE_DIR}
${FFMPEG_INCLUDE_DIRS}
${GLIB_INCLUDE_DIRS}
${UUID_INCLUDE_DIRS}
${FREETYPE_INCLUDE_DIRS}
${PROJECT_SOURCE_DIR}/
${PROJECT_SOURCE_DIR}/lmdb/libraries/liblmdb/
${PROJECT_SOURCE_DIR}/lib/onion/src/
${PROJECT_SOURCE_DIR}/lib/mupdf/include/
)
target_link_directories(
sist2_scan PUBLIC
${UUID_LIBRARY_DIRS}
${FFMPEG_LIBRARY_DIRS}
${OPENSSL_CRYPTO_LIBRARY}
)
target_compile_options(sist2_scan
PRIVATE
-O3
# -march=native
-fno-stack-protector
-fomit-frame-pointer
)
TARGET_LINK_LIBRARIES(
sist2_scan
${GLIB_LIBRARIES}
${GOBJECT_LIBRARIES}
${UUID_LIBRARIES}
# ffmpeg
${PROJECT_SOURCE_DIR}/lib/libavcodec.a
${PROJECT_SOURCE_DIR}/lib/libavformat.a
${PROJECT_SOURCE_DIR}/lib/libavutil.a
${PROJECT_SOURCE_DIR}/lib/libswscale.a
${PROJECT_SOURCE_DIR}/lib/libswresample.a
# mupdf
${PROJECT_SOURCE_DIR}/lib/libmupdf.a
${PROJECT_SOURCE_DIR}/lib/libmupdf-third.a
${PROJECT_SOURCE_DIR}/lib/libbz2.a
${PROJECT_SOURCE_DIR}/lib/libmagic.a
pthread
m
)
endif ()
add_custom_target(
before_sist2
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/before_build.sh
)
add_dependencies(sist2 before_sist2)
IF (WITH_SIST2)
add_dependencies(sist2 before_sist2)
else ()
add_dependencies(sist2_scan before_sist2)
endif ()

2
scripts/before_build.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
rm -rf index.sist2/

20
scripts/get_static_libs.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
cd lib
cd mupdf
@ -36,4 +36,22 @@ make -j 4
cd ../..
mv onion/build/src/onion/libonion_static.a .
#bzip2
git clone https://github.com/enthought/bzip2-1.0.6
cd bzip2-1.0.6
make -j 4
cd ..
mv bzip2-1.0.6/libbz2.a .
# magic
git clone https://github.com/threatstack/libmagic
cd libmagic
./autogen.sh
./configure --enable-static --disable-shared
make -j 4
cd ..
mv libmagic/src/.libs/libmagic.a .
cd ..

44
scripts/get_static_libs_freebsd.sh Executable file
View File

@ -0,0 +1,44 @@
#!/usr/bin/env bash
cd lib
# mupdf
cd mupdf
HAVE_X11=no HAVE_GLUT=no gmake -j 4
cd ..
mv mupdf/build/release/libmupdf.a .
mv mupdf/build/release/libmupdf-third.a .
# ffmpeg
cd ffmpeg
./configure --disable-shared --enable-static --disable-ffmpeg --disable-ffplay \
--disable-ffprobe --disable-doc\
--disable-manpages --disable-postproc --disable-avfilter \
--disable-alsa --disable-lzma --disable-xlib --disable-debug\
--disable-vdpau --disable-vaapi --disable-sdl2 --disable-network
gmake -j 4
cd ..
mv ffmpeg/libavcodec/libavcodec.a .
mv ffmpeg/libavformat/libavformat.a .
mv ffmpeg/libavutil/libavutil.a .
mv ffmpeg/libswresample/libswresample.a .
mv ffmpeg/libswscale/libswscale.a .
#bzip2
git clone https://github.com/enthought/bzip2-1.0.6
cd bzip2-1.0.6
make -j 4
cd ..
mv bzip2-1.0.6/libbz2.a .
# magic
git clone https://github.com/threatstack/libmagic
cd libmagic
./autogen.sh
./configure --enable-static --disable-shared
make -j 4
cd ..
mv libmagic/src/.libs/libmagic.a .
cd ..

22
src/cli.c
View File

@ -17,16 +17,6 @@ scan_args_t *scan_args_create() {
return args;
}
index_args_t *index_args_create() {
index_args_t *args = calloc(sizeof(index_args_t), 1);
return args;
}
web_args_t *web_args_create() {
web_args_t *args = calloc(sizeof(web_args_t), 1);
return args;
}
int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n");
@ -100,6 +90,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
return 0;
}
#ifndef SIST_SCAN_ONLY
int index_args_validate(index_args_t *args, int argc, const char **argv) {
if (argc < 2) {
@ -153,3 +144,14 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
return 0;
}
index_args_t *index_args_create() {
index_args_t *args = calloc(sizeof(index_args_t), 1);
return args;
}
web_args_t *web_args_create() {
web_args_t *args = calloc(sizeof(web_args_t), 1);
return args;
}
#endif

7
src/cli.h
View File

@ -15,6 +15,10 @@ typedef struct scan_args {
char *path;
} scan_args_t;
scan_args_t *scan_args_create();
int scan_args_validate(scan_args_t *args, int argc, const char **argv);
#ifndef SIST_SCAN_ONLY
typedef struct index_args {
char *es_url;
const char *index_path;
@ -30,12 +34,11 @@ typedef struct web_args {
const char **indices;
} web_args_t;
scan_args_t *scan_args_create();
index_args_t *index_args_create();
web_args_t *web_args_create();
int scan_args_validate(scan_args_t *args, int argc, const char **argv);
int index_args_validate(index_args_t *args, int argc, const char **argv);
int web_args_validate(web_args_t *args, int argc, const char **argv);
#endif
#endif

2
src/ctx.h
View File

@ -26,6 +26,7 @@ struct {
} ScanCtx;
#ifndef SIST_SCAN_ONLY
struct {
char *es_url;
} IndexCtx;
@ -35,6 +36,7 @@ struct {
int index_count;
struct index_t indices[16];
} WebCtx;
#endif
#endif

4
src/io/serialize.c
View File

@ -115,8 +115,8 @@ void write_document(document_t *doc) {
if (IndexFd == -1) {
char dstfile[PATH_MAX];
pid_t tid = syscall(SYS_gettid);
snprintf(dstfile, PATH_MAX, "%s_index_%d", ScanCtx.index.path, tid);
pthread_t self = pthread_self();
snprintf(dstfile, PATH_MAX, "%s_index_%lu", ScanCtx.index.path, self);
IndexFd = open(dstfile, O_CREAT | O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR);
if (IndexFd == -1) {

26
src/main.c
View File

@ -1,7 +1,12 @@
#include "sist.h"
#include "ctx.h"
#ifndef SIST_SCAN_ONLY
#define DESCRIPTION "Lightning-fast file system indexer and search tool."
#else
#define DESCRIPTION "Lightning-fast file system indexer and search tool. (SCAN ONLY)"
#endif
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
@ -14,7 +19,9 @@ static const char *const usage[] = {
};
void global_init() {
#ifndef SIST_SCAN_ONLY
curl_global_init(CURL_GLOBAL_NOTHING);
#endif
av_log_set_level(AV_LOG_QUIET);
}
@ -23,7 +30,7 @@ void init_dir(const char *dirpath) {
snprintf(path, PATH_MAX, "%sdescriptor.json", dirpath);
uuid_t uuid;
uuid_generate_time_safe(uuid);
uuid_generate(uuid);
uuid_unparse(uuid, ScanCtx.index.desc.uuid);
time(&ScanCtx.index.desc.timestamp);
strcpy(ScanCtx.index.desc.version, Version);
@ -116,6 +123,7 @@ void sist2_scan(scan_args_t *args) {
store_destroy(ScanCtx.index.store);
}
#ifndef SIST_SCAN_ONLY
void sist2_index(index_args_t *args) {
IndexCtx.es_url = args->es_url;
@ -186,6 +194,7 @@ void sist2_web(web_args_t *args) {
serve(args->bind, args->port);
}
#endif
int main(int argc, const char *argv[]) {
@ -193,8 +202,10 @@ int main(int argc, const char *argv[]) {
global_init();
scan_args_t *scan_args = scan_args_create();
#ifndef SIST_SCAN_ONLY
index_args_t *index_args = index_args_create();
web_args_t *web_args = web_args_create();
#endif
char * common_es_url = NULL;
@ -214,6 +225,7 @@ int main(int argc, const char *argv[]) {
OPT_STRING(0, "rewrite-url", &scan_args->rewrite_url, "Serve files from this url instead of from disk."),
OPT_STRING(0, "name", &scan_args->name, "Index display name. DEFAULT: (name of the directory)"),
#ifndef SIST_SCAN_ONLY
OPT_GROUP("Index options"),
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."),
@ -224,6 +236,7 @@ int main(int argc, const char *argv[]) {
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"),
OPT_STRING(0, "bind", &web_args->bind, "Listen on this address. DEFAULT=localhost"),
OPT_STRING(0, "port", &web_args->port, "Listen on this port. DEFAULT=4090"),
#endif
OPT_END(),
};
@ -233,8 +246,10 @@ int main(int argc, const char *argv[]) {
argparse_describe(&argparse, DESCRIPTION, EPILOG);
argc = argparse_parse(&argparse, argc, argv);
#ifndef SIST_SCAN_ONLY
web_args->es_url = common_es_url;
index_args->es_url = common_es_url;
#endif
if (argc == 0) {
argparse_usage(&argparse);
@ -247,7 +262,10 @@ int main(int argc, const char *argv[]) {
}
sist2_scan(scan_args);
} else if (strcmp(argv[0], "index") == 0) {
}
#ifndef SIST_SCAN_ONLY
else if (strcmp(argv[0], "index") == 0) {
int err = index_args_validate(index_args, argc, argv);
if (err != 0) {
@ -263,7 +281,9 @@ int main(int argc, const char *argv[]) {
}
sist2_web(web_args);
} else {
}
#endif
else {
fprintf(stderr, "Invalid command: '%s'\n", argv[0]);
argparse_usage(&argparse);
return 1;

2
src/parsing/parse.c
View File

@ -57,7 +57,7 @@ void parse(void *arg) {
doc.ino = job->info.st_ino;
doc.mtime = job->info.st_mtim.tv_sec;
uuid_generate_time_safe(doc.uuid);
uuid_generate(doc.uuid);
char *buf[PARSE_BUF_SIZE];
if (job->info.st_size == 0) {

11
src/sist.h
View File

@ -25,19 +25,20 @@
#include <pthread.h>
#include <sys/stat.h>
#include <wordexp.h>
#ifndef SIST_SCAN_ONLY
#include <onion/onion.h>
#include <onion/handler.h>
#include <onion/block.h>
#include <onion/shortcuts.h>
#include <curl/curl.h>
#endif
#include "cJSON/cJSON.h"
#include "types.h"
#include "tpool.h"
#include "util.h"
#include "src/index/elastic.h"
#include "io/store.h"
#include "io/serialize.h"
#include "io/walk.h"
@ -47,9 +48,13 @@
#include "parsing/pdf.h"
#include "parsing/media.h"
#include "parsing/font.h"
#include "cli.h"
#ifndef SIST_SCAN_ONLY
#include "src/index/elastic.h"
#include "index/web.h"
#include "web/serve.h"
#include "cli.h"
#endif
;

131
src/util.c
View File

@ -1,6 +1,133 @@
#define _GNU_SOURCE
#include "util.h"
dyn_buffer_t dyn_buffer_create() {
dyn_buffer_t buf;
buf.size = INITIAL_BUF_SIZE;
buf.cur = 0;
buf.buf = malloc(INITIAL_BUF_SIZE);
return buf;
}
void grow_buffer(dyn_buffer_t *buf, size_t size) {
if (buf->cur + size > buf->size) {
do {
buf->size *= 2;
} while (buf->cur + size > buf->size);
buf->buf = realloc(buf->buf, buf->size);
}
}
void grow_buffer_small(dyn_buffer_t *buf) {
if (buf->cur + sizeof(long) > buf->size) {
buf->size *= 2;
buf->buf = realloc(buf->buf, buf->size);
}
}
void dyn_buffer_write(dyn_buffer_t *buf, void *data, size_t size) {
grow_buffer(buf, size);
memcpy(buf->buf + buf->cur, data, size);
buf->cur += size;
}
void dyn_buffer_write_char(dyn_buffer_t *buf, char c) {
grow_buffer_small(buf);
*(buf->buf + buf->cur) = c;
buf->cur += sizeof(c);
}
void dyn_buffer_write_str(dyn_buffer_t *buf, char *str) {
dyn_buffer_write(buf, str, strlen(str));
dyn_buffer_write_char(buf, '\0');
}
void dyn_buffer_write_int(dyn_buffer_t *buf, int d) {
grow_buffer_small(buf);
*(int *) (buf->buf + buf->cur) = d;
buf->cur += sizeof(int);
}
void dyn_buffer_write_short(dyn_buffer_t *buf, short s) {
grow_buffer_small(buf);
*(short *) (buf->buf + buf->cur) = s;
buf->cur += sizeof(short);
}
void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l) {
grow_buffer_small(buf);
*(unsigned long *) (buf->buf + buf->cur) = l;
buf->cur += sizeof(unsigned long);
}
void dyn_buffer_destroy(dyn_buffer_t *buf) {
free(buf->buf);
}
void text_buffer_destroy(text_buffer_t *buf) {
dyn_buffer_destroy(&buf->dyn_buffer);
}
text_buffer_t text_buffer_create(int max_size) {
text_buffer_t text_buf;
text_buf.dyn_buffer = dyn_buffer_create();
text_buf.max_size = max_size;
text_buf.last_char_was_whitespace = FALSE;
return text_buf;
}
void text_buffer_terminate_string(text_buffer_t *buf) {
dyn_buffer_write_char(&buf->dyn_buffer, '\0');
}
int text_buffer_append_char(text_buffer_t *buf, int c) {
if (SHOULD_IGNORE_CHAR(c)) {
if (!buf->last_char_was_whitespace) {
dyn_buffer_write_char(&buf->dyn_buffer, ' ');
buf->last_char_was_whitespace = TRUE;
if (buf->dyn_buffer.cur >= buf->max_size) {
return TEXT_BUF_FULL;
}
}
} else {
buf->last_char_was_whitespace = FALSE;
dyn_buffer_write_char(&buf->dyn_buffer, (char) c);
if (buf->dyn_buffer.cur >= buf->max_size) {
return TEXT_BUF_FULL;
}
}
return 0;
}
void incremental_put(GHashTable *table, unsigned long inode_no, int mtime) {
g_hash_table_insert(table, (gpointer) inode_no, GINT_TO_POINTER(mtime));
}
int incremental_get(GHashTable *table, unsigned long inode_no) {
if (table != NULL) {
return GPOINTER_TO_INT(g_hash_table_lookup(table, (gpointer) inode_no));
} else {
return 0;
}
}
int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no) {
g_hash_table_insert(table, GINT_TO_POINTER(inode_no), GINT_TO_POINTER(1));
}
#define PBSTR "========================================"
#define PBWIDTH 40
@ -27,7 +154,7 @@ char *abspath(const char *path) {
wordexp_t w;
wordexp(path, &w, 0);
char *abs = canonicalize_file_name(w.we_wordv[0]);
char *abs = realpath(w.we_wordv[0], NULL);
if (abs == NULL) {
return NULL;
}

178
src/util.h
View File

@ -21,132 +21,6 @@ typedef struct text_buffer {
dyn_buffer_t dyn_buffer;
} text_buffer_t;
__always_inline
dyn_buffer_t dyn_buffer_create() {
dyn_buffer_t buf;
buf.size = INITIAL_BUF_SIZE;
buf.cur = 0;
buf.buf = malloc(INITIAL_BUF_SIZE);
return buf;
}
__always_inline
void grow_buffer(dyn_buffer_t *buf, size_t size) {
if (buf->cur + size > buf->size) {
do {
buf->size *= 2;
} while (buf->cur + size > buf->size);
buf->buf = realloc(buf->buf, buf->size);
}
}
__always_inline
void grow_buffer_small(dyn_buffer_t *buf) {
if (buf->cur + sizeof(long) > buf->size) {
buf->size *= 2;
buf->buf = realloc(buf->buf, buf->size);
}
}
__always_inline
void dyn_buffer_write(dyn_buffer_t *buf, void *data, size_t size) {
grow_buffer(buf, size);
memcpy(buf->buf + buf->cur, data, size);
buf->cur += size;
}
__always_inline
void dyn_buffer_write_char(dyn_buffer_t *buf, char c) {
grow_buffer_small(buf);
*(buf->buf + buf->cur) = c;
buf->cur += sizeof(c);
}
__always_inline
void dyn_buffer_write_str(dyn_buffer_t *buf, char *str) {
dyn_buffer_write(buf, str, strlen(str));
dyn_buffer_write_char(buf, '\0');
}
__always_inline
void dyn_buffer_write_int(dyn_buffer_t *buf, int d) {
grow_buffer_small(buf);
*(int *) (buf->buf + buf->cur) = d;
buf->cur += sizeof(int);
}
__always_inline
void dyn_buffer_write_short(dyn_buffer_t *buf, short s) {
grow_buffer_small(buf);
*(short *) (buf->buf + buf->cur) = s;
buf->cur += sizeof(short);
}
__always_inline
void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l) {
grow_buffer_small(buf);
*(unsigned long *) (buf->buf + buf->cur) = l;
buf->cur += sizeof(unsigned long);
}
__always_inline
void dyn_buffer_destroy(dyn_buffer_t *buf) {
free(buf->buf);
}
__always_inline
void text_buffer_destroy(text_buffer_t *buf) {
dyn_buffer_destroy(&buf->dyn_buffer);
}
__always_inline
text_buffer_t text_buffer_create(int max_size) {
text_buffer_t text_buf;
text_buf.dyn_buffer = dyn_buffer_create();
text_buf.max_size = max_size;
text_buf.last_char_was_whitespace = FALSE;
return text_buf;
}
__always_inline
void text_buffer_terminate_string(text_buffer_t *buf) {
dyn_buffer_write_char(&buf->dyn_buffer, '\0');
}
__always_inline
int text_buffer_append_char(text_buffer_t *buf, int c) {
if (SHOULD_IGNORE_CHAR(c)) {
if (!buf->last_char_was_whitespace) {
dyn_buffer_write_char(&buf->dyn_buffer, ' ');
buf->last_char_was_whitespace = TRUE;
if (buf->dyn_buffer.cur >= buf->max_size) {
return TEXT_BUF_FULL;
}
}
} else {
buf->last_char_was_whitespace = FALSE;
dyn_buffer_write_char(&buf->dyn_buffer, (char) c);
if (buf->dyn_buffer.cur >= buf->max_size) {
return TEXT_BUF_FULL;
}
}
return 0;
}
char *abspath(const char * path);
char *expandpath(const char *path);
dyn_buffer_t url_escape(char *str);
@ -154,27 +28,41 @@ dyn_buffer_t url_escape(char *str);
void progress_bar_print(double percentage, size_t tn_size, size_t index_size);
__always_inline
void incremental_put(GHashTable *table, unsigned long inode_no, int mtime) {
g_hash_table_insert(table, (gpointer) inode_no, GINT_TO_POINTER(mtime));
}
__always_inline
int incremental_get(GHashTable *table, unsigned long inode_no) {
if (table != NULL) {
return GPOINTER_TO_INT(g_hash_table_lookup(table, (gpointer) inode_no));
} else {
return 0;
}
}
__always_inline
int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no) {
g_hash_table_insert(table, GINT_TO_POINTER(inode_no), GINT_TO_POINTER(1));
}
GHashTable *incremental_get_table();
dyn_buffer_t dyn_buffer_create();
void grow_buffer(dyn_buffer_t *buf, size_t size);
void grow_buffer_small(dyn_buffer_t *buf);
void dyn_buffer_write(dyn_buffer_t *buf, void *data, size_t size);
void dyn_buffer_write_char(dyn_buffer_t *buf, char c);
void dyn_buffer_write_str(dyn_buffer_t *buf, char *str);
void dyn_buffer_write_int(dyn_buffer_t *buf, int d);
void dyn_buffer_write_short(dyn_buffer_t *buf, short s);
void dyn_buffer_write_long(dyn_buffer_t *buf, unsigned long l);
void dyn_buffer_destroy(dyn_buffer_t *buf);
void text_buffer_destroy(text_buffer_t *buf);
text_buffer_t text_buffer_create(int max_size);
void text_buffer_terminate_string(text_buffer_t *buf);
int text_buffer_append_char(text_buffer_t *buf, int c);
void incremental_put(GHashTable *table, unsigned long inode_no, int mtime);
int incremental_get(GHashTable *table, unsigned long inode_no);
int incremental_mark_file_for_copy(GHashTable *table, unsigned long inode_no);
#endif

2
src/web/static_generated.c
View File

File diff suppressed because one or more lines are too long

2
web/js/search.js
View File

@ -29,7 +29,7 @@ $.jsonPost("i").then(resp => {
resp["indices"].forEach(idx => {
$("#indices").append($("<option>")
.attr("value", idx.id)
.attr("selected", true)
.attr("selected", !idx.name.includes("(nsfw)"))
.append(idx.name)
);
selectedIndices.push(idx.id);