simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-10 14:06:45 +00:00
Code Issues Projects Releases Wiki Activity

Better logging, fixes #15

Browse Source
This commit is contained in:
simon987 2019-12-21 12:32:08 -05:00
parent 80fbcb2a01
commit 663f8e21c1
18 changed files with 303 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CMakeLists.txt
View File

@ -27,6 +27,9 @@ if (WITH_SIST2)
src/index/elastic.c src/index/elastic.h
src/util.c src/util.h
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
src/parsing/arc.c src/parsing/arc.h
src/parsing/doc.c src/parsing/doc.h
src/log.c src/log.h
# argparse
argparse/argparse.h argparse/argparse.c
@ -41,7 +44,7 @@ if (WITH_SIST2)
# utf8.h
utf8.h/utf8.h
src/parsing/arc.c src/parsing/arc.h src/parsing/doc.c src/parsing/doc.h)
)
endif ()
if (WITH_SIST2_SCAN)
@ -60,6 +63,9 @@ if (WITH_SIST2_SCAN)
src/parsing/text.h src/parsing/text.c
src/util.c src/util.h
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
src/parsing/arc.h src/parsing/arc.c
src/parsing/doc.h src/parsing/doc.c
src/log.h src/log.c
# argparse
argparse/argparse.h argparse/argparse.c
@ -74,7 +80,7 @@ if (WITH_SIST2_SCAN)
# utf8.h
utf8.h/utf8.h
src/parsing/arc.c src/parsing/arc.h src/parsing/doc.c src/parsing/doc.h)
)
endif ()
find_package(PkgConfig REQUIRED)

35
src/cli.c
View File

@ -1,4 +1,5 @@
#include "cli.h"
#include "ctx.h"
#define DEFAULT_OUTPUT "index.sist2/"
#define DEFAULT_CONTENT_SIZE 32768
@ -134,6 +135,19 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
fprintf(stderr, "Archive mode must be one of (skip, list, shallow, recurse), got '%s'", args->archive);
return 1;
}
LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
LOG_DEBUGF("cli.c", "arg size=%d", args->size)
LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
LOG_DEBUGF("cli.c", "arg threads=%d", args->threads)
LOG_DEBUGF("cli.c", "arg incremental=%s", args->incremental)
LOG_DEBUGF("cli.c", "arg output=%s", args->incremental)
LOG_DEBUGF("cli.c", "arg rewrite_url=%s", args->rewrite_url)
LOG_DEBUGF("cli.c", "arg name=%s", args->name)
LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
LOG_DEBUGF("cli.c", "arg path=%s", args->path)
LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
return 0;
}
@ -189,6 +203,14 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
args->batch_size = DEFAULT_BATCH_SIZE;
}
LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path)
LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
LOG_DEBUGF("cli.c", "arg script=%s", args->script)
LOG_DEBUGF("cli.c", "arg print=%d", args->print)
LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size)
LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset)
return 0;
}
@ -223,10 +245,21 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
for (int i = 0; i < args->index_count; i++) {
char *abs_path = abspath(args->indices[i]);
if (abs_path == NULL) {
fprintf(stderr, "File not found: %s\n", abs_path);
fprintf(stderr, "File not found: %s\n", args->indices[i]);
return 1;
}
}
LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
LOG_DEBUGF("cli.c", "arg bind=%s", args->bind)
LOG_DEBUGF("cli.c", "arg port=%s", args->port)
LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials)
LOG_DEBUGF("cli.c", "arg b64credentials=%s", args->b64credentials)
LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count)
for (int i = 0; i < args->index_count; i++) {
LOG_DEBUGF("cli.c", "arg indices[%d]=%s", i, args->indices[i])
}
return 0;
}

7
src/ctx.h
View File

@ -17,6 +17,8 @@ struct {
float tn_qscale;
int depth;
archive_mode_t archive_mode;
int verbose;
int very_verbose;
size_t stat_tn_size;
size_t stat_index_size;
@ -27,6 +29,11 @@ struct {
pthread_mutex_t mupdf_mu;
} ScanCtx;
struct {
int verbose;
int very_verbose;
int no_color;
} LogCtx;
#ifndef SIST_SCAN_ONLY
struct {

3
src/io/serialize.c
View File

@ -262,8 +262,7 @@ void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func
break;
}
default:
fprintf(stderr, "Invalid meta key (corrupt index): %x\n", key);
break;
LOG_FATALF("serialize.c", "Invalid meta key (corrupt index): %x", key)
}
key = getc(file);

14
src/io/store.c
View File

@ -9,9 +9,9 @@ store_t *store_create(char *path) {
mdb_env_create(&store->env);
int open_ret = mdb_env_open(store->env,
path,
MDB_WRITEMAP | MDB_MAPASYNC,
S_IRUSR | S_IWUSR
path,
MDB_WRITEMAP | MDB_MAPASYNC,
S_IRUSR | S_IWUSR
);
if (open_ret != 0) {
@ -42,6 +42,12 @@ void store_destroy(store_t *store) {
void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len) {
if (LogCtx.very_verbose) {
char uuid_str[UUID_STR_LEN];
uuid_unparse((unsigned char *) key, uuid_str);
LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", uuid_str, buf_len)
}
MDB_val mdb_key;
mdb_key.mv_data = key;
mdb_key.mv_size = key_len;
@ -68,6 +74,8 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
mdb_env_set_mapsize(store->env, store->size);
mdb_txn_begin(store->env, NULL, 0, &txn);
put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
LOG_INFOF("store.c", "Updated mdb mapsize to %lu bytes", store->size)
}
mdb_txn_commit(txn);

99
src/log.c Normal file
View File

@ -0,0 +1,99 @@
#include "log.h"
const char *log_colors[] = {
"\033[34m", "\033[01;34m", "\033[0m",
"\033[01;33m", "\033[31m", "\033[01;31m"
};
const char *log_levels[] = {
"DEBUG", "INFO", "WARNING", "ERROR", "FATAL"
};
void sist_logf(char *filepath, int level, char *format, ...) {
static int is_tty = -1;
if (is_tty == -1) {
is_tty = isatty(STDERR_FILENO);
}
char log_str[LOG_MAX_LENGTH];
unsigned long long pid = (unsigned long long) pthread_self();
char datetime[32];
time_t t;
struct tm result;
t = time(NULL);
localtime_r(&t, &result);
strftime(datetime, sizeof(datetime), "%Y-%m-%d %H:%M:%S", &result);
int log_len;
if (is_tty) {
log_len = snprintf(
log_str, sizeof(log_str),
"\033[%dm[%04X]%s [%s] [%s %s] ",
31 + ((unsigned int) (pid)) % 7, pid, log_colors[level],
datetime, log_levels[level], filepath
);
} else {
log_len = snprintf(
log_str, sizeof(log_str),
"[%04X] [%s] [%s %s] ",
pid, datetime, log_levels[level], filepath
);
}
va_list ap;
va_start(ap, format);
size_t maxsize = sizeof(log_str) - log_len;
log_len += vsnprintf(log_str + log_len, maxsize, format, ap);
va_end(ap);
if (is_tty) {
log_len += sprintf(log_str + log_len, "\033[0m\n");
} else {
*(log_str + log_len) = '\n';
log_len += 1;
}
write(STDOUT_FILENO, log_str, log_len);
}
void sist_log(char *filepath, int level, char *str) {
static int is_tty = -1;
if (is_tty == -1) {
is_tty = isatty(STDERR_FILENO);
}
char log_str[LOG_MAX_LENGTH];
unsigned long long pid = (unsigned long long) pthread_self();
char datetime[32];
time_t t;
struct tm result;
t = time(NULL);
localtime_r(&t, &result);
strftime(datetime, sizeof(datetime), "%Y-%m-%d %H:%M:%S", &result);
int log_len;
if (is_tty) {
log_len = snprintf(
log_str, sizeof(log_str),
"\033[%dm[%04X]%s [%s] [%s %s] %s \033[0m\n",
31 + ((unsigned int) (pid)) % 7, pid, log_colors[level],
datetime, log_levels[level], filepath,
str
);
} else {
log_len = snprintf(
log_str, sizeof(log_str),
"[%04X] [%s] [%s %s] %s \n",
pid, datetime, log_levels[level], filepath,
str
);
}
write(STDERR_FILENO, log_str, log_len);
}

45
src/log.h Normal file
View File

@ -0,0 +1,45 @@
#ifndef SIST2_LOG_H
#define SIST2_LOG_H
#define LOG_MAX_LENGTH 8192
#define SIST_DEBUG 0
#define SIST_INFO 1
#define SIST_WARNING 2
#define SIST_ERROR 3
#define SIST_FATAL 4
#define LOG_DEBUGF(filepath, fmt, ...) \
if (LogCtx.very_verbose) {sist_logf(filepath, SIST_DEBUG, fmt, __VA_ARGS__);}
#define LOG_DEBUG(filepath, str) \
if (LogCtx.very_verbose) {sist_log(filepath, SIST_DEBUG, str);}
#define LOG_INFOF(filepath, fmt, ...) \
if (LogCtx.verbose) {sist_logf(filepath, SIST_INFO, fmt, __VA_ARGS__);}
#define LOG_INFO(filepath, str) \
if (LogCtx.verbose) {sist_log(filepath, SIST_INFO, str);}
#define LOG_WARNINGF(filepath, fmt, ...) \
if (LogCtx.verbose) {sist_logf(filepath, SIST_WARNING, fmt, __VA_ARGS__);}
#define LOG_WARNING(filepath, str) \
if (LogCtx.verbose) {sist_log(filepath, SIST_WARNING, str);}
#define LOG_ERRORF(filepath, fmt, ...) \
if (LogCtx.verbose) {sist_logf(filepath, SIST_ERROR, fmt, __VA_ARGS__);}
#define LOG_ERROR(filepath, str) \
if (LogCtx.verbose) {sist_log(filepath, SIST_ERROR, str);}
#define LOG_FATALF(filepath, fmt, ...) \
sist_logf(filepath, SIST_FATAL, fmt, __VA_ARGS__);\
exit(-1);
#define LOG_FATAL(filepath, str) \
sist_log(filepath, SIST_FATAL, str);\
exit(-1);
#include "src/sist.h"
void sist_logf(char *filepath, int level, char *format, ...);
void sist_log(char *filepath, int level, char *str);
#endif

27
src/main.c
View File

@ -10,7 +10,7 @@
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
static const char *const Version = "1.1.11";
static const char *const Version = "1.1.12";
static const char *const usage[] = {
"sist2 scan [OPTION]... PATH",
"sist2 index [OPTION]... INDEX",
@ -40,24 +40,7 @@ void init_dir(const char *dirpath) {
}
void scan_print_header() {
printf("sist2 V%s\n", Version);
printf("---------------------\n");
printf("threads\t\t\t%d\n", ScanCtx.threads);
printf("tn_qscale\t\t%.1f/31.0\n", ScanCtx.tn_qscale);
if (ScanCtx.tn_size > 0) {
printf("tn_size\t\t\t%dpx\n", ScanCtx.tn_size);
} else {
printf("tn_size\t\t\tdisabled\n");
}
if (ScanCtx.content_size > 0) {
printf("content_size\t\t%d B\n", ScanCtx.content_size);
} else {
printf("content_size\t\t\tdisabled\n");
}
printf("output\t\t\t%s\n", ScanCtx.index.path);
LOG_INFOF("main.c", "sist2 v%s", Version)
}
void sist2_scan(scan_args_t *args) {
@ -236,6 +219,8 @@ int main(int argc, const char *argv[]) {
OPT_HELP(),
OPT_BOOLEAN('v', "version", &arg_version, "Show version and exit"),
OPT_BOOLEAN(0, "verbose", &LogCtx.verbose, "Turn on logging"),
OPT_BOOLEAN(0, "very-verbose", &LogCtx.very_verbose, "Turn on debug messages"),
OPT_GROUP("Scan options"),
OPT_INTEGER('t', "threads", &scan_args->threads, "Number of threads. DEFAULT=1"),
@ -285,6 +270,10 @@ int main(int argc, const char *argv[]) {
exit(0);
}
if (LogCtx.very_verbose != 0) {
LogCtx.verbose = 1;
}
#ifndef SIST_SCAN_ONLY
web_args->es_url = common_es_url;
index_args->es_url = common_es_url;

17
src/parsing/arc.c
View File

@ -67,18 +67,24 @@ void parse_archive(vfile_t *f, document_t *doc) {
struct archive *a;
struct archive_entry *entry;
a = archive_read_new();
archive_read_support_filter_all(a);
archive_read_support_format_all(a);
arc_data_f data;
data.f = f;
int ret = 0;
if (data.f->is_fs_file) {
a = archive_read_new();
archive_read_support_filter_all(a);
archive_read_support_format_all(a);
ret = archive_read_open_filename(a, doc->filepath, ARC_BUF_SIZE);
} else if (ScanCtx.archive_mode == ARC_MODE_RECURSE) {
a = archive_read_new();
archive_read_support_filter_all(a);
archive_read_support_format_all(a);
ret = archive_read_open(
a, &data,
vfile_open_callback,
@ -86,12 +92,11 @@ void parse_archive(vfile_t *f, document_t *doc) {
vfile_close_callback
);
} else {
archive_read_free(a);
return;
}
if (ret != ARCHIVE_OK) {
fprintf(stderr, "OPEN[%d]:%s %s\n", ret, archive_error_string(a), doc->filepath);
LOG_ERRORF(doc->filepath, "(arc.c) [%d] %s", ret, archive_error_string(a))
archive_read_free(a);
return;
}

20
src/parsing/doc.c
View File

@ -1,6 +1,7 @@
#include "doc.h"
#include "src/ctx.h"
static void dumpText(mceTextReader_t *reader, dyn_buffer_t *buf) {
static void dump_text(mceTextReader_t *reader, dyn_buffer_t *buf) {
mce_skip_attributes(reader);
@ -17,7 +18,7 @@ static void dumpText(mceTextReader_t *reader, dyn_buffer_t *buf) {
} mce_end_element(reader);
mce_start_element(reader, NULL, NULL) {
dumpText(reader, buf);
dump_text(reader, buf);
} mce_end_element(reader);
} mce_end_children(reader)
@ -50,19 +51,20 @@ int should_read_part(opcPart part) {
return FALSE;
}
void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part) {
__always_inline
void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part, document_t *doc) {
mceTextReader_t reader;
int ret = opcXmlReaderOpen(c, &reader, part, NULL, "UTF-8", 0);
if (ret != OPC_ERROR_NONE) {
//todo verbose
LOG_ERRORF(doc->filepath, "(doc.c) opcXmlReaderOpen() returned error code %d", ret);
return;
}
mce_start_document(&reader) {
mce_start_element(&reader, NULL, NULL) {
dumpText(&reader, buf);
dump_text(&reader, buf);
} mce_end_element(&reader);
}mce_end_document(&reader);
@ -71,9 +73,13 @@ void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part) {
void parse_doc(void *mem, size_t mem_len, document_t *doc) {
if (mem == NULL) {
return;
}
opcContainer *c = opcContainerOpenMem(mem, mem_len, OPC_OPEN_READ_ONLY, NULL);
if (c == NULL) {
//todo verbose
LOG_ERROR(doc->filepath, "(doc.c) Couldn't open document with opcContainerOpenMem()");
return;
}
@ -82,7 +88,7 @@ void parse_doc(void *mem, size_t mem_len, document_t *doc) {
opcPart part = opcPartGetFirst(c);
do {
if (should_read_part(part)) {
read_part(c, &buf, part);
read_part(c, &buf, part, doc);
}
} while ((part = opcPartGetNext(c, part)));

3
src/parsing/font.c
View File

@ -147,6 +147,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
FT_Face face;
FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, buf_len, 0, &face);
if (err != 0) {
LOG_ERRORF(doc->filepath, "(font.c) FT_New_Memory_Face() returned error code [%d] %s", err, ft_error_string(err));
return;
}
@ -176,6 +177,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
err = FT_Set_Pixel_Sizes(face, 0, pixel);
if (err != 0) {
LOG_WARNINGF(doc->filepath, "(font.c) FT_Set_Pixel_Sizes() returned error code [%d] %s", err, ft_error_string(err))
return;
}
@ -194,6 +196,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
c = c >= 'a' && c <= 'z' ? c - 32 : c + 32;
err = FT_Load_Char(face, c, FT_LOAD_NO_HINTING | FT_LOAD_RENDER);
if (err != 0) {
LOG_WARNINGF(doc->filepath, "(font.c) FT_Load_Char() returned error code [%d] %s", err, ft_error_string(err));
continue;
}
}

32
src/parsing/media.c
View File

@ -77,7 +77,8 @@ AVFrame *scale_frame(const AVCodecContext *decoder, const AVFrame *frame, int si
return scaled_frame;
}
AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx) {
__always_inline
AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx, document_t *doc) {
AVFrame *frame = av_frame_alloc();
AVPacket avPacket;
@ -90,9 +91,12 @@ AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int st
int read_frame_ret = av_read_frame(pFormatCtx, &avPacket);
if (read_frame_ret != 0) {
// if (read_frame_ret != AVERROR_EOF) {
// fprintf(stderr, "Error reading frame: %d\n", read_frame_ret);
// }
if (read_frame_ret != AVERROR_EOF) {
LOG_WARNINGF(doc->filepath,
"(media.c) avcodec_read_frame() returned error code [%d] %s",
read_frame_ret, av_err2str(read_frame_ret)
)
}
av_frame_free(&frame);
av_packet_unref(&avPacket);
return NULL;
@ -109,7 +113,10 @@ AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int st
// Feed it to decoder
int decode_ret = avcodec_send_packet(decoder, &avPacket);
if (decode_ret != 0) {
printf("Error decoding frame: %s\n", av_err2str(decode_ret));
LOG_WARNINGF(doc->filepath,
"(media.c) avcodec_send_packet() returned error code [%d] %s",
decode_ret, av_err2str(decode_ret)
)
}
av_packet_unref(&avPacket);
receive_ret = avcodec_receive_frame(decoder, frame);
@ -152,7 +159,8 @@ void append_audio_meta(AVFormatContext *pFormatCtx, document_t *doc) {
}
__always_inline
void append_video_meta(AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int include_audio_tags, int is_video) {
void
append_video_meta(AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int include_audio_tags, int is_video) {
if (is_video) {
meta_line_t *meta_duration = malloc(sizeof(meta_line_t));
@ -259,7 +267,7 @@ void parse_media(AVFormatContext *pFormatCtx, document_t *doc) {
}
}
AVFrame *frame = read_frame(pFormatCtx, decoder, video_stream);
AVFrame *frame = read_frame(pFormatCtx, decoder, video_stream, doc);
if (frame == NULL) {
avcodec_free_context(&decoder);
avformat_close_input(&pFormatCtx);
@ -308,12 +316,12 @@ void parse_media_filename(const char *filepath, document_t *doc) {
AVFormatContext *pFormatCtx = avformat_alloc_context();
if (pFormatCtx == NULL) {
fprintf(stderr, "Could not allocate AVFormatContext! %s \n", filepath);
LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()")
return;
}
int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL);
if (res < 0) {
fprintf(stderr, "media error: %s %s\n", filepath, av_err2str(res));
LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res))
return;
}
@ -336,7 +344,7 @@ void parse_media_vfile(struct vfile *f, document_t *doc) {
AVFormatContext *pFormatCtx = avformat_alloc_context();
if (pFormatCtx == NULL) {
fprintf(stderr, "Could not allocate AVFormatContext! %s \n", f->filepath);
LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()")
return;
}
@ -350,8 +358,8 @@ void parse_media_vfile(struct vfile *f, document_t *doc) {
if (res == -5) {
// Tried to parse media that requires seek
return;
} else if(res < 0) {
fprintf(stderr, "media error: %s %s\n", f->filepath, av_err2str(res));
} else if (res < 0) {
LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res))
return;
}

14
src/parsing/parse.c
View File

@ -9,8 +9,7 @@ int fs_read(struct vfile *f, void *buf, size_t size) {
if (f->fd == -1) {
f->fd = open(f->filepath, O_RDONLY);
if (f->fd == -1) {
perror("open");
printf("%s\n", f->filepath);
LOG_ERRORF(f->filepath, "open(): [%d] %s", errno, strerror(errno))
return -1;
}
}
@ -39,7 +38,7 @@ void *read_all(parse_job_t *job, const char *buf, int bytes_read) {
int ret = job->vfile.read(&job->vfile, full_buf + bytes_read, job->info.st_size - bytes_read);
if (ret == -1) {
perror("read");
LOG_ERRORF(job->filepath, "read(): [%d] %s", errno, strerror(errno))
return NULL;
}
}
@ -75,6 +74,12 @@ void parse(void *arg) {
uuid_generate(doc.uuid);
char *buf[PARSE_BUF_SIZE];
if (LogCtx.very_verbose) {
char uuid_str[UUID_STR_LEN];
uuid_unparse(doc.uuid, uuid_str);
LOG_DEBUGF(job->filepath, "Starting parse job {%s}", uuid_str)
}
if (job->info.st_size == 0) {
doc.mime = MIME_EMPTY;
} else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) {
@ -87,6 +92,7 @@ void parse(void *arg) {
// Get mime type with libmagic
bytes_read = job->vfile.read(&job->vfile, buf, PARSE_BUF_SIZE);
if (bytes_read == -1) {
LOG_WARNINGF(job->filepath, "read() Error: %s", strerror(errno))
CLOSE_FILE(job->vfile)
return;
}
@ -95,7 +101,7 @@ void parse(void *arg) {
if (magic_mime_str != NULL) {
doc.mime = mime_get_mime_by_string(ScanCtx.mime_table, magic_mime_str);
if (doc.mime == 0) {
fprintf(stderr, "Couldn't find mime %s, %s\n", magic_mime_str, job->filepath + job->base);
LOG_WARNINGF(job->filepath, "Couldn't find mime %s", magic_mime_str);
}
}
}

26
src/parsing/pdf.c
View File

@ -14,6 +14,7 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
if (err != 0) {
fz_drop_page(ctx, cover);
LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
return NULL;
}
@ -52,6 +53,7 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
err = ctx->error.errcode;
if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
fz_drop_page(ctx, cover);
fz_drop_pixmap(ctx, pixmap);
return NULL;
@ -76,6 +78,7 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
fz_drop_pixmap(ctx, pixmap);
if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_new_buffer_from_pixmap_as_png() returned error code [%d] %s", err, ctx->error.message)
fz_drop_page(ctx, cover);
return NULL;
}
@ -83,14 +86,23 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
return cover;
}
void fz_noop_callback(__attribute__((unused)) void *user, __attribute__((unused)) const char *message) {}
void fz_err_callback(void *user, __attribute__((unused)) const char *message) {
if (LogCtx.verbose) {
document_t *doc = (document_t*) user;
LOG_WARNINGF(doc->filepath, "FZ: %s", message)
}
}
void init_ctx(fz_context *ctx) {
__always_inline
void init_ctx(fz_context *ctx, document_t *doc) {
fz_disable_icc(ctx);
fz_register_document_handlers(ctx);
ctx->warn.print = fz_noop_callback;
ctx->error.print = fz_noop_callback;
ctx->warn.print_user = doc;
ctx->warn.print = fz_err_callback;
ctx->error.print_user = doc;
ctx->error.print = fz_err_callback;
}
int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
@ -125,7 +137,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
}
fz_context *ctx = fz_new_context(NULL, NULL, FZ_STORE_UNLIMITED);
init_ctx(ctx);
init_ctx(ctx, doc);
int err = 0;
@ -171,6 +183,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
err = ctx->error.errcode;
if (err) {
LOG_WARNINGF(doc->filepath, "fz_count_pages() returned error code [%d] %s", err, ctx->error.message)
fz_drop_stream(ctx, stream);
fz_drop_document(ctx, fzdoc);
fz_drop_context(ctx);
@ -210,6 +223,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
fz_catch(ctx)
err = ctx->error.errcode;
if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
text_buffer_destroy(&text_buf);
fz_drop_page(ctx, page);
fz_drop_stream(ctx, stream);
@ -234,6 +248,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
err = ctx->error.errcode;
if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
text_buffer_destroy(&text_buf);
fz_drop_page(ctx, page);
fz_drop_stext_page(ctx, stext);
@ -272,4 +287,3 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
fz_drop_document(ctx, fzdoc);
fz_drop_context(ctx);
}

1
src/sist.h
View File

@ -59,6 +59,7 @@
#include "parsing/arc.h"
#include "parsing/doc.h"
#include "cli.h"
#include "log.h"
#include "utf8.h/utf8.h"
#ifndef SIST_SCAN_ONLY

10
src/tpool.c
View File

@ -136,6 +136,7 @@ static void *tpool_worker(void *arg) {
pthread_mutex_unlock(&(pool->work_mutex));
}
LOG_INFO("tpool.c", "Executing cleaup function")
pool->cleanup_func();
pthread_cond_signal(&(pool->working_cond));
@ -144,6 +145,7 @@ static void *tpool_worker(void *arg) {
}
void tpool_wait(tpool_t *pool) {
LOG_INFO("tpool.c", "Waiting for worker threads to finish")
pthread_mutex_lock(&(pool->work_mutex));
while (1) {
if (pool->done_cnt < pool->work_cnt) {
@ -159,6 +161,8 @@ void tpool_wait(tpool_t *pool) {
}
progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
pthread_mutex_unlock(&(pool->work_mutex));
LOG_INFO("tpool.c", "Worker threads finished")
}
void tpool_destroy(tpool_t *pool) {
@ -166,6 +170,8 @@ void tpool_destroy(tpool_t *pool) {
return;
}
LOG_INFO("tpool.c", "Destroying thread pool")
pthread_mutex_lock(&(pool->work_mutex));
tpool_work_t *work = pool->work_head;
while (work != NULL) {
@ -185,6 +191,8 @@ void tpool_destroy(tpool_t *pool) {
}
}
LOG_INFO("tpool.c", "Final cleanup")
pthread_mutex_destroy(&(pool->work_mutex));
pthread_cond_destroy(&(pool->has_work_cond));
pthread_cond_destroy(&(pool->working_cond));
@ -220,6 +228,8 @@ tpool_t *tpool_create(size_t thread_cnt, void cleanup_func()) {
void tpool_start(tpool_t *pool) {
LOG_INFOF("tpool.c", "Starting thread pool with %d threads", pool->thread_cnt)
for (size_t i = 0; i < pool->thread_cnt; i++) {
pthread_create(&pool->threads[i], NULL, tpool_worker, pool);
}

2
src/web/static_generated.c
View File

File diff suppressed because one or more lines are too long

2
web/search.html
View File

@ -11,7 +11,7 @@
<nav class="navbar navbar-expand-lg">
<a class="navbar-brand" href="/">sist2</a>
<span class="badge badge-pill version">v1.1.11</span>
<span class="badge badge-pill version">v1.1.12</span>
<span class="tagline">Lightning-fast file system indexer and search tool </span>
<a style="margin-left: auto" id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
</nav>