simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-12-14 07:49:06 +00:00
Code Issues Projects Releases Wiki Activity

Better logging, fixes #15

Browse Source
This commit is contained in:
simon987
2019-12-21 12:32:08 -05:00
parent 80fbcb2a01
commit 663f8e21c1
18 changed files with 303 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/parsing/arc.c
View File

@@ -67,18 +67,24 @@ void parse_archive(vfile_t *f, document_t *doc) {
struct archive *a;
struct archive_entry *entry;
a = archive_read_new();
archive_read_support_filter_all(a);
archive_read_support_format_all(a);
arc_data_f data;
data.f = f;
int ret = 0;
if (data.f->is_fs_file) {
a = archive_read_new();
archive_read_support_filter_all(a);
archive_read_support_format_all(a);
ret = archive_read_open_filename(a, doc->filepath, ARC_BUF_SIZE);
} else if (ScanCtx.archive_mode == ARC_MODE_RECURSE) {
a = archive_read_new();
archive_read_support_filter_all(a);
archive_read_support_format_all(a);
ret = archive_read_open(
a, &data,
vfile_open_callback,
@@ -86,12 +92,11 @@ void parse_archive(vfile_t *f, document_t *doc) {
vfile_close_callback
);
} else {
archive_read_free(a);
return;
}
if (ret != ARCHIVE_OK) {
fprintf(stderr, "OPEN[%d]:%s %s\n", ret, archive_error_string(a), doc->filepath);
LOG_ERRORF(doc->filepath, "(arc.c) [%d] %s", ret, archive_error_string(a))
archive_read_free(a);
return;
}

20
src/parsing/doc.c
View File

@@ -1,6 +1,7 @@
#include "doc.h"
#include "src/ctx.h"
static void dumpText(mceTextReader_t *reader, dyn_buffer_t *buf) {
static void dump_text(mceTextReader_t *reader, dyn_buffer_t *buf) {
mce_skip_attributes(reader);
@@ -17,7 +18,7 @@ static void dumpText(mceTextReader_t *reader, dyn_buffer_t *buf) {
} mce_end_element(reader);
mce_start_element(reader, NULL, NULL) {
dumpText(reader, buf);
dump_text(reader, buf);
} mce_end_element(reader);
} mce_end_children(reader)
@@ -50,19 +51,20 @@ int should_read_part(opcPart part) {
return FALSE;
}
void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part) {
__always_inline
void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part, document_t *doc) {
mceTextReader_t reader;
int ret = opcXmlReaderOpen(c, &reader, part, NULL, "UTF-8", 0);
if (ret != OPC_ERROR_NONE) {
//todo verbose
LOG_ERRORF(doc->filepath, "(doc.c) opcXmlReaderOpen() returned error code %d", ret);
return;
}
mce_start_document(&reader) {
mce_start_element(&reader, NULL, NULL) {
dumpText(&reader, buf);
dump_text(&reader, buf);
} mce_end_element(&reader);
}mce_end_document(&reader);
@@ -71,9 +73,13 @@ void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part) {
void parse_doc(void *mem, size_t mem_len, document_t *doc) {
if (mem == NULL) {
return;
}
opcContainer *c = opcContainerOpenMem(mem, mem_len, OPC_OPEN_READ_ONLY, NULL);
if (c == NULL) {
//todo verbose
LOG_ERROR(doc->filepath, "(doc.c) Couldn't open document with opcContainerOpenMem()");
return;
}
@@ -82,7 +88,7 @@ void parse_doc(void *mem, size_t mem_len, document_t *doc) {
opcPart part = opcPartGetFirst(c);
do {
if (should_read_part(part)) {
read_part(c, &buf, part);
read_part(c, &buf, part, doc);
}
} while ((part = opcPartGetNext(c, part)));

3
src/parsing/font.c
View File

@@ -147,6 +147,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
FT_Face face;
FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, buf_len, 0, &face);
if (err != 0) {
LOG_ERRORF(doc->filepath, "(font.c) FT_New_Memory_Face() returned error code [%d] %s", err, ft_error_string(err));
return;
}
@@ -176,6 +177,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
err = FT_Set_Pixel_Sizes(face, 0, pixel);
if (err != 0) {
LOG_WARNINGF(doc->filepath, "(font.c) FT_Set_Pixel_Sizes() returned error code [%d] %s", err, ft_error_string(err))
return;
}
@@ -194,6 +196,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
c = c >= 'a' && c <= 'z' ? c - 32 : c + 32;
err = FT_Load_Char(face, c, FT_LOAD_NO_HINTING | FT_LOAD_RENDER);
if (err != 0) {
LOG_WARNINGF(doc->filepath, "(font.c) FT_Load_Char() returned error code [%d] %s", err, ft_error_string(err));
continue;
}
}

32
src/parsing/media.c
View File

@@ -77,7 +77,8 @@ AVFrame *scale_frame(const AVCodecContext *decoder, const AVFrame *frame, int si
return scaled_frame;
}
AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx) {
__always_inline
AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx, document_t *doc) {
AVFrame *frame = av_frame_alloc();
AVPacket avPacket;
@@ -90,9 +91,12 @@ AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int st
int read_frame_ret = av_read_frame(pFormatCtx, &avPacket);
if (read_frame_ret != 0) {
// if (read_frame_ret != AVERROR_EOF) {
// fprintf(stderr, "Error reading frame: %d\n", read_frame_ret);
// }
if (read_frame_ret != AVERROR_EOF) {
LOG_WARNINGF(doc->filepath,
"(media.c) avcodec_read_frame() returned error code [%d] %s",
read_frame_ret, av_err2str(read_frame_ret)
)
}
av_frame_free(&frame);
av_packet_unref(&avPacket);
return NULL;
@@ -109,7 +113,10 @@ AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int st
// Feed it to decoder
int decode_ret = avcodec_send_packet(decoder, &avPacket);
if (decode_ret != 0) {
printf("Error decoding frame: %s\n", av_err2str(decode_ret));
LOG_WARNINGF(doc->filepath,
"(media.c) avcodec_send_packet() returned error code [%d] %s",
decode_ret, av_err2str(decode_ret)
)
}
av_packet_unref(&avPacket);
receive_ret = avcodec_receive_frame(decoder, frame);
@@ -152,7 +159,8 @@ void append_audio_meta(AVFormatContext *pFormatCtx, document_t *doc) {
}
__always_inline
void append_video_meta(AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int include_audio_tags, int is_video) {
void
append_video_meta(AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int include_audio_tags, int is_video) {
if (is_video) {
meta_line_t *meta_duration = malloc(sizeof(meta_line_t));
@@ -259,7 +267,7 @@ void parse_media(AVFormatContext *pFormatCtx, document_t *doc) {
}
}
AVFrame *frame = read_frame(pFormatCtx, decoder, video_stream);
AVFrame *frame = read_frame(pFormatCtx, decoder, video_stream, doc);
if (frame == NULL) {
avcodec_free_context(&decoder);
avformat_close_input(&pFormatCtx);
@@ -308,12 +316,12 @@ void parse_media_filename(const char *filepath, document_t *doc) {
AVFormatContext *pFormatCtx = avformat_alloc_context();
if (pFormatCtx == NULL) {
fprintf(stderr, "Could not allocate AVFormatContext! %s \n", filepath);
LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()")
return;
}
int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL);
if (res < 0) {
fprintf(stderr, "media error: %s %s\n", filepath, av_err2str(res));
LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res))
return;
}
@@ -336,7 +344,7 @@ void parse_media_vfile(struct vfile *f, document_t *doc) {
AVFormatContext *pFormatCtx = avformat_alloc_context();
if (pFormatCtx == NULL) {
fprintf(stderr, "Could not allocate AVFormatContext! %s \n", f->filepath);
LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()")
return;
}
@@ -350,8 +358,8 @@ void parse_media_vfile(struct vfile *f, document_t *doc) {
if (res == -5) {
// Tried to parse media that requires seek
return;
} else if(res < 0) {
fprintf(stderr, "media error: %s %s\n", f->filepath, av_err2str(res));
} else if (res < 0) {
LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res))
return;
}

14
src/parsing/parse.c
View File

@@ -9,8 +9,7 @@ int fs_read(struct vfile *f, void *buf, size_t size) {
if (f->fd == -1) {
f->fd = open(f->filepath, O_RDONLY);
if (f->fd == -1) {
perror("open");
printf("%s\n", f->filepath);
LOG_ERRORF(f->filepath, "open(): [%d] %s", errno, strerror(errno))
return -1;
}
}
@@ -39,7 +38,7 @@ void *read_all(parse_job_t *job, const char *buf, int bytes_read) {
int ret = job->vfile.read(&job->vfile, full_buf + bytes_read, job->info.st_size - bytes_read);
if (ret == -1) {
perror("read");
LOG_ERRORF(job->filepath, "read(): [%d] %s", errno, strerror(errno))
return NULL;
}
}
@@ -75,6 +74,12 @@ void parse(void *arg) {
uuid_generate(doc.uuid);
char *buf[PARSE_BUF_SIZE];
if (LogCtx.very_verbose) {
char uuid_str[UUID_STR_LEN];
uuid_unparse(doc.uuid, uuid_str);
LOG_DEBUGF(job->filepath, "Starting parse job {%s}", uuid_str)
}
if (job->info.st_size == 0) {
doc.mime = MIME_EMPTY;
} else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) {
@@ -87,6 +92,7 @@ void parse(void *arg) {
// Get mime type with libmagic
bytes_read = job->vfile.read(&job->vfile, buf, PARSE_BUF_SIZE);
if (bytes_read == -1) {
LOG_WARNINGF(job->filepath, "read() Error: %s", strerror(errno))
CLOSE_FILE(job->vfile)
return;
}
@@ -95,7 +101,7 @@ void parse(void *arg) {
if (magic_mime_str != NULL) {
doc.mime = mime_get_mime_by_string(ScanCtx.mime_table, magic_mime_str);
if (doc.mime == 0) {
fprintf(stderr, "Couldn't find mime %s, %s\n", magic_mime_str, job->filepath + job->base);
LOG_WARNINGF(job->filepath, "Couldn't find mime %s", magic_mime_str);
}
}
}

26
src/parsing/pdf.c
View File

@@ -14,6 +14,7 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
if (err != 0) {
fz_drop_page(ctx, cover);
LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
return NULL;
}
@@ -52,6 +53,7 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
err = ctx->error.errcode;
if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
fz_drop_page(ctx, cover);
fz_drop_pixmap(ctx, pixmap);
return NULL;
@@ -76,6 +78,7 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
fz_drop_pixmap(ctx, pixmap);
if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_new_buffer_from_pixmap_as_png() returned error code [%d] %s", err, ctx->error.message)
fz_drop_page(ctx, cover);
return NULL;
}
@@ -83,14 +86,23 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
return cover;
}
void fz_noop_callback(__attribute__((unused)) void *user, __attribute__((unused)) const char *message) {}
void fz_err_callback(void *user, __attribute__((unused)) const char *message) {
if (LogCtx.verbose) {
document_t *doc = (document_t*) user;
LOG_WARNINGF(doc->filepath, "FZ: %s", message)
}
}
void init_ctx(fz_context *ctx) {
__always_inline
void init_ctx(fz_context *ctx, document_t *doc) {
fz_disable_icc(ctx);
fz_register_document_handlers(ctx);
ctx->warn.print = fz_noop_callback;
ctx->error.print = fz_noop_callback;
ctx->warn.print_user = doc;
ctx->warn.print = fz_err_callback;
ctx->error.print_user = doc;
ctx->error.print = fz_err_callback;
}
int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
@@ -125,7 +137,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
}
fz_context *ctx = fz_new_context(NULL, NULL, FZ_STORE_UNLIMITED);
init_ctx(ctx);
init_ctx(ctx, doc);
int err = 0;
@@ -171,6 +183,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
err = ctx->error.errcode;
if (err) {
LOG_WARNINGF(doc->filepath, "fz_count_pages() returned error code [%d] %s", err, ctx->error.message)
fz_drop_stream(ctx, stream);
fz_drop_document(ctx, fzdoc);
fz_drop_context(ctx);
@@ -210,6 +223,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
fz_catch(ctx)
err = ctx->error.errcode;
if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
text_buffer_destroy(&text_buf);
fz_drop_page(ctx, page);
fz_drop_stream(ctx, stream);
@@ -234,6 +248,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
err = ctx->error.errcode;
if (err != 0) {
LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
text_buffer_destroy(&text_buf);
fz_drop_page(ctx, page);
fz_drop_stext_page(ctx, stext);
@@ -272,4 +287,3 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
fz_drop_document(ctx, fzdoc);
fz_drop_context(ctx);
}