simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-12-14 15:59:03 +00:00
Code Issues Projects Releases Wiki Activity

bugfixes & refactoring

Browse Source
This commit is contained in:
simon
2019-10-26 12:35:01 -04:00
parent 564a17a8fa
commit c3b7a05dde
21 changed files with 440 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/parsing/media.c
View File

@@ -153,6 +153,10 @@ void parse_media(const char *filepath, document_t *doc) {
int video_stream = -1;
AVFormatContext *pFormatCtx = avformat_alloc_context();
if (pFormatCtx == NULL) {
fprintf(stderr, "Could not allocate AVFormatContext! %s \n", filepath);
return;
}
int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL);
if (res < 0) {
printf("ERR%s %s\n", filepath, av_err2str(res));

13
src/parsing/mime_generated.c
View File

@@ -360,6 +360,10 @@ enum mime {
application_x_ms_reader=655712,
application_CDFV2_corrupt=655713,
text_x_vcard=590178,
application_x_innosetup=655715,
application_winhelp=655716,
image_x_tga=524645,
application_x_wine_extension_ini=655718,
};
char *mime_get_mime_text(unsigned int mime_id) {switch (mime_id) {
case application_arj: return "application/arj";
@@ -716,6 +720,10 @@ case image_vnd_djvu: return "image/vnd.djvu";
case application_x_ms_reader: return "application/x-ms-reader";
case application_CDFV2_corrupt: return "application/CDFV2-corrupt";
case text_x_vcard: return "text/x-vcard";
case application_x_innosetup: return "application/x-innosetup";
case application_winhelp: return "application/winhelp";
case image_x_tga: return "image/x-tga";
case application_x_wine_extension_ini: return "application/x-wine-extension-ini";
default: return NULL;}}
GHashTable *mime_get_ext_table() {GHashTable *ext_table = g_hash_table_new(g_str_hash, g_str_equal);
g_hash_table_insert(ext_table, "arj", (gpointer)application_arj);
@@ -1192,6 +1200,7 @@ g_hash_table_insert(ext_table, "m4b", (gpointer)audio_mp4);
g_hash_table_insert(ext_table, "djvu", (gpointer)image_vnd_djvu);
g_hash_table_insert(ext_table, "lit", (gpointer)application_x_ms_reader);
g_hash_table_insert(ext_table, "vcf", (gpointer)text_x_vcard);
g_hash_table_insert(ext_table, "hlp", (gpointer)application_winhelp);
return ext_table;}
GHashTable *mime_get_mime_table() {GHashTable *mime_table = g_hash_table_new(g_str_hash, g_str_equal);
g_hash_table_insert(mime_table, "application/arj", (gpointer)application_arj);
@@ -1548,5 +1557,9 @@ g_hash_table_insert(mime_table, "image/vnd.djvu", (gpointer)image_vnd_djvu);
g_hash_table_insert(mime_table, "application/x-ms-reader", (gpointer)application_x_ms_reader);
g_hash_table_insert(mime_table, "application/CDFV2-corrupt", (gpointer)application_CDFV2_corrupt);
g_hash_table_insert(mime_table, "text/x-vcard", (gpointer)text_x_vcard);
g_hash_table_insert(mime_table, "application/x-innosetup", (gpointer)application_x_innosetup);
g_hash_table_insert(mime_table, "application/winhelp", (gpointer)application_winhelp);
g_hash_table_insert(mime_table, "image/x-tga", (gpointer)image_x_tga);
g_hash_table_insert(mime_table, "application/x-wine-extension-ini", (gpointer)application_x_wine_extension_ini);
return mime_table;}
#endif

16
src/parsing/parse.c
View File

@@ -94,7 +94,7 @@ void parse(void *arg) {
if (!(SHOULD_PARSE(doc.mime))) {
} else if ((mmime == MimeVideo && doc.size >= MIN_VIDEO_SIZE) || mmime == MimeAudio || mmime == MimeImage) {
parse_media(job->filepath, &doc);
// parse_media(job->filepath, &doc);
} else if (IS_PDF(doc.mime)) {
void *pdf_buf = read_all(job, (char *) buf, bytes_read, &fd);
@@ -105,15 +105,15 @@ void parse(void *arg) {
}
} else if (mmime == MimeText && ScanCtx.content_size > 0) {
parse_text(bytes_read, &fd, (char *) buf, &doc);
// parse_text(bytes_read, &fd, (char *) buf, &doc);
} else if (IS_FONT(doc.mime)) {
void *font_buf = read_all(job, (char *) buf, bytes_read, &fd);
parse_font(font_buf, doc.size, &doc);
if (font_buf != buf) {
free(font_buf);
}
// void *font_buf = read_all(job, (char *) buf, bytes_read, &fd);
// parse_font(font_buf, doc.size, &doc);
//
// if (font_buf != buf) {
// free(font_buf);
// }
}
write_document(&doc);

78
src/parsing/pdf.c
View File

@@ -1,13 +1,13 @@
#include "pdf.h"
#include "src/ctx.h"
__always_inline
fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
fz_page *cover = fz_load_page(ctx, fzdoc, 0);
fz_rect bounds = fz_bound_page(ctx, cover);
float scale;
unsigned char *tn_buf;
float w = (float) bounds.x1 - bounds.x0;
float h = (float) bounds.y1 - bounds.y0;
if (w > h) {
@@ -17,11 +17,21 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
}
fz_matrix m = fz_scale(scale, scale);
fz_pixmap *pixmap;
fz_colorspace *color_space = fz_device_rgb(ctx);
pixmap = fz_new_pixmap_from_page(ctx, cover, m, color_space, 0);
bounds = fz_transform_rect(bounds, m);
fz_irect bbox = fz_round_rect(bounds);
fz_pixmap *pixmap = fz_new_pixmap_with_bbox(ctx, ctx->colorspace->rgb, bbox, NULL, 0);
fz_clear_pixmap_with_value(ctx, pixmap, 0xFF);
fz_device *dev = fz_new_draw_device(ctx, m, pixmap);
pthread_mutex_lock(&ScanCtx.mupdf_mu);
fz_run_page(ctx, cover, dev, fz_identity, NULL);
pthread_mutex_unlock(&ScanCtx.mupdf_mu);
fz_drop_device(ctx, dev);
fz_buffer *fzbuf = fz_new_buffer_from_pixmap_as_png(ctx, pixmap, fz_default_color_params);
unsigned char *tn_buf;
size_t tn_len = fz_buffer_storage(ctx, fzbuf, &tn_buf);
store_write(ScanCtx.index.store, (char *) doc->uuid, sizeof(doc->uuid), (char *) tn_buf, tn_len);
@@ -32,29 +42,38 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
return cover;
}
void fz_noop_callback(void *user, const char *message) {
}
void fz_noop_callback(void *user, const char *message) {}
void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
//TODO error handling
static int mu_is_initialized = 0;
if (!mu_is_initialized) {
pthread_mutex_init(&ScanCtx.mupdf_mu, NULL);
mu_is_initialized = 1;
}
fz_context *ctx = fz_new_context(NULL, NULL, FZ_STORE_UNLIMITED);
fz_stream *stream = NULL;
fz_document *fzdoc = NULL;
fz_var(stream);
fz_var(fzdoc);
fz_try(ctx)
{
fz_disable_icc(ctx);
fz_register_document_handlers(ctx);
ctx->warn.print = fz_noop_callback; //disable warnings
//disable warnings
ctx->warn.print = fz_noop_callback;
ctx->error.print = fz_noop_callback;
fz_stream *stream = fz_open_memory(ctx, buf, buf_len);
fz_document *fzdoc = fz_open_document_with_stream(ctx, mime_get_mime_text(doc->mime), stream);
stream = fz_open_memory(ctx, buf, buf_len);
fzdoc = fz_open_document_with_stream(ctx, mime_get_mime_text(doc->mime), stream);
int page_count = fz_count_pages(ctx, fzdoc);
fz_page *cover = render_cover(ctx, doc, fzdoc);
fz_stext_options opts;
fz_parse_stext_options(ctx, &opts, "preserve-ligatures");
text_buffer_t text_buf = text_buffer_create(ScanCtx.content_size);
@@ -65,12 +84,34 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
} else {
page = fz_load_page(ctx, fzdoc, current_page);
}
fz_stext_page *stext = fz_new_stext_page_from_page(ctx, page, &opts);
fz_stext_page *stext;
fz_device *dev = NULL;
fz_var(dev);
stext = fz_new_stext_page(ctx, fz_bound_page(ctx, page));
fz_try(ctx)
{
dev = fz_new_stext_device(ctx, stext, &opts);
pthread_mutex_lock(&ScanCtx.mupdf_mu);
fz_run_page_contents(ctx, page, dev, fz_identity, NULL);
pthread_mutex_unlock(&ScanCtx.mupdf_mu);
fz_close_device(ctx, dev);
}
fz_always(ctx)
fz_drop_device(ctx, dev);
fz_catch(ctx)
{
fz_drop_stext_page(ctx, stext);
fz_rethrow(ctx);
}
fz_stext_block *block = stext->first_block;
while (block != NULL) {
if (block->type != FZ_STEXT_BLOCK_TEXT) {
block = block->next;
continue;
}
@@ -100,15 +141,12 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
memcpy(meta_content->strval, text_buf.dyn_buffer.buf, text_buf.dyn_buffer.cur);
text_buffer_destroy(&text_buf);
APPEND_META(doc, meta_content)
}
fz_always(ctx)
{
fz_drop_stream(ctx, stream);
fz_drop_document(ctx, fzdoc);
fz_drop_context(ctx);
}
fz_catch(ctx)
{
// printf("err");
}
} fz_catch(ctx) {}
}