mirror of
https://github.com/simon987/sist2.git
synced 2025-04-18 01:36:42 +00:00
Better logging, fixes #15
This commit is contained in:
parent
80fbcb2a01
commit
663f8e21c1
@ -27,6 +27,9 @@ if (WITH_SIST2)
|
|||||||
src/index/elastic.c src/index/elastic.h
|
src/index/elastic.c src/index/elastic.h
|
||||||
src/util.c src/util.h
|
src/util.c src/util.h
|
||||||
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
|
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
|
||||||
|
src/parsing/arc.c src/parsing/arc.h
|
||||||
|
src/parsing/doc.c src/parsing/doc.h
|
||||||
|
src/log.c src/log.h
|
||||||
|
|
||||||
# argparse
|
# argparse
|
||||||
argparse/argparse.h argparse/argparse.c
|
argparse/argparse.h argparse/argparse.c
|
||||||
@ -41,7 +44,7 @@ if (WITH_SIST2)
|
|||||||
|
|
||||||
# utf8.h
|
# utf8.h
|
||||||
utf8.h/utf8.h
|
utf8.h/utf8.h
|
||||||
src/parsing/arc.c src/parsing/arc.h src/parsing/doc.c src/parsing/doc.h)
|
)
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
if (WITH_SIST2_SCAN)
|
if (WITH_SIST2_SCAN)
|
||||||
@ -60,6 +63,9 @@ if (WITH_SIST2_SCAN)
|
|||||||
src/parsing/text.h src/parsing/text.c
|
src/parsing/text.h src/parsing/text.c
|
||||||
src/util.c src/util.h
|
src/util.c src/util.h
|
||||||
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
|
src/ctx.h src/types.h src/parsing/font.c src/parsing/font.h
|
||||||
|
src/parsing/arc.h src/parsing/arc.c
|
||||||
|
src/parsing/doc.h src/parsing/doc.c
|
||||||
|
src/log.h src/log.c
|
||||||
|
|
||||||
# argparse
|
# argparse
|
||||||
argparse/argparse.h argparse/argparse.c
|
argparse/argparse.h argparse/argparse.c
|
||||||
@ -74,7 +80,7 @@ if (WITH_SIST2_SCAN)
|
|||||||
|
|
||||||
# utf8.h
|
# utf8.h
|
||||||
utf8.h/utf8.h
|
utf8.h/utf8.h
|
||||||
src/parsing/arc.c src/parsing/arc.h src/parsing/doc.c src/parsing/doc.h)
|
)
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
find_package(PkgConfig REQUIRED)
|
find_package(PkgConfig REQUIRED)
|
||||||
|
35
src/cli.c
35
src/cli.c
@ -1,4 +1,5 @@
|
|||||||
#include "cli.h"
|
#include "cli.h"
|
||||||
|
#include "ctx.h"
|
||||||
|
|
||||||
#define DEFAULT_OUTPUT "index.sist2/"
|
#define DEFAULT_OUTPUT "index.sist2/"
|
||||||
#define DEFAULT_CONTENT_SIZE 32768
|
#define DEFAULT_CONTENT_SIZE 32768
|
||||||
@ -134,6 +135,19 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
|
|||||||
fprintf(stderr, "Archive mode must be one of (skip, list, shallow, recurse), got '%s'", args->archive);
|
fprintf(stderr, "Archive mode must be one of (skip, list, shallow, recurse), got '%s'", args->archive);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
|
||||||
|
LOG_DEBUGF("cli.c", "arg size=%d", args->size)
|
||||||
|
LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
|
||||||
|
LOG_DEBUGF("cli.c", "arg threads=%d", args->threads)
|
||||||
|
LOG_DEBUGF("cli.c", "arg incremental=%s", args->incremental)
|
||||||
|
LOG_DEBUGF("cli.c", "arg output=%s", args->incremental)
|
||||||
|
LOG_DEBUGF("cli.c", "arg rewrite_url=%s", args->rewrite_url)
|
||||||
|
LOG_DEBUGF("cli.c", "arg name=%s", args->name)
|
||||||
|
LOG_DEBUGF("cli.c", "arg depth=%d", args->depth)
|
||||||
|
LOG_DEBUGF("cli.c", "arg path=%s", args->path)
|
||||||
|
LOG_DEBUGF("cli.c", "arg archive=%s", args->archive)
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -189,6 +203,14 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
|
|||||||
args->batch_size = DEFAULT_BATCH_SIZE;
|
args->batch_size = DEFAULT_BATCH_SIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
|
||||||
|
LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path)
|
||||||
|
LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path)
|
||||||
|
LOG_DEBUGF("cli.c", "arg script=%s", args->script)
|
||||||
|
LOG_DEBUGF("cli.c", "arg print=%d", args->print)
|
||||||
|
LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size)
|
||||||
|
LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset)
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -223,10 +245,21 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) {
|
|||||||
for (int i = 0; i < args->index_count; i++) {
|
for (int i = 0; i < args->index_count; i++) {
|
||||||
char *abs_path = abspath(args->indices[i]);
|
char *abs_path = abspath(args->indices[i]);
|
||||||
if (abs_path == NULL) {
|
if (abs_path == NULL) {
|
||||||
fprintf(stderr, "File not found: %s\n", abs_path);
|
fprintf(stderr, "File not found: %s\n", args->indices[i]);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url)
|
||||||
|
LOG_DEBUGF("cli.c", "arg bind=%s", args->bind)
|
||||||
|
LOG_DEBUGF("cli.c", "arg port=%s", args->port)
|
||||||
|
LOG_DEBUGF("cli.c", "arg credentials=%s", args->credentials)
|
||||||
|
LOG_DEBUGF("cli.c", "arg b64credentials=%s", args->b64credentials)
|
||||||
|
LOG_DEBUGF("cli.c", "arg index_count=%d", args->index_count)
|
||||||
|
for (int i = 0; i < args->index_count; i++) {
|
||||||
|
LOG_DEBUGF("cli.c", "arg indices[%d]=%s", i, args->indices[i])
|
||||||
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -17,6 +17,8 @@ struct {
|
|||||||
float tn_qscale;
|
float tn_qscale;
|
||||||
int depth;
|
int depth;
|
||||||
archive_mode_t archive_mode;
|
archive_mode_t archive_mode;
|
||||||
|
int verbose;
|
||||||
|
int very_verbose;
|
||||||
|
|
||||||
size_t stat_tn_size;
|
size_t stat_tn_size;
|
||||||
size_t stat_index_size;
|
size_t stat_index_size;
|
||||||
@ -27,6 +29,11 @@ struct {
|
|||||||
pthread_mutex_t mupdf_mu;
|
pthread_mutex_t mupdf_mu;
|
||||||
} ScanCtx;
|
} ScanCtx;
|
||||||
|
|
||||||
|
struct {
|
||||||
|
int verbose;
|
||||||
|
int very_verbose;
|
||||||
|
int no_color;
|
||||||
|
} LogCtx;
|
||||||
|
|
||||||
#ifndef SIST_SCAN_ONLY
|
#ifndef SIST_SCAN_ONLY
|
||||||
struct {
|
struct {
|
||||||
|
@ -262,8 +262,7 @@ void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
fprintf(stderr, "Invalid meta key (corrupt index): %x\n", key);
|
LOG_FATALF("serialize.c", "Invalid meta key (corrupt index): %x", key)
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
key = getc(file);
|
key = getc(file);
|
||||||
|
@ -9,9 +9,9 @@ store_t *store_create(char *path) {
|
|||||||
mdb_env_create(&store->env);
|
mdb_env_create(&store->env);
|
||||||
|
|
||||||
int open_ret = mdb_env_open(store->env,
|
int open_ret = mdb_env_open(store->env,
|
||||||
path,
|
path,
|
||||||
MDB_WRITEMAP | MDB_MAPASYNC,
|
MDB_WRITEMAP | MDB_MAPASYNC,
|
||||||
S_IRUSR | S_IWUSR
|
S_IRUSR | S_IWUSR
|
||||||
);
|
);
|
||||||
|
|
||||||
if (open_ret != 0) {
|
if (open_ret != 0) {
|
||||||
@ -42,6 +42,12 @@ void store_destroy(store_t *store) {
|
|||||||
|
|
||||||
void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len) {
|
void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t buf_len) {
|
||||||
|
|
||||||
|
if (LogCtx.very_verbose) {
|
||||||
|
char uuid_str[UUID_STR_LEN];
|
||||||
|
uuid_unparse((unsigned char *) key, uuid_str);
|
||||||
|
LOG_DEBUGF("store.c", "Store write {%s} %lu bytes", uuid_str, buf_len)
|
||||||
|
}
|
||||||
|
|
||||||
MDB_val mdb_key;
|
MDB_val mdb_key;
|
||||||
mdb_key.mv_data = key;
|
mdb_key.mv_data = key;
|
||||||
mdb_key.mv_size = key_len;
|
mdb_key.mv_size = key_len;
|
||||||
@ -68,6 +74,8 @@ void store_write(store_t *store, char *key, size_t key_len, char *buf, size_t bu
|
|||||||
mdb_env_set_mapsize(store->env, store->size);
|
mdb_env_set_mapsize(store->env, store->size);
|
||||||
mdb_txn_begin(store->env, NULL, 0, &txn);
|
mdb_txn_begin(store->env, NULL, 0, &txn);
|
||||||
put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
|
put_ret = mdb_put(txn, store->dbi, &mdb_key, &mdb_value, 0);
|
||||||
|
|
||||||
|
LOG_INFOF("store.c", "Updated mdb mapsize to %lu bytes", store->size)
|
||||||
}
|
}
|
||||||
|
|
||||||
mdb_txn_commit(txn);
|
mdb_txn_commit(txn);
|
||||||
|
99
src/log.c
Normal file
99
src/log.c
Normal file
@ -0,0 +1,99 @@
|
|||||||
|
#include "log.h"
|
||||||
|
|
||||||
|
const char *log_colors[] = {
|
||||||
|
"\033[34m", "\033[01;34m", "\033[0m",
|
||||||
|
"\033[01;33m", "\033[31m", "\033[01;31m"
|
||||||
|
};
|
||||||
|
|
||||||
|
const char *log_levels[] = {
|
||||||
|
"DEBUG", "INFO", "WARNING", "ERROR", "FATAL"
|
||||||
|
};
|
||||||
|
|
||||||
|
void sist_logf(char *filepath, int level, char *format, ...) {
|
||||||
|
|
||||||
|
static int is_tty = -1;
|
||||||
|
if (is_tty == -1) {
|
||||||
|
is_tty = isatty(STDERR_FILENO);
|
||||||
|
}
|
||||||
|
|
||||||
|
char log_str[LOG_MAX_LENGTH];
|
||||||
|
|
||||||
|
unsigned long long pid = (unsigned long long) pthread_self();
|
||||||
|
|
||||||
|
char datetime[32];
|
||||||
|
time_t t;
|
||||||
|
struct tm result;
|
||||||
|
t = time(NULL);
|
||||||
|
localtime_r(&t, &result);
|
||||||
|
strftime(datetime, sizeof(datetime), "%Y-%m-%d %H:%M:%S", &result);
|
||||||
|
|
||||||
|
int log_len;
|
||||||
|
if (is_tty) {
|
||||||
|
log_len = snprintf(
|
||||||
|
log_str, sizeof(log_str),
|
||||||
|
"\033[%dm[%04X]%s [%s] [%s %s] ",
|
||||||
|
31 + ((unsigned int) (pid)) % 7, pid, log_colors[level],
|
||||||
|
datetime, log_levels[level], filepath
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
log_len = snprintf(
|
||||||
|
log_str, sizeof(log_str),
|
||||||
|
"[%04X] [%s] [%s %s] ",
|
||||||
|
pid, datetime, log_levels[level], filepath
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
va_list ap;
|
||||||
|
va_start(ap, format);
|
||||||
|
size_t maxsize = sizeof(log_str) - log_len;
|
||||||
|
log_len += vsnprintf(log_str + log_len, maxsize, format, ap);
|
||||||
|
va_end(ap);
|
||||||
|
|
||||||
|
if (is_tty) {
|
||||||
|
log_len += sprintf(log_str + log_len, "\033[0m\n");
|
||||||
|
} else {
|
||||||
|
*(log_str + log_len) = '\n';
|
||||||
|
log_len += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
write(STDOUT_FILENO, log_str, log_len);
|
||||||
|
}
|
||||||
|
|
||||||
|
void sist_log(char *filepath, int level, char *str) {
|
||||||
|
|
||||||
|
static int is_tty = -1;
|
||||||
|
if (is_tty == -1) {
|
||||||
|
is_tty = isatty(STDERR_FILENO);
|
||||||
|
}
|
||||||
|
|
||||||
|
char log_str[LOG_MAX_LENGTH];
|
||||||
|
|
||||||
|
unsigned long long pid = (unsigned long long) pthread_self();
|
||||||
|
|
||||||
|
char datetime[32];
|
||||||
|
time_t t;
|
||||||
|
struct tm result;
|
||||||
|
t = time(NULL);
|
||||||
|
localtime_r(&t, &result);
|
||||||
|
strftime(datetime, sizeof(datetime), "%Y-%m-%d %H:%M:%S", &result);
|
||||||
|
|
||||||
|
int log_len;
|
||||||
|
if (is_tty) {
|
||||||
|
log_len = snprintf(
|
||||||
|
log_str, sizeof(log_str),
|
||||||
|
"\033[%dm[%04X]%s [%s] [%s %s] %s \033[0m\n",
|
||||||
|
31 + ((unsigned int) (pid)) % 7, pid, log_colors[level],
|
||||||
|
datetime, log_levels[level], filepath,
|
||||||
|
str
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
log_len = snprintf(
|
||||||
|
log_str, sizeof(log_str),
|
||||||
|
"[%04X] [%s] [%s %s] %s \n",
|
||||||
|
pid, datetime, log_levels[level], filepath,
|
||||||
|
str
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
write(STDERR_FILENO, log_str, log_len);
|
||||||
|
}
|
45
src/log.h
Normal file
45
src/log.h
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
#ifndef SIST2_LOG_H
|
||||||
|
#define SIST2_LOG_H
|
||||||
|
|
||||||
|
#define LOG_MAX_LENGTH 8192
|
||||||
|
|
||||||
|
#define SIST_DEBUG 0
|
||||||
|
#define SIST_INFO 1
|
||||||
|
#define SIST_WARNING 2
|
||||||
|
#define SIST_ERROR 3
|
||||||
|
#define SIST_FATAL 4
|
||||||
|
|
||||||
|
#define LOG_DEBUGF(filepath, fmt, ...) \
|
||||||
|
if (LogCtx.very_verbose) {sist_logf(filepath, SIST_DEBUG, fmt, __VA_ARGS__);}
|
||||||
|
#define LOG_DEBUG(filepath, str) \
|
||||||
|
if (LogCtx.very_verbose) {sist_log(filepath, SIST_DEBUG, str);}
|
||||||
|
|
||||||
|
#define LOG_INFOF(filepath, fmt, ...) \
|
||||||
|
if (LogCtx.verbose) {sist_logf(filepath, SIST_INFO, fmt, __VA_ARGS__);}
|
||||||
|
#define LOG_INFO(filepath, str) \
|
||||||
|
if (LogCtx.verbose) {sist_log(filepath, SIST_INFO, str);}
|
||||||
|
|
||||||
|
#define LOG_WARNINGF(filepath, fmt, ...) \
|
||||||
|
if (LogCtx.verbose) {sist_logf(filepath, SIST_WARNING, fmt, __VA_ARGS__);}
|
||||||
|
#define LOG_WARNING(filepath, str) \
|
||||||
|
if (LogCtx.verbose) {sist_log(filepath, SIST_WARNING, str);}
|
||||||
|
|
||||||
|
#define LOG_ERRORF(filepath, fmt, ...) \
|
||||||
|
if (LogCtx.verbose) {sist_logf(filepath, SIST_ERROR, fmt, __VA_ARGS__);}
|
||||||
|
#define LOG_ERROR(filepath, str) \
|
||||||
|
if (LogCtx.verbose) {sist_log(filepath, SIST_ERROR, str);}
|
||||||
|
|
||||||
|
#define LOG_FATALF(filepath, fmt, ...) \
|
||||||
|
sist_logf(filepath, SIST_FATAL, fmt, __VA_ARGS__);\
|
||||||
|
exit(-1);
|
||||||
|
#define LOG_FATAL(filepath, str) \
|
||||||
|
sist_log(filepath, SIST_FATAL, str);\
|
||||||
|
exit(-1);
|
||||||
|
|
||||||
|
#include "src/sist.h"
|
||||||
|
|
||||||
|
void sist_logf(char *filepath, int level, char *format, ...);
|
||||||
|
|
||||||
|
void sist_log(char *filepath, int level, char *str);
|
||||||
|
|
||||||
|
#endif
|
27
src/main.c
27
src/main.c
@ -10,7 +10,7 @@
|
|||||||
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
|
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
|
||||||
|
|
||||||
|
|
||||||
static const char *const Version = "1.1.11";
|
static const char *const Version = "1.1.12";
|
||||||
static const char *const usage[] = {
|
static const char *const usage[] = {
|
||||||
"sist2 scan [OPTION]... PATH",
|
"sist2 scan [OPTION]... PATH",
|
||||||
"sist2 index [OPTION]... INDEX",
|
"sist2 index [OPTION]... INDEX",
|
||||||
@ -40,24 +40,7 @@ void init_dir(const char *dirpath) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void scan_print_header() {
|
void scan_print_header() {
|
||||||
printf("sist2 V%s\n", Version);
|
LOG_INFOF("main.c", "sist2 v%s", Version)
|
||||||
printf("---------------------\n");
|
|
||||||
printf("threads\t\t\t%d\n", ScanCtx.threads);
|
|
||||||
printf("tn_qscale\t\t%.1f/31.0\n", ScanCtx.tn_qscale);
|
|
||||||
|
|
||||||
if (ScanCtx.tn_size > 0) {
|
|
||||||
printf("tn_size\t\t\t%dpx\n", ScanCtx.tn_size);
|
|
||||||
} else {
|
|
||||||
printf("tn_size\t\t\tdisabled\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ScanCtx.content_size > 0) {
|
|
||||||
printf("content_size\t\t%d B\n", ScanCtx.content_size);
|
|
||||||
} else {
|
|
||||||
printf("content_size\t\t\tdisabled\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
printf("output\t\t\t%s\n", ScanCtx.index.path);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void sist2_scan(scan_args_t *args) {
|
void sist2_scan(scan_args_t *args) {
|
||||||
@ -236,6 +219,8 @@ int main(int argc, const char *argv[]) {
|
|||||||
OPT_HELP(),
|
OPT_HELP(),
|
||||||
|
|
||||||
OPT_BOOLEAN('v', "version", &arg_version, "Show version and exit"),
|
OPT_BOOLEAN('v', "version", &arg_version, "Show version and exit"),
|
||||||
|
OPT_BOOLEAN(0, "verbose", &LogCtx.verbose, "Turn on logging"),
|
||||||
|
OPT_BOOLEAN(0, "very-verbose", &LogCtx.very_verbose, "Turn on debug messages"),
|
||||||
|
|
||||||
OPT_GROUP("Scan options"),
|
OPT_GROUP("Scan options"),
|
||||||
OPT_INTEGER('t', "threads", &scan_args->threads, "Number of threads. DEFAULT=1"),
|
OPT_INTEGER('t', "threads", &scan_args->threads, "Number of threads. DEFAULT=1"),
|
||||||
@ -285,6 +270,10 @@ int main(int argc, const char *argv[]) {
|
|||||||
exit(0);
|
exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (LogCtx.very_verbose != 0) {
|
||||||
|
LogCtx.verbose = 1;
|
||||||
|
}
|
||||||
|
|
||||||
#ifndef SIST_SCAN_ONLY
|
#ifndef SIST_SCAN_ONLY
|
||||||
web_args->es_url = common_es_url;
|
web_args->es_url = common_es_url;
|
||||||
index_args->es_url = common_es_url;
|
index_args->es_url = common_es_url;
|
||||||
|
@ -67,18 +67,24 @@ void parse_archive(vfile_t *f, document_t *doc) {
|
|||||||
struct archive *a;
|
struct archive *a;
|
||||||
struct archive_entry *entry;
|
struct archive_entry *entry;
|
||||||
|
|
||||||
a = archive_read_new();
|
|
||||||
|
|
||||||
archive_read_support_filter_all(a);
|
|
||||||
archive_read_support_format_all(a);
|
|
||||||
|
|
||||||
arc_data_f data;
|
arc_data_f data;
|
||||||
data.f = f;
|
data.f = f;
|
||||||
|
|
||||||
int ret = 0;
|
int ret = 0;
|
||||||
if (data.f->is_fs_file) {
|
if (data.f->is_fs_file) {
|
||||||
|
|
||||||
|
a = archive_read_new();
|
||||||
|
archive_read_support_filter_all(a);
|
||||||
|
archive_read_support_format_all(a);
|
||||||
|
|
||||||
ret = archive_read_open_filename(a, doc->filepath, ARC_BUF_SIZE);
|
ret = archive_read_open_filename(a, doc->filepath, ARC_BUF_SIZE);
|
||||||
} else if (ScanCtx.archive_mode == ARC_MODE_RECURSE) {
|
} else if (ScanCtx.archive_mode == ARC_MODE_RECURSE) {
|
||||||
|
|
||||||
|
a = archive_read_new();
|
||||||
|
archive_read_support_filter_all(a);
|
||||||
|
archive_read_support_format_all(a);
|
||||||
|
|
||||||
ret = archive_read_open(
|
ret = archive_read_open(
|
||||||
a, &data,
|
a, &data,
|
||||||
vfile_open_callback,
|
vfile_open_callback,
|
||||||
@ -86,12 +92,11 @@ void parse_archive(vfile_t *f, document_t *doc) {
|
|||||||
vfile_close_callback
|
vfile_close_callback
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
archive_read_free(a);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (ret != ARCHIVE_OK) {
|
if (ret != ARCHIVE_OK) {
|
||||||
fprintf(stderr, "OPEN[%d]:%s %s\n", ret, archive_error_string(a), doc->filepath);
|
LOG_ERRORF(doc->filepath, "(arc.c) [%d] %s", ret, archive_error_string(a))
|
||||||
archive_read_free(a);
|
archive_read_free(a);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
#include "doc.h"
|
#include "doc.h"
|
||||||
|
#include "src/ctx.h"
|
||||||
|
|
||||||
static void dumpText(mceTextReader_t *reader, dyn_buffer_t *buf) {
|
static void dump_text(mceTextReader_t *reader, dyn_buffer_t *buf) {
|
||||||
|
|
||||||
mce_skip_attributes(reader);
|
mce_skip_attributes(reader);
|
||||||
|
|
||||||
@ -17,7 +18,7 @@ static void dumpText(mceTextReader_t *reader, dyn_buffer_t *buf) {
|
|||||||
} mce_end_element(reader);
|
} mce_end_element(reader);
|
||||||
|
|
||||||
mce_start_element(reader, NULL, NULL) {
|
mce_start_element(reader, NULL, NULL) {
|
||||||
dumpText(reader, buf);
|
dump_text(reader, buf);
|
||||||
} mce_end_element(reader);
|
} mce_end_element(reader);
|
||||||
|
|
||||||
} mce_end_children(reader)
|
} mce_end_children(reader)
|
||||||
@ -50,19 +51,20 @@ int should_read_part(opcPart part) {
|
|||||||
return FALSE;
|
return FALSE;
|
||||||
}
|
}
|
||||||
|
|
||||||
void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part) {
|
__always_inline
|
||||||
|
void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part, document_t *doc) {
|
||||||
|
|
||||||
mceTextReader_t reader;
|
mceTextReader_t reader;
|
||||||
int ret = opcXmlReaderOpen(c, &reader, part, NULL, "UTF-8", 0);
|
int ret = opcXmlReaderOpen(c, &reader, part, NULL, "UTF-8", 0);
|
||||||
|
|
||||||
if (ret != OPC_ERROR_NONE) {
|
if (ret != OPC_ERROR_NONE) {
|
||||||
//todo verbose
|
LOG_ERRORF(doc->filepath, "(doc.c) opcXmlReaderOpen() returned error code %d", ret);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
mce_start_document(&reader) {
|
mce_start_document(&reader) {
|
||||||
mce_start_element(&reader, NULL, NULL) {
|
mce_start_element(&reader, NULL, NULL) {
|
||||||
dumpText(&reader, buf);
|
dump_text(&reader, buf);
|
||||||
} mce_end_element(&reader);
|
} mce_end_element(&reader);
|
||||||
}mce_end_document(&reader);
|
}mce_end_document(&reader);
|
||||||
|
|
||||||
@ -71,9 +73,13 @@ void read_part(opcContainer *c, dyn_buffer_t *buf, opcPart part) {
|
|||||||
|
|
||||||
void parse_doc(void *mem, size_t mem_len, document_t *doc) {
|
void parse_doc(void *mem, size_t mem_len, document_t *doc) {
|
||||||
|
|
||||||
|
if (mem == NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
opcContainer *c = opcContainerOpenMem(mem, mem_len, OPC_OPEN_READ_ONLY, NULL);
|
opcContainer *c = opcContainerOpenMem(mem, mem_len, OPC_OPEN_READ_ONLY, NULL);
|
||||||
if (c == NULL) {
|
if (c == NULL) {
|
||||||
//todo verbose
|
LOG_ERROR(doc->filepath, "(doc.c) Couldn't open document with opcContainerOpenMem()");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -82,7 +88,7 @@ void parse_doc(void *mem, size_t mem_len, document_t *doc) {
|
|||||||
opcPart part = opcPartGetFirst(c);
|
opcPart part = opcPartGetFirst(c);
|
||||||
do {
|
do {
|
||||||
if (should_read_part(part)) {
|
if (should_read_part(part)) {
|
||||||
read_part(c, &buf, part);
|
read_part(c, &buf, part, doc);
|
||||||
}
|
}
|
||||||
} while ((part = opcPartGetNext(c, part)));
|
} while ((part = opcPartGetNext(c, part)));
|
||||||
|
|
||||||
|
@ -147,6 +147,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
|
|||||||
FT_Face face;
|
FT_Face face;
|
||||||
FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, buf_len, 0, &face);
|
FT_Error err = FT_New_Memory_Face(ft_lib, (unsigned char *) buf, buf_len, 0, &face);
|
||||||
if (err != 0) {
|
if (err != 0) {
|
||||||
|
LOG_ERRORF(doc->filepath, "(font.c) FT_New_Memory_Face() returned error code [%d] %s", err, ft_error_string(err));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -176,6 +177,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
|
|||||||
|
|
||||||
err = FT_Set_Pixel_Sizes(face, 0, pixel);
|
err = FT_Set_Pixel_Sizes(face, 0, pixel);
|
||||||
if (err != 0) {
|
if (err != 0) {
|
||||||
|
LOG_WARNINGF(doc->filepath, "(font.c) FT_Set_Pixel_Sizes() returned error code [%d] %s", err, ft_error_string(err))
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -194,6 +196,7 @@ void parse_font(const char *buf, size_t buf_len, document_t *doc) {
|
|||||||
c = c >= 'a' && c <= 'z' ? c - 32 : c + 32;
|
c = c >= 'a' && c <= 'z' ? c - 32 : c + 32;
|
||||||
err = FT_Load_Char(face, c, FT_LOAD_NO_HINTING | FT_LOAD_RENDER);
|
err = FT_Load_Char(face, c, FT_LOAD_NO_HINTING | FT_LOAD_RENDER);
|
||||||
if (err != 0) {
|
if (err != 0) {
|
||||||
|
LOG_WARNINGF(doc->filepath, "(font.c) FT_Load_Char() returned error code [%d] %s", err, ft_error_string(err));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -77,7 +77,8 @@ AVFrame *scale_frame(const AVCodecContext *decoder, const AVFrame *frame, int si
|
|||||||
return scaled_frame;
|
return scaled_frame;
|
||||||
}
|
}
|
||||||
|
|
||||||
AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx) {
|
__always_inline
|
||||||
|
AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int stream_idx, document_t *doc) {
|
||||||
AVFrame *frame = av_frame_alloc();
|
AVFrame *frame = av_frame_alloc();
|
||||||
|
|
||||||
AVPacket avPacket;
|
AVPacket avPacket;
|
||||||
@ -90,9 +91,12 @@ AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int st
|
|||||||
int read_frame_ret = av_read_frame(pFormatCtx, &avPacket);
|
int read_frame_ret = av_read_frame(pFormatCtx, &avPacket);
|
||||||
|
|
||||||
if (read_frame_ret != 0) {
|
if (read_frame_ret != 0) {
|
||||||
// if (read_frame_ret != AVERROR_EOF) {
|
if (read_frame_ret != AVERROR_EOF) {
|
||||||
// fprintf(stderr, "Error reading frame: %d\n", read_frame_ret);
|
LOG_WARNINGF(doc->filepath,
|
||||||
// }
|
"(media.c) avcodec_read_frame() returned error code [%d] %s",
|
||||||
|
read_frame_ret, av_err2str(read_frame_ret)
|
||||||
|
)
|
||||||
|
}
|
||||||
av_frame_free(&frame);
|
av_frame_free(&frame);
|
||||||
av_packet_unref(&avPacket);
|
av_packet_unref(&avPacket);
|
||||||
return NULL;
|
return NULL;
|
||||||
@ -109,7 +113,10 @@ AVFrame *read_frame(AVFormatContext *pFormatCtx, AVCodecContext *decoder, int st
|
|||||||
// Feed it to decoder
|
// Feed it to decoder
|
||||||
int decode_ret = avcodec_send_packet(decoder, &avPacket);
|
int decode_ret = avcodec_send_packet(decoder, &avPacket);
|
||||||
if (decode_ret != 0) {
|
if (decode_ret != 0) {
|
||||||
printf("Error decoding frame: %s\n", av_err2str(decode_ret));
|
LOG_WARNINGF(doc->filepath,
|
||||||
|
"(media.c) avcodec_send_packet() returned error code [%d] %s",
|
||||||
|
decode_ret, av_err2str(decode_ret)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
av_packet_unref(&avPacket);
|
av_packet_unref(&avPacket);
|
||||||
receive_ret = avcodec_receive_frame(decoder, frame);
|
receive_ret = avcodec_receive_frame(decoder, frame);
|
||||||
@ -152,7 +159,8 @@ void append_audio_meta(AVFormatContext *pFormatCtx, document_t *doc) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
__always_inline
|
__always_inline
|
||||||
void append_video_meta(AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int include_audio_tags, int is_video) {
|
void
|
||||||
|
append_video_meta(AVFormatContext *pFormatCtx, AVFrame *frame, document_t *doc, int include_audio_tags, int is_video) {
|
||||||
|
|
||||||
if (is_video) {
|
if (is_video) {
|
||||||
meta_line_t *meta_duration = malloc(sizeof(meta_line_t));
|
meta_line_t *meta_duration = malloc(sizeof(meta_line_t));
|
||||||
@ -259,7 +267,7 @@ void parse_media(AVFormatContext *pFormatCtx, document_t *doc) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
AVFrame *frame = read_frame(pFormatCtx, decoder, video_stream);
|
AVFrame *frame = read_frame(pFormatCtx, decoder, video_stream, doc);
|
||||||
if (frame == NULL) {
|
if (frame == NULL) {
|
||||||
avcodec_free_context(&decoder);
|
avcodec_free_context(&decoder);
|
||||||
avformat_close_input(&pFormatCtx);
|
avformat_close_input(&pFormatCtx);
|
||||||
@ -308,12 +316,12 @@ void parse_media_filename(const char *filepath, document_t *doc) {
|
|||||||
|
|
||||||
AVFormatContext *pFormatCtx = avformat_alloc_context();
|
AVFormatContext *pFormatCtx = avformat_alloc_context();
|
||||||
if (pFormatCtx == NULL) {
|
if (pFormatCtx == NULL) {
|
||||||
fprintf(stderr, "Could not allocate AVFormatContext! %s \n", filepath);
|
LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()")
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL);
|
int res = avformat_open_input(&pFormatCtx, filepath, NULL, NULL);
|
||||||
if (res < 0) {
|
if (res < 0) {
|
||||||
fprintf(stderr, "media error: %s %s\n", filepath, av_err2str(res));
|
LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res))
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -336,7 +344,7 @@ void parse_media_vfile(struct vfile *f, document_t *doc) {
|
|||||||
|
|
||||||
AVFormatContext *pFormatCtx = avformat_alloc_context();
|
AVFormatContext *pFormatCtx = avformat_alloc_context();
|
||||||
if (pFormatCtx == NULL) {
|
if (pFormatCtx == NULL) {
|
||||||
fprintf(stderr, "Could not allocate AVFormatContext! %s \n", f->filepath);
|
LOG_ERROR(doc->filepath, "(media.c) Could not allocate context with avformat_alloc_context()")
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -350,8 +358,8 @@ void parse_media_vfile(struct vfile *f, document_t *doc) {
|
|||||||
if (res == -5) {
|
if (res == -5) {
|
||||||
// Tried to parse media that requires seek
|
// Tried to parse media that requires seek
|
||||||
return;
|
return;
|
||||||
} else if(res < 0) {
|
} else if (res < 0) {
|
||||||
fprintf(stderr, "media error: %s %s\n", f->filepath, av_err2str(res));
|
LOG_ERRORF(doc->filepath, "(media.c) avformat_open_input() returned [%d] %s", res, av_err2str(res))
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -9,8 +9,7 @@ int fs_read(struct vfile *f, void *buf, size_t size) {
|
|||||||
if (f->fd == -1) {
|
if (f->fd == -1) {
|
||||||
f->fd = open(f->filepath, O_RDONLY);
|
f->fd = open(f->filepath, O_RDONLY);
|
||||||
if (f->fd == -1) {
|
if (f->fd == -1) {
|
||||||
perror("open");
|
LOG_ERRORF(f->filepath, "open(): [%d] %s", errno, strerror(errno))
|
||||||
printf("%s\n", f->filepath);
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -39,7 +38,7 @@ void *read_all(parse_job_t *job, const char *buf, int bytes_read) {
|
|||||||
|
|
||||||
int ret = job->vfile.read(&job->vfile, full_buf + bytes_read, job->info.st_size - bytes_read);
|
int ret = job->vfile.read(&job->vfile, full_buf + bytes_read, job->info.st_size - bytes_read);
|
||||||
if (ret == -1) {
|
if (ret == -1) {
|
||||||
perror("read");
|
LOG_ERRORF(job->filepath, "read(): [%d] %s", errno, strerror(errno))
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -75,6 +74,12 @@ void parse(void *arg) {
|
|||||||
uuid_generate(doc.uuid);
|
uuid_generate(doc.uuid);
|
||||||
char *buf[PARSE_BUF_SIZE];
|
char *buf[PARSE_BUF_SIZE];
|
||||||
|
|
||||||
|
if (LogCtx.very_verbose) {
|
||||||
|
char uuid_str[UUID_STR_LEN];
|
||||||
|
uuid_unparse(doc.uuid, uuid_str);
|
||||||
|
LOG_DEBUGF(job->filepath, "Starting parse job {%s}", uuid_str)
|
||||||
|
}
|
||||||
|
|
||||||
if (job->info.st_size == 0) {
|
if (job->info.st_size == 0) {
|
||||||
doc.mime = MIME_EMPTY;
|
doc.mime = MIME_EMPTY;
|
||||||
} else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) {
|
} else if (*(job->filepath + job->ext) != '\0' && (job->ext - job->base != 1)) {
|
||||||
@ -87,6 +92,7 @@ void parse(void *arg) {
|
|||||||
// Get mime type with libmagic
|
// Get mime type with libmagic
|
||||||
bytes_read = job->vfile.read(&job->vfile, buf, PARSE_BUF_SIZE);
|
bytes_read = job->vfile.read(&job->vfile, buf, PARSE_BUF_SIZE);
|
||||||
if (bytes_read == -1) {
|
if (bytes_read == -1) {
|
||||||
|
LOG_WARNINGF(job->filepath, "read() Error: %s", strerror(errno))
|
||||||
CLOSE_FILE(job->vfile)
|
CLOSE_FILE(job->vfile)
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -95,7 +101,7 @@ void parse(void *arg) {
|
|||||||
if (magic_mime_str != NULL) {
|
if (magic_mime_str != NULL) {
|
||||||
doc.mime = mime_get_mime_by_string(ScanCtx.mime_table, magic_mime_str);
|
doc.mime = mime_get_mime_by_string(ScanCtx.mime_table, magic_mime_str);
|
||||||
if (doc.mime == 0) {
|
if (doc.mime == 0) {
|
||||||
fprintf(stderr, "Couldn't find mime %s, %s\n", magic_mime_str, job->filepath + job->base);
|
LOG_WARNINGF(job->filepath, "Couldn't find mime %s", magic_mime_str);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -14,6 +14,7 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
|
|||||||
|
|
||||||
if (err != 0) {
|
if (err != 0) {
|
||||||
fz_drop_page(ctx, cover);
|
fz_drop_page(ctx, cover);
|
||||||
|
LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -52,6 +53,7 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
|
|||||||
err = ctx->error.errcode;
|
err = ctx->error.errcode;
|
||||||
|
|
||||||
if (err != 0) {
|
if (err != 0) {
|
||||||
|
LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
|
||||||
fz_drop_page(ctx, cover);
|
fz_drop_page(ctx, cover);
|
||||||
fz_drop_pixmap(ctx, pixmap);
|
fz_drop_pixmap(ctx, pixmap);
|
||||||
return NULL;
|
return NULL;
|
||||||
@ -76,6 +78,7 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
|
|||||||
fz_drop_pixmap(ctx, pixmap);
|
fz_drop_pixmap(ctx, pixmap);
|
||||||
|
|
||||||
if (err != 0) {
|
if (err != 0) {
|
||||||
|
LOG_WARNINGF(doc->filepath, "fz_new_buffer_from_pixmap_as_png() returned error code [%d] %s", err, ctx->error.message)
|
||||||
fz_drop_page(ctx, cover);
|
fz_drop_page(ctx, cover);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@ -83,14 +86,23 @@ fz_page *render_cover(fz_context *ctx, document_t *doc, fz_document *fzdoc) {
|
|||||||
return cover;
|
return cover;
|
||||||
}
|
}
|
||||||
|
|
||||||
void fz_noop_callback(__attribute__((unused)) void *user, __attribute__((unused)) const char *message) {}
|
void fz_err_callback(void *user, __attribute__((unused)) const char *message) {
|
||||||
|
if (LogCtx.verbose) {
|
||||||
|
document_t *doc = (document_t*) user;
|
||||||
|
LOG_WARNINGF(doc->filepath, "FZ: %s", message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void init_ctx(fz_context *ctx) {
|
__always_inline
|
||||||
|
void init_ctx(fz_context *ctx, document_t *doc) {
|
||||||
fz_disable_icc(ctx);
|
fz_disable_icc(ctx);
|
||||||
fz_register_document_handlers(ctx);
|
fz_register_document_handlers(ctx);
|
||||||
ctx->warn.print = fz_noop_callback;
|
|
||||||
ctx->error.print = fz_noop_callback;
|
ctx->warn.print_user = doc;
|
||||||
|
ctx->warn.print = fz_err_callback;
|
||||||
|
ctx->error.print_user = doc;
|
||||||
|
ctx->error.print = fz_err_callback;
|
||||||
}
|
}
|
||||||
|
|
||||||
int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
|
int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
|
||||||
@ -125,7 +137,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
|||||||
}
|
}
|
||||||
fz_context *ctx = fz_new_context(NULL, NULL, FZ_STORE_UNLIMITED);
|
fz_context *ctx = fz_new_context(NULL, NULL, FZ_STORE_UNLIMITED);
|
||||||
|
|
||||||
init_ctx(ctx);
|
init_ctx(ctx, doc);
|
||||||
|
|
||||||
int err = 0;
|
int err = 0;
|
||||||
|
|
||||||
@ -171,6 +183,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
|||||||
err = ctx->error.errcode;
|
err = ctx->error.errcode;
|
||||||
|
|
||||||
if (err) {
|
if (err) {
|
||||||
|
LOG_WARNINGF(doc->filepath, "fz_count_pages() returned error code [%d] %s", err, ctx->error.message)
|
||||||
fz_drop_stream(ctx, stream);
|
fz_drop_stream(ctx, stream);
|
||||||
fz_drop_document(ctx, fzdoc);
|
fz_drop_document(ctx, fzdoc);
|
||||||
fz_drop_context(ctx);
|
fz_drop_context(ctx);
|
||||||
@ -210,6 +223,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
|||||||
fz_catch(ctx)
|
fz_catch(ctx)
|
||||||
err = ctx->error.errcode;
|
err = ctx->error.errcode;
|
||||||
if (err != 0) {
|
if (err != 0) {
|
||||||
|
LOG_WARNINGF(doc->filepath, "fz_load_page() returned error code [%d] %s", err, ctx->error.message)
|
||||||
text_buffer_destroy(&text_buf);
|
text_buffer_destroy(&text_buf);
|
||||||
fz_drop_page(ctx, page);
|
fz_drop_page(ctx, page);
|
||||||
fz_drop_stream(ctx, stream);
|
fz_drop_stream(ctx, stream);
|
||||||
@ -234,6 +248,7 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
|||||||
err = ctx->error.errcode;
|
err = ctx->error.errcode;
|
||||||
|
|
||||||
if (err != 0) {
|
if (err != 0) {
|
||||||
|
LOG_WARNINGF(doc->filepath, "fz_run_page() returned error code [%d] %s", err, ctx->error.message)
|
||||||
text_buffer_destroy(&text_buf);
|
text_buffer_destroy(&text_buf);
|
||||||
fz_drop_page(ctx, page);
|
fz_drop_page(ctx, page);
|
||||||
fz_drop_stext_page(ctx, stext);
|
fz_drop_stext_page(ctx, stext);
|
||||||
@ -272,4 +287,3 @@ void parse_pdf(void *buf, size_t buf_len, document_t *doc) {
|
|||||||
fz_drop_document(ctx, fzdoc);
|
fz_drop_document(ctx, fzdoc);
|
||||||
fz_drop_context(ctx);
|
fz_drop_context(ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -59,6 +59,7 @@
|
|||||||
#include "parsing/arc.h"
|
#include "parsing/arc.h"
|
||||||
#include "parsing/doc.h"
|
#include "parsing/doc.h"
|
||||||
#include "cli.h"
|
#include "cli.h"
|
||||||
|
#include "log.h"
|
||||||
#include "utf8.h/utf8.h"
|
#include "utf8.h/utf8.h"
|
||||||
|
|
||||||
#ifndef SIST_SCAN_ONLY
|
#ifndef SIST_SCAN_ONLY
|
||||||
|
10
src/tpool.c
10
src/tpool.c
@ -136,6 +136,7 @@ static void *tpool_worker(void *arg) {
|
|||||||
pthread_mutex_unlock(&(pool->work_mutex));
|
pthread_mutex_unlock(&(pool->work_mutex));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG_INFO("tpool.c", "Executing cleaup function")
|
||||||
pool->cleanup_func();
|
pool->cleanup_func();
|
||||||
|
|
||||||
pthread_cond_signal(&(pool->working_cond));
|
pthread_cond_signal(&(pool->working_cond));
|
||||||
@ -144,6 +145,7 @@ static void *tpool_worker(void *arg) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void tpool_wait(tpool_t *pool) {
|
void tpool_wait(tpool_t *pool) {
|
||||||
|
LOG_INFO("tpool.c", "Waiting for worker threads to finish")
|
||||||
pthread_mutex_lock(&(pool->work_mutex));
|
pthread_mutex_lock(&(pool->work_mutex));
|
||||||
while (1) {
|
while (1) {
|
||||||
if (pool->done_cnt < pool->work_cnt) {
|
if (pool->done_cnt < pool->work_cnt) {
|
||||||
@ -159,6 +161,8 @@ void tpool_wait(tpool_t *pool) {
|
|||||||
}
|
}
|
||||||
progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
|
progress_bar_print(1.0, ScanCtx.stat_tn_size, ScanCtx.stat_index_size);
|
||||||
pthread_mutex_unlock(&(pool->work_mutex));
|
pthread_mutex_unlock(&(pool->work_mutex));
|
||||||
|
|
||||||
|
LOG_INFO("tpool.c", "Worker threads finished")
|
||||||
}
|
}
|
||||||
|
|
||||||
void tpool_destroy(tpool_t *pool) {
|
void tpool_destroy(tpool_t *pool) {
|
||||||
@ -166,6 +170,8 @@ void tpool_destroy(tpool_t *pool) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG_INFO("tpool.c", "Destroying thread pool")
|
||||||
|
|
||||||
pthread_mutex_lock(&(pool->work_mutex));
|
pthread_mutex_lock(&(pool->work_mutex));
|
||||||
tpool_work_t *work = pool->work_head;
|
tpool_work_t *work = pool->work_head;
|
||||||
while (work != NULL) {
|
while (work != NULL) {
|
||||||
@ -185,6 +191,8 @@ void tpool_destroy(tpool_t *pool) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG_INFO("tpool.c", "Final cleanup")
|
||||||
|
|
||||||
pthread_mutex_destroy(&(pool->work_mutex));
|
pthread_mutex_destroy(&(pool->work_mutex));
|
||||||
pthread_cond_destroy(&(pool->has_work_cond));
|
pthread_cond_destroy(&(pool->has_work_cond));
|
||||||
pthread_cond_destroy(&(pool->working_cond));
|
pthread_cond_destroy(&(pool->working_cond));
|
||||||
@ -220,6 +228,8 @@ tpool_t *tpool_create(size_t thread_cnt, void cleanup_func()) {
|
|||||||
|
|
||||||
void tpool_start(tpool_t *pool) {
|
void tpool_start(tpool_t *pool) {
|
||||||
|
|
||||||
|
LOG_INFOF("tpool.c", "Starting thread pool with %d threads", pool->thread_cnt)
|
||||||
|
|
||||||
for (size_t i = 0; i < pool->thread_cnt; i++) {
|
for (size_t i = 0; i < pool->thread_cnt; i++) {
|
||||||
pthread_create(&pool->threads[i], NULL, tpool_worker, pool);
|
pthread_create(&pool->threads[i], NULL, tpool_worker, pool);
|
||||||
}
|
}
|
||||||
|
File diff suppressed because one or more lines are too long
@ -11,7 +11,7 @@
|
|||||||
|
|
||||||
<nav class="navbar navbar-expand-lg">
|
<nav class="navbar navbar-expand-lg">
|
||||||
<a class="navbar-brand" href="/">sist2</a>
|
<a class="navbar-brand" href="/">sist2</a>
|
||||||
<span class="badge badge-pill version">v1.1.11</span>
|
<span class="badge badge-pill version">v1.1.12</span>
|
||||||
<span class="tagline">Lightning-fast file system indexer and search tool </span>
|
<span class="tagline">Lightning-fast file system indexer and search tool </span>
|
||||||
<a style="margin-left: auto" id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
|
<a style="margin-left: auto" id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
|
||||||
</nav>
|
</nav>
|
||||||
|
Loading…
x
Reference in New Issue
Block a user