Compare commits

..

No commits in common. "930361e78c7ccbb62a8d9a23e21993897c6e97d4" and "6225cf81de67cc030043a944144bb764d2c71c27" have entirely different histories.

14 changed files with 37 additions and 72 deletions

View File

@ -8,15 +8,13 @@ git submodule update --init --recursive
cd sist2-vue/ cd sist2-vue/
npm install npm install
npm run build npm run build
) & )
( (
cd sist2-admin/frontend/ cd sist2-admin/frontend/
npm install npm install
npm run build npm run build
) & )
wait
mkdir build mkdir build
( (

View File

@ -8,15 +8,13 @@ git submodule update --init --recursive
cd sist2-vue/ cd sist2-vue/
npm install npm install
npm run build npm run build
) & )
( (
cd sist2-admin/frontend/ cd sist2-admin/frontend/
npm install npm install
npm run build npm run build
) & )
wait
mkdir build mkdir build
( (

View File

@ -83,10 +83,6 @@ void exec_args_destroy(exec_args_t *args) {
free(args); free(args);
} }
void sqlite_index_args_destroy(sqlite_index_args_t *args) {
// TODO
}
int scan_args_validate(scan_args_t *args, int argc, const char **argv) { int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
if (argc < 2) { if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n"); fprintf(stderr, "Required positional argument: PATH.\n");

View File

@ -134,7 +134,5 @@ void exec_args_destroy(exec_args_t *args);
int exec_args_validate(exec_args_t *args, int argc, const char **argv); int exec_args_validate(exec_args_t *args, int argc, const char **argv);
void sqlite_index_args_destroy(sqlite_index_args_t *args);
#endif #endif

View File

@ -447,16 +447,12 @@ database_summary_stats_t database_fts_get_date_range(database_t *db) {
return stats; return stats;
} }
char *get_after_where(char **after, fts_sort_t sort, int sort_asc) { char *get_after_where(char **after, fts_sort_t sort) {
if (after == NULL) { if (after == NULL) {
return NULL; return NULL;
} }
if (sort_asc) {
return "(sort_var, doc.ROWID) > (?3, ?4)"; return "(sort_var, doc.ROWID) > (?3, ?4)";
}
return "(sort_var, doc.ROWID) < (?3, ?4)";
} }
cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min, cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
@ -473,7 +469,7 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
char *index_id_where = index_ids_where_clause(index_ids); char *index_id_where = index_ids_where_clause(index_ids);
char *mime_where = mime_types_where_clause(mime_types); char *mime_where = mime_types_where_clause(mime_types);
const char *query_where = match_where(query); const char *query_where = match_where(query);
const char *after_where = get_after_where(after, sort, sort_asc); const char *after_where = get_after_where(after, sort);
const char *tags_where = tags_where_clause(tags); const char *tags_where = tags_where_clause(tags);
if (!query_where && sort == FTS_SORT_SCORE) { if (!query_where && sort == FTS_SORT_SCORE) {

View File

@ -68,7 +68,9 @@ void database_scan_begin(scan_args_t *args) {
desc->version_patch = VersionPatch; desc->version_patch = VersionPatch;
// generate new index id based on timestamp // generate new index id based on timestamp
md5_hexdigest(&ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), ScanCtx.index.desc.id); unsigned char index_md5[MD5_DIGEST_LENGTH];
MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
database_initialize(db); database_initialize(db);
database_open(db); database_open(db);
@ -681,7 +683,6 @@ int main(int argc, const char *argv[]) {
index_args_destroy(index_args); index_args_destroy(index_args);
web_args_destroy(web_args); web_args_destroy(web_args);
exec_args_destroy(exec_args); exec_args_destroy(exec_args);
sqlite_index_args_destroy(sqlite_index_args);
return 0; return 0;
} }

View File

@ -2,18 +2,15 @@
#define SIST2_FS_UTIL_H #define SIST2_FS_UTIL_H
#include "src/sist.h" #include "src/sist.h"
#include <openssl/evp.h>
#define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));}; #define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));};
static int fs_read(struct vfile *f, void *buf, size_t size) { static int fs_read(struct vfile *f, void *buf, size_t size) {
if (f->fd == -1) { if (f->fd == -1) {
f->sha1_ctx = EVP_MD_CTX_new(); SHA1_Init(&f->sha1_ctx);
EVP_DigestInit_ex(f->sha1_ctx, EVP_sha1(), NULL);
f->fd = open(f->filepath, O_RDONLY); f->fd = open(f->filepath, O_RDONLY);
if (f->fd == -1) { if (f->fd == -1) {
EVP_MD_CTX_free(f->sha1_ctx);
return -1; return -1;
} }
} }
@ -22,7 +19,7 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
if (ret != 0 && f->calculate_checksum) { if (ret != 0 && f->calculate_checksum) {
f->has_checksum = TRUE; f->has_checksum = TRUE;
safe_digest_update(f->sha1_ctx, (unsigned char *) buf, ret); safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret);
} }
return ret; return ret;
@ -30,11 +27,8 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
static void fs_close(struct vfile *f) { static void fs_close(struct vfile *f) {
if (f->fd != -1) { if (f->fd != -1) {
EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL); SHA1_Final(f->sha1_digest, &f->sha1_ctx);
EVP_MD_CTX_free(f->sha1_ctx);
f->sha1_ctx = NULL;
close(f->fd); close(f->fd);
f->fd = -1;
} }
} }

View File

@ -7,7 +7,6 @@
#include "third-party/utf8.h/utf8.h" #include "third-party/utf8.h/utf8.h"
#include "libscan/scan.h" #include "libscan/scan.h"
#include <openssl/evp.h>
char *abspath(const char *path); char *abspath(const char *path);
@ -87,22 +86,13 @@ static void buf2hex(const unsigned char *buf, size_t buflen, char *hex_string) {
*s = '\0'; *s = '\0';
} }
static void md5_hexdigest(void *data, size_t size, char *output) {
EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
EVP_DigestInit_ex(md_ctx, EVP_md5(), NULL);
EVP_DigestUpdate(md_ctx, data, size);
unsigned char digest[MD5_DIGEST_LENGTH];
EVP_DigestFinal_ex(md_ctx, digest, NULL);
EVP_MD_CTX_free(md_ctx);
buf2hex(digest, MD5_DIGEST_LENGTH, output);
}
__always_inline __always_inline
static void generate_doc_id(const char *rel_path, char *doc_id) { static void generate_doc_id(const char *rel_path, char *doc_id) {
md5_hexdigest(rel_path, strlen(rel_path), doc_id); unsigned char md[MD5_DIGEST_LENGTH];
MD5((unsigned char *) rel_path, strlen(rel_path), md);
buf2hex(md, sizeof(md), doc_id);
} }
#define MILLISECOND 1000 #define MILLISECOND 1000

View File

@ -22,11 +22,7 @@ int should_parse_filtered_file(const char *filepath) {
} }
void arc_close(struct vfile *f) { void arc_close(struct vfile *f) {
if (f->sha1_ctx != NULL) { SHA1_Final(f->sha1_digest, &f->sha1_ctx);
EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
EVP_MD_CTX_free(f->sha1_ctx);
f->sha1_ctx = NULL;
}
if (f->rewind_buffer != NULL) { if (f->rewind_buffer != NULL) {
free(f->rewind_buffer); free(f->rewind_buffer);
@ -63,7 +59,7 @@ int arc_read(struct vfile *f, void *buf, size_t size) {
if (bytes_read != 0 && bytes_read <= size && f->calculate_checksum) { if (bytes_read != 0 && bytes_read <= size && f->calculate_checksum) {
f->has_checksum = TRUE; f->has_checksum = TRUE;
safe_digest_update(f->sha1_ctx, (unsigned char *) buf, bytes_read); safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, bytes_read);
} }
if (bytes_read != size && archive_errno(f->arc) != 0) { if (bytes_read != size && archive_errno(f->arc) != 0) {
@ -241,12 +237,9 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
sub_job->ext = (int) strlen(sub_job->filepath); sub_job->ext = (int) strlen(sub_job->filepath);
} }
sub_job->vfile.sha1_ctx = EVP_MD_CTX_new(); SHA1_Init(&sub_job->vfile.sha1_ctx);
EVP_DigestInit(sub_job->vfile.sha1_ctx, EVP_sha1());
ctx->parse(sub_job); ctx->parse(sub_job);
sub_job->vfile.close(&sub_job->vfile);
} }
} }

View File

@ -35,8 +35,7 @@ static int vfile_open_callback(struct archive *a, void *user_data) {
arc_data_t *data = (arc_data_t *) user_data; arc_data_t *data = (arc_data_t *) user_data;
if (!data->f->is_fs_file) { if (!data->f->is_fs_file) {
data->f->sha1_ctx = EVP_MD_CTX_new(); SHA1_Init(&data->f->sha1_ctx);
EVP_DigestInit(data->f->sha1_ctx, EVP_md5());
} }
return ARCHIVE_OK; return ARCHIVE_OK;
@ -50,7 +49,7 @@ static long vfile_read_callback(struct archive *a, void *user_data, const void *
if (!data->f->is_fs_file && ret > 0) { if (!data->f->is_fs_file && ret > 0) {
data->f->has_checksum = TRUE; data->f->has_checksum = TRUE;
safe_digest_update(data->f->sha1_ctx, (unsigned char *) data->buf, ret); safe_sha1_update(&data->f->sha1_ctx, (unsigned char*)data->buf, ret);
} }
return ret; return ret;
@ -60,9 +59,7 @@ static int vfile_close_callback(struct archive *a, void *user_data) {
arc_data_t *data = (arc_data_t *) user_data; arc_data_t *data = (arc_data_t *) user_data;
if (!data->f->is_fs_file) { if (!data->f->is_fs_file) {
EVP_DigestFinal_ex(data->f->sha1_ctx, data->f->sha1_digest, NULL); SHA1_Final((unsigned char *) data->f->sha1_digest, &data->f->sha1_ctx);
EVP_MD_CTX_free(data->f->sha1_ctx);
data->f->sha1_ctx = NULL;
} }
return ARCHIVE_OK; return ARCHIVE_OK;

View File

@ -230,6 +230,13 @@ static int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
return 0; return 0;
} }
static int ocr_progress(fz_context *fzctx, void *user_data, int progress) {
scan_ebook_ctx_t *ctx = user_data;
CTX_LOG_INFOF("ebook.c", "OCR PROGRESS=%d", progress);
return 0;
}
int read_stext(text_buffer_t *tex, fz_stext_page *stext) { int read_stext(text_buffer_t *tex, fz_stext_page *stext) {
int count = 0; int count = 0;
@ -407,7 +414,7 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi
page_mediabox, TRUE, page_mediabox, TRUE,
ctx->tesseract_lang, ctx->tesseract_lang,
ctx->tesseract_path, ctx->tesseract_path,
NULL, NULL); ocr_progress, ctx);
fz_var(err); fz_var(err);
fz_try(fzctx)fz_run_page(fzctx, page, ocr_dev, fz_identity, NULL); fz_try(fzctx)fz_run_page(fzctx, page, ocr_dev, fz_identity, NULL);

View File

@ -697,10 +697,9 @@ int memfile_open(vfile_t *f, memfile_t *mem) {
mem->file = fmemopen(mem->buf, mem->size, "rb"); mem->file = fmemopen(mem->buf, mem->size, "rb");
if (f->calculate_checksum) { if (f->calculate_checksum) {
safe_digest_update(f->sha1_ctx, mem->buf, mem->size); SHA1_Init(&f->sha1_ctx);
EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL); safe_sha1_update(&f->sha1_ctx, mem->buf, mem->size);
EVP_MD_CTX_free(f->sha1_ctx); SHA1_Final(f->sha1_digest, &f->sha1_ctx);
f->sha1_ctx = NULL;
f->has_checksum = TRUE; f->has_checksum = TRUE;
} }

View File

@ -8,7 +8,6 @@
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <openssl/evp.h>
#include <openssl/md5.h> #include <openssl/md5.h>
#include <openssl/sha.h> #include <openssl/sha.h>
@ -147,7 +146,7 @@ typedef struct vfile {
int mtime; int mtime;
size_t st_size; size_t st_size;
EVP_MD_CTX *sha1_ctx; SHA_CTX sha1_ctx;
unsigned char sha1_digest[SHA1_DIGEST_LENGTH]; unsigned char sha1_digest[SHA1_DIGEST_LENGTH];
void *rewind_buffer; void *rewind_buffer;

View File

@ -6,7 +6,6 @@
#include "string.h" #include "string.h"
#include "../third-party/utf8.h/utf8.h" #include "../third-party/utf8.h/utf8.h"
#include "macros.h" #include "macros.h"
#include <openssl/evp.h>
#define STR_STARTS_WITH_CONSTANT(x, y) (strncmp(y, x, sizeof(y) - 1) == 0) #define STR_STARTS_WITH_CONSTANT(x, y) (strncmp(y, x, sizeof(y) - 1) == 0)
@ -340,7 +339,7 @@ static void *read_all(vfile_t *f, size_t *size) {
#define STACK_BUFFER_SIZE (size_t)(4096 * 8) #define STACK_BUFFER_SIZE (size_t)(4096 * 8)
__always_inline __always_inline
static void safe_digest_update(EVP_MD_CTX *ctx, void *buf, size_t size) { static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) {
unsigned char stack_buf[STACK_BUFFER_SIZE]; unsigned char stack_buf[STACK_BUFFER_SIZE];
void *sha1_buf; void *sha1_buf;
@ -352,7 +351,7 @@ static void safe_digest_update(EVP_MD_CTX *ctx, void *buf, size_t size) {
} }
memcpy(sha1_buf, buf, size); memcpy(sha1_buf, buf, size);
EVP_DigestUpdate(ctx, sha1_buf, size); SHA1_Update(ctx, (const void *) sha1_buf, size);
if (sha1_buf != stack_buf) { if (sha1_buf != stack_buf) {
free(sha1_buf); free(sha1_buf);