Compare commits

..

No commits in common. "930361e78c7ccbb62a8d9a23e21993897c6e97d4" and "6225cf81de67cc030043a944144bb764d2c71c27" have entirely different histories.

14 changed files with 37 additions and 72 deletions

View File

@ -8,15 +8,13 @@ git submodule update --init --recursive
cd sist2-vue/
npm install
npm run build
) &
)
(
cd sist2-admin/frontend/
npm install
npm run build
) &
wait
)
mkdir build
(

View File

@ -8,15 +8,13 @@ git submodule update --init --recursive
cd sist2-vue/
npm install
npm run build
) &
)
(
cd sist2-admin/frontend/
npm install
npm run build
) &
wait
)
mkdir build
(

View File

@ -83,10 +83,6 @@ void exec_args_destroy(exec_args_t *args) {
free(args);
}
void sqlite_index_args_destroy(sqlite_index_args_t *args) {
// TODO
}
int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n");

View File

@ -134,7 +134,5 @@ void exec_args_destroy(exec_args_t *args);
int exec_args_validate(exec_args_t *args, int argc, const char **argv);
void sqlite_index_args_destroy(sqlite_index_args_t *args);
#endif

View File

@ -447,16 +447,12 @@ database_summary_stats_t database_fts_get_date_range(database_t *db) {
return stats;
}
char *get_after_where(char **after, fts_sort_t sort, int sort_asc) {
char *get_after_where(char **after, fts_sort_t sort) {
if (after == NULL) {
return NULL;
}
if (sort_asc) {
return "(sort_var, doc.ROWID) > (?3, ?4)";
}
return "(sort_var, doc.ROWID) < (?3, ?4)";
return "(sort_var, doc.ROWID) > (?3, ?4)";
}
cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
@ -473,7 +469,7 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
char *index_id_where = index_ids_where_clause(index_ids);
char *mime_where = mime_types_where_clause(mime_types);
const char *query_where = match_where(query);
const char *after_where = get_after_where(after, sort, sort_asc);
const char *after_where = get_after_where(after, sort);
const char *tags_where = tags_where_clause(tags);
if (!query_where && sort == FTS_SORT_SCORE) {

View File

@ -68,7 +68,9 @@ void database_scan_begin(scan_args_t *args) {
desc->version_patch = VersionPatch;
// generate new index id based on timestamp
md5_hexdigest(&ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), ScanCtx.index.desc.id);
unsigned char index_md5[MD5_DIGEST_LENGTH];
MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
database_initialize(db);
database_open(db);
@ -681,7 +683,6 @@ int main(int argc, const char *argv[]) {
index_args_destroy(index_args);
web_args_destroy(web_args);
exec_args_destroy(exec_args);
sqlite_index_args_destroy(sqlite_index_args);
return 0;
}

View File

@ -2,18 +2,15 @@
#define SIST2_FS_UTIL_H
#include "src/sist.h"
#include <openssl/evp.h>
#define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));};
static int fs_read(struct vfile *f, void *buf, size_t size) {
if (f->fd == -1) {
f->sha1_ctx = EVP_MD_CTX_new();
EVP_DigestInit_ex(f->sha1_ctx, EVP_sha1(), NULL);
SHA1_Init(&f->sha1_ctx);
f->fd = open(f->filepath, O_RDONLY);
if (f->fd == -1) {
EVP_MD_CTX_free(f->sha1_ctx);
return -1;
}
}
@ -22,7 +19,7 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
if (ret != 0 && f->calculate_checksum) {
f->has_checksum = TRUE;
safe_digest_update(f->sha1_ctx, (unsigned char *) buf, ret);
safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret);
}
return ret;
@ -30,11 +27,8 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
static void fs_close(struct vfile *f) {
if (f->fd != -1) {
EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
EVP_MD_CTX_free(f->sha1_ctx);
f->sha1_ctx = NULL;
SHA1_Final(f->sha1_digest, &f->sha1_ctx);
close(f->fd);
f->fd = -1;
}
}

View File

@ -7,7 +7,6 @@
#include "third-party/utf8.h/utf8.h"
#include "libscan/scan.h"
#include <openssl/evp.h>
char *abspath(const char *path);
@ -87,22 +86,13 @@ static void buf2hex(const unsigned char *buf, size_t buflen, char *hex_string) {
*s = '\0';
}
static void md5_hexdigest(void *data, size_t size, char *output) {
EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
EVP_DigestInit_ex(md_ctx, EVP_md5(), NULL);
EVP_DigestUpdate(md_ctx, data, size);
unsigned char digest[MD5_DIGEST_LENGTH];
EVP_DigestFinal_ex(md_ctx, digest, NULL);
EVP_MD_CTX_free(md_ctx);
buf2hex(digest, MD5_DIGEST_LENGTH, output);
}
__always_inline
static void generate_doc_id(const char *rel_path, char *doc_id) {
md5_hexdigest(rel_path, strlen(rel_path), doc_id);
unsigned char md[MD5_DIGEST_LENGTH];
MD5((unsigned char *) rel_path, strlen(rel_path), md);
buf2hex(md, sizeof(md), doc_id);
}
#define MILLISECOND 1000

View File

@ -22,11 +22,7 @@ int should_parse_filtered_file(const char *filepath) {
}
void arc_close(struct vfile *f) {
if (f->sha1_ctx != NULL) {
EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
EVP_MD_CTX_free(f->sha1_ctx);
f->sha1_ctx = NULL;
}
SHA1_Final(f->sha1_digest, &f->sha1_ctx);
if (f->rewind_buffer != NULL) {
free(f->rewind_buffer);
@ -63,7 +59,7 @@ int arc_read(struct vfile *f, void *buf, size_t size) {
if (bytes_read != 0 && bytes_read <= size && f->calculate_checksum) {
f->has_checksum = TRUE;
safe_digest_update(f->sha1_ctx, (unsigned char *) buf, bytes_read);
safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, bytes_read);
}
if (bytes_read != size && archive_errno(f->arc) != 0) {
@ -241,12 +237,9 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
sub_job->ext = (int) strlen(sub_job->filepath);
}
sub_job->vfile.sha1_ctx = EVP_MD_CTX_new();
EVP_DigestInit(sub_job->vfile.sha1_ctx, EVP_sha1());
SHA1_Init(&sub_job->vfile.sha1_ctx);
ctx->parse(sub_job);
sub_job->vfile.close(&sub_job->vfile);
}
}

View File

@ -35,8 +35,7 @@ static int vfile_open_callback(struct archive *a, void *user_data) {
arc_data_t *data = (arc_data_t *) user_data;
if (!data->f->is_fs_file) {
data->f->sha1_ctx = EVP_MD_CTX_new();
EVP_DigestInit(data->f->sha1_ctx, EVP_md5());
SHA1_Init(&data->f->sha1_ctx);
}
return ARCHIVE_OK;
@ -50,7 +49,7 @@ static long vfile_read_callback(struct archive *a, void *user_data, const void *
if (!data->f->is_fs_file && ret > 0) {
data->f->has_checksum = TRUE;
safe_digest_update(data->f->sha1_ctx, (unsigned char *) data->buf, ret);
safe_sha1_update(&data->f->sha1_ctx, (unsigned char*)data->buf, ret);
}
return ret;
@ -60,9 +59,7 @@ static int vfile_close_callback(struct archive *a, void *user_data) {
arc_data_t *data = (arc_data_t *) user_data;
if (!data->f->is_fs_file) {
EVP_DigestFinal_ex(data->f->sha1_ctx, data->f->sha1_digest, NULL);
EVP_MD_CTX_free(data->f->sha1_ctx);
data->f->sha1_ctx = NULL;
SHA1_Final((unsigned char *) data->f->sha1_digest, &data->f->sha1_ctx);
}
return ARCHIVE_OK;

View File

@ -230,6 +230,13 @@ static int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
return 0;
}
static int ocr_progress(fz_context *fzctx, void *user_data, int progress) {
scan_ebook_ctx_t *ctx = user_data;
CTX_LOG_INFOF("ebook.c", "OCR PROGRESS=%d", progress);
return 0;
}
int read_stext(text_buffer_t *tex, fz_stext_page *stext) {
int count = 0;
@ -407,7 +414,7 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi
page_mediabox, TRUE,
ctx->tesseract_lang,
ctx->tesseract_path,
NULL, NULL);
ocr_progress, ctx);
fz_var(err);
fz_try(fzctx)fz_run_page(fzctx, page, ocr_dev, fz_identity, NULL);

View File

@ -697,10 +697,9 @@ int memfile_open(vfile_t *f, memfile_t *mem) {
mem->file = fmemopen(mem->buf, mem->size, "rb");
if (f->calculate_checksum) {
safe_digest_update(f->sha1_ctx, mem->buf, mem->size);
EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
EVP_MD_CTX_free(f->sha1_ctx);
f->sha1_ctx = NULL;
SHA1_Init(&f->sha1_ctx);
safe_sha1_update(&f->sha1_ctx, mem->buf, mem->size);
SHA1_Final(f->sha1_digest, &f->sha1_ctx);
f->has_checksum = TRUE;
}

View File

@ -8,7 +8,6 @@
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <openssl/evp.h>
#include <openssl/md5.h>
#include <openssl/sha.h>
@ -147,7 +146,7 @@ typedef struct vfile {
int mtime;
size_t st_size;
EVP_MD_CTX *sha1_ctx;
SHA_CTX sha1_ctx;
unsigned char sha1_digest[SHA1_DIGEST_LENGTH];
void *rewind_buffer;

View File

@ -6,7 +6,6 @@
#include "string.h"
#include "../third-party/utf8.h/utf8.h"
#include "macros.h"
#include <openssl/evp.h>
#define STR_STARTS_WITH_CONSTANT(x, y) (strncmp(y, x, sizeof(y) - 1) == 0)
@ -340,7 +339,7 @@ static void *read_all(vfile_t *f, size_t *size) {
#define STACK_BUFFER_SIZE (size_t)(4096 * 8)
__always_inline
static void safe_digest_update(EVP_MD_CTX *ctx, void *buf, size_t size) {
static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) {
unsigned char stack_buf[STACK_BUFFER_SIZE];
void *sha1_buf;
@ -352,7 +351,7 @@ static void safe_digest_update(EVP_MD_CTX *ctx, void *buf, size_t size) {
}
memcpy(sha1_buf, buf, size);
EVP_DigestUpdate(ctx, sha1_buf, size);
SHA1_Update(ctx, (const void *) sha1_buf, size);
if (sha1_buf != stack_buf) {
free(sha1_buf);