mirror of
https://github.com/simon987/sist2.git
synced 2025-04-21 11:16:46 +00:00
Compare commits
No commits in common. "930361e78c7ccbb62a8d9a23e21993897c6e97d4" and "6225cf81de67cc030043a944144bb764d2c71c27" have entirely different histories.
930361e78c
...
6225cf81de
@ -8,15 +8,13 @@ git submodule update --init --recursive
|
||||
cd sist2-vue/
|
||||
npm install
|
||||
npm run build
|
||||
) &
|
||||
)
|
||||
|
||||
(
|
||||
cd sist2-admin/frontend/
|
||||
npm install
|
||||
npm run build
|
||||
) &
|
||||
|
||||
wait
|
||||
)
|
||||
|
||||
mkdir build
|
||||
(
|
||||
|
@ -8,15 +8,13 @@ git submodule update --init --recursive
|
||||
cd sist2-vue/
|
||||
npm install
|
||||
npm run build
|
||||
) &
|
||||
)
|
||||
|
||||
(
|
||||
cd sist2-admin/frontend/
|
||||
npm install
|
||||
npm run build
|
||||
) &
|
||||
|
||||
wait
|
||||
)
|
||||
|
||||
mkdir build
|
||||
(
|
||||
|
@ -83,10 +83,6 @@ void exec_args_destroy(exec_args_t *args) {
|
||||
free(args);
|
||||
}
|
||||
|
||||
void sqlite_index_args_destroy(sqlite_index_args_t *args) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "Required positional argument: PATH.\n");
|
||||
|
@ -134,7 +134,5 @@ void exec_args_destroy(exec_args_t *args);
|
||||
|
||||
int exec_args_validate(exec_args_t *args, int argc, const char **argv);
|
||||
|
||||
void sqlite_index_args_destroy(sqlite_index_args_t *args);
|
||||
|
||||
|
||||
#endif
|
||||
|
@ -447,16 +447,12 @@ database_summary_stats_t database_fts_get_date_range(database_t *db) {
|
||||
return stats;
|
||||
}
|
||||
|
||||
char *get_after_where(char **after, fts_sort_t sort, int sort_asc) {
|
||||
char *get_after_where(char **after, fts_sort_t sort) {
|
||||
if (after == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (sort_asc) {
|
||||
return "(sort_var, doc.ROWID) > (?3, ?4)";
|
||||
}
|
||||
|
||||
return "(sort_var, doc.ROWID) < (?3, ?4)";
|
||||
return "(sort_var, doc.ROWID) > (?3, ?4)";
|
||||
}
|
||||
|
||||
cJSON *database_fts_search(database_t *db, const char *query, const char *path, long size_min,
|
||||
@ -473,7 +469,7 @@ cJSON *database_fts_search(database_t *db, const char *query, const char *path,
|
||||
char *index_id_where = index_ids_where_clause(index_ids);
|
||||
char *mime_where = mime_types_where_clause(mime_types);
|
||||
const char *query_where = match_where(query);
|
||||
const char *after_where = get_after_where(after, sort, sort_asc);
|
||||
const char *after_where = get_after_where(after, sort);
|
||||
const char *tags_where = tags_where_clause(tags);
|
||||
|
||||
if (!query_where && sort == FTS_SORT_SCORE) {
|
||||
|
@ -68,7 +68,9 @@ void database_scan_begin(scan_args_t *args) {
|
||||
desc->version_patch = VersionPatch;
|
||||
|
||||
// generate new index id based on timestamp
|
||||
md5_hexdigest(&ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), ScanCtx.index.desc.id);
|
||||
unsigned char index_md5[MD5_DIGEST_LENGTH];
|
||||
MD5((unsigned char *) &ScanCtx.index.desc.timestamp, sizeof(ScanCtx.index.desc.timestamp), index_md5);
|
||||
buf2hex(index_md5, MD5_DIGEST_LENGTH, ScanCtx.index.desc.id);
|
||||
|
||||
database_initialize(db);
|
||||
database_open(db);
|
||||
@ -681,7 +683,6 @@ int main(int argc, const char *argv[]) {
|
||||
index_args_destroy(index_args);
|
||||
web_args_destroy(web_args);
|
||||
exec_args_destroy(exec_args);
|
||||
sqlite_index_args_destroy(sqlite_index_args);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -2,18 +2,15 @@
|
||||
#define SIST2_FS_UTIL_H
|
||||
|
||||
#include "src/sist.h"
|
||||
#include <openssl/evp.h>
|
||||
|
||||
#define CLOSE_FILE(f) if ((f).close != NULL) {(f).close(&(f));};
|
||||
|
||||
static int fs_read(struct vfile *f, void *buf, size_t size) {
|
||||
if (f->fd == -1) {
|
||||
f->sha1_ctx = EVP_MD_CTX_new();
|
||||
EVP_DigestInit_ex(f->sha1_ctx, EVP_sha1(), NULL);
|
||||
SHA1_Init(&f->sha1_ctx);
|
||||
|
||||
f->fd = open(f->filepath, O_RDONLY);
|
||||
if (f->fd == -1) {
|
||||
EVP_MD_CTX_free(f->sha1_ctx);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
@ -22,7 +19,7 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
|
||||
|
||||
if (ret != 0 && f->calculate_checksum) {
|
||||
f->has_checksum = TRUE;
|
||||
safe_digest_update(f->sha1_ctx, (unsigned char *) buf, ret);
|
||||
safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -30,11 +27,8 @@ static int fs_read(struct vfile *f, void *buf, size_t size) {
|
||||
|
||||
static void fs_close(struct vfile *f) {
|
||||
if (f->fd != -1) {
|
||||
EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
|
||||
EVP_MD_CTX_free(f->sha1_ctx);
|
||||
f->sha1_ctx = NULL;
|
||||
SHA1_Final(f->sha1_digest, &f->sha1_ctx);
|
||||
close(f->fd);
|
||||
f->fd = -1;
|
||||
}
|
||||
}
|
||||
|
||||
|
18
src/util.h
18
src/util.h
@ -7,7 +7,6 @@
|
||||
|
||||
#include "third-party/utf8.h/utf8.h"
|
||||
#include "libscan/scan.h"
|
||||
#include <openssl/evp.h>
|
||||
|
||||
|
||||
char *abspath(const char *path);
|
||||
@ -87,22 +86,13 @@ static void buf2hex(const unsigned char *buf, size_t buflen, char *hex_string) {
|
||||
*s = '\0';
|
||||
}
|
||||
|
||||
static void md5_hexdigest(void *data, size_t size, char *output) {
|
||||
EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
|
||||
EVP_DigestInit_ex(md_ctx, EVP_md5(), NULL);
|
||||
|
||||
EVP_DigestUpdate(md_ctx, data, size);
|
||||
|
||||
unsigned char digest[MD5_DIGEST_LENGTH];
|
||||
EVP_DigestFinal_ex(md_ctx, digest, NULL);
|
||||
EVP_MD_CTX_free(md_ctx);
|
||||
|
||||
buf2hex(digest, MD5_DIGEST_LENGTH, output);
|
||||
}
|
||||
|
||||
__always_inline
|
||||
static void generate_doc_id(const char *rel_path, char *doc_id) {
|
||||
md5_hexdigest(rel_path, strlen(rel_path), doc_id);
|
||||
unsigned char md[MD5_DIGEST_LENGTH];
|
||||
|
||||
MD5((unsigned char *) rel_path, strlen(rel_path), md);
|
||||
buf2hex(md, sizeof(md), doc_id);
|
||||
}
|
||||
|
||||
#define MILLISECOND 1000
|
||||
|
13
third-party/libscan/libscan/arc/arc.c
vendored
13
third-party/libscan/libscan/arc/arc.c
vendored
@ -22,11 +22,7 @@ int should_parse_filtered_file(const char *filepath) {
|
||||
}
|
||||
|
||||
void arc_close(struct vfile *f) {
|
||||
if (f->sha1_ctx != NULL) {
|
||||
EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
|
||||
EVP_MD_CTX_free(f->sha1_ctx);
|
||||
f->sha1_ctx = NULL;
|
||||
}
|
||||
SHA1_Final(f->sha1_digest, &f->sha1_ctx);
|
||||
|
||||
if (f->rewind_buffer != NULL) {
|
||||
free(f->rewind_buffer);
|
||||
@ -63,7 +59,7 @@ int arc_read(struct vfile *f, void *buf, size_t size) {
|
||||
if (bytes_read != 0 && bytes_read <= size && f->calculate_checksum) {
|
||||
f->has_checksum = TRUE;
|
||||
|
||||
safe_digest_update(f->sha1_ctx, (unsigned char *) buf, bytes_read);
|
||||
safe_sha1_update(&f->sha1_ctx, (unsigned char *) buf, bytes_read);
|
||||
}
|
||||
|
||||
if (bytes_read != size && archive_errno(f->arc) != 0) {
|
||||
@ -241,12 +237,9 @@ scan_code_t parse_archive(scan_arc_ctx_t *ctx, vfile_t *f, document_t *doc, pcre
|
||||
sub_job->ext = (int) strlen(sub_job->filepath);
|
||||
}
|
||||
|
||||
sub_job->vfile.sha1_ctx = EVP_MD_CTX_new();
|
||||
EVP_DigestInit(sub_job->vfile.sha1_ctx, EVP_sha1());
|
||||
SHA1_Init(&sub_job->vfile.sha1_ctx);
|
||||
|
||||
ctx->parse(sub_job);
|
||||
|
||||
sub_job->vfile.close(&sub_job->vfile);
|
||||
}
|
||||
}
|
||||
|
||||
|
9
third-party/libscan/libscan/arc/arc.h
vendored
9
third-party/libscan/libscan/arc/arc.h
vendored
@ -35,8 +35,7 @@ static int vfile_open_callback(struct archive *a, void *user_data) {
|
||||
arc_data_t *data = (arc_data_t *) user_data;
|
||||
|
||||
if (!data->f->is_fs_file) {
|
||||
data->f->sha1_ctx = EVP_MD_CTX_new();
|
||||
EVP_DigestInit(data->f->sha1_ctx, EVP_md5());
|
||||
SHA1_Init(&data->f->sha1_ctx);
|
||||
}
|
||||
|
||||
return ARCHIVE_OK;
|
||||
@ -50,7 +49,7 @@ static long vfile_read_callback(struct archive *a, void *user_data, const void *
|
||||
|
||||
if (!data->f->is_fs_file && ret > 0) {
|
||||
data->f->has_checksum = TRUE;
|
||||
safe_digest_update(data->f->sha1_ctx, (unsigned char *) data->buf, ret);
|
||||
safe_sha1_update(&data->f->sha1_ctx, (unsigned char*)data->buf, ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
@ -60,9 +59,7 @@ static int vfile_close_callback(struct archive *a, void *user_data) {
|
||||
arc_data_t *data = (arc_data_t *) user_data;
|
||||
|
||||
if (!data->f->is_fs_file) {
|
||||
EVP_DigestFinal_ex(data->f->sha1_ctx, data->f->sha1_digest, NULL);
|
||||
EVP_MD_CTX_free(data->f->sha1_ctx);
|
||||
data->f->sha1_ctx = NULL;
|
||||
SHA1_Final((unsigned char *) data->f->sha1_digest, &data->f->sha1_ctx);
|
||||
}
|
||||
|
||||
return ARCHIVE_OK;
|
||||
|
9
third-party/libscan/libscan/ebook/ebook.c
vendored
9
third-party/libscan/libscan/ebook/ebook.c
vendored
@ -230,6 +230,13 @@ static int read_stext_block(fz_stext_block *block, text_buffer_t *tex) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ocr_progress(fz_context *fzctx, void *user_data, int progress) {
|
||||
scan_ebook_ctx_t *ctx = user_data;
|
||||
CTX_LOG_INFOF("ebook.c", "OCR PROGRESS=%d", progress);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int read_stext(text_buffer_t *tex, fz_stext_page *stext) {
|
||||
|
||||
int count = 0;
|
||||
@ -407,7 +414,7 @@ parse_ebook_mem(scan_ebook_ctx_t *ctx, void *buf, size_t buf_len, const char *mi
|
||||
page_mediabox, TRUE,
|
||||
ctx->tesseract_lang,
|
||||
ctx->tesseract_path,
|
||||
NULL, NULL);
|
||||
ocr_progress, ctx);
|
||||
|
||||
fz_var(err);
|
||||
fz_try(fzctx)fz_run_page(fzctx, page, ocr_dev, fz_identity, NULL);
|
||||
|
7
third-party/libscan/libscan/media/media.c
vendored
7
third-party/libscan/libscan/media/media.c
vendored
@ -697,10 +697,9 @@ int memfile_open(vfile_t *f, memfile_t *mem) {
|
||||
mem->file = fmemopen(mem->buf, mem->size, "rb");
|
||||
|
||||
if (f->calculate_checksum) {
|
||||
safe_digest_update(f->sha1_ctx, mem->buf, mem->size);
|
||||
EVP_DigestFinal_ex(f->sha1_ctx, f->sha1_digest, NULL);
|
||||
EVP_MD_CTX_free(f->sha1_ctx);
|
||||
f->sha1_ctx = NULL;
|
||||
SHA1_Init(&f->sha1_ctx);
|
||||
safe_sha1_update(&f->sha1_ctx, mem->buf, mem->size);
|
||||
SHA1_Final(f->sha1_digest, &f->sha1_ctx);
|
||||
f->has_checksum = TRUE;
|
||||
}
|
||||
|
||||
|
3
third-party/libscan/libscan/scan.h
vendored
3
third-party/libscan/libscan/scan.h
vendored
@ -8,7 +8,6 @@
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/md5.h>
|
||||
#include <openssl/sha.h>
|
||||
|
||||
@ -147,7 +146,7 @@ typedef struct vfile {
|
||||
int mtime;
|
||||
size_t st_size;
|
||||
|
||||
EVP_MD_CTX *sha1_ctx;
|
||||
SHA_CTX sha1_ctx;
|
||||
unsigned char sha1_digest[SHA1_DIGEST_LENGTH];
|
||||
|
||||
void *rewind_buffer;
|
||||
|
5
third-party/libscan/libscan/util.h
vendored
5
third-party/libscan/libscan/util.h
vendored
@ -6,7 +6,6 @@
|
||||
#include "string.h"
|
||||
#include "../third-party/utf8.h/utf8.h"
|
||||
#include "macros.h"
|
||||
#include <openssl/evp.h>
|
||||
|
||||
#define STR_STARTS_WITH_CONSTANT(x, y) (strncmp(y, x, sizeof(y) - 1) == 0)
|
||||
|
||||
@ -340,7 +339,7 @@ static void *read_all(vfile_t *f, size_t *size) {
|
||||
#define STACK_BUFFER_SIZE (size_t)(4096 * 8)
|
||||
|
||||
__always_inline
|
||||
static void safe_digest_update(EVP_MD_CTX *ctx, void *buf, size_t size) {
|
||||
static void safe_sha1_update(SHA_CTX *ctx, void *buf, size_t size) {
|
||||
unsigned char stack_buf[STACK_BUFFER_SIZE];
|
||||
|
||||
void *sha1_buf;
|
||||
@ -352,7 +351,7 @@ static void safe_digest_update(EVP_MD_CTX *ctx, void *buf, size_t size) {
|
||||
}
|
||||
|
||||
memcpy(sha1_buf, buf, size);
|
||||
EVP_DigestUpdate(ctx, sha1_buf, size);
|
||||
SHA1_Update(ctx, (const void *) sha1_buf, size);
|
||||
|
||||
if (sha1_buf != stack_buf) {
|
||||
free(sha1_buf);
|
||||
|
Loading…
x
Reference in New Issue
Block a user