simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-12-18 01:39:05 +00:00
Code Issues Projects Releases Wiki Activity

use sqlite to save index, major thread pool refactor

Browse Source
This commit is contained in:
simon987
2023-04-03 21:39:50 -04:00
parent ca973d63a4
commit fc36f33d52
62 changed files with 3630 additions and 4673 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
src/index/elastic.c
View File

@@ -29,7 +29,7 @@ void destroy_indexer(es_indexer_t *indexer) {
return;
}
LOG_DEBUG("elastic.c", "Destroying indexer")
LOG_DEBUG("elastic.c", "Destroying indexer");
if (indexer->es_url != NULL) {
free(indexer->es_url);
@@ -64,26 +64,21 @@ void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) {
cJSON_Delete(line);
}
void index_json_func(tpool_work_arg_shm_t *arg) {
// Copy arg to heap because it's going to be freed immediately after this function returns
es_bulk_line_t *line = malloc(arg->arg_size);
memcpy(line, arg->arg, arg->arg_size);
elastic_index_line(line);
void index_json_func(job_t *job) {
elastic_index_line(job->bulk_line);
}
void delete_document(const char *document_id_str, void *UNUSED(_data)) {
void delete_document(const char *document_id) {
es_bulk_line_t *bulk_line = malloc(sizeof(es_bulk_line_t));
bulk_line->type = ES_BULK_LINE_DELETE;
bulk_line->next = NULL;
strcpy(bulk_line->doc_id, document_id_str);
strcpy(bulk_line->doc_id, document_id);
tpool_work_arg_t arg = {
.arg_size = sizeof(es_bulk_line_t),
.arg = bulk_line
};
tpool_add_work(IndexCtx.pool, index_json_func, &arg);
tpool_add_work(IndexCtx.pool, &(job_t) {
.type = JOB_BULK_LINE,
.bulk_line = bulk_line,
});
}
@@ -100,11 +95,10 @@ void index_json(cJSON *document, const char doc_id[SIST_DOC_ID_LEN]) {
bulk_line->next = NULL;
cJSON_free(json);
tpool_work_arg_t arg = {
.arg_size = sizeof(es_bulk_line_t) + json_len + 2,
.arg = bulk_line
};
tpool_add_work(IndexCtx.pool, index_json_func, &arg);
tpool_add_work(IndexCtx.pool, &(job_t) {
.type = JOB_BULK_LINE,
.bulk_line = bulk_line,
});
}
void execute_update_script(const char *script, int async, const char index_id[SIST_INDEX_ID_LEN]) {
@@ -278,7 +272,7 @@ void print_error(response_t *r) {
void _elastic_flush(int max) {
if (max == 0) {
LOG_WARNING("elastic.c", "calling _elastic_flush with 0 in queue")
LOG_WARNING("elastic.c", "calling _elastic_flush with 0 in queue");
return;
}
@@ -291,13 +285,13 @@ void _elastic_flush(int max) {
response_t *r = web_post(bulk_url, buf, IndexCtx.es_insecure_ssl);
if (r->status_code == 0) {
LOG_FATALF("elastic.c", "Could not connect to %s, make sure that elasticsearch is running!\n", IndexCtx.es_url)
LOG_FATALF("elastic.c", "Could not connect to %s, make sure that elasticsearch is running!\n", IndexCtx.es_url);
}
if (r->status_code == 413) {
if (max <= 1) {
LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->doc_id)
LOG_ERRORF("elastic.c", "Single document too large, giving up: {%s}", Indexer->line_head->doc_id);
free_response(r);
free(buf);
free_queue(1);
@@ -318,7 +312,7 @@ void _elastic_flush(int max) {
free_response(r);
free(buf);
LOG_WARNING("elastic.c", "Got 429 status, will retry after delay")
LOG_WARNING("elastic.c", "Got 429 status, will retry after delay");
usleep(1000000 * 20);
_elastic_flush(max);
return;
@@ -453,7 +447,7 @@ es_version_t *elastic_get_version(const char *es_url, int insecure) {
}
if (cJSON_GetObjectItem(response, "error") != NULL) {
LOG_WARNING("elastic.c", "Could not get Elasticsearch version")
LOG_WARNING("elastic.c", "Could not get Elasticsearch version");
print_error(r);
free_response(r);
return NULL;
@@ -489,7 +483,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s
IndexCtx.es_version = es_version;
if (es_version == NULL) {
LOG_FATAL("elastic.c", "Could not get ES version")
LOG_FATAL("elastic.c", "Could not get ES version");
}
LOG_INFOF("elastic.c",
@@ -497,7 +491,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s
format_es_version(es_version), IS_SUPPORTED_ES_VERSION(es_version), IS_LEGACY_VERSION(es_version));
if (!IS_SUPPORTED_ES_VERSION(es_version)) {
LOG_FATAL("elastic.c", "This elasticsearch version is not supported!")
LOG_FATAL("elastic.c", "This elasticsearch version is not supported!");
}
char *settings = NULL;
@@ -524,7 +518,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s
if (r->status_code != 200) {
print_error(r);
LOG_FATAL("elastic.c", "Could not create index")
LOG_FATAL("elastic.c", "Could not create index");
}
LOG_INFOF("elastic.c", "Create index <%d>", r->status_code);
@@ -545,7 +539,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s
LOG_INFOF("elastic.c", "Update ES settings <%d>", r->status_code);
if (r->status_code != 200) {
print_error(r);
LOG_FATAL("elastic.c", "Could not update user settings")
LOG_FATAL("elastic.c", "Could not update user settings");
}
free_response(r);
@@ -560,7 +554,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s
LOG_INFOF("elastic.c", "Update ES mappings <%d>", r->status_code);
if (r->status_code != 200) {
print_error(r);
LOG_FATAL("elastic.c", "Could not update user mappings")
LOG_FATAL("elastic.c", "Could not update user mappings");
}
free_response(r);

2
src/index/elastic.h
View File

@@ -46,7 +46,7 @@ void print_json(cJSON *document, const char index_id_str[SIST_INDEX_ID_LEN]);
void index_json(cJSON *document, const char doc_id[SIST_INDEX_ID_LEN]);
void delete_document(const char *document_id_str, void* data);
void delete_document(const char *document_id);
es_indexer_t *create_indexer(const char *url, const char *index);

8
src/index/web.c
View File

@@ -65,7 +65,7 @@ void web_post_async_poll(subreq_ctx_t *req) {
curl_easy_getinfo(req->handle, CURLINFO_RESPONSE_CODE, &req->response->status_code);
if (req->response->status_code == 0) {
LOG_ERRORF("web.c", "CURL Error: %s", req->curl_err_buffer)
LOG_ERRORF("web.c", "CURL Error: %s", req->curl_err_buffer);
}
curl_multi_cleanup(req->multi);
@@ -104,7 +104,7 @@ subreq_ctx_t *web_post_async(const char *url, char *data, int insecure) {
curl_multi_add_handle(req->multi, curl);
curl_multi_perform(req->multi, &req->running_handles);
LOG_DEBUGF("web.c", "async request POST %s", url)
LOG_DEBUGF("web.c", "async request POST %s", url);
return req;
}
@@ -136,7 +136,7 @@ response_t *web_get(const char *url, int timeout, int insecure) {
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code);
if (resp->status_code == 0) {
LOG_ERRORF("web.c", "CURL Error: %s", err_buffer)
LOG_ERRORF("web.c", "CURL Error: %s", err_buffer);
}
curl_easy_cleanup(curl);
@@ -180,7 +180,7 @@ response_t *web_post(const char *url, const char *data, int insecure) {
resp->size = buffer.cur;
if (resp->status_code == 0) {
LOG_ERRORF("web.c", "CURL Error: %s", err_buffer)
LOG_ERRORF("web.c", "CURL Error: %s", err_buffer);
}
curl_easy_cleanup(curl);