From 1205981a11c79d7ccd089a87b737e260b9869666 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 23 Apr 2022 10:29:31 -0400 Subject: [PATCH] CURL error handling, fix ES version handling, support for ES8, add --es-insecure-ssl argument --- docs/USAGE.md | 2 +- schema/settings.json | 34 +-------------- scripts/start_dev_es_6.sh | 3 ++ scripts/start_dev_es_8.sh | 3 ++ src/cli.c | 2 + src/cli.h | 3 ++ src/ctx.h | 2 + src/index/elastic.c | 80 +++++++++++++++++++++--------------- src/index/elastic.h | 8 ++-- src/index/static_generated.c | 2 +- src/index/web.c | 38 ++++++++++++++--- src/index/web.h | 11 ++--- src/main.c | 15 ++++++- src/web/serve.c | 12 +++--- 14 files changed, 126 insertions(+), 89 deletions(-) create mode 100755 scripts/start_dev_es_6.sh create mode 100755 scripts/start_dev_es_8.sh diff --git a/docs/USAGE.md b/docs/USAGE.md index be38b97..64591a4 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -292,7 +292,7 @@ Both the `root` and `rewrite_url` fields are safe to manually modify from the # Elasticsearch -Elasticsearch versions >=6.8.0, <8.0.0 are supported by sist2. +Elasticsearch versions >=6.8.0, 7.X.X and 8.X.X are supported by sist2. Using a version >=7.14.0 is recommended to enable the following features: diff --git a/schema/settings.json b/schema/settings.json index a6fc000..d197f0d 100644 --- a/schema/settings.json +++ b/schema/settings.json @@ -16,7 +16,7 @@ "delimiter": "." }, "my_nGram_tokenizer": { - "type": "nGram", + "type": "ngram", "min_gram": 3, "max_gram": 3 } @@ -55,37 +55,5 @@ ] } } - }, - "mappings": { - "dynamic_templates": [ - { - "keyword_fields": { - "match_mapping_type": "string", - "match": "kw_*", - "mapping": { - "type": "keyword" - } - } - }, - { - "integer_fields": { - "match_mapping_type": "*", - "match": "int_*", - "mapping": { - "type": "integer" - } - } - }, - { - "meta_fields": { - "match_mapping_type": "*", - "match": "mt_*", - "mapping": { - "type": "keyword", - "index": false - } - } - } - ] } } \ No newline at end of file diff --git a/scripts/start_dev_es_6.sh b/scripts/start_dev_es_6.sh new file mode 100755 index 0000000..69e4f51 --- /dev/null +++ b/scripts/start_dev_es_6.sh @@ -0,0 +1,3 @@ +docker run --rm -it --name "sist2-dev-es-6"\ + -p 9202:9200 -e "discovery.type=single-node" \ + -e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:6.8.0 diff --git a/scripts/start_dev_es_8.sh b/scripts/start_dev_es_8.sh new file mode 100755 index 0000000..0517a0d --- /dev/null +++ b/scripts/start_dev_es_8.sh @@ -0,0 +1,3 @@ +docker run --rm -it --name "sist2-dev-es"\ + -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" \ + -e "ES_JAVA_OPTS=-Xms8g -Xmx8g" elasticsearch:8.1.2 diff --git a/src/cli.c b/src/cli.c index a3b0a3d..a542fa7 100644 --- a/src/cli.c +++ b/src/cli.c @@ -398,6 +398,7 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) { LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url) LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index) + LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl) LOG_DEBUGF("cli.c", "arg index_path=%s", args->index_path) LOG_DEBUGF("cli.c", "arg script_path=%s", args->script_path) LOG_DEBUGF("cli.c", "arg async_script=%d", args->async_script) @@ -512,6 +513,7 @@ int web_args_validate(web_args_t *args, int argc, const char **argv) { LOG_DEBUGF("cli.c", "arg es_url=%s", args->es_url) LOG_DEBUGF("cli.c", "arg es_index=%s", args->es_index) + LOG_DEBUGF("cli.c", "arg es_insecure_ssl=%d", args->es_insecure_ssl) LOG_DEBUGF("cli.c", "arg tagline=%s", args->tagline) LOG_DEBUGF("cli.c", "arg dev=%d", args->dev) LOG_DEBUGF("cli.c", "arg listen=%s", args->listen_address) diff --git a/src/cli.h b/src/cli.h index e830a41..682fbd2 100644 --- a/src/cli.h +++ b/src/cli.h @@ -50,6 +50,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv); typedef struct index_args { char *es_url; char *es_index; + int es_insecure_ssl; char *index_path; const char *script_path; char *script; @@ -68,6 +69,7 @@ typedef struct index_args { typedef struct web_args { char *es_url; char *es_index; + int es_insecure_ssl; char *listen_address; char *credentials; char *tag_credentials; @@ -85,6 +87,7 @@ typedef struct web_args { typedef struct exec_args { char *es_url; char *es_index; + int es_insecure_ssl; char *index_path; const char *script_path; int async_script; diff --git a/src/ctx.h b/src/ctx.h index 6d6b224..4af7544 100644 --- a/src/ctx.h +++ b/src/ctx.h @@ -79,6 +79,7 @@ typedef struct { typedef struct { char *es_url; + int es_insecure_ssl; es_version_t *es_version; char *es_index; int batch_size; @@ -97,6 +98,7 @@ typedef struct { char *es_url; es_version_t *es_version; char *es_index; + int es_insecure_ssl; int index_count; char *auth_user; char *auth_pass; diff --git a/src/index/elastic.c b/src/index/elastic.c index c2a723d..0012767 100644 --- a/src/index/elastic.c +++ b/src/index/elastic.c @@ -53,7 +53,7 @@ void print_json(cJSON *document, const char id_str[SIST_DOC_ID_LEN]) { cJSON_AddStringToObject(line, "_id", id_str); cJSON_AddStringToObject(line, "_index", IndexCtx.es_index); - cJSON_AddStringToObject(line, "_type", "_doc"); +// cJSON_AddStringToObject(line, "_type", "_doc"); cJSON_AddItemReferenceToObject(line, "_source", document); char *json = cJSON_PrintUnformatted(line); @@ -119,7 +119,7 @@ void execute_update_script(const char *script, int async, const char index_id[SI } else { snprintf(url, sizeof(url), "%s/%s/_update_by_query", Indexer->es_url, Indexer->es_index); } - response_t *r = web_post(url, str); + response_t *r = web_post(url, str, IndexCtx.es_insecure_ssl); if (!async) { LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code); } @@ -150,7 +150,7 @@ void execute_update_script(const char *script, int async, const char index_id[SI cJSON_Delete(resp); } -void *create_bulk_buffer(int max, int *count, size_t *buf_len) { +void *create_bulk_buffer(int max, int *count, size_t *buf_len, int legacy) { es_bulk_line_t *line = Indexer->line_head; *count = 0; @@ -171,11 +171,20 @@ void *create_bulk_buffer(int max, int *count, size_t *buf_len) { while (line != NULL && *count < max) { char action_str[256]; if (line->type == ES_BULK_LINE_INDEX) { - snprintf( - action_str, sizeof(action_str), - "{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n", - line->doc_id, Indexer->es_index - ); + + if (legacy) { + snprintf( + action_str, sizeof(action_str), + "{\"index\":{\"_id\":\"%s\",\"_type\":\"_doc\",\"_index\":\"%s\"}}\n", + line->doc_id, Indexer->es_index + ); + } else { + snprintf( + action_str, sizeof(action_str), + "{\"index\":{\"_id\":\"%s\",\"_index\":\"%s\"}}\n", + line->doc_id, Indexer->es_index + ); + } size_t action_str_len = strlen(action_str); size_t line_len = strlen(line->line); @@ -263,11 +272,11 @@ void _elastic_flush(int max) { size_t buf_len; int count; - void *buf = create_bulk_buffer(max, &count, &buf_len); + void *buf = create_bulk_buffer(max, &count, &buf_len, IS_LEGACY_VERSION(IndexCtx.es_version)); char bulk_url[4096]; snprintf(bulk_url, sizeof(bulk_url), "%s/%s/_bulk?pipeline=tie", Indexer->es_url, Indexer->es_index); - response_t *r = web_post(bulk_url, buf); + response_t *r = web_post(bulk_url, buf, IndexCtx.es_insecure_ssl); if (r->status_code == 0) { LOG_FATALF("elastic.c", "Could not connect to %s, make sure that elasticsearch is running!\n", IndexCtx.es_url) @@ -393,7 +402,7 @@ void finish_indexer(char *script, int async_script, char *index_id) { char url[4096]; snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index); - response_t *r = web_post(url, ""); + response_t *r = web_post(url, "", IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code); free_response(r); @@ -402,24 +411,24 @@ void finish_indexer(char *script, int async_script, char *index_id) { free(script); snprintf(url, sizeof(url), "%s/%s/_refresh", IndexCtx.es_url, IndexCtx.es_index); - r = web_post(url, ""); + r = web_post(url, "", IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code); free_response(r); } snprintf(url, sizeof(url), "%s/%s/_forcemerge", IndexCtx.es_url, IndexCtx.es_index); - r = web_post(url, ""); + r = web_post(url, "", IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Merge index <%d>", r->status_code); free_response(r); snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index); - r = web_put(url, "{\"index\":{\"refresh_interval\":\"1s\"}}"); + r = web_put(url, "{\"index\":{\"refresh_interval\":\"1s\"}}", IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Set refresh interval <%d>", r->status_code); free_response(r); } -es_version_t *elastic_get_version(const char *es_url) { - response_t *r = web_get(es_url, 30); +es_version_t *elastic_get_version(const char *es_url, int insecure) { + response_t *r = web_get(es_url, 30, insecure); char *tmp = malloc(r->size + 1); memcpy(tmp, r->body, r->size); @@ -464,7 +473,7 @@ es_version_t *elastic_get_version(const char *es_url) { void elastic_init(int force_reset, const char *user_mappings, const char *user_settings) { - es_version_t *es_version = elastic_get_version(IndexCtx.es_url); + es_version_t *es_version = elastic_get_version(IndexCtx.es_url, IndexCtx.es_insecure_ssl); IndexCtx.es_version = es_version; if (es_version == NULL) { @@ -473,33 +482,33 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s LOG_INFOF("elastic.c", "Elasticsearch version is %s (supported=%d, legacy=%d)", - format_es_version(es_version), IS_SUPPORTED_ES_VERSION(es_version), USE_LEGACY_ES_SETTINGS(es_version)); + format_es_version(es_version), IS_SUPPORTED_ES_VERSION(es_version), IS_LEGACY_VERSION(es_version)); if (!IS_SUPPORTED_ES_VERSION(es_version)) { - LOG_FATAL("elastic.c", "sist2 only supports Elasticsearch v6.8 or newer") + LOG_FATAL("elastic.c", "This elasticsearch version is not supported!") } char *settings = NULL; - if (USE_LEGACY_ES_SETTINGS(es_version)) { - settings = settings_json; - } else { + if (IS_LEGACY_VERSION(es_version)) { settings = settings_legacy_json; + } else { + settings = settings_json; } // Check if index exists char url[4096]; snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index); - response_t *r = web_get(url, 30); + response_t *r = web_get(url, 30, IndexCtx.es_insecure_ssl); int index_exists = r->status_code == 200; free_response(r); if (!index_exists || force_reset) { - r = web_delete(url); + r = web_delete(url, IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Delete index <%d>", r->status_code); free_response(r); snprintf(url, sizeof(url), "%s/%s", IndexCtx.es_url, IndexCtx.es_index); - r = web_put(url, ""); + r = web_put(url, "", IndexCtx.es_insecure_ssl); if (r->status_code != 200) { print_error(r); @@ -510,17 +519,17 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s free_response(r); snprintf(url, sizeof(url), "%s/%s/_close", IndexCtx.es_url, IndexCtx.es_index); - r = web_post(url, ""); + r = web_post(url, "", IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Close index <%d>", r->status_code); free_response(r); snprintf(url, sizeof(url), "%s/_ingest/pipeline/tie", IndexCtx.es_url); - r = web_put(url, pipeline_json); + r = web_put(url, pipeline_json, IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Create pipeline <%d>", r->status_code); free_response(r); snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index); - r = web_put(url, user_settings ? user_settings : settings); + r = web_put(url, user_settings ? user_settings : settings, IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Update ES settings <%d>", r->status_code); if (r->status_code != 200) { print_error(r); @@ -528,8 +537,13 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s } free_response(r); - snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index); - r = web_put(url, user_mappings ? user_mappings : mappings_json); + if (IS_LEGACY_VERSION(es_version)) { + snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index); + } else { + snprintf(url, sizeof(url), "%s/%s/_mappings", IndexCtx.es_url, IndexCtx.es_index); + } + + r = web_put(url, user_mappings ? user_mappings : mappings_json, IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Update ES mappings <%d>", r->status_code); if (r->status_code != 200) { print_error(r); @@ -538,7 +552,7 @@ void elastic_init(int force_reset, const char *user_mappings, const char *user_s free_response(r); snprintf(url, sizeof(url), "%s/%s/_open", IndexCtx.es_url, IndexCtx.es_index); - r = web_post(url, ""); + r = web_post(url, "", IndexCtx.es_insecure_ssl); LOG_INFOF("elastic.c", "Open index <%d>", r->status_code); free_response(r); } @@ -548,7 +562,7 @@ cJSON *elastic_get_document(const char *id_str) { char url[4096]; snprintf(url, sizeof(url), "%s/%s/_doc/%s", WebCtx.es_url, WebCtx.es_index, id_str); - response_t *r = web_get(url, 3); + response_t *r = web_get(url, 3, WebCtx.es_insecure_ssl); cJSON *json = NULL; if (r->status_code == 200) { char *tmp = malloc(r->size + 1); @@ -566,7 +580,7 @@ char *elastic_get_status() { snprintf(url, sizeof(url), "%s/_cluster/state/metadata/%s?filter_path=metadata.indices.*.state", WebCtx.es_url, WebCtx.es_index); - response_t *r = web_get(url, 30); + response_t *r = web_get(url, 30, IndexCtx.es_insecure_ssl); cJSON *json = NULL; char *status = malloc(128 * sizeof(char)); status[0] = '\0'; diff --git a/src/index/elastic.h b/src/index/elastic.h index c7189ce..319fe71 100644 --- a/src/index/elastic.h +++ b/src/index/elastic.h @@ -20,8 +20,10 @@ typedef struct { } es_version_t; #define VERSION_GE(version, maj, min) ((version)->major > (maj) || ((version)->major == (maj) && (version)->minor >= (min))) -#define IS_SUPPORTED_ES_VERSION(es_version) ((es_version) != NULL && VERSION_GE((es_version), 6, 8)) -#define USE_LEGACY_ES_SETTINGS(es_version) ((es_version) != NULL && (!VERSION_GE((es_version), 7, 14))) +#define VERSION_LT(version, maj, min) (!VERSION_GE(version, maj, min)) + +#define IS_SUPPORTED_ES_VERSION(es_version) ((es_version) != NULL && VERSION_GE((es_version), 6, 8) && VERSION_LT((es_version), 9, 0)) +#define IS_LEGACY_VERSION(es_version) ((es_version) != NULL && VERSION_LT((es_version), 7, 14)) __always_inline static const char *format_es_version(es_version_t *version) { @@ -57,7 +59,7 @@ cJSON *elastic_get_document(const char *id_str); char *elastic_get_status(); -es_version_t *elastic_get_version(const char *es_url); +es_version_t *elastic_get_version(const char *es_url, int insecure); void execute_update_script(const char *script, int async, const char index_id[SIST_INDEX_ID_LEN]); diff --git a/src/index/static_generated.c b/src/index/static_generated.c index 999c153..2faec91 100644 --- a/src/index/static_generated.c +++ b/src/index/static_generated.c @@ -1,4 +1,4 @@ char mappings_json[2444] = {123,34,112,114,111,112,101,114,116,105,101,115,34,58,123,34,95,116,105,101,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,100,111,99,95,118,97,108,117,101,115,34,58,116,114,117,101,125,44,34,99,104,101,99,107,115,117,109,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,95,100,101,112,116,104,34,58,123,34,116,121,112,101,34,58,34,105,110,116,101,103,101,114,34,125,44,34,112,97,116,104,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,97,110,97,108,121,122,101,114,34,58,34,112,97,116,104,95,97,110,97,108,121,122,101,114,34,44,34,99,111,112,121,95,116,111,34,58,34,115,117,103,103,101,115,116,45,112,97,116,104,34,44,34,102,105,101,108,100,100,97,116,97,34,58,116,114,117,101,44,34,102,105,101,108,100,115,34,58,123,34,110,71,114,97,109,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,125,44,34,116,101,120,116,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,97,110,97,108,121,122,101,114,34,58,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,125,125,125,44,34,115,117,103,103,101,115,116,45,112,97,116,104,34,58,123,34,116,121,112,101,34,58,34,99,111,109,112,108,101,116,105,111,110,34,44,34,97,110,97,108,121,122,101,114,34,58,34,99,97,115,101,95,105,110,115,101,110,115,105,116,105,118,101,95,107,119,95,97,110,97,108,121,122,101,114,34,125,44,34,109,105,109,101,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,125,44,34,112,97,114,101,110,116,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,116,104,117,109,98,110,97,105,108,34,58,123,34,116,121,112,101,34,58,34,105,110,116,101,103,101,114,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,118,105,100,101,111,99,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,97,117,100,105,111,99,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,100,117,114,97,116,105,111,110,34,58,123,34,116,121,112,101,34,58,34,105,110,116,101,103,101,114,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,119,105,100,116,104,34,58,123,34,116,121,112,101,34,58,34,105,110,116,101,103,101,114,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,104,101,105,103,104,116,34,58,123,34,116,121,112,101,34,58,34,105,110,116,101,103,101,114,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,112,97,103,101,115,34,58,123,34,116,121,112,101,34,58,34,105,110,116,101,103,101,114,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,109,116,105,109,101,34,58,123,34,116,121,112,101,34,58,34,105,110,116,101,103,101,114,34,125,44,34,115,105,122,101,34,58,123,34,116,121,112,101,34,58,34,108,111,110,103,34,125,44,34,105,110,100,101,120,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,125,44,34,110,97,109,101,34,58,123,34,97,110,97,108,121,122,101,114,34,58,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,44,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,102,105,101,108,100,100,97,116,97,34,58,116,114,117,101,44,34,102,105,101,108,100,115,34,58,123,34,110,71,114,97,109,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,125,125,125,44,34,102,111,110,116,95,110,97,109,101,34,58,123,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,44,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,97,108,98,117,109,34,58,123,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,44,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,97,114,116,105,115,116,34,58,123,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,44,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,116,105,116,108,101,34,58,123,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,44,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,103,101,110,114,101,34,58,123,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,44,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,97,108,98,117,109,95,97,114,116,105,115,116,34,58,123,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,44,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,95,107,101,121,119,111,114,100,46,42,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,125,44,34,95,116,101,120,116,46,42,34,58,123,34,97,110,97,108,121,122,101,114,34,58,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,44,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,102,105,101,108,100,115,34,58,123,34,110,71,114,97,109,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,125,125,125,44,34,95,117,114,108,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,99,111,110,116,101,110,116,34,58,123,34,97,110,97,108,121,122,101,114,34,58,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,44,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,105,110,100,101,120,95,111,112,116,105,111,110,115,34,58,34,111,102,102,115,101,116,115,34,44,34,102,105,101,108,100,115,34,58,123,34,110,71,114,97,109,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,97,110,97,108,121,122,101,114,34,58,34,109,121,95,110,71,114,97,109,34,125,125,125,44,34,116,97,103,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,44,34,102,105,101,108,100,100,97,116,97,34,58,116,114,117,101,44,34,97,110,97,108,121,122,101,114,34,58,34,116,97,103,95,97,110,97,108,121,122,101,114,34,44,34,99,111,112,121,95,116,111,34,58,34,115,117,103,103,101,115,116,45,116,97,103,34,125,44,34,115,117,103,103,101,115,116,45,116,97,103,34,58,123,34,116,121,112,101,34,58,34,99,111,109,112,108,101,116,105,111,110,34,44,34,97,110,97,108,121,122,101,114,34,58,34,99,97,115,101,95,105,110,115,101,110,115,105,116,105,118,101,95,107,119,95,97,110,97,108,121,122,101,114,34,125,44,34,101,120,105,102,95,109,97,107,101,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,101,120,105,102,95,109,111,100,101,108,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,101,120,105,102,58,115,111,102,116,119,97,114,101,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,101,120,105,102,95,101,120,112,111,115,117,114,101,95,116,105,109,101,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,125,44,34,101,120,105,102,95,102,110,117,109,98,101,114,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,125,44,34,101,120,105,102,95,105,115,111,95,115,112,101,101,100,95,114,97,116,105,110,103,115,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,125,44,34,101,120,105,102,95,102,111,99,97,108,95,108,101,110,103,116,104,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,125,44,34,101,120,105,102,95,117,115,101,114,95,99,111,109,109,101,110,116,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,101,120,105,102,95,103,112,115,95,108,111,110,103,105,116,117,100,101,95,114,101,102,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,101,120,105,102,95,103,112,115,95,108,111,110,103,105,116,117,100,101,95,100,109,115,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,101,120,105,102,95,103,112,115,95,108,111,110,103,105,116,117,100,101,95,100,101,99,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,101,120,105,102,95,103,112,115,95,108,97,116,105,116,117,100,101,95,114,101,102,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,101,120,105,102,95,103,112,115,95,108,97,116,105,116,117,100,101,95,100,109,115,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,101,120,105,102,95,103,112,115,95,108,97,116,105,116,117,100,101,95,100,101,99,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,44,34,97,117,116,104,111,114,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,125,44,34,109,111,100,105,102,105,101,100,95,98,121,34,58,123,34,116,121,112,101,34,58,34,116,101,120,116,34,125,125,125,0}; -char settings_json[1052] = {123,34,105,110,100,101,120,34,58,123,34,114,101,102,114,101,115,104,95,105,110,116,101,114,118,97,108,34,58,34,51,48,115,34,44,34,99,111,100,101,99,34,58,34,98,101,115,116,95,99,111,109,112,114,101,115,115,105,111,110,34,44,34,110,117,109,98,101,114,95,111,102,95,114,101,112,108,105,99,97,115,34,58,48,44,34,104,105,103,104,108,105,103,104,116,46,109,97,120,95,97,110,97,108,121,122,101,100,95,111,102,102,115,101,116,34,58,49,48,48,48,48,48,48,125,44,34,97,110,97,108,121,115,105,115,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,123,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,58,123,34,116,121,112,101,34,58,34,112,97,116,104,95,104,105,101,114,97,114,99,104,121,34,44,34,100,101,108,105,109,105,116,101,114,34,58,34,47,34,125,44,34,116,97,103,95,116,111,107,101,110,105,122,101,114,34,58,123,34,116,121,112,101,34,58,34,112,97,116,104,95,104,105,101,114,97,114,99,104,121,34,44,34,100,101,108,105,109,105,116,101,114,34,58,34,46,34,125,44,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,58,123,34,116,121,112,101,34,58,34,110,71,114,97,109,34,44,34,109,105,110,95,103,114,97,109,34,58,51,44,34,109,97,120,95,103,114,97,109,34,58,51,125,125,44,34,97,110,97,108,121,122,101,114,34,58,123,34,112,97,116,104,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,93,125,44,34,116,97,103,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,116,97,103,95,116,111,107,101,110,105,122,101,114,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,93,125,44,34,99,97,115,101,95,105,110,115,101,110,115,105,116,105,118,101,95,107,119,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,107,101,121,119,111,114,100,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,93,125,44,34,109,121,95,110,71,114,97,109,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,44,34,97,115,99,105,105,102,111,108,100,105,110,103,34,93,125,44,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,115,116,97,110,100,97,114,100,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,44,34,97,115,99,105,105,102,111,108,100,105,110,103,34,93,125,125,125,44,34,109,97,112,112,105,110,103,115,34,58,123,34,100,121,110,97,109,105,99,95,116,101,109,112,108,97,116,101,115,34,58,91,123,34,107,101,121,119,111,114,100,95,102,105,101,108,100,115,34,58,123,34,109,97,116,99,104,95,109,97,112,112,105,110,103,95,116,121,112,101,34,58,34,115,116,114,105,110,103,34,44,34,109,97,116,99,104,34,58,34,107,119,95,42,34,44,34,109,97,112,112,105,110,103,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,125,125,125,44,123,34,105,110,116,101,103,101,114,95,102,105,101,108,100,115,34,58,123,34,109,97,116,99,104,95,109,97,112,112,105,110,103,95,116,121,112,101,34,58,34,42,34,44,34,109,97,116,99,104,34,58,34,105,110,116,95,42,34,44,34,109,97,112,112,105,110,103,34,58,123,34,116,121,112,101,34,58,34,105,110,116,101,103,101,114,34,125,125,125,44,123,34,109,101,116,97,95,102,105,101,108,100,115,34,58,123,34,109,97,116,99,104,95,109,97,112,112,105,110,103,95,116,121,112,101,34,58,34,42,34,44,34,109,97,116,99,104,34,58,34,109,116,95,42,34,44,34,109,97,112,112,105,110,103,34,58,123,34,116,121,112,101,34,58,34,107,101,121,119,111,114,100,34,44,34,105,110,100,101,120,34,58,102,97,108,115,101,125,125,125,93,125,125,0}; +char settings_json[730] = {123,34,105,110,100,101,120,34,58,123,34,114,101,102,114,101,115,104,95,105,110,116,101,114,118,97,108,34,58,34,51,48,115,34,44,34,99,111,100,101,99,34,58,34,98,101,115,116,95,99,111,109,112,114,101,115,115,105,111,110,34,44,34,110,117,109,98,101,114,95,111,102,95,114,101,112,108,105,99,97,115,34,58,48,44,34,104,105,103,104,108,105,103,104,116,46,109,97,120,95,97,110,97,108,121,122,101,100,95,111,102,102,115,101,116,34,58,49,48,48,48,48,48,48,125,44,34,97,110,97,108,121,115,105,115,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,123,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,58,123,34,116,121,112,101,34,58,34,112,97,116,104,95,104,105,101,114,97,114,99,104,121,34,44,34,100,101,108,105,109,105,116,101,114,34,58,34,47,34,125,44,34,116,97,103,95,116,111,107,101,110,105,122,101,114,34,58,123,34,116,121,112,101,34,58,34,112,97,116,104,95,104,105,101,114,97,114,99,104,121,34,44,34,100,101,108,105,109,105,116,101,114,34,58,34,46,34,125,44,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,58,123,34,116,121,112,101,34,58,34,110,103,114,97,109,34,44,34,109,105,110,95,103,114,97,109,34,58,51,44,34,109,97,120,95,103,114,97,109,34,58,51,125,125,44,34,97,110,97,108,121,122,101,114,34,58,123,34,112,97,116,104,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,93,125,44,34,116,97,103,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,116,97,103,95,116,111,107,101,110,105,122,101,114,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,93,125,44,34,99,97,115,101,95,105,110,115,101,110,115,105,116,105,118,101,95,107,119,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,107,101,121,119,111,114,100,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,93,125,44,34,109,121,95,110,71,114,97,109,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,44,34,97,115,99,105,105,102,111,108,100,105,110,103,34,93,125,44,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,115,116,97,110,100,97,114,100,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,44,34,97,115,99,105,105,102,111,108,100,105,110,103,34,93,125,125,125,125,0}; char settings_legacy_json[690] = {123,34,105,110,100,101,120,34,58,123,34,114,101,102,114,101,115,104,95,105,110,116,101,114,118,97,108,34,58,34,51,48,115,34,44,34,99,111,100,101,99,34,58,34,98,101,115,116,95,99,111,109,112,114,101,115,115,105,111,110,34,44,34,110,117,109,98,101,114,95,111,102,95,114,101,112,108,105,99,97,115,34,58,48,125,44,34,97,110,97,108,121,115,105,115,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,123,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,58,123,34,116,121,112,101,34,58,34,112,97,116,104,95,104,105,101,114,97,114,99,104,121,34,44,34,100,101,108,105,109,105,116,101,114,34,58,34,47,34,125,44,34,116,97,103,95,116,111,107,101,110,105,122,101,114,34,58,123,34,116,121,112,101,34,58,34,112,97,116,104,95,104,105,101,114,97,114,99,104,121,34,44,34,100,101,108,105,109,105,116,101,114,34,58,34,46,34,125,44,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,58,123,34,116,121,112,101,34,58,34,110,71,114,97,109,34,44,34,109,105,110,95,103,114,97,109,34,58,51,44,34,109,97,120,95,103,114,97,109,34,58,51,125,125,44,34,97,110,97,108,121,122,101,114,34,58,123,34,112,97,116,104,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,93,125,44,34,116,97,103,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,116,97,103,95,116,111,107,101,110,105,122,101,114,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,93,125,44,34,99,97,115,101,95,105,110,115,101,110,115,105,116,105,118,101,95,107,119,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,107,101,121,119,111,114,100,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,93,125,44,34,109,121,95,110,71,114,97,109,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,44,34,97,115,99,105,105,102,111,108,100,105,110,103,34,93,125,44,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,58,123,34,116,111,107,101,110,105,122,101,114,34,58,34,115,116,97,110,100,97,114,100,34,44,34,102,105,108,116,101,114,34,58,91,34,108,111,119,101,114,99,97,115,101,34,44,34,97,115,99,105,105,102,111,108,100,105,110,103,34,93,125,125,125,125,0}; char pipeline_json[217] = {123,34,100,101,115,99,114,105,112,116,105,111,110,34,58,34,67,111,112,121,32,95,105,100,32,116,111,32,95,116,105,101,44,32,115,97,118,101,32,112,97,116,104,32,100,101,112,116,104,34,44,34,112,114,111,99,101,115,115,111,114,115,34,58,91,123,34,115,99,114,105,112,116,34,58,123,34,115,111,117,114,99,101,34,58,34,99,116,120,46,95,116,105,101,32,61,32,99,116,120,46,95,105,100,59,32,99,116,120,46,95,100,101,112,116,104,32,61,32,99,116,120,46,112,97,116,104,46,108,101,110,103,116,104,40,41,32,61,61,32,48,32,63,32,48,32,58,32,49,32,43,32,99,116,120,46,112,97,116,104,46,108,101,110,103,116,104,40,41,32,45,32,99,116,120,46,112,97,116,104,46,114,101,112,108,97,99,101,40,92,34,47,92,34,44,32,92,34,92,34,41,46,108,101,110,103,116,104,40,41,59,34,125,125,93,125,0}; diff --git a/src/index/web.c b/src/index/web.c index b9aa7a2..f608da1 100644 --- a/src/index/web.c +++ b/src/index/web.c @@ -64,6 +64,10 @@ void web_post_async_poll(subreq_ctx_t *req) { req->response->size = req->response_buf.cur; curl_easy_getinfo(req->handle, CURLINFO_RESPONSE_CODE, &req->response->status_code); + if (req->response->status_code == 0) { + LOG_ERRORF("web.c", "CURL Error: %s", req->curl_err_buffer) + } + curl_multi_cleanup(req->multi); curl_easy_cleanup(req->handle); curl_slist_free_all(req->headers); @@ -71,7 +75,7 @@ void web_post_async_poll(subreq_ctx_t *req) { } } -subreq_ctx_t *web_post_async(const char *url, char *data) { +subreq_ctx_t *web_post_async(const char *url, char *data, int insecure) { subreq_ctx_t *req = calloc(1, sizeof(subreq_ctx_t)); req->response = calloc(1, sizeof(response_t)); req->data = data; @@ -84,6 +88,11 @@ subreq_ctx_t *web_post_async(const char *url, char *data) { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb); curl_easy_setopt(curl, CURLOPT_POST, 1); curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2"); + if (insecure) { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + } + + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, req->curl_err_buffer); struct curl_slist *headers = NULL; headers = curl_slist_append(headers, "Content-Type: application/json"); @@ -100,7 +109,7 @@ subreq_ctx_t *web_post_async(const char *url, char *data) { return req; } -response_t *web_get(const char *url, int timeout) { +response_t *web_get(const char *url, int timeout, int insecure) { response_t *resp = malloc(sizeof(response_t)); CURL *curl; @@ -112,14 +121,24 @@ response_t *web_get(const char *url, int timeout) { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb); curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2"); curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); + if (insecure) { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + } struct curl_slist *headers = NULL; headers = curl_slist_append(headers, "Content-Type: application/json"); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + char err_buffer[CURL_ERROR_SIZE + 1] = {}; + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer); + curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp->status_code); + if (resp->status_code == 0) { + LOG_ERRORF("web.c", "CURL Error: %s", err_buffer) + } + curl_easy_cleanup(curl); curl_slist_free_all(headers); @@ -128,7 +147,7 @@ response_t *web_get(const char *url, int timeout) { return resp; } -response_t *web_post(const char *url, const char *data) { +response_t *web_post(const char *url, const char *data, int insecure) { response_t *resp = malloc(sizeof(response_t)); @@ -141,6 +160,9 @@ response_t *web_post(const char *url, const char *data) { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb); curl_easy_setopt(curl, CURLOPT_POST, 1); curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2"); + if (insecure) { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + } char err_buffer[CURL_ERROR_SIZE + 1] = {}; curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err_buffer); @@ -168,7 +190,7 @@ response_t *web_post(const char *url, const char *data) { } -response_t *web_put(const char *url, const char *data) { +response_t *web_put(const char *url, const char *data, int insecure) { response_t *resp = malloc(sizeof(response_t)); @@ -183,6 +205,9 @@ response_t *web_put(const char *url, const char *data) { curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2"); curl_easy_setopt(curl, CURLOPT_DNS_USE_GLOBAL_CACHE, 0); curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURLOPT_DNS_LOCAL_IP4); + if (insecure) { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + } struct curl_slist *headers = NULL; headers = curl_slist_append(headers, "Content-Type: application/json"); @@ -201,7 +226,7 @@ response_t *web_put(const char *url, const char *data) { return resp; } -response_t *web_delete(const char *url) { +response_t *web_delete(const char *url, int insecure) { response_t *resp = malloc(sizeof(response_t)); @@ -214,6 +239,9 @@ response_t *web_delete(const char *url) { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb); curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(curl, CURLOPT_USERAGENT, "sist2"); + if (insecure) { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + } curl_easy_setopt(curl, CURLOPT_POSTFIELDS, ""); struct curl_slist *headers = NULL; diff --git a/src/index/web.h b/src/index/web.h index 7f29bc2..d4a64cf 100644 --- a/src/index/web.h +++ b/src/index/web.h @@ -25,14 +25,15 @@ typedef struct { response_t *response; int running_handles; int done; + char curl_err_buffer[CURL_ERROR_SIZE + 1]; } subreq_ctx_t; -response_t *web_get(const char *url, int timeout); -response_t *web_post(const char * url, const char * data); +response_t *web_get(const char *url, int timeout, int insecure); +response_t *web_post(const char * url, const char * data, int insecure); void web_post_async_poll(subreq_ctx_t* req); -subreq_ctx_t *web_post_async(const char *url, char *data); -response_t *web_put(const char *url, const char *data); -response_t *web_delete(const char *url); +subreq_ctx_t *web_post_async(const char *url, char *data, int insecure); +response_t *web_put(const char *url, const char *data, int insecure); +response_t *web_delete(const char *url, int insecure); void free_response(response_t *resp); diff --git a/src/main.c b/src/main.c index c95ccb1..c6124da 100644 --- a/src/main.c +++ b/src/main.c @@ -435,8 +435,8 @@ void sist2_scan(scan_args_t *args) { LOG_DEBUGF("main.c", "Skipped files: %d", ScanCtx.dbg_skipped_files_count) LOG_DEBUGF("main.c", "Excluded files: %d", ScanCtx.dbg_excluded_files_count) LOG_DEBUGF("main.c", "Failed files: %d", ScanCtx.dbg_failed_files_count) - LOG_DEBUGF("main.c", "Thumbnail store size: %d", ScanCtx.stat_tn_size) - LOG_DEBUGF("main.c", "Index size: %d", ScanCtx.stat_index_size) + LOG_DEBUGF("main.c", "Thumbnail store size: %lu", ScanCtx.stat_tn_size) + LOG_DEBUGF("main.c", "Index size: %lu", ScanCtx.stat_index_size) if (args->incremental != NULL) { save_incremental_index(args); @@ -453,6 +453,7 @@ void sist2_index(index_args_t *args) { IndexCtx.es_url = args->es_url; IndexCtx.es_index = args->es_index; + IndexCtx.es_insecure_ssl = args->es_insecure_ssl; IndexCtx.batch_size = args->batch_size; IndexCtx.needs_es_connection = !args->print; @@ -538,6 +539,7 @@ void sist2_exec_script(exec_args_t *args) { IndexCtx.es_url = args->es_url; IndexCtx.es_index = args->es_index; + IndexCtx.es_insecure_ssl = args->es_insecure_ssl; IndexCtx.needs_es_connection = TRUE; LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type) @@ -550,6 +552,7 @@ void sist2_web(web_args_t *args) { WebCtx.es_url = args->es_url; WebCtx.es_index = args->es_index; + WebCtx.es_insecure_ssl = args->es_insecure_ssl; WebCtx.index_count = args->index_count; WebCtx.auth_user = args->auth_user; WebCtx.auth_pass = args->auth_pass; @@ -620,6 +623,7 @@ int main(int argc, const char *argv[]) { int arg_version = 0; char *common_es_url = NULL; + int common_es_insecure_ssl = 0; char *common_es_index = NULL; char *common_script_path = NULL; int common_async_script = 0; @@ -685,6 +689,7 @@ int main(int argc, const char *argv[]) { OPT_GROUP("Index options"), OPT_INTEGER('t', "threads", &common_threads, "Number of threads. DEFAULT=1"), OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"), + OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."), OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."), OPT_BOOLEAN(0, "incremental-index", &index_args->incremental, @@ -699,6 +704,7 @@ int main(int argc, const char *argv[]) { OPT_GROUP("Web options"), OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"), + OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."), OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), OPT_STRING(0, "bind", &web_args->listen_address, "Listen on this address. DEFAULT=localhost:4090"), OPT_STRING(0, "auth", &web_args->credentials, "Basic auth in user:password format"), @@ -709,6 +715,7 @@ int main(int argc, const char *argv[]) { OPT_GROUP("Exec-script options"), OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url. DEFAULT=http://localhost:9200"), + OPT_BOOLEAN(0, "es-insecure-ssl", &common_es_insecure_ssl, "Do not verify SSL connections to Elasticsearch."), OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), OPT_STRING(0, "script-file", &common_script_path, "Path to user script."), OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."), @@ -738,6 +745,10 @@ int main(int argc, const char *argv[]) { index_args->es_index = common_es_index; exec_args->es_index = common_es_index; + web_args->es_insecure_ssl = common_es_insecure_ssl; + index_args->es_insecure_ssl = common_es_insecure_ssl; + exec_args->es_insecure_ssl = common_es_insecure_ssl; + index_args->script_path = common_script_path; exec_args->script_path = common_script_path; index_args->threads = common_threads; diff --git a/src/web/serve.c b/src/web/serve.c index 2abef29..69d4a01 100644 --- a/src/web/serve.c +++ b/src/web/serve.c @@ -212,7 +212,7 @@ void search(struct mg_connection *nc, struct mg_http_message *hm) { if (hm->body.len == 0) { LOG_DEBUG("serve.c", "Client sent empty body, ignoring request") - mg_http_reply(nc, 500, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Invalid request"); + mg_http_reply(nc, 400, HTTP_SERVER_HEADER HTTP_TEXT_TYPE_HEADER, "Invalid request"); return; } @@ -223,7 +223,7 @@ void search(struct mg_connection *nc, struct mg_http_message *hm) { char url[4096]; snprintf(url, 4096, "%s/%s/_search", WebCtx.es_url, WebCtx.es_index); - nc->fn_data = web_post_async(url, body); + nc->fn_data = web_post_async(url, body, WebCtx.es_insecure_ssl); } void serve_file_from_url(cJSON *json, index_t *idx, struct mg_connection *nc) { @@ -302,7 +302,7 @@ void cache_es_version() { return; } - es_version_t *es_version = elastic_get_version(WebCtx.es_url); + es_version_t *es_version = elastic_get_version(WebCtx.es_url, WebCtx.es_insecure_ssl); if (es_version != NULL) { WebCtx.es_version = es_version; is_cached = TRUE; @@ -326,7 +326,7 @@ void index_info(struct mg_connection *nc) { cJSON_AddStringToObject(json, "version", Version); cJSON_AddStringToObject(json, "esVersion", es_version); cJSON_AddBoolToObject(json, "esVersionSupported", IS_SUPPORTED_ES_VERSION(WebCtx.es_version)); - cJSON_AddBoolToObject(json, "esVersionLegacy", USE_LEGACY_ES_SETTINGS(WebCtx.es_version)); + cJSON_AddBoolToObject(json, "esVersionLegacy", IS_LEGACY_VERSION(WebCtx.es_version)); cJSON_AddStringToObject(json, "platform", QUOTE(SIST_PLATFORM)); cJSON_AddStringToObject(json, "sist2Hash", Sist2CommitHash); cJSON_AddStringToObject(json, "lang", WebCtx.lang); @@ -531,7 +531,7 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) { char url[4096]; snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id); - nc->fn_data = web_post_async(url, buf); + nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl); } else { cJSON_AddItemToArray(arr, cJSON_CreateString(arg_req->name)); @@ -551,7 +551,7 @@ void tag(struct mg_connection *nc, struct mg_http_message *hm) { char url[4096]; snprintf(url, sizeof(url), "%s/%s/_update/%s", WebCtx.es_url, WebCtx.es_index, arg_req->doc_id); - nc->fn_data = web_post_async(url, buf); + nc->fn_data = web_post_async(url, buf, WebCtx.es_insecure_ssl); } char *json_str = cJSON_PrintUnformatted(arr);