From b4199a1fd8f9983e92f28507c0ce773aee7031cf Mon Sep 17 00:00:00 2001 From: simon Date: Sat, 26 Oct 2019 19:05:47 -0400 Subject: [PATCH] fix for elasticsearch --- schema/settings.json | 8 +++++--- src/index/elastic.c | 13 +++++++++++-- src/index/static_generated.c | 2 +- src/io/serialize.c | 4 +++- src/main.c | 2 +- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/schema/settings.json b/schema/settings.json index 04284e3..45367dd 100644 --- a/schema/settings.json +++ b/schema/settings.json @@ -1,6 +1,6 @@ { "index": { - "refresh_interval": "-1", + "refresh_interval": "30s", "codec": "best_compression" }, "analysis": { @@ -24,13 +24,15 @@ "my_nGram": { "tokenizer": "my_nGram_tokenizer", "filter": [ - "lowercase" + "lowercase", + "asciifolding" ] }, "content_analyzer": { "tokenizer": "standard", "filter": [ - "lowercase" + "lowercase", + "asciifolding" ] } } diff --git a/src/index/elastic.c b/src/index/elastic.c index 01e2b5c..7f13096 100644 --- a/src/index/elastic.c +++ b/src/index/elastic.c @@ -70,7 +70,8 @@ void elastic_flush() { while (line != NULL) { char action_str[512]; - snprintf(action_str, 512, "{\"index\":{\"_id\": \"%s\", \"_type\":\"_doc\"}}", line->uuid_str); + snprintf(action_str, 512, + "{\"index\":{\"_id\":\"%s\", \"_type\":\"_doc\", \"_index\":\"sist2\"}}\n", line->uuid_str); size_t action_str_len = strlen(action_str); size_t line_len = strlen(line->line); @@ -98,6 +99,14 @@ void elastic_flush() { snprintf(bulk_url, 4096, "%s/sist2/_bulk", Indexer->es_url); response_t *r = web_post(bulk_url, buf, "Content-Type: application/x-ndjson"); printf("Indexed %3d documents (%zukB) <%d>\n", count, buf_cur / 1024, r->status_code); + + cJSON *ret_json = cJSON_Parse(r->body); + if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) { + fprintf(stderr, "%s\n", r->body); + } + + cJSON_Delete(ret_json); + free_response(r); } @@ -187,7 +196,7 @@ void elastic_init(int force_reset) { printf("Update settings <%d>\n", r->status_code); free_response(r); - snprintf(url, 4096, "%s/sist2/_mappings/_doc", IndexCtx.es_url); + snprintf(url, 4096, "%s/sist2/_mappings/_doc?include_type_name=true", IndexCtx.es_url); r = web_put(url, mappings_json, "Content-Type: application/json"); printf("Update mappings <%d>\n", r->status_code); free_response(r); diff --git a/src/index/static_generated.c b/src/index/static_generated.c index 230d140..fd32132 100644 --- a/src/index/static_generated.c +++ b/src/index/static_generated.c @@ -1,2 +1,2 @@ char mappings_json[1510] = {123,10,32,32,34,112,114,111,112,101,114,116,105,101,115,34,58,32,123,10,32,32,32,32,34,112,97,116,104,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,44,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,112,97,116,104,95,97,110,97,108,121,122,101,114,34,44,10,32,32,32,32,32,32,34,99,111,112,121,95,116,111,34,58,32,34,115,117,103,103,101,115,116,45,112,97,116,104,34,10,32,32,32,32,125,44,10,32,32,32,32,34,115,117,103,103,101,115,116,45,112,97,116,104,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,99,111,109,112,108,101,116,105,111,110,34,44,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,107,101,121,119,111,114,100,34,10,32,32,32,32,125,44,10,32,32,32,32,34,109,105,109,101,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,107,101,121,119,111,114,100,34,10,32,32,32,32,125,44,10,32,32,32,32,34,118,105,100,101,111,99,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,107,101,121,119,111,114,100,34,10,32,32,32,32,125,44,10,32,32,32,32,34,97,117,100,105,111,99,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,107,101,121,119,111,114,100,34,10,32,32,32,32,125,44,10,32,32,32,32,34,100,117,114,97,116,105,111,110,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,102,108,111,97,116,34,10,32,32,32,32,125,44,10,32,32,32,32,34,119,105,100,116,104,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,105,110,116,101,103,101,114,34,10,32,32,32,32,125,44,10,32,32,32,32,34,104,101,105,103,104,116,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,105,110,116,101,103,101,114,34,10,32,32,32,32,125,44,10,32,32,32,32,34,109,116,105,109,101,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,105,110,116,101,103,101,114,34,10,32,32,32,32,125,44,10,32,32,32,32,34,115,105,122,101,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,108,111,110,103,34,10,32,32,32,32,125,44,10,32,32,32,32,34,105,110,100,101,120,34,58,32,123,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,107,101,121,119,111,114,100,34,10,32,32,32,32,125,44,10,32,32,32,32,34,110,97,109,101,34,58,32,123,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,44,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,44,10,32,32,32,32,32,32,34,102,105,101,108,100,115,34,58,32,123,10,32,32,32,32,32,32,32,32,34,110,71,114,97,109,34,58,32,123,10,32,32,32,32,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,44,10,32,32,32,32,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,34,10,32,32,32,32,32,32,32,32,125,10,32,32,32,32,32,32,125,10,32,32,32,32,125,44,10,32,32,32,32,34,102,111,110,116,95,110,97,109,101,34,58,32,123,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,34,44,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,10,32,32,32,32,125,44,10,32,32,32,32,34,97,108,98,117,109,34,58,32,123,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,34,44,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,10,32,32,32,32,125,44,10,32,32,32,32,34,97,114,116,105,115,116,34,58,32,123,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,34,44,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,10,32,32,32,32,125,44,10,32,32,32,32,34,116,105,116,108,101,34,58,32,123,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,34,44,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,10,32,32,32,32,125,44,10,32,32,32,32,34,103,101,110,114,101,34,58,32,123,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,34,44,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,10,32,32,32,32,125,44,10,32,32,32,32,34,97,108,98,117,109,95,97,114,116,105,115,116,34,58,32,123,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,34,44,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,10,32,32,32,32,125,44,10,32,32,32,32,34,99,111,110,116,101,110,116,34,58,32,123,10,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,44,10,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,44,10,32,32,32,32,32,32,34,105,110,100,101,120,95,111,112,116,105,111,110,115,34,58,32,34,111,102,102,115,101,116,115,34,44,10,32,32,32,32,32,32,34,102,105,101,108,100,115,34,58,32,123,10,32,32,32,32,32,32,32,32,34,110,71,114,97,109,34,58,32,123,10,32,32,32,32,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,116,101,120,116,34,44,10,32,32,32,32,32,32,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,34,10,32,32,32,32,32,32,32,32,125,10,32,32,32,32,32,32,125,10,32,32,32,32,125,10,32,32,125,10,125,10}; -char settings_json[699] = {123,10,32,32,34,105,110,100,101,120,34,58,32,123,10,32,32,32,32,34,114,101,102,114,101,115,104,95,105,110,116,101,114,118,97,108,34,58,32,34,45,49,34,44,10,32,32,32,32,34,99,111,100,101,99,34,58,32,34,98,101,115,116,95,99,111,109,112,114,101,115,115,105,111,110,34,10,32,32,125,44,10,32,32,34,97,110,97,108,121,115,105,115,34,58,32,123,10,32,32,32,32,34,116,111,107,101,110,105,122,101,114,34,58,32,123,10,32,32,32,32,32,32,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,112,97,116,104,95,104,105,101,114,97,114,99,104,121,34,10,32,32,32,32,32,32,125,44,10,32,32,32,32,32,32,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,110,71,114,97,109,34,44,10,32,32,32,32,32,32,32,32,34,109,105,110,95,103,114,97,109,34,58,32,51,44,10,32,32,32,32,32,32,32,32,34,109,97,120,95,103,114,97,109,34,58,32,51,10,32,32,32,32,32,32,125,10,32,32,32,32,125,44,10,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,123,10,32,32,32,32,32,32,34,112,97,116,104,95,97,110,97,108,121,122,101,114,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,111,107,101,110,105,122,101,114,34,58,32,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,44,10,32,32,32,32,32,32,32,32,34,102,105,108,116,101,114,34,58,32,91,10,32,32,32,32,32,32,32,32,32,32,34,108,111,119,101,114,99,97,115,101,34,10,32,32,32,32,32,32,32,32,93,10,32,32,32,32,32,32,125,44,10,32,32,32,32,32,32,34,109,121,95,110,71,114,97,109,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,111,107,101,110,105,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,44,10,32,32,32,32,32,32,32,32,34,102,105,108,116,101,114,34,58,32,91,10,32,32,32,32,32,32,32,32,32,32,34,108,111,119,101,114,99,97,115,101,34,10,32,32,32,32,32,32,32,32,93,10,32,32,32,32,32,32,125,44,10,32,32,32,32,32,32,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,111,107,101,110,105,122,101,114,34,58,32,34,115,116,97,110,100,97,114,100,34,44,10,32,32,32,32,32,32,32,32,34,102,105,108,116,101,114,34,58,32,91,10,32,32,32,32,32,32,32,32,32,32,34,108,111,119,101,114,99,97,115,101,34,10,32,32,32,32,32,32,32,32,93,10,32,32,32,32,32,32,125,10,32,32,32,32,125,10,32,32,125,10,125}; +char settings_json[752] = {123,10,32,32,34,105,110,100,101,120,34,58,32,123,10,32,32,32,32,34,114,101,102,114,101,115,104,95,105,110,116,101,114,118,97,108,34,58,32,34,51,48,115,34,44,10,32,32,32,32,34,99,111,100,101,99,34,58,32,34,98,101,115,116,95,99,111,109,112,114,101,115,115,105,111,110,34,10,32,32,125,44,10,32,32,34,97,110,97,108,121,115,105,115,34,58,32,123,10,32,32,32,32,34,116,111,107,101,110,105,122,101,114,34,58,32,123,10,32,32,32,32,32,32,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,112,97,116,104,95,104,105,101,114,97,114,99,104,121,34,10,32,32,32,32,32,32,125,44,10,32,32,32,32,32,32,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,121,112,101,34,58,32,34,110,71,114,97,109,34,44,10,32,32,32,32,32,32,32,32,34,109,105,110,95,103,114,97,109,34,58,32,51,44,10,32,32,32,32,32,32,32,32,34,109,97,120,95,103,114,97,109,34,58,32,51,10,32,32,32,32,32,32,125,10,32,32,32,32,125,44,10,32,32,32,32,34,97,110,97,108,121,122,101,114,34,58,32,123,10,32,32,32,32,32,32,34,112,97,116,104,95,97,110,97,108,121,122,101,114,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,111,107,101,110,105,122,101,114,34,58,32,34,112,97,116,104,95,116,111,107,101,110,105,122,101,114,34,44,10,32,32,32,32,32,32,32,32,34,102,105,108,116,101,114,34,58,32,91,10,32,32,32,32,32,32,32,32,32,32,34,108,111,119,101,114,99,97,115,101,34,10,32,32,32,32,32,32,32,32,93,10,32,32,32,32,32,32,125,44,10,32,32,32,32,32,32,34,109,121,95,110,71,114,97,109,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,111,107,101,110,105,122,101,114,34,58,32,34,109,121,95,110,71,114,97,109,95,116,111,107,101,110,105,122,101,114,34,44,10,32,32,32,32,32,32,32,32,34,102,105,108,116,101,114,34,58,32,91,10,32,32,32,32,32,32,32,32,32,32,34,108,111,119,101,114,99,97,115,101,34,44,10,32,32,32,32,32,32,32,32,32,32,34,97,115,99,105,105,102,111,108,100,105,110,103,34,10,32,32,32,32,32,32,32,32,93,10,32,32,32,32,32,32,125,44,10,32,32,32,32,32,32,34,99,111,110,116,101,110,116,95,97,110,97,108,121,122,101,114,34,58,32,123,10,32,32,32,32,32,32,32,32,34,116,111,107,101,110,105,122,101,114,34,58,32,34,115,116,97,110,100,97,114,100,34,44,10,32,32,32,32,32,32,32,32,34,102,105,108,116,101,114,34,58,32,91,10,32,32,32,32,32,32,32,32,32,32,34,108,111,119,101,114,99,97,115,101,34,44,10,32,32,32,32,32,32,32,32,32,32,34,97,115,99,105,105,102,111,108,100,105,110,103,34,10,32,32,32,32,32,32,32,32,93,10,32,32,32,32,32,32,125,10,32,32,32,32,125,10,32,32,125,10,125}; diff --git a/src/io/serialize.c b/src/io/serialize.c index 28245c7..8dd1119 100644 --- a/src/io/serialize.c +++ b/src/io/serialize.c @@ -232,7 +232,9 @@ void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func case MetaTitle: { buf.cur = 0; while ((c = getc(file)) != 0) { - dyn_buffer_write_char(&buf, (char) c); + if (!(SHOULD_IGNORE_CHAR(c) || c == ' ')) { + dyn_buffer_write_char(&buf, (char) c); + } } dyn_buffer_write_char(&buf, '\0'); cJSON_AddStringToObject(document, get_meta_key_text(key), buf.buf); diff --git a/src/main.c b/src/main.c index 696ed78..478e736 100644 --- a/src/main.c +++ b/src/main.c @@ -5,7 +5,7 @@ #define EPILOG "Made by simon987 . Released under GPL-3.0" -static const char *const Version = "1.0.1"; +static const char *const Version = "1.0.2"; static const char *const usage[] = { "sist2 scan [OPTION]... PATH", "sist2 index [OPTION]... INDEX",