simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-19 18:26:43 +00:00
Code Issues Projects Releases Wiki Activity

fix for elasticsearch

Browse Source
This commit is contained in:
simon 2019-10-26 19:05:47 -04:00
parent ef79681024
commit b4199a1fd8
5 changed files with 21 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
schema/settings.json
View File

@ -1,6 +1,6 @@
{ {
"index": { "index": {
"refresh_interval": "-1", "refresh_interval": "30s",
"codec": "best_compression" "codec": "best_compression"
}, },
"analysis": { "analysis": {
@ -24,13 +24,15 @@
"my_nGram": { "my_nGram": {
"tokenizer": "my_nGram_tokenizer", "tokenizer": "my_nGram_tokenizer",
"filter": [ "filter": [
"lowercase" "lowercase",
"asciifolding"
] ]
}, },
"content_analyzer": { "content_analyzer": {
"tokenizer": "standard", "tokenizer": "standard",
"filter": [ "filter": [
"lowercase" "lowercase",
"asciifolding"
] ]
} }
} }

13
src/index/elastic.c
View File

@ -70,7 +70,8 @@ void elastic_flush() {
while (line != NULL) { while (line != NULL) {
char action_str[512]; char action_str[512];
snprintf(action_str, 512, "{\"index\":{\"_id\": \"%s\", \"_type\":\"_doc\"}}", line->uuid_str); snprintf(action_str, 512,
"{\"index\":{\"_id\":\"%s\", \"_type\":\"_doc\", \"_index\":\"sist2\"}}\n", line->uuid_str);
size_t action_str_len = strlen(action_str); size_t action_str_len = strlen(action_str);
size_t line_len = strlen(line->line); size_t line_len = strlen(line->line);
@ -98,6 +99,14 @@ void elastic_flush() {
snprintf(bulk_url, 4096, "%s/sist2/_bulk", Indexer->es_url); snprintf(bulk_url, 4096, "%s/sist2/_bulk", Indexer->es_url);
response_t *r = web_post(bulk_url, buf, "Content-Type: application/x-ndjson"); response_t *r = web_post(bulk_url, buf, "Content-Type: application/x-ndjson");
printf("Indexed %3d documents (%zukB) <%d>\n", count, buf_cur / 1024, r->status_code); printf("Indexed %3d documents (%zukB) <%d>\n", count, buf_cur / 1024, r->status_code);
cJSON *ret_json = cJSON_Parse(r->body);
if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) {
fprintf(stderr, "%s\n", r->body);
}
cJSON_Delete(ret_json);
free_response(r); free_response(r);
} }
@ -187,7 +196,7 @@ void elastic_init(int force_reset) {
printf("Update settings <%d>\n", r->status_code); printf("Update settings <%d>\n", r->status_code);
free_response(r); free_response(r);
snprintf(url, 4096, "%s/sist2/_mappings/_doc", IndexCtx.es_url); snprintf(url, 4096, "%s/sist2/_mappings/_doc?include_type_name=true", IndexCtx.es_url);
r = web_put(url, mappings_json, "Content-Type: application/json"); r = web_put(url, mappings_json, "Content-Type: application/json");
printf("Update mappings <%d>\n", r->status_code); printf("Update mappings <%d>\n", r->status_code);
free_response(r); free_response(r);

2
src/index/static_generated.c
View File

File diff suppressed because one or more lines are too long

4
src/io/serialize.c
View File

@ -232,7 +232,9 @@ void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func
case MetaTitle: { case MetaTitle: {
buf.cur = 0; buf.cur = 0;
while ((c = getc(file)) != 0) { while ((c = getc(file)) != 0) {
dyn_buffer_write_char(&buf, (char) c); if (!(SHOULD_IGNORE_CHAR(c) || c == ' ')) {
dyn_buffer_write_char(&buf, (char) c);
}
} }
dyn_buffer_write_char(&buf, '\0'); dyn_buffer_write_char(&buf, '\0');
cJSON_AddStringToObject(document, get_meta_key_text(key), buf.buf); cJSON_AddStringToObject(document, get_meta_key_text(key), buf.buf);

2
src/main.c
View File

@ -5,7 +5,7 @@
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0" #define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
static const char *const Version = "1.0.1"; static const char *const Version = "1.0.2";
static const char *const usage[] = { static const char *const usage[] = {
"sist2 scan [OPTION]... PATH", "sist2 scan [OPTION]... PATH",
"sist2 index [OPTION]... INDEX", "sist2 index [OPTION]... INDEX",