simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-10 14:06:45 +00:00
Code Issues Projects Releases Wiki Activity

Support for external json indices

Browse Source
This commit is contained in:
simon 2020-01-14 15:44:31 -05:00
parent 3bb24b4453
commit 7c8a3e2f9d
11 changed files with 160 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
schema/mappings.json
View File

@ -13,19 +13,24 @@
"type": "keyword"
},
"videoc": {
"type": "keyword"
"type": "keyword",
"index": false
},
"audioc": {
"type": "keyword"
"type": "keyword",
"index": false
},
"duration": {
"type": "float"
"type": "float",
"index": false
},
"width": {
"type": "integer"
"type": "integer",
"index": false
},
"height": {
"type": "integer"
"type": "integer",
"index": false
},
"mtime": {
"type": "integer"
@ -70,6 +75,23 @@
"analyzer": "my_nGram",
"type": "text"
},
"_keyword.*": {
"type": "keyword"
},
"_text.*": {
"analyzer": "content_analyzer",
"type": "text",
"fields": {
"nGram": {
"type": "text",
"analyzer": "my_nGram"
}
}
},
"_url": {
"type": "keyword",
"index": false
},
"content": {
"analyzer": "content_analyzer",
"type": "text",

4
src/cli.c
View File

@ -163,6 +163,8 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
int index_args_validate(index_args_t *args, int argc, const char **argv) {
LogCtx.verbose = 1;
if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n");
return 1;
@ -224,6 +226,8 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) {
int web_args_validate(web_args_t *args, int argc, const char **argv) {
LogCtx.verbose = 1;
if (argc < 2) {
fprintf(stderr, "Required positional argument: PATH.\n");
return 1;

34
src/index/elastic.c
View File

@ -27,13 +27,14 @@ void print_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
cJSON_AddStringToObject(line, "_id", uuid_str);
cJSON_AddStringToObject(line, "_index", "sist2");
cJSON_AddStringToObject(line, "_type", "_doc");
cJSON_AddItemToObject(line, "_source", document);
cJSON_AddItemReferenceToObject(line, "_source", document);
char *json = cJSON_PrintUnformatted(line);
printf("%s\n", json);
cJSON_free(line);
cJSON_free(json);
cJSON_Delete(line);
}
void index_json(cJSON *document, const char uuid_str[UUID_STR_LEN]) {
@ -68,7 +69,7 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
char bulk_url[4096];
snprintf(bulk_url, 4096, "%s/sist2/_update_by_query?pretty", Indexer->es_url);
response_t *r = web_post(bulk_url, str, "Content-Type: application/json");
printf("Executed user script <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Executed user script <%d>", r->status_code);
cJSON *resp = cJSON_Parse(r->body);
cJSON_free(str);
@ -79,7 +80,7 @@ void execute_update_script(const char *script, const char index_id[UUID_STR_LEN]
if (error != NULL) {
char *error_str = cJSON_Print(error);
fprintf(stderr, "User script error: \n%s\n", error_str);
LOG_ERRORF("elastic.c", "User script error: \n%s", error_str);
cJSON_free(error_str);
}
@ -132,11 +133,10 @@ void elastic_flush() {
response_t *r = web_post(bulk_url, buf, "Content-Type: application/x-ndjson");
if (r->status_code == 0) {
fprintf(stderr, "Could not connect to %s, make sure that elasticsearch is running!\n", IndexCtx.es_url);
exit(1);
LOG_FATALF("elastic.c", "Could not connect to %s, make sure that elasticsearch is running!\n", IndexCtx.es_url)
}
printf("Indexed %3d documents (%zukB) <%d>\n", count, buf_cur / 1024, r->status_code);
LOG_INFOF("elastic.c", "Indexed %3d documents (%zukB) <%d>", count, buf_cur / 1024, r->status_code);
cJSON *ret_json = cJSON_Parse(r->body);
if (cJSON_GetObjectItem(ret_json, "errors")->valueint != 0) {
@ -144,7 +144,7 @@ void elastic_flush() {
cJSON_ArrayForEach(err, cJSON_GetObjectItem(ret_json, "items")) {
if (cJSON_GetObjectItem(cJSON_GetObjectItem(err, "index"), "status")->valueint != 201) {
char* str = cJSON_Print(err);
fprintf(stderr, "%s\n", str);
LOG_ERRORF("elastic.c", "%s\n", str);
cJSON_free(str);
}
}
@ -198,7 +198,7 @@ void destroy_indexer(char * script, char index_id[UUID_STR_LEN]) {
snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
response_t *r = web_post(url, "", NULL);
printf("Refresh index <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
free_response(r);
if (script != NULL) {
@ -207,12 +207,12 @@ void destroy_indexer(char * script, char index_id[UUID_STR_LEN]) {
snprintf(url, sizeof(url), "%s/sist2/_refresh", IndexCtx.es_url);
r = web_post(url, "", NULL);
printf("Refresh index <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Refresh index <%d>", r->status_code);
free_response(r);
snprintf(url, sizeof(url), "%s/sist2/_forcemerge", IndexCtx.es_url);
r = web_post(url, "", NULL);
printf("Merge index <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Merge index <%d>", r->status_code);
free_response(r);
if (Indexer != NULL) {
@ -232,32 +232,32 @@ void elastic_init(int force_reset) {
if (!index_exists || force_reset) {
r = web_delete(url);
printf("Delete index <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Delete index <%d>", r->status_code);
free_response(r);
snprintf(url, 4096, "%s/sist2", IndexCtx.es_url);
r = web_put(url, "", NULL);
printf("Create index <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Create index <%d>", r->status_code);
free_response(r);
snprintf(url, 4096, "%s/sist2/_close", IndexCtx.es_url);
r = web_post(url, "", NULL);
printf("Close index <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Close index <%d>", r->status_code);
free_response(r);
snprintf(url, 4096, "%s/sist2/_settings", IndexCtx.es_url);
r = web_put(url, settings_json, "Content-Type: application/json");
printf("Update settings <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Update settings <%d>", r->status_code);
free_response(r);
snprintf(url, 4096, "%s/sist2/_mappings/_doc?include_type_name=true", IndexCtx.es_url);
r = web_put(url, mappings_json, "Content-Type: application/json");
printf("Update mappings <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Update mappings <%d>", r->status_code);
free_response(r);
snprintf(url, 4096, "%s/sist2/_open", IndexCtx.es_url);
r = web_post(url, "", NULL);
printf("Open index <%d>\n", r->status_code);
LOG_INFOF("elastic.c", "Open index <%d>", r->status_code);
free_response(r);
}
}

2
src/index/static_generated.c
View File

File diff suppressed because one or more lines are too long

98
src/io/serialize.c
View File

@ -34,6 +34,7 @@ void write_index_descriptor(char *path, index_descriptor_t *desc) {
cJSON_AddStringToObject(json, "version", desc->version);
cJSON_AddStringToObject(json, "root", desc->root);
cJSON_AddStringToObject(json, "name", desc->name);
cJSON_AddStringToObject(json, "type", desc->type);
cJSON_AddStringToObject(json, "rewrite_url", desc->rewrite_url);
cJSON_AddNumberToObject(json, "timestamp", (double) desc->timestamp);
@ -56,8 +57,7 @@ index_descriptor_t read_index_descriptor(char *path) {
int fd = open(path, O_RDONLY);
if (fd == -1) {
fprintf(stderr, "Invalid/corrupt index (Could not find descriptor)\n");
exit(1);
LOG_FATAL("serialize.c", "Invalid/corrupt index (Could not find descriptor)\n")
}
char *buf = malloc(info.st_size + 1);
@ -75,6 +75,11 @@ index_descriptor_t read_index_descriptor(char *path) {
descriptor.root_len = (short) strlen(descriptor.root);
strcpy(descriptor.version, cJSON_GetObjectItem(json, "version")->valuestring);
strcpy(descriptor.uuid, cJSON_GetObjectItem(json, "uuid")->valuestring);
if (cJSON_GetObjectItem(json, "type") == NULL) {
strcpy(descriptor.type, INDEX_TYPE_BIN);
} else {
strcpy(descriptor.type, cJSON_GetObjectItem(json, "type")->valuestring);
}
cJSON_Delete(json);
free(buf);
@ -172,8 +177,8 @@ void thread_cleanup() {
close(index_fd);
}
void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func func) {
void read_index_bin(const char *path, const char *index_id, index_func func) {
line_t line;
dyn_buffer_t buf = dyn_buffer_create();
@ -229,7 +234,7 @@ void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func
case MetaMediaBitrate: {
long value;
fread(&value, sizeof(long), 1, file);
cJSON_AddNumberToObject(document, get_meta_key_text(key), value);
cJSON_AddNumberToObject(document, get_meta_key_text(key), (double) value);
break;
}
case MetaMediaAudioCodec:
@ -262,7 +267,7 @@ void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func
break;
}
default:
LOG_FATALF("serialize.c", "Invalid meta key (corrupt index): %x", key)
LOG_FATALF("serialize.c", "Invalid meta key (corrupt index): %x", key)
}
key = getc(file);
@ -275,6 +280,89 @@ void read_index(const char *path, const char index_id[UUID_STR_LEN], index_func
fclose(file);
}
const char *json_type_copy_fields[] = {
"mime", "name", "path", "extension", "index", "size", "mtime", "parent",
// Meta
"title", "content", "width", "height", "duration", "audioc", "videoc",
"bitrate", "artist", "album", "album_artist", "genre", "title", "font_name",
// Special
"tag", "_url"
};
const char *json_type_array_fields[] = {
"_keyword", "_text"
};
void read_index_json(const char *path, UNUSED(const char *index_id), index_func func) {
FILE *file = fopen(path, "r");
while (1) {
char *line = NULL;
size_t len;
size_t read = getline(&line, &len, file);
if (read == -1) {
if (line) {
free(line);
}
break;
}
cJSON *input = cJSON_Parse(line);
if (input == NULL) {
LOG_FATALF("serialize.c", "Could not parse JSON line: \n%s", line)
}
if (line) {
free(line);
}
cJSON *document = cJSON_CreateObject();
const char *uuid_str = cJSON_GetObjectItem(input, "_id")->valuestring;
for (int i = 0; i < (sizeof(json_type_copy_fields) / sizeof(json_type_copy_fields[0])); i++) {
cJSON *value = cJSON_GetObjectItem(input, json_type_copy_fields[i]);
if (value != NULL) {
cJSON_AddItemReferenceToObject(document, json_type_copy_fields[i], value);
}
}
for (int i = 0; i < (sizeof(json_type_array_fields) / sizeof(json_type_array_fields[0])); i++) {
cJSON *arr = cJSON_GetObjectItem(input, json_type_array_fields[i]);
if (arr != NULL) {
cJSON *obj;
cJSON_ArrayForEach(obj, arr) {
char key[1024];
cJSON *k = cJSON_GetObjectItem(obj, "k");
cJSON *v = cJSON_GetObjectItem(obj, "v");
if (k == NULL || v == NULL || !cJSON_IsString(k) || !cJSON_IsString(v)) {
char *str = cJSON_Print(obj);
LOG_FATALF("serialize.c", "Invalid %s member: must contain .k and .v string fields: \n%s",
json_type_array_fields[i], str)
}
snprintf(key, sizeof(key), "%s.%s", json_type_array_fields[i], k->valuestring);
cJSON_AddStringToObject(document, key, v->valuestring);
}
}
}
func(document, uuid_str);
cJSON_Delete(document);
cJSON_Delete(input);
}
fclose(file);
}
void read_index(const char *path, const char index_id[UUID_STR_LEN], const char *type, index_func func) {
if (strcmp(type, INDEX_TYPE_BIN) == 0) {
read_index_bin(path, index_id, func);
} else if (strcmp(type, INDEX_TYPE_JSON) == 0) {
read_index_json(path, index_id, func);
}
}
void incremental_read(GHashTable *table, const char *filepath) {
FILE *file = fopen(filepath, "rb");
line_t line;

2
src/io/serialize.h
View File

@ -11,7 +11,7 @@ void incremental_copy(store_t *store, store_t *dst_store, const char *filepath,
void write_document(document_t *doc);
void read_index(const char *path, const char[UUID_STR_LEN], index_func);
void read_index(const char *path, const char[UUID_STR_LEN], const char *type, index_func);
void incremental_read(GHashTable *table, const char *filepath);

2
src/log.c
View File

@ -56,7 +56,7 @@ void sist_logf(char *filepath, int level, char *format, ...) {
log_len += 1;
}
write(STDOUT_FILENO, log_str, log_len);
write(STDERR_FILENO, log_str, log_len);
}
void sist_log(char *filepath, int level, char *str) {

13
src/main.c
View File

@ -6,7 +6,7 @@
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
static const char *const Version = "1.2.0";
static const char *const Version = "1.2.1";
static const char *const usage[] = {
"sist2 scan [OPTION]... PATH",
"sist2 index [OPTION]... INDEX",
@ -29,6 +29,7 @@ void init_dir(const char *dirpath) {
uuid_unparse(uuid, ScanCtx.index.desc.uuid);
time(&ScanCtx.index.desc.timestamp);
strcpy(ScanCtx.index.desc.version, Version);
strcpy(ScanCtx.index.desc.type, INDEX_TYPE_BIN);
write_index_descriptor(path, &ScanCtx.index.desc);
}
@ -130,8 +131,12 @@ void sist2_index(index_args_t *args) {
snprintf(descriptor_path, PATH_MAX, "%s/descriptor.json", args->index_path);
index_descriptor_t desc = read_index_descriptor(descriptor_path);
if (strcmp(desc.version, Version) != 0) {
fprintf(stderr, "Version mismatch! Index is v%s but executable is v%s\n", desc.version, Version);
LOG_DEBUGF("main.c", "descriptor version %s (%s)", desc.version, desc.type)
if (strcmp(desc.version, Version) != 0 && strcmp(desc.version, INDEX_VERSION_EXTERNAL) != 0) {
fprintf(stderr, "Version mismatch! Index is %s but executable is %s/%s\n",
desc.version, Version, INDEX_VERSION_EXTERNAL);
return;
}
@ -153,7 +158,7 @@ void sist2_index(index_args_t *args) {
if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
char file_path[PATH_MAX];
snprintf(file_path, PATH_MAX, "%s/%s", args->index_path, de->d_name);
read_index(file_path, desc.uuid, f);
read_index(file_path, desc.uuid, desc.type, f);
}
}
closedir(dir);

5
src/types.h
View File

@ -33,6 +33,10 @@ enum metakey {
MetaParent = 14 | META_STR_MASK,
};
#define INDEX_TYPE_BIN "binary"
#define INDEX_TYPE_JSON "json"
#define INDEX_VERSION_EXTERNAL "_external_v1"
typedef struct index_descriptor {
char uuid[UUID_STR_LEN];
char version[6];
@ -41,6 +45,7 @@ typedef struct index_descriptor {
char rewrite_url[8196];
short root_len;
char name[1024];
char type[64];
} index_descriptor_t;
typedef struct index_t {

2
src/web/static_generated.c
View File

File diff suppressed because one or more lines are too long

2
web/search.html
View File

@ -11,7 +11,7 @@
<nav class="navbar navbar-expand-lg">
<a class="navbar-brand" href="/">sist2</a>
<span class="badge badge-pill version">v1.2.0</span>
<span class="badge badge-pill version">v1.2.1</span>
<span class="tagline">Lightning-fast file system indexer and search tool </span>
<a style="margin-left: auto" id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
</nav>