From a8b6886f7b08cac7c94599b6d418e34379972fdb Mon Sep 17 00:00:00 2001 From: simon987 Date: Sun, 16 Apr 2023 19:46:01 -0400 Subject: [PATCH] Fix stats page --- CMakeLists.txt | 2 +- sist2-vue/src/Sist2Api.ts | 16 ++-- sist2-vue/src/components/D3DateHistogram.vue | 2 +- sist2-vue/src/components/D3MimeBarCount.vue | 2 +- sist2-vue/src/components/D3MimeBarSize.vue | 2 +- sist2-vue/src/components/D3SizeHistogram.vue | 2 +- sist2-vue/src/components/D3Treemap.vue | 2 +- src/database/database.h | 12 +++ src/database/database_stats.c | 83 ++++++++++++++++++++ src/database/database_stats.h | 5 -- src/web/serve.c | 45 +++++------ 11 files changed, 127 insertions(+), 46 deletions(-) delete mode 100644 src/database/database_stats.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e851db..87d3593 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,7 @@ add_executable(sist2 src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp - src/database/database_stats.c src/database/database_stats.h src/database/database_schema.c) + src/database/database_stats.c src/database/database_schema.c) set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C) target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/) diff --git a/sist2-vue/src/Sist2Api.ts b/sist2-vue/src/Sist2Api.ts index ea64884..6722b5b 100644 --- a/sist2-vue/src/Sist2Api.ts +++ b/sist2-vue/src/Sist2Api.ts @@ -361,20 +361,20 @@ class Sist2Api { }); } - getTreemapCsvUrl(indexId: string) { - return `${this.baseUrl}s/${indexId}/1`; + getTreemapStat(indexId: string) { + return `${this.baseUrl}s/${indexId}/TMAP`; } - getMimeCsvUrl(indexId: string) { - return `${this.baseUrl}s/${indexId}/2`; + getMimeStat(indexId: string) { + return `${this.baseUrl}s/${indexId}/MAGG`; } - getSizeCsv(indexId: string) { - return `${this.baseUrl}s/${indexId}/3`; + getSizeStat(indexId: string) { + return `${this.baseUrl}s/${indexId}/SAGG`; } - getDateCsv(indexId: string) { - return `${this.baseUrl}s/${indexId}/4`; + getDateStat(indexId: string) { + return `${this.baseUrl}s/${indexId}/DAGG`; } } diff --git a/sist2-vue/src/components/D3DateHistogram.vue b/sist2-vue/src/components/D3DateHistogram.vue index b333e18..56b025f 100644 --- a/sist2-vue/src/components/D3DateHistogram.vue +++ b/sist2-vue/src/components/D3DateHistogram.vue @@ -120,7 +120,7 @@ export default { update(indexId) { const svg = d3.select("#date-histogram"); - d3.csv(Sist2Api.getDateCsv(indexId)).then(tabularData => { + d3.json(Sist2Api.getDateStat(indexId)).then(tabularData => { dateHistogram(tabularData.slice(), svg, this.$t("d3.dateHistogram")); }); } diff --git a/sist2-vue/src/components/D3MimeBarCount.vue b/sist2-vue/src/components/D3MimeBarCount.vue index a5e7575..3b308ac 100644 --- a/sist2-vue/src/components/D3MimeBarCount.vue +++ b/sist2-vue/src/components/D3MimeBarCount.vue @@ -91,7 +91,7 @@ export default { const mimeSvgCount = d3.select("#agg-mime-count"); const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6; - d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => { + d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => { mimeBarCount(tabularData.slice(), mimeSvgCount, fillOpacity, this.$t("d3.mimeCount")); }); } diff --git a/sist2-vue/src/components/D3MimeBarSize.vue b/sist2-vue/src/components/D3MimeBarSize.vue index 2f0b707..2d78861 100644 --- a/sist2-vue/src/components/D3MimeBarSize.vue +++ b/sist2-vue/src/components/D3MimeBarSize.vue @@ -90,7 +90,7 @@ export default { const mimeSvgSize = d3.select("#agg-mime-size"); const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6; - d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => { + d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => { mimeBarSize(tabularData.slice(), mimeSvgSize, fillOpacity, this.$t("d3.mimeSize")); }); } diff --git a/sist2-vue/src/components/D3SizeHistogram.vue b/sist2-vue/src/components/D3SizeHistogram.vue index 7c59990..04b5dd4 100644 --- a/sist2-vue/src/components/D3SizeHistogram.vue +++ b/sist2-vue/src/components/D3SizeHistogram.vue @@ -117,7 +117,7 @@ export default { update(indexId) { const svg = d3.select("#size-histogram"); - d3.csv(Sist2Api.getSizeCsv(indexId)).then(tabularData => { + d3.json(Sist2Api.getSizeStat(indexId)).then(tabularData => { sizeHistogram(tabularData.slice(), svg, this.$t("d3.sizeHistogram")); }); } diff --git a/sist2-vue/src/components/D3Treemap.vue b/sist2-vue/src/components/D3Treemap.vue index fe7ddc1..abe9ef4 100644 --- a/sist2-vue/src/components/D3Treemap.vue +++ b/sist2-vue/src/components/D3Treemap.vue @@ -240,7 +240,7 @@ export default { .style("overflow", "visible") .style("font", "10px sans-serif"); - d3.csv(Sist2Api.getTreemapCsvUrl(indexId)).then(tabularData => { + d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => { tabularData.forEach(row => { row.taxonomy = row.path.split("/"); row.size = Number(row.size); diff --git a/src/database/database.h b/src/database/database.h index 7afe383..feef5c0 100644 --- a/src/database/database.h +++ b/src/database/database.h @@ -18,6 +18,14 @@ typedef enum { FTS_DATABASE } database_type_t; +typedef enum { + DATABASE_STAT_INVALID, + DATABASE_STAT_TREEMAP, + DATABASE_STAT_MIME_AGG, + DATABASE_STAT_SIZE_AGG, + DATABASE_STAT_DATE_AGG, +} database_stat_type_d; + typedef enum { JOB_UNDEFINED, JOB_BULK_LINE, @@ -132,12 +140,16 @@ treemap_row_t database_treemap_iter(database_iterator_t *iter); void database_generate_stats(database_t *db, double treemap_threshold); +database_stat_type_d database_get_stat_type_by_mnemonic(const char *name); + job_t *database_get_work(database_t *db, job_type_t job_type); void database_add_work(database_t *db, job_t *job); //void database_index(database_t *db); +cJSON *database_get_stats(database_t *db, database_stat_type_d type); + #define CRASH_IF_STMT_FAIL(x) do { \ int return_value = x; \ if (return_value != SQLITE_DONE && return_value != SQLITE_ROW) { \ diff --git a/src/database/database_stats.c b/src/database/database_stats.c index 2f2c73f..6dea1da 100644 --- a/src/database/database_stats.c +++ b/src/database/database_stats.c @@ -6,6 +6,7 @@ #define SIZE_BUCKET (long)(5 * 1000 * 1000) #define DATE_BUCKET (long)(2629800) // ~30 days + database_iterator_t *database_create_treemap_iterator(database_t *db, long threshold) { sqlite3_stmt *stmt; @@ -157,3 +158,85 @@ void database_generate_stats(database_t *db, double treemap_threshold) { LOG_INFO("database.c", "Done!"); } +database_stat_type_d database_get_stat_type_by_mnemonic(const char *name) { + if (strcmp(name, "TMAP") == 0) { + return DATABASE_STAT_TREEMAP; + } + if (strcmp(name, "MAGG") == 0) { + return DATABASE_STAT_MIME_AGG; + } + if (strcmp(name, "SAGG") == 0) { + return DATABASE_STAT_SIZE_AGG; + } + if (strcmp(name, "DAGG") == 0) { + return DATABASE_STAT_DATE_AGG; + } + + return DATABASE_STAT_INVALID; +} + +cJSON *database_get_stats(database_t *db, database_stat_type_d type) { + + sqlite3_stmt *stmt; + + switch (type) { + case DATABASE_STAT_TREEMAP: + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, "SELECT path,size FROM stats_treemap", -1, &stmt, NULL + )); + break; + case DATABASE_STAT_DATE_AGG: + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, "SELECT bucket,count FROM stats_date_agg", -1, &stmt, NULL + )); + break; + case DATABASE_STAT_SIZE_AGG: + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, "SELECT bucket,count FROM stats_size_agg", -1, &stmt, NULL + )); + break; + case DATABASE_STAT_MIME_AGG: + CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2( + db->db, "SELECT mime,size,count FROM stats_mime_agg", -1, &stmt, NULL + )); + break; + case DATABASE_STAT_INVALID: + default: + LOG_FATALF("database_stats.c", "Invalid stat type: %d", type); + } + + cJSON *json = cJSON_CreateArray(); + + int ret; + do { + ret = sqlite3_step(stmt); + CRASH_IF_STMT_FAIL(ret); + + if (ret == SQLITE_DONE) { + break; + } + + cJSON *row = cJSON_CreateObject(); + + switch (type) { + case DATABASE_STAT_TREEMAP: + cJSON_AddStringToObject(row, "path", (const char *) sqlite3_column_text(stmt, 0)); + cJSON_AddNumberToObject(row, "size", (double) sqlite3_column_int64(stmt, 1)); + break; + case DATABASE_STAT_DATE_AGG: + case DATABASE_STAT_SIZE_AGG: + cJSON_AddNumberToObject(row, "bucket", (double) sqlite3_column_int64(stmt, 0)); + cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 1)); + break; + case DATABASE_STAT_MIME_AGG: + cJSON_AddStringToObject(row, "mime", (const char *) sqlite3_column_text(stmt, 0)); + cJSON_AddNumberToObject(row, "size", (double) sqlite3_column_int64(stmt, 1)); + cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 2)); + break; + } + + cJSON_AddItemToArray(json, row); + } while (TRUE); + + return json; +} \ No newline at end of file diff --git a/src/database/database_stats.h b/src/database/database_stats.h deleted file mode 100644 index fe4d507..0000000 --- a/src/database/database_stats.h +++ /dev/null @@ -1,5 +0,0 @@ -#ifndef SIST2_DATABASE_STATS_H -#define SIST2_DATABASE_STATS_H - - -#endif //SIST2_DATABASE_STATS_H diff --git a/src/web/serve.c b/src/web/serve.c index ffb32cd..e3b2f6d 100644 --- a/src/web/serve.c +++ b/src/web/serve.c @@ -20,49 +20,40 @@ static struct mg_http_serve_opts DefaultServeOpts = { void stats_files(struct mg_connection *nc, struct mg_http_message *hm) { - if (hm->uri.len != SIST_INDEX_ID_LEN + 4) { + if (hm->uri.len != SIST_INDEX_ID_LEN + 7) { HTTP_REPLY_NOT_FOUND return; } char arg_index_id[SIST_INDEX_ID_LEN]; + char arg_stat_type[5]; + memcpy(arg_index_id, hm->uri.ptr + 3, SIST_INDEX_ID_LEN); *(arg_index_id + SIST_INDEX_ID_LEN - 1) = '\0'; + memcpy(arg_stat_type, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, 4); + *(arg_stat_type + sizeof(arg_stat_type) - 1) = '\0'; - index_t *index = web_get_index_by_id(arg_index_id); - if (index == NULL) { + database_stat_type_d stat_type = database_get_stat_type_by_mnemonic(arg_stat_type); + if (stat_type == DATABASE_STAT_INVALID) { HTTP_REPLY_NOT_FOUND return; } - const char *file; - switch (atoi(hm->uri.ptr + 3 + SIST_INDEX_ID_LEN)) { - case 1: - file = "treemap.csv"; - break; - case 2: - file = "mime_agg.csv"; - break; - case 3: - file = "size_agg.csv"; - break; - case 4: - file = "date_agg.csv"; - break; - default: - return; + database_t *db = web_get_database(arg_index_id); + if (db == NULL) { + LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index_id); + HTTP_REPLY_NOT_FOUND + return; } - char disposition[8192]; - snprintf(disposition, sizeof(disposition), - "Content-Disposition: inline; filename=\"%s\"\r\nCache-Control: max-age=31536000\r\n", file); + cJSON *json = database_get_stats(db, stat_type); + char *json_str = cJSON_PrintUnformatted(json); - char full_path[PATH_MAX]; - strcpy(full_path, index->path); - strcat(full_path, file); + web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json"); + mg_send(nc, json_str, strlen(json_str)); - struct mg_http_serve_opts opts = {}; - mg_http_serve_file(nc, hm, full_path, &opts); + free(json_str); + cJSON_Delete(json); } void serve_index_html(struct mg_connection *nc, struct mg_http_message *hm) {