simon987/sist2
1
0
Fork 0
mirror of https://github.com/simon987/sist2.git synced 2025-04-10 14:06:45 +00:00
Code Issues Projects Releases Wiki Activity

Fix stats page

Browse Source
This commit is contained in:
simon987 2023-04-16 19:46:01 -04:00
parent a7e9b6af96
commit a8b6886f7b
11 changed files with 127 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -55,7 +55,7 @@ add_executable(sist2
src/auth0/auth0_c_api.h src/auth0/auth0_c_api.cpp
src/database/database_stats.c src/database/database_stats.h src/database/database_schema.c)
src/database/database_stats.c src/database/database_schema.c)
set_target_properties(sist2 PROPERTIES LINKER_LANGUAGE C)
target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)

16
sist2-vue/src/Sist2Api.ts
View File

@ -361,20 +361,20 @@ class Sist2Api {
});
}
getTreemapCsvUrl(indexId: string) {
return `${this.baseUrl}s/${indexId}/1`;
getTreemapStat(indexId: string) {
return `${this.baseUrl}s/${indexId}/TMAP`;
}
getMimeCsvUrl(indexId: string) {
return `${this.baseUrl}s/${indexId}/2`;
getMimeStat(indexId: string) {
return `${this.baseUrl}s/${indexId}/MAGG`;
}
getSizeCsv(indexId: string) {
return `${this.baseUrl}s/${indexId}/3`;
getSizeStat(indexId: string) {
return `${this.baseUrl}s/${indexId}/SAGG`;
}
getDateCsv(indexId: string) {
return `${this.baseUrl}s/${indexId}/4`;
getDateStat(indexId: string) {
return `${this.baseUrl}s/${indexId}/DAGG`;
}
}

2
sist2-vue/src/components/D3DateHistogram.vue
View File

@ -120,7 +120,7 @@ export default {
update(indexId) {
const svg = d3.select("#date-histogram");
d3.csv(Sist2Api.getDateCsv(indexId)).then(tabularData => {
d3.json(Sist2Api.getDateStat(indexId)).then(tabularData => {
dateHistogram(tabularData.slice(), svg, this.$t("d3.dateHistogram"));
});
}

2
sist2-vue/src/components/D3MimeBarCount.vue
View File

@ -91,7 +91,7 @@ export default {
const mimeSvgCount = d3.select("#agg-mime-count");
const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
mimeBarCount(tabularData.slice(), mimeSvgCount, fillOpacity, this.$t("d3.mimeCount"));
});
}

2
sist2-vue/src/components/D3MimeBarSize.vue
View File

@ -90,7 +90,7 @@ export default {
const mimeSvgSize = d3.select("#agg-mime-size");
const fillOpacity = this.$store.state.optTheme === "black" ? 0.9 : 0.6;
d3.csv(Sist2Api.getMimeCsvUrl(indexId)).then(tabularData => {
d3.json(Sist2Api.getMimeStat(indexId)).then(tabularData => {
mimeBarSize(tabularData.slice(), mimeSvgSize, fillOpacity, this.$t("d3.mimeSize"));
});
}

2
sist2-vue/src/components/D3SizeHistogram.vue
View File

@ -117,7 +117,7 @@ export default {
update(indexId) {
const svg = d3.select("#size-histogram");
d3.csv(Sist2Api.getSizeCsv(indexId)).then(tabularData => {
d3.json(Sist2Api.getSizeStat(indexId)).then(tabularData => {
sizeHistogram(tabularData.slice(), svg, this.$t("d3.sizeHistogram"));
});
}

2
sist2-vue/src/components/D3Treemap.vue
View File

@ -240,7 +240,7 @@ export default {
.style("overflow", "visible")
.style("font", "10px sans-serif");
d3.csv(Sist2Api.getTreemapCsvUrl(indexId)).then(tabularData => {
d3.json(Sist2Api.getTreemapStat(indexId)).then(tabularData => {
tabularData.forEach(row => {
row.taxonomy = row.path.split("/");
row.size = Number(row.size);

12
src/database/database.h
View File

@ -18,6 +18,14 @@ typedef enum {
FTS_DATABASE
} database_type_t;
typedef enum {
DATABASE_STAT_INVALID,
DATABASE_STAT_TREEMAP,
DATABASE_STAT_MIME_AGG,
DATABASE_STAT_SIZE_AGG,
DATABASE_STAT_DATE_AGG,
} database_stat_type_d;
typedef enum {
JOB_UNDEFINED,
JOB_BULK_LINE,
@ -132,12 +140,16 @@ treemap_row_t database_treemap_iter(database_iterator_t *iter);
void database_generate_stats(database_t *db, double treemap_threshold);
database_stat_type_d database_get_stat_type_by_mnemonic(const char *name);
job_t *database_get_work(database_t *db, job_type_t job_type);
void database_add_work(database_t *db, job_t *job);
//void database_index(database_t *db);
cJSON *database_get_stats(database_t *db, database_stat_type_d type);
#define CRASH_IF_STMT_FAIL(x) do { \
int return_value = x; \
if (return_value != SQLITE_DONE && return_value != SQLITE_ROW) { \

83
src/database/database_stats.c
View File

@ -6,6 +6,7 @@
#define SIZE_BUCKET (long)(5 * 1000 * 1000)
#define DATE_BUCKET (long)(2629800) // ~30 days
database_iterator_t *database_create_treemap_iterator(database_t *db, long threshold) {
sqlite3_stmt *stmt;
@ -157,3 +158,85 @@ void database_generate_stats(database_t *db, double treemap_threshold) {
LOG_INFO("database.c", "Done!");
}
database_stat_type_d database_get_stat_type_by_mnemonic(const char *name) {
if (strcmp(name, "TMAP") == 0) {
return DATABASE_STAT_TREEMAP;
}
if (strcmp(name, "MAGG") == 0) {
return DATABASE_STAT_MIME_AGG;
}
if (strcmp(name, "SAGG") == 0) {
return DATABASE_STAT_SIZE_AGG;
}
if (strcmp(name, "DAGG") == 0) {
return DATABASE_STAT_DATE_AGG;
}
return DATABASE_STAT_INVALID;
}
cJSON *database_get_stats(database_t *db, database_stat_type_d type) {
sqlite3_stmt *stmt;
switch (type) {
case DATABASE_STAT_TREEMAP:
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db, "SELECT path,size FROM stats_treemap", -1, &stmt, NULL
));
break;
case DATABASE_STAT_DATE_AGG:
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db, "SELECT bucket,count FROM stats_date_agg", -1, &stmt, NULL
));
break;
case DATABASE_STAT_SIZE_AGG:
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db, "SELECT bucket,count FROM stats_size_agg", -1, &stmt, NULL
));
break;
case DATABASE_STAT_MIME_AGG:
CRASH_IF_NOT_SQLITE_OK(sqlite3_prepare_v2(
db->db, "SELECT mime,size,count FROM stats_mime_agg", -1, &stmt, NULL
));
break;
case DATABASE_STAT_INVALID:
default:
LOG_FATALF("database_stats.c", "Invalid stat type: %d", type);
}
cJSON *json = cJSON_CreateArray();
int ret;
do {
ret = sqlite3_step(stmt);
CRASH_IF_STMT_FAIL(ret);
if (ret == SQLITE_DONE) {
break;
}
cJSON *row = cJSON_CreateObject();
switch (type) {
case DATABASE_STAT_TREEMAP:
cJSON_AddStringToObject(row, "path", (const char *) sqlite3_column_text(stmt, 0));
cJSON_AddNumberToObject(row, "size", (double) sqlite3_column_int64(stmt, 1));
break;
case DATABASE_STAT_DATE_AGG:
case DATABASE_STAT_SIZE_AGG:
cJSON_AddNumberToObject(row, "bucket", (double) sqlite3_column_int64(stmt, 0));
cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 1));
break;
case DATABASE_STAT_MIME_AGG:
cJSON_AddStringToObject(row, "mime", (const char *) sqlite3_column_text(stmt, 0));
cJSON_AddNumberToObject(row, "size", (double) sqlite3_column_int64(stmt, 1));
cJSON_AddNumberToObject(row, "count", (double) sqlite3_column_int64(stmt, 2));
break;
}
cJSON_AddItemToArray(json, row);
} while (TRUE);
return json;
}

5
src/database/database_stats.h
View File

@ -1,5 +0,0 @@
#ifndef SIST2_DATABASE_STATS_H
#define SIST2_DATABASE_STATS_H
#endif //SIST2_DATABASE_STATS_H

45
src/web/serve.c
View File

@ -20,49 +20,40 @@ static struct mg_http_serve_opts DefaultServeOpts = {
void stats_files(struct mg_connection *nc, struct mg_http_message *hm) {
if (hm->uri.len != SIST_INDEX_ID_LEN + 4) {
if (hm->uri.len != SIST_INDEX_ID_LEN + 7) {
HTTP_REPLY_NOT_FOUND
return;
}
char arg_index_id[SIST_INDEX_ID_LEN];
char arg_stat_type[5];
memcpy(arg_index_id, hm->uri.ptr + 3, SIST_INDEX_ID_LEN);
*(arg_index_id + SIST_INDEX_ID_LEN - 1) = '\0';
memcpy(arg_stat_type, hm->uri.ptr + 3 + SIST_INDEX_ID_LEN, 4);
*(arg_stat_type + sizeof(arg_stat_type) - 1) = '\0';
index_t *index = web_get_index_by_id(arg_index_id);
if (index == NULL) {
database_stat_type_d stat_type = database_get_stat_type_by_mnemonic(arg_stat_type);
if (stat_type == DATABASE_STAT_INVALID) {
HTTP_REPLY_NOT_FOUND
return;
}
const char *file;
switch (atoi(hm->uri.ptr + 3 + SIST_INDEX_ID_LEN)) {
case 1:
file = "treemap.csv";
break;
case 2:
file = "mime_agg.csv";
break;
case 3:
file = "size_agg.csv";
break;
case 4:
file = "date_agg.csv";
break;
default:
return;
database_t *db = web_get_database(arg_index_id);
if (db == NULL) {
LOG_DEBUGF("serve.c", "Could not get database for index: %s", arg_index_id);
HTTP_REPLY_NOT_FOUND
return;
}
char disposition[8192];
snprintf(disposition, sizeof(disposition),
"Content-Disposition: inline; filename=\"%s\"\r\nCache-Control: max-age=31536000\r\n", file);
cJSON *json = database_get_stats(db, stat_type);
char *json_str = cJSON_PrintUnformatted(json);
char full_path[PATH_MAX];
strcpy(full_path, index->path);
strcat(full_path, file);
web_send_headers(nc, 200, strlen(json_str), "Content-Type: application/json");
mg_send(nc, json_str, strlen(json_str));
struct mg_http_serve_opts opts = {};
mg_http_serve_file(nc, hm, full_path, &opts);
free(json_str);
cJSON_Delete(json);
}
void serve_index_html(struct mg_connection *nc, struct mg_http_message *hm) {