mirror of
https://github.com/simon987/sist2.git
synced 2025-10-31 16:06:53 +00:00
GitHub
commit
0bbb96b149
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -4,6 +4,3 @@
|
||||
[submodule "third-party/argparse"]
|
||||
path = third-party/argparse
|
||||
url = https://github.com/cofyc/argparse
|
||||
[submodule "third-party/mongoose"]
|
||||
path = third-party/mongoose
|
||||
url = https://github.com/cesanta/mongoose/
|
||||
|
||||
@ -28,12 +28,9 @@ add_executable(
|
||||
|
||||
# argparse
|
||||
third-party/argparse/argparse.h third-party/argparse/argparse.c
|
||||
third-party/mongoose/mongoose.h third-party/mongoose/mongoose.c
|
||||
|
||||
src/cli.c src/cli.h
|
||||
)
|
||||
|
||||
target_compile_definitions(sist2 PUBLIC MG_ENABLE_SSL=1)
|
||||
src/stats.c src/stats.h)
|
||||
|
||||
target_link_directories(sist2 PRIVATE BEFORE ${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib/)
|
||||
set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
|
||||
@ -41,9 +38,10 @@ set(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib)
|
||||
find_package(lmdb CONFIG REQUIRED)
|
||||
find_package(cJSON CONFIG REQUIRED)
|
||||
find_package(unofficial-glib CONFIG REQUIRED)
|
||||
find_package(unofficial-mongoose CONFIG REQUIRED)
|
||||
find_library(UUID_LIB NAMES uuid)
|
||||
|
||||
find_package(OpenSSL REQUIRED)
|
||||
#find_package(OpenSSL REQUIRED)
|
||||
|
||||
|
||||
target_include_directories(
|
||||
@ -74,6 +72,7 @@ if (SIST_DEBUG)
|
||||
sist2
|
||||
PRIVATE
|
||||
-fsanitize=address
|
||||
# -static
|
||||
)
|
||||
set_target_properties(
|
||||
sist2
|
||||
@ -104,7 +103,8 @@ target_link_libraries(
|
||||
cjson
|
||||
argparse
|
||||
unofficial::glib::glib
|
||||
OpenSSL::SSL OpenSSL::Crypto
|
||||
unofficial::mongoose::mongoose
|
||||
# OpenSSL::SSL OpenSSL::Crypto
|
||||
|
||||
${UUID_LIB}
|
||||
pthread
|
||||
|
||||
@ -21,12 +21,16 @@ sist2 (Simple incremental search tool)
|
||||
* Automatic tagging from file attributes via [user scripts](scripting/README.md)
|
||||
* Recursive scan inside archive files \*\*
|
||||
* OCR support with tesseract \*\*\*
|
||||
* Stats page & disk utilisation visualization
|
||||
|
||||
|
||||
\* See [format support](#format-support)
|
||||
\*\* See [Archive files](#archive-files)
|
||||
\*\*\* See [OCR](#ocr)
|
||||
|
||||
|
||||
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. Have an Elasticsearch (>= 6.X.X) instance running
|
||||
|
||||
@ -94,6 +94,13 @@ Made by simon987 <me@simon987.net>. Released under GPL-3.0
|
||||
* `-e "(^/usr/)|(^/var/)|(^/media/DRIVE-A/tmp/)|(^/media/DRIVE-B/Trash/)"` Exclude the
|
||||
`/usr`, `/var`, `/media/DRIVE-A/tmp`, `/media/DRIVE-B/Trash` directories
|
||||
* `--fast` Only index file names and mime type
|
||||
* `--treemap-threshold` Directories smaller than (`treemap-threshold` * `<total size of the index>`)
|
||||
will not be considered for the disk utilisation visualization; their size will be added to
|
||||
the parent directory. If the parent directory is still smaller than the threshold, it will also be "merged upwards"
|
||||
and so on.
|
||||
|
||||
In effect, smaller `treemap-threshold` values will yield a more detailed
|
||||
(but also a more cluttered and harder to read) visualization.
|
||||
|
||||
### Scan examples
|
||||
|
||||
@ -122,6 +129,11 @@ documents.idx/
|
||||
├── _index_139965425223424
|
||||
├── _index_139965433616128
|
||||
├── _index_139965442008832
|
||||
├── _index_139965442008832
|
||||
├── treemap.csv
|
||||
├── agg_mime.csv
|
||||
├── agg_date.csv
|
||||
├── add_size.csv
|
||||
└── thumbs
|
||||
├── data.mdb
|
||||
└── lock.mdb
|
||||
@ -137,6 +149,8 @@ database containing the thumbnails.
|
||||
The `descriptor.json` file contains general information about the index. The
|
||||
following fields are safe to modify manually: `root`, `name`, [rewrite_url](#rewrite_url) and `timestamp`.
|
||||
|
||||
The `.csv` are pre-computed aggregations necessary for the stats page.
|
||||
|
||||
|
||||
*Advanced usage*
|
||||
|
||||
|
||||
BIN
docs/stats.png
Normal file
BIN
docs/stats.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 167 KiB |
@ -4,7 +4,7 @@ rm -rf index.sist2/
|
||||
|
||||
rm src/static/js/bundle.js 2> /dev/null
|
||||
cat `ls src/static/js/*.min.js` > src/static/js/bundle.js
|
||||
cat src/static/js/{util,dom,search}.js >> src/static/js/bundle.js
|
||||
cat src/static/js/{util,dom}.js >> src/static/js/bundle.js
|
||||
|
||||
rm src/static/css/bundle*.css 2> /dev/null
|
||||
cat src/static/css/*.min.css > src/static/css/bundle.css
|
||||
|
||||
@ -2,9 +2,11 @@ files = [
|
||||
"src/static/css/bundle.css",
|
||||
"src/static/css/bundle_dark.css",
|
||||
"src/static/js/bundle.js",
|
||||
"src/static/js/search.js",
|
||||
"src/static/img/sprite-skin-flat.png",
|
||||
"src/static/img/sprite-skin-flat-dark.png",
|
||||
"src/static/search.html",
|
||||
"src/static/stats.html",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#define DEFAULT_BATCH_SIZE 100
|
||||
|
||||
#define DEFAULT_LISTEN_ADDRESS "localhost:4090"
|
||||
#define DEFAULT_TREEMAP_THRESHOLD 0.0005
|
||||
|
||||
const char* TESS_DATAPATHS[] = {
|
||||
"/usr/share/tessdata/",
|
||||
@ -180,6 +181,12 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
|
||||
ScanCtx.exclude = NULL;
|
||||
}
|
||||
|
||||
if (args->treemap_threshold_str == 0) {
|
||||
args->treemap_threshold = DEFAULT_TREEMAP_THRESHOLD;
|
||||
} else {
|
||||
args->treemap_threshold = atof(args->treemap_threshold_str);
|
||||
}
|
||||
|
||||
LOG_DEBUGF("cli.c", "arg quality=%f", args->quality)
|
||||
LOG_DEBUGF("cli.c", "arg size=%d", args->size)
|
||||
LOG_DEBUGF("cli.c", "arg content_size=%d", args->content_size)
|
||||
@ -195,6 +202,7 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) {
|
||||
LOG_DEBUGF("cli.c", "arg tesseract_path=%s", args->tesseract_path)
|
||||
LOG_DEBUGF("cli.c", "arg exclude=%s", args->exclude_regex)
|
||||
LOG_DEBUGF("cli.c", "arg fast=%d", args->fast)
|
||||
LOG_DEBUGF("cli.c", "arg treemap_threshold=%f", args->treemap_threshold)
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -22,6 +22,8 @@ typedef struct scan_args {
|
||||
const char *tesseract_path;
|
||||
char *exclude_regex;
|
||||
int fast;
|
||||
const char* treemap_threshold_str;
|
||||
double treemap_threshold;
|
||||
} scan_args_t;
|
||||
|
||||
scan_args_t *scan_args_create();
|
||||
|
||||
@ -2,6 +2,9 @@
|
||||
#include "src/sist.h"
|
||||
#include "src/ctx.h"
|
||||
|
||||
#include <mongoose.h>
|
||||
#include <pthread.h>
|
||||
|
||||
|
||||
void free_response(response_t *resp) {
|
||||
if (resp->body != NULL) {
|
||||
@ -74,19 +77,7 @@ subreq_ctx_t *http_req(const char *url, const char *extra_headers, const char *p
|
||||
|
||||
char address[8196];
|
||||
snprintf(address, sizeof(address), "tcp://%.*s:%u", (int) host.len, host.p, port);
|
||||
struct mg_connect_opts opts;
|
||||
memset(&opts, 0, sizeof(opts));
|
||||
opts.ssl_key = "./a.key";
|
||||
opts.ssl_cert = "./cert.crt";
|
||||
opts.ssl_ca_cert = "*";
|
||||
opts.ssl_server_name = "dev2.simon987.net:443";
|
||||
const char* tmp[256];
|
||||
opts.error_string = tmp;
|
||||
|
||||
struct mg_connection *nc = mg_connect_opt(&ctx->mgr, address, http_req_ev, opts);
|
||||
printf("%d\n", nc->err);
|
||||
|
||||
|
||||
struct mg_connection *nc = mg_connect(&ctx->mgr, address, http_req_ev);
|
||||
nc->user_data = &ctx->ev_data;
|
||||
mg_set_protocol_http_websocket(nc);
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
#define SIST2_WEB_H
|
||||
|
||||
#include "src/sist.h"
|
||||
#include "third-party/mongoose/mongoose.h"
|
||||
#include <mongoose.h>
|
||||
|
||||
typedef struct response {
|
||||
char *body;
|
||||
|
||||
@ -15,6 +15,8 @@
|
||||
#include "parsing/mime.h"
|
||||
#include "parsing/parse.h"
|
||||
|
||||
#include "stats.h"
|
||||
|
||||
#define DESCRIPTION "Lightning-fast file system indexer and search tool."
|
||||
|
||||
#define EPILOG "Made by simon987 <me@simon987.net>. Released under GPL-3.0"
|
||||
@ -207,6 +209,8 @@ void sist2_scan(scan_args_t *args) {
|
||||
tpool_wait(ScanCtx.pool);
|
||||
tpool_destroy(ScanCtx.pool);
|
||||
|
||||
generate_stats(&ScanCtx.index, args->treemap_threshold, ScanCtx.index.path);
|
||||
|
||||
if (args->incremental != NULL) {
|
||||
char dst_path[PATH_MAX];
|
||||
snprintf(store_path, PATH_MAX, "%sthumbs", args->incremental);
|
||||
@ -352,6 +356,8 @@ int main(int argc, const char *argv[]) {
|
||||
"which are installed on your machine)"),
|
||||
OPT_STRING('e', "exclude", &scan_args->exclude_regex, "Files that match this regex will not be scanned"),
|
||||
OPT_BOOLEAN(0, "fast", &scan_args->fast, "Only index file names & mime type"),
|
||||
OPT_STRING(0, "treemap-threshold", &scan_args->treemap_threshold_str, "Relative size threshold for treemap "
|
||||
"(see USAGE.md). DEFAULT: 0.0005"),
|
||||
|
||||
OPT_GROUP("Index options"),
|
||||
OPT_STRING(0, "es-url", &common_es_url, "Elasticsearch url with port. DEFAULT=http://localhost:9200"),
|
||||
|
||||
@ -121,7 +121,7 @@ body {
|
||||
background: #546b7a;
|
||||
}
|
||||
|
||||
.navbar a:hover {
|
||||
a:hover,.btn:hover {
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
@ -480,3 +480,34 @@ svg {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.stats-card {
|
||||
text-align: center;
|
||||
margin-top: 1em;
|
||||
padding: 1em;
|
||||
|
||||
box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
|
||||
border-radius: 0;
|
||||
border: none;
|
||||
|
||||
background: #212121;
|
||||
}
|
||||
|
||||
.graph {
|
||||
display: inline-block;
|
||||
width: 40%;
|
||||
}
|
||||
|
||||
.full-screen {
|
||||
position: absolute;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.stats-btn {
|
||||
float: right;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
#graphs-card svg text {
|
||||
fill: #eee;
|
||||
}
|
||||
|
||||
@ -347,3 +347,30 @@ mark {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.stats-card {
|
||||
text-align: center;
|
||||
margin-top: 1em;
|
||||
padding: 1em;
|
||||
|
||||
box-shadow: 0 .125rem .25rem rgba(0, 0, 0, .08) !important;
|
||||
border-radius: 0;
|
||||
border: none;
|
||||
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.graph {
|
||||
display: inline-block;
|
||||
width: 40%;
|
||||
}
|
||||
|
||||
.full-screen {
|
||||
position: absolute;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.stats-btn {
|
||||
float: right;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
2
src/static/js/d3.v5.min.js
vendored
Normal file
2
src/static/js/d3.v5.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
2
src/static/js/dom-to-image.min.js
vendored
Normal file
2
src/static/js/dom-to-image.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
@ -20,16 +20,6 @@ let size_max = 10000000000000;
|
||||
let date_min = null;
|
||||
let date_max = null;
|
||||
|
||||
const CONF = new Settings();
|
||||
|
||||
const _defaults = {
|
||||
display: "grid",
|
||||
fuzzy: true,
|
||||
highlight: true,
|
||||
sort: "score",
|
||||
searchInPath: false,
|
||||
};
|
||||
|
||||
SORT_MODES = {
|
||||
score: {
|
||||
text: "Relevance",
|
||||
@ -69,36 +59,6 @@ SORT_MODES = {
|
||||
},
|
||||
};
|
||||
|
||||
function Settings() {
|
||||
this.options = {};
|
||||
|
||||
this._onUpdate = function () {
|
||||
$("#fuzzyToggle").prop("checked", this.options.fuzzy);
|
||||
};
|
||||
|
||||
this.load = function () {
|
||||
const raw = window.localStorage.getItem("options");
|
||||
if (raw === null) {
|
||||
this.options = _defaults;
|
||||
} else {
|
||||
const j = JSON.parse(raw);
|
||||
if (!j || Object.keys(_defaults).some(k => !j.hasOwnProperty(k))) {
|
||||
this.options = _defaults;
|
||||
} else {
|
||||
this.options = j;
|
||||
}
|
||||
}
|
||||
|
||||
this._onUpdate();
|
||||
};
|
||||
|
||||
this.save = function () {
|
||||
window.localStorage.setItem("options", JSON.stringify(this.options));
|
||||
this._onUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function showEsError() {
|
||||
$.toast({
|
||||
heading: "Elasticsearch connection error",
|
||||
@ -112,27 +72,7 @@ function showEsError() {
|
||||
});
|
||||
}
|
||||
|
||||
jQuery["jsonPost"] = function (url, data) {
|
||||
return jQuery.ajax({
|
||||
url: url,
|
||||
type: "post",
|
||||
data: JSON.stringify(data),
|
||||
contentType: "application/json"
|
||||
}).fail(err => {
|
||||
showEsError();
|
||||
console.log(err);
|
||||
});
|
||||
};
|
||||
|
||||
window.onload = () => {
|
||||
$("#theme").on("click", () => {
|
||||
if (!document.cookie.includes("sist")) {
|
||||
document.cookie = "sist=dark";
|
||||
} else {
|
||||
document.cookie = "sist=; Max-Age=-99999999;";
|
||||
}
|
||||
window.location.reload();
|
||||
})
|
||||
CONF.load();
|
||||
};
|
||||
|
||||
@ -608,7 +548,7 @@ function getNextDepth(node) {
|
||||
}
|
||||
},
|
||||
size: 0
|
||||
}
|
||||
};
|
||||
|
||||
if (node.depth > 0) {
|
||||
q.query.bool.must = {
|
||||
@ -687,32 +627,3 @@ function createPathTree(target) {
|
||||
pathTree.on("node.click", handlePathTreeClick(pathTree));
|
||||
}
|
||||
|
||||
function updateSettings() {
|
||||
CONF.options.display = $("#settingDisplay").val();
|
||||
CONF.options.fuzzy = $("#settingFuzzy").prop("checked");
|
||||
CONF.options.highlight = $("#settingHighlight").prop("checked");
|
||||
CONF.options.searchInPath = $("#settingSearchInPath").prop("checked");
|
||||
CONF.save();
|
||||
|
||||
searchDebounced();
|
||||
|
||||
$.toast({
|
||||
heading: "Settings updated",
|
||||
text: "Settings saved to browser storage",
|
||||
stack: 3,
|
||||
bgColor: "#00a4bc",
|
||||
textColor: "#fff",
|
||||
position: 'bottom-right',
|
||||
hideAfter: 3000,
|
||||
loaderBg: "#08c7e8",
|
||||
});
|
||||
}
|
||||
|
||||
function loadSettings() {
|
||||
CONF.load();
|
||||
|
||||
$("#settingDisplay").val(CONF.options.display);
|
||||
$("#settingFuzzy").prop("checked", CONF.options.fuzzy);
|
||||
$("#settingHighlight").prop("checked", CONF.options.highlight);
|
||||
$("#settingSearchInPath").prop("checked", CONF.options.searchInPath);
|
||||
}
|
||||
|
||||
@ -86,3 +86,114 @@ function strUnescape(str) {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const CONF = new Settings();
|
||||
|
||||
const _defaults = {
|
||||
display: "grid",
|
||||
fuzzy: true,
|
||||
highlight: true,
|
||||
sort: "score",
|
||||
searchInPath: false,
|
||||
treemapType: "cascaded",
|
||||
treemapTiling: "squarify",
|
||||
treemapGroupingDepth: 3,
|
||||
treemapColor: "PuBuGn",
|
||||
treemapSize: "large",
|
||||
};
|
||||
|
||||
function loadSettings() {
|
||||
CONF.load();
|
||||
|
||||
$("#settingDisplay").val(CONF.options.display);
|
||||
$("#settingFuzzy").prop("checked", CONF.options.fuzzy);
|
||||
$("#settingHighlight").prop("checked", CONF.options.highlight);
|
||||
$("#settingSearchInPath").prop("checked", CONF.options.searchInPath);
|
||||
$("#settingTreemapTiling").val(CONF.options.treemapTiling);
|
||||
$("#settingTreemapGroupingDepth").val(CONF.options.treemapGroupingDepth);
|
||||
$("#settingTreemapColor").val(CONF.options.treemapColor);
|
||||
$("#settingTreemapSize").val(CONF.options.treemapSize);
|
||||
$("#settingTreemapType").val(CONF.options.treemapType);
|
||||
}
|
||||
|
||||
function Settings() {
|
||||
this.options = {};
|
||||
|
||||
this._onUpdate = function () {
|
||||
$("#fuzzyToggle").prop("checked", this.options.fuzzy);
|
||||
};
|
||||
|
||||
this.load = function () {
|
||||
const raw = window.localStorage.getItem("options");
|
||||
if (raw === null) {
|
||||
this.options = _defaults;
|
||||
} else {
|
||||
const j = JSON.parse(raw);
|
||||
if (!j || Object.keys(_defaults).some(k => !j.hasOwnProperty(k))) {
|
||||
this.options = _defaults;
|
||||
} else {
|
||||
this.options = j;
|
||||
}
|
||||
}
|
||||
|
||||
this._onUpdate();
|
||||
};
|
||||
|
||||
this.save = function () {
|
||||
window.localStorage.setItem("options", JSON.stringify(this.options));
|
||||
this._onUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
function updateSettings() {
|
||||
CONF.options.display = $("#settingDisplay").val();
|
||||
CONF.options.fuzzy = $("#settingFuzzy").prop("checked");
|
||||
CONF.options.highlight = $("#settingHighlight").prop("checked");
|
||||
CONF.options.searchInPath = $("#settingSearchInPath").prop("checked");
|
||||
CONF.options.treemapTiling = $("#settingTreemapTiling").val();
|
||||
CONF.options.treemapGroupingDepth = $("#settingTreemapGroupingDepth").val();
|
||||
CONF.options.treemapColor = $("#settingTreemapColor").val();
|
||||
CONF.options.treemapSize = $("#settingTreemapSize").val();
|
||||
CONF.options.treemapType = $("#settingTreemapType").val();
|
||||
CONF.save();
|
||||
|
||||
if (typeof searchDebounced !== "undefined") {
|
||||
searchDebounced();
|
||||
}
|
||||
|
||||
if (typeof updateStats !== "undefined") {
|
||||
updateStats();
|
||||
}
|
||||
|
||||
$.toast({
|
||||
heading: "Settings updated",
|
||||
text: "Settings saved to browser storage",
|
||||
stack: 3,
|
||||
bgColor: "#00a4bc",
|
||||
textColor: "#fff",
|
||||
position: 'bottom-right',
|
||||
hideAfter: 3000,
|
||||
loaderBg: "#08c7e8",
|
||||
});
|
||||
}
|
||||
|
||||
jQuery["jsonPost"] = function (url, data) {
|
||||
return jQuery.ajax({
|
||||
url: url,
|
||||
type: "post",
|
||||
data: JSON.stringify(data),
|
||||
contentType: "application/json"
|
||||
}).fail(err => {
|
||||
showEsError();
|
||||
console.log(err);
|
||||
});
|
||||
};
|
||||
|
||||
function toggleTheme() {
|
||||
if (!document.cookie.includes("sist")) {
|
||||
document.cookie = "sist=dark;SameSite=Strict";
|
||||
} else {
|
||||
document.cookie = "sist=; Max-Age=-99999999;";
|
||||
}
|
||||
window.location.reload();
|
||||
}
|
||||
|
||||
@ -13,8 +13,9 @@
|
||||
<a class="navbar-brand" href="/">sist2</a>
|
||||
<span class="badge badge-pill version">2.3.0</span>
|
||||
<span class="tagline">Lightning-fast file system indexer and search tool </span>
|
||||
<button style="margin-left: auto" class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings</button>
|
||||
<a id="theme" class="btn" title="Toggle theme" href="/">Theme</a>
|
||||
<a class="btn ml-auto" href="/stats">Stats</a>
|
||||
<button class="btn" type="button" data-toggle="modal" data-target="#settings" onclick="loadSettings()">Settings</button>
|
||||
<button class="btn" title="Toggle theme" onclick="toggleTheme()">Theme</button>
|
||||
</nav>
|
||||
|
||||
<div class="container">
|
||||
@ -197,8 +198,64 @@
|
||||
<option value="list">List</option>
|
||||
</select>
|
||||
|
||||
<hr/>
|
||||
<h4>Stats</h4>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapType">Treemap type</label>
|
||||
<select id="settingTreemapType" class="form-control form-control-sm">
|
||||
<option value="cascaded">Cascaded</option>
|
||||
<option value="flat">Flat (compact)</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapTiling">Treemap tiling</label>
|
||||
<select id="settingTreemapTiling" class="form-control form-control-sm">
|
||||
<option value="binary">Binary</option>
|
||||
<option value="squarify">Squarify</option>
|
||||
<option value="slice">Slice</option>
|
||||
<option value="dice">Dice</option>
|
||||
<option value="sliceDice">Slide & Dice</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapGroupingDepth">Treemap color grouping depth (flat)</label>
|
||||
<input type="number" class="form-control" id="settingTreemapGroupingDepth" min="1" max="10">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapColor">Treemap color (cascaded)</label>
|
||||
<select id="settingTreemapColor" class="form-control form-control-sm">
|
||||
<option value="PuBuGn">Purple-Blue-Green</option>
|
||||
<option value="PuRd">Purple-Red</option>
|
||||
<option value="PuBu">Purple-Blue</option>
|
||||
<option value="YlOrBr">Yellow-Orange-Brown</option>
|
||||
<option value="YlOrRd">Yellow-Orange-Red</option>
|
||||
<option value="YlGn">Yellow-Green</option>
|
||||
<option value="YlGnBu">Yellow-Green-Blue</option>
|
||||
<option value="Plasma">Plasma</option>
|
||||
<option value="Magma">Magma</option>
|
||||
<option value="Inferno">Inferno</option>
|
||||
<option value="Viridis">Viridis</option>
|
||||
<option value="Turbo">Turbo</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapSize">Treemap size</label>
|
||||
<select id="settingTreemapSize" class="form-control form-control-sm">
|
||||
<option value="small">Small</option>
|
||||
<option value="medium">Medium</option>
|
||||
<option value="large">Large</option>
|
||||
<option value="x-large">X-Large</option>
|
||||
<option value="xx-large">XX-Large</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<button style="float: right" class="btn btn-primary" onclick="updateSettings()">Update settings</button>
|
||||
<button class="btn btn-primary ml-auto" onclick="updateSettings()">Update settings</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -223,6 +280,7 @@
|
||||
<div id="searchResults"></div>
|
||||
</div>
|
||||
|
||||
<script src="js" type="text/javascript"></script>
|
||||
<script src="jslib" type="text/javascript"></script>
|
||||
<script src="jssearch" type="text/javascript"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
789
src/static/stats.html
Normal file
789
src/static/stats.html
Normal file
@ -0,0 +1,789 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>sist2 - Stats</title>
|
||||
<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'/>
|
||||
<link href="css" rel="stylesheet" type="text/css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<nav class="navbar navbar-expand-lg">
|
||||
<a class="navbar-brand" href="/">sist2</a>
|
||||
<span class="badge badge-pill version">2.3.0</span>
|
||||
<span class="tagline">Lightning-fast file system indexer and search tool </span>
|
||||
<a style="margin-left: auto" class="btn" href="/">Back</a>
|
||||
<button class="btn" type="button" data-toggle="modal" data-target="#settings"
|
||||
onclick="loadSettings()">Settings
|
||||
</button>
|
||||
<button class="btn" title="Toggle theme" onclick="toggleTheme()">Theme</button>
|
||||
</nav>
|
||||
|
||||
<div class="container pb-3">
|
||||
<div class="card">
|
||||
<div class="card-body">
|
||||
|
||||
<label for="indices">Index</label>
|
||||
<select id="indices" onchange="updateStats()"></select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="treemap-card" class="stats-card">
|
||||
<button class="btn stats-btn" onclick="fullScreen('treemap-card')">Enlarge</button>
|
||||
<button class="btn stats-btn" onclick="exportTreemap()">Export</button>
|
||||
<svg id="treemap"></svg>
|
||||
</div>
|
||||
|
||||
<div id="graphs-card" class="stats-card">
|
||||
<button class="btn stats-btn" onclick="fullScreen('graphs-card')">Enlarge</button>
|
||||
<div class="graph">
|
||||
<svg id="agg_mime_size"></svg>
|
||||
</div>
|
||||
<div class="graph">
|
||||
<svg id="agg_mime_count"></svg>
|
||||
</div>
|
||||
<div class="graph">
|
||||
<svg id="date_histogram"></svg>
|
||||
</div>
|
||||
<div class="graph">
|
||||
<svg id="size_histogram"></svg>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal" id="settings" tabindex="-1" role="dialog" aria-labelledby="modal-title" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered" role="document">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title">Settings</h5>
|
||||
<button type="button" class="close" data-dismiss="modal" aria-label="Close">
|
||||
<span aria-hidden="true">×</span>
|
||||
</button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<div class="custom-control custom-checkbox">
|
||||
<input type="checkbox" class="custom-control-input" id="settingHighlight">
|
||||
<label class="custom-control-label" for="settingHighlight">Enable highlighting</label>
|
||||
</div>
|
||||
|
||||
<div class="custom-control custom-checkbox">
|
||||
<input type="checkbox" class="custom-control-input" id="settingFuzzy">
|
||||
<label class="custom-control-label" for="settingFuzzy">Set fuzzy search by default</label>
|
||||
</div>
|
||||
|
||||
<div class="custom-control custom-checkbox">
|
||||
<input type="checkbox" class="custom-control-input" id="settingSearchInPath">
|
||||
<label class="custom-control-label" for="settingSearchInPath">Enable matching query against document
|
||||
path</label>
|
||||
</div>
|
||||
|
||||
<label for="settingDisplay">Display</label>
|
||||
<select id="settingDisplay" class="form-control form-control-sm">
|
||||
<option value="grid">Grid</option>
|
||||
<option value="list">List</option>
|
||||
</select>
|
||||
|
||||
<hr/>
|
||||
<h4>Stats</h4>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapType">Treemap type</label>
|
||||
<select id="settingTreemapType" class="form-control form-control-sm">
|
||||
<option value="cascaded">Cascaded</option>
|
||||
<option value="flat">Flat (compact)</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapTiling">Treemap tiling</label>
|
||||
<select id="settingTreemapTiling" class="form-control form-control-sm">
|
||||
<option value="binary">Binary</option>
|
||||
<option value="squarify">Squarify</option>
|
||||
<option value="slice">Slice</option>
|
||||
<option value="dice">Dice</option>
|
||||
<option value="sliceDice">Slide & Dice</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapGroupingDepth">Treemap color grouping depth (flat)</label>
|
||||
<input type="number" class="form-control" id="settingTreemapGroupingDepth" min="1" max="10">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapColor">Treemap color (cascaded)</label>
|
||||
<select id="settingTreemapColor" class="form-control form-control-sm">
|
||||
<option value="PuBuGn">Purple-Blue-Green</option>
|
||||
<option value="PuRd">Purple-Red</option>
|
||||
<option value="PuBu">Purple-Blue</option>
|
||||
<option value="YlOrBr">Yellow-Orange-Brown</option>
|
||||
<option value="YlOrRd">Yellow-Orange-Red</option>
|
||||
<option value="YlGn">Yellow-Green</option>
|
||||
<option value="YlGnBu">Yellow-Green-Blue</option>
|
||||
<option value="Plasma">Plasma</option>
|
||||
<option value="Magma">Magma</option>
|
||||
<option value="Inferno">Inferno</option>
|
||||
<option value="Viridis">Viridis</option>
|
||||
<option value="Turbo">Turbo</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="settingTreemapSize">Treemap size</label>
|
||||
<select id="settingTreemapSize" class="form-control form-control-sm">
|
||||
<option value="small">Small</option>
|
||||
<option value="medium">Medium</option>
|
||||
<option value="large">Large</option>
|
||||
<option value="x-large">X-Large</option>
|
||||
<option value="xx-large">XX-Large</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<button class="btn btn-primary float-right" onclick="updateSettings()">Update settings</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="jslib" type="text/javascript"></script>
|
||||
<script>
|
||||
let width;
|
||||
let height;
|
||||
let indexMap = {};
|
||||
|
||||
const barHeight = 20;
|
||||
const ordinalColor = d3.scaleOrdinal(d3.schemeCategory10);
|
||||
|
||||
const formatSI = d3.format("~s");
|
||||
|
||||
|
||||
const TILING_MODES = {
|
||||
"squarify": d3.treemapSquarify,
|
||||
"binary": d3.treemapBinary,
|
||||
"sliceDice": d3.treemapSliceDice,
|
||||
"slice": d3.treemapSlice,
|
||||
"dice": d3.treemapDice,
|
||||
};
|
||||
|
||||
const COLORS = {
|
||||
"PuBuGn": d3.interpolatePuBuGn,
|
||||
"PuRd": d3.interpolatePuRd,
|
||||
"PuBu": d3.interpolatePuBu,
|
||||
"YlOrBr": d3.interpolateYlOrBr,
|
||||
"YlOrRd": d3.interpolateYlOrRd,
|
||||
"YlGn": d3.interpolateYlGn,
|
||||
"YlGnBu": d3.interpolateYlGnBu,
|
||||
"Plasma": d3.interpolatePlasma,
|
||||
"Magma": d3.interpolateMagma,
|
||||
"Inferno": d3.interpolateInferno,
|
||||
"Viridis": d3.interpolateViridis,
|
||||
"Turbo": d3.interpolateTurbo,
|
||||
};
|
||||
|
||||
const SIZES = {
|
||||
"small": [800, 600],
|
||||
"medium": [1300, 750],
|
||||
"large": [1900, 900],
|
||||
"x-large": [2800, 1700],
|
||||
"xx-large": [3600, 2000],
|
||||
};
|
||||
|
||||
const fillOpacity = document.cookie.includes("sist") ? 0.9 : 0.6;
|
||||
|
||||
const uids = {};
|
||||
|
||||
function uid(name) {
|
||||
let id = uids[name] || 0;
|
||||
uids[name] = id + 1;
|
||||
return name + id;
|
||||
}
|
||||
|
||||
const burrow = function (table, addSelfDir) {
|
||||
const root = {};
|
||||
table.forEach(row => {
|
||||
let layer = root;
|
||||
|
||||
row.taxonomy.forEach(key => {
|
||||
layer[key] = key in layer ? layer[key] : {};
|
||||
layer = layer[key];
|
||||
});
|
||||
if (Object.keys(layer).length === 0) {
|
||||
layer["$size$"] = row.size;
|
||||
} else if (addSelfDir) {
|
||||
layer["."] = {
|
||||
"$size$": row.size,
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
const descend = function (obj, depth) {
|
||||
return Object.keys(obj).filter(k => k !== "$size$").map(k => {
|
||||
const child = {
|
||||
name: k,
|
||||
depth: depth,
|
||||
value: 0,
|
||||
children: descend(obj[k], depth + 1)
|
||||
};
|
||||
if ("$size$" in obj[k]) {
|
||||
child.value = obj[k]["$size$"];
|
||||
}
|
||||
return child;
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
name: `[${indexMap[$("#indices").val()]}]`,
|
||||
children: descend(root, 1),
|
||||
value: 0,
|
||||
depth: 0,
|
||||
}
|
||||
};
|
||||
|
||||
function flatTreemap(data, svg) {
|
||||
const root = d3.treemap()
|
||||
.tile(TILING_MODES[CONF.options.treemapTiling])
|
||||
.size([width, height])
|
||||
.padding(1)
|
||||
.round(true)(
|
||||
d3.hierarchy(data)
|
||||
.sum(d => d.value)
|
||||
.sort((a, b) => b.value - a.value)
|
||||
);
|
||||
|
||||
const leaf = svg.selectAll("g")
|
||||
.data(root.leaves())
|
||||
.join("g")
|
||||
.attr("transform", d => `translate(${d.x0},${d.y0})`);
|
||||
|
||||
leaf.append("title")
|
||||
.text(d => `${d.ancestors().reverse().map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
|
||||
|
||||
leaf.append("rect")
|
||||
.attr("id", d => (d.leafUid = uid("leaf")))
|
||||
.attr("fill", d => {
|
||||
while (d.depth > CONF.options.treemapGroupingDepth) d = d.parent;
|
||||
return ordinalColor(d.data.name);
|
||||
})
|
||||
.attr("fill-opacity", fillOpacity)
|
||||
.attr("width", d => d.x1 - d.x0)
|
||||
.attr("height", d => d.y1 - d.y0);
|
||||
|
||||
leaf.append("clipPath")
|
||||
.attr("id", d => (d.clipUid = uid("clip")))
|
||||
.append("use")
|
||||
.attr("href", d => `#${d.leafUid}`);
|
||||
|
||||
leaf.append("text")
|
||||
.attr("clip-path", d => `url(#${d.clipUid})`)
|
||||
.selectAll("tspan")
|
||||
.data(d => {
|
||||
if (d.data.name === ".") {
|
||||
d = d.parent;
|
||||
}
|
||||
return [d.data.name, humanFileSize(d.value)]
|
||||
})
|
||||
.join("tspan")
|
||||
.attr("x", 2)
|
||||
.attr("y", (d, i, nodes) => `${i === 0 ? 1.1 : 2.3}em`)
|
||||
.text(d => d);
|
||||
}
|
||||
|
||||
function cascade(root, offset) {
|
||||
const x = new Map;
|
||||
const y = new Map;
|
||||
return root.eachAfter(d => {
|
||||
if (d.children && d.children.length !== 0) {
|
||||
x.set(d, 1 + d3.max(d.children, c => c.x1 === d.x1 - offset ? x.get(c) : NaN));
|
||||
y.set(d, 1 + d3.max(d.children, c => c.y1 === d.y1 - offset ? y.get(c) : NaN));
|
||||
} else {
|
||||
x.set(d, 0);
|
||||
y.set(d, 0);
|
||||
}
|
||||
}).eachBefore(d => {
|
||||
d.x1 -= 2 * offset * x.get(d);
|
||||
d.y1 -= 2 * offset * y.get(d);
|
||||
});
|
||||
}
|
||||
|
||||
function cascadeTreemap(data, svg) {
|
||||
|
||||
const root = cascade(
|
||||
d3.treemap()
|
||||
.size([width, height])
|
||||
.tile(TILING_MODES[CONF.options.treemapTiling])
|
||||
.paddingOuter(3)
|
||||
.paddingTop(16)
|
||||
.paddingInner(1)
|
||||
.round(true)(
|
||||
d3.hierarchy(data)
|
||||
.sum(d => d.value)
|
||||
.sort((a, b) => b.value - a.value)
|
||||
),
|
||||
3 // treemap.paddingOuter
|
||||
);
|
||||
|
||||
const maxDepth = Math.max(...root.descendants().map(d => d.depth));
|
||||
const color = d3.scaleSequential([maxDepth, -1], COLORS[CONF.options.treemapColor]);
|
||||
|
||||
svg.append("filter")
|
||||
.attr("id", "shadow")
|
||||
.append("feDropShadow")
|
||||
.attr("flood-opacity", 0.3)
|
||||
.attr("dx", 0)
|
||||
.attr("stdDeviation", 3);
|
||||
|
||||
const node = svg.selectAll("g")
|
||||
.data(
|
||||
d3.nest()
|
||||
.key(d => d.depth).sortKeys(d3.ascending)
|
||||
.entries(root.descendants())
|
||||
)
|
||||
.join("g")
|
||||
.attr("filter", "url(#shadow)")
|
||||
.selectAll("g")
|
||||
.data(d => d.values)
|
||||
.join("g")
|
||||
.attr("transform", d => `translate(${d.x0},${d.y0})`);
|
||||
|
||||
node.append("title")
|
||||
.text(d => `${d.ancestors().reverse().splice(1).map(d => d.data.name).join("/")}\n${humanFileSize(d.value)}`);
|
||||
|
||||
node.append("rect")
|
||||
.attr("id", d => (d.nodeUid = uid("node")))
|
||||
.attr("fill", d => color(d.depth))
|
||||
.attr("width", d => d.x1 - d.x0)
|
||||
.attr("height", d => d.y1 - d.y0);
|
||||
|
||||
node.append("clipPath")
|
||||
.attr("id", d => (d.clipUid = uid("clip")))
|
||||
.append("use")
|
||||
.attr("href", d => `#${d.nodeUid}`);
|
||||
|
||||
node.append("text")
|
||||
.attr("fill", d => d3.hsl(color(d.depth)).l > .5 ? "#333" : "#eee")
|
||||
.attr("clip-path", d => `url(#${d.clipUid})`)
|
||||
.selectAll("tspan")
|
||||
.data(d => [d.data.name, humanFileSize(d.value)])
|
||||
.join("tspan")
|
||||
.text(d => d);
|
||||
|
||||
node.filter(d => d.children).selectAll("tspan")
|
||||
.attr("dx", 3)
|
||||
.attr("y", 13);
|
||||
|
||||
node.filter(d => !d.children).selectAll("tspan")
|
||||
.attr("x", 3)
|
||||
.attr("y", (d, i, nodes) => `${i === 0 ? 1.1 : 2.3}em`);
|
||||
}
|
||||
|
||||
|
||||
function mimeBarSize(data, svg) {
|
||||
|
||||
const margin = {
|
||||
top: 50,
|
||||
right: 0,
|
||||
bottom: 10,
|
||||
left: Math.max(
|
||||
d3.max(data.sort((a, b) => b.count - a.count).slice(0, 15), d => d.mime.length) * 6,
|
||||
d3.max(data.sort((a, b) => b.size - a.size).slice(0, 15), d => d.mime.length) * 6,
|
||||
)
|
||||
};
|
||||
|
||||
data.forEach(d => {
|
||||
d.name = d.mime;
|
||||
d.value = Number(d.size);
|
||||
});
|
||||
data = data.sort((a, b) => b.value - a.value).slice(0, 15);
|
||||
|
||||
const width = 550;
|
||||
const height = Math.ceil((data.length + 0.1) * barHeight) + margin.top + margin.bottom;
|
||||
|
||||
svg.selectAll("*").remove();
|
||||
svg.attr("viewBox", [0, 0, width, height]);
|
||||
|
||||
const y = d3.scaleBand()
|
||||
.domain(d3.range(data.length))
|
||||
.rangeRound([margin.top, height - margin.bottom]);
|
||||
|
||||
const x = d3.scaleLinear()
|
||||
.domain([0, d3.max(data, d => d.value)])
|
||||
.range([margin.left, width - margin.right]);
|
||||
|
||||
svg.append("g")
|
||||
.attr("fill-opacity", fillOpacity)
|
||||
.selectAll("rect")
|
||||
.data(data)
|
||||
.join("rect")
|
||||
.attr("fill", d => ordinalColor(d.name))
|
||||
.attr("x", x(0))
|
||||
.attr("y", (d, i) => y(i))
|
||||
.attr("width", d => x(d.value) - x(0))
|
||||
.attr("height", y.bandwidth())
|
||||
.append("title")
|
||||
.text(d => formatSI(d.value));
|
||||
|
||||
svg.append("g")
|
||||
.attr("transform", `translate(0,${margin.top})`)
|
||||
.call(d3.axisTop(x).ticks(width / 80, data.format).tickFormat(formatSI))
|
||||
.call(g => g.select(".domain").remove());
|
||||
|
||||
svg.append("g")
|
||||
.attr("transform", `translate(${margin.left},0)`)
|
||||
.call(d3.axisLeft(y).tickFormat(i => data[i].name).tickSizeOuter(0));
|
||||
|
||||
svg.append("text")
|
||||
.attr("x", (width / 2))
|
||||
.attr("y", (margin.top / 2))
|
||||
.attr("text-anchor", "middle")
|
||||
.style("font-size", "16px")
|
||||
.text("Size distribution by MIME type");
|
||||
}
|
||||
|
||||
function mimeBarCount(data, svg) {
|
||||
|
||||
const margin = {
|
||||
top: 50,
|
||||
right: 0,
|
||||
bottom: 10,
|
||||
left: Math.max(
|
||||
d3.max(data.sort((a, b) => b.count - a.count).slice(0, 15), d => d.mime.length) * 6,
|
||||
d3.max(data.sort((a, b) => b.size - a.size).slice(0, 15), d => d.mime.length) * 6,
|
||||
)
|
||||
};
|
||||
|
||||
data.forEach(d => {
|
||||
d.name = d.mime;
|
||||
d.value = Number(d.count);
|
||||
});
|
||||
|
||||
data = data.sort((a, b) => b.value - a.value).slice(0, 15);
|
||||
|
||||
const width = 550;
|
||||
const height = Math.ceil((data.length + 0.1) * barHeight) + margin.top + margin.bottom;
|
||||
|
||||
svg.selectAll("*").remove();
|
||||
svg.attr("viewBox", [0, 0, width, height]);
|
||||
|
||||
const y = d3.scaleBand()
|
||||
.domain(d3.range(data.length))
|
||||
.rangeRound([margin.top, height - margin.bottom]);
|
||||
|
||||
const x = d3.scaleLinear()
|
||||
.domain([0, d3.max(data, d => d.value)])
|
||||
.range([margin.left, width - margin.right]);
|
||||
|
||||
svg.append("g")
|
||||
.attr("fill-opacity", fillOpacity)
|
||||
.selectAll("rect")
|
||||
.data(data)
|
||||
.join("rect")
|
||||
.attr("fill", d => ordinalColor(d.name))
|
||||
.attr("x", x(0))
|
||||
.attr("y", (d, i) => y(i))
|
||||
.attr("width", d => x(d.value) - x(0))
|
||||
.attr("height", y.bandwidth())
|
||||
.append("title")
|
||||
.text(d => d3.format(",")(d.value));
|
||||
|
||||
svg.append("g")
|
||||
.attr("transform", `translate(0,${margin.top})`)
|
||||
.call(d3.axisTop(x).ticks(width / 80, data.format).tickFormat(formatSI))
|
||||
.call(g => g.select(".domain").remove());
|
||||
|
||||
svg.append("g")
|
||||
.attr("transform", `translate(${margin.left},0)`)
|
||||
.call(d3.axisLeft(y).tickFormat(i => data[i].name).tickSizeOuter(0));
|
||||
|
||||
svg.append("text")
|
||||
.attr("x", (width / 2))
|
||||
.attr("y", (margin.top / 2))
|
||||
.attr("text-anchor", "middle")
|
||||
.style("font-size", "16px")
|
||||
.text("File count distribution by MIME type");
|
||||
}
|
||||
|
||||
function dateHistogram(data, svg) {
|
||||
|
||||
let bins = data.map(d => {
|
||||
return {
|
||||
length: Number(d.count),
|
||||
x0: Number(d.bucket),
|
||||
x1: Number(d.bucket) + 2629800
|
||||
}
|
||||
});
|
||||
bins.sort((a, b) => a.length - b.length);
|
||||
|
||||
const margin = {
|
||||
top: 50,
|
||||
right: 20,
|
||||
bottom: 70,
|
||||
left: 40
|
||||
};
|
||||
|
||||
const thresh = d3.quantile(bins, 0.9, d => d.length);
|
||||
bins = bins.filter(d => d.length > thresh);
|
||||
|
||||
const width = 550;
|
||||
const height = 450;
|
||||
|
||||
svg.selectAll("*").remove();
|
||||
svg.attr("viewBox", [0, 0, width, height]);
|
||||
|
||||
const y = d3.scaleLinear()
|
||||
.domain([0, d3.max(bins, d => d.length)]).nice()
|
||||
.range([height - margin.bottom, margin.top]);
|
||||
|
||||
const x = d3.scaleLinear()
|
||||
.domain(d3.extent(bins, d => d.x0)).nice()
|
||||
.range([margin.left, width - margin.right]);
|
||||
|
||||
svg.append("g")
|
||||
.attr("fill", "steelblue")
|
||||
.selectAll("rect")
|
||||
.data(bins)
|
||||
.join("rect")
|
||||
.attr("x", d => x(d.x0) + 1)
|
||||
.attr("width", d => Math.max(1, x(d.x1) - x(d.x0) - 1))
|
||||
.attr("y", d => y(d.length))
|
||||
.attr("height", d => y(0) - y(d.length))
|
||||
.call(g => g
|
||||
.append("title")
|
||||
.text(d => d.length)
|
||||
);
|
||||
|
||||
svg.append("g")
|
||||
.attr("transform", `translate(0,${height - margin.bottom})`)
|
||||
.call(
|
||||
d3.axisBottom(x)
|
||||
.ticks(width / 30)
|
||||
.tickSizeOuter(0)
|
||||
.tickFormat(t => d3.timeFormat("%Y-%m-%d")(d3.utcParse("%s")(t)))
|
||||
)
|
||||
.call(g => g
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("dx", "-.8em")
|
||||
.attr("dy", ".15em")
|
||||
.attr("transform", "rotate(-65)")
|
||||
)
|
||||
.call(g => g.append("text")
|
||||
.attr("x", width - margin.right)
|
||||
.attr("y", -4)
|
||||
.attr("fill", "currentColor")
|
||||
.attr("font-weight", "bold")
|
||||
.attr("text-anchor", "end")
|
||||
.text("mtime")
|
||||
);
|
||||
|
||||
svg.append("g")
|
||||
.attr("transform", `translate(${margin.left},0)`)
|
||||
.call(
|
||||
d3.axisLeft(y)
|
||||
.ticks(height / 40)
|
||||
.tickFormat(t => formatSI(t))
|
||||
)
|
||||
.call(g => g.select(".domain").remove())
|
||||
.call(g => g.select(".tick:last-of-type text").clone()
|
||||
.attr("x", 4)
|
||||
.attr("text-anchor", "start")
|
||||
.attr("font-weight", "bold")
|
||||
.text("File count"));
|
||||
|
||||
svg.append("text")
|
||||
.attr("x", (width / 2))
|
||||
.attr("y", (margin.top / 2))
|
||||
.attr("text-anchor", "middle")
|
||||
.style("font-size", "16px")
|
||||
.text("File modification time distribution");
|
||||
}
|
||||
|
||||
function sizeHistogram(data, svg) {
|
||||
|
||||
let bins = data.map(d => {
|
||||
return {
|
||||
length: Number(d.count),
|
||||
x0: Number(d.bucket),
|
||||
x1: Number(d.bucket) + (5 * 1024 * 1024)
|
||||
}
|
||||
});
|
||||
bins = bins.sort((a, b) => b.length - a.length).slice(0, 25);
|
||||
|
||||
const margin = {
|
||||
top: 50,
|
||||
right: 20,
|
||||
bottom: 70,
|
||||
left: 40
|
||||
};
|
||||
|
||||
const width = 550;
|
||||
const height = 450;
|
||||
|
||||
svg.selectAll("*").remove();
|
||||
svg.attr("viewBox", [0, 0, width, height]);
|
||||
|
||||
const y = d3.scaleLinear()
|
||||
.domain([0, d3.max(bins, d => d.length)])
|
||||
.range([height - margin.bottom, margin.top]);
|
||||
|
||||
const x = d3.scaleLinear()
|
||||
.domain(d3.extent(bins, d => d.x0)).nice()
|
||||
.range([margin.left, width - margin.right]);
|
||||
|
||||
svg.append("g")
|
||||
.attr("fill", "steelblue")
|
||||
.selectAll("rect")
|
||||
.data(bins)
|
||||
.join("rect")
|
||||
.attr("x", d => x(d.x0) + 1)
|
||||
.attr("width", d => Math.max(1, x(d.x1) - x(d.x0) - 1))
|
||||
.attr("y", d => y(d.length))
|
||||
.attr("height", d => y(0) - y(d.length))
|
||||
.call(g => g
|
||||
.append("title")
|
||||
.text(d => d.length)
|
||||
);
|
||||
|
||||
svg.append("g")
|
||||
.attr("transform", `translate(0,${height - margin.bottom})`)
|
||||
.call(
|
||||
d3.axisBottom(x)
|
||||
.ticks(width / 30)
|
||||
.tickSizeOuter(0)
|
||||
.tickFormat(formatSI)
|
||||
)
|
||||
.call(g => g
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("dx", "-.8em")
|
||||
.attr("dy", ".15em")
|
||||
.attr("transform", "rotate(-65)")
|
||||
)
|
||||
.call(g => g.append("text")
|
||||
.attr("x", width - margin.right)
|
||||
.attr("y", -4)
|
||||
.attr("fill", "currentColor")
|
||||
.attr("font-weight", "bold")
|
||||
.attr("text-anchor", "end")
|
||||
.text("size (bytes)")
|
||||
);
|
||||
|
||||
svg.append("g")
|
||||
.attr("transform", `translate(${margin.left},0)`)
|
||||
.call(
|
||||
d3.axisLeft(y)
|
||||
.ticks(height / 40)
|
||||
.tickFormat(t => formatSI(t))
|
||||
)
|
||||
.call(g => g.select(".domain").remove())
|
||||
.call(g => g.select(".tick:last-of-type text").clone()
|
||||
.attr("x", 4)
|
||||
.attr("text-anchor", "start")
|
||||
.attr("font-weight", "bold")
|
||||
.text("File count"));
|
||||
|
||||
svg.append("text")
|
||||
.attr("x", (width / 2))
|
||||
.attr("y", (margin.top / 2))
|
||||
.attr("text-anchor", "middle")
|
||||
.style("font-size", "16px")
|
||||
.text("File size distribution");
|
||||
}
|
||||
|
||||
|
||||
function updateStats() {
|
||||
width = SIZES[CONF.options.treemapSize][0];
|
||||
height = SIZES[CONF.options.treemapSize][1];
|
||||
|
||||
const treemapSvg = d3.select("#treemap");
|
||||
const mimeSvgSize = d3.select("#agg_mime_size");
|
||||
const mimeSvgCount = d3.select("#agg_mime_count");
|
||||
const dateHistogramSvg = d3.select("#date_histogram");
|
||||
const sizeHistogramSvg = d3.select("#size_histogram");
|
||||
|
||||
const indexId = $("#indices").val();
|
||||
|
||||
d3.csv(`/s/${indexId}/1`).then(tabularData => {
|
||||
tabularData.forEach(row => {
|
||||
row.taxonomy = row.path.split("/");
|
||||
row.size = Number(row.size);
|
||||
});
|
||||
|
||||
if (CONF.options.treemapType === "cascaded") {
|
||||
const data = burrow(tabularData, false);
|
||||
cascadeTreemap(data, treemapSvg);
|
||||
} else {
|
||||
const data = burrow(tabularData.sort((a, b) => b.taxonomy.length - a.taxonomy.length), true);
|
||||
flatTreemap(data, treemapSvg);
|
||||
}
|
||||
});
|
||||
|
||||
d3.csv(`/s/${indexId}/2`).then(tabularData => {
|
||||
mimeBarSize(tabularData.slice(), mimeSvgSize);
|
||||
mimeBarCount(tabularData.slice(), mimeSvgCount);
|
||||
});
|
||||
|
||||
d3.csv(`/s/${indexId}/3`).then(tabularData => {
|
||||
sizeHistogram(tabularData, sizeHistogramSvg);
|
||||
});
|
||||
|
||||
d3.csv(`/s/${indexId}/4`).then(tabularData => {
|
||||
dateHistogram(tabularData, dateHistogramSvg);
|
||||
});
|
||||
|
||||
treemapSvg.selectAll("*").remove();
|
||||
treemapSvg.attr("viewBox", [0, 0, width, height])
|
||||
.attr("xmlns", "http://www.w3.org/2000/svg")
|
||||
.attr("xmlns:xlink", "http://www.w3.org/1999/xlink")
|
||||
.attr("version", "1.1")
|
||||
.style("overflow", "visible")
|
||||
.style("font", "10px sans-serif");
|
||||
|
||||
}
|
||||
|
||||
window.onload = function () {
|
||||
CONF.load();
|
||||
|
||||
$.jsonPost("i").then(resp => {
|
||||
const select = $("#indices");
|
||||
|
||||
const urlIndices = (new URLSearchParams(location.search)).get("i");
|
||||
resp["indices"].forEach(idx => {
|
||||
indexMap[idx.id] = idx.name;
|
||||
select.append($("<option>")
|
||||
.attr("value", idx.id)
|
||||
.append(idx.name));
|
||||
|
||||
if (urlIndices && urlIndices.split(",").indexOf(idx.name) !== -1) {
|
||||
select.select(idx.name);
|
||||
}
|
||||
});
|
||||
|
||||
updateStats();
|
||||
});
|
||||
};
|
||||
|
||||
function fullScreen(selector) {
|
||||
const card = document.getElementById(selector);
|
||||
card.classList.toggle("full-screen");
|
||||
}
|
||||
|
||||
function exportTreemap() {
|
||||
domtoimage.toBlob(document.getElementById("treemap"), {width: width, height: height})
|
||||
.then(function (blob) {
|
||||
let a = document.createElement("a");
|
||||
let url = URL.createObjectURL(blob);
|
||||
|
||||
a.href = url;
|
||||
a.download = `${indexMap[$("#indices").val()]}_treemap.png`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
setTimeout(function() {
|
||||
document.body.removeChild(a);
|
||||
window.URL.revokeObjectURL(url);
|
||||
}, 0);
|
||||
});
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
329
src/stats.c
Normal file
329
src/stats.c
Normal file
@ -0,0 +1,329 @@
|
||||
#include "sist.h"
|
||||
#include "io/serialize.h"
|
||||
#include "ctx.h"
|
||||
|
||||
#include <glib.h>
|
||||
|
||||
static GHashTable *FlatTree;
|
||||
static GHashTable *BufferTable;
|
||||
|
||||
static GHashTable *AggMime;
|
||||
static GHashTable *AggSize;
|
||||
static GHashTable *AggDate;
|
||||
|
||||
#define SIZE_BUCKET (long)(5 * 1024 * 1024)
|
||||
#define DATE_BUCKET (long)(2629800)
|
||||
|
||||
static long TotalSize = 0;
|
||||
static long DocumentCount = 0;
|
||||
|
||||
typedef struct {
|
||||
long size;
|
||||
long count;
|
||||
} agg_t;
|
||||
|
||||
void fill_tables(cJSON *document, UNUSED(const char uuid_str[UUID_STR_LEN])) {
|
||||
|
||||
const char *json_path = cJSON_GetObjectItem(document, "path")->valuestring;
|
||||
char *path = malloc(strlen(json_path) + 1);
|
||||
strcpy(path, json_path);
|
||||
|
||||
const char *json_mime = cJSON_GetObjectItem(document, "mime")->valuestring;
|
||||
char *mime;
|
||||
if (json_mime == NULL) {
|
||||
mime = NULL;
|
||||
} else {
|
||||
mime = malloc(strlen(json_mime) + 1);
|
||||
strcpy(mime, json_mime);
|
||||
}
|
||||
|
||||
long size = (long) cJSON_GetObjectItem(document, "size")->valuedouble;
|
||||
int mtime = cJSON_GetObjectItem(document, "mtime")->valueint;
|
||||
|
||||
// treemap
|
||||
void *existing_path = g_hash_table_lookup(FlatTree, path);
|
||||
if (existing_path == NULL) {
|
||||
g_hash_table_insert(FlatTree, path, (gpointer) size);
|
||||
} else {
|
||||
g_hash_table_replace(FlatTree, path, (gpointer) ((long) existing_path + size));
|
||||
}
|
||||
|
||||
// mime agg
|
||||
if (mime != NULL) {
|
||||
agg_t *orig_agg = g_hash_table_lookup(AggMime, mime);
|
||||
if (orig_agg == NULL) {
|
||||
agg_t *agg = malloc(sizeof(agg_t));
|
||||
agg->size = size;
|
||||
agg->count = 1;
|
||||
g_hash_table_insert(AggMime, mime, agg);
|
||||
} else {
|
||||
orig_agg->size += size;
|
||||
orig_agg->count += 1;
|
||||
free(mime);
|
||||
}
|
||||
}
|
||||
|
||||
// size agg
|
||||
long size_bucket = size - (size % SIZE_BUCKET);
|
||||
agg_t *orig_agg = g_hash_table_lookup(AggSize, (gpointer) size_bucket);
|
||||
if (orig_agg == NULL) {
|
||||
agg_t *agg = malloc(sizeof(agg_t));
|
||||
agg->size = size;
|
||||
agg->count = 1;
|
||||
g_hash_table_insert(AggSize, (gpointer) size_bucket, agg);
|
||||
} else {
|
||||
orig_agg->count += 1;
|
||||
orig_agg->size += size;
|
||||
}
|
||||
|
||||
// date agg
|
||||
long date_bucket = mtime - (mtime % DATE_BUCKET);
|
||||
orig_agg = g_hash_table_lookup(AggDate, (gpointer) date_bucket);
|
||||
if (orig_agg == NULL) {
|
||||
agg_t *agg = malloc(sizeof(agg_t));
|
||||
agg->size = size;
|
||||
agg->count = 1;
|
||||
g_hash_table_insert(AggDate, (gpointer) date_bucket, agg);
|
||||
} else {
|
||||
orig_agg->count += 1;
|
||||
orig_agg->size += size;
|
||||
}
|
||||
|
||||
TotalSize += size;
|
||||
DocumentCount += 1;
|
||||
}
|
||||
|
||||
void read_index_into_tables(index_t *index) {
|
||||
DIR *dir = opendir(index->path);
|
||||
struct dirent *de;
|
||||
while ((de = readdir(dir)) != NULL) {
|
||||
if (strncmp(de->d_name, "_index_", sizeof("_index_") - 1) == 0) {
|
||||
char file_path[PATH_MAX];
|
||||
snprintf(file_path, PATH_MAX, "%s/%s", index->path, de->d_name);
|
||||
read_index(file_path, index->desc.uuid, index->desc.type, fill_tables);
|
||||
}
|
||||
}
|
||||
closedir(dir);
|
||||
}
|
||||
|
||||
static size_t rfind(const char *str, int c) {
|
||||
for (int i = (int)strlen(str); i >= 0; i--) {
|
||||
if (str[i] == c) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
int merge_up(double thresh) {
|
||||
long min_size = (long) (thresh * (double) TotalSize);
|
||||
|
||||
int count = 0;
|
||||
GHashTableIter iter;
|
||||
g_hash_table_iter_init(&iter, FlatTree);
|
||||
|
||||
void *key;
|
||||
void *value;
|
||||
|
||||
while (g_hash_table_iter_next(&iter, &key, &value)) {
|
||||
long size = (long) value;
|
||||
|
||||
if (size < min_size) {
|
||||
int stop = rfind(key, '/');
|
||||
if (stop == -1) {
|
||||
stop = 0;
|
||||
}
|
||||
char *parent = malloc(stop + 1);
|
||||
strncpy(parent, key, stop);
|
||||
*(parent + stop) = '\0';
|
||||
|
||||
void *existing_parent = g_hash_table_lookup(FlatTree, parent);
|
||||
if (existing_parent == NULL) {
|
||||
void *existing_parent2_key;
|
||||
void *existing_parent2_val;
|
||||
int found = g_hash_table_lookup_extended(BufferTable, parent, &existing_parent2_key,
|
||||
&existing_parent2_val);
|
||||
if (!found) {
|
||||
g_hash_table_insert(BufferTable, parent, value);
|
||||
} else {
|
||||
g_hash_table_replace(BufferTable, parent, (gpointer) ((long) existing_parent2_val + size));
|
||||
free(existing_parent2_key);
|
||||
}
|
||||
} else {
|
||||
g_hash_table_replace(FlatTree, parent, (gpointer) ((long) existing_parent + size));
|
||||
}
|
||||
|
||||
g_hash_table_iter_remove(&iter);
|
||||
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
g_hash_table_iter_init(&iter, BufferTable);
|
||||
while (g_hash_table_iter_next(&iter, &key, &value)) {
|
||||
g_hash_table_insert(FlatTree, key, value);
|
||||
g_hash_table_iter_remove(&iter);
|
||||
}
|
||||
|
||||
int size = g_hash_table_size(FlatTree);
|
||||
|
||||
LOG_DEBUGF("stats.h", "Merge up iteration (%d merged, %d in tree)", count, size)
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assumes out is at at least PATH_MAX *4
|
||||
*/
|
||||
void csv_escape(char *dst, const char *str) {
|
||||
|
||||
const char *ptr = str;
|
||||
char *out = dst;
|
||||
|
||||
if (rfind(str, ',') == -1 && rfind(str, '"') == -1) {
|
||||
strcpy(dst, str);
|
||||
return;
|
||||
}
|
||||
|
||||
while (*ptr++ != 0) {
|
||||
char c = *ptr;
|
||||
|
||||
if (c == '"') {
|
||||
*out++ = '"';
|
||||
*out++ = '"';
|
||||
} else {
|
||||
*out++ = c;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int open_or_exit(const char *path) {
|
||||
int fd = open(path, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR);
|
||||
if (fd < 0) {
|
||||
LOG_FATALF("stats.c", "Error while creating file: %s [%d]\n", strerror(errno), errno)
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
#define TREEMAP_CSV_HEADER "path,size"
|
||||
#define MIME_AGG_CSV_HEADER "mime,size,count"
|
||||
#define SIZE_AGG_CSV_HEADER "bucket,size,count"
|
||||
#define DATE_AGG_CSV_HEADER "bucket,size,count"
|
||||
|
||||
void write_treemap_csv(double thresh, const char *out_path) {
|
||||
|
||||
void *key;
|
||||
void *value;
|
||||
|
||||
long min_size = (long) (thresh * (double) TotalSize);
|
||||
|
||||
int fd = open_or_exit(out_path);
|
||||
write(fd, TREEMAP_CSV_HEADER, sizeof(TREEMAP_CSV_HEADER) - 1);
|
||||
|
||||
GHashTableIter iter;
|
||||
g_hash_table_iter_init(&iter, FlatTree);
|
||||
while (g_hash_table_iter_next(&iter, &key, &value)) {
|
||||
long size = (long) value;
|
||||
|
||||
if (size >= min_size) {
|
||||
char path_buf[PATH_MAX * 4];
|
||||
char buf[PATH_MAX * 4 + 16];
|
||||
|
||||
csv_escape(path_buf, key);
|
||||
size_t written = sprintf(buf, "\n%s,%ld", path_buf, (long) value);
|
||||
write(fd, buf, written);
|
||||
}
|
||||
}
|
||||
close(fd);
|
||||
}
|
||||
|
||||
void write_agg_csv_str(const char *out_path, const char *header, GHashTable *table) {
|
||||
void *key;
|
||||
void *value;
|
||||
char buf[4096];
|
||||
|
||||
int fd = open_or_exit(out_path);
|
||||
write(fd, header, strlen(header));
|
||||
|
||||
GHashTableIter iter;
|
||||
g_hash_table_iter_init(&iter, table);
|
||||
while (g_hash_table_iter_next(&iter, &key, &value)) {
|
||||
agg_t *agg = value;
|
||||
|
||||
size_t written = sprintf(buf, "\n%s,%ld,%ld", key, agg->size, agg->count);
|
||||
write(fd, buf, written);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
|
||||
void write_agg_csv_long(const char *out_path, const char *header, GHashTable *table) {
|
||||
void *key;
|
||||
void *value;
|
||||
char buf[4096];
|
||||
|
||||
int fd = open_or_exit(out_path);
|
||||
write(fd, header, strlen(header));
|
||||
|
||||
GHashTableIter iter;
|
||||
g_hash_table_iter_init(&iter, table);
|
||||
while (g_hash_table_iter_next(&iter, &key, &value)) {
|
||||
agg_t *agg = value;
|
||||
size_t written = sprintf(buf, "\n%ld,%ld,%ld", (long)key, agg->size, agg->count);
|
||||
write(fd, buf, written);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
|
||||
int generate_stats(index_t *index, const double threshold, const char *out_prefix) {
|
||||
|
||||
FlatTree = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL);
|
||||
BufferTable = g_hash_table_new(g_str_hash, g_str_equal);
|
||||
|
||||
AggMime = g_hash_table_new_full(g_str_hash, g_str_equal, free, free);
|
||||
AggSize = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, free);
|
||||
AggDate = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, free);
|
||||
|
||||
LOG_INFO("stats.c", "Generating stats...")
|
||||
|
||||
read_index_into_tables(index);
|
||||
|
||||
LOG_DEBUG("stats.c", "Read index into tables")
|
||||
LOG_DEBUGF("stats.c", "Total size is %ld", TotalSize)
|
||||
LOG_DEBUGF("stats.c", "Document count is %ld", DocumentCount)
|
||||
LOG_DEBUGF("stats.c", "Merging small directories upwards with a threshold of %f%%", threshold * 100)
|
||||
|
||||
while (merge_up(threshold) > 100) {}
|
||||
|
||||
char tmp[PATH_MAX];
|
||||
|
||||
strncpy(tmp, out_prefix, sizeof(tmp));
|
||||
strcat(tmp, "treemap.csv");
|
||||
write_treemap_csv(threshold, tmp);
|
||||
|
||||
strncpy(tmp, out_prefix, sizeof(tmp));
|
||||
strcat(tmp, "mime_agg.csv");
|
||||
write_agg_csv_str(tmp, MIME_AGG_CSV_HEADER, AggMime);
|
||||
|
||||
strncpy(tmp, out_prefix, sizeof(tmp));
|
||||
strcat(tmp, "size_agg.csv");
|
||||
write_agg_csv_long(tmp, SIZE_AGG_CSV_HEADER, AggSize);
|
||||
|
||||
strncpy(tmp, out_prefix, sizeof(tmp));
|
||||
strcat(tmp, "date_agg.csv");
|
||||
write_agg_csv_long(tmp, DATE_AGG_CSV_HEADER, AggDate);
|
||||
|
||||
g_hash_table_remove_all(FlatTree);
|
||||
g_hash_table_destroy(FlatTree);
|
||||
g_hash_table_destroy(BufferTable);
|
||||
|
||||
g_hash_table_remove_all(AggMime);
|
||||
g_hash_table_destroy(AggMime);
|
||||
g_hash_table_remove_all(AggSize);
|
||||
g_hash_table_destroy(AggSize);
|
||||
g_hash_table_remove_all(AggDate);
|
||||
g_hash_table_destroy(AggDate);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
6
src/stats.h
Normal file
6
src/stats.h
Normal file
@ -0,0 +1,6 @@
|
||||
#ifndef SIST2_STATS_H
|
||||
#define SIST2_STATS_H
|
||||
|
||||
int generate_stats(index_t *index, double threshold, const char* out_prefix);
|
||||
|
||||
#endif
|
||||
@ -59,12 +59,71 @@ void search_index(struct mg_connection *nc) {
|
||||
nc->flags |= MG_F_SEND_AND_CLOSE;
|
||||
}
|
||||
|
||||
void javascript(struct mg_connection *nc) {
|
||||
void stats(struct mg_connection *nc) {
|
||||
send_response_line(nc, 200, sizeof(stats_html), "Content-Type: text/html");
|
||||
mg_send(nc, stats_html, sizeof(stats_html));
|
||||
nc->flags |= MG_F_SEND_AND_CLOSE;
|
||||
}
|
||||
|
||||
void stats_files(struct mg_connection *nc, struct http_message *hm, struct mg_str *path) {
|
||||
|
||||
if (path->len != UUID_STR_LEN + 4) {
|
||||
nc->flags |= MG_F_SEND_AND_CLOSE;
|
||||
return;
|
||||
}
|
||||
|
||||
char arg_uuid[UUID_STR_LEN];
|
||||
memcpy(arg_uuid, hm->uri.p + 3, UUID_STR_LEN);
|
||||
*(arg_uuid + UUID_STR_LEN - 1) = '\0';
|
||||
|
||||
index_t *index = get_index_by_id(arg_uuid);
|
||||
if (index == NULL) {
|
||||
nc->flags |= MG_F_SEND_AND_CLOSE;
|
||||
return;
|
||||
}
|
||||
|
||||
const char *file;
|
||||
switch (atoi(hm->uri.p + 3 + UUID_STR_LEN)) {
|
||||
case 1:
|
||||
file = "treemap.csv";
|
||||
break;
|
||||
case 2:
|
||||
file = "mime_agg.csv";
|
||||
break;
|
||||
case 3:
|
||||
file = "size_agg.csv";
|
||||
break;
|
||||
case 4:
|
||||
file = "date_agg.csv";
|
||||
break;
|
||||
default:
|
||||
nc->flags |= MG_F_SEND_AND_CLOSE;
|
||||
return;
|
||||
}
|
||||
|
||||
char disposition[8196];
|
||||
snprintf(disposition, sizeof(disposition), "Content-Disposition: inline; filename=\"%s\"", file);
|
||||
|
||||
char full_path[PATH_MAX];
|
||||
strcpy(full_path, index->path);
|
||||
strcat(full_path, file);
|
||||
|
||||
mg_http_serve_file(nc, hm, full_path, mg_mk_str("text/csv"), mg_mk_str(disposition));
|
||||
nc->flags |= MG_F_SEND_AND_CLOSE;
|
||||
}
|
||||
|
||||
void javascript_lib(struct mg_connection *nc) {
|
||||
send_response_line(nc, 200, sizeof(bundle_js), "Content-Type: application/javascript");
|
||||
mg_send(nc, bundle_js, sizeof(bundle_js));
|
||||
nc->flags |= MG_F_SEND_AND_CLOSE;
|
||||
}
|
||||
|
||||
void javascript_search(struct mg_connection *nc) {
|
||||
send_response_line(nc, 200, sizeof(search_js), "Content-Type: application/javascript");
|
||||
mg_send(nc, search_js, sizeof(search_js));
|
||||
nc->flags |= MG_F_SEND_AND_CLOSE;
|
||||
}
|
||||
|
||||
int client_requested_dark_theme(struct http_message *hm) {
|
||||
struct mg_str *cookie_header = mg_get_http_header(hm, "cookie");
|
||||
if (cookie_header == NULL) {
|
||||
@ -387,8 +446,12 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
|
||||
search_index(nc);
|
||||
} else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/css")))) {
|
||||
style(nc, hm);
|
||||
} else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/js")))) {
|
||||
javascript(nc);
|
||||
} else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/stats")))) {
|
||||
stats(nc);
|
||||
} else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/jslib")))) {
|
||||
javascript_lib(nc);
|
||||
} else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/jssearch")))) {
|
||||
javascript_search(nc);
|
||||
} else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/img/sprite-skin-flat.png")))) {
|
||||
img_sprite_skin_flat(nc, hm);
|
||||
} else if (is_equal(&path, &((struct mg_str) MG_MK_STR("/es")))) {
|
||||
@ -401,6 +464,8 @@ static void ev_router(struct mg_connection *nc, int ev, void *p) {
|
||||
file(nc, hm, &path);
|
||||
} else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/t/")))) {
|
||||
thumbnail(nc, hm, &path);
|
||||
} else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/s/")))) {
|
||||
stats_files(nc, hm, &path);
|
||||
} else if (has_prefix(&path, &((struct mg_str) MG_MK_STR("/d/")))) {
|
||||
document_info(nc, hm, &path);
|
||||
} else {
|
||||
|
||||
File diff suppressed because one or more lines are too long
1
third-party/mongoose
vendored
1
third-party/mongoose
vendored
@ -1 +0,0 @@
|
||||
Subproject commit 359060a151998c5ab08e54af1420b7406005dc6b
|
||||
Loading…
x
Reference in New Issue
Block a user