simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-18 01:46:46 +00:00
Code Issues Projects Releases Wiki Activity

Dates and sizes graphs

Browse Source
This commit is contained in:
Simon 2018-06-19 19:04:12 -04:00
parent e0b5aad654
commit 8236b04c2e
3 changed files with 179 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
search/search.py
View File

@ -77,7 +77,7 @@ class ElasticSearchEngine(SearchEngine):
"path": {"analyzer": "standard", "type": "text"},
"name": {"analyzer": "standard", "type": "text",
"fields": {"nGram": {"type": "text", "analyzer": "my_nGram"}}},
"mtime": {"type": "date", "format": "epoch_millis"},
"mtime": {"type": "date", "format": "epoch_second"},
"size": {"type": "long"},
"website_id": {"type": "integer"},
"ext": {"type": "keyword"}
@ -202,7 +202,7 @@ class ElasticSearchEngine(SearchEngine):
}
},
"size": 0
})
}, index=self.index_name)
stats = dict()
stats["total_size"] = result["aggregations"]["total_size"]["value"]
@ -279,8 +279,48 @@ class ElasticSearchEngine(SearchEngine):
"size": 0
}, index=self.index_name)
size_and_date_histogram = self.es.search(body={
"query": {
"bool": {
"must_not": {
"term": {"size": -1},
},
"filter": [
{"range": {
"mtime": {
"gt": 0 # 1970-01-01
}
}},
{"range": {
"size": {
"gt": 0
}
}}
]
}
},
"aggs": {
"sizes": {
"histogram": {
"field": "size",
"interval": 10000000, # 10Mb
"min_doc_count": 5
}
},
"dates": {
"date_histogram": {
"field": "mtime",
"interval": "1M",
"min_doc_count": 5,
"format": "yyyy"
}
}
},
"size": 0
}, index=self.index_name)
es_stats = self.es.indices.stats(self.index_name)
print(es_stats)
print(size_and_date_histogram)
stats = dict()
stats["es_index_size"] = es_stats["indices"][self.index_name]["total"]["store"]["size_in_bytes"]
@ -297,6 +337,10 @@ class ElasticSearchEngine(SearchEngine):
stats["size_variance"] = total_stats["aggregations"]["file_stats"]["variance"]
stats["ext_stats"] = [(b["size"]["value"], b["doc_count"], b["key"])
for b in size_per_ext["aggregations"]["ext_group"]["buckets"]]
stats["sizes_histogram"] = [(b["key"], b["doc_count"])
for b in size_and_date_histogram["aggregations"]["sizes"]["buckets"]]
stats["dates_histogram"] = [(b["key_as_string"], b["doc_count"])
for b in size_and_date_histogram["aggregations"]["dates"]["buckets"]]
stats["base_url"] = "entire database"
return stats

127
static/js/report.js
View File

@ -1,3 +1,126 @@
function drawSizeHistogram(rData) {
let labels = [];
let dataSet = [];
for (let i in rData["sizes_histogram"]) {
let slice = rData["sizes_histogram"][i];
dataSet.push(slice[1]);
labels.push("[" + humanFileSize(slice[0]) + ", " + humanFileSize(slice[0] + 10000000) + "]")
}
console.log(dataSet);
let ctx = document.getElementById('sizeHistogram').getContext('2d');
new Chart(ctx, {
type: 'line',
data: {
datasets: [{
data: dataSet,
borderWidth: 1,
strokeColor: "#FFFFFF",
backgroundColor: "#FFFFFF"
}],
labels: labels,
title: "test"
},
options: {
title: {
display: true,
text: "Size histogram",
fontColor: "#c6c6c6",
fontSize: 16,
fontFamily: "Lato,'Helvetica Neue',Arial,Helvetica,sans-serif"
},
legend: {
display: false
},
scales: {
yAxes: [
{
id: "count",
type: "logarithmic",
ticks: {
// Include a dollar sign in the ticks
callback: function(value, index, values) {
let log10 = Math.log10(value);
if (Number.isInteger(log10)) {
return value;
}
}
}
}
]
}
}
});
}
function drawDateHistogram(rData) {
let labels = [];
let dataSet = [];
console.log(rData["dates_histogram"]);
for (let i in rData["dates_histogram"]) {
let slice = rData["dates_histogram"][i];
dataSet.push(slice[1]);
labels.push(slice[0])
}
let ctx = document.getElementById('dateHistogram').getContext('2d');
new Chart(ctx, {
type: 'line',
data: {
datasets: [{
data: dataSet,
borderWidth: 1,
strokeColor: "#FFFFFF",
backgroundColor: "#FFFFFF"
}],
labels: labels,
},
options: {
title: {
display: true,
text: "Dates histogram",
fontColor: "#c6c6c6",
fontSize: 16,
fontFamily: "Lato,'Helvetica Neue',Arial,Helvetica,sans-serif"
},
legend: {
display: false
},
scales: {
yAxes: [
{
id: "count",
type: "logarithmic",
ticks: {
// Include a dollar sign in the ticks
callback: function(value, index, values) {
let log10 = Math.log10(value);
if (Number.isInteger(log10)) {
return value;
}
}
}
}
]
}
}
});
}
function drawChart(rData) {
var dataSetSize = [];
@ -118,10 +241,6 @@ function getRandomColor() {
*/
function humanFileSize(bytes) {
if (bytes === 0) {
return "? B"
}
var thresh = 1000;
if (Math.abs(bytes) < thresh) {
return bytes + ' B';

11
templates/stats.html
View File

@ -12,8 +12,12 @@
<div id="chart-wrapper" style="margin-bottom: 1em">
<p id="loading-text">Calculating...</p>
<canvas id="typesChart"></canvas>
<script src="/static/js/Chart.min.js"></script>
<script src="/static/js/report.js"></script>
</div>
<div id="chart-wrapper" style="margin-bottom: 1em">
<canvas id="sizeHistogram"></canvas>
</div>
<div id="chart-wrapper" style="margin-bottom: 1em">
<canvas id="dateHistogram"></canvas>
</div>
<h4>Database stats</h4>
@ -114,6 +118,7 @@
</div>
<script src="/static/js/report.js"></script>
<script>
var xhttp = new XMLHttpRequest();
@ -123,6 +128,8 @@
let rData = JSON.parse(this.responseText);
drawChart(rData);
drawSizeHistogram(rData);
drawDateHistogram(rData);
fillDatabaseTable(rData);
document.getElementById("loading-text").innerHTML = "";