Dates and sizes graphs

search/search.py

@@ -77,7 +77,7 @@ class ElasticSearchEngine(SearchEngine):
             "path": {"analyzer": "standard", "type": "text"},
             "name": {"analyzer": "standard", "type": "text",
                      "fields": {"nGram": {"type": "text", "analyzer": "my_nGram"}}},
-            "mtime": {"type": "date", "format": "epoch_millis"},
+            "mtime": {"type": "date", "format": "epoch_second"},
             "size": {"type": "long"},
             "website_id": {"type": "integer"},
             "ext": {"type": "keyword"}
@@ -202,7 +202,7 @@ class ElasticSearchEngine(SearchEngine):
                 }
             },
             "size": 0
-        })
+        }, index=self.index_name)
 
         stats = dict()
         stats["total_size"] = result["aggregations"]["total_size"]["value"]
@@ -279,8 +279,48 @@ class ElasticSearchEngine(SearchEngine):
             "size": 0
         }, index=self.index_name)
 
+        size_and_date_histogram = self.es.search(body={
+            "query": {
+                "bool": {
+                    "must_not": {
+                        "term": {"size": -1},
+                    },
+                    "filter": [
+                        {"range": {
+                            "mtime": {
+                                "gt": 0  # 1970-01-01
+                            }
+                        }},
+                        {"range": {
+                            "size": {
+                                "gt": 0
+                            }
+                        }}
+                    ]
+                }
+            },
+            "aggs": {
+                "sizes": {
+                    "histogram": {
+                        "field": "size",
+                        "interval": 10000000,  # 10Mb
+                        "min_doc_count": 5
+                    }
+                },
+                "dates": {
+                    "date_histogram": {
+                        "field": "mtime",
+                        "interval": "1M",
+                        "min_doc_count": 5,
+                        "format": "yyyy"
+                    }
+                }
+            },
+            "size": 0
+        }, index=self.index_name)
+
         es_stats = self.es.indices.stats(self.index_name)
-        print(es_stats)
+        print(size_and_date_histogram)
 
         stats = dict()
         stats["es_index_size"] = es_stats["indices"][self.index_name]["total"]["store"]["size_in_bytes"]
@@ -297,6 +337,10 @@ class ElasticSearchEngine(SearchEngine):
         stats["size_variance"] = total_stats["aggregations"]["file_stats"]["variance"]
         stats["ext_stats"] = [(b["size"]["value"], b["doc_count"], b["key"])
                               for b in size_per_ext["aggregations"]["ext_group"]["buckets"]]
+        stats["sizes_histogram"] = [(b["key"], b["doc_count"])
+                                    for b in size_and_date_histogram["aggregations"]["sizes"]["buckets"]]
+        stats["dates_histogram"] = [(b["key_as_string"], b["doc_count"])
+                                    for b in size_and_date_histogram["aggregations"]["dates"]["buckets"]]
         stats["base_url"] = "entire database"
 
         return stats

static/js/report.js

@@ -1,3 +1,126 @@
+function drawSizeHistogram(rData) {
+
+    let labels = [];
+    let dataSet = [];
+
+    for (let i in rData["sizes_histogram"]) {
+
+        let slice = rData["sizes_histogram"][i];
+        dataSet.push(slice[1]);
+        labels.push("[" + humanFileSize(slice[0]) + ", " + humanFileSize(slice[0] + 10000000) + "]")
+    }
+
+    console.log(dataSet);
+
+    let ctx = document.getElementById('sizeHistogram').getContext('2d');
+    new Chart(ctx, {
+        type: 'line',
+        data: {
+            datasets: [{
+                data: dataSet,
+                borderWidth: 1,
+                strokeColor: "#FFFFFF",
+                backgroundColor: "#FFFFFF"
+            }],
+            labels: labels,
+            title: "test"
+
+        },
+        options: {
+            title: {
+                display: true,
+                text: "Size histogram",
+                fontColor: "#c6c6c6",
+                fontSize: 16,
+                fontFamily: "Lato,'Helvetica Neue',Arial,Helvetica,sans-serif"
+            },
+            legend: {
+                display: false
+            },
+            scales: {
+                yAxes: [
+                    {
+                        id: "count",
+                        type: "logarithmic",
+                        ticks: {
+                            // Include a dollar sign in the ticks
+                            callback: function(value, index, values) {
+
+                                let log10 = Math.log10(value);
+
+                                if (Number.isInteger(log10)) {
+                                    return value;
+                                }
+                            }
+                        }
+                    }
+                ]
+            }
+        }
+    });
+}
+
+function drawDateHistogram(rData) {
+
+    let labels = [];
+    let dataSet = [];
+
+    console.log(rData["dates_histogram"]);
+
+    for (let i in rData["dates_histogram"]) {
+
+        let slice = rData["dates_histogram"][i];
+        dataSet.push(slice[1]);
+        labels.push(slice[0])
+    }
+
+    let ctx = document.getElementById('dateHistogram').getContext('2d');
+    new Chart(ctx, {
+        type: 'line',
+        data: {
+            datasets: [{
+                data: dataSet,
+                borderWidth: 1,
+                strokeColor: "#FFFFFF",
+                backgroundColor: "#FFFFFF"
+            }],
+            labels: labels,
+
+        },
+        options: {
+            title: {
+                display: true,
+                text: "Dates histogram",
+                fontColor: "#c6c6c6",
+                fontSize: 16,
+                fontFamily: "Lato,'Helvetica Neue',Arial,Helvetica,sans-serif"
+            },
+            legend: {
+                display: false
+            },
+            scales: {
+                yAxes: [
+                    {
+                        id: "count",
+                        type: "logarithmic",
+                        ticks: {
+                            // Include a dollar sign in the ticks
+                            callback: function(value, index, values) {
+
+                                let log10 = Math.log10(value);
+
+                                if (Number.isInteger(log10)) {
+                                    return value;
+                                }
+                            }
+                        }
+                    }
+                ]
+            }
+        }
+    });
+}
+
 function drawChart(rData) {
 
     var dataSetSize = [];
@@ -118,10 +241,6 @@ function getRandomColor() {
  */
 function humanFileSize(bytes) {
 
-    if (bytes === 0) {
-        return "? B"
-    }
-
     var thresh = 1000;
     if (Math.abs(bytes) < thresh) {
         return bytes + ' B';

templates/stats.html

@@ -12,8 +12,12 @@
                 <div id="chart-wrapper" style="margin-bottom: 1em">
                     <p id="loading-text">Calculating...</p>
                     <canvas id="typesChart"></canvas>
-                    <script src="/static/js/Chart.min.js"></script>
+                </div>
-                    <script src="/static/js/report.js"></script>
+                <div id="chart-wrapper" style="margin-bottom: 1em">
+                    <canvas id="sizeHistogram"></canvas>
+                </div>
+                <div id="chart-wrapper" style="margin-bottom: 1em">
+                    <canvas id="dateHistogram"></canvas>
                 </div>
 
                 <h4>Database stats</h4>
@@ -114,6 +118,7 @@
 
     </div>
 
+    <script src="/static/js/report.js"></script>
     <script>
         var xhttp = new XMLHttpRequest();
 
@@ -123,6 +128,8 @@
                 let rData = JSON.parse(this.responseText);
 
                 drawChart(rData);
+                drawSizeHistogram(rData);
+                drawDateHistogram(rData);
                 fillDatabaseTable(rData);
 
                 document.getElementById("loading-text").innerHTML = "";