Dashboard + UI enhancements

This commit is contained in:
simon987 2018-04-21 17:04:53 -04:00
parent 87f35571fa
commit 6b754b4bb4
11 changed files with 242 additions and 336 deletions

View File

@ -1,6 +1,7 @@
# Do not change option names
default_options = {
"ThumbnailQuality": "85",
"ThumbnailSize": "275",
"ThumbnailSize": "272",
"ThumbnailColor": "FF00FF",
"TextFileContentLength": "2000",
"PdfFileContentLength": "2000",
@ -11,7 +12,17 @@ default_options = {
"FileParsers": "media, text, picture, font, pdf, docx, spreadsheet, ebook"
}
# Index documents after every X parsed files (Larger number will use more memory)
index_every = 10000
# See https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-ngram-tokenizer.html#_configuration_16
nGramMin = 3
nGramMax = 3
elasticsearch_url = "http://localhost:9200"
# Password hashing
bcrypt_rounds = 14
# sqlite3 database path
db_path = "./local_storage.db"
VERSION = "1.0a"

View File

@ -3,6 +3,7 @@ import elasticsearch
from threading import Thread
import subprocess
import requests
import config
class Indexer:
@ -22,8 +23,13 @@ class Indexer:
t.daemon = True
t.start()
time.sleep(10)
self.init()
time.sleep(15)
try:
requests.head("http://localhost:9200")
except requests.exceptions.ConnectionError:
print("First time setup...")
self.init()
@staticmethod
def run_elasticsearch():
@ -65,7 +71,7 @@ class Indexer:
"analysis": {"tokenizer": {"path_tokenizer": {"type": "path_hierarchy"}}}},
index=self.index_name)
self.es.indices.put_settings(body={
"analysis": {"tokenizer": {"my_nGram_tokenizer": {"type": "nGram", "min_gram": 3, "max_gram": 3}}}},
"analysis": {"tokenizer": {"my_nGram_tokenizer": {"type": "nGram", "min_gram": config.nGramMin, "max_gram": config.nGramMax}}}},
index=self.index_name)
self.es.indices.put_settings(body={
"analysis": {"analyzer": {"path_analyser": {"tokenizer": "path_tokenizer", "filter": ["lowercase"]}}}},

View File

@ -271,7 +271,7 @@ class TextFileParser(GenericFileParser):
"text/x-bibtex", "text/x-tcl", "text/x-c++", "text/x-shellscript", "text/x-msdos-batch",
"text/x-makefile", "text/rtf", "text/x-objective-c", "text/troff", "text/x-m4",
"text/x-lisp", "text/x-php", "text/x-gawk", "text/x-awk", "text/x-ruby", "text/x-po",
"text/x-makefile"
"text/x-makefile", "application/javascript"
]
def parse(self, full_path: str):

51
run.py
View File

@ -4,6 +4,8 @@ from storage import LocalStorage, DuplicateDirectoryException
from crawler import RunningTask, TaskManager
import json
import os
import shutil
import config
import humanfriendly
from search import Search
from PIL import Image
@ -11,7 +13,7 @@ from io import BytesIO
app = Flask(__name__)
app.secret_key = "A very secret key"
storage = LocalStorage("local_storage.db")
storage = LocalStorage(config.db_path)
tm = TaskManager(storage)
search = Search("changeme")
@ -56,7 +58,10 @@ def file(doc_id):
extension = "" if doc["extension"] is None or doc["extension"] == "" else "." + doc["extension"]
full_path = os.path.join(directory.path, doc["path"], doc["name"] + extension)
return send_file(full_path, mimetype=doc["mime"], as_attachment=True, attachment_filename=doc["name"])
if not os.path.exists(full_path):
return abort(404)
return send_file(full_path, mimetype=doc["mime"], as_attachment=True, attachment_filename=doc["name"] + extension)
@app.route("/file/<doc_id>")
@ -67,6 +72,9 @@ def download(doc_id):
extension = "" if doc["extension"] is None or doc["extension"] == "" else "." + doc["extension"]
full_path = os.path.join(directory.path, doc["path"], doc["name"] + extension)
if not os.path.exists(full_path):
return abort(404)
return send_file(full_path, mimetype=doc["mime"], conditional=True)
@ -170,12 +178,8 @@ def directory_manage(dir_id):
tn_size = get_dir_size("static/thumbnails/" + str(dir_id))
tn_size_formatted = humanfriendly.format_size(tn_size)
index_size = search.get_index_size()
index_size_formatted = humanfriendly.format_size(index_size)
return render_template("directory_manage.html", directory=directory, tn_size=tn_size,
tn_size_formatted=tn_size_formatted, index_size=index_size,
index_size_formatted=index_size_formatted, doc_count=search.get_doc_count())
tn_size_formatted=tn_size_formatted)
@app.route("/directory/<int:dir_id>/update")
@ -242,6 +246,8 @@ def directory_reset(dir_id):
storage.dir_cache_outdated = True
search.delete_directory(dir_id)
flash("<strong>Reset directory options to default settings</strong>", "success")
return redirect("directory/" + str(dir_id))
@ -283,10 +289,39 @@ def task_del(task_id):
return redirect("/task")
@app.route("/reset_es")
def reset_es():
flash("Elasticsearch index has been reset. Modifications made in <b>config.py</b> have been applied.", "success")
tm.indexer.init()
if os.path.exists("static/thumbnails"):
shutil.rmtree("static/thumbnails")
return redirect("/dashboard")
@app.route("/dashboard")
def dashboard():
return render_template("dashboard.html")
tn_sizes = {}
tn_size_total = 0
for directory in storage.dirs():
tn_size = get_dir_size("static/thumbnails/" + str(directory))
tn_size_formatted = humanfriendly.format_size(tn_size)
tn_sizes[directory] = tn_size_formatted
tn_size_total += tn_size
tn_size_total_formatted = humanfriendly.format_size(tn_size_total)
return render_template("dashboard.html", version=config.VERSION, tn_sizes=tn_sizes,
tn_size_total=tn_size_total_formatted,
doc_size=humanfriendly.format_size(search.get_doc_size()),
doc_count=search.get_doc_count(),
db_path=config.db_path,
elasticsearch_url=config.elasticsearch_url,
index_size=humanfriendly.format_size(search.get_index_size()))
if __name__ == "__main__":

View File

@ -2,6 +2,7 @@ import json
import os
import elasticsearch
import requests
import config
from elasticsearch import helpers
@ -12,7 +13,7 @@ class Search:
self.es = elasticsearch.Elasticsearch()
try:
requests.head("http://localhost:9200")
requests.head(config.elasticsearch_url)
print("elasticsearch is already running")
except:
print("elasticsearch is not running")
@ -35,7 +36,7 @@ class Search:
parsed_info = json.loads(info.text)
return int(parsed_info["indices"][self.index_name]["primaries"]["store"]["size_in_bytes"])
return int(parsed_info["indices"][self.index_name]["total"]["store"]["size_in_bytes"])
except:
return 0
@ -47,7 +48,23 @@ class Search:
if info.status_code == 200:
parsed_info = json.loads(info.text)
return int(parsed_info["indices"][self.index_name]["primaries"]["indexing"]["index_total"])
return int(parsed_info["indices"][self.index_name]["total"]["docs"]["count"])
except:
return 0
def get_doc_size(self):
try:
query = self.es.search(body={
"aggs": {
"total_size": {
"sum": {"field": "size"}
}
}
})
return query["aggregations"]["total_size"]["value"]
except:
return 0
@ -93,7 +110,6 @@ class Search:
def search(self, query, size_min, size_max, mime_types, must_match, directories, path):
condition = "must" if must_match else "should"
print(directories)
filters = [
{"range": {"size": {"gte": size_min, "lte": size_max}}},
@ -171,3 +187,16 @@ class Search:
return self.es.get(index=self.index_name, id=doc_id, doc_type="file")
except elasticsearch.exceptions.NotFoundError:
return None
def delete_directory(self, dir_id):
try:
self.es.delete_by_query(body={"query": {
"bool": {
"filter": {"term": {"directory": dir_id}}
}
}}, index=self.index_name)
except elasticsearch.exceptions.ConflictError:
print("Error: multiple delete tasks at the same time")

View File

@ -36,22 +36,12 @@ body {overflow-y:scroll;}
background-color: #FAAB3C;
}
.card-img-top {
display: block;
min-width: 64px;
max-width: 100%;
max-height: 256px;
width: unset;
margin: 0 auto 0;
padding: 3px 3px 0 3px;
}
.card-img-overlay {
pointer-events: none;
padding: 0.75rem;
bottom: 0;
top: unset;
bottom: unset;
top: 0;
left: unset;
right: unset;
}
@ -68,17 +58,19 @@ body {overflow-y:scroll;}
}
.fit {
width: 100%;
height: 100%;
padding: 3px;
display: block;
min-width: 64px;
max-width: 100%;
max-height: 256px;
width: 100%;
margin: 0 auto 0;
padding: 3px 3px 0 3px;
}
.audio-fit {
height: 39px;
vertical-align: bottom;
display: inline;
}
@media (min-width: 1200px) {
@ -96,6 +88,12 @@ body {overflow-y:scroll;}
}
}
@media (min-width: 1800px) {
.container {
max-width: 1550px;
}
}
.hl {
background: #fff217;
}

View File

@ -1,6 +1,5 @@
let searchBar = document.getElementById("searchBar");
let pathBar = document.getElementById("pathBar");
let must_match = true;
let scroll_id = null;
let docCount = 0;
let searchQueued = false;
@ -8,7 +7,6 @@ let coolingDown = false;
let selectedDirs = [];
function toggleSearchBar() {
must_match = !must_match;
searchQueued = true;
}
@ -117,6 +115,21 @@ function humanFileSize(bytes) {
return bytes.toFixed(1) + ' ' + units[u];
}
/**
* https://stackoverflow.com/questions/6312993
*/
function humanTime (sec_num) {
sec_num = Math.floor(sec_num);
let hours = Math.floor(sec_num / 3600);
let minutes = Math.floor((sec_num - (hours * 3600)) / 60);
let seconds = sec_num - (hours * 3600) - (minutes * 60);
if (hours < 10) {hours = "0" + hours;}
if (minutes < 10) {minutes = "0" + minutes;}
if (seconds < 10) {seconds = "0" + seconds;}
return hours + ":" + minutes + ":" + seconds;
}
function initPopover() {
$('[data-toggle="popover"]').popover({
@ -146,7 +159,7 @@ function gifOver(thumbnail, documentId) {
//Load gif
thumbnail.setAttribute("src", "/file/" + documentId);
}
}, 750); //todo grab hover time from config
}, 750);
});
@ -158,43 +171,6 @@ function gifOver(thumbnail, documentId) {
})
}
function videoOver(thumbnail, imgWrapper, thumbnailOverlay, documentId, docCard) {
docCard.addEventListener("focus", function () {
let callee = arguments.callee;
docCard.mouseStayedOver = true;
window.setTimeout(function() {
if(docCard.mouseStayedOver) {
docCard.removeEventListener('focus', callee, false);
imgWrapper.removeChild(thumbnail);
imgWrapper.removeChild(thumbnailOverlay);
let video = document.createElement("video");
let vidSource = document.createElement("source");
vidSource.setAttribute("src", "/file/" + documentId);
vidSource.setAttribute("type", "video/webm");
video.appendChild(vidSource);
video.setAttribute("class", "fit");
video.setAttribute("loop", "");
video.setAttribute("controls", "");
video.setAttribute("preload", "");
video.setAttribute("poster", "/thumb/" + documentId);
imgWrapper.appendChild(video);
video.addEventListener("dblclick", function() {
video.webkitRequestFullScreen();
})
}
}, 750);
});
docCard.addEventListener("blur", function() {
docCard.mouseStayedOver = false;
});
}
function downloadPopover(element, documentId) {
element.setAttribute("data-content",
'<a class="btn btn-sm btn-primary" href="/dl/'+ documentId +'"><i class="fas fa-download"></i> Download</a>' +
@ -254,9 +230,24 @@ function createDocCard(hit) {
switch (mimeCategory) {
case "video":
thumbnail = document.createElement("video");
let vidSource = document.createElement("source");
vidSource.setAttribute("src", "/file/" + hit["_id"]);
vidSource.setAttribute("type", "video/webm");
thumbnail.appendChild(vidSource);
thumbnail.setAttribute("class", "fit");
thumbnail.setAttribute("loop", "");
thumbnail.setAttribute("controls", "");
thumbnail.setAttribute("preload", "none");
thumbnail.setAttribute("poster", "/thumb/" + hit["_id"]);
thumbnail.addEventListener("dblclick", function() {
thumbnail.webkitRequestFullScreen();
});
break;
case "image":
thumbnail = document.createElement("img");
thumbnail.setAttribute("class", "card-img-top");
thumbnail.setAttribute("class", "card-img-top fit");
thumbnail.setAttribute("src", "/thumb/" + hit["_id"]);
break;
}
@ -291,12 +282,9 @@ function createDocCard(hit) {
//Duration
let durationBadge = document.createElement("span");
durationBadge.setAttribute("class", "badge badge-resolution");
durationBadge.appendChild(document.createTextNode(parseFloat(hit["_source"]["duration"]).toFixed(2) + "s"));
durationBadge.appendChild(document.createTextNode(humanTime(hit["_source"]["duration"])));
thumbnailOverlay.appendChild(durationBadge);
//Hover
videoOver(thumbnail, imgWrapper, thumbnailOverlay, hit["_id"], docCard)
}
//Tags
@ -522,7 +510,7 @@ function search() {
postBody.size_min = size_min;
postBody.size_max = size_max;
postBody.mime_types = getSelectedMimeTypes();
postBody.must_match = must_match;
postBody.must_match = document.getElementById("barToggle").checked;
postBody.directories = selectedDirs;
postBody.path = pathBar.value.replace(/\/$/, "").toLowerCase(); //remove trailing slashes
xhttp.setRequestHeader('content-type', 'application/json');
@ -539,7 +527,7 @@ searchBar.addEventListener("keyup", function () {
});
//Size slider
let sizeSlider = $("#sizeSlider").ionRangeSlider({
$("#sizeSlider").ionRangeSlider({
type: "double",
grid: false,
force_edges: true,
@ -569,7 +557,7 @@ let sizeSlider = $("#sizeSlider").ionRangeSlider({
searchQueued = true;
}
})[0];
});
//Directories select
function updateDirectories() {
@ -600,5 +588,8 @@ function getPathChoices() {
});
}
document.getElementById("pathBar").addEventListener("keyup", function () {
searchQueued = true;
});
window.setInterval(search, 75);

View File

@ -3,229 +3,68 @@
{% block body %}
<div class="container-fluid">
<div class="row">
<div class="col-sm-3">
<div class="chart-wrapper">
<div class="chart-title">FSE Info</div>
<div class="chart-stage">
<table class="info-table">
<tr>
<th>Version</th>
<td><pre>1.0a</pre></td>
</tr>
<tr>
<th>Total thumbnail size</th>
<td><pre>652.08 Mb</pre></td>
</tr>
<tr>
<th>Total document count</th>
<td><pre>1258902</pre></td>
</tr>
<tr>
<th>Total document size</th>
<td><pre>4.7 TB</pre></td>
</tr>
<tr>
<th>Folder count</th>
<td><pre>4</pre></td>
</tr>
<tr>
<th>User count</th>
<td><pre>1</pre></td>
</tr>
<tr>
<th>SQLite database path</th>
<td>./local_storage.db</td>
</tr>
</table>
</div>
</div>
<div class="chart-wrapper">
<div class="chart-title">Elasticsearch info</div>
<div class="chart-stage">
<table class="info-table">
<tr>
<th>Total index size</th>
<td><pre>3.7 GB</pre></td>
</tr>
<tr>
<th>HTTP port</th>
<td><pre>9200</pre></td>
</tr>
<tr>
<th>Version</th>
<td><pre>6.2.1</pre></td>
</tr>
<tr>
<th>Lucene version</th>
<td><pre>7.2.1</pre></td>
</tr>
<tr>
<th>Path</th>
<td><pre>./elasticsearch/</pre></td>
</tr>
</table>
</div>
<div class="container">
<div class="card">
<div class="card-header">FSE Information</div>
<div class="card-body">
<table class="info-table table-hover table-striped">
<tbody>
<tr>
<th>Version</th>
<td><pre>{{ version }}</pre></td>
</tr>
<tr>
<th>Total thumbnail cache size</th>
<td><pre>{{ tn_size_total }}</pre></td>
</tr>
<tr>
<th>Total document count</th>
<td><pre>{{ doc_count }}</pre></td>
</tr>
<tr>
<th>Total size of indexed documents</th>
<td><pre>{{ doc_size }}</pre></td>
</tr>
<tr>
<th>Total index size</th>
<td><pre>{{ index_size }}</pre></td>
</tr>
<tr>
<th>User count</th>
<td><pre>1</pre></td>
</tr>
<tr>
<th>SQLite database path</th>
<td><pre>{{ db_path }}</pre></td>
</tr>
<tr>
<th>Elasticsearch URL</th>
<td><pre>{{ elasticsearch_url }}</pre></td>
</tr>
</tbody>
</table>
</div>
<div class="card-footer text-muted">Change global settings in <b>config.py</b></div>
</div>
<div class="col-sm-9">
<div class="row">
<div class="col-sm-12">
<div class="chart-wrapper">
<div class="chart-title">Thumbnail cache size</div>
<div class="chart-stage">
<script>
window.chartColors = {
red: 'rgb(255, 99, 132)',
orange: 'rgb(255, 159, 64)',
yellow: 'rgb(255, 205, 86)',
green: 'rgb(75, 192, 192)',
blue: 'rgb(54, 162, 235)',
purple: 'rgb(153, 102, 255)',
grey: 'rgb(201, 203, 207)'
};
var color = Chart.helpers.color;
var barChartData = {
labels: ["Music: 12.08 MB", "Pictures: 560.8 MB", "Movies: 78.9 MB", "Documents: 1 MB"],
datasets: [{
label: 'Size',
backgroundColor: color("#8f2aa3").alpha(0.6).rgbString(),
borderColor: "#8f2aa3",
borderWidth: 1,
data: [
12080500,
560805400,
78898000,
1024000
]
}]
};
</script>
<canvas id="canvas"></canvas>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-sm-12">
<div class="chart-wrapper">
<div class="chart-title">Document size</div>
<div class="chart-stage">
<script>
var color = Chart.helpers.color;
var barChartData2 = {
labels: ["Music: 192.5 GB", "Pictures: 1.30 TB", "Movies: 3.73 TB", "Documents: 42.7 GB"],
datasets: [{
label: 'Size',
backgroundColor: color("#f4b80c").alpha(0.6).rgbString(),
borderColor: "#f4b80c",
borderWidth: 1,
data: [
192500000000,
1300000000000,
3730000000000,
42700000000
]
}]
};
</script>
<canvas id="docSizeChart"></canvas>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-sm-12">
<div class="chart-wrapper">
<div class="chart-title">Document count</div>
<div class="chart-stage"> {# todo padding 8px 10px 5px 10px #}
<script>
var color = Chart.helpers.color;
var barChartData3 = {
labels: ["Music", "Pictures", "Movies", "Documents"],
datasets: [{
label: 'Count',
backgroundColor: color("#f4225a").alpha(0.6).rgbString(),
borderColor: "#f4225a",
borderWidth: 1,
data: [
6790,
758652,
1289,
11589632
]
}]
};
window.onload = function() {
var ctx = document.getElementById("canvas").getContext("2d");
ctx.canvas.height = 50;
new Chart(ctx, {
type: 'bar',
data: barChartData,
options: {
legend: {
position: 'hidden'
},
title: {
display: false
}
}
});
ctx = document.getElementById("docSizeChart").getContext("2d");
ctx.canvas.height = 50;
new Chart(ctx, {
type: 'bar',
data: barChartData2,
options: {
legend: {
position: 'hidden'
},
title: {
display: false
}
}
});
ctx = document.getElementById("docCountChart").getContext("2d");
ctx.canvas.height = 50;
new Chart(ctx, {
type: 'bar',
data: barChartData3,
options: {
legend: {
position: 'hidden'
},
title: {
display: false
}
}
});
};
</script>
<canvas id="docCountChart"></canvas>
</div>
</div>
</div>
<div class="card">
<div class="card-header">Actions</div>
<div class="card-body">
<button class="btn btn-danger" onclick="resetAll()">
<i class="fas fa-exclamation-triangle"></i> Reset elasticsearch index
</button>
</div>
</div>
</div>
</div>
<script>
function resetAll() {
if (confirm("This will entirely reset the index and documents will need to be re-indexed.\n\n" +
"Do you want to proceed?")) {
window.location = "/reset_es"
}
}
</script>
{% endblock body %}

View File

@ -98,49 +98,13 @@
<th>Thumbnail cache size</th>
<td><pre>{{ tn_size_formatted }} ({{ tn_size }} bytes)</pre></td>
</tr>
<tr>
<th>Index size</th>
<td><pre>{{ index_size_formatted }} ({{ index_size }} bytes)</pre></td>
</tr>
<tr>
<th>Document count</th>
<td><pre>{{ doc_count }}</pre></td>
</tr>
</table>
</div>
</div>
<div class="card">
<div class="card-header">An excellent option list</div>
<div class="card-body">
<table class="info-table table-striped table-hover">
<thead>
<tr>
<th>Option</th>
<th>Value</th>
</tr>
</thead>
<tbody>
{% for option in directory.options %}
<tr>
<td style="width: 30%"><span>{{ option.key }}</span></td>
<td onclick="modifyVal({{ option.id }}, '{{ option.key }}')" title="Click to update"><pre id="val-{{ option.id }}">{{ option.value }}</pre></td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<div class="card">
<div class="card-header">An excellent control panel</div>
<div class="card-header">Actions</div>
<div class="card-body">
<div class="d-flex">
@ -174,6 +138,33 @@
</div>
</div>
<div class="card">
{# TODO: put github wiki link #}
<div class="card-header">Options <a href="#" style="float:right">Learn more <i class="fas fa-external-link-alt"></i></a></div>
<div class="card-body">
<table class="info-table table-striped table-hover">
<thead>
<tr>
<th>Option</th>
<th>Value</th>
</tr>
</thead>
<tbody>
{% for option in directory.options %}
<tr>
<td style="width: 30%"><span>{{ option.key }}</span></td>
<td onclick="modifyVal({{ option.id }}, '{{ option.key }}')" title="Click to update"><pre id="val-{{ option.id }}">{{ option.value }}</pre></td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>

View File

@ -14,7 +14,7 @@
<div class="card">
<div class="card-body">
<div class="form-group">
<input id="pathBar" type="search" class="form-control" placeholder="Path">
<input id="pathBar" type="search" class="form-control" placeholder="Filter path">
</div>
<div class="input-group">
<div class="input-group-prepend">

View File

@ -112,7 +112,13 @@
{% for task_id in tasks | sort() %}
<div class="task-wrapper container-fluid">
<a class="task-name" href="/directory/{{ tasks[task_id].dir_id }}">{{ directories[tasks[task_id].dir_id].name }}</a>
<span class="task-info"> - {{ tasks[task_id].type }}</span>
<span class="task-info"> -
{% if tasks[task_id].type == 1 %}
Indexing
{% else %}
Thumbnail generation
{% endif %}
</span>
<div class="d-flex p-2">
<div class="container-fluid p-2">