From 6b754b4bb42a38b0acaf78db8979ea69b086a787 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 21 Apr 2018 17:04:53 -0400 Subject: [PATCH] Dashboard + UI enhancements --- config.py | 13 +- indexer.py | 12 +- parsing.py | 2 +- run.py | 51 +++++- search.py | 37 ++++- static/css/search.css | 28 ++-- static/js/search.js | 87 +++++----- templates/dashboard.html | 273 +++++++------------------------- templates/directory_manage.html | 65 ++++---- templates/search.html | 2 +- templates/task.html | 8 +- 11 files changed, 242 insertions(+), 336 deletions(-) diff --git a/config.py b/config.py index 7a0f32d..11ebfa6 100644 --- a/config.py +++ b/config.py @@ -1,6 +1,7 @@ +# Do not change option names default_options = { "ThumbnailQuality": "85", - "ThumbnailSize": "275", + "ThumbnailSize": "272", "ThumbnailColor": "FF00FF", "TextFileContentLength": "2000", "PdfFileContentLength": "2000", @@ -11,7 +12,17 @@ default_options = { "FileParsers": "media, text, picture, font, pdf, docx, spreadsheet, ebook" } +# Index documents after every X parsed files (Larger number will use more memory) index_every = 10000 + +# See https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-ngram-tokenizer.html#_configuration_16 nGramMin = 3 nGramMax = 3 +elasticsearch_url = "http://localhost:9200" + +# Password hashing bcrypt_rounds = 14 +# sqlite3 database path +db_path = "./local_storage.db" + +VERSION = "1.0a" diff --git a/indexer.py b/indexer.py index 0effd28..2ac6c85 100644 --- a/indexer.py +++ b/indexer.py @@ -3,6 +3,7 @@ import elasticsearch from threading import Thread import subprocess import requests +import config class Indexer: @@ -22,8 +23,13 @@ class Indexer: t.daemon = True t.start() - time.sleep(10) - self.init() + time.sleep(15) + + try: + requests.head("http://localhost:9200") + except requests.exceptions.ConnectionError: + print("First time setup...") + self.init() @staticmethod def run_elasticsearch(): @@ -65,7 +71,7 @@ class Indexer: "analysis": {"tokenizer": {"path_tokenizer": {"type": "path_hierarchy"}}}}, index=self.index_name) self.es.indices.put_settings(body={ - "analysis": {"tokenizer": {"my_nGram_tokenizer": {"type": "nGram", "min_gram": 3, "max_gram": 3}}}}, + "analysis": {"tokenizer": {"my_nGram_tokenizer": {"type": "nGram", "min_gram": config.nGramMin, "max_gram": config.nGramMax}}}}, index=self.index_name) self.es.indices.put_settings(body={ "analysis": {"analyzer": {"path_analyser": {"tokenizer": "path_tokenizer", "filter": ["lowercase"]}}}}, diff --git a/parsing.py b/parsing.py index 981ca8e..c009f54 100644 --- a/parsing.py +++ b/parsing.py @@ -271,7 +271,7 @@ class TextFileParser(GenericFileParser): "text/x-bibtex", "text/x-tcl", "text/x-c++", "text/x-shellscript", "text/x-msdos-batch", "text/x-makefile", "text/rtf", "text/x-objective-c", "text/troff", "text/x-m4", "text/x-lisp", "text/x-php", "text/x-gawk", "text/x-awk", "text/x-ruby", "text/x-po", - "text/x-makefile" + "text/x-makefile", "application/javascript" ] def parse(self, full_path: str): diff --git a/run.py b/run.py index ddbeb43..d5157b3 100644 --- a/run.py +++ b/run.py @@ -4,6 +4,8 @@ from storage import LocalStorage, DuplicateDirectoryException from crawler import RunningTask, TaskManager import json import os +import shutil +import config import humanfriendly from search import Search from PIL import Image @@ -11,7 +13,7 @@ from io import BytesIO app = Flask(__name__) app.secret_key = "A very secret key" -storage = LocalStorage("local_storage.db") +storage = LocalStorage(config.db_path) tm = TaskManager(storage) search = Search("changeme") @@ -56,7 +58,10 @@ def file(doc_id): extension = "" if doc["extension"] is None or doc["extension"] == "" else "." + doc["extension"] full_path = os.path.join(directory.path, doc["path"], doc["name"] + extension) - return send_file(full_path, mimetype=doc["mime"], as_attachment=True, attachment_filename=doc["name"]) + if not os.path.exists(full_path): + return abort(404) + + return send_file(full_path, mimetype=doc["mime"], as_attachment=True, attachment_filename=doc["name"] + extension) @app.route("/file/") @@ -67,6 +72,9 @@ def download(doc_id): extension = "" if doc["extension"] is None or doc["extension"] == "" else "." + doc["extension"] full_path = os.path.join(directory.path, doc["path"], doc["name"] + extension) + if not os.path.exists(full_path): + return abort(404) + return send_file(full_path, mimetype=doc["mime"], conditional=True) @@ -170,12 +178,8 @@ def directory_manage(dir_id): tn_size = get_dir_size("static/thumbnails/" + str(dir_id)) tn_size_formatted = humanfriendly.format_size(tn_size) - index_size = search.get_index_size() - index_size_formatted = humanfriendly.format_size(index_size) - return render_template("directory_manage.html", directory=directory, tn_size=tn_size, - tn_size_formatted=tn_size_formatted, index_size=index_size, - index_size_formatted=index_size_formatted, doc_count=search.get_doc_count()) + tn_size_formatted=tn_size_formatted) @app.route("/directory//update") @@ -242,6 +246,8 @@ def directory_reset(dir_id): storage.dir_cache_outdated = True + search.delete_directory(dir_id) + flash("Reset directory options to default settings", "success") return redirect("directory/" + str(dir_id)) @@ -283,10 +289,39 @@ def task_del(task_id): return redirect("/task") +@app.route("/reset_es") +def reset_es(): + + flash("Elasticsearch index has been reset. Modifications made in config.py have been applied.", "success") + + tm.indexer.init() + if os.path.exists("static/thumbnails"): + shutil.rmtree("static/thumbnails") + + return redirect("/dashboard") + + @app.route("/dashboard") def dashboard(): - return render_template("dashboard.html") + tn_sizes = {} + tn_size_total = 0 + for directory in storage.dirs(): + tn_size = get_dir_size("static/thumbnails/" + str(directory)) + tn_size_formatted = humanfriendly.format_size(tn_size) + + tn_sizes[directory] = tn_size_formatted + tn_size_total += tn_size + + tn_size_total_formatted = humanfriendly.format_size(tn_size_total) + + return render_template("dashboard.html", version=config.VERSION, tn_sizes=tn_sizes, + tn_size_total=tn_size_total_formatted, + doc_size=humanfriendly.format_size(search.get_doc_size()), + doc_count=search.get_doc_count(), + db_path=config.db_path, + elasticsearch_url=config.elasticsearch_url, + index_size=humanfriendly.format_size(search.get_index_size())) if __name__ == "__main__": diff --git a/search.py b/search.py index 93a59bd..62bafee 100644 --- a/search.py +++ b/search.py @@ -2,6 +2,7 @@ import json import os import elasticsearch import requests +import config from elasticsearch import helpers @@ -12,7 +13,7 @@ class Search: self.es = elasticsearch.Elasticsearch() try: - requests.head("http://localhost:9200") + requests.head(config.elasticsearch_url) print("elasticsearch is already running") except: print("elasticsearch is not running") @@ -35,7 +36,7 @@ class Search: parsed_info = json.loads(info.text) - return int(parsed_info["indices"][self.index_name]["primaries"]["store"]["size_in_bytes"]) + return int(parsed_info["indices"][self.index_name]["total"]["store"]["size_in_bytes"]) except: return 0 @@ -47,7 +48,23 @@ class Search: if info.status_code == 200: parsed_info = json.loads(info.text) - return int(parsed_info["indices"][self.index_name]["primaries"]["indexing"]["index_total"]) + return int(parsed_info["indices"][self.index_name]["total"]["docs"]["count"]) + except: + return 0 + + def get_doc_size(self): + + try: + query = self.es.search(body={ + "aggs": { + "total_size": { + "sum": {"field": "size"} + } + } + }) + + return query["aggregations"]["total_size"]["value"] + except: return 0 @@ -93,7 +110,6 @@ class Search: def search(self, query, size_min, size_max, mime_types, must_match, directories, path): condition = "must" if must_match else "should" - print(directories) filters = [ {"range": {"size": {"gte": size_min, "lte": size_max}}}, @@ -171,3 +187,16 @@ class Search: return self.es.get(index=self.index_name, id=doc_id, doc_type="file") except elasticsearch.exceptions.NotFoundError: return None + + def delete_directory(self, dir_id): + + try: + self.es.delete_by_query(body={"query": { + "bool": { + "filter": {"term": {"directory": dir_id}} + } + }}, index=self.index_name) + except elasticsearch.exceptions.ConflictError: + print("Error: multiple delete tasks at the same time") + + diff --git a/static/css/search.css b/static/css/search.css index 0296723..3fdaa96 100644 --- a/static/css/search.css +++ b/static/css/search.css @@ -36,22 +36,12 @@ body {overflow-y:scroll;} background-color: #FAAB3C; } -.card-img-top { - display: block; - min-width: 64px; - max-width: 100%; - max-height: 256px; - width: unset; - margin: 0 auto 0; - padding: 3px 3px 0 3px; -} - .card-img-overlay { pointer-events: none; padding: 0.75rem; - bottom: 0; - top: unset; + bottom: unset; + top: 0; left: unset; right: unset; } @@ -68,17 +58,19 @@ body {overflow-y:scroll;} } .fit { - width: 100%; - height: 100%; - padding: 3px; + display: block; min-width: 64px; max-width: 100%; max-height: 256px; + width: 100%; + margin: 0 auto 0; + padding: 3px 3px 0 3px; } .audio-fit { height: 39px; vertical-align: bottom; + display: inline; } @media (min-width: 1200px) { @@ -96,6 +88,12 @@ body {overflow-y:scroll;} } } +@media (min-width: 1800px) { + .container { + max-width: 1550px; + } +} + .hl { background: #fff217; } diff --git a/static/js/search.js b/static/js/search.js index 73178f5..d805c11 100644 --- a/static/js/search.js +++ b/static/js/search.js @@ -1,6 +1,5 @@ let searchBar = document.getElementById("searchBar"); let pathBar = document.getElementById("pathBar"); -let must_match = true; let scroll_id = null; let docCount = 0; let searchQueued = false; @@ -8,7 +7,6 @@ let coolingDown = false; let selectedDirs = []; function toggleSearchBar() { - must_match = !must_match; searchQueued = true; } @@ -117,6 +115,21 @@ function humanFileSize(bytes) { return bytes.toFixed(1) + ' ' + units[u]; } +/** + * https://stackoverflow.com/questions/6312993 + */ +function humanTime (sec_num) { + sec_num = Math.floor(sec_num); + let hours = Math.floor(sec_num / 3600); + let minutes = Math.floor((sec_num - (hours * 3600)) / 60); + let seconds = sec_num - (hours * 3600) - (minutes * 60); + + if (hours < 10) {hours = "0" + hours;} + if (minutes < 10) {minutes = "0" + minutes;} + if (seconds < 10) {seconds = "0" + seconds;} + return hours + ":" + minutes + ":" + seconds; +} + function initPopover() { $('[data-toggle="popover"]').popover({ @@ -146,7 +159,7 @@ function gifOver(thumbnail, documentId) { //Load gif thumbnail.setAttribute("src", "/file/" + documentId); } - }, 750); //todo grab hover time from config + }, 750); }); @@ -158,43 +171,6 @@ function gifOver(thumbnail, documentId) { }) } -function videoOver(thumbnail, imgWrapper, thumbnailOverlay, documentId, docCard) { - - docCard.addEventListener("focus", function () { - let callee = arguments.callee; - docCard.mouseStayedOver = true; - - window.setTimeout(function() { - - if(docCard.mouseStayedOver) { - docCard.removeEventListener('focus', callee, false); - - imgWrapper.removeChild(thumbnail); - imgWrapper.removeChild(thumbnailOverlay); - - let video = document.createElement("video"); - let vidSource = document.createElement("source"); - vidSource.setAttribute("src", "/file/" + documentId); - vidSource.setAttribute("type", "video/webm"); - video.appendChild(vidSource); - video.setAttribute("class", "fit"); - video.setAttribute("loop", ""); - video.setAttribute("controls", ""); - video.setAttribute("preload", ""); - video.setAttribute("poster", "/thumb/" + documentId); - imgWrapper.appendChild(video); - - video.addEventListener("dblclick", function() { - video.webkitRequestFullScreen(); - }) - } - }, 750); - }); - docCard.addEventListener("blur", function() { - docCard.mouseStayedOver = false; - }); -} - function downloadPopover(element, documentId) { element.setAttribute("data-content", ' Download' + @@ -254,9 +230,24 @@ function createDocCard(hit) { switch (mimeCategory) { case "video": + thumbnail = document.createElement("video"); + let vidSource = document.createElement("source"); + vidSource.setAttribute("src", "/file/" + hit["_id"]); + vidSource.setAttribute("type", "video/webm"); + thumbnail.appendChild(vidSource); + thumbnail.setAttribute("class", "fit"); + thumbnail.setAttribute("loop", ""); + thumbnail.setAttribute("controls", ""); + thumbnail.setAttribute("preload", "none"); + thumbnail.setAttribute("poster", "/thumb/" + hit["_id"]); + thumbnail.addEventListener("dblclick", function() { + thumbnail.webkitRequestFullScreen(); + }); + + break; case "image": thumbnail = document.createElement("img"); - thumbnail.setAttribute("class", "card-img-top"); + thumbnail.setAttribute("class", "card-img-top fit"); thumbnail.setAttribute("src", "/thumb/" + hit["_id"]); break; } @@ -291,12 +282,9 @@ function createDocCard(hit) { //Duration let durationBadge = document.createElement("span"); durationBadge.setAttribute("class", "badge badge-resolution"); - durationBadge.appendChild(document.createTextNode(parseFloat(hit["_source"]["duration"]).toFixed(2) + "s")); + durationBadge.appendChild(document.createTextNode(humanTime(hit["_source"]["duration"]))); thumbnailOverlay.appendChild(durationBadge); - //Hover - videoOver(thumbnail, imgWrapper, thumbnailOverlay, hit["_id"], docCard) - } //Tags @@ -522,7 +510,7 @@ function search() { postBody.size_min = size_min; postBody.size_max = size_max; postBody.mime_types = getSelectedMimeTypes(); - postBody.must_match = must_match; + postBody.must_match = document.getElementById("barToggle").checked; postBody.directories = selectedDirs; postBody.path = pathBar.value.replace(/\/$/, "").toLowerCase(); //remove trailing slashes xhttp.setRequestHeader('content-type', 'application/json'); @@ -539,7 +527,7 @@ searchBar.addEventListener("keyup", function () { }); //Size slider -let sizeSlider = $("#sizeSlider").ionRangeSlider({ +$("#sizeSlider").ionRangeSlider({ type: "double", grid: false, force_edges: true, @@ -569,7 +557,7 @@ let sizeSlider = $("#sizeSlider").ionRangeSlider({ searchQueued = true; } -})[0]; +}); //Directories select function updateDirectories() { @@ -600,5 +588,8 @@ function getPathChoices() { }); } +document.getElementById("pathBar").addEventListener("keyup", function () { + searchQueued = true; +}); window.setInterval(search, 75); \ No newline at end of file diff --git a/templates/dashboard.html b/templates/dashboard.html index b2e7a21..5716451 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -3,229 +3,68 @@ {% block body %} -
-
-
-
-
FSE Info
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Version
1.0a
Total thumbnail size
652.08 Mb
Total document count
1258902
Total document size
4.7 TB
Folder count
4
User count
1
SQLite database path./local_storage.db
-
-
- -
-
Elasticsearch info
-
- - - - - - - - - - - - - - - - - - - - - -
Total index size
3.7 GB
HTTP port
9200
Version
6.2.1
Lucene version
7.2.1
Path
./elasticsearch/
-
+
+
+
FSE Information
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Version
{{ version }}
Total thumbnail cache size
{{ tn_size_total }}
Total document count
{{ doc_count }}
Total size of indexed documents
{{ doc_size }}
Total index size
{{ index_size }}
User count
1
SQLite database path
{{ db_path }}
Elasticsearch URL
{{ elasticsearch_url }}
+
Change global settings in config.py
-
-
-
-
-
Thumbnail cache size
-
- - -
-
-
-
-
-
-
-
Document size
-
- - - -
-
-
-
-
-
-
-
Document count
-
{# todo padding 8px 10px 5px 10px #} - - -
-
-
+
+
Actions
+
+
-
+ + {% endblock body %} \ No newline at end of file diff --git a/templates/directory_manage.html b/templates/directory_manage.html index ea10465..2441c1e 100644 --- a/templates/directory_manage.html +++ b/templates/directory_manage.html @@ -98,49 +98,13 @@ Thumbnail cache size 
{{ tn_size_formatted }} ({{ tn_size }} bytes)
- - - Index size - 
{{ index_size_formatted }} ({{ index_size }} bytes)
- - - - Document count - 
{{ doc_count }}
-
-
An excellent option list
-
- - - - - - - - - {% for option in directory.options %} - - - - - - - {% endfor %} - - -
OptionValue
{{ option.key }}
{{ option.value }}
- -
-
- -
-
An excellent control panel
+
Actions
@@ -174,6 +138,33 @@
+ +
+ {# TODO: put github wiki link #} +
Options Learn more
+
+ + + + + + + + + {% for option in directory.options %} + + + + + + + {% endfor %} + + +
OptionValue
{{ option.key }}
{{ option.value }}
+ +
+
diff --git a/templates/search.html b/templates/search.html index a5d90f1..45d4923 100644 --- a/templates/search.html +++ b/templates/search.html @@ -14,7 +14,7 @@
- +
diff --git a/templates/task.html b/templates/task.html index bf80b1d..fda5bd4 100644 --- a/templates/task.html +++ b/templates/task.html @@ -112,7 +112,13 @@ {% for task_id in tasks | sort() %}
{{ directories[tasks[task_id].dir_id].name }} - - {{ tasks[task_id].type }} + - + {% if tasks[task_id].type == 1 %} + Indexing + {% else %} + Thumbnail generation + {% endif %} +