simon987/Simple-Incremental-Search-Tool
1
0
Fork 0
mirror of https://github.com/simon987/Simple-Incremental-Search-Tool.git synced 2025-04-10 05:56:44 +00:00
Code Issues Projects Releases Wiki Activity

ES 7.X support. Bug fixes. UI tweaks. Config fixes

Browse Source
This commit is contained in:
simon 2019-05-26 11:31:28 -04:00
parent 64b743870f
commit 980babc5cc
12 changed files with 139 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.py
View File

@ -5,7 +5,8 @@ default_options = {
"ThumbnailColor": "FF00FF", "ThumbnailColor": "FF00FF",
"TextFileContentLength": "2000", "TextFileContentLength": "2000",
"PdfFileContentLength": "2000", "PdfFileContentLength": "2000",
"SpreadsheetContentLength": "2000", "DocxContentLength": "2000",
"SpreadSheetContentLength": "2000",
"EbookContentLength": "2000", "EbookContentLength": "2000",
"MimeGuesser": "extension", # extension, content "MimeGuesser": "extension", # extension, content
"CheckSumCalculators": "", # md5, sha1, sha256 "CheckSumCalculators": "", # md5, sha1, sha256
@ -19,6 +20,7 @@ index_every = 10000
nGramMin = 3 nGramMin = 3
nGramMax = 3 nGramMax = 3
elasticsearch_url = "http://localhost:9200" elasticsearch_url = "http://localhost:9200"
elasticsearch_index = "sist"
# Password hashing # Password hashing
bcrypt_rounds = 13 bcrypt_rounds = 13
@ -41,4 +43,4 @@ try:
except: except:
cairosvg = False cairosvg = False
VERSION = "1.1a" VERSION = "1.2a"

67
crawler.py
View File

@ -11,7 +11,7 @@ import config
from indexer import Indexer from indexer import Indexer
from parsing import GenericFileParser, Md5CheckSumCalculator, ExtensionMimeGuesser, MediaFileParser, TextFileParser, \ from parsing import GenericFileParser, Md5CheckSumCalculator, ExtensionMimeGuesser, MediaFileParser, TextFileParser, \
PictureFileParser, Sha1CheckSumCalculator, Sha256CheckSumCalculator, ContentMimeGuesser, MimeGuesser, FontParser, \ PictureFileParser, Sha1CheckSumCalculator, Sha256CheckSumCalculator, ContentMimeGuesser, MimeGuesser, FontParser, \
PdfFileParser, DocxParser, EbookParser PdfFileParser, DocxParser, EbookParser, SpreadSheetParser
from search import Search from search import Search
from storage import Directory from storage import Directory
from storage import Task, LocalStorage from storage import Task, LocalStorage
@ -53,7 +53,7 @@ class Crawler:
self.mime_guesser = mime_guesser self.mime_guesser = mime_guesser
def crawl(self, root_dir: str, counter: Value = None, total_files = None): def crawl(self, root_dir: str, counter: Value = None, total_files=None):
in_q = Queue(50000) # TODO: get from config? in_q = Queue(50000) # TODO: get from config?
out_q = Queue() out_q = Queue()
@ -154,7 +154,8 @@ class Crawler:
pass pass
finally: finally:
out_q.task_done() out_q.task_done()
self.indexer.index(self.documents, self.dir_id) if self.documents:
self.indexer.index(self.documents, self.dir_id)
class TaskManager: class TaskManager:
@ -162,7 +163,7 @@ class TaskManager:
self.current_task = None self.current_task = None
self.storage = storage self.storage = storage
self.current_process = None self.current_process = None
self.indexer = Indexer("changeme") self.indexer = Indexer(config.elasticsearch_index)
scheduler = BackgroundScheduler() scheduler = BackgroundScheduler()
scheduler.add_job(self.check_new_task, "interval", seconds=0.5) scheduler.add_job(self.check_new_task, "interval", seconds=0.5)
@ -188,34 +189,54 @@ class TaskManager:
def execute_crawl(self, directory: Directory, counter: Value, done: Value, total_files: Value): def execute_crawl(self, directory: Directory, counter: Value, done: Value, total_files: Value):
Search("changeme").delete_directory(directory.id) Search(config.elasticsearch_index).delete_directory(directory.id)
chksum_calcs = self.make_checksums_list(directory)
mime_guesser = ExtensionMimeGuesser() if directory.get_option("MimeGuesser") == "extension" \
else ContentMimeGuesser()
c = Crawler(self.make_parser_list(chksum_calcs, directory), mime_guesser, self.indexer, directory.id)
c.crawl(directory.path, counter, total_files)
done.value = 1
@staticmethod
def make_checksums_list(directory):
chksum_calcs = [] chksum_calcs = []
for arg in directory.get_option("CheckSumCalculators").split(","): for arg in directory.get_option("CheckSumCalculators").split(","):
if arg.strip() == "md5": if arg.strip() == "md5":
chksum_calcs.append(Md5CheckSumCalculator()) chksum_calcs.append(Md5CheckSumCalculator())
elif arg.strip() == "sha1": elif arg.strip() == "sha1":
chksum_calcs.append(Sha1CheckSumCalculator()) chksum_calcs.append(Sha1CheckSumCalculator())
elif arg.strip() == "sha256": elif arg.strip() == "sha256":
chksum_calcs.append(Sha256CheckSumCalculator()) chksum_calcs.append(Sha256CheckSumCalculator())
return chksum_calcs
mime_guesser = ExtensionMimeGuesser() if directory.get_option("MimeGuesser") == "extension" \ @staticmethod
else ContentMimeGuesser() def make_parser_list(chksum_calcs, directory):
p = [p.strip() for p in directory.get_option("FileParsers").split(",")]
c = Crawler([GenericFileParser(chksum_calcs, directory.path), parsers = [GenericFileParser(chksum_calcs, directory.path)]
MediaFileParser(chksum_calcs, directory.path), if "media" in p:
TextFileParser(chksum_calcs, int(directory.get_option("TextFileContentLength")), directory.path), parsers.append(MediaFileParser(chksum_calcs, directory.path))
PictureFileParser(chksum_calcs, directory.path), if "text" in p:
FontParser(chksum_calcs, directory.path), parsers.append(
PdfFileParser(chksum_calcs, int(directory.get_option("PdfFileContentLength")), directory.path), TextFileParser(chksum_calcs, int(directory.get_option("TextFileContentLength")), directory.path))
DocxParser(chksum_calcs, int(directory.get_option("SpreadsheetContentLength")), directory.path), if "picture" in p:
EbookParser(chksum_calcs, int(directory.get_option("EbookContentLength")), directory.path)], parsers.append(PictureFileParser(chksum_calcs, directory.path))
mime_guesser, self.indexer, directory.id) if "font" in p:
c.crawl(directory.path, counter, total_files) parsers.append(FontParser(chksum_calcs, directory.path))
if "pdf" in p:
done.value = 1 parsers.append(
PdfFileParser(chksum_calcs, int(directory.get_option("PdfFileContentLength")), directory.path))
if "docx" in p:
parsers.append(DocxParser(chksum_calcs, int(directory.get_option("DocxContentLength")), directory.path))
if "spreadsheet" in p:
parsers.append(
SpreadSheetParser(chksum_calcs, int(directory.get_option("SpreadSheetContentLength")), directory.path))
if "ebook" in p:
parsers.append(EbookParser(chksum_calcs, int(directory.get_option("EbookContentLength")), directory.path))
return parsers
def execute_thumbnails(self, directory: Directory, total_files: Value, counter: Value, done: Value): def execute_thumbnails(self, directory: Directory, total_files: Value, counter: Value, done: Value):
@ -223,7 +244,7 @@ class TaskManager:
if os.path.exists(dest_path): if os.path.exists(dest_path):
shutil.rmtree(dest_path) shutil.rmtree(dest_path)
docs = Search("changeme").get_all_documents(directory.id) docs = Search(config.elasticsearch_index).get_all_documents(directory.id)
tn_generator = ThumbnailGenerator(int(directory.get_option("ThumbnailSize")), tn_generator = ThumbnailGenerator(int(directory.get_option("ThumbnailSize")),
int(directory.get_option("ThumbnailQuality")), int(directory.get_option("ThumbnailQuality")),

4
indexer.py
View File

@ -13,7 +13,7 @@ class Indexer:
self.index_name = index self.index_name = index
self.es = elasticsearch.Elasticsearch() self.es = elasticsearch.Elasticsearch()
requests.head("http://localhost:9200") requests.head(config.elasticsearch_url)
if self.es.indices.exists(self.index_name): if self.es.indices.exists(self.index_name):
print("Index is already setup") print("Index is already setup")
else: else:
@ -93,7 +93,7 @@ class Indexer:
"genre": {"analyzer": "my_nGram", "type": "text"}, "genre": {"analyzer": "my_nGram", "type": "text"},
"album_artist": {"analyzer": "my_nGram", "type": "text"}, "album_artist": {"analyzer": "my_nGram", "type": "text"},
"content": {"analyzer": "content_analyser", "type": "text"}, "content": {"analyzer": "content_analyser", "type": "text"},
}}, doc_type="file", index=self.index_name) }}, doc_type="file", index=self.index_name, include_type_name=True)
self.es.indices.open(index=self.index_name) self.es.indices.open(index=self.index_name)

BIN
local_storage.db
View File

Binary file not shown.

2
run.py
View File

@ -25,7 +25,7 @@ flaskLogger = logging.getLogger('werkzeug')
flaskLogger.setLevel(logging.ERROR) flaskLogger.setLevel(logging.ERROR)
tm = TaskManager(storage) tm = TaskManager(storage)
search = Search("changeme") search = Search(config.elasticsearch_index)
def get_dir_size(path): def get_dir_size(path):

16
search.py
View File

@ -1,10 +1,12 @@
import json import json
import os import os
import elasticsearch import elasticsearch
import requests import requests
import config
from elasticsearch import helpers from elasticsearch import helpers
import config
class Search: class Search:
@ -14,9 +16,8 @@ class Search:
try: try:
requests.head(config.elasticsearch_url) requests.head(config.elasticsearch_url)
print("elasticsearch is already running")
except: except:
print("elasticsearch is not running") print("elasticsearch is not running!")
self.search_iterator = None self.search_iterator = None
@ -33,7 +34,6 @@ class Search:
info = requests.get("http://localhost:9200/" + self.index_name + "/_stats") info = requests.get("http://localhost:9200/" + self.index_name + "/_stats")
if info.status_code == 200: if info.status_code == 200:
parsed_info = json.loads(info.text) parsed_info = json.loads(info.text)
return int(parsed_info["indices"][self.index_name]["total"]["store"]["size_in_bytes"]) return int(parsed_info["indices"][self.index_name]["total"]["store"]["size_in_bytes"])
@ -171,8 +171,9 @@ class Search:
path_list = [] path_list = []
for option in suggestions["suggest"]["path"][0]["options"]: if "suggest" in suggestions:
path_list.append(option["_source"]["path"]) for option in suggestions["suggest"]["path"][0]["options"]:
path_list.append(option["_source"]["path"])
return path_list return path_list
@ -202,6 +203,3 @@ class Search:
print("Error: multiple delete tasks at the same time") print("Error: multiple delete tasks at the same time")
except Exception as e: except Exception as e:
print(e) print(e)

91
static/js/search.js
View File

@ -24,7 +24,7 @@ new InspireTreeDOM(tree, {
tree.select(); tree.select();
tree.node("any").deselect(); tree.node("any").deselect();
tree.on("node.click", function(event, node, handler) { tree.on("node.click", function (event, node, handler) {
event.preventTreeDefault(); event.preventTreeDefault();
if (node.id === "any") { if (node.id === "any") {
@ -44,23 +44,23 @@ new autoComplete({
selector: '#pathBar', selector: '#pathBar',
minChars: 1, minChars: 1,
delay: 75, delay: 75,
renderItem: function (item){ renderItem: function (item) {
return '<div class="autocomplete-suggestion" data-val="' + item + '">' + item + '</div>'; return '<div class="autocomplete-suggestion" data-val="' + item + '">' + item + '</div>';
}, },
source: async function(term, suggest) { source: async function (term, suggest) {
term = term.toLowerCase(); term = term.toLowerCase();
const choices = await getPathChoices(); const choices = await getPathChoices();
let matches = []; let matches = [];
for (let i=0; i<choices.length; i++) { for (let i = 0; i < choices.length; i++) {
if (~choices[i].toLowerCase().indexOf(term)) { if (~choices[i].toLowerCase().indexOf(term)) {
matches.push(choices[i]); matches.push(choices[i]);
} }
} }
suggest(matches); suggest(matches);
}, },
onSelect: function() { onSelect: function () {
searchQueued = true; searchQueued = true;
} }
}); });
@ -73,7 +73,9 @@ function makeStatsCard(searchResult) {
statsCardBody.setAttribute("class", "card-body"); statsCardBody.setAttribute("class", "card-body");
let stat = document.createElement("p"); let stat = document.createElement("p");
stat.appendChild(document.createTextNode(searchResult["hits"]["total"] + " results in " + searchResult["took"] + "ms")); const totalHits = searchResult["hits"]["total"].hasOwnProperty("value")
? searchResult["hits"]["total"]["value"] : searchResult["hits"]["total"];
stat.appendChild(document.createTextNode(totalHits + " results in " + searchResult["took"] + "ms"));
let sizeStat = document.createElement("span"); let sizeStat = document.createElement("span");
sizeStat.appendChild(document.createTextNode(humanFileSize(searchResult["aggregations"]["total_size"]["value"]))); sizeStat.appendChild(document.createTextNode(humanFileSize(searchResult["aggregations"]["total_size"]["value"])));
@ -97,20 +99,20 @@ function makeResultContainer() {
*/ */
function humanFileSize(bytes) { function humanFileSize(bytes) {
if(bytes === 0) { if (bytes === 0) {
return "? B" return "? B"
} }
let thresh = 1000; let thresh = 1000;
if(Math.abs(bytes) < thresh) { if (Math.abs(bytes) < thresh) {
return bytes + ' B'; return bytes + ' B';
} }
let units = ['kB','MB','GB','TB','PB','EB','ZB','YB']; let units = ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
let u = -1; let u = -1;
do { do {
bytes /= thresh; bytes /= thresh;
++u; ++u;
} while(Math.abs(bytes) >= thresh && u < units.length - 1); } while (Math.abs(bytes) >= thresh && u < units.length - 1);
return bytes.toFixed(1) + ' ' + units[u]; return bytes.toFixed(1) + ' ' + units[u];
} }
@ -118,15 +120,21 @@ function humanFileSize(bytes) {
/** /**
* https://stackoverflow.com/questions/6312993 * https://stackoverflow.com/questions/6312993
*/ */
function humanTime (sec_num) { function humanTime(sec_num) {
sec_num = Math.floor(sec_num); sec_num = Math.floor(sec_num);
let hours = Math.floor(sec_num / 3600); let hours = Math.floor(sec_num / 3600);
let minutes = Math.floor((sec_num - (hours * 3600)) / 60); let minutes = Math.floor((sec_num - (hours * 3600)) / 60);
let seconds = sec_num - (hours * 3600) - (minutes * 60); let seconds = sec_num - (hours * 3600) - (minutes * 60);
if (hours < 10) {hours = "0" + hours;} if (hours < 10) {
if (minutes < 10) {minutes = "0" + minutes;} hours = "0" + hours;
if (seconds < 10) {seconds = "0" + seconds;} }
if (minutes < 10) {
minutes = "0" + minutes;
}
if (seconds < 10) {
seconds = "0" + seconds;
}
return hours + ":" + minutes + ":" + seconds; return hours + ":" + minutes + ":" + seconds;
} }
@ -134,7 +142,7 @@ function humanTime (sec_num) {
function initPopover() { function initPopover() {
$('[data-toggle="popover"]').popover({ $('[data-toggle="popover"]').popover({
trigger: "focus", trigger: "focus",
delay: { "show": 0, "hide": 100 }, delay: {"show": 0, "hide": 100},
placement: "bottom", placement: "bottom",
html: true html: true
}); });
@ -152,7 +160,7 @@ function gifOver(thumbnail, documentId) {
thumbnail.mouseStayedOver = true; thumbnail.mouseStayedOver = true;
window.setTimeout(function() { window.setTimeout(function () {
if (thumbnail.mouseStayedOver) { if (thumbnail.mouseStayedOver) {
thumbnail.removeEventListener('mouseover', callee, false); thumbnail.removeEventListener('mouseover', callee, false);
@ -163,7 +171,7 @@ function gifOver(thumbnail, documentId) {
}); });
thumbnail.addEventListener("mouseout", function() { thumbnail.addEventListener("mouseout", function () {
//Reset timer //Reset timer
thumbnail.mouseStayedOver = false; thumbnail.mouseStayedOver = false;
thumbnail.setAttribute("src", "/thumb/" + documentId); thumbnail.setAttribute("src", "/thumb/" + documentId);
@ -173,10 +181,10 @@ function gifOver(thumbnail, documentId) {
function downloadPopover(element, documentId) { function downloadPopover(element, documentId) {
element.setAttribute("data-content", element.setAttribute("data-content",
'<a class="btn btn-sm btn-primary" href="/dl/'+ documentId +'"><i class="fas fa-download"></i> Download</a>' + '<a class="btn btn-sm btn-primary" href="/dl/' + documentId + '"><i class="fas fa-download"></i> Download</a>' +
'<a class="btn btn-sm btn-success" style="margin-left:3px;" href="/file/'+ documentId + '" target="_blank"><i class="fas fa-eye"></i> View</a>'); '<a class="btn btn-sm btn-success" style="margin-left:3px;" href="/file/' + documentId + '" target="_blank"><i class="fas fa-eye"></i> View</a>');
element.setAttribute("data-toggle", "popover"); element.setAttribute("data-toggle", "popover");
element.addEventListener("mouseover", function() { element.addEventListener("mouseover", function () {
element.focus(); element.focus();
}); });
} }
@ -242,7 +250,7 @@ function createDocCard(hit) {
thumbnail.setAttribute("controls", ""); thumbnail.setAttribute("controls", "");
thumbnail.setAttribute("preload", "none"); thumbnail.setAttribute("preload", "none");
thumbnail.setAttribute("poster", "/thumb/" + hit["_id"]); thumbnail.setAttribute("poster", "/thumb/" + hit["_id"]);
thumbnail.addEventListener("dblclick", function() { thumbnail.addEventListener("dblclick", function () {
thumbnail.webkitRequestFullScreen(); thumbnail.webkitRequestFullScreen();
}); });
@ -272,7 +280,7 @@ function createDocCard(hit) {
var format = hit["_source"]["format_name"]; var format = hit["_source"]["format_name"];
//Hover //Hover
if(format === "GIF") { if (format === "GIF") {
gifOver(thumbnail, hit["_id"]); gifOver(thumbnail, hit["_id"]);
} }
break; break;
@ -302,10 +310,12 @@ function createDocCard(hit) {
break; break;
case "image": { case "image": {
let formatTag = document.createElement("span"); if (format !== undefined) {
formatTag.setAttribute("class", "badge badge-pill badge-image"); let formatTag = document.createElement("span");
formatTag.appendChild(document.createTextNode(format)); formatTag.setAttribute("class", "badge badge-pill badge-image");
tags.push(formatTag); formatTag.appendChild(document.createTextNode(format));
tags.push(formatTag);
}
} }
break; break;
case "audio": { case "audio": {
@ -315,9 +325,7 @@ function createDocCard(hit) {
formatTag.appendChild(document.createTextNode(hit["_source"]["format_long_name"])); formatTag.appendChild(document.createTextNode(hit["_source"]["format_long_name"]));
tags.push(formatTag); tags.push(formatTag);
} }
} }
break; break;
case "text": { case "text": {
let formatTag = document.createElement("span"); let formatTag = document.createElement("span");
@ -387,13 +395,15 @@ function createDocCard(hit) {
function makePageIndicator(searchResult) { function makePageIndicator(searchResult) {
let pageIndicator = document.createElement("div"); let pageIndicator = document.createElement("div");
pageIndicator.appendChild(document.createTextNode(docCount + " / " +searchResult["hits"]["total"])); const totalHits = searchResult["hits"]["total"].hasOwnProperty("value")
? searchResult["hits"]["total"]["value"] : searchResult["hits"]["total"];
pageIndicator.appendChild(document.createTextNode(docCount + " / " + totalHits));
return pageIndicator; return pageIndicator;
} }
function insertHits(resultContainer, hits) { function insertHits(resultContainer, hits) {
for (let i = 0 ; i < hits.length; i++) { for (let i = 0; i < hits.length; i++) {
resultContainer.appendChild(createDocCard(hits[i])); resultContainer.appendChild(createDocCard(hits[i]));
docCount++; docCount++;
} }
@ -409,7 +419,7 @@ window.addEventListener("scroll", function () {
//load next page //load next page
let xhttp = new XMLHttpRequest(); let xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() { xhttp.onreadystatechange = function () {
if (this.readyState === 4 && this.status === 200) { if (this.readyState === 4 && this.status === 200) {
let searchResult = JSON.parse(this.responseText); let searchResult = JSON.parse(this.responseText);
@ -449,7 +459,7 @@ function getSelectedMimeTypes() {
for (let i = 0; i < selected.length; i++) { for (let i = 0; i < selected.length; i++) {
if(selected[i].id === "any") { if (selected[i].id === "any") {
return "any" return "any"
} }
@ -468,7 +478,7 @@ function search() {
searchQueued = false; searchQueued = false;
//Clear old search results //Clear old search results
let searchResults = document.getElementById("searchResults"); let searchResults = document.getElementById("searchResults");
while (searchResults.firstChild) { while (searchResults.firstChild) {
searchResults.removeChild(searchResults.firstChild); searchResults.removeChild(searchResults.firstChild);
} }
@ -476,7 +486,7 @@ function search() {
let query = searchBar.value; let query = searchBar.value;
let xhttp = new XMLHttpRequest(); let xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() { xhttp.onreadystatechange = function () {
if (this.readyState === 4 && this.status === 200) { if (this.readyState === 4 && this.status === 200) {
let searchResult = JSON.parse(this.responseText); let searchResult = JSON.parse(this.responseText);
@ -542,7 +552,7 @@ $("#sizeSlider").ionRangeSlider({
drag_interval: true, drag_interval: true,
prettify: function (num) { prettify: function (num) {
if(num === 0) { if (num === 0) {
return "0 B" return "0 B"
} else if (num >= 3684) { } else if (num >= 3684) {
return humanFileSize(num * num * num) + "+"; return humanFileSize(num * num * num) + "+";
@ -550,11 +560,11 @@ $("#sizeSlider").ionRangeSlider({
return humanFileSize(num * num * num) return humanFileSize(num * num * num)
}, },
onChange: function(e) { onChange: function (e) {
size_min = (e.from * e.from * e.from); size_min = (e.from * e.from * e.from);
size_max = (e.to * e.to * e.to); size_max = (e.to * e.to * e.to);
if (e.to >= 3684) { if (e.to >= 3684) {
size_max = 10000000000000; size_max = 10000000000000;
} }
@ -566,12 +576,13 @@ $("#sizeSlider").ionRangeSlider({
function updateDirectories() { function updateDirectories() {
let selected = $('#directories').find('option:selected'); let selected = $('#directories').find('option:selected');
selectedDirs = []; selectedDirs = [];
$(selected).each(function(){ $(selected).each(function () {
selectedDirs.push(parseInt($(this).val())); selectedDirs.push(parseInt($(this).val()));
}); });
searchQueued = true; searchQueued = true;
} }
document.getElementById("directories").addEventListener("change", updateDirectories); document.getElementById("directories").addEventListener("change", updateDirectories);
updateDirectories(); updateDirectories();
searchQueued = false; searchQueued = false;
@ -581,7 +592,7 @@ function getPathChoices() {
return new Promise(getPaths => { return new Promise(getPaths => {
let xhttp = new XMLHttpRequest(); let xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() { xhttp.onreadystatechange = function () {
if (this.readyState === 4 && this.status === 200) { if (this.readyState === 4 && this.status === 200) {
getPaths(JSON.parse(xhttp.responseText)) getPaths(JSON.parse(xhttp.responseText))
} }

21
storage.py
View File

@ -1,7 +1,9 @@
import sqlite3
import os import os
import flask_bcrypt import sqlite3
import time import time
import flask_bcrypt
import config import config
@ -39,7 +41,7 @@ class Option:
Data structure to hold a directory option Data structure to hold a directory option
""" """
def __init__(self, key: str, value: str, dir_id: int=None, opt_id: int = None): def __init__(self, key: str, value: str, dir_id: int = None, opt_id: int = None):
self.key = key self.key = key
self.value = value self.value = value
self.id = opt_id self.id = opt_id
@ -50,6 +52,7 @@ class Directory:
""" """
Data structure to hold directory information Data structure to hold directory information
""" """
def __init__(self, path: str, enabled: bool, options: list, name: str): def __init__(self, path: str, enabled: bool, options: list, name: str):
self.id = None self.id = None
self.path = path self.path = path
@ -59,7 +62,7 @@ class Directory:
def __str__(self): def __str__(self):
return self.path + " | enabled: " + str(self.enabled) + " | opts: " + str(self.options) return self.path + " | enabled: " + str(self.enabled) + " | opts: " + str(self.options)
def get_option(self, key): def get_option(self, key):
for option in self.options: for option in self.options:
@ -77,7 +80,6 @@ class Directory:
class Task: class Task:
INDEX = 1 INDEX = 1
GEN_THUMBNAIL = 2 GEN_THUMBNAIL = 2
@ -268,7 +270,7 @@ class LocalStorage:
c = conn.cursor() c = conn.cursor()
c.execute("UPDATE User SET is_admin=? WHERE username=?", (user.admin, user.username)) c.execute("UPDATE User SET is_admin=? WHERE username=?", (user.admin, user.username))
c.execute("DELETE FROM User_canRead_Directory WHERE username=?", (user.username, )) c.execute("DELETE FROM User_canRead_Directory WHERE username=?", (user.username,))
conn.commit() conn.commit()
for access in user.readable_directories: for access in user.readable_directories:
@ -332,7 +334,7 @@ class LocalStorage:
conn = sqlite3.connect(self.db_path) conn = sqlite3.connect(self.db_path)
c = conn.cursor() c = conn.cursor()
c.execute("DELETE FROM Option WHERE id=?", (opt_id, )) c.execute("DELETE FROM Option WHERE id=?", (opt_id,))
conn.commit() conn.commit()
conn.close() conn.close()
@ -385,7 +387,6 @@ class LocalStorage:
conn.close() conn.close()
for db_task in tasks: for db_task in tasks:
task = Task(db_task[2], db_task[1], db_task[3], db_task[4], db_task[0]) task = Task(db_task[2], db_task[1], db_task[3], db_task[4], db_task[0])
self.cached_tasks[task.id] = task self.cached_tasks[task.id] = task
@ -402,7 +403,7 @@ class LocalStorage:
conn = sqlite3.connect(self.db_path) conn = sqlite3.connect(self.db_path)
c = conn.cursor() c = conn.cursor()
c.execute("DELETE FROM Task WHERE id=?", (task_id, )) c.execute("DELETE FROM Task WHERE id=?", (task_id,))
conn.commit() conn.commit()
c.close() c.close()
@ -438,4 +439,4 @@ class LocalStorage:
for access in accesses: for access in accesses:
access_list.append(access[1]) access_list.append(access[1])
return access_list return access_list

2
templates/directory.html
View File

@ -44,7 +44,7 @@
<td>{{ directories[dir].name }}</td> <td>{{ directories[dir].name }}</td>
<td style="word-break: break-all"><pre>{{ directories[dir].path }}</pre></td> <td style="word-break: break-all"><pre>{{ directories[dir].path }}</pre></td>
<td><i class="far {{ "fa-check-square" if directories[dir].enabled else "fa-square" }}"></i></td> <td><i class="far {{ "fa-check-square" if directories[dir].enabled else "fa-square" }}"></i></td>
<td><a href="directory/{{ dir }}" class="btn btn-primary"><i class="fas fa-cog"></i> </a> Manage</td> <td><a href="directory/{{ dir }}" class="btn btn-primary"><i class="fas fa-cog"></i> Manage</a> </td>
</tr> </tr>
{% endfor %} {% endfor %}
</tbody> </tbody>

26
templates/layout.html
View File

@ -59,18 +59,20 @@
<div class="collapse navbar-collapse" id="navbarSupportedContent"> <div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav mr-auto"> <ul class="navbar-nav mr-auto">
<li class="nav-item"> {% if session["username"] %}
<a class="nav-link {% if "directory" == active_page %}active{% endif %}" href="/directory">Directories</a> <li class="nav-item">
</li> <a class="nav-link {% if "directory" == active_page %}active{% endif %}" href="/directory">Directories</a>
<li class="nav-item"> </li>
<a class="nav-link {% if "task" == active_page %}active{% endif %}" href="/task">Tasks</a> <li class="nav-item">
</li> <a class="nav-link {% if "task" == active_page %}active{% endif %}" href="/task">Tasks</a>
<li class="nav-item"> </li>
<a class="nav-link {% if "user" == active_page %}active{% endif %}" href="/user">Users</a> <li class="nav-item">
</li> <a class="nav-link {% if "user" == active_page %}active{% endif %}" href="/user">Users</a>
<li class="nav-item"> </li>
<a class="nav-link {% if "dashboard" == active_page %}active{% endif %}" href="/dashboard">Dashboard</a> <li class="nav-item">
</li> <a class="nav-link {% if "dashboard" == active_page %}active{% endif %}" href="/dashboard">Dashboard</a>
</li>
{% endif %}
</ul> </ul>
{% if session["username"] %} {% if session["username"] %}

4
templates/task.html
View File

@ -81,7 +81,7 @@
if (currentTask.total === 0) { if (currentTask.total === 0) {
document.getElementById("task-label-" + currentTask.id).innerHTML = "Calculating file count..."; document.getElementById("task-label-" + currentTask.id).innerHTML = "Initializing...";
} else { } else {
let bar = document.getElementById("task-bar-" + currentTask.id); let bar = document.getElementById("task-bar-" + currentTask.id);
@ -90,6 +90,8 @@
if (percent === 100) { if (percent === 100) {
bar.classList.add("bg-success") bar.classList.add("bg-success")
} else {
bar.classList.remove("bg-success")
} }
} }

2
thumbnail.py
View File

@ -36,7 +36,7 @@ class ThumbnailGenerator:
p.terminate() p.terminate()
print("Timed out: " + path) print("Timed out: " + path)
else: else:
self.generate_image("tmp", dest_path) self.generate_image(tmpfile, dest_path)
except Exception: except Exception:
print("Couldn't make thumbnail for " + path) print("Couldn't make thumbnail for " + path)