mirror of
https://github.com/simon987/Simple-Incremental-Search-Tool.git
synced 2025-04-10 14:06:41 +00:00
Added web interface, crawler and more work on local storage
This commit is contained in:
simon
parent
de0a835ecd
commit
165844e4ca
173
crawler.py
173
crawler.py
@ -1,152 +1,33 @@
|
||||
import os
|
||||
import hashlib
|
||||
|
||||
|
||||
class Crawler:
|
||||
pass
|
||||
|
||||
def __init__(self, enabled_parsers: list):
|
||||
self.documents = []
|
||||
self.enabled_parsers = enabled_parsers
|
||||
|
||||
def crawl(self, root_dir: str):
|
||||
for root, dirs, files in os.walk(root_dir):
|
||||
|
||||
for filename in files:
|
||||
full_path = os.path.join(root, filename)
|
||||
|
||||
parser = self.get_parser_by_ext(os.path.splitext(filename)[1])
|
||||
|
||||
doc = parser.parse(full_path)
|
||||
|
||||
self.documents.append(doc)
|
||||
|
||||
def get_parser_by_ext(self, ext: str):
|
||||
|
||||
for parser in self.enabled_parsers:
|
||||
|
||||
if ext in parser.extensions:
|
||||
return parser
|
||||
|
||||
for parser in self.enabled_parsers:
|
||||
if parser.is_default:
|
||||
return parser
|
||||
|
||||
|
||||
class FileParser:
|
||||
pass
|
||||
|
||||
|
||||
class FileCheckSumCalculator:
|
||||
|
||||
def checksum(self, path: str) -> str:
|
||||
"""
|
||||
Calculate the checksum of a file
|
||||
:param path: path of the file
|
||||
:return: checksum
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class Md5CheckSumCalculator(FileCheckSumCalculator):
|
||||
|
||||
def __init__(self):
|
||||
self.name = "md5"
|
||||
|
||||
def checksum(self, path: str) -> str:
|
||||
"""
|
||||
Calculate the md5 checksum of a file
|
||||
:param path: path of the file
|
||||
:return: md5 checksum
|
||||
"""
|
||||
result = hashlib.md5()
|
||||
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(65536), b""):
|
||||
result.update(block)
|
||||
|
||||
return result.hexdigest().upper()
|
||||
|
||||
|
||||
class Sha1CheckSumCalculator(FileCheckSumCalculator):
|
||||
|
||||
def __init__(self):
|
||||
self.name = "sha1"
|
||||
|
||||
def checksum(self, path: str) -> str:
|
||||
"""
|
||||
Calculate the sha1 checksum of a file
|
||||
:param path: path of the file
|
||||
:return: sha1 checksum
|
||||
"""
|
||||
result = hashlib.sha1()
|
||||
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(65536), b""):
|
||||
result.update(block)
|
||||
|
||||
return result.hexdigest().upper()
|
||||
|
||||
|
||||
class Sha256CheckSumCalculator(FileCheckSumCalculator):
|
||||
|
||||
def __init__(self):
|
||||
self.name = "sha256"
|
||||
|
||||
def checksum(self, path: str) -> str:
|
||||
"""
|
||||
Calculate the sha256 checksum of a file
|
||||
:param path: path of the file
|
||||
:return: sha256 checksum
|
||||
"""
|
||||
result = hashlib.sha256()
|
||||
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(65536), b""):
|
||||
result.update(block)
|
||||
|
||||
return result.hexdigest().upper()
|
||||
|
||||
|
||||
class GenericFileParser(FileParser):
|
||||
|
||||
def __init__(self, checksum_calculators: list):
|
||||
self.checksum_calculators = checksum_calculators
|
||||
|
||||
def parse(self, path: str) -> dict:
|
||||
"""
|
||||
Parse a generic file
|
||||
:param path: path of the file to parse
|
||||
:return: dict information about the file
|
||||
"""
|
||||
|
||||
info = dict()
|
||||
|
||||
info["size"] = os.path.getsize(path)
|
||||
info["name"] = os.path.splitext(path)[0]
|
||||
|
||||
for calculator in self.checksum_calculators:
|
||||
info[calculator.name] = calculator.checksum(path)
|
||||
|
||||
return info
|
||||
|
||||
|
||||
|
||||
|
||||
# def crawl(root_dir: str) -> None:
|
||||
# docs = []
|
||||
#
|
||||
# for root, dirs, files in os.walk(root_dir):
|
||||
#
|
||||
# print(root)
|
||||
#
|
||||
# for filename in files:
|
||||
# full_path = os.path.join(root, filename)
|
||||
#
|
||||
# doc = dict()
|
||||
#
|
||||
# doc["md5"] = md5sum(full_path)
|
||||
# doc["path"] = root
|
||||
# doc["name"] = filename
|
||||
# doc["size"] = os.path.getsize(full_path)
|
||||
# doc["mtime"] = int(os.path.getmtime(full_path))
|
||||
#
|
||||
# mime_type = mimetypes.guess_type(full_path)[0]
|
||||
#
|
||||
# if mime_type is not None:
|
||||
#
|
||||
# doc["mime"] = mime_type
|
||||
#
|
||||
# if mime_type.startswith("image"):
|
||||
# try:
|
||||
# width, height = Image.open(full_path).size
|
||||
#
|
||||
# doc["width"] = width
|
||||
# doc["height"] = height
|
||||
# except OSError:
|
||||
# doc.pop('mime', None)
|
||||
# pass
|
||||
# except ValueError:
|
||||
# doc.pop('mime', None)
|
||||
# pass
|
||||
#
|
||||
# docs.append(doc)
|
||||
#
|
||||
# file = open("crawler.json", "w")
|
||||
# file.write(simplejson.dumps(docs))
|
||||
# file.close()
|
||||
#
|
||||
#
|
||||
@ -4,6 +4,7 @@ PRAGMA FOREIGN_KEYS = ON;
|
||||
CREATE TABLE Directory (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
path TEXT UNIQUE,
|
||||
name TEXT,
|
||||
enabled BOOLEAN
|
||||
);
|
||||
|
||||
@ -19,10 +20,11 @@ CREATE TABLE Task (
|
||||
|
||||
-- You can set an option on a directory to change the crawler's behavior
|
||||
CREATE TABLE Option (
|
||||
name STRING,
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
key TEXT,
|
||||
value TEXT,
|
||||
directory_id INTEGER,
|
||||
FOREIGN KEY (directory_id) REFERENCES Directory(id),
|
||||
PRIMARY KEY (name, directory_id)
|
||||
FOREIGN KEY (directory_id) REFERENCES Directory(id)
|
||||
);
|
||||
|
||||
-- User accounts
|
||||
|
||||
55
indexer.py
Normal file
55
indexer.py
Normal file
@ -0,0 +1,55 @@
|
||||
import json
|
||||
import elasticsearch
|
||||
from threading import Thread
|
||||
import subprocess
|
||||
import requests
|
||||
|
||||
|
||||
class Indexer:
|
||||
|
||||
def __init__(self, index: str):
|
||||
|
||||
self.index_name = index
|
||||
self.es = elasticsearch.Elasticsearch()
|
||||
|
||||
try:
|
||||
requests.head("http://localhost:9200")
|
||||
print("elasticsearch is already running")
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
import time
|
||||
t = Thread(target=Indexer.run_elasticsearch)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
@staticmethod
|
||||
def run_elasticsearch():
|
||||
subprocess.Popen(["elasticsearch/bin/elasticsearch"])
|
||||
|
||||
@staticmethod
|
||||
def create_bulk_index_string(docs: list, index_name: str):
|
||||
"""
|
||||
Creates a insert string for sending to elasticsearch
|
||||
"""
|
||||
|
||||
result = ""
|
||||
|
||||
action_string = '{"index":{"_index":"' + index_name + '","_type":"file"}}\n'
|
||||
|
||||
for doc in docs:
|
||||
result += action_string
|
||||
result += json.dumps(doc) + "\n"
|
||||
|
||||
return result
|
||||
|
||||
def index(self, docs: list):
|
||||
|
||||
index_string = self.create_bulk_index_string(docs, self.index_name)
|
||||
self.es.bulk(index_string)
|
||||
|
||||
def clear(self):
|
||||
|
||||
self.es.indices.delete(self.index_name)
|
||||
self.es.indices.create(self.index_name)
|
||||
137
parsing.py
Normal file
137
parsing.py
Normal file
@ -0,0 +1,137 @@
|
||||
import hashlib
|
||||
import magic
|
||||
import os
|
||||
import mimetypes
|
||||
|
||||
|
||||
class MimeGuesser:
|
||||
|
||||
def guess_mime(self, full_path):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class ContentMimeGuesser(MimeGuesser):
|
||||
|
||||
def __init__(self):
|
||||
self.libmagic = magic.Magic(mime=True)
|
||||
|
||||
def guess_mime(self, full_path):
|
||||
return self.libmagic.from_file(full_path)
|
||||
|
||||
|
||||
class ExtensionMimeGuesser(MimeGuesser):
|
||||
|
||||
def guess_mime(self, full_path):
|
||||
return mimetypes.guess_type(full_path, strict=False)[0]
|
||||
|
||||
|
||||
class FileParser:
|
||||
extensions = []
|
||||
is_default = False
|
||||
pass
|
||||
|
||||
|
||||
class FileCheckSumCalculator:
|
||||
|
||||
def checksum(self, path: str) -> str:
|
||||
"""
|
||||
Calculate the checksum of a file
|
||||
:param path: path of the file
|
||||
:return: checksum
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class Md5CheckSumCalculator(FileCheckSumCalculator):
|
||||
|
||||
def __init__(self):
|
||||
self.name = "md5"
|
||||
|
||||
def checksum(self, path: str) -> str:
|
||||
"""
|
||||
Calculate the md5 checksum of a file
|
||||
:param path: path of the file
|
||||
:return: md5 checksum
|
||||
"""
|
||||
result = hashlib.md5()
|
||||
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(65536), b""):
|
||||
result.update(block)
|
||||
|
||||
return result.hexdigest().upper()
|
||||
|
||||
|
||||
class Sha1CheckSumCalculator(FileCheckSumCalculator):
|
||||
|
||||
def __init__(self):
|
||||
self.name = "sha1"
|
||||
|
||||
def checksum(self, path: str) -> str:
|
||||
"""
|
||||
Calculate the sha1 checksum of a file
|
||||
:param path: path of the file
|
||||
:return: sha1 checksum
|
||||
"""
|
||||
result = hashlib.sha1()
|
||||
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(65536), b""):
|
||||
result.update(block)
|
||||
|
||||
return result.hexdigest().upper()
|
||||
|
||||
|
||||
class Sha256CheckSumCalculator(FileCheckSumCalculator):
|
||||
|
||||
def __init__(self):
|
||||
self.name = "sha256"
|
||||
|
||||
def checksum(self, path: str) -> str:
|
||||
"""
|
||||
Calculate the sha256 checksum of a file
|
||||
:param path: path of the file
|
||||
:return: sha256 checksum
|
||||
"""
|
||||
result = hashlib.sha256()
|
||||
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(65536), b""):
|
||||
result.update(block)
|
||||
|
||||
return result.hexdigest().upper()
|
||||
|
||||
|
||||
class GenericFileParser(FileParser):
|
||||
|
||||
extensions = []
|
||||
is_default = True
|
||||
|
||||
def __init__(self, checksum_calculators: list, mime_guesser: MimeGuesser):
|
||||
|
||||
self.checksum_calculators = checksum_calculators
|
||||
self.mime_guesser = mime_guesser
|
||||
|
||||
def parse(self, full_path: str) -> dict:
|
||||
"""
|
||||
Parse a generic file
|
||||
:param full_path: path of the file to parse
|
||||
:return: dict information about the file
|
||||
"""
|
||||
|
||||
info = dict()
|
||||
|
||||
file_stat = os.stat(full_path)
|
||||
path, name = os.path.split(full_path)
|
||||
|
||||
info["size"] = file_stat.st_size
|
||||
info["path"] = path
|
||||
info["name"] = name
|
||||
info["mtime"] = file_stat.st_mtime
|
||||
info["mime"] = self.mime_guesser.guess_mime(full_path)
|
||||
|
||||
for calculator in self.checksum_calculators:
|
||||
info[calculator.name] = calculator.checksum(full_path)
|
||||
|
||||
return info
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
PIL
|
||||
simplejson
|
||||
flask
|
||||
flask_bcrypt
|
||||
flask_bcrypt
|
||||
elasticsearch
|
||||
python-magic
|
||||
requests
|
||||
91
run.py
91
run.py
@ -1,14 +1,8 @@
|
||||
from flask import Flask, render_template, send_file, request
|
||||
import pysolr
|
||||
import mimetypes
|
||||
import requests
|
||||
import json
|
||||
from PIL import Image
|
||||
import os
|
||||
from flask import Flask, render_template, send_file, request, redirect
|
||||
from indexer import Indexer
|
||||
from storage import Directory, Option
|
||||
|
||||
SOLR_URL = "http://localhost:8983/solr/test/"
|
||||
|
||||
solr = pysolr.Solr(SOLR_URL, timeout=10)
|
||||
# indexer = Indexer("fse")
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
@ -124,11 +118,86 @@ app = Flask(__name__)
|
||||
# return send_file("thumbnails/" + doc_id, mimetype=mimetypes.guess_type(thumb_path)[0])
|
||||
# else:
|
||||
# return "File not found"
|
||||
from storage import LocalStorage
|
||||
storage = LocalStorage("local_storage.db")
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def tmp_route():
|
||||
return "test"
|
||||
return "huh"
|
||||
|
||||
|
||||
@app.route("/directory")
|
||||
def directory():
|
||||
|
||||
directories = storage.dirs()
|
||||
print(directories)
|
||||
|
||||
return render_template("directory.html", directories=directories)
|
||||
|
||||
|
||||
@app.route("/directory/add")
|
||||
def directory_add():
|
||||
|
||||
path = request.args.get("path")
|
||||
name = request.args.get("name")
|
||||
|
||||
print(name)
|
||||
|
||||
if path is not None and name is not None:
|
||||
d = Directory(path, True, [], name)
|
||||
storage.save_directory(d)
|
||||
|
||||
return redirect("/directory")
|
||||
|
||||
return "Error" # todo better message
|
||||
|
||||
|
||||
@app.route("/directory/<int:dir_id>")
|
||||
def directory_manage(dir_id):
|
||||
|
||||
directory = storage.dirs()[dir_id]
|
||||
|
||||
return render_template("directory_manage.html", directory=directory)
|
||||
|
||||
|
||||
@app.route("/directory/<int:dir_id>/add_opt")
|
||||
def directory_add_opt(dir_id):
|
||||
|
||||
key = request.args.get("key")
|
||||
value = request.args.get("value")
|
||||
|
||||
if key is not None and value is not None:
|
||||
storage.save_option(Option(key, value, dir_id))
|
||||
|
||||
return redirect("/directory/" + str(dir_id))
|
||||
|
||||
|
||||
@app.route("/directory/<int:dir_id>/del_opt/<int:opt_id>")
|
||||
def directory_del_opt(dir_id, opt_id):
|
||||
|
||||
storage.del_option(opt_id)
|
||||
|
||||
return redirect("/directory/" + str(dir_id))
|
||||
|
||||
|
||||
@app.route("/directory/<int:dir_id>/del")
|
||||
def directory_del(dir_id):
|
||||
|
||||
storage.remove_directory(dir_id)
|
||||
|
||||
return redirect("/directory")
|
||||
|
||||
|
||||
@app.route("/task")
|
||||
def task():
|
||||
|
||||
return
|
||||
|
||||
@app.route("/dashboard")
|
||||
def dashboard():
|
||||
|
||||
return render_template("dashboard.html")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -1,10 +1,22 @@
|
||||
from unittest import TestCase
|
||||
|
||||
from parsing import GenericFileParser, Sha1CheckSumCalculator, ExtensionMimeGuesser
|
||||
from crawler import Crawler
|
||||
|
||||
|
||||
class CrawlerTest(TestCase):
|
||||
|
||||
def test_dir_walk(self):
|
||||
c = Crawler()
|
||||
c = Crawler([GenericFileParser([Sha1CheckSumCalculator()], ExtensionMimeGuesser())])
|
||||
|
||||
c.crawl("test_folder")
|
||||
|
||||
self.assertEqual(len(c.documents), 28)
|
||||
|
||||
def test_get_parser_by_ext(self):
|
||||
|
||||
c = Crawler([GenericFileParser([Sha1CheckSumCalculator()], ExtensionMimeGuesser())])
|
||||
|
||||
self.assertIsInstance(c.get_parser_by_ext("any"), GenericFileParser)
|
||||
|
||||
# todo add more parsers here
|
||||
|
||||
@ -1,39 +1,52 @@
|
||||
import os
|
||||
from unittest import TestCase
|
||||
|
||||
from crawler import GenericFileParser, Md5CheckSumCalculator, Sha1CheckSumCalculator, Sha256CheckSumCalculator
|
||||
|
||||
from parsing import GenericFileParser, Md5CheckSumCalculator, Sha1CheckSumCalculator, Sha256CheckSumCalculator, ExtensionMimeGuesser
|
||||
|
||||
|
||||
class GenericFileParserTest(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
if os.path.exists("test_parse"):
|
||||
os.remove("test_parse")
|
||||
if os.path.exists("test_parse.txt"):
|
||||
os.remove("test_parse.txt")
|
||||
|
||||
test_file = open("test_parse", "w")
|
||||
test_file = open("test_parse.txt", "w")
|
||||
test_file.write("12345678")
|
||||
test_file.close()
|
||||
os.utime("test_parse.txt", (1330123456, 1330654321))
|
||||
|
||||
self.parser = GenericFileParser([Md5CheckSumCalculator()])
|
||||
self.parser = GenericFileParser([Md5CheckSumCalculator()], ExtensionMimeGuesser())
|
||||
|
||||
def tearDown(self):
|
||||
os.remove("test_parse")
|
||||
os.remove("test_parse.txt")
|
||||
|
||||
def test_parse_size(self):
|
||||
result = self.parser.parse("test_parse")
|
||||
result = self.parser.parse("test_parse.txt")
|
||||
|
||||
self.assertEqual(result["size"], 8)
|
||||
|
||||
def test_parse_name(self):
|
||||
result = self.parser.parse("test_parse")
|
||||
result = self.parser.parse("test_parse.txt")
|
||||
|
||||
self.assertEqual(result["name"], "test_parse")
|
||||
self.assertEqual(result["name"], "test_parse.txt")
|
||||
|
||||
def test_parse_md5(self):
|
||||
result = self.parser.parse("test_parse")
|
||||
result = self.parser.parse("test_parse.txt")
|
||||
|
||||
self.assertEqual(result["md5"], "25D55AD283AA400AF464C76D713C07AD")
|
||||
|
||||
def test_mtime(self):
|
||||
|
||||
result = self.parser.parse("test_parse.txt")
|
||||
|
||||
self.assertEqual(result["mtime"], 1330654321)
|
||||
|
||||
def test_mime(self):
|
||||
|
||||
result = self.parser.parse("test_parse.txt")
|
||||
self.assertEqual(result["mime"], "text/plain")
|
||||
|
||||
|
||||
class Md5CheckSumCalculatorTest(TestCase):
|
||||
|
||||
@ -128,4 +141,7 @@ class Sha256CheckSumCalculatorTest(TestCase):
|
||||
self.assertEqual(result, "DA7606DC763306B700685A71E2E72A2D95F1291209E5DA344B82DA2508FC27C5")
|
||||
|
||||
result = self.calculator.checksum("test_sha256_2")
|
||||
self.assertEqual(result, "C39C7E0E7D84C9692F3C9C22E1EA0327DEBF1BF531B5738EEA8E79FE27EBC570")
|
||||
self.assertEqual(result, "C39C7E0E7D84C9692F3C9C22E1EA0327DEBF1BF531B5738EEA8E79FE27EBC570")
|
||||
|
||||
|
||||
|
||||
|
||||
16
spec/Indexer_spec.py
Normal file
16
spec/Indexer_spec.py
Normal file
@ -0,0 +1,16 @@
|
||||
from unittest import TestCase
|
||||
from indexer import Indexer
|
||||
|
||||
|
||||
class IndexerTest(TestCase):
|
||||
|
||||
def test_create_bulk_query(self):
|
||||
|
||||
docs = [{"name": "doc1"}, {"name": "doc2"}]
|
||||
|
||||
result = Indexer.create_bulk_index_string(docs, "indexName")
|
||||
|
||||
self.assertEqual(result, '{"index":{"_index":"indexName","_type":"file"}}\n'
|
||||
'{"name": "doc1"}\n'
|
||||
'{"index":{"_index":"indexName","_type":"file"}}\n'
|
||||
'{"name": "doc2"}\n')
|
||||
@ -1,10 +1,13 @@
|
||||
from unittest import TestCase
|
||||
from storage import LocalStorage, Directory, DuplicateDirectoryException, User
|
||||
from storage import LocalStorage, Directory, DuplicateDirectoryException, User, DuplicateUserException, Option
|
||||
import os
|
||||
|
||||
|
||||
class LocalStorageTest(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
if os.path.exists("test_database.db"):
|
||||
os.remove("test_database.db")
|
||||
|
||||
s = LocalStorage("test_database.db")
|
||||
s.init_db("../database.sql")
|
||||
@ -13,42 +16,59 @@ class LocalStorageTest(TestCase):
|
||||
|
||||
storage = LocalStorage("test_database.db")
|
||||
|
||||
d = Directory("/some/directory", True, ["opt1", "opt2", "opt3"])
|
||||
d = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val2")], "An excellent name")
|
||||
|
||||
storage.save_directory(d)
|
||||
dir_id = storage.save_directory(d)
|
||||
|
||||
self.assertEqual(storage.dirs()[dir_id].enabled, True)
|
||||
self.assertEqual(storage.dirs()[dir_id].options[0].key, "key1")
|
||||
self.assertEqual(storage.dirs()[dir_id].options[0].value, "val1")
|
||||
self.assertEqual(storage.dirs()[dir_id].options[0].dir_id, 1)
|
||||
|
||||
self.assertEqual(storage.dirs()["/some/directory"].enabled, True)
|
||||
self.assertEqual(storage.dirs()["/some/directory"].options[0], "opt1")
|
||||
|
||||
def test_save_and_retrieve_dir_persistent(self):
|
||||
|
||||
s1 = LocalStorage("test_database.db")
|
||||
|
||||
d = Directory("/some/directory", True, ["opt1", "opt2", "opt3"])
|
||||
d = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val2")], "An excellent name")
|
||||
|
||||
s1.save_directory(d)
|
||||
dir_id = s1.save_directory(d)
|
||||
|
||||
s2 = LocalStorage("test_database.db")
|
||||
self.assertEqual(s2.dirs()["/some/directory"].enabled, True)
|
||||
self.assertEqual(s2.dirs()["/some/directory"].options[0], "opt1")
|
||||
self.assertEqual(s2.dirs()[dir_id].enabled, True)
|
||||
|
||||
self.assertEqual(s2.dirs()[dir_id].options[0].key, "key1")
|
||||
self.assertEqual(s2.dirs()[dir_id].options[0].value, "val1")
|
||||
self.assertEqual(s2.dirs()[dir_id].options[0].dir_id, 1)
|
||||
|
||||
def test_reject_duplicate_path(self):
|
||||
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
d1 = Directory("/some/directory", True, ["opt1", "opt2"])
|
||||
d2 = Directory("/some/directory", True, ["opt1", "opt2"])
|
||||
d1 = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val2")], "An excellent name")
|
||||
d2 = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val2")], "An excellent name")
|
||||
|
||||
s.save_directory(d1)
|
||||
|
||||
with self.assertRaises(DuplicateDirectoryException) as e:
|
||||
s.save_directory(d2)
|
||||
|
||||
def test_remove_dir(self):
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
d = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val3")], "An excellent name")
|
||||
dir_id = s.save_directory(d)
|
||||
|
||||
s.remove_directory(dir_id)
|
||||
|
||||
with self.assertRaises(KeyError):
|
||||
_ = s.dirs()[dir_id]
|
||||
|
||||
def test_save_and_retrieve_user(self):
|
||||
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
u = User("bob", "anHashedPassword", True)
|
||||
u = User("bob", b"anHashedPassword", True)
|
||||
|
||||
s.save_user(u)
|
||||
|
||||
@ -66,12 +86,97 @@ class LocalStorageTest(TestCase):
|
||||
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
u = User("bob", b'$2b$14$VZEMbwAdy/HvLL/zh0.Iv.8XYnoZMz/LU9V4VKXLiuS.pthcUly2O', True)
|
||||
u = User("bob", b'$2b$10$RakMb.3n/tl76sK7iVahJuklNYkR7f2Y4dsf73tPANwYBkp4VuJ7.', True)
|
||||
|
||||
s.save_user(u)
|
||||
|
||||
self.assertTrue(s.auth_user("bob", "test"))
|
||||
self.assertFalse(s.auth_user("bob", "wrong"))
|
||||
self.assertFalse(s.auth_user("wrong", "test"))
|
||||
|
||||
pass
|
||||
|
||||
def test_reject_duplicate_user(self):
|
||||
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
u1 = User("user1", b"anHashedPassword", True)
|
||||
u2 = User("user1", b"anotherHashedPassword", True)
|
||||
|
||||
s.save_user(u1)
|
||||
|
||||
with self.assertRaises(DuplicateUserException) as e:
|
||||
s.save_user(u2)
|
||||
|
||||
def test_update_user(self):
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
u = User("neil", b"anHashedPassword", True)
|
||||
|
||||
s.save_user(u)
|
||||
|
||||
u.admin = False
|
||||
s.update_user(u)
|
||||
|
||||
self.assertFalse(s.users()["neil"].admin)
|
||||
|
||||
def test_remove_user(self):
|
||||
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
u = User("martin", b"anHashedPassword", True)
|
||||
s.save_user(u)
|
||||
|
||||
s.remove_user(u.username)
|
||||
|
||||
with self.assertRaises(KeyError):
|
||||
_ = s.users()["martin"]
|
||||
|
||||
def test_update_directory(self):
|
||||
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
d = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val2")], "An excellent name")
|
||||
|
||||
dir_id = s.save_directory(d)
|
||||
|
||||
d.name = "A modified name"
|
||||
d.path = "/another/directory"
|
||||
d.id = dir_id
|
||||
|
||||
s.update_directory(d)
|
||||
|
||||
s2 = LocalStorage("test_database.db")
|
||||
|
||||
self.assertEqual(s2.dirs()[dir_id].name, "A modified name")
|
||||
self.assertEqual(len(s2.dirs()[dir_id].options), 2)
|
||||
self.assertEqual(s2.dirs()[dir_id].path, "/another/directory")
|
||||
|
||||
def test_save_option(self):
|
||||
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
d = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val2")], "An excellent name")
|
||||
dir_id = s.save_directory(d)
|
||||
|
||||
opt_id = s.save_option(Option("key3", "val3", dir_id))
|
||||
|
||||
self.assertEqual(s.dirs()[dir_id].options[2].key, "key3")
|
||||
self.assertEqual(s.dirs()[dir_id].options[2].value, "val3")
|
||||
self.assertEqual(s.dirs()[dir_id].options[2].dir_id, dir_id)
|
||||
self.assertEqual(opt_id, 3)
|
||||
|
||||
def test_del_option(self):
|
||||
|
||||
s = LocalStorage("test_database.db")
|
||||
|
||||
d = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val2")], "An excellent name")
|
||||
dir_id = s.save_directory(d)
|
||||
s.del_option(1)
|
||||
|
||||
self.assertEqual(len(s.dirs()[dir_id].options), 1)
|
||||
self.assertEqual(s.dirs()[dir_id].options[0].key, "key2")
|
||||
self.assertEqual(s.dirs()[dir_id].options[0].value, "val2")
|
||||
self.assertEqual(s.dirs()[dir_id].options[0].dir_id, 1)
|
||||
|
||||
|
||||
|
||||
|
||||
23
spec/MimeGuesser_spec.py
Normal file
23
spec/MimeGuesser_spec.py
Normal file
@ -0,0 +1,23 @@
|
||||
from parsing import ContentMimeGuesser, ExtensionMimeGuesser
|
||||
from unittest import TestCase
|
||||
|
||||
|
||||
class MimeGuesserTest(TestCase):
|
||||
|
||||
def test_content_guesser(self):
|
||||
|
||||
guesser = ContentMimeGuesser()
|
||||
|
||||
self.assertEqual("text/x-shellscript", guesser.guess_mime("test_folder/test_utf8.sh"))
|
||||
self.assertEqual("text/plain", guesser.guess_mime("test_folder/more_books.json"))
|
||||
self.assertEqual("application/java-archive", guesser.guess_mime("test_folder/post.jar"))
|
||||
self.assertEqual("image/jpeg", guesser.guess_mime("test_folder/sample_1.jpg"))
|
||||
|
||||
def test_extension_guesser(self):
|
||||
|
||||
guesser = ExtensionMimeGuesser()
|
||||
|
||||
self.assertEqual("text/x-sh", guesser.guess_mime("test_folder/test_utf8.sh"))
|
||||
self.assertEqual("application/json", guesser.guess_mime("test_folder/more_books.json"))
|
||||
self.assertEqual("application/java-archive", guesser.guess_mime("test_folder/post.jar"))
|
||||
self.assertEqual("image/jpeg", guesser.guess_mime("test_folder/sample_1.jpg"))
|
||||
72
static/css/keen-dashboards.css
Normal file
72
static/css/keen-dashboards.css
Normal file
@ -0,0 +1,72 @@
|
||||
.keen-dashboard {
|
||||
background: #f2f2f2;
|
||||
font-family: 'Gotham Rounded SSm A', 'Gotham Rounded SSm B', 'Helvetica Neue', Helvetica, Arial, sans-serif;
|
||||
}
|
||||
|
||||
.keen-dataviz {
|
||||
background: #fff;
|
||||
border: 1px solid #e7e7e7;
|
||||
border-radius: 2px;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
.keen-dataviz-title {
|
||||
border-bottom: 1px solid #e7e7e7;
|
||||
border-radius: 2px 2px 0 0;
|
||||
font-size: 13px;
|
||||
padding: 2px 10px 0;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
.keen-dataviz-stage {
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.keen-dataviz-notes {
|
||||
background: #fbfbfb;
|
||||
border-radius: 0 0 2px 2px;
|
||||
border-top: 1px solid #e7e7e7;
|
||||
font-family: 'Helvetica Neue', Helvetica, sans-serif;
|
||||
font-size: 11px;
|
||||
padding: 0 10px;
|
||||
}
|
||||
|
||||
.keen-dataviz .keen-dataviz-metric {
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
.keen-dataviz .keen-spinner-indicator {
|
||||
border-top-color: rgba(0, 187, 222, .4);
|
||||
}
|
||||
|
||||
.keen-dashboard .chart-wrapper {
|
||||
background: #fff;
|
||||
border: 1px solid #e2e2e2;
|
||||
border-radius: 3px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.keen-dashboard .chart-wrapper .chart-title {
|
||||
border-bottom: 1px solid #d7d7d7;
|
||||
color: #666;
|
||||
font-size: 14px;
|
||||
font-weight: 200;
|
||||
padding: 7px 10px 4px;
|
||||
}
|
||||
|
||||
.keen-dashboard .chart-wrapper .chart-stage {
|
||||
overflow: hidden;
|
||||
padding: 5px 10px;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.keen-dashboard .chart-wrapper .chart-notes {
|
||||
background: #fbfbfb;
|
||||
border-top: 1px solid #e2e2e2;
|
||||
color: #808080;
|
||||
font-size: 12px;
|
||||
padding: 8px 10px 5px;
|
||||
}
|
||||
|
||||
.keen-dashboard .chart-wrapper .keen-dataviz,
|
||||
.keen-dashboard .chart-wrapper .keen-dataviz-title,
|
||||
.keen-dashboard .chart-stage .chart-title {
|
||||
border: medium none;
|
||||
}
|
||||
155
static/css/keen-static.css
Normal file
155
static/css/keen-static.css
Normal file
@ -0,0 +1,155 @@
|
||||
a, a:focus, a:hover, a:active {
|
||||
color: #00afd7;
|
||||
}
|
||||
|
||||
h1, h2, h3 {
|
||||
font-family: "Gotham Rounded", "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
margin: 12px 0;
|
||||
}
|
||||
h1 {
|
||||
font-size: 32px;
|
||||
font-weight: 100;
|
||||
letter-spacing: .02em;
|
||||
line-height: 48px;
|
||||
margin: 12px 0;
|
||||
}
|
||||
h2 {
|
||||
color: #2a333c;
|
||||
font-weight: 200;
|
||||
font-size: 21px;
|
||||
}
|
||||
h3 {
|
||||
color: rgb(84, 102, 120);
|
||||
font-size: 21px;
|
||||
font-weight: 500;
|
||||
letter-spacing: -0.28px;
|
||||
line-height: 29.39px;
|
||||
}
|
||||
|
||||
.btn {
|
||||
background: transparent;
|
||||
border: 1px solid white;
|
||||
}
|
||||
|
||||
.keen-logo {
|
||||
height: 38px;
|
||||
margin: 0 15px 0 0;
|
||||
width: 150px;
|
||||
}
|
||||
|
||||
.navbar-toggle {
|
||||
background-color: rgba(255,255,255,.25);
|
||||
}
|
||||
.navbar-toggle .icon-bar {
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
|
||||
.navbar-nav {
|
||||
margin: 5px 0 0;
|
||||
}
|
||||
.navbar-nav > li > a {
|
||||
font-size: 15px;
|
||||
font-weight: 200;
|
||||
letter-spacing: 0.03em;
|
||||
padding-top: 19px;
|
||||
text-shadow: 0 0 2px rgba(0,0,0,.1);
|
||||
}
|
||||
.navbar-nav > li > a:focus,
|
||||
.navbar-nav > li > a:hover {
|
||||
background: transparent none;
|
||||
}
|
||||
|
||||
.navbar-nav > li > a.navbar-btn {
|
||||
background-color: rgba(255,255,255,.25);
|
||||
border: medium none;
|
||||
padding: 10px 15px;
|
||||
}
|
||||
.navbar-nav > li > a.navbar-btn:focus,
|
||||
.navbar-nav > li > a.navbar-btn:hover {
|
||||
background-color: rgba(255,255,255,.35);
|
||||
}
|
||||
.navbar-collapse {
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.masthead {
|
||||
background-color: #00afd7;
|
||||
background-image: url("../img/bg-bars.png");
|
||||
background-position: 0 -290px;
|
||||
background-repeat: repeat-x;
|
||||
color: #fff;
|
||||
margin: 0 0 24px;
|
||||
padding: 20px 0;
|
||||
}
|
||||
.masthead h1 {
|
||||
margin: 0;
|
||||
}
|
||||
.masthead small,
|
||||
.masthead a,
|
||||
.masthead a:focus,
|
||||
.masthead a:hover,
|
||||
.masthead a:active {
|
||||
color: #fff;
|
||||
}
|
||||
.masthead p {
|
||||
color: #b3e7f3;
|
||||
font-weight: 100;
|
||||
letter-spacing: .05em;
|
||||
}
|
||||
|
||||
|
||||
|
||||
.hero {
|
||||
background-position: 50% 100%;
|
||||
min-height: 450px;
|
||||
text-align: center;
|
||||
}
|
||||
.hero h1 {
|
||||
font-size: 48px;
|
||||
margin: 120px 0 0;
|
||||
}
|
||||
.hero .lead {
|
||||
margin-bottom: 32px;
|
||||
}
|
||||
.hero a.hero-btn {
|
||||
border: 2px solid #fff;
|
||||
display: block;
|
||||
font-family: "Gotham Rounded", "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
font-size: 24px;
|
||||
font-weight: 200;
|
||||
margin: 0 auto 12px;
|
||||
max-width: 320px;
|
||||
padding: 12px 0 6px;
|
||||
}
|
||||
.hero a.hero-btn:focus,
|
||||
.hero a.hero-btn:hover {
|
||||
border-color: transparent;
|
||||
background-color: #fff;
|
||||
color: #00afd7;
|
||||
}
|
||||
|
||||
|
||||
.sample-item {
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
.signup {
|
||||
float: left;
|
||||
display: inline-block;
|
||||
vertical-align: middle;
|
||||
margin-top: -6px;
|
||||
margin-right: 10px;
|
||||
}
|
||||
|
||||
.love {
|
||||
border-top: 1px solid #d7d7d7;
|
||||
color: #546678;
|
||||
margin: 24px 0 0;
|
||||
padding: 15px 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.love p {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
BIN
static/img/bg-bars.png
Normal file
BIN
static/img/bg-bars.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 8.3 KiB |
10
static/js/Chart.min.js
vendored
Normal file
10
static/js/Chart.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
155
storage.py
155
storage.py
@ -14,7 +14,7 @@ class DuplicateDirectoryException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DuplicateUsernameException(Exception):
|
||||
class DuplicateUserException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@ -29,14 +29,28 @@ class User:
|
||||
self.admin = admin
|
||||
|
||||
|
||||
class Option:
|
||||
"""
|
||||
Data structure to hold a directory option
|
||||
"""
|
||||
|
||||
def __init__(self, key: str, value: str, dir_id: int=None, opt_id: int = None):
|
||||
self.key = key
|
||||
self.value = value
|
||||
self.id = opt_id
|
||||
self.dir_id = dir_id
|
||||
|
||||
|
||||
class Directory:
|
||||
"""
|
||||
Data structure to hold directory information
|
||||
"""
|
||||
def __init__(self, path: str, enabled: bool, options: list):
|
||||
def __init__(self, path: str, enabled: bool, options: list, name: str):
|
||||
self.id = None
|
||||
self.path = path
|
||||
self.enabled = enabled
|
||||
self.options = options
|
||||
self.name = name
|
||||
|
||||
def __str__(self):
|
||||
return self.path + " | enabled: " + str(self.enabled) + " | opts: " + str(self.options)
|
||||
@ -74,31 +88,49 @@ class LocalStorage:
|
||||
"""
|
||||
Save directory to storage
|
||||
:param directory: Directory to save
|
||||
:return: None
|
||||
:return: id
|
||||
"""
|
||||
|
||||
self.dir_cache_outdated = True
|
||||
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("PRAGMA FOREIGN_KEYS = ON;")
|
||||
try:
|
||||
c.execute("INSERT INTO Directory (path, enabled) VALUES (?, ?)", (directory.path, directory.enabled))
|
||||
c.execute("INSERT INTO Directory (path, enabled, name) VALUES (?, ?, ?)", (directory.path, directory.enabled, directory.name))
|
||||
c.execute("SELECT last_insert_rowid()")
|
||||
|
||||
dir_id = c.fetchone()[0]
|
||||
|
||||
for opt in directory.options:
|
||||
conn.execute("INSERT INTO Option (name, directory_id) VALUES (?, ?)", (opt, dir_id))
|
||||
|
||||
c.close()
|
||||
conn.commit()
|
||||
except sqlite3.IntegrityError:
|
||||
raise DuplicateDirectoryException("Duplicate directory path: " + directory.path)
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def dirs(self):
|
||||
for opt in directory.options:
|
||||
opt.dir_id = dir_id
|
||||
self.save_option(opt)
|
||||
|
||||
return dir_id
|
||||
except sqlite3.IntegrityError:
|
||||
c.close()
|
||||
conn.close()
|
||||
raise DuplicateDirectoryException("Duplicate directory path: " + directory.path)
|
||||
|
||||
def remove_directory(self, dir_id: int):
|
||||
"""Remove a directory from the database"""
|
||||
|
||||
self.dir_cache_outdated = True
|
||||
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
|
||||
c.execute("DELETE FROM Option WHERE directory_id=?", (dir_id,))
|
||||
c.execute("DELETE FROM Task WHERE directory_id=?", (dir_id,))
|
||||
c.execute("DELETE FROM Directory WHERE id=?", (dir_id,))
|
||||
|
||||
c.close()
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def dirs(self) -> dict:
|
||||
|
||||
if self.dir_cache_outdated:
|
||||
|
||||
@ -106,23 +138,26 @@ class LocalStorage:
|
||||
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT id, path, enabled FROM Directory")
|
||||
c.execute("SELECT id, path, enabled, name FROM Directory")
|
||||
db_directories = c.fetchall()
|
||||
c.execute("SELECT name, directory_id FROM Option")
|
||||
c.execute("SELECT key, value, directory_id, id FROM Option")
|
||||
db_options = c.fetchall()
|
||||
|
||||
for db_dir in db_directories:
|
||||
|
||||
options = []
|
||||
directory = Directory(db_dir[1], db_dir[2], options)
|
||||
directory = Directory(db_dir[1], db_dir[2], options, db_dir[3])
|
||||
|
||||
for db_opt in db_options:
|
||||
if db_opt[1] == db_dir[0]:
|
||||
options.append(db_opt[0])
|
||||
if db_opt[2] == db_dir[0]:
|
||||
options.append(Option(db_opt[0], db_opt[1], db_opt[2], db_opt[3]))
|
||||
|
||||
self.cached_dirs[directory.path] = directory
|
||||
self.dir_cache_outdated = False
|
||||
return self.cached_dirs
|
||||
directory.id = db_dir[0]
|
||||
|
||||
self.cached_dirs[directory.id] = directory
|
||||
|
||||
self.dir_cache_outdated = False
|
||||
return self.cached_dirs
|
||||
|
||||
else:
|
||||
return self.cached_dirs
|
||||
@ -135,15 +170,15 @@ class LocalStorage:
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("PRAGMA FOREIGN_KEYS = ON;")
|
||||
c.execute("INSERT INTO User (username, password, is_admin) VALUES (?,?,?)",
|
||||
(user.username, user.hashed_password, user.admin))
|
||||
c.close()
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except sqlite3.IntegrityError:
|
||||
raise DuplicateDirectoryException("Duplicate username: " + user.username)
|
||||
raise DuplicateUserException("Duplicate username: " + user.username)
|
||||
|
||||
def users(self):
|
||||
def users(self) -> dict:
|
||||
"""Get user list"""
|
||||
|
||||
if self.user_cache_outdated:
|
||||
@ -157,7 +192,7 @@ class LocalStorage:
|
||||
db_users = c.fetchall()
|
||||
|
||||
for db_user in db_users:
|
||||
self.cached_users[db_user[0]] = User(db_user[0], "", db_user[1])
|
||||
self.cached_users[db_user[0]] = User(db_user[0], b"", bool(db_user[1]))
|
||||
|
||||
conn.close()
|
||||
|
||||
@ -180,4 +215,72 @@ class LocalStorage:
|
||||
|
||||
return False
|
||||
|
||||
def update_user(self, user: User) -> None:
|
||||
"""Updates an user. Will have no effect if the user does not exist"""
|
||||
|
||||
self.user_cache_outdated = True
|
||||
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("UPDATE User SET is_admin=? WHERE username=?", (user.admin, user.username))
|
||||
|
||||
c.close()
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def remove_user(self, username: str):
|
||||
"""Remove an user from the database"""
|
||||
|
||||
self.user_cache_outdated = True
|
||||
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("DELETE FROM User WHERE username=?", (username,))
|
||||
|
||||
c.close()
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def update_directory(self, directory):
|
||||
"""Updated a directory (Options are left untouched). Will have no effect if the directory does not exist"""
|
||||
|
||||
self.dir_cache_outdated = True
|
||||
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("UPDATE Directory SET name=?, path=? WHERE id=?", (directory.name, directory.path, directory.id))
|
||||
|
||||
c.close()
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def save_option(self, option: Option):
|
||||
|
||||
self.dir_cache_outdated = True
|
||||
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("INSERT INTO Option (key, value, directory_id) VALUES (?, ?, ?)", (option.key, option.value, option.dir_id))
|
||||
c.execute("SELECT last_insert_rowid()")
|
||||
|
||||
opt_id = c.fetchone()[0]
|
||||
c.close()
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return opt_id
|
||||
|
||||
def del_option(self, opt_id):
|
||||
"""Delete an option from the database"""
|
||||
|
||||
self.dir_cache_outdated = True
|
||||
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
c = conn.cursor()
|
||||
c.execute("DELETE FROM Option WHERE id=?", (opt_id, ))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
|
||||
|
||||
230
templates/dashboard.html
Normal file
230
templates/dashboard.html
Normal file
@ -0,0 +1,230 @@
|
||||
{% extends "layout.html" %}
|
||||
|
||||
{% block body %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<div class="col-sm-3">
|
||||
<div class="chart-wrapper">
|
||||
<div class="chart-title">FSE Info</div>
|
||||
<div class="chart-stage">
|
||||
<table class="info-table">
|
||||
<tr>
|
||||
<th>Version</th>
|
||||
<td><pre>1.0a</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Total thumbnail size</th>
|
||||
<td><pre>652.08 Mb</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Total document count</th>
|
||||
<td><pre>1258902</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Total document size</th>
|
||||
<td><pre>4.7 TB</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Folder count</th>
|
||||
<td><pre>4</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>User count</th>
|
||||
<td><pre>1</pre></td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<th>SQLite database path</th>
|
||||
<td>./local_storage.db</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chart-wrapper">
|
||||
<div class="chart-title">Elasticsearch info</div>
|
||||
<div class="chart-stage">
|
||||
<table class="info-table">
|
||||
<tr>
|
||||
<th>Total index size</th>
|
||||
<td><pre>3.7 GB</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>HTTP port</th>
|
||||
<td><pre>9200</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Version</th>
|
||||
<td><pre>6.2.1</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Lucene version</th>
|
||||
<td><pre>7.2.1</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Path</th>
|
||||
<td><pre>./elasticsearch/</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-sm-9">
|
||||
<div class="row">
|
||||
<div class="col-sm-12">
|
||||
<div class="chart-wrapper">
|
||||
<div class="chart-title">Thumbnail cache size</div>
|
||||
<div class="chart-stage">
|
||||
<script>
|
||||
window.chartColors = {
|
||||
red: 'rgb(255, 99, 132)',
|
||||
orange: 'rgb(255, 159, 64)',
|
||||
yellow: 'rgb(255, 205, 86)',
|
||||
green: 'rgb(75, 192, 192)',
|
||||
blue: 'rgb(54, 162, 235)',
|
||||
purple: 'rgb(153, 102, 255)',
|
||||
grey: 'rgb(201, 203, 207)'
|
||||
};
|
||||
|
||||
var color = Chart.helpers.color;
|
||||
|
||||
var barChartData = {
|
||||
labels: ["Music: 12.08 MB", "Pictures: 560.8 MB", "Movies: 78.9 MB", "Documents: 1 MB"],
|
||||
datasets: [{
|
||||
label: 'Size',
|
||||
backgroundColor: color("#8f2aa3").alpha(0.6).rgbString(),
|
||||
borderColor: "#8f2aa3",
|
||||
borderWidth: 1,
|
||||
data: [
|
||||
12080500,
|
||||
560805400,
|
||||
78898000,
|
||||
1024000
|
||||
]
|
||||
}]
|
||||
|
||||
};
|
||||
</script>
|
||||
<canvas id="canvas"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-sm-12">
|
||||
<div class="chart-wrapper">
|
||||
<div class="chart-title">Document size</div>
|
||||
<div class="chart-stage">
|
||||
<script>
|
||||
|
||||
var color = Chart.helpers.color;
|
||||
var barChartData2 = {
|
||||
labels: ["Music: 192.5 GB", "Pictures: 1.30 TB", "Movies: 3.73 TB", "Documents: 42.7 GB"],
|
||||
datasets: [{
|
||||
label: 'Size',
|
||||
backgroundColor: color("#f4b80c").alpha(0.6).rgbString(),
|
||||
borderColor: "#f4b80c",
|
||||
borderWidth: 1,
|
||||
data: [
|
||||
192500000000,
|
||||
1300000000000,
|
||||
3730000000000,
|
||||
42700000000
|
||||
]
|
||||
}]
|
||||
|
||||
};
|
||||
</script>
|
||||
|
||||
<canvas id="docSizeChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-sm-12">
|
||||
<div class="chart-wrapper">
|
||||
<div class="chart-title">Document count</div>
|
||||
<div class="chart-stage"> {# todo padding 8px 10px 5px 10px #}
|
||||
<script>
|
||||
var color = Chart.helpers.color;
|
||||
var barChartData3 = {
|
||||
labels: ["Music", "Pictures", "Movies", "Documents"],
|
||||
datasets: [{
|
||||
label: 'Count',
|
||||
backgroundColor: color("#f4225a").alpha(0.6).rgbString(),
|
||||
borderColor: "#f4225a",
|
||||
borderWidth: 1,
|
||||
data: [
|
||||
6790,
|
||||
758652,
|
||||
1289,
|
||||
11589632
|
||||
]
|
||||
}]
|
||||
|
||||
};
|
||||
|
||||
window.onload = function() {
|
||||
var ctx = document.getElementById("canvas").getContext("2d");
|
||||
ctx.canvas.height = 50;
|
||||
|
||||
new Chart(ctx, {
|
||||
type: 'bar',
|
||||
data: barChartData,
|
||||
options: {
|
||||
legend: {
|
||||
position: 'hidden'
|
||||
},
|
||||
title: {
|
||||
display: false
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
ctx = document.getElementById("docSizeChart").getContext("2d");
|
||||
ctx.canvas.height = 50;
|
||||
|
||||
new Chart(ctx, {
|
||||
type: 'bar',
|
||||
data: barChartData2,
|
||||
options: {
|
||||
legend: {
|
||||
position: 'hidden'
|
||||
},
|
||||
title: {
|
||||
display: false
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
ctx = document.getElementById("docCountChart").getContext("2d");
|
||||
ctx.canvas.height = 50;
|
||||
|
||||
new Chart(ctx, {
|
||||
type: 'bar',
|
||||
data: barChartData3,
|
||||
options: {
|
||||
legend: {
|
||||
position: 'hidden'
|
||||
},
|
||||
title: {
|
||||
display: false
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
};
|
||||
</script>
|
||||
<canvas id="docCountChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endblock body %}
|
||||
56
templates/directory.html
Normal file
56
templates/directory.html
Normal file
@ -0,0 +1,56 @@
|
||||
{% extends "layout.html" %}
|
||||
|
||||
{% block title %}An Excellent title{% endblock title %}
|
||||
|
||||
{% block body %}
|
||||
<div class="container">
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">An excellent form</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<form method="GET" action="/directory/add">
|
||||
|
||||
<div class="form-group">
|
||||
<input type="text" class="form-control" placeholder="Display name" name="name">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<input type="text" class="form-control" placeholder="Absolute path" name="path">
|
||||
</div>
|
||||
<button type="submit" class="btn btn-primary">Add Directory</button>
|
||||
</form>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">An excellent list</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<table class="info-table">
|
||||
<tr>
|
||||
<th>Display Name</th>
|
||||
<th>Path</th>
|
||||
<th>Enabled</th>
|
||||
<th>Last indexed</th>
|
||||
<th>Action</th>
|
||||
</tr>
|
||||
|
||||
{% for dir in directories %}
|
||||
<tr>
|
||||
<td>{{ directories[dir].name }}</td>
|
||||
<td><pre style="width: 80%">{{ directories[dir].path }}</pre></td>
|
||||
<td><i class="far {{ "fa-check-square" if directories[dir].enabled else "fa-square" }}"></i></td>
|
||||
<td>2018-02-21</td>
|
||||
<td><a href="directory/{{ dir }}">Manage</a> </td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
{% endblock body %}
|
||||
108
templates/directory_manage.html
Normal file
108
templates/directory_manage.html
Normal file
@ -0,0 +1,108 @@
|
||||
{% extends "layout.html" %}
|
||||
|
||||
{% block title %}An excellent title{% endblock title %}
|
||||
|
||||
{% block body %}
|
||||
|
||||
<div class="container">
|
||||
|
||||
<div class="panel panel-default">
|
||||
|
||||
<div class="panel-heading">An excellent summary</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<table class="info-table">
|
||||
<tr>
|
||||
<th>Display name</th>
|
||||
<td>{{ directory.name }}</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<th>Path</th>
|
||||
<td><pre>{{ directory.path }}</pre></td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<th>Enabled</th>
|
||||
<td><i class="far {{ "fa-check-square" if directory.enabled else "fa-square" }}"></i></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">An excellent option list</div>
|
||||
<div class="panel-body">
|
||||
<table class="info-table">
|
||||
<tr>
|
||||
<th>Key</th>
|
||||
<th>Value</th>
|
||||
<th>Action</th>
|
||||
</tr>
|
||||
{% for option in directory.options %}
|
||||
|
||||
<tr>
|
||||
<td>{{ option.key }}</td>
|
||||
<td>{{ option.value }}</td>
|
||||
<td><a href="/directory/{{ directory.id }}/del_opt/{{ option.id }}" >Remove</a></td>
|
||||
</tr>
|
||||
|
||||
{% endfor %}
|
||||
</table>
|
||||
|
||||
<hr>
|
||||
|
||||
<form method="GET" action="/directory/{{ directory.id }}/add_opt">
|
||||
|
||||
<div class="form-group">
|
||||
<div class="col-sm-4">
|
||||
<input type="text" class="form-control" placeholder="Key" name="key">
|
||||
</div>
|
||||
|
||||
|
||||
<div class="col-sm-4">
|
||||
<input type="text" class="form-control" placeholder="Value" name="value">
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<button type="submit" class="btn btn-primary">Add option</button>
|
||||
</form>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">An excellent control panel</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<div class="btn-group">
|
||||
<a class="btn dropdown-toggle btn-primary" data-toggle="dropdown" href="#">
|
||||
Create a task
|
||||
<span class="caret"></span>
|
||||
</a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="#">Indexing task</a></li>
|
||||
<li><a href="#">Thumbnail generation task</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="btn-group">
|
||||
<a class="btn dropdown-toggle btn-danger" data-toggle="dropdown" href="#">
|
||||
Action
|
||||
<span class="caret"></span>
|
||||
</a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="/directory/{{ directory.id }}/del">Delete directory</a></li>
|
||||
<li><a href="#">Reset to default settings</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
{% endblock body %}
|
||||
@ -1,16 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Title</title>
|
||||
</head>
|
||||
<body>
|
||||
<form action="/search">
|
||||
<input name="query">
|
||||
<input type="number" name="page" value="0">
|
||||
<input type="number" name="per_page" value="50">
|
||||
<input type="submit" value="Search">
|
||||
</form>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@ -1,13 +1,78 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Layout Title</title>
|
||||
<meta charset="utf-8">
|
||||
<title>{% block title %}Default title{% endblock title %}</title>
|
||||
<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no' />
|
||||
|
||||
<!-- Demo Dependencies -->
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.3/jquery.min.js" type="text/javascript"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.2.0/js/bootstrap.min.js" type="text/javascript"></script>
|
||||
<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.2.0/css/bootstrap.min.css" rel="stylesheet" type="text/css" />
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/holder/2.3.2/holder.min.js" type="text/javascript"></script>
|
||||
<script src="/static/js/Chart.min.js" type="text/javascript"></script>
|
||||
|
||||
<script>
|
||||
Holder.add_theme("white", { background:"#fff", foreground:"#a7a7a7", size:10 });
|
||||
</script>
|
||||
|
||||
<!-- Dashboard -->
|
||||
<link rel="stylesheet" type="text/css" href="/static/css/keen-dashboards.css" />
|
||||
<link href="https://use.fontawesome.com/releases/v5.0.6/css/all.css" rel="stylesheet" type="text/css">
|
||||
|
||||
|
||||
<style>
|
||||
.info-table {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.info-table pre {
|
||||
padding: 6px;
|
||||
margin: 4px;
|
||||
}
|
||||
|
||||
.info-table td {
|
||||
padding: 4px;
|
||||
}
|
||||
|
||||
.info-table tr:nth-child(even) {
|
||||
background-color: #fafafa;
|
||||
}
|
||||
|
||||
{# todo: box-shadow 0 1px 10px 1px #1AC8DE#}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<body class="keen-dashboard" style="padding-top: 80px;">
|
||||
|
||||
<div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
|
||||
<div class="container-fluid">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
|
||||
<span class="sr-only">Toggle navigation</span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="../">
|
||||
<span class="glyphicon glyphicon-chevron-left"></span>
|
||||
</a>
|
||||
<a class="navbar-brand" href="./">Layouts » Hero Sidebar</a>
|
||||
</div>
|
||||
<div class="navbar-collapse collapse">
|
||||
<ul class="nav navbar-nav navbar-left">
|
||||
<li><a href="#">Home</a></li>
|
||||
<li><a href="#">Team</a></li>
|
||||
<li><a href="#">Source</a></li>
|
||||
<li><a href="#">Community</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
{% block body %}
|
||||
{% endblock body %}
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
|
||||
10
templates/task.html
Normal file
10
templates/task.html
Normal file
@ -0,0 +1,10 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Title</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
Loading…
x
Reference in New Issue
Block a user