diff --git a/crawler.py b/crawler.py index e6641a7..7acb289 100644 --- a/crawler.py +++ b/crawler.py @@ -1,33 +1,38 @@ import os - class Crawler: def __init__(self, enabled_parsers: list): self.documents = [] self.enabled_parsers = enabled_parsers + for parser in self.enabled_parsers: + if parser.is_default: + self.default_parser = parser + + self.ext_map = {} + + for parser in self.enabled_parsers: + for ext in parser.extensions: + self.ext_map[ext] = parser + def crawl(self, root_dir: str): for root, dirs, files in os.walk(root_dir): for filename in files: full_path = os.path.join(root, filename) - parser = self.get_parser_by_ext(os.path.splitext(filename)[1]) + parser = self.ext_map.get(os.path.splitext(filename)[1], self.default_parser) doc = parser.parse(full_path) self.documents.append(doc) - def get_parser_by_ext(self, ext: str): + def countFiles(self, root_dir: str): + count = 0 - for parser in self.enabled_parsers: - - if ext in parser.extensions: - return parser - - for parser in self.enabled_parsers: - if parser.is_default: - return parser + for root, dirs, files in os.walk(root_dir): + count += len(files) + return count diff --git a/run.py b/run.py index 16aaa4f..9e31ec5 100644 --- a/run.py +++ b/run.py @@ -1,11 +1,12 @@ -from flask import Flask, render_template, send_file, request, redirect +from flask import Flask, render_template, send_file, request, redirect, flash, session from indexer import Indexer -from storage import Directory, Option +from storage import Directory, Option, Task +import json # indexer = Indexer("fse") app = Flask(__name__) - +app.secret_key = "A very secret key" # # class Document: # def __init__(self, doc_id, name, path, size, md5): @@ -118,7 +119,7 @@ app = Flask(__name__) # return send_file("thumbnails/" + doc_id, mimetype=mimetypes.guess_type(thumb_path)[0]) # else: # return "File not found" -from storage import LocalStorage +from storage import LocalStorage, DuplicateDirectoryException storage = LocalStorage("local_storage.db") @@ -128,12 +129,9 @@ def tmp_route(): @app.route("/directory") -def directory(): +def dir_list(): - directories = storage.dirs() - print(directories) - - return render_template("directory.html", directories=directories) + return render_template("directory.html", directories=storage.dirs()) @app.route("/directory/add") @@ -142,15 +140,16 @@ def directory_add(): path = request.args.get("path") name = request.args.get("name") - print(name) - if path is not None and name is not None: d = Directory(path, True, [], name) - storage.save_directory(d) - return redirect("/directory") + try: + storage.save_directory(d) + flash("Created directory", "success") + except DuplicateDirectoryException: + flash("Couldn't create directory Make sure that the path is unique", "danger") - return "Error" # todo better message + return redirect("/directory") @app.route("/directory/") @@ -161,6 +160,30 @@ def directory_manage(dir_id): return render_template("directory_manage.html", directory=directory) +@app.route("/directory//update") +def directory_update(dir_id): + + directory = storage.dirs()[dir_id] + + name = request.args.get("name") + name = directory.name if name is None else name + + enabled = request.args.get("enabled") + enabled = directory.enabled if enabled is None else int(enabled) + + path = request.args.get("path") + path = directory.path if path is None else path + + # Only name and enabled status can be updated + updated_dir = Directory(path, enabled, directory.options, name) + updated_dir.id = dir_id + storage.update_directory(updated_dir) + + flash("Updated directory", "success") + + return redirect("/directory/" + str(dir_id)) + + @app.route("/directory//add_opt") def directory_add_opt(dir_id): @@ -169,6 +192,7 @@ def directory_add_opt(dir_id): if key is not None and value is not None: storage.save_option(Option(key, value, dir_id)) + flash("Added option", "success") return redirect("/directory/" + str(dir_id)) @@ -177,6 +201,17 @@ def directory_add_opt(dir_id): def directory_del_opt(dir_id, opt_id): storage.del_option(opt_id) + return redirect("/directory/" + str(dir_id)) + + +@app.route("/directory//update_opt") +def directory_update_opt(dir_id): + + opt_id = request.args.get("id") + opt_key = request.args.get("key") + opt_value = request.args.get("value") + + storage.update_option(Option(opt_key, opt_value, dir_id, opt_id)) return redirect("/directory/" + str(dir_id)) @@ -185,6 +220,7 @@ def directory_del_opt(dir_id, opt_id): def directory_del(dir_id): storage.remove_directory(dir_id) + flash("Deleted directory", "success") return redirect("/directory") @@ -192,10 +228,24 @@ def directory_del(dir_id): @app.route("/task") def task(): - tasks = storage.tasks() - directories = storage.dirs() + return render_template("task.html", tasks=storage.tasks(), directories=storage.dirs()) - return render_template("task.html", tasks=tasks, directories=directories) + +@app.route("/task/add") +def task_add(): + type = request.args.get("type") + directory = request.args.get("directory") + + storage.save_task(Task(type, directory)) + + return redirect("/task") + + +@app.route("/task//del") +def task_del(task_id): + storage.del_task(task_id) + + return redirect("/task") @app.route("/dashboard") diff --git a/spec/Crawler_spec.py b/spec/Crawler_spec.py index 860ddec..2639b86 100644 --- a/spec/Crawler_spec.py +++ b/spec/Crawler_spec.py @@ -7,16 +7,15 @@ from crawler import Crawler class CrawlerTest(TestCase): def test_dir_walk(self): + c = Crawler([GenericFileParser([Sha1CheckSumCalculator()], ExtensionMimeGuesser())]) c.crawl("test_folder") self.assertEqual(len(c.documents), 28) - def test_get_parser_by_ext(self): + def test_file_count(self): - c = Crawler([GenericFileParser([Sha1CheckSumCalculator()], ExtensionMimeGuesser())]) + c = Crawler([]) - self.assertIsInstance(c.get_parser_by_ext("any"), GenericFileParser) - - # todo add more parsers here + self.assertEqual(c.countFiles("test_folder"), 28) diff --git a/spec/LocalStorage_spec.py b/spec/LocalStorage_spec.py index ceafe34..5c241ff 100644 --- a/spec/LocalStorage_spec.py +++ b/spec/LocalStorage_spec.py @@ -25,7 +25,6 @@ class LocalStorageTest(TestCase): self.assertEqual(storage.dirs()[dir_id].options[0].value, "val1") self.assertEqual(storage.dirs()[dir_id].options[0].dir_id, 1) - def test_save_and_retrieve_dir_persistent(self): s1 = LocalStorage("test_database.db") @@ -140,6 +139,7 @@ class LocalStorageTest(TestCase): dir_id = s.save_directory(d) d.name = "A modified name" + d.enabled = False d.path = "/another/directory" d.id = dir_id @@ -150,6 +150,7 @@ class LocalStorageTest(TestCase): self.assertEqual(s2.dirs()[dir_id].name, "A modified name") self.assertEqual(len(s2.dirs()[dir_id].options), 2) self.assertEqual(s2.dirs()[dir_id].path, "/another/directory") + self.assertEqual(s2.dirs()[dir_id].enabled, 0) # enabled = false def test_save_option(self): @@ -178,6 +179,17 @@ class LocalStorageTest(TestCase): self.assertEqual(s.dirs()[dir_id].options[0].value, "val2") self.assertEqual(s.dirs()[dir_id].options[0].dir_id, 1) + def test_update_option(self): + + s = LocalStorage("test_database.db") + + d = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val2")], "An excellent name") + dir_id = s.save_directory(d) + + s.update_option(Option("key1", "newVal", dir_id, 1)) + + self.assertEqual(s.dirs()[dir_id].options[0].value, "newVal") + def test_save_task(self): s = LocalStorage("test_database.db") @@ -194,9 +206,13 @@ class LocalStorageTest(TestCase): dir_id = s.save_directory(Directory("/some/dir", True, [], "my dir")) task_id = s.save_task(Task(0, dir_id)) - s.del_task(task_id) + s2 = LocalStorage("test_database.db") + s2.tasks() + s2.del_task(task_id) + + self.assertEqual(len(s2.tasks()), 0) with self.assertRaises(KeyError): - _ = s.tasks()[task_id] + _ = s2.tasks()[task_id] diff --git a/storage.py b/storage.py index 22103d8..9c42a32 100644 --- a/storage.py +++ b/storage.py @@ -3,10 +3,10 @@ import os import flask_bcrypt import time + class CheckSumCalculator: def checksum(self, string: str): - return flask_bcrypt.generate_password_hash(string, 14) # todo load from config @@ -58,7 +58,8 @@ class Directory: class Task: - def __init__(self, task_type: int, dir_id: int, completed: bool = False, completed_time: time.time = None, task_id: int = None): + def __init__(self, task_type: int, dir_id: int, completed: bool = False, completed_time: time.time = None, + task_id: int = None): self.id = task_id self.type = task_type self.dir_id = dir_id @@ -107,7 +108,8 @@ class LocalStorage: conn = sqlite3.connect(self.db_path) c = conn.cursor() try: - c.execute("INSERT INTO Directory (path, enabled, name) VALUES (?, ?, ?)", (directory.path, directory.enabled, directory.name)) + c.execute("INSERT INTO Directory (path, enabled, name) VALUES (?, ?, ?)", + (directory.path, directory.enabled, directory.name)) c.execute("SELECT last_insert_rowid()") dir_id = c.fetchone()[0] @@ -259,7 +261,8 @@ class LocalStorage: conn = sqlite3.connect(self.db_path) c = conn.cursor() - c.execute("UPDATE Directory SET name=?, path=? WHERE id=?", (directory.name, directory.path, directory.id)) + c.execute("UPDATE Directory SET name=?, path=?, enabled=? WHERE id=?", + (directory.name, directory.path, directory.enabled, directory.id)) c.close() conn.commit() @@ -271,7 +274,8 @@ class LocalStorage: conn = sqlite3.connect(self.db_path) c = conn.cursor() - c.execute("INSERT INTO Option (key, value, directory_id) VALUES (?, ?, ?)", (option.key, option.value, option.dir_id)) + c.execute("INSERT INTO Option (key, value, directory_id) VALUES (?, ?, ?)", + (option.key, option.value, option.dir_id)) c.execute("SELECT last_insert_rowid()") opt_id = c.fetchone()[0] @@ -293,6 +297,18 @@ class LocalStorage: conn.commit() conn.close() + def update_option(self, option: Option): + """Updates an option""" + + self.dir_cache_outdated = True + + conn = sqlite3.connect(self.db_path) + c = conn.cursor() + c.execute("UPDATE Option SET key=?, value=? WHERE id=?", (option.key, option.value, option.id)) + + conn.commit() + conn.close() + def save_task(self, task: Task): """Save a task to the database""" @@ -317,6 +333,8 @@ class LocalStorage: if self.task_cache_outdated: + self.cached_tasks = {} + conn = sqlite3.connect(self.db_path) c = conn.cursor() c.execute("SELECT id, directory_id, type, completed, completed_time FROM Task") @@ -346,6 +364,6 @@ class LocalStorage: c = conn.cursor() c.execute("DELETE FROM Task WHERE id=?", (task_id, )) - c.close() conn.commit() - conn.close() + c.close() + conn.close() \ No newline at end of file diff --git a/templates/directory.html b/templates/directory.html index 964986e..e1f59c7 100644 --- a/templates/directory.html +++ b/templates/directory.html @@ -4,12 +4,13 @@ {% block body %}
+ + {# Add directory form #}
An excellent form
-
@@ -22,6 +23,7 @@
+ {# List of directories #}
An excellent list
@@ -52,9 +54,5 @@
- - - -
{% endblock body %} \ No newline at end of file diff --git a/templates/directory_manage.html b/templates/directory_manage.html index bb18135..673fb2b 100644 --- a/templates/directory_manage.html +++ b/templates/directory_manage.html @@ -4,27 +4,102 @@ {% block body %} + +
-
An excellent summary
+
Summary
- + - + - + - + - +
Display name{{ directory.name }} + {{ directory.name }} +
Path
{{ directory.path }}
+
{{ directory.path }}
+
Enabled + + + + + +
@@ -46,12 +121,14 @@ {% for option in directory.options %} - {{ option.key }} - {{ option.value }} - Remove + {{ option.key }} + {{ option.value }} + Remove + {% endfor %} + diff --git a/templates/layout.html b/templates/layout.html index bb956d0..fa3f3b1 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -35,9 +35,9 @@ padding: 4px; } -{# .info-table tr:nth-child(even) {#} -{# background-color: #fafafa;#} -{# }#} + {# .info-table tr:nth-child(even) {#} + {# background-color: #fafafa;#} + {# }#} {# todo: box-shadow 0 1px 10px 1px #1AC8DE#} @@ -69,6 +69,22 @@
+{% block alert_messages %} + + + {% with messages = get_flashed_messages(with_categories=true) %} + {% if messages %} +
+ {% for category, message in messages %} +
+ × + {{ message | safe }} +
+ {% endfor %} +
+ {% endif %} + {% endwith %} +{% endblock %} {% block body %} {% endblock body %} diff --git a/templates/task.html b/templates/task.html index 39a48b4..2395254 100644 --- a/templates/task.html +++ b/templates/task.html @@ -32,22 +32,29 @@
An excellent panel
- +
+ + + - {% for task in tasks %} + + {% for task_id in tasks %} - - - + + + + {% endfor %} + +
Task type Directory CompletedAction
{{ task.type }}{{ directories[task.dir_id].name }}{{ task.completed }}{{ tasks[task_id].type }}{{ directories[tasks[task_id].dir_id].name }}{{ tasks[task_id].completed }}Cancel