CRUD for tasks, dirs and options.

Added flash messages

crawler.py

@@ -1,33 +1,38 @@
 import os
 
-
 class Crawler:
 
     def __init__(self, enabled_parsers: list):
         self.documents = []
         self.enabled_parsers = enabled_parsers
 
+        for parser in self.enabled_parsers:
+            if parser.is_default:
+                self.default_parser = parser
+
+        self.ext_map = {}
+
+        for parser in self.enabled_parsers:
+            for ext in parser.extensions:
+                self.ext_map[ext] = parser
+
     def crawl(self, root_dir: str):
         for root, dirs, files in os.walk(root_dir):
 
             for filename in files:
                 full_path = os.path.join(root, filename)
 
-                parser = self.get_parser_by_ext(os.path.splitext(filename)[1])
+                parser = self.ext_map.get(os.path.splitext(filename)[1], self.default_parser)
 
                 doc = parser.parse(full_path)
 
                 self.documents.append(doc)
 
-    def get_parser_by_ext(self, ext: str):
+    def countFiles(self, root_dir: str):
+        count = 0
 
-        for parser in self.enabled_parsers:
+        for root, dirs, files in os.walk(root_dir):
-
+            count += len(files)
-            if ext in parser.extensions:
-                return parser
-
-        for parser in self.enabled_parsers:
-            if parser.is_default:
-                return parser
 
+        return count
 

run.py

@@ -1,11 +1,12 @@
-from flask import Flask, render_template, send_file, request, redirect
+from flask import Flask, render_template, send_file, request, redirect, flash, session
 from indexer import Indexer
-from storage import Directory, Option
+from storage import Directory, Option, Task
+import json
 
 # indexer = Indexer("fse")
 
 app = Flask(__name__)
-
+app.secret_key = "A very secret key"
 #
 # class Document:
 #     def __init__(self, doc_id, name, path, size, md5):
@@ -118,7 +119,7 @@ app = Flask(__name__)
 #         return send_file("thumbnails/" + doc_id, mimetype=mimetypes.guess_type(thumb_path)[0])
 #     else:
 #         return "File not found"
-from storage import LocalStorage
+from storage import LocalStorage, DuplicateDirectoryException
 storage = LocalStorage("local_storage.db")
 
 
@@ -128,12 +129,9 @@ def tmp_route():
 
 
 @app.route("/directory")
-def directory():
+def dir_list():
 
-    directories = storage.dirs()
+    return render_template("directory.html", directories=storage.dirs())
-    print(directories)
-
-    return render_template("directory.html", directories=directories)
 
 
 @app.route("/directory/add")
@@ -142,16 +140,17 @@ def directory_add():
     path = request.args.get("path")
     name = request.args.get("name")
 
-    print(name)
-
     if path is not None and name is not None:
         d = Directory(path, True, [], name)
+
+        try:
             storage.save_directory(d)
+            flash("<strong>Created directory</strong>", "success")
+        except DuplicateDirectoryException:
+            flash("<strong>Couldn't create directory</strong> Make sure that the path is unique", "danger")
 
     return redirect("/directory")
 
-    return "Error"  # todo better message
-
 
 @app.route("/directory/<int:dir_id>")
 def directory_manage(dir_id):
@@ -161,6 +160,30 @@ def directory_manage(dir_id):
     return render_template("directory_manage.html", directory=directory)
 
 
+@app.route("/directory/<int:dir_id>/update")
+def directory_update(dir_id):
+
+    directory = storage.dirs()[dir_id]
+
+    name = request.args.get("name")
+    name = directory.name if name is None else name
+
+    enabled = request.args.get("enabled")
+    enabled = directory.enabled if enabled is None else int(enabled)
+
+    path = request.args.get("path")
+    path = directory.path if path is None else path
+
+    # Only name and enabled status can be updated
+    updated_dir = Directory(path, enabled, directory.options, name)
+    updated_dir.id = dir_id
+    storage.update_directory(updated_dir)
+
+    flash("<strong>Updated directory</strong>", "success")
+
+    return redirect("/directory/" + str(dir_id))
+
+
 @app.route("/directory/<int:dir_id>/add_opt")
 def directory_add_opt(dir_id):
 
@@ -169,6 +192,7 @@ def directory_add_opt(dir_id):
 
     if key is not None and value is not None:
         storage.save_option(Option(key, value, dir_id))
+        flash("<strong>Added option</strong>", "success")
 
     return redirect("/directory/" + str(dir_id))
 
@@ -177,6 +201,17 @@ def directory_add_opt(dir_id):
 def directory_del_opt(dir_id, opt_id):
 
     storage.del_option(opt_id)
+    return redirect("/directory/" + str(dir_id))
+
+
+@app.route("/directory/<int:dir_id>/update_opt")
+def directory_update_opt(dir_id):
+
+    opt_id = request.args.get("id")
+    opt_key = request.args.get("key")
+    opt_value = request.args.get("value")
+
+    storage.update_option(Option(opt_key, opt_value, dir_id, opt_id))
 
     return redirect("/directory/" + str(dir_id))
 
@@ -185,6 +220,7 @@ def directory_del_opt(dir_id, opt_id):
 def directory_del(dir_id):
 
     storage.remove_directory(dir_id)
+    flash("<strong>Deleted directory</strong>", "success")
 
     return redirect("/directory")
 
@@ -192,10 +228,24 @@ def directory_del(dir_id):
 @app.route("/task")
 def task():
 
-    tasks = storage.tasks()
+    return render_template("task.html", tasks=storage.tasks(), directories=storage.dirs())
-    directories = storage.dirs()
 
-    return render_template("task.html", tasks=tasks, directories=directories)
+
+@app.route("/task/add")
+def task_add():
+    type = request.args.get("type")
+    directory = request.args.get("directory")
+
+    storage.save_task(Task(type, directory))
+
+    return redirect("/task")
+
+
+@app.route("/task/<int:task_id>/del")
+def task_del(task_id):
+    storage.del_task(task_id)
+
+    return redirect("/task")
 
 
 @app.route("/dashboard")

spec/Crawler_spec.py

@@ -7,16 +7,15 @@ from crawler import Crawler
 class CrawlerTest(TestCase):
 
     def test_dir_walk(self):
+
         c = Crawler([GenericFileParser([Sha1CheckSumCalculator()], ExtensionMimeGuesser())])
 
         c.crawl("test_folder")
 
         self.assertEqual(len(c.documents), 28)
 
-    def test_get_parser_by_ext(self):
+    def test_file_count(self):
 
-        c = Crawler([GenericFileParser([Sha1CheckSumCalculator()], ExtensionMimeGuesser())])
+        c = Crawler([])
 
-        self.assertIsInstance(c.get_parser_by_ext("any"), GenericFileParser)
+        self.assertEqual(c.countFiles("test_folder"), 28)
-
-        # todo add more parsers here

spec/LocalStorage_spec.py

@@ -25,7 +25,6 @@ class LocalStorageTest(TestCase):
         self.assertEqual(storage.dirs()[dir_id].options[0].value, "val1")
         self.assertEqual(storage.dirs()[dir_id].options[0].dir_id, 1)
 
-
     def test_save_and_retrieve_dir_persistent(self):
 
         s1 = LocalStorage("test_database.db")
@@ -140,6 +139,7 @@ class LocalStorageTest(TestCase):
         dir_id = s.save_directory(d)
 
         d.name = "A modified name"
+        d.enabled = False
         d.path = "/another/directory"
         d.id = dir_id
 
@@ -150,6 +150,7 @@ class LocalStorageTest(TestCase):
         self.assertEqual(s2.dirs()[dir_id].name, "A modified name")
         self.assertEqual(len(s2.dirs()[dir_id].options), 2)
         self.assertEqual(s2.dirs()[dir_id].path, "/another/directory")
+        self.assertEqual(s2.dirs()[dir_id].enabled, 0)  # enabled = false
 
     def test_save_option(self):
 
@@ -178,6 +179,17 @@ class LocalStorageTest(TestCase):
         self.assertEqual(s.dirs()[dir_id].options[0].value, "val2")
         self.assertEqual(s.dirs()[dir_id].options[0].dir_id, 1)
 
+    def test_update_option(self):
+
+        s = LocalStorage("test_database.db")
+
+        d = Directory("/some/directory", True, [Option("key1", "val1"), Option("key2", "val2")], "An excellent name")
+        dir_id = s.save_directory(d)
+
+        s.update_option(Option("key1", "newVal", dir_id, 1))
+
+        self.assertEqual(s.dirs()[dir_id].options[0].value, "newVal")
+
     def test_save_task(self):
 
         s = LocalStorage("test_database.db")
@@ -194,9 +206,13 @@ class LocalStorageTest(TestCase):
         dir_id = s.save_directory(Directory("/some/dir", True, [], "my dir"))
         task_id = s.save_task(Task(0, dir_id))
 
-        s.del_task(task_id)
+        s2 = LocalStorage("test_database.db")
+        s2.tasks()
+        s2.del_task(task_id)
+
+        self.assertEqual(len(s2.tasks()), 0)
 
         with self.assertRaises(KeyError):
-            _ = s.tasks()[task_id]
+            _ = s2.tasks()[task_id]
 
 

storage.py

@@ -3,10 +3,10 @@ import os
 import flask_bcrypt
 import time
 
+
 class CheckSumCalculator:
 
     def checksum(self, string: str):
-
         return flask_bcrypt.generate_password_hash(string, 14)  # todo load from config
 
 
@@ -58,7 +58,8 @@ class Directory:
 
 class Task:
 
-    def __init__(self, task_type: int, dir_id: int, completed: bool = False, completed_time: time.time = None, task_id: int = None):
+    def __init__(self, task_type: int, dir_id: int, completed: bool = False, completed_time: time.time = None,
+                 task_id: int = None):
         self.id = task_id
         self.type = task_type
         self.dir_id = dir_id
@@ -107,7 +108,8 @@ class LocalStorage:
         conn = sqlite3.connect(self.db_path)
         c = conn.cursor()
         try:
-            c.execute("INSERT INTO Directory (path, enabled, name) VALUES (?, ?, ?)", (directory.path, directory.enabled, directory.name))
+            c.execute("INSERT INTO Directory (path, enabled, name) VALUES (?, ?, ?)",
+                      (directory.path, directory.enabled, directory.name))
             c.execute("SELECT last_insert_rowid()")
 
             dir_id = c.fetchone()[0]
@@ -259,7 +261,8 @@ class LocalStorage:
 
         conn = sqlite3.connect(self.db_path)
         c = conn.cursor()
-        c.execute("UPDATE Directory SET name=?, path=? WHERE id=?", (directory.name, directory.path, directory.id))
+        c.execute("UPDATE Directory SET name=?, path=?, enabled=? WHERE id=?",
+                  (directory.name, directory.path, directory.enabled, directory.id))
 
         c.close()
         conn.commit()
@@ -271,7 +274,8 @@ class LocalStorage:
 
         conn = sqlite3.connect(self.db_path)
         c = conn.cursor()
-        c.execute("INSERT INTO Option (key, value, directory_id) VALUES (?, ?, ?)", (option.key, option.value, option.dir_id))
+        c.execute("INSERT INTO Option (key, value, directory_id) VALUES (?, ?, ?)",
+                  (option.key, option.value, option.dir_id))
         c.execute("SELECT last_insert_rowid()")
 
         opt_id = c.fetchone()[0]
@@ -293,6 +297,18 @@ class LocalStorage:
         conn.commit()
         conn.close()
 
+    def update_option(self, option: Option):
+        """Updates an option"""
+
+        self.dir_cache_outdated = True
+
+        conn = sqlite3.connect(self.db_path)
+        c = conn.cursor()
+        c.execute("UPDATE Option SET key=?, value=? WHERE id=?", (option.key, option.value, option.id))
+
+        conn.commit()
+        conn.close()
+
     def save_task(self, task: Task):
         """Save a task to the database"""
 
@@ -317,6 +333,8 @@ class LocalStorage:
 
         if self.task_cache_outdated:
 
+            self.cached_tasks = {}
+
             conn = sqlite3.connect(self.db_path)
             c = conn.cursor()
             c.execute("SELECT id, directory_id, type, completed, completed_time FROM Task")
@@ -346,6 +364,6 @@ class LocalStorage:
         c = conn.cursor()
         c.execute("DELETE FROM Task WHERE id=?", (task_id, ))
 
-        c.close()
         conn.commit()
+        c.close()
         conn.close()

templates/directory.html

@@ -4,12 +4,13 @@
 
 {% block body %}
     <div class="container">
+
+        {# Add directory form #}
         <div class="panel panel-default">
             <div class="panel-heading">An excellent form</div>
             <div class="panel-body">
 
                 <form method="GET" action="/directory/add">
-
                     <div class="form-group">
                         <input type="text" class="form-control" placeholder="Display name" name="name">
                     </div>
@@ -22,6 +23,7 @@
             </div>
         </div>
 
+        {# List of directories #}
         <div class="panel panel-default">
             <div class="panel-heading">An excellent list</div>
             <div class="panel-body">
@@ -52,9 +54,5 @@
 
             </div>
         </div>
-
-
-
-
     </div>
 {% endblock body %}

templates/directory_manage.html

@@ -4,27 +4,102 @@
 
 {% block body %}
 
+    <script>
+
+        function swapToForm(elem, fields, formAction, inputName) {
+
+            var form = document.createElement("form");
+            form.setAttribute("action", formAction);
+
+            for (var i in fields) {
+
+                var hiddenInput = document.createElement("input");
+                hiddenInput.setAttribute("type", "hidden");
+                hiddenInput.setAttribute("value", fields[i].value);
+                hiddenInput.setAttribute("name", fields[i].name);
+                form.appendChild(hiddenInput);
+            }
+
+            var input = document.createElement("input");
+            input.setAttribute("class", "form-control");
+            input.setAttribute("type", "text");
+            input.setAttribute("name", inputName);
+            input.value = elem.innerHTML;
+            form.appendChild(input);
+
+            elem.parentNode.insertBefore(form, elem);
+            elem.remove();
+
+            input.focus();
+            input.addEventListener("focusout", function () {
+                form.submit();
+            });
+
+            return input;
+        }
+
+        function modifyKey(optId, value) {
+
+            swapToForm(document.getElementById("key-" + optId), [
+                {name: "id", value: optId},
+                {name: "value", value: value},
+                {name: "dir_id", value: {{ directory.id }}}
+            ], "/directory/{{ directory.id }}/update_opt", "key");
+
+        }
+
+        function modifyVal(optId, key) {
+            swapToForm(document.getElementById("val-" + optId), [
+                {name: "id", value: optId},
+                {name: "key", value: key},
+                {name: "dir_id", value: {{ directory.id }}}
+            ], "/directory/{{ directory.id }}/update_opt", "value");
+        }
+
+        function modifyDisplayName() {
+            swapToForm(document.getElementById("display-name"), [],
+                "/directory/{{ directory.id }}/update", "name");
+        }
+
+        function modifyPath() {
+            swapToForm(document.getElementById("path"), [],
+                "/directory/{{ directory.id }}/update", "path");
+        }
+
+    </script>
+
     <div class="container">
 
         <div class="panel panel-default">
 
-            <div class="panel-heading">An excellent summary</div>
+            <div class="panel-heading">Summary</div>
             <div class="panel-body">
 
                 <table class="info-table">
-                    <tr>
+                    <tr onclick="modifyDisplayName()">
                         <th>Display name</th>
-                        <td>{{ directory.name }}</td>
+                        <td>
+                            <span id="display-name" title="Click to update">{{ directory.name }}</span>
+                        </td>
                     </tr>
 
-                    <tr>
+                    <tr onclick="modifyPath()">
                         <th>Path</th>
-                        <td><pre>{{ directory.path }}</pre></td>
+                        <td>
+                            <pre id="path" title="Click to update">{{ directory.path }}</pre>
+                        </td>
                     </tr>
 
                     <tr>
                         <th>Enabled</th>
-                        <td><i class="far {{ "fa-check-square" if directory.enabled else "fa-square" }}"></i></td>
+                        <td>
+                            <i class="far {{ "fa-check-square" if directory.enabled else "fa-square" }}"></i>
+                            <form action="/directory/{{ directory.id }}/update"  style="display: inline;">
+                                <input type="hidden" name="enabled" value="{{ "0" if directory.enabled else "1" }}">
+                                <input type="submit" class="btn {{ "btn-danger" if directory.enabled else "btn-success" }}"
+                                       value="{{ "Disable" if directory.enabled else "Enable" }}">
+                            </form>
+                        </td>
                     </tr>
                 </table>
             </div>
@@ -46,12 +121,14 @@
                     {% for option in directory.options %}
 
                         <tr>
-                            <td>{{ option.key }}</td>
+                            <td style="width: 40%" onclick="modifyKey({{ option.id }}, '{{ option.value }}')" title="Click to update"><span id="key-{{ option.id }}">{{ option.key }}</span></td>
-                            <td>{{ option.value }}</td>
+                            <td style="width: 40%" onclick="modifyVal({{ option.id }}, '{{ option.key }}')" title="Click to update"><span id="val-{{ option.id }}">{{ option.value }}</span></td>
-                            <td><a class="btn btn-danger" href="/directory/{{ directory.id }}/del_opt/{{ option.id }}" >Remove</a></td>
+                            <td><a id="opt-{{ option.id }}-btn" class="btn btn-danger" href="/directory/{{ directory.id }}/del_opt/{{ option.id }}" >Remove</a></td>
                         </tr>
 
+
                     {% endfor %}
+
                     </tbody>
                 </table>
 

templates/layout.html

@@ -35,9 +35,9 @@
             padding: 4px;
         }
 
-{#        .info-table tr:nth-child(even) {#}
+        {#        .info-table tr:nth-child(even) {#}
-{#            background-color: #fafafa;#}
+        {#            background-color: #fafafa;#}
-{#        }#}
+        {#        }#}
 
         {#        todo: box-shadow 0 1px 10px 1px #1AC8DE#}
     </style>
@@ -69,6 +69,22 @@
     </div>
 </div>
 
+{% block alert_messages %}
+
+
+    {% with messages = get_flashed_messages(with_categories=true) %}
+        {% if messages %}
+            <div class="container">
+                {% for category, message in messages %}
+                    <div class="alert alert-{{ category }}">
+                        <a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
+                        {{ message | safe }}
+                    </div>
+                {% endfor %}
+            </div>
+        {% endif %}
+    {% endwith %}
+{% endblock %}
 
 {% block body %}
 {% endblock body %}

templates/task.html

@@ -32,22 +32,29 @@
             <div class="panel-heading">An excellent panel</div>
             <div class="panel-body">
 
-                <table class="info-table">
+                <table class="info-table table-hover table-striped">
+                    <thead>
                     <tr>
                         <th>Task type</th>
                         <th>Directory</th>
                         <th>Completed</th>
+                        <th>Action</th>
                     </tr>
+                    </thead>
 
-                    {% for task in tasks %}
+                    <tbody>
+                    {% for task_id in tasks %}
 
                         <tr>
-                            <td>{{ task.type }}</td>
+                            <td>{{ tasks[task_id].type }}</td>
-                            <td>{{ directories[task.dir_id].name }}</td>
+                            <td>{{ directories[tasks[task_id].dir_id].name }}</td>
-                            <td>{{ task.completed }}</td>
+                            <td>{{ tasks[task_id].completed }}</td>
+                            <td><a class="btn btn-danger" href="/task/{{ task_id }}/del">Cancel</a></td>
                         </tr>
 
                     {% endfor %}
+                    </tbody>
+
                 </table>