Migrated to bootstrap 4, added progress bars for tasks

crawler.py

@@ -1,4 +1,22 @@
 import os
+from storage import Task, LocalStorage
+import json
+from multiprocessing import Process, Value
+from apscheduler.schedulers.background import BackgroundScheduler
+from parsing import GenericFileParser, Md5CheckSumCalculator, ExtensionMimeGuesser
+import time
+
+
+class RunningTask:
+
+    def __init__(self, task: Task):
+        self.total_files = 0
+        self.parsed_files = Value("i", 0)
+        self.task = task
+
+    def to_json(self):
+        return json.dumps({"parsed": self.parsed_files.value, "total": self.total_files, "id": self.task.id})
+
 
 class Crawler:
 
@@ -16,7 +34,8 @@ class Crawler:
             for ext in parser.extensions:
                 self.ext_map[ext] = parser
 
-    def crawl(self, root_dir: str):
+    def crawl(self, root_dir: str, counter: Value=None):
+
         for root, dirs, files in os.walk(root_dir):
 
             for filename in files:
@@ -24,9 +43,15 @@ class Crawler:
 
                 parser = self.ext_map.get(os.path.splitext(filename)[1], self.default_parser)
 
+                try:
+                    if counter:
+                        counter.value += 1
+
                     doc = parser.parse(full_path)
 
                     self.documents.append(doc)
+                except FileNotFoundError:
+                    continue  # File was deleted
 
     def countFiles(self, root_dir: str):
         count = 0
@@ -36,3 +61,52 @@ class Crawler:
 
         return count
 
+
+class TaskManager:
+    def __init__(self, storage: LocalStorage):
+        self.current_task = None
+        self.storage = storage
+        self.current_process = None
+
+        scheduler = BackgroundScheduler()
+        scheduler.add_job(self.check_new_task, "interval", seconds=0.5)
+        scheduler.start()
+
+    def start_task(self, task: Task):
+        self.current_task = RunningTask(task)
+
+        c = Crawler([GenericFileParser([Md5CheckSumCalculator()], ExtensionMimeGuesser())])
+        path = self.storage.dirs()[task.dir_id].path
+        self.current_task.total_files = c.countFiles(path)
+
+        print("Started task - " + str(self.current_task.total_files) + " files")
+        print(path)
+
+        self.current_process = Process(target=self.execute_crawl, args=(c, path, self.current_task.parsed_files))
+        self.current_process.daemon = True
+        self.current_process.start()
+
+    def execute_crawl(self, c: Crawler, path: str, counter: Value):
+        c.crawl(path, counter)
+        print("Done")
+
+    def cancel_task(self):
+        self.current_task = None
+        self.current_process.terminate()
+
+    def check_new_task(self):
+
+        if self.current_task is None:
+            for i in sorted(self.storage.tasks(), reverse=True):
+                if not self.storage.tasks()[i].completed:
+                    self.start_task(self.storage.tasks()[i])
+        else:
+            print(self.current_task.parsed_files.value)
+
+            if self.current_task.parsed_files.value == self.current_task.total_files:
+
+                self.current_process.terminate()
+                self.storage.del_task(self.current_task.task.id)
+                self.current_task = None
+
+

requirements.txt

@@ -4,3 +4,4 @@ flask_bcrypt
 elasticsearch
 python-magic
 requests
+apscheduler

run.py

@@ -1,128 +1,19 @@
 from flask import Flask, render_template, send_file, request, redirect, flash, session
 from indexer import Indexer
 from storage import Directory, Option, Task
+from storage import LocalStorage, DuplicateDirectoryException
+from crawler import RunningTask, TaskManager
 import json
 
 # indexer = Indexer("fse")
 
 app = Flask(__name__)
 app.secret_key = "A very secret key"
-#
-# class Document:
-#     def __init__(self, doc_id, name, path, size, md5):
-#         self.doc_id = doc_id
-#         self.name = name
-#         self.path = path
-#         self.size = size
-#         self.md5 = md5
-#
-#
-# class ImageDocument(Document):
-#     def __init__(self, doc_id, name, path, size, md5):
-#         super().__init__(doc_id, name, path, size, md5)
-#         self.type = "image"
-#
-#
-# class AudioClipDocument(Document):
-#     def __init__(self, doc_id, name, path, size, md5):
-#         super().__init__(doc_id, name, path, size, md5)
-#         self.type = "audio"
-#
-#
-# def get_document(id):
-#
-#     response = requests.get(SOLR_URL + "get?id=" + id)
-#
-#     return json.loads(response.text)["doc"]
-#
-#
-# def make_thumb(doc):
-#     size = (1024, 1024)
-#
-#     thumb_path = "thumbnails/" + doc["id"]
-#
-#     if not os.path.exists(thumb_path):
-#
-#         file_path = doc["path"][0] + "/" + doc["name"][0]
-#
-#         if doc["width"][0] > size[0]:
-#
-#             image = Image.open(file_path)
-#             image.thumbnail(size, Image.ANTIALIAS)
-#
-#             if image.mode == "RGB":
-#                 image.save(thumb_path, "JPEG")
-#             elif image.mode == "RGBA":
-#                 image.save(thumb_path, "PNG")
-#             else:
-#                 image = image.convert("RGB")
-#                 image.save(thumb_path, "JPEG")
-#         else:
-#             print("Skipping thumbnail")
-#             os.symlink(file_path, thumb_path)
-#
-#     return "thumbnails/" + doc["id"]
-#
-#
-# @app.route("/search/")
-# def search():
-#
-#     query = request.args.get("query")
-#     page = int(request.args.get("page"))
-#     per_page = int(request.args.get("per_page"))
-#
-#     results = solr.search(query, None, rows=per_page, start=per_page * page)
-#
-#     docs = []
-#     for r in results:
-#
-#         if "mime" in r:
-#             mime_type = r["mime"][0]
-#         else:
-#             mime_type = ""
-#
-#         if mime_type.startswith("image"):
-#             docs.append(ImageDocument(r["id"], r["name"][0], r["path"][0], r["size"], r["md5"]))
-#
-#         elif mime_type.startswith("audio"):
-#             docs.append(AudioClipDocument(r["id"], r["name"][0], r["path"][0], r["size"], r["md5"]))
-#
-#     return render_template("search.html", docs=docs)
-#
-#
-# @app.route("/")
-# def index():
-#     return render_template("index.html")
-#
-#
-# @app.route("/files/<id>/")
-# def files(id):
-#
-#     doc = get_document(id)
-#
-#     if doc is not None:
-#         file_path = doc["path"][0] + "/" + doc["name"][0]
-#         return send_file(file_path, mimetype=mimetypes.guess_type(file_path)[0])
-#     else:
-#         return "File not found"
-#
-#
-# @app.route("/thumbs/<doc_id>/")
-# def thumbs(doc_id):
-#
-#     doc = get_document(doc_id)
-#
-#     if doc is not None:
-#
-#         thumb_path = make_thumb(doc)
-#
-#         return send_file("thumbnails/" + doc_id, mimetype=mimetypes.guess_type(thumb_path)[0])
-#     else:
-#         return "File not found"
-from storage import LocalStorage, DuplicateDirectoryException
 storage = LocalStorage("local_storage.db")
 
 
+tm = TaskManager(storage)
+
 @app.route("/")
 def tmp_route():
     return "huh"
@@ -225,10 +116,34 @@ def directory_del(dir_id):
     return redirect("/directory")
 
 
+for t in storage.tasks():
+    a_task = t
+    break
+
+# tm = None
+
 @app.route("/task")
 def task():
 
-    return render_template("task.html", tasks=storage.tasks(), directories=storage.dirs())
+    return render_template("task.html", tasks=storage.tasks(), directories=storage.dirs(),
+                           task_list=json.dumps(list(storage.tasks().keys())))
+    # return render_template("task.html", tasks=storage.tasks(), directories=storage.dirs())
+
+
+@app.route("/task/current")
+def get_current_task():
+
+    if tm and tm.current_task:
+        return tm.current_task.to_json()
+    else:
+        return ""
+
+
+@app.route("/task/current/cancel")
+def cancel_current_task():
+
+    tm.cancel_task()
+    return redirect("/task")
 
 
 @app.route("/task/add")

templates/directory.html

@@ -6,9 +6,9 @@
     <div class="container">
 
         {# Add directory form #}
-        <div class="panel panel-default">
+        <div class="card">
-            <div class="panel-heading">An excellent form</div>
+            <div class="card-header">An excellent form</div>
-            <div class="panel-body">
+            <div class="card-body">
 
                 <form method="GET" action="/directory/add">
                     <div class="form-group">
@@ -24,9 +24,9 @@
         </div>
 
         {# List of directories #}
-        <div class="panel panel-default">
+        <div class="card">
-            <div class="panel-heading">An excellent list</div>
+            <div class="card-header">An excellent list</div>
-            <div class="panel-body">
+            <div class="card-body">
 
                 <table class="info-table table-hover table-striped">
                     <thead>

templates/directory_manage.html

@@ -70,10 +70,10 @@
 
     <div class="container">
 
-        <div class="panel panel-default">
+        <div class="card">
 
-            <div class="panel-heading">Summary</div>
+            <div class="card-header">Summary</div>
-            <div class="panel-body">
+            <div class="card-body">
 
                 <table class="info-table">
                     <tr onclick="modifyDisplayName()">
@@ -106,9 +106,9 @@
 
         </div>
 
-        <div class="panel panel-default">
+        <div class="card">
-            <div class="panel-heading">An excellent option list</div>
+            <div class="card-header">An excellent option list</div>
-            <div class="panel-body">
+            <div class="card-body">
                 <table class="info-table table-striped table-hover">
                     <thead>
                     <tr>
@@ -136,48 +136,41 @@
 
                 <form method="GET" action="/directory/{{ directory.id }}/add_opt">
 
-                    <div class="form-group">
+                    <div class="form-row">
-                        <div class="col-sm-4">
+                        <div class="col">
                             <input type="text" class="form-control" placeholder="Key" name="key">
                         </div>
 
-
+                        <div class="col">
-                        <div class="col-sm-4">
                             <input type="text" class="form-control" placeholder="Value" name="value">
                         </div>
-
+                        <button type="submit" class="btn btn-success">Add option</button>
                     </div>
 
-                    <button type="submit" class="btn btn-success">Add option</button>
                 </form>
 
             </div>
         </div>
 
-        <div class="panel panel-default">
+        <div class="card">
-            <div class="panel-heading">An excellent control panel</div>
+            <div class="card-header">An excellent control panel</div>
-            <div class="panel-body">
+            <div class="card-body">
 
-                <div class="btn-group">
+                <div class="dropdown">
-                    <a class="btn dropdown-toggle btn-primary" data-toggle="dropdown" href="#">
+                    <button class="btn dropdown-toggle btn-primary" data-toggle="dropdown">Create a task</button>
-                        Create a task
+                    <div class="dropdown-menu">
-                        <span class="caret"></span>
+                        <a class="dropdown-item" href="#">Indexing task</a>
-                    </a>
+                        <a class="dropdown-item" href="#">Thumbnail generation task</a>
-                    <ul class="dropdown-menu">
+                    </div>
-                        <li><a href="#">Indexing task</a></li>
-                        <li><a href="#">Thumbnail generation task</a></li>
-                    </ul>
                 </div>
 
-                <div class="btn-group">
+                <div class="dropdown">
-                    <a class="btn dropdown-toggle btn-danger" data-toggle="dropdown" href="#">
+                    <button class="btn dropdown-toggle btn-danger" data-toggle="dropdown" aria-haspopup="true">Action</button>
-                        Action
+
-                        <span class="caret"></span>
+                    <div class="dropdown-menu">
-                    </a>
+                        <a class="dropdown-item" href="/directory/{{ directory.id }}/del">Delete directory</a>
-                    <ul class="dropdown-menu">
+                        <a class="dropdown-item" href="#">Reset to default settings</a>
-                        <li><a href="/directory/{{ directory.id }}/del">Delete directory</a></li>
+                    </div>
-                        <li><a href="#">Reset to default settings</a></li>
-                    </ul>
                 </div>
 
             </div>

templates/layout.html

@@ -7,17 +7,13 @@
 
     <!-- Demo Dependencies -->
     <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.3/jquery.min.js" type="text/javascript"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.2.0/js/bootstrap.min.js" type="text/javascript"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
-    <link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.2.0/css/bootstrap.min.css" rel="stylesheet" type="text/css" />
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/holder/2.3.2/holder.min.js" type="text/javascript"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
     <script src="/static/js/Chart.min.js" type="text/javascript"></script>
 
-    <script>
-        Holder.add_theme("white", { background:"#fff", foreground:"#a7a7a7", size:10 });
-    </script>
-
     <!-- Dashboard -->
-    <link rel="stylesheet" type="text/css" href="/static/css/keen-dashboards.css" />
+    {#    <link rel="stylesheet" type="text/css" href="/static/css/keen-dashboards.css" />#}
     <link href="https://use.fontawesome.com/releases/v5.0.6/css/all.css" rel="stylesheet" type="text/css">
 
 
@@ -35,6 +31,10 @@
             padding: 4px;
         }
 
+        .card {
+            margin-top: 1em;
+        }
+
         {#        .info-table tr:nth-child(even) {#}
         {#            background-color: #fafafa;#}
         {#        }#}
@@ -44,29 +44,8 @@
 </head>
 <body class="keen-dashboard" style="padding-top: 80px;">
 
-<div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+<div>
-    <div class="container-fluid">
+    <span>Navbar1</span>
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
-                <span class="sr-only">Toggle navigation</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="../">
-                <span class="glyphicon glyphicon-chevron-left"></span>
-            </a>
-            <a class="navbar-brand" href="./">Layouts &raquo; Hero Sidebar</a>
-        </div>
-        <div class="navbar-collapse collapse">
-            <ul class="nav navbar-nav navbar-left">
-                <li><a href="#">Home</a></li>
-                <li><a href="#">Team</a></li>
-                <li><a href="#">Source</a></li>
-                <li><a href="#">Community</a></li>
-            </ul>
-        </div>
-    </div>
 </div>
 
 {% block alert_messages %}

templates/task.html

@@ -4,11 +4,42 @@
 
 {% block body %}
 
+    <style>
+        .task-wrapper {
+            border: #dddddd 1px solid;
+            border-radius: 4px;
+            padding: 5px 10px;
+            margin-bottom: 0.5em;
+        }
+
+        .task-name {
+            color: #9CA0A2;
+        }
+
+        .task-info {
+
+        }
+
+        .progress {
+            position: relative;
+            height: 100%;
+
+        }
+
+        .progress span {
+            position: absolute;
+            display: block;
+            width: 100%;
+            color: black;
+        }
+
+    </style>
+
     <div class="container">
 
-        <div class="panel panel-default">
+        <div class="card">
-            <div class="panel-heading">An excellent form</div>
+            <div class="card-header">An excellent form</div>
-            <div class="panel-body">
+            <div class="card-body">
                 <form class="form-inline" action="/task/add">
                     <label for="type">Create </label>
                     <select class="form-control" id="type" name="type" >
@@ -28,36 +59,61 @@
             </div>
         </div>
 
-        <div class="panel panel-default">
-            <div class="panel-heading">An excellent panel</div>
-            <div class="panel-body">
 
-                <table class="info-table table-hover table-striped">
-                    <thead>
-                    <tr>
-                        <th>Task type</th>
-                        <th>Directory</th>
-                        <th>Completed</th>
-                        <th>Action</th>
-                    </tr>
-                    </thead>
 
-                    <tbody>
+        <script>
-                    {% for task_id in tasks %}
+            function updateProgressBar() {
+                var xhttp = new XMLHttpRequest();
+                xhttp.onreadystatechange = function() {
+                    if (this.readyState === 4 && this.status === 200) {
 
-                        <tr>
+                        if(this.responseText.length === 0) {
-                            <td>{{ tasks[task_id].type }}</td>
+                            return;
-                            <td>{{ directories[tasks[task_id].dir_id].name }}</td>
+                        }
-                            <td>{{ tasks[task_id].completed }}</td>
+
-                            <td><a class="btn btn-danger" href="/task/{{ task_id }}/del">Cancel</a></td>
+                        var currentTask = JSON.parse(this.responseText);
-                        </tr>
+                        var percent = currentTask.parsed / currentTask.total * 100;
+
+                        try {
+                            document.getElementById("task-bar-" + currentTask.id).setAttribute("style", "width: " + percent + "%;");
+                            document.getElementById("task-label-" + currentTask.id).innerHTML = currentTask.parsed + " / " + currentTask.total + "  (" + percent.toFixed(2) + "%)";
+                        } catch (e) {
+                            window.reload();
+                        }
+
+                    }
+                };
+                xhttp.open("GET", "/task/current", true);
+                xhttp.send();
+            }
+
+            window.setInterval(updateProgressBar, 125);
+        </script>
+
+
+        <div class="card">
+            <div class="card-header">An excellent panel</div>
+            <div class="card-body">
+                {% for task_id in tasks | sort()%}
+                    <div class="task-wrapper container-fluid">
+                        <span class="task-name">{{ directories[tasks[task_id].dir_id].name }} - </span>
+                        <span class="task-info">{{ tasks[task_id].type }}</span>
+
+                        <div class="d-flex p-2">
+                            <div class="container-fluid p-2">
+                                <div class="progress">
+                                    <div id="task-bar-{{ task_id }}" class="progress-bar" role="progressbar" style="width: 0;">
+                                        <span id="task-label-{{ task_id }}">Queued</span>
+                                    </div>
+                                </div>
+                            </div>
+
+                            <div class="p-2"><a class="btn btn-danger" href="/task/{{ task_id }}/del">Cancel</a></div>
+                        </div>
+
+                    </div>
 
                 {% endfor %}
-                    </tbody>
-
-                </table>
-
-
 
             </div>
         </div>