Added admin clear & delete buttons for websites

app.py

@@ -112,6 +112,55 @@ def websites():
     return render_template("websites.html", websites=db.get_websites(100, page))
 
 
+@app.route("/website/delete_empty")
+def admin_delete_empty_website():
+    """Delete websites with no associated files that are not queued"""
+
+    if "username" in session:
+
+        current_tasks = taskDispatcher.get_queued_tasks() + taskDispatcher.get_current_tasks()
+        queued_websites = [task.website_id for task in current_tasks]
+        all_websites = db.get_all_websites()
+        non_queued_websites = list(set(all_websites).difference(queued_websites))
+
+        empty_websites = searchEngine.are_empty(non_queued_websites)
+
+        for website in empty_websites:
+            db.delete_website(website)
+
+        flash("Deleted: " + repr(list(empty_websites)), "success")
+        return redirect("/dashboard")
+
+    else:
+        abort(403)
+
+
+@app.route("/website/<int:website_id>/clear")
+def admin_clear_website(website_id):
+
+    if "username" in session:
+
+        searchEngine.delete_docs(website_id)
+        flash("Cleared all documents associated with this website", "success")
+        return redirect("/website/" + str(website_id))
+    else:
+        abort(403)
+
+
+@app.route("/website/<int:website_id>/delete")
+def admin_delete_website(website_id):
+
+    if "username" in session:
+
+        searchEngine.delete_docs(website_id)
+        db.delete_website(website_id)
+        flash("Deleted website " + str(website_id), "success")
+        return redirect("/website/")
+
+    else:
+        abort(403)
+
+
 @app.route("/search")
 def search():
 

blacklist.txt

@@ -6,3 +6,4 @@ https://www.dropbox.com
 https://oss.jfrog.org
 http://skyarchive.info
 https://skyarchive.info
+http://ftp.ubuntu.com/

database.py

@@ -114,16 +114,6 @@ class Database:
             website_id = cursor.fetchone()
             return website_id[0] if website_id else None
 
-    def website_has_been_scanned(self, url):
-        """Check if a website has at least 1 file"""
-        # TODO: Check with SearchEngine
-        print("FIXME: website_has_been_scanned")
-
-    def clear_website(self, website_id):
-        """Remove all files from a website and update its last_updated date"""
-        # TODO: Check with SearchEngine
-        print("FIXME: clear_website")
-
     def get_websites_older(self, delta: datetime.timedelta):
         """Get websites last updated before a given date"""
         date = datetime.datetime.utcnow() - delta
@@ -158,7 +148,7 @@ class Database:
         with sqlite3.connect(self.db_path) as conn:
             cursor = conn.cursor()
 
-            hashed_pw = bcrypt.hashpw(password.encode(), bcrypt.gensalt(14))
+            hashed_pw = bcrypt.hashpw(password.encode(), bcrypt.gensalt(12))
 
             cursor.execute("INSERT INTO Admin (username, password) VALUES (?,?)", (username, hashed_pw))
             conn.commit()

od_util.py

@@ -40,7 +40,8 @@ category_map = {
     'wiz': 'application', 'wsdl': 'application', 'xlb': 'application',
     'xls': 'application', 'xpdl': 'application', 'xsl': 'application',
     'torrent': 'application', 'rpm': 'application', 'deb': 'application',
-    'atr': 'application',
+    'atr': 'application', 'class': 'application', 'ttf': 'application',
+    'img': 'application', 'msi': 'application', 'run': 'application',
     # Text category
     'java': 'text', 'cpp': 'text', 'rb': 'text',
     'bat': 'text', 'latex': 'text', 'xml': 'text',
@@ -51,7 +52,10 @@ category_map = {
     'h': 'text', 'tsv': 'text', 'rtx': 'text',
     'sgm': 'text', 'sgml': 'text', 'txt': 'text',
     'vcf': 'text', 'pdf': 'text', 'epub': 'text',
-    'srt': 'text',
+    'srt': 'text', 'inc': 'text', 'php': 'text',
+    'cbz': 'text', 'docx': 'text', 'mobi': 'text',
+    'chm': 'text', 'xlsx': "text", 'djvu': 'text',
+    'rtf': 'text', 'log': 'text', 'md': 'text',
     # Video category
     '3g2': 'video', '3gp': 'video', 'asf': 'video',
     'asx': 'video', 'avi': 'video', 'flv': 'video',
@@ -60,13 +64,16 @@ category_map = {
     'm3u': 'video', 'm3u8': 'video', 'movie': 'video',
     'mp4': 'video', 'mpa': 'video', 'mpe': 'video',
     'mpeg': 'video', 'mpg': 'video', 'mkv': 'video',
-    'wmv': 'video', 'm4s': 'video',
+    'wmv': 'video', 'm4s': 'video', 'ogv': 'video',
+    'm4b': 'video', 'm4v': 'video',
     # Audio category
     'wav': 'audio', 'snd': 'audio', 'mp2': 'audio',
     'aif': 'audio', 'iff': 'audio', 'm4a': 'audio',
     'mid': 'audio', 'midi': 'audio', 'mp3': 'audio',
     'wma': 'audio', 'ra': 'audio', 'aifc': 'audio',
     'aiff': 'audio', 'au': 'audio', 'flac': 'audio',
+    'ogg': 'audio', 'oga': 'audio', 'mka': 'video',
+    'ac3': 'audio',
     # Image category
     'bmp': 'image', 'gif': 'image', 'jpg': 'image',
     'xwd': 'image', 'tif': 'image', 'tiff': 'image',
@@ -107,7 +114,7 @@ category_map = {
     'xp3': 'archive', 'yz1': 'archive', 'zip': 'archive',
     'zipx': 'archive', 'zoo': 'archive', 'zpaq': 'archive',
     'zz': 'archive', 'xpi': 'archive', 'tgz': 'archive',
-    'tbz': 'archive',
+    'tbz': 'archive', 'tar': 'archive', 'bz': 'archive',
 }
 
 colors = {

search/search.py

@@ -352,3 +352,36 @@ class ElasticSearchEngine(SearchEngine):
                 "match_all": {}
             }
         }, scroll="5m", client=self.es, index=self.index_name)
+
+    def are_empty(self, websites):
+
+        result = self.es.search(body={
+            "query": {
+                "bool": {
+                    "filter": {
+                        "terms": {
+                            "website_id": websites
+                        },
+                    }
+                }
+            },
+            "aggs": {
+                "websites": {
+                    "terms": {
+                        "field": "website_id",
+                        "size": 100000,
+                        "min_doc_count": 1
+                    }
+                }
+            },
+            "size": 0
+        }, index=self.index_name)
+
+        non_empty_websites = [bucket["key"] for bucket in result["aggregations"]["websites"]["buckets"]]
+
+        for website in websites:
+            if website not in non_empty_websites:
+                yield website
+
+
+

static/js/report.js

@@ -126,7 +126,7 @@ function drawChart(rData) {
 
     for (let ext in rData["ext_stats"]) {
 
-        dataSetSize.push(rData["ext_stats"][ext][0]);
+        dataSetSize.push(Math.max(rData["ext_stats"][ext][0], 0));
         dataSetCount.push(rData["ext_stats"][ext][1]);
         labels.push(rData["ext_stats"][ext][2] + " x" + rData["ext_stats"][ext][1] + " (" + humanFileSize(rData["ext_stats"][ext][0]) + ")");
 
@@ -252,7 +252,7 @@ category_map = {
     'h': 'text', 'tsv': 'text', 'rtx': 'text',
     'sgm': 'text', 'sgml': 'text', 'txt': 'text',
     'vcf': 'text', 'pdf': 'text', 'epub': 'text',
-    'srt': 'text', 'cbr': 'text',
+    'srt': 'text', 'cbr': 'text', 'inc': 'text',
     //Video category
     '3g2': 'video', '3gp': 'video', 'asf': 'video',
     'asx': 'video', 'avi': 'video', 'flv': 'video',
@@ -261,7 +261,8 @@ category_map = {
     'm3u': 'video', 'm3u8': 'video', 'movie': 'video',
     'mp4': 'video', 'mpa': 'video', 'mpe': 'video',
     'mpeg': 'video', 'mpg': 'video', 'mkv': 'video',
-    'wmv': 'video', 'm4s': 'video',
+    'wmv': 'video', 'm4s': 'video', 'm4v': 'video',
+    'mp4a': 'video',
     // Audio category
     'wav': 'audio', 'snd': 'audio', 'mp2': 'audio',
     'aif': 'audio', 'iff': 'audio', 'm4a': 'audio',

templates/dashboard.html

@@ -43,9 +43,11 @@
                     </div>
                 </form>
 
+                <br>
+                <hr>
                 <h3>Misc actions</h3>
 
-                <p>TODO: </p>
+                <a class="btn btn-danger" href="/website/delete_empty">Delete websites with no associated files that are not queued</a>
 
                 <hr>
                 <a class="btn btn-info" href="/logout">Logout</a>

templates/website.html

@@ -40,6 +40,10 @@
                 <hr>
                 <a href="/website/{{ website.id }}/links" class="btn btn-shadow btn-primary">Link list</a>
                 <a href="/website/{{ website.id }}/json_chart" class="btn btn-shadow btn-primary">Summary (JSON)</a>
+                {% if "username" in session %}
+                    <a href="/website/{{ website.id }}/clear" class="btn btn-danger"><i class="fas fa-exclamation"></i> Clear</a>
+                    <a href="/website/{{ website.id }}/delete" class="btn btn-danger"><i class="fas fa-trash"></i> Delete</a>
+                {% endif %}
             </div>
         </div>
     </div>