simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-18 18:06:44 +00:00
Code Issues Projects Releases Wiki Activity

Added admin clear & delete buttons for websites

Browse Source
This commit is contained in:
Simon 2018-06-20 10:48:51 -04:00
parent cef9e2c8a1
commit 35837463cd
8 changed files with 107 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
app.py
View File

@ -112,6 +112,55 @@ def websites():
return render_template("websites.html", websites=db.get_websites(100, page))
@app.route("/website/delete_empty")
def admin_delete_empty_website():
"""Delete websites with no associated files that are not queued"""
if "username" in session:
current_tasks = taskDispatcher.get_queued_tasks() + taskDispatcher.get_current_tasks()
queued_websites = [task.website_id for task in current_tasks]
all_websites = db.get_all_websites()
non_queued_websites = list(set(all_websites).difference(queued_websites))
empty_websites = searchEngine.are_empty(non_queued_websites)
for website in empty_websites:
db.delete_website(website)
flash("Deleted: " + repr(list(empty_websites)), "success")
return redirect("/dashboard")
else:
abort(403)
@app.route("/website/<int:website_id>/clear")
def admin_clear_website(website_id):
if "username" in session:
searchEngine.delete_docs(website_id)
flash("Cleared all documents associated with this website", "success")
return redirect("/website/" + str(website_id))
else:
abort(403)
@app.route("/website/<int:website_id>/delete")
def admin_delete_website(website_id):
if "username" in session:
searchEngine.delete_docs(website_id)
db.delete_website(website_id)
flash("Deleted website " + str(website_id), "success")
return redirect("/website/")
else:
abort(403)
@app.route("/search")
def search():

1
blacklist.txt
View File

@ -6,3 +6,4 @@ https://www.dropbox.com
https://oss.jfrog.org
http://skyarchive.info
https://skyarchive.info
http://ftp.ubuntu.com/

12
database.py
View File

@ -114,16 +114,6 @@ class Database:
website_id = cursor.fetchone()
return website_id[0] if website_id else None
def website_has_been_scanned(self, url):
"""Check if a website has at least 1 file"""
# TODO: Check with SearchEngine
print("FIXME: website_has_been_scanned")
def clear_website(self, website_id):
"""Remove all files from a website and update its last_updated date"""
# TODO: Check with SearchEngine
print("FIXME: clear_website")
def get_websites_older(self, delta: datetime.timedelta):
"""Get websites last updated before a given date"""
date = datetime.datetime.utcnow() - delta
@ -158,7 +148,7 @@ class Database:
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
hashed_pw = bcrypt.hashpw(password.encode(), bcrypt.gensalt(14))
hashed_pw = bcrypt.hashpw(password.encode(), bcrypt.gensalt(12))
cursor.execute("INSERT INTO Admin (username, password) VALUES (?,?)", (username, hashed_pw))
conn.commit()

15
od_util.py
View File

@ -40,7 +40,8 @@ category_map = {
'wiz': 'application', 'wsdl': 'application', 'xlb': 'application',
'xls': 'application', 'xpdl': 'application', 'xsl': 'application',
'torrent': 'application', 'rpm': 'application', 'deb': 'application',
'atr': 'application',
'atr': 'application', 'class': 'application', 'ttf': 'application',
'img': 'application', 'msi': 'application', 'run': 'application',
# Text category
'java': 'text', 'cpp': 'text', 'rb': 'text',
'bat': 'text', 'latex': 'text', 'xml': 'text',
@ -51,7 +52,10 @@ category_map = {
'h': 'text', 'tsv': 'text', 'rtx': 'text',
'sgm': 'text', 'sgml': 'text', 'txt': 'text',
'vcf': 'text', 'pdf': 'text', 'epub': 'text',
'srt': 'text',
'srt': 'text', 'inc': 'text', 'php': 'text',
'cbz': 'text', 'docx': 'text', 'mobi': 'text',
'chm': 'text', 'xlsx': "text", 'djvu': 'text',
'rtf': 'text', 'log': 'text', 'md': 'text',
# Video category
'3g2': 'video', '3gp': 'video', 'asf': 'video',
'asx': 'video', 'avi': 'video', 'flv': 'video',
@ -60,13 +64,16 @@ category_map = {
'm3u': 'video', 'm3u8': 'video', 'movie': 'video',
'mp4': 'video', 'mpa': 'video', 'mpe': 'video',
'mpeg': 'video', 'mpg': 'video', 'mkv': 'video',
'wmv': 'video', 'm4s': 'video',
'wmv': 'video', 'm4s': 'video', 'ogv': 'video',
'm4b': 'video', 'm4v': 'video',
# Audio category
'wav': 'audio', 'snd': 'audio', 'mp2': 'audio',
'aif': 'audio', 'iff': 'audio', 'm4a': 'audio',
'mid': 'audio', 'midi': 'audio', 'mp3': 'audio',
'wma': 'audio', 'ra': 'audio', 'aifc': 'audio',
'aiff': 'audio', 'au': 'audio', 'flac': 'audio',
'ogg': 'audio', 'oga': 'audio', 'mka': 'video',
'ac3': 'audio',
# Image category
'bmp': 'image', 'gif': 'image', 'jpg': 'image',
'xwd': 'image', 'tif': 'image', 'tiff': 'image',
@ -107,7 +114,7 @@ category_map = {
'xp3': 'archive', 'yz1': 'archive', 'zip': 'archive',
'zipx': 'archive', 'zoo': 'archive', 'zpaq': 'archive',
'zz': 'archive', 'xpi': 'archive', 'tgz': 'archive',
'tbz': 'archive',
'tbz': 'archive', 'tar': 'archive', 'bz': 'archive',
}
colors = {

33
search/search.py
View File

@ -352,3 +352,36 @@ class ElasticSearchEngine(SearchEngine):
"match_all": {}
}
}, scroll="5m", client=self.es, index=self.index_name)
def are_empty(self, websites):
result = self.es.search(body={
"query": {
"bool": {
"filter": {
"terms": {
"website_id": websites
},
}
}
},
"aggs": {
"websites": {
"terms": {
"field": "website_id",
"size": 100000,
"min_doc_count": 1
}
}
},
"size": 0
}, index=self.index_name)
non_empty_websites = [bucket["key"] for bucket in result["aggregations"]["websites"]["buckets"]]
for website in websites:
if website not in non_empty_websites:
yield website

7
static/js/report.js
View File

@ -126,7 +126,7 @@ function drawChart(rData) {
for (let ext in rData["ext_stats"]) {
dataSetSize.push(rData["ext_stats"][ext][0]);
dataSetSize.push(Math.max(rData["ext_stats"][ext][0], 0));
dataSetCount.push(rData["ext_stats"][ext][1]);
labels.push(rData["ext_stats"][ext][2] + " x" + rData["ext_stats"][ext][1] + " (" + humanFileSize(rData["ext_stats"][ext][0]) + ")");
@ -252,7 +252,7 @@ category_map = {
'h': 'text', 'tsv': 'text', 'rtx': 'text',
'sgm': 'text', 'sgml': 'text', 'txt': 'text',
'vcf': 'text', 'pdf': 'text', 'epub': 'text',
'srt': 'text', 'cbr': 'text',
'srt': 'text', 'cbr': 'text', 'inc': 'text',
//Video category
'3g2': 'video', '3gp': 'video', 'asf': 'video',
'asx': 'video', 'avi': 'video', 'flv': 'video',
@ -261,7 +261,8 @@ category_map = {
'm3u': 'video', 'm3u8': 'video', 'movie': 'video',
'mp4': 'video', 'mpa': 'video', 'mpe': 'video',
'mpeg': 'video', 'mpg': 'video', 'mkv': 'video',
'wmv': 'video', 'm4s': 'video',
'wmv': 'video', 'm4s': 'video', 'm4v': 'video',
'mp4a': 'video',
// Audio category
'wav': 'audio', 'snd': 'audio', 'mp2': 'audio',
'aif': 'audio', 'iff': 'audio', 'm4a': 'audio',

4
templates/dashboard.html
View File

@ -43,9 +43,11 @@
</div>
</form>
<br>
<hr>
<h3>Misc actions</h3>
<p>TODO: </p>
<a class="btn btn-danger" href="/website/delete_empty">Delete websites with no associated files that are not queued</a>
<hr>
<a class="btn btn-info" href="/logout">Logout</a>

4
templates/website.html
View File

@ -40,6 +40,10 @@
<hr>
<a href="/website/{{ website.id }}/links" class="btn btn-shadow btn-primary">Link list</a>
<a href="/website/{{ website.id }}/json_chart" class="btn btn-shadow btn-primary">Summary (JSON)</a>
{% if "username" in session %}
<a href="/website/{{ website.id }}/clear" class="btn btn-danger"><i class="fas fa-exclamation"></i> Clear</a>
<a href="/website/{{ website.id }}/delete" class="btn btn-danger"><i class="fas fa-trash"></i> Delete</a>
{% endif %}
</div>
</div>
</div>