simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-10 14:06:45 +00:00
Code Issues Projects Releases Wiki Activity

'Go to random website' button, and navigation in the website list

Browse Source
This commit is contained in:
Simon 2018-07-08 10:42:14 -04:00
parent 9ff21e7943
commit 711e8282ef
4 changed files with 60 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
app.py
View File

@ -1,5 +1,6 @@
from flask import Flask, render_template, redirect, request, flash, abort, Response, send_from_directory, session
import json
from urllib.parse import urlparse
import os
import time
import itertools
@ -105,7 +106,26 @@ def website_links(website_id):
@app.route("/website/")
def websites():
page = int(request.args.get("p")) if "p" in request.args else 0
return render_template("websites.html", websites=db.get_websites(100, page))
url = request.args.get("url") if "url" in request.args else ""
if url:
parsed_url = urlparse(url)
if parsed_url.scheme:
search_term = (parsed_url.scheme + "://" + parsed_url.netloc)
else:
flash("Sorry, I was not able to parse this url format. "
"Make sure you include the appropriate scheme (http/https/ftp)", "warning")
search_term = ""
else:
search_term = url
return render_template("websites.html",
websites=db.get_websites(10, page, search_term),
p=page, url=search_term, per_page=10)
@app.route("/website/random")
def random_website():
return redirect("/website/" + str(db.get_random_website_id()))
@app.route("/website/redispatch_queued")

13
database.py
View File

@ -104,16 +104,25 @@ class Database:
else:
return None
def get_websites(self, per_page, page: int):
def get_websites(self, per_page, page: int, url):
"""Get all websites"""
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("SELECT Website.id, Website.url, Website.last_modified FROM Website "
"ORDER BY last_modified DESC LIMIT ? OFFSET ?", (per_page, page * per_page))
"WHERE Website.url LIKE ?"
"ORDER BY last_modified DESC LIMIT ? OFFSET ?", (url + "%", per_page, page * per_page))
return cursor.fetchall()
def get_random_website_id(self):
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("SELECT id FROM Website WHERE id >= (abs(random()) % (SELECT max(id) FROM Website)) LIMIT 1;")
return cursor.fetchone()[0]
def website_exists(self, url):
"""Check if an url or the parent directory of an url already exists"""
with sqlite3.connect(self.db_path) as conn:

2
search/search.py
View File

@ -405,7 +405,6 @@ class ElasticSearchEngine(SearchEngine):
return stats
def stream_all_docs(self):
return helpers.scan(query={
"query": {
@ -413,7 +412,6 @@ class ElasticSearchEngine(SearchEngine):
}
}, scroll="5m", client=self.es, index=self.index_name)
def are_empty(self, websites):
result = self.es.search(body={
"query": {

29
templates/websites.html
View File

@ -6,7 +6,28 @@
{% block body %}
<div class="container">
<div class="card">
<div class="card-header">Last updated websites</div>
<div class="card-header">Go to website</div>
<div class="card-body">
<a class="btn btn-secondary" href="/website/random">Go to random website</a>
<hr>
<h3>Website search</h3>
<form>
<div class="form-row">
<div class="col-md-11">
<input class="form-control" placeholder="URL" name="url" value="{{ url }}">
</div>
<div class="col-md-1">
<input class="btn btn-primary" type="submit" value="Search">
</div>
</div>
</form>
</div>
</div>
</div>
<div class="container">
<div class="card">
<div class="card-header">Websites</div>
<div class="card-body">
<table class="table table-striped">
<thead>
@ -23,6 +44,12 @@
</tr>
{% endfor %}
</table>
{% if websites|length == per_page %}
<a href="/website?url={{ url }}&p={{ p + 1 }}" class="btn btn-primary" style="float: right">Next</a>
{% endif %}
{% if p > 0 %}
<a href="/website?url={{ url }}&p={{ p - 1 }}" class="btn btn-primary">Previous</a>
{% endif %}
</div>
</div>
</div>