simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-10-30 21:36:52 +00:00
Code Issues Projects Releases Wiki Activity

Indexing is a bit gentler on server resources and some pages have been memory cached

Browse Source
This commit is contained in:
Simon 2018-07-12 11:26:20 -04:00
parent 1b743e7aba
commit 290322dfa7
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app.py
View File

@ -58,6 +58,7 @@ def stats_page():
@app.route("/stats/json_chart") @app.route("/stats/json_chart")
@cache.cached(240)
def stats_json(): def stats_json():
stats = searchEngine.get_global_stats() stats = searchEngine.get_global_stats()
db.join_website_on_stats(stats) db.join_website_on_stats(stats)
@ -82,7 +83,7 @@ def website_info(website_id):
@app.route("/website/<int:website_id>/json_chart") @app.route("/website/<int:website_id>/json_chart")
@cache.memoize(30) @cache.memoize(60)
def website_json_chart(website_id): def website_json_chart(website_id):
website = db.get_website_by_id(website_id) website = db.get_website_by_id(website_id)
@ -310,6 +311,7 @@ def contribute():
@app.route("/") @app.route("/")
@cache.cached(240)
def home(): def home():
stats = searchEngine.get_global_stats() stats = searchEngine.get_global_stats()
stats["website_count"] = len(db.get_all_websites()) stats["website_count"] = len(db.get_all_websites())
@ -393,7 +395,8 @@ def enqueue_bulk():
flash("Too few or too many urls, please submit 1-10 urls", "danger") flash("Too few or too many urls, please submit 1-10 urls", "danger")
return redirect("/submit") return redirect("/submit")
else: else:
return abort(500) flash("Too few or too many urls, please submit 1-10 urls", "danger")
return redirect("/submit")
else: else:
flash("<strong>Error:</strong> Invalid captcha please try again", "danger") flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
return redirect("/submit") return redirect("/submit")

7
search/search.py
View File

@ -114,7 +114,8 @@ class ElasticSearchEngine(SearchEngine):
def import_json(self, in_lines, website_id: int): def import_json(self, in_lines, website_id: int):
import_every = 2500 import_every = 1000
cooldown_time = 0.250
docs = [] docs = []
@ -129,6 +130,8 @@ class ElasticSearchEngine(SearchEngine):
if len(docs) >= import_every: if len(docs) >= import_every:
self._index(docs) self._index(docs)
docs.clear() docs.clear()
time.sleep(cooldown_time)
if docs: if docs:
self._index(docs) self._index(docs)
@ -265,8 +268,6 @@ class ElasticSearchEngine(SearchEngine):
def get_global_stats(self): def get_global_stats(self):
# TODO: mem cache this
size_per_ext = self.es.search(body={ size_per_ext = self.es.search(body={
"query": { "query": {
"bool": { "bool": {