simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 10:56:47 +00:00
Code Issues Projects Releases Wiki Activity

Homepage stats now work with elasticsearch

Browse Source
This commit is contained in:
Simon 2018-06-12 23:19:57 -04:00
parent 2fe81e4b06
commit e91572a06f
4 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app.py
View File

@ -133,8 +133,8 @@ def contribute():
@app.route("/") @app.route("/")
def home(): def home():
stats = {}
stats = searchEngine.get_global_stats() stats = searchEngine.get_global_stats()
stats["website_count"] = len(db.get_all_websites())
current_websites = ", ".join(task.url for task in taskDispatcher.get_current_tasks()) current_websites = ", ".join(task.url for task in taskDispatcher.get_current_tasks())
return render_template("home.html", stats=stats, current_websites=current_websites) return render_template("home.html", stats=stats, current_websites=current_websites)

4
database.py
View File

@ -191,7 +191,7 @@ class Database:
cursor.execute("DELETE FROM ApiToken WHERE token=?", (token, )) cursor.execute("DELETE FROM ApiToken WHERE token=?", (token, ))
conn.commit() conn.commit()
def _get_websites(self) -> dict: def get_all_websites(self) -> dict:
# todo: mem cache that # todo: mem cache that
with sqlite3.connect(self.db_path) as conn: with sqlite3.connect(self.db_path) as conn:
@ -208,7 +208,7 @@ class Database:
def join_search_result(self, page: dict) -> dict: def join_search_result(self, page: dict) -> dict:
websites = self._get_websites() websites = self.get_all_websites()
for hit in page["hits"]["hits"]: for hit in page["hits"]["hits"]:
hit["_source"]["website_url"] = websites[hit["_source"]["website_id"]] hit["_source"]["website_url"] = websites[hit["_source"]["website_id"]]

11
search/search.py
View File

@ -222,10 +222,15 @@ class ElasticSearchEngine(SearchEngine):
}, },
"aggs": { "aggs": {
"total_size": { "total_size": {
"extended_stats": {"field": "size"} "sum": {"field": "size"}
} }
}, },
"size": 0 "size": 0
}) }, index=self.index_name)
stats = dict()
stats["file_count"] = result["hits"]["total"]
stats["file_size"] = result["aggregations"]["total_size"]["value"]
return stats
print(result)

3
templates/home.html
View File

@ -10,8 +10,7 @@
{% if stats and stats["file_size"] %} {% if stats and stats["file_size"] %}
<p class="lead">{{ stats["file_count"] }} files totalling <p class="lead">{{ stats["file_count"] }} files totalling
~{{ stats["file_size"] | filesizeformat }} in ~{{ stats["file_size"] | filesizeformat }} from {{ stats["website_count"] }} website(s)</p>
{{ stats["website_paths"] }} folders from {{ stats["website_count"] }} website(s)</p>
{% endif %} {% endif %}
{% if current_websites %} {% if current_websites %}
<p>Currently indexing <code>{{ current_websites }}</code><span class="vim-caret">&nbsp;</span> </p> <p>Currently indexing <code>{{ current_websites }}</code><span class="vim-caret">&nbsp;</span> </p>