diff --git a/database.py b/database.py index 96c7fe4..61a236b 100644 --- a/database.py +++ b/database.py @@ -1,6 +1,7 @@ +import os import time import uuid -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin import bcrypt import psycopg2 @@ -66,9 +67,8 @@ class Database: with psycopg2.connect(self.db_conn_str) as conn: cursor = conn.cursor() - cursor.execute("INSERT INTO Website (url, logged_ip, logged_useragent) VALUES (%s,%s,%s)", + cursor.execute("INSERT INTO Website (url, logged_ip, logged_useragent) VALUES (%s,%s,%s) RETURNING id", (website.url, str(website.logged_ip), str(website.logged_useragent))) - cursor.execute("SELECT LAST_INSERT_ROWID()") website_id = cursor.fetchone()[0] conn.commit() @@ -235,7 +235,7 @@ class Database: for hit in page["hits"]["hits"]: if hit["_source"]["website_id"] in websites: - hit["_source"]["website_url"] = websites[hit["_source"]["website_id"]] + hit["_source"]["website_url"] = urljoin(websites[hit["_source"]["website_id"]], "/") else: hit["_source"]["website_url"] = "[DELETED]" @@ -247,7 +247,7 @@ class Database: for doc in docs: if doc["_source"]["website_id"] in websites: - doc["_source"]["website_url"] = websites[doc["_source"]["website_id"]] + doc["_source"]["website_url"] = urljoin(websites[doc["_source"]["website_id"]], "/") else: doc["_source"]["website_url"] = "[DELETED]" diff --git a/search/search.py b/search/search.py index 40747f1..e7f7b32 100644 --- a/search/search.py +++ b/search/search.py @@ -1,5 +1,7 @@ import os import time +from urllib.parse import urljoin + import ujson import elasticsearch @@ -82,7 +84,7 @@ class ElasticSearchEngine(SearchEngine): "type": "nGram", "min_gram": 3, "max_gram": 3 } } - }}, index=self.index_name, request_timeout=60) + }}, index=self.index_name) self.es.indices.put_settings(body={ "analysis": { "analyzer": { @@ -104,7 +106,7 @@ class ElasticSearchEngine(SearchEngine): "ext": {"type": "keyword"}, }, "_routing": {"required": True} - }, doc_type="file", index=self.index_name, request_timeout=60) + }, doc_type="file", index=self.index_name, include_type_name=True) self.es.indices.open(index=self.index_name) @@ -327,7 +329,7 @@ class ElasticSearchEngine(SearchEngine): index=self.index_name, request_timeout=20, routing=website_id) for hit in hits: src = hit["_source"] - yield base_url + src["path"] + ("/" if src["path"] != "" else "") + src["name"] + \ + yield urljoin(base_url, "/") + src["path"] + ("/" if src["path"] != "" else "") + src["name"] + \ ("." if src["ext"] != "" else "") + src["ext"] def get_global_stats(self): diff --git a/static/js/report.js b/static/js/report.js index 3d4df44..3d704cd 100644 --- a/static/js/report.js +++ b/static/js/report.js @@ -257,7 +257,7 @@ function drawChart(rData) { function fillWebsiteTable(rData) { document.getElementById("baseUrl").innerHTML = rData["base_url"]; - document.getElementById("fileCount").innerHTML = rData["total_count"]; + document.getElementById("fileCount").innerHTML = rData["total_count"].hasOwnProperty("value") ? rData["total_count"]["value"] : rData["total_count"]; document.getElementById("totalSize").innerHTML = humanFileSize(rData["total_size"]); document.getElementById("reportTime").innerHTML = rData["report_time"] + " UTC"; diff --git a/templates/search.html b/templates/search.html index bd434dc..3b495ad 100755 --- a/templates/search.html +++ b/templates/search.html @@ -122,11 +122,11 @@ - {% if results and results["hits"]["total"] > 0 %} + {% if count > 0 %}
- {{ results["hits"]["total"] }} result(s) in {{ results["took"] }}ms + {{ count }} result(s) in {{ results["took"] }}ms
@@ -147,7 +147,7 @@ {# File size & date #} @@ -177,7 +177,7 @@
{% set category = get_mime(src["ext"]) %} - {% set url = src["website_url"] + src["path"] + "/" + src["name"] + ("." if src["ext"] != "" else "") + src["ext"] %} + {% set url = src["website_url"] + "/" + src["path"] + "/" + src["name"] + ("." if src["ext"] != "" else "") + src["ext"] %} {# Preview #} {% if category == "image" %} {{ src["website_url"] }}{{ hl_path|safe }} + href="/website/{{ src["website_id"] }}">{{ src["website_url"] }}/{{ hl_path|safe }}
- {% if results["hits"]["total"] > (p + 1) * per_page %} + {% if count > (p + 1) * per_page %} {% endif %} {% if p > 0 %} @@ -282,5 +282,6 @@
+ {% endblock body %} diff --git a/views.py b/views.py index d42b9b4..eccbfea 100644 --- a/views.py +++ b/views.py @@ -3,14 +3,13 @@ import os from multiprocessing.pool import Pool from urllib.parse import urlparse -from flask import render_template, redirect, request, flash, abort, Response, session -from flask_caching import Cache - import captcha import config import od_util from common import db, taskManager, searchEngine, logger, require_role from database import Website +from flask import render_template, redirect, request, flash, abort, Response, session +from flask_caching import Cache from search.search import InvalidQueryException from tasks import Task @@ -149,6 +148,7 @@ def setup_views(app): @app.route("/search") def search(): + results = 0 q = request.args.get("q") if "q" in request.args else "" sort_order = request.args.get("sort_order") if "sort_order" in request.args else "score" @@ -207,7 +207,7 @@ def setup_views(app): flash("Query failed, this could mean that the search server is overloaded or is not reachable. " "Please try again later", "danger") - results = hits["hits"]["total"] if hits else -1 + results = hits["hits"]["total"]["value"] if not isinstance(hits["hits"]["total"], int) else hits["hits"]["total"] if hits else -1 took = hits["took"] if hits else -1 forwarded_for = request.headers["X-Forwarded-For"] if "X-Forwarded-For" in request.headers else None @@ -225,6 +225,7 @@ def setup_views(app): hits = None return render_template("search.html", + count=results, results=hits, q=q, p=page, per_page=per_page,