Bug fixes, ES7

database.py

@@ -1,6 +1,7 @@
+import os
 import time
 import uuid
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urljoin
 
 import bcrypt
 import psycopg2
@@ -66,9 +67,8 @@ class Database:
 
         with psycopg2.connect(self.db_conn_str) as conn:
             cursor = conn.cursor()
-            cursor.execute("INSERT INTO Website (url, logged_ip, logged_useragent) VALUES (%s,%s,%s)",
+            cursor.execute("INSERT INTO Website (url, logged_ip, logged_useragent) VALUES (%s,%s,%s) RETURNING id",
                            (website.url, str(website.logged_ip), str(website.logged_useragent)))
-            cursor.execute("SELECT LAST_INSERT_ROWID()")
 
             website_id = cursor.fetchone()[0]
             conn.commit()
@@ -235,7 +235,7 @@ class Database:
 
         for hit in page["hits"]["hits"]:
             if hit["_source"]["website_id"] in websites:
-                hit["_source"]["website_url"] = websites[hit["_source"]["website_id"]]
+                hit["_source"]["website_url"] = urljoin(websites[hit["_source"]["website_id"]], "/")
             else:
                 hit["_source"]["website_url"] = "[DELETED]"
 
@@ -247,7 +247,7 @@ class Database:
 
         for doc in docs:
             if doc["_source"]["website_id"] in websites:
-                doc["_source"]["website_url"] = websites[doc["_source"]["website_id"]]
+                doc["_source"]["website_url"] = urljoin(websites[doc["_source"]["website_id"]], "/")
             else:
                 doc["_source"]["website_url"] = "[DELETED]"
 

search/search.py

@@ -1,5 +1,7 @@
 import os
 import time
+from urllib.parse import urljoin
+
 import ujson
 
 import elasticsearch
@@ -82,7 +84,7 @@ class ElasticSearchEngine(SearchEngine):
                         "type": "nGram", "min_gram": 3, "max_gram": 3
                     }
                 }
-            }}, index=self.index_name, request_timeout=60)
+            }}, index=self.index_name)
         self.es.indices.put_settings(body={
             "analysis": {
                 "analyzer": {
@@ -104,7 +106,7 @@ class ElasticSearchEngine(SearchEngine):
                 "ext": {"type": "keyword"},
             },
             "_routing": {"required": True}
-        }, doc_type="file", index=self.index_name, request_timeout=60)
+        }, doc_type="file", index=self.index_name, include_type_name=True)
 
         self.es.indices.open(index=self.index_name)
 
@@ -327,7 +329,7 @@ class ElasticSearchEngine(SearchEngine):
                             index=self.index_name, request_timeout=20, routing=website_id)
         for hit in hits:
             src = hit["_source"]
-            yield base_url + src["path"] + ("/" if src["path"] != "" else "") + src["name"] + \
+            yield urljoin(base_url, "/") + src["path"] + ("/" if src["path"] != "" else "") + src["name"] + \
                   ("." if src["ext"] != "" else "") + src["ext"]
 
     def get_global_stats(self):

static/js/report.js

@@ -257,7 +257,7 @@ function drawChart(rData) {
 function fillWebsiteTable(rData) {
 
     document.getElementById("baseUrl").innerHTML = rData["base_url"];
-    document.getElementById("fileCount").innerHTML = rData["total_count"];
+    document.getElementById("fileCount").innerHTML = rData["total_count"].hasOwnProperty("value") ? rData["total_count"]["value"] : rData["total_count"];
     document.getElementById("totalSize").innerHTML = humanFileSize(rData["total_size"]);
     document.getElementById("reportTime").innerHTML = rData["report_time"] + " UTC";
 

templates/search.html

@@ -122,11 +122,11 @@
             </div>
         </div>
 
-        {% if results and results["hits"]["total"] > 0 %}
+        {% if count > 0 %}
             <div class="card">
                 <div class="card-body">
 
-                    <span class="text-muted">{{ results["hits"]["total"] }} result(s) in {{ results["took"] }}ms</span>
+                    <span class="text-muted">{{ count }} result(s) in {{ results["took"] }}ms</span>
 
                     <div class="table-responsive">
                         <table class="table">
@@ -147,7 +147,7 @@
                                 <tr>
                                     <td>
                                         {% set category = get_mime(src["ext"]) %}
-                                        {% set url = src["website_url"] + src["path"] + "/" + src["name"] + ("." if src["ext"] != "" else "") + src["ext"] %}
+                                        {% set url = src["website_url"] + "/" + src["path"] + "/" +  src["name"] + ("." if src["ext"] != "" else "") + src["ext"] %}
                                         {# Preview #}
                                         {% if category == "image" %}
                                             <i class="fas fa-eye prev-icon" data-toggle="popover" tabindex="-1"
@@ -164,7 +164,7 @@
                                         {# File path #}
                                         <div class="text-muted" title="{{ path }}" style="font-size: 10px;">
                                             <a title="See files from this website"
-                                               href="/website/{{ src["website_id"] }}">{{ src["website_url"] }}</a>{{ hl_path|safe }}
+                                               href="/website/{{ src["website_id"] }}">{{ src["website_url"] }}/</a>{{ hl_path|safe }}
                                         </div>
                                     </td>
                                     {# File size & date #}
@@ -177,7 +177,7 @@
                             </tbody>
                         </table>
                     </div>
-                    {% if results["hits"]["total"] > (p + 1) * per_page %}
+                    {% if count > (p + 1) * per_page %}
                         <button class="btn btn-primary" onclick="nextPage()" style="float: right">Next</button>
                     {% endif %}
                     {% if p > 0 %}
@@ -282,5 +282,6 @@
 
         </script>
     </div>
+    <!-- test -->
 
 {% endblock body %}

views.py

@@ -3,14 +3,13 @@ import os
 from multiprocessing.pool import Pool
 from urllib.parse import urlparse
 
-from flask import render_template, redirect, request, flash, abort, Response, session
-from flask_caching import Cache
-
 import captcha
 import config
 import od_util
 from common import db, taskManager, searchEngine, logger, require_role
 from database import Website
+from flask import render_template, redirect, request, flash, abort, Response, session
+from flask_caching import Cache
 from search.search import InvalidQueryException
 from tasks import Task
 
@@ -149,6 +148,7 @@ def setup_views(app):
 
     @app.route("/search")
     def search():
+        results = 0
         q = request.args.get("q") if "q" in request.args else ""
         sort_order = request.args.get("sort_order") if "sort_order" in request.args else "score"
 
@@ -207,7 +207,7 @@ def setup_views(app):
                     flash("Query failed, this could mean that the search server is overloaded or is not reachable. "
                           "Please try again later", "danger")
 
-                results = hits["hits"]["total"] if hits else -1
+                results = hits["hits"]["total"]["value"] if not isinstance(hits["hits"]["total"], int) else hits["hits"]["total"] if hits else -1
                 took = hits["took"] if hits else -1
                 forwarded_for = request.headers["X-Forwarded-For"] if "X-Forwarded-For" in request.headers else None
 
@@ -225,6 +225,7 @@ def setup_views(app):
             hits = None
 
         return render_template("search.html",
+                               count=results,
                                results=hits,
                                q=q,
                                p=page, per_page=per_page,