simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-04 06:52:59 +00:00
Code Issues Projects Releases Wiki Activity

Bug fixes, ES7

Browse Source
This commit is contained in:
simon 2019-06-14 13:27:41 -04:00
parent 41ba6a35a4
commit 31877283b3
5 changed files with 22 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
database.py
View File

@ -1,6 +1,7 @@
import os
import time
import uuid
from urllib.parse import urlparse
from urllib.parse import urlparse, urljoin
import bcrypt
import psycopg2
@ -66,9 +67,8 @@ class Database:
with psycopg2.connect(self.db_conn_str) as conn:
cursor = conn.cursor()
cursor.execute("INSERT INTO Website (url, logged_ip, logged_useragent) VALUES (%s,%s,%s)",
cursor.execute("INSERT INTO Website (url, logged_ip, logged_useragent) VALUES (%s,%s,%s) RETURNING id",
(website.url, str(website.logged_ip), str(website.logged_useragent)))
cursor.execute("SELECT LAST_INSERT_ROWID()")
website_id = cursor.fetchone()[0]
conn.commit()
@ -235,7 +235,7 @@ class Database:
for hit in page["hits"]["hits"]:
if hit["_source"]["website_id"] in websites:
hit["_source"]["website_url"] = websites[hit["_source"]["website_id"]]
hit["_source"]["website_url"] = urljoin(websites[hit["_source"]["website_id"]], "/")
else:
hit["_source"]["website_url"] = "[DELETED]"
@ -247,7 +247,7 @@ class Database:
for doc in docs:
if doc["_source"]["website_id"] in websites:
doc["_source"]["website_url"] = websites[doc["_source"]["website_id"]]
doc["_source"]["website_url"] = urljoin(websites[doc["_source"]["website_id"]], "/")
else:
doc["_source"]["website_url"] = "[DELETED]"

8
search/search.py
View File

@ -1,5 +1,7 @@
import os
import time
from urllib.parse import urljoin
import ujson
import elasticsearch
@ -82,7 +84,7 @@ class ElasticSearchEngine(SearchEngine):
"type": "nGram", "min_gram": 3, "max_gram": 3
}
}
}}, index=self.index_name, request_timeout=60)
}}, index=self.index_name)
self.es.indices.put_settings(body={
"analysis": {
"analyzer": {
@ -104,7 +106,7 @@ class ElasticSearchEngine(SearchEngine):
"ext": {"type": "keyword"},
},
"_routing": {"required": True}
}, doc_type="file", index=self.index_name, request_timeout=60)
}, doc_type="file", index=self.index_name, include_type_name=True)
self.es.indices.open(index=self.index_name)
@ -327,7 +329,7 @@ class ElasticSearchEngine(SearchEngine):
index=self.index_name, request_timeout=20, routing=website_id)
for hit in hits:
src = hit["_source"]
yield base_url + src["path"] + ("/" if src["path"] != "" else "") + src["name"] + \
yield urljoin(base_url, "/") + src["path"] + ("/" if src["path"] != "" else "") + src["name"] + \
("." if src["ext"] != "" else "") + src["ext"]
def get_global_stats(self):

2
static/js/report.js
View File

@ -257,7 +257,7 @@ function drawChart(rData) {
function fillWebsiteTable(rData) {
document.getElementById("baseUrl").innerHTML = rData["base_url"];
document.getElementById("fileCount").innerHTML = rData["total_count"];
document.getElementById("fileCount").innerHTML = rData["total_count"].hasOwnProperty("value") ? rData["total_count"]["value"] : rData["total_count"];
document.getElementById("totalSize").innerHTML = humanFileSize(rData["total_size"]);
document.getElementById("reportTime").innerHTML = rData["report_time"] + " UTC";

11
templates/search.html
View File

@ -122,11 +122,11 @@
</div>
</div>
{% if results and results["hits"]["total"] > 0 %}
{% if count > 0 %}
<div class="card">
<div class="card-body">
<span class="text-muted">{{ results["hits"]["total"] }} result(s) in {{ results["took"] }}ms</span>
<span class="text-muted">{{ count }} result(s) in {{ results["took"] }}ms</span>
<div class="table-responsive">
<table class="table">
@ -147,7 +147,7 @@
<tr>
<td>
{% set category = get_mime(src["ext"]) %}
{% set url = src["website_url"] + src["path"] + "/" + src["name"] + ("." if src["ext"] != "" else "") + src["ext"] %}
{% set url = src["website_url"] + "/" + src["path"] + "/" + src["name"] + ("." if src["ext"] != "" else "") + src["ext"] %}
{# Preview #}
{% if category == "image" %}
<i class="fas fa-eye prev-icon" data-toggle="popover" tabindex="-1"
@ -164,7 +164,7 @@
{# File path #}
<div class="text-muted" title="{{ path }}" style="font-size: 10px;">
<a title="See files from this website"
href="/website/{{ src["website_id"] }}">{{ src["website_url"] }}</a>{{ hl_path|safe }}
href="/website/{{ src["website_id"] }}">{{ src["website_url"] }}/</a>{{ hl_path|safe }}
</div>
</td>
{# File size & date #}
@ -177,7 +177,7 @@
</tbody>
</table>
</div>
{% if results["hits"]["total"] > (p + 1) * per_page %}
{% if count > (p + 1) * per_page %}
<button class="btn btn-primary" onclick="nextPage()" style="float: right">Next</button>
{% endif %}
{% if p > 0 %}
@ -282,5 +282,6 @@
</script>
</div>
<!-- test -->
{% endblock body %}

9
views.py
View File

@ -3,14 +3,13 @@ import os
from multiprocessing.pool import Pool
from urllib.parse import urlparse
from flask import render_template, redirect, request, flash, abort, Response, session
from flask_caching import Cache
import captcha
import config
import od_util
from common import db, taskManager, searchEngine, logger, require_role
from database import Website
from flask import render_template, redirect, request, flash, abort, Response, session
from flask_caching import Cache
from search.search import InvalidQueryException
from tasks import Task
@ -149,6 +148,7 @@ def setup_views(app):
@app.route("/search")
def search():
results = 0
q = request.args.get("q") if "q" in request.args else ""
sort_order = request.args.get("sort_order") if "sort_order" in request.args else "score"
@ -207,7 +207,7 @@ def setup_views(app):
flash("Query failed, this could mean that the search server is overloaded or is not reachable. "
"Please try again later", "danger")
results = hits["hits"]["total"] if hits else -1
results = hits["hits"]["total"]["value"] if not isinstance(hits["hits"]["total"], int) else hits["hits"]["total"] if hits else -1
took = hits["took"] if hits else -1
forwarded_for = request.headers["X-Forwarded-For"] if "X-Forwarded-For" in request.headers else None
@ -225,6 +225,7 @@ def setup_views(app):
hits = None
return render_template("search.html",
count=results,
results=hits,
q=q,
p=page, per_page=per_page,