simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-24 12:45:51 +00:00
Code Issues Projects Releases Wiki Activity

More search options

Browse Source
This commit is contained in:
Simon 2018-06-26 21:38:26 -04:00
parent b1ad39c204
commit b570e81bec
4 changed files with 102 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
app.py
View File

@ -10,6 +10,7 @@ import config
from flask_caching import Cache from flask_caching import Cache
from task import TaskDispatcher, Task, CrawlServer from task import TaskDispatcher, Task, CrawlServer
from search.search import ElasticSearchEngine from search.search import ElasticSearchEngine
from jinja2 import Undefined
app = Flask(__name__) app = Flask(__name__)
recaptcha = ReCaptcha(app=app, recaptcha = ReCaptcha(app=app,
@ -239,9 +240,27 @@ def search():
size_max = request.args.get("size_max") if "size_max" in request.args else "size_max" size_max = request.args.get("size_max") if "size_max" in request.args else "size_max"
size_max = int(size_max) if size_max.isdigit() else 0 size_max = int(size_max) if size_max.isdigit() else 0
match_all = "all" in request.args
field_name = "field_name" in request.args
field_trigram = "field_trigram" in request.args
field_path = "field_path" in request.args
if not field_name and not field_trigram and not field_path:
# If no fields are selected, search in all
field_name = field_path = field_trigram = True
fields = []
if field_path:
fields.append("path")
if field_name:
fields.append("name^5")
if field_trigram:
fields.append("name.nGram^2")
if len(q) >= 3: if len(q) >= 3:
try: try:
hits = searchEngine.search(q, page, per_page, sort_order, extensions, size_min, size_max) hits = searchEngine.search(q, page, per_page, sort_order, extensions, size_min, size_max, match_all, fields)
hits = db.join_website_on_search_result(hits) hits = db.join_website_on_search_result(hits)
except InvalidQueryException as e: except InvalidQueryException as e:
flash("<strong>Invalid query:</strong> " + str(e), "warning") flash("<strong>Invalid query:</strong> " + str(e), "warning")
@ -250,9 +269,15 @@ def search():
hits = None hits = None
return render_template("search.html", return render_template("search.html",
results=hits, q=q, p=page, sort_order=sort_order, results=hits,
per_page=per_page, results_set=config.RESULTS_PER_PAGE, q=q,
extensions=",".join(extensions), size_min=size_min, size_max=size_max) p=page, per_page=per_page,
sort_order=sort_order,
results_set=config.RESULTS_PER_PAGE,
extensions=",".join(extensions),
size_min=size_min, size_max=size_max,
match_all=match_all,
field_trigram=field_trigram, field_path=field_path, field_name=field_name)
@app.route("/contribute") @app.route("/contribute")

8
search/search.py
View File

@ -16,7 +16,7 @@ class SearchEngine:
def import_json(self, in_str: str, website_id: int): def import_json(self, in_str: str, website_id: int):
raise NotImplementedError raise NotImplementedError
def search(self, query, page, per_page, sort_order, extension, size_min, size_max) -> {}: def search(self, query, page, per_page, sort_order, extension, size_min, size_max, match_all, fields) -> {}:
raise NotImplementedError raise NotImplementedError
def reset(self): def reset(self):
@ -142,7 +142,7 @@ class ElasticSearchEngine(SearchEngine):
action_string = '{"index":{}}\n' action_string = '{"index":{}}\n'
return "\n".join("".join([action_string, ujson.dumps(doc)]) for doc in docs) return "\n".join("".join([action_string, ujson.dumps(doc)]) for doc in docs)
def search(self, query, page, per_page, sort_order, extensions, size_min, size_max) -> {}: def search(self, query, page, per_page, sort_order, extensions, size_min, size_max, match_all, fields) -> {}:
filters = [] filters = []
if extensions: if extensions:
@ -167,8 +167,8 @@ class ElasticSearchEngine(SearchEngine):
"must": { "must": {
"multi_match": { "multi_match": {
"query": query, "query": query,
"fields": ["name^5", "name.nGram^2", "path"], "fields": fields,
"operator": "or" "operator": "or" if match_all else "and"
} }
}, },
"filter": filters "filter": filters

4
static/css/main.css
View File

@ -92,8 +92,8 @@
mark { mark {
background-color: rgba(255, 255, 0, 0.4); background-color: rgba(255, 255, 0, 0.4);
border-radius: 2px; border-radius: 0;
padding: 1px; padding: 1px 0;
} }
body { body {
color: #BBBBBB; color: #BBBBBB;

95
templates/search.html
View File

@ -14,13 +14,59 @@
<div class="form-row"> <div class="form-row">
{# Query #} {# Query #}
<div class="form-group col-md-7"> <div class="input-group form-group">
<div class="input-group-prepend">
<div class="input-group-text">
<label for="matchAll" style="margin-bottom: 0">Match any word&nbsp</label>
<input title="Toggle between 'match all words' and 'match any word'" type="checkbox" id="matchAll" name="all" {{ "checked" if match_all }}>
</div>
</div>
<input class="form-control" name="q" id="q" placeholder="Query" value="{{ q }}"> <input class="form-control" name="q" id="q" placeholder="Query" value="{{ q }}">
</div> </div>
{# Sort order #} </div>
{# Size #}
<div class="text-muted" style="text-align: center">File size</div>
<input title="File size" id="sizeSlider">
<input type="hidden" name="size_min" id="sizeMin" value="{{ size_min }}">
<input type="hidden" name="size_max" id="sizeMax" value="{{ size_max }}">
<div class="form-row">
{# File extension #}
<div class="form-group col-md-6">
<div class="text-muted">File extension</div>
<div class="input-group">
<div class="input-group-prepend">
<div class="input-group-text">.</div>
</div>
<input name="ext" placeholder="Extensions, comma-separated" class="form-control"
value="{{ extensions }}">
</div>
</div>
{# Fields #}
<div class="col-md-5" style="margin-left: 1em">
<div class="text-muted">Search in</div>
<div class="form-group form-check form-check-inline">
<input class="form-check-input" type="checkbox" id="field_name" name="field_name"{{ "checked" if field_name else ""}}>
<label class="form-check-label" for="field_name">name</label>
</div>
<div class="form-group form-check form-check-inline">
<input class="form-check-input" type="checkbox" id="field_trigram" name="field_trigram"{{ "checked" if field_trigram else ""}}>
<label class="form-check-label" for="field_trigram">name.trigram</label>
</div>
<div class="form-group form-check form-check-inline">
<input class="form-check-input" type="checkbox" id="field_path" name="field_path" {{ "checked" if field_path else ""}}>
<label class="form-check-label" for="field_path">path</label>
</div>
</div>
</div>
<div class="text-muted">Display options</div>
<div class="form-row">
<div class="form-group col-md-3"> <div class="form-group col-md-3">
{# Sort order #}
<select class="form-control" name="sort_order" title="Sort order"> <select class="form-control" name="sort_order" title="Sort order">
<option disabled>Select sort order</option> <option disabled>Sort order</option>
<option value="score" {{ "selected" if sort_order == "score" else "" }}>Relevance </option> <option value="score" {{ "selected" if sort_order == "score" else "" }}>Relevance </option>
<option value="size_asc" {{ "selected" if sort_order == "size_asc" else "" }}>Size ascending </option> <option value="size_asc" {{ "selected" if sort_order == "size_asc" else "" }}>Size ascending </option>
<option value="size_dsc" {{ "selected" if sort_order == "size_dsc" else "" }}>Size descending</option> <option value="size_dsc" {{ "selected" if sort_order == "size_dsc" else "" }}>Size descending</option>
@ -38,27 +84,12 @@
{% endfor %} {% endfor %}
</select> </select>
</div> </div>
</div>
{# Filters #}
<span class="text-muted">Filters</span>
<div class="form-row">
{# File extension #} {# Search button #}
<div class="form-group col-md-6"> <div class="form-group col-md-7">
<div class="input-group"> <input class="btn btn-primary btn-shadow" type="submit" value="Search" style="float: right">
<div class="input-group-prepend">
<div class="input-group-text">.</div>
</div>
<input name="ext" placeholder="Extensions, comma-separated" class="form-control" value="{{ extensions }}">
</div>
</div> </div>
</div> </div>
{# Size #}
<input title="File size" id="sizeSlider">
<input type="hidden" name="size_min" id="sizeMin" value="{{ size_min }}">
<input type="hidden" name="size_max" id="sizeMax" value="{{ size_max }}">
<input class="btn btn-primary btn-shadow" type="submit" value="Search">
</form> </form>
</div> </div>
@ -76,7 +107,14 @@
{% for hit in results["hits"]["hits"] %} {% for hit in results["hits"]["hits"] %}
{% set src = hit["_source"] %} {% set src = hit["_source"] %}
{% set hl_name = hit["highlight"]["name"][0] if "name" in hit["highlight"] else src["name"] %} {% if "name" in hit["highlight"] %}
{% set hl_name = hit["highlight"]["name"][0] %}
{% elif "name.nGram" in hit["highlight"] %}
{% set hl_name = hit["highlight"]["name.nGram"][0] %}
{% else %}
{% set hl_name = src["name"] %}
{% endif %}
{% set hl_path = hit["highlight"]["path"][0] if "path" in hit["highlight"] else src["path"] %} {% set hl_path = hit["highlight"]["path"][0] if "path" in hit["highlight"] else src["path"] %}
<tr> <tr>
@ -98,7 +136,8 @@
{% endif %} {% endif %}
{# File path #} {# File path #}
<div class="text-muted" title="{{ path }}" style="font-size: 10px;"> <div class="text-muted" title="{{ path }}" style="font-size: 10px;">
<a title="See files from this website" href="/website/{{ src["website_id"] }}">{{ src["website_url"] }}</a>{{ hl_path|safe }} <a title="See files from this website"
href="/website/{{ src["website_id"] }}">{{ src["website_url"] }}</a>{{ hl_path|safe }}
</div> </div>
</td> </td>
{# File size & date #} {# File size & date #}
@ -112,11 +151,11 @@
</table> </table>
</div> </div>
{% if results["hits"]["total"] > (p + 1) * per_page %} {% if results["hits"]["total"] > (p + 1) * per_page %}
<a href="/search?q={{ q }}&p={{ p + 1 }}&sort_order={{ sort_order }}&per_page={{ per_page }}&ext={{ extensions }}&size_min={{ size_min }}&size_max={{ size_max }}" <a href="/search?q={{ q }}&p={{ p + 1 }}&sort_order={{ sort_order }}&per_page={{ per_page }}&ext={{ extensions }}&size_min={{ size_min }}&size_max={{ size_max }}{{ "&field_path=on" if field_path else "" }}{{ "&field_name=on" if field_name else "" }}{{ "&field_trigram" if field_trigram else "" }}"
class="btn btn-primary" style="float: right">Next</a> class="btn btn-primary" style="float: right">Next</a>
{% endif %} {% endif %}
{% if p > 0 %} {% if p > 0 %}
<a href="/search?q={{ q }}&p={{ p - 1 }}&sort_order={{ sort_order }}&per_page={{ per_page }}&ext={{ extensions }}&size_min={{ size_min }}&size_max={{ size_max }}" <a href="/search?q={{ q }}&p={{ p - 1 }}&sort_order={{ sort_order }}&per_page={{ per_page }}&ext={{ extensions }}&size_min={{ size_min }}&size_max={{ size_max }}{{ "&field_path=on" if field_path else "" }}{{ "&field_name=on" if field_name else "" }}{{ "&field_trigram" if field_trigram else "" }}"
class="btn btn-primary">Previous</a> class="btn btn-primary">Previous</a>
{% endif %} {% endif %}
@ -151,7 +190,7 @@
drag_interval: true, drag_interval: true,
prettify: function (num) { prettify: function (num) {
if(num === 0) { if (num === 0) {
return "0 B" return "0 B"
} else if (num >= 3684) { } else if (num >= 3684) {
return humanFileSize(num * num * num) + "+"; return humanFileSize(num * num * num) + "+";
@ -159,11 +198,11 @@
return humanFileSize(num * num * num) return humanFileSize(num * num * num)
}, },
onFinish: function(e) { onFinish: function (e) {
let size_min = (e.from * e.from * e.from); let size_min = (e.from * e.from * e.from);
let size_max = (e.to * e.to * e.to); let size_max = (e.to * e.to * e.to);
if (e.to >= 3684) { if (e.to >= 3684) {
size_max = null; size_max = null;
} }