simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 10:56:47 +00:00
Code Issues Projects Releases Wiki Activity

Basic searching integrated with elasticsearch + highlighting

Browse Source
This commit is contained in:
Simon 2018-06-12 16:29:05 -04:00
parent af2601ee70
commit 0127b3a51d
4 changed files with 43 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app.py
View File

@ -8,6 +8,7 @@ import od_util
import config
from flask_caching import Cache
from task import TaskDispatcher, Task
from search.search import ElasticSearchEngine
app = Flask(__name__)
recaptcha = ReCaptcha(app=app,
@ -21,6 +22,7 @@ app.jinja_env.globals.update(get_color=od_util.get_color)
app.jinja_env.globals.update(get_mime=od_util.get_mime)
taskDispatcher = TaskDispatcher()
searchEngine = ElasticSearchEngine("od-database")
@app.template_filter("datetime_format")
@ -108,14 +110,15 @@ def search():
if q:
try:
# hits = sea.search(q, per_page, page, sort_order)
print("FIXME: Search")
hits = []
hits = searchEngine.search(q, page, per_page)
except InvalidQueryException as e:
flash("<strong>Invalid query:</strong> " + str(e), "warning")
return redirect("/search")
else:
hits = None
print(hits)
return render_template("search.html",
results=hits, q=q, p=page, sort_order=sort_order,
per_page=per_page, results_set=config.RESULTS_PER_PAGE)

16
search/search.py
View File

@ -109,9 +109,10 @@ class ElasticSearchEngine(SearchEngine):
return "\n".join("".join([action_string, doc[:-1], website_id_string]) for doc in docs)
def search(self, query) -> {}:
def search(self, query, page, per_page) -> {}:
filters = []
sort_by = ["_score"]
page = self.es.search(body={
"query": {
@ -126,22 +127,15 @@ class ElasticSearchEngine(SearchEngine):
"filter": filters
}
},
"sort": [
"_score"
],
"sort": sort_by,
"highlight": {
"fields": {
"name": {"pre_tags": ["<span class='hl'>"], "post_tags": ["</span>"]},
"path": {"pre_tags": ["<span class='hl'>"], "post_tags": ["</span>"]}
}
},
"size": 40}, index=self.index_name, scroll="8m")
"size": per_page, "from": page * per_page}, index=self.index_name)
# todo get scroll time from config
# todo get size from config
return page
def scroll(self, scroll_id) -> {}:
try:
return self.es.scroll(scroll_id=scroll_id, scroll="3m") # todo get scroll time from config
except TransportError:
return None

3
static/css/main.css
View File

@ -73,3 +73,6 @@
100% {
background-color: rgba(255, 255, 255, 0.6); } }
.hl {
background: #fff217;
}

46
templates/search.html
View File

@ -18,10 +18,17 @@
<div class="form-group col-md-2">
<select class="form-control" name="sort_order" title="Sort order">
<option disabled>Select sort order</option>
<option value="score" {{ "selected" if sort_order == "score" else "" }}>Relevance</option>
<option value="size_asc" {{ "selected" if sort_order == "size_asc" else "" }}>Size ascending</option>
<option value="size_dsc" {{ "selected" if sort_order == "size_dsc" else "" }}>Size descending</option>
<option value="none" {{ "selected" if sort_order == "none" else "" }}>No order (faster)</option>
<option value="score" {{ "selected" if sort_order == "score" else "" }}>Relevance
</option>
<option value="size_asc" {{ "selected" if sort_order == "size_asc" else "" }}>Size
ascending
</option>
<option value="size_dsc" {{ "selected" if sort_order == "size_dsc" else "" }}>Size
descending
</option>
<option value="none" {{ "selected" if sort_order == "none" else "" }}>No order
(faster)
</option>
</select>
</div>
<div class="form-group col-md-2">
@ -40,7 +47,7 @@
</div>
</div>
{% if results %}
{% if results["hits"]["total"] > 0 %}
<div class="card">
<div class="card-body">
@ -48,39 +55,46 @@
<table class="table">
<tbody>
{% for hit in results %}
{% set path = hit[1] + hit[2] %}
{% for hit in results["hits"]["hits"] %}
{% set src = hit["_source"] %}
{% set hl_name = hit["highlight"]["name"][0] if "name" in hit["highlight"] else src["name"] %}
{% set hl_path = hit["highlight"]["path"][0] if "path" in hit["highlight"] else src["path"] %}
{# TODO: website url + path #}
{% set path = src["path"] %}
<tr>
<td>
{# File name & link #}
<a href="{{ path + "/" + hit[3] }}" title="{{ hit[3] }}">
{{ hit[3] | truncate(50, True, "..>") }}
<a href="{{ path + "/" + src["name"] }}" title="{{ src["name"] }}">
{{ hl_name |safe }}
</a>
{# File type badge #}
{% set mime = get_mime(hit[3]) %}
{% set mime = get_mime(src["path"]) %}
{% if mime %}
<span class="badge badge-pill {{ get_color(mime) }}">
{{ hit[3][hit[3].rfind(".") + 1:] }}
{{ src["path"][src["path"].rfind(".") + 1:] }}
</span>
{% endif %}
{# File path #}
<div class="text-muted" title="{{ path }}" style="font-size: 10px;">
<a style="color: #6c757d" title="See files from this website"
href="/website/{{ hit[4] }}">{{ hit[1] }}</a>{{ truncate_path(hit[2], 60) }}
{# todo: website url #}
href="/website/{{ src["website_id"] }}">{{ hl_path | safe }}</a>{{ truncate_path(src["path"], 60) }}
</div>
</td>
{# File size #}
<td style="white-space: nowrap; vertical-align: top; text-align: right; font-size: 14px">
{{ hit[0] | filesizeformat if hit[0] >= 0 else "?"}}
{{ src["size"] | filesizeformat if src["size"] >= 0 else "?" }}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<a href="/search?q={{ q }}&p={{ p + 1 }}&sort_order={{ sort_order }}&per_page={{ per_page }}"
class="btn btn-primary" style="float: right">Next</a>
{% if results["hits"]["total"] > (p + 1) * per_page %}
<a href="/search?q={{ q }}&p={{ p + 1 }}&sort_order={{ sort_order }}&per_page={{ per_page }}"
class="btn btn-primary" style="float: right">Next</a>
{% endif %}
{% if p > 0 %}
<a href="/search?q={{ q }}&p={{ p - 1 }}&sort_order={{ sort_order }}&per_page={{ per_page }}"
class="btn btn-primary">Previous</a>