simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-16 08:09:04 +00:00
Code Issues Projects Releases Wiki Activity

Indexing is a bit gentler on server resources and some pages have been memory cached

Browse Source
This commit is contained in:
Simon
2018-07-12 11:26:20 -04:00
parent 1b743e7aba
commit 290322dfa7
2 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
search/search.py
View File

@@ -114,7 +114,8 @@ class ElasticSearchEngine(SearchEngine):
def import_json(self, in_lines, website_id: int):
import_every = 2500
import_every = 1000
cooldown_time = 0.250
docs = []
@@ -129,6 +130,8 @@ class ElasticSearchEngine(SearchEngine):
if len(docs) >= import_every:
self._index(docs)
docs.clear()
time.sleep(cooldown_time)
if docs:
self._index(docs)
@@ -265,8 +268,6 @@ class ElasticSearchEngine(SearchEngine):
def get_global_stats(self):
# TODO: mem cache this
size_per_ext = self.es.search(body={
"query": {
"bool": {