simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 02:16:47 +00:00
Code Issues Projects Releases Wiki Activity

retry on index fail/timeout

Browse Source
This commit is contained in:
Simon 2018-11-22 10:52:42 -05:00
parent 5782a45524
commit a33d4f2dca
1 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
search/search.py
View File

@ -179,13 +179,15 @@ class ElasticSearchEngine(SearchEngine):
self._index(docs) self._index(docs)
def _index(self, docs): def _index(self, docs):
while True:
try:
logger.debug("Indexing " + str(len(docs)) + " docs") logger.debug("Indexing " + str(len(docs)) + " docs")
bulk_string = ElasticSearchEngine.create_bulk_index_string(docs) bulk_string = ElasticSearchEngine.create_bulk_index_string(docs)
result = self.es.bulk(body=bulk_string, index=self.index_name, doc_type="file", request_timeout=30) self.es.bulk(body=bulk_string, index=self.index_name, doc_type="file", request_timeout=30)
break
if result["errors"]: except Exception as e:
logger.error("Error in ES bulk index: \n" + result["errors"]) logger.error("Error in _index: " + str(e) + ", retrying")
raise IndexingError time.sleep(10)
@staticmethod @staticmethod
def create_bulk_index_string(docs: list): def create_bulk_index_string(docs: list):