simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-18 18:06:44 +00:00
Code Issues Projects Releases Wiki Activity

error handling for delete

Browse Source
This commit is contained in:
Simon 2018-11-18 11:19:34 -05:00
parent 254d6ea44e
commit 750940d148
1 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
search/search.py
View File

@ -114,30 +114,33 @@ class ElasticSearchEngine(SearchEngine):
def delete_docs(self, website_id):
try:
logger.debug("Deleting docs of " + str(website_id))
while True:
try:
logger.debug("Deleting docs of " + str(website_id))
to_delete = helpers.scan(query={
"query": {
"term": {"website_id": website_id}
}
}, scroll="1m", client=self.es, index=self.index_name, request_timeout=60)
to_delete = helpers.scan(query={
"query": {
"term": {"website_id": website_id}
}
}, scroll="1m", client=self.es, index=self.index_name, request_timeout=120)
buf = []
counter = 0
for doc in to_delete:
buf.append(doc)
counter += 1
buf = []
counter = 0
for doc in to_delete:
buf.append(doc)
counter += 1
if counter >= 400:
if counter >= 400:
self._delete(buf)
buf.clear()
counter = 0
if counter > 0:
self._delete(buf)
buf.clear()
counter = 0
if counter > 0:
self._delete(buf)
break
except Exception as e:
logger.error(str(e))
except Exception as e:
logger.error("During delete: " + str(e))
time.sleep(10)
logger.debug("Done deleting for " + str(website_id))