From 750940d148cc80d9898db560e84096f8379b758d Mon Sep 17 00:00:00 2001
From: Simon <fortier.simon@hotmail.com>
Date: Sun, 18 Nov 2018 11:19:34 -0500
Subject: [PATCH] error handling for delete

---
 search/search.py | 41 ++++++++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/search/search.py b/search/search.py
index 3b299f2..a310d40 100644
--- a/search/search.py
+++ b/search/search.py
@@ -114,30 +114,33 @@ class ElasticSearchEngine(SearchEngine):
 
     def delete_docs(self, website_id):
 
-        try:
-            logger.debug("Deleting docs of " + str(website_id))
+        while True:
+            try:
+                logger.debug("Deleting docs of " + str(website_id))
 
-            to_delete = helpers.scan(query={
-                "query": {
-                    "term": {"website_id": website_id}
-                }
-            }, scroll="1m", client=self.es, index=self.index_name, request_timeout=60)
+                to_delete = helpers.scan(query={
+                    "query": {
+                        "term": {"website_id": website_id}
+                    }
+                }, scroll="1m", client=self.es, index=self.index_name, request_timeout=120)
 
-            buf = []
-            counter = 0
-            for doc in to_delete:
-                buf.append(doc)
-                counter += 1
+                buf = []
+                counter = 0
+                for doc in to_delete:
+                    buf.append(doc)
+                    counter += 1
 
-                if counter >= 400:
+                    if counter >= 400:
+                        self._delete(buf)
+                        buf.clear()
+                        counter = 0
+                if counter > 0:
                     self._delete(buf)
-                    buf.clear()
-                    counter = 0
-            if counter > 0:
-                self._delete(buf)
+                break
 
-        except Exception as e:
-            logger.error(str(e))
+            except Exception as e:
+                logger.error("During delete: " + str(e))
+                time.sleep(10)
 
         logger.debug("Done deleting for " + str(website_id))