Should fix some FTP errors

crawl_server/crawler.py

@@ -129,7 +129,6 @@ class RemoteDirectoryCrawler:
     def _process_listings(self, url: str, in_q: Queue, files_q: Queue):
 
         directory = RemoteDirectoryFactory.get_directory(url)
-        timeout_retries = RemoteDirectoryCrawler.MAX_TIMEOUT_RETRIES
 
         while directory:
             try:
@@ -145,7 +144,6 @@ class RemoteDirectoryCrawler:
                 path_id, listing = directory.list_dir(path)
                 if len(listing) > 0 and path_id not in self.crawled_paths:
                     self.crawled_paths.append(path_id)
-                    timeout_retries = RemoteDirectoryCrawler.MAX_TIMEOUT_RETRIES
 
                     for f in listing:
                         if f.is_dir:

crawl_server/remote_ftp.py

@@ -14,11 +14,15 @@ class FtpDirectory(RemoteDirectory):
 
     SCHEMES = ("ftp", )
 
+    CANCEL_LISTING_CODE = (
+        550,  # Forbidden
+    )
+
     def __init__(self, url):
 
         host = urlparse(url).netloc
         super().__init__(host)
-        self.max_attempts = 3
+        self.max_attempts = 2
         self.ftp = None
         self.stop_when_connected()
 
@@ -69,13 +73,18 @@ class FtpDirectory(RemoteDirectory):
             except ftputil.error.ParserError as e:
                 print("TODO: fix parsing error: " + e.strerror + " @ " + str(e.file_name))
                 break
-            except ftputil.error.FTPOSError as e:
-                if e.strerror == "timed out":
-                    failed_attempts += 1
-                    continue
             except ftputil.error.FTPError as e:
+                if e.errno in FtpDirectory.CANCEL_LISTING_CODE:
+                    break
+                failed_attempts += 1
+                print(str(e.strerror) + "errno" + str(e.errno))
+                print("Error - reconnecting")
+                self.stop_when_connected()
+            except ftputil.error.PermanentError as e:
                 if e.errno == 530:
                     raise TooManyConnectionsError()
+                print(str(e.strerror) + "errno" + str(e.errno))
+                break
             except Exception as e:
                 # TODO remove that debug info
                 print("ERROR:" + str(e))

crawl_server/task_manager.py

@@ -88,11 +88,7 @@ class TaskManager:
     @staticmethod
     def task_complete(result):
 
-        try:
-            task_result, db_path, current_tasks = result
-        except Exception as e:
-            print("Exception during task " + str(e))
-            return
+        task_result, db_path, current_tasks = result
 
         print(task_result.status_code)
         print(task_result.file_count)