simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-14 23:29:04 +00:00
Code Issues Projects Releases Wiki Activity

More FTP crawler bug fixes

Browse Source
This commit is contained in:
Simon
2018-06-24 18:05:30 -04:00
parent ab35ce96cc
commit e11343de23
3 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crawl_server/remote_ftp.py
View File

@@ -30,7 +30,8 @@ class FtpDirectory(RemoteDirectory):
self.ftp = ftputil.FTPHost(self.base_url, "anonymous", "od-database", session_factory=session_factory(
use_passive_mode=True
))
self.ftp._session.timeout = 1
self.ftp.keep_alive()
self.ftp._session.timeout = 30
def stop_when_connected(self):
failed_attempts = 0
@@ -96,10 +97,8 @@ class FtpDirectory(RemoteDirectory):
return path, []
def reconnect(self):
if self.ftp:
self.ftp.close()
time.sleep(8)
self.stop_when_connected()
def try_stat(self, path):