simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 18:36:44 +00:00
Code Issues Projects Releases Wiki Activity

Re-enabled multi threaded file requests for large directories

Browse Source
This commit is contained in:
Simon 2018-07-15 08:54:36 -04:00
parent 2f1b0c96f1
commit 152a6f20fb
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
crawl_server/remote_http.py
View File

@ -89,7 +89,7 @@ class HttpDirectory(RemoteDirectory):
"../" "../"
) )
MAX_RETRIES = 3 MAX_RETRIES = 2
def __init__(self, url): def __init__(self, url):
super().__init__(url) super().__init__(url)
@ -138,7 +138,7 @@ class HttpDirectory(RemoteDirectory):
def request_files(self, urls_to_request: list) -> list: def request_files(self, urls_to_request: list) -> list:
if len(urls_to_request) > 3000000: if len(urls_to_request) > 150:
# Many urls, use multi-threaded solution # Many urls, use multi-threaded solution
pool = ThreadPool(processes=10) pool = ThreadPool(processes=10)
files = pool.starmap(HttpDirectory._request_file, zip(repeat(self), urls_to_request)) files = pool.starmap(HttpDirectory._request_file, zip(repeat(self), urls_to_request))