simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-17 08:39:03 +00:00
Code Issues Projects Releases Wiki Activity

Elasticsearch search engine (search & scroll)

Browse Source
This commit is contained in:
Simon
2018-06-11 23:06:41 -04:00
parent 72495275b0
commit 011b8455a7
4 changed files with 102 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawler/http.py
View File

@@ -76,7 +76,7 @@ class HttpDirectory(RemoteDirectory):
results = []
if len(urls_to_request) > 3:
if len(urls_to_request) > 4:
# Many urls, use multi-threaded solution
pool = ThreadPool(processes=10)
files = pool.starmap(HttpDirectory._request_file, zip(repeat(self), urls_to_request))