simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-15 23:59:05 +00:00
Code Issues Projects Releases Wiki Activity

Added reply to comments option, fixed some bugs

Browse Source
This commit is contained in:
Simon
2018-06-02 17:26:15 -04:00
parent bb872a9248
commit 270ab1335a
8 changed files with 200 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy_od_database/settings.py
View File

@@ -24,15 +24,15 @@ ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 40
RETRY_TIMES = 5
DOWNLOAD_TIMEOUT = 50
RETRY_TIMES = 6
DOWNLOAD_TIMEOUT = 90
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
CONCURRENT_REQUESTS_PER_DOMAIN = 50
CONCURRENT_REQUESTS_PER_DOMAIN = 40
# CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)