Disable cookies, added download delay
This commit is contained in:
parent
0655c01cc9
commit
214c351efc
|
@ -25,13 +25,13 @@ ROBOTSTXT_OBEY = False
|
||||||
# Configure a delay for requests for the same website (default: 0)
|
# Configure a delay for requests for the same website (default: 0)
|
||||||
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
|
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
|
||||||
# See also autothrottle settings and docs
|
# See also autothrottle settings and docs
|
||||||
# DOWNLOAD_DELAY = 3
|
DOWNLOAD_DELAY = 2 # Beetween 1 and 3 seconds : RANDOMIZE_DOWNLOAD_DELAY enabled by default
|
||||||
# The download delay setting will honor only one of:
|
# The download delay setting will honor only one of:
|
||||||
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
||||||
# CONCURRENT_REQUESTS_PER_IP = 16
|
# CONCURRENT_REQUESTS_PER_IP = 16
|
||||||
|
|
||||||
# Disable cookies (enabled by default)
|
# Disable cookies (enabled by default)
|
||||||
# COOKIES_ENABLED = False
|
COOKIES_ENABLED = False # Some sites may use cookies to spot bot behaviour
|
||||||
|
|
||||||
# Disable Telnet Console (enabled by default)
|
# Disable Telnet Console (enabled by default)
|
||||||
# TELNETCONSOLE_ENABLED = False
|
# TELNETCONSOLE_ENABLED = False
|
||||||
|
|
Loading…
Reference in New Issue