Compare commits
2 Commits
1e18327a9e
...
3771e4d507
Author | SHA1 | Date |
---|---|---|
Jordan ERNST | 3771e4d507 | |
Jordan ERNST | d42cc51088 |
|
@ -0,0 +1,13 @@
|
|||
#!/usr/bin/env ipython3
|
||||
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
|
||||
process = CrawlerProcess(get_project_settings())
|
||||
|
||||
# Will crawl simultaneously:
|
||||
process.crawl('leboncoin')
|
||||
process.crawl('seloger')
|
||||
process.crawl('pap')
|
||||
process.start() # the script will block here until the crawling is finished
|
|
@ -56,9 +56,9 @@ ROBOTSTXT_OBEY = False
|
|||
|
||||
# Enable or disable extensions
|
||||
# See https://docs.scrapy.org/en/latest/topics/extensions.html
|
||||
#EXTENSIONS = {
|
||||
# 'scrapy.extensions.telnet.TelnetConsole': None,
|
||||
#}
|
||||
EXTENSIONS = {
|
||||
'scrapy.extensions.telnet.TelnetConsole': None
|
||||
}
|
||||
|
||||
# Configure item pipelines
|
||||
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
|
|
Loading…
Reference in New Issue