SecT0uch
/
ImmoScrap
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

leboncoin: St-Etienne added

This commit is contained in:
Jordan ERNST 2020-08-02 20:32:37 +02:00
parent ccf5ddda5b
commit 7e58f445a5
1 changed files with 20 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
ImmoScrap/spiders/leboncoin.py
View File

@ -5,6 +5,12 @@ class LeboncoinSpider(scrapy.Spider):
name = "leboncoin"
apiurl = 'https://api.leboncoin.fr/api/adfinder/v1/search'
areas = (
{"lat": 45.521971, "lng": 4.869926, "radius": 1000}, # Vienne
{"lat": 45.437621, "lng": 4.388003, "radius": 1000} # Saint-Etienne
)
filters = {
"category": {"id": "9"}, # 9 : buy
"enums": {
@ -14,7 +20,6 @@ class LeboncoinSpider(scrapy.Spider):
},
"ranges": {"price": {"max": 300000}},
"location": {
"area": {"lat": 45.521971, "lng": 4.869926, "radius": 1000},
"city_zipcodes": [],
"departments": [],
"disable_region": False,
@ -24,16 +29,6 @@ class LeboncoinSpider(scrapy.Spider):
"keywords": {}
}
data = {
"pivot": "0,0,0", # page cursor
"limit": 100, # number of results par page (100 is server-side max)
"limit_alu": 1, # 0 to return only statistics, 1 to also return listings
"offset": 0,
"filters": filters,
"sort_by": "time",
"sort_order": "desc"
}
headers = {
"User-Agent": "LBC;Android;6.0;Android SDK built for x86;phone;616a1ca77ca70180;wwan;4.30.4.0;70400;3",
"Accept-Encoding": "gzip, deflate",
@ -44,7 +39,20 @@ class LeboncoinSpider(scrapy.Spider):
def start_requests(self):
url = self.apiurl
yield scrapy.http.JsonRequest(url=url, headers=self.headers, data=self.data, callback=self.parse)
for area in self.areas:
filters = self.filters.copy()
filters['location']['area'] = area
self.data = {
"pivot": "0,0,0", # page cursor
"limit": 100, # number of results par page (100 is server-side max)
"limit_alu": 1, # 0 to return only statistics, 1 to also return listings
"offset": 0,
"filters": filters,
"sort_by": "time",
"sort_order": "desc"
}
yield scrapy.http.JsonRequest(url=url, headers=self.headers, data=self.data, callback=self.parse)
def parse(self, response):
json = response.json()