From 108b5514fe795e3bbf4c76245047f5ea054c3d20 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Wed, 31 Jul 2019 17:56:53 +0200 Subject: Basic crawling --- commonnet/scrapy_setting.py | 1 + 1 file changed, 1 insertion(+) (limited to 'commonnet') diff --git a/commonnet/scrapy_setting.py b/commonnet/scrapy_setting.py index 6330705..2d2b7b7 100644 --- a/commonnet/scrapy_setting.py +++ b/commonnet/scrapy_setting.py @@ -10,6 +10,7 @@ SCRAPPY_SETTINGS = { # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html # https://doc.scrapy.org/en/latest/topics/spider-middleware.html "BOT_NAME": 'commonnet-scraper', + "DEPTH_LIMIT": 5, # Obey robots.txt rules "ROBOTSTXT_OBEY": True, -- cgit v1.2.3