diff options
Diffstat (limited to 'commonnet/scrapy_setting.py')
| -rw-r--r-- | commonnet/scrapy_setting.py | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/commonnet/scrapy_setting.py b/commonnet/scrapy_setting.py index 6330705..2d2b7b7 100644 --- a/commonnet/scrapy_setting.py +++ b/commonnet/scrapy_setting.py @@ -10,6 +10,7 @@ SCRAPPY_SETTINGS = { # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html # https://doc.scrapy.org/en/latest/topics/spider-middleware.html "BOT_NAME": 'commonnet-scraper', + "DEPTH_LIMIT": 5, # Obey robots.txt rules "ROBOTSTXT_OBEY": True, |