diff options
| author | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-07-31 17:56:53 +0200 |
|---|---|---|
| committer | Étienne Loks <etienne@peacefrogs.net> | 2019-07-31 17:56:53 +0200 |
| commit | 108b5514fe795e3bbf4c76245047f5ea054c3d20 (patch) | |
| tree | 8bb5ded34e2205583b8cb12101bc3f945252ea1d /commonnet | |
| parent | dd2dd640aa649c715a843fa431621fd955ca6767 (diff) | |
| download | Comm-on-net-108b5514fe795e3bbf4c76245047f5ea054c3d20.tar.bz2 Comm-on-net-108b5514fe795e3bbf4c76245047f5ea054c3d20.zip | |
Basic crawling
Diffstat (limited to 'commonnet')
| -rw-r--r-- | commonnet/scrapy_setting.py | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/commonnet/scrapy_setting.py b/commonnet/scrapy_setting.py index 6330705..2d2b7b7 100644 --- a/commonnet/scrapy_setting.py +++ b/commonnet/scrapy_setting.py @@ -10,6 +10,7 @@ SCRAPPY_SETTINGS = { # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html # https://doc.scrapy.org/en/latest/topics/spider-middleware.html "BOT_NAME": 'commonnet-scraper', + "DEPTH_LIMIT": 5, # Obey robots.txt rules "ROBOTSTXT_OBEY": True, |