diff options
| author | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-07 03:53:32 +0200 |
|---|---|---|
| committer | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-07 03:55:49 +0200 |
| commit | d737e04553f464966f54739ba37f9f06dab44586 (patch) | |
| tree | 2b68891ff1629b55b820312fdd3a17ce91ac5722 /commonnet/scrapy_setting.py | |
| parent | 9fbd94f70d4b819b45eef720425242c0d69b032d (diff) | |
| download | Comm-on-net-d737e04553f464966f54739ba37f9f06dab44586.tar.bz2 Comm-on-net-d737e04553f464966f54739ba37f9f06dab44586.zip | |
Save crawling results in the database
Diffstat (limited to 'commonnet/scrapy_setting.py')
| -rw-r--r-- | commonnet/scrapy_setting.py | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/commonnet/scrapy_setting.py b/commonnet/scrapy_setting.py index 2d2b7b7..0d0f15b 100644 --- a/commonnet/scrapy_setting.py +++ b/commonnet/scrapy_setting.py @@ -85,5 +85,8 @@ SCRAPPY_SETTINGS = { #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' "COOKIES_ENABLED": False, + "ITEM_PIPELINES": { + 'commcrawler.scrapy.DbPipeline': 300, + } } |