diff options
| author | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-20 12:40:12 +0200 |
|---|---|---|
| committer | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-20 12:40:12 +0200 |
| commit | d9c27b5bacc00fd5f0c6012c47688d5ff1722a29 (patch) | |
| tree | 9de4fed28455ea928459f90f78f415621e554ee2 | |
| parent | a465ac1fa4271e26bdc659aff32bb9ebeaae7922 (diff) | |
| download | Comm-on-net-d9c27b5bacc00fd5f0c6012c47688d5ff1722a29.tar.bz2 Comm-on-net-d9c27b5bacc00fd5f0c6012c47688d5ff1722a29.zip | |
Always stop after crawl
| -rw-r--r-- | commcrawler/scrapy.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index 6af1c0b..e69268b 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -432,7 +432,8 @@ def launch_crawl(crawl_item, excluded_domains=None): ) ) if has_url_to_process: - process.start(stop_after_crawl=ONLY_FIRST_PAGE) + # process.start(stop_after_crawl=ONLY_FIRST_PAGE) + process.start() page += 1 crawl_item.crawl_ended = timezone.now() if ONLY_FIRST_PAGE and page <= page_number: |