diff options
Diffstat (limited to 'commcrawler')
| -rw-r--r-- | commcrawler/scrapy.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index ff8e83a..1280642 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -367,7 +367,9 @@ def launch_crawl(crawl_item, excluded_domains=None): targets = q.values("id") page = 0 page_number = total // NUMBER_PER_PAGE - while page <= page_number and not (ONLY_FIRST_PAGE and page): + has_url_to_process = True + while page <= page_number and not (ONLY_FIRST_PAGE and page) and \ + has_url_to_process: process = CrawlerProcess(settings=scrap_settings) idx, delta = 0, 0 current_idx = page * NUMBER_PER_PAGE |