From 6cc1a867e443406818ee30b02961ccc5a340f958 Mon Sep 17 00:00:00 2001
From: Étienne Loks <etienne.loks@iggdrasil.net>
Date: Wed, 21 Aug 2019 19:50:35 +0200
Subject: Scrap: process post-process

---
 commcrawler/scrapy.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py
index ff8e83a..1280642 100644
--- a/commcrawler/scrapy.py
+++ b/commcrawler/scrapy.py
@@ -367,7 +367,9 @@ def launch_crawl(crawl_item, excluded_domains=None):
     targets = q.values("id")
     page = 0
     page_number = total // NUMBER_PER_PAGE
-    while page <= page_number and not (ONLY_FIRST_PAGE and page):
+    has_url_to_process = True
+    while page <= page_number and not (ONLY_FIRST_PAGE and page) and \
+            has_url_to_process:
         process = CrawlerProcess(settings=scrap_settings)
         idx, delta = 0, 0
         current_idx = page * NUMBER_PER_PAGE
-- 
cgit v1.2.3