From a465ac1fa4271e26bdc659aff32bb9ebeaae7922 Mon Sep 17 00:00:00 2001
From: Étienne Loks <etienne.loks@iggdrasil.net>
Date: Tue, 20 Aug 2019 11:52:02 +0200
Subject: Slice crawls - add a crontab example

---
 commcrawler/scrapy.py | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'commcrawler/scrapy.py')

diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py
index 490142c..6af1c0b 100644
--- a/commcrawler/scrapy.py
+++ b/commcrawler/scrapy.py
@@ -435,6 +435,11 @@ def launch_crawl(crawl_item, excluded_domains=None):
             process.start(stop_after_crawl=ONLY_FIRST_PAGE)
         page += 1
     crawl_item.crawl_ended = timezone.now()
+    if ONLY_FIRST_PAGE and page <= page_number:
+        crawl_item.status = "A"
+        crawl_item.save()
+        return
+
     crawl_item.status = "M"
     crawl_item.save()
     launch_match(crawl_item)
-- 
cgit v1.2.3