From f1b5889f9cf6d654175d548a67a5e415b490b182 Mon Sep 17 00:00:00 2001
From: Étienne Loks <etienne.loks@iggdrasil.net>
Date: Mon, 12 Aug 2019 17:05:59 +0200
Subject: Only first page

---
 commcrawler/scrapy.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py
index bc17225..f4be032 100644
--- a/commcrawler/scrapy.py
+++ b/commcrawler/scrapy.py
@@ -315,6 +315,10 @@ def update_db_result(result_dct, values):
     result.save()
 
 
+NUMBER_PER_PAGE = 250
+ONLY_FIRST_PAGE = True
+
+
 def launch_crawl(crawl_item, excluded_domains=None):
     scrap_settings = settings.SCRAPPY_SETTINGS.copy()
     crawl_item.started = timezone.now()
@@ -328,10 +332,9 @@ def launch_crawl(crawl_item, excluded_domains=None):
     # slice
     total = q.count()
     targets = q.values("id")
-    NUMBER_PER_PAGE = 250
     page = 0
     page_number = total // NUMBER_PER_PAGE
-    while page <= page_number:
+    while page <= page_number and not (ONLY_FIRST_PAGE and page):
         process = CrawlerProcess(settings=scrap_settings)
         idx = 0
         current_idx = page * NUMBER_PER_PAGE
-- 
cgit v1.2.3