From 391cb54e98dc2c1661a4e0ae13739ed710297d02 Mon Sep 17 00:00:00 2001
From: Étienne Loks <etienne.loks@iggdrasil.net>
Date: Mon, 12 Aug 2019 17:40:48 +0200
Subject: Settings

---
 commcrawler/scrapy.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'commcrawler/scrapy.py')

diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py
index 5fbeb43..8c78e61 100644
--- a/commcrawler/scrapy.py
+++ b/commcrawler/scrapy.py
@@ -33,6 +33,9 @@ CALENDAR_KEYS = ["agenda", "calendar"]
 MAX_LINKS = None  # if None no max
 TIMEOUT = datetime.timedelta(minutes=settings.CRAWL_TIMEOUT)
 
+NUMBER_PER_PAGE = settings.NUMBER_PER_SESSION
+ONLY_FIRST_PAGE = True
+
 
 class DefaultSpider:
     name = None
@@ -317,10 +320,6 @@ def update_db_result(result_dct, values):
     result.save()
 
 
-NUMBER_PER_PAGE = 250
-ONLY_FIRST_PAGE = True
-
-
 def launch_crawl(crawl_item, excluded_domains=None):
     scrap_settings = settings.SCRAPPY_SETTINGS.copy()
     crawl_item.started = timezone.now()
-- 
cgit v1.2.3