diff options
| -rw-r--r-- | commcrawler/scrapy.py | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index bc17225..f4be032 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -315,6 +315,10 @@ def update_db_result(result_dct, values): result.save() +NUMBER_PER_PAGE = 250 +ONLY_FIRST_PAGE = True + + def launch_crawl(crawl_item, excluded_domains=None): scrap_settings = settings.SCRAPPY_SETTINGS.copy() crawl_item.started = timezone.now() @@ -328,10 +332,9 @@ def launch_crawl(crawl_item, excluded_domains=None): # slice total = q.count() targets = q.values("id") - NUMBER_PER_PAGE = 250 page = 0 page_number = total // NUMBER_PER_PAGE - while page <= page_number: + while page <= page_number and not (ONLY_FIRST_PAGE and page): process = CrawlerProcess(settings=scrap_settings) idx = 0 current_idx = page * NUMBER_PER_PAGE |