From e8068395d642fa36d7f6c53fe8088beabe7c2a31 Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 9 Aug 2019 16:09:58 +0200 Subject: Display pre-crawl progression --- commcrawler/scrapy.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'commcrawler/scrapy.py') diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index a430f0e..39e3a3e 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -324,10 +324,15 @@ def launch_crawl(crawl_item, excluded_domains=None): scrap_settings = settings.SCRAPPY_SETTINGS.copy() process = CrawlerProcess(settings=scrap_settings) crawl_item.started = timezone.now() + crawl_item.pre_crawl_ended = None + crawl_item.crawl_ended = None crawl_item.ended = None - crawl_item.status = "P" + crawl_item.progression = 0 + crawl_item.status = "W" crawl_item.save() for target in crawl_item.targets.all(): + crawl_item.progression += 1 + crawl_item.save() result_dct = { "crawl_id": crawl_item.pk, "target_id": target.pk, @@ -370,6 +375,9 @@ def launch_crawl(crawl_item, excluded_domains=None): redirect ) ) + crawl_item.pre_crawl_ended = timezone.now() + crawl_item.status = "P" + crawl_item.save() process.start() crawl_item.crawl_ended = timezone.now() crawl_item.status = "M" -- cgit v1.2.3