From bcacb3dcae815230c106cd773130f7b0ea5f720d Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 9 Aug 2019 00:15:26 +0200 Subject: Display progress --- commcrawler/scrapy.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'commcrawler/scrapy.py') diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index d24c3c2..1fabafe 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -298,6 +298,10 @@ def create_spider(name, urls, crawl, target, excluded_domains=None): def launch_crawl(crawl_item, excluded_domains=None): scrap_settings = settings.SCRAPPY_SETTINGS.copy() process = CrawlerProcess(settings=scrap_settings) + crawl_item.started = timezone.now() + crawl_item.ended = None + crawl_item.status = "P" + crawl_item.save() for target in crawl_item.targets.all(): process.crawl( create_spider( @@ -308,3 +312,6 @@ def launch_crawl(crawl_item, excluded_domains=None): ) ) process.start() + crawl_item.ended = timezone.now() + crawl_item.status = "F" + crawl_item.save() -- cgit v1.2.3