diff options
Diffstat (limited to 'commcrawler/scrapy.py')
-rw-r--r-- | commcrawler/scrapy.py | 7 |
1 files changed, 7 insertions, 0 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index d24c3c2..1fabafe 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -298,6 +298,10 @@ def create_spider(name, urls, crawl, target, excluded_domains=None): def launch_crawl(crawl_item, excluded_domains=None): scrap_settings = settings.SCRAPPY_SETTINGS.copy() process = CrawlerProcess(settings=scrap_settings) + crawl_item.started = timezone.now() + crawl_item.ended = None + crawl_item.status = "P" + crawl_item.save() for target in crawl_item.targets.all(): process.crawl( create_spider( @@ -308,3 +312,6 @@ def launch_crawl(crawl_item, excluded_domains=None): ) ) process.start() + crawl_item.ended = timezone.now() + crawl_item.status = "F" + crawl_item.save() |