diff options
| author | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-09 00:15:26 +0200 |
|---|---|---|
| committer | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-09 00:15:26 +0200 |
| commit | bcacb3dcae815230c106cd773130f7b0ea5f720d (patch) | |
| tree | 3dcb99ff067740d04689f7baa40a58b9289b98d9 /commcrawler/scrapy.py | |
| parent | 2d946cafdf247245bb58c41dbf4e8de699f950ba (diff) | |
| download | Comm-on-net-bcacb3dcae815230c106cd773130f7b0ea5f720d.tar.bz2 Comm-on-net-bcacb3dcae815230c106cd773130f7b0ea5f720d.zip | |
Display progress
Diffstat (limited to 'commcrawler/scrapy.py')
| -rw-r--r-- | commcrawler/scrapy.py | 7 |
1 files changed, 7 insertions, 0 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index d24c3c2..1fabafe 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -298,6 +298,10 @@ def create_spider(name, urls, crawl, target, excluded_domains=None): def launch_crawl(crawl_item, excluded_domains=None): scrap_settings = settings.SCRAPPY_SETTINGS.copy() process = CrawlerProcess(settings=scrap_settings) + crawl_item.started = timezone.now() + crawl_item.ended = None + crawl_item.status = "P" + crawl_item.save() for target in crawl_item.targets.all(): process.crawl( create_spider( @@ -308,3 +312,6 @@ def launch_crawl(crawl_item, excluded_domains=None): ) ) process.start() + crawl_item.ended = timezone.now() + crawl_item.status = "F" + crawl_item.save() |