summaryrefslogtreecommitdiff
path: root/commcrawler/scrapy.py
diff options
context:
space:
mode:
Diffstat (limited to 'commcrawler/scrapy.py')
-rw-r--r--commcrawler/scrapy.py7
1 files changed, 7 insertions, 0 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py
index d24c3c2..1fabafe 100644
--- a/commcrawler/scrapy.py
+++ b/commcrawler/scrapy.py
@@ -298,6 +298,10 @@ def create_spider(name, urls, crawl, target, excluded_domains=None):
def launch_crawl(crawl_item, excluded_domains=None):
scrap_settings = settings.SCRAPPY_SETTINGS.copy()
process = CrawlerProcess(settings=scrap_settings)
+ crawl_item.started = timezone.now()
+ crawl_item.ended = None
+ crawl_item.status = "P"
+ crawl_item.save()
for target in crawl_item.targets.all():
process.crawl(
create_spider(
@@ -308,3 +312,6 @@ def launch_crawl(crawl_item, excluded_domains=None):
)
)
process.start()
+ crawl_item.ended = timezone.now()
+ crawl_item.status = "F"
+ crawl_item.save()