diff options
Diffstat (limited to 'commcrawler/scrapy.py')
| -rw-r--r-- | commcrawler/scrapy.py | 10 |
1 files changed, 9 insertions, 1 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index a430f0e..39e3a3e 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -324,10 +324,15 @@ def launch_crawl(crawl_item, excluded_domains=None): scrap_settings = settings.SCRAPPY_SETTINGS.copy() process = CrawlerProcess(settings=scrap_settings) crawl_item.started = timezone.now() + crawl_item.pre_crawl_ended = None + crawl_item.crawl_ended = None crawl_item.ended = None - crawl_item.status = "P" + crawl_item.progression = 0 + crawl_item.status = "W" crawl_item.save() for target in crawl_item.targets.all(): + crawl_item.progression += 1 + crawl_item.save() result_dct = { "crawl_id": crawl_item.pk, "target_id": target.pk, @@ -370,6 +375,9 @@ def launch_crawl(crawl_item, excluded_domains=None): redirect ) ) + crawl_item.pre_crawl_ended = timezone.now() + crawl_item.status = "P" + crawl_item.save() process.start() crawl_item.crawl_ended = timezone.now() crawl_item.status = "M" |