diff options
Diffstat (limited to 'commcrawler/scrapy.py')
| -rw-r--r-- | commcrawler/scrapy.py | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index 40fc3b7..bee5ca4 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -348,6 +348,7 @@ def launch_crawl(crawl_item, excluded_domains=None): "target_id": target.pk, } response, verify_ssl = None, True + url = target.url while response is None: try: response = requests.get(target.url, verify=verify_ssl, |