diff options
Diffstat (limited to 'commcrawler')
| -rw-r--r-- | commcrawler/scrapy.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index 9ff25c9..7147949 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -338,7 +338,8 @@ def launch_crawl(crawl_item, excluded_domains=None): response, verify_ssl = None, True while response is None: try: - response = requests.get(target.url, verify=verify_ssl) + response = requests.get(target.url, verify=verify_ssl, + timeout=20) except requests.exceptions.SSLError: if not verify_ssl: # new error on SSL response = "Try..." # scrapy is more permissive |