diff options
| author | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-09 17:06:00 +0200 |
|---|---|---|
| committer | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-09 17:06:00 +0200 |
| commit | f7fe0a457eb92611731ba93959f3fca0ceb16528 (patch) | |
| tree | 9348d132887ed0791aec44b8b65e9d97ed50e960 | |
| parent | 176917b75c3a71e9ecf955b9fdb6f9d8b1a47c7c (diff) | |
| download | Comm-on-net-f7fe0a457eb92611731ba93959f3fca0ceb16528.tar.bz2 Comm-on-net-f7fe0a457eb92611731ba93959f3fca0ceb16528.zip | |
Add a timeout to requests.get
| -rw-r--r-- | commcrawler/scrapy.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index 9ff25c9..7147949 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -338,7 +338,8 @@ def launch_crawl(crawl_item, excluded_domains=None): response, verify_ssl = None, True while response is None: try: - response = requests.get(target.url, verify=verify_ssl) + response = requests.get(target.url, verify=verify_ssl, + timeout=20) except requests.exceptions.SSLError: if not verify_ssl: # new error on SSL response = "Try..." # scrapy is more permissive |