diff options
| author | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-12 15:54:35 +0200 |
|---|---|---|
| committer | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-12 15:54:35 +0200 |
| commit | c9e95983bccccec0d7c9b901078d11494eacbe32 (patch) | |
| tree | 998ecff935f03f787824bdc15f23d677458c05c4 /commcrawler/scrapy.py | |
| parent | d4a1b958fb7635d8b6dfde15cbf57207881cfa6f (diff) | |
| download | Comm-on-net-c9e95983bccccec0d7c9b901078d11494eacbe32.tar.bz2 Comm-on-net-c9e95983bccccec0d7c9b901078d11494eacbe32.zip | |
Fix timeout check
Diffstat (limited to 'commcrawler/scrapy.py')
| -rw-r--r-- | commcrawler/scrapy.py | 1 |
1 files changed, 0 insertions, 1 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index bee5ca4..8c0d48b 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -156,7 +156,6 @@ class DefaultSpider: self.links_reached.add(url) is_file = self._parse_internal_files(url, result) if not is_file and \ - not self.is_timeout and \ not self.timeout() and ( not MAX_LINKS or len(self.links_reached) < MAX_LINKS): |