diff options
| -rw-r--r-- | commcrawler/scrapy.py | 1 |
1 files changed, 0 insertions, 1 deletions
diff --git a/commcrawler/scrapy.py b/commcrawler/scrapy.py index bee5ca4..8c0d48b 100644 --- a/commcrawler/scrapy.py +++ b/commcrawler/scrapy.py @@ -156,7 +156,6 @@ class DefaultSpider: self.links_reached.add(url) is_file = self._parse_internal_files(url, result) if not is_file and \ - not self.is_timeout and \ not self.timeout() and ( not MAX_LINKS or len(self.links_reached) < MAX_LINKS): |