path: root/commcrawler/scrapy.py

import datetime
import tldextract
from urllib.parse import urldefrag

import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import NotSupported
from scrapy.linkextractors import LinkExtractor

from django.conf import settings
from django.db import transaction, IntegrityError
from django.utils import timezone

from . import models

"""
redirection
CrawlLink
"""

FACEBOOK_DOMAINS = ("facebook.com", "facebook.net", "fbcdn.net")
TWITTER_DOMAINS = ("twitter.com", "twimg.com", "twttr.net", "twttr.com",
                   "abs.twimg.com")
INSTAGRAM_DOMAINS = ("instagram.com", "cdninstagram.com")
YOUTUBE_DOMAINS = ("youtu.be", "youtube.com")
DAILYMOTION_DOMAINS = ("dailymotion.com",)
VIMEO_DOMAINS = ("vimeo.com",)
VIDEO_EXTS = (".webm", ".mkv", ".flv", ".ogv", ".mov", ".wmv", ".avi", ".mpg",
              ".mp4", ".m4v", ".mp2", ".mpeg")
AUDIO_EXTS = (".aac", ".flac", ".m4a", ".mp3", ".ogg", ".oga", ".opus",
              ".wma", ".webm")
OFFICE_EXTS = (".csv", ".doc", ".docx", ".odt", ".rtf", ".ods", ".xls", ".xlsx")


def clean_url(url):
    url, __ = urldefrag(url)  # remove anchors
    return url


def append_to_results(results, key, value):
    if key not in results:
        results[key] = []
    results[key].append(value)


MAX_LINKS = None  # if None no max
TIMEOUT = datetime.timedelta(minutes=settings.CRAWL_TIMEOUT)


class DefaultSpider:
    name = None
    start_urls = None
    allowed_domains = []
    excluded_domains = []
    crawl_id = None
    target_id = None
    crawl_result = None
    links_reached = set()

    def start_requests(self):
        q = {
            "crawl_id": self.crawl_id,
            "target_id": self.target_id,
            "status__in": ["F", "T"],
        }
        if models.CrawlResult.objects.filter(**q).count():
            return []
        q.pop("status__in")
        if models.CrawlResult.objects.filter(**q).count():
            # delete a previous interrupted attempt
            res = models.CrawlResult.objects.get(**q)
            res.delete()

        for url in self.start_urls:
            yield scrapy.Request(url, self.parse)

    def _parse_image(self, response, result):
        if "images" not in result:
            result["images"] = []
        for img in response.css('img'):
            attributes = img.attrib
            if "src" not in attributes:
                continue
            src = attributes["src"]
            is_a_real_src = src.startswith("http") or src.startswith("/")
            if not src or not is_a_real_src or src in result["images"]:
                continue
            result["images"].append(src)

    def _parse_iframe(self, response, result):
        for img in response.css('iframe'):
            attributes = img.attrib
            if "src" not in attributes:
                continue
            src = attributes["src"]
            is_a_real_src = src.startswith("http") or src.startswith("/")
            if not src or not is_a_real_src:
                continue
            current_domain = get_domain(src)
            if current_domain in YOUTUBE_DOMAINS:
                append_to_results(result, "youtube", src)
            elif current_domain in DAILYMOTION_DOMAINS:
                append_to_results(result, "dailymotion", src)
            elif current_domain in VIMEO_DOMAINS:
                append_to_results(result, "vimeo", src)

    def _parse_internal_files(self, url, result):
        types = (("video", VIDEO_EXTS), ("audio", AUDIO_EXTS),
                 ("internal_pdf", ("pdf",)), ("internal_office", OFFICE_EXTS))
        return self._parse_files(url, result, types)

    def _parse_external_files(self, url, result):
        types = (("external_pdf", ("pdf",)), ("external_office", OFFICE_EXTS))
        return self._parse_files(url, result, types)

    def _parse_files(self, url, result, types):
        """
        Parse url for file
        :return: True if is a file
        """
        url = url.lower()
        for content_type, extensions in types:
            if [1 for ext in extensions if url.endswith(ext)]:
                append_to_results(result, content_type, url)
                return True

    def timeout(self):
        if not self.crawl_result:
            q = {
                "crawl_id": self.crawl_id,
                "target_id": self.target_id,
            }
            if not models.CrawlResult.objects.filter(**q).count():
                return
            self.crawl_result = models.CrawlResult.objects.get(**q)
        duration = timezone.now() - self.crawl_result.started
        if duration < TIMEOUT:
            return
        with transaction.atomic():
            result = models.CrawlResult.objects.select_for_update().get(
                pk=self.crawl_result.pk)
            result.status = "T"
            result.save()
        return True

    def parse(self, response):
        result = {
            "url": response.url,
        }
        if self.timeout():
            return []
        for domain in self.excluded_domains:
            if domain in response.url:
                result["is_online"] = False
        if result.get("is_online", None) is False:
            yield result
        else:
            result["is_online"] = True
            try:
                self._parse_image(response, result)
                self._parse_iframe(response, result)
                for link in LinkExtractor().extract_links(response):
                    url = clean_url(link.url)
                    if url is None or url in self.links_reached:
                        continue
                    is_internal = False
                    for domain in self.allowed_domains:
                        if domain in url:
                            is_internal = True
                            self.links_reached.add(url)
                            is_file = self._parse_internal_files(url, result)
                            if is_file:
                                pass
                            elif not MAX_LINKS or \
                                    len(self.links_reached) < MAX_LINKS:
                                yield response.follow(link.url, self.parse)
                            else:
                                print("MAX", self.allowed_domains,
                                      self.links_reached)
                    if not is_internal:
                        current_domain = get_domain(url)
                        if current_domain in FACEBOOK_DOMAINS:
                            append_to_results(result, "facebook", url)
                        elif current_domain in TWITTER_DOMAINS:
                            append_to_results(result, "twitter", url)
                        elif current_domain in INSTAGRAM_DOMAINS:
                            append_to_results(result, "instagram", url)
                        else:
                            is_file = self._parse_external_files(url, result)
                            if not is_file:
                                append_to_results(result, "external_link", url)
            except NotSupported:
                print("No response", response.url)
            yield result

    def closed(self, reason):
        DbPipeline().close(self)


class DbPipeline:
    BASE_KEYS = ["url", "crawl_id", "target_id"]
    NB_KEYS = ["external_link", "images",
               "facebook", "twitter", "instagram", "youtube",
               "dailymotion", "vimeo", "video", "audio",
               "internal_pdf", "external_pdf", "internal_office",
               "external_office"]

    def _get_result_pk(self, spider):
        """
        Atomic creation
        :param spider: current spider
        :return: result_pk, created
        """
        pks = {
            "crawl_id": spider.crawl_id,
            "target_id": spider.target_id,
        }
        created = False
        try:
            result = models.CrawlResult.objects.get(**pks)
        except models.CrawlResult.DoesNotExist:
            try:
                with transaction.atomic():
                    result = models.CrawlResult.objects.create(**pks)
                    created = True
            except IntegrityError:
                result = models.CrawlResult.objects.get(**pks)
        return result.pk, created

    def _update(self, result_pk, item, result_created):
        """
        Atomic update
        """
        with transaction.atomic():
            result = models.CrawlResult.objects.select_for_update().get(
                pk=result_pk)
            crawl_result = result.crawl_result
            if crawl_result:
                crawl_result = crawl_result[0]
            else:
                crawl_result = {}
            if "urls" not in crawl_result:
                crawl_result["urls"] = []
            url = item.pop("url")
            if url in crawl_result["urls"]:
                return
            crawl_result["urls"].append(url)
            for k, value in item.items():
                if k == "is_online":
                    if result_created:  # only update on the first link
                        result.is_online = value
                elif k in self.NB_KEYS:
                    if k not in crawl_result:
                        crawl_result[k] = []
                    for subvalue in value:
                        if subvalue in crawl_result[k]:
                            continue
                        crawl_result[k].append(subvalue)
                    setattr(result, "nb_" + k, len(crawl_result[k]))
            result.nb_internal_link = len(crawl_result["urls"]) - 1
            result.crawl_result = [crawl_result]
            result.save()
            return True

    def process_item(self, item, spider):
        result_pk, created = self._get_result_pk(spider)
        self._update(result_pk, item, created)
        return item

    def close(self, spider):
        result_pk, created = self._get_result_pk(spider)
        with transaction.atomic():
            result = models.CrawlResult.objects.select_for_update().get(
                pk=result_pk)
            if result.status == "P":
                result.status = "F"
                result.duration = (timezone.now() - result.started)
                result.save()


def get_domain(url):
    ext = tldextract.extract(url)
    return '{}.{}'.format(ext.domain, ext.suffix)


def create_spider(name, urls, crawl, target, excluded_domains=None):
    if not excluded_domains:
        excluded_domains = []
    return type(
        name, (DefaultSpider, scrapy.Spider),
        {"name": name, "start_urls": urls,
         "allowed_domains": [get_domain(url) for url in urls],
         "crawl_id": crawl.pk, "target_id": target.pk, "links_reached": set(),
         "excluded_domains": excluded_domains}
    )


def launch_crawl(crawl_item, excluded_domains=None):
    scrap_settings = settings.SCRAPPY_SETTINGS.copy()
    process = CrawlerProcess(settings=scrap_settings)
    for target in crawl_item.targets.all():
        process.crawl(
            create_spider(
                "Crawl{}Target{}".format(crawl_item.pk, target.pk),
                [target.url],
                crawl_item, target,
                excluded_domains
            )
        )
    process.start()