import scrapy from scrapy.crawler import CrawlerProcess from django.conf import settings class DefaultSpider: pass def create_spider(name, urls, target=None): return type( name, (scrapy.Spider, DefaultSpider), {"name": name, "start_urls": urls, "target": target} ) def crawl(crawl_item): process = CrawlerProcess(settings=settings.SCRAPPY_SETTINGS) for target in crawl_item.targets.all(): process.crawl( create_spider("Target{}".format(target.pk), [target.url], target) ) process.start()