diff options
author | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-07-31 17:56:53 +0200 |
---|---|---|
committer | Étienne Loks <etienne@peacefrogs.net> | 2019-07-31 17:56:53 +0200 |
commit | 108b5514fe795e3bbf4c76245047f5ea054c3d20 (patch) | |
tree | 8bb5ded34e2205583b8cb12101bc3f945252ea1d /commcrawler/management/commands/launch_crawl.py | |
parent | dd2dd640aa649c715a843fa431621fd955ca6767 (diff) | |
download | Comm-on-net-108b5514fe795e3bbf4c76245047f5ea054c3d20.tar.bz2 Comm-on-net-108b5514fe795e3bbf4c76245047f5ea054c3d20.zip |
Basic crawling
Diffstat (limited to 'commcrawler/management/commands/launch_crawl.py')
-rw-r--r-- | commcrawler/management/commands/launch_crawl.py | 39 |
1 files changed, 39 insertions, 0 deletions
diff --git a/commcrawler/management/commands/launch_crawl.py b/commcrawler/management/commands/launch_crawl.py new file mode 100644 index 0000000..050a54d --- /dev/null +++ b/commcrawler/management/commands/launch_crawl.py @@ -0,0 +1,39 @@ +import csv +import sys + +from django.core.management.base import BaseCommand + +from commcrawler.models import Crawl +from commcrawler.scrapy import launch_crawl + + +class Command(BaseCommand): + help = 'Launch a crawl' + + def add_arguments(self, parser): + parser.add_argument( + '--quiet', dest='quiet', action='store_true', + help='Quiet output') + + def handle(self, *args, **options): + quiet = options['quiet'] + q = Crawl.objects.filter(status="C") + if not q.count(): + sys.stdout.write('No crawl waiting. Exit.\n') + return + + crawls = dict([(c.pk, c) for c in q.all()]) + available_ids = crawls.keys() + c_id = None + while c_id not in available_ids: + sys.stdout.write('Which crawl to launch (type the number):\n') + for crawl_id, crawl in crawls.items(): + sys.stdout.write('* {} - {}\n'.format(crawl_id, crawl)) + sys.stdout.flush() + try: + c_id = int(input("# ")) + except ValueError: + c_id = None + current_crawl = crawls[c_id] + launch_crawl(current_crawl) + |