summaryrefslogtreecommitdiff
path: root/commcrawler
diff options
context:
space:
mode:
Diffstat (limited to 'commcrawler')
-rw-r--r--commcrawler/__init__.py1
-rw-r--r--commcrawler/admin.py124
-rw-r--r--commcrawler/apps.py7
-rw-r--r--commcrawler/lookups.py49
-rw-r--r--commcrawler/management/__init__.py0
-rw-r--r--commcrawler/management/commands/__init__.py0
-rw-r--r--commcrawler/management/commands/import_csv_autres.py107
-rw-r--r--commcrawler/management/commands/import_csv_communes.py167
-rw-r--r--commcrawler/migrations/0001_initial.py121
-rw-r--r--commcrawler/migrations/__init__.py0
-rw-r--r--commcrawler/models.py123
-rw-r--r--commcrawler/templates/admin/add_to_crawl.html16
-rw-r--r--commcrawler/tests.py3
-rw-r--r--commcrawler/views.py3
14 files changed, 721 insertions, 0 deletions
diff --git a/commcrawler/__init__.py b/commcrawler/__init__.py
new file mode 100644
index 0000000..339b125
--- /dev/null
+++ b/commcrawler/__init__.py
@@ -0,0 +1 @@
+default_app_config = 'commonnet.apps.CommCrawlerConfig'
diff --git a/commcrawler/admin.py b/commcrawler/admin.py
new file mode 100644
index 0000000..f4df5e2
--- /dev/null
+++ b/commcrawler/admin.py
@@ -0,0 +1,124 @@
+from ajax_select import make_ajax_form
+from django.contrib import admin, messages
+from django.contrib.auth.admin import GroupAdmin, UserAdmin
+from django.contrib.auth.models import Group, User
+from django import forms
+from django.http import HttpResponseRedirect
+from django.shortcuts import render
+from django.utils.translation import ugettext_lazy as _
+
+from commonnet.admin_site import admin_site
+from commcrawler import models
+
+
+admin_site.register(User, UserAdmin)
+admin_site.register(Group, GroupAdmin)
+
+
+class AreaTypeAdmin(admin.ModelAdmin):
+ model = models.AreaType
+ list_display = ('name', )
+
+
+admin_site.register(models.AreaType, AreaTypeAdmin)
+
+
+class AreaAdmin(admin.ModelAdmin):
+ model = models.Area
+ list_display = ('name', 'area_type', 'reference', 'parent')
+ list_filter = ('area_type',)
+ search_fields = ['name', 'parent__name']
+
+
+admin_site.register(models.Area, AreaAdmin)
+
+
+class OrganizationTypeAdmin(admin.ModelAdmin):
+ model = models.OrganizationType
+ list_display = ('name', 'parent')
+ list_filter = ('parent',)
+
+
+admin_site.register(models.OrganizationType, OrganizationTypeAdmin)
+
+
+class OrganizationAdmin(admin.ModelAdmin):
+ model = models.Organization
+ list_display = ('name', 'organization_type', 'area')
+ list_filter = ('organization_type',)
+ search_fields = ['name']
+ form = make_ajax_form(model, {'area': 'area'})
+
+
+admin_site.register(models.Organization, OrganizationAdmin)
+
+
+class AddToCrawlForm(forms.Form):
+ crawl = forms.ChoiceField(label=_("Crawl"), choices=tuple())
+
+ def __init__(self, *args, **kwargs):
+ super(AddToCrawlForm, self).__init__(*args, **kwargs)
+ self.fields["crawl"].choices = [(None, "--")] + [
+ (c.pk, str(c))
+ for c in models.Crawl.objects.filter(status="C").all()
+ ]
+
+
+class TargetAdmin(admin.ModelAdmin):
+ list_display = ('name', 'url', 'organization')
+ model = models.Target
+ list_filter = ('organization__organization_type',)
+ form = make_ajax_form(model, {'organization': 'organization'})
+ actions = ['add_to_crawl']
+
+ def add_to_crawl(self, request, queryset):
+ if 'apply' in request.POST:
+ form = AddToCrawlForm(request.POST)
+ if form.is_valid():
+ crawl = None
+ try:
+ crawl = models.Crawl.objects.get(
+ pk=form.cleaned_data["crawl"],
+ status="C"
+ )
+ except models.Crawl.DoesNotExist:
+ pass
+ if crawl:
+ nb_added = 0
+ targets = [
+ c["id"] for c in crawl.targets.values("id")
+ ]
+ for target in queryset.all():
+ if target.pk not in targets:
+ crawl.targets.add(target)
+ nb_added += 1
+ messages.add_message(
+ request, messages.INFO,
+ str(_("{} new targets added to {}")).format(
+ nb_added, str(crawl)
+ )
+ )
+ return HttpResponseRedirect(request.get_full_path())
+ else:
+ form = AddToCrawlForm()
+ return render(
+ request, 'admin/add_to_crawl.html',
+ context={"form": form, "query_nb": queryset.count(),
+ "items": [v['pk'] for v in queryset.values('pk').all()]})
+
+ add_to_crawl.short_description = _("Add to crawl")
+
+
+admin_site.register(models.Target, TargetAdmin)
+
+
+class CrawlAdmin(admin.ModelAdmin):
+ model = models.Crawl
+ list_display = ("name", "status", "target_nb", "created", "started",
+ "ended")
+ list_filter = ("status",)
+ readonly_fields = ("status", "created", "started", "ended")
+ exclude = ("targets", )
+
+
+admin_site.register(models.Crawl, CrawlAdmin)
diff --git a/commcrawler/apps.py b/commcrawler/apps.py
new file mode 100644
index 0000000..79e032d
--- /dev/null
+++ b/commcrawler/apps.py
@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+from django.utils.translation import ugettext_lazy as _
+
+
+class CommCrawlerConfig(AppConfig):
+ name = 'commcrawler'
+ verbose_name = _("Crawler")
diff --git a/commcrawler/lookups.py b/commcrawler/lookups.py
new file mode 100644
index 0000000..f8d4897
--- /dev/null
+++ b/commcrawler/lookups.py
@@ -0,0 +1,49 @@
+from ajax_select import register, LookupChannel as BaseLookupChannel
+
+from django.db.models import Q
+
+from . import models
+
+
+class LookupChannel(BaseLookupChannel):
+ def get_objects(self, items):
+ # TODO: why IDs are not given here? M2M issue
+ ids = []
+ for item in items:
+ if hasattr(item, 'pk'):
+ ids.append(item.pk)
+ else:
+ ids.append(item)
+ return super(LookupChannel, self).get_objects(ids)
+
+ def format_item_display(self, item):
+ return u"<span class='ajax-label'>%s</span>" % str(item)
+
+
+@register('organization')
+class OrganizationLookup(LookupChannel):
+ model = models.Organization
+
+ def get_query(self, q, request):
+ query = Q()
+ for term in q.strip().split(' '):
+ subquery = Q(name__icontains=term)
+ query &= subquery
+ return self.model.objects.filter(query).order_by('name')[:20]
+
+
+@register('area')
+class AreaLookup(LookupChannel):
+ model = models.Area
+
+ def get_query(self, q, request):
+ query = Q()
+ for term in q.strip().split(' '):
+ subquery = (
+ Q(name__icontains=term) |
+ Q(parent__name__icontains=term) |
+ Q(reference=term)
+ )
+ query &= subquery
+ return self.model.objects.filter(query).order_by('name')[:20]
+
diff --git a/commcrawler/management/__init__.py b/commcrawler/management/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commcrawler/management/__init__.py
diff --git a/commcrawler/management/commands/__init__.py b/commcrawler/management/commands/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commcrawler/management/commands/__init__.py
diff --git a/commcrawler/management/commands/import_csv_autres.py b/commcrawler/management/commands/import_csv_autres.py
new file mode 100644
index 0000000..2dd177c
--- /dev/null
+++ b/commcrawler/management/commands/import_csv_autres.py
@@ -0,0 +1,107 @@
+import csv
+import sys
+
+from django.core.management.base import BaseCommand
+
+from commcrawler.models import Organization, OrganizationType, Target
+
+header_len = 1
+expected_header = [
+ 'Secteur', 'Type', 'Nom', 'URL', 'Localisation siège (département)',
+ 'Localisation siège (code postal)', 'Localisation siège (commune)']
+
+
+class Command(BaseCommand):
+ help = 'Import depuis CSV communes'
+
+ def add_arguments(self, parser):
+ parser.add_argument('csv_file')
+ parser.add_argument(
+ '--quiet', dest='quiet', action='store_true',
+ help='Quiet output')
+
+ def handle(self, *args, **options):
+ csv_file = options['csv_file']
+ quiet = options['quiet']
+ if not quiet:
+ sys.stdout.write('* opening file {}\n'.format(csv_file))
+ nb_created = 0
+ nb_organization_created = 0
+ nb_tt_created = 0
+ with open(csv_file, 'r') as csvfile:
+ reader = csv.reader(csvfile)
+ for idx, row in enumerate(reader):
+ if idx < header_len:
+ if not idx:
+ if expected_header != row:
+ sys.stdout.write('ERROR: expected header differs '
+ 'from the one provided\n')
+ sys.stdout.write('* expected header is:\n')
+ sys.stdout.write(str(expected_header))
+ sys.stdout.write('\n* header provided is:\n')
+ sys.stdout.write(str(row) + "\n")
+ return
+ continue
+ sec, sec_tpe, name, site, address = row[0:5]
+ address = address.strip()
+ if " " in address:
+ organization_name = " ".join(address.split(" ")[1:])
+ else:
+ organization_name = address
+
+ if not quiet:
+ sys.stdout.write('-> processing line %d.\r' % (idx + 1))
+ sys.stdout.flush()
+
+ tpe, c = OrganizationType.objects.get_or_create(
+ name=sec.strip(),
+ parent=None
+ )
+ if c:
+ nb_tt_created += 1
+ tpe, c = OrganizationType.objects.get_or_create(
+ name=sec_tpe.strip(),
+ parent=tpe
+ )
+ if c:
+ nb_tt_created += 1
+
+ organization_values = {
+ "organization_type": tpe,
+ "name": "{} - {}".format(name.strip(), organization_name)
+ }
+
+ default = dict()
+ default["address"] = address
+ organization_values["defaults"] = default
+
+ organization, c = Organization.objects.get_or_create(
+ **organization_values)
+ if c:
+ nb_organization_created += 1
+ else:
+ for k in default.keys():
+ setattr(organization, k, default[k])
+ organization.save()
+
+ site = site.strip()
+ if site == "0" or "." not in site:
+ site = None
+ elif not site.startswith('http'):
+ site = "http://" + site
+ values = {
+ "name": name.strip(),
+ "organization": organization,
+ "url": site,
+ }
+ target, created = Target.objects.get_or_create(**values)
+ if created:
+ nb_created += 1
+ if not quiet:
+ sys.stdout.write(
+ '\n* {} organization types created.\n'.format(nb_tt_created))
+ sys.stdout.write(
+ '* {} organizations created.\n'.format(nb_organization_created))
+ sys.stdout.write(
+ '* {} targets created.\n'.format(nb_created))
+ sys.stdout.flush()
diff --git a/commcrawler/management/commands/import_csv_communes.py b/commcrawler/management/commands/import_csv_communes.py
new file mode 100644
index 0000000..4024067
--- /dev/null
+++ b/commcrawler/management/commands/import_csv_communes.py
@@ -0,0 +1,167 @@
+import csv
+import sys
+
+from django.core.management.base import BaseCommand
+
+from commcrawler.models import Area, AreaType, Organization, OrganizationType,\
+ Target
+
+header_len = 1
+expected_header = [
+ 'DÉPARTEMENT', 'NOM EPCI OU MAIRIE', 'CODE POSTAL', 'COMMUNE',
+ 'C.C. / C.A. / C.U.', 'SITE INTERNET', 'POPULATION COMMUNAUTAIRE',
+ 'Type de collectivité', 'Code INSEE']
+
+
+class Command(BaseCommand):
+ help = 'Import depuis CSV communes'
+
+ def add_arguments(self, parser):
+ parser.add_argument('csv_file')
+ parser.add_argument(
+ '--quiet', dest='quiet', action='store_true',
+ help='Quiet output')
+
+ def handle(self, *args, **options):
+ csv_file = options['csv_file']
+ quiet = options['quiet']
+ if not quiet:
+ sys.stdout.write('* opening file {}\n'.format(csv_file))
+ nb_created = 0
+ nb_organization_created = 0
+ nb_area_created = 0
+ nb_tt_created = 0
+ nb_at_created = 0
+ with open(csv_file, 'r') as csvfile:
+ reader = csv.reader(csvfile)
+ for idx, row in enumerate(reader):
+ if idx < header_len:
+ if not idx:
+ if expected_header != row:
+ sys.stdout.write('ERROR: expected header differs '
+ 'from the one provided\n')
+ sys.stdout.write('* expected header is:\n')
+ sys.stdout.write(str(expected_header))
+ sys.stdout.write('\n* header provided is:\n')
+ sys.stdout.write(str(row) + "\n")
+ return
+ continue
+ dpt, name, code_postal, commune, comcom, site, pop = row[0:7]
+ type_coll, insee = row[7:9]
+ if insee.strip() == "NA":
+ insee = ""
+ if not quiet:
+ sys.stdout.write('-> processing line %d.\r' % (idx + 1))
+ sys.stdout.flush()
+ try:
+ pop = int(pop.replace(" ", ""))
+ except ValueError:
+ pop = None
+
+ p_tpe, c = OrganizationType.objects.get_or_create(
+ name=type_coll.strip()
+ )
+ if c:
+ nb_tt_created += 1
+ if comcom.strip():
+ tpe, c = OrganizationType.objects.get_or_create(
+ name=comcom.strip(),
+ parent=p_tpe
+ )
+ if c:
+ nb_tt_created += 1
+ else:
+ tpe = p_tpe
+ atpe, c = AreaType.objects.get_or_create(
+ name=str(tpe)
+ )
+ if c:
+ nb_at_created += 1
+
+ top_area = None
+ if dpt.strip():
+ dpt_tpe, c = AreaType.objects.get_or_create(
+ name="Département",
+ )
+ if c:
+ nb_tt_created += 1
+ top_area, c = Area.objects.get_or_create(
+ name=dpt.strip(),
+ area_type=dpt_tpe
+ )
+ if c:
+ nb_area_created += 1
+
+ area_values = {"area_type": atpe}
+ area_defaults = {"population": pop, "parent": top_area}
+ if insee.strip():
+ area_values['reference'] = insee.strip()
+ area_defaults['name'] = commune.strip()
+ else:
+ area_values['name'] = name.strip()
+ area_values["defaults"] = area_defaults
+
+ area, c = Area.objects.get_or_create(
+ **area_values)
+ if c:
+ nb_area_created += 1
+ else:
+ for k in area_defaults:
+ setattr(area, k, area_defaults[k])
+ area.save()
+
+ organization_values = {
+ "organization_type": tpe,
+ "name": name,
+ "area": area,
+ }
+ default = dict()
+ default["address"] = "{} {}".format(code_postal.strip() or "",
+ commune.strip() or "")
+ organization_values["defaults"] = default
+
+ organization, c = Organization.objects.get_or_create(
+ **organization_values)
+ if c:
+ nb_organization_created += 1
+ else:
+ for k in default:
+ setattr(organization, k, default[k])
+ organization.save()
+
+ site = site.strip()
+ if site == "0" or "." not in site:
+ site = None
+ elif not site.startswith('http'):
+ site = "http://" + site
+ values = {
+ "name": name.strip(),
+ "organization": organization,
+ "url": site,
+ }
+
+ target, created = Target.objects.get_or_create(**values)
+ if created:
+ nb_created += 1
+ if not quiet:
+ sys.stdout.write("\n")
+ if nb_at_created:
+ sys.stdout.write(
+ '* {} area types created.\n'.format(nb_at_created)
+ )
+ if nb_area_created:
+ sys.stdout.write(
+ '* {} areas created.\n'.format(
+ nb_area_created))
+ if nb_tt_created:
+ sys.stdout.write(
+ '* {} organization types created.\n'.format(nb_tt_created)
+ )
+ if nb_organization_created:
+ sys.stdout.write(
+ '* {} organizations created.\n'.format(
+ nb_organization_created))
+ if nb_created:
+ sys.stdout.write(
+ '* {} targets created.\n'.format(nb_created))
+ sys.stdout.flush()
diff --git a/commcrawler/migrations/0001_initial.py b/commcrawler/migrations/0001_initial.py
new file mode 100644
index 0000000..2fbb9fe
--- /dev/null
+++ b/commcrawler/migrations/0001_initial.py
@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2019-07-30 15:47
+from __future__ import unicode_literals
+
+import datetime
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='Area',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=200, verbose_name='Name')),
+ ('population', models.IntegerField(blank=True, null=True, verbose_name='Population')),
+ ('reference', models.CharField(blank=True, help_text='For instance, INSEE code for towns', max_length=100, null=True, verbose_name='Reference')),
+ ],
+ options={
+ 'verbose_name_plural': 'Areas',
+ 'ordering': ('name',),
+ 'verbose_name': 'Area',
+ },
+ ),
+ migrations.CreateModel(
+ name='AreaType',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=200, verbose_name='Name')),
+ ],
+ options={
+ 'verbose_name_plural': 'Area types',
+ 'ordering': ('name',),
+ 'verbose_name': 'Area type',
+ },
+ ),
+ migrations.CreateModel(
+ name='Crawl',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=200, verbose_name='Name')),
+ ('created', models.DateTimeField(default=datetime.datetime.now, verbose_name='Creation date')),
+ ('started', models.DateTimeField(blank=True, null=True, verbose_name='Start date')),
+ ('ended', models.DateTimeField(blank=True, null=True, verbose_name='End date')),
+ ('status', models.CharField(choices=[('C', 'Created'), ('P', 'In progress'), ('F', 'Finished')], default='C', max_length=1)),
+ ],
+ options={
+ 'verbose_name_plural': 'Crawls',
+ 'ordering': ('created', 'name'),
+ 'verbose_name': 'Crawl',
+ },
+ ),
+ migrations.CreateModel(
+ name='Organization',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=200, verbose_name='Name')),
+ ('address', models.TextField(blank=True, null=True, verbose_name='Address')),
+ ('area', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commcrawler.Area', verbose_name='Area')),
+ ],
+ options={
+ 'verbose_name_plural': 'Organizations',
+ 'ordering': ('name',),
+ 'verbose_name': 'Organization',
+ },
+ ),
+ migrations.CreateModel(
+ name='OrganizationType',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=200, verbose_name='Name')),
+ ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commcrawler.OrganizationType', verbose_name='Parent')),
+ ],
+ options={
+ 'verbose_name_plural': 'Organization types',
+ 'ordering': ('parent__name', 'name'),
+ 'verbose_name': 'Organization type',
+ },
+ ),
+ migrations.CreateModel(
+ name='Target',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=200, verbose_name='Name')),
+ ('url', models.URLField(blank=True, null=True, verbose_name='URL')),
+ ('organization', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commcrawler.Organization', verbose_name='Organization')),
+ ],
+ options={
+ 'verbose_name_plural': 'Targets',
+ 'ordering': ('name',),
+ 'verbose_name': 'Target',
+ },
+ ),
+ migrations.AddField(
+ model_name='organization',
+ name='organization_type',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.OrganizationType', verbose_name='Type'),
+ ),
+ migrations.AddField(
+ model_name='crawl',
+ name='targets',
+ field=models.ManyToManyField(blank=True, to='commcrawler.Target'),
+ ),
+ migrations.AddField(
+ model_name='area',
+ name='area_type',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.AreaType', verbose_name='Type'),
+ ),
+ migrations.AddField(
+ model_name='area',
+ name='parent',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commcrawler.Area', verbose_name='Parent'),
+ ),
+ ]
diff --git a/commcrawler/migrations/__init__.py b/commcrawler/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commcrawler/migrations/__init__.py
diff --git a/commcrawler/models.py b/commcrawler/models.py
new file mode 100644
index 0000000..f62157e
--- /dev/null
+++ b/commcrawler/models.py
@@ -0,0 +1,123 @@
+import datetime
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+
+class AreaType(models.Model):
+ name = models.CharField(_("Name"), max_length=200)
+
+ class Meta:
+ verbose_name = _("Area type")
+ verbose_name_plural = _("Area types")
+ ordering = ("name", )
+
+ def __str__(self):
+ return self.name
+
+
+class Area(models.Model):
+ name = models.CharField(verbose_name=_("Name"), max_length=200)
+ area_type = models.ForeignKey(
+ AreaType, on_delete=models.CASCADE, verbose_name=_("Type"))
+ population = models.IntegerField(verbose_name=_("Population"), blank=True,
+ null=True)
+ reference = models.CharField(
+ verbose_name=_("Reference"), max_length=100, blank=True, null=True,
+ help_text=_("For instance, INSEE code for towns")
+ )
+ parent = models.ForeignKey("Area", verbose_name=_("Parent"),
+ blank=True, null=True)
+
+ class Meta:
+ verbose_name = _("Area")
+ verbose_name_plural = _("Areas")
+ ordering = ("name", )
+
+ def __str__(self):
+ if not self.parent:
+ return self.name
+ return "{} / {}".format(self.parent, self.name)
+
+
+class OrganizationType(models.Model):
+ name = models.CharField(_("Name"), max_length=200)
+ parent = models.ForeignKey("OrganizationType", verbose_name=_("Parent"),
+ blank=True, null=True)
+
+ class Meta:
+ verbose_name = _("Organization type")
+ verbose_name_plural = _("Organization types")
+ ordering = ("parent__name", "name", )
+
+ def __str__(self):
+ if not self.parent:
+ return self.name
+ return "{} / {}".format(self.parent, self.name)
+
+
+class Organization(models.Model):
+ name = models.CharField(verbose_name=_("Name"), max_length=200)
+ area = models.ForeignKey(
+ Area, on_delete=models.SET_NULL, verbose_name=_("Area"), blank=True,
+ null=True
+ )
+ organization_type = models.ForeignKey(
+ OrganizationType, on_delete=models.CASCADE, verbose_name=_("Type"))
+ address = models.TextField(verbose_name=_("Address"), blank=True, null=True)
+
+ class Meta:
+ verbose_name = _("Organization")
+ verbose_name_plural = _("Organizations")
+ ordering = ("name", )
+
+ def __str__(self):
+ if not self.area:
+ return "{} ({})".format(
+ self.name, self.organization_type)
+ return "{} - {} ({})".format(
+ self.name, self.area, self.organization_type)
+
+
+class Target(models.Model):
+ name = models.CharField(verbose_name=_("Name"), max_length=200)
+ url = models.URLField(verbose_name=_("URL"), blank=True, null=True)
+ organization = models.ForeignKey(
+ Organization, verbose_name=_("Organization"), blank=True,
+ null=True, on_delete=models.SET_NULL)
+
+ class Meta:
+ verbose_name = _("Target")
+ verbose_name_plural = _("Targets")
+ ordering = ("name",)
+
+ def __str__(self):
+ return "{} ({})".format(self.name, self.organization)
+
+
+class Crawl(models.Model):
+ STATUS = (
+ ('C', _("Created")), ('P', _("In progress")),
+ ('F', _("Finished"))
+ )
+ name = models.CharField(verbose_name=_("Name"), max_length=200)
+ created = models.DateTimeField(
+ verbose_name=_("Creation date"), default=datetime.datetime.now)
+ started = models.DateTimeField(
+ verbose_name=_("Start date"), blank=True, null=True)
+ ended = models.DateTimeField(
+ verbose_name=_("End date"), blank=True, null=True)
+ status = models.CharField(max_length=1, choices=STATUS, default='C')
+ targets = models.ManyToManyField("Target", blank=True)
+
+ class Meta:
+ verbose_name = _("Crawl")
+ verbose_name_plural = _("Crawls")
+ ordering = ("created", "name")
+
+ def __str__(self):
+ return self.name
+
+ @property
+ def target_nb(self):
+ return self.targets.count()
diff --git a/commcrawler/templates/admin/add_to_crawl.html b/commcrawler/templates/admin/add_to_crawl.html
new file mode 100644
index 0000000..3822a73
--- /dev/null
+++ b/commcrawler/templates/admin/add_to_crawl.html
@@ -0,0 +1,16 @@
+{% extends "admin/base_site.html" %}{% load i18n %}
+
+{% block content %}
+<h1>{% blocktrans %}Add {{query_nb}} selected items to a crawl{% endblocktrans %}</h1>
+<div id="content-main">
+ <form action="." method="post">
+ {% csrf_token %}
+ {{form}}
+ &nbsp;
+ <input type="hidden" name="action" value="add_to_crawl" />
+ <input type="submit" name="apply" value="{% trans 'Add' %}"/>
+ {% for pk in items %}
+ <input type="hidden" name="_selected_action" value="{{ pk }}" />{% endfor %}
+ </form>
+</div>
+{% endblock %} \ No newline at end of file
diff --git a/commcrawler/tests.py b/commcrawler/tests.py
new file mode 100644
index 0000000..7ce503c
--- /dev/null
+++ b/commcrawler/tests.py
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
diff --git a/commcrawler/views.py b/commcrawler/views.py
new file mode 100644
index 0000000..91ea44a
--- /dev/null
+++ b/commcrawler/views.py
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.