First commit

22 files changed, 902 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..9ea83a1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,7 @@
+*.ignore
+*.swp
+*.pyc
+*.mo
+*~
+.idea
+*.sqlite3
diff --git a/commcrawler/__init__.py b/commcrawler/__init__.py
new file mode 100644
index 0000000..339b125
--- /dev/null
+++ b/commcrawler/__init__.py
@@ -0,0 +1 @@
+default_app_config = 'commonnet.apps.CommCrawlerConfig'
diff --git a/commcrawler/admin.py b/commcrawler/admin.py
new file mode 100644
index 0000000..f4df5e2
--- /dev/null
+++ b/commcrawler/admin.py
@@ -0,0 +1,124 @@
+from ajax_select import make_ajax_form
+from django.contrib import admin, messages
+from django.contrib.auth.admin import GroupAdmin, UserAdmin
+from django.contrib.auth.models import Group, User
+from django import forms
+from django.http import HttpResponseRedirect
+from django.shortcuts import render
+from django.utils.translation import ugettext_lazy as _
+
+from commonnet.admin_site import admin_site
+from commcrawler import models
+
+
+admin_site.register(User, UserAdmin)
+admin_site.register(Group, GroupAdmin)
+
+
+class AreaTypeAdmin(admin.ModelAdmin):
+    model = models.AreaType
+    list_display = ('name', )
+
+
+admin_site.register(models.AreaType, AreaTypeAdmin)
+
+
+class AreaAdmin(admin.ModelAdmin):
+    model = models.Area
+    list_display = ('name', 'area_type', 'reference', 'parent')
+    list_filter = ('area_type',)
+    search_fields = ['name', 'parent__name']
+
+
+admin_site.register(models.Area, AreaAdmin)
+
+
+class OrganizationTypeAdmin(admin.ModelAdmin):
+    model = models.OrganizationType
+    list_display = ('name', 'parent')
+    list_filter = ('parent',)
+
+
+admin_site.register(models.OrganizationType, OrganizationTypeAdmin)
+
+
+class OrganizationAdmin(admin.ModelAdmin):
+    model = models.Organization
+    list_display = ('name', 'organization_type', 'area')
+    list_filter = ('organization_type',)
+    search_fields = ['name']
+    form = make_ajax_form(model, {'area': 'area'})
+
+
+admin_site.register(models.Organization, OrganizationAdmin)
+
+
+class AddToCrawlForm(forms.Form):
+    crawl = forms.ChoiceField(label=_("Crawl"), choices=tuple())
+
+    def __init__(self, *args, **kwargs):
+        super(AddToCrawlForm, self).__init__(*args, **kwargs)
+        self.fields["crawl"].choices = [(None, "--")] + [
+            (c.pk, str(c))
+            for c in models.Crawl.objects.filter(status="C").all()
+        ]
+
+
+class TargetAdmin(admin.ModelAdmin):
+    list_display = ('name', 'url', 'organization')
+    model = models.Target
+    list_filter = ('organization__organization_type',)
+    form = make_ajax_form(model, {'organization': 'organization'})
+    actions = ['add_to_crawl']
+
+    def add_to_crawl(self, request, queryset):
+        if 'apply' in request.POST:
+            form = AddToCrawlForm(request.POST)
+            if form.is_valid():
+                crawl = None
+                try:
+                    crawl = models.Crawl.objects.get(
+                        pk=form.cleaned_data["crawl"],
+                        status="C"
+                    )
+                except models.Crawl.DoesNotExist:
+                    pass
+                if crawl:
+                    nb_added = 0
+                    targets = [
+                        c["id"] for c in crawl.targets.values("id")
+                    ]
+                    for target in queryset.all():
+                        if target.pk not in targets:
+                            crawl.targets.add(target)
+                            nb_added += 1
+                    messages.add_message(
+                        request, messages.INFO,
+                        str(_("{} new targets added to {}")).format(
+                            nb_added, str(crawl)
+                        )
+                    )
+                    return HttpResponseRedirect(request.get_full_path())
+        else:
+            form = AddToCrawlForm()
+        return render(
+            request, 'admin/add_to_crawl.html',
+            context={"form": form, "query_nb": queryset.count(),
+                     "items": [v['pk'] for v in queryset.values('pk').all()]})
+
+    add_to_crawl.short_description = _("Add to crawl")
+
+
+admin_site.register(models.Target, TargetAdmin)
+
+
+class CrawlAdmin(admin.ModelAdmin):
+    model = models.Crawl
+    list_display = ("name", "status", "target_nb", "created", "started",
+                    "ended")
+    list_filter = ("status",)
+    readonly_fields = ("status", "created", "started", "ended")
+    exclude = ("targets", )
+
+
+admin_site.register(models.Crawl, CrawlAdmin)
diff --git a/commcrawler/apps.py b/commcrawler/apps.py
new file mode 100644
index 0000000..79e032d
--- /dev/null
+++ b/commcrawler/apps.py
@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+from django.utils.translation import ugettext_lazy as _
+
+
+class CommCrawlerConfig(AppConfig):
+    name = 'commcrawler'
+    verbose_name = _("Crawler")
diff --git a/commcrawler/lookups.py b/commcrawler/lookups.py
new file mode 100644
index 0000000..f8d4897
--- /dev/null
+++ b/commcrawler/lookups.py
@@ -0,0 +1,49 @@
+from ajax_select import register, LookupChannel as BaseLookupChannel
+
+from django.db.models import Q
+
+from . import models
+
+
+class LookupChannel(BaseLookupChannel):
+    def get_objects(self, items):
+        # TODO: why IDs are not given here? M2M issue
+        ids = []
+        for item in items:
+            if hasattr(item, 'pk'):
+                ids.append(item.pk)
+            else:
+                ids.append(item)
+        return super(LookupChannel, self).get_objects(ids)
+
+    def format_item_display(self, item):
+        return u"<span class='ajax-label'>%s</span>" % str(item)
+
+
+@register('organization')
+class OrganizationLookup(LookupChannel):
+    model = models.Organization
+
+    def get_query(self, q, request):
+        query = Q()
+        for term in q.strip().split(' '):
+            subquery = Q(name__icontains=term)
+            query &= subquery
+        return self.model.objects.filter(query).order_by('name')[:20]
+
+
+@register('area')
+class AreaLookup(LookupChannel):
+    model = models.Area
+
+    def get_query(self, q, request):
+        query = Q()
+        for term in q.strip().split(' '):
+            subquery = (
+                    Q(name__icontains=term) |
+                    Q(parent__name__icontains=term) |
+                    Q(reference=term)
+            )
+            query &= subquery
+        return self.model.objects.filter(query).order_by('name')[:20]
+
diff --git a/commcrawler/management/__init__.py b/commcrawler/management/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commcrawler/management/__init__.py
diff --git a/commcrawler/management/commands/__init__.py b/commcrawler/management/commands/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commcrawler/management/commands/__init__.py
diff --git a/commcrawler/management/commands/import_csv_autres.py b/commcrawler/management/commands/import_csv_autres.py
new file mode 100644
index 0000000..2dd177c
--- /dev/null
+++ b/commcrawler/management/commands/import_csv_autres.py
@@ -0,0 +1,107 @@
+import csv
+import sys
+
+from django.core.management.base import BaseCommand
+
+from commcrawler.models import Organization, OrganizationType, Target
+
+header_len = 1
+expected_header = [
+    'Secteur', 'Type', 'Nom', 'URL', 'Localisation siège (département)',
+    'Localisation siège (code postal)', 'Localisation siège (commune)']
+
+
+class Command(BaseCommand):
+    help = 'Import depuis CSV communes'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_file')
+        parser.add_argument(
+            '--quiet', dest='quiet', action='store_true',
+            help='Quiet output')
+
+    def handle(self, *args, **options):
+        csv_file = options['csv_file']
+        quiet = options['quiet']
+        if not quiet:
+            sys.stdout.write('* opening file {}\n'.format(csv_file))
+        nb_created = 0
+        nb_organization_created = 0
+        nb_tt_created = 0
+        with open(csv_file, 'r') as csvfile:
+            reader = csv.reader(csvfile)
+            for idx, row in enumerate(reader):
+                if idx < header_len:
+                    if not idx:
+                        if expected_header != row:
+                            sys.stdout.write('ERROR: expected header differs '
+                                             'from the one provided\n')
+                            sys.stdout.write('* expected header is:\n')
+                            sys.stdout.write(str(expected_header))
+                            sys.stdout.write('\n* header provided is:\n')
+                            sys.stdout.write(str(row) + "\n")
+                            return
+                    continue
+                sec, sec_tpe, name, site, address = row[0:5]
+                address = address.strip()
+                if " " in address:
+                    organization_name = " ".join(address.split(" ")[1:])
+                else:
+                    organization_name = address
+
+                if not quiet:
+                    sys.stdout.write('-> processing line %d.\r' % (idx + 1))
+                    sys.stdout.flush()
+
+                tpe, c = OrganizationType.objects.get_or_create(
+                    name=sec.strip(),
+                    parent=None
+                )
+                if c:
+                    nb_tt_created += 1
+                tpe, c = OrganizationType.objects.get_or_create(
+                    name=sec_tpe.strip(),
+                    parent=tpe
+                )
+                if c:
+                    nb_tt_created += 1
+
+                organization_values = {
+                    "organization_type": tpe,
+                    "name": "{} - {}".format(name.strip(), organization_name)
+                }
+
+                default = dict()
+                default["address"] = address
+                organization_values["defaults"] = default
+
+                organization, c = Organization.objects.get_or_create(
+                    **organization_values)
+                if c:
+                    nb_organization_created += 1
+                else:
+                    for k in default.keys():
+                        setattr(organization, k, default[k])
+                    organization.save()
+
+                site = site.strip()
+                if site == "0" or "." not in site:
+                    site = None
+                elif not site.startswith('http'):
+                    site = "http://" + site
+                values = {
+                    "name": name.strip(),
+                    "organization": organization,
+                    "url": site,
+                }
+                target, created = Target.objects.get_or_create(**values)
+                if created:
+                    nb_created += 1
+        if not quiet:
+            sys.stdout.write(
+                '\n* {} organization types created.\n'.format(nb_tt_created))
+            sys.stdout.write(
+                '* {} organizations created.\n'.format(nb_organization_created))
+            sys.stdout.write(
+                '* {} targets created.\n'.format(nb_created))
+            sys.stdout.flush()
diff --git a/commcrawler/management/commands/import_csv_communes.py b/commcrawler/management/commands/import_csv_communes.py
new file mode 100644
index 0000000..4024067
--- /dev/null
+++ b/commcrawler/management/commands/import_csv_communes.py
@@ -0,0 +1,167 @@
+import csv
+import sys
+
+from django.core.management.base import BaseCommand
+
+from commcrawler.models import Area, AreaType, Organization, OrganizationType,\
+    Target
+
+header_len = 1
+expected_header = [
+    'DÉPARTEMENT', 'NOM EPCI OU MAIRIE', 'CODE POSTAL', 'COMMUNE',
+    'C.C. / C.A. / C.U.', 'SITE INTERNET', 'POPULATION COMMUNAUTAIRE',
+    'Type de collectivité', 'Code INSEE']
+
+
+class Command(BaseCommand):
+    help = 'Import depuis CSV communes'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_file')
+        parser.add_argument(
+            '--quiet', dest='quiet', action='store_true',
+            help='Quiet output')
+
+    def handle(self, *args, **options):
+        csv_file = options['csv_file']
+        quiet = options['quiet']
+        if not quiet:
+            sys.stdout.write('* opening file {}\n'.format(csv_file))
+        nb_created = 0
+        nb_organization_created = 0
+        nb_area_created = 0
+        nb_tt_created = 0
+        nb_at_created = 0
+        with open(csv_file, 'r') as csvfile:
+            reader = csv.reader(csvfile)
+            for idx, row in enumerate(reader):
+                if idx < header_len:
+                    if not idx:
+                        if expected_header != row:
+                            sys.stdout.write('ERROR: expected header differs '
+                                             'from the one provided\n')
+                            sys.stdout.write('* expected header is:\n')
+                            sys.stdout.write(str(expected_header))
+                            sys.stdout.write('\n* header provided is:\n')
+                            sys.stdout.write(str(row) + "\n")
+                            return
+                    continue
+                dpt, name, code_postal, commune, comcom, site, pop = row[0:7]
+                type_coll, insee = row[7:9]
+                if insee.strip() == "NA":
+                    insee = ""
+                if not quiet:
+                    sys.stdout.write('-> processing line %d.\r' % (idx + 1))
+                    sys.stdout.flush()
+                try:
+                    pop = int(pop.replace(" ", ""))
+                except ValueError:
+                    pop = None
+
+                p_tpe, c = OrganizationType.objects.get_or_create(
+                    name=type_coll.strip()
+                )
+                if c:
+                    nb_tt_created += 1
+                if comcom.strip():
+                    tpe, c = OrganizationType.objects.get_or_create(
+                        name=comcom.strip(),
+                        parent=p_tpe
+                    )
+                    if c:
+                        nb_tt_created += 1
+                else:
+                    tpe = p_tpe
+                atpe, c = AreaType.objects.get_or_create(
+                    name=str(tpe)
+                )
+                if c:
+                    nb_at_created += 1
+
+                top_area = None
+                if dpt.strip():
+                    dpt_tpe, c = AreaType.objects.get_or_create(
+                        name="Département",
+                    )
+                    if c:
+                        nb_tt_created += 1
+                    top_area, c = Area.objects.get_or_create(
+                        name=dpt.strip(),
+                        area_type=dpt_tpe
+                    )
+                    if c:
+                        nb_area_created += 1
+
+                area_values = {"area_type": atpe}
+                area_defaults = {"population": pop, "parent": top_area}
+                if insee.strip():
+                    area_values['reference'] = insee.strip()
+                    area_defaults['name'] = commune.strip()
+                else:
+                    area_values['name'] = name.strip()
+                area_values["defaults"] = area_defaults
+
+                area, c = Area.objects.get_or_create(
+                    **area_values)
+                if c:
+                    nb_area_created += 1
+                else:
+                    for k in area_defaults:
+                        setattr(area, k, area_defaults[k])
+                    area.save()
+
+                organization_values = {
+                    "organization_type": tpe,
+                    "name": name,
+                    "area": area,
+                }
+                default = dict()
+                default["address"] = "{} {}".format(code_postal.strip() or "",
+                                                    commune.strip() or "")
+                organization_values["defaults"] = default
+
+                organization, c = Organization.objects.get_or_create(
+                    **organization_values)
+                if c:
+                    nb_organization_created += 1
+                else:
+                    for k in default:
+                        setattr(organization, k, default[k])
+                    organization.save()
+
+                site = site.strip()
+                if site == "0" or "." not in site:
+                    site = None
+                elif not site.startswith('http'):
+                    site = "http://" + site
+                values = {
+                    "name": name.strip(),
+                    "organization": organization,
+                    "url": site,
+                }
+
+                target, created = Target.objects.get_or_create(**values)
+                if created:
+                    nb_created += 1
+        if not quiet:
+            sys.stdout.write("\n")
+            if nb_at_created:
+                sys.stdout.write(
+                    '* {} area types created.\n'.format(nb_at_created)
+                )
+            if nb_area_created:
+                sys.stdout.write(
+                    '* {} areas created.\n'.format(
+                        nb_area_created))
+            if nb_tt_created:
+                sys.stdout.write(
+                    '* {} organization types created.\n'.format(nb_tt_created)
+                )
+            if nb_organization_created:
+                sys.stdout.write(
+                    '* {} organizations created.\n'.format(
+                        nb_organization_created))
+            if nb_created:
+                sys.stdout.write(
+                    '* {} targets created.\n'.format(nb_created))
+            sys.stdout.flush()
diff --git a/commcrawler/migrations/0001_initial.py b/commcrawler/migrations/0001_initial.py
new file mode 100644
index 0000000..2fbb9fe
--- /dev/null
+++ b/commcrawler/migrations/0001_initial.py
@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2019-07-30 15:47
+from __future__ import unicode_literals
+
+import datetime
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Area',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('population', models.IntegerField(blank=True, null=True, verbose_name='Population')),
+                ('reference', models.CharField(blank=True, help_text='For instance, INSEE code for towns', max_length=100, null=True, verbose_name='Reference')),
+            ],
+            options={
+                'verbose_name_plural': 'Areas',
+                'ordering': ('name',),
+                'verbose_name': 'Area',
+            },
+        ),
+        migrations.CreateModel(
+            name='AreaType',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+            ],
+            options={
+                'verbose_name_plural': 'Area types',
+                'ordering': ('name',),
+                'verbose_name': 'Area type',
+            },
+        ),
+        migrations.CreateModel(
+            name='Crawl',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('created', models.DateTimeField(default=datetime.datetime.now, verbose_name='Creation date')),
+                ('started', models.DateTimeField(blank=True, null=True, verbose_name='Start date')),
+                ('ended', models.DateTimeField(blank=True, null=True, verbose_name='End date')),
+                ('status', models.CharField(choices=[('C', 'Created'), ('P', 'In progress'), ('F', 'Finished')], default='C', max_length=1)),
+            ],
+            options={
+                'verbose_name_plural': 'Crawls',
+                'ordering': ('created', 'name'),
+                'verbose_name': 'Crawl',
+            },
+        ),
+        migrations.CreateModel(
+            name='Organization',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('address', models.TextField(blank=True, null=True, verbose_name='Address')),
+                ('area', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commcrawler.Area', verbose_name='Area')),
+            ],
+            options={
+                'verbose_name_plural': 'Organizations',
+                'ordering': ('name',),
+                'verbose_name': 'Organization',
+            },
+        ),
+        migrations.CreateModel(
+            name='OrganizationType',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commcrawler.OrganizationType', verbose_name='Parent')),
+            ],
+            options={
+                'verbose_name_plural': 'Organization types',
+                'ordering': ('parent__name', 'name'),
+                'verbose_name': 'Organization type',
+            },
+        ),
+        migrations.CreateModel(
+            name='Target',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('url', models.URLField(blank=True, null=True, verbose_name='URL')),
+                ('organization', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commcrawler.Organization', verbose_name='Organization')),
+            ],
+            options={
+                'verbose_name_plural': 'Targets',
+                'ordering': ('name',),
+                'verbose_name': 'Target',
+            },
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='organization_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.OrganizationType', verbose_name='Type'),
+        ),
+        migrations.AddField(
+            model_name='crawl',
+            name='targets',
+            field=models.ManyToManyField(blank=True, to='commcrawler.Target'),
+        ),
+        migrations.AddField(
+            model_name='area',
+            name='area_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.AreaType', verbose_name='Type'),
+        ),
+        migrations.AddField(
+            model_name='area',
+            name='parent',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commcrawler.Area', verbose_name='Parent'),
+        ),
+    ]
diff --git a/commcrawler/migrations/__init__.py b/commcrawler/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commcrawler/migrations/__init__.py
diff --git a/commcrawler/models.py b/commcrawler/models.py
new file mode 100644
index 0000000..f62157e
--- /dev/null
+++ b/commcrawler/models.py
@@ -0,0 +1,123 @@
+import datetime
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+
+class AreaType(models.Model):
+    name = models.CharField(_("Name"), max_length=200)
+
+    class Meta:
+        verbose_name = _("Area type")
+        verbose_name_plural = _("Area types")
+        ordering = ("name", )
+
+    def __str__(self):
+        return self.name
+
+
+class Area(models.Model):
+    name = models.CharField(verbose_name=_("Name"), max_length=200)
+    area_type = models.ForeignKey(
+        AreaType, on_delete=models.CASCADE, verbose_name=_("Type"))
+    population = models.IntegerField(verbose_name=_("Population"), blank=True,
+                                     null=True)
+    reference = models.CharField(
+        verbose_name=_("Reference"), max_length=100, blank=True, null=True,
+        help_text=_("For instance, INSEE code for towns")
+    )
+    parent = models.ForeignKey("Area", verbose_name=_("Parent"),
+                               blank=True, null=True)
+
+    class Meta:
+        verbose_name = _("Area")
+        verbose_name_plural = _("Areas")
+        ordering = ("name", )
+
+    def __str__(self):
+        if not self.parent:
+            return self.name
+        return "{} / {}".format(self.parent, self.name)
+
+
+class OrganizationType(models.Model):
+    name = models.CharField(_("Name"), max_length=200)
+    parent = models.ForeignKey("OrganizationType", verbose_name=_("Parent"),
+                               blank=True, null=True)
+
+    class Meta:
+        verbose_name = _("Organization type")
+        verbose_name_plural = _("Organization types")
+        ordering = ("parent__name", "name", )
+
+    def __str__(self):
+        if not self.parent:
+            return self.name
+        return "{} / {}".format(self.parent, self.name)
+
+
+class Organization(models.Model):
+    name = models.CharField(verbose_name=_("Name"), max_length=200)
+    area = models.ForeignKey(
+        Area, on_delete=models.SET_NULL, verbose_name=_("Area"), blank=True,
+        null=True
+    )
+    organization_type = models.ForeignKey(
+        OrganizationType, on_delete=models.CASCADE, verbose_name=_("Type"))
+    address = models.TextField(verbose_name=_("Address"), blank=True, null=True)
+
+    class Meta:
+        verbose_name = _("Organization")
+        verbose_name_plural = _("Organizations")
+        ordering = ("name", )
+
+    def __str__(self):
+        if not self.area:
+            return "{} ({})".format(
+                self.name, self.organization_type)
+        return "{} - {} ({})".format(
+            self.name, self.area, self.organization_type)
+
+
+class Target(models.Model):
+    name = models.CharField(verbose_name=_("Name"), max_length=200)
+    url = models.URLField(verbose_name=_("URL"), blank=True, null=True)
+    organization = models.ForeignKey(
+        Organization, verbose_name=_("Organization"), blank=True,
+        null=True, on_delete=models.SET_NULL)
+
+    class Meta:
+        verbose_name = _("Target")
+        verbose_name_plural = _("Targets")
+        ordering = ("name",)
+
+    def __str__(self):
+        return "{} ({})".format(self.name, self.organization)
+
+
+class Crawl(models.Model):
+    STATUS = (
+        ('C', _("Created")), ('P', _("In progress")),
+        ('F', _("Finished"))
+    )
+    name = models.CharField(verbose_name=_("Name"), max_length=200)
+    created = models.DateTimeField(
+        verbose_name=_("Creation date"), default=datetime.datetime.now)
+    started = models.DateTimeField(
+        verbose_name=_("Start date"), blank=True, null=True)
+    ended = models.DateTimeField(
+        verbose_name=_("End date"), blank=True, null=True)
+    status = models.CharField(max_length=1, choices=STATUS, default='C')
+    targets = models.ManyToManyField("Target", blank=True)
+
+    class Meta:
+        verbose_name = _("Crawl")
+        verbose_name_plural = _("Crawls")
+        ordering = ("created", "name")
+
+    def __str__(self):
+        return self.name
+
+    @property
+    def target_nb(self):
+        return self.targets.count()
diff --git a/commcrawler/templates/admin/add_to_crawl.html b/commcrawler/templates/admin/add_to_crawl.html
new file mode 100644
index 0000000..3822a73
--- /dev/null
+++ b/commcrawler/templates/admin/add_to_crawl.html
@@ -0,0 +1,16 @@
+{% extends "admin/base_site.html" %}{% load i18n %}
+
+{% block content %}
+<h1>{% blocktrans %}Add {{query_nb}} selected items to a crawl{% endblocktrans %}</h1>
+<div id="content-main">
+  <form action="." method="post">
+    {% csrf_token %}
+    {{form}}
+    &nbsp;
+    <input type="hidden" name="action" value="add_to_crawl" />
+    <input type="submit" name="apply" value="{% trans 'Add' %}"/>
+    {% for pk in items %}
+    <input type="hidden" name="_selected_action" value="{{ pk }}" />{% endfor %}
+  </form>
+</div>
+{% endblock %}
\ No newline at end of file
diff --git a/commcrawler/tests.py b/commcrawler/tests.py
new file mode 100644
index 0000000..7ce503c
--- /dev/null
+++ b/commcrawler/tests.py
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
diff --git a/commcrawler/views.py b/commcrawler/views.py
new file mode 100644
index 0000000..91ea44a
--- /dev/null
+++ b/commcrawler/views.py
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.
diff --git a/commonnet/__init__.py b/commonnet/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commonnet/__init__.py
diff --git a/commonnet/admin_site.py b/commonnet/admin_site.py
new file mode 100644
index 0000000..1e36ade
--- /dev/null
+++ b/commonnet/admin_site.py
@@ -0,0 +1,11 @@
+from django.contrib.admin import AdminSite
+from django.utils.translation import ugettext_lazy as _
+
+
+class CommOnNetAdminSite(AdminSite):
+    site_header = _('Comm-on-net administration')
+    site_title = _("Comm-on-net administration")
+
+
+admin_site = CommOnNetAdminSite(name='commonnetadmin')
+
diff --git a/commonnet/settings.py b/commonnet/settings.py
new file mode 100644
index 0000000..4ca0fbf
--- /dev/null
+++ b/commonnet/settings.py
@@ -0,0 +1,112 @@
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = '!lh+r$hzd(_-aj8a2&@)34bat=w&=!k+9w%$_+&^gjhf#n6z42'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'commcrawler.apps.CommCrawlerConfig',
+    'ajax_select',
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'commonnet.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'commonnet.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.11/topics/i18n/
+
+LANGUAGE_CODE = 'fr-fr'
+
+TIME_ZONE = 'Europe/Paris'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+DATA_UPLOAD_MAX_NUMBER_FIELDS = 5000
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.11/howto/static-files/
+
+STATIC_URL = '/static/'
diff --git a/commonnet/urls.py b/commonnet/urls.py
new file mode 100644
index 0000000..d38b3b7
--- /dev/null
+++ b/commonnet/urls.py
@@ -0,0 +1,10 @@
+from django.conf.urls import include, url
+
+from ajax_select import urls as ajax_select_urls
+from commonnet.admin_site import admin_site
+
+
+urlpatterns = [
+    url(r'^admin/', admin_site.urls),
+    url(r'^ajax-select/', include(ajax_select_urls)),
+]
diff --git a/commonnet/wsgi.py b/commonnet/wsgi.py
new file mode 100644
index 0000000..bfccf8c
--- /dev/null
+++ b/commonnet/wsgi.py
@@ -0,0 +1,16 @@
+"""
+WSGI config for commonnet project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "commonnet.settings")
+
+application = get_wsgi_application()
diff --git a/manage.py b/manage.py
new file mode 100755
index 0000000..bcd0b3c
--- /dev/null
+++ b/manage.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "commonnet.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError:
+        # The above import may fail for some other reason. Ensure that the
+        # issue is really that Django is missing to avoid masking other
+        # exceptions on Python 2.
+        try:
+            import django
+        except ImportError:
+            raise ImportError(
+                "Couldn't import Django. Are you sure it's installed and "
+                "available on your PYTHONPATH environment variable? Did you "
+                "forget to activate a virtual environment?"
+            )
+        raise
+    execute_from_command_line(sys.argv)
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..7028b7e
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
+django==1.11
+scrapy==1.5
+django-ajax-selects==1.6.0
\ No newline at end of file