Models refactoring

13 files changed, 640 insertions, 0 deletions

diff --git a/commorganization/__init__.py b/commorganization/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commorganization/__init__.py
diff --git a/commorganization/admin.py b/commorganization/admin.py
new file mode 100644
index 0000000..2b3c042
--- /dev/null
+++ b/commorganization/admin.py
@@ -0,0 +1,112 @@
+from ajax_select import make_ajax_form
+from django.contrib import admin, messages
+from django.contrib.auth.admin import GroupAdmin, UserAdmin
+from django.contrib.auth.models import Group, User
+from django import forms
+from django.http import HttpResponseRedirect
+from django.shortcuts import render
+from django.utils.translation import ugettext_lazy as _
+
+from commonnet.admin_site import admin_site
+from commorganization import models
+
+
+admin_site.register(User, UserAdmin)
+admin_site.register(Group, GroupAdmin)
+
+
+class AreaTypeAdmin(admin.ModelAdmin):
+    model = models.AreaType
+    list_display = ('name', )
+
+
+admin_site.register(models.AreaType, AreaTypeAdmin)
+
+
+class AreaAdmin(admin.ModelAdmin):
+    model = models.Area
+    list_display = ('name', 'area_type', 'reference', 'parent')
+    list_filter = ('area_type',)
+    search_fields = ['name', 'parent__name']
+
+
+admin_site.register(models.Area, AreaAdmin)
+
+
+class OrganizationTypeAdmin(admin.ModelAdmin):
+    model = models.OrganizationType
+    list_display = ('name', 'parent')
+    list_filter = ('parent',)
+
+
+admin_site.register(models.OrganizationType, OrganizationTypeAdmin)
+
+
+class OrganizationAdmin(admin.ModelAdmin):
+    model = models.Organization
+    list_display = ('name', 'organization_type', 'area')
+    list_filter = ('organization_type',)
+    search_fields = ['name']
+    form = make_ajax_form(model, {'area': 'area'})
+
+
+admin_site.register(models.Organization, OrganizationAdmin)
+
+
+class AddToCrawlForm(forms.Form):
+    crawl = forms.ChoiceField(label=_("Crawl"), choices=tuple())
+
+    def __init__(self, *args, **kwargs):
+        super(AddToCrawlForm, self).__init__(*args, **kwargs)
+        self.fields["crawl"].choices = [(None, "--")] + [
+            (c.pk, str(c))
+            for c in models.Crawl.objects.filter(status="C").all()
+        ]
+
+
+class TargetAdmin(admin.ModelAdmin):
+    list_display = ('name', 'url', 'organization')
+    model = models.Target
+    list_filter = ('organization__organization_type',)
+    form = make_ajax_form(model, {'organization': 'organization'})
+    actions = ['add_to_crawl']
+
+    def add_to_crawl(self, request, queryset):
+        if 'apply' in request.POST:
+            form = AddToCrawlForm(request.POST)
+            if form.is_valid():
+                crawl = None
+                try:
+                    crawl = models.Crawl.objects.get(
+                        pk=form.cleaned_data["crawl"],
+                        status="C"
+                    )
+                except models.Crawl.DoesNotExist:
+                    pass
+                if crawl:
+                    nb_added = 0
+                    targets = [
+                        c["id"] for c in crawl.targets.values("id")
+                    ]
+                    for target in queryset.all():
+                        if target.pk not in targets:
+                            crawl.targets.add(target)
+                            nb_added += 1
+                    messages.add_message(
+                        request, messages.INFO,
+                        str(_("{} new targets added to {}")).format(
+                            nb_added, str(crawl)
+                        )
+                    )
+                    return HttpResponseRedirect(request.get_full_path())
+        else:
+            form = AddToCrawlForm()
+        return render(
+            request, 'admin/add_to_crawl.html',
+            context={"form": form, "query_nb": queryset.count(),
+                     "items": [v['pk'] for v in queryset.values('pk').all()]})
+
+    add_to_crawl.short_description = _("Add to crawl")
+
+
+admin_site.register(models.Target, TargetAdmin)
diff --git a/commorganization/apps.py b/commorganization/apps.py
new file mode 100644
index 0000000..32c623d
--- /dev/null
+++ b/commorganization/apps.py
@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+from django.utils.translation import ugettext_lazy as _
+
+
+class CommOrganizationConfig(AppConfig):
+    name = 'commorganization'
+    verbose_name = _("Organization")
diff --git a/commorganization/lookups.py b/commorganization/lookups.py
new file mode 100644
index 0000000..f8d4897
--- /dev/null
+++ b/commorganization/lookups.py
@@ -0,0 +1,49 @@
+from ajax_select import register, LookupChannel as BaseLookupChannel
+
+from django.db.models import Q
+
+from . import models
+
+
+class LookupChannel(BaseLookupChannel):
+    def get_objects(self, items):
+        # TODO: why IDs are not given here? M2M issue
+        ids = []
+        for item in items:
+            if hasattr(item, 'pk'):
+                ids.append(item.pk)
+            else:
+                ids.append(item)
+        return super(LookupChannel, self).get_objects(ids)
+
+    def format_item_display(self, item):
+        return u"<span class='ajax-label'>%s</span>" % str(item)
+
+
+@register('organization')
+class OrganizationLookup(LookupChannel):
+    model = models.Organization
+
+    def get_query(self, q, request):
+        query = Q()
+        for term in q.strip().split(' '):
+            subquery = Q(name__icontains=term)
+            query &= subquery
+        return self.model.objects.filter(query).order_by('name')[:20]
+
+
+@register('area')
+class AreaLookup(LookupChannel):
+    model = models.Area
+
+    def get_query(self, q, request):
+        query = Q()
+        for term in q.strip().split(' '):
+            subquery = (
+                    Q(name__icontains=term) |
+                    Q(parent__name__icontains=term) |
+                    Q(reference=term)
+            )
+            query &= subquery
+        return self.model.objects.filter(query).order_by('name')[:20]
+
diff --git a/commorganization/management/__init__.py b/commorganization/management/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commorganization/management/__init__.py
diff --git a/commorganization/management/commands/__init__.py b/commorganization/management/commands/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commorganization/management/commands/__init__.py
diff --git a/commorganization/management/commands/import_csv_autres.py b/commorganization/management/commands/import_csv_autres.py
new file mode 100644
index 0000000..6874cad
--- /dev/null
+++ b/commorganization/management/commands/import_csv_autres.py
@@ -0,0 +1,107 @@
+import csv
+import sys
+
+from django.core.management.base import BaseCommand
+
+from commorganization.models import Organization, OrganizationType, Target
+
+header_len = 1
+expected_header = [
+    'Secteur', 'Type', 'Nom', 'URL', 'Localisation siège (département)',
+    'Localisation siège (code postal)', 'Localisation siège (commune)']
+
+
+class Command(BaseCommand):
+    help = 'Import depuis CSV communes'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_file')
+        parser.add_argument(
+            '--quiet', dest='quiet', action='store_true',
+            help='Quiet output')
+
+    def handle(self, *args, **options):
+        csv_file = options['csv_file']
+        quiet = options['quiet']
+        if not quiet:
+            sys.stdout.write('* opening file {}\n'.format(csv_file))
+        nb_created = 0
+        nb_organization_created = 0
+        nb_tt_created = 0
+        with open(csv_file, 'r') as csvfile:
+            reader = csv.reader(csvfile)
+            for idx, row in enumerate(reader):
+                if idx < header_len:
+                    if not idx:
+                        if expected_header != row:
+                            sys.stdout.write('ERROR: expected header differs '
+                                             'from the one provided\n')
+                            sys.stdout.write('* expected header is:\n')
+                            sys.stdout.write(str(expected_header))
+                            sys.stdout.write('\n* header provided is:\n')
+                            sys.stdout.write(str(row) + "\n")
+                            return
+                    continue
+                sec, sec_tpe, name, site, address = row[0:5]
+                address = address.strip()
+                if " " in address:
+                    organization_name = " ".join(address.split(" ")[1:])
+                else:
+                    organization_name = address
+
+                if not quiet:
+                    sys.stdout.write('-> processing line %d.\r' % (idx + 1))
+                    sys.stdout.flush()
+
+                tpe, c = OrganizationType.objects.get_or_create(
+                    name=sec.strip(),
+                    parent=None
+                )
+                if c:
+                    nb_tt_created += 1
+                tpe, c = OrganizationType.objects.get_or_create(
+                    name=sec_tpe.strip(),
+                    parent=tpe
+                )
+                if c:
+                    nb_tt_created += 1
+
+                organization_values = {
+                    "organization_type": tpe,
+                    "name": "{} - {}".format(name.strip(), organization_name)
+                }
+
+                default = dict()
+                default["address"] = address
+                organization_values["defaults"] = default
+
+                organization, c = Organization.objects.get_or_create(
+                    **organization_values)
+                if c:
+                    nb_organization_created += 1
+                else:
+                    for k in default.keys():
+                        setattr(organization, k, default[k])
+                    organization.save()
+
+                site = site.strip()
+                if site == "0" or "." not in site:
+                    site = None
+                elif not site.startswith('http'):
+                    site = "http://" + site
+                values = {
+                    "name": name.strip(),
+                    "organization": organization,
+                    "url": site,
+                }
+                target, created = Target.objects.get_or_create(**values)
+                if created:
+                    nb_created += 1
+        if not quiet:
+            sys.stdout.write(
+                '\n* {} organization types created.\n'.format(nb_tt_created))
+            sys.stdout.write(
+                '* {} organizations created.\n'.format(nb_organization_created))
+            sys.stdout.write(
+                '* {} targets created.\n'.format(nb_created))
+            sys.stdout.flush()
diff --git a/commorganization/management/commands/import_csv_communes.py b/commorganization/management/commands/import_csv_communes.py
new file mode 100644
index 0000000..d0ff23a
--- /dev/null
+++ b/commorganization/management/commands/import_csv_communes.py
@@ -0,0 +1,167 @@
+import csv
+import sys
+
+from django.core.management.base import BaseCommand
+
+from commorganization.models import Area, AreaType, Organization, \
+    OrganizationType, Target
+
+header_len = 1
+expected_header = [
+    'DÉPARTEMENT', 'NOM EPCI OU MAIRIE', 'CODE POSTAL', 'COMMUNE',
+    'C.C. / C.A. / C.U.', 'SITE INTERNET', 'POPULATION COMMUNAUTAIRE',
+    'Type de collectivité', 'Code INSEE']
+
+
+class Command(BaseCommand):
+    help = 'Import depuis CSV communes'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_file')
+        parser.add_argument(
+            '--quiet', dest='quiet', action='store_true',
+            help='Quiet output')
+
+    def handle(self, *args, **options):
+        csv_file = options['csv_file']
+        quiet = options['quiet']
+        if not quiet:
+            sys.stdout.write('* opening file {}\n'.format(csv_file))
+        nb_created = 0
+        nb_organization_created = 0
+        nb_area_created = 0
+        nb_tt_created = 0
+        nb_at_created = 0
+        with open(csv_file, 'r') as csvfile:
+            reader = csv.reader(csvfile)
+            for idx, row in enumerate(reader):
+                if idx < header_len:
+                    if not idx:
+                        if expected_header != row:
+                            sys.stdout.write('ERROR: expected header differs '
+                                             'from the one provided\n')
+                            sys.stdout.write('* expected header is:\n')
+                            sys.stdout.write(str(expected_header))
+                            sys.stdout.write('\n* header provided is:\n')
+                            sys.stdout.write(str(row) + "\n")
+                            return
+                    continue
+                dpt, name, code_postal, commune, comcom, site, pop = row[0:7]
+                type_coll, insee = row[7:9]
+                if insee.strip() == "NA":
+                    insee = ""
+                if not quiet:
+                    sys.stdout.write('-> processing line %d.\r' % (idx + 1))
+                    sys.stdout.flush()
+                try:
+                    pop = int(pop.replace(" ", ""))
+                except ValueError:
+                    pop = None
+
+                p_tpe, c = OrganizationType.objects.get_or_create(
+                    name=type_coll.strip()
+                )
+                if c:
+                    nb_tt_created += 1
+                if comcom.strip():
+                    tpe, c = OrganizationType.objects.get_or_create(
+                        name=comcom.strip(),
+                        parent=p_tpe
+                    )
+                    if c:
+                        nb_tt_created += 1
+                else:
+                    tpe = p_tpe
+                atpe, c = AreaType.objects.get_or_create(
+                    name=str(tpe)
+                )
+                if c:
+                    nb_at_created += 1
+
+                top_area = None
+                if dpt.strip():
+                    dpt_tpe, c = AreaType.objects.get_or_create(
+                        name="Département",
+                    )
+                    if c:
+                        nb_tt_created += 1
+                    top_area, c = Area.objects.get_or_create(
+                        name=dpt.strip(),
+                        area_type=dpt_tpe
+                    )
+                    if c:
+                        nb_area_created += 1
+
+                area_values = {"area_type": atpe}
+                area_defaults = {"population": pop, "parent": top_area}
+                if insee.strip():
+                    area_values['reference'] = insee.strip()
+                    area_defaults['name'] = commune.strip()
+                else:
+                    area_values['name'] = name.strip()
+                area_values["defaults"] = area_defaults
+
+                area, c = Area.objects.get_or_create(
+                    **area_values)
+                if c:
+                    nb_area_created += 1
+                else:
+                    for k in area_defaults:
+                        setattr(area, k, area_defaults[k])
+                    area.save()
+
+                organization_values = {
+                    "organization_type": tpe,
+                    "name": name,
+                    "area": area,
+                }
+                default = dict()
+                default["address"] = "{} {}".format(code_postal.strip() or "",
+                                                    commune.strip() or "")
+                organization_values["defaults"] = default
+
+                organization, c = Organization.objects.get_or_create(
+                    **organization_values)
+                if c:
+                    nb_organization_created += 1
+                else:
+                    for k in default:
+                        setattr(organization, k, default[k])
+                    organization.save()
+
+                site = site.strip()
+                if site == "0" or "." not in site:
+                    site = None
+                elif not site.startswith('http'):
+                    site = "http://" + site
+                values = {
+                    "name": name.strip(),
+                    "organization": organization,
+                    "url": site,
+                }
+
+                target, created = Target.objects.get_or_create(**values)
+                if created:
+                    nb_created += 1
+        if not quiet:
+            sys.stdout.write("\n")
+            if nb_at_created:
+                sys.stdout.write(
+                    '* {} area types created.\n'.format(nb_at_created)
+                )
+            if nb_area_created:
+                sys.stdout.write(
+                    '* {} areas created.\n'.format(
+                        nb_area_created))
+            if nb_tt_created:
+                sys.stdout.write(
+                    '* {} organization types created.\n'.format(nb_tt_created)
+                )
+            if nb_organization_created:
+                sys.stdout.write(
+                    '* {} organizations created.\n'.format(
+                        nb_organization_created))
+            if nb_created:
+                sys.stdout.write(
+                    '* {} targets created.\n'.format(nb_created))
+            sys.stdout.flush()
diff --git a/commorganization/migrations/0001_initial.py b/commorganization/migrations/0001_initial.py
new file mode 100644
index 0000000..891f297
--- /dev/null
+++ b/commorganization/migrations/0001_initial.py
@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2019-07-31 08:21
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Area',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('population', models.IntegerField(blank=True, null=True, verbose_name='Population')),
+                ('reference', models.CharField(blank=True, help_text='For instance, INSEE code for towns', max_length=100, null=True, verbose_name='Reference')),
+            ],
+            options={
+                'ordering': ('name',),
+                'verbose_name_plural': 'Areas',
+                'verbose_name': 'Area',
+            },
+        ),
+        migrations.CreateModel(
+            name='AreaType',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+            ],
+            options={
+                'ordering': ('name',),
+                'verbose_name_plural': 'Area types',
+                'verbose_name': 'Area type',
+            },
+        ),
+        migrations.CreateModel(
+            name='Organization',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('address', models.TextField(blank=True, null=True, verbose_name='Address')),
+                ('area', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commorganization.Area', verbose_name='Area')),
+            ],
+            options={
+                'ordering': ('name',),
+                'verbose_name_plural': 'Organizations',
+                'verbose_name': 'Organization',
+            },
+        ),
+        migrations.CreateModel(
+            name='OrganizationType',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commorganization.OrganizationType', verbose_name='Parent')),
+            ],
+            options={
+                'ordering': ('parent__name', 'name'),
+                'verbose_name_plural': 'Organization types',
+                'verbose_name': 'Organization type',
+            },
+        ),
+        migrations.CreateModel(
+            name='Target',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('url', models.URLField(blank=True, null=True, verbose_name='URL')),
+                ('organization', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commorganization.Organization', verbose_name='Organization')),
+            ],
+            options={
+                'ordering': ('name',),
+                'verbose_name_plural': 'Targets',
+                'verbose_name': 'Target',
+            },
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='organization_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commorganization.OrganizationType', verbose_name='Type'),
+        ),
+        migrations.AddField(
+            model_name='area',
+            name='area_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commorganization.AreaType', verbose_name='Type'),
+        ),
+        migrations.AddField(
+            model_name='area',
+            name='parent',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commorganization.Area', verbose_name='Parent'),
+        ),
+    ]
diff --git a/commorganization/migrations/__init__.py b/commorganization/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/commorganization/migrations/__init__.py
diff --git a/commorganization/models.py b/commorganization/models.py
new file mode 100644
index 0000000..a2b0994
--- /dev/null
+++ b/commorganization/models.py
@@ -0,0 +1,93 @@
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+
+class AreaType(models.Model):
+    name = models.CharField(_("Name"), max_length=200)
+
+    class Meta:
+        verbose_name = _("Area type")
+        verbose_name_plural = _("Area types")
+        ordering = ("name", )
+
+    def __str__(self):
+        return self.name
+
+
+class Area(models.Model):
+    name = models.CharField(verbose_name=_("Name"), max_length=200)
+    area_type = models.ForeignKey(
+        AreaType, on_delete=models.CASCADE, verbose_name=_("Type"))
+    population = models.IntegerField(verbose_name=_("Population"), blank=True,
+                                     null=True)
+    reference = models.CharField(
+        verbose_name=_("Reference"), max_length=100, blank=True, null=True,
+        help_text=_("For instance, INSEE code for towns")
+    )
+    parent = models.ForeignKey("Area", verbose_name=_("Parent"),
+                               blank=True, null=True)
+
+    class Meta:
+        verbose_name = _("Area")
+        verbose_name_plural = _("Areas")
+        ordering = ("name", )
+
+    def __str__(self):
+        if not self.parent:
+            return self.name
+        return "{} / {}".format(self.parent, self.name)
+
+
+class OrganizationType(models.Model):
+    name = models.CharField(_("Name"), max_length=200)
+    parent = models.ForeignKey("OrganizationType", verbose_name=_("Parent"),
+                               blank=True, null=True)
+
+    class Meta:
+        verbose_name = _("Organization type")
+        verbose_name_plural = _("Organization types")
+        ordering = ("parent__name", "name", )
+
+    def __str__(self):
+        if not self.parent:
+            return self.name
+        return "{} / {}".format(self.parent, self.name)
+
+
+class Organization(models.Model):
+    name = models.CharField(verbose_name=_("Name"), max_length=200)
+    area = models.ForeignKey(
+        Area, on_delete=models.SET_NULL, verbose_name=_("Area"), blank=True,
+        null=True
+    )
+    organization_type = models.ForeignKey(
+        OrganizationType, on_delete=models.CASCADE, verbose_name=_("Type"))
+    address = models.TextField(verbose_name=_("Address"), blank=True, null=True)
+
+    class Meta:
+        verbose_name = _("Organization")
+        verbose_name_plural = _("Organizations")
+        ordering = ("name", )
+
+    def __str__(self):
+        if not self.area:
+            return "{} ({})".format(
+                self.name, self.organization_type)
+        return "{} - {} ({})".format(
+            self.name, self.area, self.organization_type)
+
+
+class Target(models.Model):
+    name = models.CharField(verbose_name=_("Name"), max_length=200)
+    url = models.URLField(verbose_name=_("URL"), blank=True, null=True)
+    organization = models.ForeignKey(
+        Organization, verbose_name=_("Organization"), blank=True,
+        null=True, on_delete=models.SET_NULL)
+
+    class Meta:
+        verbose_name = _("Target")
+        verbose_name_plural = _("Targets")
+        ordering = ("name",)
+
+    def __str__(self):
+        return "{} ({})".format(self.name, self.organization)
diff --git a/commorganization/tests.py b/commorganization/tests.py
new file mode 100644
index 0000000..7ce503c
--- /dev/null
+++ b/commorganization/tests.py
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
diff --git a/commorganization/views.py b/commorganization/views.py
new file mode 100644
index 0000000..91ea44a
--- /dev/null
+++ b/commorganization/views.py
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.