From d2e3ce3fe1bca021190c8257367e8a444596724f Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Wed, 31 Jul 2019 10:46:50 +0200 Subject: Models refactoring --- commcrawler/admin.py | 115 +------------- commcrawler/lookups.py | 41 +---- commcrawler/management/__init__.py | 0 commcrawler/management/commands/__init__.py | 0 .../management/commands/import_csv_autres.py | 107 ------------- .../management/commands/import_csv_communes.py | 167 --------------------- commcrawler/migrations/0001_initial.py | 119 ++------------- commcrawler/models.py | 93 +----------- commonnet/settings.py | 6 +- commorganization/__init__.py | 0 commorganization/admin.py | 112 ++++++++++++++ commorganization/apps.py | 7 + commorganization/lookups.py | 49 ++++++ commorganization/management/__init__.py | 0 commorganization/management/commands/__init__.py | 0 .../management/commands/import_csv_autres.py | 107 +++++++++++++ .../management/commands/import_csv_communes.py | 167 +++++++++++++++++++++ commorganization/migrations/0001_initial.py | 99 ++++++++++++ commorganization/migrations/__init__.py | 0 commorganization/models.py | 93 ++++++++++++ commorganization/tests.py | 3 + commorganization/views.py | 3 + 22 files changed, 665 insertions(+), 623 deletions(-) delete mode 100644 commcrawler/management/__init__.py delete mode 100644 commcrawler/management/commands/__init__.py delete mode 100644 commcrawler/management/commands/import_csv_autres.py delete mode 100644 commcrawler/management/commands/import_csv_communes.py create mode 100644 commorganization/__init__.py create mode 100644 commorganization/admin.py create mode 100644 commorganization/apps.py create mode 100644 commorganization/lookups.py create mode 100644 commorganization/management/__init__.py create mode 100644 commorganization/management/commands/__init__.py create mode 100644 commorganization/management/commands/import_csv_autres.py create mode 100644 commorganization/management/commands/import_csv_communes.py create mode 100644 commorganization/migrations/0001_initial.py create mode 100644 commorganization/migrations/__init__.py create mode 100644 commorganization/models.py create mode 100644 commorganization/tests.py create mode 100644 commorganization/views.py diff --git a/commcrawler/admin.py b/commcrawler/admin.py index 7ff1eed..ea4ca14 100644 --- a/commcrawler/admin.py +++ b/commcrawler/admin.py @@ -1,117 +1,10 @@ from ajax_select import make_ajax_form -from django.contrib import admin, messages -from django.contrib.auth.admin import GroupAdmin, UserAdmin -from django.contrib.auth.models import Group, User -from django import forms -from django.http import HttpResponseRedirect -from django.shortcuts import render -from django.utils.translation import ugettext_lazy as _ +from django.contrib import admin from commonnet.admin_site import admin_site from commcrawler import models -admin_site.register(User, UserAdmin) -admin_site.register(Group, GroupAdmin) - - -class AreaTypeAdmin(admin.ModelAdmin): - model = models.AreaType - list_display = ('name', ) - - -admin_site.register(models.AreaType, AreaTypeAdmin) - - -class AreaAdmin(admin.ModelAdmin): - model = models.Area - list_display = ('name', 'area_type', 'reference', 'parent') - list_filter = ('area_type',) - search_fields = ['name', 'parent__name'] - - -admin_site.register(models.Area, AreaAdmin) - - -class OrganizationTypeAdmin(admin.ModelAdmin): - model = models.OrganizationType - list_display = ('name', 'parent') - list_filter = ('parent',) - - -admin_site.register(models.OrganizationType, OrganizationTypeAdmin) - - -class OrganizationAdmin(admin.ModelAdmin): - model = models.Organization - list_display = ('name', 'organization_type', 'area') - list_filter = ('organization_type',) - search_fields = ['name'] - form = make_ajax_form(model, {'area': 'area'}) - - -admin_site.register(models.Organization, OrganizationAdmin) - - -class AddToCrawlForm(forms.Form): - crawl = forms.ChoiceField(label=_("Crawl"), choices=tuple()) - - def __init__(self, *args, **kwargs): - super(AddToCrawlForm, self).__init__(*args, **kwargs) - self.fields["crawl"].choices = [(None, "--")] + [ - (c.pk, str(c)) - for c in models.Crawl.objects.filter(status="C").all() - ] - - -class TargetAdmin(admin.ModelAdmin): - list_display = ('name', 'url', 'organization') - model = models.Target - list_filter = ('organization__organization_type',) - form = make_ajax_form(model, {'organization': 'organization'}) - actions = ['add_to_crawl'] - - def add_to_crawl(self, request, queryset): - if 'apply' in request.POST: - form = AddToCrawlForm(request.POST) - if form.is_valid(): - crawl = None - try: - crawl = models.Crawl.objects.get( - pk=form.cleaned_data["crawl"], - status="C" - ) - except models.Crawl.DoesNotExist: - pass - if crawl: - nb_added = 0 - targets = [ - c["id"] for c in crawl.targets.values("id") - ] - for target in queryset.all(): - if target.pk not in targets: - crawl.targets.add(target) - nb_added += 1 - messages.add_message( - request, messages.INFO, - str(_("{} new targets added to {}")).format( - nb_added, str(crawl) - ) - ) - return HttpResponseRedirect(request.get_full_path()) - else: - form = AddToCrawlForm() - return render( - request, 'admin/add_to_crawl.html', - context={"form": form, "query_nb": queryset.count(), - "items": [v['pk'] for v in queryset.values('pk').all()]}) - - add_to_crawl.short_description = _("Add to crawl") - - -admin_site.register(models.Target, TargetAdmin) - - class CrawlAdmin(admin.ModelAdmin): model = models.Crawl list_display = ("name", "status", "target_nb", "created", "started", @@ -125,16 +18,17 @@ admin_site.register(models.Crawl, CrawlAdmin) class CrawlResultAdmin(admin.ModelAdmin): - model = models.Crawl + model = models.CrawlResult list_display = ("target", "crawl", "is_online") list_filter = ("crawl",) + form = make_ajax_form(model, {'target': 'target'}) admin_site.register(models.CrawlResult, CrawlResultAdmin) class CrawlLinkAdmin(admin.ModelAdmin): - model = models.Crawl + model = models.CrawlLink list_display = ("result", "link") @@ -146,6 +40,7 @@ class CrawlRelationAdmin(admin.ModelAdmin): list_display = ("crawl", "source", "destination", "number") list_filter = ("crawl",) search_fields = ["source__name", "destination__name"] + form = make_ajax_form(model, {'source': 'target', 'destination': 'target'}) admin_site.register(models.CrawlRelation, CrawlRelationAdmin) diff --git a/commcrawler/lookups.py b/commcrawler/lookups.py index f8d4897..106a69a 100644 --- a/commcrawler/lookups.py +++ b/commcrawler/lookups.py @@ -1,28 +1,15 @@ -from ajax_select import register, LookupChannel as BaseLookupChannel +from ajax_select import register from django.db.models import Q -from . import models - - -class LookupChannel(BaseLookupChannel): - def get_objects(self, items): - # TODO: why IDs are not given here? M2M issue - ids = [] - for item in items: - if hasattr(item, 'pk'): - ids.append(item.pk) - else: - ids.append(item) - return super(LookupChannel, self).get_objects(ids) +from commorganization.lookups import LookupChannel - def format_item_display(self, item): - return u"%s" % str(item) +from . import models -@register('organization') -class OrganizationLookup(LookupChannel): - model = models.Organization +@register('target') +class TargetLookup(LookupChannel): + model = models.Target def get_query(self, q, request): query = Q() @@ -31,19 +18,3 @@ class OrganizationLookup(LookupChannel): query &= subquery return self.model.objects.filter(query).order_by('name')[:20] - -@register('area') -class AreaLookup(LookupChannel): - model = models.Area - - def get_query(self, q, request): - query = Q() - for term in q.strip().split(' '): - subquery = ( - Q(name__icontains=term) | - Q(parent__name__icontains=term) | - Q(reference=term) - ) - query &= subquery - return self.model.objects.filter(query).order_by('name')[:20] - diff --git a/commcrawler/management/__init__.py b/commcrawler/management/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/commcrawler/management/commands/__init__.py b/commcrawler/management/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/commcrawler/management/commands/import_csv_autres.py b/commcrawler/management/commands/import_csv_autres.py deleted file mode 100644 index 2dd177c..0000000 --- a/commcrawler/management/commands/import_csv_autres.py +++ /dev/null @@ -1,107 +0,0 @@ -import csv -import sys - -from django.core.management.base import BaseCommand - -from commcrawler.models import Organization, OrganizationType, Target - -header_len = 1 -expected_header = [ - 'Secteur', 'Type', 'Nom', 'URL', 'Localisation siège (département)', - 'Localisation siège (code postal)', 'Localisation siège (commune)'] - - -class Command(BaseCommand): - help = 'Import depuis CSV communes' - - def add_arguments(self, parser): - parser.add_argument('csv_file') - parser.add_argument( - '--quiet', dest='quiet', action='store_true', - help='Quiet output') - - def handle(self, *args, **options): - csv_file = options['csv_file'] - quiet = options['quiet'] - if not quiet: - sys.stdout.write('* opening file {}\n'.format(csv_file)) - nb_created = 0 - nb_organization_created = 0 - nb_tt_created = 0 - with open(csv_file, 'r') as csvfile: - reader = csv.reader(csvfile) - for idx, row in enumerate(reader): - if idx < header_len: - if not idx: - if expected_header != row: - sys.stdout.write('ERROR: expected header differs ' - 'from the one provided\n') - sys.stdout.write('* expected header is:\n') - sys.stdout.write(str(expected_header)) - sys.stdout.write('\n* header provided is:\n') - sys.stdout.write(str(row) + "\n") - return - continue - sec, sec_tpe, name, site, address = row[0:5] - address = address.strip() - if " " in address: - organization_name = " ".join(address.split(" ")[1:]) - else: - organization_name = address - - if not quiet: - sys.stdout.write('-> processing line %d.\r' % (idx + 1)) - sys.stdout.flush() - - tpe, c = OrganizationType.objects.get_or_create( - name=sec.strip(), - parent=None - ) - if c: - nb_tt_created += 1 - tpe, c = OrganizationType.objects.get_or_create( - name=sec_tpe.strip(), - parent=tpe - ) - if c: - nb_tt_created += 1 - - organization_values = { - "organization_type": tpe, - "name": "{} - {}".format(name.strip(), organization_name) - } - - default = dict() - default["address"] = address - organization_values["defaults"] = default - - organization, c = Organization.objects.get_or_create( - **organization_values) - if c: - nb_organization_created += 1 - else: - for k in default.keys(): - setattr(organization, k, default[k]) - organization.save() - - site = site.strip() - if site == "0" or "." not in site: - site = None - elif not site.startswith('http'): - site = "http://" + site - values = { - "name": name.strip(), - "organization": organization, - "url": site, - } - target, created = Target.objects.get_or_create(**values) - if created: - nb_created += 1 - if not quiet: - sys.stdout.write( - '\n* {} organization types created.\n'.format(nb_tt_created)) - sys.stdout.write( - '* {} organizations created.\n'.format(nb_organization_created)) - sys.stdout.write( - '* {} targets created.\n'.format(nb_created)) - sys.stdout.flush() diff --git a/commcrawler/management/commands/import_csv_communes.py b/commcrawler/management/commands/import_csv_communes.py deleted file mode 100644 index 4024067..0000000 --- a/commcrawler/management/commands/import_csv_communes.py +++ /dev/null @@ -1,167 +0,0 @@ -import csv -import sys - -from django.core.management.base import BaseCommand - -from commcrawler.models import Area, AreaType, Organization, OrganizationType,\ - Target - -header_len = 1 -expected_header = [ - 'DÉPARTEMENT', 'NOM EPCI OU MAIRIE', 'CODE POSTAL', 'COMMUNE', - 'C.C. / C.A. / C.U.', 'SITE INTERNET', 'POPULATION COMMUNAUTAIRE', - 'Type de collectivité', 'Code INSEE'] - - -class Command(BaseCommand): - help = 'Import depuis CSV communes' - - def add_arguments(self, parser): - parser.add_argument('csv_file') - parser.add_argument( - '--quiet', dest='quiet', action='store_true', - help='Quiet output') - - def handle(self, *args, **options): - csv_file = options['csv_file'] - quiet = options['quiet'] - if not quiet: - sys.stdout.write('* opening file {}\n'.format(csv_file)) - nb_created = 0 - nb_organization_created = 0 - nb_area_created = 0 - nb_tt_created = 0 - nb_at_created = 0 - with open(csv_file, 'r') as csvfile: - reader = csv.reader(csvfile) - for idx, row in enumerate(reader): - if idx < header_len: - if not idx: - if expected_header != row: - sys.stdout.write('ERROR: expected header differs ' - 'from the one provided\n') - sys.stdout.write('* expected header is:\n') - sys.stdout.write(str(expected_header)) - sys.stdout.write('\n* header provided is:\n') - sys.stdout.write(str(row) + "\n") - return - continue - dpt, name, code_postal, commune, comcom, site, pop = row[0:7] - type_coll, insee = row[7:9] - if insee.strip() == "NA": - insee = "" - if not quiet: - sys.stdout.write('-> processing line %d.\r' % (idx + 1)) - sys.stdout.flush() - try: - pop = int(pop.replace(" ", "")) - except ValueError: - pop = None - - p_tpe, c = OrganizationType.objects.get_or_create( - name=type_coll.strip() - ) - if c: - nb_tt_created += 1 - if comcom.strip(): - tpe, c = OrganizationType.objects.get_or_create( - name=comcom.strip(), - parent=p_tpe - ) - if c: - nb_tt_created += 1 - else: - tpe = p_tpe - atpe, c = AreaType.objects.get_or_create( - name=str(tpe) - ) - if c: - nb_at_created += 1 - - top_area = None - if dpt.strip(): - dpt_tpe, c = AreaType.objects.get_or_create( - name="Département", - ) - if c: - nb_tt_created += 1 - top_area, c = Area.objects.get_or_create( - name=dpt.strip(), - area_type=dpt_tpe - ) - if c: - nb_area_created += 1 - - area_values = {"area_type": atpe} - area_defaults = {"population": pop, "parent": top_area} - if insee.strip(): - area_values['reference'] = insee.strip() - area_defaults['name'] = commune.strip() - else: - area_values['name'] = name.strip() - area_values["defaults"] = area_defaults - - area, c = Area.objects.get_or_create( - **area_values) - if c: - nb_area_created += 1 - else: - for k in area_defaults: - setattr(area, k, area_defaults[k]) - area.save() - - organization_values = { - "organization_type": tpe, - "name": name, - "area": area, - } - default = dict() - default["address"] = "{} {}".format(code_postal.strip() or "", - commune.strip() or "") - organization_values["defaults"] = default - - organization, c = Organization.objects.get_or_create( - **organization_values) - if c: - nb_organization_created += 1 - else: - for k in default: - setattr(organization, k, default[k]) - organization.save() - - site = site.strip() - if site == "0" or "." not in site: - site = None - elif not site.startswith('http'): - site = "http://" + site - values = { - "name": name.strip(), - "organization": organization, - "url": site, - } - - target, created = Target.objects.get_or_create(**values) - if created: - nb_created += 1 - if not quiet: - sys.stdout.write("\n") - if nb_at_created: - sys.stdout.write( - '* {} area types created.\n'.format(nb_at_created) - ) - if nb_area_created: - sys.stdout.write( - '* {} areas created.\n'.format( - nb_area_created)) - if nb_tt_created: - sys.stdout.write( - '* {} organization types created.\n'.format(nb_tt_created) - ) - if nb_organization_created: - sys.stdout.write( - '* {} organizations created.\n'.format( - nb_organization_created)) - if nb_created: - sys.stdout.write( - '* {} targets created.\n'.format(nb_created)) - sys.stdout.flush() diff --git a/commcrawler/migrations/0001_initial.py b/commcrawler/migrations/0001_initial.py index 1468584..0b4e561 100644 --- a/commcrawler/migrations/0001_initial.py +++ b/commcrawler/migrations/0001_initial.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2019-07-31 07:48 +# Generated by Django 1.11 on 2019-07-31 08:21 from __future__ import unicode_literals import datetime @@ -12,35 +12,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ + ('commorganization', '0001_initial'), ] operations = [ - migrations.CreateModel( - name='Area', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('population', models.IntegerField(blank=True, null=True, verbose_name='Population')), - ('reference', models.CharField(blank=True, help_text='For instance, INSEE code for towns', max_length=100, null=True, verbose_name='Reference')), - ], - options={ - 'verbose_name': 'Area', - 'verbose_name_plural': 'Areas', - 'ordering': ('name',), - }, - ), - migrations.CreateModel( - name='AreaType', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ], - options={ - 'verbose_name': 'Area type', - 'verbose_name_plural': 'Area types', - 'ordering': ('name',), - }, - ), migrations.CreateModel( name='Crawl', fields=[ @@ -50,11 +25,12 @@ class Migration(migrations.Migration): ('started', models.DateTimeField(blank=True, null=True, verbose_name='Start date')), ('ended', models.DateTimeField(blank=True, null=True, verbose_name='End date')), ('status', models.CharField(choices=[('C', 'Created'), ('P', 'In progress'), ('F', 'Finished')], default='C', max_length=1)), + ('targets', models.ManyToManyField(blank=True, to='commorganization.Target')), ], options={ - 'verbose_name': 'Crawl', - 'verbose_name_plural': 'Crawls', 'ordering': ('created', 'name'), + 'verbose_name_plural': 'Crawls', + 'verbose_name': 'Crawl', }, ), migrations.CreateModel( @@ -64,8 +40,8 @@ class Migration(migrations.Migration): ('link', models.URLField(verbose_name='Link')), ], options={ - 'verbose_name': 'Crawl link', 'verbose_name_plural': 'Crawl links', + 'verbose_name': 'Crawl link', }, ), migrations.CreateModel( @@ -74,10 +50,12 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('number', models.IntegerField(default=1, verbose_name='Number')), ('crawl', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.Crawl', verbose_name='Crawl')), + ('destination', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='relation_destination', to='commorganization.Target', verbose_name='Destination')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='relation_source', to='commorganization.Target', verbose_name='Source')), ], options={ - 'verbose_name': 'Crawl relation', 'verbose_name_plural': 'Crawl relations', + 'verbose_name': 'Crawl relation', }, ), migrations.CreateModel( @@ -102,91 +80,16 @@ class Migration(migrations.Migration): ('is_online', models.BooleanField(default=False, verbose_name='Website is online')), ('redirection', models.URLField(blank=True, null=True, verbose_name='Redirection')), ('crawl', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.Crawl', verbose_name='Crawl')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commorganization.Target', verbose_name='Target')), ], options={ - 'verbose_name': 'Crawl result', 'verbose_name_plural': 'Crawl results', + 'verbose_name': 'Crawl result', }, ), - migrations.CreateModel( - name='Organization', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('address', models.TextField(blank=True, null=True, verbose_name='Address')), - ('area', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commcrawler.Area', verbose_name='Area')), - ], - options={ - 'verbose_name': 'Organization', - 'verbose_name_plural': 'Organizations', - 'ordering': ('name',), - }, - ), - migrations.CreateModel( - name='OrganizationType', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commcrawler.OrganizationType', verbose_name='Parent')), - ], - options={ - 'verbose_name': 'Organization type', - 'verbose_name_plural': 'Organization types', - 'ordering': ('parent__name', 'name'), - }, - ), - migrations.CreateModel( - name='Target', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('url', models.URLField(blank=True, null=True, verbose_name='URL')), - ('organization', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commcrawler.Organization', verbose_name='Organization')), - ], - options={ - 'verbose_name': 'Target', - 'verbose_name_plural': 'Targets', - 'ordering': ('name',), - }, - ), - migrations.AddField( - model_name='organization', - name='organization_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.OrganizationType', verbose_name='Type'), - ), - migrations.AddField( - model_name='crawlresult', - name='target', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.Target', verbose_name='Target'), - ), - migrations.AddField( - model_name='crawlrelation', - name='destination', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='relation_destination', to='commcrawler.Target', verbose_name='Destination'), - ), - migrations.AddField( - model_name='crawlrelation', - name='source', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='relation_source', to='commcrawler.Target', verbose_name='Source'), - ), migrations.AddField( model_name='crawllink', name='result', field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.CrawlResult', verbose_name='Result'), ), - migrations.AddField( - model_name='crawl', - name='targets', - field=models.ManyToManyField(blank=True, to='commcrawler.Target'), - ), - migrations.AddField( - model_name='area', - name='area_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.AreaType', verbose_name='Type'), - ), - migrations.AddField( - model_name='area', - name='parent', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commcrawler.Area', verbose_name='Parent'), - ), ] diff --git a/commcrawler/models.py b/commcrawler/models.py index c15ce7e..bf73ce3 100644 --- a/commcrawler/models.py +++ b/commcrawler/models.py @@ -3,96 +3,7 @@ import datetime from django.db import models from django.utils.translation import ugettext_lazy as _ - -class AreaType(models.Model): - name = models.CharField(_("Name"), max_length=200) - - class Meta: - verbose_name = _("Area type") - verbose_name_plural = _("Area types") - ordering = ("name", ) - - def __str__(self): - return self.name - - -class Area(models.Model): - name = models.CharField(verbose_name=_("Name"), max_length=200) - area_type = models.ForeignKey( - AreaType, on_delete=models.CASCADE, verbose_name=_("Type")) - population = models.IntegerField(verbose_name=_("Population"), blank=True, - null=True) - reference = models.CharField( - verbose_name=_("Reference"), max_length=100, blank=True, null=True, - help_text=_("For instance, INSEE code for towns") - ) - parent = models.ForeignKey("Area", verbose_name=_("Parent"), - blank=True, null=True) - - class Meta: - verbose_name = _("Area") - verbose_name_plural = _("Areas") - ordering = ("name", ) - - def __str__(self): - if not self.parent: - return self.name - return "{} / {}".format(self.parent, self.name) - - -class OrganizationType(models.Model): - name = models.CharField(_("Name"), max_length=200) - parent = models.ForeignKey("OrganizationType", verbose_name=_("Parent"), - blank=True, null=True) - - class Meta: - verbose_name = _("Organization type") - verbose_name_plural = _("Organization types") - ordering = ("parent__name", "name", ) - - def __str__(self): - if not self.parent: - return self.name - return "{} / {}".format(self.parent, self.name) - - -class Organization(models.Model): - name = models.CharField(verbose_name=_("Name"), max_length=200) - area = models.ForeignKey( - Area, on_delete=models.SET_NULL, verbose_name=_("Area"), blank=True, - null=True - ) - organization_type = models.ForeignKey( - OrganizationType, on_delete=models.CASCADE, verbose_name=_("Type")) - address = models.TextField(verbose_name=_("Address"), blank=True, null=True) - - class Meta: - verbose_name = _("Organization") - verbose_name_plural = _("Organizations") - ordering = ("name", ) - - def __str__(self): - if not self.area: - return "{} ({})".format( - self.name, self.organization_type) - return "{} - {} ({})".format( - self.name, self.area, self.organization_type) - - -class Target(models.Model): - name = models.CharField(verbose_name=_("Name"), max_length=200) - url = models.URLField(verbose_name=_("URL"), blank=True, null=True) - organization = models.ForeignKey( - Organization, verbose_name=_("Organization"), blank=True, - null=True, on_delete=models.SET_NULL) - - class Meta: - verbose_name = _("Target") - verbose_name_plural = _("Targets") - ordering = ("name",) - - def __str__(self): - return "{} ({})".format(self.name, self.organization) +from commorganization.models import Target class Crawl(models.Model): @@ -108,7 +19,7 @@ class Crawl(models.Model): ended = models.DateTimeField( verbose_name=_("End date"), blank=True, null=True) status = models.CharField(max_length=1, choices=STATUS, default='C') - targets = models.ManyToManyField("Target", blank=True) + targets = models.ManyToManyField(Target, blank=True) class Meta: verbose_name = _("Crawl") diff --git a/commonnet/settings.py b/commonnet/settings.py index 4ca0fbf..26c5c66 100644 --- a/commonnet/settings.py +++ b/commonnet/settings.py @@ -1,12 +1,7 @@ import os -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ - # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = '!lh+r$hzd(_-aj8a2&@)34bat=w&=!k+9w%$_+&^gjhf#n6z42' @@ -19,6 +14,7 @@ ALLOWED_HOSTS = [] # Application definition INSTALLED_APPS = [ + 'commorganization.apps.CommOrganizationConfig', 'commcrawler.apps.CommCrawlerConfig', 'ajax_select', 'django.contrib.admin', diff --git a/commorganization/__init__.py b/commorganization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/commorganization/admin.py b/commorganization/admin.py new file mode 100644 index 0000000..2b3c042 --- /dev/null +++ b/commorganization/admin.py @@ -0,0 +1,112 @@ +from ajax_select import make_ajax_form +from django.contrib import admin, messages +from django.contrib.auth.admin import GroupAdmin, UserAdmin +from django.contrib.auth.models import Group, User +from django import forms +from django.http import HttpResponseRedirect +from django.shortcuts import render +from django.utils.translation import ugettext_lazy as _ + +from commonnet.admin_site import admin_site +from commorganization import models + + +admin_site.register(User, UserAdmin) +admin_site.register(Group, GroupAdmin) + + +class AreaTypeAdmin(admin.ModelAdmin): + model = models.AreaType + list_display = ('name', ) + + +admin_site.register(models.AreaType, AreaTypeAdmin) + + +class AreaAdmin(admin.ModelAdmin): + model = models.Area + list_display = ('name', 'area_type', 'reference', 'parent') + list_filter = ('area_type',) + search_fields = ['name', 'parent__name'] + + +admin_site.register(models.Area, AreaAdmin) + + +class OrganizationTypeAdmin(admin.ModelAdmin): + model = models.OrganizationType + list_display = ('name', 'parent') + list_filter = ('parent',) + + +admin_site.register(models.OrganizationType, OrganizationTypeAdmin) + + +class OrganizationAdmin(admin.ModelAdmin): + model = models.Organization + list_display = ('name', 'organization_type', 'area') + list_filter = ('organization_type',) + search_fields = ['name'] + form = make_ajax_form(model, {'area': 'area'}) + + +admin_site.register(models.Organization, OrganizationAdmin) + + +class AddToCrawlForm(forms.Form): + crawl = forms.ChoiceField(label=_("Crawl"), choices=tuple()) + + def __init__(self, *args, **kwargs): + super(AddToCrawlForm, self).__init__(*args, **kwargs) + self.fields["crawl"].choices = [(None, "--")] + [ + (c.pk, str(c)) + for c in models.Crawl.objects.filter(status="C").all() + ] + + +class TargetAdmin(admin.ModelAdmin): + list_display = ('name', 'url', 'organization') + model = models.Target + list_filter = ('organization__organization_type',) + form = make_ajax_form(model, {'organization': 'organization'}) + actions = ['add_to_crawl'] + + def add_to_crawl(self, request, queryset): + if 'apply' in request.POST: + form = AddToCrawlForm(request.POST) + if form.is_valid(): + crawl = None + try: + crawl = models.Crawl.objects.get( + pk=form.cleaned_data["crawl"], + status="C" + ) + except models.Crawl.DoesNotExist: + pass + if crawl: + nb_added = 0 + targets = [ + c["id"] for c in crawl.targets.values("id") + ] + for target in queryset.all(): + if target.pk not in targets: + crawl.targets.add(target) + nb_added += 1 + messages.add_message( + request, messages.INFO, + str(_("{} new targets added to {}")).format( + nb_added, str(crawl) + ) + ) + return HttpResponseRedirect(request.get_full_path()) + else: + form = AddToCrawlForm() + return render( + request, 'admin/add_to_crawl.html', + context={"form": form, "query_nb": queryset.count(), + "items": [v['pk'] for v in queryset.values('pk').all()]}) + + add_to_crawl.short_description = _("Add to crawl") + + +admin_site.register(models.Target, TargetAdmin) diff --git a/commorganization/apps.py b/commorganization/apps.py new file mode 100644 index 0000000..32c623d --- /dev/null +++ b/commorganization/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig +from django.utils.translation import ugettext_lazy as _ + + +class CommOrganizationConfig(AppConfig): + name = 'commorganization' + verbose_name = _("Organization") diff --git a/commorganization/lookups.py b/commorganization/lookups.py new file mode 100644 index 0000000..f8d4897 --- /dev/null +++ b/commorganization/lookups.py @@ -0,0 +1,49 @@ +from ajax_select import register, LookupChannel as BaseLookupChannel + +from django.db.models import Q + +from . import models + + +class LookupChannel(BaseLookupChannel): + def get_objects(self, items): + # TODO: why IDs are not given here? M2M issue + ids = [] + for item in items: + if hasattr(item, 'pk'): + ids.append(item.pk) + else: + ids.append(item) + return super(LookupChannel, self).get_objects(ids) + + def format_item_display(self, item): + return u"%s" % str(item) + + +@register('organization') +class OrganizationLookup(LookupChannel): + model = models.Organization + + def get_query(self, q, request): + query = Q() + for term in q.strip().split(' '): + subquery = Q(name__icontains=term) + query &= subquery + return self.model.objects.filter(query).order_by('name')[:20] + + +@register('area') +class AreaLookup(LookupChannel): + model = models.Area + + def get_query(self, q, request): + query = Q() + for term in q.strip().split(' '): + subquery = ( + Q(name__icontains=term) | + Q(parent__name__icontains=term) | + Q(reference=term) + ) + query &= subquery + return self.model.objects.filter(query).order_by('name')[:20] + diff --git a/commorganization/management/__init__.py b/commorganization/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/commorganization/management/commands/__init__.py b/commorganization/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/commorganization/management/commands/import_csv_autres.py b/commorganization/management/commands/import_csv_autres.py new file mode 100644 index 0000000..6874cad --- /dev/null +++ b/commorganization/management/commands/import_csv_autres.py @@ -0,0 +1,107 @@ +import csv +import sys + +from django.core.management.base import BaseCommand + +from commorganization.models import Organization, OrganizationType, Target + +header_len = 1 +expected_header = [ + 'Secteur', 'Type', 'Nom', 'URL', 'Localisation siège (département)', + 'Localisation siège (code postal)', 'Localisation siège (commune)'] + + +class Command(BaseCommand): + help = 'Import depuis CSV communes' + + def add_arguments(self, parser): + parser.add_argument('csv_file') + parser.add_argument( + '--quiet', dest='quiet', action='store_true', + help='Quiet output') + + def handle(self, *args, **options): + csv_file = options['csv_file'] + quiet = options['quiet'] + if not quiet: + sys.stdout.write('* opening file {}\n'.format(csv_file)) + nb_created = 0 + nb_organization_created = 0 + nb_tt_created = 0 + with open(csv_file, 'r') as csvfile: + reader = csv.reader(csvfile) + for idx, row in enumerate(reader): + if idx < header_len: + if not idx: + if expected_header != row: + sys.stdout.write('ERROR: expected header differs ' + 'from the one provided\n') + sys.stdout.write('* expected header is:\n') + sys.stdout.write(str(expected_header)) + sys.stdout.write('\n* header provided is:\n') + sys.stdout.write(str(row) + "\n") + return + continue + sec, sec_tpe, name, site, address = row[0:5] + address = address.strip() + if " " in address: + organization_name = " ".join(address.split(" ")[1:]) + else: + organization_name = address + + if not quiet: + sys.stdout.write('-> processing line %d.\r' % (idx + 1)) + sys.stdout.flush() + + tpe, c = OrganizationType.objects.get_or_create( + name=sec.strip(), + parent=None + ) + if c: + nb_tt_created += 1 + tpe, c = OrganizationType.objects.get_or_create( + name=sec_tpe.strip(), + parent=tpe + ) + if c: + nb_tt_created += 1 + + organization_values = { + "organization_type": tpe, + "name": "{} - {}".format(name.strip(), organization_name) + } + + default = dict() + default["address"] = address + organization_values["defaults"] = default + + organization, c = Organization.objects.get_or_create( + **organization_values) + if c: + nb_organization_created += 1 + else: + for k in default.keys(): + setattr(organization, k, default[k]) + organization.save() + + site = site.strip() + if site == "0" or "." not in site: + site = None + elif not site.startswith('http'): + site = "http://" + site + values = { + "name": name.strip(), + "organization": organization, + "url": site, + } + target, created = Target.objects.get_or_create(**values) + if created: + nb_created += 1 + if not quiet: + sys.stdout.write( + '\n* {} organization types created.\n'.format(nb_tt_created)) + sys.stdout.write( + '* {} organizations created.\n'.format(nb_organization_created)) + sys.stdout.write( + '* {} targets created.\n'.format(nb_created)) + sys.stdout.flush() diff --git a/commorganization/management/commands/import_csv_communes.py b/commorganization/management/commands/import_csv_communes.py new file mode 100644 index 0000000..d0ff23a --- /dev/null +++ b/commorganization/management/commands/import_csv_communes.py @@ -0,0 +1,167 @@ +import csv +import sys + +from django.core.management.base import BaseCommand + +from commorganization.models import Area, AreaType, Organization, \ + OrganizationType, Target + +header_len = 1 +expected_header = [ + 'DÉPARTEMENT', 'NOM EPCI OU MAIRIE', 'CODE POSTAL', 'COMMUNE', + 'C.C. / C.A. / C.U.', 'SITE INTERNET', 'POPULATION COMMUNAUTAIRE', + 'Type de collectivité', 'Code INSEE'] + + +class Command(BaseCommand): + help = 'Import depuis CSV communes' + + def add_arguments(self, parser): + parser.add_argument('csv_file') + parser.add_argument( + '--quiet', dest='quiet', action='store_true', + help='Quiet output') + + def handle(self, *args, **options): + csv_file = options['csv_file'] + quiet = options['quiet'] + if not quiet: + sys.stdout.write('* opening file {}\n'.format(csv_file)) + nb_created = 0 + nb_organization_created = 0 + nb_area_created = 0 + nb_tt_created = 0 + nb_at_created = 0 + with open(csv_file, 'r') as csvfile: + reader = csv.reader(csvfile) + for idx, row in enumerate(reader): + if idx < header_len: + if not idx: + if expected_header != row: + sys.stdout.write('ERROR: expected header differs ' + 'from the one provided\n') + sys.stdout.write('* expected header is:\n') + sys.stdout.write(str(expected_header)) + sys.stdout.write('\n* header provided is:\n') + sys.stdout.write(str(row) + "\n") + return + continue + dpt, name, code_postal, commune, comcom, site, pop = row[0:7] + type_coll, insee = row[7:9] + if insee.strip() == "NA": + insee = "" + if not quiet: + sys.stdout.write('-> processing line %d.\r' % (idx + 1)) + sys.stdout.flush() + try: + pop = int(pop.replace(" ", "")) + except ValueError: + pop = None + + p_tpe, c = OrganizationType.objects.get_or_create( + name=type_coll.strip() + ) + if c: + nb_tt_created += 1 + if comcom.strip(): + tpe, c = OrganizationType.objects.get_or_create( + name=comcom.strip(), + parent=p_tpe + ) + if c: + nb_tt_created += 1 + else: + tpe = p_tpe + atpe, c = AreaType.objects.get_or_create( + name=str(tpe) + ) + if c: + nb_at_created += 1 + + top_area = None + if dpt.strip(): + dpt_tpe, c = AreaType.objects.get_or_create( + name="Département", + ) + if c: + nb_tt_created += 1 + top_area, c = Area.objects.get_or_create( + name=dpt.strip(), + area_type=dpt_tpe + ) + if c: + nb_area_created += 1 + + area_values = {"area_type": atpe} + area_defaults = {"population": pop, "parent": top_area} + if insee.strip(): + area_values['reference'] = insee.strip() + area_defaults['name'] = commune.strip() + else: + area_values['name'] = name.strip() + area_values["defaults"] = area_defaults + + area, c = Area.objects.get_or_create( + **area_values) + if c: + nb_area_created += 1 + else: + for k in area_defaults: + setattr(area, k, area_defaults[k]) + area.save() + + organization_values = { + "organization_type": tpe, + "name": name, + "area": area, + } + default = dict() + default["address"] = "{} {}".format(code_postal.strip() or "", + commune.strip() or "") + organization_values["defaults"] = default + + organization, c = Organization.objects.get_or_create( + **organization_values) + if c: + nb_organization_created += 1 + else: + for k in default: + setattr(organization, k, default[k]) + organization.save() + + site = site.strip() + if site == "0" or "." not in site: + site = None + elif not site.startswith('http'): + site = "http://" + site + values = { + "name": name.strip(), + "organization": organization, + "url": site, + } + + target, created = Target.objects.get_or_create(**values) + if created: + nb_created += 1 + if not quiet: + sys.stdout.write("\n") + if nb_at_created: + sys.stdout.write( + '* {} area types created.\n'.format(nb_at_created) + ) + if nb_area_created: + sys.stdout.write( + '* {} areas created.\n'.format( + nb_area_created)) + if nb_tt_created: + sys.stdout.write( + '* {} organization types created.\n'.format(nb_tt_created) + ) + if nb_organization_created: + sys.stdout.write( + '* {} organizations created.\n'.format( + nb_organization_created)) + if nb_created: + sys.stdout.write( + '* {} targets created.\n'.format(nb_created)) + sys.stdout.flush() diff --git a/commorganization/migrations/0001_initial.py b/commorganization/migrations/0001_initial.py new file mode 100644 index 0000000..891f297 --- /dev/null +++ b/commorganization/migrations/0001_initial.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2019-07-31 08:21 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Area', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('population', models.IntegerField(blank=True, null=True, verbose_name='Population')), + ('reference', models.CharField(blank=True, help_text='For instance, INSEE code for towns', max_length=100, null=True, verbose_name='Reference')), + ], + options={ + 'ordering': ('name',), + 'verbose_name_plural': 'Areas', + 'verbose_name': 'Area', + }, + ), + migrations.CreateModel( + name='AreaType', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ], + options={ + 'ordering': ('name',), + 'verbose_name_plural': 'Area types', + 'verbose_name': 'Area type', + }, + ), + migrations.CreateModel( + name='Organization', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('address', models.TextField(blank=True, null=True, verbose_name='Address')), + ('area', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commorganization.Area', verbose_name='Area')), + ], + options={ + 'ordering': ('name',), + 'verbose_name_plural': 'Organizations', + 'verbose_name': 'Organization', + }, + ), + migrations.CreateModel( + name='OrganizationType', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commorganization.OrganizationType', verbose_name='Parent')), + ], + options={ + 'ordering': ('parent__name', 'name'), + 'verbose_name_plural': 'Organization types', + 'verbose_name': 'Organization type', + }, + ), + migrations.CreateModel( + name='Target', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('url', models.URLField(blank=True, null=True, verbose_name='URL')), + ('organization', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='commorganization.Organization', verbose_name='Organization')), + ], + options={ + 'ordering': ('name',), + 'verbose_name_plural': 'Targets', + 'verbose_name': 'Target', + }, + ), + migrations.AddField( + model_name='organization', + name='organization_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commorganization.OrganizationType', verbose_name='Type'), + ), + migrations.AddField( + model_name='area', + name='area_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commorganization.AreaType', verbose_name='Type'), + ), + migrations.AddField( + model_name='area', + name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='commorganization.Area', verbose_name='Parent'), + ), + ] diff --git a/commorganization/migrations/__init__.py b/commorganization/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/commorganization/models.py b/commorganization/models.py new file mode 100644 index 0000000..a2b0994 --- /dev/null +++ b/commorganization/models.py @@ -0,0 +1,93 @@ +from django.db import models +from django.utils.translation import ugettext_lazy as _ + + +class AreaType(models.Model): + name = models.CharField(_("Name"), max_length=200) + + class Meta: + verbose_name = _("Area type") + verbose_name_plural = _("Area types") + ordering = ("name", ) + + def __str__(self): + return self.name + + +class Area(models.Model): + name = models.CharField(verbose_name=_("Name"), max_length=200) + area_type = models.ForeignKey( + AreaType, on_delete=models.CASCADE, verbose_name=_("Type")) + population = models.IntegerField(verbose_name=_("Population"), blank=True, + null=True) + reference = models.CharField( + verbose_name=_("Reference"), max_length=100, blank=True, null=True, + help_text=_("For instance, INSEE code for towns") + ) + parent = models.ForeignKey("Area", verbose_name=_("Parent"), + blank=True, null=True) + + class Meta: + verbose_name = _("Area") + verbose_name_plural = _("Areas") + ordering = ("name", ) + + def __str__(self): + if not self.parent: + return self.name + return "{} / {}".format(self.parent, self.name) + + +class OrganizationType(models.Model): + name = models.CharField(_("Name"), max_length=200) + parent = models.ForeignKey("OrganizationType", verbose_name=_("Parent"), + blank=True, null=True) + + class Meta: + verbose_name = _("Organization type") + verbose_name_plural = _("Organization types") + ordering = ("parent__name", "name", ) + + def __str__(self): + if not self.parent: + return self.name + return "{} / {}".format(self.parent, self.name) + + +class Organization(models.Model): + name = models.CharField(verbose_name=_("Name"), max_length=200) + area = models.ForeignKey( + Area, on_delete=models.SET_NULL, verbose_name=_("Area"), blank=True, + null=True + ) + organization_type = models.ForeignKey( + OrganizationType, on_delete=models.CASCADE, verbose_name=_("Type")) + address = models.TextField(verbose_name=_("Address"), blank=True, null=True) + + class Meta: + verbose_name = _("Organization") + verbose_name_plural = _("Organizations") + ordering = ("name", ) + + def __str__(self): + if not self.area: + return "{} ({})".format( + self.name, self.organization_type) + return "{} - {} ({})".format( + self.name, self.area, self.organization_type) + + +class Target(models.Model): + name = models.CharField(verbose_name=_("Name"), max_length=200) + url = models.URLField(verbose_name=_("URL"), blank=True, null=True) + organization = models.ForeignKey( + Organization, verbose_name=_("Organization"), blank=True, + null=True, on_delete=models.SET_NULL) + + class Meta: + verbose_name = _("Target") + verbose_name_plural = _("Targets") + ordering = ("name",) + + def __str__(self): + return "{} ({})".format(self.name, self.organization) diff --git a/commorganization/tests.py b/commorganization/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/commorganization/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/commorganization/views.py b/commorganization/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/commorganization/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. -- cgit v1.2.3