✨ Update IGN/INSEE import script

2 files changed, 68 insertions, 19 deletions

diff --git a/ishtar_common/management/commands/import_geofla_csv.py b/ishtar_common/management/commands/import_geofla_csv.py
index 0464d3480..c808c0dae 100644
--- a/ishtar_common/management/commands/import_geofla_csv.py
+++ b/ishtar_common/management/commands/import_geofla_csv.py
@@ -25,7 +25,17 @@ from django.contrib.gis.geos import GEOSGeometry, Point
 from django.db import transaction
 from django.db.utils import DataError
 
-from ishtar_common.models import Town
+from django.contrib.contenttypes.models import ContentType
+from ishtar_common.models import Town, GeoVectorData, GeoDataType, GeoProviderType
+
+
+town_content_type = ContentType.objects.get(app_label="ishtar_common", model="town")
+data_type, __ = GeoDataType.objects.get_or_create(
+    txt_idx="town-limit", defaults={"label": "Limites commune"}
+)
+provider, __ = GeoProviderType.objects.get_or_create(
+    txt_idx="france-ign", defaults={"label": "IGN"}
+)
 
 
 class Command(BaseCommand):
@@ -40,6 +50,9 @@ class Command(BaseCommand):
             '--quiet', dest='quiet', action='store_true',
             help='Quiet output')
         parser.add_argument(
+            '--create-only', dest='create_only', action='store_true',
+            help='Create only missing geo')
+        parser.add_argument(
             '--srid', type=int, default=2154, dest='srid',
             help='SRID uses. Default: 2154.')
 
@@ -64,11 +77,12 @@ class Command(BaseCommand):
         default_year = options['year']
         srid = options['srid']
         quiet = options['quiet']
+        create_only = options["create_only"]
         if not quiet:
             sys.stdout.write('* using year {} as a default\n'.format(
                 default_year))
             sys.stdout.write('* Opening file {}\n'.format(csv_file))
-        nb_created, nb_changed, nb_error = 0, 0, 0
+        nb_created, nb_changed, nb_error, nb_geo = 0, 0, 0, 0
         with open(csv_file, 'rt') as csvfile:
             header = csvfile.readline()
             geom_colum = header.split(",")[0]
@@ -84,52 +98,79 @@ class Command(BaseCommand):
                     num_insee = '0' + num_insee
                 if 'NOM_COM_M' in row:
                     name = row['NOM_COM_M']
+                elif 'NOM_M' in row:
+                    name = row['NOM_M']
+                elif 'NOM' in row:
+                    name = row['NOM'].upper()
                 else:
                     name = row['NOM_COM'].upper()
                 town, created = self.get_town(num_insee, name, default_year)
                 if created:
                     nb_created += 1
+                else:
+                    nb_changed += 1
+                    if create_only and town.main_geodata:
+                        continue
+
                 geom = row[geom_colum].upper()
                 if 'MULTI' not in geom:
                     geom = geom.replace('POLYGON', 'MULTIPOLYGON(') + ')'
-                values = {'limit': GEOSGeometry(geom, srid=srid)}
+                limit = GEOSGeometry(geom, srid=srid)
                 if 'X_CENTROID' in row:
-                    values['center'] = Point(
+                    center = Point(
                         float(row['X_CENTROID']), float(row['Y_CENTROID']),
                         srid=srid)
                 else:
-                    values['center'] = None
+                    center = None
+                values = {}
+                values['center'] = None
                 if not town.year and default_year:
                     values['year'] = default_year
                 if 'SUPERFICIE' in row:
                     values['surface'] = row['SUPERFICIE']
                 else:
                     values['surface'] = None
-                if not created:
-                    nb_changed += 1
                 for k in values:
                     setattr(town, k, values[k])
                 try:
                     with transaction.atomic():
                         town.save()
                 except DataError:
-                    """
-                    new_limit = str(GEOSGeometry(geom, srid=srid).simplify(
-                        preserve_topology=True))
-                    if 'MULTI' not in new_limit:
-                        new_limit = new_limit.replace(
-                            'POLYGON', 'MULTIPOLYGON(') + ')'
-                    town.limit = new_limit
-                    """
                     nb_error += 1
                     town, created = self.get_town(num_insee, name, default_year)
-                    values.pop('limit')
                     for k in values:
                         setattr(town, k, values[k])
                     town.save()
+                attrs = {
+                    "name": town._generate_cached_label(),
+                    "source_content_type": town_content_type,
+                    "source_id": town.pk,
+                    "data_type": data_type,
+                    "provider": provider,
+                }
+                if limit:
+                    attrs["multi_polygon"] = limit
+                else:
+                    attrs["point_2d"] = center
+                try:
+                    data, created = GeoVectorData.objects.get_or_create(**attrs)
+                except DataError:
+                    if limit:
+                        print(f"\nError {town} polygon\n")
+                        attrs.pop("multi_polygon")
+                        attrs["point_2d"] = center
+                        data, created = GeoVectorData.objects.get_or_create(**attrs)
+                    else:
+                        print(f"\nError {town} center\n")
+                town.main_geodata = data
+                town._post_save_geo_ok = False
+                town.save()
+                if created:
+                    nb_geo += 1
         if quiet:
             return
         sys.stdout.write('\n* {} town created'.format(nb_created))
+        sys.stdout.write('\n* {} geo created'.format(nb_geo))
         sys.stdout.write('\n* {} town changed'.format(nb_changed))
         sys.stdout.write('\n* {} town with geometry error\n'.format(nb_error))
         sys.stdout.flush()
diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py
index 7962efd37..c6fec3e96 100644
--- a/ishtar_common/management/commands/import_insee_comm_csv.py
+++ b/ishtar_common/management/commands/import_insee_comm_csv.py
@@ -55,6 +55,17 @@ class Command(BaseCommand):
         with open(csv_file, 'rt') as csvfile:
             reader = csv.DictReader(csvfile)
             for idx, row in enumerate(reader):
+                new_insee = row['DepComN']
+                if len(new_insee) < 5:
+                    new_insee = '0' + new_insee
+
+                if not idx:  # test if first do not exist
+                    q = Town.objects.filter(numero_insee=new_insee,
+                                            year=default_year)
+                    if q.count():
+                        print("First town already exists for this year....")
+                        return
+
                 if not quiet:
                     sys.stdout.write('Processing town %d.\r' % (idx + 1))
                     sys.stdout.flush()
@@ -74,9 +85,6 @@ class Command(BaseCommand):
                         continue
                 old_town = q.all()[0]
 
-                new_insee = row['DepComN']
-                if len(new_insee) < 5:
-                    new_insee = '0' + new_insee
                 q = Town.objects.filter(numero_insee=new_insee,
                                         year=default_year)
                 if not q.count():