diff options
Diffstat (limited to 'ishtar_common')
| -rw-r--r-- | ishtar_common/data_importer.py | 44 | ||||
| -rw-r--r-- | ishtar_common/forms_common.py | 1 | ||||
| -rw-r--r-- | ishtar_common/ignf_utils.py | 181 | ||||
| -rw-r--r-- | ishtar_common/management/commands/ishtar_migrate_srid.py | 62 | ||||
| -rw-r--r-- | ishtar_common/migrations/0219_auto_20220407_1952.py | 39 | ||||
| -rw-r--r-- | ishtar_common/models.py | 16 | ||||
| -rw-r--r-- | ishtar_common/models_common.py | 39 | ||||
| -rw-r--r-- | ishtar_common/models_imports.py | 188 | ||||
| -rw-r--r-- | ishtar_common/templates/ishtar/import_table.html | 2 | ||||
| -rw-r--r-- | ishtar_common/views.py | 13 |
10 files changed, 569 insertions, 16 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 9dbd45e93..1c99782e3 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -26,8 +26,10 @@ import re import sys import zipfile +from django.apps import apps from django.conf import settings from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType from django.contrib.gis.geos.error import GEOSException from django.db.models.fields import FieldDoesNotExist from django.core.exceptions import FieldError @@ -831,6 +833,8 @@ class Importer(object): SLUG = "" NAME = "" DESC = "" + TYPE = "" + MAIN_GEO = False LINE_FORMAT = [] OBJECT_CLS = None UNICITY_KEYS = [] @@ -1338,6 +1342,7 @@ class Importer(object): n2 = n self.c_errors = False c_row = [] + idx_col = 0 for idx_col, val in enumerate(line): try: self._row_processing(c_row, idx_col, idx_line, val, data) @@ -1368,6 +1373,10 @@ class Importer(object): self.new_objects, self.updated_objects = [], [] self.ambiguous_objects, self.not_find_objects = [], [] + geodata = {} + if self.TYPE == "gis": + if "geodata" in data: + geodata = data.pop("geodata") obj, created = self.get_object(self.OBJECT_CLS, data, idx_line=idx_line) if self.simulate: return data @@ -1391,6 +1400,41 @@ class Importer(object): obj._no_post_save = True obj.save() self._add_to_post_save(obj.__class__, obj.pk, idx_line) + + if self.TYPE == "gis": + content_type = ContentType.objects.get( + app_label=obj.__class__._meta.app_label, + model=obj.__class__.__name__.lower() + ) + geodata.update({ + "source_id": obj.pk, + "source_content_type": content_type, + }) + GeoVectorData = apps.get_model("ishtar_common", "GeoVectorData") + item = None + if "import_key" in geodata: + q = GeoVectorData.objects.filter( + import_key=geodata["import_key"], + source_id=obj.pk, + source_content_type=content_type, + ) + if q.count(): + item = q.all()[0] + if item: + for k in geodata: + setattr(item, k, geodata[k]) + item.save() + else: + item = GeoVectorData.objects.create(**geodata) + if self.import_instance: + item.imports.add(self.import_instance) + if self.MAIN_GEO: + obj.main_geodata = item + obj._post_saved_geo = True + obj._no_move = True + obj.skip_history_when_saving = True + obj.save() + n = datetime.datetime.now() logger.debug("* %s - Item saved" % (str(n - n2))) n2 = n diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index c3367eaac..6e9e11006 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -364,6 +364,7 @@ class NewImportGISForm(BaseImportForm): return value def save(self, user, commit=True): + self.instance.user = user item = super(NewImportGISForm, self).save(commit) return item diff --git a/ishtar_common/ignf_utils.py b/ishtar_common/ignf_utils.py new file mode 100644 index 000000000..2f167299c --- /dev/null +++ b/ishtar_common/ignf_utils.py @@ -0,0 +1,181 @@ +import xml.etree.ElementTree as ET + + +# from the bellow script +IGNF = { + "BORA01": 5607, + "CAD97": 4473, + "CAD97UTM38S": 5879, + "MAYO50UTM38S": 2980, + "CSG67UTM21": 3312, + "CSG67UTM22": 2971, + "STPM50": 5792, + "ED50UTM30": 23030, + "ED50UTM32": 23032, + "ED50UTM31": 23031, + "ETRS89": 4936, + "ETRS89LAEA": 3035, + "ETRS89LCC": 3034, + "ETRS89UTM28": 25828, + "ETRS89UTM29": 25829, + "ETRS89UTM30": 25830, + "ETRS89UTM31": 25831, + "ETRS89UTM32": 25832, + "ETRS89UTM33": 25833, + "ETRS89UTM34": 25834, + "ETRS89UTM35": 25835, + "ETRS89UTM36": 25836, + "ETRS89UTM37": 25837, + "EVRF2000": 5730, + "EVRF2007": 5621, + "GUADFMUTM20": 2969, + "GUADANNUTM20": 2970, + "HUAH01": 5605, + "ATUO63UTM7S": 3302, + "TAHITI66": 5601, + "MART87": 5756, + "GUAD88": 5757, + "GUAD88LS": 5616, + "GUAD88MG": 5617, + "GUAD88SB": 5619, + "GUAD88SM": 5620, + "REUN89": 5758, + "GUAD92LD": 5618, + "IGN72UTM58S": 3060, + "KERG62UTM42S": 3336, + "LIFOU56UT58S": 2981, + "MARE53UTM58S": 2995, + "MARTFDUTM20": 2973, + "MAUPITIUTM5S": 3306, + "MAUPITI01": 5604, + "MOOREA81": 5602, + "MOORE87UTM6S": 3305, + "NEA74LBTNM2": 3166, + "IGN69": 5720, + "IGN78C": 5721, + "NGF84": 5719, + "GUYA77": 5755, + "NGC48": 5791, + "NCAL69": 5753, + "NNLUX": 5774, + "LURESGKL": 2169, + "NTFLAMB1": 27561, + "NTFLAMB1C": 27571, + "NTFLAMB2": 27562, + "NTFLAMB2C": 27572, + "NTFLAMB2E": 27572, + "NTFLAMB3": 27563, + "NTFLAMB3C": 27573, + "NTFLAMB4": 27564, + "NTFLAMB4C": 27574, + "PETRE72SPSTA": 2985, + "RAIA01": 5603, + "RGAF09": 5487, + "RGAF09GEODD": 7085, + "RGAF09GDD": 7086, + "RGAF09G": 7086, + "RGAF09GEO": 7085, + "RGAF09UTM20": 5490, + "RGF93": 4964, + "RGF93CC42": 3942, + "RGF93CC43": 3943, + "RGF93CC44": 3944, + "RGF93CC45": 3945, + "RGF93CC46": 3946, + "RGF93CC47": 3947, + "RGF93CC48": 3948, + "RGF93CC49": 3949, + "RGF93CC50": 3950, + "RGF93GDD": 7084, + "RGF93GEODD": 7042, + "RGF93GEO": 7042, + "RGF93G": 7084, + "RGFG95": 4966, + "RGFG95GDD": 7041, + "RGFG95GEODD": 7040, + "RGFG95G": 7041, + "RGFG95GEO": 7040, + "RGFG95UTM21": 3313, + "RGFG95UTM22": 2972, + "RGM04": 4468, + "RGM04GDD": 7039, + "RGM04GEODD": 7038, + "RGM04G": 7039, + "RGM04GEO": 7038, + "RGM04UTM38S": 4471, + "RGNC": 4906, + "RGNCLAMBNC": 3163, + "RGNCUTM57S": 3169, + "RGNCUTM58S": 3170, + "RGNCUTM59S": 3171, + "RGPF": 4998, + "RGPFUTM5S": 3296, + "RGPFUTM6S": 3297, + "RGPFUTM7S": 3298, + "RGPFUTM8S": 3299, + "RGR92": 4970, + "RGR92GEODD": 7036, + "RGR92GDD": 7037, + "RGR92GEO": 7036, + "RGR92G": 7037, + "RGR92UTM40S": 2975, + "RGSPM06": 4465, + "RGSPM06GDD": 7035, + "RGSPM06GEODD": 7034, + "RGSPM06GEO": 7034, + "RGSPM06G": 7035, + "RGSPM06U21": 4467, + "RGTAAF07": 7071, + "RGTAAFGEODD": 7087, + "RGTAAF07GDD": 7133, + "RGTAAF07G": 7133, + "RGTAAFGEO": 7087, + "RGTAAFUTM37S": 7074, + "RGTAAFUTM39S": 7076, + "RGTAAFUTM42S": 7079, + "RGTAAFUTM43S": 7080, + "MAYO53": 5793, + "ST84UTM58S": 2996, + "ST87UTM58S": 3164, + "STPM50UTM21": 2987, + "TAHAA01": 5606, + "TAHI79UTM6S": 3304, + "TAHI51UTM6S": 2976, + "PGP50STEREPS": 2986, + "WALL78UTM1S": 2988, + "WGS72": 4984, + "WGS72UTM30": 32230, + "WGS72UTM31": 32231, + "WGS72UTM32": 32232, + "WGS72UTM1S": 32301, + "WGS84": 4978, + "WGS84UTM30": 32630, + "WGS84UTM31": 32631, + "WGS84UTM32": 32632, + "WGS84UTM1S": 32701, + "WGS84UTM39S": 32739, + "WGS84UTM42S": 32742, + "WGS84WMSV": 3857, +} + + +def extract_from_csv(filename): + tree = ET.parse(filename) + root = tree.getroot() + ns = "{http://www.isotc211.org/2005/gmx}" + ns_gml = "{http://www.opengis.net/gml}" + equiv = {} + base_srid = "urn:ogc:def:crs:EPSG:9.1.1:" + + for crs_type in root.findall(f"{ns}crs"): + for crs in list(crs_type): + ignf_id = crs.attrib[f"{ns_gml}id"] + for epsg in crs.findall(f"{ns_gml}name"): + if epsg.text.startswith(base_srid): + equiv[ignf_id] = epsg.text[len(base_srid) :] + return equiv + + +if __name__ == "__main__": + equiv = extract_from_csv("IGNF.xml") + print(equiv) diff --git a/ishtar_common/management/commands/ishtar_migrate_srid.py b/ishtar_common/management/commands/ishtar_migrate_srid.py new file mode 100644 index 000000000..19de96cb8 --- /dev/null +++ b/ishtar_common/management/commands/ishtar_migrate_srid.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2022 Étienne Loks <etienne.loks_AT_peacefrogsDOTnet> + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# See the file COPYING for details. + +from django.core.management.base import BaseCommand +from optparse import make_option +import sys + +from ishtar_common.models import GeoVectorData, IshtarSiteProfile +from ishtar_common.utils import BColors + + +class Command(BaseCommand): + help = "Update SRID - change from profile" + option_list = BaseCommand.option_list + ( + make_option('--quiet', + action='store_true', + dest='quiet', + default=False, + help='Proceed silently with no interactive input.'), + ) + + def interactive_start(self): + sys.stdout.write( + BColors.HEADER + BColors.BOLD + + "Update to new SRID\n") + sys.stdout.write(BColors.ENDC) + yes = None + while yes != "yes": + sys.stdout.write( + "Are you sure you want to proceed? (yes/[n])\n") + yes = input() + if not yes or yes == "n": + sys.stdout.write(BColors.FAIL + "Aborting\n") + sys.stdout.write(BColors.ENDC) + sys.exit() + + def handle(self, *args, **options): + quiet = options['quiet'] + if not quiet: + self.interactive_start() + profile = IshtarSiteProfile.get_current_profile() + if not profile.srs or not profile.srs.srid: + sys.stdout.write(BColors.FAIL + "No SRS set\n") + sys.stdout.write(BColors.ENDC) + GeoVectorData.migrate_srid(profile.srs.srid) + diff --git a/ishtar_common/migrations/0219_auto_20220407_1952.py b/ishtar_common/migrations/0219_auto_20220407_1952.py new file mode 100644 index 000000000..4ec916d63 --- /dev/null +++ b/ishtar_common/migrations/0219_auto_20220407_1952.py @@ -0,0 +1,39 @@ +# Generated by Django 2.2.24 on 2022-04-07 19:52 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('ishtar_common', '0218_auto_20220329_1121'), + ] + + operations = [ + migrations.AddField( + model_name='geovectordata', + name='import_key', + field=models.TextField(blank=True, help_text='Use this for update imports', null=True, verbose_name='Import key'), + ), + migrations.AddField( + model_name='geovectordata', + name='imports', + field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_geovectordata', to='ishtar_common.Import'), + ), + migrations.AddField( + model_name='ishtarsiteprofile', + name='srs', + field=models.ForeignKey(blank=True, help_text='Set it to the most used spatial reference system. Warning: after change launch the ishtar_migrate_srid script.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='profile_srs', to='ishtar_common.SpatialReferenceSystem', verbose_name='Spatial Reference System in database'), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='display_srs', + field=models.ForeignKey(blank=True, help_text='Spatial Reference System used for display when no SRS is defined', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='profile_display_srs', to='ishtar_common.SpatialReferenceSystem', verbose_name='Spatial Reference System for display'), + ), + migrations.AlterUniqueTogether( + name='geovectordata', + unique_together={('source_content_type', 'source_id', 'import_key')}, + ), + ] diff --git a/ishtar_common/models.py b/ishtar_common/models.py index 5b9d48357..2bd983906 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -203,6 +203,7 @@ __all__ = [ "Department", "State", "CompleteIdentifierItem", + "GeoVectorData", ] logger = logging.getLogger(__name__) @@ -1413,9 +1414,22 @@ class IshtarSiteProfile(models.Model, Cached): blank=True, null=True, help_text=_( - "Spatial Reference System used for display when no SRS is " "defined" + "Spatial Reference System used for display when no SRS is defined" ), on_delete=models.SET_NULL, + related_name="profile_display_srs" + ) + srs = models.ForeignKey( + SpatialReferenceSystem, + verbose_name=_("Spatial Reference System in database"), + blank=True, + null=True, + help_text=_( + "Set it to the most used spatial reference system. Warning: after change " + "launch the ishtar_migrate_srid script." + ), + on_delete=models.SET_NULL, + related_name="profile_srs" ) default_language = models.ForeignKey( Language, diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py index 104e46100..12f8b0507 100644 --- a/ishtar_common/models_common.py +++ b/ishtar_common/models_common.py @@ -23,6 +23,7 @@ from django.conf import settings from django.contrib.auth.models import User, Group from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.fields import GenericForeignKey +from django.db import migrations as db_migrations from django.contrib.gis.db import models from django.contrib.gis.geos import Point from django.contrib.gis.gdal.error import GDALException @@ -2078,13 +2079,15 @@ GEOJSON_POINT_TPL = { } -class GeoVectorData(models.Model): +class GeoVectorData(Imported): name = models.TextField(_("Name"), default=_("Default")) source_content_type = models.ForeignKey( ContentType, related_name="content_type_geovectordata", on_delete=models.CASCADE ) source_id = models.PositiveIntegerField() source = GenericForeignKey("source_content_type", "source_id") + import_key = models.TextField(_("Import key"), blank=True, null=True, + help_text=_("Use this for update imports")) origin = models.ForeignKey( GeoOriginType, blank=True, @@ -2141,6 +2144,7 @@ class GeoVectorData(models.Model): class Meta: verbose_name = _("Geographic - Vector data") verbose_name_plural = _("Geographic - Vector data") + unique_together = ("source_content_type", "source_id", "import_key") def __str__(self): name = self.name @@ -2197,11 +2201,20 @@ class GeoVectorData(models.Model): if dim == 2: return [None, None] return [None, None, None] - point = geom.transform(srid, clone=True) + + point = geom + if not srid or srid != geom.srid: + point = geom.transform(srid, clone=True) + x, y = point.x, point.y + # Coordinates are reversed - should be fixed on Django 3.2 + if srid in (4326, 4979): + x, y = y, x + else: + x, y = point.x, point.y if dim == 2: - coordinates = [point.x, point.y] + coordinates = [x, y] else: - coordinates = [point.x, point.y, point.z] + coordinates = [x, y, point.z] if not rounded: return coordinates return [round(coord, rounded) for coord in coordinates] @@ -2362,6 +2375,24 @@ class GeoVectorData(models.Model): return self._geojson_serialize("multi_polygon") return "{}" + @classmethod + def migrate_srid(cls, new_srid): + fields = ( + "point_2d", "point_3d", "multi_points", "multi_line", "multi_polygon", + ) + with connection.cursor() as cursor: + for name in fields: + cursor.execute( + "UPDATE ishtar_common_geovectordata SET %s=ST_SetSRID(%s, %s);", + [name, name, new_srid] + ) + items = cls.objects.all() + for item in items: + for name in fields: + getattr(item, name).transform(new_srid) + item._no_geo_check = True + item.save() + post_save.connect(post_save_geodata, sender=GeoVectorData) diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index b931cc4b4..b8c842284 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -19,9 +19,13 @@ import csv import datetime +import fiona +from fiona import crs as fiona_crs import json +from osgeo import ogr import os import logging +from pyproj import CRS import shutil import re import tempfile @@ -30,6 +34,9 @@ import zipfile from django.apps import apps from django.conf import settings from django.contrib.gis.db import models +from django.contrib.gis.gdal.error import GDALException +from django.contrib.gis.geos import GEOSGeometry +from django.contrib.gis.geos.error import GEOSException from django.core.exceptions import ValidationError from django.core.files import File from django.core.files.base import ContentFile @@ -75,6 +82,7 @@ from ishtar_common.data_importer import ( LowerCaseFormater, ) from ishtar_common.utils import task +from ishtar_common.ignf_utils import IGNF logger = logging.getLogger(__name__) @@ -124,6 +132,8 @@ IMPORT_TYPES = ( ("gis", _("GIS")), ) +IMPORT_TYPES_DICT = dict(IMPORT_TYPES) + class ImporterType(models.Model): """ @@ -185,6 +195,12 @@ class ImporterType(models.Model): def __str__(self): return self.name + @property + def type_label(self): + if self.type in IMPORT_TYPES_DICT: + return IMPORT_TYPES_DICT[self.type] + return "" + def get_libreoffice_template(self): if not UnoCalc: return @@ -320,6 +336,8 @@ class ImporterType(models.Model): "UNICITY_KEYS": UNICITY_KEYS, "LINE_EXPORT_FORMAT": LINE_EXPORT_FORMAT, "MODEL_CREATION_LIMIT": MODEL_CREATION_LIMIT, + "TYPE": self.type, + "MAIN_GEO": self.is_main_geometry, } name = str( "".join( @@ -1068,6 +1086,45 @@ def delayed_check(import_pk): imp.check_modified() +RE_NUMBER = r"[+-]?\d+(?:\.\d*)?" +RE_COORDS = r"(" + RE_NUMBER + r") (" + RE_NUMBER + r")" + + +def _reverse_coordinates(wkt): + # TODO: à effacer + return re.sub(RE_COORDS, r"\2 \1", wkt) + + +def convert_geom(feature, srid): + geo_type = feature["type"] + if geo_type in ("LineString", "Polygon"): + feature["type"] = "Multi" + geo_type + feature["coordinates"] = [feature["coordinates"]] + feature = GEOSGeometry(json.dumps(feature)) + has_z = feature.hasz + feature = f"SRID={srid};{feature.wkt}" + + IshtarSiteProfile = apps.get_model("ishtar_common", "IshtarSiteProfile") + profile = IshtarSiteProfile.get_current_profile() + srs = 4326 if not has_z else 4979 + if profile.srs and profile.srs.srid: + srs = profile.srs.srid + if srs != srid: + feature = GEOSGeometry(feature).transform(srs, clone=True).ewkt + return feature + + +IMPORT_GEOMETRY = { + "Point": "point_2d", + "3D Point": "point_3d", + "MultiPoint": "multi_points", + "LineString": "multi_line", + "MultiLineString": "multi_line", + "Polygon": "multi_polygon", + "MultiPolygon": "multi_polygon", +} + + class Import(models.Model): user = models.ForeignKey( "IshtarUser", blank=True, null=True, on_delete=models.SET_NULL @@ -1208,6 +1265,8 @@ class Import(models.Model): return errors def get_number_of_lines(self): + if self.importer_type.type == "gis": + return if self.number_of_line: return self.number_of_line if not self.imported_file or not self.imported_file.path: @@ -1291,13 +1350,11 @@ class Import(models.Model): IshtarSiteProfile = apps.get_model("ishtar_common", "IshtarSiteProfile") profile = IshtarSiteProfile.get_current_profile() actions = [] - if self.state == 'C': - actions.append(('A', _("Analyse"))) - if self.state in ('C', 'A'): - actions.append(('ED', _("Edit"))) - if self.state in ('A', 'PI'): - actions.append(('A', _("Re-analyse"))) - actions.append(('I', _("Launch import"))) + if self.state == "C": + actions.append(("A", _("Analyse"))) + if self.state in ("A", "PI"): + actions.append(("A", _("Re-analyse"))) + actions.append(("I", _("Launch import"))) if profile.experimental_feature: if self.changed_checked: actions.append(("IS", _("Step by step import"))) @@ -1317,6 +1374,8 @@ class Import(models.Model): if self.state == "AC": state = "FE" if self.error_file else "F" actions.append((state, _("Unarchive"))) + if self.state in ("C", "A"): + actions.append(("ED", _("Edit"))) actions.append(("D", _("Delete"))) return actions @@ -1337,8 +1396,7 @@ class Import(models.Model): conservative_import=self.conservative_import, ) - @property - def data_table(self): + def _data_table_tab(self): imported_file = self.imported_file.path tmpdir = None if zipfile.is_zipfile(imported_file): @@ -1373,6 +1431,118 @@ class Import(models.Model): shutil.rmtree(tmpdir) return [] + def get_gis_attr(self): + return self._data_table_gis(get_gis_attr=True) + + def _data_table_gis(self, get_gis_attr=False): + self.gis_attr = None + imported_file = self.imported_file.path + tmp_dir = None + file_type = "gpkg" + if zipfile.is_zipfile(imported_file): + z = zipfile.ZipFile(imported_file) + imported_file = None + filenames = [] + for name in z.namelist(): + # get first CSV file found + name_l = name.lower() + if name_l.endswith(".gpkg"): + filenames = [name] + break + if name_l.endswith(".shp"): + file_type = "shp" + filenames.append(name) + continue + for end in [".cpg", ".prj", ".shx", ".dbf"]: + if name_l.endswith(end): + filenames.append(name) + continue + if not filenames: + return [] + tmp_dir = tempfile.mkdtemp(prefix="tmp-ishtar-") + for filename in filenames: + if filename.lower().endswith(".shp") or filename.lower().endswith( + ".gpkg" + ): + imported_file = z.extract(filename, tmp_dir) + else: + z.extract(filename, tmp_dir) + elif not imported_file.endswith(".gpkg"): + raise ImporterError(_("Incorrect GIS file.")) + + if not imported_file: + raise ImporterError(_("Incorrect GIS file.")) + + kwargs = {} + if self.importer_type.layer_name: + kwargs["layer"] = self.importer_type.layer_name + try: + with fiona.open(imported_file, **kwargs) as collection: + schema = collection.schema + geometry = schema["geometry"] + if geometry not in IMPORT_GEOMETRY: + raise ImporterError(_(f'Geometry "{geometry}" not managed.')) + self.gis_attr = IMPORT_GEOMETRY[geometry] + if get_gis_attr: + if tmp_dir: + shutil.rmtree(tmp_dir) + return self.gis_attr + properties = schema["properties"].keys() + + crs = fiona_crs.to_string(collection.crs) + if not crs: + driver_type = {"shp": "ESRI Shapefile", "gpkg": "GPKG"} + driver = ogr.GetDriverByName(driver_type[file_type]) + shape = driver.Open(imported_file) + layer = shape.GetLayer() + crs = layer.GetSpatialRef() + auth = crs.GetAttrValue("AUTHORITY", 0) + srid = crs.GetAttrValue("AUTHORITY", 1) + if auth == "EPSG": + srid = int(srid) + elif auth == "IGNF" and srid in IGNF: + srid = IGNF[srid] + else: + raise ImporterError(_("CRS not managed.")) + + # https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6 + elif crs.startswith("+init=epsg:"): + srid = crs[len("+init=epsg:"):] + else: + srid = CRS.from_proj4(crs).to_epsg() + data = [] + # Warning: RuntimeWarning: Sequential read of iterator was interrupted. + # Resetting iterator. + # not relevant -> bug in fiona 1.8.18 (fixed in 1.8.19) + for idx, feature in enumerate(collection): + try: + line = [ + convert_geom(feature["geometry"], srid) + ] + for prop in properties: + value = feature["properties"][prop] + if value is None: + value = "" + line.append(str(value)) + data.append(line) + except (TypeError, GDALException, GEOSException) as e: + raise ImporterError( + _(f"Error reading feature {idx + 1} - {e}") + ) + except fiona.errors.DriverError: + raise ImporterError(_("Incorrect GIS file.")) + + if tmp_dir: + shutil.rmtree(tmp_dir) + return data + + @property + def data_table(self): + if self.importer_type.type == "tab": + return self._data_table_tab(), None + if self.importer_type.type == "gis": + return self._data_table_gis() + def initialize(self, user=None, session_key=None): self.state = "AP" self.end_date = datetime.datetime.now() diff --git a/ishtar_common/templates/ishtar/import_table.html b/ishtar_common/templates/ishtar/import_table.html index 1f7b242b6..e951e28c4 100644 --- a/ishtar_common/templates/ishtar/import_table.html +++ b/ishtar_common/templates/ishtar/import_table.html @@ -42,7 +42,7 @@ $("#import-list").find('input').prop('disabled', true); {{import.name|default:"-"}} </td> <td> - {{import.importer_type}} + {{import.importer_type}} ({% trans import.importer_type.type_label %}) </td> <td> {% if import.imported_file %} diff --git a/ishtar_common/views.py b/ishtar_common/views.py index 50964f462..b6686e0bd 100644 --- a/ishtar_common/views.py +++ b/ishtar_common/views.py @@ -66,6 +66,7 @@ from archaeological_operations.models import Operation, ArchaeologicalSite from archaeological_warehouse.models import Warehouse from ishtar_common import forms_common as forms from ishtar_common import wizards +from ishtar_common.data_importer import ImporterError from ishtar_common.forms import FinalForm, FinalDeleteForm from ishtar_common.models import get_current_profile from ishtar_common.templatetags.link_to_window import simple_link_to_window @@ -1473,7 +1474,17 @@ class ImportListView(IshtarMixin, LoginRequiredMixin, ListView): if settings.USE_BACKGROUND_TASK: imprt.delayed_importation(request, request.session.session_key) else: - imprt.importation() + try: + imprt.importation() + except ImporterError as e: + imprt.state = "FE" + imprt.end_date = datetime.datetime.now() + imprt.save() + put_session_message( + request.session.session_key, + f"{imprt} - {e}", + "warning", + ) elif action == "CH": if settings.USE_BACKGROUND_TASK: imprt.delayed_check_modified(request.session.session_key) |