diff options
| -rw-r--r-- | ishtar_common/admin.py | 2 | ||||
| -rw-r--r-- | ishtar_common/data_importer.py | 31 | ||||
| -rw-r--r-- | ishtar_common/forms_common.py | 47 | ||||
| -rw-r--r-- | ishtar_common/models_common.py | 7 | ||||
| -rw-r--r-- | ishtar_common/models_imports.py | 5 | ||||
| -rw-r--r-- | ishtar_common/version.py | 4 |
6 files changed, 69 insertions, 27 deletions
diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index c883a744c..602cf23a7 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -480,7 +480,7 @@ admin_site.register(Site, SiteAdmin) class AdminIshtarSiteProfileForm(forms.ModelForm): class Meta: model = models.IshtarSiteProfile - exclude = [] + exclude = ["display_srs"] default_center = PointField(label=_("Maps - default center"), widget=OSMWidget) diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py index 5090fb15b..61c294b28 100644 --- a/ishtar_common/data_importer.py +++ b/ishtar_common/data_importer.py @@ -39,7 +39,7 @@ from django.db.models import Q from django.template.defaultfilters import slugify from django.utils.translation import ugettext_lazy as _ -from ishtar_common.utils import debug_line_no, get_all_field_names, update_data +from ishtar_common.utils import get_all_field_names, update_data, get_current_profile NEW_LINE_BREAK = "#####@@@#####" @@ -864,6 +864,9 @@ class Importer(object): "{} with values {} doesn't exist in the " "database. Create it first or fix your source file." ), + "gis_missing_obj": _( + "Related object not found. Can not create the geographic item. Context: {}." + ) } def _create_models(self, force=False): @@ -1373,11 +1376,24 @@ class Importer(object): self.new_objects, self.updated_objects = [], [] self.ambiguous_objects, self.not_find_objects = [], [] - geodata = {} + geodata, main_geodata = {}, {} if self.TYPE == "gis": + profile = get_current_profile() + default_srs = profile.srs if profile.srs else None + if "geodata" in data: geodata = self._defaults.get(("geodata",), {}) geodata.update(data.pop("geodata")) + if default_srs and not [ + 1 for k in geodata if k.startswith("spatial_reference_system")]: + geodata["spatial_reference_system"] = default_srs + + if "main_geodata" in data: + main_geodata = self._defaults.get(("main_geodata",), {}) + main_geodata.update(data.pop("main_geodata")) + if default_srs and not [ + 1 for k in main_geodata if k.startswith("spatial_reference_system")]: + main_geodata["spatial_reference_system"] = default_srs obj, created = self.get_object(self.OBJECT_CLS, data, idx_line=idx_line) if self.simulate: return data @@ -1401,6 +1417,17 @@ class Importer(object): self._add_to_post_save(obj.__class__, obj.pk, idx_line) if self.TYPE == "gis": + if not obj and not self.c_errors: + self.c_errors = True + self.errors.append( + ( + idx_line + 1, + "-", + self.ERRORS["gis_missing_obj"].format(str(data)), + ) + ) + return + content_type = ContentType.objects.get( app_label=obj.__class__._meta.app_label, model=obj.__class__.__name__.lower() diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py index e4f432b4f..822efadda 100644 --- a/ishtar_common/forms_common.py +++ b/ishtar_common/forms_common.py @@ -215,6 +215,27 @@ class BaseImportForm(IshtarForm, forms.ModelForm): self.fields["imported_file"].validators = [file_size_validator] self._post_init() + def _clean_csv(self, is_csv=False): + imported_file = self.cleaned_data.get("imported_file", None) + encoding = self.cleaned_data.get("encoding", None) + print(encoding) + if imported_file and encoding: + try: + if not imported_file.name.lower().endswith(".csv"): + if is_csv: + raise AssertionError() + else: + return + imported_file.seek(0) + reader = csv.reader(StringIO(imported_file.read().decode(encoding))) + for __ in reader: + break + imported_file.seek(0) + except (AssertionError, UnicodeDecodeError) as e: + raise forms.ValidationError( + _("This is not a valid CSV file. Check file format and encoding.") + ) + def clean(self): data = self.cleaned_data if ( @@ -273,21 +294,7 @@ class NewImportForm(BaseImportForm): raise forms.ValidationError( _("\"Associated images\" field must be a valid zip file.") ) - imported_file = self.cleaned_data.get("imported_file", None) - encoding = self.cleaned_data.get("encoding", None) - if imported_file and encoding: - try: - assert imported_file.name.lower().endswith(".csv") - imported_file.seek(0) - reader = csv.reader(StringIO(imported_file.read().decode(encoding))) - for __ in reader: - break - imported_file.seek(0) - except (AssertionError, UnicodeDecodeError) as e: - raise forms.ValidationError( - _("This is not a valid CSV file. Check file format and encoding.") - ) - + self._clean_csv(is_csv=True) return data def clean_imported_images_link(self): @@ -333,6 +340,8 @@ class NewImportGISForm(BaseImportForm): "name", "importer_type", "imported_file", + "encoding", + "csv_sep", "associated_group", ) @@ -345,7 +354,7 @@ class NewImportGISForm(BaseImportForm): if value: try: ext = value.name.lower().split(".")[-1] - assert ext in ("zip", "gpkg") + assert ext in ("zip", "gpkg", "csv") if ext == "zip": zip_file = zipfile.ZipFile(value) assert not zip_file.testzip() @@ -353,7 +362,6 @@ class NewImportGISForm(BaseImportForm): for name in zip_file.namelist(): in_ext = name.lower().split(".")[-1] if in_ext in ("shp", "gpkg"): - filename = name has_correct_file = True break assert has_correct_file @@ -363,6 +371,11 @@ class NewImportGISForm(BaseImportForm): ) return value + def clean(self): + data = super().clean() + self._clean_csv() + return data + def save(self, user, commit=True): self.instance.user = user item = super(NewImportGISForm, self).save(commit) diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py index cea327e13..c88c71c35 100644 --- a/ishtar_common/models_common.py +++ b/ishtar_common/models_common.py @@ -2374,10 +2374,9 @@ class GeoVectorData(Imported, OwnPerms): if profile.display_srs and profile.display_srs.srid: srid = profile.display_srs.srid if not srid: - x, y = point_2d.x, point_2d.y - else: - point = point_2d.transform(srid, clone=True) - x, y = point.x, point.y + srid = 4326 + point = point_2d.transform(srid, clone=True) + x, y = point.x, point.y if rounded: return [round(x, rounded), round(y, rounded)] return [x, y] diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index f7a35f849..1d987c35c 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -1153,7 +1153,8 @@ class Import(models.Model): ), ) encoding = models.CharField( - _("Encoding"), choices=ENCODINGS, default="utf-8", max_length=15 + _("Encoding"), choices=ENCODINGS, default="utf-8", max_length=15, + help_text=_("Only required for CSV file"), ) csv_sep = models.CharField( _("CSV separator"), @@ -1463,6 +1464,8 @@ class Import(models.Model): imported_file = z.extract(filename, tmp_dir) else: z.extract(filename, tmp_dir) + elif imported_file.endswith(".csv"): + return self._data_table_tab() elif not imported_file.endswith(".gpkg"): raise ImporterError(_("Invalid GIS file.")) diff --git a/ishtar_common/version.py b/ishtar_common/version.py index 9425d458a..605cb0580 100644 --- a/ishtar_common/version.py +++ b/ishtar_common/version.py @@ -1,5 +1,5 @@ -# 4.0.13 -VERSION = (4, 0, 13) +# 4.0.14 +VERSION = (4, 0, 14) def get_version(): |