diff options
| -rw-r--r-- | archaeological_context_records/models.py | 58 | ||||
| -rw-r--r-- | archaeological_finds/models_finds.py | 9 | ||||
| -rw-r--r-- | ishtar_common/management/commands/migrate_to_geo_v4.py | 152 | ||||
| -rw-r--r-- | ishtar_common/utils.py | 18 |
4 files changed, 210 insertions, 27 deletions
diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py index 29e8793dc..e9bd70aa8 100644 --- a/archaeological_context_records/models.py +++ b/archaeological_context_records/models.py @@ -397,12 +397,61 @@ class CRBulkView(object): """ +class GeographicSubTownItem(GeoItem): + class Meta: + abstract = True + + def _get_geo_town(self): + raise NotImplementedError() + + def post_save_geo(self, save=True): + # manage geodata towns + if getattr(self, "_post_save_geo_ok", False): + # prevent infinite loop - should not happen, but... + return + self._post_save_geo_ok = True + + q_geodata_current_town = self.geodata.filter( + source_content_type__model="town", + source_content_type__app_label="ishtar_common", + ) + town = self._get_geo_town() + has_geo_town = ( + town and town.main_geodata and town.main_geodata.multi_polygon + ) + if has_geo_town: + bad_towns = q_geodata_current_town + else: + bad_towns = q_geodata_current_town.exclude(source_id=town.id) + + modified = False + for bad_town in bad_towns.all(): + self.geodata.remove(bad_town) + if self.main_geodata == bad_town: + self.main_geodata = None + modified = True + + if not has_geo_town: + if modified and save: + self.skip_history_when_saving = True + self._no_move = True + self.save() + return + + if not q_geodata_current_town.filter(source_id=town.id).count(): + self.geodata.add(town.main_geodata) + if save: + self.skip_history_when_saving = True + self._no_move = True + self.save() + + class ContextRecord( BulkUpdatedItem, DocumentItem, BaseHistorizedItem, CompleteIdentifierItem, - GeoItem, + GeographicSubTownItem, OwnPerms, ValueGetter, MainItem, @@ -510,7 +559,7 @@ class ContextRecord( ), "operation__common_name": SearchAltName( pgettext_lazy("key for text search", "operation-name"), - "operation__common_name__iexact" + "operation__common_name__iexact", ), "operation__code_patriarche": SearchAltName( pgettext_lazy("key for text search", "patriarche"), @@ -810,6 +859,11 @@ class ContextRecord( if self.surface: return self.surface / 10000.0 + def _get_geo_town(self): + if self.parcel: + return self.parcel.town + return self.town + def public_representation(self): dct = super(ContextRecord, self).public_representation() dct.update( diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py index bc1d64ee6..ce9a5e4af 100644 --- a/archaeological_finds/models_finds.py +++ b/archaeological_finds/models_finds.py @@ -55,7 +55,6 @@ from ishtar_common.models import ( ValueGetter, get_current_profile, IshtarSiteProfile, - GeoItem, BulkUpdatedItem, QuickAction, MainItem, @@ -76,7 +75,8 @@ from archaeological_operations.models import ( Operation, CulturalAttributionType, ) -from archaeological_context_records.models import ContextRecord, Dating +from archaeological_context_records.models import ContextRecord, Dating, \ + GeographicSubTownItem from archaeological_warehouse.models import Warehouse @@ -386,7 +386,7 @@ class BFBulkView(object): class BaseFind( BulkUpdatedItem, BaseHistorizedItem, - GeoItem, + GeographicSubTownItem, CompleteIdentifierItem, OwnPerms, ValueGetter, @@ -486,6 +486,9 @@ class BaseFind( def __str__(self): return self.label + def _get_geo_town(self): + return self.context_record._get_geo_town() + def natural_key(self): return (self.uuid,) diff --git a/ishtar_common/management/commands/migrate_to_geo_v4.py b/ishtar_common/management/commands/migrate_to_geo_v4.py index ed1a877f3..2c27eebe9 100644 --- a/ishtar_common/management/commands/migrate_to_geo_v4.py +++ b/ishtar_common/management/commands/migrate_to_geo_v4.py @@ -10,10 +10,11 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand -from ishtar_common.utils import ugettext_lazy as _ +from ishtar_common.utils import ugettext_lazy as _, get_log_time, get_percent, get_eta from ishtar_common import models_common, models from archaeological_operations.models import Operation, ArchaeologicalSite +from archaeological_context_records.models import ContextRecord log_path = os.sep.join([settings.ROOT_PATH, "logs"]) @@ -21,6 +22,14 @@ if not os.path.exists(log_path): os.mkdir(log_path, mode=0o770) +def write_output(model_name, idx, nb, ref_time=None): + lbl = f"\r[{get_percent(idx, nb)}] Migrate {model_name}s {idx + 1}/{nb}" + if ref_time: + lbl += f" ({get_eta(idx, nb, ref_time, datetime.datetime.now())} left)" + sys.stdout.write(lbl) + sys.stdout.flush() + + def migrate(quiet=False, log=True): changed = [] # create towns @@ -35,10 +44,10 @@ def migrate(quiet=False, log=True): provider, __ = models_common.GeoProviderType.objects.get_or_create( txt_idx="france-ign", defaults={"label": "IGN"} ) + ref_time = datetime.datetime.now() for idx, town in enumerate(q.all()): if not quiet: - sys.stdout.write(f"\r[{percent(idx, nb)}] Migrate towns {idx + 1}/{nb}") - sys.stdout.flush() + write_output("town", idx, nb, ref_time) attrs = { "name": town._generate_cached_label(), "source_content_type": town_content_type, @@ -56,7 +65,7 @@ def migrate(quiet=False, log=True): town.main_geodata = data town.save() if not quiet and nb: - sys.stdout.write(f"\r[{get_time()}] Towns migrated\n") + sys.stdout.write(f"\r[{get_log_time()}] Towns migrated \n") sys.stdout.flush() model_list = [ @@ -75,14 +84,13 @@ def migrate(quiet=False, log=True): defaults={"label": f"Emprise {model_full_name}"}, ) data_type_center, __ = models_common.GeoDataType.objects.get_or_create( - txt_idx="operation-center", defaults={"label": f"Centre {model_full_name}"} + txt_idx=f"{model_slug}-center", + defaults={"label": f"Centre {model_full_name}"}, ) + ref_time = datetime.datetime.now() for idx, obj in enumerate(q.all()): if not quiet: - sys.stdout.write( - f"\r[{percent(idx, nb)}] Migrate {model_name}s {idx + 1}/{nb}" - ) - sys.stdout.flush() + write_output(model_name, idx, nb, ref_time) obj._no_move = True obj.skip_history_when_saving = True @@ -155,11 +163,119 @@ def migrate(quiet=False, log=True): ["geovectordata", data.name, data.pk, f"Point {model_name}"] ) if not quiet and nb: - sys.stdout.write(f"\r[{get_time()}] {model_name.capitalize()} migrated\n") + sys.stdout.write( + f"\r[{get_log_time()}] {model_name.capitalize()} migrated" + + " " * 20 + + "\n" + ) sys.stdout.flush() + model_list = [ + ( + "archaeological_context_records", + "contextrecord", + "unité d'enregistrement", + "de l'unité d'enregistrement", + ContextRecord, + ), + ( + "archaeological_finds", + "basefind", + "mobilier d'origine", + "du mobilier d'origine", + ContextRecord, + ), + ] + for app, model_slug, model_name, model_full_name, model in model_list: + model_content_type = ContentType.objects.get(app_label=app, model=model_slug) + q = model.objects.exclude(main_geodata__isnull=False) + nb = q.count() + data_type_outline, __ = models_common.GeoDataType.objects.get_or_create( + txt_idx=f"{model_slug}-outline", + defaults={"label": f"Contour d'{model_name}"}, + ) + data_type_center, __ = models_common.GeoDataType.objects.get_or_create( + txt_idx=f"{model_slug}-center", + defaults={"label": f"Centre {model_full_name}"}, + ) + ref_time = datetime.datetime.now() + for idx, obj in enumerate(q.all()[:100]): + if not quiet: + write_output(model_name, idx, nb, ref_time) + obj._no_move = True + obj.skip_history_when_saving = True + obj.save() # auto manage geo town association + + if obj.main_geodata: + changed.append( + [model_slug, str(obj), obj.pk, "Association géo de zone communale"] + ) + if obj.multi_polygon_source == "P" and obj.multi_polygon: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "multi_polygon": obj.multi_polygon, + "data_type": data_type_outline, + } + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj.save() + changed.append( + [ + "geovectordata", + data.name, + data.pk, + f"Multi-polygone {model_name}", + ] + ) + if obj.point_source == "P" and obj.point_2d: + if obj.x and obj.y: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "data_type": data_type_center, + "x": obj.x, + "y": obj.y, + "z": obj.z, + } + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj.save() + changed.append( + [ + "geovectordata", + data.name, + data.pk, + f"Coordonnées {model_name}", + ] + ) + elif obj.point_2d: + attrs = { + "name": f"{_(model_name.capitalize())}{_(':')} {str(obj)}", + "source_content_type": model_content_type, + "source_id": obj.pk, + "data_type": data_type_center, + } + if obj.point: + attrs["point_3d"] = obj.point + else: + attrs["point_2d"] = obj.point_2d + data = models_common.GeoVectorData.objects.create(**attrs) + obj.main_geodata = data + obj.save() + changed.append( + ["geovectordata", data.name, data.pk, f"Point {model_name}"] + ) + if not quiet and nb: + sys.stdout.write( + f"\r[{get_log_time()}] {model_name.capitalize()} migrated" + " " * 20 + "\n" + ) + sys.stdout.flush() + if log and changed: - filename = f"geo_migration-created-{get_time().replace(':', '')}.csv" + filename = f"geo_migration-created-{get_log_time().replace(':', '')}.csv" path = os.sep.join([log_path, filename]) with open(path, "w+") as fle: writer = csv.writer(fle) @@ -167,15 +283,7 @@ def migrate(quiet=False, log=True): for change in changed: writer.writerow(change) if not quiet: - sys.stdout.write(f"[{get_time()}] Log: {path} written\n") - - -def percent(current, total): - return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%" - - -def get_time(): - return datetime.datetime.now().isoformat().split(".")[0] + sys.stdout.write(f"[{get_log_time()}] Log: {path} written\n") class Command(BaseCommand): @@ -193,11 +301,11 @@ class Command(BaseCommand): log = options["log"] quiet = options["quiet"] if not quiet: - sys.stdout.write(f"[{get_time()}] Processing migration\n") + sys.stdout.write(f"[{get_log_time()}] Processing migration\n") errors = migrate(quiet=quiet, log=log) if not errors: if not quiet: - sys.stdout.write(f"[{get_time()}] Migration finished\n") + sys.stdout.write(f"[{get_log_time()}] Migration finished\n") sys.exit() if not quiet: sys.stdout.write("\n".join(errors)) diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index fc302166b..a7e07ebde 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -2335,3 +2335,21 @@ def create_osm_town(rel_id, name, numero_insee=None): town.center = town.limit.centroid town.save() return town + + +def get_percent(current, total): + return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%" + + +def get_log_time(): + return datetime.datetime.now().isoformat().split(".")[0] + + +def get_eta(current, total, base_time, current_time): + if current < 5: + return "-" + elapsed_time = current_time - base_time + eta = elapsed_time.seconds / current * (total - current) + if eta < 1: + return "-" + return f"{int(eta // 3600):02d}:{int(eta % 3600 // 60):02d}:{int(eta % 60):02d}" |