✨ scripts: ishtar_operation_fix_merged_towns

2 files changed, 130 insertions, 6 deletions

diff --git a/ishtar_common/management/commands/ishtar_operation_fix_merged_towns.py b/ishtar_common/management/commands/ishtar_operation_fix_merged_towns.py
new file mode 100644
index 000000000..f7613b1b4
--- /dev/null
+++ b/ishtar_common/management/commands/ishtar_operation_fix_merged_towns.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import csv
+import datetime
+import os
+import sys
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ishtar_common.models import Town
+from archaeological_operations.models import Operation
+
+log_path = os.sep.join([settings.ROOT_PATH, "logs"])
+if not os.path.exists(log_path):
+    os.mkdir(log_path, mode=0o770)
+
+
+def percent(current, total):
+    return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%"
+
+
+def get_time():
+    return datetime.datetime.now().isoformat().split(".")[0]
+
+
+class Command(BaseCommand):
+    help = "Verification of city attachment (old/new) of operations by year"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--log", dest="log", action="store_true", help="Log into a file"
+        )
+
+    def handle(self, *args, **options):
+        q = Town.objects.filter(numero_insee__contains='-')
+        checked = []
+        ln = q.count()
+        fixed = 0
+        store_results = []
+        for idx, result in enumerate(q.values_list("numero_insee", "id")):
+            sys.stdout.write(f"\r* checking towns and associated operations {idx + 1}/{ln}" + " " * 20)
+            sys.stdout.flush()
+            num_insee, id_town = result
+            base_insee, year = num_insee.split("-")
+            if base_insee in checked:
+                continue
+            try:
+                years = [(int(year), id_town)]
+            except ValueError:
+                continue
+            checked.append(base_insee)
+            q2 = Town.objects.filter(numero_insee__startswith=base_insee + "-").exclude(pk=id_town)
+            for other_insee, other_id in q2.values_list("numero_insee", "id"):
+                years.append((int(other_insee.split("-")[-1]), other_id))
+            years = list(sorted(years))
+            q3 = Town.objects.filter(numero_insee=base_insee)
+            if not q3.count():
+                continue
+            base_town = q3.all()[0]
+            previous_year = 0
+            ids = [base_town.pk]
+            years.append((base_town.year, base_town.pk))
+            all_towns = [base_town] + [t for t in Town.objects.filter(numero_insee__startswith=base_insee + "-")]
+            for idx_y, yt in enumerate(years):
+                year, town_id = yt
+                is_last = idx_y == len(years) - 1
+                if is_last:  # last town do not take the current town
+                    ids = ids[1:]
+                q4 = Operation.objects.filter(towns__pk__in=ids)
+                if is_last:
+                    q4 = q4.filter(year__gt=year)
+                else:
+                    q4 = q4.filter(year__lte=years[idx_y + 1][0])
+                if previous_year:
+                    q4 = q4.filter(year__gt=previous_year)
+                if not q4.count():
+                    previous_year = year
+                    ids.append(town_id)
+                    continue
+                town = Town.objects.get(pk=town_id)
+                other_town = [t for t in all_towns if t.pk != town_id]
+                for operation in q4.all():
+                    sys.stdout.write(f"\r* {idx + 1}/{ln} - fixing {operation}" + " " * 20)
+                    sys.stdout.flush()
+                    fixed += 1
+                    store_results.append((operation.pk, str(operation), f'{town.name} ({town.numero_insee})'))
+                    operation.skip_history_when_saving = True
+                    if town not in list(operation.towns.all()):
+                        operation.towns.add(town)
+                    if town.main_geodata:
+                        if town.main_geodata_id not in operation.geodata.values_list("id", flat=True):
+                            operation.geodata.add(town.main_geodata)
+                        if not operation.main_geodata or operation.main_geodata in [t.main_geodata for t in other_town]:
+                            operation.main_geodata = town.main_geodata
+                            operation.save()
+                    for t in other_town:
+                        operation.towns.remove(t)
+                previous_year = year
+                ids.append(town_id)
+
+        if fixed:
+            cmsg = f"\r{fixed} operation(s) fixed" + 120 * " " + "\n"
+        else:
+            cmsg = "\r" + " " * 120 + "\n"
+        sys.stdout.write(cmsg)
+        log = options.get("log", False)
+        if log and fixed:
+            log_name = "operation-fix-merged-towns"
+            csv_cols = ["id", "operation", "town"]
+            filename = f"{get_time().replace(':', '')}-{log_name}.csv"
+            path = os.sep.join([log_path, filename])
+            with open(path, 'w+') as fle:
+                writer = csv.writer(fle)
+                writer.writerow(csv_cols)
+                writer.writerows(store_results)
+            sys.stdout.write(f"log: {path} written.\n")
+
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index e4438be43..928b22630 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -2946,12 +2946,17 @@ class GeographicItem(models.Model):
                     self.geodata.add(self.main_geodata)
             except (OperationalError, IntegrityError):
                 pass
-        elif not self.main_geodata and self.geodata.count():
-            # arbitrary associate the first to geodata
-            self.main_geodata = self.geodata.order_by("pk").all()[0]
-            self.skip_history_when_saving = True
-            self._no_move = True
-            self.save()
+        elif not self.main_geodata:
+            try:
+                with transaction.atomic():
+                    if self.geodata.count():
+                        # arbitrary associate the first to geodata
+                        self.main_geodata = self.geodata.order_by("pk").all()[0]
+                        self.skip_history_when_saving = True
+                        self._no_move = True
+                        self.save()
+            except (OperationalError, IntegrityError, IndexError):
+                pass
 
     @property
     def geodata_list(self):