diff options
| -rw-r--r-- | changelog/en/changelog_2022-06-15.md | 1 | ||||
| -rw-r--r-- | changelog/fr/changelog_2023-01-25.md | 1 | ||||
| -rw-r--r-- | ishtar_common/management/commands/ishtar_operation_fix_merged_towns.py | 119 | ||||
| -rw-r--r-- | ishtar_common/models_common.py | 17 |
4 files changed, 132 insertions, 6 deletions
diff --git a/changelog/en/changelog_2022-06-15.md b/changelog/en/changelog_2022-06-15.md index 19767fb1e..2549b90c4 100644 --- a/changelog/en/changelog_2022-06-15.md +++ b/changelog/en/changelog_2022-06-15.md @@ -32,6 +32,7 @@ v4.0.XX - 2099-12-31 ### Technical ### +- correction script grouped communes allocation to operations #### Imports ##### - update relationship between imports and main items diff --git a/changelog/fr/changelog_2023-01-25.md b/changelog/fr/changelog_2023-01-25.md index 106207f96..2fc8f7915 100644 --- a/changelog/fr/changelog_2023-01-25.md +++ b/changelog/fr/changelog_2023-01-25.md @@ -34,6 +34,7 @@ v4.0.XX - 2099-12-31 ### Technique ### +- script de correction des affectations de communes regroupées aux opérations #### Imports ##### - relation de mise à jour entre imports et les éléments principaux diff --git a/ishtar_common/management/commands/ishtar_operation_fix_merged_towns.py b/ishtar_common/management/commands/ishtar_operation_fix_merged_towns.py new file mode 100644 index 000000000..f7613b1b4 --- /dev/null +++ b/ishtar_common/management/commands/ishtar_operation_fix_merged_towns.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import csv +import datetime +import os +import sys + +from django.conf import settings +from django.core.management.base import BaseCommand + +from ishtar_common.models import Town +from archaeological_operations.models import Operation + +log_path = os.sep.join([settings.ROOT_PATH, "logs"]) +if not os.path.exists(log_path): + os.mkdir(log_path, mode=0o770) + + +def percent(current, total): + return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%" + + +def get_time(): + return datetime.datetime.now().isoformat().split(".")[0] + + +class Command(BaseCommand): + help = "Verification of city attachment (old/new) of operations by year" + + def add_arguments(self, parser): + parser.add_argument( + "--log", dest="log", action="store_true", help="Log into a file" + ) + + def handle(self, *args, **options): + q = Town.objects.filter(numero_insee__contains='-') + checked = [] + ln = q.count() + fixed = 0 + store_results = [] + for idx, result in enumerate(q.values_list("numero_insee", "id")): + sys.stdout.write(f"\r* checking towns and associated operations {idx + 1}/{ln}" + " " * 20) + sys.stdout.flush() + num_insee, id_town = result + base_insee, year = num_insee.split("-") + if base_insee in checked: + continue + try: + years = [(int(year), id_town)] + except ValueError: + continue + checked.append(base_insee) + q2 = Town.objects.filter(numero_insee__startswith=base_insee + "-").exclude(pk=id_town) + for other_insee, other_id in q2.values_list("numero_insee", "id"): + years.append((int(other_insee.split("-")[-1]), other_id)) + years = list(sorted(years)) + q3 = Town.objects.filter(numero_insee=base_insee) + if not q3.count(): + continue + base_town = q3.all()[0] + previous_year = 0 + ids = [base_town.pk] + years.append((base_town.year, base_town.pk)) + all_towns = [base_town] + [t for t in Town.objects.filter(numero_insee__startswith=base_insee + "-")] + for idx_y, yt in enumerate(years): + year, town_id = yt + is_last = idx_y == len(years) - 1 + if is_last: # last town do not take the current town + ids = ids[1:] + q4 = Operation.objects.filter(towns__pk__in=ids) + if is_last: + q4 = q4.filter(year__gt=year) + else: + q4 = q4.filter(year__lte=years[idx_y + 1][0]) + if previous_year: + q4 = q4.filter(year__gt=previous_year) + if not q4.count(): + previous_year = year + ids.append(town_id) + continue + town = Town.objects.get(pk=town_id) + other_town = [t for t in all_towns if t.pk != town_id] + for operation in q4.all(): + sys.stdout.write(f"\r* {idx + 1}/{ln} - fixing {operation}" + " " * 20) + sys.stdout.flush() + fixed += 1 + store_results.append((operation.pk, str(operation), f'{town.name} ({town.numero_insee})')) + operation.skip_history_when_saving = True + if town not in list(operation.towns.all()): + operation.towns.add(town) + if town.main_geodata: + if town.main_geodata_id not in operation.geodata.values_list("id", flat=True): + operation.geodata.add(town.main_geodata) + if not operation.main_geodata or operation.main_geodata in [t.main_geodata for t in other_town]: + operation.main_geodata = town.main_geodata + operation.save() + for t in other_town: + operation.towns.remove(t) + previous_year = year + ids.append(town_id) + + if fixed: + cmsg = f"\r{fixed} operation(s) fixed" + 120 * " " + "\n" + else: + cmsg = "\r" + " " * 120 + "\n" + sys.stdout.write(cmsg) + log = options.get("log", False) + if log and fixed: + log_name = "operation-fix-merged-towns" + csv_cols = ["id", "operation", "town"] + filename = f"{get_time().replace(':', '')}-{log_name}.csv" + path = os.sep.join([log_path, filename]) + with open(path, 'w+') as fle: + writer = csv.writer(fle) + writer.writerow(csv_cols) + writer.writerows(store_results) + sys.stdout.write(f"log: {path} written.\n") + diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py index e4438be43..928b22630 100644 --- a/ishtar_common/models_common.py +++ b/ishtar_common/models_common.py @@ -2946,12 +2946,17 @@ class GeographicItem(models.Model): self.geodata.add(self.main_geodata) except (OperationalError, IntegrityError): pass - elif not self.main_geodata and self.geodata.count(): - # arbitrary associate the first to geodata - self.main_geodata = self.geodata.order_by("pk").all()[0] - self.skip_history_when_saving = True - self._no_move = True - self.save() + elif not self.main_geodata: + try: + with transaction.atomic(): + if self.geodata.count(): + # arbitrary associate the first to geodata + self.main_geodata = self.geodata.order_by("pk").all()[0] + self.skip_history_when_saving = True + self._no_move = True + self.save() + except (OperationalError, IntegrityError, IndexError): + pass @property def geodata_list(self): |