#!/usr/bin/env python3 # -*- coding: utf-8 -*- import csv import datetime import os import sys from django.conf import settings from django.core.management.base import BaseCommand from ishtar_common.models import Town from archaeological_operations.models import Operation log_path = os.sep.join([settings.ROOT_PATH, "logs"]) if not os.path.exists(log_path): os.mkdir(log_path, mode=0o770) def percent(current, total): return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%" def get_time(): return datetime.datetime.now().isoformat().split(".")[0] class Command(BaseCommand): help = "Verification of city attachment (old/new) of operations by year" def add_arguments(self, parser): parser.add_argument( "--log", dest="log", action="store_true", help="Log into a file" ) def handle(self, *args, **options): q = Town.objects.filter(numero_insee__contains='-') checked = [] ln = q.count() fixed = 0 store_results = [] for idx, result in enumerate(q.values_list("numero_insee", "id")): sys.stdout.write(f"\r* checking towns and associated operations {idx + 1}/{ln}" + " " * 20) sys.stdout.flush() num_insee, id_town = result base_insee, year = num_insee.split("-") if base_insee in checked: continue try: years = [(int(year), id_town)] except ValueError: continue checked.append(base_insee) q2 = Town.objects.filter(numero_insee__startswith=base_insee + "-").exclude(pk=id_town) for other_insee, other_id in q2.values_list("numero_insee", "id"): years.append((int(other_insee.split("-")[-1]), other_id)) years = list(sorted(years)) q3 = Town.objects.filter(numero_insee=base_insee) if not q3.count(): continue base_town = q3.all()[0] previous_year = 0 ids = [base_town.pk] years.append((base_town.year, base_town.pk)) all_towns = [base_town] + [t for t in Town.objects.filter(numero_insee__startswith=base_insee + "-")] for idx_y, yt in enumerate(years): year, town_id = yt is_last = idx_y == len(years) - 1 if is_last: # last town do not take the current town ids = ids[1:] q4 = Operation.objects.filter(towns__pk__in=ids) if is_last: q4 = q4.filter(year__gt=year) else: q4 = q4.filter(year__lte=years[idx_y + 1][0]) if previous_year: q4 = q4.filter(year__gt=previous_year) if not q4.count(): previous_year = year ids.append(town_id) continue town = Town.objects.get(pk=town_id) other_town = [t for t in all_towns if t.pk != town_id] for operation in q4.all(): sys.stdout.write(f"\r* {idx + 1}/{ln} - fixing {operation}" + " " * 20) sys.stdout.flush() fixed += 1 store_results.append((operation.pk, str(operation), f'{town.name} ({town.numero_insee})')) operation.skip_history_when_saving = True if town not in list(operation.towns.all()): operation.towns.add(town) if town.main_geodata: if town.main_geodata_id not in operation.geodata.values_list("id", flat=True): operation.geodata.add(town.main_geodata) if not operation.main_geodata or operation.main_geodata in [t.main_geodata for t in other_town]: operation.main_geodata = town.main_geodata operation.save() for t in other_town: operation.towns.remove(t) previous_year = year ids.append(town_id) if fixed: cmsg = f"\r{fixed} operation(s) fixed" + 120 * " " + "\n" else: cmsg = "\r" + " " * 120 + "\n" sys.stdout.write(cmsg) log = options.get("log", False) if log and fixed: log_name = "operation-fix-merged-towns" csv_cols = ["id", "operation", "town"] filename = f"{get_time().replace(':', '')}-{log_name}.csv" path = os.sep.join([log_path, filename]) with open(path, 'w+') as fle: writer = csv.writer(fle) writer.writerow(csv_cols) writer.writerows(store_results) sys.stdout.write(f"log: {path} written.\n")