1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import csv
import datetime
import os
import sys
from django.conf import settings
from django.core.management.base import BaseCommand
from ishtar_common.models import Town
from archaeological_operations.models import Operation
log_path = os.sep.join([settings.ROOT_PATH, "logs"])
if not os.path.exists(log_path):
os.mkdir(log_path, mode=0o770)
def percent(current, total):
return f"{(current + 1) / total * 100:.1f}".rjust(4, "0") + "%"
def get_time():
return datetime.datetime.now().isoformat().split(".")[0]
class Command(BaseCommand):
help = "Verification of city attachment (old/new) of operations by year"
def add_arguments(self, parser):
parser.add_argument(
"--log", dest="log", action="store_true", help="Log into a file"
)
def handle(self, *args, **options):
q = Town.objects.filter(numero_insee__contains='-')
checked = []
ln = q.count()
fixed = 0
store_results = []
for idx, result in enumerate(q.values_list("numero_insee", "id")):
sys.stdout.write(f"\r* checking towns and associated operations {idx + 1}/{ln}" + " " * 20)
sys.stdout.flush()
num_insee, id_town = result
base_insee, year = num_insee.split("-")
if base_insee in checked:
continue
try:
years = [(int(year), id_town)]
except ValueError:
continue
checked.append(base_insee)
q2 = Town.objects.filter(numero_insee__startswith=base_insee + "-").exclude(pk=id_town)
for other_insee, other_id in q2.values_list("numero_insee", "id"):
years.append((int(other_insee.split("-")[-1]), other_id))
years = list(sorted(years))
q3 = Town.objects.filter(numero_insee=base_insee)
if not q3.count():
continue
base_town = q3.all()[0]
previous_year = 0
ids = [base_town.pk]
years.append((base_town.year, base_town.pk))
all_towns = [base_town] + [t for t in Town.objects.filter(numero_insee__startswith=base_insee + "-")]
for idx_y, yt in enumerate(years):
year, town_id = yt
is_last = idx_y == len(years) - 1
if is_last: # last town do not take the current town
ids = ids[1:]
q4 = Operation.objects.filter(towns__pk__in=ids)
if is_last:
q4 = q4.filter(year__gt=year)
else:
q4 = q4.filter(year__lte=years[idx_y + 1][0])
if previous_year:
q4 = q4.filter(year__gt=previous_year)
if not q4.count():
previous_year = year
ids.append(town_id)
continue
town = Town.objects.get(pk=town_id)
other_town = [t for t in all_towns if t.pk != town_id]
for operation in q4.all():
sys.stdout.write(f"\r* {idx + 1}/{ln} - fixing {operation}" + " " * 20)
sys.stdout.flush()
fixed += 1
store_results.append((operation.pk, str(operation), f'{town.name} ({town.numero_insee})'))
operation.skip_history_when_saving = True
if town not in list(operation.towns.all()):
operation.towns.add(town)
if town.main_geodata:
if town.main_geodata_id not in operation.geodata.values_list("id", flat=True):
operation.geodata.add(town.main_geodata)
if not operation.main_geodata or operation.main_geodata in [t.main_geodata for t in other_town]:
operation.main_geodata = town.main_geodata
operation.save()
for t in other_town:
operation.towns.remove(t)
previous_year = year
ids.append(town_id)
if fixed:
cmsg = f"\r{fixed} operation(s) fixed" + 120 * " " + "\n"
else:
cmsg = "\r" + " " * 120 + "\n"
sys.stdout.write(cmsg)
log = options.get("log", False)
if log and fixed:
log_name = "operation-fix-merged-towns"
csv_cols = ["id", "operation", "town"]
filename = f"{get_time().replace(':', '')}-{log_name}.csv"
path = os.sep.join([log_path, filename])
with open(path, 'w+') as fle:
writer = csv.writer(fle)
writer.writerow(csv_cols)
writer.writerows(store_results)
sys.stdout.write(f"log: {path} written.\n")
|
