diff options
author | Étienne Loks <etienne.loks@iggdrasil.net> | 2025-05-12 14:59:21 +0200 |
---|---|---|
committer | Étienne Loks <etienne.loks@iggdrasil.net> | 2025-05-13 10:40:31 +0200 |
commit | 92ea61d5bc4713b151eb3c7d513f9e3699ccb2eb (patch) | |
tree | 7b256e3a950fbf07911a60a3ca6e1c831624a377 /ishtar_common/management/commands/import_insee_comm_csv.py | |
parent | f3f0498e7001abe9676b6506baea07e820cbf60b (diff) | |
download | Ishtar-92ea61d5bc4713b151eb3c7d513f9e3699ccb2eb.tar.bz2 Ishtar-92ea61d5bc4713b151eb3c7d513f9e3699ccb2eb.zip |
✨ import_geofla_csv, import_insee_comm_csv: logging, better verbose CLI
Diffstat (limited to 'ishtar_common/management/commands/import_insee_comm_csv.py')
-rw-r--r-- | ishtar_common/management/commands/import_insee_comm_csv.py | 119 |
1 files changed, 68 insertions, 51 deletions
diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py index 2e990c7ce..4584da890 100644 --- a/ishtar_common/management/commands/import_insee_comm_csv.py +++ b/ishtar_common/management/commands/import_insee_comm_csv.py @@ -18,13 +18,17 @@ # See the file COPYING for details. import csv +import os import re import sys +from django.conf import settings from django.core.management.base import BaseCommand from ishtar_common.models import Town +from ishtar_common.utils import BColors, get_log_time + class Command(BaseCommand): help = 'Import INSEE csv' @@ -39,75 +43,88 @@ class Command(BaseCommand): help='Quiet output') def handle(self, *args, **options): + log_path = os.sep.join([settings.ROOT_PATH, "logs"]) + if not os.path.exists(log_path): + os.mkdir(log_path, mode=0o770) + csv_file = options['csv_file'] default_year = options['year'] quiet = options['quiet'] if not quiet: - sys.stdout.write('* using year {} for new towns\n'.format( - default_year)) - sys.stdout.write('* opening file {}\n'.format(csv_file)) + sys.stdout.write(BColors.OKGREEN) + sys.stdout.write(f'* Using year {default_year} for new towns\n') + sys.stdout.write(f'* Opening file {csv_file}{BColors.ENDC}\n') r = re.compile(r"(.*)\((.*)\)") nb_created = 0 nb_link = 0 missing = [] strange = [] linked = set() - with open(csv_file, 'rt') as csvfile: - reader = csv.DictReader(csvfile) - for idx, row in enumerate(reader): - new_insee = row['DepComN'] - if len(new_insee) < 5: - new_insee = '0' + new_insee - - if not idx: # test if first do not exist + + log_filename = f"import_insee-{get_log_time().replace(':', '')}.csv" + log_path = os.sep.join([log_path, log_filename]) + with open(log_path, "w+") as fle: + writer = csv.writer(fle) + writer.writerow(["new insee", "new town"]) + with open(csv_file, 'rt') as csvfile: + reader = csv.DictReader(csvfile) + for idx, row in enumerate(reader): + new_insee = row['DepComN'] + if len(new_insee) < 5: + new_insee = '0' + new_insee + + if not idx: # test if first do not exist + q = Town.objects.filter(numero_insee=new_insee, + year=default_year) + if q.count(): + sys.stdout.write( + f"{BColors.FAIL}First town already exists for this year{BColors.ENDC}\n") + return + + if not quiet: + sys.stdout.write('Processing town %d.\r' % (idx + 1)) + sys.stdout.flush() + + old_insee = row['DepComA'] + if len(old_insee) < 5: + old_insee = '0' + old_insee + q = Town.objects.filter(numero_insee=old_insee) + + if not q.count(): + missing.append((old_insee, row['NomCA'])) + continue + if q.count() > 1: + q = q.filter(year__lt=default_year).order_by('-year') + if not q.count(): + strange.append((old_insee, row['NomCA'])) + continue + old_town = q.all()[0] + q = Town.objects.filter(numero_insee=new_insee, year=default_year) - if q.count(): - print("First town already exists for this year....") - return - - if not quiet: - sys.stdout.write('Processing town %d.\r' % (idx + 1)) - sys.stdout.flush() - - old_insee = row['DepComA'] - if len(old_insee) < 5: - old_insee = '0' + old_insee - q = Town.objects.filter(numero_insee=old_insee) - - if not q.count(): - missing.append((old_insee, row['NomCA'])) - continue - if q.count() > 1: - q = q.filter(year__lt=default_year).order_by('-year') if not q.count(): - strange.append((old_insee, row['NomCA'])) - continue - old_town = q.all()[0] - - q = Town.objects.filter(numero_insee=new_insee, - year=default_year) - if not q.count(): - nb_created += 1 - name = row['NomCN'].strip() - name = r.sub(r"\2 \1", name).strip() - new_town = Town.objects.create(name=name, year=default_year, - numero_insee=new_insee) - else: - new_town = q.all()[0] - if new_town in old_town.children.all(): - continue # link already created - nb_link += 1 - old_town.children.add(new_town) - linked.add(new_town) + nb_created += 1 + name = row['NomCN'].strip() + name = r.sub(r"\2 \1", name).strip() + new_town = Town.objects.create(name=name, year=default_year, + numero_insee=new_insee) + writer.writerow([new_town.numero_insee, new_town.name]) + else: + new_town = q.all()[0] + if new_town in old_town.children.all(): + continue # link already created + nb_link += 1 + old_town.children.add(new_town) + linked.add(new_town) nb_limit = 0 if not quiet: - sys.stdout.write('\nGenerate limits...'.format(nb_created)) + sys.stdout.write('\nGenerate limits...') for town in linked: if town.generate_geo(): nb_limit += 1 if quiet: return + sys.stdout.write(BColors.OKGREEN) sys.stdout.write('\n* {} town created\n'.format(nb_created)) sys.stdout.write('* {} link created\n'.format(nb_link)) sys.stdout.write('* {} limit generated\n'.format(nb_limit)) @@ -119,6 +136,6 @@ class Command(BaseCommand): sys.stdout.write('* these towns have newer version:\n') for insee, name in strange: sys.stdout.write('* {} ({})\n'.format(name, insee)) + sys.stdout.write(f"\n[{get_log_time()}] log file:") + sys.stdout.write(f"\n{BColors.WARNING}{log_path}{BColors.ENDC}\n") sys.stdout.flush() - - |