summaryrefslogtreecommitdiff
path: root/ishtar_common/management/commands/import_insee_comm_csv.py
diff options
context:
space:
mode:
authorÉtienne Loks <etienne.loks@iggdrasil.net>2025-05-12 14:59:21 +0200
committerÉtienne Loks <etienne.loks@iggdrasil.net>2025-05-13 10:40:31 +0200
commit92ea61d5bc4713b151eb3c7d513f9e3699ccb2eb (patch)
tree7b256e3a950fbf07911a60a3ca6e1c831624a377 /ishtar_common/management/commands/import_insee_comm_csv.py
parentf3f0498e7001abe9676b6506baea07e820cbf60b (diff)
downloadIshtar-92ea61d5bc4713b151eb3c7d513f9e3699ccb2eb.tar.bz2
Ishtar-92ea61d5bc4713b151eb3c7d513f9e3699ccb2eb.zip
✨ import_geofla_csv, import_insee_comm_csv: logging, better verbose CLI
Diffstat (limited to 'ishtar_common/management/commands/import_insee_comm_csv.py')
-rw-r--r--ishtar_common/management/commands/import_insee_comm_csv.py119
1 files changed, 68 insertions, 51 deletions
diff --git a/ishtar_common/management/commands/import_insee_comm_csv.py b/ishtar_common/management/commands/import_insee_comm_csv.py
index 2e990c7ce..4584da890 100644
--- a/ishtar_common/management/commands/import_insee_comm_csv.py
+++ b/ishtar_common/management/commands/import_insee_comm_csv.py
@@ -18,13 +18,17 @@
# See the file COPYING for details.
import csv
+import os
import re
import sys
+from django.conf import settings
from django.core.management.base import BaseCommand
from ishtar_common.models import Town
+from ishtar_common.utils import BColors, get_log_time
+
class Command(BaseCommand):
help = 'Import INSEE csv'
@@ -39,75 +43,88 @@ class Command(BaseCommand):
help='Quiet output')
def handle(self, *args, **options):
+ log_path = os.sep.join([settings.ROOT_PATH, "logs"])
+ if not os.path.exists(log_path):
+ os.mkdir(log_path, mode=0o770)
+
csv_file = options['csv_file']
default_year = options['year']
quiet = options['quiet']
if not quiet:
- sys.stdout.write('* using year {} for new towns\n'.format(
- default_year))
- sys.stdout.write('* opening file {}\n'.format(csv_file))
+ sys.stdout.write(BColors.OKGREEN)
+ sys.stdout.write(f'* Using year {default_year} for new towns\n')
+ sys.stdout.write(f'* Opening file {csv_file}{BColors.ENDC}\n')
r = re.compile(r"(.*)\((.*)\)")
nb_created = 0
nb_link = 0
missing = []
strange = []
linked = set()
- with open(csv_file, 'rt') as csvfile:
- reader = csv.DictReader(csvfile)
- for idx, row in enumerate(reader):
- new_insee = row['DepComN']
- if len(new_insee) < 5:
- new_insee = '0' + new_insee
-
- if not idx: # test if first do not exist
+
+ log_filename = f"import_insee-{get_log_time().replace(':', '')}.csv"
+ log_path = os.sep.join([log_path, log_filename])
+ with open(log_path, "w+") as fle:
+ writer = csv.writer(fle)
+ writer.writerow(["new insee", "new town"])
+ with open(csv_file, 'rt') as csvfile:
+ reader = csv.DictReader(csvfile)
+ for idx, row in enumerate(reader):
+ new_insee = row['DepComN']
+ if len(new_insee) < 5:
+ new_insee = '0' + new_insee
+
+ if not idx: # test if first do not exist
+ q = Town.objects.filter(numero_insee=new_insee,
+ year=default_year)
+ if q.count():
+ sys.stdout.write(
+ f"{BColors.FAIL}First town already exists for this year{BColors.ENDC}\n")
+ return
+
+ if not quiet:
+ sys.stdout.write('Processing town %d.\r' % (idx + 1))
+ sys.stdout.flush()
+
+ old_insee = row['DepComA']
+ if len(old_insee) < 5:
+ old_insee = '0' + old_insee
+ q = Town.objects.filter(numero_insee=old_insee)
+
+ if not q.count():
+ missing.append((old_insee, row['NomCA']))
+ continue
+ if q.count() > 1:
+ q = q.filter(year__lt=default_year).order_by('-year')
+ if not q.count():
+ strange.append((old_insee, row['NomCA']))
+ continue
+ old_town = q.all()[0]
+
q = Town.objects.filter(numero_insee=new_insee,
year=default_year)
- if q.count():
- print("First town already exists for this year....")
- return
-
- if not quiet:
- sys.stdout.write('Processing town %d.\r' % (idx + 1))
- sys.stdout.flush()
-
- old_insee = row['DepComA']
- if len(old_insee) < 5:
- old_insee = '0' + old_insee
- q = Town.objects.filter(numero_insee=old_insee)
-
- if not q.count():
- missing.append((old_insee, row['NomCA']))
- continue
- if q.count() > 1:
- q = q.filter(year__lt=default_year).order_by('-year')
if not q.count():
- strange.append((old_insee, row['NomCA']))
- continue
- old_town = q.all()[0]
-
- q = Town.objects.filter(numero_insee=new_insee,
- year=default_year)
- if not q.count():
- nb_created += 1
- name = row['NomCN'].strip()
- name = r.sub(r"\2 \1", name).strip()
- new_town = Town.objects.create(name=name, year=default_year,
- numero_insee=new_insee)
- else:
- new_town = q.all()[0]
- if new_town in old_town.children.all():
- continue # link already created
- nb_link += 1
- old_town.children.add(new_town)
- linked.add(new_town)
+ nb_created += 1
+ name = row['NomCN'].strip()
+ name = r.sub(r"\2 \1", name).strip()
+ new_town = Town.objects.create(name=name, year=default_year,
+ numero_insee=new_insee)
+ writer.writerow([new_town.numero_insee, new_town.name])
+ else:
+ new_town = q.all()[0]
+ if new_town in old_town.children.all():
+ continue # link already created
+ nb_link += 1
+ old_town.children.add(new_town)
+ linked.add(new_town)
nb_limit = 0
if not quiet:
- sys.stdout.write('\nGenerate limits...'.format(nb_created))
+ sys.stdout.write('\nGenerate limits...')
for town in linked:
if town.generate_geo():
nb_limit += 1
if quiet:
return
+ sys.stdout.write(BColors.OKGREEN)
sys.stdout.write('\n* {} town created\n'.format(nb_created))
sys.stdout.write('* {} link created\n'.format(nb_link))
sys.stdout.write('* {} limit generated\n'.format(nb_limit))
@@ -119,6 +136,6 @@ class Command(BaseCommand):
sys.stdout.write('* these towns have newer version:\n')
for insee, name in strange:
sys.stdout.write('* {} ({})\n'.format(name, insee))
+ sys.stdout.write(f"\n[{get_log_time()}] log file:")
+ sys.stdout.write(f"\n{BColors.WARNING}{log_path}{BColors.ENDC}\n")
sys.stdout.flush()
-
-