✨ import_geofla_csv, import_insee_comm_csv: logging, better verbose CLI

1 files changed, 28 insertions, 0 deletions

diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index 23495c2ec..04cda150b 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -3158,6 +3158,34 @@ def get_eta(current, total, base_time, current_time):
     return f"{int(eta // 3600):02d}:{int(eta % 3600 // 60):02d}:{int(eta % 60):02d}"
 
 
+def get_progress(base_lbl, idx, total, ref_time):
+    """
+    Output progress for a long task.
+    - base_lbl: label to display
+    - idx: current item number
+    - total: number of items
+    - ref_time: time the task has been started
+    """
+    lbl = f"\r{BColors.OKBLUE}[{get_percent(idx, total)}] {base_lbl} {idx + 1}/{total}"
+    lbl += f" ({get_eta(idx, total, ref_time, datetime.datetime.now())} left){BColors.ENDC}"
+    return lbl
+
+
+def fast_line_count(filename):
+    """
+    Efficient line counter for a file
+    """
+    CHUNK_SIZE = 1024 * 1024
+    def _count(reader):
+        b = reader(CHUNK_SIZE)
+        while b:
+            yield b
+            b = reader(CHUNK_SIZE)
+    with open(filename, 'rb') as fp:
+        count = sum(buffer.count(b"\n") for buffer in _count(fp.raw.read))
+    return count + 1
+
+
 RE_NUMBER = r"[+-]?\d+(?:\.\d*)?"
 RE_COORDS = r"(" + RE_NUMBER + r") (" + RE_NUMBER + r")"