Operations import: import periods

1 files changed, 102 insertions, 30 deletions

diff --git a/archaeological_operations/import_from_dbf.py b/archaeological_operations/import_from_dbf.py
index 36871baac..d0f55cdbe 100644
--- a/archaeological_operations/import_from_dbf.py
+++ b/archaeological_operations/import_from_dbf.py
@@ -45,6 +45,61 @@ def parse_ha(value):
         val = 0
     return val * 10000
 
+period_types = {
+    '':'not_yet_documented',
+    'IND':'indetermined',
+    'CON': 'contemporan',
+    'MOD': 'modern',
+    'REC': 'recent_times',
+    'BMA': 'low_middle_age',
+    'MAC': 'classic_middle_age',
+    'HMA': 'high_middle_age',
+    'MA' : 'middle_age',
+    'MED': 'middle_age',
+    'BAS': 'low_empire',
+    'HAU': 'high-empire',
+    'NRE': 'republic',
+    'GAL': 'gallo-roman',
+    'FE2': 'second_iron_age',
+    'FE1': 'first_iron_age',
+    'BRF': 'final_bronze_age',
+    'BRM': 'middle_bronze_age',
+    'BRA': 'old_bronze_age',
+    'FER': 'iron_age',
+    'BRO': 'bronze_age',
+    'PRO': 'protohistory',
+    'NEF': 'final_neolithic',
+    'NER': 'recent_neolithic',
+    'NEM': 'middle_neolithic',
+    'NEA': 'old_neolithic',
+    'NEO': 'neolithic',
+    'MER': 'recent_mesolithic',
+    'MEM': 'middle_mesolithic',
+    'MEA': 'old_mesolithic',
+    'MES': 'mesolithic',
+    'PAF': 'final_paleolithic',
+    'PAS': 'late_paleolithic',
+    'PAM': 'middle_paleolithic',
+    'PAA': 'ancien_paleolithic',
+    'PAL': 'paleolithic'
+}
+
+_CACHED_PERIOD_TYPES = {}
+
+def _prepare_period_types():
+    for k in period_types.keys():
+        _CACHED_PERIOD_TYPES[k] = Period.objects.get(txt_idx=period_types[k])
+
+_period_re_filter = re.compile('^EUR')
+_period2_re_filter = re.compile('-*$')
+
+def parse_period(value):
+    value = _period_re_filter.sub('', value)
+    value = _period2_re_filter.sub('', value)
+    if value not in _CACHED_PERIOD_TYPES.keys():
+        value = ''
+    return _CACHED_PERIOD_TYPES[value]
+
 ope_types = {
     'AET':('other_study',
            'Autre étude', True),
@@ -99,7 +154,6 @@ def _prepare_ope_types():
 
 def parse_patriarche_operationtype(value):
     if value not in _CACHED_OPE_TYPES.keys():
-        print value
         return None
     return _CACHED_OPE_TYPES[value]
 
@@ -114,32 +168,38 @@ def parse_ope_name(value):
     value = _dpt_re_filter.sub('', value)
     return value
 
+class Column:
+    def __init__(self, col_models, format, associated_cols=None, multi=False):
+        self.col_models, self.format = col_models, format
+        self.associated_cols, self.multi = associated_cols, multi
+
+
 PATRIARCHE_DBF_OPE_COLS = [
- (('operation_type',), 'parse_patriarche_operationtype', []),
- (('common_name',), 'parse_ope_name', []),
- [],
- (('in_charge',), 'parse_person', [2]),
- [],  #'etat',
- (('comment',), unicode, []),  #'adresse',
- [],  #'origine C(3)',
- [],  # 'chronologi C(12)',
- [],  #'programme C(254)',
- [],  # 'rattach_pc C(254)',
- [],  # 'code_dossi N(8,0)',
- (('administrative_act', 'ref_sra'), unicode, []),
- (('administrative_act', 'signature_date'), parse_date, []),
- (('start_date',), parse_date, []),
- (('end_date',), parse_date, []),
- (('year',), int, []),
- [],  # 'identifica C(254)',
- (('code_patriarche',), int, []),
- [],  # 'x_degre N(16,6)',
- [],  # 'y_degre N(16,6)',
- [],  # 'x_saisi C(12)',
- [],  # 'y_saisi C(12)',
- [],  # 'georeferen C(3)',
- [],  # 'geometrie C(3)',
- (('surface',), parse_ha, [])
+    Column(('operation_type',), 'parse_patriarche_operationtype'),
+    Column(('common_name',), 'parse_ope_name'),
+    [],
+    Column(('in_charge',), 'parse_person', [2]),
+    [],  # 'etat'
+    Column(('comment',), unicode),  #'adresse'
+    [],  #'origine C(3)'
+    Column(('periods',), 'parse_period', multi=True),
+    [],  #'programme C(254)'
+    [],  # 'rattach_pc C(254)'
+    [],  # 'code_dossi N(8,0)'
+    Column(('administrative_act', 'ref_sra'), unicode),
+    Column(('administrative_act', 'signature_date'), parse_date),
+    Column(('start_date',), parse_date),
+    Column(('end_date',), parse_date),
+    Column(('year',), int, []),
+    [],  # 'identifica C(254)'
+    Column(('code_patriarche',), int),
+    [],  # 'x_degre N(16,6)'),
+    [],  # 'y_degre N(16,6)'),
+    [],  # 'x_saisi C(12)'),
+    [],  # 'y_saisi C(12)'),
+    [],  # 'georeferen C(3)'),
+    [],  # 'geometrie C(3)'),
+    Column(('surface',), parse_ha)
 ]
 
 DBF_OPE_COLS = PATRIARCHE_DBF_OPE_COLS
@@ -172,8 +232,8 @@ def import_operations_dbf(values, col_defs=DBF_OPE_COLS, update=False,
                            'act_type':ActType.objects.get(
                                       txt_idx='excavation_order')}, 'operation'),
     }
-    # prepare ope types
     _prepare_ope_types()
+    _prepare_period_types()
 
     ope_default = {'history_modifier':default_person}
     current_import = []
@@ -182,6 +242,8 @@ def import_operations_dbf(values, col_defs=DBF_OPE_COLS, update=False,
     for error in ERROR_LBLS.keys():
         errors_nb[error] = 0
     error_ope, error_reversed, error_multis = [], [], []
+    multi_keys = set([column.col_models[0]
+                      for column in col_defs if column and column.multi])
     for line_idx, vals in enumerate(values):
         if stdout:
             stdout.write("\r* line %d" % (line_idx))
@@ -191,7 +253,9 @@ def import_operations_dbf(values, col_defs=DBF_OPE_COLS, update=False,
         for col_idx, val in enumerate(vals):
             if len(col_defs) <= col_idx or not col_defs[col_idx]:
                 continue
-            attrs, typ, extra_cols = col_defs[col_idx]
+            col_def = col_defs[col_idx]
+            attrs, typ = col_def.col_models, col_def.format
+            extra_cols = col_def.associated_cols
             if not callable(typ):
                 typ = globals()[typ]
             c_args = args
@@ -245,7 +309,7 @@ def import_operations_dbf(values, col_defs=DBF_OPE_COLS, update=False,
                     obj.save()
                 transaction.commit()
                 args[k] = obj
-            elif type(args[k]) == list:
+            elif type(args[k]) == list or k in multi_keys:
                 multis.append((k, args[k]))
                 args.pop(k)
         op = None
@@ -262,6 +326,7 @@ def import_operations_dbf(values, col_defs=DBF_OPE_COLS, update=False,
         # check
         if not args.get('year') and args.get('start_date'):
             args['year'] = args['start_date'].year
+        updated = False
         # creation
         if not op:
             args.update(ope_default)
@@ -271,7 +336,6 @@ def import_operations_dbf(values, col_defs=DBF_OPE_COLS, update=False,
             op = Operation.objects.create(**args)
             new_ops += 1
             transaction.commit()
-            current_import.append(op.pk)
         else: # mise à jour
             try:
                 for k in args:
@@ -285,25 +349,33 @@ def import_operations_dbf(values, col_defs=DBF_OPE_COLS, update=False,
                 transaction.rollback()
                 continue
             transaction.commit()
+            updated = True
         try:
             for cls, default, reverse in reversed_items:
                 default[reverse] = op
                 it = cls(**default).save()
         except:
             transaction.rollback()
+            current_import.append(op.pk)
             error_reversed.append((line_idx, reversed_items))
             continue
         transaction.commit()
         try:
             for k, vals in multis:
+                if op.pk not in current_import and updated:
+                    getattr(op, k).clear()
+                if type(vals) not in (list, tuple):
+                    vals = [vals]
                 for v in vals:
                     getattr(op, k).add(v)
                     op.save()
         except:
             transaction.rollback()
+            current_import.append(op.pk)
             error_multis.append((line_idx, multis))
             continue
         transaction.commit()
+        current_import.append(op.pk)
 
     errors = []
     for error_key in errors_nb: