Operations: work on import

1 files changed, 59 insertions, 25 deletions

diff --git a/archaeological_operations/import_from_csv.py b/archaeological_operations/import_from_csv.py
index 9d7a57734..d58c0439e 100644
--- a/archaeological_operations/import_from_csv.py
+++ b/archaeological_operations/import_from_csv.py
@@ -66,12 +66,18 @@ for k in settings.ISHTAR_OPE_TYPES.keys():
                                        'preventive':k[0]==u'préventive'})
     ope_types[k] = ot
 
-def parse_string(value):
-    value = value.strip()
-    if value == '#EMPTY':
-        value = ''
-    value = value.replace('  ', ' ')
-    return value
+def _get_parse_string(trunc_number=None):
+    def parse_string(value):
+        value = value.strip()
+        if value == '#EMPTY':
+            value = ''
+        value = value.replace('  ', ' ')
+        if trunc_number:
+            value = value[:trunc_number]
+        return value
+    return parse_string
+
+parse_string = _get_parse_string()
 
 def parse_multivalue(value):
     s1 = re.sub('(.)([A-Z][a-z]+)', r'\1 \2', name)
@@ -280,8 +286,8 @@ def parse_name_surname(value):
     if len(items) > 1:
         name = " ".join(items[:-1])
         surname = items[-1]
-    values = {"surname":parse_title(surname),
-              "name":parse_title(name)}
+    values = {"surname":parse_title(surname)[:30],
+              "name":parse_title(name)[:30]}
     if not values['surname'] and not values['name']:
         return
     q = Person.objects.filter(**values)
@@ -431,6 +437,8 @@ def parse_parcels(parcel_str, insee_code, owner):
     for parcel in PARCEL_SECTION_REGEXP.findall(parcel_str):
         sector, nums = parcel[0], parcel[1]
         for num in PARCEL_NB_REGEXP.findall(nums):
+            if len(unicode(num)) > 6:
+                continue
             parcels.append({'year':year, 'town':town, 'section':sector,
                             'parcel_number':num, 'history_modifier':owner})
         for parcel_ranges in PARCEL_NB_RANGE_REGEXP.findall(nums):
@@ -519,21 +527,34 @@ class BreakIt(Exception):
 
 class RelatedClass:
     def __init__(self, key, cls, default_data={}, reverse_key=False,
-                 unique_keys=[], extra_data=[], multi=None):
+                 unique_keys=[], extra_data=[], mandatory_fields=[],
+                 multi=None):
         self.key, self.cls, self.default_data = key, cls, default_data
         self.reverse_key, self.unique_keys = reverse_key, unique_keys
         self.extra_data, self.multi = extra_data, multi
+        self.mandatory_fields = mandatory_fields
 
     def create_object(self, data):
+        for mandatory_field in self.mandatory_fields:
+            if not data.get(mandatory_field):
+                return None
         if self.unique_keys:
             unique_data = {}
             for k in self.unique_keys:
                 unique_data[k] = data.pop(k)
+            if self.cls.objects.filter(**unique_data).count() > 1:
+                return None
             unique_data['defaults'] = data
-            obj, created = self.cls.objects.get_or_create(**unique_data)
+            try:
+                obj, created = self.cls.objects.get_or_create(**unique_data)
+            except ValueError:
+                return None
             if not created:
                 for k in unique_data['defaults']:
-                    setattr(obj, k, unique_data['defaults'][k])
+                    try:
+                        setattr(obj, k, unique_data['defaults'][k])
+                    except ValueError:
+                        continue
                 obj.save()
         else:
             obj = self.cls.objects.create(**data)
@@ -602,18 +623,25 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None,
                       for column in col_defs if column and column.multi])
 
     restrict_lines = []
+    start_line, end_line = None, None
     if lines:
         if '-' not in lines:
             restrict_lines = [int(line) for line in lines.split(',')]
         else:
             start_line, end_line = lines.split('-')
-            restrict_lines = list(xrange(int(start_line), int(end_line)+1))
+            start_line, end_line = int(start_line), int(end_line)+1
+    if start_line:
+        values = list(values)[start_line:]
+    if end_line:
+        values = list(values)[:end_line+1]
     for line_idx, vals in enumerate(values):
         if restrict_lines:
             if line_idx > max(restrict_lines):
                 break
             if line_idx not in restrict_lines:
                 continue
+        if start_line:
+            line_idx = line_idx + start_line
         if stdout:
             stdout.write("\r* line %d" % (line_idx))
         if not line_idx:
@@ -626,7 +654,10 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None,
             attrs, typ = col_def.col_models, col_def.format
             extra_cols = col_def.associated_cols
             if not callable(typ):
-                typ = globals()[typ]
+                if typ.startswith('parse_string_'):
+                    typ = _get_parse_string(int(typ.split('_')[-1]))
+                else:
+                    typ = globals()[typ]
             c_args = args
             for attr in attrs:
                 if attr not in c_args:
@@ -694,21 +725,23 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None,
                     mod, value = k.split('__')
                     attached_models[(mod, value)] = args.pop(k)
             except BreakIt:
+                args.pop(k)
                 continue
         op = None
-        if not update and not args.get('operation_type'):
+        if not args.get('operation_type'):
             #print "Pas d'operation_type"
             continue
         #transaction.commit()
-        q = Operation.objects.filter(code_patriarche=args['code_patriarche'])
         try:
-            if q.count():
-                if not update:
-                    #print "Code patriarche existant"
-                    continue
-                op = q.all()[0]
-        except:
+            int(args['code_patriarche'])
+        except ValueError:
             continue
+        q = Operation.objects.filter(code_patriarche=args['code_patriarche'])
+        if q.count():
+            if not update:
+                #print "Code patriarche existant"
+                continue
+            op = q.all()[0]
         # check
         if not args.get('year') and args.get('start_date'):
             args['year'] = args['start_date'].year
@@ -761,10 +794,11 @@ def import_operations_csv(values, col_defs=OPE_COLS, update=True, person=None,
                 field.save()
         #transaction.commit()
         for rel_cls, data, attr in related_items:
-            try:
-                rel_cls.create(op, data, attr)
-            except:
-                error_reversed.append((line_idx, data))
+            rel_cls.create(op, data, attr)
+            #try:
+            #    rel_cls.create(op, data, attr)
+            #except:
+            #    error_reversed.append((line_idx, data))
         ope_postimportfix(op, args)
         #transaction.commit()