Imports: improve JSON importer for complex JSON sources

1 files changed, 93 insertions, 25 deletions

diff --git a/chimere/utils.py b/chimere/utils.py
index d6a6152..98fb7c7 100644
--- a/chimere/utils.py
+++ b/chimere/utils.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012-2016  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+# Copyright (C) 2012-2017  Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as
@@ -21,6 +21,7 @@
 Utilitaries
 """
 
+from copy import deepcopy
 import csv
 import collections
 import datetime
@@ -749,6 +750,32 @@ class JsonManager(ImportManager):
     This manager only gets and do not produce Json feed
     """
 
+    def extract_dict_values(self, item, filtr):
+        """
+        Extract values from a dict.
+
+        :param item: the source dictionary
+        :param filtr: the filter, a dictionary that contains keys or dictionary,
+        each dictionary is parsed for each values
+        :return: an iterator giving tuple of final keys and values.
+
+        example:
+        item = {'comment': {'fr': "Commentaire", 'en': "Comment"},
+                'latitude': 1.0, 'longitude': -1.0}
+        filtr = {'comment': {'fr': "description"}, 'latitude': 'y',
+                 'longitude': 'x'}
+        print(list(extract_dict_values(item, filtr)))
+        [("description", "Commentaire"), ("y", 1.0), ("x", -1.0)]
+        """
+        for k in filtr:
+            if k not in item:
+                continue
+            if not isinstance(filtr[k], dict):
+                yield filtr[k], item[k]
+                continue
+            for key, value in self.extract_dict_values(item[k], filtr[k]):
+                yield key, value
+
     def get(self):
         """
         Get data from a json simple source
@@ -771,16 +798,43 @@ class JsonManager(ImportManager):
         except ValueError as e:
             return (new_item, updated_item,
                     _("JSON file is not well formed: ") + str(e))
+
+        filtr = self.importer_instance.filtr
+        # a left part before "{" indicate keys to be used to access to the
+        # event list - separated by ";"
+        left_part = filtr.split('{')[0]
+        if left_part:
+            filtr = filtr[len(left_part):]
+            for key in left_part.split(';'):
+                if key not in values:
+                    return (
+                        new_item, updated_item,
+                        _("Bad filter configuration a key doesn't "
+                          "match with json source: ") + key)
+                values = values[key]
+
         # configuration in filtr
         try:
-            filtr = json.JSONDecoder().decode(self.importer_instance.filtr)
+            filtr = json.JSONDecoder().decode(filtr)
         except ValueError:
             return (
                 new_item, updated_item,
                 _("Bad configuration: filter field must be a valid "
                   "JSON string"))
 
-        vls = filtr.values()
+        # check that mandatory fields are available
+        vls = []
+        cvalues = filtr.copy()
+        while cvalues:
+            new_values = {}
+            for idx, val in enumerate(cvalues.values()):
+                if isinstance(val, dict):
+                    for k in val:
+                        new_values["{}-{}".format(idx, k)] = val[k]
+                else:
+                    vls.append(val)
+            cvalues = new_values
+
         for k in ('name', 'id', 'description'):
             if k not in vls:
                 return (
@@ -789,7 +843,8 @@ class JsonManager(ImportManager):
                       "filter.") % k)
 
         default_dct = {'origin': self.importer_instance.origin,
-                       'license': self.importer_instance.license}
+                       'license': self.importer_instance.license,
+                       'description': ""}
         if 'prefix_name' in filtr:
             default_dct['name'] = filtr.pop('prefix_name')
         if 'prefix_description' in filtr:
@@ -799,31 +854,38 @@ class JsonManager(ImportManager):
 
         for item in values:
             dct = default_dct.copy()
-            for k in filtr:
-                if k.startswith('prefix_') or k.startswith('suffix_'):
+
+            for key, value in self.extract_dict_values(item, filtr):
+                """
+                for k in filtr:
+                """
+                if key.startswith('prefix_') or key.startswith('suffix_'):
                     continue
-                if k in item and item[k]:
-                    if filtr[k] not in dct:
-                        dct[filtr[k]] = ""
+                if key == 'external_image':
+                    value = '<img src="{}">'.format(value)
+                if key not in dct:
+                    dct[key] = ""
+                else:
+                    if key == 'description':
+                        dct[key] += "<br/>"
                     else:
-                        if filtr[k] == 'description':
-                            dct[filtr[k]] += "<br/>"
-                        else:
-                            dct[filtr[k]] += " "
-                    dct[filtr[k]] += item[k]
-            if 'point' in item:
-                x, y = item['point'].split(",")
+                        dct[key] += " "
+                dct[key] += str(value) if value else ""
+
+            if 'point' in dct and isinstance(dct['point'], str):
+                x, y = dct['point'].split(",")
                 dct['point'] = 'SRID=4326;POINT(%s %s)' % (x, y)
-            elif 'lat' in item and item['lat'] \
-                    and 'lon' in item and item['lon']:
-                dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['lon'],
-                                                           item['lat'])
-            elif 'x' in item and item['x'] \
-                    and 'y' in item and item['y']:
-                dct['point'] = 'SRID=4326;POINT(%s %s)' % (item['x'],
-                                                           item['y'])
+            elif 'lat' in dct and dct['lat'] \
+                    and 'lon' in dct and dct['lon']:
+                dct['point'] = 'SRID=4326;POINT(%s %s)' % (dct.pop('lon'),
+                                                           dct.pop('lat'))
+            elif 'x' in dct and dct['x'] \
+                    and 'y' in dct and dct['y']:
+                dct['point'] = 'SRID=4326;POINT(%s %s)' % (dct['x'],
+                                                           dct['y'])
             if not dct['point']:
                 continue
+            # manage prefixes and suffixes
             for k in filtr:
                 if k.startswith('prefix_') or k.startswith('suffix_'):
                     pos = k.split('_')[0]
@@ -833,15 +895,21 @@ class JsonManager(ImportManager):
                             dct[key] = filtr[k] + dct[key]
                         else:
                             dct[key] += filtr[k]
+
+            if 'external_image' in dct:
+                dct['description'] = \
+                    dct.pop('external_image') + dct['description']
+
             cls = Marker
             pl_id = (dct.pop('id') if 'id' in dct else dct['name']) \
                 + "-" + str(self.importer_instance.pk)
+
             it, updated, created = self.create_or_update_item(cls, dct, pl_id)
             if updated:
                 updated_item += 1
             if created:
                 new_item += 1
-        return (new_item, updated_item, msg)
+        return new_item, updated_item, msg
 
 RE_HOOK = re.compile('\[([^\]]*)\]')