diff options
Diffstat (limited to 'ishtar_common/qfield_functions.py')
| -rw-r--r-- | ishtar_common/qfield_functions.py | 1085 |
1 files changed, 1085 insertions, 0 deletions
diff --git a/ishtar_common/qfield_functions.py b/ishtar_common/qfield_functions.py new file mode 100644 index 000000000..36ab5c541 --- /dev/null +++ b/ishtar_common/qfield_functions.py @@ -0,0 +1,1085 @@ +import os +import logging +from osgeo import ogr, osr +from itertools import permutations +from django.apps import apps +from django.conf import settings + +logger = logging.getLogger(__name__) + +ENCODING = settings.ENCODING or "utf-8" + +HIERARCHIC_LEVELS = 5 + +LIST_FIELDS = { # key: hierarchic depth + "conservatory_states": HIERARCHIC_LEVELS, + "identifications": HIERARCHIC_LEVELS, + "material_types": HIERARCHIC_LEVELS, + "material_type": HIERARCHIC_LEVELS, + "object_types": HIERARCHIC_LEVELS, + "period": HIERARCHIC_LEVELS, + "periods": HIERARCHIC_LEVELS, + "source_type": HIERARCHIC_LEVELS, + "unit": HIERARCHIC_LEVELS, + "museum_collection_entry_mode": HIERARCHIC_LEVELS, + "shooting_angle": HIERARCHIC_LEVELS, + "technical_processes": HIERARCHIC_LEVELS, + "structures": HIERARCHIC_LEVELS, + "textures": HIERARCHIC_LEVELS, + "inclusions": HIERARCHIC_LEVELS, + "colors": HIERARCHIC_LEVELS, + "development_type": HIERARCHIC_LEVELS, + "monitoring_justification": HIERARCHIC_LEVELS, + "documentations": HIERARCHIC_LEVELS, + "excavation_technics": HIERARCHIC_LEVELS, + "treatment_types": HIERARCHIC_LEVELS, + "discovery_method": 0, + "discovery_status": 0, + "current_status": 0, + "nature_of_site": 0, + "interpretation_level": 0, + "museum_inventory_marking_presence": 0, + "museum_marking_type": 0, + "museum_collection": 0, + "batch": 0, + "preservation_to_considers": 0, + "integrities": 0, + "remarkabilities": 0, + "checked_type": 0, + "material_type_quality": 0, + "object_type_quality": 0, + "communicabilities": 0, + "alterations": 0, + "alteration_causes": 0, + "treatment_emergency": 0, + "cultural_attributions": 0, + "remains": 0, + "dating_type": 0, + "quality": 0, + "operation_type": 0, + "report_processing": 0, + "record_quality_type": 0, + "data_type": 0, + "origin": 0, + "provider": 0, + "activity": 0, + "person_types": 0, + "relation_types": 0, + "types": HIERARCHIC_LEVELS, # keep it at the end to not mess with other types +} + +HIERARCHIC_FIELDS = list(LIST_FIELDS.keys()) + +def gpkg_creation(model, root, table_cols, col_names, datas): + """ + :param model: Table from the query + :param root: Path to the folder to create the geopackage + :param table_cols: List of the columns used in the query + :param col_names: Name of the columns in the new geopackage + :param datas: Data from the query + :function: Creation of the Finds and Context_Records geopackages when the query come from one of these two tables + :return finds: Geopackage for the Finds + :return cr: Geopackage for the Context_Records + :return list_ope: List of the different operations linked to the Finds and Context_Records + :return list_cr: List of the labels/names of the Context_Records used + """ + # Preparation of important values and parameters for the geopackages + finds = '' + cr = '' + list_ope = [] + list_crea = [] + driver = ogr.GetDriverByName('GPKG') + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + # I. Case where the extraction come from Finds + if str(model._meta) == 'archaeological_finds.find': + # 1) Creation of the Finds geopackage + finds = os.path.join(root, 'export', 'Finds.gpkg') + # Verification to delete it if already existing + if os.path.exists(finds): + os.remove(finds) + # 2) Creation of the finds layer and its attributes + datasource = driver.CreateDataSource(finds) + layer = datasource.CreateLayer('Finds', srs, ogr.wkbPoint25D) + layer = attributes_creation_finds_query(layer, col_names, table_cols) + # 4a) Populating the finds layer with the datas + list_cr = populating_layer_finds_query(layer,table_cols,col_names,datas) + datasource = None + # 3) Creation of the Context Records file + cr = os.path.join(root, 'export', 'Context_records.gpkg') + # Verification to delete it if already existing + if os.path.exists(cr): + os.remove(cr) + datasource = driver.CreateDataSource(cr) + # 4) Creation of the Context_Records layer and a list of default attributes + layer = datasource.CreateLayer('Context_records', srs, ogr.wkbMultiPolygon) + list_crea = ['Unité_Enregistrement', 'Opération', 'INSEE_Commune', 'Type', 'Interprétation', + 'Description', 'Localisation', 'Document_associe', 'Media', 'Periode', 'Type_Activité', + 'Type_Identification', 'Commentaires', 'WKT', 'Infos_Parcelles_UE'] + layer = attributes_creation_cr_default(layer, list_crea) + # 5) Populating the Context_Records layer with datas from the Context_Records of the extracted finds + list_ope = populating_layer_cr_default(layer, list_crea, list_cr) + datasource = None + # 6) Preparation of a list of the attributes names for the style modifications + list_crea = ['cr', list_crea] + # II. Case where the extraction come from Context_Recods + elif str(model._meta) == 'archaeological_context_records.contextrecord': + # 1) Creation of the Context Records geopackage + cr = os.path.join(root, 'export', 'Context_records.gpkg') + # Verification to delete it if already existing + if os.path.exists(cr): + os.remove(cr) + datasource = driver.CreateDataSource(cr) + # 2) Creation of the Context_Records layer and its attributes + layer = datasource.CreateLayer('Context_records', srs, ogr.wkbMultiPolygon) + layer = attributes_creation_cr_query(layer, col_names, table_cols) + # 3) Populating the Finds layer with the datas + list_ope, list_cr = populating_layer_cr_query(layer, table_cols, col_names, datas) + datasource = None + # 4) Creation of the Finds geopackage + finds = os.path.join(root, 'export', 'Finds.gpkg') + # Verification to delete it if already existing + if os.path.exists(finds): + os.remove(finds) + datasource = driver.CreateDataSource(finds) + layer = datasource.CreateLayer('Finds', srs, ogr.wkbPoint25D) + list_crea = ['Identifiant', 'UE', 'Date', 'X', 'Y', 'Z', 'Matériaux', 'Type_objets', 'Description', 'Media', + 'Commentaires', 'WKT_point', 'Infos_Parcelle_Mobilier'] + attributes_creation_finds_default(layer, list_crea) + # 5) Populating the finds layer with the datas + populating_layer_finds_default(layer, list_crea, list_cr) + # 6) Preparation of a list of the attributes names for the style modifications + list_crea = ['finds', list_crea] + return finds, cr, list_ope, list_crea + + +def gpkg_creation_sites(model, root, table_cols, col_names, datas): + """ + :param model: Table from the query + :param root: Path to the folder to create the geopackage + :param table_cols: List of the columns used in the query + :param col_names: Name of the columns in the new geopackage + :param datas: Data from the query + :function: Specific version for the creation of the needed geopackages when the query come from the + Archaeological_Sites table + :return sites: Geopackage for the Sites + :return finds: Geopackage for the Finds + :return cr: Geopackage for the Context_Records + :return list_ope: List of the different operations linked to the Finds and Context_Records + :return list_cr: List of the labels/names of the Context_Records used + """ + # Preparation of important values and parameters for the geopackages + finds = '' + cr = '' + list_ope = [] + list_crea = [] + driver = ogr.GetDriverByName('GPKG') + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + # 1) Creation of the sites layer + sites = os.path.join(root, 'export', 'Sites.gpkg') + if os.path.exists(sites): + os.remove(sites) + datasource = driver.CreateDataSource(sites) + layer = datasource.CreateLayer('Sites', srs, ogr.wkbPoint) + # Specific case if the query come from the Operations, and all element linked to them must be searched + if str(model._meta) == 'archaeological_operations.operation': + # Creation of the attributes + list_s = ['Référence', 'Nom', 'Operation', 'Commune', 'Période', 'Type', 'X', 'Y', 'Commentaires', 'WKT', 'Infos_Parcelles_Sites'] + layer = attributes_creation_sites_default(layer, list_s) + #Populating the finds layer with the datas + list_ope, list_cr = populating_layer_sites_default(layer, list_s, table_cols, datas) + else: + # Creation of the attributes + layer = attributes_creation_sites_query(layer, col_names, table_cols) + # Creation of the entities + list_ope, list_cr = populating_layer_sites_query(layer, table_cols, col_names, datas) + # 2) Creation of the Context_Records layer + cr = os.path.join(root, 'export', 'Context_records.gpkg') + # Verification to delete it if already existing + if os.path.exists(cr): + os.remove(cr) + datasource = driver.CreateDataSource(cr) + layer = datasource.CreateLayer('Context_records', srs, ogr.wkbMultiPolygon) + # Creation of the attributes + list_a = ['Unité_Enregistrement', 'Opération', 'INSEE_Commune', 'Type', 'Interprétation', 'Description', + 'Localisation', 'Document_associe', 'Media', 'Periode', 'Type_Activité', 'Type_Identification', + 'Commentaires', 'WKT', 'Infos_Parcelles_UE'] + layer = attributes_creation_cr_default(layer, list_a) + # # Creation of the entities + populating_layer_cr_default(layer, list_a, list_cr) + datasource = None + # 3) Creation of the finds layer + finds = os.path.join(root, 'export', 'Finds.gpkg') + # Verification to delete it if already existing + if os.path.exists(finds): + os.remove(finds) + datasource = driver.CreateDataSource(finds) + layer = datasource.CreateLayer('Finds', srs, ogr.wkbPoint25D) + # Creation of the attributes + list_b = ['Identifiant', 'UE', 'Date', 'X', 'Y', 'Z', 'Matériaux', 'Type_objets', 'Description', 'Media', + 'Commentaires', 'WKT_point', 'Infos_Parcelles_Mobilier'] + layer = attributes_creation_finds_default(layer, list_b) + # Creation of the entities + populating_layer_finds_default(layer, list_b, list_cr) + # Recuperation of all created attributes + if str(model._meta) == 'archaeological_operations.operation': + list_crea = list_s + list_a + list_b + else: + list_crea = list_a + list_b + return sites, finds, cr, list_ope, list_crea + + +def attributes_creation_finds_query(layer, col_names, table_cols): + """ + :param layer: Finds layer from the linked geopackage + :param col_names: Name of the columns in the new layer + :param table_cols: List of the columns used in the query + :function: Creation of the attributes of the Finds layer with the information from the exporter + :return layer: Finds layer with attributes + """ + # print(table_cols) # debugtest + # print(col_names) # debugtest + # print(datas) # debugtest + # Looping on all the attributes + for idx in range(0, len(col_names)): + # Prevent missing values (case in some .gpkg) + if table_cols[idx] != '': + # print(table_cols[idx]) # debugtest + # print(col_names[idx]) # debugtest + # Gestion of specific formats of attributes + if any(elem in table_cols[idx] for elem in ['index', 'order', 'quantity', 'taq', 'tpq', 'year']): + layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTInteger64)) + elif any(elem in table_cols[idx] for elem in + ['_x', '_y', '_z', 'circumference', 'cost', 'depth', 'diameter', 'height', 'length', 'number', + 'surface', 'side', 'thickness', 'value', 'volume', 'weight', 'width']): + layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTReal)) + elif '_date' in table_cols[idx]: + layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTDate)) + elif '_datetime' in table_cols[idx]: + layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTDateTime)) + elif any(elem in table_cols[idx] for elem in ['large_area_prescription', 'is_complete', 'executed']): + layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTBinary)) + else: + layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTString)) + return layer + + +def populating_layer_finds_query(layer, table_cols, col_names, datas): + """ + :param layer: Finds layer from the linked geopackage with attributes + :param table_cols: List of the columns used in the query + :param col_names: Name of the columns in the new layer + :param datas: Data from the query + :function: Population of the Finds layer using all the data from the query + :return list_cr: List of all the Context_Records linked to the Finds from the query + """ + max = len(col_names) + list_cr = [] + # Looping on all the datas extracted to create features + for data in datas: + # Creation of a new feature + feature = ogr.Feature(layer.GetLayerDefn()) + # Preparations for the geometry + point = '' + # Looping on the attributes to add them to the feature + for idx in range(0, max): + if col_names[idx] != '': + # print(col_names[idx]) # debugtest + # print(data[idx + 1]) # debugtest + # 4) Completion of the attributes + if any(elem == table_cols[idx] for elem in ['_date', '_datetime']): + # Preparations for specific values for the date and date_time + try: + # First version if it has all the data necessary for an ogr.OFTDateTime + # +1 because the first value in the attributes is '' + feature.SetField(col_names[idx], data[idx + 1]) + except: + # Second version if some values are missing + # +1 because the first value in the attributes is '' + feature.SetField(col_names[idx], data[idx + 1].year, data[idx + 1].month, + data[idx + 1].day, 0, 0, 0) + # Completing the list of the Context_Records + elif 'context_record__label' in table_cols[idx] and data[idx + 1] not in list_cr: + list_cr.append(data[idx + 1]) + feature.SetField(col_names[idx], str(data[idx + 1])) + else: + # +1 because the first value in the attributes is '' + feature.SetField(col_names[idx], str(data[idx + 1])) + # Gestion of the geometry + id_label = table_cols.index('label') + BaseFind = apps.get_model('archaeological_finds', 'BaseFind') + name = data[id_label + 1] + try: + # Searching for the element itself, especially if the geometry was not exported + object, __ = BaseFind.objects.get_or_create( + label=name, + ) + try: + # Completing the list of the Context_Records + if str(object.context_record).split(' | ')[-1] not in list_cr: + list_cr.append(str(object.context_record).split(' | ')[-1]) + except: + pass + try: + # Creating the geometry from the coordinates X, Y and Z + point = ogr.Geometry(ogr.wkbPoint25D) + point.AddPoint(float(object.x), float(object.y), float(object.z)) + except: + try: + # Case if the Z coordinates doesn't exist + point = ogr.Geometry(ogr.wkbPoint25D) + point.AddPoint(float(object.x), float(object.y), float(0.0)) + except: + try: + # Extreme case if no coordinates but the WKT exists + point = ogr.CreateGeometryFromWkt(str(object.point_3d).split(';')[-1]) + except: + pass + except: + pass + if point != '': + feature.SetGeometry(point) + # Addition of the new feature + layer.CreateFeature(feature) + feature = None + return list_cr + + +def attributes_creation_finds_default(layer, list_crea): + """ + :param layer: Finds layer from the linked geopackage + :param list_crea: Name of the columns by default + :function: Population of the Finds layer using default attributes + :return layer: Finds layer with attributes + """ + # Gestion of specific types of attributes for the default values + for attribute in list_crea: + if attribute == 'Date': + layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTDate)) + elif attribute in ['X', 'Y', 'Z']: + layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTReal)) + else: + layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTString)) + return layer + + +def populating_layer_finds_default(layer, list_crea, list_cr): + """ + :param layer: Finds layer from the linked geopackage with attributes + :param list_crea: Name of the columns by default + :param list_cr: List of all the Context_Records linked to the Finds from the query + :function: Population of the Finds layer using all the data from a specific query + :return layer: Populated Finds layer + """ + # Preparations for the queries + ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord') + BaseFind = apps.get_model('archaeological_finds', 'BaseFind') + Find = apps.get_model('archaeological_finds', 'Find') + # Looping on the different Context_Records names to add them + for name in list_cr: + # Recuperation of the information of the searched Context_Record + cr, __ = ContextRecord.objects.get_or_create(label=name) + # Recuperation of all the BaseFind linked to the searched Context_Record + finds = list(BaseFind.objects.filter(context_record=cr.id)) + # Looping on all the Finds from the query + for elem in finds: + point = '' + try: + # Recuperation of the information of the corresponding Find for some attributes + find = list(Find.objects.filter(base_finds=elem))[0] + list_attributes = [] + try:list_attributes.append(elem.label) + except:list_attributes.append('') + try:list_attributes.append(str(elem.context_record).split(' | ')[-1]) + except:list_attributes.append('') + try:list_attributes.append(elem.discovery_date) + except:list_attributes.append('') + try:list_attributes.append(elem.x) + except:list_attributes.append('') + try:list_attributes.append(elem.y) + except:list_attributes.append('') + try:list_attributes.append(elem.z) + except:list_attributes.append('') + try:list_attributes.append(find.cached_materials) + except:list_attributes.append('') + try:list_attributes.append(find.cached_object_types) + except:list_attributes.append('') + try:list_attributes.append(find.description) + except:list_attributes.append('') + try:list_attributes.append(find.document.image) + except:list_attributes.append('') + try:list_attributes.append(find.comment) + except:list_attributes.append('') + try:list_attributes.append(str(elem.point_3d).split(';')[-1]) + except:list_attributes.append('Point Z ({} {} {})'.format(elem.x, elem.y, elem.z)) + try:list_attributes.append(elem.parcel.external_id) + except:list_attributes.append('') + # Creation of a new feature + feature = ogr.Feature(layer.GetLayerDefn()) + for idx in range(0, len(list_crea)): + if idx == 2 : + # Gestion of the dates + try: + # First version if it has all the data necessary for an ogr.OFTDateTime + feature.SetField(list_crea[idx], list_attributes[idx]) + except: + # Second version if some values are missing + feature.SetField(list_crea[idx], int(list_attributes[idx].year), + int(list_attributes[idx].month), int(list_attributes[idx].day), 0, 0, 0.0, 0) + elif idx in [3,4,5]: + # Gestion of the coordinates + try: + feature.SetField(list_crea[idx], float(list_attributes[idx])) + except: + pass + else: + feature.SetField(list_crea[idx], str(list_attributes[idx])) + try: + # Creating the geometry from the coordinates X, Y and Z + point = ogr.Geometry(ogr.wkbPoint25D) + point.AddPoint(float(elem.x), float(elem.y), float(elem.z)) + except: + try: + # Case if the Z coordinates doesn't exist + point = ogr.Geometry(ogr.wkbPoint25D) + point.AddPoint(float(elem.x), float(elem.y), float(0.0)) + except: + try: + # Extreme case if no coordinates but the WKT exists + point = ogr.CreateGeometryFromWkt(str(elem.point_3d).split(';')[-1]) + except: + pass + if point != '': + feature.SetGeometry(point) + layer.CreateFeature(feature) + feature = None + except: + pass + return layer + + +def attributes_creation_cr_default(layer, list_crea): + """ + :param layer: Context_Records layer from the linked geopackage + :param list_crea: Name of the columns by default + :function: Population of the Context_Records layer using default attributes + :return layer: Populated Context_Records layer + """ + for idx in range(0, len(list_crea)): + layer.CreateField(ogr.FieldDefn(list_crea[idx], ogr.OFTString)) + return layer + + +def populating_layer_cr_default(layer, list_crea, list_cr): + """ + :param layer: Context_Records layer from the linked geopackage with attributes + :param list_crea: Name of the columns by default + :param list_cr: List of all the Context_Records linked to the Finds from the query + :function: Population of the Context_Records layer using all the data from a specific query + :return list_ope: List of all the Operations linked to the Context_Records from the query + """ + list_ope = [] + # Query in the DataBase to get information on the Context_Records of the Finds exported + ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord') + for name in list_cr: + geom = '' + try: + # Recuperation of the information of the searched Context_Record + cr, __ = ContextRecord.objects.get_or_create(label=name) + list_attributes = [] + try:list_attributes.append(cr.label) + except:list_attributes.append('') + try:list_attributes.append(str(cr.operation.code_patriarche)) + except:list_attributes.append('') + try:list_attributes.append(cr.town.numero_insee) + except:list_attributes.append('') + try:list_attributes.append(str(cr.unit)) + except:list_attributes.append('') + try:list_attributes.append(cr.interpretation) + except:list_attributes.append('') + try:list_attributes.append(cr.description) + except:list_attributes.append('') + try:list_attributes.append(cr.location) + except:list_attributes.append('') + try:list_attributes.append(cr.documentations.values('label')[0]['label']) + except:list_attributes.append('') + try:list_attributes.append(cr.documents.image) + except:list_attributes.append('') + try:list_attributes.append(cr.cached_periods) + except:list_attributes.append('Non-renseigné') + try:list_attributes.append(cr.activity) + except:list_attributes.append('') + try:list_attributes.append(cr.identifications.values('label')[0]['label']) + except:list_attributes.append('') + try:list_attributes.append(cr.comment) + except:list_attributes.append('') + try:list_attributes.append(str(cr.main_geodata.multi_polygon).split(';')[-1]) + except:list_attributes.append('') + try:list_attributes.append(cr.parcel.external_id) + except:list_attributes.append('') + # Creation of a new feature + feature = ogr.Feature(layer.GetLayerDefn()) + for idx in range(0, len(list_crea)): + try: + feature.SetField(list_crea[idx], str(list_attributes[idx])) + except: + pass + # Completion of the list of Operations linked to the exported Context_Records + if cr.operation.code_patriarche not in list_ope: + list_ope.append(cr.operation.code_patriarche) + # Gestion of the geometry + try: + geom = ogr.CreateGeometryFromWkt(str(cr.main_geodata.multi_polygon).split(';')[-1]) + feature.SetGeometry(geom) + except: + pass + layer.CreateFeature(feature) + feature = None + except: + pass + return list_ope + + +def attributes_creation_cr_query(layer, col_names, table_cols): + """ + :param layer: Context_Records layer from the linked geopackage + :param col_names: Name of the columns in the new layer + :param table_cols: List of the columns used in the query + :function: Creation of the attributes of the Context_Records layer with the data from the exporter + :return layer: Layer with attributes + """ + for idx in range(0, len(col_names)): + if table_cols[idx] != '': + # print(table_cols[idx]) # debugtest + # print(col_names[idx]) # debugtest + layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTString)) + return layer + + +def populating_layer_cr_query(layer, table_cols, col_names, datas): + """ + :param layer: Context_Records layer from the linked geopackage with attributes + :param table_cols: List of the columns used in the query + :param col_names: Name of the columns in the new layer + :param datas: Data from the query + :function: Population of the Context_Records layer using all the data from the query + :return list_ope: List of all the Operations linked to the Context_Records from the query + """ + #print(table_cols) #debugtest + #print(col_names) #debugtest + #print(datas) #debugtest + list_ope = [] + list_cr = [] + max = len(col_names) + # Looping on all the datas extracted to create features + for data in datas: + # Creation of a new feature + feature = ogr.Feature(layer.GetLayerDefn()) + for idx in range(0, max): + if col_names[idx] != '': + # Completing the list of the Operations + if 'operation__code_patriarche' in table_cols[idx] and data[idx + 1] not in list_ope: + list_ope.append(data[idx + 1]) + feature.SetField(col_names[idx], str(data[idx + 1])) + else: + feature.SetField(col_names[idx], str(data[idx + 1])) + # Getting the name of the Context_Records + id_label = table_cols.index(['label']) + name = data[id_label + 1] + list_cr.append(name) + # Searching for the element itself, especially if the geometry was not exported + ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord') + cr, __ = ContextRecord.objects.get_or_create( + label=name + ) + # Gestion of the geometry + try: + geom = ogr.CreateGeometryFromWkt(str(cr.main_geodata.multi_polygon).split(';')[-1]) + feature.SetGeometry(geom) + except: + pass + layer.CreateFeature(feature) + feature = None + return list_ope, list_cr + + +def attributes_creation_sites_query(layer, col_names, table_cols): + """ + :param layer: Sites layer from the linked geopackage + :param col_names: Name of the columns in the new layer + :param table_cols: List of the columns used in the query + :function: Creation of the attributes of the Sites layer with the data from the exporter + :return layer: Layer with attributes + """ + for idx in range(0, len(col_names)): + if table_cols[idx] != '': + # Gestion of the attribute's type + if table_cols[idx] in ['geodata__x', 'geodata__y']: + layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTReal)) + else: + layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTString)) + return layer + + +def populating_layer_sites_query(layer, table_cols, col_names, datas): + """ + :param layer: Sites layer from the linked geopackage with attributes + :param table_cols: List of the columns used in the query + :param col_names: Name of the columns in the new layer + :param datas: Data from the query + :function: Population of the Sites layer using all the data from the query + :return list_cr: List of all the Context_Records linked to the Sites from the query + """ + max = len(col_names) + list_ope = [] + # Looping on all the datas extracted to create features + for data in datas: + point = '' + # Creation of a new feature + feature = ogr.Feature(layer.GetLayerDefn()) + # Looping on the attributes to add them to the feature + for idx in range(0, max): + if col_names[idx] != '': + # print(table_cols[idx]) # debugtest + # print(data[idx + 1]) # debugtest + # Completing the list of the Operations + if table_cols[idx] == ['operations__code_patriarche'] and data[idx + 1] not in list_ope: + list_ope.append(data[idx + 1]) + feature.SetField(col_names[idx], str(data[idx + 1])) + # Gestion of the information linked to the geometry + elif any(elem in table_cols[idx][0] for elem in ['_x', '_y']): + feature.SetField(col_names[idx], str(data[idx + 1]).split(' & ')[-1]) + elif '_point_2d' in table_cols[idx][0]: + feature.SetField(col_names[idx], str(data[idx + 1]).split(';')[-1]) + else: + # +1 because the first value in the attributes is '' + feature.SetField(col_names[idx], str(data[idx + 1])) + # Recuperation of the information of the searched Sites + ArchaeologicalSite = apps.get_model("archaeological_operations", "ArchaeologicalSite") + try: + # First version with the name of the Site + id_label = table_cols.index(['name']) + label = data[id_label + 1] + object = list(ArchaeologicalSite.objects.filter(name=label))[0] + except: + # Second version with the reference of the Site + id_label = table_cols.index(['reference']) + label = data[id_label + 1] + object = list(ArchaeologicalSite.objects.filter(reference=label))[0] + # Completion of the list of Operations + try: + ope = str(list(object.operations.iterator())).split(' | OA')[-1].replace('>]', '') + if ope not in list_ope: + list_ope.append(ope) + else: + pass + except: + pass + try: + # Creating the geometry from the coordinates X and Y + point = ogr.Geometry(ogr.wkbPoint) + point.AddPoint(float(object.main_geodata.x), float(object.main_geodata.y)) + except: + try: + # Extreme case if no coordinates but the WKT exists + point = ogr.CreateGeometryFromWkt(str(object.main_geodata.point_2d).split(';')[-1]) + except: + pass + if point != '': + feature.SetGeometry(point) + # Addition of the new feature + layer.CreateFeature(feature) + feature = None + Operation = apps.get_model("archaeological_operations", "Operation") + ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord') + # Completion of the list of Context_Records linked to the extracted Sites + list_cr = [] + for name in list_ope: + ope, __ = Operation.objects.get_or_create(code_patriarche=name) + search = ContextRecord.objects.filter(operation=ope) + for cr in search: + if cr.label not in list_cr: + list_cr.append(cr.label) + return list_ope, list_cr + + +def attributes_creation_sites_default(layer, list_crea): + """ + :param layer: Sites layer from the linked geopackage + :param list_crea: Name of the columns by default + :function: Population of the Sites layer using default attributes + :return layer: Sites layer with attributes + """ + # Gestion of specific types of attributes for the default values + for attribute in list_crea: + if attribute in ['X', 'Y']: + layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTReal)) + else: + layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTString)) + return layer + + +def populating_layer_sites_default(layer, list_crea, table_cols, datas): + """ + :param layer: Sites layer from the linked geopackage with attributes + :param list_crea: Name of the columns by default + :param table_cols: List of the columns used in the query + :param datas: Data from the query + :function: Population of the Sites layer using all the data from a specific query + :return list_ope: List of all the Operations linked to the Sites from the query + """ + list_ope = [] + list_cr = [] + # Looping on all the datas to get the code_patriarche of the Operations + for data in datas: + idx = 0 + for col in table_cols: + if 'code_patriarche' in col[0]: + list_ope.append(data[idx + 1]) + idx += 1 + Operation = apps.get_model("archaeological_operations", "Operation") + ArchaeologicalSite = apps.get_model("archaeological_operations", "ArchaeologicalSite") + for name in list_ope: + # Recuperation of the information of the searched Sites + ope, __ = Operation.objects.get_or_create(code_patriarche=name) + sites = list(ArchaeologicalSite.objects.filter(operations=ope.id)) + for site in sites: + list_attributes = [] + point = '' + try:list_attributes.append(site.reference) + except:list_attributes.append('') + try:list_attributes.append(site.name) + except:list_attributes.append('') + try:list_attributes.append(name) + except:list_attributes.append('') + try:list_attributes.append(str(site.cached_towns_label).split('(')[-1].replace(')','')) + except:list_attributes.append('') + try:list_attributes.append(site.cached_periods) + except:list_attributes.append('') + try:list_attributes.append(site.cached_remains) + except:list_attributes.append('') + try:list_attributes.append(site.main_geodata.x) + except:list_attributes.append('') + try:list_attributes.append(site.main_geodata.y) + except:list_attributes.append('') + try:list_attributes.append(site.comment) + except:list_attributes.append('') + if str(site.point) == 'None': + list_attributes.append('Point ({} {})'.format(site.main_geodata.x, site.main_geodata.y)) + else: + list_attributes.append(str(site.point).split(';')[-1]) + # Creation of a new feature + feature = ogr.Feature(layer.GetLayerDefn()) + for idx in range(0, len(list_attributes)): + if idx in [6, 7]: + # Gestion of the coordinates + try: + feature.SetField(list_crea[idx], float(list_attributes[idx])) + except: + pass + else: + feature.SetField(list_crea[idx], str(list_attributes[idx])) + try: + # Creating the geometry from the coordinates X and Y + point = ogr.CreateGeometryFromWkt(str(site.main_geodata.point).split(';')[-1]) + except: + try: + # Extreme case if no coordinates but the WKT exists + point = ogr.Geometry(ogr.wkbPoint) + point.AddPoint(float(site.main_geodata.x), float(site.main_geodata.y)) + except: + pass + if point != '': + feature.SetGeometry(point) + layer.CreateFeature(feature) + feature = None + # Completion of the list of Context_Records linked to the extracted Sites + ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord') + search = ContextRecord.objects.filter(operation=ope) + for cr in search: + if cr.label not in list_cr: + list_cr.append(cr.label) + return list_ope, list_cr + + +def format_identification(default_value, text): + """ + :param default_value: Name of the default value of the targeted list + :param text: Text version of the .qgs + :function: Find the list entry to modify, and get the good order of element to be sure of the modification + :return old_text: List entry to modify, with arguments in the good order, or Fail if not found + """ + # All parameters of an entry in a list for a QGIS attribute + bricks = ['value="{}"'.format(default_value), 'name="{}"'.format(default_value), 'type="QString"'] + old_text = '' + # Getting all the configurations of attributes to be sure to find it in the .qgs + tests = list(permutations(bricks, 3)) + # Trying all configurations, and preparing the good one + for test in tests: + if ' '.join(test) in text: + old_text = ' <Option type="Map">\n <Option {}/>\n </Option>\n'.format(' '.join(test)) + return old_text + if old_text == '': + return 'Fail' + + +def list_recuperation(col, old_name, default_value, base, model, table_cols, col_names, text): + """ + :param col: Column that will have a list + :param old_name: Default_name of the attribute in the .qgs + :param default_value: Name of the default value of the targeted list + :param base: Name of the database to query + :param model: Name of the specific model to query + :param table_cols: List of the columns used in the query + :param col_names: Name of the columns in the new layer + :param text: Text version of the .qgs + :function: Get all possible value for specific attributes, and placing them in lists in the .qgs + :return text: Modified QGIS project + """ + # Replacing the default name of the attribute with a list + idx = table_cols.index(col) + new = col_names[idx] + text = text.replace(old_name, new) + # Recuperation of the different entries linked to the searched attribute + model = apps.get_model(base, model) + query = model.objects.all() + # Recuperation of the entry to modify, with arguments in the good order + old_text = format_identification(default_value, text) + if old_text != 'Fail': + new_text = '' + # Preparing all the entries for the list and adding to the .qgs + for elem in query: + choice = ' <Option type="Map">\n <Option name="{}" value="{}" type="QString"/>\n </Option>\n'.format( + elem, elem) + new_text += choice + text = text.replace(old_text, new_text) + return text + + +def modification_style(qgs_path, table_cols, col_names, list_ope, list_crea): + """ + :param qgs_path: Path to the QGIS project, containing the layers style + :param table_cols: List of the columns used in the query to spot specific ones + :param col_names: Name of the columns in the new layer to add their name to the style of the layer + :param list_ope: List of the Operations linked to the entities from the query, to add them as a list + :param list_crea: List of created attributes for the Finds or Context_Records layers + :function: Modification of the QGIS project style to assure the autocompletion/automations for some attributes + :return text: Modified QGIS project + """ + # Lists of default names in the style, attribut names of the datas and new default names + list_ref = ['finds_date', 'finds_time', 'finds_x', 'finds_y', 'finds_z', 'finds_cr', 'finds_parcel', + 'cr_operation', 'cr_insee', 'cr_section', 'cr_parcel', 'cr_full_parcel','cr_wkt'] + list_search = ['_date', '_datetime', '__x', '__y', '__z', 'context_record__label', 'parcel__external_id', + 'operation__code_patriarche', 'town__numero_insee', 'parcel__section', 'parcel__parcel_number', + 'parcel__external_id', 'geodata__multi_polygon'] + # Opening of the style + text = open(qgs_path, encoding='utf-8').read() + # Adding the different Operations linked of the Contexts Records and/or Finds exported to a list of possible values + old_text = format_identification("Default_value", text) + if len(list_ope) > 0: + new_text = '' + for ope in list_ope: + choice = ' <Option type="Map">\n <Option name="{}" value="{}" type="QString"/>\n </Option>\n'.format(ope, ope) + new_text += choice + text = text.replace(old_text, new_text) + else: + text = text.replace("Test_choice", "Null") + # Verification of all the possible attributes with lists, and application of modification if present + if 'material_types' in table_cols: + text = list_recuperation('material_types', 'finds_material', 'Default_material', + "archaeological_finds", "MaterialType", table_cols, col_names, text) + if 'conservatory_states' in table_cols: + text = list_recuperation('conservatory_states', 'finds_conservation', 'Default_conserv', + "archaeological_finds", "ConservatoryState", table_cols, col_names, text) + if 'object_types' in table_cols: + text = list_recuperation('object_types', 'finds_object', 'Default_object', + "archaeological_finds", "ObjectType", table_cols, col_names, text) + if ['unit'] in table_cols: + text = list_recuperation(['unit'], 'cr_type', 'Default_unit', + "archaeological_context_records", "Unit", table_cols, col_names, text) + if ['activity'] in table_cols: + text = list_recuperation(['activity'], 'cr_activity', 'Default_activity', + "archaeological_context_records", "ActivityType", table_cols, col_names, text) + if ['identifications'] in table_cols: + text = list_recuperation(['identifications'], 'cr_identif', 'Default_identif', + "archaeological_context_records", "IdentificationType", table_cols, col_names, text) + if ['documentations'] in table_cols: + text = list_recuperation(['documentations'], 'cr_doc', 'Default_doc', + "archaeological_context_records", "DocumentationType", table_cols, col_names, text) + if ['datings__period'] in table_cols: + text = list_recuperation(['datings__period'], 'cr_periode', 'Default_periode', + "archaeological_operations", "Period", table_cols, col_names, text) + + # Specifics modifications if the datas don't come from Finds + if list_crea[0] == 'finds': + for ref in list_ref: + id_ref = list_ref.index(ref) + for col in table_cols: + if col != '' and list_search[id_ref] in col[0]: + id_new = table_cols.index(col) + new = col_names[id_new] + text = text.replace(ref, new) + # List of corresponding default names in the style linked to the default names used for the Finds + list_corr = ['finds_id', 'finds_cr', 'finds_date', 'finds_x', 'finds_y', 'finds_z', 'finds_material', + 'finds_object', 'finds_description', 'finds_media', 'finds_comment', 'finds_wkt_modif', 'finds_parcel'] + # Gestion of the link between the Finds and Context Records layers + id_label = table_cols.index(['label']) + new = col_names[id_label] + text = text.replace("cr_name", new) + if ['documents__image'] in table_cols: + id_media = table_cols.index(['documents__image']) + # Gestion of the link between the Finds and Context Records layers + new = col_names[id_media] + text = text.replace("cr_media", new) + # Modification of the default attributes with lists + text = list_recuperation('finds_material', 'finds_material', 'Default_material', + "archaeological_finds", "MaterialType", list_corr, list_crea[1], text) + text = list_recuperation('finds_object', 'finds_object', 'Default_object', + "archaeological_finds", "ObjectType", list_corr, list_crea[1], text) + # Replacement of the values from the default names used for the Finds + n = 0 + for elem in list_crea[1]: + old = list_corr[n] + text = text.replace(old, elem) + n += 1 + # Specifics modifications if the datas don't come from Context_Records + elif list_crea[0] == 'cr': + for ref in list_ref: + id_ref = list_ref.index(ref) + for col in table_cols: + if col != '' and list_search[id_ref] in col: + id_new = table_cols.index(col) + new = col_names[id_new] + text = text.replace(ref, new) + # List of corresponding default names in the style linked to the default names used for the Finds + list_corr = ['cr_name', 'cr_operation', 'cr_insee', 'cr_type', 'cr_occupation', 'cr_description', + 'cr_localisation', 'cr_doc', 'cr_media', 'cr_periode', 'cr_activity', 'cr_identif', 'cr_comment', + 'cr_wkt', 'cr_full_parcel'] + # Test in case the all names of attributes are in lists + try: + id_label = table_cols.index(['label']) + except: + id_label = table_cols.index('label') + # Gestion of the link between the Finds and Context Records layers + new = col_names[id_label] + text = text.replace('finds_id', new) + if 'documents__image' in table_cols: + try: + id_media = table_cols.index(['documents__image']) + except: + id_media = table_cols.index('documents__image') + # Gestion of the link between the Finds and Context Records layers + new = col_names[id_media] + text = text.replace("finds_media", new) + # Modification of the default attributes with lists + text = list_recuperation('cr_type', 'cr_type', 'Default_unit', + "archaeological_context_records", "Unit", list_corr, list_crea[1], text) + text = list_recuperation('cr_activity', 'cr_activity', 'Default_activity', + "archaeological_context_records", "ActivityType", list_corr, list_crea[1], text) + text = list_recuperation('cr_periode', 'cr_periode', 'Default_periode', + "archaeological_operations", "Period", list_corr, list_crea[1], text) + text = list_recuperation('cr_identif', 'cr_identif', 'Default_identif', + "archaeological_context_records", "IdentificationType", list_corr, list_crea[1], text) + text = list_recuperation('cr_doc', 'cr_doc', 'Default_doc', + "archaeological_context_records", "DocumentationType", list_corr, list_crea[1], text) + # Specific case to assure the good registration of the z coordinate + if any('__point_3d' in elem for elem in table_cols): + id_new = [i for i, element in enumerate(table_cols) if 'geodata__point_3d' in element][0] + #id_new = table_cols.index('geodata__point_3d') + if any('__z' in elem for elem in table_cols): + ref = "finds_wkt_modif" + new = col_names[id_new] + else: + ref = "finds_wkt_simple" + new = col_names[id_new] + text = text.replace(ref, new) + # Replacement of the values from the default names used for the Context Records + n = 0 + for elem in list_crea[1]: + old = list_corr[n] + text = text.replace(old, elem) + n += 1 + else: + pass + return text + + +def modification_style_sites(qgs_path, table_cols, col_names, list_ope, list_crea): + """ + :param qgs_path: Path to the QGIS project, containing the layers style + :param table_cols: List of the columns used in the query to spot specific ones + :param col_names: Name of the columns in the new layer to add their name to the style of the layer + :param list_ope: List of the Operations linked to the entities from the query, to add them as a list + :param list_crea: List of created attributes for the Finds and Context_Records layers + :function: Modification of the QGIS project style to assure the autocompletion/automations for some attributes + :return text: Modified QGIS project + """ + # Lists of default names in the style, attribut names of the datas and new default names + list_ref = ['sites_operation', 'sites_parcel', 'sites_insee', 'sites_x', 'sites_y', 'sites_wkt'] + list_search = ['operations__code_patriarche', 'parcel__external_id', 'towns__numero_insee', 'geodata__x', + 'geodata__y', 'geodata__point_2d'] + # Opening of the style + text = open(qgs_path, encoding='utf-8').read() + # Adding the different Operations linked of the Contexts Records and/or Finds exported to a list of possible values + old_text = format_identification("Default_value", text) + # Adding the different Operations linked of the Contexts Records and/or Finds exported to a list of possible values + if len(list_ope) > 0: + new_text = '' + for ope in list_ope: + choice = ' <Option type="Map">\n <Option name="{}" value="{}" type="QString"/>\n </Option>\n'.format( + ope, ope) + new_text += choice + text = text.replace(old_text, new_text) + else: + text = text.replace("Test_choice", "Default") + # Verification of all the possible attributes with lists, and application of modification if present + if ['periods'] in table_cols: + text = list_recuperation(['periods'], 'sites_periodes', 'Default_speriode', + "archaeological_operations", "Period", table_cols, col_names, text) + if ['remains'] in table_cols: + text = list_recuperation(['remains'], 'sites_type', 'Default_rtype', + "archaeological_operations", "RemainType", table_cols, col_names, text) + # Only loop and modify the name of attributes if the query was done on the Sites, not the Operations + # If from Operations, the first name of default columns is the Reference of the Site + if list_crea[0] != 'Référence': + for ref in list_ref: + id_ref = list_ref.index(ref) + for col in table_cols: + if col != '' and list_search[id_ref] in col: + id_new = table_cols.index(col) + new = col_names[id_new] + text = text.replace(ref, new) + # Default names of the common attributes to modify + list_ref = ['cr_name', 'cr_operation', 'cr_insee', 'cr_type', 'cr_occupation', 'cr_description', + 'cr_localisation', 'cr_doc', 'cr_media', 'cr_periode', 'cr_activity', 'cr_identif', 'cr_comment', + 'cr_wkt', 'cr_full_parcel', 'finds_id', 'finds_cr', 'finds_date', 'finds_x', 'finds_y', 'finds_z', + 'finds_material', 'finds_object', 'finds_description', 'finds_media', 'find_comment', 'finds_wkt_modif', + 'finds_parcel'] + # Modification of the default attributes with lists + if list_crea[0] == 'Référence': + # Adding the names of the specific attributes for the sites + list_ref = ['sites_reference', 'sites_name', 'sites_operation', 'sites_insee', 'sites_periodes', 'sites_type', + 'sites_x', 'sites_y', 'sites_comment', 'sites_wkt', 'sites_parcel'] + list_ref + text = list_recuperation('sites_periodes', 'sites_periodes', 'Default_speriode', + "archaeological_operations", "Period", list_ref, list_crea, text) + text = list_recuperation('sites_type', 'sites_type', 'Default_rtype', + "archaeological_operations", "RemainType", list_ref, list_crea, text) + text = list_recuperation('finds_material', 'finds_material', 'Default_material', + "archaeological_finds", "MaterialType", list_ref, list_crea, text) + text = list_recuperation('finds_object', 'finds_object', 'Default_object', + "archaeological_finds", "ObjectType", list_ref, list_crea, text) + text = list_recuperation('cr_type', 'cr_type', 'Default_unit', + "archaeological_context_records", "Unit", list_ref, list_crea, text) + text = list_recuperation('cr_activity', 'cr_activity', 'Default_activity', + "archaeological_context_records", "ActivityType", list_ref, list_crea, text) + text = list_recuperation('cr_identif', 'cr_identif', 'Default_identif', + "archaeological_context_records", "IdentificationType", list_ref, list_crea, text) + text = list_recuperation('cr_doc', 'cr_doc', 'Default_doc', + "archaeological_context_records", "DocumentationType", list_ref, list_crea, text) + text = list_recuperation('cr_periode', 'cr_periode', 'Default_periode', + "archaeological_operations", "Period", list_ref, list_crea, text) + # Modification of the othe attributes names + for id in range(0, len(list_crea)): + text = text.replace(list_ref[id], list_crea[id]) + return text
\ No newline at end of file |