path: root/ishtar_common/qfield_functions.py

import os
import logging
from osgeo import ogr, osr
from itertools import permutations
from django.apps import apps
from django.conf import settings

logger = logging.getLogger(__name__)

ENCODING = settings.ENCODING or "utf-8"

HIERARCHIC_LEVELS = 5

LIST_FIELDS = {  # key: hierarchic depth
    "conservatory_states": HIERARCHIC_LEVELS,
    "identifications": HIERARCHIC_LEVELS,
    "material_types": HIERARCHIC_LEVELS,
    "material_type": HIERARCHIC_LEVELS,
    "object_types": HIERARCHIC_LEVELS,
    "period": HIERARCHIC_LEVELS,
    "periods": HIERARCHIC_LEVELS,
    "source_type": HIERARCHIC_LEVELS,
    "unit": HIERARCHIC_LEVELS,
    "museum_collection_entry_mode": HIERARCHIC_LEVELS,
    "shooting_angle": HIERARCHIC_LEVELS,
    "technical_processes": HIERARCHIC_LEVELS,
    "structures": HIERARCHIC_LEVELS,
    "textures": HIERARCHIC_LEVELS,
    "inclusions": HIERARCHIC_LEVELS,
    "colors": HIERARCHIC_LEVELS,
    "development_type": HIERARCHIC_LEVELS,
    "monitoring_justification": HIERARCHIC_LEVELS,
    "documentations": HIERARCHIC_LEVELS,
    "excavation_technics": HIERARCHIC_LEVELS,
    "treatment_types": HIERARCHIC_LEVELS,
    "discovery_method": 0,
    "discovery_status": 0,
    "current_status": 0,
    "nature_of_site": 0,
    "interpretation_level": 0,
    "museum_inventory_marking_presence": 0,
    "museum_marking_type": 0,
    "museum_collection": 0,
    "batch": 0,
    "preservation_to_considers": 0,
    "integrities": 0,
    "remarkabilities": 0,
    "checked_type": 0,
    "material_type_quality": 0,
    "object_type_quality": 0,
    "communicabilities": 0,
    "alterations": 0,
    "alteration_causes": 0,
    "treatment_emergency": 0,
    "cultural_attributions": 0,
    "remains": 0,
    "dating_type": 0,
    "quality": 0,
    "operation_type": 0,
    "report_processing": 0,
    "record_quality_type": 0,
    "data_type": 0,
    "origin": 0,
    "provider": 0,
    "activity": 0,
    "person_types": 0,
    "relation_types": 0,
    "types": HIERARCHIC_LEVELS,  # keep it at the end to not mess with other types
}

HIERARCHIC_FIELDS = list(LIST_FIELDS.keys())

def gpkg_creation(model, root, table_cols, col_names, datas):
    """
    :param model: Table from the query
    :param root: Path to the folder to create the geopackage
    :param table_cols: List of the columns used in the query
    :param col_names: Name of the columns in the new geopackage
    :param datas: Data from the query
    :function: Creation of the Finds and Context_Records geopackages when the query come from one of these two tables
    :return finds: Geopackage for the Finds
    :return cr: Geopackage for the Context_Records
    :return list_ope: List of the different operations linked to the Finds and Context_Records
    :return list_cr: List of the labels/names of the Context_Records used
    """
    # Preparation of important values and parameters for the geopackages
    finds = ''
    cr = ''
    list_ope = []
    list_crea = []
    driver = ogr.GetDriverByName('GPKG')
    srs = osr.SpatialReference()
    srs.ImportFromEPSG(4326)
    # I. Case where the extraction come from Finds
    if str(model._meta) == 'archaeological_finds.find':
        # 1) Creation of the Finds geopackage
        finds = os.path.join(root, 'export', 'Finds.gpkg')
        # Verification to delete it if already existing
        if os.path.exists(finds):
            os.remove(finds)
        # 2) Creation of the finds layer and its attributes
        datasource = driver.CreateDataSource(finds)
        layer = datasource.CreateLayer('Finds', srs, ogr.wkbPoint25D)
        layer = attributes_creation_finds_query(layer, col_names, table_cols)
        # 4a) Populating the finds layer with the datas
        list_cr = populating_layer_finds_query(layer,table_cols,col_names,datas)
        datasource = None
        # 3) Creation of the Context Records file
        cr = os.path.join(root, 'export', 'Context_records.gpkg')
        # Verification to delete it if already existing
        if os.path.exists(cr):
            os.remove(cr)
        datasource = driver.CreateDataSource(cr)
        # 4) Creation of the Context_Records layer and a list of default attributes
        layer = datasource.CreateLayer('Context_records', srs, ogr.wkbMultiPolygon)
        list_crea = ['Unité_Enregistrement', 'Opération', 'INSEE_Commune', 'Type', 'Interprétation',
                     'Description', 'Localisation', 'Document_associe', 'Media', 'Periode', 'Type_Activité',
                     'Type_Identification', 'Commentaires', 'WKT', 'Infos_Parcelles_UE']
        layer = attributes_creation_cr_default(layer, list_crea)
        # 5) Populating the Context_Records layer with datas from the Context_Records of the extracted finds
        list_ope = populating_layer_cr_default(layer, list_crea, list_cr)
        datasource = None
        # 6) Preparation of a list of the attributes names for the style modifications
        list_crea = ['cr', list_crea]
    # II. Case where the extraction come from Context_Recods
    elif str(model._meta) == 'archaeological_context_records.contextrecord':
        # 1) Creation of the Context Records geopackage
        cr = os.path.join(root, 'export', 'Context_records.gpkg')
        # Verification to delete it if already existing
        if os.path.exists(cr):
            os.remove(cr)
        datasource = driver.CreateDataSource(cr)
        # 2) Creation of the Context_Records layer and its attributes
        layer = datasource.CreateLayer('Context_records', srs, ogr.wkbMultiPolygon)
        layer = attributes_creation_cr_query(layer, col_names, table_cols)
        # 3) Populating the Finds layer with the datas
        list_ope, list_cr = populating_layer_cr_query(layer, table_cols, col_names, datas)
        datasource = None
        # 4) Creation of the Finds geopackage
        finds = os.path.join(root, 'export', 'Finds.gpkg')
        # Verification to delete it if already existing
        if os.path.exists(finds):
            os.remove(finds)
        datasource = driver.CreateDataSource(finds)
        layer = datasource.CreateLayer('Finds', srs, ogr.wkbPoint25D)
        list_crea = ['Identifiant', 'UE', 'Date', 'X', 'Y', 'Z', 'Matériaux', 'Type_objets', 'Description', 'Media',
                     'Commentaires', 'WKT_point', 'Infos_Parcelle_Mobilier']
        attributes_creation_finds_default(layer, list_crea)
        # 5) Populating the finds layer with the datas
        populating_layer_finds_default(layer, list_crea, list_cr)
        # 6) Preparation of a list of the attributes names for the style modifications
        list_crea = ['finds', list_crea]
    return finds, cr, list_ope, list_crea


def gpkg_creation_sites(model, root, table_cols, col_names, datas):
    """
    :param model: Table from the query
    :param root: Path to the folder to create the geopackage
    :param table_cols: List of the columns used in the query
    :param col_names: Name of the columns in the new geopackage
    :param datas: Data from the query
    :function: Specific version for the creation of the needed geopackages when the query come from the
                Archaeological_Sites table
    :return sites: Geopackage for the Sites
    :return finds: Geopackage for the Finds
    :return cr: Geopackage for the Context_Records
    :return list_ope: List of the different operations linked to the Finds and Context_Records
    :return list_cr: List of the labels/names of the Context_Records used
    """
    # Preparation of important values and parameters for the geopackages
    finds = ''
    cr = ''
    list_ope = []
    list_crea = []
    driver = ogr.GetDriverByName('GPKG')
    srs = osr.SpatialReference()
    srs.ImportFromEPSG(4326)
    # 1) Creation of the sites layer
    sites = os.path.join(root, 'export', 'Sites.gpkg')
    if os.path.exists(sites):
        os.remove(sites)
    datasource = driver.CreateDataSource(sites)
    layer = datasource.CreateLayer('Sites', srs, ogr.wkbPoint)
    # Specific case if the query come from the Operations, and all element linked to them must be searched
    if str(model._meta) == 'archaeological_operations.operation':
        # Creation of the attributes
        list_s = ['Référence', 'Nom', 'Operation', 'Commune', 'Période', 'Type', 'X', 'Y', 'Commentaires', 'WKT', 'Infos_Parcelles_Sites']
        layer = attributes_creation_sites_default(layer, list_s)
        #Populating the finds layer with the datas
        list_ope, list_cr = populating_layer_sites_default(layer, list_s, table_cols, datas)
    else:
        # Creation of the attributes
        layer = attributes_creation_sites_query(layer, col_names, table_cols)
        # Creation of the entities
        list_ope, list_cr = populating_layer_sites_query(layer, table_cols, col_names, datas)
    # 2) Creation of the Context_Records layer
    cr = os.path.join(root, 'export', 'Context_records.gpkg')
    # Verification to delete it if already existing
    if os.path.exists(cr):
        os.remove(cr)
    datasource = driver.CreateDataSource(cr)
    layer = datasource.CreateLayer('Context_records', srs, ogr.wkbMultiPolygon)
    # Creation of the attributes
    list_a = ['Unité_Enregistrement', 'Opération', 'INSEE_Commune', 'Type', 'Interprétation', 'Description',
              'Localisation', 'Document_associe', 'Media', 'Periode', 'Type_Activité', 'Type_Identification',
              'Commentaires', 'WKT', 'Infos_Parcelles_UE']
    layer = attributes_creation_cr_default(layer, list_a)
    # # Creation of the entities
    populating_layer_cr_default(layer, list_a, list_cr)
    datasource = None
    # 3) Creation of the finds layer
    finds = os.path.join(root, 'export', 'Finds.gpkg')
    # Verification to delete it if already existing
    if os.path.exists(finds):
        os.remove(finds)
    datasource = driver.CreateDataSource(finds)
    layer = datasource.CreateLayer('Finds', srs, ogr.wkbPoint25D)
    # Creation of the attributes
    list_b = ['Identifiant', 'UE', 'Date', 'X', 'Y', 'Z', 'Matériaux', 'Type_objets', 'Description', 'Media',
              'Commentaires', 'WKT_point', 'Infos_Parcelles_Mobilier']
    layer = attributes_creation_finds_default(layer, list_b)
    # Creation of the entities
    populating_layer_finds_default(layer, list_b, list_cr)
    # Recuperation of all created attributes
    if str(model._meta) == 'archaeological_operations.operation':
        list_crea = list_s + list_a + list_b
    else:
        list_crea = list_a + list_b
    return sites, finds, cr, list_ope, list_crea


def attributes_creation_finds_query(layer, col_names, table_cols):
    """
    :param layer: Finds layer from the linked geopackage
    :param col_names: Name of the columns in the new layer
    :param table_cols: List of the columns used in the query
    :function: Creation of the attributes of the Finds layer with the information from the exporter
    :return layer: Finds layer with attributes
    """
    # print(table_cols) # debugtest
    # print(col_names) # debugtest
    # print(datas) # debugtest
    # Looping on all the attributes
    for idx in range(0, len(col_names)):
        # Prevent missing values (case in some .gpkg)
        if table_cols[idx] != '':
            # print(table_cols[idx]) # debugtest
            # print(col_names[idx]) # debugtest
            # Gestion of specific formats of attributes
            if any(elem in table_cols[idx] for elem in ['index', 'order', 'quantity', 'taq', 'tpq', 'year']):
                layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTInteger64))
            elif any(elem in table_cols[idx] for elem in
                     ['_x', '_y', '_z', 'circumference', 'cost', 'depth', 'diameter', 'height', 'length', 'number',
                      'surface', 'side', 'thickness', 'value', 'volume', 'weight', 'width']):
                layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTReal))
            elif '_date' in table_cols[idx]:
                layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTDate))
            elif '_datetime' in table_cols[idx]:
                layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTDateTime))
            elif any(elem in table_cols[idx] for elem in ['large_area_prescription', 'is_complete', 'executed']):
                layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTBinary))
            else:
                layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTString))
    return layer


def populating_layer_finds_query(layer, table_cols, col_names, datas):
    """
    :param layer: Finds layer from the linked geopackage with attributes
    :param table_cols: List of the columns used in the query
    :param col_names: Name of the columns in the new layer
    :param datas: Data from the query
    :function: Population of the Finds layer using all the data from the query
    :return list_cr: List of all the Context_Records linked to the Finds from the query
    """
    max = len(col_names)
    list_cr = []
    # Looping on all the datas extracted to create features
    for data in datas:
        # Creation of a new feature
        feature = ogr.Feature(layer.GetLayerDefn())
        # Preparations for the geometry
        point = ''
        # Looping on the attributes to add them to the feature
        for idx in range(0, max):
            if col_names[idx] != '':
                # print(col_names[idx]) # debugtest
                # print(data[idx + 1]) # debugtest
                # 4) Completion of the attributes
                if any(elem == table_cols[idx] for elem in ['_date', '_datetime']):
                    # Preparations for specific values for the date and date_time
                    try:
                        # First version if it has all the data necessary for an ogr.OFTDateTime
                        # +1 because the first value in the attributes is ''
                        feature.SetField(col_names[idx], data[idx + 1])
                    except:
                        # Second version if some values are missing
                        # +1 because the first value in the attributes is ''
                        feature.SetField(col_names[idx], data[idx + 1].year, data[idx + 1].month,
                                         data[idx + 1].day, 0, 0, 0)
                # Completing the list of the Context_Records
                elif 'context_record__label' in table_cols[idx] and data[idx + 1] not in list_cr:
                    list_cr.append(data[idx + 1])
                    feature.SetField(col_names[idx], str(data[idx + 1]))
                else:
                    # +1 because the first value in the attributes is ''
                    feature.SetField(col_names[idx], str(data[idx + 1]))
        # Gestion of the geometry
        id_label = table_cols.index('label')
        BaseFind = apps.get_model('archaeological_finds', 'BaseFind')
        name = data[id_label + 1]
        try:
            # Searching for the element itself, especially if the geometry was not exported
            object, __ = BaseFind.objects.get_or_create(
                label=name,
             )
            try:
                # Completing the list of the Context_Records
                if str(object.context_record).split(' | ')[-1] not in list_cr:
                    list_cr.append(str(object.context_record).split(' | ')[-1])
            except:
                pass
            try:
                # Creating the geometry from the coordinates X, Y and Z
                point = ogr.Geometry(ogr.wkbPoint25D)
                point.AddPoint(float(object.x), float(object.y), float(object.z))
            except:
                try:
                    # Case if the Z coordinates doesn't exist
                    point = ogr.Geometry(ogr.wkbPoint25D)
                    point.AddPoint(float(object.x), float(object.y), float(0.0))
                except:
                    try:
                        # Extreme case if no coordinates but the WKT exists
                        point = ogr.CreateGeometryFromWkt(str(object.point_3d).split(';')[-1])
                    except:
                        pass
        except:
            pass
        if point != '':
            feature.SetGeometry(point)
        # Addition of the new feature
        layer.CreateFeature(feature)
        feature = None
    return list_cr


def attributes_creation_finds_default(layer, list_crea):
    """
    :param layer: Finds layer from the linked geopackage
    :param list_crea: Name of the columns by default
    :function: Population of the Finds layer using default attributes
    :return layer: Finds layer with attributes
    """
    # Gestion of specific types of attributes for the default values
    for attribute in list_crea:
        if attribute == 'Date':
            layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTDate))
        elif attribute in ['X', 'Y', 'Z']:
            layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTReal))
        else:
            layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTString))
    return layer


def populating_layer_finds_default(layer, list_crea, list_cr):
    """
    :param layer: Finds layer from the linked geopackage with attributes
    :param list_crea: Name of the columns by default
    :param list_cr: List of all the Context_Records linked to the Finds from the query
    :function: Population of the Finds layer using all the data from a specific query
    :return layer: Populated Finds layer
    """
    # Preparations for the queries
    ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord')
    BaseFind = apps.get_model('archaeological_finds', 'BaseFind')
    Find = apps.get_model('archaeological_finds', 'Find')
    # Looping on the different Context_Records names to add them
    for name in list_cr:
        # Recuperation of the information of the searched Context_Record
        cr, __ = ContextRecord.objects.get_or_create(label=name)
        # Recuperation of all the BaseFind linked to the searched Context_Record
        finds = list(BaseFind.objects.filter(context_record=cr.id))
        # Looping on all the Finds from the query
        for elem in finds:
            point = ''
            try:
                # Recuperation of the information of the corresponding Find for some attributes
                find = list(Find.objects.filter(base_finds=elem))[0]
                list_attributes = []
                try:list_attributes.append(elem.label)
                except:list_attributes.append('')
                try:list_attributes.append(str(elem.context_record).split(' | ')[-1])
                except:list_attributes.append('')
                try:list_attributes.append(elem.discovery_date)
                except:list_attributes.append('')
                try:list_attributes.append(elem.x)
                except:list_attributes.append('')
                try:list_attributes.append(elem.y)
                except:list_attributes.append('')
                try:list_attributes.append(elem.z)
                except:list_attributes.append('')
                try:list_attributes.append(find.cached_materials)
                except:list_attributes.append('')
                try:list_attributes.append(find.cached_object_types)
                except:list_attributes.append('')
                try:list_attributes.append(find.description)
                except:list_attributes.append('')
                try:list_attributes.append(find.document.image)
                except:list_attributes.append('')
                try:list_attributes.append(find.comment)
                except:list_attributes.append('')
                try:list_attributes.append(str(elem.point_3d).split(';')[-1])
                except:list_attributes.append('Point Z ({} {} {})'.format(elem.x, elem.y, elem.z))
                try:list_attributes.append(elem.parcel.external_id)
                except:list_attributes.append('')
                # Creation of a new feature
                feature = ogr.Feature(layer.GetLayerDefn())
                for idx in range(0, len(list_crea)):
                    if idx == 2 :
                        # Gestion of the dates
                        try:
                            # First version if it has all the data necessary for an ogr.OFTDateTime
                            feature.SetField(list_crea[idx], list_attributes[idx])
                        except:
                            # Second version if some values are missing
                            feature.SetField(list_crea[idx], int(list_attributes[idx].year),
                                             int(list_attributes[idx].month), int(list_attributes[idx].day), 0, 0, 0.0, 0)
                    elif idx in [3,4,5]:
                        # Gestion of the coordinates
                        try:
                            feature.SetField(list_crea[idx], float(list_attributes[idx]))
                        except:
                            pass
                    else:
                        feature.SetField(list_crea[idx], str(list_attributes[idx]))
                try:
                    # Creating the geometry from the coordinates X, Y and Z
                    point = ogr.Geometry(ogr.wkbPoint25D)
                    point.AddPoint(float(elem.x), float(elem.y), float(elem.z))
                except:
                    try:
                        # Case if the Z coordinates doesn't exist
                        point = ogr.Geometry(ogr.wkbPoint25D)
                        point.AddPoint(float(elem.x), float(elem.y), float(0.0))
                    except:
                        try:
                            # Extreme case if no coordinates but the WKT exists
                            point = ogr.CreateGeometryFromWkt(str(elem.point_3d).split(';')[-1])
                        except:
                            pass
                if point != '':
                    feature.SetGeometry(point)
                layer.CreateFeature(feature)
                feature = None
            except:
                pass
    return layer


def attributes_creation_cr_default(layer, list_crea):
    """
    :param layer: Context_Records layer from the linked geopackage
    :param list_crea: Name of the columns by default
    :function: Population of the Context_Records layer using default attributes
    :return layer: Populated Context_Records layer
    """
    for idx in range(0, len(list_crea)):
        layer.CreateField(ogr.FieldDefn(list_crea[idx], ogr.OFTString))
    return layer


def populating_layer_cr_default(layer, list_crea, list_cr):
    """
    :param layer: Context_Records layer from the linked geopackage with attributes
    :param list_crea: Name of the columns by default
    :param list_cr: List of all the Context_Records linked to the Finds from the query
    :function: Population of the Context_Records layer using all the data from a specific query
    :return list_ope: List of all the Operations linked to the Context_Records from the query
    """
    list_ope = []
    # Query in the DataBase to get information on the Context_Records of the Finds exported
    ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord')
    for name in list_cr:
        geom = ''
        try:
            # Recuperation of the information of the searched Context_Record
            cr, __ = ContextRecord.objects.get_or_create(label=name)
            list_attributes = []
            try:list_attributes.append(cr.label)
            except:list_attributes.append('')
            try:list_attributes.append(str(cr.operation.code_patriarche))
            except:list_attributes.append('')
            try:list_attributes.append(cr.town.numero_insee)
            except:list_attributes.append('')
            try:list_attributes.append(str(cr.unit))
            except:list_attributes.append('')
            try:list_attributes.append(cr.interpretation)
            except:list_attributes.append('')
            try:list_attributes.append(cr.description)
            except:list_attributes.append('')
            try:list_attributes.append(cr.location)
            except:list_attributes.append('')
            try:list_attributes.append(cr.documentations.values('label')[0]['label'])
            except:list_attributes.append('')
            try:list_attributes.append(cr.documents.image)
            except:list_attributes.append('')
            try:list_attributes.append(cr.cached_periods)
            except:list_attributes.append('Non-renseigné')
            try:list_attributes.append(cr.activity)
            except:list_attributes.append('')
            try:list_attributes.append(cr.identifications.values('label')[0]['label'])
            except:list_attributes.append('')
            try:list_attributes.append(cr.comment)
            except:list_attributes.append('')
            try:list_attributes.append(str(cr.main_geodata.multi_polygon).split(';')[-1])
            except:list_attributes.append('')
            try:list_attributes.append(cr.parcel.external_id)
            except:list_attributes.append('')
            # Creation of a new feature
            feature = ogr.Feature(layer.GetLayerDefn())
            for idx in range(0, len(list_crea)):
                try:
                    feature.SetField(list_crea[idx], str(list_attributes[idx]))
                except:
                    pass
            # Completion of the list of Operations linked to the exported Context_Records
            if cr.operation.code_patriarche not in list_ope:
                list_ope.append(cr.operation.code_patriarche)
            # Gestion of the geometry
            try:
                geom = ogr.CreateGeometryFromWkt(str(cr.main_geodata.multi_polygon).split(';')[-1])
                feature.SetGeometry(geom)
            except:
                pass
            layer.CreateFeature(feature)
            feature = None
        except:
            pass
    return list_ope


def attributes_creation_cr_query(layer, col_names, table_cols):
    """
    :param layer: Context_Records layer from the linked geopackage
    :param col_names: Name of the columns in the new layer
    :param table_cols: List of the columns used in the query
    :function: Creation of the attributes of the Context_Records layer with the data from the exporter
    :return layer: Layer with attributes
    """
    for idx in range(0, len(col_names)):
        if table_cols[idx] != '':
            # print(table_cols[idx]) # debugtest
            # print(col_names[idx]) # debugtest
            layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTString))
    return layer


def populating_layer_cr_query(layer, table_cols, col_names, datas):
    """
    :param layer: Context_Records layer from the linked geopackage with attributes
    :param table_cols: List of the columns used in the query
    :param col_names: Name of the columns in the new layer
    :param datas: Data from the query
    :function: Population of the Context_Records layer using all the data from the query
    :return list_ope: List of all the Operations linked to the Context_Records from the query
    """
    #print(table_cols) #debugtest
    #print(col_names) #debugtest
    #print(datas) #debugtest
    list_ope = []
    list_cr = []
    max = len(col_names)
    # Looping on all the datas extracted to create features
    for data in datas:
        # Creation of a new feature
        feature = ogr.Feature(layer.GetLayerDefn())
        for idx in range(0, max):
            if col_names[idx] != '':
                # Completing the list of the Operations
                if 'operation__code_patriarche' in table_cols[idx] and data[idx + 1] not in list_ope:
                    list_ope.append(data[idx + 1])
                    feature.SetField(col_names[idx], str(data[idx + 1]))
                else:
                    feature.SetField(col_names[idx], str(data[idx + 1]))
        # Getting the name of the Context_Records
        id_label = table_cols.index(['label'])
        name = data[id_label + 1]
        list_cr.append(name)
        # Searching for the element itself, especially if the geometry was not exported
        ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord')
        cr, __ = ContextRecord.objects.get_or_create(
            label=name
        )
        # Gestion of the geometry
        try:
            geom = ogr.CreateGeometryFromWkt(str(cr.main_geodata.multi_polygon).split(';')[-1])
            feature.SetGeometry(geom)
        except:
            pass
        layer.CreateFeature(feature)
        feature = None
    return list_ope, list_cr


def attributes_creation_sites_query(layer, col_names, table_cols):
    """
    :param layer: Sites layer from the linked geopackage
    :param col_names: Name of the columns in the new layer
    :param table_cols: List of the columns used in the query
    :function: Creation of the attributes of the Sites layer with the data from the exporter
    :return layer: Layer with attributes
    """
    for idx in range(0, len(col_names)):
        if table_cols[idx] != '':
            # Gestion of the attribute's type
            if table_cols[idx] in ['geodata__x', 'geodata__y']:
                layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTReal))
            else:
                layer.CreateField(ogr.FieldDefn(col_names[idx], ogr.OFTString))
    return layer


def populating_layer_sites_query(layer, table_cols, col_names, datas):
    """
    :param layer: Sites layer from the linked geopackage with attributes
    :param table_cols: List of the columns used in the query
    :param col_names: Name of the columns in the new layer
    :param datas: Data from the query
    :function: Population of the Sites layer using all the data from the query
    :return list_cr: List of all the Context_Records linked to the Sites from the query
    """
    max = len(col_names)
    list_ope = []
    # Looping on all the datas extracted to create features
    for data in datas:
        point = ''
        # Creation of a new feature
        feature = ogr.Feature(layer.GetLayerDefn())
        # Looping on the attributes to add them to the feature
        for idx in range(0, max):
            if col_names[idx] != '':
                # print(table_cols[idx]) # debugtest
                # print(data[idx + 1]) # debugtest
                # Completing the list of the Operations
                if table_cols[idx] == ['operations__code_patriarche'] and data[idx + 1] not in list_ope:
                    list_ope.append(data[idx + 1])
                    feature.SetField(col_names[idx], str(data[idx + 1]))
                # Gestion of the information linked to the geometry
                elif any(elem in table_cols[idx][0] for elem in ['_x', '_y']):
                    feature.SetField(col_names[idx], str(data[idx + 1]).split(' & ')[-1])
                elif '_point_2d' in table_cols[idx][0]:
                    feature.SetField(col_names[idx], str(data[idx + 1]).split(';')[-1])
                else:
                    # +1 because the first value in the attributes is ''
                    feature.SetField(col_names[idx], str(data[idx + 1]))
        # Recuperation of the information of the searched Sites
        ArchaeologicalSite = apps.get_model("archaeological_operations", "ArchaeologicalSite")
        try:
            # First version with the name of the Site
            id_label = table_cols.index(['name'])
            label = data[id_label + 1]
            object = list(ArchaeologicalSite.objects.filter(name=label))[0]
        except:
            # Second version with the reference of the Site
            id_label = table_cols.index(['reference'])
            label = data[id_label + 1]
            object = list(ArchaeologicalSite.objects.filter(reference=label))[0]
        # Completion of the list of Operations
        try:
            ope = str(list(object.operations.iterator())).split(' | OA')[-1].replace('>]', '')
            if ope not in list_ope:
                list_ope.append(ope)
            else:
                pass
        except:
            pass
        try:
            # Creating the geometry from the coordinates X and Y
            point = ogr.Geometry(ogr.wkbPoint)
            point.AddPoint(float(object.main_geodata.x), float(object.main_geodata.y))
        except:
            try:
                # Extreme case if no coordinates but the WKT exists
                point = ogr.CreateGeometryFromWkt(str(object.main_geodata.point_2d).split(';')[-1])
            except:
                pass
        if point != '':
            feature.SetGeometry(point)
        # Addition of the new feature
        layer.CreateFeature(feature)
        feature = None
    Operation = apps.get_model("archaeological_operations", "Operation")
    ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord')
    # Completion of the list of Context_Records linked to the extracted Sites
    list_cr = []
    for name in list_ope:
        ope, __ = Operation.objects.get_or_create(code_patriarche=name)
        search = ContextRecord.objects.filter(operation=ope)
        for cr in search:
            if cr.label not in list_cr:
                list_cr.append(cr.label)
    return list_ope, list_cr


def attributes_creation_sites_default(layer, list_crea):
    """
   :param layer: Sites layer from the linked geopackage
   :param list_crea: Name of the columns by default
   :function: Population of the Sites layer using default attributes
   :return layer: Sites layer with attributes
   """
    # Gestion of specific types of attributes for the default values
    for attribute in list_crea:
        if attribute in ['X', 'Y']:
            layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTReal))
        else:
            layer.CreateField(ogr.FieldDefn(attribute, ogr.OFTString))
    return layer


def populating_layer_sites_default(layer, list_crea, table_cols, datas):
    """
    :param layer: Sites layer from the linked geopackage with attributes
    :param list_crea: Name of the columns by default
    :param table_cols: List of the columns used in the query
    :param datas: Data from the query
    :function: Population of the Sites layer using all the data from a specific query
    :return list_ope: List of all the Operations linked to the Sites from the query
    """
    list_ope = []
    list_cr = []
    # Looping on all the datas to get the code_patriarche of the Operations
    for data in datas:
        idx = 0
        for col in table_cols:
            if 'code_patriarche' in col[0]:
                list_ope.append(data[idx + 1])
            idx += 1
    Operation = apps.get_model("archaeological_operations", "Operation")
    ArchaeologicalSite = apps.get_model("archaeological_operations", "ArchaeologicalSite")
    for name in list_ope:
        # Recuperation of the information of the searched Sites
        ope, __ = Operation.objects.get_or_create(code_patriarche=name)
        sites = list(ArchaeologicalSite.objects.filter(operations=ope.id))
        for site in sites:
            list_attributes = []
            point = ''
            try:list_attributes.append(site.reference)
            except:list_attributes.append('')
            try:list_attributes.append(site.name)
            except:list_attributes.append('')
            try:list_attributes.append(name)
            except:list_attributes.append('')
            try:list_attributes.append(str(site.cached_towns_label).split('(')[-1].replace(')',''))
            except:list_attributes.append('')
            try:list_attributes.append(site.cached_periods)
            except:list_attributes.append('')
            try:list_attributes.append(site.cached_remains)
            except:list_attributes.append('')
            try:list_attributes.append(site.main_geodata.x)
            except:list_attributes.append('')
            try:list_attributes.append(site.main_geodata.y)
            except:list_attributes.append('')
            try:list_attributes.append(site.comment)
            except:list_attributes.append('')
            if str(site.point) == 'None':
                list_attributes.append('Point ({} {})'.format(site.main_geodata.x, site.main_geodata.y))
            else:
                list_attributes.append(str(site.point).split(';')[-1])
            # Creation of a new feature
            feature = ogr.Feature(layer.GetLayerDefn())
            for idx in range(0, len(list_attributes)):
                if idx in [6, 7]:
                    # Gestion of the coordinates
                    try:
                        feature.SetField(list_crea[idx], float(list_attributes[idx]))
                    except:
                        pass
                else:
                    feature.SetField(list_crea[idx], str(list_attributes[idx]))
            try:
                # Creating the geometry from the coordinates X and Y
                point = ogr.CreateGeometryFromWkt(str(site.main_geodata.point).split(';')[-1])
            except:
                try:
                    # Extreme case if no coordinates but the WKT exists
                    point = ogr.Geometry(ogr.wkbPoint)
                    point.AddPoint(float(site.main_geodata.x), float(site.main_geodata.y))
                except:
                    pass
            if point != '':
                feature.SetGeometry(point)
            layer.CreateFeature(feature)
            feature = None
            # Completion of the list of Context_Records linked to the extracted Sites
            ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord')
            search = ContextRecord.objects.filter(operation=ope)
            for cr in search:
                if cr.label not in list_cr:
                    list_cr.append(cr.label)
    return list_ope, list_cr


def format_identification(default_value, text):
    """
    :param default_value: Name of the default value of the targeted list
    :param text: Text version of the .qgs
    :function: Find the list entry to modify, and get the good order of element to be sure of the modification
    :return old_text: List entry to modify, with arguments in the good order, or Fail if not found
    """
    # All parameters of an entry in a list for a QGIS attribute
    bricks = ['value="{}"'.format(default_value), 'name="{}"'.format(default_value), 'type="QString"']
    old_text = ''
    # Getting all the configurations of attributes to be sure to find it in the .qgs
    tests = list(permutations(bricks, 3))
    # Trying all configurations, and preparing the good one
    for test in tests:
        if ' '.join(test) in text:
            old_text = '                  <Option type="Map">\n                    <Option {}/>\n                  </Option>\n'.format(' '.join(test))
            return old_text
    if old_text == '':
        return 'Fail'


def list_recuperation(col, old_name, default_value, base, model, table_cols, col_names, text):
    """
    :param col: Column that will have a list
    :param old_name: Default_name of the attribute in the .qgs
    :param default_value: Name of the default value of the targeted list
    :param base: Name of the database to query
    :param model: Name of the specific model to query
    :param table_cols: List of the columns used in the query
    :param col_names: Name of the columns in the new layer
    :param text: Text version of the .qgs
    :function: Get all possible value for specific attributes, and placing them in lists in the .qgs
    :return text: Modified QGIS project
    """
    # Replacing the default name of the attribute with a list
    idx = table_cols.index(col)
    new = col_names[idx]
    text = text.replace(old_name, new)
    # Recuperation of the different entries linked to the searched attribute
    model = apps.get_model(base, model)
    query = model.objects.all()
    # Recuperation of the entry to modify, with arguments in the good order
    old_text = format_identification(default_value, text)
    if old_text != 'Fail':
        new_text = ''
        # Preparing all the entries for the list and adding to the .qgs
        for elem in query:
            choice = '                  <Option type="Map">\n                    <Option name="{}" value="{}" type="QString"/>\n                  </Option>\n'.format(
                elem, elem)
            new_text += choice
        text = text.replace(old_text, new_text)
    return text


def modification_style(qgs_path, table_cols, col_names, list_ope, list_crea):
    """
    :param qgs_path: Path to the QGIS project, containing the layers style
    :param table_cols: List of the columns used in the query to spot specific ones
    :param col_names: Name of the columns in the new layer to add their name to the style of the layer
    :param list_ope: List of the Operations linked to the entities from the query, to add them as a list
    :param list_crea: List of created attributes for the Finds or Context_Records layers
    :function: Modification of the QGIS project style to assure the autocompletion/automations for some attributes
    :return text: Modified QGIS project
    """
    # Lists of default names in the style, attribut names of the datas and new default names
    list_ref = ['finds_date', 'finds_time', 'finds_x', 'finds_y', 'finds_z', 'finds_cr', 'finds_parcel',
                'cr_operation', 'cr_insee', 'cr_section', 'cr_parcel', 'cr_full_parcel','cr_wkt']
    list_search = ['_date', '_datetime', '__x', '__y', '__z', 'context_record__label', 'parcel__external_id',
                   'operation__code_patriarche', 'town__numero_insee', 'parcel__section', 'parcel__parcel_number',
                   'parcel__external_id', 'geodata__multi_polygon']
    # Opening of the style
    text = open(qgs_path, encoding='utf-8').read()
    # Adding the different Operations linked of the Contexts Records and/or Finds exported to a list of possible values
    old_text = format_identification("Default_value", text)
    if len(list_ope) > 0:
        new_text = ''
        for ope in list_ope:
            choice = '                  <Option type="Map">\n                    <Option name="{}" value="{}" type="QString"/>\n                  </Option>\n'.format(ope, ope)
            new_text += choice
        text = text.replace(old_text, new_text)
    else:
        text = text.replace("Test_choice", "Null")
    # Verification of all the possible attributes with lists, and application of modification if present
    if 'material_types' in table_cols:
        text = list_recuperation('material_types', 'finds_material', 'Default_material',
                                 "archaeological_finds", "MaterialType", table_cols, col_names, text)
    if 'conservatory_states' in table_cols:
        text = list_recuperation('conservatory_states', 'finds_conservation', 'Default_conserv',
                                 "archaeological_finds", "ConservatoryState", table_cols, col_names, text)
    if 'object_types' in table_cols:
        text = list_recuperation('object_types', 'finds_object', 'Default_object',
                                 "archaeological_finds", "ObjectType", table_cols, col_names, text)
    if ['unit'] in table_cols:
        text = list_recuperation(['unit'], 'cr_type', 'Default_unit',
                                 "archaeological_context_records", "Unit", table_cols, col_names, text)
    if ['activity'] in table_cols:
        text = list_recuperation(['activity'], 'cr_activity', 'Default_activity',
                                 "archaeological_context_records", "ActivityType", table_cols, col_names, text)
    if ['identifications'] in table_cols:
        text = list_recuperation(['identifications'], 'cr_identif', 'Default_identif',
                                 "archaeological_context_records", "IdentificationType", table_cols, col_names, text)
    if ['documentations'] in table_cols:
        text = list_recuperation(['documentations'], 'cr_doc', 'Default_doc',
                                 "archaeological_context_records", "DocumentationType", table_cols, col_names, text)
    if ['datings__period'] in table_cols:
        text = list_recuperation(['datings__period'], 'cr_periode', 'Default_periode',
                                 "archaeological_operations", "Period", table_cols, col_names, text)

    # Specifics modifications if the datas don't come from Finds
    if list_crea[0] == 'finds':
        for ref in list_ref:
            id_ref = list_ref.index(ref)
            for col in table_cols:
                if col != '' and list_search[id_ref] in col[0]:
                    id_new = table_cols.index(col)
                    new = col_names[id_new]
                    text = text.replace(ref, new)
        # List of corresponding default names in the style linked to the default names used for the Finds
        list_corr = ['finds_id', 'finds_cr', 'finds_date', 'finds_x', 'finds_y', 'finds_z', 'finds_material',
                     'finds_object', 'finds_description', 'finds_media', 'finds_comment', 'finds_wkt_modif', 'finds_parcel']
        # Gestion of the link between the Finds and Context Records layers
        id_label = table_cols.index(['label'])
        new = col_names[id_label]
        text = text.replace("cr_name", new)
        if ['documents__image'] in table_cols:
            id_media = table_cols.index(['documents__image'])
            # Gestion of the link between the Finds and Context Records layers
            new = col_names[id_media]
            text = text.replace("cr_media", new)
        # Modification of the default attributes with lists
        text = list_recuperation('finds_material', 'finds_material', 'Default_material',
                                "archaeological_finds", "MaterialType", list_corr, list_crea[1], text)
        text = list_recuperation('finds_object', 'finds_object', 'Default_object',
                                 "archaeological_finds", "ObjectType", list_corr, list_crea[1], text)
        # Replacement of the values from the default names used for the Finds
        n = 0
        for elem in list_crea[1]:
            old = list_corr[n]
            text = text.replace(old, elem)
            n += 1
    # Specifics modifications if the datas don't come from Context_Records
    elif list_crea[0] == 'cr':
        for ref in list_ref:
            id_ref = list_ref.index(ref)
            for col in table_cols:
                if col != '' and list_search[id_ref] in col:
                    id_new = table_cols.index(col)
                    new = col_names[id_new]
                    text = text.replace(ref, new)
        # List of corresponding default names in the style linked to the default names used for the Finds
        list_corr = ['cr_name', 'cr_operation', 'cr_insee', 'cr_type', 'cr_occupation', 'cr_description',
                     'cr_localisation', 'cr_doc', 'cr_media', 'cr_periode', 'cr_activity', 'cr_identif', 'cr_comment',
                     'cr_wkt', 'cr_full_parcel']
        # Test in case the all names of attributes are in lists
        try:
            id_label = table_cols.index(['label'])
        except:
            id_label = table_cols.index('label')
        # Gestion of the link between the Finds and Context Records layers
        new = col_names[id_label]
        text = text.replace('finds_id', new)
        if 'documents__image' in table_cols:
            try:
                id_media = table_cols.index(['documents__image'])
            except:
                id_media = table_cols.index('documents__image')
            # Gestion of the link between the Finds and Context Records layers
            new = col_names[id_media]
            text = text.replace("finds_media", new)
        # Modification of the default attributes with lists
        text = list_recuperation('cr_type', 'cr_type', 'Default_unit',
                                 "archaeological_context_records", "Unit", list_corr, list_crea[1], text)
        text = list_recuperation('cr_activity', 'cr_activity', 'Default_activity',
                                 "archaeological_context_records", "ActivityType", list_corr, list_crea[1], text)
        text = list_recuperation('cr_periode', 'cr_periode', 'Default_periode',
                                 "archaeological_operations", "Period", list_corr, list_crea[1], text)
        text = list_recuperation('cr_identif', 'cr_identif', 'Default_identif',
                                 "archaeological_context_records", "IdentificationType", list_corr, list_crea[1], text)
        text = list_recuperation('cr_doc', 'cr_doc', 'Default_doc',
                                 "archaeological_context_records", "DocumentationType", list_corr, list_crea[1], text)
        # Specific case to assure the good registration of the z coordinate
        if any('__point_3d' in elem for elem in table_cols):
            id_new = [i for i, element in enumerate(table_cols) if 'geodata__point_3d' in element][0]
            #id_new = table_cols.index('geodata__point_3d')
            if any('__z' in elem for elem in table_cols):
                ref = "finds_wkt_modif"
                new = col_names[id_new]
            else:
                ref = "finds_wkt_simple"
                new = col_names[id_new]
            text = text.replace(ref, new)
        # Replacement of the values from the default names used for the Context Records
        n = 0
        for elem in list_crea[1]:
            old = list_corr[n]
            text = text.replace(old, elem)
            n += 1
    else:
        pass
    return text


def modification_style_sites(qgs_path, table_cols, col_names, list_ope, list_crea):
    """
    :param qgs_path: Path to the QGIS project, containing the layers style
    :param table_cols: List of the columns used in the query to spot specific ones
    :param col_names: Name of the columns in the new layer to add their name to the style of the layer
    :param list_ope: List of the Operations linked to the entities from the query, to add them as a list
    :param list_crea: List of created attributes for the Finds and Context_Records layers
    :function: Modification of the QGIS project style to assure the autocompletion/automations for some attributes
    :return text: Modified QGIS project
    """
    # Lists of default names in the style, attribut names of the datas and new default names
    list_ref = ['sites_operation', 'sites_parcel', 'sites_insee', 'sites_x', 'sites_y', 'sites_wkt']
    list_search = ['operations__code_patriarche', 'parcel__external_id', 'towns__numero_insee', 'geodata__x',
                   'geodata__y', 'geodata__point_2d']
    # Opening of the style
    text = open(qgs_path, encoding='utf-8').read()
    # Adding the different Operations linked of the Contexts Records and/or Finds exported to a list of possible values
    old_text = format_identification("Default_value", text)
    # Adding the different Operations linked of the Contexts Records and/or Finds exported to a list of possible values
    if len(list_ope) > 0:
        new_text = ''
        for ope in list_ope:
            choice = '                  <Option type="Map">\n                    <Option name="{}" value="{}" type="QString"/>\n                  </Option>\n'.format(
                ope, ope)
            new_text += choice
        text = text.replace(old_text, new_text)
    else:
        text = text.replace("Test_choice", "Default")
    # Verification of all the possible attributes with lists, and application of modification if present
    if ['periods'] in table_cols:
        text = list_recuperation(['periods'], 'sites_periodes', 'Default_speriode',
                                 "archaeological_operations", "Period", table_cols, col_names, text)
    if ['remains'] in table_cols:
        text = list_recuperation(['remains'], 'sites_type', 'Default_rtype',
                                 "archaeological_operations", "RemainType", table_cols, col_names, text)
    # Only loop and modify the name of attributes if the query was done on the Sites, not the Operations
    # If from Operations, the first name of default columns is the Reference of the Site
    if list_crea[0] != 'Référence':
        for ref in list_ref:
            id_ref = list_ref.index(ref)
            for col in table_cols:
                if col != '' and list_search[id_ref] in col:
                    id_new = table_cols.index(col)
                    new = col_names[id_new]
                    text = text.replace(ref, new)
    # Default names of the common attributes to modify
    list_ref = ['cr_name', 'cr_operation', 'cr_insee', 'cr_type', 'cr_occupation', 'cr_description',
                'cr_localisation', 'cr_doc', 'cr_media', 'cr_periode', 'cr_activity', 'cr_identif', 'cr_comment',
                'cr_wkt', 'cr_full_parcel', 'finds_id', 'finds_cr', 'finds_date', 'finds_x', 'finds_y', 'finds_z',
                'finds_material', 'finds_object', 'finds_description', 'finds_media', 'find_comment', 'finds_wkt_modif',
                'finds_parcel']
    # Modification of the default attributes with lists
    if list_crea[0] == 'Référence':
        # Adding the names of the specific attributes for the sites
        list_ref = ['sites_reference', 'sites_name', 'sites_operation', 'sites_insee', 'sites_periodes', 'sites_type',
                    'sites_x', 'sites_y', 'sites_comment', 'sites_wkt', 'sites_parcel'] + list_ref
        text = list_recuperation('sites_periodes', 'sites_periodes', 'Default_speriode',
                                 "archaeological_operations", "Period", list_ref, list_crea, text)
        text = list_recuperation('sites_type', 'sites_type', 'Default_rtype',
                                 "archaeological_operations", "RemainType", list_ref, list_crea, text)
    text = list_recuperation('finds_material', 'finds_material', 'Default_material',
                             "archaeological_finds", "MaterialType", list_ref, list_crea, text)
    text = list_recuperation('finds_object', 'finds_object', 'Default_object',
                             "archaeological_finds", "ObjectType", list_ref, list_crea, text)
    text = list_recuperation('cr_type', 'cr_type', 'Default_unit',
                             "archaeological_context_records", "Unit", list_ref, list_crea, text)
    text = list_recuperation('cr_activity', 'cr_activity', 'Default_activity',
                             "archaeological_context_records", "ActivityType", list_ref, list_crea, text)
    text = list_recuperation('cr_identif', 'cr_identif', 'Default_identif',
                             "archaeological_context_records", "IdentificationType", list_ref, list_crea, text)
    text = list_recuperation('cr_doc', 'cr_doc', 'Default_doc',
                             "archaeological_context_records", "DocumentationType", list_ref, list_crea, text)
    text = list_recuperation('cr_periode', 'cr_periode', 'Default_periode',
                             "archaeological_operations", "Period", list_ref, list_crea, text)
    # Modification of the othe attributes names
    for id in range(0, len(list_crea)):
        text = text.replace(list_ref[id], list_crea[id])
    return text