From e0e2cc9c916a122cc6e810620bbc5fda974f41bd Mon Sep 17 00:00:00 2001 From: Thomas André Date: Tue, 10 Jun 2025 11:54:22 +0200 Subject: Corrections in the importation of SIG data + Application of modifications and optimisation for the exportation of SIG data --- ishtar_common/models_imports.py | 4 +- ishtar_common/qfield/model/Context_records.gpkg | Bin 106496 -> 106496 bytes ishtar_common/qfield/model/Prospections.qgs | 2208 ++++++++++---------- .../qfield/model/Prospections_attachments.zip | Bin 1124 -> 1124 bytes ishtar_common/tests.py | 16 +- ishtar_common/views_item.py | 48 +- 6 files changed, 1191 insertions(+), 1085 deletions(-) diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 2a7039d51..02a00c469 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -2346,7 +2346,6 @@ class Import(BaseImport): return last_column = max(col_numbers) filename = ".".join(imported_file_path.split('.')[:-1]) + f"-{random.randint(1, 10000):05d}.csv" - try: data.to_csv(filename, index=False, columns=data.columns[range(last_column)], float_format=np_format_int_float) @@ -2674,7 +2673,8 @@ class Import(BaseImport): shutil.rmtree(tmp_dir) if imported_values.endswith(".gpkg"): print(data) - del data[0][0] + for elem in data: + del elem[0] print(data) return data diff --git a/ishtar_common/qfield/model/Context_records.gpkg b/ishtar_common/qfield/model/Context_records.gpkg index 0d00f6158..9bca2656c 100644 Binary files a/ishtar_common/qfield/model/Context_records.gpkg and b/ishtar_common/qfield/model/Context_records.gpkg differ diff --git a/ishtar_common/qfield/model/Prospections.qgs b/ishtar_common/qfield/model/Prospections.qgs index 4e042d631..5b54d86be 100644 --- a/ishtar_common/qfield/model/Prospections.qgs +++ b/ishtar_common/qfield/model/Prospections.qgs @@ -1,5 +1,5 @@ - + @@ -30,48 +30,48 @@ false - + - + - + - + - + - + - + - + - + @@ -83,15 +83,15 @@ Ortho_20_cm_f8497a68_e819_4170_a2ce_17cc699ac2d2 Parcelles_b677c8a6_df4d_4e0a_9f9f_42287fd8263e Communes_4866eaff_96f1_4af8_820c_6a6e1c2a757b - Context_records_94782d53_b0c1_4448_9720_44f58c1a13fa + Context_records_321d8648_b561_45da_9000_6df5d7562f17 - + - - - - + + + + @@ -99,10 +99,10 @@ degrees - -4.78929591895689732 - 43.59536848099315165 - 8.23398533104295716 - 49.74895054059680177 + -5.01831950074398492 + 42.81767541344365924 + 10.32258959016502331 + 50.06635116171296573 0 @@ -123,43 +123,43 @@ - + - + - - + + - + - - + + - + - + Annotations_61e154b2_8160_463f_af4c_71565a806b89 @@ -220,7 +220,7 @@ - + Communes_4866eaff_96f1_4af8_820c_6a6e1c2a757b pagingEnabled='default' preferCoordinatesForWfsT11='false' restrictToRequestBBOX='1' srsname='EPSG:4326' typename='ADMINEXPRESS-COG.2024:commune' url='https://data.geopf.fr/wfs/ows' url='https://data.geopf.fr/wfs/ows?VERSION=2.0.0' version='auto' @@ -282,173 +282,173 @@ 1 0 - + - + - + - - - + - - - + - - - + - + - - - + - - - @@ -525,17 +525,17 @@ 0 0 1 - + - + @@ -669,17 +669,17 @@ - - - - - - - - - - - + + + + + + + + + + + @@ -698,19 +698,19 @@ - + - @@ -756,38 +756,38 @@ def my_form_open(dialog, layer, feature): - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + "nom" - - Context_records_94782d53_b0c1_4448_9720_44f58c1a13fa + + Context_records_321d8648_b561_45da_9000_6df5d7562f17 ./Context_records.gpkg|layername=Context_records @@ -840,7 +840,7 @@ def my_form_open(dialog, layer, feature): - + @@ -865,173 +865,173 @@ def my_form_open(dialog, layer, feature): 1 0 - + - + - + - - - + - - - + - - - + - + - - - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - @@ -1107,17 +1198,17 @@ def my_form_open(dialog, layer, feature): 0 0 1 - + - + @@ -1140,7 +1231,7 @@ def my_form_open(dialog, layer, feature): @@ -1151,8 +1242,8 @@ def my_form_open(dialog, layer, feature): @@ -1161,8 +1252,8 @@ def my_form_open(dialog, layer, feature): @@ -1170,7 +1261,10 @@ def my_form_open(dialog, layer, feature): - @@ -1178,8 +1272,8 @@ def my_form_open(dialog, layer, feature): @@ -1202,20 +1296,20 @@ def my_form_open(dialog, layer, feature): - @@ -1242,6 +1336,16 @@ def my_form_open(dialog, layer, feature): + + + + + + + @@ -1257,6 +1361,7 @@ def my_form_open(dialog, layer, feature): + @@ -1272,6 +1377,7 @@ def my_form_open(dialog, layer, feature): + @@ -1287,13 +1393,14 @@ def my_form_open(dialog, layer, feature): + - - + + @@ -1302,21 +1409,23 @@ def my_form_open(dialog, layer, feature): + - - - - - - - - - - - - - + + + + + + + + + + + + + + @@ -1332,26 +1441,28 @@ def my_form_open(dialog, layer, feature): + - + - @@ -1398,6 +1509,7 @@ def my_form_open(dialog, layer, feature): + @@ -1413,69 +1525,71 @@ def my_form_open(dialog, layer, feature): - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + "description" - + Finds_7595ca92_4128_456d_a700_416cedf9aaa4 ./Finds.gpkg|layername=Finds @@ -1529,7 +1643,7 @@ def my_form_open(dialog, layer, feature): - + @@ -1554,181 +1668,181 @@ def my_form_open(dialog, layer, feature): 1 0 - + - + - + - - - + - - - + - - - + - + - - - + - - - 0 0 1 - + - + @@ -1835,8 +1949,8 @@ def my_form_open(dialog, layer, feature): @@ -1845,12 +1959,12 @@ def my_form_open(dialog, layer, feature): @@ -1859,12 +1973,12 @@ def my_form_open(dialog, layer, feature): @@ -1873,8 +1987,8 @@ def my_form_open(dialog, layer, feature): @@ -1883,8 +1997,8 @@ def my_form_open(dialog, layer, feature): @@ -1893,8 +2007,8 @@ def my_form_open(dialog, layer, feature): @@ -1903,20 +2017,20 @@ def my_form_open(dialog, layer, feature): - @@ -1926,8 +2040,8 @@ def my_form_open(dialog, layer, feature): @@ -1936,8 +2050,8 @@ def my_form_open(dialog, layer, feature): @@ -1946,8 +2060,8 @@ def my_form_open(dialog, layer, feature): @@ -2006,17 +2120,17 @@ def my_form_open(dialog, layer, feature): - - - - - - - - - - - + + + + + + + + + + + @@ -2035,19 +2149,19 @@ def my_form_open(dialog, layer, feature): - + - @@ -2096,43 +2210,43 @@ def my_form_open(dialog, layer, feature): - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + "champ_id" - + -20037508.34278924390673637 -20037508.34278924763202667 @@ -2173,7 +2287,7 @@ def my_form_open(dialog, layer, feature): Tuiles OpenStreetMap OpenStreetMap est construit par une communauté de cartographes qui contribuent et maintiennent des données sur les routes, les sentiers, les cafés, les gares, et bien plus encore, dans le monde entier. - + @@ -2195,12 +2309,12 @@ def my_form_open(dialog, layer, feature): - + wms - + @@ -2212,97 +2326,97 @@ def my_form_open(dialog, layer, feature): 0 0 - + - + - + - - - + - - - + - + None @@ -2349,14 +2463,14 @@ def my_form_open(dialog, layer, feature): 2 - - + + resamplingFilter 0 - + -20037508.34278924390673637 -15538711.09630922041833401 @@ -2416,7 +2530,7 @@ def my_form_open(dialog, layer, feature): wms - + @@ -2428,97 +2542,97 @@ def my_form_open(dialog, layer, feature): 1 0 - + - + - + - - - + - - - + - + None @@ -2554,14 +2668,14 @@ def my_form_open(dialog, layer, feature): 2 - - + + resamplingFilter 0 - + Parcelles_b677c8a6_df4d_4e0a_9f9f_42287fd8263e pagingEnabled='default' preferCoordinatesForWfsT11='false' restrictToRequestBBOX='1' srsname='EPSG:4326' typename='BDPARCELLAIRE-VECTEUR_WLD_BDD_WGS84G:parcelle' url='https://data.geopf.fr/wfs/ows' url='https://data.geopf.fr/wfs/ows?VERSION=2.0.0' version='auto' @@ -2623,173 +2737,173 @@ def my_form_open(dialog, layer, feature): 1 0 - + - + - + - - - + - - - + - - - + - + - - - + - - - @@ -2866,17 +2980,17 @@ def my_form_open(dialog, layer, feature): 0 0 1 - + - + @@ -2977,14 +3091,14 @@ def my_form_open(dialog, layer, feature): - - - - - - - - + + + + + + + + @@ -3000,19 +3114,19 @@ def my_form_open(dialog, layer, feature): - + - @@ -3058,30 +3172,30 @@ def my_form_open(dialog, layer, feature): - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + @@ -3095,7 +3209,7 @@ def my_form_open(dialog, layer, feature): - + @@ -3155,9 +3269,9 @@ def my_form_open(dialog, layer, feature): @@ -3181,9 +3295,9 @@ def my_form_open(dialog, layer, feature): - + - + GEOGCRS["WGS 84",ENSEMBLE["World Geodetic System 1984 ensemble",MEMBER["World Geodetic System 1984 (Transit)"],MEMBER["World Geodetic System 1984 (G730)"],MEMBER["World Geodetic System 1984 (G873)"],MEMBER["World Geodetic System 1984 (G1150)"],MEMBER["World Geodetic System 1984 (G1674)"],MEMBER["World Geodetic System 1984 (G1762)"],MEMBER["World Geodetic System 1984 (G2139)"],ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]],ENSEMBLEACCURACY[2.0]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],USAGE[SCOPE["Horizontal component of 3D system."],AREA["World."],BBOX[-90,-180,90,180]],ID["EPSG",4326]] +proj=longlat +datum=WGS84 +no_defs @@ -3197,10 +3311,10 @@ def my_form_open(dialog, layer, feature): - + - + @@ -3210,27 +3324,27 @@ def my_form_open(dialog, layer, feature): @@ -3248,7 +3362,7 @@ def my_form_open(dialog, layer, feature): - + diff --git a/ishtar_common/qfield/model/Prospections_attachments.zip b/ishtar_common/qfield/model/Prospections_attachments.zip index 5648d20d8..463d5b82d 100644 Binary files a/ishtar_common/qfield/model/Prospections_attachments.zip and b/ishtar_common/qfield/model/Prospections_attachments.zip differ diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 909380e77..3540f9cfe 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -2862,14 +2862,15 @@ class BaseImportTest(TestCase): # List of modified and new values to verify the absence/presence depending on the situation if mode == 1: old = ['champ_id', 'champ_ue', 'champ_x', 'champ_y', 'champ_z', 'champ_wkt_modif', 'champ_nom', 'champ_ope', - 'champ_insee', 'champ_parc', 'champ_type', 'champ_occup', 'champ_desc', 'champ_loca', 'champ_media_cr', - 'champ_periode', 'champ_acti', 'champ_geom'] + 'champ_insee', 'champ_section', 'champ_parc', 'champ_type', 'champ_occup', 'champ_desc', 'champ_loca', + 'champ_media_cr', 'champ_periode', 'champ_acti', 'champ_geom'] new = ['Identifiant', 'UE', 'X', 'Y', 'Z', 'WKT_point', 'Unité_Enregistrement', 'Opération', 'INSEE_Commune', - 'Parcelle', 'Type', 'Interprétation', 'Description', 'Localisation', 'Media', 'Periode', 'Type_Activité', 'WKT'] + 'Section', 'Parcelle', 'Type', 'Interprétation', 'Description', 'Localisation', 'Media', 'Periode', 'Type_Activité', 'WKT'] if mode == 2: - old = ['champ_id', 'champ_ue', 'champ_wkt_simple', 'champ_nom', 'champ_ope', 'champ_insee', 'champ_parc', - 'champ_type', 'champ_occup', 'champ_desc', 'champ_loca', 'champ_media_cr', 'champ_periode', 'champ_acti', 'champ_geom'] - new = ['Identifiant', 'UE', 'WKT_point', 'Unité_Enregistrement', 'Opération', 'INSEE_Commune', 'Parcelle', + old = ['champ_id', 'champ_ue', 'champ_wkt_simple', 'champ_nom', 'champ_ope', 'champ_insee', 'champ_section', + 'champ_parc', 'champ_type', 'champ_occup', 'champ_desc', 'champ_loca', 'champ_media_cr', + 'champ_periode', 'champ_acti', 'champ_geom'] + new = ['Identifiant', 'UE', 'WKT_point', 'Unité_Enregistrement', 'Opération', 'INSEE_Commune', 'Section', 'Parcelle', 'Type', 'Interprétation', 'Description', 'Localisation', 'Media', 'Periode', 'Type_Activité', 'WKT'] if mode == 3: old = ['champ_nom', 'champ_ope', 'champ_geom', 'champ_id', 'champ_ue', 'champ_date', 'champ_x', 'champ_y', @@ -2878,13 +2879,10 @@ class BaseImportTest(TestCase): with open(new_qgs, 'r', encoding='utf-8') as file: style = file.read() for elem in old: - print(mode, elem) bool = f'name="{elem}"' in style self.assertEqual(bool, False) for elem in new: bool = f'name="{elem}"' in style - if not bool: - print(elem) self.assertEqual(bool, True) for file in dir_list: os.remove(os.path.join(folder, file)) diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py index e697a2435..001a1437f 100644 --- a/ishtar_common/views_item.py +++ b/ishtar_common/views_item.py @@ -3295,7 +3295,7 @@ def gpkg_creation(root, table_cols, col_names, datas): if origin == 'finds': # 4a) Creation of the finds layer and its attributes layer = datasource.CreateLayer('Finds', srs, ogr.wkbPoint25D) - layer = attributes_creation_finds_a(layer, col_names, table_cols) + layer = attributes_creation_finds_manual(layer, col_names, table_cols) # 5a) Populating the finds layer with the datas list_cr = populating_layer_finds(layer,table_cols,col_names,datas) datasource = None @@ -3309,11 +3309,11 @@ def gpkg_creation(root, table_cols, col_names, datas): srs.ImportFromEPSG(4326) # 7a) Creation of the cr's layer and a list of default attrbutes layer = datasource.CreateLayer('Context_records', srs, ogr.wkbMultiPolygon) - list_crea = ['Unité_Enregistrement', 'Opération', 'INSEE_Commune', 'Parcelle', 'Type', 'Interprétation', + list_crea = ['Unité_Enregistrement', 'Opération', 'INSEE_Commune', 'Section', 'Parcelle', 'Type', 'Interprétation', 'Description', 'Localisation', 'Media', 'Periode', 'Type_Activité', 'WKT'] - layer = attributes_creation_cr_a(layer, list_crea) + layer = attributes_creation_cr_automatic(layer, list_crea) # 8a) Populating the cr's layer with datas from the cr of the extracted finds - list_ope = populating_layer_cr_a(layer, list_crea, list_cr) + list_ope = populating_layer_cr_automatic(layer, list_crea, list_cr) datasource = None # 9a) Preparation of a list of the attributes names for the style modifications list_crea = ['cr', list_crea] @@ -3321,7 +3321,7 @@ def gpkg_creation(root, table_cols, col_names, datas): # 4b) Creation of the finds layer and a list of default attrbutes layer = datasource.CreateLayer('Finds', srs, ogr.wkbPoint25D) list_crea = ['Identifiant', 'UE', 'Date', 'X', 'Y', 'Z', 'Matériaux', 'Description', 'Media', 'WKT_point'] - attributes_creation_finds_b(layer, list_crea) + attributes_creation_finds_automatic(layer, list_crea) datasource = None # 5b) Creation of the Context Records file cr = os.path.join(root, 'export', 'Context_records.gpkg') @@ -3333,9 +3333,9 @@ def gpkg_creation(root, table_cols, col_names, datas): srs.ImportFromEPSG(4326) # 6a) Creation of the cr's layer and its attributes layer = datasource.CreateLayer('Context_records', srs, ogr.wkbMultiPolygon) - layer = attributes_creation_cr_b(layer, col_names, table_cols) + layer = attributes_creation_cr_manual(layer, col_names, table_cols) # 7b) Populating the cr's layer with the datas - list_ope = populating_layer_cr_b(layer, table_cols, col_names, datas) + list_ope = populating_layer_cr_manual(layer, table_cols, col_names, datas) # 8b) Preparation of a list of the attributes names for the style modifications list_crea = ['finds', list_crea] return finds, cr, list_ope, list_crea @@ -3360,7 +3360,7 @@ def geometry_attribution (table_cols, datas): return origin -def attributes_creation_finds_a(layer, col_names, table_cols): +def attributes_creation_finds_manual(layer, col_names, table_cols): # print(table_cols) # debugtest # print(col_names) # debugtest # print(datas) # debugtest @@ -3444,7 +3444,7 @@ def populating_layer_finds(layer, table_cols, col_names, datas): return list_cr -def attributes_creation_finds_b(layer, list_crea): +def attributes_creation_finds_automatic(layer, list_crea): # Gestion of specific types of attributes for the default values for attribute in list_crea: if attribute == 'Date': @@ -3456,13 +3456,13 @@ def attributes_creation_finds_b(layer, list_crea): return layer -def attributes_creation_cr_a(layer, list_crea): +def attributes_creation_cr_automatic(layer, list_crea): for idx in range(0, len(list_crea)): layer.CreateField(ogr.FieldDefn(list_crea[idx], ogr.OFTString)) return layer -def populating_layer_cr_a(layer, list_crea, list_cr): +def populating_layer_cr_automatic(layer, list_crea, list_cr): list_ope = [] # Query in the DataBase to get information on the Context Records of the Finds exported ContextRecord = apps.get_model('archaeological_context_records', 'ContextRecord') @@ -3514,7 +3514,7 @@ def populating_layer_cr_a(layer, list_crea, list_cr): return list_ope -def attributes_creation_cr_b(layer, col_names, table_cols): +def attributes_creation_cr_manual(layer, col_names, table_cols): for idx in range(0, len(col_names)): if table_cols[idx] != '': # print(table_cols[idx]) # debugtest @@ -3523,7 +3523,7 @@ def attributes_creation_cr_b(layer, col_names, table_cols): return layer -def populating_layer_cr_b(layer, table_cols, col_names, datas): +def populating_layer_cr_manual(layer, table_cols, col_names, datas): #print(table_cols) #debugtest #print(col_names) #debugtest #print(datas) #debugtest @@ -3546,7 +3546,6 @@ def populating_layer_cr_b(layer, table_cols, col_names, datas): cr, __ = ContextRecord.objects.get_or_create( label=name ) - print(str(cr.multi_polygon)) try: geom = ogr.CreateGeometryFromWkt(str(cr.geodata.multi_polygon).split(';')[-1]) except: @@ -3564,11 +3563,11 @@ def populating_layer_cr_b(layer, table_cols, col_names, datas): def modification_style(qgs_path, table_cols, col_names, list_ope, list_crea): # Lists of default names in the style, attribut names of the datas and new default names list_ref = ['champ_date', 'champ_datetime', 'champ_x', 'champ_y', 'champ_z', 'champ_ue', - 'champ_ope', 'champ_insee', 'champ_parc', 'champ_geom'] + 'champ_ope', 'champ_insee', 'champ_section', 'champ_parc', 'champ_geom'] list_search = ['_date', '_datetime', 'base_finds__x', 'base_finds__y', 'base_finds__z', 'context_record__label', - 'operation__code_patriarche', 'town__numero_insee', 'parcel__external_id', + 'operation__code_patriarche', 'town__numero_insee', 'parcel__section', 'parcel__parcel_number', 'geodata__multi_polygon'] - list_auto = ['Date', 'Date/Temps', 'X', 'Y', 'Z', 'UE', 'Opération', 'INSEE_Commune', 'Parcelle', 'WKT'] + list_auto = ['Date', 'Date/Temps', 'X', 'Y', 'Z', 'UE', 'Opération', 'INSEE_Commune', 'Section', 'Parcelle', 'WKT'] # Opening of the style text = open(qgs_path, encoding='utf-8').read() # Adding the different Operations linked of the Contexts Records and/or Finds exported to a list of possible values @@ -3577,27 +3576,21 @@ def modification_style(qgs_path, table_cols, col_names, list_ope, list_crea): for ope in list_ope: choice = ' \n'.format(ope, ope) new_text += choice - old_text = ' \n' + old_text = ' \n' text = text.replace(old_text, new_text) else: text = text.replace("Test_choice", "Null") # Specifics modifications if the datas don't come from Finds if list_crea[0] == 'finds': - print(table_cols) for ref in list_ref: id_ref = list_ref.index(ref) new = '' for col in table_cols: - if list_search[id_ref] in col[0]: - print('A') + if col != '' and list_search[id_ref] in col[0]: id_new = table_cols.index(col) new = col_names[id_new] - print(col, new) if new == '': - print('B') new = list_auto[id_ref] - print(new) - print(ref, new) text = text.replace(ref, new) # List of corresponding default names in the style linked to the default names used for the Finds list_corr = ['champ_id', 'champ_ue', 'champ_date', 'champ_x', 'champ_y', 'champ_z', 'champ_matériaux', @@ -3623,13 +3616,14 @@ def modification_style(qgs_path, table_cols, col_names, list_ope, list_crea): id_ref = list_ref.index(ref) new = '' for col in table_cols: - if list_search[id_ref] in col: + if col != '' and list_search[id_ref] in col: id_new = table_cols.index(col) + new = col_names[id_new] if new == '': new = list_auto[id_ref] text = text.replace(ref, new) # List of corresponding default names in the style linked to the default names used for the Finds - list_corr = ['champ_nom', 'champ_ope', 'champ_insee', 'champ_parc', 'champ_type', 'champ_occup', 'champ_desc', + list_corr = ['champ_nom', 'champ_ope', 'champ_insee', 'champ_section', 'champ_parc', 'champ_type', 'champ_occup', 'champ_desc', 'champ_loca', 'champ_media_cr', 'champ_periode', 'champ_acti', 'champ_geom'] # Test in case the all names of attributes are in lists try: -- cgit v1.2.3