diff options
| -rw-r--r-- | archaeological_operations/tests.py | 35 | ||||
| -rw-r--r-- | archaeological_operations/tests/external_source_types_2.json | 1 | ||||
| -rw-r--r-- | ishtar_common/admin.py | 33 | ||||
| -rw-r--r-- | ishtar_common/migrations/0217_auto_20211018_1741.py (renamed from ishtar_common/migrations/0217_auto_20211015_1728.py) | 7 | ||||
| -rw-r--r-- | ishtar_common/models_rest.py | 75 |
5 files changed, 120 insertions, 31 deletions
diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index e58868c1f..74ae5819a 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -4629,8 +4629,39 @@ class ApiTest(OperationInitTest, APITestCase): mock_get.return_value.text = tpes.read() response = self.client.post(url, params, follow=True) - # create and update matches from an external source - # delete old matches! + result = {"created": "XXXXX"} + msg_created = str(_(f"{result['created']} matches created")).replace( + result["created"], "" + ) + self.assertIn(msg_created, response.content.decode()) + # test incoherent search_model and types + with open( + "../archaeological_operations/tests/external_source_types_2.json", "r" + ) as tpes: + mock_get.return_value.text = tpes.read() + response = self.client.post(url, params, follow=True) + + content = response.content.decode() + self.assertNotIn(msg_created, content) # no new creation + missing_search_models = "archaeological_operations.strangeoperation" + msg = str( + _( + f"Theses search models have not been found: {missing_search_models}. Not the same Ishtar version?" + ) + ).replace(result["created"], "") + self.assertIn(msg, content) + missing_types = "ishtar_common.oddtype" + msg = str( + _( + f"Theses types have not been found: {missing_types}. Not the same Ishtar version?" + ) + ) + self.assertIn(msg, content) + # delete old matches + result = {"deleted": 2} + msg = str(_(f"{result['deleted']} matches deleted")) + self.assertIn(msg, response.content.decode()) + pass def test_query_transformation(self): diff --git a/archaeological_operations/tests/external_source_types_2.json b/archaeological_operations/tests/external_source_types_2.json new file mode 100644 index 000000000..bc7b404e3 --- /dev/null +++ b/archaeological_operations/tests/external_source_types_2.json @@ -0,0 +1 @@ +{"archaeological_operations.operation":[["ishtar_common.oddtype",["oddtype","oddtype"],[]],["ishtar_common.operationtype",["type","type"],[["arch_diagnostic","Diagnostic archéologique"],["documents_study","Étude documentaire (préventif)"],["evaluation","Évaluation"],["ancient_excavation","Fouille ancienne"],["prev_excavation","Fouille archéologique préventive"],["emergency_excavation","Sauvetage urgent"],["unknown","À déterminer"],["assistance_preparation_help","Aide à la préparation de publication"],["AE","Aide à l'édition"],["arch_diagnostic_research","Diagnostic archéologique (programmé)"],["communication","Diffusion (pour les colloques, expo, séminaires...)"],["building_study_research","Étude de bâti (programmé)"],["documents_study_research","Étude documentaire (programmé)"],["prog_excavation","Fouille archéologique programmée"],["prog_excavation_multiyear","Fouille archéologique programmée pluriannuelle"],["analysis_program","Programme d'analyses"],["collective_research_project","Projet Collectif de Recherche"],["aerial_survey_research","Prospection aérienne"],["specialized_eqp_prospection","Prospection avec matériel spécialisé"],["rock_art_survey","Prospection avec relevé d'art rupestre"],["metal_detector_prospection","Prospection détecteur de métaux"],["inventory_survey_research","Prospection inventaire"],["survey_dig","Prospection sondage"],["thematic_survey","Prospection thématique"],["cave_art_record","Relevé d'art rupestre"],["sampling","Sondage (programmé)"]]],["archaeological_operations.reportstate",["traitement-rapport","report-processing"],[["not_received","Non reçu"],["received","Reçu et non dépouillé"],["received__not_processed","Reçus et non dépouillés"],["reports_received_partially_processed","Reçus, partiellement dépouillés"],["one_report_received_and_processed","Reçu et dépouillé"],["no-report","Rapport inexistant"]]],["archaeological_operations.remaintype",["vestige","remain"],[["aile","aile"],["aire-de-battage","aire de battage"],["amas","amas"],["amas-coquillier","amas coquillier"],["amas-de-debitage","amas de débitage"],["amenagement-indetermine","aménagement indéterminé"],["anomalie","anomalie"],["arc","arc"],["arcasse","arcasse"],["arche","arche"],["aula","aula"],["autel","autel"],["banquette","banquette"],["barbacane","barbacane"],["barrot","barrot"],["basse-cour","basse-cour"],["bassin","bassin"],["bastion","bastion"],["batiment","bâtiment"],["bloc","bloc"],["bloc-orne","bloc orné"],["bloc-ouvrage","bloc ouvragé"],["boisage","boisage"],["boniferie","boniferie"],["borde","bordé"],["borde-a-clin","bordé à clin"],["borde-a-franc-bord","bordé à franc-bord"],["borne","borne"],["butte","butte"],["cairn","cairn"],["calage","calage"],["calfatage","calfatage"],["calvaire","calvaire"],["canalisation","canalisation"],["cargaison","cargaison"],["cave","cave"],["cercueil","cercueil"],["charpente","charpente"],["castle","château"],["cloitre","cloître"],["cockpit","cockpit"],["colonnade","colonnade"],["colonne-element-de","colonne (élément de)"],["construction","construction"],["coque","coque"],["coque-borde-premier","coque bordé-premier"],["coque-membrure-premiere","coque membrure-première"],["couple","couple"],["cour","cour"],["courtine","courtine"],["croix","croix"],["crypte","crypte"],["cryptoportique","cryptoportique"],["cuve","cuve"],["cuvelage","cuvelage"],["dallage","dallage"],["demi-lune","demi-lune"],["depotoir","dépotoir"],["derive","dérive"],["doublage","doublage"],["church","église"],["empierrement","empierrement"],["emplanture","emplanture"],["empreinte","empreinte"],["enclos","enclos"],["enclosure","enclos"],["enclos-systeme-d","enclos (système d')"],["entree-amenagee","entrée aménagée"],["escalier","escalier"],["etambot","étambot"],["etrave","étrave"],["etrave-a-eperon","étrave à éperon"],["etuve","étuve"],["ferrier","ferrier"],["fondation","fondation"],["fosse","fosse"],["ditch","fossé"],["fosses-reseau-de","fossés (réseau de)"],["four","four"],["foyer","foyer"],["foyer-vidange-de","foyer (vidange de)"],["front-de-taille","front de taille"],["fuselage","fuselage"],["galerie","galerie"],["garenne","garenne"],["glaciere","glacière"],["gouvernail","gouvernail"],["gradins","gradins"],["grange","grange"],["grenier","grenier"],["helice","hélice"],["hypocauste","hypocauste"],["incineration","incinération"],["inhumation","inhumation"],["inscription","inscription"],["jardin","jardin"],["laraire","laraire"],["latrines","latrines"],["lest","lest"],["levee","levée"],["mat","mât"],["membrure","membrure"],["menhir","menhir"],["meule-fixe","meule fixe"],["mosaique","mosaïque"],["moteur","moteur"],["motte-castrale","motte castrale"],["moule-a-cloches","moule à cloches"],["mur","mur"],["niche","niche"],["niveau-doccupation","niveau d'occupation"],["oratoire","oratoire"],["orniere","ornière"],["ouvrage-avance","ouvrage avancé"],["palissade","palissade"],["paroi-ornee","paroi ornée"],["pavage","pavage"],["pieu","pieu"],["pigeonnier","pigeonnier"],["pile","pile"],["plafond","plafond"],["plancher","plancher"],["polissoir-fixe","polissoir fixe"],["pont-de-bateau","pont de bateau"],["porte","porte"],["portique","portique"],["preceinte","préceinte"],["pressoir","pressoir"],["puits","puits"],["purgerie","purgerie"],["quille","quille"],["recipient-fixe","récipient fixe"],["rempart","rempart"],["sarcophage","sarcophage"],["silo","silo"],["sol-doccupation","sol d'occupation"],["sol-orne","sol orné"],["source-amenagee","source aménagée"],["souterrain","souterrain"],["statue","statue"],["statue-menhir","statue-menhir"],["stele","stèle"],["talus","talus"],["tour","tour"],["trace-agraire","trace agraire"],["train-datterrissage","train d'atterrissage"],["tranchee-dextraction","tranchée d'extraction"],["trou-de-jauge","trou de jauge"],["trou-de-poteau","trou de poteau"],["trous-de-poteau-ensemble-de","trous de poteau (ensemble de)"],["tunnel","tunnel"],["urne","urne"],["vaigre","vaigre"],["verriere","verrière"],["villa","villa"],["virure","virure"],["zone-de-rejet","zone de rejet"]]],["archaeological_operations.period",["periode","period"],[["not-yet-documented","Non-renseigné"],["indeterminate","Époque indéterminée"],["recent-times","Période récente"],["contemporary","Époque contemporaine"],["modern","Époque moderne"],["middle-age","Moyen Âge"],["low-middle-age","Bas Moyen Âge"],["classic-middle-age","Moyen Âge classique"],["high-middle-age","Haut Moyen Âge"],["gallo-roman","Gallo-romain"],["low-empire","Bas Empire"],["high-empire","Haut Empire"],["republic","République"],["protohistory","Protohistoire"],["iron-age","Âge du Fer"],["second-iron-age","Deuxième Âge du Fer"],["first-iron-age","Premier Âge du Fer"],["bronze-age","Âge du Bronze"],["final-bronze-age","Âge du Bronze final"],["middle-bronze-age","Âge du Bronze moyen"],["old-bronze-age","Âge du Bronze ancien"],["neolithic","Néolithique"],["final-neolithic","Néolithique final"],["recent-neolithic","Néolithique récent"],["middle-neolithic","Néolithique moyen"],["old-neolithic","Néolithique ancien"],["mesolithic","Mésolithique"],["recent-mesolithic","Mésolithique récent"],["middle-mesolithic","Mésolithique moyen"],["old-mesolithic","Mésolithique ancien"],["paleolithic","Paléolithique"],["final-paleolithic","Paléolithique supérieur final"],["late-paleolithic","Paléolithique supérieur"],["middle-paleolithic","Paléolithique moyen"],["ancien-paleolithic","Paléolithique ancien"],["mediterranean","Méditerranéen"],["mediterranean-antiq","Antiquité méditerranéenne"],["late-antiq-med","Antiquité tardive (Méditerranée)"],["hellen-epoc","Époque hellénistique"],["classic-epoc","Époque classique"],["archaic-epoc","Époque archaïque"],["antilles","Antilles"],["pre-pottery-antilles","Précéramique (Antilles)"],["archaic-antilles","Archaïque (Antilles)"],["lithic-antilles","Lithique (Antilles)"],["pottery-antilles","Céramique (Antilles)"],["recent-post-saladoide","Post-saladoïde récent (Antilles)"],["old-post-saladoide","Post-saladoïde ancien (Antilles)"],["saladoide","Saladoïde (Antilles)"],["huecoide","Huécoïde (Antilles)"],["historic-antilles","Historique (Antilles)"],["contemporary-antilles","Période contemporaine (Antilles)"],["modern-colonial-antilles","Colonial moderne (Antilles)"],["precolonial-antilles","Pré-colonial (Antilles)"],["indeterminate-antilles","Époque indéterminée (Antilles)"],["guiana","Guyane"],["prehistory-guiana","Préhistoire (Guyane)"],["prepottery-guiana","Acéramique (Guyane)"],["amerindian-guiana","Période amérindienne (Guyane)"],["precolonial-guiana","Pré-colonial (Guyane)"],["pre-contact-guiana","Pré-contact (Guyane)"],["old-amerindian-guiana","Période amérindienne ancienne (Guyane)"],["colonial-guiana","Période coloniale et départementale (Guyane)"],["contemporary-guiana","Contemporain (Guyane)"],["asserted-colonisation-guiana","Colonisation affirmée (Guyane)"],["colonisation-guiana","Période de colonisation (Guyane)"],["indeterminate-guiana","Époque indéterminée (Guyane)"]]],["archaeological_operations.recordqualitytype",["qualite-enregistrement","record-quality"],[["not-documented","Non documenté"],["arbitrary","Arbitraire"],["reliable","Fiable"]]],["archaeological_operations.relationtype",["type-relation","relation-types"],[["has_got","Comprend"],["is_in","Comprise dans"],["fuzzy_relation","Relation diffuse"]]],["ishtar_common.area",["zone","area"],[]]],"archaeological_operations.strangeoperation":[]}
\ No newline at end of file diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index 39d7de021..59dc99f15 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -2183,7 +2183,38 @@ def update_types_from_source(modeladmin, request, queryset): return_url ) result = source.update_matches(content) - print(result) + if result.get("created", None): + messages.add_message( + request, + messages.INFO, + str(_(f"{result['created']} matches created")), + ) + if result.get("updated", None): + messages.add_message( + request, + messages.INFO, + str(_(f"{result['updated']} matches updated")), + ) + if result.get("deleted", None): + messages.add_message( + request, + messages.INFO, + str(_(f"{result['deleted']} matches deleted")), + ) + if result.get("search_model do not exist", None): + missing_search_models = ", ".join(result['search_model do not exist']) + messages.add_message( + request, + messages.INFO, + str(_(f"Theses search models have not been found: {missing_search_models}. Not the same Ishtar version?")), + ) + if result.get("type do not exist", None): + missing_types = ", ".join(result['type do not exist']) + messages.add_message( + request, + messages.INFO, + str(_(f"Theses types have not been found: {missing_types}. Not the same Ishtar version?")), + ) return response diff --git a/ishtar_common/migrations/0217_auto_20211015_1728.py b/ishtar_common/migrations/0217_auto_20211018_1741.py index 47bfa7c6e..d79224b5a 100644 --- a/ishtar_common/migrations/0217_auto_20211015_1728.py +++ b/ishtar_common/migrations/0217_auto_20211018_1741.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.24 on 2021-10-15 17:28 +# Generated by Django 2.2.24 on 2021-10-18 17:41 from django.conf import settings import django.contrib.postgres.fields @@ -10,8 +10,8 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), ('auth', '0011_update_proxy_permissions'), + ('contenttypes', '0002_remove_content_type_name'), ('ishtar_common', '0216_auto_20210805_1703'), ] @@ -198,7 +198,8 @@ class Migration(migrations.Migration): ('local_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Local key')), ('local_label', models.TextField(blank=True, default='', verbose_name='Local value')), ('do_not_match', models.BooleanField(default=False, verbose_name='Disable match for this search')), - ('search_model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='Search model')), + ('associated_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='key_match_types', to='contenttypes.ContentType', verbose_name='Associated type')), + ('search_model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='key_match_search_models', to='contenttypes.ContentType', verbose_name='Search model')), ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiExternalSource')), ], options={ diff --git a/ishtar_common/models_rest.py b/ishtar_common/models_rest.py index a74df2c8a..566a73127 100644 --- a/ishtar_common/models_rest.py +++ b/ishtar_common/models_rest.py @@ -6,9 +6,7 @@ from django.contrib.postgres.fields import ArrayField from ishtar_common.utils import ugettext_lazy as _ -MAIN_CONTENT_TYPES = ( - ("archaeological_operations", "operation"), -) +MAIN_CONTENT_TYPES = (("archaeological_operations", "operation"),) class ApiUser(models.Model): @@ -29,8 +27,10 @@ class ApiSearchModel(models.Model): user = models.ForeignKey(ApiUser, on_delete=models.CASCADE) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) limit_query = models.TextField( - verbose_name=_("Limit query"), blank=True, null=True, - help_text=_("Search query add to each request") + verbose_name=_("Limit query"), + blank=True, + null=True, + help_text=_("Search query add to each request"), ) class Meta: @@ -55,7 +55,6 @@ class ApiExternalSource(models.Model): "search_model do not exist": [], "type do not exist": [], } - updated = [] for search_model in content: app, model_name = search_model.split(".") try: @@ -66,17 +65,22 @@ class ApiExternalSource(models.Model): for ct_type, keys, values in content[search_model]: tapp, tmodel_name = ct_type.split(".") try: - ct_type = ContentType.objects.get( - app_label=tapp, model=tmodel_name - ) + ct_type = ContentType.objects.get(app_label=tapp, model=tmodel_name) except ContentType.DoesNotExist: - result["type do not exist"].append(search_model) + result["type do not exist"].append(ct_type) continue t_model = ct_type.model_class() + current_matches = [] for slug, label in values: + current_matches.append(slug) m, created = ApiKeyMatch.objects.get_or_create( - source=self, search_model=ct, search_keys=keys, - distant_slug=slug, defaults={"distant_label": label}) + source=self, + search_model=ct, + associated_type=ct_type, + search_keys=keys, + distant_slug=slug, + defaults={"distant_label": label}, + ) updated = False if not created and m.distant_label != label: updated = True @@ -97,25 +101,46 @@ class ApiExternalSource(models.Model): result["updated"] += 1 if created: result["created"] += 1 + # delete removed keys + q = ApiKeyMatch.objects.filter( + source=self, search_model=ct, associated_type=ct_type, + ).exclude(distant_slug__in=current_matches) + result["deleted"] += q.count() + q.delete() return result + def generate_match_csv(self): + pass + class ApiKeyMatch(models.Model): source = models.ForeignKey(ApiExternalSource, on_delete=models.CASCADE) - search_model = models.ForeignKey(ContentType, on_delete=models.CASCADE, - verbose_name=_("Search model")) - search_keys = ArrayField(models.CharField(max_length=200), - verbose_name=_("Search keys"), blank=True) - distant_slug = models.SlugField(verbose_name=_("Distant key"), max_length=200, - allow_unicode=True) - distant_label = models.TextField(verbose_name=_("Distant value"), blank=True, - default="") - local_slug = models.SlugField(verbose_name=_("Local key"), max_length=200, - allow_unicode=True) - local_label = models.TextField(verbose_name=_("Local value"), blank=True, - default="") + search_model = models.ForeignKey( + ContentType, on_delete=models.CASCADE, verbose_name=_("Search model"), + related_name="key_match_search_models" + ) + associated_type = models.ForeignKey( + ContentType, on_delete=models.CASCADE, verbose_name=_("Associated type"), + related_name="key_match_types" + ) + search_keys = ArrayField( + models.CharField(max_length=200), verbose_name=_("Search keys"), blank=True + ) + distant_slug = models.SlugField( + verbose_name=_("Distant key"), max_length=200, allow_unicode=True + ) + distant_label = models.TextField( + verbose_name=_("Distant value"), blank=True, default="" + ) + local_slug = models.SlugField( + verbose_name=_("Local key"), max_length=200, allow_unicode=True + ) + local_label = models.TextField( + verbose_name=_("Local value"), blank=True, default="" + ) do_not_match = models.BooleanField( - verbose_name=_("Disable match for this search"), default=False) + verbose_name=_("Disable match for this search"), default=False + ) class Meta: verbose_name = _("API - Key match") |