From 3221ede6e709fc680e1bebebcfc334cc46e889ca Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Fri, 15 Oct 2021 17:54:03 +0200 Subject: Syndication - initialize type match with a source --- archaeological_operations/tests.py | 97 ++++++++-- .../tests/external_source_types_1.json | 1 + ishtar_common/admin.py | 75 +++++++- .../migrations/0217_auto_20211013_1517.py | 208 -------------------- .../migrations/0217_auto_20211015_1728.py | 209 +++++++++++++++++++++ ishtar_common/models_rest.py | 54 ++++++ ishtar_common/rest.py | 8 +- 7 files changed, 425 insertions(+), 227 deletions(-) create mode 100644 archaeological_operations/tests/external_source_types_1.json delete mode 100644 ishtar_common/migrations/0217_auto_20211013_1517.py create mode 100644 ishtar_common/migrations/0217_auto_20211015_1728.py diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index d13a21011..e58868c1f 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -23,8 +23,10 @@ from subprocess import Popen, PIPE from io import StringIO, BytesIO import tempfile import locale +from unittest.mock import patch import zipfile +import requests from django.apps import apps from django.conf import settings from django.contrib.auth.models import Group @@ -44,6 +46,8 @@ from rest_framework.authtoken.models import Token from . import models +from ishtar_common import models_rest +from ishtar_common.admin import update_types_from_source from ishtar_common.views import document_deletion_steps from ishtar_common.serializers import document_serialization from archaeological_operations import views, serializers @@ -4452,13 +4456,12 @@ class ApiTest(OperationInitTest, APITestCase): self.auth_token = "Token " + Token.objects.create(user=self.user).key self.api_user = ApiUser.objects.create(user_ptr=self.user, ip="127.0.0.1") - def create_api_search_model(self, app_label="archaeological_operations", - model="operation"): + def create_api_search_model( + self, app_label="archaeological_operations", model="operation" + ): return ApiSearchModel.objects.create( user=self.api_user, - content_type=ContentType.objects.get( - app_label=app_label, model=model - ), + content_type=ContentType.objects.get(app_label=app_label, model=model), ) def test_api_permissions(self): @@ -4531,19 +4534,24 @@ class ApiTest(OperationInitTest, APITestCase): self.create_api_search_model() url = reverse("api-facets-operation") response = self.client.get( - url, format="json", HTTP_AUTHORIZATION=self.auth_token, + url, + format="json", + HTTP_AUTHORIZATION=self.auth_token, ) self.assertEqual(response.status_code, 200) j = json.loads(response.content.decode()) - self.assertIn('archaeological_operations.operation', j) + self.assertIn("archaeological_operations.operation", j) # no permissions for archaeological_site - self.assertNotIn('archaeological_operations.archaeological_site', j) + self.assertNotIn("archaeological_operations.archaeological_site", j) has_type = False - for content in j['archaeological_operations.operation']: - if content[0] == ["type", "type"]: + for content in j["archaeological_operations.operation"]: + if content[0] == "ishtar_common.operationtype" and content[1] == [ + "type", + "type", + ]: has_type = True - lst = content[1] + lst = content[2] for tpe in models.OperationType.objects.filter(available=True).all(): self.assertIn([tpe.txt_idx, tpe.label], lst) # do not send not available @@ -4552,18 +4560,75 @@ class ApiTest(OperationInitTest, APITestCase): break self.assertTrue(has_type) - self.create_api_search_model(app_label="archaeological_operations", - model="archaeologicalsite") + self.create_api_search_model( + app_label="archaeological_operations", model="archaeologicalsite" + ) url = reverse("api-facets-operation") response = self.client.get( - url, format="json", HTTP_AUTHORIZATION=self.auth_token, + url, + format="json", + HTTP_AUTHORIZATION=self.auth_token, ) self.assertEqual(response.status_code, 200) j = json.loads(response.content.decode()) - self.assertIn('archaeological_operations.archaeologicalsite', j) + self.assertIn("archaeological_operations.archaeologicalsite", j) - def test_type_admin(self): + def _get_source(self): + src, __ = models_rest.ApiExternalSource.objects.get_or_create( + url="http://localhost", name="Default source", key="xxx" + ) + return src + + @patch("requests.get") + def test_type_admin(self, mock_get): # POV: local + # receive from a source + + source = self._get_source() + url = "/admin/{}/{}/".format( + "ishtar_common", models_rest.ApiExternalSource.__name__.lower() + ) + params = { + "action": "update_types_from_source", + "_selected_action": [source.pk], + } + response = self.client.post(url, params) + self.assertEqual( + response["Location"], + "/admin/login/?next=/admin/ishtar_common/apiexternalsource/", + ) + + mock_get.return_value.status_code = 404 + self.client.login(username=self.username, password=self.password) + response = self.client.post(url, params, follow=True) + self.assertEqual( + response.status_code, + 200, + ) + self.assertIn( + "404 != 200", + response.content.decode(), + ) + mock_get.return_value.status_code = 200 + + def __json(): + return json.loads(mock_get.return_value.text) + + mock_get.return_value.json = __json + mock_get.return_value.text = "NOK" + + response = self.client.post(url, params, follow=True) + self.assertIn( + str(_("is not a valid JSON message")), + response.content.decode(), + ) + + with open( + "../archaeological_operations/tests/external_source_types_1.json", "r" + ) as tpes: + mock_get.return_value.text = tpes.read() + response = self.client.post(url, params, follow=True) + # create and update matches from an external source # delete old matches! pass diff --git a/archaeological_operations/tests/external_source_types_1.json b/archaeological_operations/tests/external_source_types_1.json new file mode 100644 index 000000000..327ad35f2 --- /dev/null +++ b/archaeological_operations/tests/external_source_types_1.json @@ -0,0 +1 @@ +{"archaeological_operations.operation":[["ishtar_common.operationtype",["type","type"],[["arch_diagnostic","Diagnostic archéologique"],["building_study","Étude de bâti (préventif)"],["documents_study","Étude documentaire (préventif)"],["evaluation","Évaluation"],["ancient_excavation","Fouille ancienne"],["prev_excavation","Fouille archéologique préventive"],["emergency_excavation","Sauvetage urgent"],["unknown","À déterminer"],["assistance_preparation_help","Aide à la préparation de publication"],["AE","Aide à l'édition"],["arch_diagnostic_research","Diagnostic archéologique (programmé)"],["communication","Diffusion (pour les colloques, expo, séminaires...)"],["building_study_research","Étude de bâti (programmé)"],["documents_study_research","Étude documentaire (programmé)"],["prog_excavation","Fouille archéologique programmée"],["prog_excavation_multiyear","Fouille archéologique programmée pluriannuelle"],["analysis_program","Programme d'analyses"],["collective_research_project","Projet Collectif de Recherche"],["aerial_survey_research","Prospection aérienne"],["specialized_eqp_prospection","Prospection avec matériel spécialisé"],["rock_art_survey","Prospection avec relevé d'art rupestre"],["metal_detector_prospection","Prospection détecteur de métaux"],["inventory_survey_research","Prospection inventaire"],["survey_dig","Prospection sondage"],["thematic_survey","Prospection thématique"],["cave_art_record","Relevé d'art rupestre"],["sampling","Sondage (programmé)"]]],["archaeological_operations.reportstate",["traitement-rapport","report-processing"],[["not_received","Non reçu"],["received","Reçu et non dépouillé"],["received__not_processed","Reçus et non dépouillés"],["reports_received_partially_processed","Reçus, partiellement dépouillés"],["one_report_received_and_processed","Reçu et dépouillé"],["received_and_processed","Reçus et dépouillés"],["no-report","Rapport inexistant"]]],["archaeological_operations.remaintype",["vestige","remain"],[["aile","aile"],["aire-de-battage","aire de battage"],["amas","amas"],["amas-coquillier","amas coquillier"],["amas-de-debitage","amas de débitage"],["amenagement-indetermine","aménagement indéterminé"],["anomalie","anomalie"],["arc","arc"],["arcasse","arcasse"],["arche","arche"],["aula","aula"],["autel","autel"],["banquette","banquette"],["barbacane","barbacane"],["barrot","barrot"],["basse-cour","basse-cour"],["bassin","bassin"],["bastion","bastion"],["batiment","bâtiment"],["bloc","bloc"],["bloc-orne","bloc orné"],["bloc-ouvrage","bloc ouvragé"],["boisage","boisage"],["boniferie","boniferie"],["borde","bordé"],["borde-a-clin","bordé à clin"],["borde-a-franc-bord","bordé à franc-bord"],["borne","borne"],["butte","butte"],["cairn","cairn"],["calage","calage"],["calfatage","calfatage"],["calvaire","calvaire"],["canalisation","canalisation"],["cargaison","cargaison"],["cave","cave"],["cercueil","cercueil"],["charpente","charpente"],["castle","château"],["cloitre","cloître"],["cockpit","cockpit"],["colonnade","colonnade"],["colonne-element-de","colonne (élément de)"],["construction","construction"],["coque","coque"],["coque-borde-premier","coque bordé-premier"],["coque-membrure-premiere","coque membrure-première"],["couple","couple"],["cour","cour"],["courtine","courtine"],["croix","croix"],["crypte","crypte"],["cryptoportique","cryptoportique"],["cuve","cuve"],["cuvelage","cuvelage"],["dallage","dallage"],["demi-lune","demi-lune"],["depotoir","dépotoir"],["derive","dérive"],["doublage","doublage"],["church","église"],["empierrement","empierrement"],["emplanture","emplanture"],["empreinte","empreinte"],["enclos","enclos"],["enclosure","enclos"],["enclos-systeme-d","enclos (système d')"],["entree-amenagee","entrée aménagée"],["escalier","escalier"],["etambot","étambot"],["etrave","étrave"],["etrave-a-eperon","étrave à éperon"],["etuve","étuve"],["ferrier","ferrier"],["fondation","fondation"],["fosse","fosse"],["ditch","fossé"],["fosses-reseau-de","fossés (réseau de)"],["four","four"],["foyer","foyer"],["foyer-vidange-de","foyer (vidange de)"],["front-de-taille","front de taille"],["fuselage","fuselage"],["galerie","galerie"],["garenne","garenne"],["glaciere","glacière"],["gouvernail","gouvernail"],["gradins","gradins"],["grange","grange"],["grenier","grenier"],["helice","hélice"],["hypocauste","hypocauste"],["incineration","incinération"],["inhumation","inhumation"],["inscription","inscription"],["jardin","jardin"],["laraire","laraire"],["latrines","latrines"],["lest","lest"],["levee","levée"],["mat","mât"],["membrure","membrure"],["menhir","menhir"],["meule-fixe","meule fixe"],["mosaique","mosaïque"],["moteur","moteur"],["motte-castrale","motte castrale"],["moule-a-cloches","moule à cloches"],["mur","mur"],["niche","niche"],["niveau-doccupation","niveau d'occupation"],["oratoire","oratoire"],["orniere","ornière"],["ouvrage-avance","ouvrage avancé"],["palissade","palissade"],["paroi-ornee","paroi ornée"],["pavage","pavage"],["pieu","pieu"],["pigeonnier","pigeonnier"],["pile","pile"],["plafond","plafond"],["plancher","plancher"],["polissoir-fixe","polissoir fixe"],["pont-de-bateau","pont de bateau"],["porte","porte"],["portique","portique"],["preceinte","préceinte"],["pressoir","pressoir"],["puits","puits"],["purgerie","purgerie"],["quille","quille"],["recipient-fixe","récipient fixe"],["rempart","rempart"],["sarcophage","sarcophage"],["silo","silo"],["sol-doccupation","sol d'occupation"],["sol-orne","sol orné"],["source-amenagee","source aménagée"],["souterrain","souterrain"],["statue","statue"],["statue-menhir","statue-menhir"],["stele","stèle"],["talus","talus"],["tour","tour"],["trace-agraire","trace agraire"],["train-datterrissage","train d'atterrissage"],["tranchee-dextraction","tranchée d'extraction"],["trou-de-jauge","trou de jauge"],["trou-de-poteau","trou de poteau"],["trous-de-poteau-ensemble-de","trous de poteau (ensemble de)"],["tunnel","tunnel"],["urne","urne"],["vaigre","vaigre"],["verriere","verrière"],["villa","villa"],["virure","virure"],["zone-de-rejet","zone de rejet"]]],["archaeological_operations.period",["periode","period"],[["not-yet-documented","Non-renseigné"],["indeterminate","Époque indéterminée"],["recent-times","Période récente"],["contemporary","Époque contemporaine"],["modern","Époque moderne"],["middle-age","Moyen Âge"],["low-middle-age","Bas Moyen Âge"],["classic-middle-age","Moyen Âge classique"],["high-middle-age","Haut Moyen Âge"],["gallo-roman","Gallo-romain"],["low-empire","Bas Empire"],["high-empire","Haut Empire"],["republic","République"],["protohistory","Protohistoire"],["iron-age","Âge du Fer"],["second-iron-age","Deuxième Âge du Fer"],["first-iron-age","Premier Âge du Fer"],["bronze-age","Âge du Bronze"],["final-bronze-age","Âge du Bronze final"],["middle-bronze-age","Âge du Bronze moyen"],["old-bronze-age","Âge du Bronze ancien"],["neolithic","Néolithique"],["final-neolithic","Néolithique final"],["recent-neolithic","Néolithique récent"],["middle-neolithic","Néolithique moyen"],["old-neolithic","Néolithique ancien"],["mesolithic","Mésolithique"],["recent-mesolithic","Mésolithique récent"],["middle-mesolithic","Mésolithique moyen"],["old-mesolithic","Mésolithique ancien"],["paleolithic","Paléolithique"],["final-paleolithic","Paléolithique supérieur final"],["late-paleolithic","Paléolithique supérieur"],["middle-paleolithic","Paléolithique moyen"],["ancien-paleolithic","Paléolithique ancien"],["mediterranean","Méditerranéen"],["mediterranean-antiq","Antiquité méditerranéenne"],["late-antiq-med","Antiquité tardive (Méditerranée)"],["hellen-epoc","Époque hellénistique"],["classic-epoc","Époque classique"],["archaic-epoc","Époque archaïque"],["antilles","Antilles"],["pre-pottery-antilles","Précéramique (Antilles)"],["archaic-antilles","Archaïque (Antilles)"],["lithic-antilles","Lithique (Antilles)"],["pottery-antilles","Céramique (Antilles)"],["recent-post-saladoide","Post-saladoïde récent (Antilles)"],["old-post-saladoide","Post-saladoïde ancien (Antilles)"],["saladoide","Saladoïde (Antilles)"],["huecoide","Huécoïde (Antilles)"],["historic-antilles","Historique (Antilles)"],["contemporary-antilles","Période contemporaine (Antilles)"],["modern-colonial-antilles","Colonial moderne (Antilles)"],["precolonial-antilles","Pré-colonial (Antilles)"],["indeterminate-antilles","Époque indéterminée (Antilles)"],["guiana","Guyane"],["prehistory-guiana","Préhistoire (Guyane)"],["prepottery-guiana","Acéramique (Guyane)"],["amerindian-guiana","Période amérindienne (Guyane)"],["precolonial-guiana","Pré-colonial (Guyane)"],["pre-contact-guiana","Pré-contact (Guyane)"],["old-amerindian-guiana","Période amérindienne ancienne (Guyane)"],["colonial-guiana","Période coloniale et départementale (Guyane)"],["contemporary-guiana","Contemporain (Guyane)"],["asserted-colonisation-guiana","Colonisation affirmée (Guyane)"],["colonisation-guiana","Période de colonisation (Guyane)"],["indeterminate-guiana","Époque indéterminée (Guyane)"]]],["archaeological_operations.recordqualitytype",["qualite-enregistrement","record-quality"],[["not-documented","Non documenté"],["arbitrary","Arbitraire"],["reliable","Fiable"]]],["archaeological_operations.relationtype",["type-relation","relation-types"],[["has_got","Comprend"],["is_in","Comprise dans"],["fuzzy_relation","Relation diffuse"]]],["ishtar_common.area",["zone","area"],[]]]} \ No newline at end of file diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index 7444aa9cd..39d7de021 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -21,6 +21,7 @@ import csv import json from io import TextIOWrapper, BytesIO import os +import requests import shutil import tempfile import urllib @@ -1482,7 +1483,8 @@ serialize_importer_action.short_description = SERIALIZE_DESC @admin.register(models.ImporterType, site=admin_site) class ImporterTypeAdmin(ImportJSONActionAdmin): list_display = ("name", "associated_models", "available") - actions = importer_type_actions + [serialize_importer_action, + actions = importer_type_actions + [ + serialize_importer_action, change_value("available", True, _("Make available")), change_value("available", False, _("Make unavailable")), ] @@ -2110,6 +2112,7 @@ class ApiSearchModelAdminForm(forms.ModelForm): class Meta: model = models.ApiUser exclude = [] + content_type = forms.ModelChoiceField( label=_("Content type"), queryset=get_main_content_types_query() ) @@ -2121,3 +2124,73 @@ class ApiSearchModelAdmin(admin.ModelAdmin): admin_site.register(models.ApiSearchModel, ApiSearchModelAdmin) + + +def send_error_message(request, msg, return_url, message_type=messages.ERROR): + messages.add_message( + request, + message_type, + msg, + ) + return HttpResponseRedirect(return_url) + + +def update_types_from_source(modeladmin, request, queryset): + return_url = ( + reverse( + "admin:%s_%s_changelist" + % (modeladmin.model._meta.app_label, modeladmin.model._meta.model_name) + ) + + "?" + + urllib.parse.urlencode(request.GET) + ) + if queryset.count() != 1: + return send_error_message( + request, + str(_("Select only one source.")), + return_url, + message_type=messages.WARNING, + ) + source = queryset.all()[0] + try: + response = requests.get( + source.url, + timeout=20, + headers={"Authorization": f"access_token {source.key}"}, + ) + except requests.exceptions.Timeout: + return send_error_message( + request, + str(_("Timeout: failed to join {}.")).format(source.url), + return_url + ) + if response.status_code != 200: + return send_error_message( + request, + str( + _( + "Bad response for {}. Response status code {} != 200. Check your key?" + ) + ).format(source.name, response.status_code), + return_url + ) + try: + content = response.json() + except ValueError: + return send_error_message( + request, + str(_("Response of {} is not a valid JSON message.")).format(source.url), + return_url + ) + result = source.update_matches(content) + print(result) + return response + + +class ApiExternalSource(admin.ModelAdmin): + model = models_rest.ApiExternalSource + actions = [update_types_from_source] + list_display = ("name", "url", "key") + + +admin_site.register(models_rest.ApiExternalSource, ApiExternalSource) diff --git a/ishtar_common/migrations/0217_auto_20211013_1517.py b/ishtar_common/migrations/0217_auto_20211013_1517.py deleted file mode 100644 index 1429da5ef..000000000 --- a/ishtar_common/migrations/0217_auto_20211013_1517.py +++ /dev/null @@ -1,208 +0,0 @@ -# Generated by Django 2.2.24 on 2021-10-13 15:17 - -from django.conf import settings -import django.contrib.postgres.fields -import django.contrib.postgres.fields.jsonb -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('auth', '0011_update_proxy_permissions'), - ('ishtar_common', '0216_auto_20210805_1703'), - ] - - operations = [ - migrations.CreateModel( - name='ApiExternalSource', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('url', models.URLField(verbose_name='URL')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('key', models.CharField(max_length=40, verbose_name='Key')), - ], - options={ - 'verbose_name': 'API - External source', - 'verbose_name_plural': 'API - External sources', - }, - ), - migrations.CreateModel( - name='ApiUser', - fields=[ - ('user_ptr', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='apiuser', serialize=False, to=settings.AUTH_USER_MODEL)), - ('ip', models.GenericIPAddressField(verbose_name='IP')), - ], - options={ - 'verbose_name': 'API - User', - 'verbose_name_plural': 'API - Users', - }, - ), - migrations.AlterField( - model_name='author', - name='author_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.AuthorType', verbose_name='Author type'), - ), - migrations.AlterField( - model_name='document', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='document', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='document', - name='language', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Language', verbose_name='Language'), - ), - migrations.AlterField( - model_name='document', - name='publisher', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='publish', to='ishtar_common.Organization', verbose_name='Publisher'), - ), - migrations.AlterField( - model_name='document', - name='source', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='ishtar_common.Document', verbose_name='Source'), - ), - migrations.AlterField( - model_name='historicalorganization', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='historicalorganization', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='historicalperson', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='historicalperson', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='ishtarsiteprofile', - name='default_language', - field=models.ForeignKey(blank=True, help_text='If set, by default the selected language will be set for localized documents.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Language', verbose_name='Default language for documentation'), - ), - migrations.AlterField( - model_name='ishtarsiteprofile', - name='display_srs', - field=models.ForeignKey(blank=True, help_text='Spatial Reference System used for display when no SRS is defined', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.SpatialReferenceSystem', verbose_name='Spatial Reference System for display'), - ), - migrations.AlterField( - model_name='itemkey', - name='group', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.TargetKeyGroup'), - ), - migrations.AlterField( - model_name='itemkey', - name='importer', - field=models.ForeignKey(blank=True, help_text='Specific key to an import', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Import'), - ), - migrations.AlterField( - model_name='itemkey', - name='user', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser'), - ), - migrations.AlterField( - model_name='organization', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='organization', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='organization', - name='organization_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.OrganizationType', verbose_name='Type'), - ), - migrations.AlterField( - model_name='organization', - name='precise_town', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Town', verbose_name='Town (precise)'), - ), - migrations.AlterField( - model_name='person', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='person', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='person', - name='precise_town', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Town', verbose_name='Town (precise)'), - ), - migrations.AlterField( - model_name='statscache', - name='values', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='targetkey', - name='associated_group', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.TargetKeyGroup'), - ), - migrations.AlterField( - model_name='targetkey', - name='associated_import', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Import'), - ), - migrations.AlterField( - model_name='targetkey', - name='associated_user', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser'), - ), - migrations.AlterField( - model_name='userprofile', - name='profile_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.ProfileType', verbose_name='Profile type'), - ), - migrations.CreateModel( - name='ApiSearchModel', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('limit_query', models.TextField(blank=True, help_text='Search query add to each request', null=True, verbose_name='Limit query')), - ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiUser')), - ], - options={ - 'verbose_name': 'API - Search model', - 'verbose_name_plural': 'API - Search models', - }, - ), - migrations.CreateModel( - name='ApiKeyMatch', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('search_keys', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, size=None, verbose_name='Search keys')), - ('distant_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Distant key')), - ('distant_label', models.TextField(blank=True, default='', verbose_name='Distant value')), - ('local_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Local key')), - ('local_label', models.TextField(blank=True, default='', verbose_name='Local value')), - ('search_model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='Search model')), - ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiExternalSource')), - ], - options={ - 'verbose_name': 'API - Key match', - 'verbose_name_plural': 'API - Keys matches', - }, - ), - ] diff --git a/ishtar_common/migrations/0217_auto_20211015_1728.py b/ishtar_common/migrations/0217_auto_20211015_1728.py new file mode 100644 index 000000000..47bfa7c6e --- /dev/null +++ b/ishtar_common/migrations/0217_auto_20211015_1728.py @@ -0,0 +1,209 @@ +# Generated by Django 2.2.24 on 2021-10-15 17:28 + +from django.conf import settings +import django.contrib.postgres.fields +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('auth', '0011_update_proxy_permissions'), + ('ishtar_common', '0216_auto_20210805_1703'), + ] + + operations = [ + migrations.CreateModel( + name='ApiExternalSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(verbose_name='URL')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('key', models.CharField(max_length=40, verbose_name='Key')), + ], + options={ + 'verbose_name': 'API - External source', + 'verbose_name_plural': 'API - External sources', + }, + ), + migrations.CreateModel( + name='ApiUser', + fields=[ + ('user_ptr', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='apiuser', serialize=False, to=settings.AUTH_USER_MODEL)), + ('ip', models.GenericIPAddressField(verbose_name='IP')), + ], + options={ + 'verbose_name': 'API - User', + 'verbose_name_plural': 'API - Users', + }, + ), + migrations.AlterField( + model_name='author', + name='author_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.AuthorType', verbose_name='Author type'), + ), + migrations.AlterField( + model_name='document', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='document', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='document', + name='language', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Language', verbose_name='Language'), + ), + migrations.AlterField( + model_name='document', + name='publisher', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='publish', to='ishtar_common.Organization', verbose_name='Publisher'), + ), + migrations.AlterField( + model_name='document', + name='source', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='ishtar_common.Document', verbose_name='Source'), + ), + migrations.AlterField( + model_name='historicalorganization', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='historicalorganization', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='historicalperson', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='historicalperson', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='default_language', + field=models.ForeignKey(blank=True, help_text='If set, by default the selected language will be set for localized documents.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Language', verbose_name='Default language for documentation'), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='display_srs', + field=models.ForeignKey(blank=True, help_text='Spatial Reference System used for display when no SRS is defined', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.SpatialReferenceSystem', verbose_name='Spatial Reference System for display'), + ), + migrations.AlterField( + model_name='itemkey', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.TargetKeyGroup'), + ), + migrations.AlterField( + model_name='itemkey', + name='importer', + field=models.ForeignKey(blank=True, help_text='Specific key to an import', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Import'), + ), + migrations.AlterField( + model_name='itemkey', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser'), + ), + migrations.AlterField( + model_name='organization', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='organization', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='organization', + name='organization_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.OrganizationType', verbose_name='Type'), + ), + migrations.AlterField( + model_name='organization', + name='precise_town', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Town', verbose_name='Town (precise)'), + ), + migrations.AlterField( + model_name='person', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='person', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='person', + name='precise_town', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Town', verbose_name='Town (precise)'), + ), + migrations.AlterField( + model_name='statscache', + name='values', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='targetkey', + name='associated_group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.TargetKeyGroup'), + ), + migrations.AlterField( + model_name='targetkey', + name='associated_import', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Import'), + ), + migrations.AlterField( + model_name='targetkey', + name='associated_user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser'), + ), + migrations.AlterField( + model_name='userprofile', + name='profile_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.ProfileType', verbose_name='Profile type'), + ), + migrations.CreateModel( + name='ApiSearchModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('limit_query', models.TextField(blank=True, help_text='Search query add to each request', null=True, verbose_name='Limit query')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiUser')), + ], + options={ + 'verbose_name': 'API - Search model', + 'verbose_name_plural': 'API - Search models', + }, + ), + migrations.CreateModel( + name='ApiKeyMatch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('search_keys', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, size=None, verbose_name='Search keys')), + ('distant_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Distant key')), + ('distant_label', models.TextField(blank=True, default='', verbose_name='Distant value')), + ('local_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Local key')), + ('local_label', models.TextField(blank=True, default='', verbose_name='Local value')), + ('do_not_match', models.BooleanField(default=False, verbose_name='Disable match for this search')), + ('search_model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='Search model')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiExternalSource')), + ], + options={ + 'verbose_name': 'API - Key match', + 'verbose_name_plural': 'API - Keys matches', + }, + ), + ] diff --git a/ishtar_common/models_rest.py b/ishtar_common/models_rest.py index db7d21834..a74df2c8a 100644 --- a/ishtar_common/models_rest.py +++ b/ishtar_common/models_rest.py @@ -47,6 +47,58 @@ class ApiExternalSource(models.Model): verbose_name = _("API - External source") verbose_name_plural = _("API - External sources") + def update_matches(self, content): + result = { + "created": 0, + "updated": 0, + "deleted": 0, + "search_model do not exist": [], + "type do not exist": [], + } + updated = [] + for search_model in content: + app, model_name = search_model.split(".") + try: + ct = ContentType.objects.get(app_label=app, model=model_name) + except ContentType.DoesNotExist: + result["search_model do not exist"].append(search_model) + continue + for ct_type, keys, values in content[search_model]: + tapp, tmodel_name = ct_type.split(".") + try: + ct_type = ContentType.objects.get( + app_label=tapp, model=tmodel_name + ) + except ContentType.DoesNotExist: + result["type do not exist"].append(search_model) + continue + t_model = ct_type.model_class() + for slug, label in values: + m, created = ApiKeyMatch.objects.get_or_create( + source=self, search_model=ct, search_keys=keys, + distant_slug=slug, defaults={"distant_label": label}) + updated = False + if not created and m.distant_label != label: + updated = True + m.distant_label = label + if not m.do_not_match and not m.local_slug: + slug_key = "txt_idx" + if hasattr(t_model, "slug"): + slug_key = "slug" + q = t_model.objects.filter(**{slug_key: m.distant_slug}) + if q.count(): + local_value = q.all()[0] + setattr(m, "local_slug", getattr(local_value, slug_key)) + m.local_value = str(local_value) + updated = True + if updated: + m.save() + if not created: + result["updated"] += 1 + if created: + result["created"] += 1 + return result + class ApiKeyMatch(models.Model): source = models.ForeignKey(ApiExternalSource, on_delete=models.CASCADE) @@ -62,6 +114,8 @@ class ApiKeyMatch(models.Model): allow_unicode=True) local_label = models.TextField(verbose_name=_("Local value"), blank=True, default="") + do_not_match = models.BooleanField( + verbose_name=_("Disable match for this search"), default=False) class Meta: verbose_name = _("API - Key match") diff --git a/ishtar_common/rest.py b/ishtar_common/rest.py index 602bba293..0ae951dd1 100644 --- a/ishtar_common/rest.py +++ b/ishtar_common/rest.py @@ -1,3 +1,5 @@ +import json + from django.conf import settings from django.db.models import Q from django.utils.translation import activate, deactivate @@ -94,9 +96,11 @@ class FacetAPIView(APIView): search_keys.append(str(search_key).lower()) deactivate() values[ct_model].append( - [search_keys, values_ct] + [f"{type.model._meta.app_label}.{type.model._meta.model_name}", + search_keys, values_ct] ) - return Response(values) + #values = json.dumps(values) + return Response(values, content_type="json") def _get_base_search_model_queries(self): """ -- cgit v1.2.3