From 66ba898ce0797fd51a826f6ea654b87a3487e8bb Mon Sep 17 00:00:00 2001 From: Étienne Loks Date: Tue, 19 Oct 2021 22:46:56 +0200 Subject: Syndication - attach the generated match document to the source --- archaeological_operations/tests.py | 1 - ishtar_common/admin.py | 6 + .../migrations/0217_auto_20211018_1741.py | 210 -------------------- .../migrations/0217_auto_20211019_2242.py | 211 +++++++++++++++++++++ ishtar_common/models_rest.py | 21 +- 5 files changed, 235 insertions(+), 214 deletions(-) delete mode 100644 ishtar_common/migrations/0217_auto_20211018_1741.py create mode 100644 ishtar_common/migrations/0217_auto_20211019_2242.py diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index ee1f2645c..409ee5079 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -4686,7 +4686,6 @@ class ApiTest(OperationInitTest, APITestCase): params["action"] = "generate_match_document" response = self.client.post(url, params, follow=True) - #src_doc = source.generate_match_document() zip_file = zipfile.ZipFile(BytesIO(response.content)) self.assertIsNone( diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index 8124bbeed..761d07b4d 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -2217,6 +2217,8 @@ def update_types_from_source(modeladmin, request, queryset): ) return response +update_types_from_source.short_description = _("Update types from source") + def generate_match_document(modeladmin, request, queryset): return_url = ( @@ -2251,6 +2253,9 @@ def generate_match_document(modeladmin, request, queryset): return response +generate_match_document.short_description = _("Generate match document") + + class ApiExternalSourceAdmin(admin.ModelAdmin): model = models_rest.ApiExternalSource actions = [update_types_from_source, generate_match_document] @@ -2272,4 +2277,5 @@ class ApiKeyMatchAdmin(admin.ModelAdmin): change_value("do_not_match", True, _("Disable match")), ] + admin_site.register(models_rest.ApiKeyMatch, ApiKeyMatchAdmin) diff --git a/ishtar_common/migrations/0217_auto_20211018_1741.py b/ishtar_common/migrations/0217_auto_20211018_1741.py deleted file mode 100644 index d79224b5a..000000000 --- a/ishtar_common/migrations/0217_auto_20211018_1741.py +++ /dev/null @@ -1,210 +0,0 @@ -# Generated by Django 2.2.24 on 2021-10-18 17:41 - -from django.conf import settings -import django.contrib.postgres.fields -import django.contrib.postgres.fields.jsonb -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('auth', '0011_update_proxy_permissions'), - ('contenttypes', '0002_remove_content_type_name'), - ('ishtar_common', '0216_auto_20210805_1703'), - ] - - operations = [ - migrations.CreateModel( - name='ApiExternalSource', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('url', models.URLField(verbose_name='URL')), - ('name', models.CharField(max_length=200, verbose_name='Name')), - ('key', models.CharField(max_length=40, verbose_name='Key')), - ], - options={ - 'verbose_name': 'API - External source', - 'verbose_name_plural': 'API - External sources', - }, - ), - migrations.CreateModel( - name='ApiUser', - fields=[ - ('user_ptr', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='apiuser', serialize=False, to=settings.AUTH_USER_MODEL)), - ('ip', models.GenericIPAddressField(verbose_name='IP')), - ], - options={ - 'verbose_name': 'API - User', - 'verbose_name_plural': 'API - Users', - }, - ), - migrations.AlterField( - model_name='author', - name='author_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.AuthorType', verbose_name='Author type'), - ), - migrations.AlterField( - model_name='document', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='document', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='document', - name='language', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Language', verbose_name='Language'), - ), - migrations.AlterField( - model_name='document', - name='publisher', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='publish', to='ishtar_common.Organization', verbose_name='Publisher'), - ), - migrations.AlterField( - model_name='document', - name='source', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='ishtar_common.Document', verbose_name='Source'), - ), - migrations.AlterField( - model_name='historicalorganization', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='historicalorganization', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='historicalperson', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='historicalperson', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='ishtarsiteprofile', - name='default_language', - field=models.ForeignKey(blank=True, help_text='If set, by default the selected language will be set for localized documents.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Language', verbose_name='Default language for documentation'), - ), - migrations.AlterField( - model_name='ishtarsiteprofile', - name='display_srs', - field=models.ForeignKey(blank=True, help_text='Spatial Reference System used for display when no SRS is defined', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.SpatialReferenceSystem', verbose_name='Spatial Reference System for display'), - ), - migrations.AlterField( - model_name='itemkey', - name='group', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.TargetKeyGroup'), - ), - migrations.AlterField( - model_name='itemkey', - name='importer', - field=models.ForeignKey(blank=True, help_text='Specific key to an import', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Import'), - ), - migrations.AlterField( - model_name='itemkey', - name='user', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser'), - ), - migrations.AlterField( - model_name='organization', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='organization', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='organization', - name='organization_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.OrganizationType', verbose_name='Type'), - ), - migrations.AlterField( - model_name='organization', - name='precise_town', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Town', verbose_name='Town (precise)'), - ), - migrations.AlterField( - model_name='person', - name='data', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='person', - name='history_m2m', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='person', - name='precise_town', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Town', verbose_name='Town (precise)'), - ), - migrations.AlterField( - model_name='statscache', - name='values', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='targetkey', - name='associated_group', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.TargetKeyGroup'), - ), - migrations.AlterField( - model_name='targetkey', - name='associated_import', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Import'), - ), - migrations.AlterField( - model_name='targetkey', - name='associated_user', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser'), - ), - migrations.AlterField( - model_name='userprofile', - name='profile_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.ProfileType', verbose_name='Profile type'), - ), - migrations.CreateModel( - name='ApiSearchModel', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('limit_query', models.TextField(blank=True, help_text='Search query add to each request', null=True, verbose_name='Limit query')), - ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiUser')), - ], - options={ - 'verbose_name': 'API - Search model', - 'verbose_name_plural': 'API - Search models', - }, - ), - migrations.CreateModel( - name='ApiKeyMatch', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('search_keys', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, size=None, verbose_name='Search keys')), - ('distant_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Distant key')), - ('distant_label', models.TextField(blank=True, default='', verbose_name='Distant value')), - ('local_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Local key')), - ('local_label', models.TextField(blank=True, default='', verbose_name='Local value')), - ('do_not_match', models.BooleanField(default=False, verbose_name='Disable match for this search')), - ('associated_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='key_match_types', to='contenttypes.ContentType', verbose_name='Associated type')), - ('search_model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='key_match_search_models', to='contenttypes.ContentType', verbose_name='Search model')), - ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiExternalSource')), - ], - options={ - 'verbose_name': 'API - Key match', - 'verbose_name_plural': 'API - Keys matches', - }, - ), - ] diff --git a/ishtar_common/migrations/0217_auto_20211019_2242.py b/ishtar_common/migrations/0217_auto_20211019_2242.py new file mode 100644 index 000000000..4a0851691 --- /dev/null +++ b/ishtar_common/migrations/0217_auto_20211019_2242.py @@ -0,0 +1,211 @@ +# Generated by Django 2.2.24 on 2021-10-19 22:42 + +from django.conf import settings +import django.contrib.postgres.fields +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('auth', '0011_update_proxy_permissions'), + ('ishtar_common', '0216_auto_20210805_1703'), + ] + + operations = [ + migrations.CreateModel( + name='ApiExternalSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(verbose_name='URL')), + ('name', models.CharField(max_length=200, verbose_name='Name')), + ('key', models.CharField(max_length=40, verbose_name='Key')), + ('match_document', models.FileField(blank=True, help_text='First use the "Update types from source" action. Then use the action "Generate match document" action to create a default match document. Complete it and attach it back to the source to finally use the action "Modify association from match document".', null=True, upload_to='', verbose_name='Match document')), + ], + options={ + 'verbose_name': 'API - Search - External source', + 'verbose_name_plural': 'API - Search - External sources', + }, + ), + migrations.CreateModel( + name='ApiUser', + fields=[ + ('user_ptr', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='apiuser', serialize=False, to=settings.AUTH_USER_MODEL)), + ('ip', models.GenericIPAddressField(verbose_name='IP')), + ], + options={ + 'verbose_name': 'API - Remote access - User', + 'verbose_name_plural': 'API - Remote access - Users', + }, + ), + migrations.AlterField( + model_name='author', + name='author_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.AuthorType', verbose_name='Author type'), + ), + migrations.AlterField( + model_name='document', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='document', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='document', + name='language', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Language', verbose_name='Language'), + ), + migrations.AlterField( + model_name='document', + name='publisher', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='publish', to='ishtar_common.Organization', verbose_name='Publisher'), + ), + migrations.AlterField( + model_name='document', + name='source', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='ishtar_common.Document', verbose_name='Source'), + ), + migrations.AlterField( + model_name='historicalorganization', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='historicalorganization', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='historicalperson', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='historicalperson', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='default_language', + field=models.ForeignKey(blank=True, help_text='If set, by default the selected language will be set for localized documents.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Language', verbose_name='Default language for documentation'), + ), + migrations.AlterField( + model_name='ishtarsiteprofile', + name='display_srs', + field=models.ForeignKey(blank=True, help_text='Spatial Reference System used for display when no SRS is defined', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.SpatialReferenceSystem', verbose_name='Spatial Reference System for display'), + ), + migrations.AlterField( + model_name='itemkey', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.TargetKeyGroup'), + ), + migrations.AlterField( + model_name='itemkey', + name='importer', + field=models.ForeignKey(blank=True, help_text='Specific key to an import', null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Import'), + ), + migrations.AlterField( + model_name='itemkey', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser'), + ), + migrations.AlterField( + model_name='organization', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='organization', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='organization', + name='organization_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.OrganizationType', verbose_name='Type'), + ), + migrations.AlterField( + model_name='organization', + name='precise_town', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Town', verbose_name='Town (precise)'), + ), + migrations.AlterField( + model_name='person', + name='data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='person', + name='history_m2m', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='person', + name='precise_town', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Town', verbose_name='Town (precise)'), + ), + migrations.AlterField( + model_name='statscache', + name='values', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), + ), + migrations.AlterField( + model_name='targetkey', + name='associated_group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.TargetKeyGroup'), + ), + migrations.AlterField( + model_name='targetkey', + name='associated_import', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.Import'), + ), + migrations.AlterField( + model_name='targetkey', + name='associated_user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ishtar_common.IshtarUser'), + ), + migrations.AlterField( + model_name='userprofile', + name='profile_type', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='ishtar_common.ProfileType', verbose_name='Profile type'), + ), + migrations.CreateModel( + name='ApiSearchModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('limit_query', models.TextField(blank=True, help_text='Search query add to each request', null=True, verbose_name='Limit query')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiUser')), + ], + options={ + 'verbose_name': 'API - Remote access - Search model', + 'verbose_name_plural': 'API - Remote access - Search models', + }, + ), + migrations.CreateModel( + name='ApiKeyMatch', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('search_keys', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, size=None, verbose_name='Search keys')), + ('distant_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Distant key')), + ('distant_label', models.TextField(blank=True, default='', verbose_name='Distant value')), + ('local_slug', models.SlugField(allow_unicode=True, max_length=200, verbose_name='Local key')), + ('local_label', models.TextField(blank=True, default='', verbose_name='Local value')), + ('do_not_match', models.BooleanField(default=False, verbose_name='Disable match for this search')), + ('associated_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='key_match_types', to='contenttypes.ContentType', verbose_name='Associated type')), + ('search_model', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='key_match_search_models', to='contenttypes.ContentType', verbose_name='Search model')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.ApiExternalSource')), + ], + options={ + 'verbose_name': 'API - Search - Key match', + 'verbose_name_plural': 'API - Search - Keys matches', + }, + ), + ] diff --git a/ishtar_common/models_rest.py b/ishtar_common/models_rest.py index bb260c30d..61b3ce042 100644 --- a/ishtar_common/models_rest.py +++ b/ishtar_common/models_rest.py @@ -55,6 +55,17 @@ class ApiExternalSource(models.Model): url = models.URLField(verbose_name=_("URL")) name = models.CharField(verbose_name=_("Name"), max_length=200) key = models.CharField(_("Key"), max_length=40) + match_document = models.FileField( + _("Match document"), + blank=True, + null=True, + help_text=_( + 'First use the "Update types from source" action. Then use the action ' + '"Generate match document" action to create a default match document. ' + "Complete it and attach it back to the source to finally use the action " + '"Modify association from match document".' + ), + ) class Meta: verbose_name = _("API - Search - External source") @@ -146,10 +157,14 @@ class ApiExternalSource(models.Model): for idx, tpe in enumerate(types): self._generate_match_page(idx, tpe, uno, calc, lst_sheet) tmpdir = tempfile.mkdtemp(prefix="ishtar-matches-") - dest_filename = "{}{}{}-{}.ods".format(tmpdir, os.sep, - datetime.date.today().isoformat(), - slugify(self.name)) + base = "{}-{}.ods".format(datetime.date.today().isoformat(), slugify(self.name)) + dest_filename = "{}{}{}".format(tmpdir, os.sep, base) uno.save_calc(calc, dest_filename) + from django.core.files import File + + with open(dest_filename, "rb") as fle: + self.match_document = File(fle, base) + self.save() return dest_filename def _generate_match_page(self, page_number, tpe, uno, calc, lst_sheet): -- cgit v1.2.3