✨ imports: allow media import from a web location

6 files changed, 237 insertions, 39 deletions

diff --git a/archaeological_finds/tests.py b/archaeological_finds/tests.py
index 45f178856..d5f763467 100644
--- a/archaeological_finds/tests.py
+++ b/archaeological_finds/tests.py
@@ -31,6 +31,7 @@ from django.contrib.auth.models import User, Permission, ContentType
 from django.core.files import File
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db.utils import IntegrityError
+from django.test import LiveServerTestCase
 from django.test.client import Client
 from django.urls import reverse
 from ishtar_common.models import (
@@ -626,15 +627,20 @@ class TreatmentWizardCreationTest(WizardTest, FindInit, TestCase):
         #                 treat)
 
 
-class ImportFindTest(ImportTest, FindInit, TestCase):
+class BaseImportFindTest(ImportTest, FindInit, TestCase):
     fixtures = FIND_TOWNS_FIXTURES + [
         settings.LIB_BASE_PATH + "archaeological_finds/tests/import_loca_test.json",
     ]
 
     def setUp(self):
-        super(ImportFindTest, self).setUp()
+        super().setUp()
         self.tmpdir = tempfile.TemporaryDirectory()
 
+    def tearDown(self):
+        self.tmpdir.cleanup()
+
+
+class ImportFindTest(BaseImportFindTest):
     def test_geo_import_csv(self):
         self._test_geo_import("importer-GIS-find", 1)
 
@@ -992,8 +998,62 @@ class ImportFindTest(ImportTest, FindInit, TestCase):
         # check errors
         self.assertEqual(len(impt.errors), 0)
 
-    def tearDown(self):
-        self.tmpdir.cleanup()
+
+class ImportFindLiveServerTest(LiveServerTestCase, BaseImportFindTest):
+
+    def setUp(self):
+        import ishtar_common
+        path = ["/"] + ishtar_common.__file__.split(os.sep)[:-1] + ["tests", "test.png"]
+        shutil.copy(
+            os.path.join(*path),
+            os.path.join(settings.ROOT_PATH, "media", "image-1.jpg"),
+        )
+        super().setUp()
+
+    def test_mcc_import_finds_with_image_link(self):
+        self.init_context_record()
+
+        old_nb = models.BaseFind.objects.count()
+        old_nb_find = models.Find.objects.count()
+        MCC = ImporterType.objects.get(name="MCC - Mobilier")
+
+        col = ImporterColumn.objects.create(col_number=25, importer_type_id=MCC.pk)
+        formater = FormaterType.objects.filter(formater_type="FileFormater").all()[0]
+        ImportTarget.objects.create(
+            target="documents__image", formater_type_id=formater.pk, column_id=col.pk
+        )
+        mcc_file = open(
+            settings.LIB_BASE_PATH + "archaeological_finds/tests/MCC-finds-example.csv",
+            "rb",
+            )
+        file_dict = {
+            "imported_file": SimpleUploadedFile(mcc_file.name, mcc_file.read()),
+        }
+        post_dict = {
+            "importer_type": MCC.pk,
+            "skip_lines": 1,
+            "encoding": "utf-8",
+            "name": "init_find_import",
+            "csv_sep": ",",
+            "imported_media_link": f"{self.live_server_url}/media/"
+        }
+        form = forms_common.BaseImportForm(
+            data=post_dict, files=file_dict, user=self.user
+        )
+        form.is_valid()
+        self.assertTrue(form.is_valid())
+        impt = form.save(self.ishtar_user)
+        impt.initialize()
+        impt.importation()
+        # new finds has now been imported
+        current_nb = models.BaseFind.objects.count()
+        self.assertEqual(current_nb, (old_nb + 4))
+        current_nb = models.Find.objects.count()
+        self.assertEqual(current_nb, (old_nb_find + 4))
+        images = []
+        for find in models.Find.objects.all():
+            images += [1 for im in find.images.all() if im.image.name]
+        self.assertEqual(len(images), 1)
 
 
 class FindTest(FindInit, TestCase):
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 546d29b2d..d16d878e5 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -39,7 +39,12 @@ from django.db.models import Q
 from django.template.defaultfilters import slugify
 from django.utils.translation import ugettext_lazy as _
 
-from ishtar_common.utils import get_all_field_names, update_data, get_current_profile
+from ishtar_common.utils import (
+    get_all_field_names,
+    get_current_profile,
+    get_file_from_link,
+    update_data,
+)
 
 
 NEW_LINE_BREAK = "#####@@@#####"
@@ -620,6 +625,29 @@ class FileFormater(Formater):
         value = value.strip()
         if not value:
             return
+        if isinstance(archive, str) and (archive.startswith("http://") or
+                                         archive.startswith("https://")):
+            return self._format_url(value, archive)
+        return self._format_zip(value, archive)
+
+    def _format_url(self, value, link):
+        print(link)
+        if not link.endswith("/"):
+            link += "/"
+        full_link = link + value
+        try:
+            filename, tmp_file = get_file_from_link(full_link)
+        except ValueError:
+            raise ValueError(
+                _('"%(full_link)s" is not a valid path')
+                % {"full_link": full_link}
+            )
+        my_file = File(tmp_file, name=filename)
+        # manually set the file size because of an issue with TempFile
+        my_file.size = os.stat(tmp_file.name).st_size
+        return my_file
+
+    def _format_zip(self, value, archive):
         zp = zipfile.ZipFile(archive)
         value = value.strip().replace("\\", "/")
         items = value.replace("/", "_").split(".")
@@ -639,7 +667,7 @@ class FileFormater(Formater):
             return my_file
         except KeyError:
             raise ValueError(
-                _('"%(value)s" is not a valid path for the ' "given archive")
+                _('"%(value)s" is not a valid path for the given archive')
                 % {"value": value}
             )
 
@@ -830,8 +858,12 @@ class Importer(object):
         self.concats = set()
         self.concat_str = {}
         self.to_be_close = []
-        if import_instance and import_instance.get_imported_images():
-            self.archive = import_instance.get_imported_images()
+        if import_instance:
+            imported_images = import_instance.get_imported_images()
+            if imported_images:
+                self.archive = imported_images
+            elif import_instance.imported_media_link:
+                self.archive = import_instance.imported_media_link
         self._defaults = self.DEFAULTS.copy()
         self._pre_import_values = self.PRE_IMPORT_VALUES.copy()
         self.history_modifier = history_modifier
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index 6e0c23188..5b2c83f93 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -77,8 +77,9 @@ from .forms import (
 )
 from ishtar_common.data_importer import ImporterError
 from ishtar_common.utils import (
-    is_downloadable,
     clean_session_cache,
+    get_file_from_link,
+    is_downloadable,
     max_size_help,
     max_value_current_year,
     reverse_coordinates,
@@ -215,8 +216,8 @@ class BaseImportForm(IshtarForm, forms.ModelForm):
     error_css_class = "error"
     required_css_class = "required"
     importer_type = "tab"
-    imported_images_link = forms.URLField(
-        label=_("Associated images (web link to a zip file)"), required=False
+    imported_media_link = forms.URLField(
+        label=_("Associated media (web link to a zip file or a path)"), required=False
     )
 
     class Meta:
@@ -230,7 +231,7 @@ class BaseImportForm(IshtarForm, forms.ModelForm):
             "encoding",
             "csv_sep",
             "imported_images",
-            "imported_images_link",
+            "imported_media_link",
         )
         widgets = {
             "imported_file": widgets.BSClearableFileInput,
@@ -337,20 +338,9 @@ class BaseImportForm(IshtarForm, forms.ModelForm):
         return data
 
 
-def get_archive_from_link(archive_link):
-    request = requests.get(archive_link, stream=True)
-    ntf = tempfile.NamedTemporaryFile()
-    for block in request.iter_content(1024 * 8):
-        if not block:
-            break
-        ntf.write(block)
-    file_name = archive_link.split("/")[-1]
-    return file_name, ntf
-
-
 class NewImportForm(BaseImportForm):
-    imported_images_link = forms.URLField(
-        label=_("Associated documents (web link to a zip file)"), required=False
+    imported_media_link = forms.URLField(
+        label=_("Associated media (web link to a zip file or a path)"), required=False
     )
 
     class Meta:
@@ -364,7 +354,7 @@ class NewImportForm(BaseImportForm):
             "encoding",
             "csv_sep",
             "imported_images",
-            "imported_images_link",
+            "imported_media_link",
         )
 
     HEADERS = {
@@ -373,16 +363,20 @@ class NewImportForm(BaseImportForm):
         "imported_images": FormHeader(_("Documents/Images")),
     }
 
+    def __init__(self, *args, **kwargs):
+        self.media_link_is_zip = False
+        super().__init__(*args, **kwargs)
+
     def _need_archive(self, data):
         tpe = data["importer_type"]
         return tpe.archive_required
 
     def clean(self):
         data = super().clean()
-        if data.get("imported_images_link", None) and data.get("imported_images", None):
+        if data.get("imported_media_link", None) and data.get("imported_images", None):
             raise forms.ValidationError(
                 _(
-                    "You put either a file or a download link for images "
+                    "You put either a file or a download link for media "
                     "but not both."
                 )
             )
@@ -400,14 +394,14 @@ class NewImportForm(BaseImportForm):
         archive_required = self._need_archive(data)
         if archive_required and (
             not data.get("imported_images", None)
-            and not data.get("imported_images_link", None)
+            and not data.get("imported_media_link", None)
         ):
             raise forms.ValidationError(_("This importer need a document archive."))
         return data
 
     def clean_imported_images_link(self):
-        value = self.cleaned_data.get("imported_images_link", None)
-        if value:
+        value = self.cleaned_data.get("imported_media_link", None)
+        if value and value.lower().endswith(".zip"):
             try:
                 if not is_downloadable(value):
                     raise forms.ValidationError("")
@@ -415,20 +409,24 @@ class NewImportForm(BaseImportForm):
                 raise forms.ValidationError(
                     _("Invalid link or no file is available for this link.")
                 )
+            self.media_link_is_zip = True
         return value
 
     def save(self, user, commit=True):
         self.instance.user = user
-        imported_images_link = (
-            self.cleaned_data.pop("imported_images_link")
-            if "imported_images_link" in self.cleaned_data
-            else None
-        )
         item = super().save(commit)
-        if not imported_images_link:
+        if not self.media_link_is_zip:
             return item
-        file_name, temp_file = get_archive_from_link(imported_images_link)
+        try:
+            file_name, temp_file = get_file_from_link(item.imported_media_link)
+        except ValueError:
+            raise forms.ValidationError(
+                _("Bad link for the archive.")
+            )
         item.imported_images.save(file_name, File(temp_file))
+        # media is downloaded - clean the link
+        item.imported_media_link = None
+        item.save()
         return item
 
 
@@ -511,7 +509,7 @@ class NewImportGroupForm(NewImportForm):
             "encoding",
             "csv_sep",
             "imported_images",
-            "imported_images_link",
+            "imported_media_link",
         )
 
     HEADERS = {
diff --git a/ishtar_common/migrations/0244_imports_media_link.py b/ishtar_common/migrations/0244_imports_media_link.py
new file mode 100644
index 000000000..88a36049f
--- /dev/null
+++ b/ishtar_common/migrations/0244_imports_media_link.py
@@ -0,0 +1,87 @@
+# Generated by Django 2.2.24 on 2024-04-18 17:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0243_default_biographicalnote_permissions'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='biographicalnote',
+            options={'permissions': (('view_own_biographicalnote', 'Can view own Biographical note'), ('add_own_biographicalnote', 'Can add own Biographical note'), ('change_own_biographicalnote', 'Can change own Biographical note'), ('delete_own_biographicalnote', 'Can delete own Biographical note')), 'verbose_name': 'Biographical note', 'verbose_name_plural': 'Biographical notes'},
+        ),
+        migrations.AddField(
+            model_name='import',
+            name='imported_media_link',
+            field=models.URLField(blank=True, null=True, verbose_name='Associated media (web link to a zip file or a path'),
+        ),
+        migrations.AddField(
+            model_name='importgroup',
+            name='imported_media_link',
+            field=models.URLField(blank=True, null=True, verbose_name='Associated media (web link to a zip file or a path'),
+        ),
+        migrations.AlterField(
+            model_name='biographicalnote',
+            name='imports',
+            field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_biographicalnote', to='ishtar_common.Import', verbose_name='Created by imports'),
+        ),
+        migrations.AlterField(
+            model_name='biographicalnote',
+            name='imports_updated',
+            field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_biographicalnote', to='ishtar_common.Import', verbose_name='Updated by imports'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='imports',
+            field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_document', to='ishtar_common.Import', verbose_name='Created by imports'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='imports_updated',
+            field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_document', to='ishtar_common.Import', verbose_name='Updated by imports'),
+        ),
+        migrations.AlterField(
+            model_name='geovectordata',
+            name='imports',
+            field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_geovectordata', to='ishtar_common.Import', verbose_name='Created by imports'),
+        ),
+        migrations.AlterField(
+            model_name='geovectordata',
+            name='imports_updated',
+            field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_geovectordata', to='ishtar_common.Import', verbose_name='Updated by imports'),
+        ),
+        migrations.AlterField(
+            model_name='organization',
+            name='imports',
+            field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_organization', to='ishtar_common.Import', verbose_name='Created by imports'),
+        ),
+        migrations.AlterField(
+            model_name='organization',
+            name='imports_updated',
+            field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_organization', to='ishtar_common.Import', verbose_name='Updated by imports'),
+        ),
+        migrations.AlterField(
+            model_name='person',
+            name='imports',
+            field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_person', to='ishtar_common.Import', verbose_name='Created by imports'),
+        ),
+        migrations.AlterField(
+            model_name='person',
+            name='imports_updated',
+            field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_person', to='ishtar_common.Import', verbose_name='Updated by imports'),
+        ),
+        migrations.AlterField(
+            model_name='town',
+            name='imports',
+            field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_town', to='ishtar_common.Import', verbose_name='Created by imports'),
+        ),
+        migrations.AlterField(
+            model_name='town',
+            name='imports_updated',
+            field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_town', to='ishtar_common.Import', verbose_name='Updated by imports'),
+        ),
+    ]
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 22890d484..a37fd2afe 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -1428,6 +1428,10 @@ class BaseImport(models.Model, OwnPerms, SheetItem):
         default=1,
         help_text=_("Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile)."),
     )
+    imported_media_link = models.URLField(
+        _("Associated media (web link to a zip file or a path)"),
+        blank=True, null=True
+    )
     creation_date = models.DateTimeField(
         _("Creation date"), auto_now_add=True, blank=True, null=True
     )
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index f734a9b2d..08ef84831 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -644,6 +644,23 @@ def is_downloadable(curl):
     return True
 
 
+def get_file_from_link(file_link):
+    """
+    return filename and temp_file object from a web link
+    """
+    try:
+        request = requests.get(file_link, stream=True)
+    except requests.exceptions.RequestException:
+        raise ValueError()
+    ntf = tempfile.NamedTemporaryFile()
+    for block in request.iter_content(1024 * 8):
+        if not block:
+            break
+        ntf.write(block)
+    file_name = file_link.split("/")[-1]
+    return file_name, ntf
+
+
 def get_current_year():
     return datetime.datetime.now().year