summaryrefslogtreecommitdiff
path: root/ishtar_common
diff options
context:
space:
mode:
Diffstat (limited to 'ishtar_common')
-rw-r--r--ishtar_common/data_importer.py40
-rw-r--r--ishtar_common/forms_common.py60
-rw-r--r--ishtar_common/migrations/0244_imports_media_link.py87
-rw-r--r--ishtar_common/models_imports.py4
-rw-r--r--ishtar_common/utils.py17
5 files changed, 173 insertions, 35 deletions
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 546d29b2d..d16d878e5 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -39,7 +39,12 @@ from django.db.models import Q
from django.template.defaultfilters import slugify
from django.utils.translation import ugettext_lazy as _
-from ishtar_common.utils import get_all_field_names, update_data, get_current_profile
+from ishtar_common.utils import (
+ get_all_field_names,
+ get_current_profile,
+ get_file_from_link,
+ update_data,
+)
NEW_LINE_BREAK = "#####@@@#####"
@@ -620,6 +625,29 @@ class FileFormater(Formater):
value = value.strip()
if not value:
return
+ if isinstance(archive, str) and (archive.startswith("http://") or
+ archive.startswith("https://")):
+ return self._format_url(value, archive)
+ return self._format_zip(value, archive)
+
+ def _format_url(self, value, link):
+ print(link)
+ if not link.endswith("/"):
+ link += "/"
+ full_link = link + value
+ try:
+ filename, tmp_file = get_file_from_link(full_link)
+ except ValueError:
+ raise ValueError(
+ _('"%(full_link)s" is not a valid path')
+ % {"full_link": full_link}
+ )
+ my_file = File(tmp_file, name=filename)
+ # manually set the file size because of an issue with TempFile
+ my_file.size = os.stat(tmp_file.name).st_size
+ return my_file
+
+ def _format_zip(self, value, archive):
zp = zipfile.ZipFile(archive)
value = value.strip().replace("\\", "/")
items = value.replace("/", "_").split(".")
@@ -639,7 +667,7 @@ class FileFormater(Formater):
return my_file
except KeyError:
raise ValueError(
- _('"%(value)s" is not a valid path for the ' "given archive")
+ _('"%(value)s" is not a valid path for the given archive')
% {"value": value}
)
@@ -830,8 +858,12 @@ class Importer(object):
self.concats = set()
self.concat_str = {}
self.to_be_close = []
- if import_instance and import_instance.get_imported_images():
- self.archive = import_instance.get_imported_images()
+ if import_instance:
+ imported_images = import_instance.get_imported_images()
+ if imported_images:
+ self.archive = imported_images
+ elif import_instance.imported_media_link:
+ self.archive = import_instance.imported_media_link
self._defaults = self.DEFAULTS.copy()
self._pre_import_values = self.PRE_IMPORT_VALUES.copy()
self.history_modifier = history_modifier
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index 6e0c23188..5b2c83f93 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -77,8 +77,9 @@ from .forms import (
)
from ishtar_common.data_importer import ImporterError
from ishtar_common.utils import (
- is_downloadable,
clean_session_cache,
+ get_file_from_link,
+ is_downloadable,
max_size_help,
max_value_current_year,
reverse_coordinates,
@@ -215,8 +216,8 @@ class BaseImportForm(IshtarForm, forms.ModelForm):
error_css_class = "error"
required_css_class = "required"
importer_type = "tab"
- imported_images_link = forms.URLField(
- label=_("Associated images (web link to a zip file)"), required=False
+ imported_media_link = forms.URLField(
+ label=_("Associated media (web link to a zip file or a path)"), required=False
)
class Meta:
@@ -230,7 +231,7 @@ class BaseImportForm(IshtarForm, forms.ModelForm):
"encoding",
"csv_sep",
"imported_images",
- "imported_images_link",
+ "imported_media_link",
)
widgets = {
"imported_file": widgets.BSClearableFileInput,
@@ -337,20 +338,9 @@ class BaseImportForm(IshtarForm, forms.ModelForm):
return data
-def get_archive_from_link(archive_link):
- request = requests.get(archive_link, stream=True)
- ntf = tempfile.NamedTemporaryFile()
- for block in request.iter_content(1024 * 8):
- if not block:
- break
- ntf.write(block)
- file_name = archive_link.split("/")[-1]
- return file_name, ntf
-
-
class NewImportForm(BaseImportForm):
- imported_images_link = forms.URLField(
- label=_("Associated documents (web link to a zip file)"), required=False
+ imported_media_link = forms.URLField(
+ label=_("Associated media (web link to a zip file or a path)"), required=False
)
class Meta:
@@ -364,7 +354,7 @@ class NewImportForm(BaseImportForm):
"encoding",
"csv_sep",
"imported_images",
- "imported_images_link",
+ "imported_media_link",
)
HEADERS = {
@@ -373,16 +363,20 @@ class NewImportForm(BaseImportForm):
"imported_images": FormHeader(_("Documents/Images")),
}
+ def __init__(self, *args, **kwargs):
+ self.media_link_is_zip = False
+ super().__init__(*args, **kwargs)
+
def _need_archive(self, data):
tpe = data["importer_type"]
return tpe.archive_required
def clean(self):
data = super().clean()
- if data.get("imported_images_link", None) and data.get("imported_images", None):
+ if data.get("imported_media_link", None) and data.get("imported_images", None):
raise forms.ValidationError(
_(
- "You put either a file or a download link for images "
+ "You put either a file or a download link for media "
"but not both."
)
)
@@ -400,14 +394,14 @@ class NewImportForm(BaseImportForm):
archive_required = self._need_archive(data)
if archive_required and (
not data.get("imported_images", None)
- and not data.get("imported_images_link", None)
+ and not data.get("imported_media_link", None)
):
raise forms.ValidationError(_("This importer need a document archive."))
return data
def clean_imported_images_link(self):
- value = self.cleaned_data.get("imported_images_link", None)
- if value:
+ value = self.cleaned_data.get("imported_media_link", None)
+ if value and value.lower().endswith(".zip"):
try:
if not is_downloadable(value):
raise forms.ValidationError("")
@@ -415,20 +409,24 @@ class NewImportForm(BaseImportForm):
raise forms.ValidationError(
_("Invalid link or no file is available for this link.")
)
+ self.media_link_is_zip = True
return value
def save(self, user, commit=True):
self.instance.user = user
- imported_images_link = (
- self.cleaned_data.pop("imported_images_link")
- if "imported_images_link" in self.cleaned_data
- else None
- )
item = super().save(commit)
- if not imported_images_link:
+ if not self.media_link_is_zip:
return item
- file_name, temp_file = get_archive_from_link(imported_images_link)
+ try:
+ file_name, temp_file = get_file_from_link(item.imported_media_link)
+ except ValueError:
+ raise forms.ValidationError(
+ _("Bad link for the archive.")
+ )
item.imported_images.save(file_name, File(temp_file))
+ # media is downloaded - clean the link
+ item.imported_media_link = None
+ item.save()
return item
@@ -511,7 +509,7 @@ class NewImportGroupForm(NewImportForm):
"encoding",
"csv_sep",
"imported_images",
- "imported_images_link",
+ "imported_media_link",
)
HEADERS = {
diff --git a/ishtar_common/migrations/0244_imports_media_link.py b/ishtar_common/migrations/0244_imports_media_link.py
new file mode 100644
index 000000000..88a36049f
--- /dev/null
+++ b/ishtar_common/migrations/0244_imports_media_link.py
@@ -0,0 +1,87 @@
+# Generated by Django 2.2.24 on 2024-04-18 17:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('ishtar_common', '0243_default_biographicalnote_permissions'),
+ ]
+
+ operations = [
+ migrations.AlterModelOptions(
+ name='biographicalnote',
+ options={'permissions': (('view_own_biographicalnote', 'Can view own Biographical note'), ('add_own_biographicalnote', 'Can add own Biographical note'), ('change_own_biographicalnote', 'Can change own Biographical note'), ('delete_own_biographicalnote', 'Can delete own Biographical note')), 'verbose_name': 'Biographical note', 'verbose_name_plural': 'Biographical notes'},
+ ),
+ migrations.AddField(
+ model_name='import',
+ name='imported_media_link',
+ field=models.URLField(blank=True, null=True, verbose_name='Associated media (web link to a zip file or a path'),
+ ),
+ migrations.AddField(
+ model_name='importgroup',
+ name='imported_media_link',
+ field=models.URLField(blank=True, null=True, verbose_name='Associated media (web link to a zip file or a path'),
+ ),
+ migrations.AlterField(
+ model_name='biographicalnote',
+ name='imports',
+ field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_biographicalnote', to='ishtar_common.Import', verbose_name='Created by imports'),
+ ),
+ migrations.AlterField(
+ model_name='biographicalnote',
+ name='imports_updated',
+ field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_biographicalnote', to='ishtar_common.Import', verbose_name='Updated by imports'),
+ ),
+ migrations.AlterField(
+ model_name='document',
+ name='imports',
+ field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_document', to='ishtar_common.Import', verbose_name='Created by imports'),
+ ),
+ migrations.AlterField(
+ model_name='document',
+ name='imports_updated',
+ field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_document', to='ishtar_common.Import', verbose_name='Updated by imports'),
+ ),
+ migrations.AlterField(
+ model_name='geovectordata',
+ name='imports',
+ field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_geovectordata', to='ishtar_common.Import', verbose_name='Created by imports'),
+ ),
+ migrations.AlterField(
+ model_name='geovectordata',
+ name='imports_updated',
+ field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_geovectordata', to='ishtar_common.Import', verbose_name='Updated by imports'),
+ ),
+ migrations.AlterField(
+ model_name='organization',
+ name='imports',
+ field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_organization', to='ishtar_common.Import', verbose_name='Created by imports'),
+ ),
+ migrations.AlterField(
+ model_name='organization',
+ name='imports_updated',
+ field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_organization', to='ishtar_common.Import', verbose_name='Updated by imports'),
+ ),
+ migrations.AlterField(
+ model_name='person',
+ name='imports',
+ field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_person', to='ishtar_common.Import', verbose_name='Created by imports'),
+ ),
+ migrations.AlterField(
+ model_name='person',
+ name='imports_updated',
+ field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_person', to='ishtar_common.Import', verbose_name='Updated by imports'),
+ ),
+ migrations.AlterField(
+ model_name='town',
+ name='imports',
+ field=models.ManyToManyField(blank=True, related_name='imported_ishtar_common_town', to='ishtar_common.Import', verbose_name='Created by imports'),
+ ),
+ migrations.AlterField(
+ model_name='town',
+ name='imports_updated',
+ field=models.ManyToManyField(blank=True, related_name='import_updated_ishtar_common_town', to='ishtar_common.Import', verbose_name='Updated by imports'),
+ ),
+ ]
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 22890d484..a37fd2afe 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -1428,6 +1428,10 @@ class BaseImport(models.Model, OwnPerms, SheetItem):
default=1,
help_text=_("Number of header lines in your file (can be 0 and should be 0 for geopackage or Shapefile)."),
)
+ imported_media_link = models.URLField(
+ _("Associated media (web link to a zip file or a path)"),
+ blank=True, null=True
+ )
creation_date = models.DateTimeField(
_("Creation date"), auto_now_add=True, blank=True, null=True
)
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index f734a9b2d..08ef84831 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -644,6 +644,23 @@ def is_downloadable(curl):
return True
+def get_file_from_link(file_link):
+ """
+ return filename and temp_file object from a web link
+ """
+ try:
+ request = requests.get(file_link, stream=True)
+ except requests.exceptions.RequestException:
+ raise ValueError()
+ ntf = tempfile.NamedTemporaryFile()
+ for block in request.iter_content(1024 * 8):
+ if not block:
+ break
+ ntf.write(block)
+ file_name = file_link.split("/")[-1]
+ return file_name, ntf
+
+
def get_current_year():
return datetime.datetime.now().year
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-16 20:41:09 +0000