diff options
35 files changed, 898 insertions, 560 deletions
diff --git a/archaeological_context_records/models.py b/archaeological_context_records/models.py index a5608c8d0..df6d22ff2 100644 --- a/archaeological_context_records/models.py +++ b/archaeological_context_records/models.py @@ -25,10 +25,10 @@ from django.conf import settings from django.contrib.gis.db import models from django.contrib.gis.geos import Point from django.contrib.postgres.indexes import GinIndex -from django.core.urlresolvers import reverse from django.db import connection from django.db.models import Q from django.db.models.signals import post_delete, post_save, m2m_changed +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _, pgettext_lazy, pgettext from django.utils.text import slugify @@ -100,7 +100,9 @@ post_delete.connect(post_save_cache, sender=DatingQuality) class Dating(models.Model): uuid = models.UUIDField(default=uuid.uuid4) - period = models.ForeignKey(Period, verbose_name=_("Period")) + period = models.ForeignKey( + Period, verbose_name=_("Period"), on_delete=models.PROTECT + ) start_date = models.IntegerField(_("Start date"), blank=True, null=True) end_date = models.IntegerField(_("End date"), blank=True, null=True) dating_type = models.ForeignKey( @@ -638,7 +640,10 @@ class ContextRecord( null=True, ) operation = models.ForeignKey( - Operation, verbose_name=_("Operation"), related_name="context_record" + Operation, + verbose_name=_("Operation"), + related_name="context_record", + on_delete=models.CASCADE, ) archaeological_site = models.ForeignKey( ArchaeologicalSite, @@ -1222,9 +1227,13 @@ class RecordRelationsManager(models.Manager): class RecordRelations(GeneralRecordRelations, models.Model): MAIN_ATTR = "left_record" - left_record = models.ForeignKey(ContextRecord, related_name="right_relations") - right_record = models.ForeignKey(ContextRecord, related_name="left_relations") - relation_type = models.ForeignKey(RelationType) + left_record = models.ForeignKey( + ContextRecord, related_name="right_relations", on_delete=models.CASCADE + ) + right_record = models.ForeignKey( + ContextRecord, related_name="left_relations", on_delete=models.CASCADE + ) + relation_type = models.ForeignKey(RelationType, on_delete=models.PROTECT) objects = RecordRelationsManager() TABLE_COLS = [ "left_record__label", diff --git a/archaeological_context_records/tests.py b/archaeological_context_records/tests.py index 8e1a14ed4..353716ed2 100644 --- a/archaeological_context_records/tests.py +++ b/archaeological_context_records/tests.py @@ -26,9 +26,9 @@ from django.apps import apps from django.conf import settings from django.contrib.auth.models import Permission from django.core.exceptions import ValidationError, ImproperlyConfigured -from django.core.urlresolvers import reverse from django.test import tag from django.test.client import Client +from django.urls import reverse from django.utils.translation import pgettext_lazy from ishtar_common.models import ( diff --git a/archaeological_context_records/views.py b/archaeological_context_records/views.py index 8d6030af3..da2dde835 100644 --- a/archaeological_context_records/views.py +++ b/archaeological_context_records/views.py @@ -19,10 +19,10 @@ import json -from django.core.urlresolvers import reverse from django.db.models import Q from django.http import HttpResponse, HttpResponseRedirect, Http404 from django.shortcuts import redirect +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _ from django.views.generic import RedirectView diff --git a/archaeological_files/models.py b/archaeological_files/models.py index c8a328633..3a6c2d8c5 100644 --- a/archaeological_files/models.py +++ b/archaeological_files/models.py @@ -28,7 +28,7 @@ from django.core.cache import cache from django.core.validators import MinValueValidator, MaxValueValidator from django.db.models import Q, Count, Sum, Max from django.db.models.signals import post_save, m2m_changed, post_delete -from django.core.urlresolvers import reverse +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _, pgettext_lazy from ishtar_common.utils import ( @@ -107,7 +107,7 @@ class Job(GeneralType): null=True, verbose_name=_("Child"), help_text=_("Auto-add this job when a parent is added"), - related_name="parents", + related_name="parents", on_delete=models.SET_NULL, ) class Meta: @@ -166,7 +166,7 @@ class GenericEquipmentServiceType(GeneralType): class EquipmentServiceType(GeneralType): generic_equipment_type = models.ForeignKey( - GenericEquipmentServiceType, verbose_name=_("Generic type") + GenericEquipmentServiceType, verbose_name=_("Generic type"), on_delete=models.CASCADE ) order = models.IntegerField(_("Order"), default=10) @@ -202,7 +202,7 @@ class EquipmentServiceCost(models.Model): on_delete=models.CASCADE ) equipment_service_type = models.ForeignKey( - EquipmentServiceType, verbose_name=_("Equipment/Service") + EquipmentServiceType, verbose_name=_("Equipment/Service"), on_delete=models.CASCADE ) slug = models.SlugField( _("Textual ID"), @@ -532,7 +532,7 @@ class File( _("External ID is set automatically"), default=False ) name = models.TextField(_("Name"), blank=True, default="") - file_type = models.ForeignKey(FileType, verbose_name=_("File type")) + file_type = models.ForeignKey(FileType, verbose_name=_("File type"), on_delete=models.PROTECT) in_charge = models.ForeignKey( Person, related_name="file_responsability", @@ -1313,7 +1313,7 @@ class FileByDepartment(models.Model): DELETE_SQL = """ DROP VIEW IF EXISTS file_department; """ - file = models.ForeignKey(File, verbose_name=_("File")) + file = models.ForeignKey(File, verbose_name=_("File"), on_delete=models.DO_NOTHING) department = models.ForeignKey( Department, verbose_name=_("Department"), @@ -1505,8 +1505,8 @@ class ManDays(models.Model): class PreventiveFileGroundJob(ManDays): - file = models.ForeignKey(File, related_name="ground_jobs") - job = models.ForeignKey(Job, verbose_name=_("Job")) + file = models.ForeignKey(File, related_name="ground_jobs", on_delete=models.CASCADE) + job = models.ForeignKey(Job, on_delete=models.CASCADE, verbose_name=_("Job")) class Meta: ordering = ("job",) @@ -1521,8 +1521,8 @@ class PreventiveFileGroundJob(ManDays): class PreventiveFileJob(ManDays): - file = models.ForeignKey(File, related_name="jobs") - job = models.ForeignKey(Job, verbose_name=_("Job")) + file = models.ForeignKey(File, related_name="jobs", on_delete=models.CASCADE) + job = models.ForeignKey(Job, on_delete=models.CASCADE, verbose_name=_("Job")) class Meta: ordering = ("job",) @@ -1537,8 +1537,8 @@ class PreventiveFileJob(ManDays): class PreventiveFileEquipmentServiceCost(models.Model): - file = models.ForeignKey(File, related_name="equipment_costs") - equipment_service_cost = models.ForeignKey(EquipmentServiceCost) + file = models.ForeignKey(File, related_name="equipment_costs", on_delete=models.CASCADE) + equipment_service_cost = models.ForeignKey(EquipmentServiceCost, on_delete=models.CASCADE) quantity_by_day_planned = models.FloatField( _("Quantity by day - planned"), null=True, blank=True ) diff --git a/archaeological_files/tests.py b/archaeological_files/tests.py index fc9483830..947f328a8 100644 --- a/archaeological_files/tests.py +++ b/archaeological_files/tests.py @@ -22,8 +22,8 @@ import json from django.conf import settings from django.contrib.auth.models import User -from django.core.urlresolvers import reverse from django.test.client import Client +from django.urls import reverse from ishtar_common.tests import ( TestCase, diff --git a/archaeological_files/views.py b/archaeological_files/views.py index eaeccd1ad..95aa0c9c1 100644 --- a/archaeological_files/views.py +++ b/archaeological_files/views.py @@ -20,12 +20,12 @@ import json import re -from django.core.urlresolvers import reverse from django.db.models import Q, F from django.http import HttpResponse, Http404, HttpResponseRedirect from django.forms.formsets import formset_factory from django.views.generic.edit import UpdateView from django.shortcuts import redirect, render +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _ from ishtar_common.views import wizard_is_available diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py index 2c2fffd00..63afb9dff 100644 --- a/archaeological_finds/models_finds.py +++ b/archaeological_finds/models_finds.py @@ -25,11 +25,11 @@ from django.apps import apps from django.conf import settings from django.contrib.gis.db import models from django.contrib.postgres.indexes import GinIndex -from django.core.urlresolvers import reverse from django.db import connection from django.db.models import Max, Q, F from django.db.models.signals import m2m_changed, post_save, post_delete, pre_delete from django.core.exceptions import ObjectDoesNotExist +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _, pgettext_lazy from ishtar_common.data_importer import post_importer_action, ImporterError @@ -407,7 +407,10 @@ class BaseFind( _("Special interest"), blank=True, default="", max_length=120 ) context_record = models.ForeignKey( - ContextRecord, related_name="base_finds", verbose_name=_("Context Record") + ContextRecord, + related_name="base_finds", + verbose_name=_("Context Record"), + on_delete=models.CASCADE, ) discovery_date = models.DateField( _("Discovery date (exact or TPQ)"), blank=True, null=True @@ -3244,12 +3247,17 @@ class FindInsideContainer(models.Model): SLUG = "find_inside_container" find = models.OneToOneField( - Find, verbose_name=_("Find"), related_name="inside_container", primary_key=True + Find, + verbose_name=_("Find"), + related_name="inside_container", + primary_key=True, + on_delete=models.DO_NOTHING, ) container = models.ForeignKey( "archaeological_warehouse.Container", verbose_name=_("Container"), related_name="container_content", + on_delete=models.DO_NOTHING, ) class Meta: @@ -3262,12 +3270,17 @@ for attr in Find.HISTORICAL_M2M: class Property(LightHistorizedItem): - find = models.ForeignKey(Find, verbose_name=_("Find")) + find = models.ForeignKey(Find, verbose_name=_("Find"), on_delete=models.CASCADE) administrative_act = models.ForeignKey( - AdministrativeAct, verbose_name=_("Administrative act") + AdministrativeAct, + verbose_name=_("Administrative act"), + on_delete=models.CASCADE, ) person = models.ForeignKey( - Person, verbose_name=_("Person"), related_name="properties" + Person, + verbose_name=_("Person"), + related_name="properties", + on_delete=models.CASCADE, ) start_date = models.DateField(_("Start date")) end_date = models.DateField(_("End date")) diff --git a/archaeological_finds/models_treatments.py b/archaeological_finds/models_treatments.py index 5d4597b19..7cbf0334a 100644 --- a/archaeological_finds/models_treatments.py +++ b/archaeological_finds/models_treatments.py @@ -22,10 +22,10 @@ import datetime from django.conf import settings from django.contrib.gis.db import models from django.contrib.postgres.indexes import GinIndex -from django.core.urlresolvers import reverse from django.db.models import Max, Q from django.db.models.signals import post_save, post_delete, pre_delete, m2m_changed from django.template.defaultfilters import slugify +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _, pgettext_lazy from archaeological_finds.models_finds import Find, FindBasket, TreatmentType @@ -208,7 +208,10 @@ class Treatment( TreatmentType, verbose_name=_("Treatment type") ) treatment_state = models.ForeignKey( - TreatmentState, verbose_name=_("State"), default=TreatmentState.get_default + TreatmentState, + verbose_name=_("State"), + default=TreatmentState.get_default, + on_delete=models.PROTECT, ) executed = models.BooleanField(_("Treatment have been executed"), default=False) location = models.ForeignKey( @@ -728,10 +731,16 @@ for attr in Treatment.HISTORICAL_M2M: class AbsFindTreatments(models.Model): find = models.ForeignKey( - Find, verbose_name=_("Find"), related_name="%(class)s_related" + Find, + verbose_name=_("Find"), + related_name="%(class)s_related", + on_delete=models.DO_NOTHING, ) treatment = models.OneToOneField( - Treatment, verbose_name=_("Treatment"), primary_key=True + Treatment, + verbose_name=_("Treatment"), + primary_key=True, + on_delete=models.DO_NOTHING, ) # primary_key is set to prevent django to ask for an id column # treatment is not a real primary key @@ -981,7 +990,9 @@ class FindTreatments(AbsFindTreatments): class TreatmentFileType(GeneralType): - treatment_type = models.ForeignKey(TreatmentType, blank=True, null=True) + treatment_type = models.ForeignKey( + TreatmentType, blank=True, null=True, on_delete=models.SET_NULL + ) class Meta: verbose_name = _("Treatment request type") @@ -1091,7 +1102,9 @@ class TreatmentFile( ) name = models.TextField(_("Name"), blank=True, default="") type = models.ForeignKey( - TreatmentFileType, verbose_name=_("Treatment request type") + TreatmentFileType, + verbose_name=_("Treatment request type"), + on_delete=models.PROTECT, ) in_charge = models.ForeignKey( Person, diff --git a/archaeological_finds/tests.py b/archaeological_finds/tests.py index 0949972b3..3fd7d56e2 100644 --- a/archaeological_finds/tests.py +++ b/archaeological_finds/tests.py @@ -33,9 +33,9 @@ from django.contrib.auth.models import User, Permission, ContentType from django.contrib.gis.geos import GEOSGeometry from django.core.files import File from django.core.files.uploadedfile import SimpleUploadedFile -from django.core.urlresolvers import reverse from django.test import tag from django.test.client import Client +from django.urls import reverse from ishtar_common.models import ( ImporterType, IshtarUser, diff --git a/archaeological_finds/views.py b/archaeological_finds/views.py index 3a8d291ab..4f5e64475 100644 --- a/archaeological_finds/views.py +++ b/archaeological_finds/views.py @@ -26,10 +26,11 @@ from rest_framework.response import Response from django.conf import settings from django.core.exceptions import PermissionDenied -from django.core.urlresolvers import reverse from django.db.models import Q from django.http import HttpResponseRedirect, HttpResponse, Http404 from django.shortcuts import redirect +from django.urls import reverse + from ishtar_common.utils import ugettext_lazy as _ from django.views.generic import TemplateView from django.views.generic.edit import CreateView, FormView diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py index 731d6cb34..568917c97 100644 --- a/archaeological_operations/models.py +++ b/archaeological_operations/models.py @@ -28,11 +28,11 @@ from django.contrib.gis.db import models from django.contrib.gis.db.models.aggregates import Union from django.contrib.gis.db.models.functions import Centroid from django.contrib.postgres.indexes import GinIndex -from django.core.urlresolvers import reverse from django.db import IntegrityError, transaction from django.db.models import Q, Count, Sum, Max, Avg from django.db.models.signals import post_save, m2m_changed, post_delete from django.forms import ValidationError +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _, pgettext_lazy from ishtar_common.models import ( @@ -130,7 +130,7 @@ post_save.connect(post_save_cache, sender=ReportState) post_delete.connect(post_save_cache, sender=ReportState) -class SiteManager(models.GeoManager): +class SiteManager(models.Manager): def get_by_natural_key(self, txt_idx): return self.get(reference=txt_idx) @@ -1174,7 +1174,10 @@ class Operation( null=True, ) operation_type = models.ForeignKey( - OperationType, related_name="+", verbose_name=_("Operation type") + OperationType, + related_name="+", + verbose_name=_("Operation type"), + on_delete=models.PROTECT, ) surface = models.FloatField(_("Surface (m2)"), blank=True, null=True) remains = models.ManyToManyField( @@ -1322,6 +1325,7 @@ class Operation( blank=True, null=True, related_name="operation_protagonist", + on_delete=models.SET_NULL, ) applicant_authority = models.ForeignKey( Organization, @@ -1329,6 +1333,7 @@ class Operation( blank=True, null=True, related_name="operation_applicant_authority", + on_delete=models.SET_NULL, ) minutes_writer = models.ForeignKey( Person, @@ -1336,6 +1341,7 @@ class Operation( blank=True, null=True, related_name="minutes_writer", + on_delete=models.SET_NULL, ) cached_towns_label = models.TextField( _("Cached town label"), @@ -2122,9 +2128,13 @@ class OperationRecordRelationManager(models.Manager): class RecordRelations(GeneralRecordRelations, models.Model): MAIN_ATTR = "left_record" - left_record = models.ForeignKey(Operation, related_name="right_relations") - right_record = models.ForeignKey(Operation, related_name="left_relations") - relation_type = models.ForeignKey(RelationType) + left_record = models.ForeignKey( + Operation, related_name="right_relations", on_delete=models.CASCADE + ) + right_record = models.ForeignKey( + Operation, related_name="left_relations", on_delete=models.CASCADE + ) + relation_type = models.ForeignKey(RelationType, on_delete=models.PROTECT) objects = OperationRecordRelationManager() class Meta: @@ -2171,7 +2181,9 @@ class OperationByDepartment(models.Model): DROP VIEW IF EXISTS operation_department; """ - operation = models.ForeignKey(Operation, verbose_name=_("Operation")) + operation = models.ForeignKey( + Operation, verbose_name=_("Operation"), on_delete=models.DO_NOTHING + ) department = models.ForeignKey( Department, verbose_name=_("Department"), @@ -2462,7 +2474,9 @@ class AdministrativeAct(DocumentItem, BaseHistorizedItem, OwnPerms, ValueGetter) } # fields - act_type = models.ForeignKey(ActType, verbose_name=_("Act type")) + act_type = models.ForeignKey( + ActType, verbose_name=_("Act type"), on_delete=models.PROTECT + ) in_charge = models.ForeignKey( Person, blank=True, @@ -2502,6 +2516,7 @@ class AdministrativeAct(DocumentItem, BaseHistorizedItem, OwnPerms, ValueGetter) null=True, related_name="administrative_act", verbose_name=_("Operation"), + on_delete=models.CASCADE, ) associated_file = models.ForeignKey( "archaeological_files.File", @@ -2509,6 +2524,7 @@ class AdministrativeAct(DocumentItem, BaseHistorizedItem, OwnPerms, ValueGetter) null=True, related_name="administrative_act", verbose_name=_("Archaeological file"), + on_delete=models.CASCADE, ) treatment_file = models.ForeignKey( "archaeological_finds.TreatmentFile", @@ -2516,6 +2532,7 @@ class AdministrativeAct(DocumentItem, BaseHistorizedItem, OwnPerms, ValueGetter) null=True, related_name="administrative_act", verbose_name=_("Treatment request"), + on_delete=models.CASCADE, ) treatment = models.ForeignKey( "archaeological_finds.Treatment", @@ -2523,6 +2540,7 @@ class AdministrativeAct(DocumentItem, BaseHistorizedItem, OwnPerms, ValueGetter) null=True, related_name="administrative_act", verbose_name=_("Treatment"), + on_delete=models.CASCADE, ) signature_date = models.DateField(_("Signature date"), blank=True, null=True) year = models.IntegerField(_("Year"), blank=True, null=True) @@ -2792,7 +2810,9 @@ class Parcel(LightHistorizedItem): on_delete=models.SET_NULL, ) year = models.IntegerField(_("Year"), blank=True, null=True) - town = models.ForeignKey(Town, related_name="parcels", verbose_name=_("Town")) + town = models.ForeignKey( + Town, related_name="parcels", verbose_name=_("Town"), on_delete=models.PROTECT + ) section = models.CharField(_("Section"), max_length=4, null=True, blank=True) parcel_number = models.CharField( _("Parcel number"), max_length=6, null=True, blank=True @@ -3050,9 +3070,17 @@ post_save.connect(parcel_post_save, sender=Parcel) class ParcelOwner(LightHistorizedItem): uuid = models.UUIDField(default=uuid.uuid4) owner = models.ForeignKey( - Person, verbose_name=_("Owner"), related_name="parcel_owner" + Person, + verbose_name=_("Owner"), + related_name="parcel_owner", + on_delete=models.PROTECT, + ) + parcel = models.ForeignKey( + Parcel, + verbose_name=_("Parcel"), + related_name="owners", + on_delete=models.CASCADE, ) - parcel = models.ForeignKey(Parcel, verbose_name=_("Parcel"), related_name="owners") start_date = models.DateField(_("Start date")) end_date = models.DateField(_("End date")) objects = UUIDModelManager() diff --git a/archaeological_operations/tests.py b/archaeological_operations/tests.py index e7cbe670c..e70b701c4 100644 --- a/archaeological_operations/tests.py +++ b/archaeological_operations/tests.py @@ -30,10 +30,10 @@ from django.conf import settings from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType from django.core.files.uploadedfile import SimpleUploadedFile -from django.core.urlresolvers import reverse from django.db.models import Q from django.test import tag from django.test.client import Client +from django.urls import reverse from django.utils.text import slugify from django.contrib.auth.models import User, Permission diff --git a/archaeological_operations/views.py b/archaeological_operations/views.py index 51cedbc9c..ec741e434 100644 --- a/archaeological_operations/views.py +++ b/archaeological_operations/views.py @@ -21,10 +21,10 @@ import json from jinja2 import TemplateSyntaxError from django.conf import settings -from django.core.urlresolvers import reverse from django.db.models import Q from django.http import HttpResponse, HttpResponseRedirect, Http404 from django.shortcuts import render, redirect +from django.urls import reverse from django.views.generic import RedirectView from ishtar_common.utils import ugettext_lazy as _, pgettext_lazy diff --git a/archaeological_operations/wizards.py b/archaeological_operations/wizards.py index a58e7437d..7afe73442 100644 --- a/archaeological_operations/wizards.py +++ b/archaeological_operations/wizards.py @@ -21,9 +21,9 @@ import logging from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.core.urlresolvers import reverse from django.http import Http404 from django.shortcuts import render +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _ from archaeological_files.models import File diff --git a/archaeological_warehouse/models.py b/archaeological_warehouse/models.py index 66386113b..ccc16d206 100644 --- a/archaeological_warehouse/models.py +++ b/archaeological_warehouse/models.py @@ -25,10 +25,10 @@ from django.conf import settings from django.contrib.gis.db import models from django.contrib.postgres.indexes import GinIndex from django.core.exceptions import ObjectDoesNotExist -from django.core.urlresolvers import reverse from django.db.models import Q, Max, Count from django.db.models.signals import post_save, post_delete, m2m_changed, pre_delete from django.template.defaultfilters import slugify +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _, pgettext_lazy from django.apps import apps @@ -344,7 +344,9 @@ class Warehouse( uuid = models.UUIDField(default=uuid.uuid4) name = models.CharField(_("Name"), max_length=200) - warehouse_type = models.ForeignKey(WarehouseType, verbose_name=_("Warehouse type")) + warehouse_type = models.ForeignKey( + WarehouseType, verbose_name=_("Warehouse type"), on_delete=models.PROTECT + ) person_in_charge = models.ForeignKey( "ishtar_common.Person", on_delete=models.SET_NULL, @@ -667,10 +669,18 @@ class WarehouseDivisionLink(models.Model): RELATED_SET_NAME = "divisions" RELATED_ATTRS = ["order", "container_type"] RELATIVE_MODELS = {Warehouse: "warehouse"} - warehouse = models.ForeignKey(Warehouse, related_name="divisions") - container_type = models.ForeignKey(ContainerType, blank=True, null=True) + warehouse = models.ForeignKey( + Warehouse, related_name="divisions", on_delete=models.CASCADE + ) + container_type = models.ForeignKey( + ContainerType, blank=True, null=True, on_delete=models.PROTECT + ) division = models.ForeignKey( - WarehouseDivision, help_text=_("Deprecated - do not use"), blank=True, null=True + WarehouseDivision, + help_text=_("Deprecated - do not use"), + blank=True, + null=True, + on_delete=models.SET_NULL, ) order = models.IntegerField(_("Order"), default=10) objects = WarehouseDivisionLinkManager() @@ -744,11 +754,13 @@ class ContainerTree(models.Model): verbose_name=_("Container"), related_name="container_tree_child", primary_key=True, + on_delete=models.DO_NOTHING, ) container_parent = models.ForeignKey( "archaeological_warehouse.Container", verbose_name=_("Container parent"), related_name="container_tree_parent", + on_delete=models.DO_NOTHING, ) class Meta: @@ -1011,7 +1023,10 @@ class Container( # fields uuid = models.UUIDField(default=uuid.uuid4) location = models.ForeignKey( - Warehouse, verbose_name=_("Location (warehouse)"), related_name="containers" + Warehouse, + verbose_name=_("Location (warehouse)"), + related_name="containers", + on_delete=models.CASCADE, ) responsible = models.ForeignKey( Warehouse, @@ -1020,6 +1035,7 @@ class Container( blank=True, null=True, help_text=_("Deprecated - do not use"), + on_delete=models.SET_NULL, ) responsibility = models.ForeignKey( Warehouse, @@ -1028,9 +1044,13 @@ class Container( blank=True, null=True, help_text=_("Warehouse that owns the container"), + on_delete=models.SET_NULL, ) container_type = models.ForeignKey( - ContainerType, verbose_name=_("Container type"), related_name="containers" + ContainerType, + verbose_name=_("Container type"), + related_name="containers", + on_delete=models.PROTECT, ) reference = models.TextField(_("Container ref.")) comment = models.TextField(_("Comment"), blank=True, default="") @@ -1868,9 +1888,14 @@ class ContainerLocalisationManager(models.Manager): class ContainerLocalisation(models.Model): # TODO: to be deleted.... container = models.ForeignKey( - Container, verbose_name=_("Container"), related_name="division" + Container, + verbose_name=_("Container"), + related_name="division", + on_delete=models.CASCADE, + ) + division = models.ForeignKey( + WarehouseDivisionLink, verbose_name=_("Division"), on_delete=models.CASCADE ) - division = models.ForeignKey(WarehouseDivisionLink, verbose_name=_("Division")) reference = models.CharField(_("Reference"), max_length=200, default="") objects = ContainerLocalisationManager() TO_BE_DELETED = True diff --git a/archaeological_warehouse/tests.py b/archaeological_warehouse/tests.py index a4336d409..d84fd5fb9 100644 --- a/archaeological_warehouse/tests.py +++ b/archaeological_warehouse/tests.py @@ -20,9 +20,9 @@ import json from django.contrib.auth.models import Permission -from django.core.urlresolvers import reverse from django.db.utils import IntegrityError from django.test.client import Client +from django.urls import reverse from archaeological_finds.tests import FindInit diff --git a/archaeological_warehouse/views.py b/archaeological_warehouse/views.py index 9b8fe1728..cb389e6e5 100644 --- a/archaeological_warehouse/views.py +++ b/archaeological_warehouse/views.py @@ -21,12 +21,12 @@ import json import itertools from unidecode import unidecode -from django.core.urlresolvers import reverse from django.db.models import Q from django.views.generic import RedirectView from django.views.generic.edit import FormView from django.http import HttpResponse, Http404, HttpResponseRedirect from django.shortcuts import redirect +from django.urls import reverse from ishtar_common.utils import ugettext_lazy as _ from archaeological_warehouse import models diff --git a/example_project/settings.py b/example_project/settings.py index 53f15cc38..16a63d49a 100644 --- a/example_project/settings.py +++ b/example_project/settings.py @@ -154,7 +154,7 @@ ROOT_URLCONF = "example_project.urls" AUTHENTICATION_BACKENDS = ("ishtar_common.backend.ObjectPermBackend",) INSTALLED_APPS = [ - "registration", + "django_registration", "ishtar_common", "archaeological_files_pdl", "archaeological_files", diff --git a/example_project/urls.py b/example_project/urls.py index 79645b8ae..8ccc28815 100644 --- a/example_project/urls.py +++ b/example_project/urls.py @@ -11,7 +11,7 @@ admin.autodiscover() urlpatterns = [ - url(r'^accounts/', include('registration.urls')), + url(r'^accounts/', include('django_registration.urls')), ] diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py index 16d3bd2f1..752594616 100644 --- a/ishtar_common/admin.py +++ b/ishtar_common/admin.py @@ -48,7 +48,6 @@ from django.contrib.gis.geos.error import GEOSException from django.core.cache import cache from django.core.exceptions import FieldError from django.core.serializers import serialize -from django.core.urlresolvers import reverse from django.db.models.fields import ( BooleanField, IntegerField, @@ -60,6 +59,7 @@ from django.db.models.fields.related import ForeignKey from django.forms import BaseInlineFormSet from django.http import HttpResponseRedirect, HttpResponse from django.shortcuts import render +from django.urls import reverse from django.utils.decorators import method_decorator from django.utils.text import slugify, mark_safe from django.utils.translation import ugettext_lazy as _ diff --git a/ishtar_common/forms.py b/ishtar_common/forms.py index d31febd95..aede44637 100644 --- a/ishtar_common/forms.py +++ b/ishtar_common/forms.py @@ -30,9 +30,9 @@ from django.apps import apps from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import ValidationError -from django.core.urlresolvers import reverse from django.core import validators from django.forms.formsets import BaseFormSet, DELETION_FIELD_NAME +from django.urls import reverse from django.utils import formats, translation from django.utils.functional import lazy from django.utils.safestring import mark_safe diff --git a/ishtar_common/menus.py b/ishtar_common/menus.py index 466ed18c4..b167c1765 100644 --- a/ishtar_common/menus.py +++ b/ishtar_common/menus.py @@ -26,7 +26,7 @@ import datetime from django.conf import settings from django.core.cache import cache -from django.core.urlresolvers import reverse +from django.urls import reverse from django.contrib.auth.models import User diff --git a/ishtar_common/model_managers.py b/ishtar_common/model_managers.py index 1857b2079..a0bf86040 100644 --- a/ishtar_common/model_managers.py +++ b/ishtar_common/model_managers.py @@ -2,10 +2,10 @@ # -*- coding: utf-8 -*- -from django.contrib.gis.db.models import Manager, GeoManager +from django.contrib.gis.db.models import Manager -class ExternalIdManager(GeoManager): +class ExternalIdManager(Manager): def get_by_natural_key(self, external_id): return self.get(external_id=external_id) diff --git a/ishtar_common/models.py b/ishtar_common/models.py index bf35e213e..6196e1efe 100644 --- a/ishtar_common/models.py +++ b/ishtar_common/models.py @@ -63,13 +63,13 @@ from django.core.exceptions import ( ) from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile -from django.core.urlresolvers import reverse from django.db import connection from django.db.models import Q, Max, Count from django.db.models.signals import post_save, post_delete, m2m_changed from django.db.utils import DatabaseError from django.template import Context, Template from django.template.defaultfilters import slugify +from django.urls import reverse from django.utils.functional import lazy from ishtar_common.utils import ( ugettext_lazy as _, @@ -427,14 +427,22 @@ class TinyUrl(models.Model): class ItemKey(models.Model): key = models.TextField(_("Key")) - content_type = models.ForeignKey(ContentType) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField() content_object = GenericForeignKey("content_type", "object_id") importer = models.ForeignKey( - Import, null=True, blank=True, help_text=_("Specific key to an import") + Import, + null=True, + blank=True, + help_text=_("Specific key to an import"), + on_delete=models.SET_NULL, + ) + user = models.ForeignKey( + "IshtarUser", blank=True, null=True, on_delete=models.SET_NULL + ) + group = models.ForeignKey( + TargetKeyGroup, blank=True, null=True, on_delete=models.SET_NULL ) - user = models.ForeignKey("IshtarUser", blank=True, null=True) - group = models.ForeignKey(TargetKeyGroup, blank=True, null=True) def __str__(self): return self.key @@ -658,7 +666,7 @@ class JsonDataSectionManager(models.Manager): class JsonDataSection(models.Model): - content_type = models.ForeignKey(ContentType) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) name = models.CharField(_("Name"), max_length=200) order = models.IntegerField(_("Order"), default=10) objects = JsonDataSectionManager() @@ -699,7 +707,7 @@ class JsonDataFieldManager(models.Manager): class JsonDataField(models.Model): name = models.CharField(_("Name"), max_length=200) - content_type = models.ForeignKey(ContentType) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) key = models.CharField( _("Key"), max_length=200, @@ -760,7 +768,11 @@ class GeneralRelationType(GeneralType): symmetrical = models.BooleanField(_("Symmetrical")) tiny_label = models.CharField(_("Tiny label"), max_length=50, blank=True, null=True) inverse_relation = models.ForeignKey( - "self", verbose_name=_("Inverse relation"), blank=True, null=True + "self", + verbose_name=_("Inverse relation"), + blank=True, + null=True, + on_delete=models.SET_NULL, ) logical_relation = models.CharField( verbose_name=_("Logical relation"), @@ -869,9 +881,7 @@ class RelationsViews(models.Model): if not settings.USE_BACKGROUND_TASK: return relation_view_update(cls, {"item_id": item_id}) else: - sender, kwargs = serialize_args_for_tasks( - cls, None, {"item_id": item_id} - ) + sender, kwargs = serialize_args_for_tasks(cls, None, {"item_id": item_id}) return relation_view_update.delay(sender, kwargs) @classmethod @@ -909,28 +919,30 @@ class RelationsViews(models.Model): assert cls.DELETE_SQL assert cls.CREATE_TABLE_SQL profile = get_current_profile(force=True) - table_type = '' + table_type = "" with connection.cursor() as cursor: - q = "select table_type from information_schema.tables WHERE " \ + q = ( + "select table_type from information_schema.tables WHERE " "table_name=%s;" + ) cursor.execute(q, [cls._meta.db_table]) q = cursor.fetchall() if q: table_type = q[0][0] if profile.parent_relations_engine == "V": - if table_type == 'VIEW': + if table_type == "VIEW": return - elif 'TABLE' in table_type: + elif "TABLE" in table_type: q = "DROP TABLE IF EXISTS %s" % cls._meta.db_table cursor.execute(q) cursor.execute(cls.CREATE_SQL) return True if profile.parent_relations_engine == "T": - if 'TABLE' in table_type: + if "TABLE" in table_type: return - elif table_type == 'VIEW': + elif table_type == "VIEW": cursor.execute(cls.DELETE_SQL) cursor.execute(cls.CREATE_TABLE_SQL) return True @@ -939,8 +951,12 @@ class RelationsViews(models.Model): class SearchQuery(models.Model): label = models.TextField(_("Label"), blank=True, default="") query = models.TextField(_("Query"), blank=True, default="") - content_type = models.ForeignKey(ContentType, verbose_name=_("Content type")) - profile = models.ForeignKey("UserProfile", verbose_name=_("Profile")) + content_type = models.ForeignKey( + ContentType, verbose_name=_("Content type"), on_delete=models.CASCADE + ) + profile = models.ForeignKey( + "UserProfile", verbose_name=_("Profile"), on_delete=models.CASCADE + ) is_alert = models.BooleanField(_("Is an alert"), default=False) class Meta: @@ -1037,11 +1053,13 @@ class IshtarSiteProfile(models.Model, Cached): ), default="V", max_length=1, - help_text=_("If you experience performance problems with complex relations " - "(for instance: complex statigraphic relations), set it to " - "\"Cache tables\" in order to use static cache tables. Do not " - "forget to update theses table with the " - "\"relations_update_cache_tables\" manage.py command.") + help_text=_( + "If you experience performance problems with complex relations " + "(for instance: complex statigraphic relations), set it to " + '"Cache tables" in order to use static cache tables. Do not ' + "forget to update theses table with the " + '"relations_update_cache_tables" manage.py command.' + ), ) config = models.CharField( _("Alternate configuration"), @@ -1104,8 +1122,9 @@ class IshtarSiteProfile(models.Model, Cached): _("Use town to locate when coordinates are missing"), default=True ) relation_graph = models.BooleanField(_("Generate relation graph"), default=False) - preventive_operator = models.BooleanField(_("Preventive operator module"), - default=False) + preventive_operator = models.BooleanField( + _("Preventive operator module"), default=False + ) underwater = models.BooleanField(_("Underwater module"), default=False) parcel_mandatory = models.BooleanField( _("Parcel are mandatory for context records"), default=True @@ -1385,6 +1404,7 @@ class IshtarSiteProfile(models.Model, Cached): help_text=_( "Spatial Reference System used for display when no SRS is " "defined" ), + on_delete=models.SET_NULL, ) default_language = models.ForeignKey( Language, @@ -1395,6 +1415,7 @@ class IshtarSiteProfile(models.Model, Cached): "If set, by default the selected language will be set for " "localized documents." ), + on_delete=models.SET_NULL, ) objects = SlugModelManager() @@ -1633,7 +1654,9 @@ class ExcludedFieldManager(models.Manager): class ExcludedField(models.Model): - custom_form = models.ForeignKey(CustomForm, related_name="excluded_fields") + custom_form = models.ForeignKey( + CustomForm, related_name="excluded_fields", on_delete=models.CASCADE + ) field = models.CharField(_("Field"), max_length=250) objects = ExcludedFieldManager() @@ -1665,8 +1688,12 @@ class CustomFormJsonFieldManager(models.Manager): class CustomFormJsonField(models.Model): - custom_form = models.ForeignKey(CustomForm, related_name="json_fields") - json_field = models.ForeignKey(JsonDataField, related_name="custom_form_details") + custom_form = models.ForeignKey( + CustomForm, related_name="json_fields", on_delete=models.CASCADE + ) + json_field = models.ForeignKey( + JsonDataField, related_name="custom_form_details", on_delete=models.CASCADE + ) label = models.CharField(_("Label"), max_length=200, blank=True, default="") order = models.IntegerField(verbose_name=_("Order"), default=1) help_text = models.TextField(_("Help"), blank=True, default="") @@ -1878,7 +1905,7 @@ class Dashboard(object): class DocumentTemplate(models.Model): name = models.CharField(_("Name"), max_length=100) slug = models.SlugField(_("Slug"), max_length=100, unique=True) - associated_model = models.ForeignKey(ImporterModel) + associated_model = models.ForeignKey(ImporterModel, on_delete=models.CASCADE) template = models.FileField( _("Template"), upload_to="templates/%Y/", @@ -2358,7 +2385,9 @@ class Organization(Address, Merge, OwnPerms, BaseGenderedType, ValueGetter, Main # fields uuid = models.UUIDField(default=uuid.uuid4) name = models.CharField(_("Name"), max_length=500) - organization_type = models.ForeignKey(OrganizationType, verbose_name=_("Type")) + organization_type = models.ForeignKey( + OrganizationType, verbose_name=_("Type"), on_delete=models.PROTECT + ) url = models.URLField(verbose_name=_("Web address"), blank=True, null=True) grammatical_gender = models.CharField( _("Grammatical gender"), @@ -2969,7 +2998,9 @@ class ProfileTypeSummary(ProfileType): class UserProfile(models.Model): name = models.CharField(_("Name"), blank=True, default="", max_length=100) - profile_type = models.ForeignKey(ProfileType, verbose_name=_("Profile type")) + profile_type = models.ForeignKey( + ProfileType, verbose_name=_("Profile type"), on_delete=models.PROTECT + ) areas = models.ManyToManyField( "Area", verbose_name=_("Areas"), blank=True, related_name="profiles" ) @@ -2978,7 +3009,10 @@ class UserProfile(models.Model): auto_pin = models.BooleanField(_("Automatically pin"), default=False) display_pin_menu = models.BooleanField(_("Display pin menu"), default=False) person = models.ForeignKey( - Person, verbose_name=_("Person"), related_name="profiles" + Person, + verbose_name=_("Person"), + related_name="profiles", + on_delete=models.CASCADE, ) class Meta: @@ -3122,9 +3156,14 @@ class IshtarUser(FullSearch): } # fields - user_ptr = models.OneToOneField(User, primary_key=True, related_name="ishtaruser") + user_ptr = models.OneToOneField( + User, primary_key=True, related_name="ishtaruser", on_delete=models.CASCADE + ) person = models.OneToOneField( - Person, verbose_name=_("Person"), related_name="ishtaruser" + Person, + verbose_name=_("Person"), + related_name="ishtaruser", + on_delete=models.CASCADE, ) advanced_shortcut_menu = models.BooleanField( _("Advanced shortcut menu"), default=False @@ -3343,8 +3382,15 @@ class Author(FullSearch): PARENT_SEARCH_VECTORS = ["person"] uuid = models.UUIDField(default=uuid.uuid4) - person = models.ForeignKey(Person, verbose_name=_("Person"), related_name="author") - author_type = models.ForeignKey(AuthorType, verbose_name=_("Author type")) + person = models.ForeignKey( + Person, + verbose_name=_("Person"), + related_name="author", + on_delete=models.CASCADE, + ) + author_type = models.ForeignKey( + AuthorType, verbose_name=_("Author type"), on_delete=models.PROTECT + ) cached_label = models.TextField( _("Cached name"), blank=True, default="", db_index=True ) @@ -3896,6 +3942,7 @@ class Document( blank=True, null=True, related_name="publish", + on_delete=models.SET_NULL, ) publishing_year = models.PositiveIntegerField( _("Year of publication"), blank=True, null=True @@ -3905,7 +3952,11 @@ class Document( ) tags = models.ManyToManyField(DocumentTag, verbose_name=_("Tags"), blank=True) language = models.ForeignKey( - Language, verbose_name=_("Language"), blank=True, null=True + Language, + verbose_name=_("Language"), + blank=True, + null=True, + on_delete=models.SET_NULL, ) issn = models.CharField(_("ISSN"), blank=True, null=True, max_length=10) isbn = models.CharField(_("ISBN"), blank=True, null=True, max_length=17) @@ -3915,6 +3966,7 @@ class Document( blank=True, null=True, related_name="children", + on_delete=models.SET_NULL, ) source_free_input = models.CharField( verbose_name=_("Source - free input"), @@ -4050,13 +4102,10 @@ class Document( if len(operations) != 1: return current_operation = operations[0] - q = ( - Document.objects.exclude(pk=self.pk) - .filter( - Q(operations__id=current_operation) - | Q(context_records__operation_id=current_operation) - | Q(finds__base_finds__context_record__operation_id=current_operation) - ) + q = Document.objects.exclude(pk=self.pk).filter( + Q(operations__id=current_operation) + | Q(context_records__operation_id=current_operation) + | Q(finds__base_finds__context_record__operation_id=current_operation) ) if extra_filters: q = q.filter(**extra_filters) @@ -4746,7 +4795,7 @@ SCRIPT_STATE_DCT = dict(SCRIPT_STATE) class AdministrationTask(models.Model): - script = models.ForeignKey(AdministrationScript) + script = models.ForeignKey(AdministrationScript, on_delete=models.CASCADE) state = models.CharField( _("State"), max_length=2, choices=SCRIPT_STATE, default="S" ) diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py index 22244e4e2..6b137f2fd 100644 --- a/ishtar_common/models_common.py +++ b/ishtar_common/models_common.py @@ -31,7 +31,7 @@ from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist from django.core.files import File from django.core.serializers import serialize -from django.core.urlresolvers import reverse, NoReverseMatch +from django.urls import reverse from django.core.validators import validate_slug from django.db import connection from django.db.models import Q, Count, Max @@ -1878,7 +1878,9 @@ class Department(models.Model): class Arrondissement(models.Model): name = models.CharField("Nom", max_length=30) - department = models.ForeignKey(Department, verbose_name="Département") + department = models.ForeignKey( + Department, verbose_name="Département", on_delete=models.CASCADE + ) def __str__(self): return settings.JOINT.join((self.name, str(self.department))) @@ -1886,13 +1888,15 @@ class Arrondissement(models.Model): class Canton(models.Model): name = models.CharField("Nom", max_length=30) - arrondissement = models.ForeignKey(Arrondissement, verbose_name="Arrondissement") + arrondissement = models.ForeignKey( + Arrondissement, verbose_name="Arrondissement", on_delete=models.CASCADE + ) def __str__(self): return settings.JOINT.join((self.name, str(self.arrondissement))) -class TownManager(models.GeoManager): +class TownManager(models.Manager): def get_by_natural_key(self, numero_insee, year): return self.get(numero_insee=numero_insee, year=year) @@ -2114,7 +2118,11 @@ class Address(BaseHistorizedItem): ) town = models.CharField(_("Town (freeform)"), max_length=150, null=True, blank=True) precise_town = models.ForeignKey( - Town, verbose_name=_("Town (precise)"), null=True, blank=True + Town, + verbose_name=_("Town (precise)"), + null=True, + blank=True, + on_delete=models.SET_NULL, ) country = models.CharField(_("Country"), max_length=30, null=True, blank=True) alt_address = models.TextField(_("Other address: address"), blank=True, default="") @@ -2802,6 +2810,7 @@ class GeoItem(models.Model): verbose_name=_("Spatial Reference System"), blank=True, null=True, + on_delete=models.PROTECT ) point = models.PointField(_("Point"), blank=True, null=True, dim=3) point_2d = models.PointField(_("Point (2D)"), blank=True, null=True) diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 3779b656e..54409e2c9 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -48,13 +48,32 @@ except (AssertionError, ImportError): UnoCalc = None from ishtar_common.model_managers import SlugModelManager -from ishtar_common.utils import create_slug, \ - get_all_related_m2m_objects_with_model, put_session_message, \ - put_session_var, get_session_var, num2col, max_size_help, import_class -from ishtar_common.data_importer import Importer, ImportFormater, \ - IntegerFormater, FloatFormater, UnicodeFormater, DateFormater, \ - TypeFormater, YearFormater, StrToBoolean, FileFormater, InseeFormater, \ - ImporterError, UpperCaseFormater, LowerCaseFormater +from ishtar_common.utils import ( + create_slug, + get_all_related_m2m_objects_with_model, + put_session_message, + put_session_var, + get_session_var, + num2col, + max_size_help, + import_class, +) +from ishtar_common.data_importer import ( + Importer, + ImportFormater, + IntegerFormater, + FloatFormater, + UnicodeFormater, + DateFormater, + TypeFormater, + YearFormater, + StrToBoolean, + FileFormater, + InseeFormater, + ImporterError, + UpperCaseFormater, + LowerCaseFormater, +) from ishtar_common.utils import task @@ -86,13 +105,13 @@ class ImporterModel(models.Model): class Meta: verbose_name = _("Model") verbose_name_plural = _("Models") - ordering = ('name',) + ordering = ("name",) def __str__(self): return self.name def natural_key(self): - return (self.klass, ) + return (self.klass,) class ImporterTypeManager(models.Manager): @@ -104,23 +123,32 @@ class ImporterType(models.Model): """ Description of a table to be mapped with ishtar database """ + name = models.CharField(_("Name"), max_length=200) slug = models.SlugField(_("Slug"), unique=True, max_length=100) - description = models.CharField(_("Description"), blank=True, null=True, - max_length=500) - users = models.ManyToManyField('IshtarUser', verbose_name=_("Users"), - blank=True) + description = models.CharField( + _("Description"), blank=True, null=True, max_length=500 + ) + users = models.ManyToManyField("IshtarUser", verbose_name=_("Users"), blank=True) associated_models = models.ForeignKey( - ImporterModel, verbose_name=_("Associated model"), + ImporterModel, + verbose_name=_("Associated model"), on_delete=models.SET_NULL, - related_name='importer_type_associated', blank=True, null=True) + related_name="importer_type_associated", + blank=True, + null=True, + ) created_models = models.ManyToManyField( - ImporterModel, verbose_name=_("Models that can accept new items"), - blank=True, help_text=_("Leave blank for no restrictions"), - related_name='importer_type_created') + ImporterModel, + verbose_name=_("Models that can accept new items"), + blank=True, + help_text=_("Leave blank for no restrictions"), + related_name="importer_type_created", + ) is_template = models.BooleanField(_("Can be exported"), default=False) - unicity_keys = models.CharField(_("Unicity keys (separator \";\")"), - blank=True, null=True, max_length=500) + unicity_keys = models.CharField( + _('Unicity keys (separator ";")'), blank=True, null=True, max_length=500 + ) available = models.BooleanField(_("Available"), default=True) objects = ImporterTypeManager() SERIALIZATION_EXCLUDE = ["users"] @@ -128,10 +156,10 @@ class ImporterType(models.Model): class Meta: verbose_name = _("Importer - Type") verbose_name_plural = _("Importer - Types") - ordering = ('name',) + ordering = ("name",) def natural_key(self): - return (self.slug, ) + return (self.slug,) def __str__(self): return self.name @@ -150,7 +178,7 @@ class ImporterType(models.Model): return col_number = 1 # user number so we start with 1 lst_col_number = 0 - for column in self.columns.order_by('col_number').all(): + for column in self.columns.order_by("col_number").all(): while column.col_number > col_number: col_number += 1 # header @@ -171,7 +199,7 @@ class ImporterType(models.Model): if not ft: continue # first we only manage TypeFormater - if ft.formater_type != 'TypeFormater': + if ft.formater_type != "TypeFormater": continue if not ft.options: # bad configuration continue @@ -181,11 +209,18 @@ class ImporterType(models.Model): lst = [] for typ in model.get_types(instances=True): lst.append(str(typ)) - end_row = uno.create_list(lst_sheet, lst_col_number, 0, - str(model._meta.verbose_name), lst) + end_row = uno.create_list( + lst_sheet, lst_col_number, 0, str(model._meta.verbose_name), lst + ) uno.set_cell_validation_list( - main_sheet, col_number, 2, ROW_NUMBER + 2, - lst_sheet, lst_col_number, [1, end_row]) + main_sheet, + col_number, + 2, + ROW_NUMBER + 2, + lst_sheet, + lst_col_number, + [1, end_row], + ) lst_col_number += 1 tmpdir = tempfile.mkdtemp(prefix="ishtar-templates-") dest_filename = "{}{}{}.ods".format(tmpdir, os.sep, self.name) @@ -194,12 +229,13 @@ class ImporterType(models.Model): def get_importer_class(self, import_instance=None): OBJECT_CLS = import_class(self.associated_models.klass) - DEFAULTS = dict([(default.keys, default.values) - for default in self.defaults.all()]) + DEFAULTS = dict( + [(default.keys, default.values) for default in self.defaults.all()] + ) LINE_FORMAT = [] LINE_EXPORT_FORMAT = [] idx = 0 - for column in self.columns.order_by('col_number').all(): + for column in self.columns.order_by("col_number").all(): idx += 1 while column.col_number > idx: LINE_FORMAT.append(None) @@ -212,8 +248,7 @@ class ImporterType(models.Model): LINE_FORMAT.append(None) if column.export_field_name: LINE_EXPORT_FORMAT.append( - ImportFormater(column.export_field_name, - label=column.label) + ImportFormater(column.export_field_name, label=column.label) ) continue force_news = [] @@ -221,7 +256,8 @@ class ImporterType(models.Model): concat = [] for target in column.targets.order_by("pk").all(): ft = target.formater_type.get_formater_type( - target, import_instance=import_instance) + target, import_instance=import_instance + ) if not ft: continue formater_types.append(ft) @@ -231,42 +267,46 @@ class ImporterType(models.Model): concat.append(target.concat) formater_kwargs = {} if column.regexp_pre_filter: - formater_kwargs['regexp'] = re.compile( - column.regexp_pre_filter.regexp) + formater_kwargs["regexp"] = re.compile(column.regexp_pre_filter.regexp) if column.value_format: - formater_kwargs['value_format'] = \ - column.value_format.format_string - formater_kwargs['concat'] = concat - formater_kwargs['concat_str'] = concat_str - formater_kwargs['duplicate_fields'] = [ - (field.field_name, field.force_new, field.concat, - field.concat_str) - for field in column.duplicate_fields.all()] - formater_kwargs['label'] = column.label - formater_kwargs['required'] = column.required - formater_kwargs['force_new'] = force_news - formater_kwargs['comment'] = column.description + formater_kwargs["value_format"] = column.value_format.format_string + formater_kwargs["concat"] = concat + formater_kwargs["concat_str"] = concat_str + formater_kwargs["duplicate_fields"] = [ + (field.field_name, field.force_new, field.concat, field.concat_str) + for field in column.duplicate_fields.all() + ] + formater_kwargs["label"] = column.label + formater_kwargs["required"] = column.required + formater_kwargs["force_new"] = force_news + formater_kwargs["comment"] = column.description if column.export_field_name: - formater_kwargs['export_field_name'] = [ - column.export_field_name] - formater = ImportFormater(targets, formater_types, - **formater_kwargs) + formater_kwargs["export_field_name"] = [column.export_field_name] + formater = ImportFormater(targets, formater_types, **formater_kwargs) LINE_FORMAT.append(formater) LINE_EXPORT_FORMAT.append(formater) UNICITY_KEYS = [] if self.unicity_keys: - UNICITY_KEYS = [un.strip() for un in self.unicity_keys.split(';')] + UNICITY_KEYS = [un.strip() for un in self.unicity_keys.split(";")] MODEL_CREATION_LIMIT = [] for modls in self.created_models.all(): MODEL_CREATION_LIMIT.append(import_class(modls.klass)) - args = {'OBJECT_CLS': OBJECT_CLS, 'DESC': self.description, - 'DEFAULTS': DEFAULTS, 'LINE_FORMAT': LINE_FORMAT, - 'UNICITY_KEYS': UNICITY_KEYS, - 'LINE_EXPORT_FORMAT': LINE_EXPORT_FORMAT, - 'MODEL_CREATION_LIMIT': MODEL_CREATION_LIMIT} - name = str(''.join( - x for x in slugify(self.name).replace('-', ' ').title() - if not x.isspace())) + args = { + "OBJECT_CLS": OBJECT_CLS, + "DESC": self.description, + "DEFAULTS": DEFAULTS, + "LINE_FORMAT": LINE_FORMAT, + "UNICITY_KEYS": UNICITY_KEYS, + "LINE_EXPORT_FORMAT": LINE_EXPORT_FORMAT, + "MODEL_CREATION_LIMIT": MODEL_CREATION_LIMIT, + } + name = str( + "".join( + x + for x in slugify(self.name).replace("-", " ").title() + if not x.isspace() + ) + ) newclass = type(name, (Importer,), args) return newclass @@ -289,25 +329,32 @@ def get_associated_model(parent_model, keys): elif not idx: if item not in fields: raise ImporterError( - str(_("Importer configuration error: " - "\"{}\" is not available for \"{}\"." - " Check your default and column " - "configuration")).format( - item, OBJECT_CLS.__name__)) + str( + _( + "Importer configuration error: " + '"{}" is not available for "{}".' + " Check your default and column " + "configuration" + ) + ).format(item, OBJECT_CLS.__name__) + ) field = fields[item] - if hasattr(field, 'rel') and hasattr(field.rel, 'to'): + if hasattr(field, "rel") and hasattr(field.rel, "to"): model = field.rel.to if type(field) == ModelBase: model = field else: if not model: raise ImporterError( - str(_("Importer configuration error: " - "\"{}\" is not available for \"{}\"." - " Check your default and column " - "configuration")).format( - "__".join(keys[1:]), - OBJECT_CLS.__name__)) + str( + _( + "Importer configuration error: " + '"{}" is not available for "{}".' + " Check your default and column " + "configuration" + ) + ).format("__".join(keys[1:]), OBJECT_CLS.__name__) + ) return get_associated_model(model, keys[1:]) return model @@ -321,13 +368,17 @@ class ImporterDefault(models.Model): """ Targets of default values in an import """ - importer_type = models.ForeignKey(ImporterType, related_name='defaults') + + importer_type = models.ForeignKey( + ImporterType, related_name="defaults", on_delete=models.CASCADE + ) target = models.CharField("Target", max_length=500) class Meta: verbose_name = _("Importer - Default") verbose_name_plural = _("Importer - Defaults") - unique_together = ('importer_type', 'target') + unique_together = ("importer_type", "target") + objects = ImporterDefaultManager() def __str__(self): @@ -338,14 +389,15 @@ class ImporterDefault(models.Model): @property def keys(self): - return tuple(t for t in self.target.split('__') if t not in ("-", "")) + return tuple(t for t in self.target.split("__") if t not in ("-", "")) @property def associated_model(self): if not self.keys: return import_class(self.importer_type.associated_models.klass) - return get_associated_model(self.importer_type.associated_models.klass, - self.keys) + return get_associated_model( + self.importer_type.associated_models.klass, self.keys + ) @property def values(self): @@ -360,17 +412,21 @@ class ImporterDefault(models.Model): class ImporterDefaultValuesManager(models.Manager): def get_by_natural_key(self, def_target_type, def_target, target): - return self.get(default_target__importer_type__slug=def_target_type, - default_target__target=def_target, - target=target) + return self.get( + default_target__importer_type__slug=def_target_type, + default_target__target=def_target, + target=target, + ) class ImporterDefaultValues(models.Model): """ Default values in an import """ - default_target = models.ForeignKey(ImporterDefault, - related_name='default_values') + + default_target = models.ForeignKey( + ImporterDefault, related_name="default_values", on_delete=models.CASCADE + ) target = models.CharField("Target", max_length=500) value = models.CharField("Value", max_length=500) objects = ImporterDefaultValuesManager() @@ -378,12 +434,14 @@ class ImporterDefaultValues(models.Model): class Meta: verbose_name = _("Importer - Default value") verbose_name_plural = _("Importer - Default values") - unique_together = ('default_target', 'target') + unique_together = ("default_target", "target") def natural_key(self): - return (self.default_target.importer_type.slug, - self.default_target.target, - self.target) + return ( + self.default_target.importer_type.slug, + self.default_target.target, + self.target, + ) def __str__(self): return "{} - {}".format(self.default_target, self.target, self.value) @@ -397,7 +455,7 @@ class ImporterDefaultValues(models.Model): if target not in fields: return field = fields[target] - if not hasattr(field, 'rel') or not hasattr(field.rel, 'to'): + if not hasattr(field, "rel") or not hasattr(field.rel, "to"): return self.value model = field.rel.to # if value is an id @@ -415,40 +473,52 @@ class ImporterDefaultValues(models.Model): class ImporterColumnManager(models.Manager): def get_by_natural_key(self, importer_type, col_number): - return self.get(importer_type__slug=importer_type, - col_number=col_number) + return self.get(importer_type__slug=importer_type, col_number=col_number) class ImporterColumn(models.Model): """ Import file column description """ - label = models.CharField(_("Label"), blank=True, null=True, - max_length=200) - importer_type = models.ForeignKey(ImporterType, related_name='columns') + + label = models.CharField(_("Label"), blank=True, null=True, max_length=200) + importer_type = models.ForeignKey( + ImporterType, related_name="columns", on_delete=models.CASCADE + ) col_number = models.IntegerField(_("Column number"), default=1) description = models.TextField(_("Description"), blank=True, null=True) regexp_pre_filter = models.ForeignKey( - "Regexp", blank=True, null=True, on_delete=models.SET_NULL, + "Regexp", + blank=True, + null=True, + on_delete=models.SET_NULL, related_name="columns", ) value_format = models.ForeignKey( - "ValueFormater", blank=True, null=True, on_delete=models.SET_NULL, - related_name="columns" + "ValueFormater", + blank=True, + null=True, + on_delete=models.SET_NULL, + related_name="columns", ) required = models.BooleanField(_("Required"), default=False) export_field_name = models.CharField( - _("Export field name"), blank=True, null=True, max_length=200, - help_text=_("Fill this field if the field name is ambiguous for " - "export. For instance: concatenated fields.") + _("Export field name"), + blank=True, + null=True, + max_length=200, + help_text=_( + "Fill this field if the field name is ambiguous for " + "export. For instance: concatenated fields." + ), ) objects = ImporterColumnManager() class Meta: verbose_name = _("Importer - Column") verbose_name_plural = _("Importer - Columns") - ordering = ('importer_type', 'col_number') - unique_together = ('importer_type', 'col_number') + ordering = ("importer_type", "col_number") + unique_together = ("importer_type", "col_number") def __str__(self): return "{} - {}".format(self.importer_type, self.col_number) @@ -461,11 +531,10 @@ class ImporterColumn(models.Model): return self.importer_type.slug, self.col_number def targets_lbl(self): - return ', '.join([target.target for target in self.targets.all()]) + return ", ".join([target.target for target in self.targets.all()]) def duplicate_fields_lbl(self): - return ', '.join([dp.field_name or "" - for dp in self.duplicate_fields.all()]) + return ", ".join([dp.field_name or "" for dp in self.duplicate_fields.all()]) def formater_type_lbl(self): return ', '.join([str(target.formater_type) for target in self.targets.all()]) @@ -473,35 +542,39 @@ class ImporterColumn(models.Model): class ImporterDuplicateFieldManager(models.Manager): def get_by_natural_key(self, importer_type, col_number, field_name): - return self.get(column__importer_type__slug=importer_type, - column__col_number=col_number, - field_name=field_name) + return self.get( + column__importer_type__slug=importer_type, + column__col_number=col_number, + field_name=field_name, + ) class ImporterDuplicateField(models.Model): """ Direct copy of result in other fields """ - column = models.ForeignKey(ImporterColumn, related_name='duplicate_fields') - field_name = models.CharField(_("Field name"), blank=True, null=True, - max_length=200) - force_new = models.BooleanField(_("Force creation of new items"), - default=False) - concat = models.BooleanField(_("Concatenate with existing"), - default=False) - concat_str = models.CharField(_("Concatenate character"), max_length=5, - blank=True, null=True) + + column = models.ForeignKey( + ImporterColumn, related_name="duplicate_fields", on_delete=models.CASCADE + ) + field_name = models.CharField( + _("Field name"), blank=True, null=True, max_length=200 + ) + force_new = models.BooleanField(_("Force creation of new items"), default=False) + concat = models.BooleanField(_("Concatenate with existing"), default=False) + concat_str = models.CharField( + _("Concatenate character"), max_length=5, blank=True, null=True + ) objects = ImporterDuplicateFieldManager() class Meta: verbose_name = _("Importer - Duplicate field") verbose_name_plural = _("Importer - Duplicate fields") - ordering = ('column', 'field_name') - unique_together = ('column', 'field_name') + ordering = ("column", "field_name") + unique_together = ("column", "field_name") def natural_key(self): - return self.column.importer_type.slug, self.column.col_number, \ - self.field_name + return self.column.importer_type.slug, self.column.col_number, self.field_name class NamedManager(models.Manager): @@ -523,7 +596,7 @@ class Regexp(models.Model): return self.name def natural_key(self): - return (self.name, ) + return (self.name,) class ValueFormater(models.Model): @@ -531,11 +604,14 @@ class ValueFormater(models.Model): slug = models.SlugField(_("Slug"), unique=True, max_length=100) description = models.TextField(_("Description"), blank=True, null=True) format_string = models.CharField( - _("Format string"), max_length=100, - help_text=_("A string used to format a value using the Python " - "\"format()\" method. The site https://pyformat.info/ " - "provide good examples of usage. Only one \"{}\" entry " - "is managed. The input is assumed to be a string.") + _("Format string"), + max_length=100, + help_text=_( + "A string used to format a value using the Python " + '"format()" method. The site https://pyformat.info/ ' + 'provide good examples of usage. Only one "{}" entry ' + "is managed. The input is assumed to be a string." + ), ) objects = SlugModelManager() @@ -551,41 +627,50 @@ class ValueFormater(models.Model): self.format_string.format("sample value") except ValueError: raise ValidationError( - {'format_string': _("The string provided generate an error. " - "Fix it.")} + { + "format_string": _( + "The string provided generate an error. " "Fix it." + ) + } ) def natural_key(self): - return (self.slug, ) + return (self.slug,) class ImportTargetManager(models.Manager): def get_by_natural_key(self, importer_type, col_number, target): - return self.get(column__importer_type__slug=importer_type, - column__col_number=col_number, - target=target) + return self.get( + column__importer_type__slug=importer_type, + column__col_number=col_number, + target=target, + ) class ImportTarget(models.Model): """ Ishtar database target for a column """ - column = models.ForeignKey(ImporterColumn, related_name='targets') + + column = models.ForeignKey( + ImporterColumn, related_name="targets", on_delete=models.CASCADE + ) target = models.CharField("Target", max_length=500) - formater_type = models.ForeignKey("FormaterType", related_name='targets') - force_new = models.BooleanField(_("Force creation of new items"), - default=False) - concat = models.BooleanField(_("Concatenate with existing"), - default=False) - concat_str = models.CharField(_("Concatenate character"), max_length=5, - blank=True, null=True) + formater_type = models.ForeignKey( + "FormaterType", related_name="targets", on_delete=models.CASCADE + ) + force_new = models.BooleanField(_("Force creation of new items"), default=False) + concat = models.BooleanField(_("Concatenate with existing"), default=False) + concat_str = models.CharField( + _("Concatenate character"), max_length=5, blank=True, null=True + ) comment = models.TextField(_("Comment"), blank=True, null=True) objects = ImportTargetManager() class Meta: verbose_name = _("Importer - Target") verbose_name_plural = _("Importer - Targets") - unique_together = ('column', 'target') + unique_together = ("column", "target") def __str__(self): return self.target[:50] if self.target else self.comment @@ -599,8 +684,7 @@ class ImportTarget(models.Model): return "{} - {}".format(self.target[:50], desc) def natural_key(self): - return self.column.importer_type.slug, self.column.col_number, \ - self.target + return self.column.importer_type.slug, self.column.col_number, self.target @property def associated_model(self): @@ -609,24 +693,24 @@ class ImportTarget(models.Model): try: return get_associated_model( self.column.importer_type.associated_models.klass, - self.target.split('__')) + self.target.split("__"), + ) except KeyError: return def get_choices(self): - if self.formater_type.formater_type == 'UnknowType' \ - and self.column.importer_type.slug: + if ( + self.formater_type.formater_type == "UnknowType" + and self.column.importer_type.slug + ): cls = self.column.importer_type.get_importer_class() formt = cls().line_format[self.column.col_number - 1] - if hasattr(formt.formater, 'choices'): - return [('', '--' * 8)] + list(formt.formater.choices) - return [('', '--' * 8)] - if self.formater_type.formater_type == 'StrToBoolean': - return [('', '--' * 8), - ('True', _("True")), - ('False', _("False"))] - if not self.associated_model or not hasattr(self.associated_model, - 'get_types'): + if hasattr(formt.formater, "choices"): + return [("", "--" * 8)] + list(formt.formater.choices) + return [("", "--" * 8)] + if self.formater_type.formater_type == "StrToBoolean": + return [("", "--" * 8), ("True", _("True")), ("False", _("False"))] + if not self.associated_model or not hasattr(self.associated_model, "get_types"): return [] return self.associated_model.get_types() @@ -635,11 +719,12 @@ class TargetKeyGroup(models.Model): """ Group of target keys for imports. """ + name = models.TextField(_("Name"), unique=True) - all_user_can_use = models.BooleanField(_("All users can use it"), - default=False) - all_user_can_modify = models.BooleanField(_("All users can modify it"), - default=False) + all_user_can_use = models.BooleanField(_("All users can use it"), default=False) + all_user_can_modify = models.BooleanField( + _("All users can modify it"), default=False + ) available = models.BooleanField(_("Available"), default=True) class Meta: @@ -660,20 +745,33 @@ class TargetKey(models.Model): one particular group (associated_group) or to all imports (associated_import, associated_user and associated_group are empty). """ - target = models.ForeignKey(ImportTarget, related_name='keys') + + target = models.ForeignKey( + ImportTarget, related_name="keys", on_delete=models.CASCADE + ) key = models.TextField(_("Key")) value = models.TextField(_("Value"), blank=True, null=True) is_set = models.BooleanField(_("Is set"), default=False) - associated_import = models.ForeignKey('Import', blank=True, null=True) - associated_user = models.ForeignKey('IshtarUser', blank=True, null=True) - associated_group = models.ForeignKey(TargetKeyGroup, blank=True, null=True) + associated_import = models.ForeignKey( + "Import", blank=True, null=True, on_delete=models.SET_NULL + ) + associated_user = models.ForeignKey( + "IshtarUser", blank=True, null=True, on_delete=models.SET_NULL + ) + associated_group = models.ForeignKey( + TargetKeyGroup, blank=True, null=True, on_delete=models.SET_NULL + ) class Meta: - unique_together = ('target', 'key', 'associated_user', - 'associated_import',) + unique_together = ( + "target", + "key", + "associated_user", + "associated_import", + ) verbose_name = _("Importer - Target key") verbose_name_plural = _("Importer - Targets keys") - ordering = ('target', 'key') + ordering = ("target", "key") def __str__(self): return " - ".join([str(self.target), self.key[:50]]) @@ -689,8 +787,8 @@ class TargetKey(models.Model): def format(self): if not self.is_set: return None - if self.target.formater_type.formater_type == 'StrToBoolean': - if self.value in ('False', '0'): + if self.target.formater_type.formater_type == "StrToBoolean": + if self.value in ("False", "0"): return False elif self.value: return True @@ -703,162 +801,165 @@ class TargetKey(models.Model): return obj v = None associated_model = self.target.associated_model - if associated_model and hasattr(self.target.associated_model, - "add_key"): + if associated_model and hasattr(self.target.associated_model, "add_key"): # pk is given try: - v = self.target.associated_model.objects.get( - pk=str(int(self.value))) + v = self.target.associated_model.objects.get(pk=str(int(self.value))) except (ValueError, self.target.associated_model.DoesNotExist): # try with txt_idx try: v = self.target.associated_model.objects.get( - txt_idx=str(self.value)) + txt_idx=str(self.value) + ) except self.target.associated_model.DoesNotExist: pass if v: keys = {} if self.associated_group: - keys['group'] = self.associated_group + keys["group"] = self.associated_group if self.associated_user: - keys['user'] = self.associated_user + keys["user"] = self.associated_user else: - keys['importer'] = self.associated_import + keys["importer"] = self.associated_import v.add_key(self.key, **keys) return obj TARGET_MODELS = [ - ('OrganizationType', _("Organization type")), - ('ishtar_common.models.OrganizationType', _("Organization type")), - ('ishtar_common.models.PersonType', _("Person type")), - ('TitleType', _("Title")), - ('SourceType', _("Source type")), - ('ishtar_common.models.SourceType', _("Source type")), - ('AuthorType', _("Author type")), - ('Format', _("Format")), - ('ishtar_common.models.Format', _("Format")), - ('ishtar_common.models.LicenseType', _("License type")), - ('ishtar_common.models.DocumentTag', _("Document tag")), - ('ishtar_common.models.Language', _("Language")), - ('ishtar_common.models.SupportType', _("Support type")), - ('archaeological_operations.models.OperationType', _("Operation type")), - ('archaeological_operations.models.Period', _("Period")), - ('archaeological_operations.models.ReportState', _("Report state")), - ('archaeological_operations.models.RemainType', _("Remain type")), - ('archaeological_operations.models.RelationType', - _("Operation relation type")), + ("OrganizationType", _("Organization type")), + ("ishtar_common.models.OrganizationType", _("Organization type")), + ("ishtar_common.models.PersonType", _("Person type")), + ("TitleType", _("Title")), + ("SourceType", _("Source type")), + ("ishtar_common.models.SourceType", _("Source type")), + ("AuthorType", _("Author type")), + ("Format", _("Format")), + ("ishtar_common.models.Format", _("Format")), + ("ishtar_common.models.LicenseType", _("License type")), + ("ishtar_common.models.DocumentTag", _("Document tag")), + ("ishtar_common.models.Language", _("Language")), + ("ishtar_common.models.SupportType", _("Support type")), + ("archaeological_operations.models.OperationType", _("Operation type")), + ("archaeological_operations.models.Period", _("Period")), + ("archaeological_operations.models.ReportState", _("Report state")), + ("archaeological_operations.models.RemainType", _("Remain type")), + ("archaeological_operations.models.RelationType", _("Operation relation type")), ("archaeological_operations.models.ActType", _("Act type")), - ('archaeological_context_records.models.Unit', _("Unit")), - ('archaeological_context_records.models.ActivityType', - _("Activity type")), - ('archaeological_context_records.models.DocumentationType', - _("Documentation type")), - ("archaeological_context_records.models.DatingQuality", - _("Dating quality")), - ('archaeological_finds.models.MaterialType', _("Material")), - ('archaeological_finds.models.ConservatoryState', - _("Conservatory state")), - ('archaeological_warehouse.models.ContainerType', _("Container type")), - ('archaeological_warehouse.models.WarehouseDivision', - _("Warehouse division")), - ('archaeological_warehouse.models.WarehouseType', _("Warehouse type")), - ('archaeological_finds.models.TreatmentType', _("Treatment type")), - ('archaeological_finds.models.TreatmentEmergencyType', - _("Treatment emergency type")), - ('archaeological_finds.models.ObjectType', _("Object type")), - ('archaeological_finds.models.IntegrityType', _("Integrity type")), - ('archaeological_finds.models.RemarkabilityType', - _("Remarkability type")), - ('archaeological_finds.models.AlterationType', _("Alteration type")), - ('archaeological_finds.models.AlterationCauseType', - _("Alteration cause type")), - ('archaeological_finds.models.BatchType', _("Batch type")), - ('archaeological_finds.models.CheckedType', _("Checked type")), - ('archaeological_finds.models.MaterialTypeQualityType', - _("Material type quality")), + ("archaeological_context_records.models.Unit", _("Unit")), + ("archaeological_context_records.models.ActivityType", _("Activity type")), + ( + "archaeological_context_records.models.DocumentationType", + _("Documentation type"), + ), + ("archaeological_context_records.models.DatingQuality", _("Dating quality")), + ("archaeological_finds.models.MaterialType", _("Material")), + ("archaeological_finds.models.ConservatoryState", _("Conservatory state")), + ("archaeological_warehouse.models.ContainerType", _("Container type")), + ("archaeological_warehouse.models.WarehouseDivision", _("Warehouse division")), + ("archaeological_warehouse.models.WarehouseType", _("Warehouse type")), + ("archaeological_finds.models.TreatmentType", _("Treatment type")), + ( + "archaeological_finds.models.TreatmentEmergencyType", + _("Treatment emergency type"), + ), + ("archaeological_finds.models.ObjectType", _("Object type")), + ("archaeological_finds.models.IntegrityType", _("Integrity type")), + ("archaeological_finds.models.RemarkabilityType", _("Remarkability type")), + ("archaeological_finds.models.AlterationType", _("Alteration type")), + ("archaeological_finds.models.AlterationCauseType", _("Alteration cause type")), + ("archaeological_finds.models.BatchType", _("Batch type")), + ("archaeological_finds.models.CheckedType", _("Checked type")), + ("archaeological_finds.models.MaterialTypeQualityType", _("Material type quality")), ("archaeological_finds.models.FunctionalArea", _("Functional area")), - ('archaeological_context_records.models.IdentificationType', - _("Identification type")), - ('archaeological_context_records.models.RelationType', - _("Context record relation type")), - ('SpatialReferenceSystem', _("Spatial reference system")), - ('SupportType', _("Support type")), - ('TitleType', _("Title type")), + ( + "archaeological_context_records.models.IdentificationType", + _("Identification type"), + ), + ( + "archaeological_context_records.models.RelationType", + _("Context record relation type"), + ), + ("SpatialReferenceSystem", _("Spatial reference system")), + ("SupportType", _("Support type")), + ("TitleType", _("Title type")), ] TARGET_MODELS_KEYS = [tm[0] for tm in TARGET_MODELS] IMPORTER_TYPES = ( - ('IntegerFormater', _("Integer")), - ('FloatFormater', _("Float")), - ('UnicodeFormater', _("String")), - ('DateFormater', _("Date")), - ('TypeFormater', _("Type")), - ('YearFormater', _("Year")), - ('InseeFormater', _("INSEE code")), - ('UpperFormater', _("Upper case")), - ('LowerFormater', _("Lower case")), - ('StrToBoolean', _("String to boolean")), - ('FileFormater', pgettext_lazy("filesystem", "File")), - ('UnknowType', _("Unknow type")) + ("IntegerFormater", _("Integer")), + ("FloatFormater", _("Float")), + ("UnicodeFormater", _("String")), + ("DateFormater", _("Date")), + ("TypeFormater", _("Type")), + ("YearFormater", _("Year")), + ("InseeFormater", _("INSEE code")), + ("UpperFormater", _("Upper case")), + ("LowerFormater", _("Lower case")), + ("StrToBoolean", _("String to boolean")), + ("FileFormater", pgettext_lazy("filesystem", "File")), + ("UnknowType", _("Unknow type")), ) IMPORTER_TYPES_DCT = { - 'IntegerFormater': IntegerFormater, - 'FloatFormater': FloatFormater, - 'UnicodeFormater': UnicodeFormater, - 'DateFormater': DateFormater, - 'TypeFormater': TypeFormater, - 'YearFormater': YearFormater, - 'StrToBoolean': StrToBoolean, - 'FileFormater': FileFormater, - 'InseeFormater': InseeFormater, - 'UpperFormater': UpperCaseFormater, - 'LowerFormater': LowerCaseFormater, - 'UnknowType': None, + "IntegerFormater": IntegerFormater, + "FloatFormater": FloatFormater, + "UnicodeFormater": UnicodeFormater, + "DateFormater": DateFormater, + "TypeFormater": TypeFormater, + "YearFormater": YearFormater, + "StrToBoolean": StrToBoolean, + "FileFormater": FileFormater, + "InseeFormater": InseeFormater, + "UpperFormater": UpperCaseFormater, + "LowerFormater": LowerCaseFormater, + "UnknowType": None, } DATE_FORMATS = ( - ('%Y', _("4 digit year. e.g.: \"2015\"")), - ('%Y/%m/%d', _("4 digit year/month/day. e.g.: \"2015/02/04\"")), - ('%d/%m/%Y', _("Day/month/4 digit year. e.g.: \"04/02/2015\"")), + ("%Y", _('4 digit year. e.g.: "2015"')), + ("%Y/%m/%d", _('4 digit year/month/day. e.g.: "2015/02/04"')), + ("%d/%m/%Y", _('Day/month/4 digit year. e.g.: "04/02/2015"')), ) -IMPORTER_TYPES_CHOICES = {'TypeFormater': TARGET_MODELS, - 'DateFormater': DATE_FORMATS} +IMPORTER_TYPES_CHOICES = {"TypeFormater": TARGET_MODELS, "DateFormater": DATE_FORMATS} class FormaterTypeManager(models.Manager): def get_by_natural_key(self, formater_type, options, many_split): - return self.get(formater_type=formater_type, - options=options, many_split=many_split) + return self.get( + formater_type=formater_type, options=options, many_split=many_split + ) class FormaterType(models.Model): - formater_type = models.CharField("Formater type", max_length=20, - choices=IMPORTER_TYPES) - options = models.CharField(_("Options"), max_length=500, blank=True, - null=True) - many_split = models.CharField(_("Split character(s)"), max_length=10, - blank=True, null=True) + formater_type = models.CharField( + "Formater type", max_length=20, choices=IMPORTER_TYPES + ) + options = models.CharField(_("Options"), max_length=500, blank=True, null=True) + many_split = models.CharField( + _("Split character(s)"), max_length=10, blank=True, null=True + ) objects = FormaterTypeManager() class Meta: verbose_name = _("Importer - Formater type") verbose_name_plural = _("Importer - Formater types") - unique_together = ('formater_type', 'options', 'many_split') - ordering = ('formater_type', 'options') + unique_together = ("formater_type", "options", "many_split") + ordering = ("formater_type", "options") def natural_key(self): return self.formater_type, self.options, self.many_split def __str__(self): return " - ".join( - [str(dict(IMPORTER_TYPES)[self.formater_type]) - if self.formater_type in IMPORTER_TYPES_DCT else ''] + - [getattr(self, k) for k in ('options', 'many_split') - if getattr(self, k)]) + [ + str(dict(IMPORTER_TYPES)[self.formater_type]) + if self.formater_type in IMPORTER_TYPES_DCT + else "" + ] + + [getattr(self, k) for k in ("options", "many_split") if getattr(self, k)] + ) def get_choices(self): if self.format_type in IMPORTER_TYPES_CHOICES: @@ -867,35 +968,36 @@ class FormaterType(models.Model): def get_formater_type(self, target, import_instance=None): if self.formater_type not in IMPORTER_TYPES_DCT.keys(): return - kwargs = {'db_target': target, 'import_instance': import_instance} + kwargs = {"db_target": target, "import_instance": import_instance} if self.many_split: - kwargs['many_split'] = self.many_split - if self.formater_type == 'TypeFormater': + kwargs["many_split"] = self.many_split + if self.formater_type == "TypeFormater": if self.options not in TARGET_MODELS_KEYS: logger.warning( "**WARN FormaterType.get_formater_type**: {} " - "is not in TARGET_MODELS_KEYS".format(self.options)) + "is not in TARGET_MODELS_KEYS".format(self.options) + ) return if self.options in dir(): model = dir()[self.options] else: model = import_class(self.options) return TypeFormater(model, **kwargs) - elif self.formater_type == 'UnicodeFormater': + elif self.formater_type == "UnicodeFormater": if self.options: try: return UnicodeFormater(int(self.options.strip()), **kwargs) except ValueError: pass return UnicodeFormater(**kwargs) - elif self.formater_type == 'DateFormater': + elif self.formater_type == "DateFormater": date_formats = self.options if self.many_split: - date_formats = self.options.split(kwargs.pop('many_split')) + date_formats = self.options.split(kwargs.pop("many_split")) return DateFormater(date_formats, **kwargs) - elif self.formater_type == 'StrToBoolean': + elif self.formater_type == "StrToBoolean": return StrToBoolean(**kwargs) - elif self.formater_type == 'UnknowType': + elif self.formater_type == "UnknowType": return else: return IMPORTER_TYPES_DCT[self.formater_type](**kwargs) @@ -916,13 +1018,13 @@ IMPORT_STATE = ( ) IMPORT_STATE_DCT = dict(IMPORT_STATE) -ENCODINGS = [(settings.ENCODING, settings.ENCODING), - (settings.ALT_ENCODING, settings.ALT_ENCODING), - ('utf-8', 'utf-8')] +ENCODINGS = [ + (settings.ENCODING, settings.ENCODING), + (settings.ALT_ENCODING, settings.ALT_ENCODING), + ("utf-8", "utf-8"), +] -CSV_SEPS = ((",", ","), - (";", ";"), - ("|", "|")) +CSV_SEPS = ((",", ","), (";", ";"), ("|", "|")) @task() @@ -944,68 +1046,117 @@ def delayed_check(import_pk): class Import(models.Model): - user = models.ForeignKey('IshtarUser', blank=True, null=True, - on_delete=models.SET_NULL) + user = models.ForeignKey( + "IshtarUser", blank=True, null=True, on_delete=models.SET_NULL + ) name = models.CharField(_("Name"), max_length=500, null=True) - importer_type = models.ForeignKey(ImporterType) + importer_type = models.ForeignKey(ImporterType, on_delete=models.CASCADE) imported_file = models.FileField( - _("Imported file"), upload_to="upload/imports/%Y/%m/", max_length=220, - help_text=max_size_help(), blank=True, null=True) + _("Imported file"), + upload_to="upload/imports/%Y/%m/", + max_length=220, + help_text=max_size_help(), + blank=True, + null=True, + ) imported_images = models.FileField( - _("Associated images (zip file)"), upload_to="upload/imports/%Y/%m/", - blank=True, null=True, max_length=220, help_text=max_size_help()) + _("Associated images (zip file)"), + upload_to="upload/imports/%Y/%m/", + blank=True, + null=True, + max_length=220, + help_text=max_size_help(), + ) associated_group = models.ForeignKey( - TargetKeyGroup, blank=True, null=True, + TargetKeyGroup, + blank=True, + null=True, on_delete=models.SET_NULL, - help_text=_("If a group is selected, target key saved in this group " - "will be used.") + help_text=_( + "If a group is selected, target key saved in this group " "will be used." + ), + ) + encoding = models.CharField( + _("Encoding"), choices=ENCODINGS, default="utf-8", max_length=15 ) - encoding = models.CharField(_("Encoding"), choices=ENCODINGS, - default='utf-8', max_length=15) csv_sep = models.CharField( - _("CSV separator"), choices=CSV_SEPS, default=',', max_length=1, - help_text=_("Separator for CSV file. Standard is comma but Microsoft " - "Excel do not follow this standard and use semi-colon.") + _("CSV separator"), + choices=CSV_SEPS, + default=",", + max_length=1, + help_text=_( + "Separator for CSV file. Standard is comma but Microsoft " + "Excel do not follow this standard and use semi-colon." + ), ) skip_lines = models.IntegerField( - _("Skip lines"), default=1, - help_text=_("Number of header lines in your file (can be 0).")) + _("Skip lines"), + default=1, + help_text=_("Number of header lines in your file (can be 0)."), + ) error_file = models.FileField( - _("Error file"), upload_to="upload/imports/%Y/%m/", - blank=True, null=True, max_length=255, help_text=max_size_help()) + _("Error file"), + upload_to="upload/imports/%Y/%m/", + blank=True, + null=True, + max_length=255, + help_text=max_size_help(), + ) result_file = models.FileField( - _("Result file"), upload_to="upload/imports/%Y/%m/", - blank=True, null=True, max_length=255, help_text=max_size_help()) + _("Result file"), + upload_to="upload/imports/%Y/%m/", + blank=True, + null=True, + max_length=255, + help_text=max_size_help(), + ) match_file = models.FileField( - _("Match file"), upload_to="upload/imports/%Y/%m/", blank=True, - null=True, max_length=255, help_text=max_size_help()) + _("Match file"), + upload_to="upload/imports/%Y/%m/", + blank=True, + null=True, + max_length=255, + help_text=max_size_help(), + ) archive_file = models.FileField( - _("Archive file"), upload_to="upload/imports/%Y/%m/", blank=True, - null=True, max_length=255, help_text=max_size_help()) - state = models.CharField(_("State"), max_length=2, choices=IMPORT_STATE, - default='C') + _("Archive file"), + upload_to="upload/imports/%Y/%m/", + blank=True, + null=True, + max_length=255, + help_text=max_size_help(), + ) + state = models.CharField( + _("State"), max_length=2, choices=IMPORT_STATE, default="C" + ) conservative_import = models.BooleanField( - _("Conservative import"), default=False, - help_text=_('If set to true, do not overload existing values.')) + _("Conservative import"), + default=False, + help_text=_("If set to true, do not overload existing values."), + ) creation_date = models.DateTimeField( - _("Creation date"), auto_now_add=True, blank=True, null=True) - end_date = models.DateTimeField(_("End date"), auto_now_add=True, - blank=True, null=True, editable=False) + _("Creation date"), auto_now_add=True, blank=True, null=True + ) + end_date = models.DateTimeField( + _("End date"), auto_now_add=True, blank=True, null=True, editable=False + ) seconds_remaining = models.IntegerField( - _("Remaining seconds"), blank=True, null=True, editable=False) - current_line = models.IntegerField(_("Current line"), blank=True, - null=True) - number_of_line = models.IntegerField(_("Number of line"), blank=True, - null=True) + _("Remaining seconds"), blank=True, null=True, editable=False + ) + current_line = models.IntegerField(_("Current line"), blank=True, null=True) + number_of_line = models.IntegerField(_("Number of line"), blank=True, null=True) imported_line_numbers = models.TextField( - _("Imported line numbers"), blank=True, null=True, - validators=[validate_comma_separated_integer_list] + _("Imported line numbers"), + blank=True, + null=True, + validators=[validate_comma_separated_integer_list], ) - changed_checked = models.BooleanField(_("Changed have been checked"), - default=False) + changed_checked = models.BooleanField(_("Changed have been checked"), default=False) changed_line_numbers = models.TextField( - _("Changed line numbers"), blank=True, null=True, - validators=[validate_comma_separated_integer_list] + _("Changed line numbers"), + blank=True, + null=True, + validators=[validate_comma_separated_integer_list], ) class Meta: @@ -1016,17 +1167,17 @@ class Import(models.Model): return "{} | {}".format(self.name or "-", self.importer_type) def need_matching(self): - return bool(TargetKey.objects.filter(associated_import=self, - is_set=False).count()) + return bool( + TargetKey.objects.filter(associated_import=self, is_set=False).count() + ) @property def errors(self): if not self.error_file: return [] errors = [] - with open(self.error_file.path, 'rt') as csvfile: - reader = csv.DictReader( - csvfile, fieldnames=['line', 'column', 'error']) + with open(self.error_file.path, "rt") as csvfile: + reader = csv.DictReader(csvfile, fieldnames=["line", "column", "error"]) for idx, row in enumerate(reader): if not idx: # pass the header continue @@ -1040,11 +1191,10 @@ class Import(models.Model): return filename = self.imported_file.path encodings = [self.encoding] - encodings += [coding for coding, c in ENCODINGS - if coding != self.encoding] + encodings += [coding for coding, c in ENCODINGS if coding != self.encoding] for encoding in encodings: try: - with open(filename, 'r', encoding=encoding) as f: + with open(filename, "r", encoding=encoding) as f: reader = csv.reader(f, delimiter=self.csv_sep) nb = sum(1 for __ in reader) - self.skip_lines except UnicodeDecodeError: @@ -1059,14 +1209,14 @@ class Import(models.Model): def progress_percent(self): if not self.current_line or not self.number_of_line: return 0 - return int((float(self.current_line) / float(self.number_of_line)) - * 100) + return int((float(self.current_line) / float(self.number_of_line)) * 100) def add_imported_line(self, idx_line): if not self.number_of_line: self.get_number_of_lines() - if self.imported_line_numbers and \ - str(idx_line) in self.imported_line_numbers.split(','): + if self.imported_line_numbers and str( + idx_line + ) in self.imported_line_numbers.split(","): return if self.imported_line_numbers: self.imported_line_numbers += "," @@ -1077,8 +1227,9 @@ class Import(models.Model): self.save() def add_changed_line(self, idx_line): - if self.changed_line_numbers and \ - str(idx_line) in self.changed_line_numbers.split(','): + if self.changed_line_numbers and str( + idx_line + ) in self.changed_line_numbers.split(","): return if self.changed_line_numbers: self.changed_line_numbers += "," @@ -1090,7 +1241,7 @@ class Import(models.Model): def remove_changed_line(self, idx_line): if not self.changed_line_numbers: return - line_numbers = self.changed_line_numbers.split(',') + line_numbers = self.changed_line_numbers.split(",") if str(idx_line) not in line_numbers: return line_numbers.pop(line_numbers.index(str(idx_line))) @@ -1102,12 +1253,13 @@ class Import(models.Model): return True if not self.changed_line_numbers: return - line_numbers = self.changed_line_numbers.split(',') + line_numbers = self.changed_line_numbers.split(",") return str(idx_line) in line_numbers def line_is_imported(self, idx_line): - return self.imported_line_numbers and \ - str(idx_line) in self.imported_line_numbers.split(',') + return self.imported_line_numbers and str( + idx_line + ) in self.imported_line_numbers.split(",") def get_actions(self): """ @@ -1125,24 +1277,24 @@ class Import(models.Model): actions.append(('I', _("Launch import"))) if profile.experimental_feature: if self.changed_checked: - actions.append(('IS', _("Step by step import"))) - actions.append(('CH', _("Re-check for changes"))) + actions.append(("IS", _("Step by step import"))) + actions.append(("CH", _("Re-check for changes"))) else: - actions.append(('CH', _("Check for changes"))) - if self.state in ('F', 'FE'): - actions.append(('A', _("Re-analyse"))) - actions.append(('I', _("Re-import"))) + actions.append(("CH", _("Check for changes"))) + if self.state in ("F", "FE"): + actions.append(("A", _("Re-analyse"))) + actions.append(("I", _("Re-import"))) if profile.experimental_feature: if self.changed_checked: - actions.append(('IS', _("Step by step re-import"))) - actions.append(('CH', _("Re-check for changes"))) + actions.append(("IS", _("Step by step re-import"))) + actions.append(("CH", _("Re-check for changes"))) else: - actions.append(('CH', _("Check for changes"))) - actions.append(('AC', _("Archive"))) - if self.state == 'AC': + actions.append(("CH", _("Check for changes"))) + actions.append(("AC", _("Archive"))) + if self.state == "AC": state = "FE" if self.error_file else "F" actions.append((state, _("Unarchive"))) - actions.append(('D', _("Delete"))) + actions.append(("D", _("Delete"))) return actions @property @@ -1157,8 +1309,10 @@ class Import(models.Model): def get_importer_instance(self): return self.importer_type.get_importer_class(import_instance=self)( - skip_lines=self.skip_lines, import_instance=self, - conservative_import=self.conservative_import) + skip_lines=self.skip_lines, + import_instance=self, + conservative_import=self.conservative_import, + ) @property def data_table(self): @@ -1169,23 +1323,21 @@ class Import(models.Model): filename = None for name in z.namelist(): # get first CSV file found - if name.endswith('.csv'): + if name.endswith(".csv"): filename = name break if not filename: return [] - tmpdir = tempfile.mkdtemp(prefix='tmp-ishtar-') + tmpdir = tempfile.mkdtemp(prefix="tmp-ishtar-") imported_file = z.extract(filename, tmpdir) encodings = [self.encoding] - encodings += [coding for coding, c in ENCODINGS - if coding != self.encoding] + encodings += [coding for coding, c in ENCODINGS if coding != self.encoding] for encoding in encodings: try: with open(imported_file, encoding=encoding) as csv_file: vals = [ - line for line in csv.reader(csv_file, - delimiter=self.csv_sep) + line for line in csv.reader(csv_file, delimiter=self.csv_sep) ] if tmpdir: shutil.rmtree(tmpdir) @@ -1199,19 +1351,20 @@ class Import(models.Model): return [] def initialize(self, user=None, session_key=None): - self.state = 'AP' + self.state = "AP" self.end_date = datetime.datetime.now() self.save() try: self.get_importer_instance().initialize( - self.data_table, user=user, output='db') + self.data_table, user=user, output="db" + ) except ImporterError as e: if session_key: put_session_message(session_key, e.msg, "danger") - self.state = 'C' + self.state = "C" self.save() return - self.state = 'A' + self.state = "A" self.end_date = datetime.datetime.now() self.save() @@ -1220,17 +1373,16 @@ class Import(models.Model): return self.check_modified(session_key=session_key) put_session_message( session_key, - str( - _("Modification check {} added to the queue") - ).format(self.name), - "info") - self.state = 'HQ' + str(_("Modification check {} added to the queue")).format(self.name), + "info", + ) + self.state = "HQ" self.end_date = datetime.datetime.now() self.save() return delayed_check.delay(self.pk) def check_modified(self, session_key=None): - self.state = 'HP' + self.state = "HP" self.end_date = datetime.datetime.now() self.changed_line_numbers = "" self.changed_checked = False @@ -1239,9 +1391,7 @@ class Import(models.Model): for idx in range(self.skip_lines, self.get_number_of_lines() + 1): try: imprt, data = self.importation( - simulate=True, - line_to_process=idx, - return_importer_and_data=True + simulate=True, line_to_process=idx, return_importer_and_data=True ) except IOError as e: # error is identified as a change @@ -1268,7 +1418,7 @@ class Import(models.Model): break current_value = getattr(obj, k) updated_value = updated_values[k] - if hasattr(current_value, 'all'): + if hasattr(current_value, "all"): current_value = list(current_value.all()) changed = False for v in updated_value: @@ -1292,85 +1442,95 @@ class Import(models.Model): put_session_message( session_key, str(_("Import {} added to the queue")).format(self.name), - "info") - self.state = 'IQ' + "info", + ) + self.state = "IQ" self.end_date = datetime.datetime.now() self.save() return delayed_import.delay(self.pk) - def importation(self, session_key=None, line_to_process=None, - simulate=False, return_importer_and_data=False, - request=None): - self.state = 'IP' + def importation( + self, + session_key=None, + line_to_process=None, + simulate=False, + return_importer_and_data=False, + request=None, + ): + self.state = "IP" self.end_date = datetime.datetime.now() if not line_to_process: # full import - self.imported_line_numbers = '' + self.imported_line_numbers = "" self.current_line = 0 self.save() importer = self.get_importer_instance() try: data = importer.importation( - self.data_table, user=self.user, - line_to_process=line_to_process, simulate=simulate) + self.data_table, + user=self.user, + line_to_process=line_to_process, + simulate=simulate, + ) except IOError: error_message = str(_("Error on imported file: {}")).format( - self.imported_file) + self.imported_file + ) importer.errors = [error_message] if session_key: put_session_message(session_key, error_message, "warning") - ids = get_session_var(session_key, 'current_import_id') + ids = get_session_var(session_key, "current_import_id") if not ids: ids = [] ids.append(self.pk) - put_session_var(session_key, 'current_import_id', ids) + put_session_var(session_key, "current_import_id", ids) if line_to_process: - self.state = 'PI' + self.state = "PI" else: - self.state = 'FE' + self.state = "FE" self.save() if not return_importer_and_data: return return importer, None # result file filename = slugify(self.importer_type.name) - now = datetime.datetime.now().isoformat('-').replace(':', '') + now = datetime.datetime.now().isoformat("-").replace(":", "") result_file = filename + "_result_%s.csv" % now self.result_file.save( - result_file, ContentFile(importer.get_csv_result().encode('utf-8'))) + result_file, ContentFile(importer.get_csv_result().encode("utf-8")) + ) if importer.errors: if line_to_process: - self.state = 'PI' + self.state = "PI" else: - self.state = 'FE' + self.state = "FE" error_file = filename + "_errors_%s.csv" % now self.error_file.save( - error_file, - ContentFile(importer.get_csv_errors().encode('utf-8')) + error_file, ContentFile(importer.get_csv_errors().encode("utf-8")) ) - msg = str(_("Import {} finished with errors")).format( - self.name) + msg = str(_("Import {} finished with errors")).format(self.name) msg_cls = "warning" else: if line_to_process: - self.state = 'PI' + self.state = "PI" else: - self.state = 'F' + self.state = "F" self.error_file = None - msg = str(_("Import {} finished with no errors")).format( - self.name) + msg = str(_("Import {} finished with no errors")).format(self.name) msg_cls = "primary" if session_key and request: put_session_message(session_key, msg, msg_cls) - ids = request.session['current_import_id'] \ - if 'current_import_id' in request.session else [] + ids = ( + request.session["current_import_id"] + if "current_import_id" in request.session + else [] + ) ids.append(self.pk) - put_session_var(session_key, 'current_import_id', ids) + put_session_var(session_key, "current_import_id", ids) if importer.match_table: match_file = filename + "_match_%s.csv" % now self.match_file.save( - match_file, - ContentFile(importer.get_csv_matches().encode('utf-8')) + match_file, ContentFile(importer.get_csv_matches().encode("utf-8")) ) self.end_date = datetime.datetime.now() self.save() @@ -1382,7 +1542,7 @@ class Import(models.Model): return with tempfile.TemporaryDirectory() as tmp_dir_name: # extract the current archive - current_zip = zipfile.ZipFile(self.archive_file.path, 'r') + current_zip = zipfile.ZipFile(self.archive_file.path, "r") name_list = current_zip.namelist() if "content.json" not in name_list: return @@ -1402,19 +1562,19 @@ class Import(models.Model): with open(full_filename, "rb") as raw_file: getattr(self, attr).save( "upload/imports/{}/{:02d}/{}".format( - today.year, today.month, filename), - File(raw_file) + today.year, today.month, filename + ), + File(raw_file), ) os.remove(self.archive_file.path) - setattr(self, 'archive_file', None) + setattr(self, "archive_file", None) self.state = "FE" if self.error_file else "F" self.save() return True def _archive(self): - file_attr = ["imported_file", "error_file", "result_file", - "match_file"] + file_attr = ["imported_file", "error_file", "result_file", "match_file"] files = [ (k, getattr(self, k).path, getattr(self, k).name.split(os.sep)[-1]) for k in file_attr @@ -1438,11 +1598,15 @@ class Import(models.Model): current_zip.write(content_name, arcname="content.json") today = datetime.date.today() - with open(archive_name, "rb", ) as raw_file: + with open( + archive_name, + "rb", + ) as raw_file: self.archive_file.save( "upload/imports/{}/{:02d}/{}".format( - today.year, today.month, base_name), - File(raw_file) + today.year, today.month, base_name + ), + File(raw_file), ) IshtarSiteProfile = apps.get_model("ishtar_common", "IshtarSiteProfile") profile = IshtarSiteProfile.get_current_profile() @@ -1460,7 +1624,7 @@ class Import(models.Model): self._archive_pending = False def archive(self): - self.state = 'AC' + self.state = "AC" self.end_date = datetime.datetime.now() self._archive() @@ -1473,20 +1637,22 @@ class Import(models.Model): imported = [] for related, zorg in get_all_related_m2m_objects_with_model(self): accessor = related.get_accessor_name() - imported += [(accessor, obj) - for obj in getattr(self, accessor).all()] + imported += [(accessor, obj) for obj in getattr(self, accessor).all()] return imported def save(self, *args, **kwargs): super(Import, self).save(*args, **kwargs) - if self.state == "AC" and not getattr( - self, "_archive_pending", False) and not self.archive_file: + if ( + self.state == "AC" + and not getattr(self, "_archive_pending", False) + and not self.archive_file + ): self._archive() def pre_delete_import(sender, **kwargs): # deleted imported items when an import is delete - instance = kwargs.get('instance') + instance = kwargs.get("instance") if not instance: return to_delete = [] diff --git a/ishtar_common/templatetags/link_to_window.py b/ishtar_common/templatetags/link_to_window.py index c971ded26..692b9dd8d 100644 --- a/ishtar_common/templatetags/link_to_window.py +++ b/ishtar_common/templatetags/link_to_window.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from django.core.urlresolvers import reverse +from django.urls import reverse from django.template import Library from django.utils.safestring import mark_safe diff --git a/ishtar_common/templatetags/window_header.py b/ishtar_common/templatetags/window_header.py index 8b717e65c..5d0e85f72 100644 --- a/ishtar_common/templatetags/window_header.py +++ b/ishtar_common/templatetags/window_header.py @@ -1,5 +1,4 @@ from django import template -from django.core.urlresolvers import reverse from django.utils.safestring import mark_safe register = template.Library() diff --git a/ishtar_common/templatetags/window_tables.py b/ishtar_common/templatetags/window_tables.py index f91339cb2..d19a72f96 100644 --- a/ishtar_common/templatetags/window_tables.py +++ b/ishtar_common/templatetags/window_tables.py @@ -3,9 +3,9 @@ import time from django import template from django.conf import settings -from django.core.urlresolvers import resolve from django.template.defaultfilters import slugify from django.template.loader import get_template +from django.urls import reverse from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ diff --git a/ishtar_common/tests.py b/ishtar_common/tests.py index 50aec88b3..1b430cacd 100644 --- a/ishtar_common/tests.py +++ b/ishtar_common/tests.py @@ -45,7 +45,6 @@ from django.core.exceptions import ValidationError from django.core.files import File as DjangoFile from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management import call_command -from django.core.urlresolvers import reverse from django.db.models.fields import BooleanField from django.db.models.fields.related import ForeignKey from django.template.defaultfilters import slugify @@ -53,6 +52,7 @@ from django.test import tag, TestCase as BaseTestCase from django.test.client import Client from django.test.runner import DiscoverRunner from django.utils.translation import ugettext_lazy as _ +from django.urls import reverse from ishtar_common import models, models_common from ishtar_common import views diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py index b7a78a226..cbf4242b2 100644 --- a/ishtar_common/utils.py +++ b/ishtar_common/utils.py @@ -54,7 +54,7 @@ from django.core.exceptions import SuspiciousOperation, ObjectDoesNotExist from django.core.files import File from django.core.files.storage import FileSystemStorage from django.core.validators import EMPTY_VALUES -from django.core.urlresolvers import reverse +from django.urls import reverse from django.db import models from django.http import HttpResponseRedirect from django.utils.datastructures import MultiValueDict as BaseMultiValueDict diff --git a/ishtar_common/views.py b/ishtar_common/views.py index 2dc42041f..24ef044bb 100644 --- a/ishtar_common/views.py +++ b/ishtar_common/views.py @@ -33,7 +33,6 @@ from django.contrib.auth.decorators import login_required from django.contrib.auth.views import redirect_to_login from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist -from django.core.urlresolvers import reverse, NoReverseMatch from django.db.models import Q from django.template import loader from django.forms.models import modelformset_factory @@ -45,6 +44,7 @@ from django.http import ( JsonResponse, ) from django.shortcuts import redirect, render, get_object_or_404 +from django.urls import reverse from django.utils.decorators import method_decorator from django.utils.translation import ugettext, ugettext_lazy as _ from django.views.generic import ListView, TemplateView, View diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py index e9d28e306..abe7fa11b 100644 --- a/ishtar_common/views_item.py +++ b/ishtar_common/views_item.py @@ -17,7 +17,6 @@ from django.contrib.gis.geos import GEOSException from django.contrib.staticfiles.templatetags.staticfiles import static from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist -from django.core.urlresolvers import reverse, NoReverseMatch from django.db.models import ( Q, Count, @@ -36,6 +35,7 @@ from django.forms.models import model_to_dict from django.http import HttpResponse from django.shortcuts import render from django.template import loader +from django.urls import reverse, NoReverseMatch from django.utils.translation import ( ugettext, ugettext_lazy as _, diff --git a/ishtar_common/widgets.py b/ishtar_common/widgets.py index 4852f4ceb..bab6b3e00 100644 --- a/ishtar_common/widgets.py +++ b/ishtar_common/widgets.py @@ -25,7 +25,6 @@ from django import forms from django.conf import settings from django.core.exceptions import ValidationError from django.core.files import File -from django.core.urlresolvers import reverse, NoReverseMatch from django.db.models import fields from django.forms import ClearableFileInput from django.forms.utils import flatatt @@ -35,6 +34,7 @@ from django.forms.widgets import ( ) from django.template import loader from django.template.defaultfilters import slugify +from django.urls import reverse, NoReverseMatch from django.utils.encoding import smart_text from django.utils.functional import lazy from django.utils.html import escape diff --git a/requirements.txt b/requirements.txt index 72b40cd31..4f5630fb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,36 +1,62 @@ six>=1.9 + +# gdal==2.4.0 +# 3.2 + psycopg2-binary==2.7.7 -django-registration==2.2 -django==1.11.28 +# 2.8.6 + +django-registration==3.2 +django==2.2.24 + Pillow==5.4.1 +# 8.1.2 + WeasyPrint==0.42.3 html5lib==1.0.1 - +# 1.1 pyqrcode==1.2.1 + pypng==0.0.19 +# 0.0.20 + xmltodict==0.11 +# 0.12 requests==2.21 +# 2.25 # python-memcached==1.59 ## production # celery==4.2.1 ## not mandatory +# 5.0.0 + djangorestframework==3.9 +# 3.12 pytidylib==0.3.2 lxml==4.3.2 +# 4.6.3 Jinja2==2.10 - +# 2.11 django-extra-views==0.12.0 +# 0.13 beautifulsoup4==4.7.1 +# 4.9.3 markdown==3.0.1 +# 3.3.4 django-ajax-selects==1.7.0 +# -> à remplacer - intégré nativement dans Django django-compressor==2.2 +# 2.4 django-formtools==2.0 +# 2.2 secretary==0.2.19 unidecode +# 1.2 -e git+https://github.com/treyhunner/django-simple-history.git@2.7.0#egg=django-simple-history django-extensions==2.1.4 +# 3.0.3 |