♻️ django 3.2 - new version of libraries: fix errors and deprecation warnings

14 files changed, 66 insertions, 56 deletions

diff --git a/Makefile.example b/Makefile.example
index aba1818d3..aa49d27ec 100644
--- a/Makefile.example
+++ b/Makefile.example
@@ -123,21 +123,21 @@ test_verbose: clean  ## launch tests with verbose
 	cd $(project); $(PYTHON) manage.py test -v 2 $(apps)
 
 soft_test: clean  ## launch tests without database regeneration
-	cd $(project); $(PYTHON) manage.py test -k --tag gis --exclude-tag ui --exclude-tag libreoffice $(apps)
-	cd $(project); $(PYTHON) manage.py test -k --exclude-tag gis --exclude-tag ui $(apps)
+	cd $(project); $(PYTHON) manage.py test --keep --tag gis --exclude-tag ui --exclude-tag libreoffice $(apps)
+	cd $(project); $(PYTHON) manage.py test --keep --exclude-tag gis --exclude-tag ui $(apps)
 
 soft_test_verbose: clean  ## launch tests without database regeneration - verbose
-	cd $(project); $(PYTHON) manage.py test -k --verbosity 2 $(apps)
+	cd $(project); $(PYTHON) manage.py test --keep --verbosity 2 $(apps)
 
 soft_test_multi: clean  ## launch multi-process tests without database regeneration
-	cd $(project); $(PYTHON) manage.py test -k --parallel $(NB_PROCESS) --exclude-tag libreoffice --exclude-tag ui --tag gis $(apps)
-	cd $(project); $(PYTHON) manage.py test -k --parallel $(NB_PROCESS) --exclude-tag libreoffice --exclude-tag ui --exclude-tag gis $(apps)
+	cd $(project); $(PYTHON) manage.py test --keep --parallel $(NB_PROCESS) --exclude-tag libreoffice --exclude-tag ui --tag gis $(apps)
+	cd $(project); $(PYTHON) manage.py test --keep --parallel $(NB_PROCESS) --exclude-tag libreoffice --exclude-tag ui --exclude-tag gis $(apps)
 
 test_ui: clean  ## launch tests for UI
 	cd $(project); $(PYTHON) manage.py test --tag ui $(apps)
 
 soft_test_ui: clean  ## launch soft tests for UI
-	cd $(project); $(PYTHON) manage.py test -k --tag ui $(apps)
+	cd $(project); $(PYTHON) manage.py test --keep --tag ui $(apps)
 
 build_gitlab: clean collectstatic makemessages  ## specific build for gitlab
 	cd $(project); $(PYTHON) ./manage.py migrate
@@ -146,7 +146,7 @@ test_gitlab: clean collectstatic makemessages ## specific test for gitlab
 	cd $(project); $(PYTHON) manage.py test --exclude-tag no_ci --exclude-tag gis --exclude-tag ui --exclude-tag libreoffice $(apps)
 
 soft_test_gitlab: build_gitlab  ## run test for gitlab - do not erase previous database
-	cd $(project); $(PYTHON) manage.py test -k --exclude-tag no_ci --exclude-tag gis --exclude-tag ui --exclude-tag libreoffice $(apps)
+	cd $(project); $(PYTHON) manage.py test --keep --exclude-tag no_ci --exclude-tag gis --exclude-tag ui --exclude-tag libreoffice $(apps)
 
 run_libreoffice:  ## run libreoffice daemon for testing purpose
 	/usr/bin/libreoffice --headless --accept="socket,host=127.0.0.1,port=8101;urp;" --nodefault --nofirststartwizard --nolockcheck --nologo --norestore --invisible --pidfile=/var/run/libreoffice.pid
diff --git a/archaeological_files/models.py b/archaeological_files/models.py
index 1aeb191cf..1f3475537 100644
--- a/archaeological_files/models.py
+++ b/archaeological_files/models.py
@@ -99,7 +99,7 @@ class Job(GeneralType):
     )
     ground_daily_cost = models.FloatField(_("Ground daily cost"), blank=True, null=True)
     daily_cost = models.FloatField(_("Daily cost"), blank=True, null=True)
-    permanent_contract = models.NullBooleanField(
+    permanent_contract = models.BooleanField(
         _("Permanent contract"), blank=True, null=True
     )
     default_daily_need_on_ground = models.FloatField(
@@ -790,16 +790,16 @@ class File(
     research_comment = models.TextField(
         _("Research archaeology comment"), blank=True, default=""
     )
-    classified_area = models.NullBooleanField(
+    classified_area = models.BooleanField(
         _("Classified area"), blank=True, null=True
     )
-    protected_area = models.NullBooleanField(_("Protected area"), blank=True, null=True)
+    protected_area = models.BooleanField(_("Protected area"), blank=True, null=True)
     if settings.COUNTRY == "fr":
-        cira_advised = models.NullBooleanField("Passage en CIRA", blank=True, null=True)
-        mh_register = models.NullBooleanField(
+        cira_advised = models.BooleanField("Passage en CIRA", blank=True, null=True)
+        mh_register = models.BooleanField(
             "Sur Monument Historique classé", blank=True, null=True
         )
-        mh_listing = models.NullBooleanField(
+        mh_listing = models.BooleanField(
             "Sur Monument Historique inscrit", blank=True, null=True
         )
     # <-- research archaeology
diff --git a/archaeological_finds/models_finds.py b/archaeological_finds/models_finds.py
index 83069bafb..902444a4a 100644
--- a/archaeological_finds/models_finds.py
+++ b/archaeological_finds/models_finds.py
@@ -2169,7 +2169,7 @@ class Find(
         _("Reference container - first full location"), default="", blank=True,
         help_text=_("Updated as long as no packaging is attached")
     )
-    is_complete = models.NullBooleanField(_("Is complete?"), blank=True, null=True)
+    is_complete = models.BooleanField(_("Is complete?"), blank=True, null=True)
     object_types = models.ManyToManyField(
         ObjectType, verbose_name=_("Object types"), related_name="find", blank=True
     )
diff --git a/archaeological_operations/models.py b/archaeological_operations/models.py
index 72838e815..bd6ae8e6e 100644
--- a/archaeological_operations/models.py
+++ b/archaeological_operations/models.py
@@ -1635,14 +1635,14 @@ class Operation(
     # preventive
     fnap_cost = models.IntegerField("Financement FNAP (€)", blank=True, null=True)
     # preventive diag
-    zoning_prescription = models.NullBooleanField(
+    zoning_prescription = models.BooleanField(
         _("Prescription on zoning"), blank=True, null=True
     )
     # preventive diag
-    large_area_prescription = models.NullBooleanField(
+    large_area_prescription = models.BooleanField(
         _("Prescription on large area"), blank=True, null=True
     )
-    geoarchaeological_context_prescription = models.NullBooleanField(
+    geoarchaeological_context_prescription = models.BooleanField(
         _("Prescription on geoarchaeological context"), blank=True, null=True
     )  # preventive diag
     cira_rapporteur = models.ForeignKey(
@@ -1653,7 +1653,7 @@ class Operation(
         on_delete=models.SET_NULL,
         verbose_name="Rapporteur CTRA/CIRA",
     )
-    negative_result = models.NullBooleanField(
+    negative_result = models.BooleanField(
         _("Result considered negative"), blank=True, null=True
     )
     cira_date = models.DateField("Date avis CTRA/CIRA", null=True, blank=True)
@@ -1713,14 +1713,14 @@ class Operation(
     documentation_deadline = models.DateField(
         _("Deadline for submission of the documentation"), blank=True, null=True
     )
-    documentation_received = models.NullBooleanField(
+    documentation_received = models.BooleanField(
         _("Documentation provided"), blank=True, null=True
     )
     finds_deposit_date = models.DateField(_("Finds deposit date"), blank=True, null=True)
     finds_deadline = models.DateField(
         _("Deadline for submission of the finds"), blank=True, null=True
     )
-    finds_received = models.NullBooleanField(_("Finds provided"), blank=True, null=True)
+    finds_received = models.BooleanField(_("Finds provided"), blank=True, null=True)
 
     # underwater
     drassm_code = models.CharField(
diff --git a/docs/generate_values_doc.py b/docs/generate_values_doc.py
index 6d9ca6d42..e71278ded 100644
--- a/docs/generate_values_doc.py
+++ b/docs/generate_values_doc.py
@@ -23,7 +23,6 @@ TYPES = {  # put more precise fields after generic field. e.g. IntegerField befo
     'DateTimeField': "Date/heure",
     'DateField': "Date",
     'EmailField': "Courriel",
-    'NullBooleanField': "Booléen",
     'BooleanField': "Booléen",
     'ImageField': "Image",
     'FileField': "Fichier",
diff --git a/example_project/settings.py b/example_project/settings.py
index 5b4e023cc..1667f1942 100644
--- a/example_project/settings.py
+++ b/example_project/settings.py
@@ -20,6 +20,7 @@ DEBUG_TO_CONSOLE = False
 SQL_DEBUG = False
 DJANGO_EXTENSIONS = False
 TEST_VIEWS = True
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
 if "test" in sys.argv:
     sys.path.insert(0, "..")
diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py
index 36331e2f7..cafc9d6e2 100644
--- a/ishtar_common/admin.py
+++ b/ishtar_common/admin.py
@@ -2507,11 +2507,16 @@ class JsonContentTypeFormMixin(object):
         exclude = []
 
     def __init__(self, *args, **kwargs):
-        super(JsonContentTypeFormMixin, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         choices = []
-        for pk, label in self.fields["content_type"].choices:
-            if not pk:
-                choices.append((pk, label))
+        for choice, label in self.fields["content_type"].choices:
+            if not choice:
+                choices.append(('', label))
+                continue
+            try:
+                pk = int(choice.value)
+            except ValueError:
+                choices.append(('', label))
                 continue
             ct = ContentType.objects.get(pk=pk)
             model_class = ct.model_class()
diff --git a/ishtar_common/data_importer.py b/ishtar_common/data_importer.py
index 9d470a084..78147ab8a 100644
--- a/ishtar_common/data_importer.py
+++ b/ishtar_common/data_importer.py
@@ -31,8 +31,7 @@ from django.conf import settings
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.gis.geos.error import GEOSException
-from django.db.models.fields import FieldDoesNotExist
-from django.core.exceptions import FieldError, MultipleObjectsReturned
+from django.core.exceptions import FieldDoesNotExist, FieldError, MultipleObjectsReturned
 from django.core.files import File
 from django.db import IntegrityError, DatabaseError, transaction
 from django.db.models import Q
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index aa18f6182..d81d25ca5 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -60,7 +60,7 @@ from django.contrib.gis.db import models
 from django.contrib.gis.db.models.aggregates import Union
 from django.contrib.gis.geos.polygon import Polygon
 from django.contrib.gis.geos.collections import MultiPolygon
-from django.contrib.postgres.fields import JSONField, ArrayField
+from django.contrib.postgres.fields import ArrayField
 from django.contrib.postgres.indexes import GinIndex
 from django.contrib.sites.models import Site
 from django.core.cache import cache
@@ -72,7 +72,7 @@ from django.core.exceptions import (
 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import connection, transaction
-from django.db.models import Q, Max, Count
+from django.db.models import JSONField, Q, Max, Count
 from django.db.models.signals import post_save, post_delete, pre_delete, m2m_changed
 from django.db.utils import DatabaseError
 from django.template import Context, Template
@@ -2961,7 +2961,7 @@ class Person(Address, Merge, OwnPerms, ValueGetter, MainItem):
         "ishtaruser__isnull": "ishtaruser__isnull",
         "attached_to": "attached_to",
     }
-    COL_LABELS = {"attached_to": _("Organization")}
+    COL_LABELS = {"attached_to": _("Organization"), "person_types_list": _("Types")}
 
     # alternative names of fields for searches
     ALT_NAMES = {
@@ -5220,7 +5220,7 @@ class Document(
     additional_information = models.TextField(
         _("Additional information"), blank=True, default=""
     )
-    duplicate = models.NullBooleanField(_("Has a duplicate"), blank=True, null=True)
+    duplicate = models.BooleanField(_("Has a duplicate"), blank=True, null=True)
     associated_links = models.TextField(_("Symbolic links"), blank=True, default="")
     cache_related_label = models.TextField(
         _("Related"),
diff --git a/ishtar_common/models_common.py b/ishtar_common/models_common.py
index 13447a4d3..9b9b974d4 100644
--- a/ishtar_common/models_common.py
+++ b/ishtar_common/models_common.py
@@ -30,7 +30,6 @@ from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.gis.db import models
 from django.contrib.gis.geos import GEOSGeometry, Point
 from django.contrib.gis.gdal.error import GDALException
-from django.contrib.postgres.fields import JSONField
 from django.contrib.postgres.search import SearchVectorField, SearchVector
 from django.contrib.sites.models import Site
 from django.core.cache import cache as django_cache
@@ -40,7 +39,7 @@ from django.core.serializers import serialize
 from django.urls import reverse, NoReverseMatch
 from django.core.validators import validate_slug
 from django.db import connection, transaction, OperationalError, IntegrityError
-from django.db.models import Q, Count, Max
+from django.db.models import JSONField, Q, Count, Max
 from django.db.models.signals import post_save, post_delete, m2m_changed
 from django.template import loader
 from django.template.defaultfilters import slugify
@@ -4351,7 +4350,7 @@ class Merge(models.Model):
     merge_key = models.TextField(_("Merge key"), blank=True, null=True)
     merge_candidate = models.ManyToManyField("self", blank=True)
     merge_exclusion = models.ManyToManyField("self", blank=True)
-    archived = models.NullBooleanField(default=False, blank=True, null=True)
+    archived = models.BooleanField(default=False, blank=True, null=True)
     # 1 for one word similarity, 2 for two word similarity, etc.
     MERGE_CLEMENCY = None
     EMPTY_MERGE_KEY = "--"
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py
index 55bbd1906..b66fe7a72 100644
--- a/ishtar_common/models_imports.py
+++ b/ishtar_common/models_imports.py
@@ -1352,6 +1352,7 @@ def convert_geom(feature, srid):
     if not feature:
         return feature
     srid = int(srid)
+    feature = dict(feature)
     geo_type = feature["type"]
     if geo_type in ("LineString", "Polygon"):
         feature["type"] = "Multi" + geo_type
@@ -2251,7 +2252,8 @@ class Import(BaseImport):
         if self.number_of_line:
             return self.number_of_line
         imported_values = self.get_imported_values()
-        if (not imported_values or not imported_values.path or not imported_values.path.endswith(".csv")) \
+        if (not imported_values or not imported_values.path
+                or not imported_values.path.endswith(".csv")) \
                 and self.importer_type.type == "gis":
             return self._data_table_gis(get_number_only=True)
         if not imported_values or not imported_values.path:
@@ -2260,8 +2262,11 @@ class Import(BaseImport):
         encodings = [self.encoding]
         encodings += [coding for coding, c in ENCODINGS if coding != self.encoding]
         for encoding in encodings:
+            options = {}
+            if encoding:
+                options["encoding"] = encoding
             try:
-                with open(filename, "r", encoding=encoding) as f:
+                with open(filename, "r", **options) as f:
                     reader = csv.reader(f, delimiter=self.csv_sep)
                     nb = sum(1 for __ in reader) - self.skip_lines
             except UnicodeDecodeError:
@@ -2499,6 +2504,8 @@ class Import(BaseImport):
                 # https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6
                 elif crs.startswith("+init=epsg:"):
                     srid = crs[len("+init=epsg:"):]
+                elif crs.lower().startswith("epsg:"):
+                    srid = crs[len("epsg:"):]
                 else:
                     srid = CRS.from_proj4(crs).to_epsg()
                 data = []
diff --git a/ishtar_common/utils.py b/ishtar_common/utils.py
index 04cda150b..4e558ebd0 100644
--- a/ishtar_common/utils.py
+++ b/ishtar_common/utils.py
@@ -30,7 +30,7 @@ import hashlib
 from importlib import import_module
 import io
 from jinja2 import Template
-from jinja2.filters import FILTERS, environmentfilter
+from jinja2.filters import FILTERS, pass_environment
 import locale
 import math
 import numpy
@@ -2723,7 +2723,7 @@ PARSE_JINJA = re.compile("{{([^}]*)}")
 PARSE_JINJA_IF = re.compile("{% if ([^}]*)}")
 
 
-@environmentfilter
+@pass_environment
 def _deduplicate(*args):
     value = args[0] if len(args) == 1 else args[1]  # jinja simple filter
     new_values = []
@@ -2742,55 +2742,55 @@ def _padding(formt, args):
     return formt.format(value)
 
 
-@environmentfilter
+@pass_environment
 def _padding03(*args):
     return _padding("{:0>3}", args)
 
 
-@environmentfilter
+@pass_environment
 def _padding04(*args):
     return _padding("{:0>4}", args)
 
 
-@environmentfilter
+@pass_environment
 def _padding05(*args):
     return _padding("{:0>5}", args)
 
 
-@environmentfilter
+@pass_environment
 def _padding06(*args):
     return _padding( "{:0>6}", args)
 
 
-@environmentfilter
+@pass_environment
 def _padding07(*args):
     return _padding( "{:0>7}", args)
 
 
-@environmentfilter
+@pass_environment
 def _padding08(*args):
     return _padding( "{:0>8}", args)
 
 
-@environmentfilter
+@pass_environment
 def _upper(*args):
     value = args[0] if len(args) == 1 else args[1]  # jinja simple filter
     return value.upper()
 
 
-@environmentfilter
+@pass_environment
 def _lower(*args):
     value = args[0] if len(args) == 1 else args[1]  # jinja simple filter
     return value.lower()
 
 
-@environmentfilter
+@pass_environment
 def _capitalize(*args):
     value = args[0] if len(args) == 1 else args[1]  # jinja simple filter
     return value.capitalize()
 
 
-@environmentfilter
+@pass_environment
 def _slug(*args):
     value = args[0] if len(args) == 1 else args[1]  # jinja simple filter
     return slugify(value)
diff --git a/ishtar_common/views_item.py b/ishtar_common/views_item.py
index e731d42ba..f7a2ca1ea 100644
--- a/ishtar_common/views_item.py
+++ b/ishtar_common/views_item.py
@@ -261,7 +261,7 @@ def new_qa_item(
     model, frm, many=False, template="ishtar/forms/qa_new_item.html", page_name="",
     callback=None
 ):
-    def func(request, parent_name, limits=""):
+    def func(request, parent_name=None, limits=""):
         not_permitted_msg = ugettext("Operation not permitted.")
         meta = model._meta
         permission = f"{meta.app_label}.add_{meta.model_name}"
@@ -898,7 +898,7 @@ def _parse_parentheses_groups(
         exc_dct = {}
     if not extra_distinct_q:
         extra_distinct_q = []
-    if type(groups) is not list:
+    if not isinstance(groups, list):
         string = groups.strip()
         if string.startswith('"') and string.endswith('"') and string.count('"') == 2:
             string = string[1:-1]
@@ -2743,8 +2743,10 @@ def get_item(
                 and_reqs[:]
             )
 
-            # print("ishtar_common/views_item.py - 2515")
-            # print(query, distinct_queries, base_query, exc_query, extras)
+            # print("ishtar_common/views_item.py - 2745")
+            # print(f"query: {query}", f"distinct_queries: {distinct_queries}",
+            #       f"base_query: {base_query}", f"exc_query: {exc_query}",
+            #       f"extras: {extras}")
             sub_items = model.objects.filter(query)
             for d_q in distinct_queries:
                 sub_items = sub_items.filter(d_q)
diff --git a/ishtar_common/widgets.py b/ishtar_common/widgets.py
index d0ade6fea..3e1f9d0f1 100644
--- a/ishtar_common/widgets.py
+++ b/ishtar_common/widgets.py
@@ -22,10 +22,8 @@ import logging
 from django import forms
 from django.conf import settings
 from django.contrib.gis import forms as gis_forms
-from django.contrib.gis.db import models as gis_models
-from django.core.exceptions import ValidationError
+from django.core.exceptions import FieldDoesNotExist, ValidationError
 from django.core.files import File
-from django.db.models import fields
 from django.forms import ClearableFileInput
 from django.forms.utils import flatatt
 from django.forms.widgets import (
@@ -1251,7 +1249,7 @@ class DataTable(Select2Media, forms.RadioSelect):
                     try:
                         field = field._meta.get_field(key)
                         field_verbose_name = field.verbose_name
-                    except (fields.FieldDoesNotExist, AttributeError):
+                    except (FieldDoesNotExist, AttributeError, KeyError):
                         if hasattr(field, key + "_lbl"):
                             field_verbose_name = getattr(field, key + "_lbl")
                         else: