Container: add parents - localisation refactoring

8 files changed, 233 insertions, 9 deletions

diff --git a/archaeological_warehouse/forms.py b/archaeological_warehouse/forms.py
index a7b6c575e..e03918965 100644
--- a/archaeological_warehouse/forms.py
+++ b/archaeological_warehouse/forms.py
@@ -259,14 +259,23 @@ class ContainerForm(CustomForm, ManageOldType, forms.Form):
     form_admin_name = _(u"Container - 010 - General")
     form_slug = "container-010-general"
     file_upload = True
-    extra_form_modals = ["warehouse", "organization", "person"]
+    extra_form_modals = ["warehouse", "organization", "person", "container"]
     associated_models = {'container_type': models.ContainerType,
                          'location': models.Warehouse,
+                         'parent': models.Container,
                          'responsible': models.Warehouse}
     reference = forms.CharField(label=_(u"Ref."), max_length=200)
     old_reference = forms.CharField(label=_(u"Old reference"), required=False,
                                     max_length=200)
     container_type = forms.ChoiceField(label=_(u"Container type"), choices=[])
+    parent = forms.IntegerField(
+        label=_("Parent container"),
+        widget=widgets.JQueryAutoComplete(
+            reverse_lazy('autocomplete-container'),
+            associated_model=models.Container, new=True),
+        validators=[valid_id(models.Container)],
+        required=False
+    )
     responsible = forms.IntegerField(
         label=_(u"Responsible warehouse"),
         widget=widgets.JQueryAutoComplete(
diff --git a/archaeological_warehouse/management/__init__.py b/archaeological_warehouse/management/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/archaeological_warehouse/management/__init__.py
diff --git a/archaeological_warehouse/management/commands/__init__.py b/archaeological_warehouse/management/commands/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/archaeological_warehouse/management/commands/__init__.py
diff --git a/archaeological_warehouse/management/commands/migrate_to_new_container_management.py b/archaeological_warehouse/management/commands/migrate_to_new_container_management.py
new file mode 100644
index 000000000..b5885cbf0
--- /dev/null
+++ b/archaeological_warehouse/management/commands/migrate_to_new_container_management.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2020 Étienne Loks  <etienne.loks_AT_peacefrogsDOTnet>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# See the file COPYING for details.
+
+import csv
+import sys
+
+from django.core.management.base import BaseCommand
+from django.template.defaultfilters import slugify
+from archaeological_warehouse import models
+
+
+class Command(BaseCommand):
+    help = 'Migrate to new container management (v3.0.6)'
+
+    def handle(self, *args, **options):
+        to_update = models.Container.objects.filter(
+                division__pk__isnull=False)
+        container_types = {}
+        created_nb = 0
+        for div_type in models.WarehouseDivision.objects.all():
+            container_type, c = models.ContainerType.objects.get_or_create(
+                txt_idx=slugify(div_type.label),
+                defaults={"label": div_type.label})
+            if c:
+                created_nb += 1
+                sys.stdout.write("-> {} created\n".format(
+                    div_type.label))
+            container_types[div_type.pk] = container_type
+        if created_nb:
+            sys.stdout.write("* {} container types created\n".format(
+                created_nb))
+        to_be_done = to_update.count()
+        created_nb = 0
+        potential_duplicate = {}
+        data = [("id", "warehouse", "reference",
+                 "old cached division", "new cached division")]
+        for idx, container in enumerate(models.Container.objects.filter(
+                division__pk__isnull=False).all()):
+            sys.stdout.write("* Updating: {}/{}\r".format(idx + 1, to_be_done))
+            sys.stdout.flush()
+            if container.responsible_id not in potential_duplicate:
+                potential_duplicate[container.responsible_id] = {}
+            parent = None
+            cached_division = container.cached_division
+            for division in container.division.order_by(
+                    "division__order").all():
+                ref = division.reference.strip()
+                if not ref or ref == "-":
+                    continue
+                new_container, created = models.Container.objects.get_or_create(
+                    reference=division.reference.strip(),
+                    parent=parent,
+                    container_type=container_types[
+                        division.division.division_id],
+                    location=container.responsible,
+                    responsible=container.responsible)
+                if created:
+                    created_nb += 1
+                    ref = "{} || {}".format(str(new_container.container_type),
+                                            slugify(division.reference.strip()))
+                    if ref not in potential_duplicate[container.responsible_id]:
+                        potential_duplicate[container.responsible_id][ref] = []
+                    if division.reference.strip() not in \
+                            potential_duplicate[container.responsible_id][ref]:
+                        potential_duplicate[container.responsible_id][
+                            ref].append(division.reference.strip())
+                parent = new_container
+            if parent:
+                container.parent = parent
+                container.save()
+                data.append((container.id, str(container.responsible),
+                             container.reference, cached_division,
+                             container._generate_cached_division()))
+        sys.stdout.write("\n* Potential duplicate:")
+        for warehouse_id in potential_duplicate.keys():
+            warehouse = models.Warehouse.objects.get(pk=warehouse_id)
+            for ref in potential_duplicate[warehouse_id]:
+                items = potential_duplicate[warehouse_id][ref]
+                if len(items) > 1:
+                    sys.stdout.write(
+                        "\n-> {}: {}".format(warehouse, " ; ".join(items)))
+        print("")
+        sys.stdout.write("* {} container created\n".format(created_nb))
+        if not data:
+            return
+        with open("new_containers.csv", 'w+') as f:
+            w = csv.writer(f)
+            w.writerows(data)
+        sys.stdout.write("-> check new containers in \"new_containers.csv\"\n")
+
diff --git a/archaeological_warehouse/migrations/0102_auto_20200324_1142.py b/archaeological_warehouse/migrations/0102_auto_20200324_1142.py
new file mode 100644
index 000000000..4006a9701
--- /dev/null
+++ b/archaeological_warehouse/migrations/0102_auto_20200324_1142.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.27 on 2020-03-24 11:42
+from __future__ import unicode_literals
+
+from django.conf import settings
+import django.contrib.gis.db.models.fields
+import django.contrib.postgres.fields.jsonb
+import django.contrib.postgres.search
+from django.db import migrations, models
+import django.db.models.deletion
+import simple_history.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('ishtar_common', '0202_auto_20200129_1941'),
+        ('archaeological_warehouse', '0101_squashed'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='HistoricalWarehouse',
+            fields=[
+                ('id', models.IntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')),
+                ('data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default={})),
+                ('search_vector', django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector')),
+                ('qrcode', models.TextField(blank=True, max_length=255, null=True)),
+                ('x', models.FloatField(blank=True, null=True, verbose_name='X')),
+                ('y', models.FloatField(blank=True, null=True, verbose_name='Y')),
+                ('z', models.FloatField(blank=True, null=True, verbose_name='Z')),
+                ('estimated_error_x', models.FloatField(blank=True, null=True, verbose_name='Estimated error for X')),
+                ('estimated_error_y', models.FloatField(blank=True, null=True, verbose_name='Estimated error for Y')),
+                ('estimated_error_z', models.FloatField(blank=True, null=True, verbose_name='Estimated error for Z')),
+                ('point', django.contrib.gis.db.models.fields.PointField(blank=True, dim=3, null=True, srid=4326, verbose_name='Point')),
+                ('point_2d', django.contrib.gis.db.models.fields.PointField(blank=True, null=True, srid=4326, verbose_name='Point (2D)')),
+                ('point_source', models.CharField(blank=True, choices=[('T', 'Town'), ('P', 'Precise'), ('M', 'Polygon')], max_length=1, null=True, verbose_name='Point source')),
+                ('point_source_item', models.CharField(blank=True, max_length=100, null=True, verbose_name='Point source item')),
+                ('multi_polygon', django.contrib.gis.db.models.fields.MultiPolygonField(blank=True, null=True, srid=4326, verbose_name='Multi polygon')),
+                ('multi_polygon_source', models.CharField(blank=True, choices=[('T', 'Town'), ('P', 'Precise'), ('M', 'Polygon')], max_length=1, null=True, verbose_name='Multi-polygon source')),
+                ('multi_polygon_source_item', models.CharField(blank=True, max_length=100, null=True, verbose_name='Multi polygon source item')),
+                ('last_modified', models.DateTimeField(blank=True, editable=False)),
+                ('history_m2m', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default={})),
+                ('need_update', models.BooleanField(default=False, verbose_name='Need update')),
+                ('locked', models.BooleanField(default=False, verbose_name='Item locked for edition')),
+                ('address', models.TextField(blank=True, null=True, verbose_name='Address')),
+                ('address_complement', models.TextField(blank=True, null=True, verbose_name='Address complement')),
+                ('postal_code', models.CharField(blank=True, max_length=10, null=True, verbose_name='Postal code')),
+                ('town', models.CharField(blank=True, max_length=150, null=True, verbose_name='Town (freeform)')),
+                ('country', models.CharField(blank=True, max_length=30, null=True, verbose_name='Country')),
+                ('alt_address', models.TextField(blank=True, null=True, verbose_name='Other address: address')),
+                ('alt_address_complement', models.TextField(blank=True, null=True, verbose_name='Other address: address complement')),
+                ('alt_postal_code', models.CharField(blank=True, max_length=10, null=True, verbose_name='Other address: postal code')),
+                ('alt_town', models.CharField(blank=True, max_length=70, null=True, verbose_name='Other address: town')),
+                ('alt_country', models.CharField(blank=True, max_length=30, null=True, verbose_name='Other address: country')),
+                ('phone', models.CharField(blank=True, max_length=18, null=True, verbose_name='Phone')),
+                ('phone_desc', models.CharField(blank=True, max_length=300, null=True, verbose_name='Phone description')),
+                ('phone2', models.CharField(blank=True, max_length=18, null=True, verbose_name='Phone description 2')),
+                ('phone_desc2', models.CharField(blank=True, max_length=300, null=True, verbose_name='Phone description 2')),
+                ('phone3', models.CharField(blank=True, max_length=18, null=True, verbose_name='Phone 3')),
+                ('phone_desc3', models.CharField(blank=True, max_length=300, null=True, verbose_name='Phone description 3')),
+                ('raw_phone', models.TextField(blank=True, null=True, verbose_name='Raw phone')),
+                ('mobile_phone', models.CharField(blank=True, max_length=18, null=True, verbose_name='Mobile phone')),
+                ('email', models.EmailField(blank=True, max_length=300, null=True, verbose_name='Email')),
+                ('alt_address_is_prefered', models.BooleanField(default=False, verbose_name='Alternative address is prefered')),
+                ('uuid', models.UUIDField(default=uuid.uuid4)),
+                ('name', models.CharField(max_length=200, verbose_name='Name')),
+                ('comment', models.TextField(blank=True, null=True, verbose_name='Comment')),
+                ('external_id', models.TextField(blank=True, null=True, verbose_name='External ID')),
+                ('auto_external_id', models.BooleanField(default=False, verbose_name='External ID is set automatically')),
+                ('history_id', models.AutoField(primary_key=True, serialize=False)),
+                ('history_date', models.DateTimeField()),
+                ('history_change_reason', models.CharField(max_length=100, null=True)),
+                ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+                ('history_creator', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to=settings.AUTH_USER_MODEL, verbose_name='Creator')),
+                ('history_modifier', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to=settings.AUTH_USER_MODEL, verbose_name='Last editor')),
+                ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+                ('lock_user', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to=settings.AUTH_USER_MODEL, verbose_name='Locked by')),
+                ('main_image', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='ishtar_common.Document', verbose_name='Main image')),
+                ('organization', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='ishtar_common.Organization', verbose_name='Organization')),
+                ('person_in_charge', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='ishtar_common.Person', verbose_name='Person in charge')),
+                ('precise_town', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='ishtar_common.Town', verbose_name='Town (precise)')),
+                ('spatial_reference_system', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='ishtar_common.SpatialReferenceSystem', verbose_name='Spatial Reference System')),
+                ('warehouse_type', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='archaeological_warehouse.WarehouseType', verbose_name='Warehouse type')),
+            ],
+            options={
+                'verbose_name': 'historical Warehouse',
+                'ordering': ('-history_date', '-history_id'),
+                'get_latest_by': 'history_date',
+            },
+            bases=(simple_history.models.HistoricalChanges, models.Model),
+        ),
+        migrations.AddField(
+            model_name='container',
+            name='parent',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='archaeological_warehouse.Container', verbose_name='Parent container'),
+        ),
+    ]
diff --git a/archaeological_warehouse/models.py b/archaeological_warehouse/models.py
index 7b98a385a..839ca1a6e 100644
--- a/archaeological_warehouse/models.py
+++ b/archaeological_warehouse/models.py
@@ -653,6 +653,9 @@ class Container(DocumentItem, LightHistorizedItem, QRCodeItem, GeoItem,
                                        null=True, blank=True, db_index=True)
     cached_division = models.TextField(_(u"Cached division"),
                                        null=True, blank=True, db_index=True)
+    parent = models.ForeignKey("Container", verbose_name=_("Parent container"),
+                               on_delete=models.SET_NULL,
+                               blank=True, null=True)
     index = models.IntegerField(u"Container ID", default=0)
     old_reference = models.TextField(_(u"Old reference"), blank=True, null=True)
     external_id = models.TextField(_(u"External ID"), blank=True, null=True)
@@ -695,16 +698,24 @@ class Container(DocumentItem, LightHistorizedItem, QRCodeItem, GeoItem,
 
     def _generate_cached_location(self):
         items = [self.location.name, str(self.index)]
-        cached_label = u" - ".join(items)
+        cached_label = " - ".join(items)
         return cached_label
 
     def _generate_cached_division(self):
+        parents = []
+        parent = self.parent
+        c_ids = []
+        while parent:
+            if parent.id in c_ids:  # prevent cyclic
+                break
+            c_ids.append(parent.id)
+            parents.append(parent)
+            parent = parent.parent
         locas = [
-            u"{} {}".format(loca.division.division, loca.reference)
-            for loca in ContainerLocalisation.objects.filter(
-                container=self)
+            "{} {}".format(loca.container_type.name, loca.reference)
+            for loca in reversed(parents)
         ]
-        return u" | ".join(locas)
+        return " | ".join(locas)
 
     def _get_base_image_path(self):
         return self.responsible._get_base_image_path() + u"/" + self.external_id
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index a22a9307f..4b4753fe3 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -4358,7 +4358,6 @@ class Organization(Address, Merge, OwnPerms, ValueGetter, MainItem):
     url = models.URLField(verbose_name=_("Web address"), blank=True, null=True)
     cached_label = models.TextField(_("Cached name"), null=True, blank=True,
                                     db_index=True)
-    history = HistoricalRecords()
 
     DOWN_MODEL_UPDATE = ['members']
 
@@ -4563,7 +4562,6 @@ class Person(Address, Merge, OwnPerms, ValueGetter, MainItem):
         verbose_name=_("Is attached to"), blank=True, null=True)
     cached_label = models.TextField(_("Cached name"), null=True, blank=True,
                                     db_index=True)
-    history = HistoricalRecords()
     DOWN_MODEL_UPDATE = ["author"]
 
     class Meta:
diff --git a/requirements.txt b/requirements.txt
index 680b2986a..45ae30150 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 six>=1.9
-psycopg2==2.7.7
+psycopg2-binary==2.7.7
 django-registration==2.2
 django==1.11.28
 Pillow==5.4.1