Migrate images and sources to new document model (refs #4107)

5 files changed, 265 insertions, 85 deletions

diff --git a/ishtar_common/admin.py b/ishtar_common/admin.py
index fd0efd455..88583ad29 100644
--- a/ishtar_common/admin.py
+++ b/ishtar_common/admin.py
@@ -474,8 +474,7 @@ class GeneralTypeAdmin(admin.ModelAdmin):
 
 general_models = [models.OrganizationType, models.SourceType,
                   models.AuthorType, models.TitleType, models.Format,
-                  models.SupportType, models.PersonType, models.LicenseType,
-                  models.ImageType]
+                  models.SupportType, models.PersonType, models.LicenseType]
 for model in general_models:
     admin_site.register(model, GeneralTypeAdmin)
 
@@ -963,8 +962,7 @@ admin_site.register(models.AdministrationScript, AdministrationScriptAdmin)
 
 
 class ImageAdmin(admin.ModelAdmin):
-    list_display = ('name', 'image_type', 'reference', 'internal_reference')
-    list_filter = ('image_type',)
+    list_display = ('title', 'reference', 'internal_reference')
     search_fields = ('name', 'reference', 'internal_reference')
 
 
diff --git a/ishtar_common/forms_common.py b/ishtar_common/forms_common.py
index 92c4c5797..134e7ceb1 100644
--- a/ishtar_common/forms_common.py
+++ b/ishtar_common/forms_common.py
@@ -1190,7 +1190,6 @@ class BaseImageForm(ManageOldType):
     form_label = _(u"Images")
     base_model = 'image'
     associated_models = {'licence': models.LicenseType,
-                         'image_type': models.ImageType,
                          'authors': models.Author}
 
     pk = forms.IntegerField(label=" ", widget=forms.HiddenInput,
@@ -1198,7 +1197,6 @@ class BaseImageForm(ManageOldType):
     image = forms.ImageField(
         label=_(u"Image"), help_text=mark_safe(get_image_help()),
         max_length=255, required=False, widget=widgets.ImageFileInput())
-    image_type = forms.ChoiceField(label=_(u"Type"), required=False, choices=[])
     name = forms.CharField(label=_(u"Name"), max_length=250, required=False)
     authors = widgets.Select2MultipleField(
         model=models.Author, remote=True, label=_(u"Authors"),
@@ -1219,7 +1217,6 @@ class BaseImageForm(ManageOldType):
                                  initial=False)
 
     TYPES = [
-        FieldType('image_type', models.ImageType, False),
         FieldType('licence', models.LicenseType, True),
     ]
 
diff --git a/ishtar_common/migrations/0055_auto_20180530_1900.py b/ishtar_common/migrations/0055_auto_20180530_1900.py
new file mode 100644
index 000000000..8a806fe21
--- /dev/null
+++ b/ishtar_common/migrations/0055_auto_20180530_1900.py
@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.10 on 2018-05-30 19:00
+from __future__ import unicode_literals
+
+import django.contrib.postgres.search
+from django.db import migrations, models
+import django.db.models.deletion
+import ishtar_common.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ishtar_common', '0054_auto_20180525_1249'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Document',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('image', models.ImageField(blank=True, max_length=255, null=True, upload_to=ishtar_common.models.get_image_path)),
+                ('thumbnail', models.ImageField(blank=True, max_length=255, null=True, upload_to=ishtar_common.models.get_image_path)),
+                ('search_vector', django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector')),
+                ('title', models.TextField(blank=True, default=b'', verbose_name='Title')),
+                ('index', models.IntegerField(blank=True, null=True, verbose_name='Index')),
+                ('external_id', models.TextField(blank=True, null=True, verbose_name='External ID')),
+                ('reference', models.TextField(blank=True, null=True, verbose_name='Ref.')),
+                ('internal_reference', models.TextField(blank=True, null=True, verbose_name='Internal ref.')),
+                ('scale', models.CharField(blank=True, max_length=30, null=True, verbose_name='Scale')),
+                ('authors_raw', models.CharField(blank=True, max_length=250, null=True, verbose_name='Authors (raw)')),
+                ('associated_url', models.URLField(blank=True, max_length=1000, null=True, verbose_name='Numerical ressource (web address)')),
+                ('receipt_date', models.DateField(blank=True, null=True, verbose_name='Receipt date')),
+                ('creation_date', models.DateField(blank=True, null=True, verbose_name='Creation date')),
+                ('receipt_date_in_documentation', models.DateField(blank=True, null=True, verbose_name='Receipt date in documentation')),
+                ('item_number', models.IntegerField(default=1, verbose_name='Item number')),
+                ('description', models.TextField(blank=True, null=True, verbose_name='Description')),
+                ('comment', models.TextField(blank=True, null=True, verbose_name='Comment')),
+                ('additional_information', models.TextField(blank=True, null=True, verbose_name='Additional information')),
+                ('duplicate', models.BooleanField(default=False, verbose_name='Has a duplicate')),
+                ('associated_links', models.TextField(blank=True, null=True, verbose_name='Symbolic links')),
+                ('authors', models.ManyToManyField(related_name='document_related', to='ishtar_common.Author', verbose_name='Authors')),
+                ('format_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.Format', verbose_name='Format')),
+                ('licenses', models.ManyToManyField(blank=True, to='ishtar_common.LicenseType', verbose_name='License')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(ishtar_common.models.OwnPerms, models.Model),
+        ),
+        migrations.AlterModelOptions(
+            name='ishtarimage',
+            options={'ordering': ('title',), 'verbose_name': 'Image', 'verbose_name_plural': 'Images'},
+        ),
+        migrations.RemoveField(
+            model_name='ishtarimage',
+            name='image_type',
+        ),
+        migrations.RemoveField(
+            model_name='ishtarimage',
+            name='name',
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='additional_information',
+            field=models.TextField(blank=True, null=True, verbose_name='Additional information'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='associated_url',
+            field=models.URLField(blank=True, max_length=1000, null=True, verbose_name='Numerical ressource (web address)'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='comment',
+            field=models.TextField(blank=True, null=True, verbose_name='Comment'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='duplicate',
+            field=models.BooleanField(default=False, verbose_name='Has a duplicate'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='external_id',
+            field=models.TextField(blank=True, null=True, verbose_name='External ID'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='format_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.Format', verbose_name='Format'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='index',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Index'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='item_number',
+            field=models.IntegerField(default=1, verbose_name='Item number'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='receipt_date',
+            field=models.DateField(blank=True, null=True, verbose_name='Receipt date'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='receipt_date_in_documentation',
+            field=models.DateField(blank=True, null=True, verbose_name='Receipt date in documentation'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='scale',
+            field=models.CharField(blank=True, max_length=30, null=True, verbose_name='Scale'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='search_vector',
+            field=django.contrib.postgres.search.SearchVectorField(blank=True, help_text='Auto filled at save', null=True, verbose_name='Search vector'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='source_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.SourceType', verbose_name='Type'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='support_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.SupportType', verbose_name='Support'),
+        ),
+        migrations.AddField(
+            model_name='ishtarimage',
+            name='title',
+            field=models.TextField(blank=True, default=b'', verbose_name='Title'),
+        ),
+        migrations.AddField(
+            model_name='sourcetype',
+            name='parent',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.SourceType', verbose_name='Parent'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarimage',
+            name='authors',
+            field=models.ManyToManyField(related_name='ishtarimage_related', to='ishtar_common.Author', verbose_name='Authors'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarimage',
+            name='internal_reference',
+            field=models.TextField(blank=True, null=True, verbose_name='Internal ref.'),
+        ),
+        migrations.AlterField(
+            model_name='ishtarimage',
+            name='reference',
+            field=models.TextField(blank=True, null=True, verbose_name='Ref.'),
+        ),
+        migrations.DeleteModel(
+            name='ImageType',
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='source_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.SourceType', verbose_name='Type'),
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='support_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='ishtar_common.SupportType', verbose_name='Support'),
+        ),
+    ]
diff --git a/ishtar_common/models.py b/ishtar_common/models.py
index bdc621528..ef4ae63e2 100644
--- a/ishtar_common/models.py
+++ b/ishtar_common/models.py
@@ -20,54 +20,50 @@
 """
 Models description
 """
-from cStringIO import StringIO
 import copy
 import datetime
 import inspect
-from PIL import Image
 import logging
 import os
 import re
-from secretary import Renderer as SecretaryRenderer
 import shutil
-from subprocess import Popen, PIPE
 import tempfile
 import time
-from unidecode import unidecode
+from cStringIO import StringIO
+from subprocess import Popen, PIPE
 
+from PIL import Image
 from django.conf import settings
+from django.contrib.auth.models import User, Group
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType
+from django.contrib.gis.db import models
 from django.contrib.postgres.fields import JSONField
 from django.contrib.postgres.search import SearchVectorField, SearchVector
 from django.core.cache import cache
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.core.files.uploadedfile import SimpleUploadedFile
-from django.core.validators import validate_slug
 from django.core.urlresolvers import reverse, NoReverseMatch
-from django.db.utils import DatabaseError
+from django.core.validators import validate_slug
 from django.db.models import Q, Max, Count
 from django.db.models.signals import post_save, post_delete, m2m_changed
+from django.db.utils import DatabaseError
+from django.template.defaultfilters import slugify
 from django.utils.functional import lazy
-from django.utils.translation import ugettext_lazy as _
-
 from django.utils.safestring import SafeUnicode, mark_safe
-from django.template.defaultfilters import slugify
-
-from django.contrib.auth.models import User, Group
-from django.contrib.contenttypes.models import ContentType
-from django.contrib.contenttypes.fields import GenericForeignKey
-from django.contrib.gis.db import models
-
+from django.utils.translation import ugettext_lazy as _
+from secretary import Renderer as SecretaryRenderer
 from simple_history.models import HistoricalRecords as BaseHistoricalRecords
+from unidecode import unidecode
 
 from ishtar_common.model_merging import merge_model_objects
-from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug,\
-    get_all_field_names, merge_tsvectors, cached_label_changed, \
-    generate_relation_graph
-
 from ishtar_common.models_imports import ImporterModel, ImporterType, \
     ImporterDefault, ImporterDefaultValues, ImporterColumn, \
     ImporterDuplicateField, Regexp, ImportTarget, TargetKey, FormaterType, \
     Import, TargetKeyGroup
+from ishtar_common.utils import get_cache, disable_for_loaddata, create_slug, \
+    get_all_field_names, merge_tsvectors, cached_label_changed, \
+    generate_relation_graph
 
 __all__ = [
     'ImporterModel', 'ImporterType', 'ImporterDefault', 'ImporterDefaultValues',
@@ -3038,7 +3034,7 @@ class Author(FullSearch):
             list(self.contextrecordsource_related.all())
 
 
-class SourceType(GeneralType):
+class SourceType(HierarchicalType):
     class Meta:
         verbose_name = _(u"Source type")
         verbose_name_plural = _(u"Source types")
@@ -3070,11 +3066,31 @@ post_save.connect(post_save_cache, sender=Format)
 post_delete.connect(post_save_cache, sender=Format)
 
 
-class Source(OwnPerms, ImageModel, FullSearch):
-    title = models.CharField(_(u"Title"), max_length=300)
-    external_id = models.TextField(_(u"External ID"), max_length=300, null=True,
-                                   blank=True)
-    source_type = models.ForeignKey(SourceType, verbose_name=_(u"Type"))
+class LicenseType(GeneralType):
+    url = models.URLField(_(u"URL"), blank=True, null=True)
+
+    class Meta:
+        verbose_name = _(u"License type")
+        verbose_name_plural = _(u"License types")
+        ordering = ('label',)
+
+
+post_save.connect(post_save_cache, sender=LicenseType)
+post_delete.connect(post_save_cache, sender=LicenseType)
+
+
+class BaseSource(OwnPerms, ImageModel, FullSearch):
+    title = models.TextField(_(u"Title"), blank=True, default='')
+    index = models.IntegerField(verbose_name=_(u"Index"), blank=True,
+                                null=True)
+    external_id = models.TextField(_(u"External ID"), null=True, blank=True)
+    reference = models.TextField(_(u"Ref."), null=True, blank=True)
+    internal_reference = models.TextField(_(u"Internal ref."), null=True,
+                                          blank=True)
+    source_type = models.ForeignKey(SourceType, verbose_name=_(u"Type"),
+                                    null=True, blank=True)
+    licenses = models.ManyToManyField(LicenseType, verbose_name=_(u"License"),
+                                      blank=True)
     support_type = models.ForeignKey(SupportType, verbose_name=_(u"Support"),
                                      blank=True, null=True,)
     format_type = models.ForeignKey(Format, verbose_name=_(u"Format"),
@@ -3083,6 +3099,8 @@ class Source(OwnPerms, ImageModel, FullSearch):
                              blank=True)
     authors = models.ManyToManyField(Author, verbose_name=_(u"Authors"),
                                      related_name="%(class)s_related")
+    authors_raw = models.CharField(verbose_name=_(u"Authors (raw)"),
+                                   blank=True, null=True, max_length=250)
     associated_url = models.URLField(
         blank=True, null=True, max_length=1000,
         verbose_name=_(u"Numerical ressource (web address)"))
@@ -3094,15 +3112,13 @@ class Source(OwnPerms, ImageModel, FullSearch):
         blank=True, null=True,
         verbose_name=_(u"Receipt date in documentation"))
     item_number = models.IntegerField(_(u"Item number"), default=1)
-    reference = models.CharField(_(u"Ref."), max_length=100, null=True,
-                                 blank=True)
-    internal_reference = models.CharField(
-        _(u"Internal ref."), max_length=100, null=True, blank=True)
     description = models.TextField(_(u"Description"), blank=True, null=True)
     comment = models.TextField(_(u"Comment"), blank=True, null=True)
     additional_information = models.TextField(_(u"Additional information"),
                                               blank=True, null=True)
     duplicate = models.BooleanField(_(u"Has a duplicate"), default=False)
+    associated_links = models.TextField(_(u"Symbolic links"), blank=True,
+                                        null=True)
 
     TABLE_COLS = ['title', 'source_type', 'authors', 'associated_url']
     COL_LINK = ['associated_url']
@@ -3132,22 +3148,11 @@ class Source(OwnPerms, ImageModel, FullSearch):
         return slugify(u"-".join(values))
 
 
-class LicenseType(GeneralType):
-    url = models.URLField(_(u"URL"), blank=True, null=True)
-    class Meta:
-        verbose_name = _(u"License type")
-        verbose_name_plural = _(u"License types")
-        ordering = ('label',)
-
-
-class ImageType(GeneralType):
-    class Meta:
-        verbose_name = _(u"Image type")
-        verbose_name_plural = _(u"Image types")
-        ordering = ('label',)
+class Document(BaseSource):
+    pass
 
 
-class IshtarImage(ImageModel):
+class IshtarImage(BaseSource):
     # order is important: put the image in the first match found
     # other will be symbolic links
     RELATED_MODELS = [
@@ -3156,30 +3161,10 @@ class IshtarImage(ImageModel):
     ]
     LINK_SPLIT = u"<||>"
 
-    name = models.CharField(_(u"Name"), max_length=250, blank=True, null=True)
-    description = models.TextField(_(u"Description"), blank=True, null=True)
-    associated_links = models.TextField(_(u"Symbolic links"), blank=True,
-                                        null=True)
-    licenses = models.ManyToManyField(LicenseType, verbose_name=_(u"License"),
-                                      blank=True)
-    authors = models.ManyToManyField(Author, verbose_name=_(u"Authors"),
-                                     blank=True)
-    authors_raw = models.CharField(verbose_name=_(u"Authors (raw)"),
-                                   blank=True, null=True, max_length=250)
-
-    image_type = models.ForeignKey(ImageType, verbose_name=_(u"Type"),
-                                   blank=True, null=True)
-    creation_date = models.DateField(blank=True, null=True,
-                                     verbose_name=_(u"Creation date"))
-    reference = models.CharField(_(u"Ref."), max_length=250, null=True,
-                                 blank=True)
-    internal_reference = models.CharField(
-        _(u"Internal ref."), max_length=250, null=True, blank=True)
-
     class Meta:
         verbose_name = _(u"Image")
         verbose_name_plural = _(u"Images")
-        ordering = ('name',)
+        ordering = ('title',)
 
     def _get_base_image_paths(self):
         for related_model in self.RELATED_MODELS:
@@ -3314,22 +3299,22 @@ class ThroughImage(models.Model):
         self.image.save()
 
 
-if settings.COUNTRY == 'fr':
-    class Arrondissement(models.Model):
-        name = models.CharField(u"Nom", max_length=30)
-        department = models.ForeignKey(Department, verbose_name=u"Département")
+class Arrondissement(models.Model):
+    name = models.CharField(u"Nom", max_length=30)
+    department = models.ForeignKey(Department, verbose_name=u"Département")
 
-        def __unicode__(self):
-            return settings.JOINT.join((self.name, unicode(self.department)))
+    def __unicode__(self):
+        return settings.JOINT.join((self.name, unicode(self.department)))
 
-    class Canton(models.Model):
-        name = models.CharField(u"Nom", max_length=30)
-        arrondissement = models.ForeignKey(Arrondissement,
-                                           verbose_name=u"Arrondissement")
 
-        def __unicode__(self):
-            return settings.JOINT.join(
-                (self.name, unicode(self.arrondissement)))
+class Canton(models.Model):
+    name = models.CharField(u"Nom", max_length=30)
+    arrondissement = models.ForeignKey(Arrondissement,
+                                       verbose_name=u"Arrondissement")
+
+    def __unicode__(self):
+        return settings.JOINT.join(
+            (self.name, unicode(self.arrondissement)))
 
 
 class TownManager(models.GeoManager):
diff --git a/ishtar_common/utils_migrations.py b/ishtar_common/utils_migrations.py
index 9f514ea48..48e9e4f9b 100644
--- a/ishtar_common/utils_migrations.py
+++ b/ishtar_common/utils_migrations.py
@@ -46,3 +46,32 @@ def migrate_simple_image_to_m2m(base_model, image_model, rel_model,
         print(base_model)
         print("{} missing".format(missing))
         print("{} moved".format(moved))
+
+
+def migrate_images(apps, base_model, rel_model):
+    IshtarImage = apps.get_model('ishtar_common', 'IshtarImage')
+    Document = apps.get_model('ishtar_common', 'Document')
+    for image_rel in rel_model.objects.order_by('is_main').all():
+        image = IshtarImage.objects.get(pk=image_rel.image.pk)
+        doc = Document.objects.create(image=image.image,
+                                      thumbnail=image.thumbnail)
+        item = base_model.objects.get(pk=image_rel.item.pk)
+        item.documents.add(doc)
+
+
+def migrate_sources(apps, base_model, source_model, item_attr):
+    Document = apps.get_model('ishtar_common', 'Document')
+    for source in source_model.objects.all():
+        doc = Document.objects.create()
+        for attr in ['title', 'index', 'external_id', 'reference',
+                     'internal_reference', 'source_type', 'support_type',
+                     'format_type', 'scale', 'associated_url', 'receipt_date',
+                     'creation_date', 'receipt_date_in_documentation',
+                     'item_number', 'description', 'comment',
+                     'additional_information', 'duplicate']:
+            setattr(doc, attr, getattr(source, attr))
+        doc.save()
+        for author in source.authors.all():
+            doc.authors.add(author)
+        item = base_model.objects.get(pk=getattr(source, item_attr).pk)
+        item.documents.add(doc)