diff options
Diffstat (limited to 'ishtar_common/models_imports.py')
| -rw-r--r-- | ishtar_common/models_imports.py | 169 |
1 files changed, 152 insertions, 17 deletions
diff --git a/ishtar_common/models_imports.py b/ishtar_common/models_imports.py index 9afd435da..84008bb01 100644 --- a/ishtar_common/models_imports.py +++ b/ishtar_common/models_imports.py @@ -760,16 +760,19 @@ class FormaterType(models.Model): return IMPORTER_TYPES_DCT[self.formater_type](**kwargs) -IMPORT_STATE = (("C", _(u"Created")), - ("AP", _(u"Analyse in progress")), - ("A", _(u"Analysed")), - ("P", _(u"Import pending")), - ("IQ", _(u"Import in queue")), - ("IP", _(u"Import in progress")), - ("FE", _(u"Finished with errors")), - ("F", _(u"Finished")), - ("AC", _(u"Archived")), - ) +IMPORT_STATE = ( + ("C", _(u"Created")), + ("AP", _(u"Analyse in progress")), + ("A", _(u"Analysed")), + ("HQ", _(u"Check modified in queue")), + ("IQ", _(u"Import in queue")), + ("HP", _(u"Check modified in progress")), + ("IP", _(u"Import in progress")), + ("PI", _(u"Partially imported")), + ("FE", _(u"Finished with errors")), + ("F", _(u"Finished")), + ("AC", _(u"Archived")), +) IMPORT_STATE_DCT = dict(IMPORT_STATE) ENCODINGS = [(settings.ENCODING, settings.ENCODING), @@ -777,6 +780,7 @@ ENCODINGS = [(settings.ENCODING, settings.ENCODING), ('utf-8', 'utf-8')] delayed_import = None +delayed_check = None if settings.USE_BACKGROUND_TASK: @@ -788,6 +792,14 @@ if settings.USE_BACKGROUND_TASK: pass imp.importation(session_key=session_key) + @background(schedule=1) + def delayed_check(import_pk, session_key): + try: + imp = Import.objects.get(pk=import_pk) + except Import.DoesNotExist: + pass + imp.check_modified(session_key=session_key) + class Import(models.Model): user = models.ForeignKey('IshtarUser', blank=True, null=True, @@ -838,6 +850,12 @@ class Import(models.Model): _(u"Imported line numbers"), blank=True, null=True, validators=[validate_comma_separated_integer_list] ) + changed_checked = models.BooleanField(_(u"Changed have been checked"), + default=False) + changed_line_numbers = models.TextField( + _(u"Changed line numbers"), blank=True, null=True, + validators=[validate_comma_separated_integer_list] + ) class Meta: verbose_name = _(u"Import") @@ -879,7 +897,7 @@ class Import(models.Model): def add_imported_line(self, idx_line): if self.imported_line_numbers and \ - idx_line in self.imported_line_numbers.split(','): + str(idx_line) in self.imported_line_numbers.split(','): return if self.imported_line_numbers: self.imported_line_numbers += "," @@ -888,6 +906,35 @@ class Import(models.Model): self.imported_line_numbers += str(idx_line) self.save() + def add_changed_line(self, idx_line): + if self.changed_line_numbers and \ + str(idx_line) in self.changed_line_numbers.split(','): + return + if self.changed_line_numbers: + self.changed_line_numbers += "," + else: + self.changed_line_numbers = "" + self.changed_line_numbers += str(idx_line) + self.save() + + def remove_changed_line(self, idx_line): + if not self.changed_line_numbers: + return + line_numbers = self.changed_line_numbers.split(',') + if str(idx_line) not in line_numbers: + return + line_numbers.pop(line_numbers.index(str(idx_line))) + self.changed_line_numbers = ",".join(line_numbers) + self.save() + + def has_changes(self, idx_line): + if not self.changed_checked: + return True + if not self.changed_line_numbers: + return + line_numbers = self.changed_line_numbers.split(',') + return str(idx_line) in line_numbers + def line_is_imported(self, idx_line): return self.imported_line_numbers and \ str(idx_line) in self.imported_line_numbers.split(',') @@ -901,16 +948,24 @@ class Import(models.Model): actions = [] if self.state == 'C': actions.append(('A', _(u"Analyse"))) - if self.state == 'A': + if self.state in ('A', 'PI'): actions.append(('A', _(u"Re-analyse"))) actions.append(('I', _(u"Launch import"))) if profile.experimental_feature: - actions.append(('IS', _(u"Step by step import"))) + if self.changed_checked: + actions.append(('IS', _(u"Step by step import"))) + actions.append(('CH', _(u"Re-check for changes"))) + else: + actions.append(('CH', _(u"Check for changes"))) if self.state in ('F', 'FE'): actions.append(('A', _(u"Re-analyse"))) actions.append(('I', _(u"Re-import"))) if profile.experimental_feature: - actions.append(('IS', _(u"Step by step import"))) + if self.changed_checked: + actions.append(('IS', _(u"Step by step re-import"))) + actions.append(('CH', _(u"Re-check for changes"))) + else: + actions.append(('CH', _(u"Check for changes"))) actions.append(('AC', _(u"Archive"))) if self.state == 'AC': actions.append(('A', _(u"Unarchive"))) @@ -977,6 +1032,77 @@ class Import(models.Model): self.end_date = datetime.datetime.now() self.save() + def delayed_check_modified(self, session_key): + if not settings.USE_BACKGROUND_TASK: + return self.check_modified(session_key=session_key) + put_session_message( + session_key, + unicode( + _(u"Modification check {} added to the queue")).format( + self.name), + "info") + self.state = 'HQ' + self.end_date = datetime.datetime.now() + self.save() + return delayed_check(self.pk, session_key) + + def check_modified(self, session_key=None): + self.state = 'HP' + self.end_date = datetime.datetime.now() + self.changed_line_numbers = "" + self.changed_checked = False + self.save() + + for idx in range(self.skip_lines, self.get_number_of_lines() + 1): + try: + imprt, data = self.importation( + simulate=True, + line_to_process=idx, + return_importer_and_data=True + ) + except IOError as e: + # error is identified as a change + self.add_changed_line(idx) + continue + + # no data is not normal and an error is identified as a change + if not data or not data[0]: + self.add_changed_line(idx) + continue + + # new objects is a change + if imprt.new_objects: + self.add_changed_line(idx) + continue + + # check all updated fields + changed = False + for path, obj, values, updated_values in imprt.updated_objects: + if changed: + break + for k in updated_values.keys(): + if changed: + break + current_value = getattr(obj, k) + updated_value = updated_values[k] + if hasattr(current_value, 'all'): + current_value = list(current_value.all()) + changed = False + for v in updated_value: + if v not in current_value: + changed = True + break + else: + if current_value != updated_value: + changed = True + break + if changed: + self.add_changed_line(idx) + continue + self.remove_changed_line(idx) + self.changed_checked = True + self.save() + def delayed_importation(self, session_key): if not settings.USE_BACKGROUND_TASK: return self.importation(session_key=session_key) @@ -1010,7 +1136,10 @@ class Import(models.Model): ids = [] ids.append(self.pk) put_session_var(session_key, 'current_import_id', ids) - self.state = 'FE' + if line_to_process: + self.state = 'PI' + else: + self.state = 'FE' self.save() if not return_importer_and_data: return @@ -1023,7 +1152,10 @@ class Import(models.Model): result_file, ContentFile(importer.get_csv_result().encode('utf-8'))) if importer.errors: - self.state = 'FE' + if line_to_process: + self.state = 'PI' + else: + self.state = 'FE' error_file = filename + "_errors_%s.csv" % now self.error_file.save( error_file, @@ -1033,7 +1165,10 @@ class Import(models.Model): self.name) msg_cls = "warning" else: - self.state = 'F' + if line_to_process: + self.state = 'PI' + else: + self.state = 'F' self.error_file = None msg = unicode(_(u"Import {} finished with no errors")).format( self.name) |