diff options
Diffstat (limited to 'misc')
| -rw-r--r-- | misc/history_duplicate_clean.py | 31 | ||||
| -rw-r--r-- | misc/pre_import_sra_files.py | 79 | ||||
| -rwxr-xr-x | misc/simple_ooo_replace.py | 79 |
3 files changed, 0 insertions, 189 deletions
diff --git a/misc/history_duplicate_clean.py b/misc/history_duplicate_clean.py deleted file mode 100644 index 61d358720..000000000 --- a/misc/history_duplicate_clean.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Clean duplicate in history. -This should be unecessary now. -""" - -import datetime -from archaeological_operations.models import Operation, AdministrativeAct -from archaeological_files.models import File -from archaeological_context_records.models import ContextRecord -from archaeological_finds.models import Find, BaseFind, Treatment - -nb_deleted = {} -to_delete = [] -for model in [Operation, File, ContextRecord, AdministrativeAct, Find, - BaseFind, Treatment]: - nb_deleted[model.__name__] = 0 - for item in model.objects.all()[0:]: - c_user, c_date = None, None - for h in item.history.order_by('-history_modifier_id', '-history_date', - '-history_id').all(): - if c_user and c_date and h.history_modifier_id == c_user and \ - c_date - h.history_date < datetime.timedelta(seconds=5): - to_delete.append(h) - c_user = h.history_modifier_id - c_date = h.history_date - nb_deleted[model.__name__] += len(to_delete) - -for item in to_delete: - item.delete() -for m in nb_deleted: - print "* %d deleted for %s" % (nb_deleted[m], m) diff --git a/misc/pre_import_sra_files.py b/misc/pre_import_sra_files.py deleted file mode 100644 index df00d3ef5..000000000 --- a/misc/pre_import_sra_files.py +++ /dev/null @@ -1,79 +0,0 @@ -import unicodecsv -import datetime - -from django.conf import settings - -from ishtar_common.data_importer import Importer - - -def get_year(value): - try: - for fmt in ['%d/%m/%Y', '%d/%m/%Y']: - return datetime.datetime.strptime(value, fmt).year - except: - pass - -index_list = [] - - -def treatment(data): - internal_ref = data[37].strip() - creation = data[34].strip() - reception = data[19].strip() - yr = get_year(creation) - if not yr: - yr = get_year(reception) - - idx, year = None, None - if '-' in internal_ref: - year, y_idx = internal_ref.split('-') - if len(year) == 4: # 2007-XXXX - try: - year = int(year) - idx = int(y_idx) - except ValueError: - pass - elif '.' in internal_ref: - year, y_idx = internal_ref.split('.') - if len(year) == 4: # 2011.XXXX - try: - year = int(year) - idx = int(y_idx) - except ValueError: - pass - if not idx: - idx = int(internal_ref) - if year and year != yr: - yr = year - assert yr # we should absolutly have a year! - - external_id = "{}{}-{}".format(settings.ISHTAR_LOCAL_PREFIX, yr, idx) - assert (yr, external_id) not in index_list - index_list.append((yr, external_id)) - return yr, idx, external_id - - -new_datas = [] -with open('plouf.csv') as csv_file: - datas = [line for line in unicodecsv.reader(csv_file, - encoding='utf-8')] - for idx, data in enumerate(datas): - if idx < 3: - # headers - data.append('annee') - data.append('identifiant numerique') - data.append('external_id') - new_datas.append(data) - continue - try: - year, idx, external_id = treatment(data) - data.append(year) - data.append(idx) - data.append(external_id) - new_datas.append(data) - except Exception as e: - print("Line {}: {}".format(idx + 1, e)) - -csv = Importer()._get_csv(new_datas, empty=u'') -with open('plouf2.csv', 'w') as fle: - fle.write(csv.encode('utf-8')) diff --git a/misc/simple_ooo_replace.py b/misc/simple_ooo_replace.py deleted file mode 100755 index 62cbab7a5..000000000 --- a/misc/simple_ooo_replace.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import shutil -import sys -from zipfile import ZipFile, ZIP_DEFLATED - -rpl_lst = [ - ('adminact_associated_file_general_contractor_attached_to_name', - 'adminact_associated_file_corporation_general_contractor_name'), - ('adminact_associated_file_general_contractor_' - 'attached_to_address', - 'adminact_associated_file_corporation_general_contractor_' - 'address'), - ('adminact_associated_file_general_contractor_' - 'address_complement', - 'adminact_associated_file_corporation_general_contractor_' - 'address_complement '), - ('adminact_associated_file_general_contractor_' - 'attached_to_postal_code', - 'adminact_associated_file_corporation_general_contractor_' - 'postal_code '), - ('adminact_associated_file_general_contractor_attached_to_town', - 'adminact_associated_file_corporation_general_contractor_town', - ), - ('adminact_associated_file_address', - 'adminact_associated_file_get_locality', - ) -] - -context = dict(rpl_lst) - - -def value_replace(content): - value = content - modified = False - for key in context: - if key in value: - modified = True - value = value.replace(key, context[key]) - return value, modified - - -def replace(directory, infile): - print("Processing {}".format(infile)) - outfile = "PREPROCESS--" + infile - infile = directory + os.sep + infile - outfile = directory + os.sep + outfile - - inzip = ZipFile(infile, 'r', ZIP_DEFLATED) - outzip = ZipFile(outfile, 'w', ZIP_DEFLATED) - - values = {} - idx = 0 - for xml_file in ('content.xml', 'styles.xml'): - content = inzip.read(xml_file) - values[xml_file], modified = value_replace(content) - if modified: - idx += 1 - - for f in inzip.infolist(): - if f.filename in values: - outzip.writestr(f.filename, values[f.filename]) - else: - outzip.writestr(f, inzip.read(f.filename)) - - inzip.close() - outzip.close() - # replace original by PREPROCESS - shutil.move(outfile, infile) - return idx - -directory = sys.argv[-1] -idx = 0 - - -for fle in os.listdir(directory): - if fle.endswith('.odt'): - idx += replace(directory, fle) - -print("{} modifications".format(idx)) |