diff --git a/rog/models.py b/rog/models.py index 3470ba9..e2080f6 100644 --- a/rog/models.py +++ b/rog/models.py @@ -19,6 +19,10 @@ from .choices import LAYER_CHOICES from django.contrib.gis.utils import LayerMapping from django.apps import apps +import csv +import codecs +import sys + env = environ.Env(DEBUG=(bool, False)) environ.Env.read_env(env_file=".env") @@ -317,6 +321,26 @@ class ShapeLayers(models.Model): return self.name +def remove_bom_inplace(path): + """Removes BOM mark, if it exists, from a file and rewrites it in-place""" + buffer_size = 4096 + bom_length = len(codecs.BOM_UTF8) + + with open(path, "r+b") as fp: + chunk = fp.read(buffer_size) + if chunk.startswith(codecs.BOM_UTF8): + i = 0 + chunk = chunk[bom_length:] + while chunk: + fp.seek(i) + fp.write(chunk) + i += len(chunk) + fp.seek(bom_length, os.SEEK_CUR) + chunk = fp.read(buffer_size) + fp.seek(-bom_length, os.SEEK_CUR) + fp.truncate() + + @receiver(pre_save, sender=ShapeLayers) @@ -355,28 +379,52 @@ def publish_date(sender, instance, created, **kwargs): print('##################',e) try: - csv = glob.glob(r'{}/**/*.csv'.format(file_path), recursive=True)[0] + csv_f = glob.glob(r'{}/**/*.csv'.format(file_path), recursive=True)[0] + + remove_bom_inplace(csv_f) mdl = apps.get_model(app_label="rog", model_name=LAYER_CHOICES[instance.layerof -1][1]) print(mdl) print(f"#### instance.layerof - {instance.layerof}") - with open(csv, encoding='"utf-8-sig"') as txt_file: - #with open(csv, 'r') as txt_file: - lns = txt_file.readlines() - for ln in lns: - fields = ln.split(",") + with open(csv_f, 'r') as txt_file: + reader = csv.reader(txt_file, delimiter=',') + for fields in reader: + #fields = ln.split(",") if instance.layerof == 2: updateLineTable(mdl, fields) if instance.layerof == 3: - updatePolygonTable(mdl, fields) + updatePolygonTable(mdl, fields) + # with open(csv_f, 'r') as txt_file: + # reader = csv.reader(codecs.EncodedFile(txt_file, 'utf-8', 'utf-8-sig'), delimiter=",") + # for ln in reader: + # pass + # print(str(ln)) + # fields = ln.split(",") + # if instance.layerof == 2: + # updateLineTable(mdl, fields) + # if instance.layerof == 3: + # updatePolygonTable(mdl, fields) + #with open(csv, encoding='"utf-8-sig"') as txt_file: + #with open(csv, 'r') as txt_file: + # lns = txt_file.readlines() + # print(lns) + # for ln in lns: + # fields = ln.split(",") + # if instance.layerof == 2: + # updateLineTable(mdl, fields) + # if instance.layerof == 3: + # updatePolygonTable(mdl, fields) except Exception as e: print('##################',e) def updateLineTable(mdl, fields): - #print(f"Updated {fields[0]} - {fields[1]}") + print("Done") + print(f"Updated {fields[0]} - {fields[1]}") print(len(fields)) + print(fields[0]) + print(mdl.objects.filter(location_id = int(fields[0]))) mdl.objects.filter(location_id = int(fields[0])).update( location_name= fields[1] if len(fields) > 1 else '', category=fields[2] if len(fields) > 2 else '', @@ -404,24 +452,24 @@ def updateLineTable(mdl, fields): def updatePolygonTable(mdl, fields): #print(f"Updated {fields[0]} - {fields[1]}") mdl.objects.filter(location_id = fields[0]).update( - location_name=fields[1], - category=fields[2], - zip=fields[3], - address=fields[4], - prefecture=fields[5], - area=fields[6], - city=fields[7], - photos=fields[8], - videos=fields[9], - webcontents=fields[10], - status=fields[11], - portal=fields[12], - group=fields[13], - phone=fields[14], - fax=fields[15], - email=fields[16], - facility=fields[17], - remark=fields[18], - tags=fields[19], - parammeters=fields[20] + location_name= fields[1] if len(fields) > 1 else '', + category=fields[2] if len(fields) > 2 else '', + zip=fields[3] if len(fields) > 3 else '', + address=fields[4] if len(fields) > 4 else '', + prefecture=fields[5] if len(fields) > 5 else '', + area=fields[6] if len(fields) > 6 else '', + city=fields[7] if len(fields) > 7 else '', + photos=fields[8] if len(fields) > 8 else '', + videos=fields[9] if len(fields) > 9 else '', + webcontents=fields[10] if len(fields) > 10 else '', + status=fields[11] if len(fields) > 11 else '', + portal=fields[12] if len(fields) > 12 else '', + group=fields[13] if len(fields) > 13 else '', + phone=fields[14] if len(fields) > 14 else '', + fax=fields[15] if len(fields) > 15 else '', + email=fields[16] if len(fields) > 16 else '', + facility=fields[17] if len(fields) > 17 else '', + remark=fields[18] if len(fields) > 18 else '', + tags=fields[19] if len(fields) > 19 else '', + parammeters=fields[20] if len(fields) > 20 else '' )