update

rog/models.py

@@ -19,6 +19,10 @@ from .choices import LAYER_CHOICES
 from django.contrib.gis.utils import LayerMapping
 from django.apps import apps
 
+import csv
+import codecs
+import sys
+
 env = environ.Env(DEBUG=(bool, False))
 environ.Env.read_env(env_file=".env")
 
@@ -317,6 +321,26 @@ class ShapeLayers(models.Model):
         return self.name
 
 
+def remove_bom_inplace(path):
+    """Removes BOM mark, if it exists, from a file and rewrites it in-place"""
+    buffer_size = 4096
+    bom_length = len(codecs.BOM_UTF8)
+ 
+    with open(path, "r+b") as fp:
+        chunk = fp.read(buffer_size)
+        if chunk.startswith(codecs.BOM_UTF8):
+            i = 0
+            chunk = chunk[bom_length:]
+            while chunk:
+                fp.seek(i)
+                fp.write(chunk)
+                i += len(chunk)
+                fp.seek(bom_length, os.SEEK_CUR)
+                chunk = fp.read(buffer_size)
+            fp.seek(-bom_length, os.SEEK_CUR)
+            fp.truncate()
+
+
 
 
 @receiver(pre_save, sender=ShapeLayers)
@@ -355,28 +379,52 @@ def publish_date(sender, instance, created, **kwargs):
         print('##################',e)
 
     try:
-        csv = glob.glob(r'{}/**/*.csv'.format(file_path), recursive=True)[0]
+        csv_f = glob.glob(r'{}/**/*.csv'.format(file_path), recursive=True)[0]
+
+        remove_bom_inplace(csv_f)
         
         mdl = apps.get_model(app_label="rog", model_name=LAYER_CHOICES[instance.layerof -1][1])
         print(mdl)
         print(f"#### instance.layerof - {instance.layerof}")
-        with open(csv, encoding='"utf-8-sig"') as txt_file:
+        with open(csv_f, 'r') as txt_file:
-        #with open(csv, 'r') as txt_file:
+            reader = csv.reader(txt_file, delimiter=',')
-            lns = txt_file.readlines()
+            for fields in reader:
-            for ln in lns:
+                #fields = ln.split(",")
-                fields = ln.split(",")
                 if instance.layerof == 2:
                     updateLineTable(mdl, fields)
                 if instance.layerof == 3:
                     updatePolygonTable(mdl, fields)
+        # with open(csv_f, 'r') as txt_file:
+        #     reader = csv.reader(codecs.EncodedFile(txt_file, 'utf-8', 'utf-8-sig'), delimiter=",")
+        #     for ln in reader:
+        #         pass
+                # print(str(ln))
+                # fields = ln.split(",")
+                # if instance.layerof == 2:
+                #     updateLineTable(mdl, fields)
+                # if instance.layerof == 3:
+                #     updatePolygonTable(mdl, fields)
+        #with open(csv, encoding='"utf-8-sig"') as txt_file:
+        #with open(csv, 'r') as txt_file:
+            # lns = txt_file.readlines()
+            # print(lns)
+            # for ln in lns:
+            #     fields = ln.split(",")
+            #     if instance.layerof == 2:
+            #         updateLineTable(mdl, fields)
+            #     if instance.layerof == 3:
+            #         updatePolygonTable(mdl, fields)    
     
     except Exception as e:
         print('##################',e)
 
         
 def updateLineTable(mdl, fields):
-    #print(f"Updated {fields[0]} - {fields[1]}")
+    print("Done")
+    print(f"Updated {fields[0]} - {fields[1]}")
     print(len(fields))
+    print(fields[0])
+    print(mdl.objects.filter(location_id = int(fields[0])))
     mdl.objects.filter(location_id = int(fields[0])).update(
     location_name= fields[1] if len(fields) > 1 else '',
     category=fields[2] if len(fields) > 2 else '',
@@ -404,24 +452,24 @@ def updateLineTable(mdl, fields):
 def updatePolygonTable(mdl, fields):
     #print(f"Updated {fields[0]} - {fields[1]}")
     mdl.objects.filter(location_id = fields[0]).update(
-    location_name=fields[1],
+    location_name= fields[1] if len(fields) > 1 else '',
-    category=fields[2],
+    category=fields[2] if len(fields) > 2 else '',
-    zip=fields[3],
+    zip=fields[3] if len(fields) > 3 else '',
-    address=fields[4],
+    address=fields[4] if len(fields) > 4 else '',
-    prefecture=fields[5],
+    prefecture=fields[5] if len(fields) > 5 else '',
-    area=fields[6],
+    area=fields[6] if len(fields) > 6 else '',
-    city=fields[7],
+    city=fields[7] if len(fields) > 7 else '',
-    photos=fields[8],
+    photos=fields[8] if len(fields) > 8 else '',
-    videos=fields[9],
+    videos=fields[9] if len(fields) > 9 else '',
-    webcontents=fields[10],
+    webcontents=fields[10] if len(fields) > 10 else '',
-    status=fields[11],
+    status=fields[11] if len(fields) > 11 else '',
-    portal=fields[12],
+    portal=fields[12] if len(fields) > 12 else '',
-    group=fields[13],
+    group=fields[13] if len(fields) > 13 else '',
-    phone=fields[14],
+    phone=fields[14] if len(fields) > 14 else '',
-    fax=fields[15],
+    fax=fields[15] if len(fields) > 15 else '',
-    email=fields[16],
+    email=fields[16] if len(fields) > 16 else '',
-    facility=fields[17],
+    facility=fields[17] if len(fields) > 17 else '',
-    remark=fields[18],
+    remark=fields[18] if len(fields) > 18 else '',
-    tags=fields[19],
+    tags=fields[19] if len(fields) > 19 else '',
-    parammeters=fields[20]
+    parammeters=fields[20] if len(fields) > 20 else ''
     )