398 lines
16 KiB
Python
398 lines
16 KiB
Python
#!/usr/bin/env python
|
||
"""
|
||
LocationからLocation2025への完全データ移行スクリプト（統計検証付き）
|
||
|
||
機能：
|
||
- 全フィールド対応の完全データ移行
|
||
- リアルタイム統計検証
|
||
- データ品質チェック
|
||
- 移行前後の比較
|
||
- 詳細レポート生成
|
||
"""
|
||
|
||
import os
|
||
import django
|
||
from collections import defaultdict, Counter
|
||
|
||
# Django設定
|
||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
|
||
django.setup()
|
||
|
||
from rog.models import Location, Location2025, NewEvent2
|
||
from django.contrib.auth import get_user_model
|
||
from django.contrib.gis.geos import Point
|
||
|
||
def analyze_source_data():
|
||
"""移行前のデータ分析"""
|
||
print('=== 移行前データ分析 ===')
|
||
|
||
total_locations = Location.objects.count()
|
||
print(f'総Location件数: {total_locations}件')
|
||
|
||
# グループ別統計
|
||
with_group = Location.objects.exclude(group__isnull=True).exclude(group='').count()
|
||
without_group = total_locations - with_group
|
||
print(f'groupありLocation: {with_group}件')
|
||
print(f'groupなしLocation: {without_group}件')
|
||
|
||
# 座標データ統計
|
||
with_geom = Location.objects.exclude(geom__isnull=True).count()
|
||
with_lat_lng = Location.objects.exclude(longitude__isnull=True).exclude(latitude__isnull=True).count()
|
||
print(f'geom座標あり: {with_geom}件')
|
||
print(f'lat/lng座標あり: {with_lat_lng}件')
|
||
|
||
# フィールド統計
|
||
fields_stats = {}
|
||
text_fields = ['photos', 'videos', 'remark', 'tags', 'evaluation_value', 'sub_loc_id', 'subcategory']
|
||
numeric_fields = ['checkin_point', 'buy_point']
|
||
boolean_fields = ['hidden_location']
|
||
|
||
for field in text_fields:
|
||
if hasattr(Location, field):
|
||
count = Location.objects.exclude(**{f'{field}__isnull': True}).exclude(**{field: ''}).count()
|
||
fields_stats[field] = count
|
||
print(f'{field}データあり: {count}件')
|
||
|
||
for field in numeric_fields:
|
||
if hasattr(Location, field):
|
||
count = Location.objects.exclude(**{f'{field}__isnull': True}).exclude(**{field: 0}).count()
|
||
fields_stats[field] = count
|
||
print(f'{field}データあり: {count}件')
|
||
|
||
for field in boolean_fields:
|
||
if hasattr(Location, field):
|
||
count = Location.objects.filter(**{field: True}).count()
|
||
fields_stats[field] = count
|
||
print(f'{field}データあり: {count}件')
|
||
|
||
return {
|
||
'total': total_locations,
|
||
'with_group': with_group,
|
||
'without_group': without_group,
|
||
'with_geom': with_geom,
|
||
'with_lat_lng': with_lat_lng,
|
||
'fields': fields_stats
|
||
}
|
||
|
||
def validate_migration_data(source_stats):
|
||
"""移行後データ検証"""
|
||
print('\n=== 移行後データ検証 ===')
|
||
|
||
total_migrated = Location2025.objects.count()
|
||
print(f'移行完了件数: {total_migrated}件')
|
||
|
||
# フィールド検証
|
||
migrated_stats = {}
|
||
field_mapping = {
|
||
'photos': 'photos',
|
||
'videos': 'videos',
|
||
'remark': 'remark',
|
||
'tags': 'tags',
|
||
'evaluation_value': 'evaluation_value',
|
||
'hidden_location': 'hidden_location',
|
||
'sub_loc_id': 'sub_loc_id',
|
||
'subcategory': 'subcategory'
|
||
}
|
||
|
||
for source_field, target_field in field_mapping.items():
|
||
if source_field == 'hidden_location':
|
||
count = Location2025.objects.filter(**{target_field: True}).count()
|
||
else:
|
||
count = Location2025.objects.exclude(**{f'{target_field}__isnull': True}).exclude(**{target_field: ''}).count()
|
||
migrated_stats[source_field] = count
|
||
print(f'{target_field}データあり: {count}件')
|
||
|
||
# 座標検証
|
||
with_location = Location2025.objects.exclude(location__isnull=True).count()
|
||
with_lat_lng = Location2025.objects.exclude(longitude__isnull=True).exclude(latitude__isnull=True).count()
|
||
print(f'location座標あり: {with_location}件')
|
||
print(f'lat/lng座標あり: {with_lat_lng}件')
|
||
|
||
# 必須フィールド検証
|
||
with_event = Location2025.objects.exclude(event__isnull=True).count()
|
||
with_cp_name = Location2025.objects.exclude(cp_name__isnull=True).exclude(cp_name='').count()
|
||
print(f'eventリンクあり: {with_event}件')
|
||
print(f'cp_nameあり: {with_cp_name}件')
|
||
|
||
return {
|
||
'total': total_migrated,
|
||
'fields': migrated_stats,
|
||
'with_location': with_location,
|
||
'with_lat_lng': with_lat_lng,
|
||
'with_event': with_event,
|
||
'with_cp_name': with_cp_name
|
||
}
|
||
|
||
def generate_comparison_report(source_stats, migrated_stats):
|
||
"""移行前後比較レポート"""
|
||
print('\n=== 移行前後比較レポート ===')
|
||
|
||
print(f'総件数比較:')
|
||
print(f' 移行前: {source_stats["total"]:,}件')
|
||
print(f' 移行後: {migrated_stats["total"]:,}件')
|
||
print(f' 移行率: {(migrated_stats["total"] / source_stats["total"] * 100):.1f}%')
|
||
|
||
print(f'\nフィールド別データ保持率:')
|
||
for field in source_stats['fields']:
|
||
if field in migrated_stats['fields']:
|
||
source_count = source_stats['fields'][field]
|
||
migrated_count = migrated_stats['fields'][field]
|
||
if source_count > 0:
|
||
retention_rate = (migrated_count / source_count * 100)
|
||
print(f' {field}: {migrated_count:,}/{source_count:,}件 ({retention_rate:.1f}%)')
|
||
else:
|
||
print(f' {field}: {migrated_count:,}/0件 (N/A)')
|
||
|
||
def analyze_event_distribution():
|
||
"""イベント別分布分析"""
|
||
print('\n=== イベント別分布分析 ===')
|
||
|
||
event_stats = {}
|
||
for location in Location2025.objects.select_related('event'):
|
||
event_name = location.event.event_name if location.event else 'No Event'
|
||
event_code = location.event.event_code if location.event else 'No Code'
|
||
key = f"{event_code} ({event_name})"
|
||
event_stats[key] = event_stats.get(key, 0) + 1
|
||
|
||
# 件数順でソート
|
||
sorted_events = sorted(event_stats.items(), key=lambda x: x[1], reverse=True)
|
||
|
||
print(f'総イベント数: {len(sorted_events)}件')
|
||
print(f'上位イベント:')
|
||
for i, (event_key, count) in enumerate(sorted_events[:10], 1):
|
||
print(f' {i:2d}. {event_key}: {count:,}件')
|
||
|
||
return event_stats
|
||
|
||
def sample_data_verification():
|
||
"""サンプルデータ検証"""
|
||
print('\n=== サンプルデータ検証 ===')
|
||
|
||
# 各種データパターンのサンプルを取得
|
||
samples = []
|
||
|
||
# 写真データありのサンプル
|
||
photo_sample = Location2025.objects.filter(photos__isnull=False).exclude(photos='').first()
|
||
if photo_sample:
|
||
samples.append(('写真データあり', photo_sample))
|
||
|
||
# remarkデータありのサンプル
|
||
remark_sample = Location2025.objects.filter(remark__isnull=False).exclude(remark='').first()
|
||
if remark_sample:
|
||
samples.append(('詳細説明あり', remark_sample))
|
||
|
||
# 高ポイントのサンプル
|
||
high_point_sample = Location2025.objects.filter(cp_point__gt=50).first()
|
||
if high_point_sample:
|
||
samples.append(('高ポイント', high_point_sample))
|
||
|
||
# 通常サンプル
|
||
if not samples:
|
||
normal_sample = Location2025.objects.first()
|
||
if normal_sample:
|
||
samples.append(('通常データ', normal_sample))
|
||
|
||
for sample_type, sample in samples[:3]:
|
||
print(f'\n【{sample_type}サンプル】')
|
||
print(f' CP番号: {sample.cp_number}')
|
||
print(f' CP名: {sample.cp_name}')
|
||
print(f' CPポイント: {sample.cp_point}')
|
||
print(f' フォトポイント: {sample.photo_point}')
|
||
print(f' sub_loc_id: {sample.sub_loc_id}')
|
||
print(f' subcategory: {sample.subcategory}')
|
||
|
||
# データ長を制限して表示
|
||
def truncate_text(text, max_len=30):
|
||
if not text:
|
||
return '(空)'
|
||
return text[:max_len] + '...' if len(text) > max_len else text
|
||
|
||
print(f' 写真: {truncate_text(sample.photos)}')
|
||
print(f' 動画: {truncate_text(sample.videos)}')
|
||
print(f' 詳細: {truncate_text(sample.remark)}')
|
||
print(f' タグ: {truncate_text(sample.tags)}')
|
||
print(f' 評価値: {truncate_text(sample.evaluation_value)}')
|
||
print(f' 隠し: {sample.hidden_location}')
|
||
print(f' イベント: {sample.event.event_name if sample.event else "None"}')
|
||
|
||
def main():
|
||
"""メイン実行関数"""
|
||
User = get_user_model()
|
||
default_user = User.objects.first()
|
||
|
||
print('='*60)
|
||
print('Location → Location2025 完全移行スクリプト（統計検証付き）')
|
||
print('='*60)
|
||
|
||
# 1. 移行前データ分析
|
||
source_stats = analyze_source_data()
|
||
|
||
# 2. 既存Location2025データ削除
|
||
print('\n=== 既存データクリア ===')
|
||
deleted_count = Location2025.objects.count()
|
||
Location2025.objects.all().delete()
|
||
print(f'削除済み: {deleted_count}件')
|
||
|
||
# 3. NewEvent2のevent_codeマップ作成
|
||
print('\n=== Event Code マッピング ===')
|
||
events = NewEvent2.objects.filter(event_code__isnull=False).exclude(event_code='')
|
||
event_code_map = {}
|
||
for event in events:
|
||
event_code_map[event.event_code] = event
|
||
print(f'有効なevent_code数: {len(event_code_map)}件')
|
||
|
||
# 4. データ移行実行
|
||
print('\n=== データ移行実行 ===')
|
||
locations = Location.objects.all()
|
||
processed_combinations = set()
|
||
migrated_count = 0
|
||
skipped_count = 0
|
||
error_count = 0
|
||
event_migration_stats = defaultdict(int)
|
||
|
||
for location in locations:
|
||
try:
|
||
# groupが空の場合はスキップ
|
||
if not location.group:
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# location.groupに含まれるevent_codeを検索
|
||
matched_event = None
|
||
matched_event_code = None
|
||
|
||
for event_code, event in event_code_map.items():
|
||
if event_code in location.group:
|
||
matched_event = event
|
||
matched_event_code = event_code
|
||
break
|
||
|
||
# マッチするevent_codeがない場合はスキップ
|
||
if not matched_event:
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# cp_number + event_idの組み合わせを確認
|
||
combination_key = (location.cp, matched_event.id)
|
||
if combination_key in processed_combinations:
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# この組み合わせを処理済みとしてマーク
|
||
processed_combinations.add(combination_key)
|
||
|
||
# MultiPointからPointに変換
|
||
point_location = None
|
||
if location.geom and len(location.geom) > 0:
|
||
first_point = location.geom[0]
|
||
point_location = Point(first_point.x, first_point.y)
|
||
elif location.longitude and location.latitude:
|
||
point_location = Point(location.longitude, location.latitude)
|
||
|
||
# Location2025レコードを作成
|
||
location2025, created = Location2025.objects.update_or_create(
|
||
cp_number=location.cp,
|
||
event=matched_event,
|
||
defaults={
|
||
'cp_name': location.location_name or '',
|
||
'sub_loc_id': location.sub_loc_id or '',
|
||
'subcategory': location.subcategory or '',
|
||
'latitude': location.latitude or 0.0,
|
||
'longitude': location.longitude or 0.0,
|
||
'location': point_location,
|
||
'cp_point': int(location.checkin_point) if location.checkin_point else 0,
|
||
'photo_point': int(location.checkin_point) if location.checkin_point else 0,
|
||
'buy_point': int(location.buy_point) if location.buy_point else 0,
|
||
'checkin_radius': location.checkin_radius or 100.0,
|
||
'auto_checkin': location.auto_checkin or False,
|
||
'shop_closed': location.shop_closed or False,
|
||
'shop_shutdown': location.shop_shutdown or False,
|
||
'opening_hours': '',
|
||
'address': location.address or '',
|
||
'phone': location.phone or '',
|
||
'website': '',
|
||
'description': location.remark or '',
|
||
# 追加フィールド
|
||
'photos': location.photos or '',
|
||
'videos': location.videos or '',
|
||
'remark': location.remark or '',
|
||
'tags': location.tags or '',
|
||
'evaluation_value': location.evaluation_value or '',
|
||
'hidden_location': location.hidden_location or False,
|
||
# 管理情報
|
||
'is_active': True,
|
||
'sort_order': 0,
|
||
'csv_source_file': 'migration_from_location',
|
||
'created_by': default_user,
|
||
'updated_by': default_user,
|
||
}
|
||
)
|
||
|
||
if created:
|
||
migrated_count += 1
|
||
event_migration_stats[matched_event_code] += 1
|
||
|
||
if migrated_count % 100 == 0:
|
||
print(f'進捗: {migrated_count:,}件完了')
|
||
|
||
except Exception as e:
|
||
print(f'❌ エラー: CP {location.cp} - {str(e)}')
|
||
error_count += 1
|
||
|
||
# 5. 移行結果サマリー
|
||
print(f'\n=== 移行結果サマリー ===')
|
||
print(f'移行完了: {migrated_count:,}件')
|
||
print(f'スキップ: {skipped_count:,}件')
|
||
print(f'エラー: {error_count:,}件')
|
||
print(f'総処理: {migrated_count + skipped_count + error_count:,}件')
|
||
|
||
# 6. 移行後データ検証
|
||
migrated_stats = validate_migration_data(source_stats)
|
||
|
||
# 7. 比較レポート生成
|
||
generate_comparison_report(source_stats, migrated_stats)
|
||
|
||
# 8. イベント別分布分析
|
||
event_distribution = analyze_event_distribution()
|
||
|
||
# 9. サンプルデータ検証
|
||
sample_data_verification()
|
||
|
||
# 10. 最終検証サマリー
|
||
print('\n' + '='*60)
|
||
print('🎯 移行完了検証サマリー')
|
||
print('='*60)
|
||
|
||
success_rate = (migrated_count / source_stats['total'] * 100) if source_stats['total'] > 0 else 0
|
||
print(f'✅ 総移行成功率: {success_rate:.1f}% ({migrated_count:,}/{source_stats["total"]:,}件)')
|
||
print(f'✅ エラー率: {(error_count / source_stats["total"] * 100):.1f}% ({error_count:,}件)')
|
||
print(f'✅ 最終Location2025件数: {Location2025.objects.count():,}件')
|
||
print(f'✅ 対応イベント数: {len(event_distribution)}件')
|
||
|
||
# データ品質スコア算出
|
||
quality_score = 0
|
||
if migrated_stats['with_event'] == migrated_stats['total']:
|
||
quality_score += 25 # 全てにイベントがリンクされている
|
||
if migrated_stats['with_cp_name'] >= migrated_stats['total'] * 0.95:
|
||
quality_score += 25 # 95%以上にCP名がある
|
||
if migrated_stats['fields']['photos'] >= migrated_stats['total'] * 0.8:
|
||
quality_score += 25 # 80%以上に写真データがある
|
||
if migrated_stats['fields']['remark'] >= migrated_stats['total'] * 0.8:
|
||
quality_score += 25 # 80%以上に詳細説明がある
|
||
|
||
print(f'✅ データ品質スコア: {quality_score}/100点')
|
||
|
||
if quality_score >= 90:
|
||
print('🏆 優秀：本格運用準備完了')
|
||
elif quality_score >= 70:
|
||
print('🥉 良好：運用可能レベル')
|
||
elif quality_score >= 50:
|
||
print('⚠️ 要改善：一部データ補完推奨')
|
||
else:
|
||
print('❌ 要対応：データ品質に課題あり')
|
||
|
||
print('\n✅ 全フィールド対応の完全データ移行が正常に完了しました')
|
||
|
||
if __name__ == "__main__":
|
||
main()
|