#!/usr/bin/env python """ LocationからLocation2025への完全データ移行スクリプト（統計検証付き） 機能： - 全フィールド対応の完全データ移行 - リアルタイム統計検証 - データ品質チェック - 移行前後の比較 - 詳細レポート生成 """ import os import django from collections import defaultdict, Counter # Django設定 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') django.setup() from rog.models import Location, Location2025, NewEvent2 from django.contrib.auth import get_user_model from django.contrib.gis.geos import Point def analyze_source_data(): """移行前のデータ分析""" print('=== 移行前データ分析 ===') total_locations = Location.objects.count() print(f'総Location件数: {total_locations}件') # グループ別統計 with_group = Location.objects.exclude(group__isnull=True).exclude(group='').count() without_group = total_locations - with_group print(f'groupありLocation: {with_group}件') print(f'groupなしLocation: {without_group}件') # 座標データ統計 with_geom = Location.objects.exclude(geom__isnull=True).count() with_lat_lng = Location.objects.exclude(longitude__isnull=True).exclude(latitude__isnull=True).count() print(f'geom座標あり: {with_geom}件') print(f'lat/lng座標あり: {with_lat_lng}件') # フィールド統計 fields_stats = {} text_fields = ['photos', 'videos', 'remark', 'tags', 'evaluation_value', 'sub_loc_id', 'subcategory'] numeric_fields = ['checkin_point', 'buy_point'] boolean_fields = ['hidden_location'] for field in text_fields: if hasattr(Location, field): count = Location.objects.exclude(**{f'{field}__isnull': True}).exclude(**{field: ''}).count() fields_stats[field] = count print(f'{field}データあり: {count}件') for field in numeric_fields: if hasattr(Location, field): count = Location.objects.exclude(**{f'{field}__isnull': True}).exclude(**{field: 0}).count() fields_stats[field] = count print(f'{field}データあり: {count}件') for field in boolean_fields: if hasattr(Location, field): count = Location.objects.filter(**{field: True}).count() fields_stats[field] = count print(f'{field}データあり: {count}件') return { 'total': total_locations, 'with_group': with_group, 'without_group': without_group, 'with_geom': with_geom, 'with_lat_lng': with_lat_lng, 'fields': fields_stats } def validate_migration_data(source_stats): """移行後データ検証""" print('\n=== 移行後データ検証 ===') total_migrated = Location2025.objects.count() print(f'移行完了件数: {total_migrated}件') # フィールド検証 migrated_stats = {} field_mapping = { 'photos': 'photos', 'videos': 'videos', 'remark': 'remark', 'tags': 'tags', 'evaluation_value': 'evaluation_value', 'hidden_location': 'hidden_location', 'sub_loc_id': 'sub_loc_id', 'subcategory': 'subcategory' } for source_field, target_field in field_mapping.items(): if source_field == 'hidden_location': count = Location2025.objects.filter(**{target_field: True}).count() else: count = Location2025.objects.exclude(**{f'{target_field}__isnull': True}).exclude(**{target_field: ''}).count() migrated_stats[source_field] = count print(f'{target_field}データあり: {count}件') # 座標検証 with_location = Location2025.objects.exclude(location__isnull=True).count() with_lat_lng = Location2025.objects.exclude(longitude__isnull=True).exclude(latitude__isnull=True).count() print(f'location座標あり: {with_location}件') print(f'lat/lng座標あり: {with_lat_lng}件') # 必須フィールド検証 with_event = Location2025.objects.exclude(event__isnull=True).count() with_cp_name = Location2025.objects.exclude(cp_name__isnull=True).exclude(cp_name='').count() print(f'eventリンクあり: {with_event}件') print(f'cp_nameあり: {with_cp_name}件') return { 'total': total_migrated, 'fields': migrated_stats, 'with_location': with_location, 'with_lat_lng': with_lat_lng, 'with_event': with_event, 'with_cp_name': with_cp_name } def generate_comparison_report(source_stats, migrated_stats): """移行前後比較レポート""" print('\n=== 移行前後比較レポート ===') print(f'総件数比較:') print(f' 移行前: {source_stats["total"]:,}件') print(f' 移行後: {migrated_stats["total"]:,}件') print(f' 移行率: {(migrated_stats["total"] / source_stats["total"] * 100):.1f}%') print(f'\nフィールド別データ保持率:') for field in source_stats['fields']: if field in migrated_stats['fields']: source_count = source_stats['fields'][field] migrated_count = migrated_stats['fields'][field] if source_count > 0: retention_rate = (migrated_count / source_count * 100) print(f' {field}: {migrated_count:,}/{source_count:,}件 ({retention_rate:.1f}%)') else: print(f' {field}: {migrated_count:,}/0件 (N/A)') def analyze_event_distribution(): """イベント別分布分析""" print('\n=== イベント別分布分析 ===') event_stats = {} for location in Location2025.objects.select_related('event'): event_name = location.event.event_name if location.event else 'No Event' event_code = location.event.event_code if location.event else 'No Code' key = f"{event_code} ({event_name})" event_stats[key] = event_stats.get(key, 0) + 1 # 件数順でソート sorted_events = sorted(event_stats.items(), key=lambda x: x[1], reverse=True) print(f'総イベント数: {len(sorted_events)}件') print(f'上位イベント:') for i, (event_key, count) in enumerate(sorted_events[:10], 1): print(f' {i:2d}. {event_key}: {count:,}件') return event_stats def sample_data_verification(): """サンプルデータ検証""" print('\n=== サンプルデータ検証 ===') # 各種データパターンのサンプルを取得 samples = [] # 写真データありのサンプル photo_sample = Location2025.objects.filter(photos__isnull=False).exclude(photos='').first() if photo_sample: samples.append(('写真データあり', photo_sample)) # remarkデータありのサンプル remark_sample = Location2025.objects.filter(remark__isnull=False).exclude(remark='').first() if remark_sample: samples.append(('詳細説明あり', remark_sample)) # 高ポイントのサンプル high_point_sample = Location2025.objects.filter(cp_point__gt=50).first() if high_point_sample: samples.append(('高ポイント', high_point_sample)) # 通常サンプル if not samples: normal_sample = Location2025.objects.first() if normal_sample: samples.append(('通常データ', normal_sample)) for sample_type, sample in samples[:3]: print(f'\n【{sample_type}サンプル】') print(f' CP番号: {sample.cp_number}') print(f' CP名: {sample.cp_name}') print(f' CPポイント: {sample.cp_point}') print(f' フォトポイント: {sample.photo_point}') print(f' sub_loc_id: {sample.sub_loc_id}') print(f' subcategory: {sample.subcategory}') # データ長を制限して表示 def truncate_text(text, max_len=30): if not text: return '(空)' return text[:max_len] + '...' if len(text) > max_len else text print(f' 写真: {truncate_text(sample.photos)}') print(f' 動画: {truncate_text(sample.videos)}') print(f' 詳細: {truncate_text(sample.remark)}') print(f' タグ: {truncate_text(sample.tags)}') print(f' 評価値: {truncate_text(sample.evaluation_value)}') print(f' 隠し: {sample.hidden_location}') print(f' イベント: {sample.event.event_name if sample.event else "None"}') def main(): """メイン実行関数""" User = get_user_model() default_user = User.objects.first() print('='*60) print('Location → Location2025 完全移行スクリプト（統計検証付き）') print('='*60) # 1. 移行前データ分析 source_stats = analyze_source_data() # 2. 既存Location2025データ削除 print('\n=== 既存データクリア ===') deleted_count = Location2025.objects.count() Location2025.objects.all().delete() print(f'削除済み: {deleted_count}件') # 3. NewEvent2のevent_codeマップ作成 print('\n=== Event Code マッピング ===') events = NewEvent2.objects.filter(event_code__isnull=False).exclude(event_code='') event_code_map = {} for event in events: event_code_map[event.event_code] = event print(f'有効なevent_code数: {len(event_code_map)}件') # 4. データ移行実行 print('\n=== データ移行実行 ===') locations = Location.objects.all() processed_combinations = set() migrated_count = 0 skipped_count = 0 error_count = 0 event_migration_stats = defaultdict(int) for location in locations: try: # groupが空の場合はスキップ if not location.group: skipped_count += 1 continue # location.groupに含まれるevent_codeを検索 matched_event = None matched_event_code = None for event_code, event in event_code_map.items(): if event_code in location.group: matched_event = event matched_event_code = event_code break # マッチするevent_codeがない場合はスキップ if not matched_event: skipped_count += 1 continue # cp_number + event_idの組み合わせを確認 combination_key = (location.cp, matched_event.id) if combination_key in processed_combinations: skipped_count += 1 continue # この組み合わせを処理済みとしてマーク processed_combinations.add(combination_key) # MultiPointからPointに変換 point_location = None if location.geom and len(location.geom) > 0: first_point = location.geom[0] point_location = Point(first_point.x, first_point.y) elif location.longitude and location.latitude: point_location = Point(location.longitude, location.latitude) # Location2025レコードを作成 location2025, created = Location2025.objects.update_or_create( cp_number=location.cp, event=matched_event, defaults={ 'cp_name': location.location_name or '', 'sub_loc_id': location.sub_loc_id or '', 'subcategory': location.subcategory or '', 'latitude': location.latitude or 0.0, 'longitude': location.longitude or 0.0, 'location': point_location, 'cp_point': int(location.checkin_point) if location.checkin_point else 0, 'photo_point': int(location.checkin_point) if location.checkin_point else 0, 'buy_point': int(location.buy_point) if location.buy_point else 0, 'checkin_radius': location.checkin_radius or 100.0, 'auto_checkin': location.auto_checkin or False, 'shop_closed': location.shop_closed or False, 'shop_shutdown': location.shop_shutdown or False, 'opening_hours': '', 'address': location.address or '', 'phone': location.phone or '', 'website': '', 'description': location.remark or '', # 追加フィールド 'photos': location.photos or '', 'videos': location.videos or '', 'remark': location.remark or '', 'tags': location.tags or '', 'evaluation_value': location.evaluation_value or '', 'hidden_location': location.hidden_location or False, # 管理情報 'is_active': True, 'sort_order': 0, 'csv_source_file': 'migration_from_location', 'created_by': default_user, 'updated_by': default_user, } ) if created: migrated_count += 1 event_migration_stats[matched_event_code] += 1 if migrated_count % 100 == 0: print(f'進捗: {migrated_count:,}件完了') except Exception as e: print(f'❌ エラー: CP {location.cp} - {str(e)}') error_count += 1 # 5. 移行結果サマリー print(f'\n=== 移行結果サマリー ===') print(f'移行完了: {migrated_count:,}件') print(f'スキップ: {skipped_count:,}件') print(f'エラー: {error_count:,}件') print(f'総処理: {migrated_count + skipped_count + error_count:,}件') # 6. 移行後データ検証 migrated_stats = validate_migration_data(source_stats) # 7. 比較レポート生成 generate_comparison_report(source_stats, migrated_stats) # 8. イベント別分布分析 event_distribution = analyze_event_distribution() # 9. サンプルデータ検証 sample_data_verification() # 10. 最終検証サマリー print('\n' + '='*60) print('🎯 移行完了検証サマリー') print('='*60) success_rate = (migrated_count / source_stats['total'] * 100) if source_stats['total'] > 0 else 0 print(f'✅ 総移行成功率: {success_rate:.1f}% ({migrated_count:,}/{source_stats["total"]:,}件)') print(f'✅ エラー率: {(error_count / source_stats["total"] * 100):.1f}% ({error_count:,}件)') print(f'✅ 最終Location2025件数: {Location2025.objects.count():,}件') print(f'✅ 対応イベント数: {len(event_distribution)}件') # データ品質スコア算出 quality_score = 0 if migrated_stats['with_event'] == migrated_stats['total']: quality_score += 25 # 全てにイベントがリンクされている if migrated_stats['with_cp_name'] >= migrated_stats['total'] * 0.95: quality_score += 25 # 95%以上にCP名がある if migrated_stats['fields']['photos'] >= migrated_stats['total'] * 0.8: quality_score += 25 # 80%以上に写真データがある if migrated_stats['fields']['remark'] >= migrated_stats['total'] * 0.8: quality_score += 25 # 80%以上に詳細説明がある print(f'✅ データ品質スコア: {quality_score}/100点') if quality_score >= 90: print('🏆 優秀：本格運用準備完了') elif quality_score >= 70: print('🥉 良好：運用可能レベル') elif quality_score >= 50: print('⚠️ 要改善：一部データ補完推奨') else: print('❌ 要対応：データ品質に課題あり') print('\n✅ 全フィールド対応の完全データ移行が正常に完了しました') if __name__ == "__main__": main()