diff --git a/MIGRATE_ENHANCED_README.md b/MIGRATE_ENHANCED_README.md new file mode 100644 index 0000000..26d58b1 --- /dev/null +++ b/MIGRATE_ENHANCED_README.md @@ -0,0 +1,293 @@ +# Old RogDB → RogDB 移行手順書 + +## 概要 + +old_rogdb から rogdb へのデータ移行を行います。テーブル構造の違いにより、一部テーブルは専用スクリプトで処理します。 + +## 移行対象テーブル + +### 通常移行（migrate_old_rogdb_to_rogdb.py） +- rog_customuser +- rog_newcategory +- rog_newevent2 +- rog_member +- rog_useractions +- その他 rog_* テーブル + +### 専用移行スクリプト + +#### 1. rog_team (migrate_rog_team_enhanced.py) +**理由**: 新DBで追加フィールドあり +- `class_name` (character varying(100)) +- `event_id` (bigint) - rog_newevent2への外部キー +- `location` (geometry(Point,4326)) - PostGIS座標 +- `password` (character varying(100)) +- `trial` (boolean) +- `zekken_number` (character varying(50)) +- `created_at` (timestamp with time zone) +- `updated_at` (timestamp with time zone) + +#### 2. rog_entry (migrate_rog_entry_enhanced.py) +**理由**: camelCaseカラム名の予約語問題 +- `hasGoaled` (boolean) +- `hasParticipated` (boolean) + +#### 3. rog_goalimages (migrate_rog_goalimages_enhanced.py) +**理由**: team_name → zekken_number 変換ロジック +- 旧DBで`zekken_number`がブランク/NULLの場合 +- `team_name`を使用してrog_entryから対応する`zekken_number`を検索・取得 +- team_name → zekken_numberマッピングキャッシュを事前構築 + +## 移行手順 + +### 事前チェック + +```bash +# NULL値チェック +make null-check + +# カラム名チェック +make column-check + +# Docker コンテナ状況確認 +docker compose ps +``` + +### 段階的移行 + +#### ステップ1: 基本テーブル移行 +```bash +# 通常テーブル移行（rog_team, rog_entry除く） +make migrate-old-rogdb +``` + +#### ステップ2: rog_team構造変換移行 +```bash +# rog_team専用移行 +make migrate-rog-team +``` + +#### ステップ3: rog_entry camelCase対応移行 +```bash +# rog_entry専用移行 +make migrate-rog-entry +``` + +#### ステップ4: rog_goalimages team_name変換移行 +```bash +# rog_goalimages専用移行（team_name→zekken_number変換） +make migrate-rog-goalimages +``` + +### 一括移行 + +```bash +# 全テーブル一括移行 +make migrate-full +``` + +## 外部キー依存関係 + +移行順序に注意が必要な依存関係： + +1. **rog_customuser** → 他テーブルのowner_id, user_id参照 +2. **rog_newcategory** → rog_team, rog_entryのcategory_id参照 +3. **rog_newevent2** → rog_team, rog_entryのevent_id参照 +4. **rog_team** → rog_entryのteam_id参照 +5. **rog_entry** → rog_entrymemberのentry_id参照、rog_goalimadesのzekken_number解決 +6. **rog_goalimages** → rog_customuserのuser_id参照、team_name→zekken_number変換 + +## トラブルシューティング + +### エラー対応 + +#### NULL値制約違反 +```bash +# NULL値の詳細チェック +docker compose exec app python check_null_values.py + +# 個別テーブルのNULL値確認 +docker compose exec postgres-db psql -U admin -d old_rogdb -c " +SELECT column_name, COUNT(*) +FROM rog_team t, information_schema.columns c +WHERE c.table_name = 'rog_team' AND t.column_name IS NULL +GROUP BY column_name; +" +``` + +#### 外部キー制約違反 +```bash +# 参照整合性チェック +docker compose exec postgres-db psql -U admin -d old_rogdb -c " +SELECT t.team_id, COUNT(*) +FROM rog_entry t +LEFT JOIN rog_team tt ON t.team_id = tt.id +WHERE tt.id IS NULL +GROUP BY t.team_id; +" +``` + +#### team_name → zekken_number変換失敗 +```bash +# rog_goalimagesのteam_name一覧確認 +docker compose exec postgres-db psql -U admin -d old_rogdb -c " +SELECT DISTINCT team_name, zekken_number +FROM rog_goalimages +WHERE zekken_number IS NULL OR zekken_number = '' +ORDER BY team_name; +" + +# 新DBでのteam_name → zekken_numberマッピング確認 +docker compose exec postgres-db psql -U admin -d rogdb -c " +SELECT t.team_name, e.zekken_number +FROM rog_team t +JOIN rog_entry e ON t.id = e.team_id +ORDER BY t.team_name; +" +``` + +#### PostgreSQL予約語エラー +- camelCaseカラムや予約語は自動でダブルクォートで囲まれます +- エラーが発生した場合は該当スクリプトで quote_column_if_needed() を確認 + +### ログ確認 + +```bash +# 移行ログのリアルタイム確認 +docker compose logs -f app + +# 特定期間のログ確認 +docker compose logs --since="2025-08-25T08:00:00" app +``` + +## 設定値 + +### 環境変数 + +```bash +# データベース接続設定 +OLD_ROGDB_HOST=postgres-db +OLD_ROGDB_NAME=old_rogdb +OLD_ROGDB_USER=admin +OLD_ROGDB_PASSWORD=admin123456 + +ROGDB_HOST=postgres-db +ROGDB_NAME=rogdb +ROGDB_USER=admin +ROGDB_PASSWORD=admin123456 + +# 除外テーブル設定（カンマ区切り） +EXCLUDE_TABLES=rog_session,django_migrations +``` + +### デフォルト値設定 + +#### rog_team +- `trial`: False +- `event_id`: 最初のイベントID +- `location`: NULL +- `password`: '' +- `class_name`: '' +- `zekken_number`: '' + +#### rog_entry +- `hasGoaled`: False +- `hasParticipated`: False +- `is_active`: True +- `is_trial`: False +- `zekken_label`: '' + +## 移行後確認 + +### データ件数確認 + +```bash +# テーブル別レコード数比較 +docker compose exec postgres-db psql -U admin -d old_rogdb -c " +SELECT 'rog_team' as table_name, COUNT(*) as old_count FROM rog_team +UNION ALL +SELECT 'rog_entry', COUNT(*) FROM rog_entry +UNION ALL +SELECT 'rog_goalimages', COUNT(*) FROM rog_goalimages; +" + +docker compose exec postgres-db psql -U admin -d rogdb -c " +SELECT 'rog_team' as table_name, COUNT(*) as new_count FROM rog_team +UNION ALL +SELECT 'rog_entry', COUNT(*) FROM rog_entry +UNION ALL +SELECT 'rog_goalimages', COUNT(*) FROM rog_goalimages; +" +``` + +### 制約確認 + +```bash +# 外部キー制約確認 +docker compose exec postgres-db psql -U admin -d rogdb -c " +SELECT conname, contype +FROM pg_constraint +WHERE conrelid IN ( + SELECT oid FROM pg_class WHERE relname IN ('rog_team', 'rog_entry', 'rog_goalimages') +); +" +``` + +### team_name → zekken_number 変換確認 + +```bash +# rog_goalimadesでzekken_number変換結果確認 +docker compose exec postgres-db psql -U admin -d rogdb -c " +SELECT team_name, zekken_number, COUNT(*) as count +FROM rog_goalimages +GROUP BY team_name, zekken_number +ORDER BY team_name; +" + +# 変換できなかったレコード確認 +docker compose exec postgres-db psql -U admin -d rogdb -c " +SELECT team_name, COUNT(*) as blank_zekken_count +FROM rog_goalimages +WHERE zekken_number IS NULL OR zekken_number = '' +GROUP BY team_name +ORDER BY blank_zekken_count DESC; +" +``` + +## バックアップ・ロールバック + +### 移行前バックアップ + +```bash +# rogdbのバックアップ +docker compose exec postgres-db pg_dump -U admin rogdb > rogdb_backup_$(date +%Y%m%d_%H%M%S).sql +``` + +### ロールバック + +```bash +# 移行テーブルのクリア +docker compose exec postgres-db psql -U admin -d rogdb -c " +TRUNCATE rog_team, rog_entry, rog_goalimages CASCADE; +" + +# バックアップからの復元 +docker compose exec -T postgres-db psql -U admin -d rogdb < rogdb_backup_YYYYMMDD_HHMMSS.sql +``` + +## よくある問題 + +1. **メモリ不足**: docker-compose.ymlでPostgreSQLのメモリ制限を確認 +2. **コンテナ再起動**: 移行中にコンテナが再起動する場合はresources設定を調整 +3. **文字化け**: PostgreSQLの文字エンコーディング設定確認 +4. **タイムアウト**: 大量データの場合はバッチサイズを調整 + +## 参考ファイル + +- `docker-compose.yml`: データベース設定 +- `migrate_old_rogdb_to_rogdb.py`: 通常テーブル移行 +- `migrate_rog_team_enhanced.py`: rog_team専用移行 +- `migrate_rog_entry_enhanced.py`: rog_entry専用移行 +- `migrate_rog_goalimages_enhanced.py`: rog_goalimages専用移行（team_name→zekken変換） +- `check_null_values.py`: NULL値事前チェック +- `Makefile`: 移行タスク定義 diff --git a/Makefile b/Makefile index cd18721..d8fa85f 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,30 @@ migration-data-protection: migrate-old-rogdb: docker compose exec app python migrate_old_rogdb_to_rogdb.py +# rog_team 専用移行 (構造変換) +migrate-rog-team: + docker compose exec app python migrate_rog_team_enhanced.py + +# rog_entry 専用移行 (camelCase対応) +migrate-rog-entry: + docker compose exec app python migrate_rog_entry_enhanced.py + +# rog_goalimages 専用移行 (team_name→zekken_number変換) +migrate-rog-goalimages: + docker compose exec app python migrate_rog_goalimages_enhanced.py + +# 完全移行（通常テーブル + 特殊テーブル） +migrate-full: + @echo "=== 1. 通常テーブル移行 (特殊テーブル除く) ===" + $(MAKE) migrate-old-rogdb + @echo "=== 2. rog_team構造変換移行 ===" + $(MAKE) migrate-rog-team + @echo "=== 3. rog_entry camelCase対応移行 ===" + $(MAKE) migrate-rog-entry + @echo "=== 4. rog_goalimages team_name→zekken変換移行 ===" + $(MAKE) migrate-rog-goalimages + @echo "=== 移行完了 ===" + # カラム名チェック check-columns: docker compose exec app python check_column_names.py diff --git a/migrate_old_rogdb_to_rogdb.py b/migrate_old_rogdb_to_rogdb.py index 944d034..a77592e 100644 --- a/migrate_old_rogdb_to_rogdb.py +++ b/migrate_old_rogdb_to_rogdb.py @@ -404,8 +404,14 @@ class RogTableMigrator: if exclude_tables is None: exclude_tables = [] + # 特殊処理が必要なテーブルは専用スクリプトで処理 + exclude_tables.extend(['rog_team', 'rog_entry', 'rog_goalimages']) + logger.info("=" * 80) logger.info("Old RogDB → RogDB データ移行開始") + logger.info("⚠️ rog_team は migrate_rog_team_enhanced.py で別途処理してください") + logger.info("⚠️ rog_entry は migrate_rog_entry_enhanced.py で別途処理してください") + logger.info("⚠️ rog_goalimages は migrate_rog_goalimages_enhanced.py で別途処理してください") logger.info("=" * 80) try: diff --git a/migrate_rog_entry_enhanced.py b/migrate_rog_entry_enhanced.py new file mode 100644 index 0000000..5eba45d --- /dev/null +++ b/migrate_rog_entry_enhanced.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +""" +rog_entry テーブル専用移行スクリプト (予約語・NULL値対応) +old_rogdb の rog_entry から rogdb の rog_entry へデータ移行 +""" + +import os +import sys +import psycopg2 +from datetime import datetime, timezone +import logging + +# ログ設定 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# データベース設定 +OLD_ROGDB_CONFIG = { + 'host': os.getenv('OLD_ROGDB_HOST', 'postgres-db'), + 'database': os.getenv('OLD_ROGDB_NAME', 'old_rogdb'), + 'user': os.getenv('OLD_ROGDB_USER', 'admin'), + 'password': os.getenv('OLD_ROGDB_PASSWORD', 'admin123456'), + 'port': int(os.getenv('OLD_ROGDB_PORT', 5432)) +} + +ROGDB_CONFIG = { + 'host': os.getenv('ROGDB_HOST', 'postgres-db'), + 'database': os.getenv('ROGDB_NAME', 'rogdb'), + 'user': os.getenv('ROGDB_USER', 'admin'), + 'password': os.getenv('ROGDB_PASSWORD', 'admin123456'), + 'port': int(os.getenv('ROGDB_PORT', 5432)) +} + +class RogEntryMigrator: + """rog_entry テーブル専用移行クラス""" + + def __init__(self): + self.old_conn = None + self.new_conn = None + self.old_cursor = None + self.new_cursor = None + + def connect_databases(self): + """データベース接続""" + try: + logger.info("データベースに接続中...") + self.old_conn = psycopg2.connect(**OLD_ROGDB_CONFIG) + self.new_conn = psycopg2.connect(**ROGDB_CONFIG) + + self.old_conn.autocommit = True + self.new_conn.autocommit = False + + self.old_cursor = self.old_conn.cursor() + self.new_cursor = self.new_conn.cursor() + + logger.info("✅ データベース接続成功") + return True + + except Exception as e: + logger.error(f"❌ データベース接続エラー: {e}") + return False + + def close_connections(self): + """データベース接続クローズ""" + try: + if self.old_cursor: + self.old_cursor.close() + if self.new_cursor: + self.new_cursor.close() + if self.old_conn: + self.old_conn.close() + if self.new_conn: + self.new_conn.close() + logger.info("データベース接続をクローズしました") + except Exception as e: + logger.warning(f"接続クローズ時の警告: {e}") + + def quote_column_if_needed(self, column_name): + """予約語やキャメルケースの場合はダブルクォートで囲む""" + # camelCaseの場合はクォート + if any(c.isupper() for c in column_name): + return f'"{column_name}"' + return column_name + + def handle_null_values(self, column_name, value): + """NULL値の処理とデフォルト値設定""" + if value is not None: + return value + + # rog_entryテーブル固有のデフォルト値 + defaults = { + 'is_active': True, + 'is_trial': False, + 'hasGoaled': False, + 'hasParticipated': False, + 'zekken_label': '', + 'zekken_number': 0 + } + + if column_name in defaults: + default_value = defaults[column_name] + logger.debug(f"NULL値をデフォルト値に変換: {column_name} = {default_value}") + return default_value + + # デフォルト値が見つからない場合はNULLを返す + logger.warning(f"デフォルト値が設定されていません: {column_name}") + return None + + def validate_foreign_keys(self): + """外部キー参照の整合性をチェック""" + logger.info("外部キー参照の整合性をチェック中...") + + # team_id の存在確認 + self.old_cursor.execute("SELECT DISTINCT team_id FROM rog_entry WHERE team_id IS NOT NULL") + old_team_ids = [row[0] for row in self.old_cursor.fetchall()] + + self.new_cursor.execute("SELECT id FROM rog_team") + new_team_ids = [row[0] for row in self.new_cursor.fetchall()] + + missing_teams = set(old_team_ids) - set(new_team_ids) + if missing_teams: + logger.warning(f"⚠️ 新DBに存在しないteam_id: {missing_teams}") + logger.warning("先にrog_teamの移行を完了してください") + return False + + # event_id の存在確認 + self.old_cursor.execute("SELECT DISTINCT event_id FROM rog_entry WHERE event_id IS NOT NULL") + old_event_ids = [row[0] for row in self.old_cursor.fetchall()] + + self.new_cursor.execute("SELECT id FROM rog_newevent2") + new_event_ids = [row[0] for row in self.new_cursor.fetchall()] + + missing_events = set(old_event_ids) - set(new_event_ids) + if missing_events: + logger.warning(f"⚠️ 新DBに存在しないevent_id: {missing_events}") + logger.warning("先にrog_newevent2の移行を完了してください") + return False + + # category_id の存在確認 + self.old_cursor.execute("SELECT DISTINCT category_id FROM rog_entry WHERE category_id IS NOT NULL") + old_category_ids = [row[0] for row in self.old_cursor.fetchall()] + + self.new_cursor.execute("SELECT id FROM rog_newcategory") + new_category_ids = [row[0] for row in self.new_cursor.fetchall()] + + missing_categories = set(old_category_ids) - set(new_category_ids) + if missing_categories: + logger.warning(f"⚠️ 新DBに存在しないcategory_id: {missing_categories}") + logger.warning("先にrog_newcategoryの移行を完了してください") + return False + + logger.info("✅ 外部キー参照の整合性チェック完了") + return True + + def migrate_rog_entry(self): + """rog_entry テーブルのデータ移行""" + logger.info("=" * 60) + logger.info("rog_entry テーブルデータ移行開始") + logger.info("=" * 60) + + try: + # 外部キー整合性チェック + if not self.validate_foreign_keys(): + logger.error("❌ 外部キー整合性チェックに失敗しました") + return False + + # 旧データ取得（camelCaseカラム名をクォート） + logger.info("旧rog_entryデータを取得中...") + self.old_cursor.execute(""" + SELECT id, date, category_id, event_id, owner_id, team_id, + is_active, zekken_number, "hasGoaled", "hasParticipated", + zekken_label, is_trial + FROM rog_entry + ORDER BY id + """) + old_records = self.old_cursor.fetchall() + + if not old_records: + logger.info("✅ 移行対象データがありません") + return True + + logger.info(f"移行対象レコード数: {len(old_records)}件") + + # 統計情報 + inserted_count = 0 + updated_count = 0 + error_count = 0 + + # レコード別処理 + for i, old_record in enumerate(old_records): + try: + # レコードデータの展開とNULL値処理 + entry_id, date, category_id, event_id, owner_id, team_id, \ + is_active, zekken_number, hasGoaled, hasParticipated, \ + zekken_label, is_trial = old_record + + # NULL値処理 + processed_record = { + 'id': entry_id, + 'date': date, + 'category_id': category_id, + 'event_id': event_id, + 'owner_id': owner_id, + 'team_id': team_id, + 'is_active': self.handle_null_values('is_active', is_active), + 'zekken_number': self.handle_null_values('zekken_number', zekken_number), + 'hasGoaled': self.handle_null_values('hasGoaled', hasGoaled), + 'hasParticipated': self.handle_null_values('hasParticipated', hasParticipated), + 'zekken_label': self.handle_null_values('zekken_label', zekken_label), + 'is_trial': self.handle_null_values('is_trial', is_trial) + } + + # 既存レコード確認 + self.new_cursor.execute( + "SELECT COUNT(*) FROM rog_entry WHERE id = %s", + (entry_id,) + ) + exists = self.new_cursor.fetchone()[0] > 0 + + if exists: + # UPDATE処理（camelCaseカラムをクォート） + update_query = """ + UPDATE rog_entry SET + date = %s, + category_id = %s, + event_id = %s, + owner_id = %s, + team_id = %s, + is_active = %s, + zekken_number = %s, + "hasGoaled" = %s, + "hasParticipated" = %s, + zekken_label = %s, + is_trial = %s + WHERE id = %s + """ + + self.new_cursor.execute(update_query, ( + processed_record['date'], + processed_record['category_id'], + processed_record['event_id'], + processed_record['owner_id'], + processed_record['team_id'], + processed_record['is_active'], + processed_record['zekken_number'], + processed_record['hasGoaled'], + processed_record['hasParticipated'], + processed_record['zekken_label'], + processed_record['is_trial'], + entry_id + )) + updated_count += 1 + + else: + # INSERT処理（camelCaseカラムをクォート） + insert_query = """ + INSERT INTO rog_entry ( + id, date, category_id, event_id, owner_id, team_id, + is_active, zekken_number, "hasGoaled", "hasParticipated", + zekken_label, is_trial + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + ) + """ + + self.new_cursor.execute(insert_query, ( + processed_record['id'], + processed_record['date'], + processed_record['category_id'], + processed_record['event_id'], + processed_record['owner_id'], + processed_record['team_id'], + processed_record['is_active'], + processed_record['zekken_number'], + processed_record['hasGoaled'], + processed_record['hasParticipated'], + processed_record['zekken_label'], + processed_record['is_trial'] + )) + inserted_count += 1 + + # 進捗表示とコミット + if (i + 1) % 100 == 0: + self.new_conn.commit() + logger.info(f" 進捗: {i + 1}/{len(old_records)} 件処理完了") + + except Exception as e: + error_count += 1 + logger.error(f" レコード処理エラー (ID: {entry_id}): {e}") + + # トランザクションロールバック + try: + self.new_conn.rollback() + except: + pass + + if error_count > 10: + logger.error("❌ エラー数が上限を超えました") + break + + # 最終コミット + self.new_conn.commit() + + # 結果サマリー + logger.info("=" * 60) + logger.info("rog_entry 移行完了") + logger.info("=" * 60) + logger.info(f"挿入: {inserted_count}件") + logger.info(f"更新: {updated_count}件") + logger.info(f"エラー: {error_count}件") + logger.info(f"総処理: {len(old_records)}件") + + if error_count == 0: + logger.info("✅ rog_entry移行が正常に完了しました！") + return True + else: + logger.warning(f"⚠️ {error_count}件のエラーがありました") + return False + + except Exception as e: + logger.error(f"❌ rog_entry移行エラー: {e}") + try: + self.new_conn.rollback() + except: + pass + return False + + def run(self): + """移行実行""" + try: + if not self.connect_databases(): + return False + + return self.migrate_rog_entry() + + finally: + self.close_connections() + +def main(): + """メイン処理""" + logger.info("rog_entry テーブル移行スクリプト") + + migrator = RogEntryMigrator() + success = migrator.run() + + if success: + logger.info("🎉 移行が正常に完了しました！") + else: + logger.error("💥 移行中にエラーが発生しました") + + sys.exit(0 if success else 1) + +if __name__ == "__main__": + main() diff --git a/migrate_rog_goalimages_enhanced.py b/migrate_rog_goalimages_enhanced.py new file mode 100644 index 0000000..57d7021 --- /dev/null +++ b/migrate_rog_goalimages_enhanced.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +""" +rog_goalimages テーブル専用移行スクリプト (team_name → zekken_number変換対応) +old_rogdb の rog_goalimages から rogdb の rog_goalimages へ +zekken_numberがブランクの場合、team_nameからrog_entryを検索してzekken_numberを取得 +""" + +import os +import sys +import psycopg2 +from datetime import datetime, timezone +import logging + +# ログ設定 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# データベース設定 +OLD_ROGDB_CONFIG = { + 'host': os.getenv('OLD_ROGDB_HOST', 'postgres-db'), + 'database': os.getenv('OLD_ROGDB_NAME', 'old_rogdb'), + 'user': os.getenv('OLD_ROGDB_USER', 'admin'), + 'password': os.getenv('OLD_ROGDB_PASSWORD', 'admin123456'), + 'port': int(os.getenv('OLD_ROGDB_PORT', 5432)) +} + +ROGDB_CONFIG = { + 'host': os.getenv('ROGDB_HOST', 'postgres-db'), + 'database': os.getenv('ROGDB_NAME', 'rogdb'), + 'user': os.getenv('ROGDB_USER', 'admin'), + 'password': os.getenv('ROGDB_PASSWORD', 'admin123456'), + 'port': int(os.getenv('ROGDB_PORT', 5432)) +} + +class RogGoalImagesMigrator: + """rog_goalimages テーブル専用移行クラス""" + + def __init__(self): + self.old_conn = None + self.new_conn = None + self.old_cursor = None + self.new_cursor = None + self.team_zekken_cache = {} # team_name → zekken_number キャッシュ + + def connect_databases(self): + """データベース接続""" + try: + logger.info("データベースに接続中...") + self.old_conn = psycopg2.connect(**OLD_ROGDB_CONFIG) + self.new_conn = psycopg2.connect(**ROGDB_CONFIG) + + self.old_conn.autocommit = True + self.new_conn.autocommit = False + + self.old_cursor = self.old_conn.cursor() + self.new_cursor = self.new_conn.cursor() + + logger.info("✅ データベース接続成功") + return True + + except Exception as e: + logger.error(f"❌ データベース接続エラー: {e}") + return False + + def close_connections(self): + """データベース接続クローズ""" + try: + if self.old_cursor: + self.old_cursor.close() + if self.new_cursor: + self.new_cursor.close() + if self.old_conn: + self.old_conn.close() + if self.new_conn: + self.new_conn.close() + logger.info("データベース接続をクローズしました") + except Exception as e: + logger.warning(f"接続クローズ時の警告: {e}") + + def build_team_zekken_cache(self): + """team_name → zekken_number のキャッシュを構築""" + logger.info("team_name → zekken_number キャッシュを構築中...") + + try: + # 新DBのrog_entryから team_name → zekken_number マッピングを取得 + self.new_cursor.execute(""" + SELECT DISTINCT t.team_name, e.zekken_number + FROM rog_entry e + JOIN rog_team t ON e.team_id = t.id + WHERE t.team_name IS NOT NULL + AND e.zekken_number IS NOT NULL + ORDER BY t.team_name, e.zekken_number + """) + + team_zekken_pairs = self.new_cursor.fetchall() + + for team_name, zekken_number in team_zekken_pairs: + if team_name not in self.team_zekken_cache: + self.team_zekken_cache[team_name] = zekken_number + logger.debug(f"キャッシュ追加: {team_name} → {zekken_number}") + + logger.info(f"✅ キャッシュ構築完了: {len(self.team_zekken_cache)}件のteam_name → zekken_numberマッピング") + + # キャッシュ内容の一部をログ出力 + if self.team_zekken_cache: + sample_items = list(self.team_zekken_cache.items())[:5] + logger.info(f"キャッシュサンプル: {sample_items}") + + return True + + except Exception as e: + logger.error(f"❌ キャッシュ構築エラー: {e}") + return False + + def resolve_zekken_number(self, old_zekken_number, team_name): + """zekken_numberを解決（ブランクの場合はteam_nameから取得）""" + # zekken_numberが既に設定されている場合はそのまま使用 + if old_zekken_number and old_zekken_number.strip(): + return old_zekken_number.strip() + + # team_nameからzekken_numberを検索 + if team_name and team_name.strip(): + clean_team_name = team_name.strip() + + if clean_team_name in self.team_zekken_cache: + resolved_zekken = self.team_zekken_cache[clean_team_name] + logger.debug(f"team_name '{clean_team_name}' → zekken_number '{resolved_zekken}'") + return str(resolved_zekken) + else: + logger.warning(f"team_name '{clean_team_name}' に対応するzekken_numberが見つかりません") + + # 解決できない場合は空文字列を返す + logger.warning(f"zekken_number解決失敗: zekken='{old_zekken_number}', team='{team_name}'") + return '' + + def get_old_goalimages_structure(self): + """旧DBのrog_goalimagesテーブル構造を取得""" + try: + self.old_cursor.execute(""" + SELECT column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_name = 'rog_goalimages' + AND table_schema = 'public' + ORDER BY ordinal_position + """) + + columns = self.old_cursor.fetchall() + column_names = [col[0] for col in columns] + + logger.info(f"旧DBのrog_goalimagesカラム: {column_names}") + return column_names + + except Exception as e: + logger.error(f"❌ 旧DBテーブル構造取得エラー: {e}") + return [] + + def migrate_rog_goalimages(self): + """rog_goalimages テーブルのデータ移行""" + logger.info("=" * 60) + logger.info("rog_goalimages テーブルデータ移行開始") + logger.info("=" * 60) + + try: + # team_name → zekken_number キャッシュ構築 + if not self.build_team_zekken_cache(): + logger.error("❌ キャッシュ構築に失敗しました") + return False + + # 旧DBテーブル構造確認 + old_columns = self.get_old_goalimages_structure() + if not old_columns: + logger.error("❌ 旧DBのテーブル構造を取得できませんでした") + return False + + # 旧データ取得 + logger.info("旧rog_goalimagesデータを取得中...") + + # カラム存在チェック + has_zekken_number = 'zekken_number' in old_columns + + if has_zekken_number: + select_query = """ + SELECT id, goalimage, goaltime, team_name, event_code, + cp_number, user_id, zekken_number + FROM rog_goalimages + ORDER BY id + """ + else: + select_query = """ + SELECT id, goalimage, goaltime, team_name, event_code, + cp_number, user_id, NULL as zekken_number + FROM rog_goalimages + ORDER BY id + """ + + self.old_cursor.execute(select_query) + old_records = self.old_cursor.fetchall() + + if not old_records: + logger.info("✅ 移行対象データがありません") + return True + + logger.info(f"移行対象レコード数: {len(old_records)}件") + + # 統計情報 + inserted_count = 0 + updated_count = 0 + error_count = 0 + zekken_resolved_count = 0 + + # レコード別処理 + for i, old_record in enumerate(old_records): + try: + # レコードデータの展開 + record_id, goalimage, goaltime, team_name, event_code, \ + cp_number, user_id, old_zekken_number = old_record + + # zekken_number解決 + resolved_zekken_number = self.resolve_zekken_number(old_zekken_number, team_name) + + if not old_zekken_number and resolved_zekken_number: + zekken_resolved_count += 1 + + # 新レコードデータ + new_record = { + 'id': record_id, + 'goalimage': goalimage, + 'goaltime': goaltime, + 'team_name': team_name or '', + 'event_code': event_code or '', + 'cp_number': cp_number or 0, + 'user_id': user_id, + 'zekken_number': resolved_zekken_number + } + + # 既存レコード確認 + self.new_cursor.execute( + "SELECT COUNT(*) FROM rog_goalimages WHERE id = %s", + (record_id,) + ) + exists = self.new_cursor.fetchone()[0] > 0 + + if exists: + # UPDATE処理 + update_query = """ + UPDATE rog_goalimages SET + goalimage = %s, + goaltime = %s, + team_name = %s, + event_code = %s, + cp_number = %s, + user_id = %s, + zekken_number = %s + WHERE id = %s + """ + + self.new_cursor.execute(update_query, ( + new_record['goalimage'], + new_record['goaltime'], + new_record['team_name'], + new_record['event_code'], + new_record['cp_number'], + new_record['user_id'], + new_record['zekken_number'], + record_id + )) + updated_count += 1 + + else: + # INSERT処理 + insert_query = """ + INSERT INTO rog_goalimages ( + id, goalimage, goaltime, team_name, event_code, + cp_number, user_id, zekken_number + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s + ) + """ + + self.new_cursor.execute(insert_query, ( + new_record['id'], + new_record['goalimage'], + new_record['goaltime'], + new_record['team_name'], + new_record['event_code'], + new_record['cp_number'], + new_record['user_id'], + new_record['zekken_number'] + )) + inserted_count += 1 + + # 進捗表示とコミット + if (i + 1) % 100 == 0: + self.new_conn.commit() + logger.info(f" 進捗: {i + 1}/{len(old_records)} 件処理完了") + + except Exception as e: + error_count += 1 + logger.error(f" レコード処理エラー (ID: {record_id}): {e}") + + # トランザクションロールバック + try: + self.new_conn.rollback() + except: + pass + + if error_count > 10: + logger.error("❌ エラー数が上限を超えました") + break + + # 最終コミット + self.new_conn.commit() + + # 結果サマリー + logger.info("=" * 60) + logger.info("rog_goalimages 移行完了") + logger.info("=" * 60) + logger.info(f"挿入: {inserted_count}件") + logger.info(f"更新: {updated_count}件") + logger.info(f"エラー: {error_count}件") + logger.info(f"zekken_number解決: {zekken_resolved_count}件") + logger.info(f"総処理: {len(old_records)}件") + + if error_count == 0: + logger.info("✅ rog_goalimages移行が正常に完了しました！") + return True + else: + logger.warning(f"⚠️ {error_count}件のエラーがありました") + return False + + except Exception as e: + logger.error(f"❌ rog_goalimages移行エラー: {e}") + try: + self.new_conn.rollback() + except: + pass + return False + + def run(self): + """移行実行""" + try: + if not self.connect_databases(): + return False + + return self.migrate_rog_goalimages() + + finally: + self.close_connections() + +def main(): + """メイン処理""" + logger.info("rog_goalimages テーブル移行スクリプト (team_name → zekken_number変換対応)") + + migrator = RogGoalImagesMigrator() + success = migrator.run() + + if success: + logger.info("🎉 移行が正常に完了しました！") + else: + logger.error("💥 移行中にエラーが発生しました") + + sys.exit(0 if success else 1) + +if __name__ == "__main__": + main() diff --git a/migrate_rog_team_enhanced.py b/migrate_rog_team_enhanced.py new file mode 100644 index 0000000..c91795b --- /dev/null +++ b/migrate_rog_team_enhanced.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +""" +rog_team テーブル専用移行スクリプト (構造変換対応) +old_rogdb の rog_team から rogdb の rog_team へ構造変換を行いながらデータ移行 +""" + +import os +import sys +import psycopg2 +from datetime import datetime, timezone +import logging + +# ログ設定 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# データベース設定 +OLD_ROGDB_CONFIG = { + 'host': os.getenv('OLD_ROGDB_HOST', 'postgres-db'), + 'database': os.getenv('OLD_ROGDB_NAME', 'old_rogdb'), + 'user': os.getenv('OLD_ROGDB_USER', 'admin'), + 'password': os.getenv('OLD_ROGDB_PASSWORD', 'admin123456'), + 'port': int(os.getenv('OLD_ROGDB_PORT', 5432)) +} + +ROGDB_CONFIG = { + 'host': os.getenv('ROGDB_HOST', 'postgres-db'), + 'database': os.getenv('ROGDB_NAME', 'rogdb'), + 'user': os.getenv('ROGDB_USER', 'admin'), + 'password': os.getenv('ROGDB_PASSWORD', 'admin123456'), + 'port': int(os.getenv('ROGDB_PORT', 5432)) +} + +class RogTeamMigrator: + """rog_team テーブル専用移行クラス""" + + def __init__(self): + self.old_conn = None + self.new_conn = None + self.old_cursor = None + self.new_cursor = None + self.default_event_id = None + + def connect_databases(self): + """データベース接続""" + try: + logger.info("データベースに接続中...") + self.old_conn = psycopg2.connect(**OLD_ROGDB_CONFIG) + self.new_conn = psycopg2.connect(**ROGDB_CONFIG) + + self.old_conn.autocommit = True + self.new_conn.autocommit = False + + self.old_cursor = self.old_conn.cursor() + self.new_cursor = self.new_conn.cursor() + + logger.info("✅ データベース接続成功") + return True + + except Exception as e: + logger.error(f"❌ データベース接続エラー: {e}") + return False + + def close_connections(self): + """データベース接続クローズ""" + try: + if self.old_cursor: + self.old_cursor.close() + if self.new_cursor: + self.new_cursor.close() + if self.old_conn: + self.old_conn.close() + if self.new_conn: + self.new_conn.close() + logger.info("データベース接続をクローズしました") + except Exception as e: + logger.warning(f"接続クローズ時の警告: {e}") + + def get_default_event_id(self): + """デフォルトのevent_idを取得または作成""" + try: + # 既存のイベントを探す + self.new_cursor.execute(""" + SELECT id FROM rog_newevent2 + WHERE event_name LIKE '%移行%' OR event_name LIKE '%default%' + ORDER BY id LIMIT 1 + """) + result = self.new_cursor.fetchone() + + if result: + event_id = result[0] + logger.info(f"既存のデフォルトイベントを使用: event_id = {event_id}") + return event_id + + # なければ最初のイベントを使用 + self.new_cursor.execute(""" + SELECT id FROM rog_newevent2 + ORDER BY id LIMIT 1 + """) + result = self.new_cursor.fetchone() + + if result: + event_id = result[0] + logger.info(f"最初のイベントをデフォルトとして使用: event_id = {event_id}") + return event_id + + # イベントがない場合はエラー + logger.error("❌ rog_newevent2 テーブルにイベントが存在しません") + return None + + except Exception as e: + logger.error(f"❌ デフォルトevent_id取得エラー: {e}") + return None + + def get_category_mapping(self): + """カテゴリIDのマッピングを確認""" + try: + # 旧DBのカテゴリ + self.old_cursor.execute("SELECT id, category_name FROM rog_newcategory ORDER BY id") + old_categories = dict(self.old_cursor.fetchall()) + + # 新DBのカテゴリ + self.new_cursor.execute("SELECT id, category_name FROM rog_newcategory ORDER BY id") + new_categories = dict(self.new_cursor.fetchall()) + + logger.info(f"旧DB カテゴリ: {old_categories}") + logger.info(f"新DB カテゴリ: {new_categories}") + + # 名前ベースでマッピング作成 + category_mapping = {} + for old_id, old_name in old_categories.items(): + for new_id, new_name in new_categories.items(): + if old_name == new_name: + category_mapping[old_id] = new_id + break + else: + # マッチしない場合は最初のカテゴリを使用 + if new_categories: + category_mapping[old_id] = min(new_categories.keys()) + logger.warning(f"カテゴリマッピング失敗 - デフォルト使用: {old_id} -> {category_mapping[old_id]}") + + return category_mapping + + except Exception as e: + logger.error(f"❌ カテゴリマッピング取得エラー: {e}") + return {} + + def convert_team_record(self, old_record, category_mapping): + """旧レコードを新レコード形式に変換""" + old_id, old_team_name, old_category_id, old_owner_id = old_record + + # 新しいレコード作成 + new_record = { + 'id': old_id, + 'team_name': old_team_name, + 'category_id': category_mapping.get(old_category_id, old_category_id), + 'owner_id': old_owner_id, + # 新しいフィールドにデフォルト値を設定 + 'class_name': '', # 空文字列 + 'event_id': self.default_event_id, # デフォルトイベント + 'location': None, # PostGIS座標は後で設定可能 + 'password': '', # パスワードなし + 'trial': False, # 本番チーム + 'zekken_number': '', # ゼッケン番号なし + 'created_at': datetime.now(timezone.utc), + 'updated_at': datetime.now(timezone.utc) + } + + return new_record + + def migrate_rog_team(self): + """rog_team テーブルのデータ移行""" + logger.info("=" * 60) + logger.info("rog_team テーブル構造変換移行開始") + logger.info("=" * 60) + + try: + # デフォルトevent_id取得 + self.default_event_id = self.get_default_event_id() + if not self.default_event_id: + return False + + # カテゴリマッピング取得 + category_mapping = self.get_category_mapping() + + # 旧データ取得 + logger.info("旧rog_teamデータを取得中...") + self.old_cursor.execute(""" + SELECT id, team_name, category_id, owner_id + FROM rog_team + ORDER BY id + """) + old_records = self.old_cursor.fetchall() + + if not old_records: + logger.info("✅ 移行対象データがありません") + return True + + logger.info(f"移行対象レコード数: {len(old_records)}件") + + # 統計情報 + inserted_count = 0 + updated_count = 0 + error_count = 0 + + # レコード別処理 + for i, old_record in enumerate(old_records): + try: + # レコード変換 + new_record = self.convert_team_record(old_record, category_mapping) + team_id = new_record['id'] + + # 既存レコード確認 + self.new_cursor.execute( + "SELECT COUNT(*) FROM rog_team WHERE id = %s", + (team_id,) + ) + exists = self.new_cursor.fetchone()[0] > 0 + + if exists: + # UPDATE処理 + update_query = """ + UPDATE rog_team SET + team_name = %s, + category_id = %s, + owner_id = %s, + class_name = %s, + event_id = %s, + location = %s, + password = %s, + trial = %s, + zekken_number = %s, + updated_at = %s + WHERE id = %s + """ + + self.new_cursor.execute(update_query, ( + new_record['team_name'], + new_record['category_id'], + new_record['owner_id'], + new_record['class_name'], + new_record['event_id'], + new_record['location'], + new_record['password'], + new_record['trial'], + new_record['zekken_number'], + new_record['updated_at'], + team_id + )) + updated_count += 1 + + else: + # INSERT処理 + insert_query = """ + INSERT INTO rog_team ( + id, team_name, category_id, owner_id, + class_name, event_id, location, password, + trial, zekken_number, created_at, updated_at + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + ) + """ + + self.new_cursor.execute(insert_query, ( + new_record['id'], + new_record['team_name'], + new_record['category_id'], + new_record['owner_id'], + new_record['class_name'], + new_record['event_id'], + new_record['location'], + new_record['password'], + new_record['trial'], + new_record['zekken_number'], + new_record['created_at'], + new_record['updated_at'] + )) + inserted_count += 1 + + # 進捗表示 + if (i + 1) % 50 == 0: + self.new_conn.commit() + logger.info(f" 進捗: {i + 1}/{len(old_records)} 件処理完了") + + except Exception as e: + error_count += 1 + logger.error(f" レコード処理エラー (ID: {old_record[0]}): {e}") + + # トランザクションロールバック + try: + self.new_conn.rollback() + except: + pass + + if error_count > 10: + logger.error("❌ エラー数が上限を超えました") + break + + # 最終コミット + self.new_conn.commit() + + # 結果サマリー + logger.info("=" * 60) + logger.info("rog_team 移行完了") + logger.info("=" * 60) + logger.info(f"挿入: {inserted_count}件") + logger.info(f"更新: {updated_count}件") + logger.info(f"エラー: {error_count}件") + logger.info(f"総処理: {len(old_records)}件") + + if error_count == 0: + logger.info("✅ rog_team移行が正常に完了しました！") + return True + else: + logger.warning(f"⚠️ {error_count}件のエラーがありました") + return False + + except Exception as e: + logger.error(f"❌ rog_team移行エラー: {e}") + try: + self.new_conn.rollback() + except: + pass + return False + + def run(self): + """移行実行""" + try: + if not self.connect_databases(): + return False + + return self.migrate_rog_team() + + finally: + self.close_connections() + +def main(): + """メイン処理""" + logger.info("rog_team テーブル構造変換移行スクリプト") + + migrator = RogTeamMigrator() + success = migrator.run() + + if success: + logger.info("🎉 移行が正常に完了しました！") + else: + logger.error("💥 移行中にエラーが発生しました") + + sys.exit(0 if success else 1) + +if __name__ == "__main__": + main()