add migrate_old_team/entry/goalimages_enhanced
This commit is contained in:
293
MIGRATE_ENHANCED_README.md
Normal file
293
MIGRATE_ENHANCED_README.md
Normal file
@ -0,0 +1,293 @@
|
||||
# Old RogDB → RogDB 移行手順書
|
||||
|
||||
## 概要
|
||||
|
||||
old_rogdb から rogdb へのデータ移行を行います。テーブル構造の違いにより、一部テーブルは専用スクリプトで処理します。
|
||||
|
||||
## 移行対象テーブル
|
||||
|
||||
### 通常移行（migrate_old_rogdb_to_rogdb.py）
|
||||
- rog_customuser
|
||||
- rog_newcategory
|
||||
- rog_newevent2
|
||||
- rog_member
|
||||
- rog_useractions
|
||||
- その他 rog_* テーブル
|
||||
|
||||
### 専用移行スクリプト
|
||||
|
||||
#### 1. rog_team (migrate_rog_team_enhanced.py)
|
||||
**理由**: 新DBで追加フィールドあり
|
||||
- `class_name` (character varying(100))
|
||||
- `event_id` (bigint) - rog_newevent2への外部キー
|
||||
- `location` (geometry(Point,4326)) - PostGIS座標
|
||||
- `password` (character varying(100))
|
||||
- `trial` (boolean)
|
||||
- `zekken_number` (character varying(50))
|
||||
- `created_at` (timestamp with time zone)
|
||||
- `updated_at` (timestamp with time zone)
|
||||
|
||||
#### 2. rog_entry (migrate_rog_entry_enhanced.py)
|
||||
**理由**: camelCaseカラム名の予約語問題
|
||||
- `hasGoaled` (boolean)
|
||||
- `hasParticipated` (boolean)
|
||||
|
||||
#### 3. rog_goalimages (migrate_rog_goalimages_enhanced.py)
|
||||
**理由**: team_name → zekken_number 変換ロジック
|
||||
- 旧DBで`zekken_number`がブランク/NULLの場合
|
||||
- `team_name`を使用してrog_entryから対応する`zekken_number`を検索・取得
|
||||
- team_name → zekken_numberマッピングキャッシュを事前構築
|
||||
|
||||
## 移行手順
|
||||
|
||||
### 事前チェック
|
||||
|
||||
```bash
|
||||
# NULL値チェック
|
||||
make null-check
|
||||
|
||||
# カラム名チェック
|
||||
make column-check
|
||||
|
||||
# Docker コンテナ状況確認
|
||||
docker compose ps
|
||||
```
|
||||
|
||||
### 段階的移行
|
||||
|
||||
#### ステップ1: 基本テーブル移行
|
||||
```bash
|
||||
# 通常テーブル移行（rog_team, rog_entry除く）
|
||||
make migrate-old-rogdb
|
||||
```
|
||||
|
||||
#### ステップ2: rog_team構造変換移行
|
||||
```bash
|
||||
# rog_team専用移行
|
||||
make migrate-rog-team
|
||||
```
|
||||
|
||||
#### ステップ3: rog_entry camelCase対応移行
|
||||
```bash
|
||||
# rog_entry専用移行
|
||||
make migrate-rog-entry
|
||||
```
|
||||
|
||||
#### ステップ4: rog_goalimages team_name変換移行
|
||||
```bash
|
||||
# rog_goalimages専用移行（team_name→zekken_number変換）
|
||||
make migrate-rog-goalimages
|
||||
```
|
||||
|
||||
### 一括移行
|
||||
|
||||
```bash
|
||||
# 全テーブル一括移行
|
||||
make migrate-full
|
||||
```
|
||||
|
||||
## 外部キー依存関係
|
||||
|
||||
移行順序に注意が必要な依存関係：
|
||||
|
||||
1. **rog_customuser** → 他テーブルのowner_id, user_id参照
|
||||
2. **rog_newcategory** → rog_team, rog_entryのcategory_id参照
|
||||
3. **rog_newevent2** → rog_team, rog_entryのevent_id参照
|
||||
4. **rog_team** → rog_entryのteam_id参照
|
||||
5. **rog_entry** → rog_entrymemberのentry_id参照、rog_goalimadesのzekken_number解決
|
||||
6. **rog_goalimages** → rog_customuserのuser_id参照、team_name→zekken_number変換
|
||||
|
||||
## トラブルシューティング
|
||||
|
||||
### エラー対応
|
||||
|
||||
#### NULL値制約違反
|
||||
```bash
|
||||
# NULL値の詳細チェック
|
||||
docker compose exec app python check_null_values.py
|
||||
|
||||
# 個別テーブルのNULL値確認
|
||||
docker compose exec postgres-db psql -U admin -d old_rogdb -c "
|
||||
SELECT column_name, COUNT(*)
|
||||
FROM rog_team t, information_schema.columns c
|
||||
WHERE c.table_name = 'rog_team' AND t.column_name IS NULL
|
||||
GROUP BY column_name;
|
||||
"
|
||||
```
|
||||
|
||||
#### 外部キー制約違反
|
||||
```bash
|
||||
# 参照整合性チェック
|
||||
docker compose exec postgres-db psql -U admin -d old_rogdb -c "
|
||||
SELECT t.team_id, COUNT(*)
|
||||
FROM rog_entry t
|
||||
LEFT JOIN rog_team tt ON t.team_id = tt.id
|
||||
WHERE tt.id IS NULL
|
||||
GROUP BY t.team_id;
|
||||
"
|
||||
```
|
||||
|
||||
#### team_name → zekken_number変換失敗
|
||||
```bash
|
||||
# rog_goalimagesのteam_name一覧確認
|
||||
docker compose exec postgres-db psql -U admin -d old_rogdb -c "
|
||||
SELECT DISTINCT team_name, zekken_number
|
||||
FROM rog_goalimages
|
||||
WHERE zekken_number IS NULL OR zekken_number = ''
|
||||
ORDER BY team_name;
|
||||
"
|
||||
|
||||
# 新DBでのteam_name → zekken_numberマッピング確認
|
||||
docker compose exec postgres-db psql -U admin -d rogdb -c "
|
||||
SELECT t.team_name, e.zekken_number
|
||||
FROM rog_team t
|
||||
JOIN rog_entry e ON t.id = e.team_id
|
||||
ORDER BY t.team_name;
|
||||
"
|
||||
```
|
||||
|
||||
#### PostgreSQL予約語エラー
|
||||
- camelCaseカラムや予約語は自動でダブルクォートで囲まれます
|
||||
- エラーが発生した場合は該当スクリプトで quote_column_if_needed() を確認
|
||||
|
||||
### ログ確認
|
||||
|
||||
```bash
|
||||
# 移行ログのリアルタイム確認
|
||||
docker compose logs -f app
|
||||
|
||||
# 特定期間のログ確認
|
||||
docker compose logs --since="2025-08-25T08:00:00" app
|
||||
```
|
||||
|
||||
## 設定値
|
||||
|
||||
### 環境変数
|
||||
|
||||
```bash
|
||||
# データベース接続設定
|
||||
OLD_ROGDB_HOST=postgres-db
|
||||
OLD_ROGDB_NAME=old_rogdb
|
||||
OLD_ROGDB_USER=admin
|
||||
OLD_ROGDB_PASSWORD=admin123456
|
||||
|
||||
ROGDB_HOST=postgres-db
|
||||
ROGDB_NAME=rogdb
|
||||
ROGDB_USER=admin
|
||||
ROGDB_PASSWORD=admin123456
|
||||
|
||||
# 除外テーブル設定（カンマ区切り）
|
||||
EXCLUDE_TABLES=rog_session,django_migrations
|
||||
```
|
||||
|
||||
### デフォルト値設定
|
||||
|
||||
#### rog_team
|
||||
- `trial`: False
|
||||
- `event_id`: 最初のイベントID
|
||||
- `location`: NULL
|
||||
- `password`: ''
|
||||
- `class_name`: ''
|
||||
- `zekken_number`: ''
|
||||
|
||||
#### rog_entry
|
||||
- `hasGoaled`: False
|
||||
- `hasParticipated`: False
|
||||
- `is_active`: True
|
||||
- `is_trial`: False
|
||||
- `zekken_label`: ''
|
||||
|
||||
## 移行後確認
|
||||
|
||||
### データ件数確認
|
||||
|
||||
```bash
|
||||
# テーブル別レコード数比較
|
||||
docker compose exec postgres-db psql -U admin -d old_rogdb -c "
|
||||
SELECT 'rog_team' as table_name, COUNT(*) as old_count FROM rog_team
|
||||
UNION ALL
|
||||
SELECT 'rog_entry', COUNT(*) FROM rog_entry
|
||||
UNION ALL
|
||||
SELECT 'rog_goalimages', COUNT(*) FROM rog_goalimages;
|
||||
"
|
||||
|
||||
docker compose exec postgres-db psql -U admin -d rogdb -c "
|
||||
SELECT 'rog_team' as table_name, COUNT(*) as new_count FROM rog_team
|
||||
UNION ALL
|
||||
SELECT 'rog_entry', COUNT(*) FROM rog_entry
|
||||
UNION ALL
|
||||
SELECT 'rog_goalimages', COUNT(*) FROM rog_goalimages;
|
||||
"
|
||||
```
|
||||
|
||||
### 制約確認
|
||||
|
||||
```bash
|
||||
# 外部キー制約確認
|
||||
docker compose exec postgres-db psql -U admin -d rogdb -c "
|
||||
SELECT conname, contype
|
||||
FROM pg_constraint
|
||||
WHERE conrelid IN (
|
||||
SELECT oid FROM pg_class WHERE relname IN ('rog_team', 'rog_entry', 'rog_goalimages')
|
||||
);
|
||||
"
|
||||
```
|
||||
|
||||
### team_name → zekken_number 変換確認
|
||||
|
||||
```bash
|
||||
# rog_goalimadesでzekken_number変換結果確認
|
||||
docker compose exec postgres-db psql -U admin -d rogdb -c "
|
||||
SELECT team_name, zekken_number, COUNT(*) as count
|
||||
FROM rog_goalimages
|
||||
GROUP BY team_name, zekken_number
|
||||
ORDER BY team_name;
|
||||
"
|
||||
|
||||
# 変換できなかったレコード確認
|
||||
docker compose exec postgres-db psql -U admin -d rogdb -c "
|
||||
SELECT team_name, COUNT(*) as blank_zekken_count
|
||||
FROM rog_goalimages
|
||||
WHERE zekken_number IS NULL OR zekken_number = ''
|
||||
GROUP BY team_name
|
||||
ORDER BY blank_zekken_count DESC;
|
||||
"
|
||||
```
|
||||
|
||||
## バックアップ・ロールバック
|
||||
|
||||
### 移行前バックアップ
|
||||
|
||||
```bash
|
||||
# rogdbのバックアップ
|
||||
docker compose exec postgres-db pg_dump -U admin rogdb > rogdb_backup_$(date +%Y%m%d_%H%M%S).sql
|
||||
```
|
||||
|
||||
### ロールバック
|
||||
|
||||
```bash
|
||||
# 移行テーブルのクリア
|
||||
docker compose exec postgres-db psql -U admin -d rogdb -c "
|
||||
TRUNCATE rog_team, rog_entry, rog_goalimages CASCADE;
|
||||
"
|
||||
|
||||
# バックアップからの復元
|
||||
docker compose exec -T postgres-db psql -U admin -d rogdb < rogdb_backup_YYYYMMDD_HHMMSS.sql
|
||||
```
|
||||
|
||||
## よくある問題
|
||||
|
||||
1. **メモリ不足**: docker-compose.ymlでPostgreSQLのメモリ制限を確認
|
||||
2. **コンテナ再起動**: 移行中にコンテナが再起動する場合はresources設定を調整
|
||||
3. **文字化け**: PostgreSQLの文字エンコーディング設定確認
|
||||
4. **タイムアウト**: 大量データの場合はバッチサイズを調整
|
||||
|
||||
## 参考ファイル
|
||||
|
||||
- `docker-compose.yml`: データベース設定
|
||||
- `migrate_old_rogdb_to_rogdb.py`: 通常テーブル移行
|
||||
- `migrate_rog_team_enhanced.py`: rog_team専用移行
|
||||
- `migrate_rog_entry_enhanced.py`: rog_entry専用移行
|
||||
- `migrate_rog_goalimages_enhanced.py`: rog_goalimages専用移行（team_name→zekken変換）
|
||||
- `check_null_values.py`: NULL値事前チェック
|
||||
- `Makefile`: 移行タスク定義
|
||||
Reference in New Issue
Block a user