From f81f8bf61a17a6635526f11c9a969b402a3acf90 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Fri, 8 Sep 2023 15:22:54 +0900 Subject: [PATCH 1/3] [DO NOT MERGE] account migration script --- server/cmd/migrate/main.go | 237 +++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 server/cmd/migrate/main.go diff --git a/server/cmd/migrate/main.go b/server/cmd/migrate/main.go new file mode 100644 index 0000000000..e83474faac --- /dev/null +++ b/server/cmd/migrate/main.go @@ -0,0 +1,237 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + + mongodoc2 "github.com/reearth/reearth-cms/server/internal/infrastructure/mongo/mongodoc" + "github.com/reearth/reearthx/account/accountinfrastructure/accountmongo/mongodoc" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +type subject string +type cmsUserID string +type accountUserID string +type cmsWorkspaceID string +type accountWorkspaceID string + +const ( + reearthAccount = "reearth-account" + reearthCMS = "reearth_cms" +) + +func run() error { + dry := flag.Bool("dry", false, "dry run") + flag.Parse() + + ctx := context.Background() + dbURI := os.Getenv("DB_URI") + client, err := mongo.Connect(ctx, options.Client().ApplyURI(dbURI)) + if err != nil { + return fmt.Errorf("connect: %w", err) + } + accountUsers, err := allUsers(ctx, client, reearthAccount) + if err != nil { + return fmt.Errorf("reearth-account all accountUsers: %w", err) + } + cmsUsers, err := allUsers(ctx, client, reearthCMS) + if err != nil { + return fmt.Errorf("reearth_cms all accountUsers: %w", err) + } + + // subject => account user_id + accountUserMap := map[subject]accountUserID{} + // account user_id = account personal ws_id + personalAccountWorkspaces := map[accountUserID]accountWorkspaceID{} + + for _, u := range accountUsers { + auID, awsID, subs := fromAccountUser(u) + for _, s := range subs { + accountUserMap[s] = auID + } + personalAccountWorkspaces[auID] = awsID + } + + // cms ws_id => account ws_id + migratedWorkspace := map[cmsWorkspaceID]accountWorkspaceID{} + // cms user_id => account user_id + migratedUser := map[cmsUserID]accountUserID{} + + var userWrites []mongo.WriteModel + for _, u := range cmsUsers { + cuID, cwsID, subs := fromCMSUser(u) + var migrated []subject + for _, s := range subs { + if _, ok := accountUserMap[s]; ok { + migrated = append(migrated, s) + } + } + if len(migrated) == len(subs) { + // すべての subject が reearth-account に存在する場合は reearth-account 側の個人ワークスペースを cms 側の個人ワークスペースとする + // cms 側の個人ワークスペースに関する参照を書き換える必要が生じる + auID := accountUserMap[subs[0]] + awsID := personalAccountWorkspaces[auID] + migratedUser[cuID] = auID + migratedWorkspace[cwsID] = awsID + log.Printf("%s: u:%v -> u:%v", u.Name, cuID, auID) + log.Printf("%s: ws:%v -> ws:%v", u.Name, cwsID, awsID) + } else if len(migrated) > 0 { + // subject に紐づく account user_id があるが欠損している sub があるので subs を追加する必要がある + panic("unreachable") + } else { + // すべての subject に紐づく account user_id がない場合は新規に挿入する + log.Printf("%s: insert u:%s %v", u.Name, cuID, subs) + log.Printf("%s: insert ws:%s", u.Name, cwsID) + + migratedUser[cuID] = accountUserID(cuID) + migratedWorkspace[cwsID] = accountWorkspaceID(cwsID) + + userWrites = append(userWrites, mongo.NewInsertOneModel().SetDocument(u)) + } + } + + cmsWSs, err := allWorkspaces(ctx, client, reearthCMS) + if err != nil { + return fmt.Errorf("reearth_cms all workspaces: %w", err) + } + + var wsWrites []mongo.WriteModel + for _, ws := range cmsWSs { + cwsID, cmsMembers := fromCMSWorkspace(ws) + members := map[string]mongodoc.WorkspaceMemberDocument{} + for cuID, v := range cmsMembers { + auID := migratedUser[cuID] + if !isSameUserID(auID, cuID) { + log.Printf("%s: member u:%s -> u:%s", ws.Name, cuID, auID) + } + members[string(auID)] = v + } + + if awsID, ok := migratedWorkspace[cwsID]; ok { + // TODO: reearth_cms 側の個人ワークスペースと reearth-account 側の個人ワークスペースのメンバーが異なる場合どうするか？ + wsWrites = append(wsWrites, mongo.NewUpdateOneModel(). + SetFilter(bson.M{"id": awsID}). + SetUpdate(bson.M{"$set": bson.M{"members": members}}), + ) + } else { // 個人ワークスペース以外の場合 reearth-account 側に追加する必要がある + migratedWorkspace[cwsID] = accountWorkspaceID(cwsID) + log.Printf("%s: insert ws:%s", ws.Name, cwsID) + aws := *ws + aws.Members = members + wsWrites = append(wsWrites, mongo.NewInsertOneModel().SetDocument(aws)) + } + } + + ps, err := allProjects(ctx, client) + if err != nil { + return fmt.Errorf("reearth_cms all projects: %w", err) + } + + var projectWrites []mongo.WriteModel + for _, p := range ps { + cwsID := cmsWorkspaceID(p.Workspace) + awsID := migratedWorkspace[cwsID] + if isSameWorkspaceID(awsID, cwsID) { + continue + } + // reearth-account 側に統合したワークスペースの場合は ID が変わっているので project からの参照を書き換える必要がある + log.Printf("%s: workspace ws:%s -> ws:%s", p.Name, cwsID, awsID) + projectWrites = append(projectWrites, mongo.NewUpdateOneModel(). + SetFilter(bson.M{"id": p.ID}). + SetUpdate(bson.M{"$set": bson.M{"workspace": awsID}}), + ) + } + + if !*dry { + if _, err := client.Database(reearthAccount).Collection("user").BulkWrite(ctx, userWrites); err != nil { + return fmt.Errorf("write user: %w", err) + } + if _, err := client.Database(reearthAccount).Collection("workspace").BulkWrite(ctx, wsWrites); err != nil { + return fmt.Errorf("write workspace: %w", err) + } + if _, err := client.Database(reearthCMS).Collection("project").BulkWrite(ctx, projectWrites); err != nil { + return fmt.Errorf("write project: %w", err) + } + } + + return nil +} + +func fromAccountUser(u *mongodoc.UserDocument) (accountUserID, accountWorkspaceID, []subject) { + return accountUserID(u.ID), accountWorkspaceID(u.Workspace), toSubs(u.Subs) +} + +func fromCMSUser(u *mongodoc.UserDocument) (cmsUserID, cmsWorkspaceID, []subject) { + return cmsUserID(u.ID), cmsWorkspaceID(u.Workspace), toSubs(u.Subs) +} + +func fromCMSWorkspace(ws *mongodoc.WorkspaceDocument) (cmsWorkspaceID, map[cmsUserID]mongodoc.WorkspaceMemberDocument) { + members := map[cmsUserID]mongodoc.WorkspaceMemberDocument{} + for k, v := range ws.Members { + members[cmsUserID(k)] = v + } + return cmsWorkspaceID(ws.ID), members +} + +func allUsers(ctx context.Context, c *mongo.Client, db string) ([]*mongodoc.UserDocument, error) { + cur, err := c.Database(db).Collection("user").Find(ctx, bson.M{}) + if err != nil { + return nil, fmt.Errorf("find: %w", err) + } + var users []*mongodoc.UserDocument + if err := cur.All(ctx, &users); err != nil { + return nil, fmt.Errorf("all: %w", err) + } + return users, nil +} + +func allWorkspaces(ctx context.Context, c *mongo.Client, db string) ([]*mongodoc.WorkspaceDocument, error) { + cur, err := c.Database(db).Collection("workspace").Find(ctx, bson.M{}) + if err != nil { + return nil, fmt.Errorf("find: %w", err) + } + var ws []*mongodoc.WorkspaceDocument + if err := cur.All(ctx, &ws); err != nil { + return nil, fmt.Errorf("all: %w", err) + } + return ws, nil +} + +func allProjects(ctx context.Context, c *mongo.Client) ([]*mongodoc2.ProjectDocument, error) { + cur, err := c.Database(reearthCMS).Collection("project").Find(ctx, bson.M{}) + if err != nil { + return nil, fmt.Errorf("find: %w", err) + } + var ps []*mongodoc2.ProjectDocument + if err := cur.All(ctx, &ps); err != nil { + return nil, fmt.Errorf("all: %w", err) + } + return ps, nil +} + +func toSubs(ss []string) []subject { + var subs []subject + for _, s := range ss { + subs = append(subs, subject(s)) + } + return subs +} + +func isSameUserID(a accountUserID, c cmsUserID) bool { + return string(a) == string(c) +} + +func isSameWorkspaceID(a accountWorkspaceID, c cmsWorkspaceID) bool { + return string(a) == string(c) +} From ce0b886bb29159cbd2382daaa56f32e5f82996a9 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Fri, 8 Sep 2023 22:28:56 +0900 Subject: [PATCH 2/3] refactor: add commnet --- server/cmd/migrate/main.go | 182 ++++++++++++++++++++----------------- 1 file changed, 98 insertions(+), 84 deletions(-) diff --git a/server/cmd/migrate/main.go b/server/cmd/migrate/main.go index e83474faac..3ad4563eca 100644 --- a/server/cmd/migrate/main.go +++ b/server/cmd/migrate/main.go @@ -41,26 +41,25 @@ func run() error { if err != nil { return fmt.Errorf("connect: %w", err) } - accountUsers, err := allUsers(ctx, client, reearthAccount) - if err != nil { - return fmt.Errorf("reearth-account all accountUsers: %w", err) - } - cmsUsers, err := allUsers(ctx, client, reearthCMS) - if err != nil { - return fmt.Errorf("reearth_cms all accountUsers: %w", err) - } // subject => account user_id accountUserMap := map[subject]accountUserID{} // account user_id = account personal ws_id personalAccountWorkspaces := map[accountUserID]accountWorkspaceID{} - for _, u := range accountUsers { - auID, awsID, subs := fromAccountUser(u) - for _, s := range subs { - accountUserMap[s] = auID + // reearth-account 側の全ユーザーから subject とユーザーIDの紐づけ, 個人ワークスペースの紐づけを作る + { + accountUsers, err := allUsers(ctx, client, reearthAccount) + if err != nil { + return fmt.Errorf("reearth-account all accountUsers: %w", err) + } + for _, u := range accountUsers { + auID, awsID, subs := fromAccountUser(u) + for _, s := range subs { + accountUserMap[s] = auID + } + personalAccountWorkspaces[auID] = awsID } - personalAccountWorkspaces[auID] = awsID } // cms ws_id => account ws_id @@ -68,91 +67,105 @@ func run() error { // cms user_id => account user_id migratedUser := map[cmsUserID]accountUserID{} + // reearth_cms 側の全ユーザーから reearth-account に subject が存在しないユーザーを抽出する + // subject が同一のユーザーが存在した場合 reearth-account 側のユーザに統合する var userWrites []mongo.WriteModel - for _, u := range cmsUsers { - cuID, cwsID, subs := fromCMSUser(u) - var migrated []subject - for _, s := range subs { - if _, ok := accountUserMap[s]; ok { - migrated = append(migrated, s) - } + { + cmsUsers, err := allUsers(ctx, client, reearthCMS) + if err != nil { + return fmt.Errorf("reearth_cms all accountUsers: %w", err) } - if len(migrated) == len(subs) { - // すべての subject が reearth-account に存在する場合は reearth-account 側の個人ワークスペースを cms 側の個人ワークスペースとする - // cms 側の個人ワークスペースに関する参照を書き換える必要が生じる - auID := accountUserMap[subs[0]] - awsID := personalAccountWorkspaces[auID] - migratedUser[cuID] = auID - migratedWorkspace[cwsID] = awsID - log.Printf("%s: u:%v -> u:%v", u.Name, cuID, auID) - log.Printf("%s: ws:%v -> ws:%v", u.Name, cwsID, awsID) - } else if len(migrated) > 0 { - // subject に紐づく account user_id があるが欠損している sub があるので subs を追加する必要がある - panic("unreachable") - } else { - // すべての subject に紐づく account user_id がない場合は新規に挿入する - log.Printf("%s: insert u:%s %v", u.Name, cuID, subs) - log.Printf("%s: insert ws:%s", u.Name, cwsID) - - migratedUser[cuID] = accountUserID(cuID) - migratedWorkspace[cwsID] = accountWorkspaceID(cwsID) - - userWrites = append(userWrites, mongo.NewInsertOneModel().SetDocument(u)) + for _, u := range cmsUsers { + cuID, cwsID, subs := fromCMSUser(u) + var migrated []subject + for _, s := range subs { + if _, ok := accountUserMap[s]; ok { + migrated = append(migrated, s) + } + } + if len(migrated) == len(subs) { + // すべての subject が reearth-account に存在する場合は reearth-account 側の個人ワークスペースを cms 側の個人ワークスペースとする + // cms 側の個人ワークスペースに関する参照を書き換える必要が生じる + auID := accountUserMap[subs[0]] + awsID := personalAccountWorkspaces[auID] + migratedUser[cuID] = auID + migratedWorkspace[cwsID] = awsID + log.Printf("%s: u:%v -> u:%v", u.Name, cuID, auID) + log.Printf("%s: ws:%v -> ws:%v", u.Name, cwsID, awsID) + } else if len(migrated) > 0 { + // subject に紐づく account user_id があるが欠損している sub があるので subs を追加する必要がある + panic("unreachable") + } else { + // すべての subject に紐づく account user_id がない場合は新規に挿入する + log.Printf("%s: insert u:%s %v", u.Name, cuID, subs) + log.Printf("%s: insert ws:%s", u.Name, cwsID) + + migratedUser[cuID] = accountUserID(cuID) + migratedWorkspace[cwsID] = accountWorkspaceID(cwsID) + + userWrites = append(userWrites, mongo.NewInsertOneModel().SetDocument(u)) + } } } - cmsWSs, err := allWorkspaces(ctx, client, reearthCMS) - if err != nil { - return fmt.Errorf("reearth_cms all workspaces: %w", err) - } - + // reearth-account へのユーザー統合に伴う統合が必要な個人ワークスペースの抽出を行う + // 個人ワークスペース以外はそのまま reearth-account に追加する + // すべてのワークスペースはユーザーへの参照を保持しているので書き換える必要がある var wsWrites []mongo.WriteModel - for _, ws := range cmsWSs { - cwsID, cmsMembers := fromCMSWorkspace(ws) - members := map[string]mongodoc.WorkspaceMemberDocument{} - for cuID, v := range cmsMembers { - auID := migratedUser[cuID] - if !isSameUserID(auID, cuID) { - log.Printf("%s: member u:%s -> u:%s", ws.Name, cuID, auID) - } - members[string(auID)] = v + { + cmsWSs, err := allWorkspaces(ctx, client, reearthCMS) + if err != nil { + return fmt.Errorf("reearth_cms all workspaces: %w", err) } + for _, ws := range cmsWSs { + cwsID, cmsMembers := fromCMSWorkspace(ws) + members := map[string]mongodoc.WorkspaceMemberDocument{} + for cuID, v := range cmsMembers { + auID := migratedUser[cuID] + if !isSameUserID(auID, cuID) { + log.Printf("%s: member u:%s -> u:%s", ws.Name, cuID, auID) + } + members[string(auID)] = v + } - if awsID, ok := migratedWorkspace[cwsID]; ok { - // TODO: reearth_cms 側の個人ワークスペースと reearth-account 側の個人ワークスペースのメンバーが異なる場合どうするか？ - wsWrites = append(wsWrites, mongo.NewUpdateOneModel(). - SetFilter(bson.M{"id": awsID}). - SetUpdate(bson.M{"$set": bson.M{"members": members}}), - ) - } else { // 個人ワークスペース以外の場合 reearth-account 側に追加する必要がある - migratedWorkspace[cwsID] = accountWorkspaceID(cwsID) - log.Printf("%s: insert ws:%s", ws.Name, cwsID) - aws := *ws - aws.Members = members - wsWrites = append(wsWrites, mongo.NewInsertOneModel().SetDocument(aws)) + if awsID, ok := migratedWorkspace[cwsID]; ok { + // TODO: reearth_cms 側の個人ワークスペースと reearth-account 側の個人ワークスペースのメンバーが異なる場合どうするか？ + wsWrites = append(wsWrites, mongo.NewUpdateOneModel(). + SetFilter(bson.M{"id": awsID}). + SetUpdate(bson.M{"$set": bson.M{"members": members}}), + ) + } else { // 個人ワークスペース以外の場合 reearth-account 側に追加する必要がある + migratedWorkspace[cwsID] = accountWorkspaceID(cwsID) + log.Printf("%s: insert ws:%s", ws.Name, cwsID) + aws := *ws + aws.Members = members + wsWrites = append(wsWrites, mongo.NewInsertOneModel().SetDocument(aws)) + } } } - ps, err := allProjects(ctx, client) - if err != nil { - return fmt.Errorf("reearth_cms all projects: %w", err) - } - + // ワークスペースの統合に伴って reearth_cms 側でワークスペースを参照しているプロジェクトを書き換える var projectWrites []mongo.WriteModel - for _, p := range ps { - cwsID := cmsWorkspaceID(p.Workspace) - awsID := migratedWorkspace[cwsID] - if isSameWorkspaceID(awsID, cwsID) { - continue + { + ps, err := allProjects(ctx, client) + if err != nil { + return fmt.Errorf("reearth_cms all projects: %w", err) } - // reearth-account 側に統合したワークスペースの場合は ID が変わっているので project からの参照を書き換える必要がある - log.Printf("%s: workspace ws:%s -> ws:%s", p.Name, cwsID, awsID) - projectWrites = append(projectWrites, mongo.NewUpdateOneModel(). - SetFilter(bson.M{"id": p.ID}). - SetUpdate(bson.M{"$set": bson.M{"workspace": awsID}}), - ) - } + for _, p := range ps { + cwsID := cmsWorkspaceID(p.Workspace) + awsID := migratedWorkspace[cwsID] + if isSameWorkspaceID(awsID, cwsID) { + continue + } + // reearth-account 側に統合したワークスペースの場合は ID が変わっているので project からの参照を書き換える必要がある + log.Printf("%s: workspace ws:%s -> ws:%s", p.Name, cwsID, awsID) + projectWrites = append(projectWrites, mongo.NewUpdateOneModel(). + SetFilter(bson.M{"id": p.ID}). + SetUpdate(bson.M{"$set": bson.M{"workspace": awsID}}), + ) + } + } if !*dry { if _, err := client.Database(reearthAccount).Collection("user").BulkWrite(ctx, userWrites); err != nil { return fmt.Errorf("write user: %w", err) @@ -160,6 +173,7 @@ func run() error { if _, err := client.Database(reearthAccount).Collection("workspace").BulkWrite(ctx, wsWrites); err != nil { return fmt.Errorf("write workspace: %w", err) } + // プロジェクトだけ reearth_cms 側の更新が必要になる if _, err := client.Database(reearthCMS).Collection("project").BulkWrite(ctx, projectWrites); err != nil { return fmt.Errorf("write project: %w", err) } From 2294ba2da3e6a056d039d1c5baaa6a4b02e6f47f Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Thu, 21 Sep 2023 18:35:37 +0900 Subject: [PATCH 3/3] chore: update --- server/cmd/migrate/main.go | 309 +++++++++++++++++++++++++++++++++---- server/go.mod | 2 +- server/go.sum | 4 + 3 files changed, 283 insertions(+), 32 deletions(-) diff --git a/server/cmd/migrate/main.go b/server/cmd/migrate/main.go index 3ad4563eca..e846b509c6 100644 --- a/server/cmd/migrate/main.go +++ b/server/cmd/migrate/main.go @@ -1,11 +1,13 @@ package main import ( + "bufio" "context" "flag" "fmt" "log" "os" + "strings" mongodoc2 "github.com/reearth/reearth-cms/server/internal/infrastructure/mongo/mongodoc" "github.com/reearth/reearthx/account/accountinfrastructure/accountmongo/mongodoc" @@ -27,29 +29,99 @@ type cmsWorkspaceID string type accountWorkspaceID string const ( + reearth = "reearth" reearthAccount = "reearth-account" reearthCMS = "reearth_cms" ) +func rewriteTeam(ctx context.Context, client *mongo.Client, from, to string) error { + if _, err := client.Database(reearth).Collection("project").UpdateMany(ctx, bson.M{"team": from}, bson.M{"$set": bson.M{"team": to}}); err != nil { + return err + } + if _, err := client.Database(reearth).Collection("scene").UpdateMany(ctx, bson.M{"team": from}, bson.M{"$set": bson.M{"team": to}}); err != nil { + return err + } + return nil +} + func run() error { + log.SetOutput(os.Stdout) dry := flag.Bool("dry", false, "dry run") flag.Parse() ctx := context.Background() - dbURI := os.Getenv("DB_URI") - client, err := mongo.Connect(ctx, options.Client().ApplyURI(dbURI)) + + databaseURI := os.Getenv("DB_URI") + client, err := mongo.Connect(ctx, options.Client().ApplyURI(databaseURI)) if err != nil { return fmt.Errorf("connect: %w", err) } + if false { + if err := restoreProjects(ctx, client); err != nil { + return fmt.Errorf("restore p: %w", err) + } + if err := restoreIntegrations(ctx, client); err != nil { + return fmt.Errorf("restore i: %w", err) + } + return nil + } + + if false { + if err := client.Database(reearthAccount).Drop(ctx); err != nil { + return fmt.Errorf("drop reearth-account: %w", err) + } + } + + var migrateUserWrites []mongo.WriteModel + { + users, err := allUsers(ctx, client, reearth) + if err != nil { + return fmt.Errorf("reearth all users: %w", err) + } + for _, u := range users { + migrateUserWrites = append(migrateUserWrites, mongo.NewInsertOneModel().SetDocument(u)) + } + } + var migrateWorkspaceWrites []mongo.WriteModel + { + ws, err := allWorkspaces(ctx, client, reearth, "team") + if err != nil { + return fmt.Errorf("reearth all teams: %w", err) + } + for _, w := range ws { + migrateWorkspaceWrites = append(migrateWorkspaceWrites, mongo.NewInsertOneModel().SetDocument(w)) + } + } + if !*dry { + if len(migrateUserWrites) > 0 { + if _, err := client.Database(reearthAccount).Collection("user").BulkWrite(ctx, migrateUserWrites); err != nil { + return fmt.Errorf("write users: %w", err) + } + } + if len(migrateWorkspaceWrites) > 0 { + if _, err := client.Database(reearthAccount).Collection("workspace").BulkWrite(ctx, migrateWorkspaceWrites); err != nil { + return fmt.Errorf("write workspaces: %w", err) + } + } + } + return nil + + cmsDatabaseURI := os.Getenv("CMS_DB_URI") + cmsClient, err := mongo.Connect(ctx, options.Client().ApplyURI(cmsDatabaseURI)) + if err != nil { + return fmt.Errorf("connect cms: %w", err) + } + // subject => account user_id accountUserMap := map[subject]accountUserID{} // account user_id = account personal ws_id personalAccountWorkspaces := map[accountUserID]accountWorkspaceID{} + emails := map[string]bool{} // reearth-account 側の全ユーザーから subject とユーザーIDの紐づけ, 個人ワークスペースの紐づけを作る { - accountUsers, err := allUsers(ctx, client, reearthAccount) + accountUsers, err := allUsers(ctx, cmsClient, reearthAccount) if err != nil { return fmt.Errorf("reearth-account all accountUsers: %w", err) } @@ -58,6 +130,7 @@ func run() error { for _, s := range subs { accountUserMap[s] = auID } + emails[u.Email] = true personalAccountWorkspaces[auID] = awsID } } @@ -70,8 +143,8 @@ func run() error { // reearth_cms 側の全ユーザーから reearth-account に subject が存在しないユーザーを抽出する // subject が同一のユーザーが存在した場合 reearth-account 側のユーザに統合する var userWrites []mongo.WriteModel - { - cmsUsers, err := allUsers(ctx, client, reearthCMS) + if false { + cmsUsers, err := allUsers(ctx, cmsClient, reearthCMS) if err != nil { return fmt.Errorf("reearth_cms all accountUsers: %w", err) } @@ -91,14 +164,24 @@ func run() error { migratedUser[cuID] = auID migratedWorkspace[cwsID] = awsID log.Printf("%s: u:%v -> u:%v", u.Name, cuID, auID) - log.Printf("%s: ws:%v -> ws:%v", u.Name, cwsID, awsID) + log.Printf("%s: ws:%v -> ws:%v personal", u.Name, cwsID, awsID) } else if len(migrated) > 0 { // subject に紐づく account user_id があるが欠損している sub があるので subs を追加する必要がある - panic("unreachable") + diff := []subject{} + ss := map[subject]bool{} + for _, sub := range migrated { + ss[sub] = true + } + for _, s := range subs { + if !ss[s] { + diff = append(diff, s) + } + } + log.Printf("WARN: reearth-cms-only user: %s: u:%s %+v", u.Name, cuID, diff) } else { // すべての subject に紐づく account user_id がない場合は新規に挿入する log.Printf("%s: insert u:%s %v", u.Name, cuID, subs) - log.Printf("%s: insert ws:%s", u.Name, cwsID) + log.Printf("%s: insert ws:%s personal", u.Name, cwsID) migratedUser[cuID] = accountUserID(cuID) migratedWorkspace[cwsID] = accountWorkspaceID(cwsID) @@ -112,33 +195,58 @@ func run() error { // 個人ワークスペース以外はそのまま reearth-account に追加する // すべてのワークスペースはユーザーへの参照を保持しているので書き換える必要がある var wsWrites []mongo.WriteModel - { - cmsWSs, err := allWorkspaces(ctx, client, reearthCMS) + if false { + cmsWSs, err := allWorkspaces(ctx, cmsClient, reearthCMS, "workspace") if err != nil { return fmt.Errorf("reearth_cms all workspaces: %w", err) } for _, ws := range cmsWSs { - cwsID, cmsMembers := fromCMSWorkspace(ws) + cwsID, cmsMembers, cmsIntegrations := fromCMSWorkspace(ws) members := map[string]mongodoc.WorkspaceMemberDocument{} for cuID, v := range cmsMembers { - auID := migratedUser[cuID] + auID, ok := migratedUser[cuID] + if !ok { + log.Printf("WARN: member not found: ws:%s u:%s", cwsID, cuID) + continue + } if !isSameUserID(auID, cuID) { log.Printf("%s: member u:%s -> u:%s", ws.Name, cuID, auID) } + invitedBy := cmsUserID(v.InvitedBy) + if migrated := migratedUser[invitedBy]; !isSameUserID(migrated, invitedBy) { + v.InvitedBy = string(migrated) + } members[string(auID)] = v } + // メンバーが 0 なら作らない + integrations := map[string]mongodoc.WorkspaceMemberDocument{} + for id, v := range cmsIntegrations { + invitedBy := cmsUserID(v.InvitedBy) + if migrated := migratedUser[invitedBy]; !isSameUserID(migrated, invitedBy) { + v.InvitedBy = string(migrated) + } + integrations[id] = v + } if awsID, ok := migratedWorkspace[cwsID]; ok { - // TODO: reearth_cms 側の個人ワークスペースと reearth-account 側の個人ワークスペースのメンバーが異なる場合どうするか？ - wsWrites = append(wsWrites, mongo.NewUpdateOneModel(). - SetFilter(bson.M{"id": awsID}). - SetUpdate(bson.M{"$set": bson.M{"members": members}}), - ) + if isSameWorkspaceID(awsID, cwsID) { + log.Printf("%s: insert ws:%s", ws.Name, cwsID) + aws := *ws + aws.Members = members + aws.Integrations = integrations + wsWrites = append(wsWrites, mongo.NewInsertOneModel().SetDocument(aws)) + } else { + wsWrites = append(wsWrites, mongo.NewUpdateOneModel(). + SetFilter(bson.M{"id": awsID}). + SetUpdate(bson.M{"$set": bson.M{"members": members, "integrations": integrations}}), + ) + } } else { // 個人ワークスペース以外の場合 reearth-account 側に追加する必要がある migratedWorkspace[cwsID] = accountWorkspaceID(cwsID) log.Printf("%s: insert ws:%s", ws.Name, cwsID) aws := *ws aws.Members = members + aws.Integrations = integrations wsWrites = append(wsWrites, mongo.NewInsertOneModel().SetDocument(aws)) } } @@ -147,15 +255,21 @@ func run() error { // ワークスペースの統合に伴って reearth_cms 側でワークスペースを参照しているプロジェクトを書き換える var projectWrites []mongo.WriteModel { - ps, err := allProjects(ctx, client) + ps, err := allProjects(ctx, cmsClient) if err != nil { return fmt.Errorf("reearth_cms all projects: %w", err) } for _, p := range ps { + log.Printf("PROJECT: id:%s ws:%s", p.ID, p.Workspace) cwsID := cmsWorkspaceID(p.Workspace) awsID := migratedWorkspace[cwsID] if isSameWorkspaceID(awsID, cwsID) { + log.Printf("DEBUG:: %s: p:%s ws:%s -> ws:%s same", p.Name, p.ID, p.Workspace, awsID) + continue + } + if awsID == "" { + log.Printf("WARN: MISSING WORKSPACE p:%s ws:%s", p.ID, cwsID) continue } // reearth-account 側に統合したワークスペースの場合は ID が変わっているので project からの参照を書き換える必要がある @@ -166,16 +280,52 @@ func run() error { ) } } + var integrationWrites []mongo.WriteModel + { + is, err := allIntegrations(ctx, cmsClient) + if err != nil { + return fmt.Errorf("reearth_cms all integrations: %w", err) + } + for _, i := range is { + log.Printf("INTEGRATION: id:%s u:%s", i.ID, i.Developer) + cuID := cmsUserID(i.Developer) + auID := migratedUser[cuID] + if isSameUserID(auID, cuID) { + continue + } + if auID == "" { + log.Printf("WARN: MISSING USER i:%s u:%s", i.ID, cuID) + continue + } + log.Printf("%s: developer u:%s -> u:%s", i.Name, cuID, auID) + integrationWrites = append(integrationWrites, mongo.NewUpdateOneModel(). + SetFilter(bson.M{"id": i.ID}). + SetUpdate(bson.M{"$set": bson.M{"developer": auID}}), + ) + } + } + if !*dry { - if _, err := client.Database(reearthAccount).Collection("user").BulkWrite(ctx, userWrites); err != nil { - return fmt.Errorf("write user: %w", err) + if len(userWrites) > 0 { + if _, err := cmsClient.Database(reearthAccount).Collection("user").BulkWrite(ctx, userWrites); err != nil { + return fmt.Errorf("write user: %w", err) + } } - if _, err := client.Database(reearthAccount).Collection("workspace").BulkWrite(ctx, wsWrites); err != nil { - return fmt.Errorf("write workspace: %w", err) + if len(wsWrites) > 0 { + if _, err := cmsClient.Database(reearthAccount).Collection("workspace").BulkWrite(ctx, wsWrites); err != nil { + return fmt.Errorf("write workspace: %w", err) + } } - // プロジェクトだけ reearth_cms 側の更新が必要になる - if _, err := client.Database(reearthCMS).Collection("project").BulkWrite(ctx, projectWrites); err != nil { - return fmt.Errorf("write project: %w", err) + if false && len(projectWrites) > 0 { + // プロジェクトだけ reearth_cms 側の更新が必要になる + if _, err := cmsClient.Database(reearthCMS).Collection("project").BulkWrite(ctx, projectWrites); err != nil { + return fmt.Errorf("write project: %w", err) + } + } + if false && len(integrationWrites) > 0 { + if _, err := cmsClient.Database(reearthCMS).Collection("integration").BulkWrite(ctx, integrationWrites); err != nil { + return fmt.Errorf("write integration: %w", err) + } } } @@ -190,12 +340,31 @@ func fromCMSUser(u *mongodoc.UserDocument) (cmsUserID, cmsWorkspaceID, []subject return cmsUserID(u.ID), cmsWorkspaceID(u.Workspace), toSubs(u.Subs) } -func fromCMSWorkspace(ws *mongodoc.WorkspaceDocument) (cmsWorkspaceID, map[cmsUserID]mongodoc.WorkspaceMemberDocument) { +func fromCMSWorkspace(ws *mongodoc.WorkspaceDocument) (cmsWorkspaceID, map[cmsUserID]mongodoc.WorkspaceMemberDocument, map[string]mongodoc.WorkspaceMemberDocument) { members := map[cmsUserID]mongodoc.WorkspaceMemberDocument{} for k, v := range ws.Members { members[cmsUserID(k)] = v } - return cmsWorkspaceID(ws.ID), members + integrations := map[string]mongodoc.WorkspaceMemberDocument{} + for k, v := range ws.Integrations { + integrations[k] = v + } + return cmsWorkspaceID(ws.ID), members, integrations +} + +type UserDocument struct { + ID string + Name string + Email string + Auth0Sublist []string + Subs []string + Workspace string + Team string + Lang string + Theme string + Password []byte + PasswordReset *mongodoc.PasswordResetDocument + Verification *mongodoc.UserVerificationDoc } func allUsers(ctx context.Context, c *mongo.Client, db string) ([]*mongodoc.UserDocument, error) { @@ -203,15 +372,39 @@ func allUsers(ctx context.Context, c *mongo.Client, db string) ([]*mongodoc.User if err != nil { return nil, fmt.Errorf("find: %w", err) } - var users []*mongodoc.UserDocument + var users []UserDocument if err := cur.All(ctx, &users); err != nil { return nil, fmt.Errorf("all: %w", err) } - return users, nil + var newUsers []*mongodoc.UserDocument + for _, u := range users { + subs := u.Subs + if len(subs) == 0 { + subs = u.Auth0Sublist + } + if u.Team != "" { + u.Workspace = u.Team + u.Team = "" + } + newUsers = append(newUsers, &mongodoc.UserDocument{ + ID: u.ID, + Name: u.Name, + Email: u.Email, + Subs: subs, + Workspace: u.Workspace, + Team: u.Team, + Lang: u.Lang, + Theme: u.Theme, + Password: u.Password, + PasswordReset: u.PasswordReset, + Verification: u.Verification, + }) + } + return newUsers, nil } -func allWorkspaces(ctx context.Context, c *mongo.Client, db string) ([]*mongodoc.WorkspaceDocument, error) { - cur, err := c.Database(db).Collection("workspace").Find(ctx, bson.M{}) +func allWorkspaces(ctx context.Context, c *mongo.Client, db string, name string) ([]*mongodoc.WorkspaceDocument, error) { + cur, err := c.Database(db).Collection(name).Find(ctx, bson.M{}) if err != nil { return nil, fmt.Errorf("find: %w", err) } @@ -234,6 +427,18 @@ func allProjects(ctx context.Context, c *mongo.Client) ([]*mongodoc2.ProjectDocu return ps, nil } +func allIntegrations(ctx context.Context, c *mongo.Client) ([]*mongodoc2.IntegrationDocument, error) { + cur, err := c.Database(reearthCMS).Collection("integration").Find(ctx, bson.M{}) + if err != nil { + return nil, fmt.Errorf("find: %w", err) + } + var ps []*mongodoc2.IntegrationDocument + if err := cur.All(ctx, &ps); err != nil { + return nil, fmt.Errorf("all: %w", err) + } + return ps, nil +} + func toSubs(ss []string) []subject { var subs []subject for _, s := range ss { @@ -249,3 +454,45 @@ func isSameUserID(a accountUserID, c cmsUserID) bool { func isSameWorkspaceID(a accountWorkspaceID, c cmsWorkspaceID) bool { return string(a) == string(c) } + +func restoreProjects(ctx context.Context, client *mongo.Client) error { + f, err := os.Open("projects.txt") + if err != nil { + return fmt.Errorf("projects.txt: %w", err) + } + defer f.Close() + + s := bufio.NewScanner(f) + var writes []mongo.WriteModel + for s.Scan() { + ts := strings.Split(s.Text(), " ") + id := strings.TrimPrefix(ts[3], "id:") + ws := strings.TrimPrefix(ts[4], "ws:") + writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"id": id}).SetUpdate(bson.M{"$set": bson.M{"workspace": ws}})) + } + if _, err := client.Database(reearthCMS).Collection("project").BulkWrite(ctx, writes); err != nil { + return fmt.Errorf("write project: %w", err) + } + return nil +} + +func restoreIntegrations(ctx context.Context, client *mongo.Client) error { + f, err := os.Open("integrations.txt") + if err != nil { + return fmt.Errorf("integrations.txt: %w", err) + } + defer f.Close() + + s := bufio.NewScanner(f) + var writes []mongo.WriteModel + for s.Scan() { + ts := strings.Split(s.Text(), " ") + id := strings.TrimPrefix(ts[3], "id:") + u := strings.TrimPrefix(ts[4], "u:") + writes = append(writes, mongo.NewUpdateOneModel().SetFilter(bson.M{"id": id}).SetUpdate(bson.M{"$set": bson.M{"developer": u}})) + } + if _, err := client.Database(reearthCMS).Collection("integration").BulkWrite(ctx, writes); err != nil { + return fmt.Errorf("write integration: %w", err) + } + return nil +} diff --git a/server/go.mod b/server/go.mod index 1699f2c6c1..37199c5ecc 100644 --- a/server/go.mod +++ b/server/go.mod @@ -23,7 +23,7 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 github.com/labstack/echo/v4 v4.10.2 github.com/ravilushqa/otelgqlgen v0.13.1 - github.com/reearth/reearthx v0.0.0-20230803135203-6fb13c1d7d8b + github.com/reearth/reearthx v0.0.0-20230921095435-ee497ccf3f7e github.com/robbiet480/go.sns v0.0.0-20230523235941-e8d832c79d68 github.com/samber/lo v1.38.1 github.com/sendgrid/sendgrid-go v3.12.0+incompatible diff --git a/server/go.sum b/server/go.sum index c6895c784b..ecadc82e16 100644 --- a/server/go.sum +++ b/server/go.sum @@ -443,6 +443,10 @@ github.com/ravilushqa/otelgqlgen v0.13.1 h1:V+zFE75iDd2/CSzy5kKnb+Fi09SsE5535wv9 github.com/ravilushqa/otelgqlgen v0.13.1/go.mod h1:ZIyWykK2paCuNi9k8gk5edcNSwDJuxZaW90vZXpafxw= github.com/reearth/reearthx v0.0.0-20230803135203-6fb13c1d7d8b h1:EmpPvS/fC0w3qF7vI9MTfBqa7ch5CgpEIzSyq/v/2J0= github.com/reearth/reearthx v0.0.0-20230803135203-6fb13c1d7d8b/go.mod h1:O4zFr6ks8jDKfgy9NBpZKp31jpmAPnfOtMQK9B6xX54= +github.com/reearth/reearthx v0.0.0-20230825072907-543ac159f33b h1:7IsqB7KdGzp0hwlOIGb2imQW8mWJ9Ua9Ad5pAGNYaRs= +github.com/reearth/reearthx v0.0.0-20230825072907-543ac159f33b/go.mod h1:b8EygPZ9VcMv9vTbnl/oz3PU/wHz3wpBa4rc7W7URjw= +github.com/reearth/reearthx v0.0.0-20230921095435-ee497ccf3f7e h1:WaKfv4akNs2OI2vh2d6YIf/W812Y2/4tVv3HtC4xstU= +github.com/reearth/reearthx v0.0.0-20230921095435-ee497ccf3f7e/go.mod h1:b8EygPZ9VcMv9vTbnl/oz3PU/wHz3wpBa4rc7W7URjw= github.com/robbiet480/go.sns v0.0.0-20230523235941-e8d832c79d68 h1:Jknsfy5cqCH6qAuoU1qNZ51hfBJfMSJYwsH9j9mdVnw= github.com/robbiet480/go.sns v0.0.0-20230523235941-e8d832c79d68/go.mod h1:9CDhL7uDVy8vEVDNPJzxq89dPaPBWP6hxQcC8woBHus= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=