diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index 230c2a0236..9a107f88b7 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -45,6 +45,7 @@ var serviceCommands = []func(cmd *cobra.Command) *cobra.Command{ addOneDriveCommands, addSharePointCommands, addGroupsCommands, + addTeamsChatsCommands, } // AddCommands attaches all `corso backup * *` commands to the parent. diff --git a/src/cli/backup/groups.go b/src/cli/backup/groups.go index 2a8ab3024e..fad35ce5d0 100644 --- a/src/cli/backup/groups.go +++ b/src/cli/backup/groups.go @@ -310,7 +310,7 @@ func groupsBackupCreateSelectors( group, cats []string, ) *selectors.GroupsBackup { if filters.PathContains(group).Compare(flags.Wildcard) { - return includeAllGroupWithCategories(ins, cats) + return includeAllGroupsWithCategories(ins, cats) } sel := selectors.NewGroupsBackup(slices.Clone(group)) @@ -318,6 +318,6 @@ func groupsBackupCreateSelectors( return utils.AddGroupsCategories(sel, cats) } -func includeAllGroupWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup { +func includeAllGroupsWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup { return utils.AddGroupsCategories(selectors.NewGroupsBackup(ins.IDs()), categories) } diff --git a/src/cli/backup/teamschats.go b/src/cli/backup/teamschats.go new file mode 100644 index 0000000000..1901893809 --- /dev/null +++ b/src/cli/backup/teamschats.go @@ -0,0 +1,305 @@ +package backup + +import ( + "context" + "fmt" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "golang.org/x/exp/slices" + + "github.com/alcionai/corso/src/cli/flags" + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/filters" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365" +) + +// ------------------------------------------------------------------------------------------------ +// setup and globals +// ------------------------------------------------------------------------------------------------ + +const ( + teamschatsServiceCommand = "chats" + teamschatsServiceCommandCreateUseSuffix = "--user | '" + flags.Wildcard + "'" + teamschatsServiceCommandDeleteUseSuffix = "--backups " + teamschatsServiceCommandDetailsUseSuffix = "--backup " +) + +const ( + teamschatsServiceCommandCreateExamples = `# Backup all chats with bob@company.hr +corso backup create chats --user bob@company.hr + +# Backup all chats for all users +corso backup create chats --user '*'` + + teamschatsServiceCommandDeleteExamples = `# Delete chats backup with ID 1234abcd-12ab-cd34-56de-1234abcd \ +and 1234abcd-12ab-cd34-56de-1234abce +corso backup delete chats --backups 1234abcd-12ab-cd34-56de-1234abcd,1234abcd-12ab-cd34-56de-1234abce` + + teamschatsServiceCommandDetailsExamples = `# Explore chats in Bob's latest backup (1234abcd...) +corso backup details chats --backup 1234abcd-12ab-cd34-56de-1234abcd` +) + +// called by backup.go to map subcommands to provider-specific handling. +func addTeamsChatsCommands(cmd *cobra.Command) *cobra.Command { + var c *cobra.Command + + switch cmd.Use { + case createCommand: + c, _ = utils.AddCommand(cmd, teamschatsCreateCmd(), utils.MarkPreReleaseCommand()) + + c.Use = c.Use + " " + teamschatsServiceCommandCreateUseSuffix + c.Example = teamschatsServiceCommandCreateExamples + + // Flags addition ordering should follow the order we want them to appear in help and docs: + flags.AddUserFlag(c) + flags.AddDataFlag(c, []string{flags.DataChats}, false) + flags.AddGenericBackupFlags(c) + + case listCommand: + c, _ = utils.AddCommand(cmd, teamschatsListCmd(), utils.MarkPreReleaseCommand()) + + flags.AddBackupIDFlag(c, false) + flags.AddAllBackupListFlags(c) + + case detailsCommand: + c, _ = utils.AddCommand(cmd, teamschatsDetailsCmd(), utils.MarkPreReleaseCommand()) + + c.Use = c.Use + " " + teamschatsServiceCommandDetailsUseSuffix + c.Example = teamschatsServiceCommandDetailsExamples + + flags.AddSkipReduceFlag(c) + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + flags.AddBackupIDFlag(c, true) + flags.AddTeamsChatsDetailsAndRestoreFlags(c) + + case deleteCommand: + c, _ = utils.AddCommand(cmd, teamschatsDeleteCmd(), utils.MarkPreReleaseCommand()) + + c.Use = c.Use + " " + teamschatsServiceCommandDeleteUseSuffix + c.Example = teamschatsServiceCommandDeleteExamples + + flags.AddMultipleBackupIDsFlag(c, false) + flags.AddBackupIDFlag(c, false) + } + + return c +} + +// ------------------------------------------------------------------------------------------------ +// backup create +// ------------------------------------------------------------------------------------------------ + +// `corso backup create chats [...]` +func teamschatsCreateCmd() *cobra.Command { + return &cobra.Command{ + Use: teamschatsServiceCommand, + Aliases: []string{teamsServiceCommand}, + Short: "Backup M365 Chats data", + RunE: createTeamsChatsCmd, + Args: cobra.NoArgs, + } +} + +// processes a teamschats backup. +func createTeamsChatsCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + if err := validateTeamsChatsBackupCreateFlags(flags.UserFV, flags.CategoryDataFV); err != nil { + return err + } + + r, acct, err := utils.AccountConnectAndWriteRepoConfig( + ctx, + cmd, + path.TeamsChatsService) + if err != nil { + return Only(ctx, err) + } + + defer utils.CloseRepo(ctx, r) + + // TODO: log/print recoverable errors + errs := fault.New(false) + + svcCli, err := m365.NewM365Client(ctx, *acct) + if err != nil { + return Only(ctx, clues.Stack(err)) + } + + ins, err := svcCli.AC.Users().GetAllIDsAndNames(ctx, errs) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 teamschats")) + } + + sel := teamschatsBackupCreateSelectors(ctx, ins, flags.UserFV, flags.CategoryDataFV) + selectorSet := []selectors.Selector{} + + for _, discSel := range sel.SplitByResourceOwner(ins.IDs()) { + selectorSet = append(selectorSet, discSel.Selector) + } + + return genericCreateCommand( + ctx, + r, + "Chats", + selectorSet, + ins) +} + +// ------------------------------------------------------------------------------------------------ +// backup list +// ------------------------------------------------------------------------------------------------ + +// `corso backup list teamschats [...]` +func teamschatsListCmd() *cobra.Command { + return &cobra.Command{ + Use: teamschatsServiceCommand, + Short: "List the history of M365 Chats backups", + RunE: listTeamsChatsCmd, + Args: cobra.NoArgs, + } +} + +// lists the history of backup operations +func listTeamsChatsCmd(cmd *cobra.Command, args []string) error { + return genericListCommand(cmd, flags.BackupIDFV, path.TeamsChatsService, args) +} + +// ------------------------------------------------------------------------------------------------ +// backup details +// ------------------------------------------------------------------------------------------------ + +// `corso backup details teamschats [...]` +func teamschatsDetailsCmd() *cobra.Command { + return &cobra.Command{ + Use: teamschatsServiceCommand, + Short: "Shows the details of a M365 Chats backup", + RunE: detailsTeamsChatsCmd, + Args: cobra.NoArgs, + } +} + +// processes a teamschats backup. +func detailsTeamsChatsCmd(cmd *cobra.Command, args []string) error { + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + return runDetailsTeamsChatsCmd(cmd) +} + +func runDetailsTeamsChatsCmd(cmd *cobra.Command) error { + ctx := cmd.Context() + opts := utils.MakeTeamsChatsOpts(cmd) + + sel := utils.IncludeTeamsChatsRestoreDataSelectors(ctx, opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) + utils.FilterTeamsChatsRestoreInfoSelectors(sel, opts) + + ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector) + if err != nil { + return Only(ctx, err) + } + + if len(ds.Entries) > 0 { + ds.PrintEntries(ctx) + } else { + Info(ctx, selectors.ErrorNoMatchingItems) + } + + return nil +} + +// ------------------------------------------------------------------------------------------------ +// backup delete +// ------------------------------------------------------------------------------------------------ + +// `corso backup delete teamschats [...]` +func teamschatsDeleteCmd() *cobra.Command { + return &cobra.Command{ + Use: teamschatsServiceCommand, + Short: "Delete backed-up M365 Chats data", + RunE: deleteTeamsChatsCmd, + Args: cobra.NoArgs, + } +} + +// deletes an teamschats backup. +func deleteTeamsChatsCmd(cmd *cobra.Command, args []string) error { + backupIDValue := []string{} + + if len(flags.BackupIDsFV) > 0 { + backupIDValue = flags.BackupIDsFV + } else if len(flags.BackupIDFV) > 0 { + backupIDValue = append(backupIDValue, flags.BackupIDFV) + } else { + return clues.New("either --backup or --backups flag is required") + } + + return genericDeleteCommand(cmd, path.TeamsChatsService, "TeamsChats", backupIDValue, args) +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +func validateTeamsChatsBackupCreateFlags(teamschats, cats []string) error { + if len(teamschats) == 0 { + return clues.New( + "requires one or more --" + + flags.UserFN + " ids, or the wildcard --" + + flags.UserFN + " *") + } + + msg := fmt.Sprintf( + " is an unrecognized data type; only %s is supported", + flags.DataChats) + + allowedCats := utils.TeamsChatsAllowedCategories() + + for _, d := range cats { + if _, ok := allowedCats[d]; !ok { + return clues.New(d + msg) + } + } + + return nil +} + +func teamschatsBackupCreateSelectors( + ctx context.Context, + ins idname.Cacher, + users, cats []string, +) *selectors.TeamsChatsBackup { + if filters.PathContains(users).Compare(flags.Wildcard) { + return includeAllTeamsChatsWithCategories(ins, cats) + } + + sel := selectors.NewTeamsChatsBackup(slices.Clone(users)) + + return utils.AddTeamsChatsCategories(sel, cats) +} + +func includeAllTeamsChatsWithCategories(ins idname.Cacher, categories []string) *selectors.TeamsChatsBackup { + return utils.AddTeamsChatsCategories(selectors.NewTeamsChatsBackup(ins.IDs()), categories) +} diff --git a/src/cli/backup/teamschats_e2e_test.go b/src/cli/backup/teamschats_e2e_test.go new file mode 100644 index 0000000000..2e829e628e --- /dev/null +++ b/src/cli/backup/teamschats_e2e_test.go @@ -0,0 +1,631 @@ +package backup_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/alcionai/clues" + "github.com/google/uuid" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli" + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/print" + cliTD "github.com/alcionai/corso/src/cli/testdata" + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/config" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" + storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" +) + +// --------------------------------------------------------------------------- +// tests that require no existing backups +// --------------------------------------------------------------------------- + +type NoBackupTeamsChatsE2ESuite struct { + tester.Suite + dpnd dependencies + its intgTesterSetup +} + +func TestNoBackupTeamsChatsE2ESuite(t *testing.T) { + suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite( + t, + [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})}) +} + +func (suite *NoBackupTeamsChatsE2ESuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + suite.its = newIntegrationTesterSetup(t) + suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService) +} + +func (suite *NoBackupTeamsChatsE2ESuite) TestTeamsChatsBackupListCmd_noBackups() { + t := suite.T() + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + suite.dpnd.recorder.Reset() + + cmd := cliTD.StubRootCmd( + "backup", "list", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath) + cli.BuildCommandTree(cmd) + + cmd.SetErr(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + result := suite.dpnd.recorder.String() + + // as an offhand check: the result should contain the m365 teamschat id + assert.True(t, strings.HasSuffix(result, "No backups available\n")) +} + +// --------------------------------------------------------------------------- +// tests with no prior backup +// --------------------------------------------------------------------------- + +type BackupTeamsChatsE2ESuite struct { + tester.Suite + dpnd dependencies + its intgTesterSetup +} + +func TestBackupTeamsChatsE2ESuite(t *testing.T) { + suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite( + t, + [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})}) +} + +func (suite *BackupTeamsChatsE2ESuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + suite.its = newIntegrationTesterSetup(t) + suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService) +} + +func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_chats() { + runTeamsChatsBackupCategoryTest(suite, flags.DataChats) +} + +func runTeamsChatsBackupCategoryTest(suite *BackupTeamsChatsE2ESuite, category string) { + recorder := strings.Builder{} + recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd, ctx := buildTeamsChatsBackupCmd( + ctx, + suite.dpnd.configFilePath, + suite.its.user.ID, + category, + &recorder) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + result := recorder.String() + t.Log("backup results", result) +} + +func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_teamschatNotFound_chats() { + runTeamsChatsBackupTeamsChatNotFoundTest(suite, flags.DataChats) +} + +func runTeamsChatsBackupTeamsChatNotFoundTest(suite *BackupTeamsChatsE2ESuite, category string) { + recorder := strings.Builder{} + recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd, ctx := buildTeamsChatsBackupCmd( + ctx, + suite.dpnd.configFilePath, + "foo@not-there.com", + category, + &recorder) + + // run the command + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) + assert.Contains( + t, + err.Error(), + "not found", + "error missing user not found") + assert.NotContains(t, err.Error(), "runtime error", "panic happened") + + t.Logf("backup error message: %s", err.Error()) + + result := recorder.String() + t.Log("backup results", result) +} + +func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAzureClientIDFlag() { + t := suite.T() + ctx, flush := tester.NewContext(t) + + defer flush() + + suite.dpnd.recorder.Reset() + + cmd := cliTD.StubRootCmd( + "backup", "create", "chats", + "--teamschat", suite.its.user.ID, + "--azure-client-id", "invalid-value") + cli.BuildCommandTree(cmd) + + cmd.SetErr(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_fromConfigFile() { + t := suite.T() + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + suite.dpnd.recorder.Reset() + + cmd := cliTD.StubRootCmd( + "backup", "create", "chats", + "--teamschat", suite.its.user.ID, + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath) + cli.BuildCommandTree(cmd) + + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) +} + +// AWS flags +func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAWSFlags() { + t := suite.T() + ctx, flush := tester.NewContext(t) + + defer flush() + + suite.dpnd.recorder.Reset() + + cmd := cliTD.StubRootCmd( + "backup", "create", "chats", + "--teamschat", suite.its.user.ID, + "--aws-access-key", "invalid-value", + "--aws-secret-access-key", "some-invalid-value") + cli.BuildCommandTree(cmd) + + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + // since invalid aws creds are explicitly set, should see a failure + require.Error(t, err, clues.ToCore(err)) +} + +// --------------------------------------------------------------------------- +// tests prepared with a previous backup +// --------------------------------------------------------------------------- + +type PreparedBackupTeamsChatsE2ESuite struct { + tester.Suite + dpnd dependencies + backupOps map[path.CategoryType]string + its intgTesterSetup +} + +func TestPreparedBackupTeamsChatsE2ESuite(t *testing.T) { + suite.Run(t, &PreparedBackupTeamsChatsE2ESuite{ + Suite: tester.NewE2ESuite( + t, + [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + suite.its = newIntegrationTesterSetup(t) + suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService) + suite.backupOps = make(map[path.CategoryType]string) + + var ( + teamschats = []string{suite.its.user.ID} + ins = idname.NewCache(map[string]string{suite.its.user.ID: suite.its.user.ID}) + cats = []path.CategoryType{ + path.ChatsCategory, + } + ) + + for _, set := range cats { + var ( + sel = selectors.NewTeamsChatsBackup(teamschats) + scopes []selectors.TeamsChatsScope + ) + + switch set { + case path.ChatsCategory: + scopes = selTD.TeamsChatsBackupChatScope(sel) + } + + sel.Include(scopes) + + bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins) + require.NoError(t, err, clues.ToCore(err)) + + err = bop.Run(ctx) + require.NoError(t, err, clues.ToCore(err)) + + bIDs := string(bop.Results.BackupID) + + // sanity check, ensure we can find the backup and its details immediately + b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID)) + require.NoError(t, err, "retrieving recent backup by ID") + require.Equal(t, bIDs, string(b.ID), "repo backup matches results id") + + _, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs) + require.NoError(t, errs.Failure(), "retrieving recent backup details by ID") + require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID") + require.Equal(t, bIDs, string(b.ID), "repo details matches results id") + + suite.backupOps[set] = string(b.ID) + } +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_chats() { + runTeamsChatsListCmdTest(suite, path.ChatsCategory) +} + +func runTeamsChatsListCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) { + suite.dpnd.recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "list", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath) + cli.BuildCommandTree(cmd) + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // compare the output + result := suite.dpnd.recorder.String() + assert.Contains(t, result, suite.backupOps[category]) +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_singleID_chats() { + runTeamsChatsListSingleCmdTest(suite, path.ChatsCategory) +} + +func runTeamsChatsListSingleCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) { + suite.dpnd.recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + bID := suite.backupOps[category] + + cmd := cliTD.StubRootCmd( + "backup", "list", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--backup", string(bID)) + cli.BuildCommandTree(cmd) + + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // compare the output + result := suite.dpnd.recorder.String() + assert.Contains(t, result, bID) +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_badID() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "list", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--backup", "smarfs") + cli.BuildCommandTree(cmd) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsDetailsCmd_chats() { + runTeamsChatsDetailsCmdTest(suite, path.ChatsCategory) +} + +func runTeamsChatsDetailsCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) { + suite.dpnd.recorder.Reset() + + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + bID := suite.backupOps[category] + + // fetch the details from the repo first + deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID)) + require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure())) + require.Empty(t, errs.Recovered()) + + cmd := cliTD.StubRootCmd( + "backup", "details", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--"+flags.BackupFN, string(bID)) + cli.BuildCommandTree(cmd) + cmd.SetOut(&suite.dpnd.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // compare the output + result := suite.dpnd.recorder.String() + + i := 0 + foundFolders := 0 + + for _, ent := range deets.Entries { + // Skip folders as they don't mean anything to the end teamschat. + if ent.Folder != nil { + foundFolders++ + continue + } + + suite.Run(fmt.Sprintf("detail %d", i), func() { + assert.Contains(suite.T(), result, ent.ShortRef) + }) + + i++ + } + + // We only backup the default folder for each category so there should be at + // least that folder (we don't make details entries for prefix folders). + assert.GreaterOrEqual(t, foundFolders, 1) +} + +// --------------------------------------------------------------------------- +// tests for deleting backups +// --------------------------------------------------------------------------- + +type BackupDeleteTeamsChatsE2ESuite struct { + tester.Suite + dpnd dependencies + backupOps [3]operations.BackupOperation +} + +func TestBackupDeleteTeamsChatsE2ESuite(t *testing.T) { + suite.Run(t, &BackupDeleteTeamsChatsE2ESuite{ + Suite: tester.NewE2ESuite( + t, + [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService) + + m365TeamsChatID := tconfig.M365TeamID(t) + teamschats := []string{m365TeamsChatID} + + // some tests require an existing backup + sel := selectors.NewTeamsChatsBackup(teamschats) + sel.Include(selTD.TeamsChatsBackupChatScope(sel)) + + for i := 0; i < cap(suite.backupOps); i++ { + backupOp, err := suite.dpnd.repo.NewBackup(ctx, sel.Selector) + require.NoError(t, err, clues.ToCore(err)) + + suite.backupOps[i] = backupOp + + err = suite.backupOps[i].Run(ctx) + require.NoError(t, err, clues.ToCore(err)) + } +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "delete", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--"+flags.BackupIDsFN, + fmt.Sprintf("%s,%s", + string(suite.backupOps[0].Results.BackupID), + string(suite.backupOps[1].Results.BackupID))) + cli.BuildCommandTree(cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // a follow-up details call should fail, due to the backup ID being deleted + cmd = cliTD.StubRootCmd( + "backup", "details", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--backups", string(suite.backupOps[0].Results.BackupID)) + cli.BuildCommandTree(cmd) + + err = cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_SingleID() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "delete", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--"+flags.BackupFN, + string(suite.backupOps[2].Results.BackupID)) + cli.BuildCommandTree(cmd) + + // run the command + err := cmd.ExecuteContext(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // a follow-up details call should fail, due to the backup ID being deleted + cmd = cliTD.StubRootCmd( + "backup", "details", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--backup", string(suite.backupOps[2].Results.BackupID)) + cli.BuildCommandTree(cmd) + + err = cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_UnknownID() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "delete", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath, + "--"+flags.BackupIDsFN, uuid.NewString()) + cli.BuildCommandTree(cmd) + + // unknown backupIDs should error since the modelStore can't find the backup + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_NoBackupID() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + ctx = config.SetViper(ctx, suite.dpnd.vpr) + + defer flush() + + cmd := cliTD.StubRootCmd( + "backup", "delete", "chats", + "--"+flags.ConfigFileFN, suite.dpnd.configFilePath) + cli.BuildCommandTree(cmd) + + // empty backupIDs should error since no data provided + err := cmd.ExecuteContext(ctx) + require.Error(t, err, clues.ToCore(err)) +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +func buildTeamsChatsBackupCmd( + ctx context.Context, + configFile, resource, category string, + recorder *strings.Builder, +) (*cobra.Command, context.Context) { + cmd := cliTD.StubRootCmd( + "backup", "create", "chats", + "--"+flags.ConfigFileFN, configFile, + "--"+flags.UserFN, resource, + "--"+flags.CategoryDataFN, category) + cli.BuildCommandTree(cmd) + cmd.SetOut(recorder) + + return cmd, print.SetRootCmd(ctx, cmd) +} diff --git a/src/cli/backup/teamschats_test.go b/src/cli/backup/teamschats_test.go new file mode 100644 index 0000000000..177cc29e7d --- /dev/null +++ b/src/cli/backup/teamschats_test.go @@ -0,0 +1,248 @@ +package backup + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + flagsTD "github.com/alcionai/corso/src/cli/flags/testdata" + cliTD "github.com/alcionai/corso/src/cli/testdata" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" +) + +type TeamsChatsUnitSuite struct { + tester.Suite +} + +func TestTeamsChatsUnitSuite(t *testing.T) { + suite.Run(t, &TeamsChatsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *TeamsChatsUnitSuite) TestAddTeamsChatsCommands() { + expectUse := teamschatsServiceCommand + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + { + name: "create teamschats", + use: createCommand, + expectUse: expectUse + " " + teamschatsServiceCommandCreateUseSuffix, + expectShort: teamschatsCreateCmd().Short, + expectRunE: createTeamsChatsCmd, + }, + { + name: "list teamschats", + use: listCommand, + expectUse: expectUse, + expectShort: teamschatsListCmd().Short, + expectRunE: listTeamsChatsCmd, + }, + { + name: "details teamschats", + use: detailsCommand, + expectUse: expectUse + " " + teamschatsServiceCommandDetailsUseSuffix, + expectShort: teamschatsDetailsCmd().Short, + expectRunE: detailsTeamsChatsCmd, + }, + { + name: "delete teamschats", + use: deleteCommand, + expectUse: expectUse + " " + teamschatsServiceCommandDeleteUseSuffix, + expectShort: teamschatsDeleteCmd().Short, + expectRunE: deleteTeamsChatsCmd, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + c := addTeamsChatsCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + }) + } +} + +func (suite *TeamsChatsUnitSuite) TestValidateTeamsChatsBackupCreateFlags() { + table := []struct { + name string + cats []string + expect assert.ErrorAssertionFunc + }{ + { + name: "none", + cats: []string{}, + expect: assert.NoError, + }, + { + name: "chats", + cats: []string{flags.DataChats}, + expect: assert.NoError, + }, + { + name: "all allowed", + cats: []string{ + flags.DataChats, + }, + expect: assert.NoError, + }, + { + name: "bad inputs", + cats: []string{"foo"}, + expect: assert.Error, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + err := validateTeamsChatsBackupCreateFlags([]string{"*"}, test.cats) + test.expect(suite.T(), err, clues.ToCore(err)) + }) + } +} + +func (suite *TeamsChatsUnitSuite) TestBackupCreateFlags() { + t := suite.T() + + cmd := cliTD.SetUpCmdHasFlags( + t, + &cobra.Command{Use: createCommand}, + addTeamsChatsCommands, + []cliTD.UseCobraCommandFn{ + flags.AddAllProviderFlags, + flags.AddAllStorageFlags, + }, + flagsTD.WithFlags( + teamschatsServiceCommand, + []string{ + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.UserFN, flagsTD.FlgInputs(flagsTD.UsersInput), + "--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.TeamsChatsCategoryDataInput), + }, + flagsTD.PreparedGenericBackupFlags(), + flagsTD.PreparedProviderFlags(), + flagsTD.PreparedStorageFlags())) + + opts := utils.MakeTeamsChatsOpts(cmd) + co := utils.Control() + backupOpts := utils.ParseBackupOptions() + + // TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once + // restore flags are switched over too and we no longer parse flags beyond + // connection info into control.Options. + assert.Equal(t, control.FailFast, backupOpts.FailureHandling) + assert.True(t, backupOpts.Incrementals.ForceFullEnumeration) + assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh) + + assert.Equal(t, control.FailFast, co.FailureHandling) + assert.True(t, co.ToggleFeatures.DisableIncrementals) + assert.True(t, co.ToggleFeatures.ForceItemDataDownload) + + assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users) + flagsTD.AssertGenericBackupFlags(t, cmd) + flagsTD.AssertProviderFlags(t, cmd) + flagsTD.AssertStorageFlags(t, cmd) +} + +func (suite *TeamsChatsUnitSuite) TestBackupListFlags() { + t := suite.T() + + cmd := cliTD.SetUpCmdHasFlags( + t, + &cobra.Command{Use: listCommand}, + addTeamsChatsCommands, + []cliTD.UseCobraCommandFn{ + flags.AddAllProviderFlags, + flags.AddAllStorageFlags, + }, + flagsTD.WithFlags( + teamschatsServiceCommand, + []string{ + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, flagsTD.BackupInput, + }, + flagsTD.PreparedBackupListFlags(), + flagsTD.PreparedProviderFlags(), + flagsTD.PreparedStorageFlags())) + + assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV) + flagsTD.AssertBackupListFlags(t, cmd) + flagsTD.AssertProviderFlags(t, cmd) + flagsTD.AssertStorageFlags(t, cmd) +} + +func (suite *TeamsChatsUnitSuite) TestBackupDetailsFlags() { + t := suite.T() + + cmd := cliTD.SetUpCmdHasFlags( + t, + &cobra.Command{Use: detailsCommand}, + addTeamsChatsCommands, + []cliTD.UseCobraCommandFn{ + flags.AddAllProviderFlags, + flags.AddAllStorageFlags, + }, + flagsTD.WithFlags( + teamschatsServiceCommand, + []string{ + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, flagsTD.BackupInput, + "--" + flags.SkipReduceFN, + }, + flagsTD.PreparedTeamsChatsFlags(), + flagsTD.PreparedProviderFlags(), + flagsTD.PreparedStorageFlags())) + + co := utils.Control() + + assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV) + assert.True(t, co.SkipReduce) + flagsTD.AssertProviderFlags(t, cmd) + flagsTD.AssertStorageFlags(t, cmd) + flagsTD.AssertTeamsChatsFlags(t, cmd) +} + +func (suite *TeamsChatsUnitSuite) TestBackupDeleteFlags() { + t := suite.T() + + cmd := cliTD.SetUpCmdHasFlags( + t, + &cobra.Command{Use: deleteCommand}, + addTeamsChatsCommands, + []cliTD.UseCobraCommandFn{ + flags.AddAllProviderFlags, + flags.AddAllStorageFlags, + }, + flagsTD.WithFlags( + teamschatsServiceCommand, + []string{ + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, flagsTD.BackupInput, + }, + flagsTD.PreparedProviderFlags(), + flagsTD.PreparedStorageFlags())) + + assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV) + flagsTD.AssertProviderFlags(t, cmd) + flagsTD.AssertStorageFlags(t, cmd) +} diff --git a/src/cli/flags/teamschats.go b/src/cli/flags/teamschats.go new file mode 100644 index 0000000000..39fde2276f --- /dev/null +++ b/src/cli/flags/teamschats.go @@ -0,0 +1,13 @@ +package flags + +import ( + "github.com/spf13/cobra" +) + +const ( + DataChats = "chats" +) + +func AddTeamsChatsDetailsAndRestoreFlags(cmd *cobra.Command) { + // TODO: add details flags +} diff --git a/src/cli/flags/testdata/flags.go b/src/cli/flags/testdata/flags.go index 26ea8085be..1fe8a3c1da 100644 --- a/src/cli/flags/testdata/flags.go +++ b/src/cli/flags/testdata/flags.go @@ -21,6 +21,7 @@ var ( ExchangeCategoryDataInput = []string{"email", "events", "contacts"} SharepointCategoryDataInput = []string{"files", "lists", "pages"} GroupsCategoryDataInput = []string{"files", "lists", "pages", "messages"} + TeamsChatsCategoryDataInput = []string{"chats"} ChannelInput = []string{"channel1", "channel2"} MessageInput = []string{"message1", "message2"} diff --git a/src/cli/flags/testdata/teamschats.go b/src/cli/flags/testdata/teamschats.go new file mode 100644 index 0000000000..9203ad5864 --- /dev/null +++ b/src/cli/flags/testdata/teamschats.go @@ -0,0 +1,25 @@ +package testdata + +import ( + "testing" + + "github.com/spf13/cobra" +) + +func PreparedTeamsChatsFlags() []string { + return []string{ + // FIXME: populate when adding filters + // "--" + flags.ChatCreatedAfterFN, ChatCreatedAfterInput, + // "--" + flags.ChatCreatedBeforeFN, ChatCreatedBeforeInput, + // "--" + flags.ChatLastMessageAfterFN, ChatLastMessageAfterInput, + // "--" + flags.ChatLastMessageBeforeFN, ChatLastMessageBeforeInput, + } +} + +func AssertTeamsChatsFlags(t *testing.T, cmd *cobra.Command) { + // FIXME: populate when adding filters + // assert.Equal(t, ChatCreatedAfterInput, flags.ChatCreatedAfterFV) + // assert.Equal(t, ChatCreatedBeforeInput, flags.ChatCreatedBeforeFV) + // assert.Equal(t, ChatLastMessageAfterInput, flags.ChatLastMessageAfterFV) + // assert.Equal(t, ChatLastMessageBeforeInput, flags.ChatLastMessageBeforeFV) +} diff --git a/src/cli/utils/teamschats.go b/src/cli/utils/teamschats.go new file mode 100644 index 0000000000..1467949a33 --- /dev/null +++ b/src/cli/utils/teamschats.go @@ -0,0 +1,101 @@ +package utils + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/pkg/selectors" +) + +type TeamsChatsOpts struct { + Users []string + + ExportCfg ExportCfgOpts + + Populated flags.PopulatedFlags +} + +func TeamsChatsAllowedCategories() map[string]struct{} { + return map[string]struct{}{ + flags.DataChats: {}, + } +} + +func AddTeamsChatsCategories(sel *selectors.TeamsChatsBackup, cats []string) *selectors.TeamsChatsBackup { + if len(cats) == 0 { + sel.Include(sel.AllData()) + } + + for _, d := range cats { + switch d { + case flags.DataChats: + sel.Include(sel.Chats(selectors.Any())) + } + } + + return sel +} + +func MakeTeamsChatsOpts(cmd *cobra.Command) TeamsChatsOpts { + return TeamsChatsOpts{ + Users: flags.UserFV, + + ExportCfg: makeExportCfgOpts(cmd), + + // populated contains the list of flags that appear in the + // command, according to pflags. Use this to differentiate + // between an "empty" and a "missing" value. + Populated: flags.GetPopulatedFlags(cmd), + } +} + +// ValidateTeamsChatsRestoreFlags checks common flags for correctness and interdependencies +func ValidateTeamsChatsRestoreFlags(backupID string, opts TeamsChatsOpts, isRestore bool) error { + if len(backupID) == 0 { + return clues.New("a backup ID is required") + } + + // restore isn't currently supported + if isRestore { + return clues.New("restore not supported") + } + + return nil +} + +// AddTeamsChatsFilter adds the scope of the provided values to the selector's +// filter set +func AddTeamsChatsFilter( + sel *selectors.TeamsChatsRestore, + v string, + f func(string) []selectors.TeamsChatsScope, +) { + if len(v) == 0 { + return + } + + sel.Filter(f(v)) +} + +// IncludeTeamsChatsRestoreDataSelectors builds the common data-selector +// inclusions for teamschats commands. +func IncludeTeamsChatsRestoreDataSelectors(ctx context.Context, opts TeamsChatsOpts) *selectors.TeamsChatsRestore { + users := opts.Users + + if len(opts.Users) == 0 { + users = selectors.Any() + } + + return selectors.NewTeamsChatsRestore(users) +} + +// FilterTeamsChatsRestoreInfoSelectors builds the common info-selector filters. +func FilterTeamsChatsRestoreInfoSelectors( + sel *selectors.TeamsChatsRestore, + opts TeamsChatsOpts, +) { + // TODO: populate when adding filters +} diff --git a/src/internal/m365/backup.go b/src/internal/m365/backup.go index b6bab41202..5ea76dd4c7 100644 --- a/src/internal/m365/backup.go +++ b/src/internal/m365/backup.go @@ -13,7 +13,10 @@ import ( "github.com/alcionai/corso/src/internal/m365/service/groups" "github.com/alcionai/corso/src/internal/m365/service/onedrive" "github.com/alcionai/corso/src/internal/m365/service/sharepoint" + "github.com/alcionai/corso/src/internal/m365/service/teamschats" + "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/pkg/account" bupMD "github.com/alcionai/corso/src/pkg/backup/metadata" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" @@ -22,9 +25,33 @@ import ( "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) +type backupHandler interface { + produceBackupCollectionser +} + +type produceBackupCollectionser interface { + ProduceBackupCollections( + ctx context.Context, + bpc inject.BackupProducerConfig, + ac api.Client, + creds account.M365Config, + su support.StatusUpdater, + counter *count.Bus, + errs *fault.Bus, + ) ( + collections []data.BackupCollection, + excludeItems *prefixmatcher.StringSetMatcher, + // canUsePreviousBacukp can be always returned true for impelementations + // that always return a tombstone collection when the metadata read fails + canUsePreviousBackup bool, + err error, + ) +} + // --------------------------------------------------------------------------- // Data Collections // --------------------------------------------------------------------------- @@ -63,67 +90,40 @@ func (ctrl *Controller) ProduceBackupCollections( canUsePreviousBackup bool ) + var handler backupHandler + switch service { case path.ExchangeService: - colls, excludeItems, canUsePreviousBackup, err = exchange.ProduceBackupCollections( - ctx, - bpc, - ctrl.AC, - ctrl.credentials, - ctrl.UpdateStatus, - counter, - errs) - if err != nil { - return nil, nil, false, err - } + handler = exchange.NewBackup() case path.OneDriveService: - colls, excludeItems, canUsePreviousBackup, err = onedrive.ProduceBackupCollections( - ctx, - bpc, - ctrl.AC, - ctrl.credentials, - ctrl.UpdateStatus, - counter, - errs) - if err != nil { - return nil, nil, false, err - } + handler = onedrive.NewBackup() case path.SharePointService: - colls, excludeItems, canUsePreviousBackup, err = sharepoint.ProduceBackupCollections( - ctx, - bpc, - ctrl.AC, - ctrl.credentials, - ctrl.UpdateStatus, - counter, - errs) - if err != nil { - return nil, nil, false, err - } + handler = sharepoint.NewBackup() case path.GroupsService: - colls, excludeItems, err = groups.ProduceBackupCollections( - ctx, - bpc, - ctrl.AC, - ctrl.credentials, - ctrl.UpdateStatus, - counter, - errs) - if err != nil { - return nil, nil, false, err - } + handler = groups.NewBackup() - // canUsePreviousBacukp can be always returned true for groups as we - // return a tombstone collection in case the metadata read fails - canUsePreviousBackup = true + case path.TeamsChatsService: + handler = teamschats.NewBackup() default: return nil, nil, false, clues.Wrap(clues.NewWC(ctx, service.String()), "service not supported") } + colls, excludeItems, canUsePreviousBackup, err = handler.ProduceBackupCollections( + ctx, + bpc, + ctrl.AC, + ctrl.credentials, + ctrl.UpdateStatus, + counter, + errs) + if err != nil { + return nil, nil, false, err + } + for _, c := range colls { // kopia doesn't stream Items() from deleted collections, // and so they never end up calling the UpdateStatus closer. @@ -153,25 +153,27 @@ func (ctrl *Controller) IsServiceEnabled( return sharepoint.IsServiceEnabled(ctx, ctrl.AC.Sites(), resourceOwner) case path.GroupsService: return groups.IsServiceEnabled(ctx, ctrl.AC.Groups(), resourceOwner) + case path.TeamsChatsService: + return teamschats.IsServiceEnabled(ctx, ctrl.AC.Users(), resourceOwner) } return false, clues.Wrap(clues.NewWC(ctx, service.String()), "service not supported") } -func verifyBackupInputs(sels selectors.Selector, cachedIDs []string) error { +func verifyBackupInputs(sel selectors.Selector, cachedIDs []string) error { var ids []string - switch sels.Service { + switch sel.Service { case selectors.ServiceExchange, selectors.ServiceOneDrive: // Exchange and OneDrive user existence now checked in checkServiceEnabled. return nil - case selectors.ServiceSharePoint, selectors.ServiceGroups: + case selectors.ServiceSharePoint, selectors.ServiceGroups, selectors.ServiceTeamsChats: ids = cachedIDs } - if !filters.Contains(ids).Compare(sels.ID()) { - return clues.Stack(core.ErrNotFound).With("selector_protected_resource", sels.DiscreteOwner) + if !filters.Contains(ids).Compare(sel.ID()) { + return clues.Stack(core.ErrNotFound).With("selector_protected_resource", sel.ID()) } return nil diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index 5df7d5fff4..2a95edda10 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -139,7 +139,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() { Selector: sel, } - collections, excludes, canUsePreviousBackup, err := exchange.ProduceBackupCollections( + collections, excludes, canUsePreviousBackup, err := exchange.NewBackup().ProduceBackupCollections( ctx, bpc, suite.ac, @@ -309,7 +309,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { Selector: sel, } - collections, excludes, canUsePreviousBackup, err := sharepoint.ProduceBackupCollections( + collections, excludes, canUsePreviousBackup, err := sharepoint.NewBackup().ProduceBackupCollections( ctx, bpc, suite.ac, diff --git a/src/internal/m365/collection/teamsChats/backup.go b/src/internal/m365/collection/teamsChats/backup.go index 5908dbc6ea..2a464d322c 100644 --- a/src/internal/m365/collection/teamsChats/backup.go +++ b/src/internal/m365/collection/teamsChats/backup.go @@ -131,9 +131,10 @@ func populateCollection[I chatsItemer]( for _, item := range items { if !bh.includeItem(item, scope) { cl.Inc(count.SkippedItems) - } else { - includedItems = append(includedItems, item) + continue } + + includedItems = append(includedItems, item) } cl.Add(count.ItemsAdded, int64(len(includedItems))) @@ -159,8 +160,7 @@ func populateCollection[I chatsItemer]( qp.ProtectedResource.ID(), includedItems, container, - statusUpdater, - useLazyReader) + statusUpdater) return collection, clues.Stack(errs.Failure()).OrNil() } diff --git a/src/internal/m365/collection/teamsChats/chat_handler.go b/src/internal/m365/collection/teamsChats/chat_handler.go index a6ad0341ae..93ef8316a7 100644 --- a/src/internal/m365/collection/teamsChats/chat_handler.go +++ b/src/internal/m365/collection/teamsChats/chat_handler.go @@ -58,6 +58,13 @@ func (bh usersChatsBackupHandler) includeItem( ch models.Chatable, scope selectors.TeamsChatsScope, ) bool { + // corner case: many Topics are empty, and empty inputs are automatically + // set to non-matching in the selectors code. This allows us to include + // everything without needing to check the topic value in that case. + if scope.IsAny(selectors.TeamsChatsChat) { + return true + } + return scope.Matches(selectors.TeamsChatsChat, ptr.Val(ch.GetTopic())) } diff --git a/src/internal/m365/collection/teamsChats/collection.go b/src/internal/m365/collection/teamsChats/collection.go index d9bc75ff1a..a7f25c6011 100644 --- a/src/internal/m365/collection/teamsChats/collection.go +++ b/src/internal/m365/collection/teamsChats/collection.go @@ -18,7 +18,6 @@ import ( "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/errs/core" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" @@ -26,10 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -var ( - _ data.BackupCollection = &prefetchCollection[chatsItemer]{} - _ data.BackupCollection = &lazyFetchCollection[chatsItemer]{} -) +var _ data.BackupCollection = &lazyFetchCollection[chatsItemer]{} const ( collectionChannelBufferSize = 1000 @@ -63,24 +59,6 @@ func updateStatus( statusUpdater(status) } -// ----------------------------------------------------------------------------- -// prefetchCollection -// ----------------------------------------------------------------------------- - -type prefetchCollection[I chatsItemer] struct { - data.BaseCollection - protectedResource string - stream chan data.Item - - contains container[I] - - items []I - - getAndAugment getItemAndAugmentInfoer[I] - - statusUpdater support.StatusUpdater -} - // State of the collection is set as an observation of the current // and previous paths. If the curr path is nil, the state is assumed // to be deleted. If the prev path is nil, it is assumed newly created. @@ -93,21 +71,8 @@ func NewCollection[I chatsItemer]( items []I, contains container[I], statusUpdater support.StatusUpdater, - useLazyReader bool, ) data.BackupCollection { - if useLazyReader { - return &lazyFetchCollection[I]{ - BaseCollection: baseCol, - items: items, - contains: contains, - getAndAugment: getAndAugment, - statusUpdater: statusUpdater, - stream: make(chan data.Item, collectionChannelBufferSize), - protectedResource: protectedResource, - } - } - - return &prefetchCollection[I]{ + return &lazyFetchCollection[I]{ BaseCollection: baseCol, items: items, contains: contains, @@ -118,128 +83,6 @@ func NewCollection[I chatsItemer]( } } -func (col *prefetchCollection[I]) Items(ctx context.Context, errs *fault.Bus) <-chan data.Item { - go col.streamItems(ctx, errs) - return col.stream -} - -func (col *prefetchCollection[I]) streamItems(ctx context.Context, errs *fault.Bus) { - var ( - streamedItems int64 - totalBytes int64 - wg sync.WaitGroup - progressMessage chan<- struct{} - el = errs.Local() - ) - - ctx = clues.Add(ctx, "category", col.Category().String()) - - defer func() { - close(col.stream) - logger.Ctx(ctx).Infow( - "finished stream backup collection items", - "stats", col.Counter.Values()) - - updateStatus( - ctx, - col.statusUpdater, - len(col.items), - streamedItems, - totalBytes, - col.FullPath().Folder(false), - errs.Failure()) - }() - - if len(col.items) > 0 { - progressMessage = observe.CollectionProgress( - ctx, - col.Category().HumanString(), - col.LocationPath().Elements()) - defer close(progressMessage) - } - - semaphoreCh := make(chan struct{}, col.Opts().Parallelism.ItemFetch) - defer close(semaphoreCh) - - // add any new items - for _, item := range col.items { - if el.Failure() != nil { - break - } - - wg.Add(1) - semaphoreCh <- struct{}{} - - itemID := ptr.Val(item.GetId()) - - go func(id string) { - defer wg.Done() - defer func() { <-semaphoreCh }() - - writer := kjson.NewJsonSerializationWriter() - defer writer.Close() - - item, info, err := col.getAndAugment.getItem( - ctx, - col.protectedResource, - id) - if err != nil { - err = clues.Wrap(err, "getting channel message data").Label(fault.LabelForceNoBackupCreation) - el.AddRecoverable(ctx, err) - - return - } - - col.getAndAugment.augmentItemInfo(info, col.contains.container) - - if err := writer.WriteObjectValue("", item); err != nil { - err = clues.Wrap(err, "writing channel message to serializer").Label(fault.LabelForceNoBackupCreation) - el.AddRecoverable(ctx, err) - - return - } - - itemData, err := writer.GetSerializedContent() - if err != nil { - err = clues.Wrap(err, "serializing channel message").Label(fault.LabelForceNoBackupCreation) - el.AddRecoverable(ctx, err) - - return - } - - info.ParentPath = col.LocationPath().String() - - storeItem, err := data.NewPrefetchedItemWithInfo( - io.NopCloser(bytes.NewReader(itemData)), - id, - details.ItemInfo{TeamsChats: info}) - if err != nil { - err := clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation) - el.AddRecoverable(ctx, err) - - return - } - - col.stream <- storeItem - - atomic.AddInt64(&streamedItems, 1) - atomic.AddInt64(&totalBytes, int64(info.Chat.MessageCount)) - - if col.Counter.Inc(count.StreamItemsAdded)%1000 == 0 { - logger.Ctx(ctx).Infow("item stream progress", "stats", col.Counter.Values()) - } - - col.Counter.Add(count.StreamBytesAdded, int64(info.Chat.MessageCount)) - - if progressMessage != nil { - progressMessage <- struct{}{} - } - }(itemID) - } - - wg.Wait() -} - // ----------------------------------------------------------------------------- // lazyFetchCollection // ----------------------------------------------------------------------------- diff --git a/src/internal/m365/collection/teamsChats/collection_test.go b/src/internal/m365/collection/teamsChats/collection_test.go index cda6908a7b..15b0dac778 100644 --- a/src/internal/m365/collection/teamsChats/collection_test.go +++ b/src/internal/m365/collection/teamsChats/collection_test.go @@ -133,14 +133,13 @@ func (suite *CollectionUnitSuite) TestNewCollection_state() { "g", nil, container[models.Chatable]{}, - nil, - false) + nil) assert.Equal(t, test.expect, c.State(), "collection state") assert.Equal(t, test.curr, c.FullPath(), "full path") assert.Equal(t, test.prev, c.PreviousPath(), "prev path") - prefetch, ok := c.(*prefetchCollection[models.Chatable]) + prefetch, ok := c.(*lazyFetchCollection[models.Chatable]) require.True(t, ok, "collection type") assert.Equal(t, test.loc, prefetch.LocationPath(), "location path") @@ -170,79 +169,6 @@ func (getAndAugmentChat) augmentItemInfo(*details.TeamsChatsInfo, models.Chatabl // no-op } -func (suite *CollectionUnitSuite) TestPrefetchCollection_streamItems() { - var ( - t = suite.T() - statusUpdater = func(*support.ControllerOperationStatus) {} - ) - - fullPath, err := path.Build("t", "pr", path.TeamsChatsService, path.ChatsCategory, false, "fnords", "smarf") - require.NoError(t, err, clues.ToCore(err)) - - locPath, err := path.Build("t", "pr", path.TeamsChatsService, path.ChatsCategory, false, "fnords", "smarf") - require.NoError(t, err, clues.ToCore(err)) - - table := []struct { - name string - items []models.Chatable - }{ - { - name: "no items", - }, - { - name: "items", - items: testdata.StubChats("fisher", "flannigan", "fitzbog"), - }, - } - for _, test := range table { - suite.Run(test.name, func() { - var ( - t = suite.T() - errs = fault.New(true) - itemCount int - ) - - ctx, flush := tester.NewContext(t) - defer flush() - - col := &prefetchCollection[models.Chatable]{ - BaseCollection: data.NewBaseCollection( - fullPath, - nil, - locPath.ToBuilder(), - control.DefaultOptions(), - false, - count.New()), - items: test.items, - contains: container[models.Chatable]{}, - getAndAugment: getAndAugmentChat{}, - stream: make(chan data.Item), - statusUpdater: statusUpdater, - } - - go col.streamItems(ctx, errs) - - for item := range col.stream { - itemCount++ - - ok := slices.ContainsFunc(test.items, func(mc models.Chatable) bool { - return ptr.Val(mc.GetId()) == item.ID() - }) - - require.True(t, ok, "item must be either added or removed: %q", item.ID()) - assert.False(t, item.Deleted(), "additions should not be marked as deleted") - } - - assert.NoError(t, errs.Failure()) - assert.Equal( - t, - len(test.items), - itemCount, - "should see all expected items") - }) - } -} - func (suite *CollectionUnitSuite) TestLazyFetchCollection_Items_LazyFetch() { var ( t = suite.T() diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index 4057b24240..536f24f786 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -112,7 +112,7 @@ func (ctrl *Controller) setResourceHandler( var rh *resourceGetter switch serviceInOperation { - case path.ExchangeService, path.OneDriveService: + case path.ExchangeService, path.OneDriveService, path.TeamsChatsService: rh = &resourceGetter{ enum: resource.Users, getter: ctrl.AC.Users(), diff --git a/src/internal/m365/service/exchange/backup.go b/src/internal/m365/service/exchange/backup.go index 879160375c..999b2441dd 100644 --- a/src/internal/m365/service/exchange/backup.go +++ b/src/internal/m365/service/exchange/backup.go @@ -19,9 +19,17 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) +type exchangeBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *exchangeBackup { + return &exchangeBackup{} +} + // ProduceBackupCollections returns a DataCollection which the caller can // use to read mailbox data out for the specified user -func ProduceBackupCollections( +func (exchangeBackup) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, ac api.Client, diff --git a/src/internal/m365/service/groups/backup.go b/src/internal/m365/service/groups/backup.go index 07b1e1adad..9f5ac8e85c 100644 --- a/src/internal/m365/service/groups/backup.go +++ b/src/internal/m365/service/groups/backup.go @@ -34,7 +34,15 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -func ProduceBackupCollections( +type groupsBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *groupsBackup { + return &groupsBackup{} +} + +func (groupsBackup) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, ac api.Client, @@ -42,10 +50,10 @@ func ProduceBackupCollections( su support.StatusUpdater, counter *count.Bus, errs *fault.Bus, -) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, error) { +) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) { b, err := bpc.Selector.ToGroupsBackup() if err != nil { - return nil, nil, clues.Wrap(err, "groupsDataCollection: parsing selector") + return nil, nil, true, clues.Wrap(err, "groupsDataCollection: parsing selector") } var ( @@ -66,7 +74,7 @@ func ProduceBackupCollections( bpc.ProtectedResource.ID(), api.CallConfig{}) if err != nil { - return nil, nil, clues.WrapWC(ctx, err, "getting group") + return nil, nil, true, clues.WrapWC(ctx, err, "getting group") } bc := backupCommon{ac, bpc, creds, group, sitesPreviousPaths, su} @@ -129,7 +137,7 @@ func ProduceBackupCollections( counter, errs) if err != nil { - return nil, nil, err + return nil, nil, true, err } collections = append(collections, baseCols...) @@ -143,7 +151,7 @@ func ProduceBackupCollections( su, counter) if err != nil { - return nil, nil, err + return nil, nil, true, err } collections = append(collections, md) @@ -152,7 +160,7 @@ func ProduceBackupCollections( logger.Ctx(ctx).Infow("produced collections", "stats", counter.Values()) - return collections, globalItemIDExclusions.ToReader(), el.Failure() + return collections, globalItemIDExclusions.ToReader(), true, el.Failure() } type backupCommon struct { diff --git a/src/internal/m365/service/onedrive/backup.go b/src/internal/m365/service/onedrive/backup.go index 0c19020ed7..ece8c21a66 100644 --- a/src/internal/m365/service/onedrive/backup.go +++ b/src/internal/m365/service/onedrive/backup.go @@ -22,7 +22,15 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -func ProduceBackupCollections( +type oneDriveBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *oneDriveBackup { + return &oneDriveBackup{} +} + +func (oneDriveBackup) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, ac api.Client, diff --git a/src/internal/m365/service/sharepoint/backup.go b/src/internal/m365/service/sharepoint/backup.go index 9cebb74a34..46706a180b 100644 --- a/src/internal/m365/service/sharepoint/backup.go +++ b/src/internal/m365/service/sharepoint/backup.go @@ -20,7 +20,15 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -func ProduceBackupCollections( +type sharePointBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *sharePointBackup { + return &sharePointBackup{} +} + +func (sharePointBackup) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, ac api.Client, diff --git a/src/internal/m365/service/teamschats/backup.go b/src/internal/m365/service/teamschats/backup.go index 88ce9ac307..da17cc8f43 100644 --- a/src/internal/m365/service/teamschats/backup.go +++ b/src/internal/m365/service/teamschats/backup.go @@ -22,7 +22,15 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -func ProduceBackupCollections( +type teamsChatsBackup struct{} + +// NewBackup provides a struct that matches standard apis +// across m365/service handlers. +func NewBackup() *teamsChatsBackup { + return &teamsChatsBackup{} +} + +func (teamsChatsBackup) ProduceBackupCollections( ctx context.Context, bpc inject.BackupProducerConfig, ac api.Client, @@ -30,10 +38,10 @@ func ProduceBackupCollections( su support.StatusUpdater, counter *count.Bus, errs *fault.Bus, -) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, error) { +) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) { b, err := bpc.Selector.ToTeamsChatsBackup() if err != nil { - return nil, nil, clues.WrapWC(ctx, err, "parsing selector") + return nil, nil, true, clues.WrapWC(ctx, err, "parsing selector") } var ( @@ -98,7 +106,7 @@ func ProduceBackupCollections( counter, errs) if err != nil { - return nil, nil, err + return nil, nil, true, err } collections = append(collections, baseCols...) @@ -108,7 +116,7 @@ func ProduceBackupCollections( logger.Ctx(ctx).Infow("produced collections", "stats", counter.Values()) - return collections, nil, clues.Stack(el.Failure()).OrNil() + return collections, nil, true, clues.Stack(el.Failure()).OrNil() } type backupCommon struct { @@ -126,9 +134,7 @@ func backupChats( counter *count.Bus, errs *fault.Bus, ) ([]data.BackupCollection, error) { - var ( - colls []data.BackupCollection - ) + var colls []data.BackupCollection progressMessage := observe.MessageWithCompletion( ctx, diff --git a/src/internal/m365/service/teamschats/enabled.go b/src/internal/m365/service/teamschats/enabled.go index c97417bda5..a4613feab4 100644 --- a/src/internal/m365/service/teamschats/enabled.go +++ b/src/internal/m365/service/teamschats/enabled.go @@ -3,7 +3,6 @@ package teamschats import ( "context" - "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -14,5 +13,6 @@ func IsServiceEnabled( gbi api.GetByIDer[models.Userable], resource string, ) (bool, error) { - return true, clues.New("needs implementation") + // TODO(rkeepers): investgate service enablement checks + return true, nil } diff --git a/src/internal/m365/service/teamschats/enabled_test.go b/src/internal/m365/service/teamschats/enabled_test.go index 820591e3a2..6c79549aa1 100644 --- a/src/internal/m365/service/teamschats/enabled_test.go +++ b/src/internal/m365/service/teamschats/enabled_test.go @@ -49,7 +49,7 @@ func (suite *EnabledUnitSuite) TestIsServiceEnabled() { return mockGU{} }, expect: assert.True, - expectErr: assert.Error, + expectErr: assert.NoError, }, } for _, test := range table { diff --git a/src/pkg/path/service_category_test.go b/src/pkg/path/service_category_test.go index ce58d61c33..21b1684cb3 100644 --- a/src/pkg/path/service_category_test.go +++ b/src/pkg/path/service_category_test.go @@ -163,9 +163,10 @@ func (suite *ServiceCategoryUnitSuite) TestToServiceType() { } for _, test := range table { suite.Run(test.name, func() { - t := suite.T() - - assert.Equal(t, test.expected, ToServiceType(test.service)) + assert.Equal( + suite.T(), + test.expected.String(), + ToServiceType(test.service).String()) }) } } @@ -189,9 +190,10 @@ func (suite *ServiceCategoryUnitSuite) TestToCategoryType() { } for _, test := range table { suite.Run(test.name, func() { - t := suite.T() - - assert.Equal(t, test.expected, ToCategoryType(test.category)) + assert.Equal( + suite.T(), + test.expected.String(), + ToCategoryType(test.category).String()) }) } } diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 4c49f637b2..03654d8d11 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -355,6 +355,9 @@ func selectorAsIface[T any](s Selector) (T, error) { case ServiceGroups: a, err = func() (any, error) { return s.ToGroupsRestore() }() t = a.(T) + case ServiceTeamsChats: + a, err = func() (any, error) { return s.ToTeamsChatsRestore() }() + t = a.(T) default: err = clues.Stack(ErrorUnrecognizedService, clues.New(s.Service.String())) }