feat: support more than 100 clusters (#229)

.gitignore

@@ -39,3 +39,4 @@ _testmain.go
 
 *.log
 dist
+e1s_*

internal/api/cluster.go

@@ -3,24 +3,40 @@ package api
 import (
 	"context"
 	"log/slog"
+	"math"
 	"sort"
 
 	"github.com/aws/aws-sdk-go-v2/service/ecs"
 	"github.com/aws/aws-sdk-go-v2/service/ecs/types"
+	"golang.org/x/sync/errgroup"
 )
 
 // Equivalent to
 // aws ecs list-clusters
 // aws ecs describe-clusters --clusters ${clusters}
 func (store *Store) ListClusters() ([]types.Cluster, error) {
-	limit := int32(100)
-	clustersOutput, err := store.ecs.ListClusters(context.Background(), &ecs.ListClustersInput{
+	batchSize := 100
+	limit := int32(batchSize)
+	clusterARNs := []string{}
+	params := &ecs.ListClustersInput{
 		MaxResults: &limit,
-	})
+	}
 
-	if err != nil {
-		slog.Warn("failed to run aws api to list clusters", "error", err)
-		return []types.Cluster{}, err
+	for {
+		clustersOutput, err := store.ecs.ListClusters(context.Background(), params)
+		if err != nil {
+			slog.Warn("failed to run aws api to list clusters", "error", err)
+			if len(clusterARNs) == 0 {
+				return []types.Cluster{}, err
+			}
+			continue
+		}
+		clusterARNs = append(clusterARNs, clustersOutput.ClusterArns...)
+		if clustersOutput.NextToken != nil {
+			params.NextToken = clustersOutput.NextToken
+		} else {
+			break
+		}
 	}
 
 	include := []types.ClusterField{
@@ -30,20 +46,40 @@ func (store *Store) ListClusters() ([]types.Cluster, error) {
 		types.ClusterFieldStatistics,
 		types.ClusterFieldTags,
 	}
-	describeInput := &ecs.DescribeClustersInput{
-		Clusters: clustersOutput.ClusterArns,
-		Include:  include,
-	}
 
 	results := []types.Cluster{}
+	g := new(errgroup.Group)
 
-	describeClusterOutput, err := store.ecs.DescribeClusters(context.Background(), describeInput)
-	if err != nil {
-		slog.Warn("failed to run aws api to describe clusters", "error", err)
-		return []types.Cluster{}, err
+	clusterCount := len(clusterARNs)
+	loopCount := clusterCount / batchSize
+
+	if clusterCount%batchSize == 0 {
+		loopCount = loopCount - 1
 	}
 
-	results = append(results, describeClusterOutput.Clusters...)
+	for i := 0; i <= loopCount; i++ {
+		i := i
+		g.Go(func() error {
+			clusters := clusterARNs[i*batchSize : int(math.Min(float64((i+1)*batchSize), float64(clusterCount)))]
+
+			// If describe more than 100, InvalidParameterException: Clusters cannot have more than 100 elements
+			describeClusterOutput, err := store.ecs.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{
+				Clusters: clusters,
+				Include:  include,
+			})
+			if err != nil {
+				slog.Warn("failed to run aws api to describe clusters", "error", err)
+				return err
+			}
+
+			results = append(results, describeClusterOutput.Clusters...)
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		return []types.Cluster{}, err
+	}
 
 	// sort by running task count, name ascending
 	sort.Slice(results, func(i, j int) bool {

internal/api/service.go

@@ -15,7 +15,11 @@ import (
 // aws ecs list-services --cluster ${cluster}
 // aws ecs describe-services --cluster ${cluster} --services ${service}
 func (store *Store) ListServices(clusterName *string) ([]types.Service, error) {
-	limit := int32(100)
+	// You may specify up to 100 services to describe.
+	// If there are > 100 services in the cluster, loop and slice by 100
+	// to describe them in batches of <= 100.
+	batchSize := 100
+	limit := int32(batchSize)
 	params := &ecs.ListServicesInput{
 		Cluster:    clusterName,
 		MaxResults: &limit,
@@ -47,10 +51,6 @@ func (store *Store) ListServices(clusterName *string) ([]types.Service, error) {
 		}
 	}
 
-	// You may specify up to 10 services to describe.
-	// If there are > 10 services in the cluster, loop and slice by 10
-	// to describe them in batches of <= 10.
-	batchSize := 10
 	serviceCount := len(serviceARNs)
 	loopCount := serviceCount / batchSize
 
@@ -89,8 +89,7 @@ func (store *Store) ListServices(clusterName *string) ([]types.Service, error) {
 		})
 	}
 
-	err := g.Wait()
-	if err != nil {
+	if err := g.Wait(); err != nil {
 		return []types.Service{}, err
 	}