Add integ tests for EKS windows (aws#387)

### Changelog * Add terraform for eks windows * Add terraform templates for both 2019 and 2022
okankoAMZ · Mar 7, 2024 · 1f5a5d7 · 1f5a5d7
1 parent 3c62d6b
commit 1f5a5d7
Show file tree

Hide file tree

Showing 24 changed files with 1,711 additions and 194 deletions.
diff --git a/environment/metadata.go b/environment/metadata.go
@@ -41,6 +41,7 @@ type MetaData struct {
 	ProxyUrl                  string
 	AssumeRoleArn             string
 	InstanceId                string
+	InstancePlatform          string
 	AgentStartCommand         string
 }
 
@@ -62,6 +63,7 @@ type MetaDataStrings struct {
 	ProxyUrl                  string
 	AssumeRoleArn             string
 	InstanceId                string
+	InstancePlatform          string
 	AgentStartCommand         string
 }
 
@@ -122,6 +124,10 @@ func registerInstanceId(dataString *MetaDataStrings) {
 	flag.StringVar(&(dataString.InstanceId), "instanceId", "", "ec2 instance ID that is being used by a test")
 }
 
+func registerInstancePlatform(dataString *MetaDataStrings) {
+	flag.StringVar(&(dataString.InstancePlatform), "instancePlatform", "linux", "ec2 instance OS that is being used for a test")
+}
+
 func registerAgentStartCommand(dataString *MetaDataStrings) {
 	flag.StringVar(&(dataString.AgentStartCommand), "agentStartCommand",
 		DefaultEC2AgentStartCommand,
@@ -218,6 +224,7 @@ func RegisterEnvironmentMetaDataFlags() *MetaDataStrings {
 	registerProxyUrl(registeredMetaDataStrings)
 	registerAssumeRoleArn(registeredMetaDataStrings)
 	registerInstanceId(registeredMetaDataStrings)
+	registerInstancePlatform(registeredMetaDataStrings)
 	registerAgentStartCommand(registeredMetaDataStrings)
 
 	return registeredMetaDataStrings
@@ -241,6 +248,7 @@ func GetEnvironmentMetaData() *MetaData {
 	metaDataStorage.ProxyUrl = registeredMetaDataStrings.ProxyUrl
 	metaDataStorage.AssumeRoleArn = registeredMetaDataStrings.AssumeRoleArn
 	metaDataStorage.InstanceId = registeredMetaDataStrings.InstanceId
+	metaDataStorage.InstancePlatform = registeredMetaDataStrings.InstancePlatform
 	metaDataStorage.AgentStartCommand = registeredMetaDataStrings.AgentStartCommand
 
 	return metaDataStorage

diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go
@@ -187,6 +187,16 @@ var testTypeToTestConfig = map[string][]testConfig{
 			testDir: "./test/metric_value_benchmark",
 			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
 		},
+		{
+			testDir:      "./test/metric_value_benchmark",
+			terraformDir: "terraform/eks/daemon/windows/2019",
+			targets:      map[string]map[string]struct{}{"arc": {"amd64": {}}},
+		},
+		{
+			testDir:      "./test/metric_value_benchmark",
+			terraformDir: "terraform/eks/daemon/windows/2022",
+			targets:      map[string]map[string]struct{}{"arc": {"amd64": {}}},
+		},
 		{
 			testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd",
 			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
@@ -203,6 +213,7 @@ var testTypeToTestConfig = map[string][]testConfig{
 		{testDir: "./test/app_signals", terraformDir: "terraform/eks/daemon/app_signals",
 			targets: map[string]map[string]struct{}{"arc": {"amd64": {}}},
 		},
+		{testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/windows/2022"},
 	},
 	"eks_deployment": {
 		{testDir: "./test/metric_value_benchmark"},

diff --git a/terraform/eks/daemon/default_resources/cwagent-windows.yaml b/terraform/eks/daemon/default_resources/cwagent-windows.yaml
@@ -0,0 +1,62 @@
+# create amazon-cloudwatch namespace
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cloudwatch-agent-windows
+  namespace: amazon-cloudwatch
+spec:
+  selector:
+    matchLabels:
+      app: cloudwatch-agent
+  template:
+    metadata:
+      labels:
+        app: cloudwatch-agent
+    spec:
+      securityContext:
+        windowsOptions:
+          hostProcess: true
+          runAsUserName: "NT AUTHORITY\\SYSTEM"
+      hostNetwork: true
+      containers:
+        - name: cloudwatch-agent
+          image: CW_TEST_IMAGE
+          volumeMounts:
+            - name: cwagentconfig
+              mountPath: C:\Program Files\Amazon\AmazonCloudWatchAgent\cwagentconfig
+          resources:
+            limits:
+              cpu:  400m
+              memory: 400Mi
+            requests:
+              cpu: 400m
+              memory: 400Mi
+          imagePullPolicy: Always
+          env:
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: "status.hostIP"
+            - name: HOST_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: K8S_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CI_VERSION
+              value: "k8s/1.3.20"
+            - name: CWAGENT_LOG_LEVEL
+              value: DEBUG
+            - name: RUN_IN_CONTAINER
+              value: "True"
+            - name: RUN_AS_HOST_PROCESS_CONTAINER
+              value: "True"
+      nodeSelector:
+        kubernetes.io/os: windows
+      serviceAccountName: cloudwatch-agent
+      volumes:
+        - name: cwagentconfig
+          configMap:
+            name: cwagentconfig
diff --git a/terraform/eks/daemon/default_resources/fluenbit-windows-configmap.yaml b/terraform/eks/daemon/default_resources/fluenbit-windows-configmap.yaml
@@ -0,0 +1,148 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluent-bit-windows-config
+  namespace: amazon-cloudwatch
+  labels:
+    k8s-app: fluent-bit
+data:
+  fluent-bit.conf: |
+    [SERVICE]
+        Flush                       5
+        Log_Level                   error
+        Daemon                      off
+        net.dns.resolver            LEGACY
+        Parsers_File                parsers.conf
+
+    @INCLUDE application-log.conf
+    @INCLUDE dataplane-log.conf
+    @INCLUDE host-log.conf
+
+  application-log.conf: |
+    [INPUT]
+        Name                tail
+        Tag                 application.*
+        Exclude_Path        C:\\var\\log\\containers\\fluent-bit*, C:\\var\\log\\containers\\cloudwatch-agent*
+        Path                C:\\var\\log\\containers\\*.log
+        Parser              docker
+        DB                  C:\\var\\fluent-bit\\state\\flb_container.db
+        Mem_Buf_Limit       50MB
+        Skip_Long_Lines     On
+        Rotate_Wait         30
+        Refresh_Interval    10
+        Read_from_Head      ${READ_FROM_HEAD}
+
+    [INPUT]
+        Name                tail
+        Tag                 application.*
+        Path                C:\\var\\log\\containers\\fluent-bit*
+        Parser              docker
+        DB                  C:\\var\\fluent-bit\\state\\flb_log.db
+        Mem_Buf_Limit       5MB
+        Skip_Long_Lines     On
+        Rotate_Wait         30
+        Refresh_Interval    10
+        Read_from_Head      ${READ_FROM_HEAD}
+
+    [INPUT]
+        Name                tail
+        Tag                 application.*
+        Path                C:\\var\\log\\containers\\cloudwatch-agent*
+        Parser              docker
+        DB                  C:\\var\\fluent-bit\\state\\flb_cwagent.db
+        Mem_Buf_Limit       5MB
+        Skip_Long_Lines     On
+        Rotate_Wait         30
+        Refresh_Interval    10
+        Read_from_Head      ${READ_FROM_HEAD}
+
+    [OUTPUT]
+        Name                cloudwatch_logs
+        Match               application.*
+        region              ${AWS_REGION}
+        log_group_name      /aws/containerinsights/${CLUSTER_NAME}/application
+        log_stream_prefix   ${HOST_NAME}-
+        auto_create_group   true
+        extra_user_agent    container-insights
+
+  dataplane-log.conf: |
+    [INPUT]
+        Name                tail
+        Tag                 dataplane.tail.*
+        Path                C:\\ProgramData\\containerd\\root\\*.log, C:\\ProgramData\\Amazon\\EKS\\logs\\*.log
+        Parser              dataplane_firstline
+        DB                  C:\\var\\fluent-bit\\state\\flb_dataplane_tail.db
+        Mem_Buf_Limit       5MB
+        Skip_Long_Lines     On
+        Rotate_Wait         30
+        Refresh_Interval    10
+        Read_from_Head      ${READ_FROM_HEAD}
+
+    [INPUT]
+        Name                tail
+        Tag                 dataplane.tail.C.ProgramData.Amazon.EKS.logs.vpc-bridge
+        Path                C:\\ProgramData\\Amazon\\EKS\\logs\\*.log.*
+        Path_Key            file_name
+        Parser              dataplane_firstline
+        DB                  C:\\var\\fluent-bit\\state\\flb_dataplane_cni_tail.db
+        Mem_Buf_Limit       5MB
+        Skip_Long_Lines     On
+        Rotate_Wait         30
+        Refresh_Interval    10
+        Read_from_Head      ${READ_FROM_HEAD}
+
+    [FILTER]
+        Name                aws
+        Match               dataplane.*
+        imds_version        v2
+
+    [OUTPUT]
+        Name                cloudwatch_logs
+        Match               dataplane.*
+        region              ${AWS_REGION}
+        log_group_name      /aws/containerinsights/${CLUSTER_NAME}/dataplane
+        log_stream_prefix   ${HOST_NAME}-
+        auto_create_group   true
+        extra_user_agent    container-insights
+
+  host-log.conf: |
+    [INPUT]
+        Name                winlog
+        Channels            EKS, System
+        DB                  C:\\var\\fluent-bit\\state\\flb_system_winlog.db
+        Interval_Sec        60
+
+    [FILTER]
+        Name                aws
+        Match               winlog.*
+        imds_version        v2
+
+    [OUTPUT]
+        Name                cloudwatch_logs
+        Match               winlog.*
+        region              ${AWS_REGION}
+        log_group_name      /aws/containerinsights/${CLUSTER_NAME}/host
+        log_stream_prefix   ${HOST_NAME}.
+        auto_create_group   true
+        extra_user_agent    container-insights
+
+  parsers.conf: |
+    [PARSER]
+        Name                docker
+        Format              json
+        Time_Key            time
+        Time_Format         %b %d %H:%M:%S
+
+    [PARSER]
+        Name                container_firstline
+        Format              regex
+        Regex               (?<log>(?<="log":")\S(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
+        Time_Key            time
+        Time_Format         %Y-%m-%dT%H:%M:%S.%LZ
+
+    [PARSER]
+        Name                dataplane_firstline
+        Format              regex
+        Regex               (?<log>(?<="log":")\S(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
+        Time_Key            time
+        Time_Format         %Y-%m-%dT%H:%M:%S.%LZ
diff --git a/terraform/eks/daemon/default_resources/fluenbit-windows-daemonset.yaml b/terraform/eks/daemon/default_resources/fluenbit-windows-daemonset.yaml
@@ -0,0 +1,76 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluent-bit-windows
+  namespace: amazon-cloudwatch
+  labels:
+    k8s-app: fluent-bit
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    matchLabels:
+      k8s-app: fluent-bit
+  template:
+    metadata:
+      labels:
+        k8s-app: fluent-bit
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      securityContext:
+        windowsOptions:
+          hostProcess: true
+          runAsUserName: "NT AUTHORITY\\System"
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: windows
+      containers:
+        - name: fluent-bit
+          image: public.ecr.aws/aws-observability/aws-for-fluent-bit:windowsservercore-stable
+          imagePullPolicy: Always
+          command: ["powershell.exe", "-Command", "New-Item -ItemType Directory -Path C:\\var\\fluent-bit\\state -Force;", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/bin/fluent-bit.exe", "-e", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/kinesis.dll", "-e", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/firehose.dll", "-e", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/cloudwatch.dll", "-c", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/configuration/fluent-bit.conf"]
+          env:
+            - name: AWS_REGION
+              valueFrom:
+                configMapKeyRef:
+                  name: "fluent-bit-cluster-info"
+                  key: "logs.region"
+            - name: CLUSTER_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: "fluent-bit-cluster-info"
+                  key: "cluster.name"
+            - name: READ_FROM_HEAD
+              valueFrom:
+                configMapKeyRef:
+                  name: "fluent-bit-cluster-info"
+                  key: "read.head"
+            - name: HOST_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: CI_VERSION
+              value: "k8s/1.3.17"
+          resources:
+            limits:
+              cpu: 500m
+              memory: 600Mi
+            requests:
+              cpu: 500m
+              memory: 600Mi
+          volumeMounts:
+            - name: fluent-bit-config
+              mountPath: fluent-bit\configuration\
+      volumes:
+        - name: fluent-bit-config
+          configMap:
+            name: fluent-bit-windows-config
+      terminationGracePeriodSeconds: 10
+      dnsPolicy: ClusterFirstWithHostNet
+      serviceAccountName: cloudwatch-agent
diff --git a/terraform/eks/daemon/default_resources/test-sample-windows.yaml b/terraform/eks/daemon/default_resources/test-sample-windows.yaml
@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: windows-test-deployment
+spec:
+  selector:
+    matchLabels:
+      app: windows-test-deployment
+      tier: backend
+      track: stable
+  replicas: REPLICAS
+  template:
+    metadata:
+      labels:
+        app: windows-test-deployment
+        tier: backend
+        track: stable
+    spec:
+      containers:
+        - name: windows-test-deployment
+          image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltscWINDOWS_SERVER_VERSION
+          command:
+            - powershell.exe
+            - -command
+            - "Add-WindowsFeature Web-Server; Invoke-WebRequest -UseBasicParsing -Uri 'https://dotnetbinaries.blob.core.windows.net/servicemonitor/2.0.1.6/ServiceMonitor.exe' -OutFile 'C:\\ServiceMonitor.exe'; echo '<html><body><br/><br/><H1>Windows Container Workshop - Windows LTSC2019!!!<H1></body><html>' > C:\\inetpub\\wwwroot\\iisstart.htm; C:\\ServiceMonitor.exe 'w3svc'; "
+      nodeSelector:
+        kubernetes.io/os: windows
+