Skip to content

Commit

Permalink
allow to set resource requests and limits for deamonset through operator
Browse files Browse the repository at this point in the history
  • Loading branch information
@staerion
staerion committed Aug 3, 2023
1 parent 31c02b2 commit 3882803
Show file tree
Hide file tree
Showing 6 changed files with 317 additions and 113 deletions.
45 changes: 24 additions & 21 deletions cli/k8s_client/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/version"

netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
versionutils "github.com/netapp/trident/utils/version"
)

Expand Down Expand Up @@ -168,27 +169,29 @@ type DeploymentYAMLArguments struct {
}

type DaemonsetYAMLArguments struct {
DaemonsetName string `json:"daemonsetName"`
TridentImage string `json:"tridentImage"`
ImageRegistry string `json:"imageRegistry"`
KubeletDir string `json:"kubeletDir"`
LogFormat string `json:"logFormat"`
LogLevel string `json:"logLevel"`
LogWorkflows string `json:"logWorkflows"`
LogLayers string `json:"logLayers"`
ProbePort string `json:"probePort"`
ImagePullSecrets []string `json:"imagePullSecrets"`
Labels map[string]string `json:"labels"`
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
EnableForceDetach bool `json:"enableForceDetach"`
DisableAuditLog bool `json:"disableAuditLog"`
Debug bool `json:"debug"`
Version *versionutils.Version `json:"version"`
HTTPRequestTimeout string `json:"httpRequestTimeout"`
NodeSelector map[string]string `json:"nodeSelector"`
Tolerations []map[string]string `json:"tolerations"`
ServiceAccountName string `json:"serviceAccountName"`
ImagePullPolicy string `json:"imagePullPolicy"`
DaemonsetName string `json:"daemonsetName"`
TridentImage string `json:"tridentImage"`
ImageRegistry string `json:"imageRegistry"`
KubeletDir string `json:"kubeletDir"`
LogFormat string `json:"logFormat"`
LogLevel string `json:"logLevel"`
LogWorkflows string `json:"logWorkflows"`
LogLayers string `json:"logLayers"`
ProbePort string `json:"probePort"`
ImagePullSecrets []string `json:"imagePullSecrets"`
Labels map[string]string `json:"labels"`
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
EnableForceDetach bool `json:"enableForceDetach"`
DisableAuditLog bool `json:"disableAuditLog"`
Debug bool `json:"debug"`
Version *versionutils.Version `json:"version"`
HTTPRequestTimeout string `json:"httpRequestTimeout"`
NodeSelector map[string]string `json:"nodeSelector"`
Tolerations []map[string]string `json:"tolerations"`
TridentResourceSpec netappv1.ResourceSpec `json:"tridentResourceSpec"`
RegistrarResourceSpec netappv1.ResourceSpec `json:"registrarSpec"`
ServiceAccountName string `json:"serviceAccountName"`
ImagePullPolicy string `json:"imagePullPolicy"`
}

type TridentVersionPodYAMLArguments struct {
Expand Down
83 changes: 65 additions & 18 deletions cli/k8s_client/yaml_factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ import (

commonconfig "github.com/netapp/trident/config"
. "github.com/netapp/trident/logging"
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
"github.com/netapp/trident/utils"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
k8syaml "sigs.k8s.io/yaml"
)

const (
Expand Down Expand Up @@ -726,6 +730,25 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
}
}

defaultRegistrarResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("100Mi"),
},
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("40Mi"),
},
}}
defaultTridentResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("400Mi"),
},
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("10m"),
v1.ResourceMemory: resource.MustParse("20Mi"),
},
}}

kubeletDir := strings.TrimRight(args.KubeletDir, "/")
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage)
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName)
Expand All @@ -745,6 +768,10 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "REGISTRAR_RESOURCE_CLAIMS", constructResourceClaim(
"DsRegistrar", args.RegistrarResourceSpec, defaultRegistrarResources))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "TRIDENT_RESOURCE_CLAIMS", constructResourceClaim(
"DsTrident", args.TridentResourceSpec, defaultTridentResources))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
// Log before secrets are inserted into YAML.
Expand Down Expand Up @@ -799,6 +826,13 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
}
}

defaultRegistrarResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{},
}}
defaultTridentResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{},
}}

kubeletDir := strings.TrimRight(args.KubeletDir, "/")
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage)
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName)
Expand All @@ -819,6 +853,8 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "REGISTRAR_RESOURCE_CLAIMS", constructResourceClaim("DsRegistrar", args.RegistrarResourceSpec, defaultRegistrarResources))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "TRIDENT_RESOURCE_CLAIMS", constructResourceClaim("DsTrident", args.TridentResourceSpec, defaultTridentResources))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))

Expand Down Expand Up @@ -907,6 +943,7 @@ spec:
failureThreshold: 5
initialDelaySeconds: 10
periodSeconds: 10
{TRIDENT_RESOURCE_CLAIMS}
env:
- name: KUBE_NODE_NAME
valueFrom:
Expand Down Expand Up @@ -948,6 +985,7 @@ spec:
- "--v={SIDECAR_LOG_LEVEL}"
- "--csi-address=$(ADDRESS)"
- "--kubelet-registration-path=$(REGISTRATION_PATH)"
{REGISTRAR_RESOURCE_CLAIMS}
env:
- name: ADDRESS
value: /plugin/csi.sock
Expand Down Expand Up @@ -1137,12 +1175,7 @@ spec:
mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1
- name: csi-proxy-smb-pipe-v1beta1
mountPath: \\.\pipe\csi-proxy-smb-v1beta1
resources:
limits:
memory: 400Mi
requests:
cpu: 10m
memory: 20Mi
{TRIDENT_RESOURCE_CLAIMS}
- name: node-driver-registrar
image: {CSI_SIDECAR_REGISTRY}/csi-node-driver-registrar:v2.8.0
imagePullPolicy: {IMAGE_PULL_POLICY}
Expand Down Expand Up @@ -1174,12 +1207,7 @@ spec:
mountPath: C:\csi
- name: registration-dir
mountPath: C:\registration
resources:
limits:
memory: 200Mi
requests:
cpu: 10m
memory: 20Mi
{REGISTRAR_RESOURCE_CLAIMS}
- name: liveness-probe
volumeMounts:
- mountPath: C:\csi
Expand All @@ -1193,12 +1221,6 @@ spec:
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
resources:
limits:
memory: 100Mi
requests:
cpu: 10m
memory: 40Mi
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
Expand Down Expand Up @@ -2509,6 +2531,31 @@ func constructNodeSelector(nodeLabels map[string]string) string {
return nodeSelector
}

func constructResourceClaim(
containerName string, resourceClaim netappv1.ResourceSpec, defaultResourceClaim netappv1.ResourceSpec,
) string {
userData, err := k8syaml.Marshal(resourceClaim)
if err != nil {
Log().WithField("container", containerName).Errorf("Cannot marshal resource claim: %s", err)
return ""
}

if err := k8syaml.Unmarshal(userData, &defaultResourceClaim); err != nil {
Log().WithField("container", containerName).Errorf(
"Cannot unmarshal provided resource claim into default resource claim: %s", err,
)
return ""
}
mergedData, err := k8syaml.Marshal(defaultResourceClaim)

if err != nil {
Log().WithField("container", containerName).Errorf("Cannot marshal resource claim: %s", err)
return ""
}

return string(mergedData)
}

func constructTolerations(tolerations []map[string]string) string {
var tolerationsString string

Expand Down
137 changes: 137 additions & 0 deletions cli/k8s_client/yaml_factory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@ import (
pspv1beta1 "k8s.io/api/policy/v1beta1"
csiv1 "k8s.io/api/storage/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/netapp/trident/config"
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
versionutils "github.com/netapp/trident/utils/version"
)

Expand Down Expand Up @@ -652,6 +654,43 @@ func TestGetCSIDaemonSetYAMLLinux_NodeSelectors(t *testing.T) {
fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData))
}

func TestGetCSIDaemonSetYAMLLinux_ResourceClaims(t *testing.T) {
daemonsetArgs := &DaemonsetYAMLArguments{
RegistrarResourceSpec: netappv1.ResourceSpec{},
TridentResourceSpec: netappv1.ResourceSpec{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("125Mi"),
},
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("50m"),
v1.ResourceMemory: resource.MustParse("50Mi"),
},
},
},
}
expectedRegistrarResourceClaimString := "resources: {}\n"
expectedTridentResourceClaimString := `
resources:
limits:
memory: 125Mi
requests:
cpu: 50m
memory: 50Mi
`

yamlData := GetCSIDaemonSetYAMLLinux(daemonsetArgs)
_, err := yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML, got %s", yamlData)
}
assert.Contains(t, yamlData, expectedRegistrarResourceClaimString,
fmt.Sprintf("expected empty registrar resource claims in final YAML: %s", yamlData))

assert.Contains(t, yamlData, expectedTridentResourceClaimString,
fmt.Sprintf("expected trident resource claims in final YAML: %s", yamlData))
}

func TestGetCSIDaemonSetYAMLLinux_Tolerations(t *testing.T) {
daemonsetArgs := &DaemonsetYAMLArguments{
Tolerations: []map[string]string{
Expand Down Expand Up @@ -800,6 +839,56 @@ func TestGetCSIDaemonSetYAMLWindowsImagePullPolicy(t *testing.T) {
}
}

func TestGetCSIDaemonSetYAMWindows_ResourceClaims(t *testing.T) {
versions := []string{"1.26.0"}
registrarResourceSpec := netappv1.ResourceSpec{}
tridentResourceSpec := netappv1.ResourceSpec{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: resource.MustParse("125Mi"),
},
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("50m"),
},
},
}

expectedRegistrarResourceClaimString := `
resources:
limits:
memory: 100Mi
requests:
cpu: 10m
memory: 40Mi
`
expectedTridentResourceClaimString := `
resources:
limits:
memory: 125Mi
requests:
cpu: 50m
memory: 20Mi
`
for _, versionString := range versions {
version := versionutils.MustParseSemantic(versionString)
daemonsetArgs := &DaemonsetYAMLArguments{
Version: version,
RegistrarResourceSpec: registrarResourceSpec,
TridentResourceSpec: tridentResourceSpec,
}
yamlData := GetCSIDaemonSetYAMLWindows(daemonsetArgs)
_, err := yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML, got %s", yamlData)
}
assert.Contains(t, yamlData, expectedRegistrarResourceClaimString,
fmt.Sprintf("expected empty registrar resource claims in final YAML: %s", yamlData))

assert.Contains(t, yamlData, expectedTridentResourceClaimString,
fmt.Sprintf("expected trident resource claims in final YAML: %s", yamlData))
}
}

func TestConstructNodeSelector(t *testing.T) {
nodeSelMap := map[string]string{"node-label-name": "master"}

Expand All @@ -809,6 +898,54 @@ func TestConstructNodeSelector(t *testing.T) {
assert.Equal(t, expectedNodeSelString, result)
}

func TestConstructResourceClaim(t *testing.T) {
defaultResourceClaim := netappv1.ResourceSpec{
Resources: v1.ResourceRequirements{},
}
resourceClaim := netappv1.ResourceSpec{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("100m"),
v1.ResourceMemory: resource.MustParse("50Mi"),
},
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("20m"),
v1.ResourceMemory: resource.MustParse("20Mi"),
},
},
}

expectedResourceClaimString := `resources:
limits:
cpu: 100m
memory: 50Mi
requests:
cpu: 20m
memory: 20Mi
`

result := constructResourceClaim("test", resourceClaim, defaultResourceClaim)
assert.Equal(t, expectedResourceClaimString, result)

resourceClaim = netappv1.ResourceSpec{
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("100m"),
},
},
}

expectedResourceClaimString = "resources:\n limits:\n cpu: 100m\n"
result = constructResourceClaim("test", resourceClaim, defaultResourceClaim)
assert.Equal(t, expectedResourceClaimString, result)

resourceClaim = netappv1.ResourceSpec{}

expectedResourceClaimString = "resources: {}\n"
result = constructResourceClaim("test", resourceClaim, defaultResourceClaim)
assert.Equal(t, expectedResourceClaimString, result)
}

func TestGetNamespaceYAML(t *testing.T) {
expected := v1.Namespace{
TypeMeta: metav1.TypeMeta{
Expand Down
Loading

0 comments on commit 3882803

Please sign in to comment.