On premises flow was added for all cluster resources

instaclustr · Dec 11, 2023 · d7fcad4 · d7fcad4
1 parent b504941
commit d7fcad4
Show file tree

Hide file tree

Showing 54 changed files with 4,466 additions and 1,023 deletions.
diff --git a/Makefile b/Makefile
@@ -5,6 +5,9 @@ IMG_TAG ?= latest
 OPERATOR_NAMESPACE ?= instaclustr-operator
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.24.2
+KUBEVIRT_TAG?=v1.57.0
+KUBEVIRT_VERSION?=v1.0.0
+ARCH?=linux-amd64
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -196,3 +199,28 @@ dev-build: docker-build kind-load deploy ## builds docker-image, loads it to kin
 kind-load: ## loads given image to kind cluster
 	kind load docker-image ${IMG}
 
+.PHONY: deploy-kubevirt
+deploy-kubevirt: ## Deploy kubevirt operator and custom resources
+	kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml
+	kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml
+	kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-operator.yaml
+	kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-cr.yaml
+
+.PHONY: undeploy-kubevirt
+undeploy-kubevirt: ## Delete kubevirt operator and custom resources
+	kubectl delete apiservices v1.subresources.kubevirt.io --ignore-not-found
+	kubectl delete mutatingwebhookconfigurations virt-api-mutator --ignore-not-found
+	kubectl delete validatingwebhookconfigurations virt-operator-validator --ignore-not-found
+	kubectl delete validatingwebhookconfigurations virt-api-validator --ignore-not-found
+	kubectl delete -n kubevirt kubevirt kubevirt --ignore-not-found
+	kubectl delete -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-cr.yaml --ignore-not-found
+	kubectl delete -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-operator.yaml --ignore-not-found
+	kubectl delete -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml --ignore-not-found
+	kubectl delete -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml --ignore-not-found
+
+.PHONY: install-virtctl
+install-virtctl: ## Install virtctl tool
+	curl -L -o virtctl https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/virtctl-${KUBEVIRT_VERSION}-${ARCH}
+	chmod +x virtctl
+	sudo install virtctl /usr/local/bin
+	rm virtctl
diff --git a/apis/clusters/v1beta1/cadence_types.go b/apis/clusters/v1beta1/cadence_types.go
@@ -22,9 +22,11 @@ import (
 	"fmt"
 	"regexp"
 
+	k8scorev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/instaclustr/operator/pkg/models"
@@ -62,7 +64,8 @@ type BundledOpenSearchSpec struct {
 
 // CadenceSpec defines the desired state of Cadence
 type CadenceSpec struct {
-	Cluster `json:",inline"`
+	Cluster        `json:",inline"`
+	OnPremisesSpec *OnPremisesSpec `json:"onPremisesSpec,omitempty"`
 	//+kubebuilder:validation:MinItems:=1
 	//+kubebuilder:validation:MaxItems:=1
 	DataCentres          []*CadenceDataCentre    `json:"dataCentres"`
@@ -793,3 +796,72 @@ func (o *BundledOpenSearchSpec) validate() error {
 
 	return nil
 }
+
+func (c *Cadence) GetExposePorts() []k8scorev1.ServicePort {
+	var exposePorts []k8scorev1.ServicePort
+	if !c.Spec.PrivateNetworkCluster {
+		exposePorts = []k8scorev1.ServicePort{
+			{
+				Name: models.CadenceTChannel,
+				Port: models.Port7933,
+				TargetPort: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: models.Port7933,
+				},
+			},
+			{
+				Name: models.CadenceHHTPAPI,
+				Port: models.Port8088,
+				TargetPort: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: models.Port8088,
+				},
+			},
+			{
+				Name: models.CadenceWeb,
+				Port: models.Port443,
+				TargetPort: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: models.Port443,
+				},
+			},
+		}
+		if c.Spec.DataCentres[0].ClientEncryption {
+			sslPort := k8scorev1.ServicePort{
+				Name: models.CadenceGRPC,
+				Port: models.Port7833,
+				TargetPort: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: models.Port7833,
+				},
+			}
+			exposePorts = append(exposePorts, sslPort)
+		}
+	}
+	return exposePorts
+}
+
+func (c *Cadence) GetHeadlessPorts() []k8scorev1.ServicePort {
+	headlessPorts := []k8scorev1.ServicePort{
+		{
+			Name: models.CadenceTChannel,
+			Port: models.Port7933,
+			TargetPort: intstr.IntOrString{
+				Type:   intstr.Int,
+				IntVal: models.Port7933,
+			},
+		},
+	}
+	if c.Spec.DataCentres[0].ClientEncryption {
+		sslPort := k8scorev1.ServicePort{
+			Name: models.CadenceGRPC,
+			Port: models.Port7833,
+			TargetPort: intstr.IntOrString{
+				Type:   intstr.Int,
+				IntVal: models.Port7833,
+			},
+		}
+		headlessPorts = append(headlessPorts, sslPort)
+	}
+	return headlessPorts
+}
diff --git a/apis/clusters/v1beta1/cadence_webhook.go b/apis/clusters/v1beta1/cadence_webhook.go
@@ -85,6 +85,19 @@ func (cv *cadenceValidator) ValidateCreate(ctx context.Context, obj runtime.Obje
 		return err
 	}
 
+	if c.Spec.OnPremisesSpec != nil {
+		err = c.Spec.OnPremisesSpec.ValidateCreation()
+		if err != nil {
+			return err
+		}
+		if c.Spec.PrivateNetworkCluster {
+			err = c.Spec.OnPremisesSpec.ValidateSSHGatewayCreation()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	appVersions, err := cv.API.ListAppVersions(models.CadenceAppKind)
 	if err != nil {
 		return fmt.Errorf("cannot list versions for kind: %v, err: %w",
@@ -173,10 +186,22 @@ func (cv *cadenceValidator) ValidateCreate(ctx context.Context, obj runtime.Obje
 		return fmt.Errorf("data centres field is empty")
 	}
 
+	//TODO: add support of multiple DCs for OnPrem clusters
+	if len(c.Spec.DataCentres) > 1 && c.Spec.OnPremisesSpec != nil {
+		return fmt.Errorf("on-premises cluster can be provisioned with only one data centre")
+	}
+
 	for _, dc := range c.Spec.DataCentres {
-		err := dc.DataCentre.ValidateCreation()
-		if err != nil {
-			return err
+		if c.Spec.OnPremisesSpec != nil {
+			err := dc.DataCentre.ValidateOnPremisesCreation()
+			if err != nil {
+				return err
+			}
+		} else {
+			err := dc.DataCentre.ValidateCreation()
+			if err != nil {
+				return err
+			}
 		}
 
 		if !c.Spec.PrivateNetworkCluster && dc.PrivateLink != nil {

diff --git a/apis/clusters/v1beta1/cassandra_types.go b/apis/clusters/v1beta1/cassandra_types.go
@@ -21,7 +21,9 @@ import (
 	"fmt"
 	"strconv"
 
+	k8scorev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -52,6 +54,7 @@ type CassandraRestoreFrom struct {
 // CassandraSpec defines the desired state of Cassandra
 type CassandraSpec struct {
 	RestoreFrom         *CassandraRestoreFrom `json:"restoreFrom,omitempty"`
+	OnPremisesSpec      *OnPremisesSpec       `json:"onPremisesSpec,omitempty"`
 	Cluster             `json:",inline"`
 	DataCentres         []*CassandraDataCentre `json:"dataCentres,omitempty"`
 	LuceneEnabled       bool                   `json:"luceneEnabled,omitempty"`
@@ -539,3 +542,80 @@ func (c *Cassandra) SetClusterID(id string) {
 func init() {
 	SchemeBuilder.Register(&Cassandra{}, &CassandraList{})
 }
+
+func (c *Cassandra) GetExposePorts() []k8scorev1.ServicePort {
+	var exposePorts []k8scorev1.ServicePort
+	if !c.Spec.PrivateNetworkCluster {
+		exposePorts = []k8scorev1.ServicePort{
+			{
+				Name: models.CassandraInterNode,
+				Port: models.Port7000,
+				TargetPort: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: models.Port7000,
+				},
+			},
+			{
+				Name: models.CassandraCQL,
+				Port: models.Port9042,
+				TargetPort: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: models.Port9042,
+				},
+			},
+			{
+				Name: models.CassandraJMX,
+				Port: models.Port7199,
+				TargetPort: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: models.Port7199,
+				},
+			},
+		}
+		if c.Spec.DataCentres[0].ClientToClusterEncryption {
+			sslPort := k8scorev1.ServicePort{
+				Name: models.CassandraSSL,
+				Port: models.Port7001,
+				TargetPort: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: models.Port7001,
+				},
+			}
+			exposePorts = append(exposePorts, sslPort)
+		}
+	}
+	return exposePorts
+}
+
+func (c *Cassandra) GetHeadlessPorts() []k8scorev1.ServicePort {
+	headlessPorts := []k8scorev1.ServicePort{
+		{
+			Name: models.CassandraInterNode,
+			Port: models.Port7000,
+			TargetPort: intstr.IntOrString{
+				Type:   intstr.Int,
+				IntVal: models.Port7000,
+			},
+		},
+		{
+			Name: models.CassandraCQL,
+			Port: models.Port9042,
+			TargetPort: intstr.IntOrString{
+				Type:   intstr.Int,
+				IntVal: models.Port9042,
+			},
+		},
+	}
+	if c.Spec.DataCentres[0].ClientToClusterEncryption {
+		sslPort := k8scorev1.ServicePort{
+			Name: models.CassandraSSL,
+			Port: models.Port7001,
+			TargetPort: intstr.IntOrString{
+				Type:   intstr.Int,
+				IntVal: models.Port7001,
+			},
+		}
+		headlessPorts = append(headlessPorts, sslPort)
+	}
+	return headlessPorts
+}
diff --git a/apis/clusters/v1beta1/cassandra_webhook.go b/apis/clusters/v1beta1/cassandra_webhook.go
@@ -91,6 +91,19 @@ func (cv *cassandraValidator) ValidateCreate(ctx context.Context, obj runtime.Ob
 		return err
 	}
 
+	if c.Spec.OnPremisesSpec != nil {
+		err = c.Spec.OnPremisesSpec.ValidateCreation()
+		if err != nil {
+			return err
+		}
+		if c.Spec.PrivateNetworkCluster {
+			err = c.Spec.OnPremisesSpec.ValidateSSHGatewayCreation()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	appVersions, err := cv.API.ListAppVersions(models.CassandraAppKind)
 	if err != nil {
 		return fmt.Errorf("cannot list versions for kind: %v, err: %w",
@@ -106,10 +119,22 @@ func (cv *cassandraValidator) ValidateCreate(ctx context.Context, obj runtime.Ob
 		return fmt.Errorf("data centres field is empty")
 	}
 
+	//TODO: add support of multiple DCs for OnPrem clusters
+	if len(c.Spec.DataCentres) > 1 && c.Spec.OnPremisesSpec != nil {
+		return fmt.Errorf("on-premises cluster can be provisioned with only one data centre")
+	}
+
 	for _, dc := range c.Spec.DataCentres {
-		err := dc.DataCentre.ValidateCreation()
-		if err != nil {
-			return err
+		if c.Spec.OnPremisesSpec != nil {
+			err := dc.DataCentre.ValidateOnPremisesCreation()
+			if err != nil {
+				return err
+			}
+		} else {
+			err := dc.DataCentre.ValidateCreation()
+			if err != nil {
+				return err
+			}
 		}
 
 		if !c.Spec.PrivateNetworkCluster && dc.PrivateIPBroadcastForDiscovery {