From d7fcad441566b474d5b3be8cea4da0b6ff1e87b9 Mon Sep 17 00:00:00 2001 From: testisnullus Date: Wed, 15 Nov 2023 10:29:43 +0200 Subject: [PATCH] On premises flow was added for all cluster resources --- Makefile | 28 + apis/clusters/v1beta1/cadence_types.go | 74 +- apis/clusters/v1beta1/cadence_webhook.go | 31 +- apis/clusters/v1beta1/cassandra_types.go | 80 ++ apis/clusters/v1beta1/cassandra_webhook.go | 31 +- apis/clusters/v1beta1/kafka_types.go | 72 ++ apis/clusters/v1beta1/kafka_webhook.go | 31 +- apis/clusters/v1beta1/kafkaconnect_types.go | 40 +- apis/clusters/v1beta1/kafkaconnect_webhook.go | 39 +- apis/clusters/v1beta1/postgresql_types.go | 34 + apis/clusters/v1beta1/postgresql_webhook.go | 31 +- apis/clusters/v1beta1/redis_types.go | 54 +- apis/clusters/v1beta1/redis_webhook.go | 30 +- apis/clusters/v1beta1/structs.go | 12 + apis/clusters/v1beta1/validation.go | 48 + .../clusters/v1beta1/zz_generated.deepcopy.go | 50 + .../clusters.instaclustr.com_cadences.yaml | 39 + .../clusters.instaclustr.com_cassandras.yaml | 39 + ...lusters.instaclustr.com_kafkaconnects.yaml | 39 + .../clusters.instaclustr.com_kafkas.yaml | 39 + .../clusters.instaclustr.com_postgresqls.yaml | 39 + .../bases/clusters.instaclustr.com_redis.yaml | 39 + config/rbac/role.yaml | 66 ++ .../samples/clusters_v1beta1_cassandra.yaml | 31 +- .../onpremises/clusters_v1beta1_cadence.yaml | 34 + .../clusters_v1beta1_cassandra.yaml | 37 + .../onpremises/clusters_v1beta1_kafka.yaml | 40 + .../clusters_v1beta1_kafkaconnect.yaml | 36 + .../clusters_v1beta1_postgresql.yaml | 34 + .../onpremises/clusters_v1beta1_redis.yaml | 32 + .../awsendpointserviceprincipal_controller.go | 2 +- .../clusterbackup_controller.go | 2 +- controllers/clusters/cadence_controller.go | 715 ++++++++------ controllers/clusters/cassandra_controller.go | 614 +++++++----- controllers/clusters/helpers.go | 27 +- controllers/clusters/kafka_controller.go | 394 +++++--- .../clusters/kafkaconnect_controller.go | 157 ++- controllers/clusters/on_premises.go | 927 ++++++++++++++++++ controllers/clusters/opensearch_controller.go | 2 +- controllers/clusters/postgresql_controller.go | 257 +++-- controllers/clusters/redis_controller.go | 217 +++- doc/clusters/cadence.md | 153 ++- doc/clusters/cassandra.md | 130 ++- doc/clusters/kafka-connect.md | 120 ++- doc/clusters/kafka.md | 155 ++- doc/clusters/postgresql.md | 128 ++- doc/clusters/redis.md | 125 ++- main.go | 9 +- pkg/models/on_premises.go | 94 ++ pkg/models/operator.go | 2 +- pkg/models/validation.go | 2 + pkg/scheduler/scheduler.go | 9 +- scripts/cloud-init-script-example.sh | 13 + scripts/cloud-init-secret.yaml | 6 + 54 files changed, 4466 insertions(+), 1023 deletions(-) create mode 100644 config/samples/onpremises/clusters_v1beta1_cadence.yaml create mode 100644 config/samples/onpremises/clusters_v1beta1_cassandra.yaml create mode 100644 config/samples/onpremises/clusters_v1beta1_kafka.yaml create mode 100644 config/samples/onpremises/clusters_v1beta1_kafkaconnect.yaml create mode 100644 config/samples/onpremises/clusters_v1beta1_postgresql.yaml create mode 100644 config/samples/onpremises/clusters_v1beta1_redis.yaml create mode 100644 controllers/clusters/on_premises.go create mode 100644 pkg/models/on_premises.go create mode 100644 scripts/cloud-init-script-example.sh create mode 100644 scripts/cloud-init-secret.yaml diff --git a/Makefile b/Makefile index 49f81dad1..154101031 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,9 @@ IMG_TAG ?= latest OPERATOR_NAMESPACE ?= instaclustr-operator # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.24.2 +KUBEVIRT_TAG?=v1.57.0 +KUBEVIRT_VERSION?=v1.0.0 +ARCH?=linux-amd64 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -196,3 +199,28 @@ dev-build: docker-build kind-load deploy ## builds docker-image, loads it to kin kind-load: ## loads given image to kind cluster kind load docker-image ${IMG} +.PHONY: deploy-kubevirt +deploy-kubevirt: ## Deploy kubevirt operator and custom resources + kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml + kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml + kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-operator.yaml + kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-cr.yaml + +.PHONY: undeploy-kubevirt +undeploy-kubevirt: ## Delete kubevirt operator and custom resources + kubectl delete apiservices v1.subresources.kubevirt.io --ignore-not-found + kubectl delete mutatingwebhookconfigurations virt-api-mutator --ignore-not-found + kubectl delete validatingwebhookconfigurations virt-operator-validator --ignore-not-found + kubectl delete validatingwebhookconfigurations virt-api-validator --ignore-not-found + kubectl delete -n kubevirt kubevirt kubevirt --ignore-not-found + kubectl delete -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-cr.yaml --ignore-not-found + kubectl delete -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-operator.yaml --ignore-not-found + kubectl delete -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml --ignore-not-found + kubectl delete -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml --ignore-not-found + +.PHONY: install-virtctl +install-virtctl: ## Install virtctl tool + curl -L -o virtctl https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/virtctl-${KUBEVIRT_VERSION}-${ARCH} + chmod +x virtctl + sudo install virtctl /usr/local/bin + rm virtctl \ No newline at end of file diff --git a/apis/clusters/v1beta1/cadence_types.go b/apis/clusters/v1beta1/cadence_types.go index f3b394f27..03af314ed 100644 --- a/apis/clusters/v1beta1/cadence_types.go +++ b/apis/clusters/v1beta1/cadence_types.go @@ -22,9 +22,11 @@ import ( "fmt" "regexp" + k8scorev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/instaclustr/operator/pkg/models" @@ -62,7 +64,8 @@ type BundledOpenSearchSpec struct { // CadenceSpec defines the desired state of Cadence type CadenceSpec struct { - Cluster `json:",inline"` + Cluster `json:",inline"` + OnPremisesSpec *OnPremisesSpec `json:"onPremisesSpec,omitempty"` //+kubebuilder:validation:MinItems:=1 //+kubebuilder:validation:MaxItems:=1 DataCentres []*CadenceDataCentre `json:"dataCentres"` @@ -793,3 +796,72 @@ func (o *BundledOpenSearchSpec) validate() error { return nil } + +func (c *Cadence) GetExposePorts() []k8scorev1.ServicePort { + var exposePorts []k8scorev1.ServicePort + if !c.Spec.PrivateNetworkCluster { + exposePorts = []k8scorev1.ServicePort{ + { + Name: models.CadenceTChannel, + Port: models.Port7933, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port7933, + }, + }, + { + Name: models.CadenceHHTPAPI, + Port: models.Port8088, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port8088, + }, + }, + { + Name: models.CadenceWeb, + Port: models.Port443, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port443, + }, + }, + } + if c.Spec.DataCentres[0].ClientEncryption { + sslPort := k8scorev1.ServicePort{ + Name: models.CadenceGRPC, + Port: models.Port7833, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port7833, + }, + } + exposePorts = append(exposePorts, sslPort) + } + } + return exposePorts +} + +func (c *Cadence) GetHeadlessPorts() []k8scorev1.ServicePort { + headlessPorts := []k8scorev1.ServicePort{ + { + Name: models.CadenceTChannel, + Port: models.Port7933, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port7933, + }, + }, + } + if c.Spec.DataCentres[0].ClientEncryption { + sslPort := k8scorev1.ServicePort{ + Name: models.CadenceGRPC, + Port: models.Port7833, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port7833, + }, + } + headlessPorts = append(headlessPorts, sslPort) + } + return headlessPorts +} diff --git a/apis/clusters/v1beta1/cadence_webhook.go b/apis/clusters/v1beta1/cadence_webhook.go index bae85db9f..bec8a5c26 100644 --- a/apis/clusters/v1beta1/cadence_webhook.go +++ b/apis/clusters/v1beta1/cadence_webhook.go @@ -85,6 +85,19 @@ func (cv *cadenceValidator) ValidateCreate(ctx context.Context, obj runtime.Obje return err } + if c.Spec.OnPremisesSpec != nil { + err = c.Spec.OnPremisesSpec.ValidateCreation() + if err != nil { + return err + } + if c.Spec.PrivateNetworkCluster { + err = c.Spec.OnPremisesSpec.ValidateSSHGatewayCreation() + if err != nil { + return err + } + } + } + appVersions, err := cv.API.ListAppVersions(models.CadenceAppKind) if err != nil { return fmt.Errorf("cannot list versions for kind: %v, err: %w", @@ -173,10 +186,22 @@ func (cv *cadenceValidator) ValidateCreate(ctx context.Context, obj runtime.Obje return fmt.Errorf("data centres field is empty") } + //TODO: add support of multiple DCs for OnPrem clusters + if len(c.Spec.DataCentres) > 1 && c.Spec.OnPremisesSpec != nil { + return fmt.Errorf("on-premises cluster can be provisioned with only one data centre") + } + for _, dc := range c.Spec.DataCentres { - err := dc.DataCentre.ValidateCreation() - if err != nil { - return err + if c.Spec.OnPremisesSpec != nil { + err := dc.DataCentre.ValidateOnPremisesCreation() + if err != nil { + return err + } + } else { + err := dc.DataCentre.ValidateCreation() + if err != nil { + return err + } } if !c.Spec.PrivateNetworkCluster && dc.PrivateLink != nil { diff --git a/apis/clusters/v1beta1/cassandra_types.go b/apis/clusters/v1beta1/cassandra_types.go index 784a5f75a..c39e1a017 100644 --- a/apis/clusters/v1beta1/cassandra_types.go +++ b/apis/clusters/v1beta1/cassandra_types.go @@ -21,7 +21,9 @@ import ( "fmt" "strconv" + k8scorev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -52,6 +54,7 @@ type CassandraRestoreFrom struct { // CassandraSpec defines the desired state of Cassandra type CassandraSpec struct { RestoreFrom *CassandraRestoreFrom `json:"restoreFrom,omitempty"` + OnPremisesSpec *OnPremisesSpec `json:"onPremisesSpec,omitempty"` Cluster `json:",inline"` DataCentres []*CassandraDataCentre `json:"dataCentres,omitempty"` LuceneEnabled bool `json:"luceneEnabled,omitempty"` @@ -539,3 +542,80 @@ func (c *Cassandra) SetClusterID(id string) { func init() { SchemeBuilder.Register(&Cassandra{}, &CassandraList{}) } + +func (c *Cassandra) GetExposePorts() []k8scorev1.ServicePort { + var exposePorts []k8scorev1.ServicePort + if !c.Spec.PrivateNetworkCluster { + exposePorts = []k8scorev1.ServicePort{ + { + Name: models.CassandraInterNode, + Port: models.Port7000, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port7000, + }, + }, + { + Name: models.CassandraCQL, + Port: models.Port9042, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port9042, + }, + }, + { + Name: models.CassandraJMX, + Port: models.Port7199, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port7199, + }, + }, + } + if c.Spec.DataCentres[0].ClientToClusterEncryption { + sslPort := k8scorev1.ServicePort{ + Name: models.CassandraSSL, + Port: models.Port7001, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port7001, + }, + } + exposePorts = append(exposePorts, sslPort) + } + } + return exposePorts +} + +func (c *Cassandra) GetHeadlessPorts() []k8scorev1.ServicePort { + headlessPorts := []k8scorev1.ServicePort{ + { + Name: models.CassandraInterNode, + Port: models.Port7000, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port7000, + }, + }, + { + Name: models.CassandraCQL, + Port: models.Port9042, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port9042, + }, + }, + } + if c.Spec.DataCentres[0].ClientToClusterEncryption { + sslPort := k8scorev1.ServicePort{ + Name: models.CassandraSSL, + Port: models.Port7001, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port7001, + }, + } + headlessPorts = append(headlessPorts, sslPort) + } + return headlessPorts +} diff --git a/apis/clusters/v1beta1/cassandra_webhook.go b/apis/clusters/v1beta1/cassandra_webhook.go index 18ae6393b..a96d82983 100644 --- a/apis/clusters/v1beta1/cassandra_webhook.go +++ b/apis/clusters/v1beta1/cassandra_webhook.go @@ -91,6 +91,19 @@ func (cv *cassandraValidator) ValidateCreate(ctx context.Context, obj runtime.Ob return err } + if c.Spec.OnPremisesSpec != nil { + err = c.Spec.OnPremisesSpec.ValidateCreation() + if err != nil { + return err + } + if c.Spec.PrivateNetworkCluster { + err = c.Spec.OnPremisesSpec.ValidateSSHGatewayCreation() + if err != nil { + return err + } + } + } + appVersions, err := cv.API.ListAppVersions(models.CassandraAppKind) if err != nil { return fmt.Errorf("cannot list versions for kind: %v, err: %w", @@ -106,10 +119,22 @@ func (cv *cassandraValidator) ValidateCreate(ctx context.Context, obj runtime.Ob return fmt.Errorf("data centres field is empty") } + //TODO: add support of multiple DCs for OnPrem clusters + if len(c.Spec.DataCentres) > 1 && c.Spec.OnPremisesSpec != nil { + return fmt.Errorf("on-premises cluster can be provisioned with only one data centre") + } + for _, dc := range c.Spec.DataCentres { - err := dc.DataCentre.ValidateCreation() - if err != nil { - return err + if c.Spec.OnPremisesSpec != nil { + err := dc.DataCentre.ValidateOnPremisesCreation() + if err != nil { + return err + } + } else { + err := dc.DataCentre.ValidateCreation() + if err != nil { + return err + } } if !c.Spec.PrivateNetworkCluster && dc.PrivateIPBroadcastForDiscovery { diff --git a/apis/clusters/v1beta1/kafka_types.go b/apis/clusters/v1beta1/kafka_types.go index d4c51261c..2af6c17e1 100644 --- a/apis/clusters/v1beta1/kafka_types.go +++ b/apis/clusters/v1beta1/kafka_types.go @@ -19,7 +19,9 @@ package v1beta1 import ( "encoding/json" + k8scorev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/instaclustr/operator/pkg/models" @@ -63,6 +65,7 @@ type KarapaceSchemaRegistry struct { // KafkaSpec defines the desired state of Kafka type KafkaSpec struct { Cluster `json:",inline"` + OnPremisesSpec *OnPremisesSpec `json:"onPremisesSpec,omitempty"` SchemaRegistry []*SchemaRegistry `json:"schemaRegistry,omitempty"` // ReplicationFactor to use for new topic. @@ -484,3 +487,72 @@ func (k *Kafka) GetClusterID() string { func (k *Kafka) SetClusterID(id string) { k.Status.ID = id } + +func (k *Kafka) GetExposePorts() []k8scorev1.ServicePort { + var exposePorts []k8scorev1.ServicePort + if !k.Spec.PrivateNetworkCluster { + exposePorts = []k8scorev1.ServicePort{ + { + Name: models.KafkaClient, + Port: models.Port9092, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port9092, + }, + }, + { + Name: models.KafkaControlPlane, + Port: models.Port9093, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port9093, + }, + }, + } + if k.Spec.ClientToClusterEncryption { + sslPort := k8scorev1.ServicePort{ + Name: models.KafkaBroker, + Port: models.Port9094, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port9094, + }, + } + exposePorts = append(exposePorts, sslPort) + } + } + return exposePorts +} + +func (k *Kafka) GetHeadlessPorts() []k8scorev1.ServicePort { + headlessPorts := []k8scorev1.ServicePort{ + { + Name: models.KafkaClient, + Port: models.Port9092, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port9092, + }, + }, + { + Name: models.KafkaControlPlane, + Port: models.Port9093, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port9093, + }, + }, + } + if k.Spec.ClientToClusterEncryption { + kafkaBrokerPort := k8scorev1.ServicePort{ + Name: models.KafkaBroker, + Port: models.Port9094, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port9094, + }, + } + headlessPorts = append(headlessPorts, kafkaBrokerPort) + } + return headlessPorts +} diff --git a/apis/clusters/v1beta1/kafka_webhook.go b/apis/clusters/v1beta1/kafka_webhook.go index 74fbd508e..e088dc317 100644 --- a/apis/clusters/v1beta1/kafka_webhook.go +++ b/apis/clusters/v1beta1/kafka_webhook.go @@ -85,6 +85,19 @@ func (kv *kafkaValidator) ValidateCreate(ctx context.Context, obj runtime.Object return err } + if k.Spec.OnPremisesSpec != nil { + err = k.Spec.OnPremisesSpec.ValidateCreation() + if err != nil { + return err + } + if k.Spec.PrivateNetworkCluster { + err = k.Spec.OnPremisesSpec.ValidateSSHGatewayCreation() + if err != nil { + return err + } + } + } + appVersions, err := kv.API.ListAppVersions(models.KafkaAppKind) if err != nil { return fmt.Errorf("cannot list versions for kind: %v, err: %w", @@ -100,10 +113,22 @@ func (kv *kafkaValidator) ValidateCreate(ctx context.Context, obj runtime.Object return models.ErrZeroDataCentres } + //TODO: add support of multiple DCs for OnPrem clusters + if len(k.Spec.DataCentres) > 1 && k.Spec.OnPremisesSpec != nil { + return fmt.Errorf("on-premises cluster can be provisioned with only one data centre") + } + for _, dc := range k.Spec.DataCentres { - err := dc.DataCentre.ValidateCreation() - if err != nil { - return err + if k.Spec.OnPremisesSpec != nil { + err := dc.DataCentre.ValidateOnPremisesCreation() + if err != nil { + return err + } + } else { + err := dc.DataCentre.ValidateCreation() + if err != nil { + return err + } } if len(dc.PrivateLink) > 1 { diff --git a/apis/clusters/v1beta1/kafkaconnect_types.go b/apis/clusters/v1beta1/kafkaconnect_types.go index a101ee454..9c2e99f22 100644 --- a/apis/clusters/v1beta1/kafkaconnect_types.go +++ b/apis/clusters/v1beta1/kafkaconnect_types.go @@ -20,8 +20,10 @@ import ( "encoding/json" "fmt" + k8scorev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/instaclustr/operator/pkg/models" @@ -105,9 +107,10 @@ type KafkaConnectDataCentre struct { // KafkaConnectSpec defines the desired state of KafkaConnect type KafkaConnectSpec struct { - Cluster `json:",inline"` - DataCentres []*KafkaConnectDataCentre `json:"dataCentres"` - TargetCluster []*TargetCluster `json:"targetCluster"` + Cluster `json:",inline"` + OnPremisesSpec *OnPremisesSpec `json:"onPremisesSpec,omitempty"` + DataCentres []*KafkaConnectDataCentre `json:"dataCentres"` + TargetCluster []*TargetCluster `json:"targetCluster"` // CustomConnectors defines the location for custom connector storage and access info. CustomConnectors []*CustomConnectors `json:"customConnectors,omitempty"` @@ -720,3 +723,34 @@ func (k *KafkaConnect) NewDefaultUserSecret(username, password string) *v1.Secre }, } } + +func (k *KafkaConnect) GetExposePorts() []k8scorev1.ServicePort { + var exposePorts []k8scorev1.ServicePort + if !k.Spec.PrivateNetworkCluster { + exposePorts = []k8scorev1.ServicePort{ + { + Name: models.KafkaConnectAPI, + Port: models.Port8083, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port8083, + }, + }, + } + } + return exposePorts +} + +func (k *KafkaConnect) GetHeadlessPorts() []k8scorev1.ServicePort { + headlessPorts := []k8scorev1.ServicePort{ + { + Name: models.KafkaConnectAPI, + Port: models.Port8083, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port8083, + }, + }, + } + return headlessPorts +} diff --git a/apis/clusters/v1beta1/kafkaconnect_webhook.go b/apis/clusters/v1beta1/kafkaconnect_webhook.go index fd7f7180d..595a7d4b8 100644 --- a/apis/clusters/v1beta1/kafkaconnect_webhook.go +++ b/apis/clusters/v1beta1/kafkaconnect_webhook.go @@ -86,6 +86,19 @@ func (kcv *kafkaConnectValidator) ValidateCreate(ctx context.Context, obj runtim return err } + if kc.Spec.OnPremisesSpec != nil { + err = kc.Spec.OnPremisesSpec.ValidateCreation() + if err != nil { + return err + } + if kc.Spec.PrivateNetworkCluster { + err = kc.Spec.OnPremisesSpec.ValidateSSHGatewayCreation() + if err != nil { + return err + } + } + } + appVersions, err := kcv.API.ListAppVersions(models.KafkaConnectAppKind) if err != nil { return fmt.Errorf("cannot list versions for kind: %v, err: %w", @@ -97,10 +110,6 @@ func (kcv *kafkaConnectValidator) ValidateCreate(ctx context.Context, obj runtim return err } - if len(kc.Spec.DataCentres) == 0 { - return fmt.Errorf("data centres field is empty") - } - if len(kc.Spec.TargetCluster) > 1 { return fmt.Errorf("targetCluster array size must be between 0 and 1") } @@ -130,10 +139,26 @@ func (kcv *kafkaConnectValidator) ValidateCreate(ctx context.Context, obj runtim return fmt.Errorf("customConnectors array size must be between 0 and 1") } + if len(kc.Spec.DataCentres) == 0 { + return fmt.Errorf("data centres field is empty") + } + + //TODO: add support of multiple DCs for OnPrem clusters + if len(kc.Spec.DataCentres) > 1 && kc.Spec.OnPremisesSpec != nil { + return fmt.Errorf("on-premises cluster can be provisioned with only one data centre") + } + for _, dc := range kc.Spec.DataCentres { - err := dc.ValidateCreation() - if err != nil { - return err + if kc.Spec.OnPremisesSpec != nil { + err := dc.DataCentre.ValidateOnPremisesCreation() + if err != nil { + return err + } + } else { + err := dc.DataCentre.ValidateCreation() + if err != nil { + return err + } } err = validateReplicationFactor(models.KafkaConnectReplicationFactors, dc.ReplicationFactor) diff --git a/apis/clusters/v1beta1/postgresql_types.go b/apis/clusters/v1beta1/postgresql_types.go index bde017933..4aae8eb93 100644 --- a/apis/clusters/v1beta1/postgresql_types.go +++ b/apis/clusters/v1beta1/postgresql_types.go @@ -24,9 +24,11 @@ import ( "unicode" k8sCore "k8s.io/api/core/v1" + k8scorev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -75,6 +77,7 @@ type PgRestoreFrom struct { type PgSpec struct { PgRestoreFrom *PgRestoreFrom `json:"pgRestoreFrom,omitempty"` Cluster `json:",inline"` + OnPremisesSpec *OnPremisesSpec `json:"onPremisesSpec,omitempty"` DataCentres []*PgDataCentre `json:"dataCentres,omitempty"` ClusterConfigurations map[string]string `json:"clusterConfigurations,omitempty"` SynchronousModeStrict bool `json:"synchronousModeStrict,omitempty"` @@ -645,3 +648,34 @@ func GetDefaultPgUserSecret( return userSecret, nil } + +func (pg *PostgreSQL) GetExposePorts() []k8scorev1.ServicePort { + var exposePorts []k8scorev1.ServicePort + if !pg.Spec.PrivateNetworkCluster { + exposePorts = []k8scorev1.ServicePort{ + { + Name: models.PostgreSQLDB, + Port: models.Port5432, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port5432, + }, + }, + } + } + return exposePorts +} + +func (pg *PostgreSQL) GetHeadlessPorts() []k8scorev1.ServicePort { + headlessPorts := []k8scorev1.ServicePort{ + { + Name: models.PostgreSQLDB, + Port: models.Port5432, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port5432, + }, + }, + } + return headlessPorts +} diff --git a/apis/clusters/v1beta1/postgresql_webhook.go b/apis/clusters/v1beta1/postgresql_webhook.go index 733fe5866..f6358341e 100644 --- a/apis/clusters/v1beta1/postgresql_webhook.go +++ b/apis/clusters/v1beta1/postgresql_webhook.go @@ -95,6 +95,19 @@ func (pgv *pgValidator) ValidateCreate(ctx context.Context, obj runtime.Object) return err } + if pg.Spec.OnPremisesSpec != nil { + err = pg.Spec.OnPremisesSpec.ValidateCreation() + if err != nil { + return err + } + if pg.Spec.PrivateNetworkCluster { + err = pg.Spec.OnPremisesSpec.ValidateSSHGatewayCreation() + if err != nil { + return err + } + } + } + if pg.Spec.UserRefs != nil { err = pgv.validatePostgreSQLUsers(ctx, pg) if err != nil { @@ -117,10 +130,22 @@ func (pgv *pgValidator) ValidateCreate(ctx context.Context, obj runtime.Object) return models.ErrZeroDataCentres } + //TODO: add support of multiple DCs for OnPrem clusters + if len(pg.Spec.DataCentres) > 1 && pg.Spec.OnPremisesSpec != nil { + return fmt.Errorf("on-premises cluster can be provisioned with only one data centre") + } + for _, dc := range pg.Spec.DataCentres { - err = dc.DataCentre.ValidateCreation() - if err != nil { - return err + if pg.Spec.OnPremisesSpec != nil { + err := dc.DataCentre.ValidateOnPremisesCreation() + if err != nil { + return err + } + } else { + err := dc.DataCentre.ValidateCreation() + if err != nil { + return err + } } err = dc.ValidatePGBouncer() diff --git a/apis/clusters/v1beta1/redis_types.go b/apis/clusters/v1beta1/redis_types.go index 02c1df4b4..34c5d9624 100644 --- a/apis/clusters/v1beta1/redis_types.go +++ b/apis/clusters/v1beta1/redis_types.go @@ -21,7 +21,9 @@ import ( "fmt" "strconv" + k8scorev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -60,8 +62,9 @@ type RedisRestoreFrom struct { // RedisSpec defines the desired state of Redis type RedisSpec struct { - RestoreFrom *RedisRestoreFrom `json:"restoreFrom,omitempty"` - Cluster `json:",inline"` + RestoreFrom *RedisRestoreFrom `json:"restoreFrom,omitempty"` + Cluster `json:",inline"` + OnPremisesSpec *OnPremisesSpec `json:"onPremisesSpec,omitempty"` // Enables client to node encryption ClientEncryption bool `json:"clientEncryption,omitempty"` @@ -492,3 +495,50 @@ func (r *Redis) SetClusterID(id string) { func init() { SchemeBuilder.Register(&Redis{}, &RedisList{}) } + +func (r *Redis) GetExposePorts() []k8scorev1.ServicePort { + var exposePorts []k8scorev1.ServicePort + if !r.Spec.PrivateNetworkCluster { + exposePorts = []k8scorev1.ServicePort{ + { + Name: models.RedisDB, + Port: models.Port6379, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port6379, + }, + }, + { + Name: models.RedisBus, + Port: models.Port16379, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port16379, + }, + }, + } + } + return exposePorts +} + +func (r *Redis) GetHeadlessPorts() []k8scorev1.ServicePort { + headlessPorts := []k8scorev1.ServicePort{ + { + Name: models.RedisDB, + Port: models.Port6379, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port6379, + }, + }, + { + Name: models.RedisBus, + Port: models.Port16379, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port16379, + }, + }, + } + return headlessPorts +} diff --git a/apis/clusters/v1beta1/redis_webhook.go b/apis/clusters/v1beta1/redis_webhook.go index e9e902376..f2629409e 100644 --- a/apis/clusters/v1beta1/redis_webhook.go +++ b/apis/clusters/v1beta1/redis_webhook.go @@ -96,6 +96,19 @@ func (rv *redisValidator) ValidateCreate(ctx context.Context, obj runtime.Object return err } + if r.Spec.OnPremisesSpec != nil { + err = r.Spec.OnPremisesSpec.ValidateCreation() + if err != nil { + return err + } + if r.Spec.PrivateNetworkCluster { + err = r.Spec.OnPremisesSpec.ValidateSSHGatewayCreation() + if err != nil { + return err + } + } + } + err = r.Spec.ValidatePrivateLink() if err != nil { return err @@ -116,10 +129,21 @@ func (rv *redisValidator) ValidateCreate(ctx context.Context, obj runtime.Object return fmt.Errorf("data centres field is empty") } + if len(r.Spec.DataCentres) > 1 && r.Spec.OnPremisesSpec != nil { + return fmt.Errorf("on-premises cluster can be provisioned with only one data centre") + } + for _, dc := range r.Spec.DataCentres { - err = dc.ValidateCreate() - if err != nil { - return err + if r.Spec.OnPremisesSpec != nil { + err := dc.DataCentre.ValidateOnPremisesCreation() + if err != nil { + return err + } + } else { + err := dc.DataCentre.ValidateCreation() + if err != nil { + return err + } } if !r.Spec.PrivateNetworkCluster && dc.PrivateLink != nil { diff --git a/apis/clusters/v1beta1/structs.go b/apis/clusters/v1beta1/structs.go index 082bd126e..7c8016c40 100644 --- a/apis/clusters/v1beta1/structs.go +++ b/apis/clusters/v1beta1/structs.go @@ -120,6 +120,18 @@ type ClusteredMaintenanceEvent struct { Upcoming []*clusterresource.MaintenanceEventStatus `json:"upcoming"` } +type OnPremisesSpec struct { + StorageClassName string `json:"storageClassName"` + OSDiskSize string `json:"osDiskSize"` + DataDiskSize string `json:"dataDiskSize"` + SSHGatewayCPU int64 `json:"sshGatewayCPU,omitempty"` + SSHGatewayMemory string `json:"sshGatewayMemory,omitempty"` + NodeCPU int64 `json:"nodeCPU"` + NodeMemory string `json:"nodeMemory"` + OSImageURL string `json:"osImageURL"` + CloudInitScriptRef *Reference `json:"cloudInitScriptRef"` +} + type TwoFactorDelete struct { // Email address which will be contacted when the cluster is requested to be deleted. Email string `json:"email"` diff --git a/apis/clusters/v1beta1/validation.go b/apis/clusters/v1beta1/validation.go index 697b6ad15..ab95a6f1e 100644 --- a/apis/clusters/v1beta1/validation.go +++ b/apis/clusters/v1beta1/validation.go @@ -94,6 +94,40 @@ func (dc *DataCentre) ValidateCreation() error { return nil } +func (ops *OnPremisesSpec) ValidateCreation() error { + osDiskSizeMatched, err := regexp.Match(models.StorageRegExp, []byte(ops.OSDiskSize)) + if !osDiskSizeMatched || err != nil { + return fmt.Errorf("disk size field for node OS must fit pattern: %s", + models.StorageRegExp) + } + + dataDiskSizeMatched, err := regexp.Match(models.StorageRegExp, []byte(ops.DataDiskSize)) + if !dataDiskSizeMatched || err != nil { + return fmt.Errorf("disk size field for storring cluster data must fit pattern: %s", + models.StorageRegExp) + } + + nodeMemoryMatched, err := regexp.Match(models.MemoryRegExp, []byte(ops.DataDiskSize)) + if !nodeMemoryMatched || err != nil { + return fmt.Errorf("node memory field must fit pattern: %s", + models.MemoryRegExp) + } + + return nil +} + +func (ops *OnPremisesSpec) ValidateSSHGatewayCreation() error { + if ops.SSHGatewayCPU == 0 || ops.SSHGatewayMemory == "" { + return fmt.Errorf("fields SSHGatewayCPU and SSHGatewayMemory must not be empty") + } + sshGatewayMemoryMatched, err := regexp.Match(models.MemoryRegExp, []byte(ops.DataDiskSize)) + if !sshGatewayMemoryMatched || err != nil { + return fmt.Errorf("ssh gateway memory field must fit pattern: %s", + models.MemoryRegExp) + } + return nil +} + func (dc *DataCentre) validateImmutableCloudProviderSettingsUpdate(oldSettings []*CloudProviderSettings) error { if len(oldSettings) != len(dc.CloudProviderSettings) { return models.ErrImmutableCloudProviderSettings @@ -205,3 +239,17 @@ func validateSingleConcurrentResize(concurrentResizes int) error { return nil } + +func (dc *DataCentre) ValidateOnPremisesCreation() error { + if dc.CloudProvider != models.ONPREMISES { + return fmt.Errorf("cloud provider %s is unavailable for data centre: %s, available value: %s", + dc.CloudProvider, dc.Name, models.ONPREMISES) + } + + if dc.Region != models.CLIENTDC { + return fmt.Errorf("region %s is unavailable for data centre: %s, available value: %s", + dc.Region, dc.Name, models.CLIENTDC) + } + + return nil +} diff --git a/apis/clusters/v1beta1/zz_generated.deepcopy.go b/apis/clusters/v1beta1/zz_generated.deepcopy.go index e19d6b661..4526c650b 100644 --- a/apis/clusters/v1beta1/zz_generated.deepcopy.go +++ b/apis/clusters/v1beta1/zz_generated.deepcopy.go @@ -232,6 +232,11 @@ func (in *CadenceList) DeepCopyObject() runtime.Object { func (in *CadenceSpec) DeepCopyInto(out *CadenceSpec) { *out = *in in.Cluster.DeepCopyInto(&out.Cluster) + if in.OnPremisesSpec != nil { + in, out := &in.OnPremisesSpec, &out.OnPremisesSpec + *out = new(OnPremisesSpec) + (*in).DeepCopyInto(*out) + } if in.DataCentres != nil { in, out := &in.DataCentres, &out.DataCentres *out = make([]*CadenceDataCentre, len(*in)) @@ -462,6 +467,11 @@ func (in *CassandraSpec) DeepCopyInto(out *CassandraSpec) { *out = new(CassandraRestoreFrom) (*in).DeepCopyInto(*out) } + if in.OnPremisesSpec != nil { + in, out := &in.OnPremisesSpec, &out.OnPremisesSpec + *out = new(OnPremisesSpec) + (*in).DeepCopyInto(*out) + } in.Cluster.DeepCopyInto(&out.Cluster) if in.DataCentres != nil { in, out := &in.DataCentres, &out.DataCentres @@ -1006,6 +1016,11 @@ func (in *KafkaConnectList) DeepCopyObject() runtime.Object { func (in *KafkaConnectSpec) DeepCopyInto(out *KafkaConnectSpec) { *out = *in in.Cluster.DeepCopyInto(&out.Cluster) + if in.OnPremisesSpec != nil { + in, out := &in.OnPremisesSpec, &out.OnPremisesSpec + *out = new(OnPremisesSpec) + (*in).DeepCopyInto(*out) + } if in.DataCentres != nil { in, out := &in.DataCentres, &out.DataCentres *out = make([]*KafkaConnectDataCentre, len(*in)) @@ -1130,6 +1145,11 @@ func (in *KafkaList) DeepCopyObject() runtime.Object { func (in *KafkaSpec) DeepCopyInto(out *KafkaSpec) { *out = *in in.Cluster.DeepCopyInto(&out.Cluster) + if in.OnPremisesSpec != nil { + in, out := &in.OnPremisesSpec, &out.OnPremisesSpec + *out = new(OnPremisesSpec) + (*in).DeepCopyInto(*out) + } if in.SchemaRegistry != nil { in, out := &in.SchemaRegistry, &out.SchemaRegistry *out = make([]*SchemaRegistry, len(*in)) @@ -1348,6 +1368,26 @@ func (in *Node) DeepCopy() *Node { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OnPremisesSpec) DeepCopyInto(out *OnPremisesSpec) { + *out = *in + if in.CloudInitScriptRef != nil { + in, out := &in.CloudInitScriptRef, &out.CloudInitScriptRef + *out = new(Reference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OnPremisesSpec. +func (in *OnPremisesSpec) DeepCopy() *OnPremisesSpec { + if in == nil { + return nil + } + out := new(OnPremisesSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenSearch) DeepCopyInto(out *OpenSearch) { *out = *in @@ -1800,6 +1840,11 @@ func (in *PgSpec) DeepCopyInto(out *PgSpec) { (*in).DeepCopyInto(*out) } in.Cluster.DeepCopyInto(&out.Cluster) + if in.OnPremisesSpec != nil { + in, out := &in.OnPremisesSpec, &out.OnPremisesSpec + *out = new(OnPremisesSpec) + (*in).DeepCopyInto(*out) + } if in.DataCentres != nil { in, out := &in.DataCentres, &out.DataCentres *out = make([]*PgDataCentre, len(*in)) @@ -2108,6 +2153,11 @@ func (in *RedisSpec) DeepCopyInto(out *RedisSpec) { (*in).DeepCopyInto(*out) } in.Cluster.DeepCopyInto(&out.Cluster) + if in.OnPremisesSpec != nil { + in, out := &in.OnPremisesSpec, &out.OnPremisesSpec + *out = new(OnPremisesSpec) + (*in).DeepCopyInto(*out) + } if in.DataCentres != nil { in, out := &in.DataCentres, &out.DataCentres *out = make([]*RedisDataCentre, len(*in)) diff --git a/config/crd/bases/clusters.instaclustr.com_cadences.yaml b/config/crd/bases/clusters.instaclustr.com_cadences.yaml index f91754c22..dcb64f211 100644 --- a/config/crd/bases/clusters.instaclustr.com_cadences.yaml +++ b/config/crd/bases/clusters.instaclustr.com_cadences.yaml @@ -125,6 +125,45 @@ spec: name: description: Name [ 3 .. 32 ] characters. type: string + onPremisesSpec: + properties: + cloudInitScriptRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + dataDiskSize: + type: string + nodeCPU: + format: int64 + type: integer + nodeMemory: + type: string + osDiskSize: + type: string + osImageURL: + type: string + sshGatewayCPU: + format: int64 + type: integer + sshGatewayMemory: + type: string + storageClassName: + type: string + required: + - cloudInitScriptRef + - dataDiskSize + - nodeCPU + - nodeMemory + - osDiskSize + - osImageURL + - storageClassName + type: object packagedProvisioning: items: properties: diff --git a/config/crd/bases/clusters.instaclustr.com_cassandras.yaml b/config/crd/bases/clusters.instaclustr.com_cassandras.yaml index 7bf382dcd..0554645d9 100644 --- a/config/crd/bases/clusters.instaclustr.com_cassandras.yaml +++ b/config/crd/bases/clusters.instaclustr.com_cassandras.yaml @@ -131,6 +131,45 @@ spec: name: description: Name [ 3 .. 32 ] characters. type: string + onPremisesSpec: + properties: + cloudInitScriptRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + dataDiskSize: + type: string + nodeCPU: + format: int64 + type: integer + nodeMemory: + type: string + osDiskSize: + type: string + osImageURL: + type: string + sshGatewayCPU: + format: int64 + type: integer + sshGatewayMemory: + type: string + storageClassName: + type: string + required: + - cloudInitScriptRef + - dataDiskSize + - nodeCPU + - nodeMemory + - osDiskSize + - osImageURL + - storageClassName + type: object passwordAndUserAuth: type: boolean pciCompliance: diff --git a/config/crd/bases/clusters.instaclustr.com_kafkaconnects.yaml b/config/crd/bases/clusters.instaclustr.com_kafkaconnects.yaml index 412f5bd96..12afb57f9 100644 --- a/config/crd/bases/clusters.instaclustr.com_kafkaconnects.yaml +++ b/config/crd/bases/clusters.instaclustr.com_kafkaconnects.yaml @@ -179,6 +179,45 @@ spec: name: description: Name [ 3 .. 32 ] characters. type: string + onPremisesSpec: + properties: + cloudInitScriptRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + dataDiskSize: + type: string + nodeCPU: + format: int64 + type: integer + nodeMemory: + type: string + osDiskSize: + type: string + osImageURL: + type: string + sshGatewayCPU: + format: int64 + type: integer + sshGatewayMemory: + type: string + storageClassName: + type: string + required: + - cloudInitScriptRef + - dataDiskSize + - nodeCPU + - nodeMemory + - osDiskSize + - osImageURL + - storageClassName + type: object pciCompliance: description: The PCI compliance standards relate to the security of user data and transactional information. Can only be applied clusters diff --git a/config/crd/bases/clusters.instaclustr.com_kafkas.yaml b/config/crd/bases/clusters.instaclustr.com_kafkas.yaml index a2b0cfbfe..68774390f 100644 --- a/config/crd/bases/clusters.instaclustr.com_kafkas.yaml +++ b/config/crd/bases/clusters.instaclustr.com_kafkas.yaml @@ -165,6 +165,45 @@ spec: name: description: Name [ 3 .. 32 ] characters. type: string + onPremisesSpec: + properties: + cloudInitScriptRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + dataDiskSize: + type: string + nodeCPU: + format: int64 + type: integer + nodeMemory: + type: string + osDiskSize: + type: string + osImageURL: + type: string + sshGatewayCPU: + format: int64 + type: integer + sshGatewayMemory: + type: string + storageClassName: + type: string + required: + - cloudInitScriptRef + - dataDiskSize + - nodeCPU + - nodeMemory + - osDiskSize + - osImageURL + - storageClassName + type: object partitionsNumber: description: PartitionsNumber number of partitions to use when created new topics. diff --git a/config/crd/bases/clusters.instaclustr.com_postgresqls.yaml b/config/crd/bases/clusters.instaclustr.com_postgresqls.yaml index 208795a36..9d928bc2e 100644 --- a/config/crd/bases/clusters.instaclustr.com_postgresqls.yaml +++ b/config/crd/bases/clusters.instaclustr.com_postgresqls.yaml @@ -132,6 +132,45 @@ spec: name: description: Name [ 3 .. 32 ] characters. type: string + onPremisesSpec: + properties: + cloudInitScriptRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + dataDiskSize: + type: string + nodeCPU: + format: int64 + type: integer + nodeMemory: + type: string + osDiskSize: + type: string + osImageURL: + type: string + sshGatewayCPU: + format: int64 + type: integer + sshGatewayMemory: + type: string + storageClassName: + type: string + required: + - cloudInitScriptRef + - dataDiskSize + - nodeCPU + - nodeMemory + - osDiskSize + - osImageURL + - storageClassName + type: object pciCompliance: description: The PCI compliance standards relate to the security of user data and transactional information. Can only be applied clusters diff --git a/config/crd/bases/clusters.instaclustr.com_redis.yaml b/config/crd/bases/clusters.instaclustr.com_redis.yaml index 21c7ea9cd..0f6b7ae93 100644 --- a/config/crd/bases/clusters.instaclustr.com_redis.yaml +++ b/config/crd/bases/clusters.instaclustr.com_redis.yaml @@ -115,6 +115,45 @@ spec: name: description: Name [ 3 .. 32 ] characters. type: string + onPremisesSpec: + properties: + cloudInitScriptRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + dataDiskSize: + type: string + nodeCPU: + format: int64 + type: integer + nodeMemory: + type: string + osDiskSize: + type: string + osImageURL: + type: string + sshGatewayCPU: + format: int64 + type: integer + sshGatewayMemory: + type: string + storageClassName: + type: string + required: + - cloudInitScriptRef + - dataDiskSize + - nodeCPU + - nodeMemory + - osDiskSize + - osImageURL + - storageClassName + type: object passwordAndUserAuth: type: boolean pciCompliance: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ee225136e..167097051 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -32,6 +32,31 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -39,6 +64,7 @@ rules: verbs: - create - delete + - deletecollection - get - list - patch @@ -51,6 +77,20 @@ rules: verbs: - create - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - cdi.kubevirt.io + resources: + - datavolumes + verbs: + - create + - delete + - deletecollection - get - list - patch @@ -796,3 +836,29 @@ rules: - get - patch - update +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch diff --git a/config/samples/clusters_v1beta1_cassandra.yaml b/config/samples/clusters_v1beta1_cassandra.yaml index c81e23181..784b5eb55 100644 --- a/config/samples/clusters_v1beta1_cassandra.yaml +++ b/config/samples/clusters_v1beta1_cassandra.yaml @@ -42,20 +42,19 @@ spec: pciCompliance: false luceneEnabled: false # can be enabled only on 3.11.13 version of Cassandra passwordAndUserAuth: true -# userRefs: -# - namespace: default -# name: cassandrauser-sample -# - namespace: default -# name: cassandrauser-sample2 -# - namespace: default -# name: cassandrauser-sample3 + # userRefs: + # - namespace: default + # name: cassandrauser-sample + # - namespace: default + # name: cassandrauser-sample2 + # - namespace: default + # name: cassandrauser-sample3 slaTier: "NON_PRODUCTION" -# resizeSettings: -# - notifySupportContacts: false -# concurrency: 2 -# description: "this is a sample of description" -# twoFactorDelete: -# - email: "rostyslp@netapp.com" - #spark: - # - version: "2.3.2" # 3.0.1 for 4.0.4 version of Cassandra | 2.3.2 for 3.11.13 version of Cassandra - + # resizeSettings: + # - notifySupportContacts: false + # concurrency: 2 + # description: "this is a sample of description" + # twoFactorDelete: + # - email: "rostyslp@netapp.com" + #spark: + # - version: "2.3.2" # 3.0.1 for 4.0.4 version of Cassandra | 2.3.2 for 3.11.13 version of Cassandra \ No newline at end of file diff --git a/config/samples/onpremises/clusters_v1beta1_cadence.yaml b/config/samples/onpremises/clusters_v1beta1_cadence.yaml new file mode 100644 index 000000000..5652834a4 --- /dev/null +++ b/config/samples/onpremises/clusters_v1beta1_cadence.yaml @@ -0,0 +1,34 @@ +apiVersion: clusters.instaclustr.com/v1beta1 +kind: Cadence +metadata: + name: cadence-sample +spec: + name: "cadence-test" + version: "1.0.0" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: instaclustr-cloud-init-secret + standardProvisioning: + - targetCassandra: + dependencyCdcId: "9d43ac54-7317-4ce5-859a-e9d0443508a4" + dependencyVpcType: "TARGET_VPC" + privateNetworkCluster: false + dataCentres: + - region: "CLIENT_DC" + network: "10.1.0.0/16" + cloudProvider: "ONPREMISES" + name: "testdc" + nodeSize: "CAD-DEV-OP.4.8-200" + nodesNumber: 2 + clientEncryption: false + slaTier: "NON_PRODUCTION" + useCadenceWebAuth: false \ No newline at end of file diff --git a/config/samples/onpremises/clusters_v1beta1_cassandra.yaml b/config/samples/onpremises/clusters_v1beta1_cassandra.yaml new file mode 100644 index 000000000..8b7968872 --- /dev/null +++ b/config/samples/onpremises/clusters_v1beta1_cassandra.yaml @@ -0,0 +1,37 @@ +apiVersion: clusters.instaclustr.com/v1beta1 +kind: Cassandra +metadata: + name: cassandra-on-prem-cluster +spec: + name: "danylo-on-prem-cassandra" + version: "4.0.10" + privateNetworkCluster: false + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: instaclustr-cloud-init-secret + dataCentres: + - name: "onPremCassandra" + region: "CLIENT_DC" + cloudProvider: "ONPREMISES" + continuousBackup: false + nodesNumber: 2 + replicationFactor: 2 + privateIpBroadcastForDiscovery: false + network: "192.168.0.0/16" + tags: + "onprem": "test" + clientToClusterEncryption: false + nodeSize: "CAS-PRD-OP.4.8-200" + pciCompliance: false + luceneEnabled: false # can be enabled only on 3.11.13 version of Cassandra + passwordAndUserAuth: false + slaTier: "NON_PRODUCTION" diff --git a/config/samples/onpremises/clusters_v1beta1_kafka.yaml b/config/samples/onpremises/clusters_v1beta1_kafka.yaml new file mode 100644 index 000000000..79f5574b6 --- /dev/null +++ b/config/samples/onpremises/clusters_v1beta1_kafka.yaml @@ -0,0 +1,40 @@ +apiVersion: clusters.instaclustr.com/v1beta1 +kind: Kafka +metadata: + name: danylo-kafka +spec: + name: "danylo-kafka" + version: "3.3.1" + pciCompliance: false + replicationFactor: 3 + partitionsNumber: 3 + allowDeleteTopics: true + autoCreateTopics: true + clientToClusterEncryption: false + privateNetworkCluster: false + slaTier: "NON_PRODUCTION" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: instaclustr-cloud-init-secret + dataCentres: + - name: "onPremKafka" + nodesNumber: 3 + cloudProvider: "ONPREMISES" + tags: + tag: "oneTag" + tag2: "twoTags" + nodeSize: "KFK-DEV-OP.4.8-200" + network: "10.0.0.0/16" + region: "CLIENT_DC" + resizeSettings: + - notifySupportContacts: false + concurrency: 1 \ No newline at end of file diff --git a/config/samples/onpremises/clusters_v1beta1_kafkaconnect.yaml b/config/samples/onpremises/clusters_v1beta1_kafkaconnect.yaml new file mode 100644 index 000000000..e334b11ba --- /dev/null +++ b/config/samples/onpremises/clusters_v1beta1_kafkaconnect.yaml @@ -0,0 +1,36 @@ +apiVersion: clusters.instaclustr.com/v1beta1 +kind: KafkaConnect +metadata: + name: kafkaconnect-sample +spec: + name: "kafkaconnect-onprem" + version: "3.1.2" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: instaclustr-cloud-init-secret + privateNetworkCluster: false + slaTier: "NON_PRODUCTION" + targetCluster: + - managedCluster: + - targetKafkaClusterId: "34dfc53c-c8c1-4be8-bd2f-cfdb77ec7349" + kafkaConnectVpcType: "KAFKA_VPC" + dataCentres: + - name: "kafkaconnect-onprem" + nodesNumber: 3 + cloudProvider: "ONPREMISES" + replicationFactor: 3 + tags: + tag: "oneTag" + tag2: "twoTags" + nodeSize: "KCN-DEV-OP.4.8-200" + network: "10.15.0.0/16" + region: "CLIENT_DC" diff --git a/config/samples/onpremises/clusters_v1beta1_postgresql.yaml b/config/samples/onpremises/clusters_v1beta1_postgresql.yaml new file mode 100644 index 000000000..d729db449 --- /dev/null +++ b/config/samples/onpremises/clusters_v1beta1_postgresql.yaml @@ -0,0 +1,34 @@ +apiVersion: clusters.instaclustr.com/v1beta1 +kind: PostgreSQL +metadata: + name: postgresql-sample +spec: + name: "postgresql-sample" + version: "15.4.0" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: instaclustr-cloud-init-secret + dataCentres: + - region: "CLIENT_DC" + network: "10.1.0.0/16" + cloudProvider: "ONPREMISES" + nodeSize: "PGS-DEV-OP.4.8-200" + nodesNumber: 2 + clientEncryption: false + name: "testDC1" + intraDataCentreReplication: + - replicationMode: "SYNCHRONOUS" + interDataCentreReplication: + - isPrimaryDataCentre: true + slaTier: "NON_PRODUCTION" + privateNetworkCluster: false + synchronousModeStrict: false \ No newline at end of file diff --git a/config/samples/onpremises/clusters_v1beta1_redis.yaml b/config/samples/onpremises/clusters_v1beta1_redis.yaml new file mode 100644 index 000000000..3c5722b62 --- /dev/null +++ b/config/samples/onpremises/clusters_v1beta1_redis.yaml @@ -0,0 +1,32 @@ +apiVersion: clusters.instaclustr.com/v1beta1 +kind: Redis +metadata: + name: danylo-redis +spec: + name: "danylo-redis" + version: "7.0.14" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: instaclustr-cloud-init-secret + slaTier: "NON_PRODUCTION" + clientEncryption: false + passwordAndUserAuth: true + privateNetworkCluster: false + dataCentres: + - region: "CLIENT_DC" + name: "onPremRedis" + cloudProvider: "ONPREMISES" + network: "10.1.0.0/16" + nodeSize: "RDS-PRD-OP.8.64-400" + masterNodes: 3 + nodesNumber: 0 + replicationFactor: 0 \ No newline at end of file diff --git a/controllers/clusterresources/awsendpointserviceprincipal_controller.go b/controllers/clusterresources/awsendpointserviceprincipal_controller.go index c28c11b78..85f58b4b9 100644 --- a/controllers/clusterresources/awsendpointserviceprincipal_controller.go +++ b/controllers/clusterresources/awsendpointserviceprincipal_controller.go @@ -101,7 +101,7 @@ func (r *AWSEndpointServicePrincipalReconciler) handleCreate(ctx context.Context err = json.Unmarshal(b, &principal.Status) if err != nil { l.Error(err, "failed to parse an AWS endpoint service principal resource response from Instaclustr") - r.EventRecorder.Eventf(principal, models.Warning, models.ConvertionFailed, + r.EventRecorder.Eventf(principal, models.Warning, models.ConversionFailed, "Failed to parse an AWS endpoint service principal resource response from Instaclustr. Reason: %v", err, ) diff --git a/controllers/clusterresources/clusterbackup_controller.go b/controllers/clusterresources/clusterbackup_controller.go index aa25574f9..db42d76d2 100644 --- a/controllers/clusterresources/clusterbackup_controller.go +++ b/controllers/clusterresources/clusterbackup_controller.go @@ -172,7 +172,7 @@ func (r *ClusterBackupReconciler) Reconcile(ctx context.Context, req ctrl.Reques ) r.EventRecorder.Eventf( - backup, models.Warning, models.ConvertionFailed, + backup, models.Warning, models.ConversionFailed, "Start timestamp annotation convertion to int is failed. Reason: %v", err, ) diff --git a/controllers/clusters/cadence_controller.go b/controllers/clusters/cadence_controller.go index a22f993dd..ce4b61f91 100644 --- a/controllers/clusters/cadence_controller.go +++ b/controllers/clusters/cadence_controller.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/instaclustr/operator/apis/clusters/v1beta1" "github.com/instaclustr/operator/pkg/exposeservice" @@ -56,8 +57,15 @@ type CadenceReconciler struct { //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=cadences,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=cadences/status,verbs=get;update;patch //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=cadences/finalizers,verbs=update -//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;patch //+kubebuilder:rbac:groups="",resources=events,verbs=create +//+kubebuilder:rbac:groups="",resources=endpoints,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachines,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete;deletecollection // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -65,92 +73,92 @@ type CadenceReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile func (r *CadenceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) + l := log.FromContext(ctx) - cadenceCluster := &v1beta1.Cadence{} - err := r.Client.Get(ctx, req.NamespacedName, cadenceCluster) + c := &v1beta1.Cadence{} + err := r.Client.Get(ctx, req.NamespacedName, c) if err != nil { if k8serrors.IsNotFound(err) { - logger.Info("Cadence resource is not found", + l.Info("Cadence resource is not found", "resource name", req.NamespacedName, ) return ctrl.Result{}, nil } - logger.Error(err, "Unable to fetch Cadence resource", + l.Error(err, "Unable to fetch Cadence resource", "resource name", req.NamespacedName, ) return ctrl.Result{}, err } - switch cadenceCluster.Annotations[models.ResourceStateAnnotation] { + switch c.Annotations[models.ResourceStateAnnotation] { case models.CreatingEvent: - return r.HandleCreateCluster(ctx, cadenceCluster, logger) + return r.handleCreateCluster(ctx, c, l) case models.UpdatingEvent: - return r.HandleUpdateCluster(ctx, cadenceCluster, logger) + return r.handleUpdateCluster(ctx, c, l) case models.DeletingEvent: - return r.HandleDeleteCluster(ctx, cadenceCluster, logger) + return r.handleDeleteCluster(ctx, c, l) case models.GenericEvent: - logger.Info("Generic event isn't handled", + l.Info("Generic event isn't handled", "request", req, - "event", cadenceCluster.Annotations[models.ResourceStateAnnotation], + "event", c.Annotations[models.ResourceStateAnnotation], ) return ctrl.Result{}, nil default: - logger.Info("Unknown event isn't handled", + l.Info("Unknown event isn't handled", "request", req, - "event", cadenceCluster.Annotations[models.ResourceStateAnnotation], + "event", c.Annotations[models.ResourceStateAnnotation], ) return ctrl.Result{}, nil } } -func (r *CadenceReconciler) HandleCreateCluster( +func (r *CadenceReconciler) handleCreateCluster( ctx context.Context, - cadence *v1beta1.Cadence, - logger logr.Logger, + c *v1beta1.Cadence, + l logr.Logger, ) (ctrl.Result, error) { - if cadence.Status.ID == "" { - patch := cadence.NewPatch() + if c.Status.ID == "" { + patch := c.NewPatch() - for _, packagedProvisioning := range cadence.Spec.PackagedProvisioning { - requeueNeeded, err := r.preparePackagedSolution(ctx, cadence, packagedProvisioning) + for _, packagedProvisioning := range c.Spec.PackagedProvisioning { + requeueNeeded, err := r.preparePackagedSolution(ctx, c, packagedProvisioning) if err != nil { - logger.Error(err, "Cannot prepare packaged solution for Cadence cluster", - "cluster name", cadence.Spec.Name, + l.Error(err, "Cannot prepare packaged solution for Cadence cluster", + "cluster name", c.Spec.Name, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.CreationFailed, + r.EventRecorder.Eventf(c, models.Warning, models.CreationFailed, "Cannot prepare packaged solution for Cadence cluster. Reason: %v", err) return ctrl.Result{}, err } if requeueNeeded { - logger.Info("Waiting for bundled clusters to be created", - "cadence cluster name", cadence.Spec.Name) + l.Info("Waiting for bundled clusters to be created", + "c cluster name", c.Spec.Name) - r.EventRecorder.Event(cadence, models.Normal, "Waiting", + r.EventRecorder.Event(c, models.Normal, "Waiting", "Waiting for bundled clusters to be created") return models.ReconcileRequeue, nil } } - logger.Info( + l.Info( "Creating Cadence cluster", - "cluster name", cadence.Spec.Name, - "data centres", cadence.Spec.DataCentres, + "cluster name", c.Spec.Name, + "data centres", c.Spec.DataCentres, ) - cadenceAPISpec, err := cadence.Spec.ToInstAPI(ctx, r.Client) + cadenceAPISpec, err := c.Spec.ToInstAPI(ctx, r.Client) if err != nil { - logger.Error(err, "Cannot convert Cadence cluster manifest to API spec", - "cluster manifest", cadence.Spec) + l.Error(err, "Cannot convert Cadence cluster manifest to API spec", + "cluster manifest", c.Spec) - r.EventRecorder.Eventf(cadence, models.Warning, models.ConvertionFailed, + r.EventRecorder.Eventf(c, models.Warning, models.ConversionFailed, "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", err) return ctrl.Result{}, err @@ -158,141 +166,218 @@ func (r *CadenceReconciler) HandleCreateCluster( id, err := r.API.CreateCluster(instaclustr.CadenceEndpoint, cadenceAPISpec) if err != nil { - logger.Error( + l.Error( err, "Cannot create Cadence cluster", - "cadence manifest", cadence.Spec, + "c manifest", c.Spec, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.CreationFailed, + r.EventRecorder.Eventf(c, models.Warning, models.CreationFailed, "Cluster creation on the Instaclustr is failed. Reason: %v", err) return ctrl.Result{}, err } - cadence.Status.ID = id - err = r.Status().Patch(ctx, cadence, patch) + c.Status.ID = id + err = r.Status().Patch(ctx, c, patch) if err != nil { - logger.Error(err, "Cannot update Cadence cluster status", - "cluster name", cadence.Spec.Name, - "cluster status", cadence.Status, + l.Error(err, "Cannot update Cadence cluster status", + "cluster name", c.Spec.Name, + "cluster status", c.Status, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.PatchFailed, + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, "Cluster resource status patch is failed. Reason: %v", err) return ctrl.Result{}, err } - if cadence.Spec.Description != "" { - err = r.API.UpdateDescriptionAndTwoFactorDelete(instaclustr.ClustersEndpointV1, id, cadence.Spec.Description, nil) + if c.Spec.Description != "" { + err = r.API.UpdateDescriptionAndTwoFactorDelete(instaclustr.ClustersEndpointV1, id, c.Spec.Description, nil) if err != nil { - logger.Error(err, "Cannot update Cadence cluster description and TwoFactorDelete", - "cluster name", cadence.Spec.Name, - "description", cadence.Spec.Description, - "twoFactorDelete", cadence.Spec.TwoFactorDelete, + l.Error(err, "Cannot update Cadence cluster description and TwoFactorDelete", + "cluster name", c.Spec.Name, + "description", c.Spec.Description, + "twoFactorDelete", c.Spec.TwoFactorDelete, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.CreationFailed, + r.EventRecorder.Eventf(c, models.Warning, models.CreationFailed, "Cluster description and TwoFactoDelete update is failed. Reason: %v", err) } } - cadence.Annotations[models.ResourceStateAnnotation] = models.CreatedEvent - controllerutil.AddFinalizer(cadence, models.DeletionFinalizer) + c.Annotations[models.ResourceStateAnnotation] = models.CreatedEvent + controllerutil.AddFinalizer(c, models.DeletionFinalizer) - err = r.Patch(ctx, cadence, patch) + err = r.Patch(ctx, c, patch) if err != nil { - logger.Error(err, "Cannot patch Cadence cluster", - "cluster name", cadence.Spec.Name, "patch", patch) + l.Error(err, "Cannot patch Cadence cluster", + "cluster name", c.Spec.Name, "patch", patch) - r.EventRecorder.Eventf(cadence, models.Warning, models.PatchFailed, + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, "Cluster resource status patch is failed. Reason: %v", err) return ctrl.Result{}, err } - logger.Info( + l.Info( "Cadence resource has been created", - "cluster name", cadence.Name, - "cluster ID", cadence.Status.ID, - "kind", cadence.Kind, - "api version", cadence.APIVersion, - "namespace", cadence.Namespace, + "cluster name", c.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api version", c.APIVersion, + "namespace", c.Namespace, ) - r.EventRecorder.Eventf(cadence, models.Normal, models.Created, + r.EventRecorder.Eventf(c, models.Normal, models.Created, "Cluster creation request is sent. Cluster ID: %s", id) } - if cadence.Status.State != models.DeletedStatus { - err := r.startClusterStatusJob(cadence) + if c.Status.State != models.DeletedStatus { + err := r.startClusterStatusJob(c) if err != nil { - logger.Error(err, "Cannot start cluster status job", - "cadence cluster ID", cadence.Status.ID, + l.Error(err, "Cannot start cluster status job", + "c cluster ID", c.Status.ID, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.CreationFailed, + r.EventRecorder.Eventf(c, models.Warning, models.CreationFailed, "Cluster status check job is failed. Reason: %v", err) return ctrl.Result{}, err } - r.EventRecorder.Event(cadence, models.Normal, models.Created, + r.EventRecorder.Event(c, models.Normal, models.Created, "Cluster status check job is started") } + if c.Spec.OnPremisesSpec != nil { + iData, err := r.API.GetCadence(c.Status.ID) + if err != nil { + l.Error(err, "Cannot get cluster from the Instaclustr API", + "cluster name", c.Spec.Name, + "data centres", c.Spec.DataCentres, + "cluster ID", c.Status.ID, + ) + r.EventRecorder.Eventf( + c, models.Warning, models.FetchFailed, + "Cluster fetch from the Instaclustr API is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + iCadence, err := c.FromInstAPI(iData) + if err != nil { + l.Error( + err, "Cannot convert cluster from the Instaclustr API", + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + ) + r.EventRecorder.Eventf( + c, models.Warning, models.ConversionFailed, + "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + bootstrap := newOnPremisesBootstrap( + r.Client, + c, + r.EventRecorder, + iCadence.Status.ClusterStatus, + c.Spec.OnPremisesSpec, + newExposePorts(c.GetExposePorts()), + c.GetHeadlessPorts(), + c.Spec.PrivateNetworkCluster, + ) + + err = handleCreateOnPremisesClusterResources(ctx, bootstrap) + if err != nil { + l.Error( + err, "Cannot create resources for on-premises cluster", + "cluster spec", c.Spec.OnPremisesSpec, + ) + r.EventRecorder.Eventf( + c, models.Warning, models.CreationFailed, + "Resources creation for on-premises cluster is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + err = r.startClusterOnPremisesIPsJob(c, bootstrap) + if err != nil { + l.Error(err, "Cannot start on-premises cluster IPs check job", + "cluster ID", c.Status.ID, + ) + + r.EventRecorder.Eventf( + c, models.Warning, models.CreationFailed, + "On-premises cluster IPs check job is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + l.Info( + "On-premises resources have been created", + "cluster name", c.Spec.Name, + "on-premises Spec", c.Spec.OnPremisesSpec, + "cluster ID", c.Status.ID, + ) + return models.ExitReconcile, nil + } return ctrl.Result{}, nil } -func (r *CadenceReconciler) HandleUpdateCluster( +func (r *CadenceReconciler) handleUpdateCluster( ctx context.Context, - cadence *v1beta1.Cadence, - logger logr.Logger, + c *v1beta1.Cadence, + l logr.Logger, ) (ctrl.Result, error) { - iData, err := r.API.GetCadence(cadence.Status.ID) + iData, err := r.API.GetCadence(c.Status.ID) if err != nil { - logger.Error( + l.Error( err, "Cannot get Cadence cluster from the Instaclustr API", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.FetchFailed, + r.EventRecorder.Eventf(c, models.Warning, models.FetchFailed, "Cluster fetch from the Instaclustr API is failed. Reason: %v", err) return ctrl.Result{}, err } - iCadence, err := cadence.FromInstAPI(iData) + iCadence, err := c.FromInstAPI(iData) if err != nil { - logger.Error( + l.Error( err, "Cannot convert Cadence cluster from the Instaclustr API", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.ConvertionFailed, + r.EventRecorder.Eventf(c, models.Warning, models.ConversionFailed, "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", err) return ctrl.Result{}, err } if iCadence.Status.CurrentClusterOperationStatus != models.NoOperation { - logger.Info("Cadence cluster is not ready to update", + l.Info("Cadence cluster is not ready to update", "cluster name", iCadence.Spec.Name, "cluster state", iCadence.Status.State, "current operation status", iCadence.Status.CurrentClusterOperationStatus, ) - patch := cadence.NewPatch() - cadence.Annotations[models.ResourceStateAnnotation] = models.UpdatingEvent - cadence.Annotations[models.UpdateQueuedAnnotation] = models.True - err = r.Patch(ctx, cadence, patch) + patch := c.NewPatch() + c.Annotations[models.ResourceStateAnnotation] = models.UpdatingEvent + c.Annotations[models.UpdateQueuedAnnotation] = models.True + err = r.Patch(ctx, c, patch) if err != nil { - logger.Error(err, "Cannot patch Cadence cluster", - "cluster name", cadence.Spec.Name, + l.Error(err, "Cannot patch Cadence cluster", + "cluster name", c.Spec.Name, "patch", patch) - r.EventRecorder.Eventf(cadence, models.Warning, models.PatchFailed, + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err) return ctrl.Result{}, err @@ -301,236 +386,271 @@ func (r *CadenceReconciler) HandleUpdateCluster( return models.ReconcileRequeue, nil } - if cadence.Annotations[models.ExternalChangesAnnotation] == models.True { - return r.handleExternalChanges(cadence, iCadence, logger) + if c.Annotations[models.ExternalChangesAnnotation] == models.True { + return r.handleExternalChanges(c, iCadence, l) } - if cadence.Spec.ClusterSettingsNeedUpdate(iCadence.Spec.Cluster) { - logger.Info("Updating cluster settings", + if c.Spec.ClusterSettingsNeedUpdate(iCadence.Spec.Cluster) { + l.Info("Updating cluster settings", "instaclustr description", iCadence.Spec.Description, "instaclustr two factor delete", iCadence.Spec.TwoFactorDelete) - err = r.API.UpdateClusterSettings(cadence.Status.ID, cadence.Spec.ClusterSettingsUpdateToInstAPI()) + err = r.API.UpdateClusterSettings(c.Status.ID, c.Spec.ClusterSettingsUpdateToInstAPI()) if err != nil { - logger.Error(err, "Cannot update cluster settings", - "cluster ID", cadence.Status.ID, "cluster spec", cadence.Spec) - r.EventRecorder.Eventf(cadence, models.Warning, models.UpdateFailed, + l.Error(err, "Cannot update cluster settings", + "cluster ID", c.Status.ID, "cluster spec", c.Spec) + r.EventRecorder.Eventf(c, models.Warning, models.UpdateFailed, "Cannot update cluster settings. Reason: %v", err) return ctrl.Result{}, err } } - logger.Info("Update request to Instaclustr API has been sent", - "spec data centres", cadence.Spec.DataCentres, - "resize settings", cadence.Spec.ResizeSettings, + l.Info("Update request to Instaclustr API has been sent", + "spec data centres", c.Spec.DataCentres, + "resize settings", c.Spec.ResizeSettings, ) - err = r.API.UpdateCluster(cadence.Status.ID, instaclustr.CadenceEndpoint, cadence.Spec.NewDCsUpdate()) + err = r.API.UpdateCluster(c.Status.ID, instaclustr.CadenceEndpoint, c.Spec.NewDCsUpdate()) if err != nil { - logger.Error(err, "Cannot update Cadence cluster", - "cluster ID", cadence.Status.ID, - "update request", cadence.Spec.NewDCsUpdate(), + l.Error(err, "Cannot update Cadence cluster", + "cluster ID", c.Status.ID, + "update request", c.Spec.NewDCsUpdate(), ) - r.EventRecorder.Eventf(cadence, models.Warning, models.UpdateFailed, + r.EventRecorder.Eventf(c, models.Warning, models.UpdateFailed, "Cluster update on the Instaclustr API is failed. Reason: %v", err) return ctrl.Result{}, err } - patch := cadence.NewPatch() - cadence.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent - cadence.Annotations[models.UpdateQueuedAnnotation] = "" - err = r.Patch(ctx, cadence, patch) + patch := c.NewPatch() + c.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent + c.Annotations[models.UpdateQueuedAnnotation] = "" + err = r.Patch(ctx, c, patch) if err != nil { - logger.Error(err, "Cannot patch Cadence cluster", - "cluster name", cadence.Spec.Name, + l.Error(err, "Cannot patch Cadence cluster", + "cluster name", c.Spec.Name, "patch", patch) - r.EventRecorder.Eventf(cadence, models.Warning, models.PatchFailed, + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err) return ctrl.Result{}, err } - logger.Info( + l.Info( "Cluster has been updated", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID, - "data centres", cadence.Spec.DataCentres, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "data centres", c.Spec.DataCentres, ) return ctrl.Result{}, nil } -func (r *CadenceReconciler) handleExternalChanges(cadence, iCadence *v1beta1.Cadence, l logr.Logger) (ctrl.Result, error) { - if !cadence.Spec.AreDCsEqual(iCadence.Spec.DataCentres) { +func (r *CadenceReconciler) handleExternalChanges(c, iCadence *v1beta1.Cadence, l logr.Logger) (ctrl.Result, error) { + if !c.Spec.AreDCsEqual(iCadence.Spec.DataCentres) { l.Info(msgExternalChanges, "instaclustr data", iCadence.Spec.DataCentres, - "k8s resource spec", cadence.Spec.DataCentres) + "k8s resource spec", c.Spec.DataCentres) - msgDiffSpecs, err := createSpecDifferenceMessage(cadence.Spec.DataCentres, iCadence.Spec.DataCentres) + msgDiffSpecs, err := createSpecDifferenceMessage(c.Spec.DataCentres, iCadence.Spec.DataCentres) if err != nil { l.Error(err, "Cannot create specification difference message", - "instaclustr data", iCadence.Spec, "k8s resource spec", cadence.Spec) + "instaclustr data", iCadence.Spec, "k8s resource spec", c.Spec) return ctrl.Result{}, err } - r.EventRecorder.Eventf(cadence, models.Warning, models.ExternalChanges, msgDiffSpecs) + r.EventRecorder.Eventf(c, models.Warning, models.ExternalChanges, msgDiffSpecs) return ctrl.Result{}, nil } - patch := cadence.NewPatch() + patch := c.NewPatch() - cadence.Annotations[models.ExternalChangesAnnotation] = "" + c.Annotations[models.ExternalChangesAnnotation] = "" - err := r.Patch(context.Background(), cadence, patch) + err := r.Patch(context.Background(), c, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "cluster name", cadence.Spec.Name, "cluster ID", cadence.Status.ID) + "cluster name", c.Spec.Name, "cluster ID", c.Status.ID) - r.EventRecorder.Eventf(cadence, models.Warning, models.PatchFailed, + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err) return ctrl.Result{}, err } - l.Info("External changes have been reconciled", "resource ID", cadence.Status.ID) - r.EventRecorder.Event(cadence, models.Normal, models.ExternalChanges, "External changes have been reconciled") + l.Info("External changes have been reconciled", "resource ID", c.Status.ID) + r.EventRecorder.Event(c, models.Normal, models.ExternalChanges, "External changes have been reconciled") return ctrl.Result{}, nil } -func (r *CadenceReconciler) HandleDeleteCluster( +func (r *CadenceReconciler) handleDeleteCluster( ctx context.Context, - cadence *v1beta1.Cadence, - logger logr.Logger, + c *v1beta1.Cadence, + l logr.Logger, ) (ctrl.Result, error) { - _, err := r.API.GetCadence(cadence.Status.ID) + _, err := r.API.GetCadence(c.Status.ID) if err != nil && !errors.Is(err, instaclustr.NotFound) { - logger.Error( + l.Error( err, "Cannot get Cadence cluster status from the Instaclustr API", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.FetchFailed, + r.EventRecorder.Eventf(c, models.Warning, models.FetchFailed, "Cluster resource fetch from the Instaclustr API is failed. Reason: %v", err) return ctrl.Result{}, err } if !errors.Is(err, instaclustr.NotFound) { - logger.Info("Sending cluster deletion to the Instaclustr API", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID) + l.Info("Sending cluster deletion to the Instaclustr API", + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID) - err = r.API.DeleteCluster(cadence.Status.ID, instaclustr.CadenceEndpoint) + err = r.API.DeleteCluster(c.Status.ID, instaclustr.CadenceEndpoint) if err != nil { - logger.Error(err, "Cannot delete Cadence cluster", - "cluster name", cadence.Spec.Name, - "cluster status", cadence.Status, + l.Error(err, "Cannot delete Cadence cluster", + "cluster name", c.Spec.Name, + "cluster status", c.Status, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.DeletionFailed, + r.EventRecorder.Eventf(c, models.Warning, models.DeletionFailed, "Cluster deletion is failed on the Instaclustr. Reason: %v", err) return ctrl.Result{}, err } - r.EventRecorder.Event(cadence, models.Normal, models.DeletionStarted, + r.EventRecorder.Event(c, models.Normal, models.DeletionStarted, "Cluster deletion request is sent to the Instaclustr API.") - if cadence.Spec.TwoFactorDelete != nil { - patch := cadence.NewPatch() + if c.Spec.TwoFactorDelete != nil { + patch := c.NewPatch() - cadence.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent - cadence.Annotations[models.ClusterDeletionAnnotation] = models.Triggered - err = r.Patch(ctx, cadence, patch) + c.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent + c.Annotations[models.ClusterDeletionAnnotation] = models.Triggered + err = r.Patch(ctx, c, patch) if err != nil { - logger.Error(err, "Cannot patch cluster resource", - "cluster name", cadence.Spec.Name, - "cluster state", cadence.Status.State) - r.EventRecorder.Eventf(cadence, models.Warning, models.PatchFailed, + l.Error(err, "Cannot patch cluster resource", + "cluster name", c.Spec.Name, + "cluster state", c.Status.State) + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err) return ctrl.Result{}, err } - logger.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", cadence.Status.ID) + l.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", c.Status.ID) - r.EventRecorder.Event(cadence, models.Normal, models.DeletionStarted, + r.EventRecorder.Event(c, models.Normal, models.DeletionStarted, "Two-Factor Delete is enabled, please confirm cluster deletion via email or phone.") - return ctrl.Result{}, nil } + if c.Spec.OnPremisesSpec != nil { + err = deleteOnPremResources(ctx, r.Client, c.Status.ID, c.Namespace) + if err != nil { + l.Error(err, "Cannot delete cluster on-premises resources", + "cluster ID", c.Status.ID) + r.EventRecorder.Eventf(c, models.Warning, models.DeletionFailed, + "Cluster on-premises resources deletion is failed. Reason: %v", err) + return reconcile.Result{}, err + } + + l.Info("Cluster on-premises resources are deleted", + "cluster ID", c.Status.ID) + r.EventRecorder.Eventf(c, models.Normal, models.Deleted, + "Cluster on-premises resources are deleted") + + patch := c.NewPatch() + controllerutil.RemoveFinalizer(c, models.DeletionFinalizer) + + err = r.Patch(ctx, c, patch) + if err != nil { + l.Error(err, "Cannot patch cluster resource", + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api Version", c.APIVersion, + "namespace", c.Namespace, + "cluster metadata", c.ObjectMeta, + ) + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, + "Cluster resource patch is failed. Reason: %v", err) + return reconcile.Result{}, err + } + + return reconcile.Result{}, err + } + r.Scheduler.RemoveJob(c.GetJobID(scheduler.OnPremisesIPsChecker)) } - logger.Info("Cadence cluster is being deleted", - "cluster name", cadence.Spec.Name, - "cluster status", cadence.Status) + l.Info("Cadence cluster is being deleted", + "cluster name", c.Spec.Name, + "cluster status", c.Status) - for _, packagedProvisioning := range cadence.Spec.PackagedProvisioning { - err = r.deletePackagedResources(ctx, cadence, packagedProvisioning) + for _, packagedProvisioning := range c.Spec.PackagedProvisioning { + err = r.deletePackagedResources(ctx, c, packagedProvisioning) if err != nil { - logger.Error( + l.Error( err, "Cannot delete Cadence packaged resources", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) - r.EventRecorder.Eventf(cadence, models.Warning, models.DeletionFailed, + r.EventRecorder.Eventf(c, models.Warning, models.DeletionFailed, "Cannot delete Cadence packaged resources. Reason: %v", err) return ctrl.Result{}, err } } - r.Scheduler.RemoveJob(cadence.GetJobID(scheduler.StatusChecker)) - patch := cadence.NewPatch() - controllerutil.RemoveFinalizer(cadence, models.DeletionFinalizer) - cadence.Annotations[models.ResourceStateAnnotation] = models.DeletedEvent + r.Scheduler.RemoveJob(c.GetJobID(scheduler.StatusChecker)) + patch := c.NewPatch() + controllerutil.RemoveFinalizer(c, models.DeletionFinalizer) + c.Annotations[models.ResourceStateAnnotation] = models.DeletedEvent - err = r.Patch(ctx, cadence, patch) + err = r.Patch(ctx, c, patch) if err != nil { - logger.Error(err, "Cannot patch Cadence cluster", - "cluster name", cadence.Spec.Name, + l.Error(err, "Cannot patch Cadence cluster", + "cluster name", c.Spec.Name, "patch", patch, ) return ctrl.Result{}, err } - err = exposeservice.Delete(r.Client, cadence.Name, cadence.Namespace) + err = exposeservice.Delete(r.Client, c.Name, c.Namespace) if err != nil { - logger.Error(err, "Cannot delete Cadence cluster expose service", - "cluster ID", cadence.Status.ID, - "cluster name", cadence.Spec.Name, + l.Error(err, "Cannot delete Cadence cluster expose service", + "cluster ID", c.Status.ID, + "cluster name", c.Spec.Name, ) return ctrl.Result{}, err } - logger.Info("Cadence cluster was deleted", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID, + l.Info("Cadence cluster was deleted", + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) - r.EventRecorder.Event(cadence, models.Normal, models.Deleted, "Cluster resource is deleted") + r.EventRecorder.Event(c, models.Normal, models.Deleted, "Cluster resource is deleted") return ctrl.Result{}, nil } func (r *CadenceReconciler) preparePackagedSolution( ctx context.Context, - cluster *v1beta1.Cadence, + c *v1beta1.Cadence, packagedProvisioning *v1beta1.PackagedProvisioning, ) (bool, error) { - if len(cluster.Spec.DataCentres) < 1 { + if len(c.Spec.DataCentres) < 1 { return false, models.ErrZeroDataCentres } - labelsToQuery := fmt.Sprintf("%s=%s", models.ControlledByLabel, cluster.Name) + labelsToQuery := fmt.Sprintf("%s=%s", models.ControlledByLabel, c.Name) selector, err := labels.Parse(labelsToQuery) if err != nil { return false, err @@ -555,7 +675,7 @@ func (r *CadenceReconciler) preparePackagedSolution( models.CassandraAppKind) } - cassandraSpec, err := r.newCassandraSpec(cluster, cassandraVersions[len(cassandraVersions)-1].String()) + cassandraSpec, err := r.newCassandraSpec(c, cassandraVersions[len(cassandraVersions)-1].String()) if err != nil { return false, err } @@ -593,7 +713,7 @@ func (r *CadenceReconciler) preparePackagedSolution( models.KafkaAppType) } - kafkaSpec, err := r.newKafkaSpec(cluster, kafkaVersions[len(kafkaVersions)-1].String()) + kafkaSpec, err := r.newKafkaSpec(c, kafkaVersions[len(kafkaVersions)-1].String()) if err != nil { return false, err } @@ -631,7 +751,7 @@ func (r *CadenceReconciler) preparePackagedSolution( } // For OpenSearch we cannot use the latest version because is not supported by Cadence. So we use the oldest one. - osSpec, err := r.newOpenSearchSpec(cluster, openSearchVersions[0].String()) + osSpec, err := r.newOpenSearchSpec(c, openSearchVersions[0].String()) if err != nil { return false, err } @@ -657,7 +777,7 @@ func (r *CadenceReconciler) preparePackagedSolution( return true, nil } - cluster.Spec.StandardProvisioning = append(cluster.Spec.StandardProvisioning, &v1beta1.StandardProvisioning{ + c.Spec.StandardProvisioning = append(c.Spec.StandardProvisioning, &v1beta1.StandardProvisioning{ AdvancedVisibility: advancedVisibilities, TargetCassandra: &v1beta1.TargetCassandra{ DependencyCDCID: cassandraList.Items[0].Status.DataCentres[0].ID, @@ -668,42 +788,42 @@ func (r *CadenceReconciler) preparePackagedSolution( return false, nil } -func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence, latestCassandraVersion string) (*v1beta1.Cassandra, error) { +func (r *CadenceReconciler) newCassandraSpec(c *v1beta1.Cadence, latestCassandraVersion string) (*v1beta1.Cassandra, error) { typeMeta := v1.TypeMeta{ Kind: models.CassandraKind, APIVersion: models.ClustersV1beta1APIVersion, } metadata := v1.ObjectMeta{ - Name: models.CassandraChildPrefix + cadence.Name, - Labels: map[string]string{models.ControlledByLabel: cadence.Name}, + Name: models.CassandraChildPrefix + c.Name, + Labels: map[string]string{models.ControlledByLabel: c.Name}, Annotations: map[string]string{models.ResourceStateAnnotation: models.CreatingEvent}, - Namespace: cadence.ObjectMeta.Namespace, + Namespace: c.ObjectMeta.Namespace, Finalizers: []string{}, } - if len(cadence.Spec.DataCentres) == 0 { + if len(c.Spec.DataCentres) == 0 { return nil, models.ErrZeroDataCentres } - slaTier := cadence.Spec.SLATier - privateClusterNetwork := cadence.Spec.PrivateNetworkCluster - pciCompliance := cadence.Spec.PCICompliance + slaTier := c.Spec.SLATier + privateClusterNetwork := c.Spec.PrivateNetworkCluster + pciCompliance := c.Spec.PCICompliance var twoFactorDelete []*v1beta1.TwoFactorDelete - if len(cadence.Spec.TwoFactorDelete) > 0 { + if len(c.Spec.TwoFactorDelete) > 0 { twoFactorDelete = []*v1beta1.TwoFactorDelete{ { - Email: cadence.Spec.TwoFactorDelete[0].Email, - Phone: cadence.Spec.TwoFactorDelete[0].Phone, + Email: c.Spec.TwoFactorDelete[0].Email, + Phone: c.Spec.TwoFactorDelete[0].Phone, }, } } var cassNodeSize, network string var cassNodesNumber, cassReplicationFactor int var cassPrivateIPBroadcastForDiscovery, cassPasswordAndUserAuth bool - for _, dc := range cadence.Spec.DataCentres { - for _, pp := range cadence.Spec.PackagedProvisioning { + for _, dc := range c.Spec.DataCentres { + for _, pp := range c.Spec.PackagedProvisioning { cassNodeSize = pp.BundledCassandraSpec.NodeSize network = pp.BundledCassandraSpec.Network cassNodesNumber = pp.BundledCassandraSpec.NodesNumber @@ -722,9 +842,9 @@ func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence, latestCas } dcName := models.CassandraChildDCName - dcRegion := cadence.Spec.DataCentres[0].Region - cloudProvider := cadence.Spec.DataCentres[0].CloudProvider - providerAccountName := cadence.Spec.DataCentres[0].ProviderAccountName + dcRegion := c.Spec.DataCentres[0].Region + cloudProvider := c.Spec.DataCentres[0].CloudProvider + providerAccountName := c.Spec.DataCentres[0].ProviderAccountName cassandraDataCentres := []*v1beta1.CassandraDataCentre{ { @@ -743,7 +863,7 @@ func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence, latestCas } spec := v1beta1.CassandraSpec{ Cluster: v1beta1.Cluster{ - Name: models.CassandraChildPrefix + cadence.Name, + Name: models.CassandraChildPrefix + c.Name, Version: latestCassandraVersion, SLATier: slaTier, PrivateNetworkCluster: privateClusterNetwork, @@ -762,10 +882,21 @@ func (r *CadenceReconciler) newCassandraSpec(cadence *v1beta1.Cadence, latestCas }, nil } -func (r *CadenceReconciler) startClusterStatusJob(cadence *v1beta1.Cadence) error { - job := r.newWatchStatusJob(cadence) +func (r *CadenceReconciler) startClusterOnPremisesIPsJob(c *v1beta1.Cadence, b *onPremisesBootstrap) error { + job := newWatchOnPremisesIPsJob(c.Kind, b) + + err := r.Scheduler.ScheduleJob(c.GetJobID(scheduler.OnPremisesIPsChecker), scheduler.ClusterStatusInterval, job) + if err != nil { + return err + } + + return nil +} + +func (r *CadenceReconciler) startClusterStatusJob(c *v1beta1.Cadence) error { + job := r.newWatchStatusJob(c) - err := r.Scheduler.ScheduleJob(cadence.GetJobID(scheduler.StatusChecker), scheduler.ClusterStatusInterval, job) + err := r.Scheduler.ScheduleJob(c.GetJobID(scheduler.StatusChecker), scheduler.ClusterStatusInterval, job) if err != nil { return err } @@ -773,66 +904,66 @@ func (r *CadenceReconciler) startClusterStatusJob(cadence *v1beta1.Cadence) erro return nil } -func (r *CadenceReconciler) newWatchStatusJob(cadence *v1beta1.Cadence) scheduler.Job { +func (r *CadenceReconciler) newWatchStatusJob(c *v1beta1.Cadence) scheduler.Job { l := log.Log.WithValues("component", "cadenceStatusClusterJob") return func() error { - namespacedName := client.ObjectKeyFromObject(cadence) - err := r.Get(context.Background(), namespacedName, cadence) + namespacedName := client.ObjectKeyFromObject(c) + err := r.Get(context.Background(), namespacedName, c) if k8serrors.IsNotFound(err) { l.Info("Resource is not found in the k8s cluster. Closing Instaclustr status sync.", "namespaced name", namespacedName) - r.Scheduler.RemoveJob(cadence.GetJobID(scheduler.StatusChecker)) + r.Scheduler.RemoveJob(c.GetJobID(scheduler.StatusChecker)) return nil } if err != nil { l.Error(err, "Cannot get Cadence custom resource", - "resource name", cadence.Name, + "resource name", c.Name, ) return err } - iData, err := r.API.GetCadence(cadence.Status.ID) + iData, err := r.API.GetCadence(c.Status.ID) if err != nil { if errors.Is(err, instaclustr.NotFound) { - if cadence.DeletionTimestamp != nil { - _, err = r.HandleDeleteCluster(context.Background(), cadence, l) + if c.DeletionTimestamp != nil { + _, err = r.handleDeleteCluster(context.Background(), c, l) return err } - return r.handleExternalDelete(context.Background(), cadence) + return r.handleExternalDelete(context.Background(), c) } l.Error(err, "Cannot get Cadence cluster from the Instaclustr API", - "clusterID", cadence.Status.ID, + "clusterID", c.Status.ID, ) return err } - iCadence, err := cadence.FromInstAPI(iData) + iCadence, err := c.FromInstAPI(iData) if err != nil { l.Error(err, "Cannot convert Cadence cluster from the Instaclustr API", - "clusterID", cadence.Status.ID, + "clusterID", c.Status.ID, ) return err } - if !areStatusesEqual(&iCadence.Status.ClusterStatus, &cadence.Status.ClusterStatus) || - !areSecondaryCadenceTargetsEqual(cadence.Status.TargetSecondaryCadence, iCadence.Status.TargetSecondaryCadence) { + if !areStatusesEqual(&iCadence.Status.ClusterStatus, &c.Status.ClusterStatus) || + !areSecondaryCadenceTargetsEqual(c.Status.TargetSecondaryCadence, iCadence.Status.TargetSecondaryCadence) { l.Info("Updating Cadence cluster status", "new status", iCadence.Status.ClusterStatus, - "old status", cadence.Status.ClusterStatus, + "old status", c.Status.ClusterStatus, ) - areDCsEqual := areDataCentresEqual(iCadence.Status.ClusterStatus.DataCentres, cadence.Status.ClusterStatus.DataCentres) + areDCsEqual := areDataCentresEqual(iCadence.Status.ClusterStatus.DataCentres, c.Status.ClusterStatus.DataCentres) - patch := cadence.NewPatch() - cadence.Status.ClusterStatus = iCadence.Status.ClusterStatus - cadence.Status.TargetSecondaryCadence = iCadence.Status.TargetSecondaryCadence - err = r.Status().Patch(context.Background(), cadence, patch) + patch := c.NewPatch() + c.Status.ClusterStatus = iCadence.Status.ClusterStatus + c.Status.TargetSecondaryCadence = iCadence.Status.TargetSecondaryCadence + err = r.Status().Patch(context.Background(), c, patch) if err != nil { l.Error(err, "Cannot patch Cadence cluster", - "cluster name", cadence.Spec.Name, - "status", cadence.Status.State, + "cluster name", c.Spec.Name, + "status", c.Status.State, ) return err } @@ -845,9 +976,9 @@ func (r *CadenceReconciler) newWatchStatusJob(cadence *v1beta1.Cadence) schedule } err = exposeservice.Create(r.Client, - cadence.Name, - cadence.Namespace, - cadence.Spec.PrivateNetworkCluster, + c.Name, + c.Namespace, + c.Spec.PrivateNetworkCluster, nodes, models.CadenceConnectionPort) if err != nil { @@ -857,50 +988,50 @@ func (r *CadenceReconciler) newWatchStatusJob(cadence *v1beta1.Cadence) schedule } if iCadence.Status.CurrentClusterOperationStatus == models.NoOperation && - cadence.Annotations[models.ResourceStateAnnotation] != models.UpdatingEvent && - cadence.Annotations[models.UpdateQueuedAnnotation] != models.True && - !cadence.Spec.AreDCsEqual(iCadence.Spec.DataCentres) { + c.Annotations[models.ResourceStateAnnotation] != models.UpdatingEvent && + c.Annotations[models.UpdateQueuedAnnotation] != models.True && + !c.Spec.AreDCsEqual(iCadence.Spec.DataCentres) { l.Info(msgExternalChanges, "instaclustr data", iCadence.Spec.DataCentres, - "k8s resource spec", cadence.Spec.DataCentres) + "k8s resource spec", c.Spec.DataCentres) - patch := cadence.NewPatch() - cadence.Annotations[models.ExternalChangesAnnotation] = models.True + patch := c.NewPatch() + c.Annotations[models.ExternalChangesAnnotation] = models.True - err = r.Patch(context.Background(), cadence, patch) + err = r.Patch(context.Background(), c, patch) if err != nil { l.Error(err, "Cannot patch cluster cluster", - "cluster name", cadence.Spec.Name, "cluster state", cadence.Status.State) + "cluster name", c.Spec.Name, "cluster state", c.Status.State) return err } - msgDiffSpecs, err := createSpecDifferenceMessage(cadence.Spec.DataCentres, iCadence.Spec.DataCentres) + msgDiffSpecs, err := createSpecDifferenceMessage(c.Spec.DataCentres, iCadence.Spec.DataCentres) if err != nil { l.Error(err, "Cannot create specification difference message", - "instaclustr data", iCadence.Spec, "k8s resource spec", cadence.Spec) + "instaclustr data", iCadence.Spec, "k8s resource spec", c.Spec) return err } - r.EventRecorder.Eventf(cadence, models.Warning, models.ExternalChanges, msgDiffSpecs) + r.EventRecorder.Eventf(c, models.Warning, models.ExternalChanges, msgDiffSpecs) } //TODO: change all context.Background() and context.TODO() to ctx from Reconcile - err = r.reconcileMaintenanceEvents(context.Background(), cadence) + err = r.reconcileMaintenanceEvents(context.Background(), c) if err != nil { l.Error(err, "Cannot reconcile cluster maintenance events", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) return err } - if cadence.Status.State == models.RunningStatus && cadence.Status.CurrentClusterOperationStatus == models.OperationInProgress { - patch := cadence.NewPatch() - for _, dc := range cadence.Status.DataCentres { + if c.Status.State == models.RunningStatus && c.Status.CurrentClusterOperationStatus == models.OperationInProgress { + patch := c.NewPatch() + for _, dc := range c.Status.DataCentres { resizeOperations, err := r.API.GetResizeOperationsByClusterDataCentreID(dc.ID) if err != nil { l.Error(err, "Cannot get data centre resize operations", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, "data centre ID", dc.ID, ) @@ -908,11 +1039,11 @@ func (r *CadenceReconciler) newWatchStatusJob(cadence *v1beta1.Cadence) schedule } dc.ResizeOperations = resizeOperations - err = r.Status().Patch(context.Background(), cadence, patch) + err = r.Status().Patch(context.Background(), c, patch) if err != nil { l.Error(err, "Cannot patch data centre resize operations", - "cluster name", cadence.Spec.Name, - "cluster ID", cadence.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, "data centre ID", dc.ID, ) @@ -925,36 +1056,36 @@ func (r *CadenceReconciler) newWatchStatusJob(cadence *v1beta1.Cadence) schedule } } -func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence, latestKafkaVersion string) (*v1beta1.Kafka, error) { +func (r *CadenceReconciler) newKafkaSpec(c *v1beta1.Cadence, latestKafkaVersion string) (*v1beta1.Kafka, error) { typeMeta := v1.TypeMeta{ Kind: models.KafkaKind, APIVersion: models.ClustersV1beta1APIVersion, } metadata := v1.ObjectMeta{ - Name: models.KafkaChildPrefix + cadence.Name, - Labels: map[string]string{models.ControlledByLabel: cadence.Name}, + Name: models.KafkaChildPrefix + c.Name, + Labels: map[string]string{models.ControlledByLabel: c.Name}, Annotations: map[string]string{models.ResourceStateAnnotation: models.CreatingEvent}, - Namespace: cadence.ObjectMeta.Namespace, + Namespace: c.ObjectMeta.Namespace, Finalizers: []string{}, } - if len(cadence.Spec.DataCentres) == 0 { + if len(c.Spec.DataCentres) == 0 { return nil, models.ErrZeroDataCentres } var kafkaTFD []*v1beta1.TwoFactorDelete - for _, cadenceTFD := range cadence.Spec.TwoFactorDelete { + for _, cadenceTFD := range c.Spec.TwoFactorDelete { twoFactorDelete := &v1beta1.TwoFactorDelete{ Email: cadenceTFD.Email, Phone: cadenceTFD.Phone, } kafkaTFD = append(kafkaTFD, twoFactorDelete) } - bundledKafkaSpec := cadence.Spec.PackagedProvisioning[0].BundledKafkaSpec + bundledKafkaSpec := c.Spec.PackagedProvisioning[0].BundledKafkaSpec kafkaNetwork := bundledKafkaSpec.Network - for _, cadenceDC := range cadence.Spec.DataCentres { + for _, cadenceDC := range c.Spec.DataCentres { isKafkaNetworkOverlaps, err := cadenceDC.IsNetworkOverlaps(kafkaNetwork) if err != nil { return nil, err @@ -967,9 +1098,9 @@ func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence, latestKafkaVe kafkaNodeSize := bundledKafkaSpec.NodeSize kafkaNodesNumber := bundledKafkaSpec.NodesNumber dcName := models.KafkaChildDCName - dcRegion := cadence.Spec.DataCentres[0].Region - cloudProvider := cadence.Spec.DataCentres[0].CloudProvider - providerAccountName := cadence.Spec.DataCentres[0].ProviderAccountName + dcRegion := c.Spec.DataCentres[0].Region + cloudProvider := c.Spec.DataCentres[0].CloudProvider + providerAccountName := c.Spec.DataCentres[0].ProviderAccountName kafkaDataCentres := []*v1beta1.KafkaDataCentre{ { DataCentre: v1beta1.DataCentre{ @@ -984,13 +1115,13 @@ func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence, latestKafkaVe }, } - slaTier := cadence.Spec.SLATier - privateClusterNetwork := cadence.Spec.PrivateNetworkCluster - pciCompliance := cadence.Spec.PCICompliance - clientEncryption := cadence.Spec.DataCentres[0].ClientEncryption + slaTier := c.Spec.SLATier + privateClusterNetwork := c.Spec.PrivateNetworkCluster + pciCompliance := c.Spec.PCICompliance + clientEncryption := c.Spec.DataCentres[0].ClientEncryption spec := v1beta1.KafkaSpec{ Cluster: v1beta1.Cluster{ - Name: models.KafkaChildPrefix + cadence.Name, + Name: models.KafkaChildPrefix + c.Name, Version: latestKafkaVersion, SLATier: slaTier, PrivateNetworkCluster: privateClusterNetwork, @@ -1013,25 +1144,25 @@ func (r *CadenceReconciler) newKafkaSpec(cadence *v1beta1.Cadence, latestKafkaVe }, nil } -func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence, oldestOpenSearchVersion string) (*v1beta1.OpenSearch, error) { +func (r *CadenceReconciler) newOpenSearchSpec(c *v1beta1.Cadence, oldestOpenSearchVersion string) (*v1beta1.OpenSearch, error) { typeMeta := v1.TypeMeta{ Kind: models.OpenSearchKind, APIVersion: models.ClustersV1beta1APIVersion, } metadata := v1.ObjectMeta{ - Name: models.OpenSearchChildPrefix + cadence.Name, - Labels: map[string]string{models.ControlledByLabel: cadence.Name}, + Name: models.OpenSearchChildPrefix + c.Name, + Labels: map[string]string{models.ControlledByLabel: c.Name}, Annotations: map[string]string{models.ResourceStateAnnotation: models.CreatingEvent}, - Namespace: cadence.ObjectMeta.Namespace, + Namespace: c.ObjectMeta.Namespace, Finalizers: []string{}, } - if len(cadence.Spec.DataCentres) < 1 { + if len(c.Spec.DataCentres) < 1 { return nil, models.ErrZeroDataCentres } - bundledOpenSearchSpec := cadence.Spec.PackagedProvisioning[0].BundledOpenSearchSpec + bundledOpenSearchSpec := c.Spec.PackagedProvisioning[0].BundledOpenSearchSpec managerNodes := []*v1beta1.ClusterManagerNodes{{ NodeSize: bundledOpenSearchSpec.NodeSize, @@ -1039,22 +1170,22 @@ func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence, oldestOp }} osReplicationFactor := bundledOpenSearchSpec.ReplicationFactor - slaTier := cadence.Spec.SLATier - privateClusterNetwork := cadence.Spec.PrivateNetworkCluster - pciCompliance := cadence.Spec.PCICompliance + slaTier := c.Spec.SLATier + privateClusterNetwork := c.Spec.PrivateNetworkCluster + pciCompliance := c.Spec.PCICompliance var twoFactorDelete []*v1beta1.TwoFactorDelete - if len(cadence.Spec.TwoFactorDelete) > 0 { + if len(c.Spec.TwoFactorDelete) > 0 { twoFactorDelete = []*v1beta1.TwoFactorDelete{ { - Email: cadence.Spec.TwoFactorDelete[0].Email, - Phone: cadence.Spec.TwoFactorDelete[0].Phone, + Email: c.Spec.TwoFactorDelete[0].Email, + Phone: c.Spec.TwoFactorDelete[0].Phone, }, } } osNetwork := bundledOpenSearchSpec.Network - isOsNetworkOverlaps, err := cadence.Spec.DataCentres[0].IsNetworkOverlaps(osNetwork) + isOsNetworkOverlaps, err := c.Spec.DataCentres[0].IsNetworkOverlaps(osNetwork) if err != nil { return nil, err } @@ -1063,9 +1194,9 @@ func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence, oldestOp } dcName := models.OpenSearchChildDCName - dcRegion := cadence.Spec.DataCentres[0].Region - cloudProvider := cadence.Spec.DataCentres[0].CloudProvider - providerAccountName := cadence.Spec.DataCentres[0].ProviderAccountName + dcRegion := c.Spec.DataCentres[0].Region + cloudProvider := c.Spec.DataCentres[0].CloudProvider + providerAccountName := c.Spec.DataCentres[0].ProviderAccountName osDataCentres := []*v1beta1.OpenSearchDataCentre{ { @@ -1079,7 +1210,7 @@ func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence, oldestOp } spec := v1beta1.OpenSearchSpec{ Cluster: v1beta1.Cluster{ - Name: models.OpenSearchChildPrefix + cadence.Name, + Name: models.OpenSearchChildPrefix + c.Name, Version: oldestOpenSearchVersion, SLATier: slaTier, PrivateNetworkCluster: privateClusterNetwork, @@ -1100,10 +1231,10 @@ func (r *CadenceReconciler) newOpenSearchSpec(cadence *v1beta1.Cadence, oldestOp func (r *CadenceReconciler) deletePackagedResources( ctx context.Context, - cadence *v1beta1.Cadence, + c *v1beta1.Cadence, packagedProvisioning *v1beta1.PackagedProvisioning, ) error { - labelsToQuery := fmt.Sprintf("%s=%s", models.ControlledByLabel, cadence.Name) + labelsToQuery := fmt.Sprintf("%s=%s", models.ControlledByLabel, c.Name) selector, err := labels.Parse(labelsToQuery) if err != nil { return err diff --git a/controllers/clusters/cassandra_controller.go b/controllers/clusters/cassandra_controller.go index 539e2c3fe..a1230b7b7 100644 --- a/controllers/clusters/cassandra_controller.go +++ b/controllers/clusters/cassandra_controller.go @@ -45,10 +45,6 @@ import ( "github.com/instaclustr/operator/pkg/scheduler" ) -const ( - StatusRUNNING = "RUNNING" -) - // CassandraReconciler reconciles a Cassandra object type CassandraReconciler struct { client.Client @@ -62,8 +58,15 @@ type CassandraReconciler struct { //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=cassandras/status,verbs=get;update;patch //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=cassandras/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch -//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=endpoints,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachines,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete;deletecollection // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -73,8 +76,8 @@ type CassandraReconciler struct { func (r *CassandraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { l := log.FromContext(ctx) - cassandra := &v1beta1.Cassandra{} - err := r.Client.Get(ctx, req.NamespacedName, cassandra) + c := &v1beta1.Cassandra{} + err := r.Client.Get(ctx, req.NamespacedName, c) if err != nil { if k8serrors.IsNotFound(err) { l.Info("Cassandra resource is not found", @@ -87,23 +90,23 @@ func (r *CassandraReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return reconcile.Result{}, err } - switch cassandra.Annotations[models.ResourceStateAnnotation] { + switch c.Annotations[models.ResourceStateAnnotation] { case models.CreatingEvent: - return r.handleCreateCluster(ctx, l, cassandra) + return r.handleCreateCluster(ctx, l, c) case models.UpdatingEvent: - return r.handleUpdateCluster(ctx, l, cassandra) + return r.handleUpdateCluster(ctx, l, c) case models.DeletingEvent: - return r.handleDeleteCluster(ctx, l, cassandra) + return r.handleDeleteCluster(ctx, l, c) case models.GenericEvent: l.Info("Event isn't handled", - "cluster name", cassandra.Spec.Name, + "cluster name", c.Spec.Name, "request", req, - "event", cassandra.Annotations[models.ResourceStateAnnotation]) + "event", c.Annotations[models.ResourceStateAnnotation]) return models.ExitReconcile, nil default: l.Info("Event isn't handled", "request", req, - "event", cassandra.Annotations[models.ResourceStateAnnotation]) + "event", c.Annotations[models.ResourceStateAnnotation]) return models.ExitReconcile, nil } @@ -112,27 +115,27 @@ func (r *CassandraReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( func (r *CassandraReconciler) handleCreateCluster( ctx context.Context, l logr.Logger, - cassandra *v1beta1.Cassandra, + c *v1beta1.Cassandra, ) (reconcile.Result, error) { l = l.WithName("Cassandra creation event") var err error - patch := cassandra.NewPatch() - if cassandra.Status.ID == "" { + patch := c.NewPatch() + if c.Status.ID == "" { var id string - if cassandra.Spec.HasRestore() { + if c.Spec.HasRestore() { l.Info( "Creating cluster from backup", - "original cluster ID", cassandra.Spec.RestoreFrom.ClusterID, + "original cluster ID", c.Spec.RestoreFrom.ClusterID, ) - id, err = r.API.RestoreCluster(cassandra.RestoreInfoToInstAPI(cassandra.Spec.RestoreFrom), models.CassandraAppKind) + id, err = r.API.RestoreCluster(c.RestoreInfoToInstAPI(c.Spec.RestoreFrom), models.CassandraAppKind) if err != nil { l.Error(err, "Cannot restore cluster from backup", - "original cluster ID", cassandra.Spec.RestoreFrom.ClusterID, + "original cluster ID", c.Spec.RestoreFrom.ClusterID, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.CreationFailed, + c, models.Warning, models.CreationFailed, "Cluster restore from backup on the Instaclustr is failed. Reason: %v", err, ) @@ -141,26 +144,26 @@ func (r *CassandraReconciler) handleCreateCluster( } r.EventRecorder.Eventf( - cassandra, models.Normal, models.Created, + c, models.Normal, models.Created, "Cluster restore request is sent. Original cluster ID: %s, new cluster ID: %s", - cassandra.Spec.RestoreFrom.ClusterID, + c.Spec.RestoreFrom.ClusterID, id, ) } else { l.Info( "Creating cluster", - "cluster name", cassandra.Spec.Name, - "data centres", cassandra.Spec.DataCentres, + "cluster name", c.Spec.Name, + "data centres", c.Spec.DataCentres, ) - id, err = r.API.CreateCluster(instaclustr.CassandraEndpoint, cassandra.Spec.ToInstAPI()) + id, err = r.API.CreateCluster(instaclustr.CassandraEndpoint, c.Spec.ToInstAPI()) if err != nil { l.Error( err, "Cannot create cluster", - "cluster spec", cassandra.Spec, + "cluster spec", c.Spec, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.CreationFailed, + c, models.Warning, models.CreationFailed, "Cluster creation on the Instaclustr is failed. Reason: %v", err, ) @@ -168,45 +171,45 @@ func (r *CassandraReconciler) handleCreateCluster( } r.EventRecorder.Eventf( - cassandra, models.Normal, models.Created, + c, models.Normal, models.Created, "Cluster creation request is sent. Cluster ID: %s", id, ) } - cassandra.Status.ID = id - err = r.Status().Patch(ctx, cassandra, patch) + c.Status.ID = id + err = r.Status().Patch(ctx, c, patch) if err != nil { l.Error(err, "Cannot patch cluster status", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, - "kind", cassandra.Kind, - "api Version", cassandra.APIVersion, - "namespace", cassandra.Namespace, - "cluster metadata", cassandra.ObjectMeta, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api Version", c.APIVersion, + "namespace", c.Namespace, + "cluster metadata", c.ObjectMeta, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.PatchFailed, + c, models.Warning, models.PatchFailed, "Cluster resource status patch is failed. Reason: %v", err, ) return reconcile.Result{}, err } - controllerutil.AddFinalizer(cassandra, models.DeletionFinalizer) - cassandra.Annotations[models.ResourceStateAnnotation] = models.CreatedEvent - err = r.Patch(ctx, cassandra, patch) + controllerutil.AddFinalizer(c, models.DeletionFinalizer) + c.Annotations[models.ResourceStateAnnotation] = models.CreatedEvent + err = r.Patch(ctx, c, patch) if err != nil { l.Error(err, "Cannot patch cluster", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, - "kind", cassandra.Kind, - "api Version", cassandra.APIVersion, - "namespace", cassandra.Namespace, - "cluster metadata", cassandra.ObjectMeta, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api Version", c.APIVersion, + "namespace", c.Namespace, + "cluster metadata", c.ObjectMeta, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.PatchFailed, + c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err, ) @@ -215,22 +218,22 @@ func (r *CassandraReconciler) handleCreateCluster( l.Info( "Cluster has been created", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, - "kind", cassandra.Kind, - "api Version", cassandra.APIVersion, - "namespace", cassandra.Namespace, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api Version", c.APIVersion, + "namespace", c.Namespace, ) } - if cassandra.Status.State != models.DeletedStatus { - err = r.startClusterStatusJob(cassandra) + if c.Status.State != models.DeletedStatus { + err = r.startClusterStatusJob(c) if err != nil { l.Error(err, "Cannot start cluster status job", - "cassandra cluster ID", cassandra.Status.ID) + "c cluster ID", c.Status.ID) r.EventRecorder.Eventf( - cassandra, models.Warning, models.CreationFailed, + c, models.Warning, models.CreationFailed, "Cluster status check job is failed. Reason: %v", err, ) @@ -238,41 +241,118 @@ func (r *CassandraReconciler) handleCreateCluster( } r.EventRecorder.Eventf( - cassandra, models.Normal, models.Created, + c, models.Normal, models.Created, "Cluster status check job is started", ) + } + if c.Spec.OnPremisesSpec != nil { + iData, err := r.API.GetCassandra(c.Status.ID) + if err != nil { + l.Error(err, "Cannot get cluster from the Instaclustr API", + "cluster name", c.Spec.Name, + "data centres", c.Spec.DataCentres, + "cluster ID", c.Status.ID, + ) + r.EventRecorder.Eventf( + c, models.Warning, models.FetchFailed, + "Cluster fetch from the Instaclustr API is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + iCassandra, err := c.FromInstAPI(iData) + if err != nil { + l.Error( + err, "Cannot convert cluster from the Instaclustr API", + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + ) + r.EventRecorder.Eventf( + c, models.Warning, models.ConversionFailed, + "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + bootstrap := newOnPremisesBootstrap( + r.Client, + c, + r.EventRecorder, + iCassandra.Status.ClusterStatus, + c.Spec.OnPremisesSpec, + newExposePorts(c.GetExposePorts()), + c.GetHeadlessPorts(), + c.Spec.PrivateNetworkCluster, + ) + + err = handleCreateOnPremisesClusterResources(ctx, bootstrap) + if err != nil { + l.Error( + err, "Cannot create resources for on-premises cluster", + "cluster spec", c.Spec.OnPremisesSpec, + ) + r.EventRecorder.Eventf( + c, models.Warning, models.CreationFailed, + "Resources creation for on-premises cluster is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } - err = r.startClusterBackupsJob(cassandra) + err = r.startClusterOnPremisesIPsJob(c, bootstrap) if err != nil { - l.Error(err, "Cannot start cluster backups check job", - "cluster ID", cassandra.Status.ID, + l.Error(err, "Cannot start on-premises cluster IPs check job", + "cluster ID", c.Status.ID, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.CreationFailed, - "Cluster backups check job is failed. Reason: %v", + c, models.Warning, models.CreationFailed, + "On-premises cluster IPs check job is failed. Reason: %v", err, ) return reconcile.Result{}, err } + l.Info( + "On-premises resources have been created", + "cluster name", c.Spec.Name, + "on-premises Spec", c.Spec.OnPremisesSpec, + "cluster ID", c.Status.ID, + ) + return models.ExitReconcile, nil + } + + err = r.startClusterBackupsJob(c) + if err != nil { + l.Error(err, "Cannot start cluster backups check job", + "cluster ID", c.Status.ID, + ) + r.EventRecorder.Eventf( - cassandra, models.Normal, models.Created, - "Cluster backups check job is started", + c, models.Warning, models.CreationFailed, + "Cluster backups check job is failed. Reason: %v", + err, ) + return reconcile.Result{}, err + } - if cassandra.Spec.UserRefs != nil && cassandra.Status.AvailableUsers == nil { - err = r.startUsersCreationJob(cassandra) - if err != nil { - l.Error(err, "Failed to start user creation job") - r.EventRecorder.Eventf(cassandra, models.Warning, models.CreationFailed, - "User creation job is failed. Reason: %v", err) - return reconcile.Result{}, err - } + r.EventRecorder.Eventf( + c, models.Normal, models.Created, + "Cluster backups check job is started", + ) - r.EventRecorder.Event(cassandra, models.Normal, models.Created, - "Cluster user creation job is started") + if c.Spec.UserRefs != nil && c.Status.AvailableUsers == nil { + err = r.startUsersCreationJob(c) + if err != nil { + l.Error(err, "Failed to start user creation job") + r.EventRecorder.Eventf(c, models.Warning, models.CreationFailed, + "User creation job is failed. Reason: %v", err) + return reconcile.Result{}, err } + + r.EventRecorder.Event(c, models.Normal, models.Created, + "Cluster user creation job is started") } return models.ExitReconcile, nil @@ -281,99 +361,99 @@ func (r *CassandraReconciler) handleCreateCluster( func (r *CassandraReconciler) handleUpdateCluster( ctx context.Context, l logr.Logger, - cassandra *v1beta1.Cassandra, + c *v1beta1.Cassandra, ) (reconcile.Result, error) { l = l.WithName("Cassandra update event") - iData, err := r.API.GetCassandra(cassandra.Status.ID) + iData, err := r.API.GetCassandra(c.Status.ID) if err != nil { l.Error(err, "Cannot get cluster from the Instaclustr API", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.FetchFailed, + c, models.Warning, models.FetchFailed, "Cluster fetch from the Instaclustr API is failed. Reason: %v", err, ) return reconcile.Result{}, err } - iCassandra, err := cassandra.FromInstAPI(iData) + iCassandra, err := c.FromInstAPI(iData) if err != nil { l.Error( err, "Cannot convert cluster from the Instaclustr API", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.ConvertionFailed, + c, models.Warning, models.ConversionFailed, "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", err, ) return reconcile.Result{}, err } - if cassandra.Annotations[models.ExternalChangesAnnotation] == models.True { - return r.handleExternalChanges(cassandra, iCassandra, l) + if c.Annotations[models.ExternalChangesAnnotation] == models.True { + return r.handleExternalChanges(c, iCassandra, l) } - patch := cassandra.NewPatch() + patch := c.NewPatch() - if cassandra.Spec.ClusterSettingsNeedUpdate(iCassandra.Spec.Cluster) { + if c.Spec.ClusterSettingsNeedUpdate(iCassandra.Spec.Cluster) { l.Info("Updating cluster settings", "instaclustr description", iCassandra.Spec.Description, "instaclustr two factor delete", iCassandra.Spec.TwoFactorDelete) - err = r.API.UpdateClusterSettings(cassandra.Status.ID, cassandra.Spec.ClusterSettingsUpdateToInstAPI()) + err = r.API.UpdateClusterSettings(c.Status.ID, c.Spec.ClusterSettingsUpdateToInstAPI()) if err != nil { l.Error(err, "Cannot update cluster settings", - "cluster ID", cassandra.Status.ID, "cluster spec", cassandra.Spec) - r.EventRecorder.Eventf(cassandra, models.Warning, models.UpdateFailed, + "cluster ID", c.Status.ID, "cluster spec", c.Spec) + r.EventRecorder.Eventf(c, models.Warning, models.UpdateFailed, "Cannot update cluster settings. Reason: %v", err) return reconcile.Result{}, err } } - if !cassandra.Spec.AreDCsEqual(iCassandra.Spec.DataCentres) { + if !c.Spec.AreDCsEqual(iCassandra.Spec.DataCentres) { l.Info("Update request to Instaclustr API has been sent", - "spec data centres", cassandra.Spec.DataCentres, - "resize settings", cassandra.Spec.ResizeSettings, + "spec data centres", c.Spec.DataCentres, + "resize settings", c.Spec.ResizeSettings, ) - err = r.API.UpdateCassandra(cassandra.Status.ID, cassandra.Spec.DCsUpdateToInstAPI()) + err = r.API.UpdateCassandra(c.Status.ID, c.Spec.DCsUpdateToInstAPI()) if err != nil { l.Error(err, "Cannot update cluster", - "cluster ID", cassandra.Status.ID, - "cluster name", cassandra.Spec.Name, - "cluster spec", cassandra.Spec, - "cluster state", cassandra.Status.State, + "cluster ID", c.Status.ID, + "cluster name", c.Spec.Name, + "cluster spec", c.Spec, + "cluster state", c.Status.State, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.UpdateFailed, + c, models.Warning, models.UpdateFailed, "Cluster update on the Instaclustr API is failed. Reason: %v", err, ) if errors.Is(err, instaclustr.ClusterIsNotReadyToResize) { - patch := cassandra.NewPatch() - cassandra.Annotations[models.UpdateQueuedAnnotation] = models.True - err = r.Patch(ctx, cassandra, patch) + patch := c.NewPatch() + c.Annotations[models.UpdateQueuedAnnotation] = models.True + err = r.Patch(ctx, c, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, - "kind", cassandra.Kind, - "api Version", cassandra.APIVersion, - "namespace", cassandra.Namespace, - "cluster metadata", cassandra.ObjectMeta, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api Version", c.APIVersion, + "namespace", c.Namespace, + "cluster metadata", c.ObjectMeta, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.PatchFailed, + c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err, ) @@ -385,29 +465,29 @@ func (r *CassandraReconciler) handleUpdateCluster( } } - err = handleUsersChanges(ctx, r.Client, r, cassandra) + err = handleUsersChanges(ctx, r.Client, r, c) if err != nil { l.Error(err, "Failed to handle users changes") - r.EventRecorder.Eventf(cassandra, models.Warning, models.PatchFailed, + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, "Handling users changes is failed. Reason: %w", err, ) return reconcile.Result{}, err } - cassandra.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent - cassandra.Annotations[models.UpdateQueuedAnnotation] = "" - err = r.Patch(ctx, cassandra, patch) + c.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent + c.Annotations[models.UpdateQueuedAnnotation] = "" + err = r.Patch(ctx, c, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, - "kind", cassandra.Kind, - "api Version", cassandra.APIVersion, - "namespace", cassandra.Namespace, - "cluster metadata", cassandra.ObjectMeta, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api Version", c.APIVersion, + "namespace", c.Namespace, + "cluster metadata", c.ObjectMeta, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.PatchFailed, + c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err, ) @@ -416,49 +496,49 @@ func (r *CassandraReconciler) handleUpdateCluster( l.Info( "Cluster has been updated", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, - "data centres", cassandra.Spec.DataCentres, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "data centres", c.Spec.DataCentres, ) return models.ExitReconcile, nil } -func (r *CassandraReconciler) handleExternalChanges(cassandra, iCassandra *v1beta1.Cassandra, l logr.Logger) (reconcile.Result, error) { - if !cassandra.Spec.IsEqual(iCassandra.Spec) { +func (r *CassandraReconciler) handleExternalChanges(c, iCassandra *v1beta1.Cassandra, l logr.Logger) (reconcile.Result, error) { + if !c.Spec.IsEqual(iCassandra.Spec) { l.Info(msgSpecStillNoMatch, - "specification of k8s resource", cassandra.Spec, + "specification of k8s resource", c.Spec, "data from Instaclustr ", iCassandra.Spec) - msgDiffSpecs, err := createSpecDifferenceMessage(cassandra.Spec, iCassandra.Spec) + msgDiffSpecs, err := createSpecDifferenceMessage(c.Spec, iCassandra.Spec) if err != nil { l.Error(err, "Cannot create specification difference message", - "instaclustr data", iCassandra.Spec, "k8s resource spec", cassandra.Spec) + "instaclustr data", iCassandra.Spec, "k8s resource spec", c.Spec) return models.ExitReconcile, nil } - r.EventRecorder.Eventf(cassandra, models.Warning, models.ExternalChanges, msgDiffSpecs) + r.EventRecorder.Eventf(c, models.Warning, models.ExternalChanges, msgDiffSpecs) return models.ExitReconcile, nil } - patch := cassandra.NewPatch() + patch := c.NewPatch() - cassandra.Annotations[models.ExternalChangesAnnotation] = "" + c.Annotations[models.ExternalChangesAnnotation] = "" - err := r.Patch(context.Background(), cassandra, patch) + err := r.Patch(context.Background(), c, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "cluster name", cassandra.Spec.Name, "cluster ID", cassandra.Status.ID) + "cluster name", c.Spec.Name, "cluster ID", c.Status.ID) - r.EventRecorder.Eventf(cassandra, models.Warning, models.PatchFailed, + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err) return reconcile.Result{}, err } - l.Info("External changes have been reconciled", "resource ID", cassandra.Status.ID) - r.EventRecorder.Event(cassandra, models.Normal, models.ExternalChanges, "External changes have been reconciled") + l.Info("External changes have been reconciled", "resource ID", c.Status.ID) + r.EventRecorder.Event(c, models.Normal, models.ExternalChanges, "External changes have been reconciled") return models.ExitReconcile, nil } @@ -466,91 +546,108 @@ func (r *CassandraReconciler) handleExternalChanges(cassandra, iCassandra *v1bet func (r *CassandraReconciler) handleDeleteCluster( ctx context.Context, l logr.Logger, - cassandra *v1beta1.Cassandra, + c *v1beta1.Cassandra, ) (reconcile.Result, error) { l = l.WithName("Cassandra deletion event") - _, err := r.API.GetCassandra(cassandra.Status.ID) + _, err := r.API.GetCassandra(c.Status.ID) if err != nil && !errors.Is(err, instaclustr.NotFound) { l.Error( err, "Cannot get cluster from the Instaclustr API", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, - "kind", cassandra.Kind, - "api Version", cassandra.APIVersion, - "namespace", cassandra.Namespace, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api Version", c.APIVersion, + "namespace", c.Namespace, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.FetchFailed, + c, models.Warning, models.FetchFailed, "Cluster fetch from the Instaclustr API is failed. Reason: %v", err, ) return reconcile.Result{}, err } - patch := cassandra.NewPatch() + patch := c.NewPatch() if !errors.Is(err, instaclustr.NotFound) { l.Info("Sending cluster deletion to the Instaclustr API", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID) + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID) - err = r.API.DeleteCluster(cassandra.Status.ID, instaclustr.CassandraEndpoint) + err = r.API.DeleteCluster(c.Status.ID, instaclustr.CassandraEndpoint) if err != nil { l.Error(err, "Cannot delete cluster", - "cluster name", cassandra.Spec.Name, - "state", cassandra.Status.State, - "kind", cassandra.Kind, - "api Version", cassandra.APIVersion, - "namespace", cassandra.Namespace, + "cluster name", c.Spec.Name, + "state", c.Status.State, + "kind", c.Kind, + "api Version", c.APIVersion, + "namespace", c.Namespace, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.DeletionFailed, + c, models.Warning, models.DeletionFailed, "Cluster deletion on the Instaclustr API is failed. Reason: %v", err, ) return reconcile.Result{}, err } - r.EventRecorder.Event(cassandra, models.Normal, models.DeletionStarted, + r.EventRecorder.Event(c, models.Normal, models.DeletionStarted, "Cluster deletion request is sent to the Instaclustr API.") - if cassandra.Spec.TwoFactorDelete != nil { - cassandra.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent - cassandra.Annotations[models.ClusterDeletionAnnotation] = models.Triggered - err = r.Patch(ctx, cassandra, patch) + if c.Spec.TwoFactorDelete != nil { + c.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent + c.Annotations[models.ClusterDeletionAnnotation] = models.Triggered + err = r.Patch(ctx, c, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "cluster name", cassandra.Spec.Name, - "cluster state", cassandra.Status.State) - r.EventRecorder.Eventf(cassandra, models.Warning, models.PatchFailed, + "cluster name", c.Spec.Name, + "cluster state", c.Status.State) + r.EventRecorder.Eventf(c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err) return reconcile.Result{}, err } - l.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", cassandra.Status.ID) + l.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", c.Status.ID) - r.EventRecorder.Event(cassandra, models.Normal, models.DeletionStarted, + r.EventRecorder.Event(c, models.Normal, models.DeletionStarted, "Two-Factor Delete is enabled, please confirm cluster deletion via email or phone.") return models.ExitReconcile, nil } } - r.Scheduler.RemoveJob(cassandra.GetJobID(scheduler.UserCreator)) - r.Scheduler.RemoveJob(cassandra.GetJobID(scheduler.BackupsChecker)) - r.Scheduler.RemoveJob(cassandra.GetJobID(scheduler.StatusChecker)) + r.Scheduler.RemoveJob(c.GetJobID(scheduler.UserCreator)) + r.Scheduler.RemoveJob(c.GetJobID(scheduler.BackupsChecker)) + r.Scheduler.RemoveJob(c.GetJobID(scheduler.StatusChecker)) + + if c.Spec.OnPremisesSpec != nil { + err = deleteOnPremResources(ctx, r.Client, c.Status.ID, c.Namespace) + if err != nil { + l.Error(err, "Cannot delete cluster on-premises resources", + "cluster ID", c.Status.ID) + r.EventRecorder.Eventf(c, models.Warning, models.DeletionFailed, + "Cluster on-premises resources deletion is failed. Reason: %v", err) + return reconcile.Result{}, err + } - l.Info("Deleting cluster backup resources", "cluster ID", cassandra.Status.ID) + l.Info("Cluster on-premises resources are deleted", + "cluster ID", c.Status.ID) + r.EventRecorder.Eventf(c, models.Normal, models.Deleted, + "Cluster on-premises resources are deleted") + r.Scheduler.RemoveJob(c.GetJobID(scheduler.OnPremisesIPsChecker)) + } - err = r.deleteBackups(ctx, cassandra.Status.ID, cassandra.Namespace) + l.Info("Deleting cluster backup resources", "cluster ID", c.Status.ID) + + err = r.deleteBackups(ctx, c.Status.ID, c.Namespace) if err != nil { l.Error(err, "Cannot delete cluster backup resources", - "cluster ID", cassandra.Status.ID, + "cluster ID", c.Status.ID, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.DeletionFailed, + c, models.Warning, models.DeletionFailed, "Cluster backups deletion is failed. Reason: %v", err, ) @@ -558,72 +655,72 @@ func (r *CassandraReconciler) handleDeleteCluster( } l.Info("Cluster backup resources were deleted", - "cluster ID", cassandra.Status.ID, + "cluster ID", c.Status.ID, ) r.EventRecorder.Eventf( - cassandra, models.Normal, models.Deleted, + c, models.Normal, models.Deleted, "Cluster backup resources are deleted", ) - err = detachUsers(ctx, r.Client, r, cassandra) + err = detachUsers(ctx, r.Client, r, c) if err != nil { l.Error(err, "Failed to detach users from the cluster") - r.EventRecorder.Eventf(cassandra, models.Warning, models.DeletionFailed, + r.EventRecorder.Eventf(c, models.Warning, models.DeletionFailed, "Detaching users from the cluster is failed. Reason: %w", err, ) return reconcile.Result{}, err } - controllerutil.RemoveFinalizer(cassandra, models.DeletionFinalizer) - cassandra.Annotations[models.ResourceStateAnnotation] = models.DeletedEvent - err = r.Patch(ctx, cassandra, patch) + controllerutil.RemoveFinalizer(c, models.DeletionFinalizer) + c.Annotations[models.ResourceStateAnnotation] = models.DeletedEvent + err = r.Patch(ctx, c, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, - "kind", cassandra.Kind, - "api Version", cassandra.APIVersion, - "namespace", cassandra.Namespace, - "cluster metadata", cassandra.ObjectMeta, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api Version", c.APIVersion, + "namespace", c.Namespace, + "cluster metadata", c.ObjectMeta, ) r.EventRecorder.Eventf( - cassandra, models.Warning, models.PatchFailed, + c, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err, ) return reconcile.Result{}, err } - err = exposeservice.Delete(r.Client, cassandra.Name, cassandra.Namespace) + err = exposeservice.Delete(r.Client, c.Name, c.Namespace) if err != nil { l.Error(err, "Cannot delete Cassandra cluster expose service", - "cluster ID", cassandra.Status.ID, - "cluster name", cassandra.Spec.Name, + "cluster ID", c.Status.ID, + "cluster name", c.Spec.Name, ) return reconcile.Result{}, err } l.Info("Cluster has been deleted", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, - "kind", cassandra.Kind, - "api Version", cassandra.APIVersion) + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, + "kind", c.Kind, + "api Version", c.APIVersion) r.EventRecorder.Eventf( - cassandra, models.Normal, models.Deleted, + c, models.Normal, models.Deleted, "Cluster resource is deleted", ) return models.ExitReconcile, nil } -func (r *CassandraReconciler) startClusterStatusJob(cassandraCluster *v1beta1.Cassandra) error { - job := r.newWatchStatusJob(cassandraCluster) +func (r *CassandraReconciler) startClusterStatusJob(c *v1beta1.Cassandra) error { + job := r.newWatchStatusJob(c) - err := r.Scheduler.ScheduleJob(cassandraCluster.GetJobID(scheduler.StatusChecker), scheduler.ClusterStatusInterval, job) + err := r.Scheduler.ScheduleJob(c.GetJobID(scheduler.StatusChecker), scheduler.ClusterStatusInterval, job) if err != nil { return err } @@ -653,55 +750,66 @@ func (r *CassandraReconciler) startUsersCreationJob(cluster *v1beta1.Cassandra) return nil } -func (r *CassandraReconciler) newWatchStatusJob(cassandra *v1beta1.Cassandra) scheduler.Job { +func (r *CassandraReconciler) startClusterOnPremisesIPsJob(c *v1beta1.Cassandra, b *onPremisesBootstrap) error { + job := newWatchOnPremisesIPsJob(c.Kind, b) + + err := r.Scheduler.ScheduleJob(c.GetJobID(scheduler.OnPremisesIPsChecker), scheduler.ClusterStatusInterval, job) + if err != nil { + return err + } + + return nil +} + +func (r *CassandraReconciler) newWatchStatusJob(c *v1beta1.Cassandra) scheduler.Job { l := log.Log.WithValues("component", "CassandraStatusClusterJob") return func() error { - namespacedName := client.ObjectKeyFromObject(cassandra) - err := r.Get(context.Background(), namespacedName, cassandra) + namespacedName := client.ObjectKeyFromObject(c) + err := r.Get(context.Background(), namespacedName, c) if k8serrors.IsNotFound(err) { l.Info("Resource is not found in the k8s cluster. Closing Instaclustr status sync.", "namespaced name", namespacedName) - r.Scheduler.RemoveJob(cassandra.GetJobID(scheduler.BackupsChecker)) - r.Scheduler.RemoveJob(cassandra.GetJobID(scheduler.UserCreator)) - r.Scheduler.RemoveJob(cassandra.GetJobID(scheduler.StatusChecker)) + r.Scheduler.RemoveJob(c.GetJobID(scheduler.BackupsChecker)) + r.Scheduler.RemoveJob(c.GetJobID(scheduler.UserCreator)) + r.Scheduler.RemoveJob(c.GetJobID(scheduler.StatusChecker)) return nil } - iData, err := r.API.GetCassandra(cassandra.Status.ID) + iData, err := r.API.GetCassandra(c.Status.ID) if err != nil { if errors.Is(err, instaclustr.NotFound) { - if cassandra.DeletionTimestamp != nil { - _, err = r.handleDeleteCluster(context.Background(), l, cassandra) + if c.DeletionTimestamp != nil { + _, err = r.handleDeleteCluster(context.Background(), l, c) return err } - return r.handleExternalDelete(context.Background(), cassandra) + return r.handleExternalDelete(context.Background(), c) } l.Error(err, "Cannot get cluster from the Instaclustr API", - "clusterID", cassandra.Status.ID) + "clusterID", c.Status.ID) return err } - iCassandra, err := cassandra.FromInstAPI(iData) + iCassandra, err := c.FromInstAPI(iData) if err != nil { l.Error(err, "Cannot convert cluster from the Instaclustr API", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) return err } - if !areStatusesEqual(&iCassandra.Status.ClusterStatus, &cassandra.Status.ClusterStatus) { + if !areStatusesEqual(&iCassandra.Status.ClusterStatus, &c.Status.ClusterStatus) { l.Info("Updating cluster status", "status from Instaclustr", iCassandra.Status.ClusterStatus, - "status from k8s", cassandra.Status.ClusterStatus) + "status from k8s", c.Status.ClusterStatus) - areDCsEqual := areDataCentresEqual(iCassandra.Status.ClusterStatus.DataCentres, cassandra.Status.ClusterStatus.DataCentres) + areDCsEqual := areDataCentresEqual(iCassandra.Status.ClusterStatus.DataCentres, c.Status.ClusterStatus.DataCentres) - patch := cassandra.NewPatch() - cassandra.Status.ClusterStatus = iCassandra.Status.ClusterStatus - err = r.Status().Patch(context.Background(), cassandra, patch) + patch := c.NewPatch() + c.Status.ClusterStatus = iCassandra.Status.ClusterStatus + err = r.Status().Patch(context.Background(), c, patch) if err != nil { return err } @@ -714,9 +822,9 @@ func (r *CassandraReconciler) newWatchStatusJob(cassandra *v1beta1.Cassandra) sc } err = exposeservice.Create(r.Client, - cassandra.Name, - cassandra.Namespace, - cassandra.Spec.PrivateNetworkCluster, + c.Name, + c.Namespace, + c.Spec.PrivateNetworkCluster, nodes, models.CassandraConnectionPort) if err != nil { @@ -726,10 +834,10 @@ func (r *CassandraReconciler) newWatchStatusJob(cassandra *v1beta1.Cassandra) sc } if iCassandra.Status.CurrentClusterOperationStatus == models.NoOperation && - cassandra.Annotations[models.ResourceStateAnnotation] != models.UpdatingEvent && - cassandra.Annotations[models.UpdateQueuedAnnotation] != models.True && - !cassandra.Spec.IsEqual(iCassandra.Spec) { - k8sData, err := removeRedundantFieldsFromSpec(cassandra.Spec, "userRefs") + c.Annotations[models.ResourceStateAnnotation] != models.UpdatingEvent && + c.Annotations[models.UpdateQueuedAnnotation] != models.True && + !c.Spec.IsEqual(iCassandra.Spec) { + k8sData, err := removeRedundantFieldsFromSpec(c.Spec, "userRefs") if err != nil { l.Error(err, "Cannot remove redundant fields from k8s Spec") return err @@ -737,44 +845,44 @@ func (r *CassandraReconciler) newWatchStatusJob(cassandra *v1beta1.Cassandra) sc l.Info(msgExternalChanges, "instaclustr data", iCassandra.Spec, "k8s resource spec", string(k8sData)) - patch := cassandra.NewPatch() - cassandra.Annotations[models.ExternalChangesAnnotation] = models.True + patch := c.NewPatch() + c.Annotations[models.ExternalChangesAnnotation] = models.True - err = r.Patch(context.Background(), cassandra, patch) + err = r.Patch(context.Background(), c, patch) if err != nil { l.Error(err, "Cannot patch cluster cluster", - "cluster name", cassandra.Spec.Name, "cluster state", cassandra.Status.State) + "cluster name", c.Spec.Name, "cluster state", c.Status.State) return err } - msgDiffSpecs, err := createSpecDifferenceMessage(cassandra.Spec, iCassandra.Spec) + msgDiffSpecs, err := createSpecDifferenceMessage(c.Spec, iCassandra.Spec) if err != nil { l.Error(err, "Cannot create specification difference message", - "instaclustr data", iCassandra.Spec, "k8s resource spec", cassandra.Spec) + "instaclustr data", iCassandra.Spec, "k8s resource spec", c.Spec) return err } - r.EventRecorder.Eventf(cassandra, models.Warning, models.ExternalChanges, msgDiffSpecs) + r.EventRecorder.Eventf(c, models.Warning, models.ExternalChanges, msgDiffSpecs) } //TODO: change all context.Background() and context.TODO() to ctx from Reconcile - err = r.reconcileMaintenanceEvents(context.Background(), cassandra) + err = r.reconcileMaintenanceEvents(context.Background(), c) if err != nil { l.Error(err, "Cannot reconcile cluster maintenance events", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) return err } - if cassandra.Status.State == models.RunningStatus && cassandra.Status.CurrentClusterOperationStatus == models.OperationInProgress { - patch := cassandra.NewPatch() - for _, dc := range cassandra.Status.DataCentres { + if c.Status.State == models.RunningStatus && c.Status.CurrentClusterOperationStatus == models.OperationInProgress { + patch := c.NewPatch() + for _, dc := range c.Status.DataCentres { resizeOperations, err := r.API.GetResizeOperationsByClusterDataCentreID(dc.ID) if err != nil { l.Error(err, "Cannot get data centre resize operations", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, "data centre ID", dc.ID, ) @@ -782,11 +890,11 @@ func (r *CassandraReconciler) newWatchStatusJob(cassandra *v1beta1.Cassandra) sc } dc.ResizeOperations = resizeOperations - err = r.Status().Patch(context.Background(), cassandra, patch) + err = r.Status().Patch(context.Background(), c, patch) if err != nil { l.Error(err, "Cannot patch data centre resize operations", - "cluster name", cassandra.Spec.Name, - "cluster ID", cassandra.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, "data centre ID", dc.ID, ) @@ -799,12 +907,12 @@ func (r *CassandraReconciler) newWatchStatusJob(cassandra *v1beta1.Cassandra) sc } } -func (r *CassandraReconciler) newWatchBackupsJob(cluster *v1beta1.Cassandra) scheduler.Job { +func (r *CassandraReconciler) newWatchBackupsJob(c *v1beta1.Cassandra) scheduler.Job { l := log.Log.WithValues("component", "cassandraBackupsClusterJob") return func() error { ctx := context.Background() - err := r.Get(ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: cluster.Name}, cluster) + err := r.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: c.Name}, c) if err != nil { if k8serrors.IsNotFound(err) { return nil @@ -813,11 +921,11 @@ func (r *CassandraReconciler) newWatchBackupsJob(cluster *v1beta1.Cassandra) sch return err } - iBackups, err := r.API.GetClusterBackups(cluster.Status.ID, models.ClusterKindsMap[cluster.Kind]) + iBackups, err := r.API.GetClusterBackups(c.Status.ID, models.ClusterKindsMap[c.Kind]) if err != nil { l.Error(err, "Cannot get cluster backups", - "cluster name", cluster.Spec.Name, - "cluster ID", cluster.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) return err @@ -825,11 +933,11 @@ func (r *CassandraReconciler) newWatchBackupsJob(cluster *v1beta1.Cassandra) sch iBackupEvents := iBackups.GetBackupEvents(models.CassandraKind) - k8sBackupList, err := r.listClusterBackups(ctx, cluster.Status.ID, cluster.Namespace) + k8sBackupList, err := r.listClusterBackups(ctx, c.Status.ID, c.Namespace) if err != nil { l.Error(err, "Cannot list cluster backups", - "cluster name", cluster.Spec.Name, - "cluster ID", cluster.Status.ID, + "cluster name", c.Spec.Name, + "cluster ID", c.Status.ID, ) return err @@ -893,7 +1001,7 @@ func (r *CassandraReconciler) newWatchBackupsJob(cluster *v1beta1.Cassandra) sch continue } - backupSpec := cluster.NewBackupSpec(start) + backupSpec := c.NewBackupSpec(start) err = r.Create(ctx, backupSpec) if err != nil { return err diff --git a/controllers/clusters/helpers.go b/controllers/clusters/helpers.go index 1ff09ec84..9678e1a22 100644 --- a/controllers/clusters/helpers.go +++ b/controllers/clusters/helpers.go @@ -117,26 +117,31 @@ func isDataCentreNodesEqual(a, b []*v1beta1.Node) bool { if a == nil && b == nil { return true } - if len(a) != len(b) { return false } for i := range a { - if a[i].ID != b[i].ID { - continue - } + var eq bool + for j := range b { + if a[i].ID != b[j].ID { + continue + } - if a[i].Size != b[i].Size || - a[i].PublicAddress != b[i].PublicAddress || - a[i].PrivateAddress != b[i].PrivateAddress || - a[i].Status != b[i].Status || - !slices.Equal(a[i].Roles, b[i].Roles) || - a[i].Rack != b[i].Rack { + if a[i].Size != b[j].Size || + a[i].PublicAddress != b[j].PublicAddress || + a[i].PrivateAddress != b[j].PrivateAddress || + a[i].Status != b[j].Status || + !slices.Equal(a[i].Roles, b[j].Roles) || + a[i].Rack != b[j].Rack { + return false + } + eq = true + } + if !eq { return false } } - return true } diff --git a/controllers/clusters/kafka_controller.go b/controllers/clusters/kafka_controller.go index d076e73c0..9acd59b04 100644 --- a/controllers/clusters/kafka_controller.go +++ b/controllers/clusters/kafka_controller.go @@ -57,6 +57,14 @@ type KafkaReconciler struct { //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=kafkas/status,verbs=get;update;patch //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=kafkas/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch +//+kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachines,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete;deletecollection // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -66,8 +74,8 @@ type KafkaReconciler struct { func (r *KafkaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { l := log.FromContext(ctx) - var kafka v1beta1.Kafka - err := r.Client.Get(ctx, req.NamespacedName, &kafka) + var k v1beta1.Kafka + err := r.Client.Get(ctx, req.NamespacedName, &k) if err != nil { if k8serrors.IsNotFound(err) { l.Info("Kafka custom resource is not found", "namespaced name ", req.NamespacedName) @@ -78,39 +86,39 @@ func (r *KafkaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return reconcile.Result{}, err } - switch kafka.Annotations[models.ResourceStateAnnotation] { + switch k.Annotations[models.ResourceStateAnnotation] { case models.CreatingEvent: - return r.handleCreateCluster(ctx, &kafka, l) + return r.handleCreateCluster(ctx, &k, l) case models.UpdatingEvent: - return r.handleUpdateCluster(ctx, &kafka, l) + return r.handleUpdateCluster(ctx, &k, l) case models.DeletingEvent: - return r.handleDeleteCluster(ctx, &kafka, l) + return r.handleDeleteCluster(ctx, &k, l) case models.GenericEvent: - l.Info("Event isn't handled", "cluster name", kafka.Spec.Name, "request", req, - "event", kafka.Annotations[models.ResourceStateAnnotation]) + l.Info("Event isn't handled", "cluster name", k.Spec.Name, "request", req, + "event", k.Annotations[models.ResourceStateAnnotation]) return models.ExitReconcile, nil } return models.ExitReconcile, nil } -func (r *KafkaReconciler) handleCreateCluster(ctx context.Context, kafka *v1beta1.Kafka, l logr.Logger) (reconcile.Result, error) { +func (r *KafkaReconciler) handleCreateCluster(ctx context.Context, k *v1beta1.Kafka, l logr.Logger) (reconcile.Result, error) { l = l.WithName("Kafka creation Event") var err error - if kafka.Status.ID == "" { + if k.Status.ID == "" { l.Info("Creating cluster", - "cluster name", kafka.Spec.Name, - "data centres", kafka.Spec.DataCentres) + "cluster name", k.Spec.Name, + "data centres", k.Spec.DataCentres) - patch := kafka.NewPatch() - kafka.Status.ID, err = r.API.CreateCluster(instaclustr.KafkaEndpoint, kafka.Spec.ToInstAPI()) + patch := k.NewPatch() + k.Status.ID, err = r.API.CreateCluster(instaclustr.KafkaEndpoint, k.Spec.ToInstAPI()) if err != nil { l.Error(err, "Cannot create cluster", - "spec", kafka.Spec, + "spec", k.Spec, ) r.EventRecorder.Eventf( - kafka, models.Warning, models.CreationFailed, + k, models.Warning, models.CreationFailed, "Cluster creation on the Instaclustr is failed. Reason: %v", err, ) @@ -118,33 +126,33 @@ func (r *KafkaReconciler) handleCreateCluster(ctx context.Context, kafka *v1beta } r.EventRecorder.Eventf( - kafka, models.Normal, models.Created, + k, models.Normal, models.Created, "Cluster creation request is sent. Cluster ID: %s", - kafka.Status.ID, + k.Status.ID, ) - err = r.Status().Patch(ctx, kafka, patch) + err = r.Status().Patch(ctx, k, patch) if err != nil { l.Error(err, "Cannot patch cluster status", - "spec", kafka.Spec, + "spec", k.Spec, ) r.EventRecorder.Eventf( - kafka, models.Warning, models.PatchFailed, + k, models.Warning, models.PatchFailed, "Cluster resource status patch is failed. Reason: %v", err, ) return reconcile.Result{}, err } - kafka.Annotations[models.ResourceStateAnnotation] = models.CreatedEvent - controllerutil.AddFinalizer(kafka, models.DeletionFinalizer) - err = r.Patch(ctx, kafka, patch) + k.Annotations[models.ResourceStateAnnotation] = models.CreatedEvent + controllerutil.AddFinalizer(k, models.DeletionFinalizer) + err = r.Patch(ctx, k, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "name", kafka.Spec.Name, + "name", k.Spec.Name, ) r.EventRecorder.Eventf( - kafka, models.Warning, models.PatchFailed, + k, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err, ) @@ -152,17 +160,17 @@ func (r *KafkaReconciler) handleCreateCluster(ctx context.Context, kafka *v1beta } l.Info("Cluster has been created", - "cluster ID", kafka.Status.ID, + "cluster ID", k.Status.ID, ) } - if kafka.Status.State != models.DeletedStatus { - err = r.startClusterStatusJob(kafka) + if k.Status.State != models.DeletedStatus { + err = r.startClusterStatusJob(k) if err != nil { l.Error(err, "Cannot start cluster status job", - "cluster ID", kafka.Status.ID) + "cluster ID", k.Status.ID) r.EventRecorder.Eventf( - kafka, models.Warning, models.CreationFailed, + k, models.Warning, models.CreationFailed, "Cluster status check job creation is failed. Reason: %v", err, ) @@ -170,24 +178,102 @@ func (r *KafkaReconciler) handleCreateCluster(ctx context.Context, kafka *v1beta } r.EventRecorder.Eventf( - kafka, models.Normal, models.Created, + k, models.Normal, models.Created, "Cluster status check job is started", ) - if kafka.Spec.UserRefs != nil { - err = r.startUsersCreationJob(kafka) + if k.Spec.UserRefs != nil { + err = r.startUsersCreationJob(k) if err != nil { l.Error(err, "Failed to start user creation job") - r.EventRecorder.Eventf(kafka, models.Warning, models.CreationFailed, + r.EventRecorder.Eventf(k, models.Warning, models.CreationFailed, "User creation job is failed. Reason: %v", err, ) return reconcile.Result{}, err } - r.EventRecorder.Event(kafka, models.Normal, models.Created, + r.EventRecorder.Event(k, models.Normal, models.Created, "Cluster user creation job is started", ) } + + if k.Spec.OnPremisesSpec != nil { + iData, err := r.API.GetKafka(k.Status.ID) + if err != nil { + l.Error(err, "Cannot get cluster from the Instaclustr API", + "cluster name", k.Spec.Name, + "data centres", k.Spec.DataCentres, + "cluster ID", k.Status.ID, + ) + r.EventRecorder.Eventf( + k, models.Warning, models.FetchFailed, + "Cluster fetch from the Instaclustr API is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + iKafka, err := k.FromInstAPI(iData) + if err != nil { + l.Error( + err, "Cannot convert cluster from the Instaclustr API", + "cluster name", k.Spec.Name, + "cluster ID", k.Status.ID, + ) + r.EventRecorder.Eventf( + k, models.Warning, models.ConversionFailed, + "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + bootstrap := newOnPremisesBootstrap( + r.Client, + k, + r.EventRecorder, + iKafka.Status.ClusterStatus, + k.Spec.OnPremisesSpec, + newExposePorts(k.GetExposePorts()), + k.GetHeadlessPorts(), + k.Spec.PrivateNetworkCluster, + ) + + err = handleCreateOnPremisesClusterResources(ctx, bootstrap) + if err != nil { + l.Error( + err, "Cannot create resources for on-premises cluster", + "cluster spec", k.Spec.OnPremisesSpec, + ) + r.EventRecorder.Eventf( + k, models.Warning, models.CreationFailed, + "Resources creation for on-premises cluster is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + err = r.startClusterOnPremisesIPsJob(k, bootstrap) + if err != nil { + l.Error(err, "Cannot start on-premises cluster IPs check job", + "cluster ID", k.Status.ID, + ) + + r.EventRecorder.Eventf( + k, models.Warning, models.CreationFailed, + "On-premises cluster IPs check job is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + l.Info( + "On-premises resources have been created", + "cluster name", k.Spec.Name, + "on-premises Spec", k.Spec.OnPremisesSpec, + "cluster ID", k.Status.ID, + ) + return models.ExitReconcile, nil + } } return models.ExitReconcile, nil @@ -212,7 +298,7 @@ func (r *KafkaReconciler) handleUpdateCluster( return reconcile.Result{}, err } - if iKafka.Status.ClusterStatus.State != StatusRUNNING { + if iKafka.Status.ClusterStatus.State != models.RunningStatus { l.Error(instaclustr.ClusterNotRunning, "Unable to update cluster, cluster still not running", "cluster name", k.Spec.Name, "cluster state", iKafka.Status.ClusterStatus.State) @@ -359,35 +445,35 @@ func (r *KafkaReconciler) handleExternalChanges(k, ik *v1beta1.Kafka, l logr.Log return models.ExitReconcile, nil } -func (r *KafkaReconciler) handleDeleteCluster(ctx context.Context, kafka *v1beta1.Kafka, l logr.Logger) (reconcile.Result, error) { +func (r *KafkaReconciler) handleDeleteCluster(ctx context.Context, k *v1beta1.Kafka, l logr.Logger) (reconcile.Result, error) { l = l.WithName("Kafka deletion Event") - _, err := r.API.GetKafka(kafka.Status.ID) + _, err := r.API.GetKafka(k.Status.ID) if err != nil && !errors.Is(err, instaclustr.NotFound) { l.Error(err, "Cannot get cluster from the Instaclustr API", - "cluster name", kafka.Spec.Name, - "cluster state", kafka.Status.ClusterStatus.State) + "cluster name", k.Spec.Name, + "cluster state", k.Status.ClusterStatus.State) r.EventRecorder.Eventf( - kafka, models.Warning, models.FetchFailed, + k, models.Warning, models.FetchFailed, "Cluster resource fetch from the Instaclustr API is failed. Reason: %v", err, ) return reconcile.Result{}, err } - patch := kafka.NewPatch() + patch := k.NewPatch() if !errors.Is(err, instaclustr.NotFound) { l.Info("Sending cluster deletion to the Instaclustr API", - "cluster name", kafka.Spec.Name, - "cluster ID", kafka.Status.ID) + "cluster name", k.Spec.Name, + "cluster ID", k.Status.ID) - err = r.API.DeleteCluster(kafka.Status.ID, instaclustr.KafkaEndpoint) + err = r.API.DeleteCluster(k.Status.ID, instaclustr.KafkaEndpoint) if err != nil { l.Error(err, "Cannot delete cluster", - "cluster name", kafka.Spec.Name, - "cluster state", kafka.Status.ClusterStatus.State) + "cluster name", k.Spec.Name, + "cluster state", k.Status.ClusterStatus.State) r.EventRecorder.Eventf( - kafka, models.Warning, models.DeletionFailed, + k, models.Warning, models.DeletionFailed, "Cluster deletion is failed on the Instaclustr. Reason: %v", err, ) @@ -395,82 +481,110 @@ func (r *KafkaReconciler) handleDeleteCluster(ctx context.Context, kafka *v1beta } r.EventRecorder.Eventf( - kafka, models.Normal, models.DeletionStarted, + k, models.Normal, models.DeletionStarted, "Cluster deletion request is sent to the Instaclustr API.", ) - if kafka.Spec.TwoFactorDelete != nil { - kafka.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent - kafka.Annotations[models.ClusterDeletionAnnotation] = models.Triggered - err = r.Patch(ctx, kafka, patch) + if k.Spec.TwoFactorDelete != nil { + k.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent + k.Annotations[models.ClusterDeletionAnnotation] = models.Triggered + err = r.Patch(ctx, k, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "cluster name", kafka.Spec.Name, - "cluster state", kafka.Status.State) + "cluster name", k.Spec.Name, + "cluster state", k.Status.State) r.EventRecorder.Eventf( - kafka, models.Warning, models.PatchFailed, + k, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err, ) return reconcile.Result{}, err } - l.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", kafka.Status.ID) + l.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", k.Status.ID) - r.EventRecorder.Event(kafka, models.Normal, models.DeletionStarted, + r.EventRecorder.Event(k, models.Normal, models.DeletionStarted, "Two-Factor Delete is enabled, please confirm cluster deletion via email or phone.") return models.ExitReconcile, nil } } - err = detachUsers(ctx, r.Client, r, kafka) + err = detachUsers(ctx, r.Client, r, k) if err != nil { l.Error(err, "Failed to detach users from the cluster") - r.EventRecorder.Eventf(kafka, models.Warning, models.DeletionFailed, + r.EventRecorder.Eventf(k, models.Warning, models.DeletionFailed, "Detaching users from the cluster is failed. Reason: %w", err, ) return reconcile.Result{}, err } - r.Scheduler.RemoveJob(kafka.GetJobID(scheduler.StatusChecker)) - r.Scheduler.RemoveJob(kafka.GetJobID(scheduler.UserCreator)) - controllerutil.RemoveFinalizer(kafka, models.DeletionFinalizer) - kafka.Annotations[models.ResourceStateAnnotation] = models.DeletedEvent - err = r.Patch(ctx, kafka, patch) + if k.Spec.OnPremisesSpec != nil { + err = deleteOnPremResources(ctx, r.Client, k.Status.ID, k.Namespace) + if err != nil { + l.Error(err, "Cannot delete cluster on-premises resources", + "cluster ID", k.Status.ID) + r.EventRecorder.Eventf(k, models.Warning, models.DeletionFailed, + "Cluster on-premises resources deletion is failed. Reason: %v", err) + return reconcile.Result{}, err + } + + l.Info("Cluster on-premises resources are deleted", + "cluster ID", k.Status.ID) + r.EventRecorder.Eventf(k, models.Normal, models.Deleted, + "Cluster on-premises resources are deleted") + r.Scheduler.RemoveJob(k.GetJobID(scheduler.OnPremisesIPsChecker)) + } + + r.Scheduler.RemoveJob(k.GetJobID(scheduler.StatusChecker)) + r.Scheduler.RemoveJob(k.GetJobID(scheduler.UserCreator)) + controllerutil.RemoveFinalizer(k, models.DeletionFinalizer) + k.Annotations[models.ResourceStateAnnotation] = models.DeletedEvent + err = r.Patch(ctx, k, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "cluster name", kafka.Spec.Name) + "cluster name", k.Spec.Name) r.EventRecorder.Eventf( - kafka, models.Warning, models.PatchFailed, + k, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err, ) return reconcile.Result{}, err } - err = exposeservice.Delete(r.Client, kafka.Name, kafka.Namespace) + err = exposeservice.Delete(r.Client, k.Name, k.Namespace) if err != nil { l.Error(err, "Cannot delete Kafka cluster expose service", - "cluster ID", kafka.Status.ID, - "cluster name", kafka.Spec.Name, + "cluster ID", k.Status.ID, + "cluster name", k.Spec.Name, ) return reconcile.Result{}, err } l.Info("Cluster was deleted", - "cluster name", kafka.Spec.Name, - "cluster ID", kafka.Status.ID) + "cluster name", k.Spec.Name, + "cluster ID", k.Status.ID) r.EventRecorder.Eventf( - kafka, models.Normal, models.Deleted, + k, models.Normal, models.Deleted, "Cluster resource is deleted", ) return models.ExitReconcile, nil } +func (r *KafkaReconciler) startClusterOnPremisesIPsJob(k *v1beta1.Kafka, b *onPremisesBootstrap) error { + job := newWatchOnPremisesIPsJob(k.Kind, b) + + err := r.Scheduler.ScheduleJob(k.GetJobID(scheduler.OnPremisesIPsChecker), scheduler.ClusterStatusInterval, job) + if err != nil { + return err + } + + return nil +} + func (r *KafkaReconciler) startClusterStatusJob(kafka *v1beta1.Kafka) error { job := r.newWatchStatusJob(kafka) @@ -482,61 +596,61 @@ func (r *KafkaReconciler) startClusterStatusJob(kafka *v1beta1.Kafka) error { return nil } -func (r *KafkaReconciler) newWatchStatusJob(kafka *v1beta1.Kafka) scheduler.Job { +func (r *KafkaReconciler) newWatchStatusJob(k *v1beta1.Kafka) scheduler.Job { l := log.Log.WithValues("component", "kafkaStatusClusterJob") return func() error { - namespacedName := client.ObjectKeyFromObject(kafka) - err := r.Get(context.Background(), namespacedName, kafka) + namespacedName := client.ObjectKeyFromObject(k) + err := r.Get(context.Background(), namespacedName, k) if k8serrors.IsNotFound(err) { l.Info("Resource is not found in the k8s cluster. Closing Instaclustr status sync.", "namespaced name", namespacedName) - r.Scheduler.RemoveJob(kafka.GetJobID(scheduler.StatusChecker)) - r.Scheduler.RemoveJob(kafka.GetJobID(scheduler.UserCreator)) - r.Scheduler.RemoveJob(kafka.GetJobID(scheduler.BackupsChecker)) + r.Scheduler.RemoveJob(k.GetJobID(scheduler.StatusChecker)) + r.Scheduler.RemoveJob(k.GetJobID(scheduler.UserCreator)) + r.Scheduler.RemoveJob(k.GetJobID(scheduler.BackupsChecker)) return nil } if err != nil { l.Error(err, "Cannot get cluster resource", - "resource name", kafka.Name) + "resource name", k.Name) return err } - iData, err := r.API.GetKafka(kafka.Status.ID) + iData, err := r.API.GetKafka(k.Status.ID) if err != nil { if errors.Is(err, instaclustr.NotFound) { - if kafka.DeletionTimestamp != nil { - _, err = r.handleDeleteCluster(context.Background(), kafka, l) + if k.DeletionTimestamp != nil { + _, err = r.handleDeleteCluster(context.Background(), k, l) return err } - return r.handleExternalDelete(context.Background(), kafka) + return r.handleExternalDelete(context.Background(), k) } - l.Error(err, "Cannot get cluster from the Instaclustr", "cluster ID", kafka.Status.ID) + l.Error(err, "Cannot get cluster from the Instaclustr", "cluster ID", k.Status.ID) return err } - iKafka, err := kafka.FromInstAPI(iData) + iKafka, err := k.FromInstAPI(iData) if err != nil { l.Error(err, "Cannot convert cluster from the Instaclustr API", - "cluster ID", kafka.Status.ID, + "cluster ID", k.Status.ID, ) return err } - if !areStatusesEqual(&kafka.Status.ClusterStatus, &iKafka.Status.ClusterStatus) { + if !areStatusesEqual(&k.Status.ClusterStatus, &iKafka.Status.ClusterStatus) { l.Info("Kafka status of k8s is different from Instaclustr. Reconcile k8s resource status..", "instacluster status", iKafka.Status, - "k8s status", kafka.Status.ClusterStatus) + "k8s status", k.Status.ClusterStatus) - areDCsEqual := areDataCentresEqual(iKafka.Status.ClusterStatus.DataCentres, kafka.Status.ClusterStatus.DataCentres) + areDCsEqual := areDataCentresEqual(iKafka.Status.ClusterStatus.DataCentres, k.Status.ClusterStatus.DataCentres) - patch := kafka.NewPatch() - kafka.Status.ClusterStatus = iKafka.Status.ClusterStatus - err = r.Status().Patch(context.Background(), kafka, patch) + patch := k.NewPatch() + k.Status.ClusterStatus = iKafka.Status.ClusterStatus + err = r.Status().Patch(context.Background(), k, patch) if err != nil { l.Error(err, "Cannot patch cluster cluster", - "cluster name", kafka.Spec.Name, "cluster state", kafka.Status.State) + "cluster name", k.Spec.Name, "cluster state", k.Status.State) return err } @@ -548,9 +662,9 @@ func (r *KafkaReconciler) newWatchStatusJob(kafka *v1beta1.Kafka) scheduler.Job } err = exposeservice.Create(r.Client, - kafka.Name, - kafka.Namespace, - kafka.Spec.PrivateNetworkCluster, + k.Name, + k.Namespace, + k.Spec.PrivateNetworkCluster, nodes, models.KafkaConnectionPort) if err != nil { @@ -560,49 +674,49 @@ func (r *KafkaReconciler) newWatchStatusJob(kafka *v1beta1.Kafka) scheduler.Job } if iKafka.Status.CurrentClusterOperationStatus == models.NoOperation && - kafka.Annotations[models.UpdateQueuedAnnotation] != models.True && - !kafka.Spec.IsEqual(iKafka.Spec) { + k.Annotations[models.UpdateQueuedAnnotation] != models.True && + !k.Spec.IsEqual(iKafka.Spec) { - patch := kafka.NewPatch() - kafka.Annotations[models.ExternalChangesAnnotation] = models.True + patch := k.NewPatch() + k.Annotations[models.ExternalChangesAnnotation] = models.True - err = r.Patch(context.Background(), kafka, patch) + err = r.Patch(context.Background(), k, patch) if err != nil { l.Error(err, "Cannot patch cluster cluster", - "cluster name", kafka.Spec.Name, "cluster state", kafka.Status.State) + "cluster name", k.Spec.Name, "cluster state", k.Status.State) return err } l.Info("The k8s specification is different from Instaclustr Console. Update operations are blocked.", - "instaclustr data", iKafka.Spec, "k8s resource spec", kafka.Spec) + "instaclustr data", iKafka.Spec, "k8s resource spec", k.Spec) - msgDiffSpecs, err := createSpecDifferenceMessage(kafka.Spec, iKafka.Spec) + msgDiffSpecs, err := createSpecDifferenceMessage(k.Spec, iKafka.Spec) if err != nil { l.Error(err, "Cannot create specification difference message", - "instaclustr data", iKafka.Spec, "k8s resource spec", kafka.Spec) + "instaclustr data", iKafka.Spec, "k8s resource spec", k.Spec) return err } - r.EventRecorder.Eventf(kafka, models.Warning, models.ExternalChanges, msgDiffSpecs) + r.EventRecorder.Eventf(k, models.Warning, models.ExternalChanges, msgDiffSpecs) } //TODO: change all context.Background() and context.TODO() to ctx from Reconcile - err = r.reconcileMaintenanceEvents(context.Background(), kafka) + err = r.reconcileMaintenanceEvents(context.Background(), k) if err != nil { l.Error(err, "Cannot reconcile cluster maintenance events", - "cluster name", kafka.Spec.Name, - "cluster ID", kafka.Status.ID, + "cluster name", k.Spec.Name, + "cluster ID", k.Status.ID, ) return err } - if kafka.Status.State == models.RunningStatus && kafka.Status.CurrentClusterOperationStatus == models.OperationInProgress { - patch := kafka.NewPatch() - for _, dc := range kafka.Status.DataCentres { + if k.Status.State == models.RunningStatus && k.Status.CurrentClusterOperationStatus == models.OperationInProgress { + patch := k.NewPatch() + for _, dc := range k.Status.DataCentres { resizeOperations, err := r.API.GetResizeOperationsByClusterDataCentreID(dc.ID) if err != nil { l.Error(err, "Cannot get data centre resize operations", - "cluster name", kafka.Spec.Name, - "cluster ID", kafka.Status.ID, + "cluster name", k.Spec.Name, + "cluster ID", k.Status.ID, "data centre ID", dc.ID, ) @@ -610,11 +724,11 @@ func (r *KafkaReconciler) newWatchStatusJob(kafka *v1beta1.Kafka) scheduler.Job } dc.ResizeOperations = resizeOperations - err = r.Status().Patch(context.Background(), kafka, patch) + err = r.Status().Patch(context.Background(), k, patch) if err != nil { l.Error(err, "Cannot patch data centre resize operations", - "cluster name", kafka.Spec.Name, - "cluster ID", kafka.Status.ID, + "cluster name", k.Spec.Name, + "cluster ID", k.Status.ID, "data centre ID", dc.ID, ) @@ -627,25 +741,25 @@ func (r *KafkaReconciler) newWatchStatusJob(kafka *v1beta1.Kafka) scheduler.Job } } -func (r *KafkaReconciler) startUsersCreationJob(kafka *v1beta1.Kafka) error { - job := r.newUsersCreationJob(kafka) +func (r *KafkaReconciler) startUsersCreationJob(k *v1beta1.Kafka) error { + job := r.newUsersCreationJob(k) - err := r.Scheduler.ScheduleJob(kafka.GetJobID(scheduler.UserCreator), scheduler.UserCreationInterval, job) + err := r.Scheduler.ScheduleJob(k.GetJobID(scheduler.UserCreator), scheduler.UserCreationInterval, job) if err != nil { return err } return nil } -func (r *KafkaReconciler) newUsersCreationJob(kafka *v1beta1.Kafka) scheduler.Job { +func (r *KafkaReconciler) newUsersCreationJob(k *v1beta1.Kafka) scheduler.Job { l := log.Log.WithValues("component", "kafkaUsersCreationJob") return func() error { ctx := context.Background() err := r.Get(ctx, types.NamespacedName{ - Namespace: kafka.Namespace, - Name: kafka.Name, - }, kafka) + Namespace: k.Namespace, + Name: k.Name, + }, k) if err != nil { if k8serrors.IsNotFound(err) { @@ -655,50 +769,50 @@ func (r *KafkaReconciler) newUsersCreationJob(kafka *v1beta1.Kafka) scheduler.Jo return err } - if kafka.Status.State != models.RunningStatus { + if k.Status.State != models.RunningStatus { l.Info("User creation job is scheduled") - r.EventRecorder.Eventf(kafka, models.Normal, models.CreationFailed, + r.EventRecorder.Eventf(k, models.Normal, models.CreationFailed, "User creation job is scheduled, cluster is not in the running state", ) return nil } - err = handleUsersChanges(ctx, r.Client, r, kafka) + err = handleUsersChanges(ctx, r.Client, r, k) if err != nil { l.Error(err, "Failed to create users for the cluster") - r.EventRecorder.Eventf(kafka, models.Warning, models.CreationFailed, + r.EventRecorder.Eventf(k, models.Warning, models.CreationFailed, "Failed to create users for the cluster. Reason: %v", err) return err } l.Info("User creation job successfully finished") - r.EventRecorder.Eventf(kafka, models.Normal, models.Created, + r.EventRecorder.Eventf(k, models.Normal, models.Created, "User creation job successfully finished", ) - r.Scheduler.RemoveJob(kafka.GetJobID(scheduler.UserCreator)) + r.Scheduler.RemoveJob(k.GetJobID(scheduler.UserCreator)) return nil } } -func (r *KafkaReconciler) handleExternalDelete(ctx context.Context, kafka *v1beta1.Kafka) error { +func (r *KafkaReconciler) handleExternalDelete(ctx context.Context, k *v1beta1.Kafka) error { l := log.FromContext(ctx) - patch := kafka.NewPatch() - kafka.Status.State = models.DeletedStatus - err := r.Status().Patch(ctx, kafka, patch) + patch := k.NewPatch() + k.Status.State = models.DeletedStatus + err := r.Status().Patch(ctx, k, patch) if err != nil { return err } l.Info(instaclustr.MsgInstaclustrResourceNotFound) - r.EventRecorder.Eventf(kafka, models.Warning, models.ExternalDeleted, instaclustr.MsgInstaclustrResourceNotFound) + r.EventRecorder.Eventf(k, models.Warning, models.ExternalDeleted, instaclustr.MsgInstaclustrResourceNotFound) - r.Scheduler.RemoveJob(kafka.GetJobID(scheduler.BackupsChecker)) - r.Scheduler.RemoveJob(kafka.GetJobID(scheduler.UserCreator)) - r.Scheduler.RemoveJob(kafka.GetJobID(scheduler.StatusChecker)) + r.Scheduler.RemoveJob(k.GetJobID(scheduler.BackupsChecker)) + r.Scheduler.RemoveJob(k.GetJobID(scheduler.UserCreator)) + r.Scheduler.RemoveJob(k.GetJobID(scheduler.StatusChecker)) return nil } diff --git a/controllers/clusters/kafkaconnect_controller.go b/controllers/clusters/kafkaconnect_controller.go index e249b7640..423688fdd 100644 --- a/controllers/clusters/kafkaconnect_controller.go +++ b/controllers/clusters/kafkaconnect_controller.go @@ -55,6 +55,14 @@ type KafkaConnectReconciler struct { //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=kafkaconnects/status,verbs=get;update;patch //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=kafkaconnects/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch +//+kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachines,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete;deletecollection // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -64,8 +72,8 @@ type KafkaConnectReconciler struct { func (r *KafkaConnectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { l := log.FromContext(ctx) - kafkaConnect := &v1beta1.KafkaConnect{} - err := r.Client.Get(ctx, req.NamespacedName, kafkaConnect) + kc := &v1beta1.KafkaConnect{} + err := r.Client.Get(ctx, req.NamespacedName, kc) if err != nil { if k8serrors.IsNotFound(err) { l.Error(err, "KafkaConnect resource is not found", "request", req) @@ -76,16 +84,16 @@ func (r *KafkaConnectReconciler) Reconcile(ctx context.Context, req ctrl.Request return reconcile.Result{}, err } - switch kafkaConnect.Annotations[models.ResourceStateAnnotation] { + switch kc.Annotations[models.ResourceStateAnnotation] { case models.CreatingEvent: - return r.handleCreateCluster(ctx, kafkaConnect, l) + return r.handleCreateCluster(ctx, kc, l) case models.UpdatingEvent: - return r.handleUpdateCluster(ctx, kafkaConnect, l) + return r.handleUpdateCluster(ctx, kc, l) case models.DeletingEvent: - return r.handleDeleteCluster(ctx, kafkaConnect, l) + return r.handleDeleteCluster(ctx, kc, l) default: - l.Info("Event isn't handled", "cluster name", kafkaConnect.Spec.Name, - "request", req, "event", kafkaConnect.Annotations[models.ResourceStateAnnotation]) + l.Info("Event isn't handled", "cluster name", kc.Spec.Name, + "request", req, "event", kc.Annotations[models.ResourceStateAnnotation]) return models.ExitReconcile, nil } } @@ -178,6 +186,83 @@ func (r *KafkaConnectReconciler) handleCreateCluster(ctx context.Context, kc *v1 "Cluster status check job is started", ) } + if kc.Spec.OnPremisesSpec != nil { + iData, err := r.API.GetKafkaConnect(kc.Status.ID) + if err != nil { + l.Error(err, "Cannot get cluster from the Instaclustr API", + "cluster name", kc.Spec.Name, + "data centres", kc.Spec.DataCentres, + "cluster ID", kc.Status.ID, + ) + r.EventRecorder.Eventf( + kc, models.Warning, models.FetchFailed, + "Cluster fetch from the Instaclustr API is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + iKafkaConnect, err := kc.FromInst(iData) + if err != nil { + l.Error( + err, "Cannot convert cluster from the Instaclustr API", + "cluster name", kc.Spec.Name, + "cluster ID", kc.Status.ID, + ) + r.EventRecorder.Eventf( + kc, models.Warning, models.ConversionFailed, + "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + bootstrap := newOnPremisesBootstrap( + r.Client, + kc, + r.EventRecorder, + iKafkaConnect.Status.ClusterStatus, + kc.Spec.OnPremisesSpec, + newExposePorts(kc.GetExposePorts()), + kc.GetHeadlessPorts(), + kc.Spec.PrivateNetworkCluster, + ) + + err = handleCreateOnPremisesClusterResources(ctx, bootstrap) + if err != nil { + l.Error( + err, "Cannot create resources for on-premises cluster", + "cluster spec", kc.Spec.OnPremisesSpec, + ) + r.EventRecorder.Eventf( + kc, models.Warning, models.CreationFailed, + "Resources creation for on-premises cluster is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + err = r.startClusterOnPremisesIPsJob(kc, bootstrap) + if err != nil { + l.Error(err, "Cannot start on-premises cluster IPs check job", + "cluster ID", kc.Status.ID, + ) + + r.EventRecorder.Eventf( + kc, models.Warning, models.CreationFailed, + "On-premises cluster IPs check job is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + l.Info( + "On-premises resources have been created", + "cluster name", kc.Spec.Name, + "on-premises Spec", kc.Spec.OnPremisesSpec, + "cluster ID", kc.Status.ID, + ) + return models.ExitReconcile, nil + } return models.ExitReconcile, nil } @@ -202,7 +287,7 @@ func (r *KafkaConnectReconciler) handleUpdateCluster(ctx context.Context, kc *v1 l.Error(err, "Cannot convert Kafka Connect from Instaclustr", "ClusterID", kc.Status.ID) r.EventRecorder.Eventf( - kc, models.Warning, models.ConvertionFailed, + kc, models.Warning, models.ConversionFailed, "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", err, ) @@ -292,39 +377,39 @@ func (r *KafkaConnectReconciler) handleUpdateCluster(ctx context.Context, kc *v1 return models.ExitReconcile, nil } -func (r *KafkaConnectReconciler) handleExternalChanges(k, ik *v1beta1.KafkaConnect, l logr.Logger) (reconcile.Result, error) { - if !k.Spec.IsEqual(ik.Spec) { +func (r *KafkaConnectReconciler) handleExternalChanges(kc, ik *v1beta1.KafkaConnect, l logr.Logger) (reconcile.Result, error) { + if !kc.Spec.IsEqual(ik.Spec) { l.Info(msgSpecStillNoMatch, - "specification of k8s resource", k.Spec, + "specification of k8s resource", kc.Spec, "data from Instaclustr ", ik.Spec) - msgDiffSpecs, err := createSpecDifferenceMessage(k.Spec, ik.Spec) + msgDiffSpecs, err := createSpecDifferenceMessage(kc.Spec, ik.Spec) if err != nil { l.Error(err, "Cannot create specification difference message", - "instaclustr data", ik.Spec, "k8s resource spec", k.Spec) + "instaclustr data", ik.Spec, "k8s resource spec", kc.Spec) return models.ExitReconcile, nil } - r.EventRecorder.Eventf(k, models.Warning, models.ExternalChanges, msgDiffSpecs) + r.EventRecorder.Eventf(kc, models.Warning, models.ExternalChanges, msgDiffSpecs) return models.ExitReconcile, nil } - patch := k.NewPatch() + patch := kc.NewPatch() - k.Annotations[models.ExternalChangesAnnotation] = "" + kc.Annotations[models.ExternalChangesAnnotation] = "" - err := r.Patch(context.Background(), k, patch) + err := r.Patch(context.Background(), kc, patch) if err != nil { l.Error(err, "Cannot patch cluster resource", - "cluster name", k.Spec.Name, "cluster ID", k.Status.ID) + "cluster name", kc.Spec.Name, "cluster ID", kc.Status.ID) - r.EventRecorder.Eventf(k, models.Warning, models.PatchFailed, + r.EventRecorder.Eventf(kc, models.Warning, models.PatchFailed, "Cluster resource patch is failed. Reason: %v", err) return reconcile.Result{}, err } - l.Info("External changes have been reconciled", "resource ID", k.Status.ID) - r.EventRecorder.Event(k, models.Normal, models.ExternalChanges, "External changes have been reconciled") + l.Info("External changes have been reconciled", "resource ID", kc.Status.ID) + r.EventRecorder.Event(kc, models.Normal, models.ExternalChanges, "External changes have been reconciled") return models.ExitReconcile, nil } @@ -390,6 +475,23 @@ func (r *KafkaConnectReconciler) handleDeleteCluster(ctx context.Context, kc *v1 return models.ExitReconcile, nil } + + if kc.Spec.OnPremisesSpec != nil { + err = deleteOnPremResources(ctx, r.Client, kc.Status.ID, kc.Namespace) + if err != nil { + l.Error(err, "Cannot delete cluster on-premises resources", + "cluster ID", kc.Status.ID) + r.EventRecorder.Eventf(kc, models.Warning, models.DeletionFailed, + "Cluster on-premises resources deletion is failed. Reason: %v", err) + return reconcile.Result{}, err + } + + l.Info("Cluster on-premises resources are deleted", + "cluster ID", kc.Status.ID) + r.EventRecorder.Eventf(kc, models.Normal, models.Deleted, + "Cluster on-premises resources are deleted") + r.Scheduler.RemoveJob(kc.GetJobID(scheduler.OnPremisesIPsChecker)) + } } err = deleteDefaultUserSecret(ctx, r.Client, client.ObjectKeyFromObject(kc)) @@ -472,6 +574,17 @@ func (r *KafkaConnectReconciler) createDefaultSecret(ctx context.Context, kc *v1 return nil } +func (r *KafkaConnectReconciler) startClusterOnPremisesIPsJob(k *v1beta1.KafkaConnect, b *onPremisesBootstrap) error { + job := newWatchOnPremisesIPsJob(k.Kind, b) + + err := r.Scheduler.ScheduleJob(k.GetJobID(scheduler.OnPremisesIPsChecker), scheduler.ClusterStatusInterval, job) + if err != nil { + return err + } + + return nil +} + func (r *KafkaConnectReconciler) startClusterStatusJob(kc *v1beta1.KafkaConnect) error { job := r.newWatchStatusJob(kc) diff --git a/controllers/clusters/on_premises.go b/controllers/clusters/on_premises.go new file mode 100644 index 000000000..18fefaf67 --- /dev/null +++ b/controllers/clusters/on_premises.go @@ -0,0 +1,927 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusters + +import ( + "context" + "fmt" + "strings" + + k8scorev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" + virtcorev1 "kubevirt.io/api/core/v1" + cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/instaclustr/operator/apis/clusters/v1beta1" + "github.com/instaclustr/operator/pkg/models" + "github.com/instaclustr/operator/pkg/scheduler" +) + +type onPremisesBootstrap struct { + K8sClient client.Client + K8sObject client.Object + EventRecorder record.EventRecorder + ClusterStatus v1beta1.ClusterStatus + OnPremisesSpec *v1beta1.OnPremisesSpec + ExposePorts []k8scorev1.ServicePort + HeadlessPorts []k8scorev1.ServicePort + PrivateNetworkCluster bool +} + +func newOnPremisesBootstrap( + k8sClient client.Client, + o client.Object, + e record.EventRecorder, + status v1beta1.ClusterStatus, + onPremisesSpec *v1beta1.OnPremisesSpec, + exposePorts, + headlessPorts []k8scorev1.ServicePort, + privateNetworkCluster bool, +) *onPremisesBootstrap { + return &onPremisesBootstrap{ + K8sClient: k8sClient, + K8sObject: o, + EventRecorder: e, + ClusterStatus: status, + OnPremisesSpec: onPremisesSpec, + ExposePorts: exposePorts, + HeadlessPorts: headlessPorts, + PrivateNetworkCluster: privateNetworkCluster, + } +} + +func handleCreateOnPremisesClusterResources(ctx context.Context, b *onPremisesBootstrap) error { + if len(b.ClusterStatus.DataCentres) < 1 { + return fmt.Errorf("datacenter ID is empty") + } + + if b.PrivateNetworkCluster { + err := reconcileSSHGatewayResources(ctx, b) + if err != nil { + return err + } + } + + err := reconcileNodesResources(ctx, b) + if err != nil { + return err + } + + return nil +} + +func reconcileSSHGatewayResources(ctx context.Context, b *onPremisesBootstrap) error { + gatewayDVSize, err := resource.ParseQuantity(b.OnPremisesSpec.OSDiskSize) + if err != nil { + return err + } + + gatewayDVName := fmt.Sprintf("%s-%s", models.GatewayDVPrefix, strings.ToLower(b.K8sObject.GetName())) + gatewayDV, err := createDV( + ctx, + b, + gatewayDVName, + b.ClusterStatus.DataCentres[0].ID, + gatewayDVSize, + true, + ) + if err != nil { + return err + } + + gatewayCPU := resource.Quantity{} + gatewayCPU.Set(b.OnPremisesSpec.SSHGatewayCPU) + + gatewayMemory, err := resource.ParseQuantity(b.OnPremisesSpec.SSHGatewayMemory) + if err != nil { + return err + } + + gatewayName := fmt.Sprintf("%s-%s", models.GatewayVMPrefix, strings.ToLower(b.K8sObject.GetName())) + + gatewayVM := &virtcorev1.VirtualMachine{} + err = b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: gatewayName, + }, gatewayVM) + if client.IgnoreNotFound(err) != nil { + return err + } + if k8serrors.IsNotFound(err) { + gatewayVM, err = newVM( + ctx, + b, + gatewayName, + b.ClusterStatus.DataCentres[0].ID, + models.GatewayRack, + gatewayDV.Name, + gatewayCPU, + gatewayMemory) + if err != nil { + return err + } + err = b.K8sClient.Create(ctx, gatewayVM) + if err != nil { + return err + } + } + + gatewaySvcName := fmt.Sprintf("%s-%s", models.GatewaySvcPrefix, gatewayName) + gatewayExposeService := &k8scorev1.Service{} + err = b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: gatewaySvcName, + }, gatewayExposeService) + + if client.IgnoreNotFound(err) != nil { + return err + } + if k8serrors.IsNotFound(err) { + gatewayExposeService = newExposeService( + b, + gatewaySvcName, + gatewayName, + b.ClusterStatus.DataCentres[0].ID, + ) + err = b.K8sClient.Create(ctx, gatewayExposeService) + if err != nil { + return err + } + } + + clusterIPServiceName := fmt.Sprintf("cluster-ip-%s", gatewayName) + nodeExposeService := &k8scorev1.Service{} + err = b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: clusterIPServiceName, + }, nodeExposeService) + if client.IgnoreNotFound(err) != nil { + return err + } + if k8serrors.IsNotFound(err) { + nodeExposeService = newClusterIPService( + b, + clusterIPServiceName, + gatewayName, + b.ClusterStatus.DataCentres[0].ID, + ) + err = b.K8sClient.Create(ctx, nodeExposeService) + if err != nil { + return err + } + } + + return nil +} + +func reconcileNodesResources(ctx context.Context, b *onPremisesBootstrap) error { + for i, node := range b.ClusterStatus.DataCentres[0].Nodes { + nodeOSDiskSize, err := resource.ParseQuantity(b.OnPremisesSpec.OSDiskSize) + if err != nil { + return err + } + + nodeOSDiskDVName := fmt.Sprintf("%s-%d-%s", models.NodeOSDVPrefix, i, strings.ToLower(b.K8sObject.GetName())) + nodeOSDV, err := createDV( + ctx, + b, + nodeOSDiskDVName, + node.ID, + nodeOSDiskSize, + true, + ) + if err != nil { + return err + } + + nodeDataDiskDVSize, err := resource.ParseQuantity(b.OnPremisesSpec.DataDiskSize) + if err != nil { + return err + } + + nodeDataDiskDVName := fmt.Sprintf("%s-%d-%s", models.NodeDVPrefix, i, strings.ToLower(b.K8sObject.GetName())) + nodeDataDV, err := createDV( + ctx, + b, + nodeDataDiskDVName, + node.ID, + nodeDataDiskDVSize, + false, + ) + if err != nil { + return err + } + + nodeCPU := resource.Quantity{} + nodeCPU.Set(b.OnPremisesSpec.NodeCPU) + + nodeMemory, err := resource.ParseQuantity(b.OnPremisesSpec.NodeMemory) + if err != nil { + return err + } + + nodeName := fmt.Sprintf("%s-%d-%s", models.NodeVMPrefix, i, strings.ToLower(b.K8sObject.GetName())) + + nodeVM := &virtcorev1.VirtualMachine{} + err = b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: nodeName, + }, nodeVM) + if client.IgnoreNotFound(err) != nil { + return err + } + if k8serrors.IsNotFound(err) { + nodeVM, err = newVM( + ctx, + b, + nodeName, + node.ID, + node.Rack, + nodeOSDV.Name, + nodeCPU, + nodeMemory, + nodeDataDV.Name, + ) + if err != nil { + return err + } + err = b.K8sClient.Create(ctx, nodeVM) + if err != nil { + return err + } + } + + if !b.PrivateNetworkCluster { + nodeExposeName := fmt.Sprintf("%s-%s", models.NodeSvcPrefix, nodeName) + nodeExposeService := &k8scorev1.Service{} + err = b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: nodeExposeName, + }, nodeExposeService) + if client.IgnoreNotFound(err) != nil { + return err + } + if k8serrors.IsNotFound(err) { + nodeExposeService = newExposeService( + b, + nodeExposeName, + nodeName, + node.ID, + ) + err = b.K8sClient.Create(ctx, nodeExposeService) + if err != nil { + return err + } + } + } + + headlessServiceName := fmt.Sprintf("%s-%s", models.KubevirtSubdomain, strings.ToLower(b.K8sObject.GetName())) + headlessSVC := &k8scorev1.Service{} + err = b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: headlessServiceName, + }, headlessSVC) + + if client.IgnoreNotFound(err) != nil { + return err + } + if k8serrors.IsNotFound(err) { + headlessSVC = newHeadlessService( + b, + headlessServiceName, + ) + err = b.K8sClient.Create(ctx, headlessSVC) + if err != nil { + return err + } + } + + clusterIPServiceName := fmt.Sprintf("cluster-ip-%s", nodeName) + nodeExposeService := &k8scorev1.Service{} + err = b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: clusterIPServiceName, + }, nodeExposeService) + if client.IgnoreNotFound(err) != nil { + return err + } + if k8serrors.IsNotFound(err) { + nodeExposeService = newClusterIPService( + b, + clusterIPServiceName, + nodeName, + node.ID, + ) + err = b.K8sClient.Create(ctx, nodeExposeService) + if err != nil { + return err + } + } + + } + return nil +} + +func createDV( + ctx context.Context, + b *onPremisesBootstrap, + name, + nodeID string, + size resource.Quantity, + isOSDisk bool, +) (*cdiv1beta1.DataVolume, error) { + dv := &cdiv1beta1.DataVolume{} + err := b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: name, + }, dv) + if client.IgnoreNotFound(err) != nil { + return nil, err + } + if k8serrors.IsNotFound(err) { + dv = newDataDiskDV( + b, + name, + nodeID, + size, + isOSDisk, + ) + err = b.K8sClient.Create(ctx, dv) + if err != nil { + return nil, err + } + } + return dv, nil +} + +func newDataDiskDV( + b *onPremisesBootstrap, + name, + nodeID string, + storageSize resource.Quantity, + isOSDisk bool, +) *cdiv1beta1.DataVolume { + dvSource := &cdiv1beta1.DataVolumeSource{} + + if isOSDisk { + dvSource.HTTP = &cdiv1beta1.DataVolumeSourceHTTP{URL: b.OnPremisesSpec.OSImageURL} + } else { + dvSource.Blank = &cdiv1beta1.DataVolumeBlankImage{} + } + + return &cdiv1beta1.DataVolume{ + TypeMeta: metav1.TypeMeta{ + Kind: models.DVKind, + APIVersion: models.CDIKubevirtV1beta1APIVersion, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: b.K8sObject.GetNamespace(), + Labels: map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeIDLabel: nodeID, + }, + Finalizers: []string{models.DeletionFinalizer}, + }, + Spec: cdiv1beta1.DataVolumeSpec{ + Source: dvSource, + PVC: &k8scorev1.PersistentVolumeClaimSpec{ + AccessModes: []k8scorev1.PersistentVolumeAccessMode{ + k8scorev1.ReadWriteOnce, + }, + Resources: k8scorev1.ResourceRequirements{ + Requests: k8scorev1.ResourceList{ + models.Storage: storageSize, + }, + }, + StorageClassName: &b.OnPremisesSpec.StorageClassName, + }, + }, + } +} + +func newVM( + ctx context.Context, + b *onPremisesBootstrap, + vmName, + nodeID, + nodeRack, + OSDiskDVName string, + cpu, + memory resource.Quantity, + storageDVNames ...string, +) (*virtcorev1.VirtualMachine, error) { + runStrategy := virtcorev1.RunStrategyAlways + bootOrder1 := uint(1) + + cloudInitSecret := &k8scorev1.Secret{} + err := b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.OnPremisesSpec.CloudInitScriptRef.Namespace, + Name: b.OnPremisesSpec.CloudInitScriptRef.Name, + }, cloudInitSecret) + if err != nil { + return nil, err + } + + labelSet := map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeIDLabel: nodeID, + models.NodeRackLabel: nodeRack, + models.KubevirtDomainLabel: vmName, + } + + if nodeRack != models.GatewayRack { + labelSet[models.NodeLabel] = models.WorkerNode + } + + vm := &virtcorev1.VirtualMachine{ + TypeMeta: metav1.TypeMeta{ + Kind: models.VirtualMachineKind, + APIVersion: models.KubevirtV1APIVersion, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: vmName, + Namespace: b.K8sObject.GetNamespace(), + Labels: labelSet, + Finalizers: []string{models.DeletionFinalizer}, + }, + Spec: virtcorev1.VirtualMachineSpec{ + RunStrategy: &runStrategy, + Template: &virtcorev1.VirtualMachineInstanceTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labelSet, + }, + Spec: virtcorev1.VirtualMachineInstanceSpec{ + Hostname: vmName, + Subdomain: fmt.Sprintf("%s-%s", models.KubevirtSubdomain, b.K8sObject.GetName()), + Domain: virtcorev1.DomainSpec{ + Resources: virtcorev1.ResourceRequirements{ + Requests: k8scorev1.ResourceList{ + models.CPU: cpu, + models.Memory: memory, + }, + }, + Devices: virtcorev1.Devices{ + Disks: []virtcorev1.Disk{ + { + Name: models.Boot, + BootOrder: &bootOrder1, + IO: models.Native, + Cache: models.None, + DiskDevice: virtcorev1.DiskDevice{ + Disk: &virtcorev1.DiskTarget{ + Bus: models.Virtio, + }, + }, + }, + { + Name: models.CloudInit, + DiskDevice: virtcorev1.DiskDevice{}, + Cache: models.None, + }, + }, + Interfaces: []virtcorev1.Interface{ + { + Name: models.Default, + InterfaceBindingMethod: virtcorev1.InterfaceBindingMethod{ + Bridge: &virtcorev1.InterfaceBridge{}, + }, + }, + }, + }, + }, + Volumes: []virtcorev1.Volume{ + { + Name: models.Boot, + VolumeSource: virtcorev1.VolumeSource{ + PersistentVolumeClaim: &virtcorev1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: k8scorev1.PersistentVolumeClaimVolumeSource{ + ClaimName: OSDiskDVName, + }, + }, + }, + }, + { + Name: models.CloudInit, + VolumeSource: virtcorev1.VolumeSource{ + CloudInitNoCloud: &virtcorev1.CloudInitNoCloudSource{ + UserDataSecretRef: &k8scorev1.LocalObjectReference{ + Name: b.OnPremisesSpec.CloudInitScriptRef.Name, + }, + }, + }, + }, + }, + Networks: []virtcorev1.Network{ + { + Name: models.Default, + NetworkSource: virtcorev1.NetworkSource{ + Pod: &virtcorev1.PodNetwork{}, + }, + }, + }, + }, + }, + }, + } + + for i, dvName := range storageDVNames { + diskName := fmt.Sprintf("%s-%d-%s", models.DataDisk, i, vm.Name) + + vm.Spec.Template.Spec.Domain.Devices.Disks = append(vm.Spec.Template.Spec.Domain.Devices.Disks, virtcorev1.Disk{ + Name: diskName, + IO: models.Native, + Cache: models.None, + DiskDevice: virtcorev1.DiskDevice{ + Disk: &virtcorev1.DiskTarget{ + Bus: models.Virtio, + }, + }, + Serial: models.DataDiskSerial, + }) + + vm.Spec.Template.Spec.Volumes = append(vm.Spec.Template.Spec.Volumes, virtcorev1.Volume{ + Name: diskName, + VolumeSource: virtcorev1.VolumeSource{ + PersistentVolumeClaim: &virtcorev1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: k8scorev1.PersistentVolumeClaimVolumeSource{ + ClaimName: dvName, + }, + }, + }, + }) + } + + return vm, nil +} + +func newExposeService( + b *onPremisesBootstrap, + svcName, + vmName, + nodeID string, +) *k8scorev1.Service { + return &k8scorev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: models.ServiceKind, + APIVersion: models.K8sAPIVersionV1, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: b.K8sObject.GetNamespace(), + Labels: map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeIDLabel: nodeID, + }, + Finalizers: []string{models.DeletionFinalizer}, + }, + Spec: k8scorev1.ServiceSpec{ + Ports: b.ExposePorts, + Selector: map[string]string{ + models.KubevirtDomainLabel: vmName, + models.NodeIDLabel: nodeID, + }, + Type: k8scorev1.ServiceTypeLoadBalancer, + }, + } +} + +func newHeadlessService( + b *onPremisesBootstrap, + svcName string, +) *k8scorev1.Service { + return &k8scorev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: models.ServiceKind, + APIVersion: models.K8sAPIVersionV1, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: b.K8sObject.GetNamespace(), + Labels: map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + }, + Finalizers: []string{models.DeletionFinalizer}, + }, + Spec: k8scorev1.ServiceSpec{ + ClusterIP: k8scorev1.ClusterIPNone, + Ports: b.HeadlessPorts, + Selector: map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeLabel: models.WorkerNode, + }, + }, + } +} + +func deleteOnPremResources(ctx context.Context, K8sClient client.Client, clusterID, ns string) error { + vms := &virtcorev1.VirtualMachineList{} + err := K8sClient.List(ctx, vms, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.ClusterIDLabel: clusterID, + }), + Namespace: ns, + }) + if err != nil { + return err + } + + for _, vm := range vms.Items { + err = K8sClient.Delete(ctx, &vm) + if err != nil { + return err + } + + patch := client.MergeFrom(vm.DeepCopy()) + controllerutil.RemoveFinalizer(&vm, models.DeletionFinalizer) + err = K8sClient.Patch(ctx, &vm, patch) + if err != nil { + return err + } + } + + vmis := &virtcorev1.VirtualMachineInstanceList{} + err = K8sClient.List(ctx, vmis, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.ClusterIDLabel: clusterID, + }), + Namespace: ns, + }) + if err != nil { + return err + } + + for _, vmi := range vmis.Items { + err = K8sClient.Delete(ctx, &vmi) + if err != nil { + return err + } + + patch := client.MergeFrom(vmi.DeepCopy()) + controllerutil.RemoveFinalizer(&vmi, models.DeletionFinalizer) + err = K8sClient.Patch(ctx, &vmi, patch) + if err != nil { + return err + } + } + + dvs := &cdiv1beta1.DataVolumeList{} + err = K8sClient.List(ctx, dvs, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.ClusterIDLabel: clusterID, + }), + Namespace: ns, + }) + if err != nil { + return err + } + + for _, dv := range dvs.Items { + err = K8sClient.Delete(ctx, &dv) + if err != nil { + return err + } + + patch := client.MergeFrom(dv.DeepCopy()) + controllerutil.RemoveFinalizer(&dv, models.DeletionFinalizer) + err = K8sClient.Patch(ctx, &dv, patch) + if err != nil { + return err + } + } + + svcs := &k8scorev1.ServiceList{} + err = K8sClient.List(ctx, svcs, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.ClusterIDLabel: clusterID, + }), + Namespace: ns, + }) + if err != nil { + return err + } + + for _, svc := range svcs.Items { + err = K8sClient.Delete(ctx, &svc) + if err != nil { + return err + } + + patch := client.MergeFrom(svc.DeepCopy()) + controllerutil.RemoveFinalizer(&svc, models.DeletionFinalizer) + err = K8sClient.Patch(ctx, &svc, patch) + if err != nil { + return err + } + } + + return nil +} + +func newExposePorts(sp []k8scorev1.ServicePort) []k8scorev1.ServicePort { + var ports []k8scorev1.ServicePort + ports = []k8scorev1.ServicePort{{ + Name: models.SSH, + Port: models.Port22, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: models.Port22, + }, + }, + } + + ports = append(ports, sp...) + + return ports +} + +func newWatchOnPremisesIPsJob(kind string, b *onPremisesBootstrap) scheduler.Job { + l := log.Log.WithValues("component", fmt.Sprintf("%sOnPremisesIPsCheckerJob", kind)) + + return func() error { + allNodePods := &k8scorev1.PodList{} + err := b.K8sClient.List(context.Background(), allNodePods, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeLabel: models.WorkerNode, + }), + Namespace: b.K8sObject.GetNamespace(), + }) + if err != nil { + l.Error(err, "Cannot get on-premises cluster pods", + "cluster name", b.K8sObject.GetName(), + "clusterID", b.ClusterStatus.ID, + ) + + b.EventRecorder.Eventf( + b.K8sObject, models.Warning, models.CreationFailed, + "Fetching on-premises cluster pods is failed. Reason: %v", + err, + ) + return err + } + + if len(allNodePods.Items) != len(b.ClusterStatus.DataCentres[0].Nodes) { + err = fmt.Errorf("the quantity of pods does not match the number of on-premises nodes") + l.Error(err, "Cannot compare private IPs for the cluster", + "cluster name", b.K8sObject.GetName(), + "clusterID", b.ClusterStatus.ID, + ) + + b.EventRecorder.Eventf( + b.K8sObject, models.Warning, models.CreationFailed, + "Comparing of cluster's private IPs is failing. Reason: %v", + err, + ) + return err + } + + for _, node := range b.ClusterStatus.DataCentres[0].Nodes { + nodePods := &k8scorev1.PodList{} + err = b.K8sClient.List(context.Background(), nodePods, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeIDLabel: node.ID, + }), + Namespace: b.K8sObject.GetNamespace(), + }) + if err != nil { + l.Error(err, "Cannot get on-premises cluster pods", + "cluster name", b.K8sObject.GetName(), + "clusterID", b.ClusterStatus.ID, + ) + + b.EventRecorder.Eventf( + b.K8sObject, models.Warning, models.CreationFailed, + "Fetching on-premises cluster pods is failed. Reason: %v", + err, + ) + return err + } + + for _, pod := range nodePods.Items { + if (pod.Status.PodIP != "" && node.PrivateAddress != "") && + (pod.Status.PodIP != node.PrivateAddress) { + + err = fmt.Errorf("private IPs was changed") + l.Error(err, "Node's private IP addresses are not equal", + "cluster name", b.K8sObject.GetName(), + "clusterID", b.ClusterStatus.ID, + "nodeID", node.ID, + "nodeIP", node.PrivateAddress, + "podIP", pod.Status.PodIP, + ) + + b.EventRecorder.Eventf( + b.K8sObject, models.Warning, models.CreationFailed, + "The private IP addresses of the node are not matching. Reason: %v", + err, + ) + return err + } + } + + if !b.PrivateNetworkCluster { + nodeSVCs := &k8scorev1.ServiceList{} + err = b.K8sClient.List(context.Background(), nodeSVCs, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeIDLabel: node.ID, + }), + Namespace: b.K8sObject.GetNamespace(), + }) + if err != nil { + l.Error(err, "Cannot get services backed by on-premises cluster pods", + "cluster name", b.K8sObject.GetName(), + "clusterID", b.ClusterStatus.ID, + ) + + b.EventRecorder.Eventf( + b.K8sObject, models.Warning, models.CreationFailed, + "Fetching services backed by on-premises cluster pods is failed. Reason: %v", + err, + ) + return err + } + for _, svc := range nodeSVCs.Items { + if len(svc.Status.LoadBalancer.Ingress) != 0 { + if (svc.Status.LoadBalancer.Ingress[0].IP != "" && node.PublicAddress != "") && + (svc.Status.LoadBalancer.Ingress[0].IP != node.PublicAddress) { + + err = fmt.Errorf("public IPs was changed") + l.Error(err, "Node's public IP addresses are not equal", + "cluster name", b.K8sObject.GetName(), + "clusterID", b.ClusterStatus.ID, + "nodeID", node.ID, + "nodeIP", node.PrivateAddress, + "svcIP", svc.Status.LoadBalancer.Ingress[0].IP, + ) + + b.EventRecorder.Eventf( + b.K8sObject, models.Warning, models.CreationFailed, + "The public IP addresses of the node are not matching. Reason: %v", + err, + ) + return err + } + } + } + } + } + return nil + } +} + +func newClusterIPService( + b *onPremisesBootstrap, + svcName, + vmName, + nodeID string, +) *k8scorev1.Service { + return &k8scorev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: models.ServiceKind, + APIVersion: models.K8sAPIVersionV1, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: b.K8sObject.GetNamespace(), + Labels: map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeIDLabel: nodeID, + }, + Finalizers: []string{models.DeletionFinalizer}, + }, + Spec: k8scorev1.ServiceSpec{ + Ports: b.ExposePorts, + Selector: map[string]string{ + models.KubevirtDomainLabel: vmName, + models.NodeIDLabel: nodeID, + }, + Type: k8scorev1.ServiceTypeClusterIP, + }, + } +} diff --git a/controllers/clusters/opensearch_controller.go b/controllers/clusters/opensearch_controller.go index cf4012adb..895f3a0d2 100644 --- a/controllers/clusters/opensearch_controller.go +++ b/controllers/clusters/opensearch_controller.go @@ -276,7 +276,7 @@ func (r *OpenSearchReconciler) HandleUpdateCluster( "cluster ID", o.Status.ID, ) - r.EventRecorder.Eventf(o, models.Warning, models.ConvertionFailed, + r.EventRecorder.Eventf(o, models.Warning, models.ConversionFailed, "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", err) return reconcile.Result{}, err diff --git a/controllers/clusters/postgresql_controller.go b/controllers/clusters/postgresql_controller.go index e6fb6c743..e1a25c0f7 100644 --- a/controllers/clusters/postgresql_controller.go +++ b/controllers/clusters/postgresql_controller.go @@ -62,9 +62,16 @@ type PostgreSQLReconciler struct { //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=postgresqls/status,verbs=get;update;patch //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=postgresqls/finalizers,verbs=update //+kubebuilder:rbac:groups=clusterresources.instaclustr.com,resources=clusterbackups,verbs=get;list;create;update;patch;deletecollection;delete -//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;watch;create;delete;update //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch //+kubebuilder:rbac:groups="",resources=nodes,verbs=get;watch;list +//+kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachines,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete;deletecollection // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -72,19 +79,19 @@ type PostgreSQLReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile func (r *PostgreSQLReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) + l := log.FromContext(ctx) pg := &v1beta1.PostgreSQL{} err := r.Client.Get(ctx, req.NamespacedName, pg) if err != nil { if k8serrors.IsNotFound(err) { - logger.Info("PostgreSQL custom resource is not found", + l.Info("PostgreSQL custom resource is not found", "resource name", req.NamespacedName, ) return models.ExitReconcile, nil } - logger.Error(err, "Unable to fetch PostgreSQL cluster", + l.Error(err, "Unable to fetch PostgreSQL cluster", "resource name", req.NamespacedName, ) @@ -93,22 +100,22 @@ func (r *PostgreSQLReconciler) Reconcile(ctx context.Context, req ctrl.Request) switch pg.Annotations[models.ResourceStateAnnotation] { case models.CreatingEvent: - return r.handleCreateCluster(ctx, pg, logger) + return r.handleCreateCluster(ctx, pg, l) case models.UpdatingEvent: - return r.handleUpdateCluster(ctx, pg, logger) + return r.handleUpdateCluster(ctx, pg, l) case models.DeletingEvent: - return r.handleDeleteCluster(ctx, pg, logger) + return r.handleDeleteCluster(ctx, pg, l) case models.SecretEvent: - return r.handleUpdateDefaultUserPassword(ctx, pg, logger) + return r.handleUpdateDefaultUserPassword(ctx, pg, l) case models.GenericEvent: - logger.Info("PostgreSQL resource generic event isn't handled", + l.Info("PostgreSQL resource generic event isn't handled", "cluster name", pg.Spec.Name, "request", req, "event", pg.Annotations[models.ResourceStateAnnotation], ) return models.ExitReconcile, nil default: - logger.Info("PostgreSQL resource event isn't handled", + l.Info("PostgreSQL resource event isn't handled", "cluster name", pg.Spec.Name, "request", req, "event", pg.Annotations[models.ResourceStateAnnotation], @@ -120,23 +127,23 @@ func (r *PostgreSQLReconciler) Reconcile(ctx context.Context, req ctrl.Request) func (r *PostgreSQLReconciler) handleCreateCluster( ctx context.Context, pg *v1beta1.PostgreSQL, - logger logr.Logger, + l logr.Logger, ) (reconcile.Result, error) { - logger = logger.WithName("PostgreSQL creation event") + l = l.WithName("PostgreSQL creation event") var err error patch := pg.NewPatch() if pg.Status.ID == "" { if pg.Spec.HasRestore() { - logger.Info( + l.Info( "Creating PostgreSQL cluster from backup", "original cluster ID", pg.Spec.PgRestoreFrom.ClusterID, ) pg.Status.ID, err = r.API.RestoreCluster(pg.RestoreInfoToInstAPI(pg.Spec.PgRestoreFrom), models.PgRestoreValue) if err != nil { - logger.Error(err, "Cannot restore PostgreSQL cluster from backup", + l.Error(err, "Cannot restore PostgreSQL cluster from backup", "original cluster ID", pg.Spec.PgRestoreFrom.ClusterID, ) @@ -156,7 +163,7 @@ func (r *PostgreSQLReconciler) handleCreateCluster( pg.Status.ID, ) } else { - logger.Info( + l.Info( "Creating PostgreSQL cluster", "cluster name", pg.Spec.Name, "data centres", pg.Spec.DataCentres, @@ -166,7 +173,7 @@ func (r *PostgreSQLReconciler) handleCreateCluster( pg.Status.ID, err = r.API.CreateCluster(instaclustr.PGSQLEndpoint, pgSpec) if err != nil { - logger.Error( + l.Error( err, "Cannot create PostgreSQL cluster", "spec", pg.Spec, ) @@ -189,7 +196,7 @@ func (r *PostgreSQLReconciler) handleCreateCluster( err = r.Status().Patch(ctx, pg, patch) if err != nil { - logger.Error(err, "Cannot patch PostgreSQL resource status", + l.Error(err, "Cannot patch PostgreSQL resource status", "cluster name", pg.Spec.Name, "status", pg.Status, ) @@ -203,7 +210,7 @@ func (r *PostgreSQLReconciler) handleCreateCluster( return reconcile.Result{}, err } - logger.Info( + l.Info( "PostgreSQL resource has been created", "cluster name", pg.Name, "cluster ID", pg.Status.ID, @@ -218,7 +225,7 @@ func (r *PostgreSQLReconciler) handleCreateCluster( pg.Annotations[models.ResourceStateAnnotation] = models.CreatedEvent err = r.Patch(ctx, pg, patch) if err != nil { - logger.Error(err, "Cannot patch PostgreSQL resource", + l.Error(err, "Cannot patch PostgreSQL resource", "cluster name", pg.Spec.Name, "status", pg.Status) @@ -232,7 +239,7 @@ func (r *PostgreSQLReconciler) handleCreateCluster( if pg.Status.State != models.DeletedStatus { err = r.startClusterStatusJob(pg) if err != nil { - logger.Error(err, "Cannot start PostgreSQL cluster status check job", + l.Error(err, "Cannot start PostgreSQL cluster status check job", "cluster ID", pg.Status.ID, ) @@ -249,9 +256,87 @@ func (r *PostgreSQLReconciler) handleCreateCluster( "Cluster status check job is started", ) + if pg.Spec.OnPremisesSpec != nil { + iData, err := r.API.GetPostgreSQL(pg.Status.ID) + if err != nil { + l.Error(err, "Cannot get cluster from the Instaclustr API", + "cluster name", pg.Spec.Name, + "data centres", pg.Spec.DataCentres, + "cluster ID", pg.Status.ID, + ) + r.EventRecorder.Eventf( + pg, models.Warning, models.FetchFailed, + "Cluster fetch from the Instaclustr API is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + iPostgreSQL, err := pg.FromInstAPI(iData) + if err != nil { + l.Error( + err, "Cannot convert cluster from the Instaclustr API", + "cluster name", pg.Spec.Name, + "cluster ID", pg.Status.ID, + ) + r.EventRecorder.Eventf( + pg, models.Warning, models.ConversionFailed, + "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + bootstrap := newOnPremisesBootstrap( + r.Client, + pg, + r.EventRecorder, + iPostgreSQL.Status.ClusterStatus, + pg.Spec.OnPremisesSpec, + newExposePorts(pg.GetExposePorts()), + pg.GetHeadlessPorts(), + pg.Spec.PrivateNetworkCluster, + ) + + err = handleCreateOnPremisesClusterResources(ctx, bootstrap) + if err != nil { + l.Error( + err, "Cannot create resources for on-premises cluster", + "cluster spec", pg.Spec.OnPremisesSpec, + ) + r.EventRecorder.Eventf( + pg, models.Warning, models.CreationFailed, + "Resources creation for on-premises cluster is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + err = r.startClusterOnPremisesIPsJob(pg, bootstrap) + if err != nil { + l.Error(err, "Cannot start on-premises cluster IPs check job", + "cluster ID", pg.Status.ID, + ) + + r.EventRecorder.Eventf( + pg, models.Warning, models.CreationFailed, + "On-premises cluster IPs check job is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + l.Info( + "On-premises resources have been created", + "cluster name", pg.Spec.Name, + "on-premises Spec", pg.Spec.OnPremisesSpec, + "cluster ID", pg.Status.ID, + ) + return models.ExitReconcile, nil + } + err = r.startClusterBackupsJob(pg) if err != nil { - logger.Error(err, "Cannot start PostgreSQL cluster backups check job", + l.Error(err, "Cannot start PostgreSQL cluster backups check job", "cluster ID", pg.Status.ID, ) @@ -271,7 +356,7 @@ func (r *PostgreSQLReconciler) handleCreateCluster( if pg.Spec.UserRefs != nil { err = r.startUsersCreationJob(pg) if err != nil { - logger.Error(err, "Failed to start user PostreSQL creation job") + l.Error(err, "Failed to start user PostreSQL creation job") r.EventRecorder.Eventf(pg, models.Warning, models.CreationFailed, "User creation job is failed. Reason: %v", err) return reconcile.Result{}, err @@ -282,9 +367,9 @@ func (r *PostgreSQLReconciler) handleCreateCluster( } } - err = r.createDefaultPassword(ctx, pg, logger) + err = r.createDefaultPassword(ctx, pg, l) if err != nil { - logger.Error(err, "Cannot create default password for PostgreSQL", + l.Error(err, "Cannot create default password for PostgreSQL", "cluster name", pg.Spec.Name, "clusterID", pg.Status.ID, ) @@ -304,13 +389,13 @@ func (r *PostgreSQLReconciler) handleCreateCluster( func (r *PostgreSQLReconciler) handleUpdateCluster( ctx context.Context, pg *v1beta1.PostgreSQL, - logger logr.Logger, + l logr.Logger, ) (reconcile.Result, error) { - logger = logger.WithName("PostgreSQL update event") + l = l.WithName("PostgreSQL update event") iData, err := r.API.GetPostgreSQL(pg.Status.ID) if err != nil { - logger.Error( + l.Error( err, "Cannot get PostgreSQL cluster status from the Instaclustr API", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, @@ -326,14 +411,14 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( iPg, err := pg.FromInstAPI(iData) if err != nil { - logger.Error( + l.Error( err, "Cannot convert PostgreSQL cluster status from the Instaclustr API", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, ) r.EventRecorder.Eventf( - pg, models.Warning, models.ConvertionFailed, + pg, models.Warning, models.ConversionFailed, "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", err, ) @@ -341,7 +426,7 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( } if iPg.Status.CurrentClusterOperationStatus != models.NoOperation { - logger.Info("PostgreSQL cluster is not ready to update", + l.Info("PostgreSQL cluster is not ready to update", "cluster name", pg.Spec.Name, "cluster status", iPg.Status.State, "current operation status", iPg.Status.CurrentClusterOperationStatus, @@ -350,7 +435,7 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( pg.Annotations[models.UpdateQueuedAnnotation] = models.True err = r.Patch(ctx, pg, patch) if err != nil { - logger.Error(err, "Cannot patch cluster resource", + l.Error(err, "Cannot patch cluster resource", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID) r.EventRecorder.Eventf( @@ -364,17 +449,17 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( } if pg.Annotations[models.ExternalChangesAnnotation] == models.True { - return r.handleExternalChanges(pg, iPg, logger) + return r.handleExternalChanges(pg, iPg, l) } if pg.Spec.ClusterSettingsNeedUpdate(iPg.Spec.Cluster) { - logger.Info("Updating cluster settings", + l.Info("Updating cluster settings", "instaclustr description", iPg.Spec.Description, "instaclustr two factor delete", iPg.Spec.TwoFactorDelete) err = r.API.UpdateClusterSettings(pg.Status.ID, pg.Spec.ClusterSettingsUpdateToInstAPI()) if err != nil { - logger.Error(err, "Cannot update cluster settings", + l.Error(err, "Cannot update cluster settings", "cluster ID", pg.Status.ID, "cluster spec", pg.Spec) r.EventRecorder.Eventf(pg, models.Warning, models.UpdateFailed, "Cannot update cluster settings. Reason: %v", err) @@ -384,14 +469,14 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( } if !pg.Spec.AreDCsEqual(iPg.Spec.DataCentres) { - logger.Info("Update request to Instaclustr API has been sent", + l.Info("Update request to Instaclustr API has been sent", "spec data centres", pg.Spec.DataCentres, "resize settings", pg.Spec.ResizeSettings, ) err = r.updateCluster(pg) if err != nil { - logger.Error(err, "Cannot update Data Centres", + l.Error(err, "Cannot update Data Centres", "cluster name", pg.Spec.Name, ) @@ -405,7 +490,7 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( pg.Annotations[models.UpdateQueuedAnnotation] = models.True err = r.Patch(ctx, pg, patch) if err != nil { - logger.Error(err, "Cannot patch PostgreSQL metadata", + l.Error(err, "Cannot patch PostgreSQL metadata", "cluster name", pg.Spec.Name, "cluster metadata", pg.ObjectMeta, ) @@ -420,7 +505,7 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( return reconcile.Result{}, err } - logger.Info( + l.Info( "Cluster has been updated", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, @@ -430,7 +515,7 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( iConfigs, err := r.API.GetPostgreSQLConfigs(pg.Status.ID) if err != nil { - logger.Error(err, "Cannot get PostgreSQL cluster configs", + l.Error(err, "Cannot get PostgreSQL cluster configs", "cluster name", pg.Spec.Name, "clusterID", pg.Status.ID, ) @@ -449,7 +534,7 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( pg.Spec.ClusterConfigurations, iConfig.ConfigurationProperties) if err != nil { - logger.Error(err, "Cannot reconcile PostgreSQL cluster configs", + l.Error(err, "Cannot reconcile PostgreSQL cluster configs", "cluster name", pg.Spec.Name, "clusterID", pg.Status.ID, "configs", pg.Spec.ClusterConfigurations, @@ -464,16 +549,16 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( return reconcile.Result{}, err } - logger.Info("PostgreSQL cluster configurations were updated", + l.Info("PostgreSQL cluster configurations were updated", "cluster name", pg.Spec.Name, ) } pg.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent pg.Annotations[models.UpdateQueuedAnnotation] = "" - err = r.patchClusterMetadata(ctx, pg, logger) + err = r.patchClusterMetadata(ctx, pg, l) if err != nil { - logger.Error(err, "Cannot patch PostgreSQL resource metadata", + l.Error(err, "Cannot patch PostgreSQL resource metadata", "cluster name", pg.Spec.Name, "cluster metadata", pg.ObjectMeta, ) @@ -486,7 +571,7 @@ func (r *PostgreSQLReconciler) handleUpdateCluster( return reconcile.Result{}, err } - logger.Info("PostgreSQL cluster was updated", + l.Info("PostgreSQL cluster was updated", "cluster name", pg.Spec.Name, "cluster status", pg.Status.State, ) @@ -790,13 +875,13 @@ func (r *PostgreSQLReconciler) handleExternalChanges(pg, iPg *v1beta1.PostgreSQL func (r *PostgreSQLReconciler) handleDeleteCluster( ctx context.Context, pg *v1beta1.PostgreSQL, - logger logr.Logger, + l logr.Logger, ) (reconcile.Result, error) { - logger = logger.WithName("PostgreSQL deletion event") + l = l.WithName("PostgreSQL deletion event") _, err := r.API.GetPostgreSQL(pg.Status.ID) if err != nil && !errors.Is(err, instaclustr.NotFound) { - logger.Error(err, "Cannot get PostgreSQL cluster status", + l.Error(err, "Cannot get PostgreSQL cluster status", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, ) @@ -810,13 +895,13 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( } if !errors.Is(err, instaclustr.NotFound) { - logger.Info("Sending cluster deletion to the Instaclustr API", + l.Info("Sending cluster deletion to the Instaclustr API", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID) err = r.API.DeleteCluster(pg.Status.ID, instaclustr.PGSQLEndpoint) if err != nil { - logger.Error(err, "Cannot delete PostgreSQL cluster", + l.Error(err, "Cannot delete PostgreSQL cluster", "cluster name", pg.Spec.Name, "cluster status", pg.Status.State, ) @@ -839,7 +924,7 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( pg.Annotations[models.ClusterDeletionAnnotation] = models.Triggered err = r.Patch(ctx, pg, patch) if err != nil { - logger.Error(err, "Cannot patch cluster resource", + l.Error(err, "Cannot patch cluster resource", "cluster name", pg.Spec.Name, "cluster state", pg.Status.State) r.EventRecorder.Eventf(pg, models.Warning, models.PatchFailed, @@ -849,26 +934,43 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( return reconcile.Result{}, err } - logger.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", pg.Status.ID) + l.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", pg.Status.ID) r.EventRecorder.Event(pg, models.Normal, models.DeletionStarted, "Two-Factor Delete is enabled, please confirm cluster deletion via email or phone.") return models.ExitReconcile, nil } + + if pg.Spec.OnPremisesSpec != nil { + err = deleteOnPremResources(ctx, r.Client, pg.Status.ID, pg.Namespace) + if err != nil { + l.Error(err, "Cannot delete cluster on-premises resources", + "cluster ID", pg.Status.ID) + r.EventRecorder.Eventf(pg, models.Warning, models.DeletionFailed, + "Cluster on-premises resources deletion is failed. Reason: %v", err) + return reconcile.Result{}, err + } + + l.Info("Cluster on-premises resources are deleted", + "cluster ID", pg.Status.ID) + r.EventRecorder.Eventf(pg, models.Normal, models.Deleted, + "Cluster on-premises resources are deleted") + r.Scheduler.RemoveJob(pg.GetJobID(scheduler.OnPremisesIPsChecker)) + } } - logger.Info("PostgreSQL cluster is being deleted. Deleting PostgreSQL default user secret", + l.Info("PostgreSQL cluster is being deleted. Deleting PostgreSQL default user secret", "cluster ID", pg.Status.ID, ) - logger.Info("Deleting cluster backup resources", + l.Info("Deleting cluster backup resources", "cluster ID", pg.Status.ID, ) err = r.deleteBackups(ctx, pg.Status.ID, pg.Namespace) if err != nil { - logger.Error(err, "Cannot delete PostgreSQL backup resources", + l.Error(err, "Cannot delete PostgreSQL backup resources", "cluster ID", pg.Status.ID, ) r.EventRecorder.Eventf( @@ -879,7 +981,7 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( return reconcile.Result{}, err } - logger.Info("Cluster backup resources were deleted", + l.Info("Cluster backup resources were deleted", "cluster ID", pg.Status.ID, ) @@ -893,7 +995,7 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( r.Scheduler.RemoveJob(pg.GetJobID(scheduler.StatusChecker)) for _, ref := range pg.Spec.UserRefs { - err = r.handleUsersDetach(ctx, logger, pg, ref) + err = r.handleUsersDetach(ctx, l, pg, ref) if err != nil { return reconcile.Result{}, err } @@ -901,9 +1003,9 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( controllerutil.RemoveFinalizer(pg, models.DeletionFinalizer) pg.Annotations[models.ResourceStateAnnotation] = models.DeletedEvent - err = r.patchClusterMetadata(ctx, pg, logger) + err = r.patchClusterMetadata(ctx, pg, l) if err != nil { - logger.Error( + l.Error( err, "Cannot patch PostgreSQL resource metadata after finalizer removal", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, @@ -919,7 +1021,7 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( err = r.deleteSecret(ctx, pg) if client.IgnoreNotFound(err) != nil { - logger.Error(err, "Cannot delete PostgreSQL default user secret", + l.Error(err, "Cannot delete PostgreSQL default user secret", "cluster ID", pg.Status.ID, ) @@ -931,7 +1033,7 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( return reconcile.Result{}, err } - logger.Info("Cluster PostgreSQL default user secret was deleted", + l.Info("Cluster PostgreSQL default user secret was deleted", "cluster ID", pg.Status.ID, ) @@ -943,7 +1045,7 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( err = exposeservice.Delete(r.Client, pg.Name, pg.Namespace) if err != nil { - logger.Error(err, "Cannot delete PostgreSQL cluster expose service", + l.Error(err, "Cannot delete PostgreSQL cluster expose service", "cluster ID", pg.Status.ID, "cluster name", pg.Spec.Name, ) @@ -951,7 +1053,7 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( return reconcile.Result{}, err } - logger.Info("PostgreSQL cluster was deleted", + l.Info("PostgreSQL cluster was deleted", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, ) @@ -967,13 +1069,13 @@ func (r *PostgreSQLReconciler) handleDeleteCluster( func (r *PostgreSQLReconciler) handleUpdateDefaultUserPassword( ctx context.Context, pg *v1beta1.PostgreSQL, - logger logr.Logger, + l logr.Logger, ) (reconcile.Result, error) { - logger = logger.WithName("PostgreSQL default user password updating event") + l = l.WithName("PostgreSQL default user password updating event") secret, err := v1beta1.GetDefaultPgUserSecret(ctx, pg.Name, pg.Namespace, r.Client) if err != nil { - logger.Error(err, "Cannot get the default secret for the PostgreSQL cluster", + l.Error(err, "Cannot get the default secret for the PostgreSQL cluster", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, ) @@ -990,7 +1092,7 @@ func (r *PostgreSQLReconciler) handleUpdateDefaultUserPassword( password := string(secret.Data[models.Password]) isValid := pg.ValidateDefaultUserPassword(password) if !isValid { - logger.Error(err, "Default PostgreSQL user password is not valid. This field must be at least 8 characters long. Must contain characters from at least 3 of the following 4 categories: Uppercase, Lowercase, Numbers, Special Characters", + l.Error(err, "Default PostgreSQL user password is not valid. This field must be at least 8 characters long. Must contain characters from at least 3 of the following 4 categories: Uppercase, Lowercase, Numbers, Special Characters", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, ) @@ -1006,7 +1108,7 @@ func (r *PostgreSQLReconciler) handleUpdateDefaultUserPassword( err = r.API.UpdatePostgreSQLDefaultUserPassword(pg.Status.ID, password) if err != nil { - logger.Error(err, "Cannot update default PostgreSQL user password", + l.Error(err, "Cannot update default PostgreSQL user password", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, ) @@ -1021,9 +1123,9 @@ func (r *PostgreSQLReconciler) handleUpdateDefaultUserPassword( } pg.Annotations[models.ResourceStateAnnotation] = models.UpdatedEvent - err = r.patchClusterMetadata(ctx, pg, logger) + err = r.patchClusterMetadata(ctx, pg, l) if err != nil { - logger.Error(err, "Cannot patch PostgreSQL resource metadata", + l.Error(err, "Cannot patch PostgreSQL resource metadata", "cluster name", pg.Spec.Name, "cluster metadata", pg.ObjectMeta, ) @@ -1036,7 +1138,7 @@ func (r *PostgreSQLReconciler) handleUpdateDefaultUserPassword( return reconcile.Result{}, err } - logger.Info("PostgreSQL default user password was updated", + l.Info("PostgreSQL default user password was updated", "cluster name", pg.Spec.Name, "cluster ID", pg.Status.ID, ) @@ -1049,6 +1151,17 @@ func (r *PostgreSQLReconciler) handleUpdateDefaultUserPassword( return models.ExitReconcile, nil } +func (r *PostgreSQLReconciler) startClusterOnPremisesIPsJob(pg *v1beta1.PostgreSQL, b *onPremisesBootstrap) error { + job := newWatchOnPremisesIPsJob(pg.Kind, b) + + err := r.Scheduler.ScheduleJob(pg.GetJobID(scheduler.OnPremisesIPsChecker), scheduler.ClusterStatusInterval, job) + if err != nil { + return err + } + + return nil +} + func (r *PostgreSQLReconciler) startClusterStatusJob(pg *v1beta1.PostgreSQL) error { job := r.newWatchStatusJob(pg) @@ -1578,7 +1691,7 @@ func (r *PostgreSQLReconciler) reconcileClusterConfigurations( func (r *PostgreSQLReconciler) patchClusterMetadata( ctx context.Context, pgCluster *v1beta1.PostgreSQL, - logger logr.Logger, + l logr.Logger, ) error { patchRequest := []*v1beta1.PatchRequest{} @@ -1616,7 +1729,7 @@ func (r *PostgreSQLReconciler) patchClusterMetadata( return err } - logger.Info("PostgreSQL cluster patched", + l.Info("PostgreSQL cluster patched", "Cluster name", pgCluster.Spec.Name, "Finalizers", pgCluster.Finalizers, "Annotations", pgCluster.Annotations, diff --git a/controllers/clusters/redis_controller.go b/controllers/clusters/redis_controller.go index ab1f4d374..ef1296507 100644 --- a/controllers/clusters/redis_controller.go +++ b/controllers/clusters/redis_controller.go @@ -58,6 +58,14 @@ type RedisReconciler struct { //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=redis/status,verbs=get;update;patch //+kubebuilder:rbac:groups=clusters.instaclustr.com,resources=redis/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch +//+kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachines,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete;deletecollection // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -65,19 +73,19 @@ type RedisReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile func (r *RedisReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) + l := log.FromContext(ctx) redis := &v1beta1.Redis{} err := r.Client.Get(ctx, req.NamespacedName, redis) if err != nil { if k8serrors.IsNotFound(err) { - logger.Info("Redis cluster resource is not found", + l.Info("Redis cluster resource is not found", "resource name", req.NamespacedName, ) return models.ExitReconcile, nil } - logger.Error(err, "Unable to fetch Redis cluster", + l.Error(err, "Unable to fetch Redis cluster", "resource name", req.NamespacedName, ) return models.ExitReconcile, nil @@ -85,19 +93,19 @@ func (r *RedisReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl switch redis.Annotations[models.ResourceStateAnnotation] { case models.CreatingEvent: - return r.handleCreateCluster(ctx, redis, logger) + return r.handleCreateCluster(ctx, redis, l) case models.UpdatingEvent: - return r.handleUpdateCluster(ctx, redis, logger) + return r.handleUpdateCluster(ctx, redis, l) case models.DeletingEvent: - return r.handleDeleteCluster(ctx, redis, logger) + return r.handleDeleteCluster(ctx, redis, l) case models.GenericEvent: - logger.Info("Redis generic event isn't handled", + l.Info("Redis generic event isn't handled", "cluster name", redis.Spec.Name, "request", req, ) return models.ExitReconcile, nil default: - logger.Info("Unknown event isn't handled", + l.Info("Unknown event isn't handled", "cluster name", redis.Spec.Name, "request", req, ) @@ -108,20 +116,20 @@ func (r *RedisReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl func (r *RedisReconciler) handleCreateCluster( ctx context.Context, redis *v1beta1.Redis, - logger logr.Logger, + l logr.Logger, ) (reconcile.Result, error) { var err error if redis.Status.ID == "" { var id string if redis.Spec.HasRestore() { - logger.Info( + l.Info( "Creating Redis cluster from backup", "original cluster ID", redis.Spec.RestoreFrom.ClusterID, ) id, err = r.API.RestoreCluster(redis.RestoreInfoToInstAPI(redis.Spec.RestoreFrom), models.RedisAppKind) if err != nil { - logger.Error( + l.Error( err, "Cannot restore Redis cluster from backup", "original cluster ID", redis.Spec.RestoreFrom.ClusterID, ) @@ -133,7 +141,7 @@ func (r *RedisReconciler) handleCreateCluster( return reconcile.Result{}, err } - logger.Info( + l.Info( "Redis cluster was created from backup", "original cluster ID", redis.Spec.RestoreFrom.ClusterID, ) @@ -145,7 +153,7 @@ func (r *RedisReconciler) handleCreateCluster( id, ) } else { - logger.Info( + l.Info( "Creating Redis cluster", "cluster name", redis.Spec.Name, "data centres", redis.Spec.DataCentres, @@ -153,7 +161,7 @@ func (r *RedisReconciler) handleCreateCluster( id, err = r.API.CreateCluster(instaclustr.RedisEndpoint, redis.Spec.ToInstAPI()) if err != nil { - logger.Error( + l.Error( err, "Cannot create Redis cluster", "cluster manifest", redis.Spec, ) @@ -165,7 +173,7 @@ func (r *RedisReconciler) handleCreateCluster( return reconcile.Result{}, err } - logger.Info( + l.Info( "Redis cluster was created", "cluster ID", id, "cluster name", redis.Spec.Name, @@ -181,7 +189,7 @@ func (r *RedisReconciler) handleCreateCluster( redis.Status.ID = id err = r.Status().Patch(ctx, redis, patch) if err != nil { - logger.Error(err, "Cannot patch Redis cluster status", + l.Error(err, "Cannot patch Redis cluster status", "cluster name", redis.Spec.Name) r.EventRecorder.Eventf(redis, models.Warning, models.PatchFailed, "Cluster resource status patch is failed. Reason: %v", err) @@ -189,7 +197,7 @@ func (r *RedisReconciler) handleCreateCluster( return reconcile.Result{}, err } - logger.Info("Redis resource has been created", + l.Info("Redis resource has been created", "cluster name", redis.Name, "cluster ID", redis.Status.ID, "api version", redis.APIVersion) @@ -200,7 +208,7 @@ func (r *RedisReconciler) handleCreateCluster( redis.Annotations[models.ResourceStateAnnotation] = models.CreatedEvent err = r.Patch(ctx, redis, patch) if err != nil { - logger.Error(err, "Cannot patch Redis cluster", + l.Error(err, "Cannot patch Redis cluster", "cluster name", redis.Spec.Name, "cluster metadata", redis.ObjectMeta, ) @@ -215,7 +223,7 @@ func (r *RedisReconciler) handleCreateCluster( if redis.Status.State != models.DeletedStatus { err = r.startClusterStatusJob(redis) if err != nil { - logger.Error(err, "Cannot start cluster status job", + l.Error(err, "Cannot start cluster status job", "redis cluster ID", redis.Status.ID, ) @@ -232,9 +240,87 @@ func (r *RedisReconciler) handleCreateCluster( "Cluster status check job is started", ) + if redis.Spec.OnPremisesSpec != nil { + iData, err := r.API.GetRedis(redis.Status.ID) + if err != nil { + l.Error(err, "Cannot get cluster from the Instaclustr API", + "cluster name", redis.Spec.Name, + "data centres", redis.Spec.DataCentres, + "cluster ID", redis.Status.ID, + ) + r.EventRecorder.Eventf( + redis, models.Warning, models.FetchFailed, + "Cluster fetch from the Instaclustr API is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + iRedis, err := redis.FromInstAPI(iData) + if err != nil { + l.Error( + err, "Cannot convert cluster from the Instaclustr API", + "cluster name", redis.Spec.Name, + "cluster ID", redis.Status.ID, + ) + r.EventRecorder.Eventf( + redis, models.Warning, models.ConversionFailed, + "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + bootstrap := newOnPremisesBootstrap( + r.Client, + redis, + r.EventRecorder, + iRedis.Status.ClusterStatus, + redis.Spec.OnPremisesSpec, + newExposePorts(redis.GetExposePorts()), + redis.GetHeadlessPorts(), + redis.Spec.PrivateNetworkCluster, + ) + + err = handleCreateOnPremisesClusterResources(ctx, bootstrap) + if err != nil { + l.Error( + err, "Cannot create resources for on-premises cluster", + "cluster spec", redis.Spec.OnPremisesSpec, + ) + r.EventRecorder.Eventf( + redis, models.Warning, models.CreationFailed, + "Resources creation for on-premises cluster is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + err = r.startClusterOnPremisesIPsJob(redis, bootstrap) + if err != nil { + l.Error(err, "Cannot start on-premises cluster IPs check job", + "cluster ID", redis.Status.ID, + ) + + r.EventRecorder.Eventf( + redis, models.Warning, models.CreationFailed, + "On-premises cluster IPs check job is failed. Reason: %v", + err, + ) + return reconcile.Result{}, err + } + + l.Info( + "On-premises resources have been created", + "cluster name", redis.Spec.Name, + "on-premises Spec", redis.Spec.OnPremisesSpec, + "cluster ID", redis.Status.ID, + ) + return models.ExitReconcile, nil + } + err = r.startClusterBackupsJob(redis) if err != nil { - logger.Error(err, "Cannot start Redis cluster backups check job", + l.Error(err, "Cannot start Redis cluster backups check job", "cluster ID", redis.Status.ID, ) @@ -255,7 +341,7 @@ func (r *RedisReconciler) handleCreateCluster( err = r.startUsersCreationJob(redis) if err != nil { - logger.Error(err, "Failed to start user creation job") + l.Error(err, "Failed to start user creation job") r.EventRecorder.Eventf(redis, models.Warning, models.CreationFailed, "User creation job is failed. Reason: %v", err, ) @@ -267,7 +353,7 @@ func (r *RedisReconciler) handleCreateCluster( } } - logger.Info( + l.Info( "Redis resource has been created", "cluster name", redis.Name, "cluster ID", redis.Status.ID, @@ -293,11 +379,11 @@ func (r *RedisReconciler) startUsersCreationJob(cluster *v1beta1.Redis) error { func (r *RedisReconciler) handleUpdateCluster( ctx context.Context, redis *v1beta1.Redis, - logger logr.Logger, + l logr.Logger, ) (reconcile.Result, error) { iData, err := r.API.GetRedis(redis.Status.ID) if err != nil { - logger.Error( + l.Error( err, "Cannot get Redis cluster from the Instaclustr API", "cluster name", redis.Spec.Name, "cluster ID", redis.Status.ID, @@ -313,14 +399,14 @@ func (r *RedisReconciler) handleUpdateCluster( iRedis, err := redis.FromInstAPI(iData) if err != nil { - logger.Error( + l.Error( err, "Cannot convert Redis cluster from the Instaclustr API", "cluster name", redis.Spec.Name, "cluster ID", redis.Status.ID, ) r.EventRecorder.Eventf( - redis, models.Warning, models.ConvertionFailed, + redis, models.Warning, models.ConversionFailed, "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", err, ) @@ -328,17 +414,17 @@ func (r *RedisReconciler) handleUpdateCluster( } if redis.Annotations[models.ExternalChangesAnnotation] == models.True { - return r.handleExternalChanges(redis, iRedis, logger) + return r.handleExternalChanges(redis, iRedis, l) } if redis.Spec.ClusterSettingsNeedUpdate(iRedis.Spec.Cluster) { - logger.Info("Updating cluster settings", + l.Info("Updating cluster settings", "instaclustr description", iRedis.Spec.Description, "instaclustr two factor delete", iRedis.Spec.TwoFactorDelete) err = r.API.UpdateClusterSettings(redis.Status.ID, redis.Spec.ClusterSettingsUpdateToInstAPI()) if err != nil { - logger.Error(err, "Cannot update cluster settings", + l.Error(err, "Cannot update cluster settings", "cluster ID", redis.Status.ID, "cluster spec", redis.Spec) r.EventRecorder.Eventf(redis, models.Warning, models.UpdateFailed, "Cannot update cluster settings. Reason: %v", err) @@ -348,14 +434,14 @@ func (r *RedisReconciler) handleUpdateCluster( } if !redis.Spec.IsEqual(iRedis.Spec) { - logger.Info("Update request to Instaclustr API has been sent", + l.Info("Update request to Instaclustr API has been sent", "spec data centres", redis.Spec.DataCentres, "resize settings", redis.Spec.ResizeSettings, ) err = r.API.UpdateRedis(redis.Status.ID, redis.Spec.DCsUpdateToInstAPI()) if err != nil { - logger.Error(err, "Cannot update Redis cluster data centres", + l.Error(err, "Cannot update Redis cluster data centres", "cluster name", redis.Spec.Name, "cluster status", redis.Status, "data centres", redis.Spec.DataCentres, @@ -370,7 +456,7 @@ func (r *RedisReconciler) handleUpdateCluster( patch := redis.NewPatch() redis.Annotations[models.UpdateQueuedAnnotation] = models.True if err := r.Patch(ctx, redis, patch); err != nil { - logger.Error(err, "Cannot patch metadata", + l.Error(err, "Cannot patch metadata", "cluster name", redis.Spec.Name, "cluster metadata", redis.ObjectMeta, ) @@ -388,7 +474,7 @@ func (r *RedisReconciler) handleUpdateCluster( err = handleUsersChanges(ctx, r.Client, r, redis) if err != nil { - logger.Error(err, "Failed to handle users changes") + l.Error(err, "Failed to handle users changes") r.EventRecorder.Eventf(redis, models.Warning, models.PatchFailed, "Handling users changes is failed. Reason: %w", err, ) @@ -400,7 +486,7 @@ func (r *RedisReconciler) handleUpdateCluster( redis.Annotations[models.UpdateQueuedAnnotation] = "" err = r.Patch(ctx, redis, patch) if err != nil { - logger.Error(err, "Cannot patch Redis cluster after update", + l.Error(err, "Cannot patch Redis cluster after update", "cluster name", redis.Spec.Name, "cluster metadata", redis.ObjectMeta, ) @@ -413,7 +499,7 @@ func (r *RedisReconciler) handleUpdateCluster( return reconcile.Result{}, err } - logger.Info( + l.Info( "Cluster has been updated", "cluster name", redis.Spec.Name, "cluster ID", redis.Status.ID, @@ -463,12 +549,12 @@ func (r *RedisReconciler) handleExternalChanges(redis, iRedis *v1beta1.Redis, l func (r *RedisReconciler) handleDeleteCluster( ctx context.Context, redis *v1beta1.Redis, - logger logr.Logger, + l logr.Logger, ) (reconcile.Result, error) { _, err := r.API.GetRedis(redis.Status.ID) if err != nil && !errors.Is(err, instaclustr.NotFound) { - logger.Error(err, "Cannot get Redis cluster status from Instaclustr", + l.Error(err, "Cannot get Redis cluster status from Instaclustr", "cluster ID", redis.Status.ID, "cluster name", redis.Spec.Name, ) @@ -482,13 +568,13 @@ func (r *RedisReconciler) handleDeleteCluster( } if !errors.Is(err, instaclustr.NotFound) { - logger.Info("Sending cluster deletion to the Instaclustr API", + l.Info("Sending cluster deletion to the Instaclustr API", "cluster name", redis.Spec.Name, "cluster ID", redis.Status.ID) err = r.API.DeleteCluster(redis.Status.ID, instaclustr.RedisEndpoint) if err != nil { - logger.Error(err, "Cannot delete Redis cluster", + l.Error(err, "Cannot delete Redis cluster", "cluster name", redis.Spec.Name, "cluster status", redis.Status.State, ) @@ -511,7 +597,7 @@ func (r *RedisReconciler) handleDeleteCluster( redis.Annotations[models.ClusterDeletionAnnotation] = models.Triggered err = r.Patch(ctx, redis, patch) if err != nil { - logger.Error(err, "Cannot patch cluster resource", + l.Error(err, "Cannot patch cluster resource", "cluster name", redis.Spec.Name, "cluster state", redis.Status.State) r.EventRecorder.Eventf(redis, models.Warning, models.PatchFailed, @@ -521,25 +607,41 @@ func (r *RedisReconciler) handleDeleteCluster( return reconcile.Result{}, err } - logger.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", redis.Status.ID) + l.Info(msgDeleteClusterWithTwoFactorDelete, "cluster ID", redis.Status.ID) r.EventRecorder.Event(redis, models.Normal, models.DeletionStarted, "Two-Factor Delete is enabled, please confirm cluster deletion via email or phone.") return models.ExitReconcile, nil } + if redis.Spec.OnPremisesSpec != nil { + err = deleteOnPremResources(ctx, r.Client, redis.Status.ID, redis.Namespace) + if err != nil { + l.Error(err, "Cannot delete cluster on-premises resources", + "cluster ID", redis.Status.ID) + r.EventRecorder.Eventf(redis, models.Warning, models.DeletionFailed, + "Cluster on-premises resources deletion is failed. Reason: %v", err) + return reconcile.Result{}, err + } + + l.Info("Cluster on-premises resources are deleted", + "cluster ID", redis.Status.ID) + r.EventRecorder.Eventf(redis, models.Normal, models.Deleted, + "Cluster on-premises resources are deleted") + r.Scheduler.RemoveJob(redis.GetJobID(scheduler.OnPremisesIPsChecker)) + } } r.Scheduler.RemoveJob(redis.GetJobID(scheduler.StatusChecker)) r.Scheduler.RemoveJob(redis.GetJobID(scheduler.BackupsChecker)) - logger.Info("Deleting cluster backup resources", + l.Info("Deleting cluster backup resources", "cluster ID", redis.Status.ID, ) err = r.deleteBackups(ctx, redis.Status.ID, redis.Namespace) if err != nil { - logger.Error(err, "Cannot delete cluster backup resources", + l.Error(err, "Cannot delete cluster backup resources", "cluster ID", redis.Status.ID, ) r.EventRecorder.Eventf( @@ -555,13 +657,13 @@ func (r *RedisReconciler) handleDeleteCluster( "Cluster backup resources are deleted", ) - logger.Info("Cluster backup resources are deleted", + l.Info("Cluster backup resources are deleted", "cluster ID", redis.Status.ID, ) err = detachUsers(ctx, r.Client, r, redis) if err != nil { - logger.Error(err, "Failed to detach users from the cluster") + l.Error(err, "Failed to detach users from the cluster") r.EventRecorder.Eventf(redis, models.Warning, models.DeletionFailed, "Detaching users from the cluster is failed. Reason: %w", err, ) @@ -573,7 +675,7 @@ func (r *RedisReconciler) handleDeleteCluster( redis.Annotations[models.ResourceStateAnnotation] = models.DeletedEvent err = r.Patch(ctx, redis, patch) if err != nil { - logger.Error(err, "Cannot patch Redis cluster metadata after finalizer removal", + l.Error(err, "Cannot patch Redis cluster metadata after finalizer removal", "cluster name", redis.Spec.Name, "cluster ID", redis.Status.ID, ) @@ -588,7 +690,7 @@ func (r *RedisReconciler) handleDeleteCluster( err = exposeservice.Delete(r.Client, redis.Name, redis.Namespace) if err != nil { - logger.Error(err, "Cannot delete Redis cluster expose service", + l.Error(err, "Cannot delete Redis cluster expose service", "cluster ID", redis.Status.ID, "cluster name", redis.Spec.Name, ) @@ -596,7 +698,7 @@ func (r *RedisReconciler) handleDeleteCluster( return reconcile.Result{}, err } - logger.Info("Redis cluster was deleted", + l.Info("Redis cluster was deleted", "cluster name", redis.Spec.Name, "cluster ID", redis.Status.ID, ) @@ -609,6 +711,17 @@ func (r *RedisReconciler) handleDeleteCluster( return models.ExitReconcile, nil } +func (r *RedisReconciler) startClusterOnPremisesIPsJob(redis *v1beta1.Redis, b *onPremisesBootstrap) error { + job := newWatchOnPremisesIPsJob(redis.Kind, b) + + err := r.Scheduler.ScheduleJob(redis.GetJobID(scheduler.OnPremisesIPsChecker), scheduler.ClusterStatusInterval, job) + if err != nil { + return err + } + + return nil +} + func (r *RedisReconciler) startClusterStatusJob(cluster *v1beta1.Redis) error { job := r.newWatchStatusJob(cluster) @@ -632,7 +745,7 @@ func (r *RedisReconciler) startClusterBackupsJob(cluster *v1beta1.Redis) error { } func (r *RedisReconciler) newUsersCreationJob(redis *v1beta1.Redis) scheduler.Job { - logger := log.Log.WithValues("component", "redisUsersCreationJob") + l := log.Log.WithValues("component", "redisUsersCreationJob") return func() error { ctx := context.Background() @@ -649,7 +762,7 @@ func (r *RedisReconciler) newUsersCreationJob(redis *v1beta1.Redis) scheduler.Jo } if redis.Status.State != models.RunningStatus { - logger.Info("User creation job is scheduled") + l.Info("User creation job is scheduled") r.EventRecorder.Eventf(redis, models.Normal, models.CreationFailed, "User creation job is scheduled, cluster is not in the running state", ) @@ -658,14 +771,14 @@ func (r *RedisReconciler) newUsersCreationJob(redis *v1beta1.Redis) scheduler.Jo err = handleUsersChanges(ctx, r.Client, r, redis) if err != nil { - logger.Error(err, "Failed to create users") + l.Error(err, "Failed to create users") r.EventRecorder.Eventf(redis, models.Warning, models.PatchFailed, "Creating users is failed. Reason: %w", err, ) return err } - logger.Info("User creation job successfully finished") + l.Info("User creation job successfully finished") r.EventRecorder.Eventf(redis, models.Normal, models.Created, "User creation job successfully finished", ) diff --git a/doc/clusters/cadence.md b/doc/clusters/cadence.md index e4b35920e..312982d5a 100644 --- a/doc/clusters/cadence.md +++ b/doc/clusters/cadence.md @@ -2,24 +2,24 @@ ## Available spec fields -| Field | Type | Description | -|-----------------------|-----------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | -| version | string
**required** | Cadence instance version.
**Available versions**: `0.22.4`, `0.24.0`. | -| pciCompliance | bool
**required** | Creates a PCI compliant cluster, see [PCI Compliance](https://www.instaclustr.com/support/documentation/useful-information/pci-compliance/) | -| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | -| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | -| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | -| dataCentres | Array of objects ([CadenceDataCentre](#CadenceDataCentreObject))
**required** | Object fields are described below as a bulleted list. | -| description | string | Description of the Cadence cluster. | -| useCadenceWebAuth | bool
**required** | Enable Authentication for Cadence Web. | -| awsArchival | Array of objects ([AWSArchival](#AWSArchivalObject)) | Cadence AWS Archival settings. | -| standardProvisioning | Array of objects ([StandardProvisioning](#StandardProvisioningObject)) | Settings for STANDARD provisioning. Must not be defined with SHARED and PACKAGED provisioning options. | -| sharedProvisioning | Array of objects ([SharedProvisioning](#SharedProvisioningObject)) | Settings for SHARED provisioning. Must not be defined with STANDARD and PACKAGED provisioning options. | -| packagedProvisioning | Array of objects ([PackagedProvisioning](#PackagedProvisioningObject)) | Settings for PACKAGED provisioning. Must not be defined with STANDARD and SHARED provisioning options. | -| targetPrimaryCadence | Array of objects ([TargetPrimaryCadence](#TargetPrimaryCadenceObject))
_mutable_ | Supporting Primary Cadence info for Multi region Cadence. | -| resizeSettings | Array of objects ([ResizeSettings](#ResizeSettingsObject))
_mutable_ | Settings to determine how resize requests will be performed for the cluster. | - +| Field | Type | Description | +|-----------------------|-----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | +| version | string
**required** | Cadence instance version.
**Available versions**: `0.22.4`, `0.24.0`. | +| pciCompliance | bool
**required** | Creates a PCI compliant cluster, see [PCI Compliance](https://www.instaclustr.com/support/documentation/useful-information/pci-compliance/) | +| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | +| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | +| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | +| dataCentres | Array of objects ([CadenceDataCentre](#CadenceDataCentreObject))
**required** | Object fields are described below as a bulleted list. | +| description | string | Description of the Cadence cluster. | +| useCadenceWebAuth | bool
**required** | Enable Authentication for Cadence Web. | +| awsArchival | Array of objects ([AWSArchival](#AWSArchivalObject)) | Cadence AWS Archival settings. | +| standardProvisioning | Array of objects ([StandardProvisioning](#StandardProvisioningObject)) | Settings for STANDARD provisioning. Must not be defined with SHARED and PACKAGED provisioning options. | +| sharedProvisioning | Array of objects ([SharedProvisioning](#SharedProvisioningObject)) | Settings for SHARED provisioning. Must not be defined with STANDARD and PACKAGED provisioning options. | +| packagedProvisioning | Array of objects ([PackagedProvisioning](#PackagedProvisioningObject)) | Settings for PACKAGED provisioning. Must not be defined with STANDARD and SHARED provisioning options. | +| targetPrimaryCadence | Array of objects ([TargetPrimaryCadence](#TargetPrimaryCadenceObject))
_mutable_ | Supporting Primary Cadence info for Multi region Cadence. | +| resizeSettings | Array of objects ([ResizeSettings](#ResizeSettingsObject))
_mutable_ | Settings to determine how resize requests will be performed for the cluster. | +| onPremisesSpec | Object ([OnPremisesSpec](#OnPremisesSpecObject)) | Specifies settings to provision on-premises cluster inside K8s cluster. | ### TargetPrimaryCadenceObject | Field | Type | Description | @@ -145,6 +145,27 @@ | privateIPBroadcastForDiscovery | bool
**required** | Enables broadcast of private IPs for auto-discovery. | | passwordAndUserAuth | bool
**required** | Enables Password Authentication and User Authorization. | +### OnPremisesSpecObject + +| Field | Type | Description | +|--------------------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| storageClassName | string
**required** | Name of the storage class that will be used to provision disks for on-premises nodes. | +| osDiskSize | string
**required** | Disk size on which OS will be installed. | +| dataDiskSize | string
**required** | Disk size on which on-premises cluster data will be stored. | +| sshGatewayCPU | int64 | Amount of CPU that will be dedicated to provision SSH Gateway node (only for private clusters). | +| sshGatewayMemory | string | Amount of RAM that will be dedicated to provision SSH Gateway node (only for private clusters). | +| nodeCPU | int64
**required** | Amount of CPU that will be dedicated to provision on-premises worker node. | +| nodeMemory | string
**required** | Amount of RAM that will be dedicated to provision on-premises worker node | +| osImageURL | string
**required** | OS image URL that will be use to dynamically provision disks with preinstalled OS (more info can be found [here](https://kubevirt.io/2020/KubeVirt-VM-Image-Usage-Patterns.html)). | +| cloudInitScriptRef | Object ([Reference](#ReferenceObject))
**required** | Reference to the secret with cloud-init script (must be located in the same namespace with the cluster). Example can be found [here](#CloudInitScript). | + +### ReferenceObject + +| Field | Type | Description | +|-----------|--------|---------------------------------------------------| +| name | string | Name of the cloud-init secret. | +| namespace | string | Namespace in which the cloud-init secret located. | + ## Cluster create flow To create a Cadence cluster instance you need to prepare the yaml manifest. Here is an example: @@ -164,25 +185,6 @@ spec: - targetCassandra: dependencyCdcId: "9d43ac54-7317-4ce5-859a-e9d0443508a4" dependencyVpcType: "VPC_PEERED" - packagedProvisioning: - - bundledCassandraSpec: - nodeSize: "CAS-DEV-t4g.small-5" - network: "10.2.0.0/16" - replicationFactor: 3 - nodesNumber: 3 - privateIPBroadcastForDiscovery: false - passwordAndUserAuth: true - useAdvancedVisibility: true - bundledKafkaSpec: - nodeSize: "KFK-DEV-t4g.small-5" - nodesNumber: 3 - network: "10.3.0.0/16" - replicationFactor: 3 - partitionsNumber: 3 - bundledOpenSearchSpec: - nodeSize: "SRH-DEV-t4g.small-5" - replicationFactor: 3 - network: "10.4.0.0/16" twoFactorDelete: - email: "example@netapp.com" privateNetworkCluster: false @@ -208,6 +210,83 @@ spec: concurrency: 1 ``` + +Or if you want to create an on-premises cluster: +```yaml +# cadence.yaml +apiVersion: clusters.instaclustr.com/v1beta1 +kind: Cadence +metadata: + name: cadence-sample +spec: + name: "cadence-sample" + version: "1.0.0" + standardProvisioning: + - targetCassandra: + dependencyCdcId: "9d43ac54-7317-4ce5-859a-e9d0443508a4" + dependencyVpcType: "VPC_PEERED" + privateNetworkCluster: false + dataCentres: + - region: "US_EAST_2" + network: "10.12.0.0/16" + cloudProvider: "AWS_VPC" + name: "testdc" + nodeSize: "CAD-DEV-t3.small-5" + nodesNumber: 2 + clientEncryption: false + slaTier: "NON_PRODUCTION" + useCadenceWebAuth: false + targetPrimaryCadence: + - dependencyCdcId: "cce79be3-7f41-4cad-837c-86d3d8b4be77" + dependencyVpcType: "SEPARATE_VPC" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: cloud-init-secret +``` + +Also, don't forget to create cloud-init script firstly. + +### CloudInitScript + +cloud-init.sh: +```shell +#!/bin/bash + +export NEW_PASS="qwerty12345" +export SSH_PUB_KEY="" +export BOOTSTRAP_SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDgeaO3zkY5v1dww3fFONPzUnEgIqJ4kUK0Usu8iFdp+TWIulw9dDeQHa+PdWXP97l5Vv1mG9ipqShEIu7/2bp13KxSblWX4iV1MYZbtimhY3UDOsPn1G3E1Ipis6y+/tosDAm8LoWaGEMcLuE5UjP6gs6K57rCEjkVGjg7vjhIypAMC0J2N2+CxK9o/Y1+LZAec+FL5cmSJoajoo9y/VYJjz/a52jJ93wRafD2uu6ObAl5gkN/+gqY4IJFSMx20sPsIRXdbiBNDqiap56ibHHPKTeZhRbdXvZfjYkHtutXnSM2xn7BjnV8CguISxS3rXlzlzRVYwSUjnKUf5SKBbeyZbCokx271vCeUd5EXfHphvW6FIOna2AI5lpCSYkw5Kho3HaPi2NjXJ9i2dCr1zpcZpCiettDuaEjxR0Cl4Jd6PrAiAEZ0Ns0u2ysVhudshVzQrq6qdd7W9/MLjbDIHdTToNjFLZA6cbE0MQf18LXwJAl+G/QrXgcVaiopUmld+68JL89Xym55LzkMhI0NiDtWufawd/NiZ6jm13Z3+atvnOimdyuqBYeFWgbtcxjs0yN9v7h7PfPh6TbiQYFx37DCfQiIucqx1GWmMijmd7HMY6Nv3UvnoTUTSn4yz1NxhEpC61N+iAInZDpeJEtULSzlEMWlbzL4t5cF+Rm1dFdq3WpZt1mi8F4DgrsgZEuLGAw22RNW3++EWYFUNnJXaYyctPrMpWQktr4DB5nQGIHF92WR8uncxTNUXfWuT29O9e+bFYh1etmq8rsCoLcxN0zFHWvcECK55aE+47lfNAR+HEjuwYW10mGU/pFmO0F9FFmcQRSw4D4tnVUgl3XjKe3bBiTa4lUrzrKkLZ6n9/buW2e7c3jbjmXdPh2R+2Msr/vfuWs9glxQf+CYEbBW6Ye4pekIyI77SaB/bVhaHtXutKxm+QWdNle8aeqiA8Ji1Ml+s75vIg+n5v6viCnl5aV33xHRFpGQJzj2ktsXl9P9d5kgal9eXJYTywC2SnVbZVLb6FGN4kPZTVwX1f+u7v7JCm4YWlbQZtwwiXKjs99AVtQnBWqQvUH5sFUkVXlHA1Y9W6wlup0r+F6URL+7Yw+d0dHByfevrJg3pvmpLb3sEpjIAZodW3dIUReE7Ku3s/q/O9foFnfRBnCcZ2QsnxI5pqNrbrundD1ApOnNXEvICvPXHBBQ44cW0hzAO+WxY5VxyG8y/kXnb48G9efkIQFkNaITJrU9SiOk6bFP4QANdS/pmaSLjJIsHixa+7vmYjRy1SVoQ/39vDUnyCbqKtO56QMH32hQLRO3Vk7NVG6o4dYjFkiaMSaqVlHKMkJQHVzlK2PW9/fjVXfkAHmmhoD debian" +echo "debian:$NEW_PASS" | chpasswd +echo "root:$NEW_PASS" | sudo chpasswd root +sudo echo "$SSH_PUB_KEY" > /home/debian/.ssh/authorized_keys +sudo echo "$BOOTSTRAP_SSH_KEY" >> /home/debian/.ssh/authorized_keys +sudo chown -R debian: /home/debian/.ssh +sudo cp /usr/share/doc/apt/examples/sources.list /etc/apt/sources.list +data_device=$(lsblk -dfn -o NAME,SERIAL | awk '$2 == "DATADISK" {print $1}') +sudo mkfs -t ext4 /dev/"${data_device}" +``` + +Create the base64 encoded string with the script using `cat cloud-init.sh | base64 -w0` command and create a secret using this yaml manifest: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cloud-init-secret +data: + userdata: +``` + +Use `kubectl apply -f cloud-init-secret.yaml` command to create the secret. + If you want to enable AWSArchival option, please create a secret **before** cluster creation: ```yaml # cadence-aws-secret.yaml diff --git a/doc/clusters/cassandra.md b/doc/clusters/cassandra.md index 83a269db5..7091d8abe 100644 --- a/doc/clusters/cassandra.md +++ b/doc/clusters/cassandra.md @@ -2,22 +2,23 @@ ## Available spec fields -| Field | Type | Description | -|---------------------------|------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | -| version | string
**required** | Cassandra instance version.
**Available versions**: `3.11.15`, `3.11.16`, `4.0.10`, `4.0.11`, `4.1.3`. | -| pciCompliance | bool
**required** | Creates a PCI compliant cluster, see [PCI Compliance](https://www.instaclustr.com/support/documentation/useful-information/pci-compliance/) | -| description | string
| A description of the cluster | -| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | -| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | -| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | -| schemaRegistry | Array of objects ([KafkaSchemaRegistryDetails](#KafkaSchemaRegistryDetailsObject))
_mutable_ | Adds the specified version of Kafka Schema Registry to this Kafka cluster. | -| luceneEnabled | bool
**required** | Adds Apache Lucene to the Cassandra cluster. | -| passwordAndUserAuth | bool
**required** | Enables Password Authentication and User Authorization. | -| bundledUseOnly | bool
**required** | Provision this cluster for Bundled Use only. | -| restoreFrom | Object ([CassandraRestoreFrom](#CassandraRestoreFromObject)) | Triggers a restore cluster operation. | -| dataCentres | Array of objects ([CassandraDataCentre](#CassandraDataCentreObject))
**required** | Object fields are described below as a bulleted list. | -| resizeSettings | Array of objects ([ResizeSettings](#ResizeSettingsObject))
_mutable_ | Settings to determine how resize requests will be performed for the cluster. | +| Field | Type | Description | +|-----------------------|-----------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | +| version | string
**required** | Cassandra instance version.
**Available versions**: `3.11.15`, `3.11.16`, `4.0.10`, `4.0.11`, `4.1.3`. | +| pciCompliance | bool
**required** | Creates a PCI compliant cluster, see [PCI Compliance](https://www.instaclustr.com/support/documentation/useful-information/pci-compliance/) | +| description | string
| A description of the cluster | +| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | +| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | +| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | +| schemaRegistry | Array of objects ([KafkaSchemaRegistryDetails](#KafkaSchemaRegistryDetailsObject))
_mutable_ | Adds the specified version of Kafka Schema Registry to this Kafka cluster. | +| luceneEnabled | bool
**required** | Adds Apache Lucene to the Cassandra cluster. | +| passwordAndUserAuth | bool
**required** | Enables Password Authentication and User Authorization. | +| bundledUseOnly | bool
**required** | Provision this cluster for Bundled Use only. | +| restoreFrom | Object ([CassandraRestoreFrom](#CassandraRestoreFromObject)) | Triggers a restore cluster operation. | +| dataCentres | Array of objects ([CassandraDataCentre](#CassandraDataCentreObject))
**required** | Object fields are described below as a bulleted list. | +| resizeSettings | Array of objects ([ResizeSettings](#ResizeSettingsObject))
_mutable_ | Settings to determine how resize requests will be performed for the cluster. | +| onPremisesSpec | Object ([OnPremisesSpec](#OnPremisesSpecObject)) | Specifies settings to provision on-premises cluster inside K8s cluster. | ### TwoFactorDeleteObject | Field | Type | Description | @@ -91,6 +92,27 @@ | version | string | Adds the specified version of Debezium Connector Cassandra to the Cassandra cluster. | +### OnPremisesSpecObject + +| Field | Type | Description | +|--------------------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| storageClassName | string
**required** | Name of the storage class that will be used to provision disks for on-premises nodes. | +| osDiskSize | string
**required** | Disk size on which OS will be installed. | +| dataDiskSize | string
**required** | Disk size on which on-premises cluster data will be stored. | +| sshGatewayCPU | int64 | Amount of CPU that will be dedicated to provision SSH Gateway node (only for private clusters). | +| sshGatewayMemory | string | Amount of RAM that will be dedicated to provision SSH Gateway node (only for private clusters). | +| nodeCPU | int64
**required** | Amount of CPU that will be dedicated to provision on-premises worker node. | +| nodeMemory | string
**required** | Amount of RAM that will be dedicated to provision on-premises worker node | +| osImageURL | string
**required** | OS image URL that will be use to dynamically provision disks with preinstalled OS (more info can be found [here](https://kubevirt.io/2020/KubeVirt-VM-Image-Usage-Patterns.html)). | +| cloudInitScriptRef | Object ([Reference](#ReferenceObject))
**required** | Reference to the secret with cloud-init script (must be located in the same namespace with the cluster). Example can be found [here](#CloudInitScript). | + +### ReferenceObject + +| Field | Type | Description | +|-----------|--------|---------------------------------------------------| +| name | string | Name of the cloud-init secret. | +| namespace | string | Namespace in which the cloud-init secret located. | + ## Cluster create flow To create a Cassandra cluster instance you need to prepare the yaml manifest. Here is an example: @@ -125,6 +147,82 @@ spec: slaTier: "NON_PRODUCTION" ``` +Or if you want to create an on-premises cluster: +```yaml +# cassandra.yaml +apiVersion: clusters.instaclustr.com/v1beta1 +kind: Cassandra +metadata: + name: cassandra-on-prem-cluster +spec: + name: "cassandra-on-prem-cluster" + version: "4.0.10" + privateNetworkCluster: true + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: cloud-init-secret + dataCentres: + - name: "onPremCassandra" + region: "CLIENT_DC" # Don't change if you want to run on-premises + cloudProvider: "ONPREMISES" # Don't change if you want to run on-premises + continuousBackup: false + nodesNumber: 2 + replicationFactor: 2 + privateIpBroadcastForDiscovery: false + network: "192.168.0.0/16" + tags: + "onprem": "test" + clientToClusterEncryption: false + nodeSize: "CAS-PRD-OP.4.8-200" + pciCompliance: false + luceneEnabled: false + passwordAndUserAuth: false + slaTier: "NON_PRODUCTION" +``` + +Also, don't forget to create cloud-init script firstly. + +### CloudInitScript + +cloud-init.sh: +```shell +#!/bin/bash + +export NEW_PASS="qwerty12345" +export SSH_PUB_KEY="" +export BOOTSTRAP_SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDgeaO3zkY5v1dww3fFONPzUnEgIqJ4kUK0Usu8iFdp+TWIulw9dDeQHa+PdWXP97l5Vv1mG9ipqShEIu7/2bp13KxSblWX4iV1MYZbtimhY3UDOsPn1G3E1Ipis6y+/tosDAm8LoWaGEMcLuE5UjP6gs6K57rCEjkVGjg7vjhIypAMC0J2N2+CxK9o/Y1+LZAec+FL5cmSJoajoo9y/VYJjz/a52jJ93wRafD2uu6ObAl5gkN/+gqY4IJFSMx20sPsIRXdbiBNDqiap56ibHHPKTeZhRbdXvZfjYkHtutXnSM2xn7BjnV8CguISxS3rXlzlzRVYwSUjnKUf5SKBbeyZbCokx271vCeUd5EXfHphvW6FIOna2AI5lpCSYkw5Kho3HaPi2NjXJ9i2dCr1zpcZpCiettDuaEjxR0Cl4Jd6PrAiAEZ0Ns0u2ysVhudshVzQrq6qdd7W9/MLjbDIHdTToNjFLZA6cbE0MQf18LXwJAl+G/QrXgcVaiopUmld+68JL89Xym55LzkMhI0NiDtWufawd/NiZ6jm13Z3+atvnOimdyuqBYeFWgbtcxjs0yN9v7h7PfPh6TbiQYFx37DCfQiIucqx1GWmMijmd7HMY6Nv3UvnoTUTSn4yz1NxhEpC61N+iAInZDpeJEtULSzlEMWlbzL4t5cF+Rm1dFdq3WpZt1mi8F4DgrsgZEuLGAw22RNW3++EWYFUNnJXaYyctPrMpWQktr4DB5nQGIHF92WR8uncxTNUXfWuT29O9e+bFYh1etmq8rsCoLcxN0zFHWvcECK55aE+47lfNAR+HEjuwYW10mGU/pFmO0F9FFmcQRSw4D4tnVUgl3XjKe3bBiTa4lUrzrKkLZ6n9/buW2e7c3jbjmXdPh2R+2Msr/vfuWs9glxQf+CYEbBW6Ye4pekIyI77SaB/bVhaHtXutKxm+QWdNle8aeqiA8Ji1Ml+s75vIg+n5v6viCnl5aV33xHRFpGQJzj2ktsXl9P9d5kgal9eXJYTywC2SnVbZVLb6FGN4kPZTVwX1f+u7v7JCm4YWlbQZtwwiXKjs99AVtQnBWqQvUH5sFUkVXlHA1Y9W6wlup0r+F6URL+7Yw+d0dHByfevrJg3pvmpLb3sEpjIAZodW3dIUReE7Ku3s/q/O9foFnfRBnCcZ2QsnxI5pqNrbrundD1ApOnNXEvICvPXHBBQ44cW0hzAO+WxY5VxyG8y/kXnb48G9efkIQFkNaITJrU9SiOk6bFP4QANdS/pmaSLjJIsHixa+7vmYjRy1SVoQ/39vDUnyCbqKtO56QMH32hQLRO3Vk7NVG6o4dYjFkiaMSaqVlHKMkJQHVzlK2PW9/fjVXfkAHmmhoD debian" +echo "debian:$NEW_PASS" | chpasswd +echo "root:$NEW_PASS" | sudo chpasswd root +sudo echo "$SSH_PUB_KEY" > /home/debian/.ssh/authorized_keys +sudo echo "$BOOTSTRAP_SSH_KEY" >> /home/debian/.ssh/authorized_keys +sudo chown -R debian: /home/debian/.ssh +sudo cp /usr/share/doc/apt/examples/sources.list /etc/apt/sources.list +data_device=$(lsblk -dfn -o NAME,SERIAL | awk '$2 == "DATADISK" {print $1}') +sudo mkfs -t ext4 /dev/"${data_device}" +``` + +Create the base64 encoded string with the script using `cat cloud-init.sh | base64 -w0` command and create a secret using this yaml manifest: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cloud-init-secret +data: + userdata: +``` + +Use `kubectl apply -f cloud-init-secret.yaml` command to create the secret. + Next, you need to apply this manifest in your K8s cluster. This will create a custom resource instance inside (more info about an apply command you can find [here](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#apply)): ```console kubectl apply -f cassandra.yaml diff --git a/doc/clusters/kafka-connect.md b/doc/clusters/kafka-connect.md index b06b58459..f9236247f 100644 --- a/doc/clusters/kafka-connect.md +++ b/doc/clusters/kafka-connect.md @@ -2,17 +2,18 @@ ## Available spec fields -| Field | Type | Description | -|------------------------|--------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | -| version | string
**required** | Kafka Connect instance version.
**Available versions**: `3.1.2`, `3.3.1`, `3.4.1`, `3.5.1`. | -| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | -| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | -| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | -| targetCluster | Array of objects ([TargetCluster](#TargetClusterObject))
**required** | Details to connect to a target Kafka Cluster cluster. | -| customConnectors | Array of objects ([CustomConnectors](#CustomConnectorsObject)) | Defines the location for custom connector storage and access info. | -| dataCentres | Array of objects ([KafkaConnectDataCentre](#KafkaConnectDataCentreObject))
**required** | Object fields are described below as a bulleted list. | -| description | string
| A description of the cluster | +| Field | Type | Description | +|-----------------------|------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | +| version | string
**required** | Kafka Connect instance version.
**Available versions**: `3.1.2`, `3.3.1`, `3.4.1`, `3.5.1`. | +| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | +| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | +| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | +| targetCluster | Array of objects ([TargetCluster](#TargetClusterObject))
**required** | Details to connect to a target Kafka Cluster cluster. | +| customConnectors | Array of objects ([CustomConnectors](#CustomConnectorsObject)) | Defines the location for custom connector storage and access info. | +| dataCentres | Array of objects ([KafkaConnectDataCentre](#KafkaConnectDataCentreObject))
**required** | Object fields are described below as a bulleted list. | +| description | string
| A description of the cluster | +| onPremisesSpec | Object ([OnPremisesSpec](#OnPremisesSpecObject)) | Specifies settings to provision on-premises cluster inside K8s cluster. | ### TwoFactorDeleteObject | Field | Type | Description | @@ -97,6 +98,27 @@ | resourceGroup | string | The name of the Azure Resource Group into which the Data Centre will be provisioned.
Cannot be provided with `customVirtualNetworkId` and `diskEncryptionKey` | | diskEncryptionKey | string | ID of a KMS encryption key to encrypt data on nodes. KMS encryption key must be set in Cluster Resources through the Instaclustr Console before provisioning an encrypted Data Centre.
Cannot be provided with `customVirtualNetworkId` | +### OnPremisesSpecObject + +| Field | Type | Description | +|--------------------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| storageClassName | string
**required** | Name of the storage class that will be used to provision disks for on-premises nodes. | +| osDiskSize | string
**required** | Disk size on which OS will be installed. | +| dataDiskSize | string
**required** | Disk size on which on-premises cluster data will be stored. | +| sshGatewayCPU | int64 | Amount of CPU that will be dedicated to provision SSH Gateway node (only for private clusters). | +| sshGatewayMemory | string | Amount of RAM that will be dedicated to provision SSH Gateway node (only for private clusters). | +| nodeCPU | int64
**required** | Amount of CPU that will be dedicated to provision on-premises worker node. | +| nodeMemory | string
**required** | Amount of RAM that will be dedicated to provision on-premises worker node | +| osImageURL | string
**required** | OS image URL that will be use to dynamically provision disks with preinstalled OS (more info can be found [here](https://kubevirt.io/2020/KubeVirt-VM-Image-Usage-Patterns.html)). | +| cloudInitScriptRef | Object ([Reference](#ReferenceObject))
**required** | Reference to the secret with cloud-init script (must be located in the same namespace with the cluster). Example can be found [here](#CloudInitScript). | + +### ReferenceObject + +| Field | Type | Description | +|-----------|--------|---------------------------------------------------| +| name | string | Name of the cloud-init secret. | +| namespace | string | Namespace in which the cloud-init secret located. | + ## Cluster create flow To create a Kafka Connect cluster instance you need to prepare the yaml manifest. Here is an example: @@ -105,7 +127,6 @@ Notice: ```yaml # kafkaconnect.yaml - apiVersion: clusters.instaclustr.com/v1beta1 kind: KafkaConnect metadata: @@ -132,6 +153,81 @@ spec: kafkaConnectVpcType: "KAFKA_VPC" ``` +Or if you want to create an on-premises cluster: +```yaml +# kafkaconnect.yaml +apiVersion: clusters.instaclustr.com/v1beta1 +kind: KafkaConnect +metadata: + name: kafkaconnect-sample +spec: + name: "kafkaconnect-sample" + version: "3.1.2" + privateNetworkCluster: false + dataCentres: + - name: "OnPremKafkaConnect" + nodesNumber: 3 + cloudProvider: "ONPREMISES" # Don't change if you want to run on-premises + replicationFactor: 3 + tags: + tag: "oneTag" + tag2: "twoTags" + nodeSize: "KCN-DEV-OP.4.8-200" + network: "10.1.0.0/16" + region: "CLIENT_DC" # Don't change if you want to run on-premises + slaTier: "NON_PRODUCTION" + targetCluster: + - managedCluster: + - targetKafkaClusterId: "34dfc53c-c8c1-4be8-bd2f-cfdb77ec7349" + kafkaConnectVpcType: "KAFKA_VPC" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: cloud-init-secret +``` + +Also, don't forget to create cloud-init script firstly. + +### CloudInitScript + +cloud-init.sh: +```shell +#!/bin/bash + +export NEW_PASS="qwerty12345" +export SSH_PUB_KEY="" +export BOOTSTRAP_SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDgeaO3zkY5v1dww3fFONPzUnEgIqJ4kUK0Usu8iFdp+TWIulw9dDeQHa+PdWXP97l5Vv1mG9ipqShEIu7/2bp13KxSblWX4iV1MYZbtimhY3UDOsPn1G3E1Ipis6y+/tosDAm8LoWaGEMcLuE5UjP6gs6K57rCEjkVGjg7vjhIypAMC0J2N2+CxK9o/Y1+LZAec+FL5cmSJoajoo9y/VYJjz/a52jJ93wRafD2uu6ObAl5gkN/+gqY4IJFSMx20sPsIRXdbiBNDqiap56ibHHPKTeZhRbdXvZfjYkHtutXnSM2xn7BjnV8CguISxS3rXlzlzRVYwSUjnKUf5SKBbeyZbCokx271vCeUd5EXfHphvW6FIOna2AI5lpCSYkw5Kho3HaPi2NjXJ9i2dCr1zpcZpCiettDuaEjxR0Cl4Jd6PrAiAEZ0Ns0u2ysVhudshVzQrq6qdd7W9/MLjbDIHdTToNjFLZA6cbE0MQf18LXwJAl+G/QrXgcVaiopUmld+68JL89Xym55LzkMhI0NiDtWufawd/NiZ6jm13Z3+atvnOimdyuqBYeFWgbtcxjs0yN9v7h7PfPh6TbiQYFx37DCfQiIucqx1GWmMijmd7HMY6Nv3UvnoTUTSn4yz1NxhEpC61N+iAInZDpeJEtULSzlEMWlbzL4t5cF+Rm1dFdq3WpZt1mi8F4DgrsgZEuLGAw22RNW3++EWYFUNnJXaYyctPrMpWQktr4DB5nQGIHF92WR8uncxTNUXfWuT29O9e+bFYh1etmq8rsCoLcxN0zFHWvcECK55aE+47lfNAR+HEjuwYW10mGU/pFmO0F9FFmcQRSw4D4tnVUgl3XjKe3bBiTa4lUrzrKkLZ6n9/buW2e7c3jbjmXdPh2R+2Msr/vfuWs9glxQf+CYEbBW6Ye4pekIyI77SaB/bVhaHtXutKxm+QWdNle8aeqiA8Ji1Ml+s75vIg+n5v6viCnl5aV33xHRFpGQJzj2ktsXl9P9d5kgal9eXJYTywC2SnVbZVLb6FGN4kPZTVwX1f+u7v7JCm4YWlbQZtwwiXKjs99AVtQnBWqQvUH5sFUkVXlHA1Y9W6wlup0r+F6URL+7Yw+d0dHByfevrJg3pvmpLb3sEpjIAZodW3dIUReE7Ku3s/q/O9foFnfRBnCcZ2QsnxI5pqNrbrundD1ApOnNXEvICvPXHBBQ44cW0hzAO+WxY5VxyG8y/kXnb48G9efkIQFkNaITJrU9SiOk6bFP4QANdS/pmaSLjJIsHixa+7vmYjRy1SVoQ/39vDUnyCbqKtO56QMH32hQLRO3Vk7NVG6o4dYjFkiaMSaqVlHKMkJQHVzlK2PW9/fjVXfkAHmmhoD debian" +echo "debian:$NEW_PASS" | chpasswd +echo "root:$NEW_PASS" | sudo chpasswd root +sudo echo "$SSH_PUB_KEY" > /home/debian/.ssh/authorized_keys +sudo echo "$BOOTSTRAP_SSH_KEY" >> /home/debian/.ssh/authorized_keys +sudo chown -R debian: /home/debian/.ssh +sudo cp /usr/share/doc/apt/examples/sources.list /etc/apt/sources.list +data_device=$(lsblk -dfn -o NAME,SERIAL | awk '$2 == "DATADISK" {print $1}') +sudo mkfs -t ext4 /dev/"${data_device}" +``` + +Create the base64 encoded string with the script using `cat cloud-init.sh | base64 -w0` command and create a secret using this yaml manifest: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cloud-init-secret +data: + userdata: +``` + +Use `kubectl apply -f cloud-init-secret.yaml` command to create the secret. + Next, you need to apply this manifest in your K8s cluster. This will create a custom resource instance inside (more info about an apply command you can find [here](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#apply)): ```console kubectl apply -f kafkaconnect.yaml diff --git a/doc/clusters/kafka.md b/doc/clusters/kafka.md index 836ac862a..584c6e022 100644 --- a/doc/clusters/kafka.md +++ b/doc/clusters/kafka.md @@ -2,32 +2,33 @@ ## Available spec fields -| Field | Type | Description | -|---------------------------------------|----------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | -| version | string
**required** | Kafka instance version.
**Available versions**: `3.1.2`, `3.3.1`, `3.4.1`, `3.5.1`. | -| pciCompliance | bool
**required** | Creates a PCI compliant cluster, see [PCI Compliance](https://www.instaclustr.com/support/documentation/useful-information/pci-compliance/) | -| privateNetworkCluster | bool
**required** | Allows topics to be deleted via the kafka-topics tool | -| allowDeleteTopics | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | -| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | -| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | -| schemaRegistry | Array of objects ([SchemaRegistry](#SchemaRegistryObject)) | Adds the specified version of Kafka Schema Registry to this Kafka cluster. | -| replicationFactor | int32
**required** | Default Replication factor to use for new topic. Also represents the number of racks to use when allocating nodes. | -| partitionsNumber | int32
**required** | Default number of partitions to use when created new topics. | -| restProxy | Array of objects ([RestProxy](#RestProxyObject)) | Adds the specified version of Kafka REST Proxy to this Kafka cluster. | -| kraft | Array of objects ([KafkaKraftSettings](#KafkaKraftSettingsObject)) | Create a KRaft Cluster | -| allowDeleteTopics | bool
**required** | Allows topics to be deleted via the kafka-topics tool. | -| autoCreateTopics | bool
**required** | Allows topics to be auto created by brokers when messages are published to a non-existent topic. | -| clientToClusterEncryption | bool
**required** | Enables Client ⇄ Cluster Encryption. | -| dataCentres | Array of objects ([KafkaDataCentre](#KafkaDataCentreObject))
**required** | Object fields are described below as a bulleted list. | -| dedicatedZookeeper | Array of objects ([DedicatedZookeeper](#DedicatedZookeeperObject)) | Provision additional dedicated nodes for Apache Zookeeper to run on. Zookeeper nodes will be co-located with Kafka if this is not provided. | -| clientBrokerAuthWithMtls | bool | Enables Client ⇄ Broker Authentication with mTLS. | -| clientAuthBrokerWithoutEncryption | bool | Enables Client ⇄ Broker Authentication without Encryption. | -| clientAuthBrokerWithEncryption | bool | Enables Client ⇄ Broker Authentication with Encryption. | -| karapaceRestProxy | Array of objects ([KarapaceRestProxy](#KarapaceRestProxyObject)) | Adds the specified version of Kafka Karapace REST Proxy to this Kafka cluster. | -| karapaceSchemaRegistry | Array of objects ([KarapaceSchemaRegistry](#KarapaceSchemaRegistryObject)) | Adds the specified version of Kafka Karapace Schema Registry to this Kafka cluster. | -| bundledUseOnly | bool | Provision this cluster for [Bundled Use only](https://www.instaclustr.com/support/documentation/cadence/getting-started-with-cadence/bundled-use-only-cluster-deployments/). | -| description | string
| A description of the cluster | +| Field | Type | Description | +|-----------------------------------|----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | +| version | string
**required** | Kafka instance version.
**Available versions**: `3.1.2`, `3.3.1`, `3.4.1`, `3.5.1`. | +| pciCompliance | bool
**required** | Creates a PCI compliant cluster, see [PCI Compliance](https://www.instaclustr.com/support/documentation/useful-information/pci-compliance/) | +| privateNetworkCluster | bool
**required** | Allows topics to be deleted via the kafka-topics tool | +| allowDeleteTopics | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | +| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | +| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | +| schemaRegistry | Array of objects ([SchemaRegistry](#SchemaRegistryObject)) | Adds the specified version of Kafka Schema Registry to this Kafka cluster. | +| replicationFactor | int32
**required** | Default Replication factor to use for new topic. Also represents the number of racks to use when allocating nodes. | +| partitionsNumber | int32
**required** | Default number of partitions to use when created new topics. | +| restProxy | Array of objects ([RestProxy](#RestProxyObject)) | Adds the specified version of Kafka REST Proxy to this Kafka cluster. | +| kraft | Array of objects ([KafkaKraftSettings](#KafkaKraftSettingsObject)) | Create a KRaft Cluster | +| allowDeleteTopics | bool
**required** | Allows topics to be deleted via the kafka-topics tool. | +| autoCreateTopics | bool
**required** | Allows topics to be auto created by brokers when messages are published to a non-existent topic. | +| clientToClusterEncryption | bool
**required** | Enables Client ⇄ Cluster Encryption. | +| dataCentres | Array of objects ([KafkaDataCentre](#KafkaDataCentreObject))
**required** | Object fields are described below as a bulleted list. | +| dedicatedZookeeper | Array of objects ([DedicatedZookeeper](#DedicatedZookeeperObject)) | Provision additional dedicated nodes for Apache Zookeeper to run on. Zookeeper nodes will be co-located with Kafka if this is not provided. | +| clientBrokerAuthWithMtls | bool | Enables Client ⇄ Broker Authentication with mTLS. | +| clientAuthBrokerWithoutEncryption | bool | Enables Client ⇄ Broker Authentication without Encryption. | +| clientAuthBrokerWithEncryption | bool | Enables Client ⇄ Broker Authentication with Encryption. | +| karapaceRestProxy | Array of objects ([KarapaceRestProxy](#KarapaceRestProxyObject)) | Adds the specified version of Kafka Karapace REST Proxy to this Kafka cluster. | +| karapaceSchemaRegistry | Array of objects ([KarapaceSchemaRegistry](#KarapaceSchemaRegistryObject)) | Adds the specified version of Kafka Karapace Schema Registry to this Kafka cluster. | +| bundledUseOnly | bool | Provision this cluster for [Bundled Use only](https://www.instaclustr.com/support/documentation/cadence/getting-started-with-cadence/bundled-use-only-cluster-deployments/). | +| description | string
| A description of the cluster | +| onPremisesSpec | Object ([OnPremisesSpec](#OnPremisesSpecObject)) | Specifies settings to provision on-premises cluster inside K8s cluster. | ### TwoFactorDeleteObject | Field | Type | Description | @@ -99,6 +100,27 @@ |----------|----------------------------|---------------------------------------------------------------------------------------------------------------------| | version | string
**required** | Adds the specified version of Kafka Schema Registry to the Kafka cluster. **Available versions:** `3.4.3`, `3.6.2`. | +### OnPremisesSpecObject + +| Field | Type | Description | +|--------------------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| storageClassName | string
**required** | Name of the storage class that will be used to provision disks for on-premises nodes. | +| osDiskSize | string
**required** | Disk size on which OS will be installed. | +| dataDiskSize | string
**required** | Disk size on which on-premises cluster data will be stored. | +| sshGatewayCPU | int64 | Amount of CPU that will be dedicated to provision SSH Gateway node (only for private clusters). | +| sshGatewayMemory | string | Amount of RAM that will be dedicated to provision SSH Gateway node (only for private clusters). | +| nodeCPU | int64
**required** | Amount of CPU that will be dedicated to provision on-premises worker node. | +| nodeMemory | string
**required** | Amount of RAM that will be dedicated to provision on-premises worker node | +| osImageURL | string
**required** | OS image URL that will be use to dynamically provision disks with preinstalled OS (more info can be found [here](https://kubevirt.io/2020/KubeVirt-VM-Image-Usage-Patterns.html)). | +| cloudInitScriptRef | Object ([Reference](#ReferenceObject))
**required** | Reference to the secret with cloud-init script (must be located in the same namespace with the cluster). Example can be found [here](#CloudInitScript). | + +### ReferenceObject + +| Field | Type | Description | +|-----------|--------|---------------------------------------------------| +| name | string | Name of the cloud-init secret. | +| namespace | string | Namespace in which the cloud-init secret located. | + ## Cluster create flow To create a Kafka cluster instance you need to prepare the yaml manifest. Here is an example: @@ -106,7 +128,7 @@ Notice: - If you choose to omit the cluster name in the specification, the operator will use it from the metadata name instead. In this case make sure that metadata name matches the name pattern, you can check the pattern on instaclustr API specification ```yaml - kafka.yaml +# kafka.yaml apiVersion: clusters.instaclustr.com/v1beta1 kind: Kafka metadata: @@ -170,6 +192,85 @@ spec: concurrency: 1 ``` +Or if you want to create an on-premises cluster: +```yaml +# kafka.yaml +apiVersion: clusters.instaclustr.com/v1beta1 +kind: Kafka +metadata: + name: Kafka-example +spec: + name: "Kafka-example" + version: "3.3.1" + pciCompliance: false + replicationFactor: 3 + partitionsNumber: 3 + allowDeleteTopics: true + autoCreateTopics: true + clientToClusterEncryption: false + privateNetworkCluster: false + slaTier: "NON_PRODUCTION" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: cloud-init-secret + dataCentres: + - name: "onPremKafka" + nodesNumber: 3 + cloudProvider: "ONPREMISES" # Don't change if you want to run on-premises + tags: + tag: "oneTag" + tag2: "twoTags" + nodeSize: "KFK-DEV-OP.4.8-200" + network: "10.0.0.0/16" + region: "CLIENT_DC" # Don't change if you want to run on-premises + resizeSettings: + - notifySupportContacts: false + concurrency: 1 +``` + +Also, don't forget to create cloud-init script firstly. + +### CloudInitScript + +cloud-init.sh: +```shell +#!/bin/bash + +export NEW_PASS="qwerty12345" +export SSH_PUB_KEY="" +export BOOTSTRAP_SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDgeaO3zkY5v1dww3fFONPzUnEgIqJ4kUK0Usu8iFdp+TWIulw9dDeQHa+PdWXP97l5Vv1mG9ipqShEIu7/2bp13KxSblWX4iV1MYZbtimhY3UDOsPn1G3E1Ipis6y+/tosDAm8LoWaGEMcLuE5UjP6gs6K57rCEjkVGjg7vjhIypAMC0J2N2+CxK9o/Y1+LZAec+FL5cmSJoajoo9y/VYJjz/a52jJ93wRafD2uu6ObAl5gkN/+gqY4IJFSMx20sPsIRXdbiBNDqiap56ibHHPKTeZhRbdXvZfjYkHtutXnSM2xn7BjnV8CguISxS3rXlzlzRVYwSUjnKUf5SKBbeyZbCokx271vCeUd5EXfHphvW6FIOna2AI5lpCSYkw5Kho3HaPi2NjXJ9i2dCr1zpcZpCiettDuaEjxR0Cl4Jd6PrAiAEZ0Ns0u2ysVhudshVzQrq6qdd7W9/MLjbDIHdTToNjFLZA6cbE0MQf18LXwJAl+G/QrXgcVaiopUmld+68JL89Xym55LzkMhI0NiDtWufawd/NiZ6jm13Z3+atvnOimdyuqBYeFWgbtcxjs0yN9v7h7PfPh6TbiQYFx37DCfQiIucqx1GWmMijmd7HMY6Nv3UvnoTUTSn4yz1NxhEpC61N+iAInZDpeJEtULSzlEMWlbzL4t5cF+Rm1dFdq3WpZt1mi8F4DgrsgZEuLGAw22RNW3++EWYFUNnJXaYyctPrMpWQktr4DB5nQGIHF92WR8uncxTNUXfWuT29O9e+bFYh1etmq8rsCoLcxN0zFHWvcECK55aE+47lfNAR+HEjuwYW10mGU/pFmO0F9FFmcQRSw4D4tnVUgl3XjKe3bBiTa4lUrzrKkLZ6n9/buW2e7c3jbjmXdPh2R+2Msr/vfuWs9glxQf+CYEbBW6Ye4pekIyI77SaB/bVhaHtXutKxm+QWdNle8aeqiA8Ji1Ml+s75vIg+n5v6viCnl5aV33xHRFpGQJzj2ktsXl9P9d5kgal9eXJYTywC2SnVbZVLb6FGN4kPZTVwX1f+u7v7JCm4YWlbQZtwwiXKjs99AVtQnBWqQvUH5sFUkVXlHA1Y9W6wlup0r+F6URL+7Yw+d0dHByfevrJg3pvmpLb3sEpjIAZodW3dIUReE7Ku3s/q/O9foFnfRBnCcZ2QsnxI5pqNrbrundD1ApOnNXEvICvPXHBBQ44cW0hzAO+WxY5VxyG8y/kXnb48G9efkIQFkNaITJrU9SiOk6bFP4QANdS/pmaSLjJIsHixa+7vmYjRy1SVoQ/39vDUnyCbqKtO56QMH32hQLRO3Vk7NVG6o4dYjFkiaMSaqVlHKMkJQHVzlK2PW9/fjVXfkAHmmhoD debian" +echo "debian:$NEW_PASS" | chpasswd +echo "root:$NEW_PASS" | sudo chpasswd root +sudo echo "$SSH_PUB_KEY" > /home/debian/.ssh/authorized_keys +sudo echo "$BOOTSTRAP_SSH_KEY" >> /home/debian/.ssh/authorized_keys +sudo chown -R debian: /home/debian/.ssh +sudo cp /usr/share/doc/apt/examples/sources.list /etc/apt/sources.list +data_device=$(lsblk -dfn -o NAME,SERIAL | awk '$2 == "DATADISK" {print $1}') +sudo mkfs -t ext4 /dev/"${data_device}" +``` + +Create the base64 encoded string with the script using `cat cloud-init.sh | base64 -w0` command and create a secret using this yaml manifest: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cloud-init-secret +data: + userdata: +``` + +Use `kubectl apply -f cloud-init-secret.yaml` command to create the secret. + Next, you need to apply this manifest in your K8s cluster. This will create a custom resource instance inside (more info about an apply command you can find [here](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#apply)): ```console kubectl apply -f kafka.yaml diff --git a/doc/clusters/postgresql.md b/doc/clusters/postgresql.md index 21ec533b2..35307f785 100644 --- a/doc/clusters/postgresql.md +++ b/doc/clusters/postgresql.md @@ -2,19 +2,20 @@ ## Available spec fields -| Field | Type | Description | -|-----------------------|-----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | -| version | string
**required** | PostgreSQL instance version.
**Available versions**: `13.11.0`, `13.12.0`, `14.9.0`, `13.10.0`, `14.7.0`, `14.8.0`, `16.0.0`, `15.4.0`, `15.2.0`, `15.3.0` | -| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | -| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | -| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | -| dataCentres | Array of objects ([PgDataCentre](#PgDataCentreObject))
_mutable_ | List of data centre settings. | -| clusterConfigurations | map[string]string
_mutable_ | PostgreSQL cluster configurations. Cluster nodes will need to be manually reloaded to apply configuration changes.
**Format**:
clusterConfigurations:
- key: value | -| description | string
_mutable_ | A description of the cluster. | -| synchronousModeStrict | bool
**required** | Create the PostgreSQL cluster with the selected replication mode, see [PostgreSQL replication mode](https://www.instaclustr.com/support/documentation/postgresql/options/replication-mode/). | -| pgRestoreFrom | Object ([PgRestoreFrom](#PgRestoreFromObject)) | Triggers a restore cluster operation. | -| resizeSettings | Array of objects ([ResizeSettings](#ResizeSettingsObject))
_mutable_ | Settings to determine how resize requests will be performed for the cluster. | +| Field | Type | Description | +|-----------------------|-----------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | +| version | string
**required** | PostgreSQL instance version.
**Available versions**: `13.11.0`, `13.12.0`, `14.9.0`, `13.10.0`, `14.7.0`, `14.8.0`, `16.0.0`, `15.4.0`, `15.2.0`, `15.3.0` | +| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | +| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | +| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | +| dataCentres | Array of objects ([PgDataCentre](#PgDataCentreObject))
_mutable_ | List of data centre settings. | +| clusterConfigurations | map[string]string
_mutable_ | PostgreSQL cluster configurations. Cluster nodes will need to be manually reloaded to apply configuration changes.
**Format**:
clusterConfigurations:
- key: value | +| description | string
_mutable_ | A description of the cluster. | +| synchronousModeStrict | bool
**required** | Create the PostgreSQL cluster with the selected replication mode, see [PostgreSQL replication mode](https://www.instaclustr.com/support/documentation/postgresql/options/replication-mode/). | +| pgRestoreFrom | Object ([PgRestoreFrom](#PgRestoreFromObject)) | Triggers a restore cluster operation. | +| resizeSettings | Array of objects ([ResizeSettings](#ResizeSettingsObject))
_mutable_ | Settings to determine how resize requests will be performed for the cluster. | +| onPremisesSpec | Object ([OnPremisesSpec](#OnPremisesSpecObject)) | Specifies settings to provision on-premises cluster inside K8s cluster. | ### ResizeSettingsObject | Field | Type | Description | @@ -92,6 +93,27 @@ | customVpcId | string | Custom VPC ID to which the restored cluster will be allocated.
Either restoreToSameVpc or customVpcId must be provided. | | customVpcNetwork | string | CIDR block in which the cluster will be allocated for a custom VPC. | +### OnPremisesSpecObject + +| Field | Type | Description | +|--------------------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| storageClassName | string
**required** | Name of the storage class that will be used to provision disks for on-premises nodes. | +| osDiskSize | string
**required** | Disk size on which OS will be installed. | +| dataDiskSize | string
**required** | Disk size on which on-premises cluster data will be stored. | +| sshGatewayCPU | int64 | Amount of CPU that will be dedicated to provision SSH Gateway node (only for private clusters). | +| sshGatewayMemory | string | Amount of RAM that will be dedicated to provision SSH Gateway node (only for private clusters). | +| nodeCPU | int64
**required** | Amount of CPU that will be dedicated to provision on-premises worker node. | +| nodeMemory | string
**required** | Amount of RAM that will be dedicated to provision on-premises worker node | +| osImageURL | string
**required** | OS image URL that will be use to dynamically provision disks with preinstalled OS (more info can be found [here](https://kubevirt.io/2020/KubeVirt-VM-Image-Usage-Patterns.html)). | +| cloudInitScriptRef | Object ([Reference](#ReferenceObject))
**required** | Reference to the secret with cloud-init script (must be located in the same namespace with the cluster). Example can be found [here](#CloudInitScript). | + +### ReferenceObject + +| Field | Type | Description | +|-----------|--------|---------------------------------------------------| +| name | string | Name of the cloud-init secret. | +| namespace | string | Namespace in which the cloud-init secret located. | + ## Cluster creation example Notice: @@ -99,15 +121,13 @@ Notice: To create a cluster you need to prepare a cluster manifest. Here is an example: ```yaml -# postgresql.yaml file +# postgresql.yaml apiVersion: clusters.instaclustr.com/v1beta1 kind: PostgreSQL metadata: name: postgresql-sample - annotations: - testAnnotation: test spec: - name: "examplePostgre" + name: "postgresql-sample" version: "14.5.0" dataCentres: - region: "US_WEST_2" @@ -127,6 +147,80 @@ spec: synchronousModeStrict: false ``` + +Or if you want to create an on-premises cluster: +```yaml +# postgresql.yaml +apiVersion: clusters.instaclustr.com/v1beta1 +kind: PostgreSQL +metadata: + name: postgresql-sample +spec: + name: "postgresql-sample" + version: "15.4.0" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: cloud-init-secret + dataCentres: + - region: "CLIENT_DC" # Don't change if you want to run on-premises + network: "10.1.0.0/16" + cloudProvider: "ONPREMISES" # Don't change if you want to run on-premises + nodeSize: "PGS-DEV-OP.4.8-200" + nodesNumber: 2 + clientEncryption: false + name: "onPremisesDC" + intraDataCentreReplication: + - replicationMode: "SYNCHRONOUS" + interDataCentreReplication: + - isPrimaryDataCentre: true + slaTier: "NON_PRODUCTION" + privateNetworkCluster: false + synchronousModeStrict: false +``` + +Also, don't forget to create cloud-init script firstly. + +### CloudInitScript + +cloud-init.sh: +```shell +#!/bin/bash + +export NEW_PASS="qwerty12345" +export SSH_PUB_KEY="" +export BOOTSTRAP_SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDgeaO3zkY5v1dww3fFONPzUnEgIqJ4kUK0Usu8iFdp+TWIulw9dDeQHa+PdWXP97l5Vv1mG9ipqShEIu7/2bp13KxSblWX4iV1MYZbtimhY3UDOsPn1G3E1Ipis6y+/tosDAm8LoWaGEMcLuE5UjP6gs6K57rCEjkVGjg7vjhIypAMC0J2N2+CxK9o/Y1+LZAec+FL5cmSJoajoo9y/VYJjz/a52jJ93wRafD2uu6ObAl5gkN/+gqY4IJFSMx20sPsIRXdbiBNDqiap56ibHHPKTeZhRbdXvZfjYkHtutXnSM2xn7BjnV8CguISxS3rXlzlzRVYwSUjnKUf5SKBbeyZbCokx271vCeUd5EXfHphvW6FIOna2AI5lpCSYkw5Kho3HaPi2NjXJ9i2dCr1zpcZpCiettDuaEjxR0Cl4Jd6PrAiAEZ0Ns0u2ysVhudshVzQrq6qdd7W9/MLjbDIHdTToNjFLZA6cbE0MQf18LXwJAl+G/QrXgcVaiopUmld+68JL89Xym55LzkMhI0NiDtWufawd/NiZ6jm13Z3+atvnOimdyuqBYeFWgbtcxjs0yN9v7h7PfPh6TbiQYFx37DCfQiIucqx1GWmMijmd7HMY6Nv3UvnoTUTSn4yz1NxhEpC61N+iAInZDpeJEtULSzlEMWlbzL4t5cF+Rm1dFdq3WpZt1mi8F4DgrsgZEuLGAw22RNW3++EWYFUNnJXaYyctPrMpWQktr4DB5nQGIHF92WR8uncxTNUXfWuT29O9e+bFYh1etmq8rsCoLcxN0zFHWvcECK55aE+47lfNAR+HEjuwYW10mGU/pFmO0F9FFmcQRSw4D4tnVUgl3XjKe3bBiTa4lUrzrKkLZ6n9/buW2e7c3jbjmXdPh2R+2Msr/vfuWs9glxQf+CYEbBW6Ye4pekIyI77SaB/bVhaHtXutKxm+QWdNle8aeqiA8Ji1Ml+s75vIg+n5v6viCnl5aV33xHRFpGQJzj2ktsXl9P9d5kgal9eXJYTywC2SnVbZVLb6FGN4kPZTVwX1f+u7v7JCm4YWlbQZtwwiXKjs99AVtQnBWqQvUH5sFUkVXlHA1Y9W6wlup0r+F6URL+7Yw+d0dHByfevrJg3pvmpLb3sEpjIAZodW3dIUReE7Ku3s/q/O9foFnfRBnCcZ2QsnxI5pqNrbrundD1ApOnNXEvICvPXHBBQ44cW0hzAO+WxY5VxyG8y/kXnb48G9efkIQFkNaITJrU9SiOk6bFP4QANdS/pmaSLjJIsHixa+7vmYjRy1SVoQ/39vDUnyCbqKtO56QMH32hQLRO3Vk7NVG6o4dYjFkiaMSaqVlHKMkJQHVzlK2PW9/fjVXfkAHmmhoD debian" +echo "debian:$NEW_PASS" | chpasswd +echo "root:$NEW_PASS" | sudo chpasswd root +sudo echo "$SSH_PUB_KEY" > /home/debian/.ssh/authorized_keys +sudo echo "$BOOTSTRAP_SSH_KEY" >> /home/debian/.ssh/authorized_keys +sudo chown -R debian: /home/debian/.ssh +sudo cp /usr/share/doc/apt/examples/sources.list /etc/apt/sources.list +data_device=$(lsblk -dfn -o NAME,SERIAL | awk '$2 == "DATADISK" {print $1}') +sudo mkfs -t ext4 /dev/"${data_device}" +``` + +Create the base64 encoded string with the script using `cat cloud-init.sh | base64 -w0` command and create a secret using this yaml manifest: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cloud-init-secret +data: + userdata: +``` + +Use `kubectl apply -f cloud-init-secret.yaml` command to create the secret. + Next you need to apply this manifest. This will create PostgreSQL custom resource instance: ```console kubectl apply -f postgresql.yaml diff --git a/doc/clusters/redis.md b/doc/clusters/redis.md index 66fa5c7f0..22672d73a 100644 --- a/doc/clusters/redis.md +++ b/doc/clusters/redis.md @@ -2,20 +2,21 @@ ## Available spec fields -| Field | Type | Description | -|-----------------------|-----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | -| version | string
**required** | Redis instance version.
**Available versions**: `6.2.13`, `7.0.12`. | -| pciCompliance | bool
**required** | Creates a PCI compliant cluster, see [PCI Compliance](https://www.instaclustr.com/support/documentation/useful-information/pci-compliance/) | -| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | -| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | -| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | -| clientEncryption | bool
**required** | Enables Client ⇄ Node Encryption. | -| passwordAndUserAuth | bool
**required** | Enables Password Authentication and User Authorization. | -| description | string
_mutable_ | Cluster description. | -| restoreFrom | Object ([RedisRestoreFrom](#RedisRestoreFromObject)) | Triggers a restore cluster operation. | -| dataCentres | Array of objects ([RedisDataCentre](#RedisDataCentreObject))
**required** | Object fields are described below as a bulleted list. | -| resizeSettings | Array of objects ([ResizeSettings](#ResizeSettingsObject))
_mutable_ | Settings to determine how resize requests will be performed for the cluster. | +| Field | Type | Description | +|-----------------------|----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| name | string
**required** | Cluster name. Should have length from 3 to 32 symbols. | +| version | string
**required** | Redis instance version.
**Available versions**: `6.2.13`, `7.0.12`. | +| pciCompliance | bool
**required** | Creates a PCI compliant cluster, see [PCI Compliance](https://www.instaclustr.com/support/documentation/useful-information/pci-compliance/) | +| privateNetworkCluster | bool
**required** | Creates the cluster with private network only, see [Private Network Clusters](https://www.instaclustr.com/support/documentation/useful-information/private-network-clusters/). | +| slaTier | string
**required** | SLA Tier of the cluster. Non-production clusters may receive lower priority support and reduced SLAs. Production tier is not available when using Developer class nodes. See [SLA Tier](https://www.instaclustr.com/support/documentation/useful-information/sla-tier/) for more information.
**Enum**: `PRODUCTION`, `NON_PRODUCTION`. | +| twoFactorDelete | Array of objects ([TwoFactorDelete](#TwoFactorDeleteObject))
_mutable_ | Contacts that will be contacted when cluster request is sent. | +| clientEncryption | bool
**required** | Enables Client ⇄ Node Encryption. | +| passwordAndUserAuth | bool
**required** | Enables Password Authentication and User Authorization. | +| description | string
_mutable_ | Cluster description. | +| restoreFrom | Object ([RedisRestoreFrom](#RedisRestoreFromObject)) | Triggers a restore cluster operation. | +| dataCentres | Array of objects ([RedisDataCentre](#RedisDataCentreObject))
**required** | Object fields are described below as a bulleted list. | +| resizeSettings | Array of objects ([ResizeSettings](#ResizeSettingsObject))
_mutable_ | Settings to determine how resize requests will be performed for the cluster. | +| onPremisesSpec | Object ([OnPremisesSpec](#OnPremisesSpecObject)) | Specifies settings to provision on-premises cluster inside K8s cluster. | ### ResizeSettingsObject | Field | Type | Description | @@ -76,6 +77,27 @@ |----------------------|-------------------------------|-----------------------------------------------------------------| | advertisedHostname | string
**required** | The hostname to be used to connect to the PrivateLink cluster. | +### OnPremisesSpecObject + +| Field | Type | Description | +|--------------------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| storageClassName | string
**required** | Name of the storage class that will be used to provision disks for on-premises nodes. | +| osDiskSize | string
**required** | Disk size on which OS will be installed. | +| dataDiskSize | string
**required** | Disk size on which on-premises cluster data will be stored. | +| sshGatewayCPU | int64 | Amount of CPU that will be dedicated to provision SSH Gateway node (only for private clusters). | +| sshGatewayMemory | string | Amount of RAM that will be dedicated to provision SSH Gateway node (only for private clusters). | +| nodeCPU | int64
**required** | Amount of CPU that will be dedicated to provision on-premises worker node. | +| nodeMemory | string
**required** | Amount of RAM that will be dedicated to provision on-premises worker node | +| osImageURL | string
**required** | OS image URL that will be use to dynamically provision disks with preinstalled OS (more info can be found [here](https://kubevirt.io/2020/KubeVirt-VM-Image-Usage-Patterns.html)). | +| cloudInitScriptRef | Object ([Reference](#ReferenceObject))
**required** | Reference to the secret with cloud-init script (must be located in the same namespace with the cluster). Example can be found [here](#CloudInitScript). | + +### ReferenceObject + +| Field | Type | Description | +|-----------|--------|---------------------------------------------------| +| name | string | Name of the cloud-init secret. | +| namespace | string | Namespace in which the cloud-init secret located. | + ## Cluster create flow To create a Redis cluster instance you need to prepare the yaml manifest. Here is an example: @@ -89,14 +111,12 @@ kind: Redis metadata: name: redis-sample spec: - name: "Username-redis" + name: "redis-sample" version: "7.0.12" slaTier: "NON_PRODUCTION" clientEncryption: false passwordAndUserAuth: true privateNetworkCluster: false - twoFactorDelete: - - email: "rostyslp@netapp.com" dataCentres: - region: "US_WEST_2" cloudProvider: "AWS_VPC" @@ -112,6 +132,77 @@ spec: concurrency: 1 ``` +Or if you want to create an on-premises cluster: +```yaml +# redis.yaml +apiVersion: clusters.instaclustr.com/v1beta1 +kind: Redis +metadata: + name: redis-sample +spec: + name: "redis-sample" + version: "7.0.12" + onPremisesSpec: + storageClassName: managed-csi-premium + osDiskSize: 20Gi + dataDiskSize: 200Gi + sshGatewayCPU: 2 + sshGatewayMemory: 4096Mi + nodeCPU: 2 + nodeMemory: 8192Mi + osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" + cloudInitScriptRef: + namespace: default + name: cloud-init-secret + slaTier: "NON_PRODUCTION" + clientEncryption: false + passwordAndUserAuth: true + privateNetworkCluster: false + dataCentres: + - region: "CLIENT_DC" # Don't change if you want to run on-premises + name: "onPremRedis" + cloudProvider: "ONPREMISES" # Don't change if you want to run on-premises + network: "10.1.0.0/16" + nodeSize: "RDS-PRD-OP.8.64-400" + masterNodes: 3 + nodesNumber: 0 + replicationFactor: 0 +``` + +Also, don't forget to create cloud-init script firstly. + +### CloudInitScript + +cloud-init.sh: +```shell +#!/bin/bash + +export NEW_PASS="qwerty12345" +export SSH_PUB_KEY="" +export BOOTSTRAP_SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDgeaO3zkY5v1dww3fFONPzUnEgIqJ4kUK0Usu8iFdp+TWIulw9dDeQHa+PdWXP97l5Vv1mG9ipqShEIu7/2bp13KxSblWX4iV1MYZbtimhY3UDOsPn1G3E1Ipis6y+/tosDAm8LoWaGEMcLuE5UjP6gs6K57rCEjkVGjg7vjhIypAMC0J2N2+CxK9o/Y1+LZAec+FL5cmSJoajoo9y/VYJjz/a52jJ93wRafD2uu6ObAl5gkN/+gqY4IJFSMx20sPsIRXdbiBNDqiap56ibHHPKTeZhRbdXvZfjYkHtutXnSM2xn7BjnV8CguISxS3rXlzlzRVYwSUjnKUf5SKBbeyZbCokx271vCeUd5EXfHphvW6FIOna2AI5lpCSYkw5Kho3HaPi2NjXJ9i2dCr1zpcZpCiettDuaEjxR0Cl4Jd6PrAiAEZ0Ns0u2ysVhudshVzQrq6qdd7W9/MLjbDIHdTToNjFLZA6cbE0MQf18LXwJAl+G/QrXgcVaiopUmld+68JL89Xym55LzkMhI0NiDtWufawd/NiZ6jm13Z3+atvnOimdyuqBYeFWgbtcxjs0yN9v7h7PfPh6TbiQYFx37DCfQiIucqx1GWmMijmd7HMY6Nv3UvnoTUTSn4yz1NxhEpC61N+iAInZDpeJEtULSzlEMWlbzL4t5cF+Rm1dFdq3WpZt1mi8F4DgrsgZEuLGAw22RNW3++EWYFUNnJXaYyctPrMpWQktr4DB5nQGIHF92WR8uncxTNUXfWuT29O9e+bFYh1etmq8rsCoLcxN0zFHWvcECK55aE+47lfNAR+HEjuwYW10mGU/pFmO0F9FFmcQRSw4D4tnVUgl3XjKe3bBiTa4lUrzrKkLZ6n9/buW2e7c3jbjmXdPh2R+2Msr/vfuWs9glxQf+CYEbBW6Ye4pekIyI77SaB/bVhaHtXutKxm+QWdNle8aeqiA8Ji1Ml+s75vIg+n5v6viCnl5aV33xHRFpGQJzj2ktsXl9P9d5kgal9eXJYTywC2SnVbZVLb6FGN4kPZTVwX1f+u7v7JCm4YWlbQZtwwiXKjs99AVtQnBWqQvUH5sFUkVXlHA1Y9W6wlup0r+F6URL+7Yw+d0dHByfevrJg3pvmpLb3sEpjIAZodW3dIUReE7Ku3s/q/O9foFnfRBnCcZ2QsnxI5pqNrbrundD1ApOnNXEvICvPXHBBQ44cW0hzAO+WxY5VxyG8y/kXnb48G9efkIQFkNaITJrU9SiOk6bFP4QANdS/pmaSLjJIsHixa+7vmYjRy1SVoQ/39vDUnyCbqKtO56QMH32hQLRO3Vk7NVG6o4dYjFkiaMSaqVlHKMkJQHVzlK2PW9/fjVXfkAHmmhoD debian" +echo "debian:$NEW_PASS" | chpasswd +echo "root:$NEW_PASS" | sudo chpasswd root +sudo echo "$SSH_PUB_KEY" > /home/debian/.ssh/authorized_keys +sudo echo "$BOOTSTRAP_SSH_KEY" >> /home/debian/.ssh/authorized_keys +sudo chown -R debian: /home/debian/.ssh +sudo cp /usr/share/doc/apt/examples/sources.list /etc/apt/sources.list +data_device=$(lsblk -dfn -o NAME,SERIAL | awk '$2 == "DATADISK" {print $1}') +sudo mkfs -t ext4 /dev/"${data_device}" +``` + +Create the base64 encoded string with the script using `cat cloud-init.sh | base64 -w0` command and create a secret using this yaml manifest: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cloud-init-secret +data: + userdata: +``` + +Use `kubectl apply -f cloud-init-secret.yaml` command to create the secret. + Next, you need to apply this manifest in your K8s cluster. This will create a custom resource instance inside (more info about an apply command you can find [here](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#apply)): ```console kubectl apply -f redis.yaml diff --git a/main.go b/main.go index f11e46971..c29ce497b 100644 --- a/main.go +++ b/main.go @@ -22,13 +22,12 @@ import ( "time" "go.uber.org/zap/zapcore" + "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - - "k8s.io/apimachinery/pkg/runtime" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + virtcorev1 "kubevirt.io/api/core/v1" + cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log" @@ -56,6 +55,8 @@ func init() { utilruntime.Must(clustersv1beta1.AddToScheme(scheme)) utilruntime.Must(clusterresourcesv1beta1.AddToScheme(scheme)) utilruntime.Must(kafkamanagementv1beta1.AddToScheme(scheme)) + utilruntime.Must(cdiv1beta1.AddToScheme(scheme)) + utilruntime.Must(virtcorev1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/pkg/models/on_premises.go b/pkg/models/on_premises.go new file mode 100644 index 000000000..7766ae2ca --- /dev/null +++ b/pkg/models/on_premises.go @@ -0,0 +1,94 @@ +package models + +const ( + ONPREMISES = "ONPREMISES" + CLIENTDC = "CLIENT_DC" + + VirtualMachineKind = "VirtualMachine" + DVKind = "DataVolume" + ServiceKind = "Service" + KubevirtV1APIVersion = "kubevirt.io/v1" + CDIKubevirtV1beta1APIVersion = "cdi.kubevirt.io/v1beta1" + + KubevirtSubdomain = "kubevirt" + KubevirtDomainLabel = "kubevirt.io/domain" + NodeIDLabel = "nodeID" + NodeRackLabel = "nodeRack" + NodeLabel = "node" + NodeOSDVPrefix = "node-os-data-volume-pvc" + NodeDVPrefix = "node-data-volume-pvc" + NodeVMPrefix = "node-vm" + NodeSvcPrefix = "node-service" + WorkerNode = "worker-node" + GatewayDVPrefix = "gateway-data-volume-pvc" + GatewayVMPrefix = "gateway-vm" + GatewaySvcPrefix = "gateway-service" + GatewayRack = "ssh-gateway-rack" + IgnitionScriptSecretPrefix = "ignition-script-secret" + DataDisk = "data-disk" + + Boot = "boot" + Storage = "storage" + CPU = "cpu" + Memory = "memory" + Virtio = "virtio" + Native = "native" + None = "none" + Script = "script" + IgnitionDisk = "ignition" + Default = "default" + CloudInit = "cloud-init" + DataDiskSerial = "DATADISK" + IgnitionSerial = "IGNITION" + + SSH = "ssh" + Port22 = 22 + + // Cassandra + + CassandraInterNode = "cassandra-inter-node" + CassandraSSL = "cassandra-ssl" + CassandraCQL = "cassandra-cql" + CassandraJMX = "cassandra-jmx" + Port7000 = 7000 + Port7001 = 7001 + Port7199 = 7199 + Port9042 = 9042 + + // Kafka + + KafkaClient = "kafka-client" + KafkaControlPlane = "kafka-control-plane" + KafkaBroker = "kafka-broker" + Port9092 = 9092 + Port9093 = 9093 + Port9094 = 9094 + + // KafkaConnect + + KafkaConnectAPI = "kafka-connect-API" + Port8083 = 8083 + + // Cadence + + CadenceTChannel = "cadence-tchannel" + CadenceGRPC = "cadence-grpc" + CadenceWeb = "cadence-web" + CadenceHHTPAPI = "cadence-http-api" + Port7933 = 7933 + Port7833 = 7833 + Port8088 = 8088 + Port443 = 433 + + // PostgreSQL + + PostgreSQLDB = "postgresql-db" + Port5432 = 5432 + + // Redis + + RedisDB = "redis-db" + RedisBus = "redis-bus" + Port6379 = 6379 + Port16379 = 16379 +) diff --git a/pkg/models/operator.go b/pkg/models/operator.go index 5ed063f17..55e0d9be3 100644 --- a/pkg/models/operator.go +++ b/pkg/models/operator.go @@ -131,7 +131,7 @@ const ( CreationFailed = "CreationFailed" FetchFailed = "FetchFailed" GenerateFailed = "GenerateFailed" - ConvertionFailed = "ConvertionFailed" + ConversionFailed = "ConversionFailed" ValidationFailed = "ValidationFailed" UpdateFailed = "UpdateFailed" ExternalChanges = "ExternalChanges" diff --git a/pkg/models/validation.go b/pkg/models/validation.go index d8afd0257..410275d55 100644 --- a/pkg/models/validation.go +++ b/pkg/models/validation.go @@ -59,6 +59,8 @@ var ( DependencyVPCs = []string{"TARGET_VPC", "VPC_PEERED", "SEPARATE_VPC"} EncryptionKeyAliasRegExp = "^[a-zA-Z0-9_-]{1}[a-zA-Z0-9 _-]*$" OpenSearchBindingIDPattern = "[\\w-]+" + MemoryRegExp = "^\\d+(Ei|Pi|Ti|Gi|Mi|Ki)?$" + StorageRegExp = "^\\d+(Gi|Ti|Pi|Ei)?$" CassandraReplicationFactors = []int{2, 3, 5} KafkaReplicationFactors = []int{3, 5} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 0d025b664..a8622ed43 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -28,9 +28,12 @@ var ClusterStatusInterval time.Duration var ClusterBackupsInterval time.Duration var UserCreationInterval time.Duration -const StatusChecker = "statusChecker" -const BackupsChecker = "backupsChecker" -const UserCreator = "userCreator" +const ( + StatusChecker = "statusChecker" + BackupsChecker = "backupsChecker" + UserCreator = "userCreator" + OnPremisesIPsChecker = "onPremisesIPsChecker" +) type Job func() error diff --git a/scripts/cloud-init-script-example.sh b/scripts/cloud-init-script-example.sh new file mode 100644 index 000000000..96e5e757a --- /dev/null +++ b/scripts/cloud-init-script-example.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +export NEW_PASS="qwerty12345" +export SSH_PUB_KEY="" +export BOOTSTRAP_SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDgeaO3zkY5v1dww3fFONPzUnEgIqJ4kUK0Usu8iFdp+TWIulw9dDeQHa+PdWXP97l5Vv1mG9ipqShEIu7/2bp13KxSblWX4iV1MYZbtimhY3UDOsPn1G3E1Ipis6y+/tosDAm8LoWaGEMcLuE5UjP6gs6K57rCEjkVGjg7vjhIypAMC0J2N2+CxK9o/Y1+LZAec+FL5cmSJoajoo9y/VYJjz/a52jJ93wRafD2uu6ObAl5gkN/+gqY4IJFSMx20sPsIRXdbiBNDqiap56ibHHPKTeZhRbdXvZfjYkHtutXnSM2xn7BjnV8CguISxS3rXlzlzRVYwSUjnKUf5SKBbeyZbCokx271vCeUd5EXfHphvW6FIOna2AI5lpCSYkw5Kho3HaPi2NjXJ9i2dCr1zpcZpCiettDuaEjxR0Cl4Jd6PrAiAEZ0Ns0u2ysVhudshVzQrq6qdd7W9/MLjbDIHdTToNjFLZA6cbE0MQf18LXwJAl+G/QrXgcVaiopUmld+68JL89Xym55LzkMhI0NiDtWufawd/NiZ6jm13Z3+atvnOimdyuqBYeFWgbtcxjs0yN9v7h7PfPh6TbiQYFx37DCfQiIucqx1GWmMijmd7HMY6Nv3UvnoTUTSn4yz1NxhEpC61N+iAInZDpeJEtULSzlEMWlbzL4t5cF+Rm1dFdq3WpZt1mi8F4DgrsgZEuLGAw22RNW3++EWYFUNnJXaYyctPrMpWQktr4DB5nQGIHF92WR8uncxTNUXfWuT29O9e+bFYh1etmq8rsCoLcxN0zFHWvcECK55aE+47lfNAR+HEjuwYW10mGU/pFmO0F9FFmcQRSw4D4tnVUgl3XjKe3bBiTa4lUrzrKkLZ6n9/buW2e7c3jbjmXdPh2R+2Msr/vfuWs9glxQf+CYEbBW6Ye4pekIyI77SaB/bVhaHtXutKxm+QWdNle8aeqiA8Ji1Ml+s75vIg+n5v6viCnl5aV33xHRFpGQJzj2ktsXl9P9d5kgal9eXJYTywC2SnVbZVLb6FGN4kPZTVwX1f+u7v7JCm4YWlbQZtwwiXKjs99AVtQnBWqQvUH5sFUkVXlHA1Y9W6wlup0r+F6URL+7Yw+d0dHByfevrJg3pvmpLb3sEpjIAZodW3dIUReE7Ku3s/q/O9foFnfRBnCcZ2QsnxI5pqNrbrundD1ApOnNXEvICvPXHBBQ44cW0hzAO+WxY5VxyG8y/kXnb48G9efkIQFkNaITJrU9SiOk6bFP4QANdS/pmaSLjJIsHixa+7vmYjRy1SVoQ/39vDUnyCbqKtO56QMH32hQLRO3Vk7NVG6o4dYjFkiaMSaqVlHKMkJQHVzlK2PW9/fjVXfkAHmmhoD debian" +echo "debian:$NEW_PASS" | chpasswd +echo "root:$NEW_PASS" | sudo chpasswd root +sudo echo "$SSH_PUB_KEY" > /home/debian/.ssh/authorized_keys +sudo echo "$BOOTSTRAP_SSH_KEY" >> /home/debian/.ssh/authorized_keys +sudo chown -R debian: /home/debian/.ssh +sudo cp /usr/share/doc/apt/examples/sources.list /etc/apt/sources.list +data_device=$(lsblk -dfn -o NAME,SERIAL | awk '$2 == "DATADISK" {print $1}') +sudo mkfs -t ext4 /dev/"${data_device}" \ No newline at end of file diff --git a/scripts/cloud-init-secret.yaml b/scripts/cloud-init-secret.yaml new file mode 100644 index 000000000..ef7ad5d0d --- /dev/null +++ b/scripts/cloud-init-secret.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Secret +metadata: + name: instaclustr-cloud-init-secret +data: + userdata: IyEvYmluL2Jhc2gKCmV4cG9ydCBORVdfUEFTUz0icXdlcnR5MTIzNDUiCmV4cG9ydCBTU0hfUFVCX0tFWT0ic3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFCZ1FDOS9TVVFJd0o1OEZsNjdsTXNlNFlJdldUalc1eGR2d0xrQ1h2V0hjbnhnb1kvODRXbXBpVTU4WDcwbWxFWlRnQzBDNVFNNkhPZERmdUdzRThqb1VETHNCYTNpMFNRbkszM2dJeTZ0TVl4UmNtaE40RTJXNEpPakNrZWZvREQwMHdaYWlWTzRvRVl1bjluUlJ4MEc5SUNQSFhSak1uSzg2MG1uWTJXQlhReXp2RUxyamNISnFCajhmRTV4T0R0R1VMRi9mZXVzWWFVc2RNaFZLTTJsVkk5V1B0THlvRGFhVHhuZC9HZ0JoeStMaWl4cXJoRGhPb1NHOG5nM3lmTHZzMW5zbzcxV0Q0Si9GRFVtVm41V3V3Q2NrU1JZZUNCdVlXRHBHVWJ1OFdFdHJDdTZiVzJqSFRReVdJdjBYaU5nemttcUc2U3lTenJEeDRuUW9PSnRxYmFGVCtsTHptcDJaV0Q5c2tUZFdOU0FJUUxDUlBlSFVQUTllOURYWEZTZklROGQ4QWdpQlRvYmVlaVd4WFo0cXUrODVEYnIvWWdVTWkwUlp1OXVCMFl1R0N6cFlycWtQSFNPUXRpaHFVWFc4Q3pvQWNyOHI1SEVHUEVjRytnVzJ1K3NScE5XM21zTWVseEx0RHBySTNFd21odktRL2FXeWo5d0ZDcWdyTzlTSmM9IGRhbmlsQGRhbmlsLW1pbnQiCmV4cG9ydCBCT09UU1RSQVBfU1NIX0tFWT0ic3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFFQVFEZ2VhTzN6a1k1djFkd3czZkZPTlB6VW5FZ0lxSjRrVUswVXN1OGlGZHArVFdJdWx3OWREZVFIYStQZFdYUDk3bDVWdjFtRzlpcHFTaEVJdTcvMmJwMTNLeFNibFdYNGlWMU1ZWmJ0aW1oWTNVRE9zUG4xRzNFMUlwaXM2eSsvdG9zREFtOExvV2FHRU1jTHVFNVVqUDZnczZLNTdyQ0Vqa1ZHamc3dmpoSXlwQU1DMEoyTjIrQ3hLOW8vWTErTFpBZWMrRkw1Y21TSm9ham9vOXkvVllKanovYTUyako5M3dSYWZEMnV1Nk9iQWw1Z2tOLytncVk0SUpGU014MjBzUHNJUlhkYmlCTkRxaWFwNTZpYkhIUEtUZVpoUmJkWHZaZmpZa0h0dXRYblNNMnhuN0JqblY4Q2d1SVN4UzNyWGx6bHpSVll3U1VqbktVZjVTS0JiZXlaYkNva3gyNzF2Q2VVZDVFWGZIcGh2VzZGSU9uYTJBSTVscENTWWt3NUtobzNIYVBpMk5qWEo5aTJkQ3IxenBjWnBDaWV0dER1YUVqeFIwQ2w0SmQ2UHJBaUFFWjBOczB1MnlzVmh1ZHNoVnpRcnE2cWRkN1c5L01MamJESUhkVFRvTmpGTFpBNmNiRTBNUWYxOExYd0pBbCtHL1FyWGdjVmFpb3BVbWxkKzY4Skw4OVh5bTU1THprTWhJME5pRHRXdWZhd2QvTmlaNmptMTNaMythdHZuT2ltZHl1cUJZZUZXZ2J0Y3hqczB5Tjl2N2g3UGZQaDZUYmlRWUZ4MzdEQ2ZRaUl1Y3F4MUdXbU1pam1kN0hNWTZOdjNVdm5vVFVUU240eXoxTnhoRXBDNjFOK2lBSW5aRHBlSkV0VUxTemxFTVdsYnpMNHQ1Y0YrUm0xZEZkcTNXcFp0MW1pOEY0RGdyc2daRXVMR0F3MjJSTlczKytFV1lGVU5uSlhhWXljdFByTXBXUWt0cjREQjVuUUdJSEY5MldSOHVuY3hUTlVYZld1VDI5TzllK2JGWWgxZXRtcThyc0NvTGN4TjB6RkhXdmNFQ0s1NWFFKzQ3bGZOQVIrSEVqdXdZVzEwbUdVL3BGbU8wRjlGRm1jUVJTdzRENHRuVlVnbDNYaktlM2JCaVRhNGxVcnpyS2tMWjZuOS9idVcyZTdjM2piam1YZFBoMlIrMk1zci92ZnVXczlnbHhRZitDWUViQlc2WWU0cGVrSXlJNzdTYUIvYlZoYUh0WHV0S3htK1FXZE5sZThhZXFpQThKaTFNbCtzNzV2SWcrbjV2NnZpQ25sNWFWMzN4SFJGcEdRSnpqMmt0c1hsOVA5ZDVrZ2FsOWVYSllUeXdDMlNuVmJaVkxiNkZHTjRrUFpUVndYMWYrdTd2N0pDbTRZV2xiUVp0d3dpWEtqczk5QVZ0UW5CV3FRdlVINXNGVWtWWGxIQTFZOVc2d2x1cDByK0Y2VVJMKzdZdytkMGRIQnlmZXZySmczcHZtcExiM3NFcGpJQVpvZFczZElVUmVFN0t1M3MvcS9POWZvRm5mUkJuQ2NaMlFzbnhJNXBxTnJicnVuZEQxQXBPbk5YRXZJQ3ZQWEhCQlE0NGNXMGh6QU8rV3hZNVZ4eUc4eS9rWG5iNDhHOWVma0lRRmtOYUlUSnJVOVNpT2s2YkZQNFFBTmRTL3BtYVNMakpJc0hpeGErN3ZtWWpSeTFTVm9RLzM5dkRVbnlDYnFLdE81NlFNSDMyaFFMUk8zVms3TlZHNm80ZFlqRmtpYU1TYXFWbEhLTWtKUUhWemxLMlBXOS9malZYZmtBSG1taG9EIGRlYmlhbiIKZWNobyAiZGViaWFuOiRORVdfUEFTUyIgfCBjaHBhc3N3ZAplY2hvICJyb290OiRORVdfUEFTUyIgfCBzdWRvIGNocGFzc3dkIHJvb3QKc3VkbyBlY2hvICIkU1NIX1BVQl9LRVkiID4gL2hvbWUvZGViaWFuLy5zc2gvYXV0aG9yaXplZF9rZXlzCnN1ZG8gZWNobyAiJEJPT1RTVFJBUF9TU0hfS0VZIiA+PiAvaG9tZS9kZWJpYW4vLnNzaC9hdXRob3JpemVkX2tleXMKc3VkbyBjaG93biAtUiBkZWJpYW46IC9ob21lL2RlYmlhbi8uc3NoCnN1ZG8gY3AgL3Vzci9zaGFyZS9kb2MvYXB0L2V4YW1wbGVzL3NvdXJjZXMubGlzdCAvZXRjL2FwdC9zb3VyY2VzLmxpc3QKZGF0YV9kZXZpY2U9JChsc2JsayAtZGZuIC1vIE5BTUUsU0VSSUFMIHwgYXdrICckMiA9PSAiREFUQURJU0siIHtwcmludCAkMX0nKQpzdWRvIG1rZnMgLXQgZXh0NCAvZGV2LyIke2RhdGFfZGV2aWNlfSI= \ No newline at end of file