From 2b448dc667d734afa6d60a4c520f6b4810d0eb5e Mon Sep 17 00:00:00 2001 From: Jakob Steiner Date: Mon, 23 Oct 2023 17:33:13 +0200 Subject: [PATCH] feat: add cloud storage backup cronjob (#350) --- .idea/kotlinc.xml | 2 +- deploy/crd/planes.glasskube.eu-v1.yml | 7 +- .../operator/apps/common/backup/BackupSpec.kt | 31 ++-- .../common/cloudstorage/CloudStorageSpec.kt | 26 +++ .../cloudstorage/HasCloudStorageSpec.kt | 8 + .../cloudstorage/ResourceWithCloudStorage.kt | 13 ++ .../glasskube/operator/apps/gitlab/Gitlab.kt | 5 + .../operator/apps/gitlab/GitlabReconciler.kt | 6 + .../apps/gitlab/GitlabRegistryStorageSpec.kt | 22 ++- .../operator/apps/gitlab/GitlabSpec.kt | 5 +- .../GitlabCloudStorageBackupCronJob.kt | 11 ++ .../operator/apps/nextcloud/Nextcloud.kt | 5 + .../apps/nextcloud/NextcloudReconciler.kt | 21 ++- .../operator/apps/nextcloud/NextcloudSpec.kt | 39 ++++- .../apps/nextcloud/NextcloudStorageSpec.kt | 33 ---- .../NextcloudCloudStorageBackupCronJob.kt | 11 ++ .../nextcloud/dependent/NextcloudCronJob.kt | 11 +- .../eu/glasskube/operator/apps/plane/Plane.kt | 5 + .../operator/apps/plane/PlaneReconciler.kt | 6 + .../operator/apps/plane/PlaneSpec.kt | 24 ++- .../PlaneCloudStorageBackupCronJob.kt | 10 ++ .../backups/BackupSpecNotNullCondition.kt | 2 +- .../backups/CloudStorageNotNullCondition.kt | 12 ++ .../DependentCloudStorageBackupCronJob.kt | 150 ++++++++++++++++++ .../processing/CompositeAndCondition.kt | 13 ++ .../kotlin/eu/glasskube/utils/Duration.kt | 25 +++ .../eu/glasskube/utils/DurationKtTest.kt | 23 +++ 27 files changed, 445 insertions(+), 81 deletions(-) create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/CloudStorageSpec.kt create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/HasCloudStorageSpec.kt create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/ResourceWithCloudStorage.kt create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/dependent/GitlabCloudStorageBackupCronJob.kt delete mode 100644 operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudStorageSpec.kt create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/dependent/NextcloudCloudStorageBackupCronJob.kt create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/apps/plane/dependent/PlaneCloudStorageBackupCronJob.kt create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/CloudStorageNotNullCondition.kt create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/DependentCloudStorageBackupCronJob.kt create mode 100644 operator/src/main/kotlin/eu/glasskube/operator/processing/CompositeAndCondition.kt create mode 100644 operator/src/main/kotlin/eu/glasskube/utils/Duration.kt create mode 100644 operator/src/test/kotlin/eu/glasskube/utils/DurationKtTest.kt diff --git a/.idea/kotlinc.xml b/.idea/kotlinc.xml index 1ee496d8..8a0b064c 100644 --- a/.idea/kotlinc.xml +++ b/.idea/kotlinc.xml @@ -1,6 +1,6 @@ - diff --git a/deploy/crd/planes.glasskube.eu-v1.yml b/deploy/crd/planes.glasskube.eu-v1.yml index c12b3112..75d6ddea 100644 --- a/deploy/crd/planes.glasskube.eu-v1.yml +++ b/deploy/crd/planes.glasskube.eu-v1.yml @@ -221,9 +221,14 @@ spec: type: object region: type: string - endpoint: + hostname: nullable: true type: string + port: + nullable: true + type: integer + useSsl: + type: boolean usePathStyle: nullable: true type: boolean diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/common/backup/BackupSpec.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/backup/BackupSpec.kt index eccc3aea..5e291779 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/common/backup/BackupSpec.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/backup/BackupSpec.kt @@ -1,6 +1,6 @@ package eu.glasskube.operator.apps.common.backup -import com.fasterxml.jackson.annotation.JsonIgnore +import eu.glasskube.operator.apps.common.cloudstorage.CloudStorageSpec import io.fabric8.generator.annotation.Nullable import io.fabric8.generator.annotation.Required import io.fabric8.kubernetes.api.model.SecretKeySelector @@ -13,29 +13,18 @@ data class BackupSpec( ) { data class S3Spec( @field:Nullable - val hostname: String?, + override val hostname: String?, @field:Nullable - val port: Int?, - val useSsl: Boolean = true, + override val port: Int?, + override val useSsl: Boolean = true, @field:Nullable - val region: String?, + override val region: String?, @field:Required - val bucket: String, + override val bucket: String, @field:Required - val accessKeySecret: SecretKeySelector, + override val accessKeySecret: SecretKeySelector, @field:Required - val secretKeySecret: SecretKeySelector, - val usePathStyle: Boolean = true - ) { - @get:JsonIgnore - val endpoint - get() = hostname?.let { - buildString { - append(if (useSsl) "https" else "http", "://", it) - if (port != null) { - append(":", port) - } - } - } - } + override val secretKeySecret: SecretKeySelector, + override val usePathStyle: Boolean = true + ) : CloudStorageSpec } diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/CloudStorageSpec.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/CloudStorageSpec.kt new file mode 100644 index 00000000..6f9f93f2 --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/CloudStorageSpec.kt @@ -0,0 +1,26 @@ +package eu.glasskube.operator.apps.common.cloudstorage + +import com.fasterxml.jackson.annotation.JsonIgnore +import io.fabric8.kubernetes.api.model.SecretKeySelector + +interface CloudStorageSpec { + val bucket: String + val accessKeySecret: SecretKeySelector + val secretKeySecret: SecretKeySelector + val region: String? + val hostname: String? + val port: Int? + val useSsl: Boolean + val usePathStyle: Boolean? + + @get:JsonIgnore + val endpoint + get() = hostname?.let { + buildString { + append(if (useSsl) "https" else "http", "://", it) + if (port != null) { + append(":", port) + } + } + } +} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/HasCloudStorageSpec.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/HasCloudStorageSpec.kt new file mode 100644 index 00000000..b8bc51ef --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/HasCloudStorageSpec.kt @@ -0,0 +1,8 @@ +package eu.glasskube.operator.apps.common.cloudstorage + +import com.fasterxml.jackson.annotation.JsonIgnore + +interface HasCloudStorageSpec { + @get:JsonIgnore + val cloudStorage: CloudStorageSpec? +} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/ResourceWithCloudStorage.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/ResourceWithCloudStorage.kt new file mode 100644 index 00000000..0d1f49db --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/common/cloudstorage/ResourceWithCloudStorage.kt @@ -0,0 +1,13 @@ +package eu.glasskube.operator.apps.common.cloudstorage + +import com.fasterxml.jackson.annotation.JsonIgnore + +interface ResourceWithCloudStorage { + @get:JsonIgnore + val backupResourceName: String + + @get:JsonIgnore + val backupResourceLabels: Map + + fun getSpec(): HasCloudStorageSpec +} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/Gitlab.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/Gitlab.kt index ca8f6304..e120643d 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/Gitlab.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/Gitlab.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore import eu.glasskube.kubernetes.client.resources import eu.glasskube.operator.Labels import eu.glasskube.operator.apps.common.backup.ResourceWithBackups +import eu.glasskube.operator.apps.common.cloudstorage.ResourceWithCloudStorage import eu.glasskube.operator.apps.common.database.ResourceWithDatabaseSpec import eu.glasskube.operator.apps.common.database.postgres.PostgresDatabaseSpec import eu.glasskube.operator.apps.gitlab.Gitlab.Postgres.postgresClusterLabelSelector @@ -21,6 +22,7 @@ class Gitlab : CustomResource(), Namespaced, ResourceWithBackups, + ResourceWithCloudStorage, ResourceWithDatabaseSpec { companion object { const val APP_NAME = "gitlab" @@ -33,6 +35,9 @@ class Gitlab : override fun getDatabaseName(primary: Gitlab) = "gitlabhq_production" } + override val backupResourceName get() = "$genericResourceName-backup" + override val backupResourceLabels get() = resourceLabels + @delegate:JsonIgnore override val velero by lazy { object : VeleroNameMapper(this) { diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabReconciler.kt index 0bd0fc64..b6f08cd9 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabReconciler.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabReconciler.kt @@ -4,6 +4,7 @@ import eu.glasskube.kubernetes.client.patchOrUpdateStatus import eu.glasskube.operator.Labels import eu.glasskube.operator.api.reconciler.getSecondaryResource import eu.glasskube.operator.api.reconciler.informerEventSource +import eu.glasskube.operator.apps.gitlab.dependent.GitlabCloudStorageBackupCronJob import eu.glasskube.operator.apps.gitlab.dependent.GitlabConfigMap import eu.glasskube.operator.apps.gitlab.dependent.GitlabDeployment import eu.glasskube.operator.apps.gitlab.dependent.GitlabIngress @@ -90,6 +91,11 @@ import io.javaoperatorsdk.operator.api.reconciler.dependent.Dependent type = GitlabRunners::class, dependsOn = ["GitlabDeployment"] ), + Dependent( + type = GitlabCloudStorageBackupCronJob::class, + name = "GitlabCloudStorageBackupCronJob", + reconcilePrecondition = GitlabCloudStorageBackupCronJob.ReconcilePrecondition::class + ), Dependent( type = GitlabVeleroSecret::class, name = "GitlabVeleroSecret", diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabRegistryStorageSpec.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabRegistryStorageSpec.kt index 4649a4ba..275c8a64 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabRegistryStorageSpec.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabRegistryStorageSpec.kt @@ -1,5 +1,7 @@ package eu.glasskube.operator.apps.gitlab +import com.fasterxml.jackson.annotation.JsonIgnore +import eu.glasskube.operator.apps.common.cloudstorage.CloudStorageSpec import io.fabric8.generator.annotation.Required import io.fabric8.kubernetes.api.model.SecretKeySelector @@ -8,16 +10,22 @@ data class GitlabRegistryStorageSpec( ) { data class S3( @field:Required - val bucket: String, + override val bucket: String, @field:Required - val accessKeySecret: SecretKeySelector, + override val accessKeySecret: SecretKeySelector, @field:Required - val secretKeySecret: SecretKeySelector, + override val secretKeySecret: SecretKeySelector, @field:Required - val region: String, + override val region: String, @field:Required - val hostname: String, + override val hostname: String, @field:Required - val usePathStyle: Boolean - ) + override val usePathStyle: Boolean + ) : CloudStorageSpec { + @field:JsonIgnore + override val port = null + + @field:JsonIgnore + override val useSsl = true + } } diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabSpec.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabSpec.kt index 48e435bc..bd52588c 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabSpec.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/GitlabSpec.kt @@ -2,6 +2,7 @@ package eu.glasskube.operator.apps.gitlab import eu.glasskube.operator.apps.common.backup.BackupSpec import eu.glasskube.operator.apps.common.backup.HasBackupSpec +import eu.glasskube.operator.apps.common.cloudstorage.HasCloudStorageSpec import eu.glasskube.operator.apps.common.database.HasDatabaseSpec import eu.glasskube.operator.apps.common.database.postgres.PostgresDatabaseSpec import eu.glasskube.operator.validation.Patterns.SEMVER @@ -35,4 +36,6 @@ data class GitlabSpec( @field:Nullable override val database: PostgresDatabaseSpec = PostgresDatabaseSpec(), override val backups: BackupSpec? -) : HasBackupSpec, HasDatabaseSpec +) : HasBackupSpec, HasCloudStorageSpec, HasDatabaseSpec { + override val cloudStorage get() = registry?.storage?.s3 +} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/dependent/GitlabCloudStorageBackupCronJob.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/dependent/GitlabCloudStorageBackupCronJob.kt new file mode 100644 index 00000000..23d61120 --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/gitlab/dependent/GitlabCloudStorageBackupCronJob.kt @@ -0,0 +1,11 @@ +package eu.glasskube.operator.apps.gitlab.dependent + +import eu.glasskube.operator.apps.gitlab.Gitlab +import eu.glasskube.operator.apps.gitlab.GitlabReconciler +import eu.glasskube.operator.generic.dependent.backups.DependentCloudStorageBackupCronJob +import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent + +@KubernetesDependent(labelSelector = GitlabReconciler.SELECTOR) +class GitlabCloudStorageBackupCronJob : DependentCloudStorageBackupCronJob() { + internal class ReconcilePrecondition : DependentCloudStorageBackupCronJob.ReconcilePrecondition() +} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/Nextcloud.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/Nextcloud.kt index b1e2345c..699f6fd2 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/Nextcloud.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/Nextcloud.kt @@ -6,6 +6,7 @@ import eu.glasskube.kubernetes.api.model.envVar import eu.glasskube.kubernetes.api.model.secretKeyRef import eu.glasskube.operator.Labels import eu.glasskube.operator.apps.common.backup.ResourceWithBackups +import eu.glasskube.operator.apps.common.cloudstorage.ResourceWithCloudStorage import eu.glasskube.operator.apps.common.database.ResourceWithDatabaseSpec import eu.glasskube.operator.apps.common.database.postgres.PostgresDatabaseSpec import eu.glasskube.operator.apps.nextcloud.Nextcloud.Postgres.postgresClusterLabelSelector @@ -28,6 +29,7 @@ class Nextcloud : CustomResource(), Namespaced, ResourceWithBackups, + ResourceWithCloudStorage, ResourceWithDatabaseSpec { internal companion object { const val APP_NAME = "nextcloud" @@ -58,6 +60,9 @@ class Nextcloud : override fun getDatabaseName(primary: Nextcloud) = "nextcloud" } + override val backupResourceName get() = "$genericResourceName-backup" + override val backupResourceLabels get() = resourceLabels + @delegate:JsonIgnore override val velero by lazy { object : VeleroNameMapper(this) { diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudReconciler.kt index fbe14eb5..16509bb4 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudReconciler.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudReconciler.kt @@ -4,6 +4,7 @@ import eu.glasskube.kubernetes.client.patchOrUpdateStatus import eu.glasskube.operator.Labels import eu.glasskube.operator.api.reconciler.getSecondaryResource import eu.glasskube.operator.api.reconciler.informerEventSource +import eu.glasskube.operator.apps.nextcloud.dependent.NextcloudCloudStorageBackupCronJob import eu.glasskube.operator.apps.nextcloud.dependent.NextcloudConfigMap import eu.glasskube.operator.apps.nextcloud.dependent.NextcloudCronJob import eu.glasskube.operator.apps.nextcloud.dependent.NextcloudDeployment @@ -25,9 +26,9 @@ import eu.glasskube.operator.generic.condition.isReady import eu.glasskube.operator.infra.postgres.PostgresCluster import eu.glasskube.operator.infra.postgres.isReady import eu.glasskube.operator.webhook.WebhookService -import eu.glasskube.utils.logger import io.fabric8.kubernetes.api.model.Service import io.fabric8.kubernetes.api.model.apps.Deployment +import io.fabric8.kubernetes.api.model.batch.v1.CronJob import io.javaoperatorsdk.operator.api.reconciler.Context import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext @@ -67,7 +68,17 @@ import kotlin.jvm.optionals.getOrDefault useEventSourceWithName = NextcloudReconciler.DEPLOYMENT_EVENT_SOURCE, dependsOn = ["NextcloudVolume", "NextcloudConfigMap", "NextcloudPostgresCluster"] ), - Dependent(type = NextcloudCronJob::class, name = "NextcloudCronJob", dependsOn = ["NextcloudDeployment"]), + Dependent( + type = NextcloudCronJob::class, + name = "NextcloudCronJob", + dependsOn = ["NextcloudDeployment"], + useEventSourceWithName = NextcloudReconciler.CRON_JOB_EVENT_SOURCE + ), + Dependent( + type = NextcloudCloudStorageBackupCronJob::class, + name = "NextcloudCloudStorageBackupCronJob", + reconcilePrecondition = NextcloudCloudStorageBackupCronJob.ReconcilePrecondition::class + ), Dependent( type = NextcloudRedisDeployment::class, name = "NextcloudRedisDeployment", @@ -128,7 +139,8 @@ class NextcloudReconciler(webhookService: WebhookService) : override fun prepareEventSources(context: EventSourceContext) = with(context) { mutableMapOf( DEPLOYMENT_EVENT_SOURCE to informerEventSource(), - SERVICE_EVENT_SOURCE to informerEventSource() + SERVICE_EVENT_SOURCE to informerEventSource(), + CRON_JOB_EVENT_SOURCE to informerEventSource() ) } @@ -142,7 +154,6 @@ class NextcloudReconciler(webhookService: WebhookService) : internal const val SERVICE_EVENT_SOURCE = "NextcloudServiceEventSource" internal const val DEPLOYMENT_EVENT_SOURCE = "NextcloudDeploymentEventSource" - - private val log = logger() + internal const val CRON_JOB_EVENT_SOURCE = "NextcloudCronJobEventSource" } } diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudSpec.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudSpec.kt index b78e0a05..ce0840bc 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudSpec.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudSpec.kt @@ -2,13 +2,17 @@ package eu.glasskube.operator.apps.nextcloud import eu.glasskube.operator.apps.common.backup.BackupSpec import eu.glasskube.operator.apps.common.backup.HasBackupSpec +import eu.glasskube.operator.apps.common.cloudstorage.CloudStorageSpec +import eu.glasskube.operator.apps.common.cloudstorage.HasCloudStorageSpec import eu.glasskube.operator.apps.common.database.HasDatabaseSpec import eu.glasskube.operator.apps.common.database.postgres.PostgresDatabaseSpec import eu.glasskube.operator.validation.Patterns import io.fabric8.generator.annotation.Nullable import io.fabric8.generator.annotation.Pattern +import io.fabric8.generator.annotation.Required import io.fabric8.kubernetes.api.model.Quantity import io.fabric8.kubernetes.api.model.ResourceRequirements +import io.fabric8.kubernetes.api.model.SecretKeySelector data class NextcloudSpec( val host: String, @@ -16,14 +20,17 @@ data class NextcloudSpec( val apps: NextcloudAppsSpec = NextcloudAppsSpec(), @field:Nullable val smtp: NextcloudSmtpSpec?, - val storage: NextcloudStorageSpec?, + val storage: StorageSpec?, @field:Pattern(Patterns.SEMVER) val version: String = "27.0.1", val server: ServerSpec = ServerSpec(), @field:Nullable override val database: PostgresDatabaseSpec = PostgresDatabaseSpec(), override val backups: BackupSpec? -) : HasBackupSpec, HasDatabaseSpec { +) : HasBackupSpec, HasCloudStorageSpec, HasDatabaseSpec { + + override val cloudStorage get() = storage?.s3 + data class ServerSpec( @field:Nullable val resources: ResourceRequirements = ResourceRequirements( @@ -36,4 +43,32 @@ data class NextcloudSpec( val minSpareServers: Int = maxChildren / 16, val maxSpareServers: Int = maxChildren / 4 ) + + data class StorageSpec( + val s3: S3? + ) { + data class S3( + @field:Required + override val bucket: String, + @field:Required + override val accessKeySecret: SecretKeySelector, + @field:Required + override val secretKeySecret: SecretKeySelector, + @field:Nullable + override val region: String?, + @field:Nullable + override val hostname: String?, + @field:Nullable + override val port: Int?, + @field:Nullable + val objectPrefix: String?, + @field:Nullable + val autoCreate: Boolean?, + override val useSsl: Boolean = true, + @field:Nullable + override val usePathStyle: Boolean?, + @field:Nullable + val legacyAuth: Boolean? + ) : CloudStorageSpec + } } diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudStorageSpec.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudStorageSpec.kt deleted file mode 100644 index e175db22..00000000 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/NextcloudStorageSpec.kt +++ /dev/null @@ -1,33 +0,0 @@ -package eu.glasskube.operator.apps.nextcloud - -import io.fabric8.generator.annotation.Nullable -import io.fabric8.generator.annotation.Required -import io.fabric8.kubernetes.api.model.SecretKeySelector - -data class NextcloudStorageSpec( - val s3: S3? -) { - data class S3( - @field:Required - val bucket: String, - @field:Required - val accessKeySecret: SecretKeySelector, - @field:Required - val secretKeySecret: SecretKeySelector, - @field:Nullable - val region: String?, - @field:Nullable - val hostname: String?, - @field:Nullable - val port: Int?, - @field:Nullable - val objectPrefix: String?, - @field:Nullable - val autoCreate: Boolean?, - val useSsl: Boolean = true, - @field:Nullable - val usePathStyle: Boolean?, - @field:Nullable - val legacyAuth: Boolean? - ) -} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/dependent/NextcloudCloudStorageBackupCronJob.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/dependent/NextcloudCloudStorageBackupCronJob.kt new file mode 100644 index 00000000..f9d11ea7 --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/dependent/NextcloudCloudStorageBackupCronJob.kt @@ -0,0 +1,11 @@ +package eu.glasskube.operator.apps.nextcloud.dependent + +import eu.glasskube.operator.apps.nextcloud.Nextcloud +import eu.glasskube.operator.generic.dependent.backups.DependentCloudStorageBackupCronJob +import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent + +@KubernetesDependent(resourceDiscriminator = NextcloudCloudStorageBackupCronJob.Discriminator::class) +class NextcloudCloudStorageBackupCronJob : DependentCloudStorageBackupCronJob() { + internal class Discriminator : DependentCloudStorageBackupCronJob.Discriminator() + internal class ReconcilePrecondition : DependentCloudStorageBackupCronJob.ReconcilePrecondition() +} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/dependent/NextcloudCronJob.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/dependent/NextcloudCronJob.kt index 0d9cb72c..97b5517f 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/dependent/NextcloudCronJob.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/nextcloud/dependent/NextcloudCronJob.kt @@ -24,11 +24,20 @@ import eu.glasskube.operator.apps.nextcloud.resourceLabels import eu.glasskube.operator.apps.nextcloud.volumeName import io.fabric8.kubernetes.api.model.batch.v1.CronJob import io.javaoperatorsdk.operator.api.reconciler.Context +import io.javaoperatorsdk.operator.api.reconciler.ResourceIDMatcherDiscriminator import io.javaoperatorsdk.operator.processing.dependent.kubernetes.CRUDKubernetesDependentResource import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent +import io.javaoperatorsdk.operator.processing.event.ResourceID -@KubernetesDependent(labelSelector = NextcloudReconciler.SELECTOR) +@KubernetesDependent( + labelSelector = NextcloudReconciler.SELECTOR, + resourceDiscriminator = NextcloudCronJob.Discriminator::class +) class NextcloudCronJob : CRUDKubernetesDependentResource(CronJob::class.java) { + + internal class Discriminator : + ResourceIDMatcherDiscriminator({ ResourceID(it.cronName, it.namespace) }) + override fun desired(primary: Nextcloud, context: Context) = CronJob().apply { metadata { name(primary.cronName) diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/Plane.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/Plane.kt index 4238e9bb..3d910d59 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/Plane.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/Plane.kt @@ -3,6 +3,7 @@ package eu.glasskube.operator.apps.plane import com.fasterxml.jackson.annotation.JsonIgnore import eu.glasskube.operator.Labels import eu.glasskube.operator.apps.common.backup.ResourceWithBackups +import eu.glasskube.operator.apps.common.cloudstorage.ResourceWithCloudStorage import eu.glasskube.operator.apps.common.database.ResourceWithDatabaseSpec import eu.glasskube.operator.apps.common.database.postgres.PostgresDatabaseSpec import eu.glasskube.operator.apps.plane.Plane.Postgres.postgresClusterLabelSelector @@ -22,6 +23,7 @@ class Plane : CustomResource(), Namespaced, ResourceWithBackups, + ResourceWithCloudStorage, ResourceWithDatabaseSpec { object Redis : RedisNameMapper() { private const val NAME = "redis" @@ -55,6 +57,9 @@ class Plane : } } + override val backupResourceName get() = "$genericResourceName-backup" + override val backupResourceLabels get() = genericResourceLabels + internal companion object { const val APP_NAME = "plane" const val FRONTEND_NAME = "frontend" diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneReconciler.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneReconciler.kt index c86c2622..144d3031 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneReconciler.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneReconciler.kt @@ -11,6 +11,7 @@ import eu.glasskube.operator.apps.plane.dependent.PlaneApiService import eu.glasskube.operator.apps.plane.dependent.PlaneBackendConfigMap import eu.glasskube.operator.apps.plane.dependent.PlaneBackendSecret import eu.glasskube.operator.apps.plane.dependent.PlaneBeatWorkerDeployment +import eu.glasskube.operator.apps.plane.dependent.PlaneCloudStorageBackupCronJob import eu.glasskube.operator.apps.plane.dependent.PlaneFrontendConfigMap import eu.glasskube.operator.apps.plane.dependent.PlaneFrontendDeployment import eu.glasskube.operator.apps.plane.dependent.PlaneFrontendService @@ -157,6 +158,11 @@ import kotlin.jvm.optionals.getOrDefault dependsOn = ["PlanePostgresCluster", "PlaneRedisDeployment", "PlaneBackendConfigMap", "PlaneBackendSecret", "PlaneWorkerConfigMap", "PlaneApiDeployment"], useEventSourceWithName = PlaneReconciler.DEPLOYMENT_EVENT_SOURCE ), + Dependent( + type = PlaneCloudStorageBackupCronJob::class, + name = "PlaneCloudStorageBackupCronJob", + reconcilePrecondition = PlaneCloudStorageBackupCronJob.ReconcilePrecondition::class + ), Dependent( type = PlaneVeleroSecret::class, name = "PlaneVeleroSecret", diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneSpec.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneSpec.kt index c024b8d5..b7278400 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneSpec.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/PlaneSpec.kt @@ -2,6 +2,8 @@ package eu.glasskube.operator.apps.plane import eu.glasskube.operator.apps.common.backup.BackupSpec import eu.glasskube.operator.apps.common.backup.HasBackupSpec +import eu.glasskube.operator.apps.common.cloudstorage.CloudStorageSpec +import eu.glasskube.operator.apps.common.cloudstorage.HasCloudStorageSpec import eu.glasskube.operator.apps.common.database.HasDatabaseSpec import eu.glasskube.operator.apps.common.database.postgres.PostgresDatabaseSpec import io.fabric8.generator.annotation.Nullable @@ -27,7 +29,10 @@ data class PlaneSpec( @field:Nullable override val database: PostgresDatabaseSpec = PostgresDatabaseSpec(), override val backups: BackupSpec? -) : HasBackupSpec, HasDatabaseSpec { +) : HasBackupSpec, HasCloudStorageSpec, HasDatabaseSpec { + + override val cloudStorage get() = s3 + data class DefaultUserSpec( @field:Required val email: String, @@ -121,16 +126,19 @@ data class PlaneSpec( data class S3Spec( @field:Required - val bucket: String, + override val bucket: String, @field:Required - val accessKeySecret: SecretKeySelector, + override val accessKeySecret: SecretKeySelector, @field:Required - val secretKeySecret: SecretKeySelector, + override val secretKeySecret: SecretKeySelector, @field:Required - val region: String, + override val region: String, @field:Nullable - val endpoint: String?, + override val hostname: String?, @field:Nullable - val usePathStyle: Boolean? - ) + override val port: Int?, + override val useSsl: Boolean = true, + @field:Nullable + override val usePathStyle: Boolean? + ) : CloudStorageSpec } diff --git a/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/dependent/PlaneCloudStorageBackupCronJob.kt b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/dependent/PlaneCloudStorageBackupCronJob.kt new file mode 100644 index 00000000..a1fb92d2 --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/operator/apps/plane/dependent/PlaneCloudStorageBackupCronJob.kt @@ -0,0 +1,10 @@ +package eu.glasskube.operator.apps.plane.dependent + +import eu.glasskube.operator.apps.plane.Plane +import eu.glasskube.operator.generic.dependent.backups.DependentCloudStorageBackupCronJob +import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent + +@KubernetesDependent +class PlaneCloudStorageBackupCronJob : DependentCloudStorageBackupCronJob() { + internal class ReconcilePrecondition : DependentCloudStorageBackupCronJob.ReconcilePrecondition() +} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/BackupSpecNotNullCondition.kt b/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/BackupSpecNotNullCondition.kt index 8e7372a7..0a049dd6 100644 --- a/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/BackupSpecNotNullCondition.kt +++ b/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/BackupSpecNotNullCondition.kt @@ -6,7 +6,7 @@ import io.javaoperatorsdk.operator.api.reconciler.Context import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource import io.javaoperatorsdk.operator.processing.dependent.workflow.Condition -abstract class BackupSpecNotNullCondition : Condition where P : HasMetadata, P : ResourceWithBackups { +open class BackupSpecNotNullCondition : Condition where P : HasMetadata, P : ResourceWithBackups { override fun isMet(dependentResource: DependentResource, primary: P, context: Context

) = primary.getSpec().backups != null } diff --git a/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/CloudStorageNotNullCondition.kt b/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/CloudStorageNotNullCondition.kt new file mode 100644 index 00000000..831141d3 --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/CloudStorageNotNullCondition.kt @@ -0,0 +1,12 @@ +package eu.glasskube.operator.generic.dependent.backups + +import eu.glasskube.operator.apps.common.cloudstorage.ResourceWithCloudStorage +import io.fabric8.kubernetes.api.model.HasMetadata +import io.javaoperatorsdk.operator.api.reconciler.Context +import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource +import io.javaoperatorsdk.operator.processing.dependent.workflow.Condition + +class CloudStorageNotNullCondition : Condition where P : HasMetadata, P : ResourceWithCloudStorage { + override fun isMet(dependentResource: DependentResource, primary: P, context: Context

) = + primary.getSpec().cloudStorage != null +} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/DependentCloudStorageBackupCronJob.kt b/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/DependentCloudStorageBackupCronJob.kt new file mode 100644 index 00000000..599c22a2 --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/operator/generic/dependent/backups/DependentCloudStorageBackupCronJob.kt @@ -0,0 +1,150 @@ +package eu.glasskube.operator.generic.dependent.backups + +import eu.glasskube.kubernetes.api.model.batch.jobTemplate +import eu.glasskube.kubernetes.api.model.batch.spec +import eu.glasskube.kubernetes.api.model.batch.template +import eu.glasskube.kubernetes.api.model.container +import eu.glasskube.kubernetes.api.model.emptyDir +import eu.glasskube.kubernetes.api.model.env +import eu.glasskube.kubernetes.api.model.envVar +import eu.glasskube.kubernetes.api.model.metadata +import eu.glasskube.kubernetes.api.model.namespace +import eu.glasskube.kubernetes.api.model.secretKeyRef +import eu.glasskube.kubernetes.api.model.securityContext +import eu.glasskube.kubernetes.api.model.spec +import eu.glasskube.kubernetes.api.model.volume +import eu.glasskube.kubernetes.api.model.volumeMount +import eu.glasskube.kubernetes.api.model.volumeMounts +import eu.glasskube.operator.apps.common.backup.BackupSpec +import eu.glasskube.operator.apps.common.backup.ResourceWithBackups +import eu.glasskube.operator.apps.common.cloudstorage.CloudStorageSpec +import eu.glasskube.operator.apps.common.cloudstorage.ResourceWithCloudStorage +import eu.glasskube.operator.processing.CompositeAndCondition +import eu.glasskube.utils.logger +import eu.glasskube.utils.parseGolangDuration +import io.fabric8.kubernetes.api.model.Container +import io.fabric8.kubernetes.api.model.HasMetadata +import io.fabric8.kubernetes.api.model.batch.v1.CronJob +import io.javaoperatorsdk.operator.api.reconciler.Context +import io.javaoperatorsdk.operator.api.reconciler.ResourceIDMatcherDiscriminator +import io.javaoperatorsdk.operator.processing.dependent.kubernetes.CRUDKubernetesDependentResource +import io.javaoperatorsdk.operator.processing.event.ResourceID + +abstract class DependentCloudStorageBackupCronJob

: + CRUDKubernetesDependentResource(CronJob::class.java) + where P : HasMetadata, P : ResourceWithBackups, P : ResourceWithCloudStorage { + + abstract class ReconcilePrecondition

: + CompositeAndCondition(CloudStorageNotNullCondition(), BackupSpecNotNullCondition()) + where P : HasMetadata, P : ResourceWithCloudStorage, P : ResourceWithBackups + + abstract class Discriminator

: + ResourceIDMatcherDiscriminator({ ResourceID(it.backupResourceName, it.namespace) }) + where P : HasMetadata, P : ResourceWithCloudStorage + + override fun desired(primary: P, context: Context

) = CronJob().apply { + metadata { + name(primary.backupResourceName) + namespace(primary.namespace) + labels(primary.backupResourceLabels) + } + + spec { + val source = (primary as ResourceWithCloudStorage).getSpec().cloudStorage!! + val backups = (primary as ResourceWithBackups).getSpec().requireBackups() + + schedule = backups.schedule + concurrencyPolicy = "Forbid" + jobTemplate { + spec { + template { + spec { + restartPolicy = "Never" + volumes = listOf( + volume(VOLUME_NAME) { emptyDir() } + ) + securityContext { + runAsNonRoot = true + runAsUser = 1009 + runAsGroup = 1009 + fsGroup = 1009 + } + initContainers = listOf( + configInitContainer(SOURCE_REMOTE_NAME, source), + configInitContainer(DESTINATION_REMOTE_NAME, backups.s3) + ) + containers = listOf( + container { + name = "cloud-storage-backup" + image = IMAGE + env { + envVar("SRC_REMOTE", SOURCE_REMOTE_NAME) + envVar("SRC_BUCKET", source.bucket) + envVar("DST_REMOTE", DESTINATION_REMOTE_NAME) + envVar("DST_BUCKET", backups.s3.bucket) + envVar("BACKUP_TTL", backups.ttlInSeconds.toString()) + } + defaultVolumeMounts() + } + ) + } + } + } + } + } + } + + private fun configInitContainer(remoteName: String, spec: CloudStorageSpec) = container { + name = "$remoteName-config" + image = IMAGE + env { + envVar("ACCESS_KEY") { + secretKeyRef(spec.accessKeySecret.name, spec.accessKeySecret.key) + } + envVar("SECRET_KEY") { + secretKeyRef(spec.secretKeySecret.name, spec.secretKeySecret.key) + } + } + defaultVolumeMounts() + command = listOf("rclone") + args = mutableListOf("config", "create", remoteName, "s3").also { argList -> + argList += when (val endpoint = spec.endpoint) { + null -> listOf("provider", "AWS") + else -> listOf("provider", "Other", "endpoint", endpoint) + } + spec.region?.let { argList += listOf("region", it) } + argList += listOf("access_key_id", "$(ACCESS_KEY)", "secret_access_key", "$(SECRET_KEY)") + } + } + + private fun Container.defaultVolumeMounts() { + volumeMounts { + volumeMount { + name = VOLUME_NAME + mountPath = VOLUME_PATH + } + } + } + + private val BackupSpec.ttlInSeconds: Long + get() = parseGolangDuration(ttl) + .also { + if (it.nano > 0) { + log.warn("fractional seconds were found in backup TTL $ttl. This is unsupported and will be ignored.") + } + if (it.isNegative) { + log.warn("negative duration will be inverted.") + } + } + .abs() + .seconds + + companion object { + private const val VOLUME_NAME = "config" + private const val VOLUME_PATH = "/config/rclone" + private const val IMAGE = "ghcr.io/glasskube/cloud-storage-backup:v0.2.1" + private const val SOURCE_REMOTE_NAME = "src" + private const val DESTINATION_REMOTE_NAME = "dst" + private val log = logger() + } +} diff --git a/operator/src/main/kotlin/eu/glasskube/operator/processing/CompositeAndCondition.kt b/operator/src/main/kotlin/eu/glasskube/operator/processing/CompositeAndCondition.kt new file mode 100644 index 00000000..338eb2a8 --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/operator/processing/CompositeAndCondition.kt @@ -0,0 +1,13 @@ +package eu.glasskube.operator.processing + +import io.fabric8.kubernetes.api.model.HasMetadata +import io.javaoperatorsdk.operator.api.reconciler.Context +import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource +import io.javaoperatorsdk.operator.processing.dependent.workflow.Condition + +abstract class CompositeAndCondition( + private vararg val delegateConditions: Condition +) : Condition { + override fun isMet(dependentResource: DependentResource, primary: P, context: Context

) = + delegateConditions.all { it.isMet(dependentResource, primary, context) } +} diff --git a/operator/src/main/kotlin/eu/glasskube/utils/Duration.kt b/operator/src/main/kotlin/eu/glasskube/utils/Duration.kt new file mode 100644 index 00000000..75963fd7 --- /dev/null +++ b/operator/src/main/kotlin/eu/glasskube/utils/Duration.kt @@ -0,0 +1,25 @@ +package eu.glasskube.utils + +import java.time.Duration +import java.time.temporal.ChronoUnit + +private val durationPattern = Regex("""^-?(\d+(ns|us|ms|s|m|h))+$""") +private val durationSegmentPattern = Regex("""(\d+)(\D+)""") + +fun parseGolangDuration(value: String): Duration { + require(value.matches(durationPattern)) { "invalid duration \"$value\"" } + return durationSegmentPattern.findAll(value) + .fold(Duration.ZERO) { duration, match -> + val amount = match.groupValues[1].toLong() + when (val unit = match.groupValues[2]) { + "ns" -> duration.plusNanos(amount) + "us" -> duration.plus(amount, ChronoUnit.MICROS) + "ms" -> duration.plusMillis(amount) + "s" -> duration.plusSeconds(amount) + "m" -> duration.plusMinutes(amount) + "h" -> duration.plusHours(amount) + else -> throw IllegalArgumentException("unknown unit \"$unit\" in duration \"$value\"") + } + } + .let { if (value[0] == '-') it.negated() else it } +} diff --git a/operator/src/test/kotlin/eu/glasskube/utils/DurationKtTest.kt b/operator/src/test/kotlin/eu/glasskube/utils/DurationKtTest.kt new file mode 100644 index 00000000..632e58eb --- /dev/null +++ b/operator/src/test/kotlin/eu/glasskube/utils/DurationKtTest.kt @@ -0,0 +1,23 @@ +package eu.glasskube.utils + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.time.Duration + +class DurationKtTest { + @Test + fun `parseGolangDuration when called with 'm1s' should throw IllegalArgumentException`() { + assertThrows { parseGolangDuration("m1s") } + } + + @Test + fun `parseGolangDuration when called with '1d1h' should throw IllegalArgumentException`() { + assertThrows { parseGolangDuration("1d1h") } + } + + @Test + fun `parseGolangDuration when called with '1h1m1s1ms1us1ns' should parse`() { + assertEquals(Duration.parse("PT1H1M1.001001001S"), parseGolangDuration("1h1m1s1ms1us1ns")) + } +}