Skip to content

Commit

Permalink
Add job to handle allowed_content_checksums update
Browse files Browse the repository at this point in the history
closes #1060
  • Loading branch information
git-hyagi committed Aug 30, 2023
1 parent 4cfeac1 commit 51ed148
Show file tree
Hide file tree
Showing 19 changed files with 329 additions and 15 deletions.
1 change: 1 addition & 0 deletions CHANGES/1060.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added a job to handle `ALLOWED_CONTENT_CHECKSUMS` modifications.
10 changes: 10 additions & 0 deletions apis/repo-manager.pulpproject.org/v1beta2/pulp_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,12 @@ type PulpSpec struct {
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:hidden"}
PulpSecretKey string `json:"pulp_secret_key,omitempty"`

// List of allowed checksum algorithms used to verify repository's integrity.
// Valid options: ["md5","sha1","sha256","sha512"].
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"}
AllowedContentChecksums []string `json:"allowed_content_checksums,omitempty"`

// Protocol used by pulp-web service when ingress_type==loadbalancer
// +kubebuilder:validation:Enum:=http;https
// +kubebuilder:validation:Optional
Expand Down Expand Up @@ -881,6 +887,10 @@ type PulpStatus struct {
TelemetryEnabled bool `json:"telemetry_enabled,omitempty"`
// Name of the Secret to provide Django cryptographic signing.
PulpSecretKey string `json:"pulp_secret_key,omitempty"`
// List of allowed checksum algorithms used to verify repository's integrity.
AllowedContentChecksums string `json:"allowed_content_checksums,omitempty"`
// Controller status to keep tracking of deployment updates
LastDeploymentUpdate string `json:"last_deployment_update,omitempty"`
}

// Pulp is the Schema for the pulps API
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 7 additions & 1 deletion bundle/manifests/pulp-operator.clusterserviceversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ metadata:
capabilities: Full Lifecycle
categories: Integration & Delivery
containerImage: quay.io/pulp/pulp-operator:devel
createdAt: "2023-08-24T11:17:51Z"
createdAt: "2023-08-29T13:45:26Z"
description: Pulp is a platform for managing repositories of software packages
and making them available to a large number of consumers.
operators.operatorframework.io/builder: operator-sdk-v1.29.0
Expand Down Expand Up @@ -301,6 +301,12 @@ spec:
x-descriptors:
- urn:alm:descriptor:io.kubernetes:Secret
- urn:alm:descriptor:com.tectonic.ui:advanced
- description: 'List of allowed checksum algorithms used to verify repository''s
integrity. Valid options: ["md5","sha1","sha256","sha512"].'
displayName: Allowed Content Checksums
path: allowed_content_checksums
x-descriptors:
- urn:alm:descriptor:com.tectonic.ui:advanced
- description: Api defines desired state of pulpcore-api resources
displayName: Api
path: api
Expand Down
13 changes: 13 additions & 0 deletions bundle/manifests/repo-manager.pulpproject.org_pulps.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,12 @@ spec:
description: 'Secret where the administrator password can be found.
Default: <operator''s name> + "-admin-password"'
type: string
allowed_content_checksums:
description: 'List of allowed checksum algorithms used to verify repository''s
integrity. Valid options: ["md5","sha1","sha256","sha512"].'
items:
type: string
type: array
api:
default:
replicas: 1
Expand Down Expand Up @@ -8627,6 +8633,10 @@ spec:
admin_password_secret:
description: Secret where the administrator password can be found
type: string
allowed_content_checksums:
description: List of allowed checksum algorithms used to verify repository's
integrity.
type: string
conditions:
items:
description: "Condition contains details for one aspect of the current
Expand Down Expand Up @@ -8718,6 +8728,9 @@ spec:
ingress_type:
description: The ingress type to use to reach the deployed instance
type: string
last_deployment_update:
description: Controller status to keep tracking of deployment updates
type: string
object_storage_azure_secret:
description: The secret for Azure compliant object storage configuration.
type: string
Expand Down
13 changes: 13 additions & 0 deletions config/crd/bases/repo-manager.pulpproject.org_pulps.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ spec:
description: 'Secret where the administrator password can be found.
Default: <operator''s name> + "-admin-password"'
type: string
allowed_content_checksums:
description: 'List of allowed checksum algorithms used to verify repository''s
integrity. Valid options: ["md5","sha1","sha256","sha512"].'
items:
type: string
type: array
api:
default:
replicas: 1
Expand Down Expand Up @@ -8628,6 +8634,10 @@ spec:
admin_password_secret:
description: Secret where the administrator password can be found
type: string
allowed_content_checksums:
description: List of allowed checksum algorithms used to verify repository's
integrity.
type: string
conditions:
items:
description: "Condition contains details for one aspect of the current
Expand Down Expand Up @@ -8719,6 +8729,9 @@ spec:
ingress_type:
description: The ingress type to use to reach the deployed instance
type: string
last_deployment_update:
description: Controller status to keep tracking of deployment updates
type: string
object_storage_azure_secret:
description: The secret for Azure compliant object storage configuration.
type: string
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,12 @@ spec:
x-descriptors:
- urn:alm:descriptor:io.kubernetes:Secret
- urn:alm:descriptor:com.tectonic.ui:advanced
- description: 'List of allowed checksum algorithms used to verify repository''s
integrity. Valid options: ["md5","sha1","sha256","sha512"].'
displayName: Allowed Content Checksums
path: allowed_content_checksums
x-descriptors:
- urn:alm:descriptor:com.tectonic.ui:advanced
- description: Api defines desired state of pulpcore-api resources
displayName: Api
path: api
Expand Down
4 changes: 4 additions & 0 deletions controllers/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -949,6 +949,10 @@ func (d *CommonDeployment) setAnnotations(pulp repomanagerpulpprojectorgv1beta2.
"kubectl.kubernetes.io/default-container": strings.ToLower(pulpcoreType),
}

if pulp.Status.LastDeploymentUpdate != "" {
d.podAnnotations["repo-manager.pulpproject.org/restartedAt"] = pulp.Status.LastDeploymentUpdate
}

d.deploymentAnnotations = map[string]string{
"email": "[email protected]",
"ignore-check.kube-linter.io/no-node-affinity": "Do not check node affinity",
Expand Down
3 changes: 3 additions & 0 deletions controllers/repo_manager/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ PulpSpec defines the desired state of Pulp
| admin_password_job | Job to reset pulp admin password | [PulpJob](#pulpjob) | false |
| migration_job | Job to run django migrations | [PulpJob](#pulpjob) | false |
| pulp_secret_key | Name of the Secret to provide Django cryptographic signing. Default: \"pulp-secret-key\" | string | false |
| allowed_content_checksums | List of allowed checksum algorithms used to verify repository's integrity. Valid options: [\"md5\",\"sha1\",\"sha256\",\"sha512\"]. | []string | false |
| loadbalancer_protocol | Protocol used by pulp-web service when ingress_type==loadbalancer | string | false |
| loadbalancer_port | Port exposed by pulp-web service when ingress_type==loadbalancer | int32 | false |
| telemetry | Telemetry defines the OpenTelemetry configuration | [Telemetry](#telemetry) | false |
Expand All @@ -239,6 +240,8 @@ PulpStatus defines the observed state of Pulp
| external_cache_secret | Name of the secret with the parameters to connect to an external Redis cluster | string | false |
| telemetry_enabled | Pulp metrics collection enabled | bool | false |
| pulp_secret_key | Name of the Secret to provide Django cryptographic signing. | string | false |
| allowed_content_checksums | List of allowed checksum algorithms used to verify repository's integrity. | string | false |
| last_deployment_update | Controller status to keep tracking of deployment updates | string | false |

[Back to Custom Resources](#custom-resources)

Expand Down
10 changes: 2 additions & 8 deletions controllers/repo_manager/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,14 +176,8 @@ func (r *RepoManagerReconciler) pulpApiController(ctx context.Context, pulp *rep
r.Get(ctx, types.NamespacedName{Name: pulp.Name + "-server", Namespace: pulp.Namespace}, serverSecret)
expectedServerSecret := pulpServerSecret(funcResources)
if requeue, err := controllers.ReconcileObject(funcResources, expectedServerSecret, serverSecret, conditionType, controllers.PulpSecret{}); err != nil || requeue {
log.Info("Reprovisioning pulpcore-api pods to get the new settings ...")
// when requeue==true it means the secret changed so we need to redeploy api and content pods to get the new settings.py
r.restartPods(pulp, apiDeployment)
contentDeployment := &appsv1.Deployment{}
r.Get(ctx, types.NamespacedName{Name: pulp.Name + "-content", Namespace: pulp.Namespace}, contentDeployment)
log.Info("Reprovisioning pulpcore-content pods to get the new settings ...")
r.restartPods(pulp, contentDeployment)

// restart pulpcore pods if the secret has changed
r.restartPulpCorePods(pulp)
return ctrl.Result{Requeue: requeue}, err
}

Expand Down
3 changes: 3 additions & 0 deletions controllers/repo_manager/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,9 @@ func (r *RepoManagerReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// create the job to reset pulp admin password in case admin_password_secret has changed
r.updateAdminPasswordJob(ctx, pulp)

// create the job to update the allowed_content_checksums
r.updateContentChecksumsJob(ctx, pulp)

// if this is the first reconciliation loop (.status.ingress_type == "") OR
// if there is no update in ingressType field
if len(pulp.Status.IngressType) == 0 || pulp.Status.IngressType == pulp.Spec.IngressType {
Expand Down
124 changes: 119 additions & 5 deletions controllers/repo_manager/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,17 @@ package repo_manager

import (
"context"
"encoding/json"
"reflect"
"time"

repomanagerpulpprojectorgv1beta2 "github.com/pulp/pulp-operator/apis/repo-manager.pulpproject.org/v1beta2"
"github.com/pulp/pulp-operator/controllers"
jobs "k8s.io/api/batch/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
Expand Down Expand Up @@ -56,13 +60,13 @@ func (r *RepoManagerReconciler) updateAdminPasswordJob(ctx context.Context, pulp
jobTTL := int32(3600)

// job definition
job := &jobs.Job{
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "reset-admin-password-",
Namespace: pulp.Namespace,
Labels: labels,
},
Spec: jobs.JobSpec{
Spec: batchv1.JobSpec{
BackoffLimit: &backOffLimit,
TTLSecondsAfterFinished: &jobTTL,
Template: corev1.PodTemplateSpec{
Expand Down Expand Up @@ -214,13 +218,13 @@ func (r *RepoManagerReconciler) migrationJob(ctx context.Context, pulp *repomana
jobTTL := int32(3600)

// job definition
job := &jobs.Job{
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "pulpcore-migration-",
Namespace: pulp.Namespace,
Labels: labels,
},
Spec: jobs.JobSpec{
Spec: batchv1.JobSpec{
BackoffLimit: &backOffLimit,
TTLSecondsAfterFinished: &jobTTL,
Template: corev1.PodTemplateSpec{
Expand Down Expand Up @@ -275,3 +279,113 @@ func jobLabels(pulp repomanagerpulpprojectorgv1beta2.Pulp) map[string]string {
"pulp_cr": pulp.Name,
}
}

// updateContentChecksumsJob creates a k8s Job to update the list of allowed content checksums
func (r *RepoManagerReconciler) updateContentChecksumsJob(ctx context.Context, pulp *repomanagerpulpprojectorgv1beta2.Pulp) {
log := r.RawLogger

if !contentChecksumsModified(pulp) {
return
}

labels := jobLabels(*pulp)
labels["app.kubernetes.io/component"] = "allowed-content-checksums"
containers := []corev1.Container{contentChecksumsContainer(pulp)}
volumes := pulpcoreVolumes(pulp, "")
backOffLimit := int32(2)
jobTTL := int32(60)

// job definition
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "update-content-checksums-",
Namespace: pulp.Namespace,
Labels: labels,
},
Spec: batchv1.JobSpec{
BackoffLimit: &backOffLimit,
TTLSecondsAfterFinished: &jobTTL,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
RestartPolicy: "Never",
Containers: containers,
Volumes: volumes,
ServiceAccountName: pulp.Name,
},
},
},
}

// create the Job
log.Info("Creating a new update content checksums Job")
if err := r.Create(ctx, job); err != nil {
log.Error(err, "Failed to create update content checksums Job!")
}

// update .status
settings, _ := json.Marshal(pulp.Spec.AllowedContentChecksums)
pulp.Status.AllowedContentChecksums = string(settings)
r.Status().Update(ctx, pulp)
}

// contentChecksumsContainer defines the container spec for the updateContentChecksums Job
func contentChecksumsContainer(pulp *repomanagerpulpprojectorgv1beta2.Pulp) corev1.Container {
// env vars
envVars := controllers.GetPostgresEnvVars(*pulp)

// volume mounts
volumeMounts := pulpcoreVolumeMounts(pulp)

// resource requirements
resources := pulp.Spec.MigrationJob.PulpContainer.ResourceRequirements

return corev1.Container{
Name: "update-checksum",
Image: pulp.Spec.Image + ":" + pulp.Spec.ImageVersion,
ImagePullPolicy: "Always",
Env: envVars,
Command: []string{"/bin/sh"},
Args: []string{
"-c",
`/usr/bin/wait_on_postgres.py
/usr/bin/wait_on_database_migrations.sh
pulpcore-manager handle-artifact-checksums`,
},
Resources: resources,
VolumeMounts: volumeMounts,
}
}

// contentChecksumsModified returns true if
// .status.AllowedContentChecksums != pulp.Spec.AllowedContentChecksums
func contentChecksumsModified(pulp *repomanagerpulpprojectorgv1beta2.Pulp) bool {
var statusAllowedChecksum []string
json.Unmarshal([]byte(pulp.Status.AllowedContentChecksums), &statusAllowedChecksum)
return !reflect.DeepEqual(pulp.Spec.AllowedContentChecksums, statusAllowedChecksum)
}

// waitJobFinishes wait until content-checksums job completes
func waitJobFinishes(resources RepoManagerReconciler, pulp *repomanagerpulpprojectorgv1beta2.Pulp, timeout time.Duration) {
labels := map[string]string{
"app.kubernetes.io/component": "allowed-content-checksums",
}
listOpts := []client.ListOption{
client.InNamespace(pulp.Namespace),
client.MatchingLabels(labels),
}

TIMEOUT:
for i := 0; i < int(timeout.Seconds()); i++ {
jobList := &batchv1.JobList{}
if err := resources.Client.List(context.TODO(), jobList, listOpts...); err != nil {
resources.RawLogger.Error(err, "Failed to find content-checksum Jobs")
return
}
for _, p := range jobList.Items {
if p.Status.CompletionTime != nil {
break TIMEOUT
}
}
time.Sleep(time.Second)
}
}
Loading

0 comments on commit 51ed148

Please sign in to comment.