diff --git a/crds/crd.yaml b/crds/crd.yaml new file mode 100644 index 000000000..511d600fa --- /dev/null +++ b/crds/crd.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: deployments.stable.example.com +spec: + group: stable.example.com + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + thiny: + type: string + scope: Cluster + names: + plural: deployments + singular: deployment + kind: Deployment diff --git a/crds/create_crds.rb b/crds/create_crds.rb new file mode 100644 index 000000000..8ace63a84 --- /dev/null +++ b/crds/create_crds.rb @@ -0,0 +1,41 @@ +require "yaml" + +1.times do |nr| + crd_yaml = { + "apiVersion" => "apiextensions.k8s.io/v1", + "kind" => "CustomResourceDefinition", + "metadata" => { + "name" => "deploymentss.stable.example.com", + }, + "spec" => { + "group" => "stable.example.com", + "versions" => [ + { + "name" => "v1", + "served" => true, + "storage" => true, + "schema" => { + "openAPIV3Schema" => { + "type" => "object", + "properties" => { + "thiny" => { + "type" => "string", + } + } + }, + }, + }, + ], + "scope" => "Cluster", + "names" => { + "plural" => "deployments", + "singular" => "deployment", + "kind" => "Deployment", + }, + }, + } + + yaml = YAML.dump(crd_yaml) + + File.write(File.expand_path("crd.yaml", __dir__), yaml) +end diff --git a/krane-manifest.yaml b/krane-manifest.yaml new file mode 100644 index 000000000..347b58aa3 --- /dev/null +++ b/krane-manifest.yaml @@ -0,0 +1,16 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: nginx-deployment +spec: + selector: + matchLabels: + deploy: example + template: + metadata: + labels: + deploy: example + spec: + containers: + - name: nginx + image: nginx diff --git a/lib/krane/cluster_resource_discovery.rb b/lib/krane/cluster_resource_discovery.rb index 00c2662d5..abc312723 100644 --- a/lib/krane/cluster_resource_discovery.rb +++ b/lib/krane/cluster_resource_discovery.rb @@ -94,6 +94,7 @@ def resource_hash(path, namespaced, blob) path_regex = %r{(/apis?/)(?[^/]*)/?(?v.+)} match = path.match(path_regex) { + "name" => blob["name"], "verbs" => blob["verbs"], "kind" => blob["kind"], "apigroup" => match[:group], diff --git a/lib/krane/common.rb b/lib/krane/common.rb index 44901e8ff..aa318d1a0 100644 --- a/lib/krane/common.rb +++ b/lib/krane/common.rb @@ -21,4 +21,12 @@ module Krane MIN_KUBE_VERSION = '1.15.0' + + def self.group_from_api_version(input) + input.include?("/") ? input.split("/").first : "" + end + + def self.group_kind(group, kind) + "#{kind}.#{group}" + end end diff --git a/lib/krane/deploy_task.rb b/lib/krane/deploy_task.rb index 3b95e476a..9f17caf8f 100644 --- a/lib/krane/deploy_task.rb +++ b/lib/krane/deploy_task.rb @@ -238,9 +238,11 @@ def secrets_from_ejson def discover_resources @logger.info("Discovering resources:") resources = [] - crds_by_kind = cluster_resource_discoverer.crds.group_by(&:kind) + crds_by_kind = cluster_resource_discoverer.crds.group_by(&:group_kind) @template_sets.with_resource_definitions(current_sha: @current_sha, bindings: @bindings) do |r_def| - crd = crds_by_kind[r_def["kind"]]&.first + group = Krane.group_from_api_version(r_def["apiVersion"]) + + crd = crds_by_kind[Krane.group_kind(group, r_def["kind"])]&.first r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger, definition: r_def, statsd_tags: @namespace_tags, crd: crd, global_names: @task_config.global_kinds) resources << r diff --git a/lib/krane/kubectl.rb b/lib/krane/kubectl.rb index 679488136..aa1909a14 100644 --- a/lib/krane/kubectl.rb +++ b/lib/krane/kubectl.rb @@ -9,7 +9,7 @@ class Kubectl empty: /\A\z/, context_deadline: /context deadline exceeded/, } - DEFAULT_TIMEOUT = 15 + DEFAULT_TIMEOUT = 30 MAX_RETRY_DELAY = 16 SERVER_DRY_RUN_MIN_VERSION = "1.13" @@ -35,6 +35,7 @@ def run(*args, log_failure: nil, use_context: true, use_namespace: true, output: (1..attempts).to_a.each do |current_attempt| logger.debug("Running command (attempt #{current_attempt}): #{cmd.join(' ')}") + cmd_string = cmd.join(' ') env = { 'KUBECONFIG' => kubeconfig } out, err, st = Open3.capture3(env, *cmd) @@ -43,9 +44,13 @@ def run(*args, log_failure: nil, use_context: true, use_namespace: true, output: out = out.dup.force_encoding(Encoding::UTF_8) end + if cmd_string.include?("kubectl get Deploy") + # logger.debug(out) + end + if logger.debug? && !output_is_sensitive # don't do the gsub unless we're going to print this - logger.debug("Kubectl out: " + out.gsub(/\s+/, ' ')) + # logger.debug("Kubectl out: " + out.gsub(/\s+/, ' ')) end break if st.success? diff --git a/lib/krane/kubernetes_resource.rb b/lib/krane/kubernetes_resource.rb index 289876454..c5d188f60 100644 --- a/lib/krane/kubernetes_resource.rb +++ b/lib/krane/kubernetes_resource.rb @@ -46,15 +46,17 @@ class KubernetesResource class << self def build(namespace: nil, context:, definition:, logger:, statsd_tags:, crd: nil, global_names: []) validate_definition_essentials(definition) + + group = Krane.group_from_api_version(definition["apiVersion"]) opts = { namespace: namespace, context: context, definition: definition, logger: logger, statsd_tags: statsd_tags } - if (klass = class_for_kind(definition["kind"])) + if (klass = class_for_group_kind(group, definition["kind"])) return klass.new(**opts) end if crd CustomResource.new(crd: crd, **opts) else - type = definition["kind"] + type = Krane.group_kind(definition["group"], definition["kind"]) inst = new(**opts) inst.type = type inst.global = global_names.map(&:downcase).include?(type.downcase) @@ -70,6 +72,26 @@ def class_for_kind(kind) nil end + def class_for_group_kind(group, kind) + if Krane.const_defined?(kind, false) + const = Krane.const_get(kind, false) + + if const_defined?("GROUPS") + groups = const.const_get("GROUPS") + + if groups.include?(group) + const + else + nil + end + else + const + end + end + rescue NameError + nil + end + def timeout self::TIMEOUT end @@ -78,6 +100,11 @@ def kind name.demodulize end + def group + const_get("GROUPS").first + # self.class.GROUPS.first + end + private def validate_definition_essentials(definition) @@ -218,7 +245,7 @@ def status end def type - @type || self.class.kind + @type || Krane.group_kind(self.class.group, self.class.kind) end def group diff --git a/lib/krane/kubernetes_resource/config_map.rb b/lib/krane/kubernetes_resource/config_map.rb index 364656922..0e9195dfb 100644 --- a/lib/krane/kubernetes_resource/config_map.rb +++ b/lib/krane/kubernetes_resource/config_map.rb @@ -2,6 +2,7 @@ module Krane class ConfigMap < KubernetesResource TIMEOUT = 30.seconds + GROUPS = [""] def deploy_succeeded? exists? diff --git a/lib/krane/kubernetes_resource/cron_job.rb b/lib/krane/kubernetes_resource/cron_job.rb index db59dd090..994bcef75 100644 --- a/lib/krane/kubernetes_resource/cron_job.rb +++ b/lib/krane/kubernetes_resource/cron_job.rb @@ -2,6 +2,7 @@ module Krane class CronJob < KubernetesResource TIMEOUT = 30.seconds + GROUPS = ["batch"] def deploy_succeeded? exists? diff --git a/lib/krane/kubernetes_resource/custom_resource.rb b/lib/krane/kubernetes_resource/custom_resource.rb index 453ce3f8d..90240111c 100644 --- a/lib/krane/kubernetes_resource/custom_resource.rb +++ b/lib/krane/kubernetes_resource/custom_resource.rb @@ -59,7 +59,7 @@ def status end def type - kind + "#{group}.#{kind}" end def validate_definition(*, **) @@ -80,6 +80,10 @@ def kind @definition["kind"] end + def group + definition["apiVersion"].include?("/") ? definition["apiVersion"].split("/").first : "" + end + def rollout_conditions @crd.rollout_conditions end diff --git a/lib/krane/kubernetes_resource/custom_resource_definition.rb b/lib/krane/kubernetes_resource/custom_resource_definition.rb index f07d84a27..6b178503c 100644 --- a/lib/krane/kubernetes_resource/custom_resource_definition.rb +++ b/lib/krane/kubernetes_resource/custom_resource_definition.rb @@ -52,6 +52,10 @@ def group @definition.dig("spec", "group") end + def group_kind + ::Krane.group_kind(group, kind) + end + def prunable? prunable = krane_annotation_value("prunable") prunable == "true" diff --git a/lib/krane/kubernetes_resource/daemon_set.rb b/lib/krane/kubernetes_resource/daemon_set.rb index 7c1c367f7..80f09e307 100644 --- a/lib/krane/kubernetes_resource/daemon_set.rb +++ b/lib/krane/kubernetes_resource/daemon_set.rb @@ -4,6 +4,7 @@ module Krane class DaemonSet < PodSetBase TIMEOUT = 5.minutes SYNC_DEPENDENCIES = %w(Pod) + GROUPS = ["apps"] attr_reader :pods def sync(cache) diff --git a/lib/krane/kubernetes_resource/deployment.rb b/lib/krane/kubernetes_resource/deployment.rb index 7c4f81d7a..38495ead2 100644 --- a/lib/krane/kubernetes_resource/deployment.rb +++ b/lib/krane/kubernetes_resource/deployment.rb @@ -8,6 +8,7 @@ class Deployment < KubernetesResource REQUIRED_ROLLOUT_ANNOTATION = "required-rollout" REQUIRED_ROLLOUT_TYPES = %w(maxUnavailable full none).freeze DEFAULT_REQUIRED_ROLLOUT = 'full' + GROUPS = ["apps"] def sync(cache) super diff --git a/lib/krane/kubernetes_resource/horizontal_pod_autoscaler.rb b/lib/krane/kubernetes_resource/horizontal_pod_autoscaler.rb index f37e8308e..aacd77fbe 100644 --- a/lib/krane/kubernetes_resource/horizontal_pod_autoscaler.rb +++ b/lib/krane/kubernetes_resource/horizontal_pod_autoscaler.rb @@ -3,6 +3,7 @@ module Krane class HorizontalPodAutoscaler < KubernetesResource TIMEOUT = 3.minutes RECOVERABLE_CONDITION_PREFIX = "FailedGet" + GROUPS = ["autoscaling"] def deploy_succeeded? scaling_active_condition["status"] == "True" || scaling_disabled? diff --git a/lib/krane/kubernetes_resource/ingress.rb b/lib/krane/kubernetes_resource/ingress.rb index e34f4231c..e4e634129 100644 --- a/lib/krane/kubernetes_resource/ingress.rb +++ b/lib/krane/kubernetes_resource/ingress.rb @@ -2,6 +2,7 @@ module Krane class Ingress < KubernetesResource TIMEOUT = 30.seconds + GROUPS = ["networking.k8s.io"] def status exists? ? "Created" : "Not Found" diff --git a/lib/krane/kubernetes_resource/job.rb b/lib/krane/kubernetes_resource/job.rb index af66c01de..7f8b77cd5 100644 --- a/lib/krane/kubernetes_resource/job.rb +++ b/lib/krane/kubernetes_resource/job.rb @@ -2,6 +2,7 @@ module Krane class Job < KubernetesResource TIMEOUT = 10.minutes + GROUP = ["batch"] def deploy_succeeded? # Don't block deploys for long running jobs, diff --git a/lib/krane/kubernetes_resource/mutating_webhook_configuration.rb b/lib/krane/kubernetes_resource/mutating_webhook_configuration.rb index 8d830fdf3..e277f3621 100644 --- a/lib/krane/kubernetes_resource/mutating_webhook_configuration.rb +++ b/lib/krane/kubernetes_resource/mutating_webhook_configuration.rb @@ -3,6 +3,7 @@ module Krane class MutatingWebhookConfiguration < KubernetesResource GLOBAL = true + GROUPS= ["admissionregistration.k8s.io"] class Webhook EQUIVALENT = 'Equivalent' diff --git a/lib/krane/kubernetes_resource/network_policy.rb b/lib/krane/kubernetes_resource/network_policy.rb index 8cbf622fa..af3f6bc1f 100644 --- a/lib/krane/kubernetes_resource/network_policy.rb +++ b/lib/krane/kubernetes_resource/network_policy.rb @@ -2,6 +2,7 @@ module Krane class NetworkPolicy < KubernetesResource TIMEOUT = 30.seconds + GROUPS= ["networking.k8s.io"] def status exists? ? "Created" : "Not Found" diff --git a/lib/krane/kubernetes_resource/persistent_volume_claim.rb b/lib/krane/kubernetes_resource/persistent_volume_claim.rb index 6a2824e15..403641648 100644 --- a/lib/krane/kubernetes_resource/persistent_volume_claim.rb +++ b/lib/krane/kubernetes_resource/persistent_volume_claim.rb @@ -2,6 +2,7 @@ module Krane class PersistentVolumeClaim < KubernetesResource TIMEOUT = 5.minutes + GROUPS = [""] def sync(cache) super diff --git a/lib/krane/kubernetes_resource/pod.rb b/lib/krane/kubernetes_resource/pod.rb index 8cddbaec7..26d2e4aa7 100644 --- a/lib/krane/kubernetes_resource/pod.rb +++ b/lib/krane/kubernetes_resource/pod.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true module Krane class Pod < KubernetesResource + GROUPS = [""] TIMEOUT = 10.minutes FAILED_PHASE_NAME = "Failed" diff --git a/lib/krane/kubernetes_resource/pod_disruption_budget.rb b/lib/krane/kubernetes_resource/pod_disruption_budget.rb index 5142b6537..7427c51e3 100644 --- a/lib/krane/kubernetes_resource/pod_disruption_budget.rb +++ b/lib/krane/kubernetes_resource/pod_disruption_budget.rb @@ -2,6 +2,7 @@ module Krane class PodDisruptionBudget < KubernetesResource TIMEOUT = 10.seconds + GROUPS = ["policy"] def status exists? ? "Available" : "Not Found" diff --git a/lib/krane/kubernetes_resource/pod_set_base.rb b/lib/krane/kubernetes_resource/pod_set_base.rb index 4a4843402..d6849e9d4 100644 --- a/lib/krane/kubernetes_resource/pod_set_base.rb +++ b/lib/krane/kubernetes_resource/pod_set_base.rb @@ -3,6 +3,7 @@ module Krane class PodSetBase < KubernetesResource + GROUPS = ["?"] # TODO def failure_message pods.map(&:failure_message).compact.uniq.join("\n") end diff --git a/lib/krane/kubernetes_resource/pod_template.rb b/lib/krane/kubernetes_resource/pod_template.rb index 64739ce82..31e7b0719 100644 --- a/lib/krane/kubernetes_resource/pod_template.rb +++ b/lib/krane/kubernetes_resource/pod_template.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true module Krane class PodTemplate < KubernetesResource + GROUPS = "" def status exists? ? "Available" : "Not Found" end diff --git a/lib/krane/kubernetes_resource/replica_set.rb b/lib/krane/kubernetes_resource/replica_set.rb index bb585bcef..a3f761b22 100644 --- a/lib/krane/kubernetes_resource/replica_set.rb +++ b/lib/krane/kubernetes_resource/replica_set.rb @@ -3,6 +3,7 @@ module Krane class ReplicaSet < PodSetBase + GROUPS = ["apps"] TIMEOUT = 5.minutes SYNC_DEPENDENCIES = %w(Pod) attr_reader :pods diff --git a/lib/krane/kubernetes_resource/resource_quota.rb b/lib/krane/kubernetes_resource/resource_quota.rb index 8a1032666..c41f2540c 100644 --- a/lib/krane/kubernetes_resource/resource_quota.rb +++ b/lib/krane/kubernetes_resource/resource_quota.rb @@ -2,6 +2,7 @@ module Krane class ResourceQuota < KubernetesResource TIMEOUT = 30.seconds + GROUPS = [""] def status exists? ? "In effect" : "Not Found" diff --git a/lib/krane/kubernetes_resource/role.rb b/lib/krane/kubernetes_resource/role.rb index 58588739a..892b46826 100644 --- a/lib/krane/kubernetes_resource/role.rb +++ b/lib/krane/kubernetes_resource/role.rb @@ -2,6 +2,7 @@ module Krane class Role < KubernetesResource TIMEOUT = 30.seconds + GROUPS = ["rbac.authorization.k8s.io"] def status exists? ? "Created" : "Not Found" diff --git a/lib/krane/kubernetes_resource/role_binding.rb b/lib/krane/kubernetes_resource/role_binding.rb index 2ff4874ae..39812347b 100644 --- a/lib/krane/kubernetes_resource/role_binding.rb +++ b/lib/krane/kubernetes_resource/role_binding.rb @@ -2,6 +2,7 @@ module Krane class RoleBinding < KubernetesResource TIMEOUT = 30.seconds + GROUP = ["rbac.authorization.k8s.io"] def status exists? ? "Created" : "Not Found" diff --git a/lib/krane/kubernetes_resource/secret.rb b/lib/krane/kubernetes_resource/secret.rb index d484b0d90..c3cc96be2 100644 --- a/lib/krane/kubernetes_resource/secret.rb +++ b/lib/krane/kubernetes_resource/secret.rb @@ -2,6 +2,7 @@ module Krane class Secret < KubernetesResource TIMEOUT = 30.seconds + GROUPS = [""] SENSITIVE_TEMPLATE_CONTENT = true SERVER_DRY_RUNNABLE = true diff --git a/lib/krane/kubernetes_resource/service.rb b/lib/krane/kubernetes_resource/service.rb index 78a7f35c3..ef79ab5fd 100644 --- a/lib/krane/kubernetes_resource/service.rb +++ b/lib/krane/kubernetes_resource/service.rb @@ -4,6 +4,7 @@ module Krane class Service < KubernetesResource TIMEOUT = 7.minutes + GROUP = [""] SYNC_DEPENDENCIES = %w(Pod Deployment StatefulSet) def sync(cache) diff --git a/lib/krane/kubernetes_resource/service_account.rb b/lib/krane/kubernetes_resource/service_account.rb index 4ef4cbcff..e21c65757 100644 --- a/lib/krane/kubernetes_resource/service_account.rb +++ b/lib/krane/kubernetes_resource/service_account.rb @@ -2,6 +2,7 @@ module Krane class ServiceAccount < KubernetesResource TIMEOUT = 30.seconds + GROUPS = [""] def status exists? ? "Created" : "Not Found" diff --git a/lib/krane/kubernetes_resource/stateful_set.rb b/lib/krane/kubernetes_resource/stateful_set.rb index 26a8aa9ef..7457e5fa8 100644 --- a/lib/krane/kubernetes_resource/stateful_set.rb +++ b/lib/krane/kubernetes_resource/stateful_set.rb @@ -3,6 +3,7 @@ module Krane class StatefulSet < PodSetBase TIMEOUT = 10.minutes + GROUP = "apps" ONDELETE = 'OnDelete' SYNC_DEPENDENCIES = %w(Pod) attr_reader :pods diff --git a/lib/krane/resource_cache.rb b/lib/krane/resource_cache.rb index baba9f8a8..e53e190a2 100644 --- a/lib/krane/resource_cache.rb +++ b/lib/krane/resource_cache.rb @@ -24,6 +24,28 @@ def get_instance(kind, resource_name, raise_if_not_found: false) {} end + def get_instance_group_kind(group_kind, resource_name, raise_if_not_found: false) + instance = use_or_populate_cache_group_kind(group_kind).fetch(resource_name, {}) + if instance.blank? && raise_if_not_found + raise Krane::Kubectl::ResourceNotFoundError, "Resource does not exist (used cache for group kind #{group_kind})" + end + instance + rescue KubectlError + {} + end + + def get_all_group_kind(group_kind, selector = nil) + instances = use_or_populate_cache_group_kind(group_kind).values + return instances unless selector + + instances.select do |r| + labels = r.dig("metadata", "labels") || {} + labels >= selector + end + rescue KubectlError + [] + end + def get_all(kind, selector = nil) instances = use_or_populate_cache(kind).values return instances unless selector @@ -55,9 +77,43 @@ def use_or_populate_cache(kind) end end + + def use_or_populate_cache_group_kind(group_kind) + @kind_fetcher_locks[group_kind].synchronize do + return @data[group_kind] if @data.key?(group_kind) + @data[group_kind] = fetch_by_group_kind(group_kind) + end + end + def fetch_by_kind(kind) resource_class = KubernetesResource.class_for_kind(kind) global_kind = @task_config.global_kinds.map(&:downcase).include?(kind.downcase) + + if kind.downcase == "deployment" + # pp kind + # pp global_kind + # exit + end + + output_is_sensitive = resource_class.nil? ? false : resource_class::SENSITIVE_TEMPLATE_CONTENT + raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", attempts: 5, output: "json", + output_is_sensitive: output_is_sensitive, use_namespace: !global_kind) + raise KubectlError unless st.success? + + instances = {} + JSON.parse(raw_json)["items"].each do |resource| + resource_name = resource.dig("metadata", "name") + instances[resource_name] = resource + end + instances + end + + + def fetch_by_group_kind(group_kind) + kind = @task_config.group_kind_to_kind(api_kind) + resource_class = KubernetesResource.class_for_kind(kind) + global_kind = @task_config.global_group_kinds.map(&:downcase).include?(group_kind.downcase) + output_is_sensitive = resource_class.nil? ? false : resource_class::SENSITIVE_TEMPLATE_CONTENT raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", attempts: 5, output: "json", output_is_sensitive: output_is_sensitive, use_namespace: !global_kind) diff --git a/lib/krane/task_config.rb b/lib/krane/task_config.rb index 113304914..2e9df686a 100644 --- a/lib/krane/task_config.rb +++ b/lib/krane/task_config.rb @@ -15,13 +15,32 @@ def initialize(context, namespace, logger = nil, kubeconfig = nil) def global_kinds @global_kinds ||= begin - cluster_resource_discoverer = ClusterResourceDiscovery.new(task_config: self) - cluster_resource_discoverer.fetch_resources(namespaced: false).map { |g| g["kind"] } + fetch_resources.map { |g| g["kind"] } end end + def global_group_kinds + @global_group_kinds ||= begin + fetch_resources.map { |g| Krane.group_kind(g["apigroup"], g["kind"]) } + end + end + + def group_kind_to_kind(group_kind) + hashy = fetch_resources.find{ |x| Krane.group_kind(x["apigroup"], x["kind"]) == group_kind } + + hashy["kind"] + end + def kubeclient_builder @kubeclient_builder ||= KubeclientBuilder.new(kubeconfig: kubeconfig) end + + def cluster_resource_discoverer + @cluster_resource_discoverer ||= ClusterResourceDiscovery.new(task_config: self) + end + + def fetch_resources + @fetch_resources ||= cluster_resource_discoverer.fetch_resources(namespaced: false) + end end end