install-upi.yaml

---
- name: Install OpenShift 4
  gather_facts: false
  hosts: localhost
  environment:
    KUBECONFIG: /tmp/{{ clustername }}/auth/kubeconfig

  tasks:
    - name: Check if virtualenv is installed
      command: virtualenv --version
      ignore_errors: yes
      changed_when: false
      register: virtualenv_cmd

    - name: Install python-virtualenv package
      yum:
        name: python-virtualenv
        state: present
      become: yes
      when: not virtualenv_cmd is success

    - name: Create virtual Python environment
      pip:
        name: awscli
        virtualenv: "{{ awscli_venv_path }}"

    - import_tasks: tasks/download-ocp-binaries.yaml

    - name: Get encrypted AMI
      ec2_ami_facts:
        filters:
          "tag:rhcos_version": "{{ rhcos_version }}"
          "tag:latest_ami": 'true'
        region: "{{ region }}"
      register: ami_encrypted

    - name: Override AMI ID
      set_fact:
        rhcos_ami: "{{ ami_encrypted.images[0].image_id }}"
      when: ami_encrypted.images

    - import_tasks: tasks/installer-config-directory.yaml

    - name: "Check if /tmp/{{ clustername }}/metadata.json exists"
      stat:
        path: /tmp/{{ clustername }}/metadata.json
      register: stat_result

    - name: render template
      template:
        src: files/install-config.yaml.j2
        dest: /tmp/{{ clustername }}/install-config.yaml
      when: stat_result.stat.exists == False

    - name: Create manifests
      command: "{{ openshift_install }} create manifests --dir=/tmp/{{ clustername }}"
      args:
        creates: /tmp/{{ clustername }}/manifests/cluster-config.yaml

    - name: Delete master machine manifests
      file:
        path: /tmp/{{ clustername }}/openshift/99_openshift-cluster-api_master-machines-{{ item }}.yaml
        state: absent
      with_sequence: start=0 count=3

    - name: Create the cluster ignition configs
      command: "{{ openshift_install }} create ignition-configs --dir=/tmp/{{ clustername }}"
      args:
        creates: /tmp/{{ clustername }}/metadata.json

    - name: read master CA info
      command: cat /tmp/{{ clustername }}/master.ign
      register: result

    - set_fact:
        master_ignition: "{{ result.stdout | from_json }}"

    - import_tasks: tasks/get_infraid.yaml

#    - debug:
#        msg: "{{ master_ignition.ignition.security.tls.certificateAuthorities[0].source }}"
#
#    - debug:
#        msg: "{{ metadata_json.infraID }}"

    - import_tasks: tasks/bootstrap-s3-bucket.yaml

    - name: upload bootstrap.ign to s3 bucket
      aws_s3:
        bucket: "{{ infraid }}-bootstrap"
        object: /bootstrap.ign
        src: /tmp/{{ clustername }}/bootstrap.ign
        mode: "{{ state | default('put') }}"
      retries: 3
      delay: 3

    - name: Create UPI Network Elements (Route53 & LBs)
      cloudformation:
        stack_name: "{{ infraid }}-cluster-infra"
        state: present
        region: "{{ region }}"
        template: "files/cloudformation/02_cluster_infra.yaml{% if api_internal_only %}{% if privatezoneid is defined %}.internalonly-existingprivzone{% else %}.internalonly{% endif %}{% elif privatezoneid is defined %}.existingprivzone{% endif %}"
        template_parameters:
          ClusterName: "{{ clustername }}"
          InfrastructureName: "{{ infraid }}"
          PrivateSubnets: "{{ privatesubnets}}"
          PrivateZoneId: "{{ privatezoneid | default(omit) }}"
          PublicSubnets: "{{ publicsubnets }}"
          HostedZoneId: "{{ publiczoneid }}"
          HostedZoneName: "{{ publiczonename }}"
          VpcId: "{{ vpcid }}"
        tags:
          clustername: "{{ clustername }}"
          infraid: "{{ infraid }}"
      register: cluster_infra_stack

    - set_fact:
        privatezoneid: "{{ cluster_infra_stack.stack_outputs.PrivateHostedZoneId }}"
      when: privatezoneid is not defined

    - name: Create UPI Security Elements (Security Groups & IAM)
      cloudformation:
        stack_name: "{{ infraid }}-cluster-security"
        state: present
        region: "{{ region }}"
        template: "files/cloudformation/03_cluster_security.yaml"
        template_parameters:
          InfrastructureName: "{{ infraid }}"
          PrivateSubnets: "{{ privatesubnets}}"
          VpcId: "{{ vpcid }}"
          VpcCidr: "{{ vpccidr }}"
        tags:
          clustername: "{{ clustername }}"
          infraid: "{{ infraid }}"
      register: cluster_security_stack

#    - debug:
#        msg: "{{ cluster_security_stack.stack_outputs.MasterSecurityGroupId }}"

    - name: Set default tags
      set_fact:
        default_tags:
          clustername: "{{ clustername }}"
          infraid: "{{ infraid }}"

    - name: Create UPI Bootstrap (EC2 Instance, Security Groups and IAM)
      cloudformation:
        stack_name: "{{ infraid }}-cluster-bootstrap"
        state: present
        region: "{{ region }}"
        template: "files/cloudformation/04_cluster_bootstrap.yaml{% if api_internal_only is sameas true %}.internalonly{% endif %}"
        template_parameters:
          InfrastructureName: "{{ infraid }}"
          BootstrapIgnitionLocation: s3://{{ infraid }}-bootstrap//bootstrap.ign
          VpcId: "{{ vpcid }}"
          AutoRegisterELB: "yes"
          AllowedBootstrapSshCidr: "0.0.0.0/0"
          MasterSecurityGroupId: "{{ cluster_security_stack.stack_outputs.MasterSecurityGroupId }}"
          PublicSubnet: "{{ publicsubnets.split(',')[0] }}"
          RhcosAmi: "{{ rhcos_ami }}"
          ExternalApiTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.ExternalApiTargetGroupArn | default(omit) }}"
          InternalApiTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.InternalApiTargetGroupArn }}"
          InternalServiceTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.InternalServiceTargetGroupArn }}"
          RegisterNlbIpTargetsLambdaArn: "{{ cluster_infra_stack.stack_outputs.RegisterNlbIpTargetsLambda }}"
        tags: "{{ default_tags | combine(extra_instance_tags) }}"

    - name: Create UPI Node Launch (EC2 master instances)
      cloudformation:
        stack_name: "{{ infraid }}-cluster-master-nodes"
        state: present
        region: "{{ region }}"
        template: "files/cloudformation/05_cluster_master_nodes.yaml{% if api_internal_only is sameas true %}.internalonly{% endif %}"
        template_parameters:
          InfrastructureName: "{{ infraid }}"
          AutoRegisterELB: "yes"
          MasterSecurityGroupId: "{{ cluster_security_stack.stack_outputs.MasterSecurityGroupId }}"
          Master0Subnet: "{{ privatesubnets.split(',')[0] }}"
          Master1Subnet: "{{ privatesubnets.split(',')[1] }}"
          Master2Subnet: "{{ privatesubnets.split(',')[2] }}"
          MasterInstanceProfileName: "{{ cluster_security_stack.stack_outputs.MasterInstanceProfile }}"
          MasterInstanceType: "{{ master_instance }}"
          RhcosAmi: "{{ rhcos_ami }}"
          ExternalApiTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.ExternalApiTargetGroupArn | default(omit) }}"
          InternalApiTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.InternalApiTargetGroupArn }}"
          InternalServiceTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.InternalServiceTargetGroupArn }}"
          RegisterNlbIpTargetsLambdaArn: "{{ cluster_infra_stack.stack_outputs.RegisterNlbIpTargetsLambda }}"
          AutoRegisterDNS: "yes"
          IgnitionLocation: "https://api-int.{{ clustername }}.{{ publiczonename }}:22623/config/master"
          PrivateHostedZoneId: "{{ privatezoneid }}"
#this is confusing but right for in some cases. in others use the privatezonename var
          PrivateHostedZoneName: "{{ clustername }}.{{ publiczonename }}"
          CertificateAuthorities: "{{ master_ignition.ignition.security.tls.certificateAuthorities[0].source }}"
        tags: "{{ default_tags | combine(extra_instance_tags) }}"

#TODO: change master ec2 instance termination protection to true and then remove old machine api objects after

    - name: tag public subnets
      command: "{{ aws }} --region {{ region }} ec2 create-tags --resources {{ item }} --tags Key=kubernetes.io/cluster/{{ infraid }},Value=shared"
      with_items: "{{ publicsubnets.split(',') }}"

    - name: tag private subnets
      command: "{{ aws }} --region {{ region }} ec2 create-tags --resources {{ item }} --tags Key=kubernetes.io/role/internal-elb,Value=''"
      with_items: "{{ privatesubnets.split(',') }}"

    - name: "Check if /tmp/{{ clustername }}/bootstrap-complete exists"
      stat:
        path: /tmp/{{ clustername }}/bootstrap-complete
      register: bootstrap_result

    - name: Wait until bootstrap is complete
      shell: >
        {{ openshift_install }} wait-for bootstrap-complete --dir=/tmp/{{ clustername }} &&
        touch /tmp/{{ clustername }}/bootstrap-complete
      when: bootstrap_result.stat.exists == False

    - name: "Check if /tmp/{{ clustername }}/bootstrap-complete exists"
      stat:
        path: /tmp/{{ clustername }}/finish
      register: finish

    - name: approve master certificates
      shell: "{{ oc }} adm certificate approve $(oc get csr -o 'jsonpath={..metadata.name}')"
      register: oc_approve
      retries: 12
      delay: 10
      until: oc_approve.rc == 0
      when: not finish.stat.exists
      ignore_errors: yes

    - name: Fix worker machineset, securitygroup, subnet, ami and instanceprofile setting
      shell: |
        {{ oc }} -n openshift-machine-api patch machineset {{ infraid }}-worker-{{ region }}{{item.key}} -p '{"spec":{"template":{"spec":{"providerSpec":{"value":{"securityGroups":[{"filters":[{"name": "group-id", "values":["{{ cluster_security_stack.stack_outputs.WorkerSecurityGroupId }}"]}]}]}}}}}}' --type merge
        {{ oc }} -n openshift-machine-api patch machineset {{ infraid }}-worker-{{ region }}{{item.key}} -p '{"spec":{"template":{"spec":{"providerSpec":{"value":{"subnet":{"filters":[{"name": "subnet-id", "values":["{{ privatesubnets.split(',')[item.value | int] }}"]}]}}}}}}}' --type merge
        {{ oc }} -n openshift-machine-api patch machineset {{ infraid }}-worker-{{ region }}{{item.key}} -p '{"spec":{"template":{"spec":{"providerSpec":{"value":{"iamInstanceProfile":{"id": "{{ cluster_security_stack.stack_outputs.WorkerInstanceProfile }}"}}}}}}}' --type merge
        {{ oc }} -n openshift-machine-api patch machineset {{ infraid }}-worker-{{ region }}{{item.key}} -p '{"spec":{"template":{"spec":{"providerSpec":{"value":{"ami":{"id": "{{ rhcos_ami }}"}}}}}}}' --type merge
      with_dict: {a: 0, b: 1, c: 2}
      when: finish.stat.exists == False

    - name: delete broken machines
      command: "{{ oc }} -n openshift-machine-api delete machine -l machine.openshift.io/cluster-api-machineset={{ infraid }}-worker-{{ region }}{{ item.key }}"
      with_dict: {a: 0, b: 1, c: 2}
      when: finish.stat.exists == False


#    - name: delete broken machines
#      command: "{{ oc }} -n openshift-machine-api delete machine -l machine.openshift.io/cluster-api-machine-role=master"
#      when: finish.stat.exists == False


#    - name: Fix master machine loadbalancer, securitygroup, subnet, ami and instanceprofile setting
#      shell: |
#        {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"loadBalancers":[{"name": "{{ cluster_infra_stack.stack_outputs.InternalApiLoadBalancerName.split('/')[1] }}", "type": "network"}{% if api_internal_only != true %},{"name": "{{ cluster_infra_stack.stack_outputs.ExternalApiLoadBalancerName.split('/')[1] }}", "type": "network"}{% endif %}]}}}}' --type merge
#        {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"securityGroups":[{"filters":[{"name": "group-id", "values":["{{ cluster_security_stack.stack_outputs.MasterSecurityGroupId }}"]}]}]}}}}' --type merge
#        {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"subnet":{"filters":[{"name": "subnet-id", "values":["{{ privatesubnets.split(',')[item | int] }}"]}]}}}}}' --type merge
#        {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"iamInstanceProfile":{"id": "{{ cluster_security_stack.stack_outputs.MasterInstanceProfile }}"}}}}}' --type merge
#        {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"ami":{"id": "{{ rhcos_ami }}"}}}}}' --type merge


#        {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"loadBalancers":[{"name": "{{ cluster_infra_stack.stack_outputs.InternalApiLoadBalancerName.split('/')[1] }}", "type": "network"},{"name": "{{ cluster_infra_stack.stack_outputs.ExternalApiLoadBalancerName.split('/')[1] }}", "type": "network"}]}}}}' --type merge
#
#       {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"subnet":[{"filters":[{"name": "subnet-id", "values":["{{ privatesubnets.split(',')[item | int] }}"]}]}]}}}}' --type merge
#      with_sequence: start=0 count=3


    - name: render DNS settings
      template:
        src: files/dnses.yaml.j2
        dest: /tmp/{{ clustername }}/dnses.yaml
      when: finish.stat.exists == False

    - name: fix dns settings
      command: "{{ oc }} replace -f /tmp/{{ clustername }}/dnses.yaml"
      when: finish.stat.exists == False
        
#    - name: workaround for missing kubeadmin-password file
#      copy:
#        dest: "/tmp/{{ clustername }}/auth/kubeadmin-password"
#        content: "REDACTED, use kubeconfig instead"

    - name: Wait for ingress controller
      command: "{{ oc }} get -n openshift-ingress-operator ingresscontroller/default"
      register: ingress_controller
      until: ingress_controller.rc == 0
      retries: 60
      delay: 10

    - name: Scale ingress controller replicas
      command: "{{ oc }} patch -n openshift-ingress-operator -p '{\"spec\":{\"replicas\":3}}' --type merge ingresscontroller/default"
      when: finish.stat.exists == False

    - name: Wait until install is complete
      shell: >
        {{ openshift_install }} wait-for install-complete --dir=/tmp/{{ clustername }} &&
        touch /tmp/{{ clustername }}/finish
      when: finish.stat.exists == False

    - name: Terminate bootstrap resources
      block:
        - name: Terminate bootstrap CloudFormation stack
          cloudformation:
            stack_name: "{{ infraid }}-cluster-bootstrap"
            region: "{{ region }}"
            state: absent

        - import_tasks: tasks/bootstrap-s3-bucket.yaml
          vars:
            state: absent
      when: bootstrap_cleanup | bool

    - debug:
        msg:
          - "OpenShift cluster {{ clustername }} has been installed successfully."
          - ""
          - "API: https://api.{{ clustername }}.{{ publiczonename }}:6443"
          - "Web console: https://console-openshift-console.apps.{{ clustername }}.{{ publiczonename }}/"
...