Skip to content

Commit

Permalink
Merge pull request #27 from banzaicloud/azure-intergration
Browse files Browse the repository at this point in the history
PKE Azure cloud proivder integration
  • Loading branch information
Ecsy authored Apr 11, 2019
2 parents f9a7f82 + 06070d5 commit 133fbf1
Show file tree
Hide file tree
Showing 15 changed files with 907 additions and 35 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ You can create PKE clusters on any of the cloud providers, Vagrant, virtual mach

- [PKE in Vagrant](/docs/vagrant.md)
- [PKE on AWS](/docs/aws.md)
- [PKE on Azure](/docs/azure.md)
- [PKE with Pipeline](https://beta.banzaicloud.io/)

### Contributing
Expand Down
20 changes: 20 additions & 0 deletions cmd/pke/app/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,32 @@ const (

// CloudProviderAmazon
CloudProviderAmazon = "aws"
// CloudProviderAzure
CloudProviderAzure = "azure"

// FlagImageRepository prefix for image repository.
FlagImageRepository = "image-repository"

// FlagAdmissionPluginPodSecurityPolicy enable admission plugin PodSecurityPolicy.
FlagAdmissionPluginPodSecurityPolicy = "with-plugin-psp"

// Azure specific flags
// FlagAzureTenantID the AAD Tenant ID for the Subscription that the cluster is deployed in.
FlagAzureTenantID = "azure-tenant-id"
// FlagAzureSubnetName the name of the subnet that the cluster is deployed in.
FlagAzureSubnetName = "azure-subnet-name"
// FlagAzureSecurityGroupName the name of the security group attached to the cluster's subnet.
FlagAzureSecurityGroupName = "azure-security-group-name"
// FlagAzureVNetName the name of the VNet that the cluster is deployed in.
FlagAzureVNetName = "azure-vnet-name"
// FlagAzureVNetResourceGroup the name of the resource group that the Vnet is deployed in.
FlagAzureVNetResourceGroup = "azure-vnet-resource-group"
// FlagAzureVMType the type of azure nodes. Candidate values are: vmss and standard.
FlagAzureVMType = "azure-vm-type"
// FlagAzureLoadBalancerSku sku of Load Balancer and Public IP. Candidate values are: basic and standard.
FlagAzureLoadBalancerSku = "azure-loadbalancer-sku"
// FlagAzureRouteTableName the name of the route table attached to the subnet that the cluster is deployed in.
FlagAzureRouteTableName = "azure-route-table-name"
)

var (
Expand Down
144 changes: 144 additions & 0 deletions cmd/pke/app/phases/kubeadm/common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
// Copyright © 2019 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubeadm

import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"text/template"
"time"

"github.com/banzaicloud/pke/cmd/pke/app/constants"
"github.com/pkg/errors"
)

const (
urlAzureAZ = "http://169.254.169.254/metadata/instance?api-version=2018-10-01"
)

func WriteKubeadmAzureConfig(out io.Writer, filename, cloudProvider, tenantID, subnetName, securityGroupName, vnetName, vnetResourceGroup, vmType, loadBalancerSku, routeTableName string) error {
if cloudProvider == constants.CloudProviderAzure {
if http.DefaultClient.Timeout < 10*time.Second {
http.DefaultClient.Timeout = 10 * time.Second
}

req, err := http.NewRequest(http.MethodGet, urlAzureAZ, nil)
if err != nil {
return err
}
req.Header.Set("Metadata", "true")

resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return errors.New(fmt.Sprintf("failed to get azure availability zone. http status code: %d", resp.StatusCode))
}
defer func() { _ = resp.Body.Close() }()

b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "failed to read response body")
}

type metadata struct {
Compute struct {
AZEnvironment string `json:"azEnvironment"`
Location string `json:"location"`
ResourceGroupName string `json:"resourceGroupName"`
SubscriptionId string `json:"subscriptionId"`
} `json:"compute"`
}
var r metadata
if err := json.Unmarshal(b, &r); err != nil {
return errors.Wrap(err, "failed to parse response")
}

if vmType == "" {
vmType = "standard"
}
if loadBalancerSku == "" {
loadBalancerSku = "basic"
}

conf := `{
"cloud":"{{ .Cloud }}",
"tenantId": "{{ .TenantId }}",
"subscriptionId": "{{ .SubscriptionId }}",
"resourceGroup": "{{ .ResourceGroup }}",
"location": "{{ .Location }}",
"subnetName": "{{ .SubnetName }}",
"securityGroupName": "{{ .SecurityGroupName }}",
"vnetName": "{{ .VNetName }}",
"vnetResourceGroup": "{{ .VNetResourceGroup }}",
"vmType": "{{ .VMType }}",
"loadBalancerSku": "{{ .LoadBalancerSku }}",
"routeTableName": "{{ .RouteTableName }}",
"cloudProviderBackoff": false,
"useManagedIdentityExtension": true,
"useInstanceMetadata": true
}`

tmpl, err := template.New("azure-config").Parse(conf)
if err != nil {
return err
}

w, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640)
if err != nil {
return err
}
defer func() { _ = w.Close() }()

type data struct {
Cloud string
TenantId string
SubscriptionId string
ResourceGroup string
Location string
SubnetName string
SecurityGroupName string
VNetName string
VNetResourceGroup string
VMType string
LoadBalancerSku string
RouteTableName string
}

d := data{
Cloud: r.Compute.AZEnvironment,
TenantId: tenantID,
SubscriptionId: r.Compute.SubscriptionId,
ResourceGroup: r.Compute.ResourceGroupName,
Location: r.Compute.Location,
SubnetName: subnetName,
SecurityGroupName: securityGroupName,
VNetName: vnetName,
VNetResourceGroup: vnetResourceGroup,
VMType: vmType,
LoadBalancerSku: loadBalancerSku,
RouteTableName: routeTableName,
}

return tmpl.Execute(w, d)
}

return nil
}
97 changes: 86 additions & 11 deletions cmd/pke/app/phases/kubeadm/controlplane/controlplane.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,16 @@ const (
use = "kubernetes-controlplane"
short = "Kubernetes Control Plane installation"

cmdKubeadm = "/bin/kubeadm"
cmdKubectl = "/bin/kubectl"
weaveNetUrl = "https://cloud.weave.works/k8s/net"
kubeConfig = "/etc/kubernetes/admin.conf"
kubeProxyConfig = "/var/lib/kube-proxy/config.conf"
kubeadmConfig = "/etc/kubernetes/kubeadm.conf"
kubeadmAmazonConfig = "/etc/kubernetes/aws.conf"
urlAWSAZ = "http://169.254.169.254/latest/meta-data/placement/availability-zone"
cmdKubeadm = "/bin/kubeadm"
cmdKubectl = "/bin/kubectl"
weaveNetUrl = "https://cloud.weave.works/k8s/net"
kubeConfig = "/etc/kubernetes/admin.conf"
kubeProxyConfig = "/var/lib/kube-proxy/config.conf"
kubeadmConfig = "/etc/kubernetes/kubeadm.conf"
kubeadmAmazonConfig = "/etc/kubernetes/aws.conf"
kubeadmAzureConfig = "/etc/kubernetes/azure.conf"
urlAWSAZ = "http://169.254.169.254/latest/meta-data/placement/availability-zone"

kubernetesCASigningCert = "/etc/kubernetes/pki/cm-signing-ca.crt"
admissionConfig = "/etc/kubernetes/admission-control.yaml"
admissionEventRateLimitConfig = "/etc/kubernetes/admission-control/event-rate-limit.yaml"
Expand Down Expand Up @@ -85,6 +87,14 @@ type ControlPlane struct {
imageRepository string
withPluginPSP bool
node *node.Node
azureTenantID string
azureSubnetName string
azureSecurityGroupName string
azureVNetName string
azureVNetResourceGroup string
azureVMType string
azureLoadBalancerSku string
azureRouteTableName string
}

func NewCommand(out io.Writer) *cobra.Command {
Expand Down Expand Up @@ -134,6 +144,15 @@ func (c *ControlPlane) RegisterFlags(flags *pflag.FlagSet) {
flags.String(constants.FlagImageRepository, "banzaicloud", "Prefix for image repository")
// PodSecurityPolicy admission plugin
flags.Bool(constants.FlagAdmissionPluginPodSecurityPolicy, false, "Enable PodSecurityPolicy admission plugin")
// Azure cloud
flags.String(constants.FlagAzureTenantID, "", "The AAD Tenant ID for the Subscription that the cluster is deployed in")
flags.String(constants.FlagAzureSubnetName, "", "The name of the subnet that the cluster is deployed in")
flags.String(constants.FlagAzureSecurityGroupName, "", "The name of the security group attached to the cluster's subnet")
flags.String(constants.FlagAzureVNetName, "", "The name of the VNet that the cluster is deployed in")
flags.String(constants.FlagAzureVNetResourceGroup, "", "The name of the resource group that the Vnet is deployed in")
flags.String(constants.FlagAzureVMType, "standard", "The type of azure nodes. Candidate values are: vmss and standard")
flags.String(constants.FlagAzureLoadBalancerSku, "basic", "Sku of Load Balancer and Public IP. Candidate values are: basic and standard")
flags.String(constants.FlagAzureRouteTableName, "kubernetes-routes", "The name of the route table attached to the subnet that the cluster is deployed in")

addHAControlPlaneFlags(flags)
}
Expand Down Expand Up @@ -172,6 +191,22 @@ func (c *ControlPlane) Validate(cmd *cobra.Command) error {
return err
}

// Azure specific required flags
if c.cloudProvider == constants.CloudProviderAzure {
if err := validator.NotEmpty(map[string]interface{}{
constants.FlagAzureTenantID: c.azureTenantID,
constants.FlagAzureSubnetName: c.azureSubnetName,
constants.FlagAzureSecurityGroupName: c.azureSecurityGroupName,
constants.FlagAzureVNetName: c.azureVNetName,
constants.FlagAzureVNetResourceGroup: c.azureVNetResourceGroup,
constants.FlagAzureVMType: c.azureVMType,
constants.FlagAzureLoadBalancerSku: c.azureLoadBalancerSku,
constants.FlagAzureRouteTableName: c.azureRouteTableName,
}); err != nil {
return err
}
}

if c.networkProvider != "weave" {
return errors.Wrapf(constants.ErrUnsupportedNetworkProvider, "network provider: %s", c.networkProvider)
}
Expand Down Expand Up @@ -218,7 +253,7 @@ func (c *ControlPlane) Run(out io.Writer) error {
return err
}

if err := installPodNetwork(out, c.podNetworkCIDR, kubeConfig); err != nil {
if err := installPodNetwork(out, c.cloudProvider, c.podNetworkCIDR, kubeConfig); err != nil {
return err
}

Expand Down Expand Up @@ -305,6 +340,38 @@ func (c *ControlPlane) masterBootstrapParameters(cmd *cobra.Command) (err error)
return
}
c.joinControlPlane, err = cmd.Flags().GetBool(constants.FlagControlPlaneJoin)
if err != nil {
return
}
c.azureTenantID, err = cmd.Flags().GetString(constants.FlagAzureTenantID)
if err != nil {
return
}
c.azureSubnetName, err = cmd.Flags().GetString(constants.FlagAzureSubnetName)
if err != nil {
return
}
c.azureSecurityGroupName, err = cmd.Flags().GetString(constants.FlagAzureSecurityGroupName)
if err != nil {
return
}
c.azureVNetName, err = cmd.Flags().GetString(constants.FlagAzureVNetName)
if err != nil {
return
}
c.azureVNetResourceGroup, err = cmd.Flags().GetString(constants.FlagAzureVNetResourceGroup)
if err != nil {
return
}
c.azureVMType, err = cmd.Flags().GetString(constants.FlagAzureVMType)
if err != nil {
return
}
c.azureLoadBalancerSku, err = cmd.Flags().GetString(constants.FlagAzureLoadBalancerSku)
if err != nil {
return
}
c.azureRouteTableName, err = cmd.Flags().GetString(constants.FlagAzureRouteTableName)

return
}
Expand Down Expand Up @@ -356,6 +423,12 @@ func (c *ControlPlane) installMaster(out io.Writer) error {
return err
}

// write kubeadm azure.conf
err = kubeadm.WriteKubeadmAzureConfig(out, kubeadmAzureConfig, c.cloudProvider, c.azureTenantID, c.azureSubnetName, c.azureSecurityGroupName, c.azureVNetName, c.azureVNetResourceGroup, c.azureVMType, c.azureLoadBalancerSku, c.azureRouteTableName)
if err != nil {
return err
}

// kubeadm init --config=/etc/kubernetes/kubeadm.conf
args := []string{
"init",
Expand Down Expand Up @@ -389,7 +462,7 @@ func (c *ControlPlane) installMaster(out io.Writer) error {
return nil
}

func installPodNetwork(out io.Writer, podNetworkCIDR, kubeConfig string) error {
func installPodNetwork(out io.Writer, cloudProvider, podNetworkCIDR, kubeConfig string) error {
// kubectl version
cmd := runner.Cmd(out, cmdKubectl, "version")
cmd.Env = append(os.Environ(), "KUBECONFIG="+kubeConfig)
Expand All @@ -408,7 +481,9 @@ func installPodNetwork(out io.Writer, podNetworkCIDR, kubeConfig string) error {
}
q := u.Query()
q.Set("k8s-version", ver)
q.Set("env.IPALLOC_RANGE", podNetworkCIDR)
if cloudProvider != constants.CloudProviderAzure {
q.Set("env.IPALLOC_RANGE", podNetworkCIDR)
}
u.RawQuery = q.Encode()

// kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')&env.IPALLOC_RANGE=10.200.0.0/16"
Expand Down
7 changes: 4 additions & 3 deletions cmd/pke/app/phases/kubeadm/controlplane/kubeadm.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ nodeRegistration:
# pod-infra-container-image: {{ .ImageRepository }}/pause:3.1 # only needed by docker
{{if .CloudProvider }}
cloud-provider: "{{ .CloudProvider }}"{{end}}
{{if eq .CloudProvider "azure" }}cloud-config: "/etc/kubernetes/{{ .CloudProvider }}.conf"{{end}}
read-only-port: "0"
anonymous-auth: "false"
streaming-connection-idle-timeout: "5m"
Expand All @@ -172,10 +173,10 @@ networking:
kubernetesVersion: "v{{ .KubernetesVersion }}"
{{ if .ControlPlaneEndpoint }}controlPlaneEndpoint: "{{ .ControlPlaneEndpoint }}"{{end}}
certificatesDir: "/etc/kubernetes/pki"
{{if .APIServerCertSANs}}apiServerCertSANs:
{{range $k, $san := .APIServerCertSANs}} - "{{ $san }}"
{{end}}{{end}}
apiServer:
{{if .APIServerCertSANs}}certSANs:
{{range $k, $san := .APIServerCertSANs}} - "{{ $san }}"
{{end}}{{end}}
extraArgs:
# anonymous-auth: "false"
profiling: "false"
Expand Down
1 change: 1 addition & 0 deletions cmd/pke/app/phases/kubeadm/node/kubeadm.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ nodeRegistration:
node-labels: "nodepool.banzaicloud.io/name={{ .Nodepool }}"{{end}}
{{if .CloudProvider }}
cloud-provider: "{{ .CloudProvider }}"{{end}}
{{if eq .CloudProvider "azure" }}cloud-config: "/etc/kubernetes/{{ .CloudProvider }}.conf"{{end}}
read-only-port: "0"
anonymous-auth: "false"
streaming-connection-idle-timeout: "5m"
Expand Down
Loading

0 comments on commit 133fbf1

Please sign in to comment.