Skip to content

Commit

Permalink
Add Cilium config and notes (#1380)
Browse files Browse the repository at this point in the history
  • Loading branch information
Richard87 authored Jun 27, 2024
1 parent baea0d8 commit c684f8a
Show file tree
Hide file tree
Showing 38 changed files with 1,584 additions and 65 deletions.
75 changes: 17 additions & 58 deletions scripts/aks/bootstrap.sh
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,9 @@ echo -e " - VNET_ADDRESS_PREFIX : $VNET_ADDRESS_PREFIX"
echo -e " - VNET_SUBNET_PREFIX : $VNET_SUBNET_PREFIX"
echo -e " - NSG_NAME : $NSG_NAME"
if [ "${CILIUM}" = true ]; then
echo -e " - NETWORK_POLICY : none"
echo -e " - NETWORK_POLICY : azure"
echo -e " - NETWORK_PLUGIN : overlay"
echo -e " - network-dataplane : cilium"
else
echo -e " - NETWORK_PLUGIN : $NETWORK_PLUGIN"
echo -e " - NETWORK_POLICY : $NETWORK_POLICY"
Expand Down Expand Up @@ -282,10 +284,20 @@ if [[ $USER_PROMPT == true ]]; then
echo ""
fi

#######################################################################################
### Prepare Terraform
###

printf "Initializing Terraform..."
terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/common" init
terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/virtualnetwork" init
terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/pre-clusters" init
terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/clusters" init
terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/post-clusters" init
#######################################################################################
### Set credentials
###
printf "Reading credentials... "

ID_AKS=$(terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/clusters" output -raw radix_id_aks_mi_id)
ID_AKSKUBELET=$(terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/clusters" output -raw radix_id_akskubelet_mi_id)
ACR_ID=$(terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/common" output -raw acr_id)
Expand Down Expand Up @@ -472,7 +484,6 @@ if [[ ${update_keyvault} == true ]]; then
printf "Done.\n"
fi

terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/pre-clusters" init
terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/pre-clusters" apply

#######################################################################################
Expand Down Expand Up @@ -549,7 +560,9 @@ fi

if [ "$CILIUM" = true ]; then
AKS_NETWORK_OPTIONS=(
--network-plugin "none"
--network-plugin "azure"
--network-plugin-mode overlay
--network-dataplane cilium
)
else
AKS_NETWORK_OPTIONS=(
Expand Down Expand Up @@ -636,59 +649,6 @@ function retry() {
done
}

if [ "$CILIUM" = true ]; then
CILIUM_VALUES="cilium-values.yaml"

cat <<EOF >"${WORK_DIR}/${CILIUM_VALUES}"
nodeinit:
enabled: true
aksbyocni:
enabled: true
azure:
resourceGroup: ${AZ_RESOURCE_GROUP_CLUSTERS}
k8sClientRateLimit:
qps: 20
burst: 20
prometheus:
enabled: true
hubble:
enabled: true
relay:
enabled: true
ui:
enabled: true
operator:
prometheus:
enabled: true
ipam:
operator:
clusterPoolIPv4PodCIDRList: ["10.200.0.0/16"]
clusterPoolIPv4MaskSize: 24
EOF

printf "Installing Cilium...\n"

retry "1m" "helm repo add cilium https://helm.cilium.io/"

retry "2m" "helm upgrade --install cilium cilium/cilium \
--version $CILIUM_VERSION \
--namespace kube-system \
--values ${WORK_DIR}/${CILIUM_VALUES}"

cilium status --wait

printf "Done.\n"

rm -f "${WORK_DIR}/${CILIUM_VALUES}"
fi

#######################################################################################
### Taint the 'systempool'
###
Expand Down Expand Up @@ -903,7 +863,6 @@ fi
### Do some terraform post tasks
###
echo "Do some terraform post tasks"
terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/post-clusters" init
terraform -chdir="../terraform/subscriptions/$AZ_SUBSCRIPTION_NAME/$RADIX_ZONE/post-clusters" apply
printf "Done."
#######################################################################################
Expand Down
1 change: 1 addition & 0 deletions scripts/cilium/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.certs
26 changes: 26 additions & 0 deletions scripts/cilium/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Work in progress!

Started cililum-26 with these network options:

AKS_NETWORK_OPTIONS=(
--network-plugin "azure"
--network-plugin-mode overlay
--network-dataplane cilium
)
Setup Advanced Networking with managed Cilium, but bring your own Grafana/Prometheus
https://learn.microsoft.com/en-us/azure/aks/advanced-network-observability-bring-your-own-cli?tabs=non-cilium

az aks update --resource-group clusters-dev --name cilium-26 --enable-advanced-network-observability
```shell
k get pods -n kube-system -l k8s-app=hubble-relay
# NAME READY STATUS RESTARTS AGE
# hubble-relay-55b65f695c-6bnwk 1/1 Running 0 4m9s
```
Level 7 / DNS & HTTP visiblity:
https://docs.cilium.io/en/latest/observability/visibility/#layer-7-protocol-visibility

Note: We should enable --hubble-redact-enabled to redact sensitive http data like query/headers/auth cookies etc

```shell
kubectl port-forward svc/hubble-ui 12000:80 -n kube-system
```
64 changes: 64 additions & 0 deletions scripts/cilium/dns-policy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: "tofqdn-dns-visibility"
namespace: kube-system
spec:
endpointSelector:
matchLabels: {}
# matchLabels:
# any:org: alliance
egress:
- toEndpoints:
- matchLabels:
"k8s:io.kubernetes.pod.namespace": kube-system
"k8s:k8s-app": kube-dns
toPorts:
- ports:
- port: "53"
protocol: ANY
rules:
dns:
- matchPattern: "*"
# - toFQDNs:
# - matchName: "equinor.com"
# - matchName: "sub.cilium.io"
# - matchPattern: "*.sub.cilium.io"
---
apiVersion: "cilium.io/v2"
kind: CiliumNetworkPolicy
metadata:
name: "l7-visibility"
spec:
endpointSelector:
matchLabels:
"k8s:io.kubernetes.pod.namespace": radix-web-console-qa
egress:
- toPorts:
- ports:
- port: "53"
protocol: ANY
rules:
dns:
- matchPattern: "*"
- toEndpoints:
- matchLabels:
"k8s:io.kubernetes.pod.namespace": default
toPorts:
- ports:
- port: "80"
protocol: TCP
- port: "8000"
protocol: TCP
- port: "8080"
protocol: TCP
- port: "3000"
protocol: TCP
- port: "3001"
protocol: TCP
- port: "3002"
protocol: TCP
- port: "3003"
protocol: TCP
rules:
http: [{}]
30 changes: 30 additions & 0 deletions scripts/cilium/download-hubble-certs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/usr/bin/env bash

set -euo pipefail
set -x

# Directory where certificates will be stored
CERT_DIR="$(pwd)/.certs"
mkdir -p "$CERT_DIR"

declare -A CERT_FILES=(
["tls.crt"]="tls-client-cert-file"
["tls.key"]="tls-client-key-file"
["ca.crt"]="tls-ca-cert-files"
)

for FILE in "${!CERT_FILES[@]}"; do
KEY="${CERT_FILES[$FILE]}"
JSONPATH="{.data['${FILE//./\\.}']}"

# Retrieve the secret and decode it
kubectl get secret hubble-relay-client-certs -n kube-system \
-o jsonpath="${JSONPATH}" | \
base64 -d > "$CERT_DIR/$FILE"

# Set the appropriate hubble CLI config
hubble config set "$KEY" "$CERT_DIR/$FILE"
done

hubble config set tls true
hubble config set tls-server-name instance.hubble-relay.cilium.io
Loading

0 comments on commit c684f8a

Please sign in to comment.