merge dev branch (#1)

.gitignore

@@ -5,6 +5,8 @@
 *.tfstate
 *.tfstate.*
 
+.terraform.lock.hcl
+
 # Crash log files
 crash.log
 
@@ -27,3 +29,7 @@ override.tf.json
 
 # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
 # example: *tfplan*
+
+*.zip
+src/lambda-payments/lambda-payments
+src/lambda-products/lambda-products

README.md

@@ -1,2 +1,77 @@
-# consul-lambda-demo
-Showcases Consul's serverless integration with AWS Lambda
+# Consul Lambda Demo
+
+This project showcases Consul's AWS multi-runtime support with AWS EKS and Lambda.
+
+Below is a list of resources for taking the next steps to integrating your AWS Lambda functions with your Consul service mesh:
+- [Docs: Consul on AWS Lambda](https://developer.hashicorp.com/consul/docs/lambda) - The official Consul documentation for AWS Lambda integration.
+- [Github: Consul on AWS Lambda](https://github.com/hashicorp/terraform-aws-consul-lambda) - OSS repository that contains all the source code for Lambda Registrator and the Consul-Lambda extension.
+- [Tutorial: Extend your Service Mesh to Support AWS Lambda](https://developer.hashicorp.com/consul/tutorials/developer-mesh/serverless-consul-with-lambda) - Official tutorial for adding AWS Lambda to your Consul service mesh.
+- [Video: Consul Service Mesh on Amazon EKS and AWS Lambda](https://www.youtube.com/watch?v=wgPPALAKuXI) - AWS Containers from the Couch: Sai Vennam (@svennam92) hosts this deep dive into Consul's new architecture with Iryna Shustava (@ishustava), and support for AWS Lambda with Chris Thain (@cthain). Includes live demos!
+
+## Architecture
+
+This demo deploys the HashiCups demo application on Amazon Elastic Kubernetes Service (EKS) and connects all the components using Consul service mesh.
+The Consul server is deployed and managed using the HashiCorp Cloud Platform (HCP).
+AWS Lambda functions are deployed during the demo and configured as part of the Consul service mesh.
+
+The architecture of the system at the end of the demo is presented in the figure below.
+
+![Consul Lambda demo architecture](consul-lambda-demo-arch.png)
+
+## Requirements
+
+This project requires the following:
+
+- [AWS account](https://aws.amazon.com/)
+- [HCP account](https://cloud.hashicorp.com/)
+- [`aws` CLI](https://aws.amazon.com/cli/)
+- [`curl`](https://curl.se/)
+- [`docker`](https://www.docker.com/)
+- [`go`](https://go.dev/)
+- [`jq`](https://stedolan.github.io/jq/)
+- [`kubectl` v1.23.8](https://kubernetes.io/docs/tasks/tools/)
+- [`terraform` v1.3.x](https://releases.hashicorp.com/terraform/)
+
+## Usage
+
+**Note: _Running the steps in this demo will create resources that are not eligible for the AWS free-tier and will generate costs to your AWS account_.**
+
+### Deploy the demo infrastructure
+
+Deploy the AWS infrastructure and the HashiCups demo application using Terraform.
+
+```shell
+terraform init && terraform apply -auto-approve
+```
+
+### Run the demo
+
+The `runbook` script automates all the steps to deploy the AWS Lambda functions into the Consul service mesh.
+It walks you through the demo and provides prompts for each step of the way.
+
+```
+./runbook
+```
+
+If you're interested in how the demo works take a look at the `runbook`. It's just a `bash` script.
+
+### Reset the demo
+
+If you want to run the demo over again, you can use the `runbook reset` command.
+This command resets the demo to its original state.
+It removes the AWS Lambda components from the service mesh and clears the database.
+
+The `runbook reset` command is idempotent and can be run repeatedly any number of times.
+
+```
+./runbook reset
+```
+
+### Clean up all resources
+
+When you're done, make sure that you clean up all your resources.
+
+```
+./runbook reset -f
+terraform destroy -auto-approve
+```

env.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+TFO=$(terraform output -json)
+
+$(echo "$TFO" | jq -r .eks_update_kubeconfig_command.value)
+
+# configure the env from the deployment
+export AWS_REGION=$(echo "$TFO" | jq -r .region.value)
+export DEPLOY_NAME=$(echo "$TFO" | jq -r .name.value)
+export CONSUL_HTTP_ADDR=$(echo "$TFO" | jq -r .consul_http_addr.value)
+export CONSUL_HTTP_TOKEN=$(echo "$TFO" | jq -r .consul_http_token.value)
+export CONSUL_DC=$(echo "$TFO" | jq -r .consul_datacenter.value)
+export HASHICUPS_ADDR=$(kubectl get svc | grep 'api-gateway.*LoadBalancer' | awk '{ print $4}')
+export POLICY_NAME="payments-lambda-tgw"
+export CONSUL_MESH_GATEWAY_URI=$(kubectl get svc | grep consul-mesh-gateway | awk '{ print $4":443"}')
+export TGW_TOKEN=$(consul acl token list -format=json | jq '.[] | select(.Roles[]?.Name | contains("terminating-gateway"))' | jq -r '.AccessorID')
+
+if ! consul acl policy list | grep -q 'payments-lambda'; then
+  # allow the terminating gateway to write policies and read intentions on the payments components
+  # only do this once.
+  consul acl policy create -name "${POLICY_NAME}" -description "Allows Terminating Gateway to pass traffic from the payments Lambda function" -rules @lambda/tgw-policy.hcl &> /dev/null
+  consul acl token update -id $TGW_TOKEN -policy-name $POLICY_NAME -merge-policies -merge-roles  &> /dev/null
+fi
+
+echo ""
+echo "Consul UI:          $CONSUL_HTTP_ADDR"
+echo "Consul login token: $CONSUL_HTTP_TOKEN"
+echo "HashiCups UI:       $HASHICUPS_ADDR"

lambda/gateway-registration.yaml

@@ -0,0 +1,7 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: TerminatingGateway
+metadata:
+  name: terminating-gateway
+spec:
+  services:
+    - name: lambda-payments

lambda/lambda-iam.tf

@@ -0,0 +1,55 @@
+resource "aws_iam_policy" "lambda" {
+  name        = "${var.name}-lambda-policy"
+  path        = "/"
+  description = "IAM policy for Lambda functions"
+
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "arn:aws:logs:*:*:*",
+      "Effect": "Allow"
+    },
+    {
+      "Action": [
+        "ssm:GetParameter"
+      ],
+      "Resource": "arn:aws:ssm:*:*:parameter/${var.extension_data_prefix}/*",
+      "Effect": "Allow"
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role" "lambda" {
+  name = "${var.name}-lambda-role"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "lambda.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "lambda" {
+  role       = aws_iam_role.lambda.name
+  policy_arn = aws_iam_policy.lambda.arn
+}
+

lambda/lambda-payments.tf

@@ -0,0 +1,11 @@
+resource "aws_lambda_function" "lambda_payments" {
+  function_name    = "lambda-payments"
+  filename         = "lambda-payments.zip"
+  source_code_hash = filebase64sha256("lambda-payments.zip")
+  role             = aws_iam_role.lambda.arn
+  handler          = "lambda-payments"
+  runtime          = "go1.x"
+  tags = {
+    "serverless.consul.hashicorp.com/v1alpha1/lambda/enabled" = "true"
+  }
+}

lambda/lambda-products.tf

@@ -0,0 +1,34 @@
+resource "aws_lambda_layer_version" "consul_lambda_extension" {
+  layer_name       = "consul-lambda-extension"
+  filename         = "consul-lambda-extension.zip"
+  source_code_hash = filebase64sha256("consul-lambda-extension.zip")
+  description      = "Consul service mesh extension for AWS Lambda"
+}
+
+resource "aws_lambda_function" "lambda_products" {
+  function_name    = "lambda-products"
+  filename         = "lambda-products.zip"
+  source_code_hash = filebase64sha256("lambda-products.zip")
+  role             = aws_iam_role.lambda.arn
+  handler          = "lambda-products"
+  runtime          = "go1.x"
+  layers           = [aws_lambda_layer_version.consul_lambda_extension.arn]
+  tags = {
+    "serverless.consul.hashicorp.com/v1alpha1/lambda/enabled" : "true"
+  }
+
+  environment {
+    variables = {
+      CONSUL_DATACENTER            = var.consul_datacenter
+      CONSUL_EXTENSION_DATA_PREFIX = "/${var.extension_data_prefix}"
+      CONSUL_MESH_GATEWAY_URI      = var.consul_mesh_gateway_uri
+      CONSUL_SERVICE_UPSTREAMS     = "product-api-db:5432:${var.consul_datacenter}"
+
+      PGHOST     = "localhost"
+      PGPORT     = "5432"
+      PGDATABASE = "products"
+      PGUSER     = "postgres"
+      PGPASSWORD = "password"
+    }
+  }
+}

lambda/service-intentions.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ServiceIntentions
+metadata:
+  name: lambda-payments
+spec:
+  destination:
+    name: lambda-payments
+  sources:
+    - name: public-api
+      action: allow
+
+---
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ServiceIntentions
+metadata:
+  name: product-api-db
+spec:
+  destination:
+    name: product-api-db
+  sources:
+    - name: product-api
+      action: allow
+    - name: lambda-products
+      action: allow

lambda/service-splitter.yaml

@@ -0,0 +1,10 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ServiceSplitter
+metadata:
+  name: payments
+spec:
+  splits:
+    - weight: 100
+      service: lambda-payments
+    - weight: 0
+      service: payments

lambda/tgw-policy.hcl

@@ -0,0 +1,9 @@
+service "lambda-payments" {
+  policy = "write"
+  intentions = "read"
+}
+
+service "payments" {
+  policy = "write"
+  intentions = "read"
+}

lambda/variables.tf

@@ -0,0 +1,24 @@
+variable "name" {
+  description = "The name of the deployment."
+  type        = string
+}
+
+variable "region" {
+  description = "The AWS region to deploy to."
+  type        = string
+}
+
+variable "consul_datacenter" {
+  description = "The Consul datacenter."
+  type        = string
+}
+
+variable "consul_mesh_gateway_uri" {
+  description = "The public address for the Consul mesh gateway."
+  type        = string
+}
+
+variable "extension_data_prefix" {
+  description = "The path prefix for the Consul Lambda extension data."
+  type        = string
+}

main.tf

@@ -0,0 +1,42 @@
+# Deploy infrastructure: AWS VPC, EKS, Consul
+module "infra" {
+  source = "./modules/infra"
+
+  name                  = var.name
+  region                = var.region
+  vpc_cidr              = var.vpc_cidr
+  private_subnets       = var.private_subnets
+  public_subnets        = var.public_subnets
+  iam_path              = var.iam_path
+  hvn_cidr              = var.hvn_cidr
+  consul_datacenter     = var.consul_datacenter
+  consul_version        = var.consul_version
+  consul_tier           = var.consul_tier
+  consul_lambda_version = var.consul_lambda_version
+  image_rgy_url         = var.image_rgy_url
+  k8s_version           = var.k8s_version
+}
+
+# Deploy HashiCups on Kubernetes
+module "k8s_hashicups" {
+  source = "./modules/k8s"
+
+  hcp_cluster_id        = module.infra.hcp_cluster_id
+  consul_hosts          = module.infra.consul_hosts
+  k8s_api_endpoint      = module.infra.k8s_api_endpoint
+  consul_version        = module.infra.consul_version
+  boostrap_acl_token    = module.infra.boostrap_acl_token
+  consul_ca_file        = module.infra.consul_ca_file
+  datacenter            = module.infra.datacenter
+  gossip_encryption_key = module.infra.gossip_encryption_key
+  eks_cluster_id        = module.infra.eks_cluster_id
+  region                = var.region
+  security_group        = module.infra.security_group
+}
+
+
+module "remove_kubernetes_backed_enis" {
+  source = "github.com/webdog/terraform-kubernetes-delete-eni"
+  vpc_id = module.infra.vpc.vpc_id
+  region = var.region
+}

modules/api-gw-crd/main.tf

@@ -0,0 +1,28 @@
+terraform {
+  required_providers {
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = "1.14.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = ">= 2.3.0"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.12.1"
+    }
+  }
+}
+
+variable "crd_path" {
+  type        = string
+  description = "Where CRDs for the API Gateway are located"
+  default     = "./modules/k8s/api-gw/crd/*.yaml"
+}
+
+resource "kubectl_manifest" "consul_api_gateway" {
+  for_each  = fileset(path.root, var.crd_path)
+  yaml_body = file(each.value)
+}
+