Merge pull request #1583 from alphagov/search_gcp

Import gcp resources creation code from search-api-v2 repo

terraform/deployments/gcp-search-api-v2/README.md

@@ -0,0 +1,44 @@
+# gcp-search-api-v2
+Terraform module to bootstrap GCP projects
+
+## Resources
+This module manages the following resources:
+- For every desired environment (dev, integration, staging, prod), through the `modules/environment`
+  child module:
+  - A GCP project
+
+## Applying this module
+This module uses Terraform Cloud for remote state storage, but is intended to be run *locally* by a
+user with "interactive" end-user access to both Terraform Cloud and Google Cloud Platform (so as to
+not have a chicken-and-egg problem around having to manually create service accounts to manage
+meta-resources like service accounts or projects).
+
+### Authentication & configuration
+Before you can use this module, you must:
+- use `terraform login` to authenticate to Terraform Cloud
+- use `gcloud auth application-default login` to authenticate to GCP
+- specify values for `google_cloud_folder` and `google_cloud_billing_account` as parameters to
+  `terraform` or through a (gitignored) `local.auto.tfvars` file
+
+## Additional information
+### Adding additional GCP quota overrides
+Quota overrides on GCP are somewhat complex to set up and use inconsistent terminology between the
+console UI, the REST API, and the (beta) Terraform provider. In particular, it can be somewhat
+confusing to figure out the `limit` value for the `google_service_usage_consumer_quota_override`
+resource (which actually corresponds to the `unit` field in the API but with different syntax), and
+to find the internal (not display) name of quotas.
+
+If you need to set up a new `google_service_usage_consumer_quota_override` resource for a Discovery
+Engine project, the best way of finding out these values is to make a GET request to the
+`consumerQuotaMetrics` endpoint like so:
+
+```bash
+curl -H "Authorization: Bearer $(gcloud auth print-access-token)" \
+-H "Content-Type: application/json" \
+"https://serviceusage.googleapis.com/v1beta1/projects/${GCP_PROJECT}/services/discoveryengine.googleapis.com/consumerQuotaMetrics" \
+| jq -r '.metrics[] | "\(.displayName): \(.consumerQuotaLimits[0].metric) (\(.consumerQuotaLimits[0].unit | gsub("[1\\{\\}]";"")))"' \
+| sort
+```
+
+This returns a list of available quotas by display name, complete with the necessary `metric` and
+`unit` values.

terraform/deployments/gcp-search-api-v2/main.tf

@@ -0,0 +1,65 @@
+terraform {
+  cloud {
+    organization = "govuk"
+    workspaces {
+      project = "govuk-search-api-v2"
+      name    = "search-api-v2-meta"
+    }
+  }
+
+  required_providers {
+    tfe = {
+      source  = "hashicorp/tfe"
+      version = "~> 0.55.0"
+    }
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 5.20"
+    }
+  }
+
+  required_version = "~> 1.7"
+}
+
+module "environment_integration" {
+  source = "./modules/search-api-v2"
+
+  name                         = "integration"
+  google_cloud_billing_account = var.google_cloud_billing_account
+  google_cloud_folder          = var.google_cloud_folder
+  tfc_project_name             = var.tfe_project_name
+  environment_workspace_name   = "search-api-v2-integration"
+}
+
+module "environment_staging" {
+  source = "./modules/search-api-v2"
+
+  name                      = "staging"
+  upstream_environment_name = "integration"
+
+  google_cloud_billing_account = var.google_cloud_billing_account
+  google_cloud_folder          = var.google_cloud_folder
+  tfc_project_name             = var.tfe_project_name
+  environment_workspace_name   = "search-api-v2-staging"
+}
+
+module "environment_production" {
+  source = "./modules/search-api-v2"
+
+  name                      = "production"
+  upstream_environment_name = "staging"
+
+  google_cloud_billing_account = var.google_cloud_billing_account
+  google_cloud_folder          = var.google_cloud_folder
+  tfc_project_name             = var.tfe_project_name
+  environment_workspace_name   = "search-api-v2-production"
+
+
+  # NOTE: There are limits on the Google side on how high we are permitted to set these quotas. If
+  # you attempt to increase these beyond the ceiling, a `COMMON_QUOTA_CONSUMER_OVERRIDE_TOO_HIGH`
+  # error will be raised (including some metadata that should tell you what the current ceiling is)
+  # and you will need to manually request a quota increase from Google through the console first
+  # (see the environment module for the exact quota names you need to request increases for).
+  discovery_engine_quota_search_requests_per_minute = 5000
+  discovery_engine_quota_documents                  = 2000000
+}

terraform/deployments/gcp-search-api-v2/modules/search-api-v2/main.tf

@@ -0,0 +1,145 @@
+terraform {
+  required_providers {
+    tfe = {
+      source  = "hashicorp/tfe"
+      version = "~> 0.55.0"
+    }
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 5.20"
+    }
+    # required for `google_service_usage_consumer_quota_override` resources
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 5.20"
+    }
+  }
+
+  required_version = "~> 1.7"
+}
+
+locals {
+  display_name = title(var.name)
+}
+
+resource "google_project" "environment_project" {
+  name       = "Search API V2 ${local.display_name}"
+  project_id = "search-api-v2-${var.name}"
+
+  folder_id       = var.google_cloud_folder
+  billing_account = var.google_cloud_billing_account
+
+  labels = {
+    "programme"         = "govuk"
+    "team"              = "govuk-search-improvement"
+    "govuk_environment" = var.name
+  }
+}
+
+resource "google_project_iam_member" "environment_project_owner" {
+  project = google_project.environment_project.project_id
+  role    = "roles/owner"
+
+  member = "group:[email protected]"
+}
+
+resource "google_project_service" "api_service" {
+  for_each = var.google_cloud_apis
+
+  project                    = google_project.environment_project.project_id
+  service                    = each.value
+  disable_dependent_services = true
+}
+
+resource "google_service_usage_consumer_quota_override" "discoveryengine_search_requests" {
+  provider = google-beta
+  project  = google_project.environment_project.project_id
+
+  service = "discoveryengine.googleapis.com"
+  metric  = urlencode("discoveryengine.googleapis.com/search_requests")
+  force   = true
+
+  # limit is equivalent to `unit` field when making a GET request against the metric, but without
+  # leading `1/` and without curly braces
+  limit          = urlencode("/min/project")
+  override_value = var.discovery_engine_quota_search_requests_per_minute
+}
+
+resource "google_service_usage_consumer_quota_override" "discoveryengine_documents" {
+  provider = google-beta
+  project  = google_project.environment_project.project_id
+
+  service = "discoveryengine.googleapis.com"
+  metric  = urlencode("discoveryengine.googleapis.com/documents")
+  force   = true
+
+  # limit is equivalent to `unit` field when making a GET request against the metric, but without
+  # leading `1/` and without curly braces
+  limit          = urlencode("/project")
+  override_value = var.discovery_engine_quota_documents
+}
+
+data "tfe_oauth_client" "github" {
+  organization     = var.tfc_organization_name
+  service_provider = "github"
+}
+
+# Set up Workload Identity Federation between Terraform Cloud and GCP
+# see https://github.com/hashicorp/terraform-dynamic-credentials-setup-examples
+resource "google_iam_workload_identity_pool" "tfc_pool" {
+  project                   = google_project.environment_project.project_id
+  workload_identity_pool_id = "terraform-cloud-id-pool"
+
+  display_name = "Terraform Cloud ID Pool"
+  description  = "Pool to enable access to project resources for Terraform Cloud"
+}
+
+resource "google_iam_workload_identity_pool_provider" "tfc_provider" {
+  project                            = google_project.environment_project.project_id
+  workload_identity_pool_id          = google_iam_workload_identity_pool.tfc_pool.workload_identity_pool_id
+  workload_identity_pool_provider_id = "terraform-cloud-provider-oidc"
+
+  display_name = "Terraform Cloud OIDC Provider"
+  description  = "Configures Terraform Cloud as an external identity provider for this project"
+
+  attribute_mapping = {
+    "google.subject"                        = "assertion.sub",
+    "attribute.aud"                         = "assertion.aud",
+    "attribute.terraform_run_phase"         = "assertion.terraform_run_phase",
+    "attribute.terraform_project_id"        = "assertion.terraform_project_id",
+    "attribute.terraform_project_name"      = "assertion.terraform_project_name",
+    "attribute.terraform_workspace_id"      = "assertion.terraform_workspace_id",
+    "attribute.terraform_workspace_name"    = "assertion.terraform_workspace_name",
+    "attribute.terraform_organization_id"   = "assertion.terraform_organization_id",
+    "attribute.terraform_organization_name" = "assertion.terraform_organization_name",
+    "attribute.terraform_run_id"            = "assertion.terraform_run_id",
+    "attribute.terraform_full_workspace"    = "assertion.terraform_full_workspace",
+  }
+
+  oidc {
+    issuer_uri = "https://${var.tfc_hostname}"
+  }
+
+  attribute_condition = "assertion.sub.startsWith(\"organization:${var.tfc_organization_name}:project:${var.tfc_project_name}:workspace:${var.environment_workspace_name}\")"
+}
+
+resource "google_service_account" "tfc_service_account" {
+  project = google_project.environment_project.project_id
+
+  account_id   = "tfc-service-account"
+  display_name = "Terraform Cloud Service Account"
+  description  = "Used by Terraform Cloud to manage resources in this project through Workload Identity Federation"
+}
+
+resource "google_service_account_iam_member" "tfc_service_account_member" {
+  service_account_id = google_service_account.tfc_service_account.name
+  role               = "roles/iam.workloadIdentityUser"
+  member             = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.tfc_pool.name}/*"
+}
+
+resource "google_project_iam_member" "tfc_project_member" {
+  project = google_project.environment_project.project_id
+
+  role   = "roles/owner"
+  member = "serviceAccount:${google_service_account.tfc_service_account.email}"
+}

terraform/deployments/gcp-search-api-v2/modules/search-api-v2/variables.tf

@@ -0,0 +1,81 @@
+variable "name" {
+  type        = string
+  description = "A short name for this environment (used in resource IDs)"
+}
+
+variable "google_cloud_folder" {
+  type        = string
+  description = "The ID of the Google Cloud folder to create projects under"
+}
+
+variable "google_cloud_billing_account" {
+  type        = string
+  description = "The ID of the Google Cloud billing account to associate projects with"
+}
+
+variable "google_cloud_apis" {
+  type        = set(string)
+  description = "The Google Cloud APIs to enable for the project"
+  default = [
+    # Required to be able to manage resources using Terraform
+    "cloudresourcemanager.googleapis.com",
+    # Required to set up service accounts and manage dynamic credentials
+    "iam.googleapis.com",
+    "iamcredentials.googleapis.com",
+    "sts.googleapis.com",
+    # Required for Discovery Engine
+    "discoveryengine.googleapis.com",
+    # Required for event data pipeline
+    "bigquery.googleapis.com",
+    "bigquerystorage.googleapis.com",
+    "storage.googleapis.com",
+    "cloudbuild.googleapis.com",
+    "artifactregistry.googleapis.com",
+    "cloudfunctions.googleapis.com",
+    "run.googleapis.com",
+    "cloudscheduler.googleapis.com",
+    # Required for observability
+    "logging.googleapis.com",
+    "monitoring.googleapis.com",
+  ]
+}
+
+variable "discovery_engine_quota_search_requests_per_minute" {
+  type        = number
+  description = "The maximum number of search requests per minute for the Discovery Engine"
+  default     = 250
+}
+
+variable "discovery_engine_quota_documents" {
+  type        = number
+  description = "The maximum number of documents across Discovery Engine datastores"
+  default     = 1000000
+}
+
+variable "upstream_environment_name" {
+  type        = string
+  description = "The name of the upstream environment, if any (used to wait for a successful apply on a 'lower' environment before applying this one)"
+  default     = null
+}
+
+variable "tfc_hostname" {
+  type        = string
+  description = "The hostname of the Terraform Cloud/Enterprise instance to use"
+  default     = "app.terraform.io"
+}
+
+variable "tfc_organization_name" {
+  type        = string
+  description = "The name of the Terraform Cloud/Enterprise organization to use"
+  default     = "govuk"
+}
+
+variable "tfc_project_name" {
+  type        = string
+  description = "The  name of the overarching terraform cloud project for all workspaces"
+}
+
+variable "environment_workspace_name" {
+  type        = string
+  description = "Provisions search-api-v2 Discovery Engine resources for the environment"
+}

terraform/deployments/gcp-search-api-v2/variables.tf

@@ -0,0 +1,32 @@
+variable "tfc_hostname" {
+  type        = string
+  default     = "app.terraform.io"
+  description = "The hostname of the TFC or TFE to use with AWS"
+}
+
+variable "tfc_organization_name" {
+  type        = string
+  default     = "govuk"
+  description = "The name of the Terraform Cloud organization"
+}
+
+variable "google_cloud_folder" {
+  type        = string
+  description = "The ID of the Google Cloud folder to create projects under"
+}
+
+variable "google_cloud_billing_account" {
+  type        = string
+  description = "The ID of the Google Cloud billing account to associate projects with"
+}
+
+variable "project_id" {
+  type        = string
+  description = "The ID of the overarching terraform cloud project for all workspaces"
+}
+
+variable "tfe_project_name" {
+  type        = string
+  default     = "govuk-search-api-v2"
+  description = "The  name of the overarching terraform cloud project for all workspaces"
+}