From 584e791addb7deea31674f7c2848b6d62e0c2022 Mon Sep 17 00:00:00 2001 From: roch Date: Tue, 14 Jan 2025 20:12:42 +0000 Subject: [PATCH] Import gcp resources creation code from search-api-v2 repo --- .../deployments/gcp-search-api-v2/README.md | 44 ++++++ .../deployments/gcp-search-api-v2/main.tf | 80 ++++++++++ .../modules/search-api-v2/main.tf | 145 ++++++++++++++++++ .../modules/search-api-v2/variables.tf | 81 ++++++++++ .../gcp-search-api-v2/variables.tf | 32 ++++ 5 files changed, 382 insertions(+) create mode 100644 terraform/deployments/gcp-search-api-v2/README.md create mode 100644 terraform/deployments/gcp-search-api-v2/main.tf create mode 100644 terraform/deployments/gcp-search-api-v2/modules/search-api-v2/main.tf create mode 100644 terraform/deployments/gcp-search-api-v2/modules/search-api-v2/variables.tf create mode 100644 terraform/deployments/gcp-search-api-v2/variables.tf diff --git a/terraform/deployments/gcp-search-api-v2/README.md b/terraform/deployments/gcp-search-api-v2/README.md new file mode 100644 index 000000000..9835e08e5 --- /dev/null +++ b/terraform/deployments/gcp-search-api-v2/README.md @@ -0,0 +1,44 @@ +# gcp-search-api-v2 +Terraform module to bootstrap GCP projects + +## Resources +This module manages the following resources: +- For every desired environment (dev, integration, staging, prod), through the `modules/environment` + child module: + - A GCP project + +## Applying this module +This module uses Terraform Cloud for remote state storage, but is intended to be run *locally* by a +user with "interactive" end-user access to both Terraform Cloud and Google Cloud Platform (so as to +not have a chicken-and-egg problem around having to manually create service accounts to manage +meta-resources like service accounts or projects). + +### Authentication & configuration +Before you can use this module, you must: +- use `terraform login` to authenticate to Terraform Cloud +- use `gcloud auth application-default login` to authenticate to GCP +- specify values for `google_cloud_folder` and `google_cloud_billing_account` as parameters to + `terraform` or through a (gitignored) `local.auto.tfvars` file + +## Additional information +### Adding additional GCP quota overrides +Quota overrides on GCP are somewhat complex to set up and use inconsistent terminology between the +console UI, the REST API, and the (beta) Terraform provider. In particular, it can be somewhat +confusing to figure out the `limit` value for the `google_service_usage_consumer_quota_override` +resource (which actually corresponds to the `unit` field in the API but with different syntax), and +to find the internal (not display) name of quotas. + +If you need to set up a new `google_service_usage_consumer_quota_override` resource for a Discovery +Engine project, the best way of finding out these values is to make a GET request to the +`consumerQuotaMetrics` endpoint like so: + +```bash +curl -H "Authorization: Bearer $(gcloud auth print-access-token)" \ +-H "Content-Type: application/json" \ +"https://serviceusage.googleapis.com/v1beta1/projects/${GCP_PROJECT}/services/discoveryengine.googleapis.com/consumerQuotaMetrics" \ +| jq -r '.metrics[] | "\(.displayName): \(.consumerQuotaLimits[0].metric) (\(.consumerQuotaLimits[0].unit | gsub("[1\\{\\}]";"")))"' \ +| sort +``` + +This returns a list of available quotas by display name, complete with the necessary `metric` and +`unit` values. diff --git a/terraform/deployments/gcp-search-api-v2/main.tf b/terraform/deployments/gcp-search-api-v2/main.tf new file mode 100644 index 000000000..88239186f --- /dev/null +++ b/terraform/deployments/gcp-search-api-v2/main.tf @@ -0,0 +1,80 @@ +terraform { + cloud { + organization = "govuk" + workspaces { + project = "govuk-search-api-v2" + name = "search-api-v2-meta" + } + } + + required_providers { + tfe = { + source = "hashicorp/tfe" + version = "~> 0.55.0" + } + google = { + source = "hashicorp/google" + version = "~> 5.20" + } + } + + required_version = "~> 1.7" +} + +# NOTE: This is used to store the state for this module itself (see `terraform` block above). It was +# initially created using a local backend, and then migrated to a remote backend. +resource "tfe_workspace" "meta_workspace" { + name = "search-api-v2-meta" + project_id = var.project_id + description = "Meta workspace for cross-environment TF Cloud resources (state backend only)" + tag_names = ["govuk", "search-api-v2"] +} + +resource "tfe_workspace_settings" "meta_workspace_settings" { + workspace_id = tfe_workspace.meta_workspace.id + + execution_mode = "local" +} + +module "environment_integration" { + source = "./modules/search-api-v2" + + name = "integration" + google_cloud_billing_account = var.google_cloud_billing_account + google_cloud_folder = var.google_cloud_folder + tfc_project_name = var.tfe_project_name + environment_workspace_name = "search-api-v2-integration" +} + +module "environment_staging" { + source = "./modules/search-api-v2" + + name = "staging" + upstream_environment_name = "integration" + + google_cloud_billing_account = var.google_cloud_billing_account + google_cloud_folder = var.google_cloud_folder + tfc_project_name = var.tfe_project_name + environment_workspace_name = "search-api-v2-staging" +} + +module "environment_production" { + source = "./modules/search-api-v2" + + name = "production" + upstream_environment_name = "staging" + + google_cloud_billing_account = var.google_cloud_billing_account + google_cloud_folder = var.google_cloud_folder + tfc_project_name = var.tfe_project_name + environment_workspace_name = "search-api-v2-production" + + + # NOTE: There are limits on the Google side on how high we are permitted to set these quotas. If + # you attempt to increase these beyond the ceiling, a `COMMON_QUOTA_CONSUMER_OVERRIDE_TOO_HIGH` + # error will be raised (including some metadata that should tell you what the current ceiling is) + # and you will need to manually request a quota increase from Google through the console first + # (see the environment module for the exact quota names you need to request increases for). + discovery_engine_quota_search_requests_per_minute = 5000 + discovery_engine_quota_documents = 2000000 +} diff --git a/terraform/deployments/gcp-search-api-v2/modules/search-api-v2/main.tf b/terraform/deployments/gcp-search-api-v2/modules/search-api-v2/main.tf new file mode 100644 index 000000000..6efa049d6 --- /dev/null +++ b/terraform/deployments/gcp-search-api-v2/modules/search-api-v2/main.tf @@ -0,0 +1,145 @@ +terraform { + required_providers { + tfe = { + source = "hashicorp/tfe" + version = "~> 0.55.0" + } + google = { + source = "hashicorp/google" + version = "~> 5.20" + } + # required for `google_service_usage_consumer_quota_override` resources + google-beta = { + source = "hashicorp/google-beta" + version = "~> 5.20" + } + } + + required_version = "~> 1.7" +} + +locals { + display_name = title(var.name) +} + +resource "google_project" "environment_project" { + name = "Search API V2 ${local.display_name}" + project_id = "search-api-v2-${var.name}" + + folder_id = var.google_cloud_folder + billing_account = var.google_cloud_billing_account + + labels = { + "programme" = "govuk" + "team" = "govuk-search-improvement" + "govuk_environment" = var.name + } +} + +resource "google_project_iam_member" "environment_project_owner" { + project = google_project.environment_project.project_id + role = "roles/owner" + + member = "group:govuk-gcp-access@digital.cabinet-office.gov.uk" +} + +resource "google_project_service" "api_service" { + for_each = var.google_cloud_apis + + project = google_project.environment_project.project_id + service = each.value + disable_dependent_services = true +} + +resource "google_service_usage_consumer_quota_override" "discoveryengine_search_requests" { + provider = google-beta + project = google_project.environment_project.project_id + + service = "discoveryengine.googleapis.com" + metric = urlencode("discoveryengine.googleapis.com/search_requests") + force = true + + # limit is equivalent to `unit` field when making a GET request against the metric, but without + # leading `1/` and without curly braces + limit = urlencode("/min/project") + override_value = var.discovery_engine_quota_search_requests_per_minute +} + +resource "google_service_usage_consumer_quota_override" "discoveryengine_documents" { + provider = google-beta + project = google_project.environment_project.project_id + + service = "discoveryengine.googleapis.com" + metric = urlencode("discoveryengine.googleapis.com/documents") + force = true + + # limit is equivalent to `unit` field when making a GET request against the metric, but without + # leading `1/` and without curly braces + limit = urlencode("/project") + override_value = var.discovery_engine_quota_documents +} + +data "tfe_oauth_client" "github" { + organization = var.tfc_organization_name + service_provider = "github" +} + +# Set up Workload Identity Federation between Terraform Cloud and GCP +# see https://github.com/hashicorp/terraform-dynamic-credentials-setup-examples +resource "google_iam_workload_identity_pool" "tfc_pool" { + project = google_project.environment_project.project_id + workload_identity_pool_id = "terraform-cloud-id-pool" + + display_name = "Terraform Cloud ID Pool" + description = "Pool to enable access to project resources for Terraform Cloud" +} + +resource "google_iam_workload_identity_pool_provider" "tfc_provider" { + project = google_project.environment_project.project_id + workload_identity_pool_id = google_iam_workload_identity_pool.tfc_pool.workload_identity_pool_id + workload_identity_pool_provider_id = "terraform-cloud-provider-oidc" + + display_name = "Terraform Cloud OIDC Provider" + description = "Configures Terraform Cloud as an external identity provider for this project" + + attribute_mapping = { + "google.subject" = "assertion.sub", + "attribute.aud" = "assertion.aud", + "attribute.terraform_run_phase" = "assertion.terraform_run_phase", + "attribute.terraform_project_id" = "assertion.terraform_project_id", + "attribute.terraform_project_name" = "assertion.terraform_project_name", + "attribute.terraform_workspace_id" = "assertion.terraform_workspace_id", + "attribute.terraform_workspace_name" = "assertion.terraform_workspace_name", + "attribute.terraform_organization_id" = "assertion.terraform_organization_id", + "attribute.terraform_organization_name" = "assertion.terraform_organization_name", + "attribute.terraform_run_id" = "assertion.terraform_run_id", + "attribute.terraform_full_workspace" = "assertion.terraform_full_workspace", + } + + oidc { + issuer_uri = "https://${var.tfc_hostname}" + } + + attribute_condition = "assertion.sub.startsWith(\"organization:${var.tfc_organization_name}:project:${var.tfc_project_name}:workspace:${var.environment_workspace_name}\")" +} + +resource "google_service_account" "tfc_service_account" { + project = google_project.environment_project.project_id + + account_id = "tfc-service-account" + display_name = "Terraform Cloud Service Account" + description = "Used by Terraform Cloud to manage resources in this project through Workload Identity Federation" +} + +resource "google_service_account_iam_member" "tfc_service_account_member" { + service_account_id = google_service_account.tfc_service_account.name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.tfc_pool.name}/*" +} + +resource "google_project_iam_member" "tfc_project_member" { + project = google_project.environment_project.project_id + + role = "roles/owner" + member = "serviceAccount:${google_service_account.tfc_service_account.email}" +} diff --git a/terraform/deployments/gcp-search-api-v2/modules/search-api-v2/variables.tf b/terraform/deployments/gcp-search-api-v2/modules/search-api-v2/variables.tf new file mode 100644 index 000000000..1073d4b72 --- /dev/null +++ b/terraform/deployments/gcp-search-api-v2/modules/search-api-v2/variables.tf @@ -0,0 +1,81 @@ +variable "name" { + type = string + description = "A short name for this environment (used in resource IDs)" +} + +variable "google_cloud_folder" { + type = string + description = "The ID of the Google Cloud folder to create projects under" +} + +variable "google_cloud_billing_account" { + type = string + description = "The ID of the Google Cloud billing account to associate projects with" +} + +variable "google_cloud_apis" { + type = set(string) + description = "The Google Cloud APIs to enable for the project" + default = [ + # Required to be able to manage resources using Terraform + "cloudresourcemanager.googleapis.com", + # Required to set up service accounts and manage dynamic credentials + "iam.googleapis.com", + "iamcredentials.googleapis.com", + "sts.googleapis.com", + # Required for Discovery Engine + "discoveryengine.googleapis.com", + # Required for event data pipeline + "bigquery.googleapis.com", + "bigquerystorage.googleapis.com", + "storage.googleapis.com", + "cloudbuild.googleapis.com", + "artifactregistry.googleapis.com", + "cloudfunctions.googleapis.com", + "run.googleapis.com", + "cloudscheduler.googleapis.com", + # Required for observability + "logging.googleapis.com", + "monitoring.googleapis.com", + ] +} + +variable "discovery_engine_quota_search_requests_per_minute" { + type = number + description = "The maximum number of search requests per minute for the Discovery Engine" + default = 250 +} + +variable "discovery_engine_quota_documents" { + type = number + description = "The maximum number of documents across Discovery Engine datastores" + default = 1000000 +} + +variable "upstream_environment_name" { + type = string + description = "The name of the upstream environment, if any (used to wait for a successful apply on a 'lower' environment before applying this one)" + default = null +} + +variable "tfc_hostname" { + type = string + description = "The hostname of the Terraform Cloud/Enterprise instance to use" + default = "app.terraform.io" +} + +variable "tfc_organization_name" { + type = string + description = "The name of the Terraform Cloud/Enterprise organization to use" + default = "govuk" +} + +variable "tfc_project_name" { + type = string + description = "The name of the overarching terraform cloud project for all workspaces" +} + +variable "environment_workspace_name" { + type = string + description = "Provisions search-api-v2 Discovery Engine resources for the environment" +} diff --git a/terraform/deployments/gcp-search-api-v2/variables.tf b/terraform/deployments/gcp-search-api-v2/variables.tf new file mode 100644 index 000000000..5f507717f --- /dev/null +++ b/terraform/deployments/gcp-search-api-v2/variables.tf @@ -0,0 +1,32 @@ +variable "tfc_hostname" { + type = string + default = "app.terraform.io" + description = "The hostname of the TFC or TFE to use with AWS" +} + +variable "tfc_organization_name" { + type = string + default = "govuk" + description = "The name of the Terraform Cloud organization" +} + +variable "google_cloud_folder" { + type = string + description = "The ID of the Google Cloud folder to create projects under" +} + +variable "google_cloud_billing_account" { + type = string + description = "The ID of the Google Cloud billing account to associate projects with" +} + +variable "project_id" { + type = string + description = "The ID of the overarching terraform cloud project for all workspaces" +} + +variable "tfe_project_name" { + type = string + default = "govuk-search-api-v2" + description = "The name of the overarching terraform cloud project for all workspaces" +}