From fc8061cf588f6509e6c8b32fa5eef6d326ff5e15 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 30 Oct 2024 09:45:48 +0000 Subject: [PATCH] move PV over to ecs task --- terraform/modules/services/pv/README.md | 8 --- terraform/modules/services/pv/cloudwatch.tf | 12 ---- terraform/modules/services/pv/ecs.tf | 70 --------------------- terraform/modules/services/pv/iam.tf | 18 ------ terraform/modules/services/pv/secrets.tf | 19 ------ terraform/modules/services/pv/variables.tf | 43 ------------- terraform/nowcasting/development/main.tf | 50 ++++++++++++--- 7 files changed, 42 insertions(+), 178 deletions(-) delete mode 100644 terraform/modules/services/pv/README.md delete mode 100644 terraform/modules/services/pv/cloudwatch.tf delete mode 100644 terraform/modules/services/pv/ecs.tf delete mode 100644 terraform/modules/services/pv/iam.tf delete mode 100644 terraform/modules/services/pv/secrets.tf delete mode 100644 terraform/modules/services/pv/variables.tf diff --git a/terraform/modules/services/pv/README.md b/terraform/modules/services/pv/README.md deleted file mode 100644 index 0aae2d3b..00000000 --- a/terraform/modules/services/pv/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Modules/Services/PV - -This module makes -- AWS task definition -- IAM role to setup application -- IAM role for running task -- get secrets for PV API -- temp: scheduled aws task, and iam roles diff --git a/terraform/modules/services/pv/cloudwatch.tf b/terraform/modules/services/pv/cloudwatch.tf deleted file mode 100644 index 89b97010..00000000 --- a/terraform/modules/services/pv/cloudwatch.tf +++ /dev/null @@ -1,12 +0,0 @@ -# set up cloudwatch log group - -resource "aws_cloudwatch_log_group" "pv" { - name = var.log-group-name - - retention_in_days = 7 - - tags = { - Environment = var.environment - Application = "nowcasting" - } -} diff --git a/terraform/modules/services/pv/ecs.tf b/terraform/modules/services/pv/ecs.tf deleted file mode 100644 index 80ff7381..00000000 --- a/terraform/modules/services/pv/ecs.tf +++ /dev/null @@ -1,70 +0,0 @@ -# define aws ecs task definition -# needs access to the internet - -resource "aws_ecs_task_definition" "pv-task-definition" { - family = "pv" - requires_compatibilities = ["FARGATE"] - network_mode = "awsvpc" - - # specific values are needed - - # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html - cpu = 256 - memory = 512 - - tags = { - name = "pv-consumer" - type = "ecs" - } - - task_role_arn = aws_iam_role.consumer-pv-iam-role.arn - execution_role_arn = var.ecs-task_execution_role_arn - container_definitions = jsonencode([ - { - name = "pv-ss-consumer" - image = "openclimatefix/pvconsumer:${var.docker_version_ss}" - # cpu = 128 - # memory = 128 - essential = true - - environment : [ - { "name" : "LOGLEVEL", "value" : "DEBUG"}, - { "name" :"DATA_SERVICE_URL", "value" : "https://pvoutput.org/"}, - { "name" : "PROVIDER", "value" : "solar_sheffield_passiv"} - ] - - secrets : [ - { - "name" : "SS_USER_ID", - "valueFrom" : "${data.aws_secretsmanager_secret_version.pv-ss-version.arn}:user_id::", - }, - { - "name" : "SS_KEY", - "valueFrom" : "${data.aws_secretsmanager_secret_version.pv-ss-version.arn}:key::", - }, - { - "name" : "SS_URL", - "valueFrom" : "${data.aws_secretsmanager_secret_version.pv-ss-version.arn}:url::", - }, - { - "name" : "DB_URL_FORECAST", - "valueFrom" : "${var.database_secret_forecast.arn}:url::", - }, - { - "name" : "DB_URL_PV_SITE", - "valueFrom" : "${data.aws_secretsmanager_secret_version.pv-sites-database-version.arn}:url::", - }, - ] - - logConfiguration : { - "logDriver" : "awslogs", - "options" : { - "awslogs-group" : var.log-group-name, - "awslogs-region" : var.region, - "awslogs-stream-prefix" : "streaming" - } - } - } - ]) - - # add volume? So we dont have to keep downloading same docker image -} diff --git a/terraform/modules/services/pv/iam.tf b/terraform/modules/services/pv/iam.tf deleted file mode 100644 index 6afe6d3d..00000000 --- a/terraform/modules/services/pv/iam.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Define the IAM task Instance role used to run the task - -data "aws_iam_policy_document" "ec2-instance-assume-role-policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["ecs-tasks.amazonaws.com"] - } - } -} - -resource "aws_iam_role" "consumer-pv-iam-role" { - name = "consumer-pv-iam-role" - path = "/consumer/" - assume_role_policy = data.aws_iam_policy_document.ec2-instance-assume-role-policy.json -} diff --git a/terraform/modules/services/pv/secrets.tf b/terraform/modules/services/pv/secrets.tf deleted file mode 100644 index 2a3b9a2c..00000000 --- a/terraform/modules/services/pv/secrets.tf +++ /dev/null @@ -1,19 +0,0 @@ -# Read in secrets for API for pvoutput.org API - -data "aws_secretsmanager_secret" "pv-ss" { - name = "${var.environment}/consumer/solar_sheffield" -} - -data "aws_secretsmanager_secret_version" "pv-ss-version" { - secret_id = data.aws_secretsmanager_secret.pv-ss.id -} - - -# This gets the URL for the pv sites database -data "aws_secretsmanager_secret" "pv-sites-database" { - name = "${var.environment}/rds/pvsite" -} - -data "aws_secretsmanager_secret_version" "pv-sites-database-version" { - secret_id = data.aws_secretsmanager_secret.pv-sites-database.id -} diff --git a/terraform/modules/services/pv/variables.tf b/terraform/modules/services/pv/variables.tf deleted file mode 100644 index 40d727a7..00000000 --- a/terraform/modules/services/pv/variables.tf +++ /dev/null @@ -1,43 +0,0 @@ - -variable "environment" { - description = "The Deployment environment" -} - - -variable "region" { - description = "The AWS region" -} - - -variable "log-group-name" { - description = "The log group name where log streams are saved" - default = "/aws/ecs/consumer/pv/" -} - -variable "public_subnet_ids" { - type = list(string) - description = "Public subnet ids" -} - -variable "database_secret_forecast" { - description = "AWS secret that gives connection details to the forecast database" -} - -variable "iam-policy-rds-read-secret_forecast" { - description = "IAM policy to be able to read the forecast RDS secret" -} - -variable "docker_version_ss" { - description = "The version of the docker that should be used for the solar sheffield pv consumer" -} - - -variable "pv_provider" { - description = "The provider that this service uses. Can be pvoutput.org or solar_sheffield_passiv" - default = "pvoutput.org" -} - -variable "ecs-task_execution_role_arn" { - description = "The arn of the ECS cluster task execution role" - type = string -} diff --git a/terraform/nowcasting/development/main.tf b/terraform/nowcasting/development/main.tf index c29ed209..b68ae153 100644 --- a/terraform/nowcasting/development/main.tf +++ b/terraform/nowcasting/development/main.tf @@ -12,6 +12,7 @@ The componentes ares: 2.1 - Database 2.2 - NWP Consumer Secret 2.3 - Satellite Consumer Secret +2.4 - PV Secret 3.2 - NWP Consumer (MetOffice National) 3.3 - NWP Consumer (ECMWF UK) 3.4 - Satellite Consumer @@ -139,6 +140,17 @@ resource "aws_secretsmanager_secret" "satellite_consumer_secret" { name = "${local.environment}/data/satellite-consumer" } +# 2.4 +# TODO remove +import { + to = aws_secretsmanager_secret.pv_consumer_secret + id = "arn:aws:secretsmanager:eu-west-1:008129123253:secret:development/consumer/solar_sheffield-2Tyskm" +} + +resource "aws_secretsmanager_secret" "pv_consumer_secret" { + name = "${local.environment}/data/solar-sheffield" +} + # 3.2 module "nwp-national" { @@ -315,15 +327,37 @@ module "sat_clean_up" { # 3.6 module "pv" { - source = "../../modules/services/pv" - - region = var.region - environment = local.environment - public_subnet_ids = module.networking.public_subnet_ids - database_secret_forecast = module.database.forecast-database-secret - docker_version_ss = var.pv_ss_version - iam-policy-rds-read-secret_forecast = module.database.iam-policy-forecast-db-read + source = "../../modules/services/ecs_task" + + ecs-task_name = "pv" + ecs-task_type = "consumer" ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn + ecs-task_size = { + cpu = 256 + memory = 512 + } + + aws-region = var.region + aws-environment = local.environment + + s3-buckets = [] + + container-env_vars = [ + { "name" : "SENTRY_DSN", "value" : var.sentry_dsn }, + { "name" : "ENVIRONMENT", "value" : local.environment }, + { "name" : "LOGLEVEL", "value" : "INFO"}, + { "name" : "PROVIDER", "value" : "solar_sheffield_passiv"}, + ] + container-secret_vars = [ + {secret_policy_arn: module.pvsite_database.secret.arn, + values: ["DB_URL"]}, + {secret_policy_arn: module.aws_secretsmanager_secret.pv_consumer_secret.arn, + values: ["SS_USER_ID", "SS_KEY", "SS_URL"]} + ] + container-tag = var.pv_ss_version + container-name = "openclimatefix/pvconsumer" + container-registry = "docker.io" + container-command = [] } # 3.7