From c8f0487a34b990442fbe392de00ad272bed414c6 Mon Sep 17 00:00:00 2001 From: Alexander Veit <53857412+alexander-veit@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:00:14 -0400 Subject: [PATCH] Add option to disable metrics collection (#407) * Disable metrics collection * Update requirements.txt * Bump version * Update CHANGELOG.rst --- CHANGELOG.rst | 6 +++++ awsf3/aws_run_workflow_generic.sh | 40 ++++++++++++++++++------------- docs/execution_json.rst | 4 ++++ pyproject.toml | 2 +- tibanna/ec2_utils.py | 4 ++++ tibanna/lambdas/requirements.txt | 2 +- 6 files changed, 40 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7cc3482d..15022667 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,12 @@ Change Log ========== +5.3.0 +===== + +* Add config option `disable_metrics_collection` + + 5.2.2 ===== diff --git a/awsf3/aws_run_workflow_generic.sh b/awsf3/aws_run_workflow_generic.sh index 4bb2f695..e3972fda 100755 --- a/awsf3/aws_run_workflow_generic.sh +++ b/awsf3/aws_run_workflow_generic.sh @@ -8,10 +8,11 @@ export ACCESS_KEY= export SECRET_KEY= export REGION= export SINGULARITY_OPTION_TO_PASS= +export DISABLE_METRICS_COLLECTION=false export S3_ENCRYPT_KEY_ID= printHelpAndExit() { - echo "Usage: ${0##*/} -i JOBID -l LOGBUCKET -V VERSION -A AWSF_IMAGE [-m SHUTDOWN_MIN] [-p PASSWORD] [-a ACCESS_KEY] [-s SECRET_KEY] [-r REGION] [-g] [-k S3_ENCRYPT_KEY_ID]" + echo "Usage: ${0##*/} -i JOBID -l LOGBUCKET -V VERSION -A AWSF_IMAGE [-m SHUTDOWN_MIN] [-p PASSWORD] [-a ACCESS_KEY] [-s SECRET_KEY] [-r REGION] [-g] [-c] [-k S3_ENCRYPT_KEY_ID]" echo "-i JOBID : awsem job id (required)" echo "-l LOGBUCKET : bucket for sending log file (required)" echo "-V TIBANNA_VERSION : tibanna version (used in the run_task lambda that launched this instance)" @@ -22,10 +23,11 @@ printHelpAndExit() { echo "-s SECRET_KEY : secret key for certian s3 bucket access (if not set, use IAM permission only)" echo "-r REGION : region for the profile set for certain s3 bucket access (if not set, use IAM permission only)" echo "-g : use singularity" + echo "-c : Metrics collection is disabled if flag is set" echo "-k S3_ENCRYPT_KEY_ID : KMS key to encrypt s3 files with" exit "$1" } -while getopts "i:m:l:p:a:s:r:gV:A:k:" opt; do +while getopts "i:m:l:p:a:s:r:gcV:A:k:" opt; do case $opt in i) export JOBID=$OPTARG;; l) export LOGBUCKET=$OPTARG;; # bucket for sending log file @@ -37,6 +39,7 @@ while getopts "i:m:l:p:a:s:r:gV:A:k:" opt; do s) export SECRET_KEY=$OPTARG;; # secret key for certian s3 bucket access r) export REGION=$OPTARG;; # region for the profile set for certian s3 bucket access g) export SINGULARITY_OPTION_TO_PASS=-g;; # use singularity + c) export DISABLE_METRICS_COLLECTION=true;; # disable metrics collection k) export S3_ENCRYPT_KEY_ID=$OPTARG;; # KMS key ID to encrypt s3 files with h) printHelpAndExit 0;; [?]) printHelpAndExit 1;; @@ -176,24 +179,29 @@ exl mkdir -p $LOCAL_OUTDIR mv $LOGFILE1 $LOGFILE2 export LOGFILE=$LOGFILE2 - exl echo -exl echo "## Installing and activating Cloudwatch agent to collect metrics" cwd0=$(pwd) cd ~ -ARCHITECTURE="$(dpkg --print-architecture)" -CW_AGENT_LINK="https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/${ARCHITECTURE}/latest/amazon-cloudwatch-agent.deb" -apt install -y wget -exl echo "Loading Cloudwatch Agent from ${CW_AGENT_LINK}" -wget "${CW_AGENT_LINK}" -sudo dpkg -i -E ./amazon-cloudwatch-agent.deb -# If we want to collect new metrics, the following file has to be modified -exl echo "## Using CW Agent config: https://raw.githubusercontent.com/4dn-dcic/tibanna/master/awsf3/cloudwatch_agent_config.json" -wget https://raw.githubusercontent.com/4dn-dcic/tibanna/master/awsf3/cloudwatch_agent_config.json -mv ./cloudwatch_agent_config.json /opt/aws/amazon-cloudwatch-agent/bin/config.json -# This starts the agent with the downloaded configuration file -sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json +if [ "$DISABLE_METRICS_COLLECTION" = false ] ; then + exl echo "## Installing and activating Cloudwatch agent to collect metrics" + ARCHITECTURE="$(dpkg --print-architecture)" + CW_AGENT_LINK="https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/${ARCHITECTURE}/latest/amazon-cloudwatch-agent.deb" + apt install -y wget + exl echo "Loading Cloudwatch Agent from ${CW_AGENT_LINK}" + wget "${CW_AGENT_LINK}" + sudo dpkg -i -E ./amazon-cloudwatch-agent.deb + # If we want to collect new metrics, the following file has to be modified + exl echo "## Using CW Agent config: https://raw.githubusercontent.com/4dn-dcic/tibanna/master/awsf3/cloudwatch_agent_config.json" + wget https://raw.githubusercontent.com/4dn-dcic/tibanna/master/awsf3/cloudwatch_agent_config.json + mv ./cloudwatch_agent_config.json /opt/aws/amazon-cloudwatch-agent/bin/config.json + # This starts the agent with the downloaded configuration file + sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json +else + exl echo "## Metrics collection is disabled" +fi + + # Set up cronjob to monitor AWS spot instance termination notice. # Works only in deployed Tibanna version >=1.6.0 since the ec2 needed more permissions to call `aws ec2 describe-spot-instance-requests` diff --git a/docs/execution_json.rst b/docs/execution_json.rst index 494df69c..bfc4e4db 100755 --- a/docs/execution_json.rst +++ b/docs/execution_json.rst @@ -572,6 +572,10 @@ The ``config`` field describes execution configuration. - type of EBS (e.g. ``gp3``, ``gp2``, ``io1``, ``io2``) - optional (default: gp3 (version >= ``1.0.0``) or gp2 (version < ``1.0.0``)) +:disable_metrics_collection: + - , default: false + - If true, the cloudwatch agent is not installed on the EC2 and CPU/memory/storage won't be collected and send to AWS CloudWatch. Disabling metrics collection can reduce CloudWatch associated costs. + :cloudwatch_dashboard: - **This option is now depricated.** - if true, Memory Used, Disk Used, CPU Utilization Cloudwatch metrics are collected into a single Cloudwatch Dashboard page. (default ``false``) diff --git a/pyproject.toml b/pyproject.toml index 92e77214..fc3f232a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tibanna" -version = "5.2.2" +version = "5.3.0" description = "Tibanna runs portable pipelines (in CWL/WDL) on the AWS Cloud." authors = ["4DN-DCIC Team "] license = "MIT" diff --git a/tibanna/ec2_utils.py b/tibanna/ec2_utils.py index 494609d5..963da512 100755 --- a/tibanna/ec2_utils.py +++ b/tibanna/ec2_utils.py @@ -325,6 +325,8 @@ def fill_default(self): self.mem_as_is = False if not hasattr(self, 'ebs_size_as_is'): # if false, add 5GB overhead self.ebs_size_as_is = False + if not hasattr(self, 'disable_metrics_collection'): + self.disable_metrics_collection = False if not hasattr(self, 'ami_id'): self.ami_id = "" # will be assigned instance architecture specific later if not hasattr(self, 'ami_per_region'): @@ -741,6 +743,8 @@ def create_userdata(self, profile=None): str += " -l $LOGBUCKET" str += " -V {version}".format(version=__version__) str += " -A {awsf_image}".format(awsf_image=cfg.awsf_image) + if cfg.disable_metrics_collection: + str += " -c" if cfg.kms_key_id: str += " -k %s" % cfg.kms_key_id if cfg.password: diff --git a/tibanna/lambdas/requirements.txt b/tibanna/lambdas/requirements.txt index 10de6cf6..a77fac96 100755 --- a/tibanna/lambdas/requirements.txt +++ b/tibanna/lambdas/requirements.txt @@ -1,4 +1,4 @@ Benchmark-4dn>=0.5.24 boto3>=1.9.0 botocore>=1.12.1 -tomlkit>=0.11.0 +tomlkit>=0.11.0,<1.0.0