diff --git a/README.md b/README.md index 78431a0..1e19b27 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,20 @@ df-.*-percent_bytes-used 1. The df.percent_bytes.used metric will be published for every file system reported by df plugin +### Flexible Dimension + +This feature will give the user the capability to report metrics with any customized dimensions. + +Basically, there is a flexible filled up dimension dictionary has been filled up every time while get_dimensions function is called. + +To expand this feature, the developer needs to: +1. Implement your own dimension class deriving from DimensionsPlugin. A callback func and args should be filled out correctly in register_plugin function. The callback function is used to fill out the dimension dictionary. +2. Initialize and register your dimension in dimensionhandler.py + +To use this feature, the user can just add the dimension name into the dimension.conf file. One dimenion per line. The dimension name must match the name in the dimension plugin implementation. If you don't want to include certain dimension, you can just not include it in the configuration file. + +The default implemented dimensions are: InstanceId, PluginInstance + ## Usage Once the plugin is configured correctly, restart collectd to load new configuration. ``` diff --git a/src/cloudwatch/config/dimensions.conf b/src/cloudwatch/config/dimensions.conf new file mode 100755 index 0000000..97391dd --- /dev/null +++ b/src/cloudwatch/config/dimensions.conf @@ -0,0 +1,2 @@ +InstanceId +PluginInstance diff --git a/src/cloudwatch/modules/client/querystringbuilder.py b/src/cloudwatch/modules/client/querystringbuilder.py index 13d9472..5150de4 100644 --- a/src/cloudwatch/modules/client/querystringbuilder.py +++ b/src/cloudwatch/modules/client/querystringbuilder.py @@ -15,6 +15,7 @@ class QuerystringBuilder(object): _METRIC_NAME_KEY = "MetricName" _NAME_KEY = "Name" _VALUE_KEY = "Value" + _UNIT_KEY = "Unit" _TIMESTAMP_KEY = "Timestamp" _STATISTICS_KEY = "StatisticValues." _STAT_MAX = _STATISTICS_KEY + "Maximum" @@ -80,3 +81,6 @@ def _add_values(self, metric, metric_map, metric_prefix): metric_map[metric_prefix + self._STAT_MIN] = metric.statistics.min metric_map[metric_prefix + self._STAT_SUM] = metric.statistics.sum metric_map[metric_prefix + self._STAT_SAMPLE] = metric.statistics.sample_count + + def _add_unit(self, metric, metric_map, metric_prefix): + metric_map[metric_prefix + self._UNIT_KEY] = metric.unit diff --git a/src/cloudwatch/modules/configuration/confighelper.py b/src/cloudwatch/modules/configuration/confighelper.py index f013085..5e356b5 100644 --- a/src/cloudwatch/modules/configuration/confighelper.py +++ b/src/cloudwatch/modules/configuration/confighelper.py @@ -6,6 +6,7 @@ from whitelist import Whitelist, WhitelistConfigReader from ..client.ec2getclient import EC2GetClient import traceback +from dimensionreader import DimensionConfigReader class ConfigHelper(object): """ @@ -30,6 +31,7 @@ class ConfigHelper(object): _METADATA_SERVICE_ADDRESS = 'http://169.254.169.254/' WHITELIST_CONFIG_PATH = _DEFAULT_AGENT_ROOT_FOLDER + 'whitelist.conf' BLOCKED_METRIC_PATH = _DEFAULT_AGENT_ROOT_FOLDER + 'blocked_metrics' + DIMENSION_CONFIG_PATH = _DEFAULT_AGENT_ROOT_FOLDER + 'dimensions.conf' def __init__(self, config_path=_DEFAULT_CONFIG_PATH, metadata_server=_METADATA_SERVICE_ADDRESS): self._config_path = config_path @@ -211,3 +213,4 @@ def _check_configuration_integrity(self): raise ValueError("AWS secret key is missing.") if not self.region: raise ValueError("Region is missing") + diff --git a/src/cloudwatch/modules/configuration/dimensionreader.py b/src/cloudwatch/modules/configuration/dimensionreader.py new file mode 100755 index 0000000..96a6bee --- /dev/null +++ b/src/cloudwatch/modules/configuration/dimensionreader.py @@ -0,0 +1,34 @@ +from ..logger.logger import get_logger + +class DimensionConfigReader(object): + """ + The DimensionConfigReader is responsible for parsing the dimension.conf file into a dimension list + used by the Dimension class. + """ + _LOGGER = get_logger(__name__) + + def __init__(self, dimension_config_path): + self.dimension_config_path = dimension_config_path + + def get_dimension_list(self): + """ + Reads dimension configuration file and returns a list of dimension name. + :return: dimension list configured + """ + try: + return self._get_dimensions_from_file(self.dimension_config_path) + except IOError as e: + self._LOGGER.warning("Could not open dimension file '" + self.dimension_config_path + "'. Reason: " + str(e)) + return None + + def _get_dimensions_from_file(self, dimension_path): + dimensions = [] + with open(dimension_path) as dimension_file: + lines = dimension_file.readlines() + for line in lines: + dimensions.append(line.rstrip()) + return dimensions + + + + diff --git a/src/cloudwatch/modules/dimensionhandler.py b/src/cloudwatch/modules/dimensionhandler.py new file mode 100644 index 0000000..e415aa6 --- /dev/null +++ b/src/cloudwatch/modules/dimensionhandler.py @@ -0,0 +1,31 @@ +from logger.logger import get_logger +from dimensionplugins import * +from configuration.dimensionreader import DimensionConfigReader + +class Dimensions(object): + """ + The Dimensions is responsible for holding all dimension plugins + """ + _LOGGER = get_logger(__name__) + + def __init__(self, config_helper, vl): + self.config = config_helper + self.vl = vl + self.dimension_handlers = dict() + self.dimension_handlers["InstanceId"] = Dimension_InstanceId(self.config, self.vl) + self.dimension_handlers["PluginInstance"] = Dimension_PluginInstance(self.config, self.vl) + self.dimension_handlers["Hostname"] = Dimension_Hostname(self.config, self.vl) + for h in self.dimension_handlers: + self.dimension_handlers[h].register_plugin() + + """ + Go through the configured dimension list and find out if there is a plugin can handle it + """ + def get_dimensions(self): + dimension_config_list = DimensionConfigReader(self.config.DIMENSION_CONFIG_PATH).get_dimension_list() + dimensions = dict() + for dm in dimension_config_list: + if dm in self.dimension_handlers: + self.dimension_handlers[dm].func(dimensions, self.dimension_handlers[dm].args) + return dimensions + diff --git a/src/cloudwatch/modules/dimensionplugins/__init__.py b/src/cloudwatch/modules/dimensionplugins/__init__.py new file mode 100644 index 0000000..ce179a1 --- /dev/null +++ b/src/cloudwatch/modules/dimensionplugins/__init__.py @@ -0,0 +1,29 @@ +""" +This Dimension Plugin abstract base class file +""" + +class DimensionPlugin(object): + """ + Base class of Dimension plugin. + Any vendor can implement a derived class + """ + def __init__(self, config_helper, vl): + self.func = None + self.args = None + self.config = config_helper + self.vl = vl + + def __str__(self): + if self.func and self.args: + return "func: %s, args: %s" % (self.func.__name__, self.args) + else: + return __name__ + + """ + Abstract method: register dimension plugin function + """ + def register_plugin(self): + pass + +from generic_dimensions import * + diff --git a/src/cloudwatch/modules/dimensionplugins/generic_dimensions.py b/src/cloudwatch/modules/dimensionplugins/generic_dimensions.py new file mode 100644 index 0000000..0123d85 --- /dev/null +++ b/src/cloudwatch/modules/dimensionplugins/generic_dimensions.py @@ -0,0 +1,54 @@ +""" +This is the file containing generic dimension plugin classes +""" + +import os +from . import DimensionPlugin + +""" +This InstanceId Dimension coming from configured host instance +""" + +def dimension_get_instance_id(dimension, args): + dimension[args['name']] = args['value'] + +class Dimension_InstanceId(DimensionPlugin): + def register_plugin(self): + self.func = dimension_get_instance_id + self.args = { + 'name': "InstanceId", + 'value': self.config.host + } + + +""" +This PluginInstance Dimension coming from collectd value plugin instance +""" + +def dimension_get_plugin_instance(dimension, args): + dimension[args['name']] = args['value'] + +class Dimension_PluginInstance(DimensionPlugin): + def register_plugin(self): + self.func = dimension_get_plugin_instance + plugin_instance = self.vl.plugin_instance if self.vl.plugin_instance else "NONE" + self.args = { + 'name': "PluginInstance", + 'value': plugin_instance + } + + +""" +Hostname Dimension report the hostname value +""" + +def dimension_get_hostname(dimension, args): + dimension[args['name']] = args['value'] + +class Dimension_Hostname(DimensionPlugin): + def register_plugin(self): + self.func = dimension_get_hostname + self.args = { + 'name': "Hostname", + 'value': os.uname()[1] + } diff --git a/src/cloudwatch/modules/metricdata.py b/src/cloudwatch/modules/metricdata.py index 1669e66..9b71715 100644 --- a/src/cloudwatch/modules/metricdata.py +++ b/src/cloudwatch/modules/metricdata.py @@ -1,6 +1,8 @@ import awsutils as awsutils import plugininfo import datetime +from logger.logger import get_logger +from dimensionhandler import Dimensions class MetricDataStatistic(object): """ @@ -86,11 +88,11 @@ def __init__(self, config_helper, vl, adjusted_time=None): def build(self): """ Builds metric data object with name and dimensions but without value or statistics """ - metric_array = [MetricDataStatistic(metric_name=self._build_metric_name(), dimensions=self._build_metric_dimensions(), timestamp=self._build_timestamp())] + metric_array = [MetricDataStatistic(metric_name=self._build_metric_name(), unit=self.vl.type_instance, dimensions=self._build_metric_dimensions(), timestamp=self._build_timestamp())] if self.config.push_asg: - metric_array.append(MetricDataStatistic(metric_name=self._build_metric_name(), dimensions=self._build_asg_dimension(), timestamp=self._build_timestamp())) + metric_array.append(MetricDataStatistic(metric_name=self._build_metric_name(), unit=self.vl.type_instance, dimensions=self._build_asg_dimension(), timestamp=self._build_timestamp())) if self.config.push_constant: - metric_array.append(MetricDataStatistic(metric_name=self._build_metric_name(), dimensions=self._build_constant_dimension(), timestamp=self._build_timestamp())) + metric_array.append(MetricDataStatistic(metric_name=self._build_metric_name(), unit=self.vl.type_instance, dimensions=self._build_constant_dimension(), timestamp=self._build_timestamp())) return metric_array def _build_timestamp(self): @@ -123,10 +125,14 @@ def _build_constant_dimension(self): return dimensions def _build_metric_dimensions(self): - dimensions = { - "Host" : self._get_host_dimension(), - "PluginInstance" : self._get_plugin_instance_dimension() - } + if self.config.DIMENSION_CONFIG_PATH != None: + d = Dimensions(self.config, self.vl) + dimensions = d.get_dimensions() + else: + dimensions = { + "Host" : self._get_host_dimension(), + "PluginInstance" : self._get_plugin_instance_dimension() + } if self.config.push_asg: dimensions["AutoScalingGroup"] = self._get_autoscaling_group() if self.config.push_constant: @@ -147,3 +153,4 @@ def _get_autoscaling_group(self): if self.config.asg_name: return self.config.asg_name return "NONE" + diff --git a/test/config_files/dimensions.conf b/test/config_files/dimensions.conf new file mode 100755 index 0000000..97391dd --- /dev/null +++ b/test/config_files/dimensions.conf @@ -0,0 +1,2 @@ +InstanceId +PluginInstance diff --git a/test/test_metricdatabuilder.py b/test/test_metricdatabuilder.py index 9221674..5e05f5e 100644 --- a/test/test_metricdatabuilder.py +++ b/test/test_metricdatabuilder.py @@ -11,6 +11,7 @@ class MetricDataBuilderTest(unittest.TestCase): CONFIG_DIR = "./test/config_files/" VALID_CONFIG_FULL = CONFIG_DIR + "valid_config_full" VALID_CONFIG_WITH_CREDS_AND_REGION = CONFIG_DIR + "valid_config_with_creds_and_region" + DIMENSION_CONFIG = CONFIG_DIR + "dimensions.conf" @classmethod def setUpClass(cls): @@ -31,6 +32,7 @@ def test_build_no_add(self): vl = self._get_vl_mock("CPU", "0", "CPU", "Steal") self.config_helper.push_asg = False self.config_helper.push_constant = False + self.config_helper.DIMENSION_CONFIG_PATH = None metric = MetricDataBuilder(self.config_helper, vl).build() self.assertEquals(None, metric[0].statistics) self.assertEquals("CPU.CPU.Steal", metric[0].metric_name) @@ -43,6 +45,7 @@ def test_build_add_asg(self): self.config_helper.push_asg = True self.config_helper.push_constant = False self.config_helper.asg_name = "MyASG" + self.config_helper.DIMENSION_CONFIG_PATH = None metric = MetricDataBuilder(self.config_helper, vl).build() self.assertEquals(None, metric[0].statistics) self.assertEquals("CPU.CPU.Steal", metric[0].metric_name) @@ -59,6 +62,7 @@ def test_build_add_constant(self): self.config_helper.push_asg = False self.config_helper.push_constant = True self.config_helper.constant_dimension_value = "somevalue" + self.config_helper.DIMENSION_CONFIG_PATH = None metric = MetricDataBuilder(self.config_helper, vl).build() self.assertEquals(None, metric[0].statistics) self.assertEquals("CPU.CPU.Steal", metric[0].metric_name) @@ -76,6 +80,7 @@ def test_build_add_constant_and_asg(self): self.config_helper.asg_name = "MyASG" self.config_helper.push_constant = True self.config_helper.constant_dimension_value = "somevalue" + self.config_helper.DIMENSION_CONFIG_PATH = None metric = MetricDataBuilder(self.config_helper, vl).build() self.assertEquals(None, metric[0].statistics) self.assertEquals("CPU.CPU.Steal", metric[0].metric_name) @@ -99,6 +104,7 @@ def test_build_with_enable_high_resolution_metrics(self): self.config_helper.host = "valid_host" self.config_helper.region = "localhost" self.config_helper.enable_high_resolution_metrics = True + self.config_helper.DIMENSION_CONFIG_PATH = None vl = self._get_vl_mock("CPU", "0", "CPU", "Steal", 112.1) metric = MetricDataBuilder(self.config_helper, vl, 160.1).build() self.assertEquals(None, metric[0].statistics) @@ -129,13 +135,31 @@ def test_build_metric_name_with_real_CPU_name_parts_only(self): self.assertEquals(expected_name, generated_name) def test_build_metric_dimensions(self): + self.config_helper.DIMENSION_CONFIG_PATH = None vl = self._get_vl_mock("aggregation", "cpu-average", "cpu", "idle") metric_data_builder = MetricDataBuilder(self.config_helper, vl) dimensions = metric_data_builder._build_metric_dimensions() self.assertEquals("cpu-average", dimensions['PluginInstance']) self.assertEquals("valid_host", dimensions['Host']) - def test_buoild_metric_dimensions_with_no_plugin_instance(self): + def test_build_metric_flex_dimensions(self): + self.config_helper.DIMENSION_CONFIG_PATH = self.DIMENSION_CONFIG + vl = self._get_vl_mock("aggregation", "cpu-average", "cpu", "idle") + metric_data_builder = MetricDataBuilder(self.config_helper, vl) + dimensions = metric_data_builder._build_metric_dimensions() + self.assertEquals("cpu-average", dimensions['PluginInstance']) + self.assertEquals("valid_host", dimensions['InstanceId']) + + def test_build_metric_flex_dimensions_without_plugin_instance(self): + self.config_helper.DIMENSION_CONFIG_PATH = self.DIMENSION_CONFIG + vl = self._get_vl_mock("aggregation", "", "cpu", "idle") + metric_data_builder = MetricDataBuilder(self.config_helper, vl) + dimensions = metric_data_builder._build_metric_dimensions() + self.assertEquals("NONE", dimensions['PluginInstance']) + self.assertEquals("valid_host", dimensions['InstanceId']) + + def test_build_metric_dimensions_with_no_plugin_instance(self): + self.config_helper.DIMENSION_CONFIG_PATH = None vl = self._get_vl_mock("plugin", "", "type", "") metric_data_builder = MetricDataBuilder(self.config_helper, vl) dimensions = metric_data_builder._build_metric_dimensions() @@ -144,6 +168,7 @@ def test_buoild_metric_dimensions_with_no_plugin_instance(self): def test_build_metric_dimensions_with_host_from_value_list(self): self.server.set_expected_response("Error", 404) self.config_helper.host = "" + self.config_helper.DIMENSION_CONFIG_PATH = None vl = self._get_vl_mock("aggregation", "cpu-average", "cpu", "idle") metric_data_builder = MetricDataBuilder(self.config_helper, vl) dimensions = metric_data_builder._build_metric_dimensions()