Flexible Dimension

README.md

@@ -102,6 +102,20 @@ df-.*-percent_bytes-used
 1. The df.percent_bytes.used metric will be published for every file system reported by df plugin
 
 
+### Flexible Dimension
+
+This feature will give the user the capability to report metrics with any customized dimensions.
+
+Basically, there is a flexible filled up dimension dictionary has been filled up every time while get_dimensions function is called. 
+
+To expand this feature, the developer needs to:
+1. Implement your own dimension class deriving from DimensionsPlugin. A callback func and args should be filled out correctly in register_plugin function. The callback function is used to fill out the dimension dictionary.
+2. Initialize and register your dimension in dimensionhandler.py
+
+To use this feature, the user can just add the dimension name into the dimension.conf file. One dimenion per line. The dimension name must match the name in the dimension plugin implementation. If you don't want to include certain dimension, you can just not include it in the configuration file.
+
+The default implemented dimensions are: InstanceId, PluginInstance
+
 ## Usage
 Once the plugin is configured correctly, restart collectd to load new configuration.
 ```

src/cloudwatch/config/dimensions.conf

@@ -0,0 +1,2 @@
+InstanceId
+PluginInstance

src/cloudwatch/modules/client/querystringbuilder.py

@@ -15,6 +15,7 @@ class QuerystringBuilder(object):
     _METRIC_NAME_KEY = "MetricName"
     _NAME_KEY = "Name"
     _VALUE_KEY = "Value"
+    _UNIT_KEY = "Unit"
     _TIMESTAMP_KEY = "Timestamp"
     _STATISTICS_KEY = "StatisticValues."
     _STAT_MAX = _STATISTICS_KEY + "Maximum"
@@ -80,3 +81,6 @@ def _add_values(self, metric, metric_map, metric_prefix):
         metric_map[metric_prefix + self._STAT_MIN] = metric.statistics.min
         metric_map[metric_prefix + self._STAT_SUM] = metric.statistics.sum
         metric_map[metric_prefix + self._STAT_SAMPLE] = metric.statistics.sample_count
+
+    def _add_unit(self, metric, metric_map, metric_prefix):
+        metric_map[metric_prefix + self._UNIT_KEY] = metric.unit

src/cloudwatch/modules/configuration/confighelper.py

@@ -6,6 +6,7 @@
 from whitelist import Whitelist, WhitelistConfigReader
 from ..client.ec2getclient import EC2GetClient
 import traceback
+from dimensionreader import DimensionConfigReader
 
 class ConfigHelper(object):
     """
@@ -30,6 +31,7 @@ class ConfigHelper(object):
     _METADATA_SERVICE_ADDRESS = 'http://169.254.169.254/' 
     WHITELIST_CONFIG_PATH = _DEFAULT_AGENT_ROOT_FOLDER + 'whitelist.conf'
     BLOCKED_METRIC_PATH = _DEFAULT_AGENT_ROOT_FOLDER + 'blocked_metrics'
+    DIMENSION_CONFIG_PATH = _DEFAULT_AGENT_ROOT_FOLDER + 'dimensions.conf'
 
     def __init__(self, config_path=_DEFAULT_CONFIG_PATH, metadata_server=_METADATA_SERVICE_ADDRESS):
         self._config_path = config_path
@@ -211,3 +213,4 @@ def _check_configuration_integrity(self):
             raise ValueError("AWS secret key is missing.") 
         if not self.region:
             raise ValueError("Region is missing")
+

src/cloudwatch/modules/configuration/dimensionreader.py

@@ -0,0 +1,34 @@
+from ..logger.logger import get_logger
+
+class DimensionConfigReader(object):
+    """
+    The DimensionConfigReader is responsible for parsing the dimension.conf file into a dimension list
+    used by the Dimension class.
+    """
+    _LOGGER = get_logger(__name__)
+
+    def __init__(self, dimension_config_path):
+        self.dimension_config_path = dimension_config_path
+
+    def get_dimension_list(self):
+        """
+        Reads dimension configuration file and returns a list of dimension name.
+        :return: dimension list configured
+        """
+        try:
+            return self._get_dimensions_from_file(self.dimension_config_path)
+        except IOError as e:
+            self._LOGGER.warning("Could not open dimension file '" + self.dimension_config_path + "'. Reason: " + str(e))
+            return None
+
+    def _get_dimensions_from_file(self, dimension_path):
+        dimensions = []
+        with open(dimension_path) as dimension_file:
+            lines = dimension_file.readlines()
+            for line in lines:
+                dimensions.append(line.rstrip())
+        return dimensions
+
+
+
+

src/cloudwatch/modules/dimensionhandler.py

@@ -0,0 +1,31 @@
+from logger.logger import get_logger
+from dimensionplugins import *
+from configuration.dimensionreader import DimensionConfigReader
+
+class Dimensions(object):
+    """
+    The Dimensions is responsible for holding all dimension plugins
+    """
+    _LOGGER = get_logger(__name__)
+
+    def __init__(self, config_helper, vl):
+        self.config = config_helper
+        self.vl = vl
+        self.dimension_handlers = dict()
+        self.dimension_handlers["InstanceId"] = Dimension_InstanceId(self.config, self.vl)
+        self.dimension_handlers["PluginInstance"] = Dimension_PluginInstance(self.config, self.vl)
+        self.dimension_handlers["Hostname"] = Dimension_Hostname(self.config, self.vl)
+        for h in self.dimension_handlers:
+            self.dimension_handlers[h].register_plugin()
+
+    """
+    Go through the configured dimension list and find out if there is a plugin can handle it
+    """
+    def get_dimensions(self):
+        dimension_config_list = DimensionConfigReader(self.config.DIMENSION_CONFIG_PATH).get_dimension_list()
+        dimensions = dict()
+        for dm in dimension_config_list:
+            if dm in self.dimension_handlers:
+                self.dimension_handlers[dm].func(dimensions, self.dimension_handlers[dm].args)
+        return dimensions
+

src/cloudwatch/modules/dimensionplugins/__init__.py

@@ -0,0 +1,29 @@
+"""
+This Dimension Plugin abstract base class file
+"""
+
+class DimensionPlugin(object):
+    """
+    Base class of Dimension plugin.
+    Any vendor can implement a derived class
+    """
+    def __init__(self, config_helper, vl):
+        self.func = None
+        self.args = None
+        self.config = config_helper
+        self.vl = vl
+
+    def __str__(self):
+        if self.func and self.args:
+            return "func: %s, args: %s" % (self.func.__name__, self.args)
+        else:
+            return __name__
+
+    """
+    Abstract method: register dimension plugin function
+    """
+    def register_plugin(self):
+        pass
+
+from generic_dimensions import *
+

src/cloudwatch/modules/dimensionplugins/generic_dimensions.py

@@ -0,0 +1,54 @@
+"""
+This is the file containing generic dimension plugin classes
+"""
+
+import os
+from . import DimensionPlugin
+
+"""
+This InstanceId Dimension coming from configured host instance
+"""
+
+def dimension_get_instance_id(dimension, args):
+    dimension[args['name']] = args['value']
+
+class Dimension_InstanceId(DimensionPlugin):
+    def register_plugin(self):
+        self.func = dimension_get_instance_id
+        self.args = {
+            'name': "InstanceId",
+            'value': self.config.host
+        }
+
+
+"""
+This PluginInstance Dimension coming from collectd value plugin instance
+"""
+
+def dimension_get_plugin_instance(dimension, args):
+    dimension[args['name']] = args['value']
+
+class Dimension_PluginInstance(DimensionPlugin):
+    def register_plugin(self):
+        self.func = dimension_get_plugin_instance
+        plugin_instance = self.vl.plugin_instance if self.vl.plugin_instance else "NONE"
+        self.args = {
+            'name': "PluginInstance",
+            'value': plugin_instance
+        }
+
+
+"""
+Hostname Dimension report the hostname value
+"""
+
+def dimension_get_hostname(dimension, args):
+    dimension[args['name']] = args['value']
+
+class Dimension_Hostname(DimensionPlugin):
+    def register_plugin(self):
+        self.func = dimension_get_hostname
+        self.args = {
+            'name': "Hostname",
+            'value': os.uname()[1]
+        }

src/cloudwatch/modules/metricdata.py

@@ -1,6 +1,8 @@
 import awsutils as awsutils
 import plugininfo
 import datetime
+from logger.logger import get_logger
+from dimensionhandler import Dimensions
 
 class MetricDataStatistic(object):
     """
@@ -86,11 +88,11 @@ def __init__(self, config_helper, vl, adjusted_time=None):
 
     def build(self):
         """ Builds metric data object with name and dimensions but without value or statistics """
-        metric_array = [MetricDataStatistic(metric_name=self._build_metric_name(), dimensions=self._build_metric_dimensions(), timestamp=self._build_timestamp())]
+        metric_array = [MetricDataStatistic(metric_name=self._build_metric_name(), unit=self.vl.type_instance, dimensions=self._build_metric_dimensions(), timestamp=self._build_timestamp())]
         if self.config.push_asg:
-            metric_array.append(MetricDataStatistic(metric_name=self._build_metric_name(), dimensions=self._build_asg_dimension(), timestamp=self._build_timestamp()))
+            metric_array.append(MetricDataStatistic(metric_name=self._build_metric_name(), unit=self.vl.type_instance, dimensions=self._build_asg_dimension(), timestamp=self._build_timestamp()))
         if self.config.push_constant:
-            metric_array.append(MetricDataStatistic(metric_name=self._build_metric_name(), dimensions=self._build_constant_dimension(), timestamp=self._build_timestamp()))
+            metric_array.append(MetricDataStatistic(metric_name=self._build_metric_name(), unit=self.vl.type_instance, dimensions=self._build_constant_dimension(), timestamp=self._build_timestamp()))
         return metric_array
 
     def _build_timestamp(self):
@@ -123,10 +125,14 @@ def _build_constant_dimension(self):
         return dimensions
 
     def _build_metric_dimensions(self):
-        dimensions = {
-              "Host" : self._get_host_dimension(),
-              "PluginInstance" : self._get_plugin_instance_dimension()
-              }
+        if self.config.DIMENSION_CONFIG_PATH != None:
+            d = Dimensions(self.config, self.vl)
+            dimensions = d.get_dimensions()
+        else:
+            dimensions = {
+                "Host" : self._get_host_dimension(),
+                "PluginInstance" : self._get_plugin_instance_dimension()
+            }
         if self.config.push_asg:
             dimensions["AutoScalingGroup"] = self._get_autoscaling_group()
         if self.config.push_constant:
@@ -147,3 +153,4 @@ def _get_autoscaling_group(self):
         if self.config.asg_name:
             return self.config.asg_name
         return "NONE"
+

test/config_files/dimensions.conf

@@ -0,0 +1,2 @@
+InstanceId
+PluginInstance

test/test_metricdatabuilder.py

@@ -11,6 +11,7 @@ class MetricDataBuilderTest(unittest.TestCase):
     CONFIG_DIR = "./test/config_files/"
     VALID_CONFIG_FULL = CONFIG_DIR + "valid_config_full"
     VALID_CONFIG_WITH_CREDS_AND_REGION = CONFIG_DIR + "valid_config_with_creds_and_region"
+    DIMENSION_CONFIG = CONFIG_DIR + "dimensions.conf"
 
     @classmethod
     def setUpClass(cls):
@@ -31,6 +32,7 @@ def test_build_no_add(self):
         vl = self._get_vl_mock("CPU", "0", "CPU", "Steal")
         self.config_helper.push_asg = False
         self.config_helper.push_constant = False
+        self.config_helper.DIMENSION_CONFIG_PATH = None
         metric = MetricDataBuilder(self.config_helper, vl).build()
         self.assertEquals(None, metric[0].statistics)
         self.assertEquals("CPU.CPU.Steal", metric[0].metric_name)
@@ -43,6 +45,7 @@ def test_build_add_asg(self):
         self.config_helper.push_asg = True
         self.config_helper.push_constant = False
         self.config_helper.asg_name = "MyASG"
+        self.config_helper.DIMENSION_CONFIG_PATH = None
         metric = MetricDataBuilder(self.config_helper, vl).build()
         self.assertEquals(None, metric[0].statistics)
         self.assertEquals("CPU.CPU.Steal", metric[0].metric_name)
@@ -59,6 +62,7 @@ def test_build_add_constant(self):
         self.config_helper.push_asg = False
         self.config_helper.push_constant = True
         self.config_helper.constant_dimension_value = "somevalue"
+        self.config_helper.DIMENSION_CONFIG_PATH = None
         metric = MetricDataBuilder(self.config_helper, vl).build()
         self.assertEquals(None, metric[0].statistics)
         self.assertEquals("CPU.CPU.Steal", metric[0].metric_name)
@@ -76,6 +80,7 @@ def test_build_add_constant_and_asg(self):
         self.config_helper.asg_name = "MyASG"
         self.config_helper.push_constant = True
         self.config_helper.constant_dimension_value = "somevalue"
+        self.config_helper.DIMENSION_CONFIG_PATH = None
         metric = MetricDataBuilder(self.config_helper, vl).build()
         self.assertEquals(None, metric[0].statistics)
         self.assertEquals("CPU.CPU.Steal", metric[0].metric_name)
@@ -99,6 +104,7 @@ def test_build_with_enable_high_resolution_metrics(self):
         self.config_helper.host = "valid_host"
         self.config_helper.region = "localhost"
         self.config_helper.enable_high_resolution_metrics = True
+        self.config_helper.DIMENSION_CONFIG_PATH = None
         vl = self._get_vl_mock("CPU", "0", "CPU", "Steal", 112.1)
         metric = MetricDataBuilder(self.config_helper, vl, 160.1).build()
         self.assertEquals(None, metric[0].statistics)
@@ -129,13 +135,31 @@ def test_build_metric_name_with_real_CPU_name_parts_only(self):
         self.assertEquals(expected_name, generated_name)
 
     def test_build_metric_dimensions(self):
+        self.config_helper.DIMENSION_CONFIG_PATH = None
         vl = self._get_vl_mock("aggregation", "cpu-average", "cpu", "idle")
         metric_data_builder = MetricDataBuilder(self.config_helper, vl)
         dimensions = metric_data_builder._build_metric_dimensions()
         self.assertEquals("cpu-average", dimensions['PluginInstance'])
         self.assertEquals("valid_host", dimensions['Host'])
 
-    def test_buoild_metric_dimensions_with_no_plugin_instance(self):
+    def test_build_metric_flex_dimensions(self):
+        self.config_helper.DIMENSION_CONFIG_PATH = self.DIMENSION_CONFIG
+        vl = self._get_vl_mock("aggregation", "cpu-average", "cpu", "idle")
+        metric_data_builder = MetricDataBuilder(self.config_helper, vl)
+        dimensions = metric_data_builder._build_metric_dimensions()
+        self.assertEquals("cpu-average", dimensions['PluginInstance'])
+        self.assertEquals("valid_host", dimensions['InstanceId'])
+
+    def test_build_metric_flex_dimensions_without_plugin_instance(self):
+        self.config_helper.DIMENSION_CONFIG_PATH = self.DIMENSION_CONFIG
+        vl = self._get_vl_mock("aggregation", "", "cpu", "idle")
+        metric_data_builder = MetricDataBuilder(self.config_helper, vl)
+        dimensions = metric_data_builder._build_metric_dimensions()
+        self.assertEquals("NONE", dimensions['PluginInstance'])
+        self.assertEquals("valid_host", dimensions['InstanceId'])
+
+    def test_build_metric_dimensions_with_no_plugin_instance(self):
+        self.config_helper.DIMENSION_CONFIG_PATH = None
         vl = self._get_vl_mock("plugin", "", "type", "")
         metric_data_builder = MetricDataBuilder(self.config_helper, vl)
         dimensions = metric_data_builder._build_metric_dimensions()
@@ -144,6 +168,7 @@ def test_buoild_metric_dimensions_with_no_plugin_instance(self):
     def test_build_metric_dimensions_with_host_from_value_list(self):
         self.server.set_expected_response("Error", 404)
         self.config_helper.host = ""
+        self.config_helper.DIMENSION_CONFIG_PATH = None
         vl = self._get_vl_mock("aggregation", "cpu-average", "cpu", "idle")
         metric_data_builder = MetricDataBuilder(self.config_helper, vl)
         dimensions = metric_data_builder._build_metric_dimensions()