diff --git a/README.rst b/README.rst index 3877aaa..1bc2260 100644 --- a/README.rst +++ b/README.rst @@ -303,7 +303,6 @@ Configuration Example name: localhost host: localhost port: 8091 - path: pools/default/buckets elasticsearch: name: clustername diff --git a/docker/base/newrelic-plugin-agent.cfg b/docker/base/newrelic-plugin-agent.cfg index 3b7a316..dd4f334 100644 --- a/docker/base/newrelic-plugin-agent.cfg +++ b/docker/base/newrelic-plugin-agent.cfg @@ -26,7 +26,6 @@ Application: # name: localhost # host: localhost # port: 8091 - # path: pools/default/buckets #elasticsearch: # name: Clustername diff --git a/etc/newrelic/newrelic-plugin-agent.cfg b/etc/newrelic/newrelic-plugin-agent.cfg index 3b7a316..dd4f334 100644 --- a/etc/newrelic/newrelic-plugin-agent.cfg +++ b/etc/newrelic/newrelic-plugin-agent.cfg @@ -26,7 +26,6 @@ Application: # name: localhost # host: localhost # port: 8091 - # path: pools/default/buckets #elasticsearch: # name: Clustername diff --git a/newrelic_plugin_agent/plugins/couchbase.py b/newrelic_plugin_agent/plugins/couchbase.py index 35c3f72..2612ef2 100644 --- a/newrelic_plugin_agent/plugins/couchbase.py +++ b/newrelic_plugin_agent/plugins/couchbase.py @@ -2,59 +2,158 @@ couchbase """ +import logging from newrelic_plugin_agent.plugins import base +LOGGER = logging.getLogger(__name__) + class Couchbase(base.JSONStatsPlugin): - GUID = 'com.meetme.couchbase' - - def add_datapoints(self, stats): - for bucket_data in stats: - bucket_name = bucket_data['name'] - bucket_stats = bucket_data['basicStats'] - if bucket_stats: - """ Example JSON - "basicStats":{ - "quotaPercentUsed":7.548735046386719, - "opsPerSec":0, - "diskFetches":0, - "itemCount":2222, - "diskUsed":229366539, - "dataUsed":203511808, - "memUsed":79154224 - } - """ - self.add_gauge_bucket_metric( - bucket_name, 'quotaPercentUsed', 'percent', bucket_stats['quotaPercentUsed'] - ) - self.add_gauge_bucket_metric( - bucket_name, 'opsPerSec', 'ops', bucket_stats['opsPerSec'] - ) - self.add_gauge_bucket_metric( - bucket_name, 'diskFetches', 'count', bucket_stats['diskFetches'] - ) - self.add_gauge_bucket_metric( - bucket_name, 'itemCount', 'count', bucket_stats['itemCount'] - ) - self.add_gauge_bucket_metric( - bucket_name, 'diskUsed', 'Byte', bucket_stats['diskUsed'] - ) - self.add_gauge_bucket_metric( - bucket_name, 'dataUsed', 'Byte', bucket_stats['dataUsed'] - ) - self.add_gauge_bucket_metric( - bucket_name, 'memUsed', 'Byte', bucket_stats['memUsed'] - ) - - # Summary metrics - self.add_gauge_value('Summary/%s/quotaPercentUsed' % bucket_name, 'percent', - bucket_stats['quotaPercentUsed'], - min_val=0, max_val=0) - self.add_gauge_value('Summary/%s/diskUsed' % bucket_name, 'byte', - bucket_stats['diskUsed']) - - def add_gauge_bucket_metric(self, bucket_name, metric_name, units, metric_value): - if metric_value: - self.add_gauge_value('%s/%s' % (bucket_name, metric_name), units, metric_value) + GUID = 'com.meetme.newrelic_couchbase_agent' + + # metrics are according to api reference v3.0 or v3.1 + METRICS = [ + {'type': 'cluster', 'label': 'storageTotals.ram.total', 'suffix': 'bytes'}, + {'type': 'cluster', 'label': 'storageTotals.ram.used', 'suffix': 'bytes'}, + {'type': 'cluster', 'label': 'storageTotals.ram.usedByData', 'suffix': 'bytes'}, + {'type': 'cluster', 'label': 'storageTotals.ram.quotaTotal', 'suffix': 'bytes'}, + {'type': 'cluster', 'label': 'storageTotals.ram.quotaUsed', 'suffix': 'bytes'}, + + {'type': 'cluster', 'label': 'storageTotals.hdd.total', 'suffix': 'bytes'}, + {'type': 'cluster', 'label': 'storageTotals.hdd.used', 'suffix': 'bytes'}, + {'type': 'cluster', 'label': 'storageTotals.hdd.usedByData', 'suffix': 'bytes'}, + {'type': 'cluster', 'label': 'storageTotals.hdd.quotaTotal', 'suffix': 'bytes'}, + {'type': 'cluster', 'label': 'storageTotals.hdd.free', 'suffix': 'bytes'}, + + {'type': 'cluster', 'label': 'counters.rebalance_success', 'suffix': 'count'}, + {'type': 'cluster', 'label': 'counters.rebalance_start', 'suffix': 'count'}, + {'type': 'cluster', 'label': 'counters.rebalance_fail', 'suffix': 'count'}, + {'type': 'cluster', 'label': 'counters.rebalance_node', 'suffix': 'count'}, + + {'type': 'nodes', 'label': 'systemStats.cpu_utilization_rate', 'suffix': 'count'}, + {'type': 'nodes', 'label': 'systemStats.swap_total', 'suffix': 'byte'}, + {'type': 'nodes', 'label': 'systemStats.swap_used', 'suffix': 'byte'}, + {'type': 'nodes', 'label': 'systemStats.mem_total', 'suffix': 'byte'}, + {'type': 'nodes', 'label': 'systemStats.mem_free', 'suffix': 'byte'}, + + {'type': 'nodes', 'label': 'interestingStats.couch_docs_actual_disk_size', 'suffix': 'byte'}, + {'type': 'nodes', 'label': 'interestingStats.couch_docs_data_size', 'suffix': 'byte'}, + {'type': 'nodes', 'label': 'interestingStats.couch_views_actual_disk_size', 'suffix': 'byte'}, + {'type': 'nodes', 'label': 'interestingStats.couch_views_data_size', 'suffix': 'byte'}, + {'type': 'nodes', 'label': 'interestingStats.mem_used', 'suffix': 'byte'}, + {'type': 'nodes', 'label': 'interestingStats.ops', 'suffix': 'count'}, + {'type': 'nodes', 'label': 'interestingStats.curr_items', 'suffix': 'count'}, + {'type': 'nodes', 'label': 'interestingStats.curr_items_tot', 'suffix': 'count'}, + {'type': 'nodes', 'label': 'interestingStats.vb_replica_curr_items', 'suffix': 'count'}, + + {'type': 'nodes', 'scoreboard': True, 'score_value': 'active', 'label': 'clusterMembership', 'suffix': 'count'}, + {'type': 'nodes', 'scoreboard': True, 'score_value': 'healthy', 'label': 'status', 'suffix': 'count'}, + + {'type': 'buckets', 'label': 'basicStats.quotaPercentUsed', 'suffix': 'percent'}, + {'type': 'buckets', 'label': 'basicStats.opsPerSec', 'suffix': 'ops'}, + {'type': 'buckets', 'label': 'basicStats.diskFetches', 'suffix': 'percent'}, + {'type': 'buckets', 'label': 'basicStats.itemCount', 'suffix': 'percent'}, + {'type': 'buckets', 'label': 'basicStats.diskUsed', 'suffix': 'byte'}, + {'type': 'buckets', 'label': 'basicStats.dataUsed', 'suffix': 'byte'}, + {'type': 'buckets', 'label': 'basicStats.memUsed', 'suffix': 'byte'}, + ] + + def add_datapoints(self, data): + """Add data points for all couchbase nodes. + + :param dict stats: stats for all nodes + + """ + # fetch metrics for each metric type (cluster, nodes, buckets) + for typ, stats in data.iteritems(): + for m in [m for m in self.METRICS if m['type'] == typ]: + if typ == 'cluster': + items = stats + name_key = 'name' + elif typ == 'nodes': + items = stats['nodes'] + name_key = 'hostname' + elif typ == 'buckets': + items = stats + name_key = 'name' + + # count score for scoreboard metrics, + # otherwise just add the gauge value from the API repsonse + if m.get('scoreboard', False): + self.add_gauge_value( + '%s/_scoreboard/%s' % (typ, m['label']), + m['suffix'], self._get_metric_score(m, items)) + else: + # add gauge value for current metric. + # NOTE cluster metrics are not repeated, + # as there is a single cluster. + # nodes and bucket metrics are repeated, + # as there are (usually) several of them + if typ == 'cluster': + self._add_gauge_value(m, typ, items[name_key], items) + else: + for item in items: + self._add_gauge_value(m, typ, item[name_key], item) + + def _add_gauge_value(self, metric, typ, name, items): + """Adds a gauge value for a nested metric. + + Some stats are missing from memcached bucket types, + thus we use dict.get() + + :param dict m: metric as defined at the top of this class. + :param str typ: metric type: cluster, nodes, buckets. + :param str name: cluster, node, or bucket name. + :param dict items: stats to lookup the metric in. + + """ + label = metric['label'] + value = items + for key in label.split('.'): + value = value.get(key, 0) + self.add_gauge_value('%s/%s/%s' % (typ, name, label), + metric['suffix'], value) + + def _get_metric_score(self, metric, items): + """Calculate metric score for scoreboard type metrics. + + A "scoreboard" metric is a counter of a specific value for a field. + E.g How many times the pair {"status": "healthy"} is found in a + list of object. + + NOTE that scoreboard metrics are meant to be used on lists of objects, + i.e. a list of node objects. + + :param dict metric: metric to calculate score for. + :param items metric: stats to calculate score from. + + :rtype: int + + """ + score = 0 + label = metric['label'] + score_value = metric['score_value'] + for d in items: + score += 1 if d[label] == score_value else 0 + return score + + def fetch_data(self): + """Fetch data from multiple couchbase stats URLs. + + Returns a dictionary with three keys: cluster, nodes, buckets. + Each key holds the JSON response from the API request. + + :rtype: dict + + """ + data = {} + for path, typ in [('pools/default', 'cluster'), + ('pools/nodes', 'nodes'), + ('pools/default/buckets', 'buckets')]: + res = self.http_get(self.stats_url + path) + if res: + data[typ] = res.json() if res else {} + + return data