Merge pull request #12 from pbenas/host_discovery

client: plugable host discovery

MANIFEST.in

@@ -0,0 +1,2 @@
+graft etc
+include rc.d/init.d/smokerd

etc/smokercli-example.yaml

@@ -0,0 +1,6 @@
+# Example configuration file for the smoker client (/etc/smokercli.yaml)
+# plugin paths: Python modules containing the host discovery plugins. It has to
+#               be valid python module path.
+
+plugin_paths:
+    - smoker.client.plugins

rc.d/init.d/smokerd-rhel → rc.d/init.d/smokerd

@@ -32,30 +32,6 @@ exit_error() {
 	exit 1
 }
 
-confgen() {
-	if [ "$GENCONFIG" -ne 1 ]; then
-		return 0
-	fi
-
-	cat $CONFDIR/common.yaml > $CONFIG || exit_error
-
-	dirs=('template' 'action' 'plugin')
-	for dir in ${dirs[*]}; do
-		echo -e "\n${dir}s:" >> $CONFIG
-
-		# Skip if directory doesn't contain YAML files
-		if [ "`ls ${CONFDIR}/${dir}.d/*.yaml 2>/dev/null|wc -l`" -eq 0 ]; then
-			continue
-		fi
-
-		for plugin in ${CONFDIR}/${dir}.d/*.yaml; do
-			name=`basename $plugin .yaml`
-
-			echo "    ${name}: !include ${dir}.d/${name}.yaml" >> $CONFIG
-		done
-	done
-}
-
 start() {
 	[ -x $BINARY ] || exit 5
 
@@ -66,7 +42,6 @@ start() {
 	[ -f "$PIDFILE" ] && exit_error "PID file $PIDFILE already exists!"
 	[ -f "$LOCKFILE" ] && exit_error "Subsys $LOCKFILE is locked!"
 
-	confgen
 	$BINARY $SMOKERD_OPTIONS
 	RETVAL=$?
 	if [ $RETVAL -eq 0 ]; then

setup.py

@@ -2,9 +2,13 @@
 # -*- coding: utf-8 -*-
 # Copyright (C) 2007-2012, GoodData(R) Corporation. All rights reserved
 
+import os
 import sys
 from setuptools import setup
 
+CONFIGDIR = '/etc/smokerd'
+INITDIR = '/etc/rc.d/init.d'
+
 # Parameters for build
 params = {
     # This package is named gdc-smoker on Pypi, use it on register or upload actions
@@ -16,6 +20,7 @@
         'smoker.server.plugins',
         'smoker.client',
         'smoker.client.out_junit',
+        'smoker.client.plugins',
         'smoker.logger',
         'smoker.util'
         ],
@@ -76,6 +81,71 @@
     'platforms' : ['POSIX'],
     'provides' : ['smoker'],
     'install_requires' : ['PyYAML', 'argparse', 'simplejson', 'psutil', 'setproctitle', 'Flask-RESTful'],
+    'data_files': [
+        (INITDIR, ['rc.d/init.d/smokerd']),
+        (CONFIGDIR, ['etc/smokerd-example.yaml', 'etc/smokercli-example.yaml'])
+    ]
 }
 
+# Get current branch
+branch = os.getenv('GIT_BRANCH')
+if not branch:
+    branch = os.popen(
+        'git branch|grep -v "no branch"|grep \*|sed s,\*\ ,,g').read().strip()
+
+    if not branch:
+        branch = 'master'
+else:
+    branch = branch.replace('origin/', '')
+
+# Get git revision hash
+revision = os.popen('git rev-parse --short HEAD').read().strip()
+
+if not revision:
+    revision = '0'
+
+# Get build number
+build = os.getenv('BUILD_NUMBER')
+if not build:
+    build = '1'
+
+try:
+    action = sys.argv[1]
+except IndexError:
+    action = None
+
+if action == 'clean':
+    # Remove MANIFEST file
+    print "Cleaning MANIFEST.."
+    try:
+        os.unlink('MANIFEST')
+    except OSError as e:
+        if e.errno == 2:
+            pass
+        else:
+            raise
+
+    # Remove dist and build directories
+    for dir in ['dist', 'build']:
+        print "Cleaning %s.." % dir
+        for root, dirs, files in os.walk(dir, topdown=False):
+            for name in files:
+                os.remove(os.path.join(root, name))
+            for name in dirs:
+                os.rmdir(os.path.join(root, name))
+        try:
+            os.rmdir(dir)
+        except OSError as e:
+            if e.errno == 2:
+                pass
+            else:
+                raise
+elif action == 'bdist_rpm':
+    # Set release number
+    sys.argv.append('--release=1.%s.%s' % (build, revision))
+    # Require same version of gdc-python-common package
+    sys.argv.append(
+        '--requires=python-argparse python-simplejson python-setproctitle '
+        'python-flask-restful')
+
 setup(**params)

smoker/client/cli.py

@@ -9,20 +9,23 @@
 """
 
 import argparse
+import datetime
+import glob
 import logging
+import os
+import simplejson
+import sys
+import yaml
 
 from smoker.client import Client
 import smoker.logger
+from smoker.client.out_junit import plugins_to_xml
+from smoker.util.tap import TapTest, Tap
+
 
 smoker.logger.init(syslog=False)
 lg = logging.getLogger('smokercli')
 
-from smoker.util.tap import TapTest, Tap
-from smoker.client.out_junit import plugins_to_xml
-
-import sys
-import simplejson
-import datetime
 
 COLORS = {
     'default' : '\033[0;0m',
@@ -36,6 +39,117 @@
     'gray' : '\033[0;37m',
 }
 
+
+CONFIG_FILE = '/etc/smokercli.yaml'
+
+
+def _get_plugins():
+    """
+    Get list of available host discovery plugin module names
+    """
+
+    plugins = []
+    conf_file = os.path.expanduser('~/.smokercli.yaml')
+
+    if not os.path.exists(conf_file):
+        conf_file = CONFIG_FILE
+
+    if not os.path.exists(conf_file):
+        return plugins
+
+    with open(conf_file) as f:
+        config = yaml.safe_load(f)
+
+    if config and 'plugin_paths' in config:
+        paths = config['plugin_paths']
+    else:
+        raise Exception('Invalid config file')
+
+
+    for path in paths:
+        try:
+            module = __import__(path)
+        except ImportError:
+            raise Exception('Invalid config file')
+
+        toplevel = os.path.dirname(module.__file__)
+        submodule = '/'.join(path.split('.')[1:])
+        plugin_dir = os.path.join(toplevel, submodule, '*.py')
+        modules = [os.path.basename(name)[:-3] for name in
+                   glob.glob(plugin_dir)]
+        modules.remove('__init__')
+        plugins += ['%s.%s' % (path, name) for name in modules]
+
+    return plugins
+
+
+def _get_plugin_arguments(name):
+    """
+    Get list of host discovery plugin specific cmdline arguments
+
+    :param name: plugin module name
+    """
+    try:
+        plugin = __import__(name, globals(), locals(), ['HostDiscoveryPlugin'])
+    except ImportError as e:
+        lg.error("Can't load module %s: %s" % (name, e))
+        raise
+    return plugin.HostDiscoveryPlugin.arguments
+
+
+def _add_plugin_arguments(parser):
+    """
+    Add host discovery plugin specific options to the cmdline argument parser
+
+    :param parser: argparse.ArgumentParser instance
+    """
+
+    plugins = _get_plugins()
+    if not plugins:
+        return
+    argument_group = parser.add_argument_group('Plugin arguments')
+
+    for plugin in plugins:
+        args = _get_plugin_arguments(plugin)
+        for argument in args:
+            argument_group.add_argument(*argument.args, **argument.kwargs)
+
+
+def _run_discovery_plugin(name, args):
+    """
+    Run the host discovery plugin
+    :param name: plugin module name
+    :param args: attribute namespace
+    :return: discovered hosts list
+    """
+    try:
+        this_plugin = __import__(name, globals(), locals(),
+                                 ['HostDiscoveryPlugin'])
+    except ImportError as e:
+        lg.error("Can't load module %s: %s" % (name, e))
+        raise
+
+    plugin=this_plugin.HostDiscoveryPlugin()
+    return plugin.get_hosts(args)
+
+
+def _host_discovery(args):
+    """
+    Run all the discovery plugins
+
+    :param args: attribute namespace
+    :return: discovered hosts list
+    """
+    discovered = []
+
+    for plugin in _get_plugins():
+        hosts = _run_discovery_plugin(plugin, args)
+        if hosts:
+            discovered += hosts
+
+    return discovered
+
+
 def main():
     """
     Main entrance
@@ -50,7 +164,7 @@ def main():
 
     # Host arguments
     group_main = parser.add_argument_group('Target host switchers')
-    group_main.add_argument('-s', '--hosts', dest='hosts', nargs='+', default=['localhost'], help="Hosts with running smokerd (default localhost)")
+    group_main.add_argument('-s', '--hosts', dest='hosts', nargs='+', help="Hosts with running smokerd (default localhost)")
 
     # Filtering options
     # List of plugins
@@ -83,6 +197,7 @@ def main():
     group_output.add_argument('-d', '--debug', dest='debug', action='store_true', help="Debug output")
     group_output.add_argument('--junit-config-file', dest='junit_config_file', help="Name of configuration file for junit xml formatter")
 
+    _add_plugin_arguments(parser)
     args = parser.parse_args()
 
     # Set log level and set args.no_progress option
@@ -336,8 +451,17 @@ def main():
     # Add status filter
     filters.append(('status', states))
 
+    hosts = ['localhost']
+    discovered_hosts = _host_discovery(args)
+    if args.hosts:
+        hosts = args.hosts
+        if discovered_hosts:
+            hosts += discovered_hosts
+    elif discovered_hosts:
+        hosts = discovered_hosts
+
     # Initialize Client
-    client = Client(args.hosts)
+    client = Client(hosts)
     plugins = client.get_plugins(filters, filters_negative=args.exclude, exclude_plugins=args.exclude_plugins)
 
     # No plugins found

smoker/client/plugins/__init__.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2007-2015, GoodData(R) Corporation. All rights reserved
+#
+# Example plugin:
+#
+# from smoker.client.plugins import SpecificArgument, HostDiscoveryPluginBase
+#
+#
+# class HostDiscoveryPlugin(HostDiscoveryPluginBase):
+#     """
+#     This is an example without any real world usability
+#     """
+#     arguments = [
+#         SpecificArgument(
+#             '-x',
+#             '--example',
+#             **{'dest': 'example',
+#                'help': 'Example parameter for host discovery'}
+#         ),
+#         SpecificArgument(
+#             '-y',
+#             '--example_prefix',
+#             **{'dest': 'prefix',
+#                'help': 'Another example for host discovery'}
+#         )
+#     ]
+#
+#     def get_hosts(self, args):
+#         if not args.example:
+#             return []
+#         if not args.prefix:
+#             return [args.example]
+#         return ['%s-%s' % (args.prefix, args.example)]
+
+
+class SpecificArgument(object):
+    """
+    Argparse argument to be added to the smoker CLI specific to this plugin
+    """
+    def __init__(self, short, long, **kwargs):
+        if short and long:
+            self.args = [short, long]
+        elif short:
+            self.args = [short]
+        else:
+            self.args = [long]
+        self.kwargs = kwargs
+
+
+class HostDiscoveryPluginBase(object):
+    """
+    Host discovery plugin interface
+    Inherit from this class when creating a host discovery plugin
+    """
+    # List of Specific Argument instances to be added
+    # to argparse.ArgumentParser
+    arguments = []
+
+    def get_hosts(self, args):
+        """
+        Override this method in your plugin
+
+        :return: discovered hosts
+        """
+        return []