diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..0a00940 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +graft etc +include rc.d/init.d/smokerd diff --git a/etc/smokercli-example.yaml b/etc/smokercli-example.yaml new file mode 100644 index 0000000..8637038 --- /dev/null +++ b/etc/smokercli-example.yaml @@ -0,0 +1,6 @@ +# Example configuration file for the smoker client (/etc/smokercli.yaml) +# plugin paths: Python modules containing the host discovery plugins. It has to +# be valid python module path. + +plugin_paths: + - smoker.client.plugins diff --git a/rc.d/init.d/smokerd-rhel b/rc.d/init.d/smokerd similarity index 82% rename from rc.d/init.d/smokerd-rhel rename to rc.d/init.d/smokerd index dcecf9f..5c2468c 100755 --- a/rc.d/init.d/smokerd-rhel +++ b/rc.d/init.d/smokerd @@ -32,30 +32,6 @@ exit_error() { exit 1 } -confgen() { - if [ "$GENCONFIG" -ne 1 ]; then - return 0 - fi - - cat $CONFDIR/common.yaml > $CONFIG || exit_error - - dirs=('template' 'action' 'plugin') - for dir in ${dirs[*]}; do - echo -e "\n${dir}s:" >> $CONFIG - - # Skip if directory doesn't contain YAML files - if [ "`ls ${CONFDIR}/${dir}.d/*.yaml 2>/dev/null|wc -l`" -eq 0 ]; then - continue - fi - - for plugin in ${CONFDIR}/${dir}.d/*.yaml; do - name=`basename $plugin .yaml` - - echo " ${name}: !include ${dir}.d/${name}.yaml" >> $CONFIG - done - done -} - start() { [ -x $BINARY ] || exit 5 @@ -66,7 +42,6 @@ start() { [ -f "$PIDFILE" ] && exit_error "PID file $PIDFILE already exists!" [ -f "$LOCKFILE" ] && exit_error "Subsys $LOCKFILE is locked!" - confgen $BINARY $SMOKERD_OPTIONS RETVAL=$? if [ $RETVAL -eq 0 ]; then diff --git a/setup.py b/setup.py index 6eb7147..9e816a8 100755 --- a/setup.py +++ b/setup.py @@ -2,9 +2,13 @@ # -*- coding: utf-8 -*- # Copyright (C) 2007-2012, GoodData(R) Corporation. All rights reserved +import os import sys from setuptools import setup +CONFIGDIR = '/etc/smokerd' +INITDIR = '/etc/rc.d/init.d' + # Parameters for build params = { # This package is named gdc-smoker on Pypi, use it on register or upload actions @@ -16,6 +20,7 @@ 'smoker.server.plugins', 'smoker.client', 'smoker.client.out_junit', + 'smoker.client.plugins', 'smoker.logger', 'smoker.util' ], @@ -76,6 +81,71 @@ 'platforms' : ['POSIX'], 'provides' : ['smoker'], 'install_requires' : ['PyYAML', 'argparse', 'simplejson', 'psutil', 'setproctitle', 'Flask-RESTful'], + 'data_files': [ + (INITDIR, ['rc.d/init.d/smokerd']), + (CONFIGDIR, ['etc/smokerd-example.yaml', 'etc/smokercli-example.yaml']) + ] } +# Get current branch +branch = os.getenv('GIT_BRANCH') +if not branch: + branch = os.popen( + 'git branch|grep -v "no branch"|grep \*|sed s,\*\ ,,g').read().strip() + + if not branch: + branch = 'master' +else: + branch = branch.replace('origin/', '') + +# Get git revision hash +revision = os.popen('git rev-parse --short HEAD').read().strip() + +if not revision: + revision = '0' + +# Get build number +build = os.getenv('BUILD_NUMBER') +if not build: + build = '1' + +try: + action = sys.argv[1] +except IndexError: + action = None + +if action == 'clean': + # Remove MANIFEST file + print "Cleaning MANIFEST.." + try: + os.unlink('MANIFEST') + except OSError as e: + if e.errno == 2: + pass + else: + raise + + # Remove dist and build directories + for dir in ['dist', 'build']: + print "Cleaning %s.." % dir + for root, dirs, files in os.walk(dir, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + try: + os.rmdir(dir) + except OSError as e: + if e.errno == 2: + pass + else: + raise +elif action == 'bdist_rpm': + # Set release number + sys.argv.append('--release=1.%s.%s' % (build, revision)) + # Require same version of gdc-python-common package + sys.argv.append( + '--requires=python-argparse python-simplejson python-setproctitle ' + 'python-flask-restful') + setup(**params) diff --git a/smoker/client/cli.py b/smoker/client/cli.py index 8080c1c..bad3e96 100644 --- a/smoker/client/cli.py +++ b/smoker/client/cli.py @@ -9,20 +9,23 @@ """ import argparse +import datetime +import glob import logging +import os +import simplejson +import sys +import yaml from smoker.client import Client import smoker.logger +from smoker.client.out_junit import plugins_to_xml +from smoker.util.tap import TapTest, Tap + smoker.logger.init(syslog=False) lg = logging.getLogger('smokercli') -from smoker.util.tap import TapTest, Tap -from smoker.client.out_junit import plugins_to_xml - -import sys -import simplejson -import datetime COLORS = { 'default' : '\033[0;0m', @@ -36,6 +39,117 @@ 'gray' : '\033[0;37m', } + +CONFIG_FILE = '/etc/smokercli.yaml' + + +def _get_plugins(): + """ + Get list of available host discovery plugin module names + """ + + plugins = [] + conf_file = os.path.expanduser('~/.smokercli.yaml') + + if not os.path.exists(conf_file): + conf_file = CONFIG_FILE + + if not os.path.exists(conf_file): + return plugins + + with open(conf_file) as f: + config = yaml.safe_load(f) + + if config and 'plugin_paths' in config: + paths = config['plugin_paths'] + else: + raise Exception('Invalid config file') + + + for path in paths: + try: + module = __import__(path) + except ImportError: + raise Exception('Invalid config file') + + toplevel = os.path.dirname(module.__file__) + submodule = '/'.join(path.split('.')[1:]) + plugin_dir = os.path.join(toplevel, submodule, '*.py') + modules = [os.path.basename(name)[:-3] for name in + glob.glob(plugin_dir)] + modules.remove('__init__') + plugins += ['%s.%s' % (path, name) for name in modules] + + return plugins + + +def _get_plugin_arguments(name): + """ + Get list of host discovery plugin specific cmdline arguments + + :param name: plugin module name + """ + try: + plugin = __import__(name, globals(), locals(), ['HostDiscoveryPlugin']) + except ImportError as e: + lg.error("Can't load module %s: %s" % (name, e)) + raise + return plugin.HostDiscoveryPlugin.arguments + + +def _add_plugin_arguments(parser): + """ + Add host discovery plugin specific options to the cmdline argument parser + + :param parser: argparse.ArgumentParser instance + """ + + plugins = _get_plugins() + if not plugins: + return + argument_group = parser.add_argument_group('Plugin arguments') + + for plugin in plugins: + args = _get_plugin_arguments(plugin) + for argument in args: + argument_group.add_argument(*argument.args, **argument.kwargs) + + +def _run_discovery_plugin(name, args): + """ + Run the host discovery plugin + :param name: plugin module name + :param args: attribute namespace + :return: discovered hosts list + """ + try: + this_plugin = __import__(name, globals(), locals(), + ['HostDiscoveryPlugin']) + except ImportError as e: + lg.error("Can't load module %s: %s" % (name, e)) + raise + + plugin=this_plugin.HostDiscoveryPlugin() + return plugin.get_hosts(args) + + +def _host_discovery(args): + """ + Run all the discovery plugins + + :param args: attribute namespace + :return: discovered hosts list + """ + discovered = [] + + for plugin in _get_plugins(): + hosts = _run_discovery_plugin(plugin, args) + if hosts: + discovered += hosts + + return discovered + + def main(): """ Main entrance @@ -50,7 +164,7 @@ def main(): # Host arguments group_main = parser.add_argument_group('Target host switchers') - group_main.add_argument('-s', '--hosts', dest='hosts', nargs='+', default=['localhost'], help="Hosts with running smokerd (default localhost)") + group_main.add_argument('-s', '--hosts', dest='hosts', nargs='+', help="Hosts with running smokerd (default localhost)") # Filtering options # List of plugins @@ -83,6 +197,7 @@ def main(): group_output.add_argument('-d', '--debug', dest='debug', action='store_true', help="Debug output") group_output.add_argument('--junit-config-file', dest='junit_config_file', help="Name of configuration file for junit xml formatter") + _add_plugin_arguments(parser) args = parser.parse_args() # Set log level and set args.no_progress option @@ -336,8 +451,17 @@ def main(): # Add status filter filters.append(('status', states)) + hosts = ['localhost'] + discovered_hosts = _host_discovery(args) + if args.hosts: + hosts = args.hosts + if discovered_hosts: + hosts += discovered_hosts + elif discovered_hosts: + hosts = discovered_hosts + # Initialize Client - client = Client(args.hosts) + client = Client(hosts) plugins = client.get_plugins(filters, filters_negative=args.exclude, exclude_plugins=args.exclude_plugins) # No plugins found diff --git a/smoker/client/plugins/__init__.py b/smoker/client/plugins/__init__.py new file mode 100644 index 0000000..9ef8422 --- /dev/null +++ b/smoker/client/plugins/__init__.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2007-2015, GoodData(R) Corporation. All rights reserved +# +# Example plugin: +# +# from smoker.client.plugins import SpecificArgument, HostDiscoveryPluginBase +# +# +# class HostDiscoveryPlugin(HostDiscoveryPluginBase): +# """ +# This is an example without any real world usability +# """ +# arguments = [ +# SpecificArgument( +# '-x', +# '--example', +# **{'dest': 'example', +# 'help': 'Example parameter for host discovery'} +# ), +# SpecificArgument( +# '-y', +# '--example_prefix', +# **{'dest': 'prefix', +# 'help': 'Another example for host discovery'} +# ) +# ] +# +# def get_hosts(self, args): +# if not args.example: +# return [] +# if not args.prefix: +# return [args.example] +# return ['%s-%s' % (args.prefix, args.example)] + + +class SpecificArgument(object): + """ + Argparse argument to be added to the smoker CLI specific to this plugin + """ + def __init__(self, short, long, **kwargs): + if short and long: + self.args = [short, long] + elif short: + self.args = [short] + else: + self.args = [long] + self.kwargs = kwargs + + +class HostDiscoveryPluginBase(object): + """ + Host discovery plugin interface + Inherit from this class when creating a host discovery plugin + """ + # List of Specific Argument instances to be added + # to argparse.ArgumentParser + arguments = [] + + def get_hosts(self, args): + """ + Override this method in your plugin + + :return: discovered hosts + """ + return [] diff --git a/smoker/server/daemon.py b/smoker/server/daemon.py index b87d2bc..a2b95b7 100644 --- a/smoker/server/daemon.py +++ b/smoker/server/daemon.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # Copyright (C) 2007-2012, GoodData(R) Corporation. All rights reserved +import glob import logging import os import signal @@ -43,6 +44,8 @@ def __init__(self, **kwargs): lg.info("Config argument not submitted, default config file will be used!") self.conf['config'] = '/etc/smokerd/smokerd.yaml' + self.conf_dirs = [os.path.dirname(self.conf['config'])] + self._load_config() def _yaml_include(self, loader, node): @@ -50,7 +53,14 @@ def _yaml_include(self, loader, node): Include another yaml file from main file This is usually done by registering !include tag """ - filepath = "%s/%s" % (os.path.dirname(self.conf['config']), node.value) + filepath = node.value + if not os.path.exists(filepath): + for dir in self.conf_dirs: + filepath = os.path.join(dir, node.value) + if os.path.exists(filepath): + break + + self.conf_dirs.append(os.path.dirname(filepath)) try: with open(filepath, 'r') as inputfile: return yaml.load(inputfile) @@ -58,21 +68,43 @@ def _yaml_include(self, loader, node): lg.error("Can't include config file %s: %s" % (filepath, e)) raise + def _yaml_include_dir(self, loader, node): + """ + Include another yaml file from main file + This is usually done by registering !include tag + """ + if not os.path.exists(node.value): + return + + yamls = glob.glob(os.path.join(node.value, '*.yaml')) + if not yamls: + return + + content = '\n' + + for file in yamls: + plugin, _ = os.path.splitext(os.path.basename(file)) + content += ' %s: !include %s\n' % (plugin, file) + + return yaml.load(content) + def _load_config(self): """ Load specified config file """ try: - fp = open(self.conf['config'], 'r') + with open(self.conf['config'], 'r') as fp: + config = fp.read() except IOError as e: lg.error("Can't read config file %s: %s" % (self.conf['config'], e)) raise - # Register include constructor + # Register include constructors + yaml.add_constructor('!include_dir', self._yaml_include_dir) yaml.add_constructor('!include', self._yaml_include) try: - conf = yaml.load(fp) + conf = yaml.load(config) except Exception as e: lg.error("Can't parse config file %s: %s" % (self.conf['config'], e)) raise