diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..7b35e57 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,20 @@ +## Additional Information +(The following information is very important in order to help us to help you. Omission of the following details may delay your support request or receive no attention at all.) + +- Version of python (python --version) + ``` + ``` + +- Version of K2HR3 OpenStack Notification Listener (k2hr3-osnl -v) + ``` + ``` + +- System information (uname -a) + ``` + ``` + +- Distro (cat /etc/issue) + ``` + ``` + +## Details about issue diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..40f1ac1 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,5 @@ +## Relevant Issue (if applicable) +(If there are Issues related to this PullRequest, please list it.) + +## Details +(Please describe the details of PullRequest.) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..d0c76da --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,56 @@ +name: CI + +on: + # Trigger the workflow on push or pull request, + # but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master ] + schedule: + - cron: '0 0 * * 1' + +jobs: + build: + + if: "! contains(toJSON(github.event.commits.*.message), '[skip ci]') && ! contains(toJSON(github.event.commits.*.message), '[ci skip]')" + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.6', '3.7', '3.8'] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + make init + - name: Lint with flake8, mypy, pylint and checkdocs + run: | + # stop the build if there are Python syntax errors or undefined names + make lint + - name: Run tests + run: | + make test + - name: Check coverage + run: | + make coverage + - name: Make docs + run: | + make docs + - name: Make dist package + run: | + make dist + - name: Install the package locally + run: | + make install + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + if: github.event_name == 'release' + run: | + make test-release diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..64240b5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,127 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +#*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..89ee6c6 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,13 @@ +======= +Credits +======= + +Development Lead +---------------- + +* Hirotaka Wakabayashi + +Contributors +------------ + +* Takeshi Nakatani diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..f4069f2 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,123 @@ +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every little bit +helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/yahoojapan/k2hr3_osnl/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help +wanted" is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +K2HR3 OpenStack Notification Listener could always use more documentation, whether as part of the +official K2HR3 OpenStack Notification Listener docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/yahoojapan/k2hr3_osnl/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `k2hr3_osnl` for local development. + +1. Fork the `k2hr3_osnl` repo on GitHub. +2. Clone your fork locally:: + + $ git clone git@github.com:your_name_here/k2hr3_osnl.git + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + + $ cd k2hr3_osnl/ + $ pip3 install pipenv + $ python3 -m pipenv install -dev --python /path/to/python3 + $ pipenv shell + +4. Create a branch for local development:: + + (k2hr3_osnl) $ git checkout -b name-of-your-bugfix-or-feature + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass flake8, + mypy, pylint and the tests, including testing other Python versions + with make lint test + + Make sure you are in virtual environment:: + + (k2hr3_osnl) $ make lint test + +6. Commit your changes and push your branch to GitHub:: + + (k2hr3_osnl) $ git add . + (k2hr3_osnl) $ git commit -m "Your detailed description of your changes." + (k2hr3_osnl) $ git push origin name-of-your-bugfix-or-feature + +7. Submit a pull request through the GitHub website. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 3.5, 3.6 and 3.7. Check + https://travis-ci.org/yahoojapan/k2hr3_osnl/pull_requests + and make sure that the tests pass for all supported Python versions. + +Tips +---- + +To run a subset of tests:: + + + $ make test + +Deploying +--------- + +A reminder for the maintainers on how to deploy. +Make sure all your changes are committed (including an entry in HISTORY.rst). +Then visit the release page at https://github.com/yahoojapan/k2hr3_osnl/releases +and create a new release note. + +Travis will then deploy to PyPI if tests pass. diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..7f813ef --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,34 @@ +======= +History +======= + +0.9.5 (2020-12-01) +------------------- + +* Removes the --user option from pip command + +0.9.4 (2020-12-01) +------------------- + +* Changes CI from Travis to GitHub Actions + +0.9.3 (2020-11-30) +------------------- + +* Supports OpenStack Ussuri + +0.9.2 (2019-03-26) +------------------- + +* Fix systemd unitfile for FHS v3 +* Fix RPM specfile for the indicated items on Fedora Review No.1663668 + +0.9.1 (2019-03-19) +------------------- + +* Fixed systemd unitfile. + +0.9.0 (2019-03-06) +------------------- + +* First release on PyPI. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..58f3279 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2018 Yahoo Japan Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..965b2dd --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.rst +include LICENSE +include README.rst + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b4b6d28 --- /dev/null +++ b/Makefile @@ -0,0 +1,174 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +.PHONY: clean clean-test clean-pyc clean-build docs help +.DEFAULT_GOAL := help + +define BROWSER_PYSCRIPT +import os, webbrowser, sys + +try: + from urllib import pathname2url +except: + from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +BROWSER := python3 -c "$$BROWSER_PYSCRIPT" + +help: + @python3 -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +# Initialize development environment +# Initialization fails if dependency problems happens or security vulnerabilities are detected. +# +# Note: +# Make sure python3-devel or python3-dev is installed. Because oslo-message(or dependent libs) requires Python.h +init: + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade pipenv + pipenv install --dev --skip-lock + pipenv graph + pipenv check + +# Lint code and docs +# lint fails if there are syntax errors or undefined names. +# +# Note: +# lint commands should be emit in virtualenv activated environment. +lint: + pipenv run flake8 --version + pipenv run flake8 k2hr3_osnl tests + pipenv run mypy k2hr3_osnl tests + pipenv run pylint k2hr3_osnl tests --py3k -r n + pipenv run python3 setup.py checkdocs + +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +clean-build: ## remove build artifacts + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + rm -f VERSION RPMSPEC_VERSION + +clean-pyc: ## remove Python file artifacts + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: ## remove test and coverage artifacts + rm -fr ..mypy_cache/ + rm -f .coverage + rm -fr htmlcov/ + rm -fr .pytest_cache + +test-only: ## run tests quickly with the default Python + pipenv run python3 -m unittest + +# Check version strings consistency +# Make sure the following version strings are same. +# 1. HISTORY.rst +# 2. python-k2hr3-osnl.spec +# 3. __init__.py +version: + @rm -f VERSION RPMSPEC_VERSION + @perl -ne 'print if /^[0-9]+.[0-9]+.[0-9]+ \([0-9]{4}-[0-9]{2}-[0-9]{2}\)$$/' HISTORY.rst \ + | head -n 1 | perl -lne 'print $$1 if /^([0-9]+.[0-9]+.[0-9]+) \(.*\)/' > VERSION + @perl -ne 'print $$2 if /^Version:(\s+)([0-9]+.[0-9]+.[0-9]+)$$/' python-k2hr3-osnl.spec > RPMSPEC_VERSION + +SOURCE_VERSION = $(shell pipenv run python3 -c 'import k2hr3_osnl; print(k2hr3_osnl.version())') +HISTORY_VERSION = $(shell cat VERSION) +RPMSPEC_VERSION = $(shell cat RPMSPEC_VERSION) + +# Check version strings consistency +# Make sure the following version strings are same. +# 1. HISTORY.rst +# 2. python-k2hr3-osnl.spec +# 3. __init__.py +test: version ## builds source and wheel package + @echo 'source ' ${SOURCE_VERSION} + @echo 'history ' ${HISTORY_VERSION} + @echo 'rpmspec ' ${RPMSPEC_VERSION} + @if test "${SOURCE_VERSION}" = "${HISTORY_VERSION}" -a "${HISTORY_VERSION}" = "${RPMSPEC_VERSION}" ; then \ + pipenv run python3 -m unittest ; \ + else \ + exit 1; \ + fi + +test-all: lint test + +coverage: ## check code coverage quickly with the default Python + pipenv run coverage run --source k2hr3_osnl -m unittest + pipenv run coverage report -m + pipenv run coverage xml + pipenv run coverage html +# $(BROWSER) htmlcov/index.html + +docs: ## generate Sphinx HTML documentation, including API docs + rm -f docs/k2hr3_osnl.rst + rm -f docs/modules.rst + pipenv run sphinx-apidoc -o docs/ k2hr3_osnl + $(MAKE) -C docs clean + $(MAKE) -C docs html +# $(BROWSER) docs/_build/html/index.html + +servedocs: docs ## compile the docs watching for changes + pipenv run watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . + +release: dist ## package and upload a release + pipenv run twine check dist/* + pipenv run twine upload dist/* + +test-release: + pipenv run twine check dist/* + pipenv run twine upload --repository-url https://test.pypi.org/legacy/ dist/* + +dist: clean version ## builds source and wheel package + @echo 'source ' ${SOURCE_VERSION} + @echo 'history ' ${HISTORY_VERSION} + @if test "${SOURCE_VERSION}" = "${HISTORY_VERSION}" -a "${HISTORY_VERSION}" = "${RPMSPEC_VERSION}" ; then \ + pipenv run python3 setup.py sdist ; \ + pipenv run python3 setup.py bdist_wheel ; \ + ls -l dist ; \ + fi + +install: clean ## install the package to the active Python's site-packages + pipenv run python3 setup.py install + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..8b88cd0 --- /dev/null +++ b/Pipfile @@ -0,0 +1,27 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +coverage = {version = ">=5.3,<5.4"} +xmlrunner = {version = ">=1.7.7,<1.7.8"} +flake8 = {version = ">=3.8.4,<3.8.5"} +flake8-docstrings = {version = ">=1.5.0,<1.5.1"} +pep8-naming = {version = ">=0.11.1,<0.11.2"} +pycodestyle = {version = ">=2.6.0,<2.6.1"} +mccabe = {version = ">=0.6.1,<0.6.2"} +sphinx = {version = ">=3.3.1,<3.3.2"} +twine = {version = ">=3.2.0,<3.2.1"} +pylint = {version = ">=2.6.0,<2.6.1"} +astroid = {version = ">=2.4.2,<2.4.3"} +typed-ast = {version = ">=1.4.1,<1.4.2"} +mypy = {version = ">=0.790,<0.791"} +importlib-metadata = {version = ">=2.1.0,<2.2.0"} +"collective.checkdocs" = "*" +argh = {version = ">=0.26.2,<0.26.3"} +watchdog = {version = ">=0.10.4,<0.10.5"} + +[packages] +oslo-config = "*" +oslo-messaging = "*" diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..5eccd61 --- /dev/null +++ b/README.rst @@ -0,0 +1,83 @@ +===================================== +K2HR3 OpenStack Notification Listener +===================================== + + +.. image:: https://img.shields.io/pypi/v/k2hr3_osnl.svg + :target: https://pypi.org/project/k2hr3-osnl + +.. image:: https://img.shields.io/travis/yahoojapan/k2hr3_osnl.svg + :target: https://travis-ci.org/yahoojapan/k2hr3_osnl + +.. image:: https://readthedocs.org/projects/k2hr3-osnl/badge/?version=latest + :target: https://k2hr3-osnl.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://img.shields.io/badge/license-MIT-blue.svg + :target: https://github.com/yahoojapan/k2hr3_osnl/blob/master/LICENSE + + +An OpenStack notification listener for the K2HR3 role-based ACL system developed in Yahoo Japan Corporation. + + +Overview +-------- + +**k2hr3_osnl** is **K2HR3** **O** pen **S** tack **N** otification **L** istener that is a part of the K2HR3_ +system, which is a role-based ACL system developed in `Yahoo Japan Corporation`_. + +.. _K2HR3: https://k2hr3.antpick.ax/ +.. _`Yahoo Japan Corporation`: https://about.yahoo.co.jp/info/en/company/ + +**k2hr3_osnl** is an OpenStack_ Notification Listener that listens to notifications from +OpenStack_ services. OpenStack_ services emit notifications to the message bus, which is provided +by oslo.messaging_. oslo.messaging_ transports notification messages to a message broker server. +The default broker server is RabbitMQ_. When **k2hr3_osnl** catches a notification message from RabbitMQ_, +it sends the payload to the K2HR3_ that is a role-based ACL system that provides access control +for OpenStack virtual machine instances. Figure 1 shows the relationship between the components. + +.. _OpenStack: https://www.openstack.org/ +.. _oslo.messaging: https://docs.openstack.org/oslo.messaging/latest/ +.. _RabbitMQ: http://www.rabbitmq.com/ + +Fig.1: overview + +.. image:: https://raw.githubusercontent.com/yahoojapan/k2hr3_osnl/master/docs/k2hr3_osnl_overview.png + + +Document +-------- + +https://k2hr3-osnl.readthedocs.io/ + + +K2HR3 - K2Hdkc based Resource and Roles and policy Rules +-------------------------------------------------------- + +K2HR3_ is a role-based ACL system developed in `Yahoo Japan Corporation`_. + +.. _`Yahoo Japan Corporation`: https://about.yahoo.co.jp/info/en/company/ + + +License +-------- + +MIT License + + +AntPickax +--------- + +**k2hr3_osnl** is a project by AntPickax_, which is an open source team in `Yahoo Japan Corporation`_. + +.. _AntPickax: https://antpick.ax/ +.. _`Yahoo Japan Corporation`: https://about.yahoo.co.jp/info/en/company/ + +Credits +------- + +This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. + +.. _Cookiecutter: https://github.com/audreyr/cookiecutter +.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage + diff --git a/RPMSPEC_VERSION b/RPMSPEC_VERSION new file mode 100644 index 0000000..0383441 --- /dev/null +++ b/RPMSPEC_VERSION @@ -0,0 +1 @@ +0.9.5 \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..b0bb878 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.9.5 diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..306f58b --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,45 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = pipenv run python3 -msphinx +SPHINXPROJ = k2hr3_osnl +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/docs/authors.rst b/docs/authors.rst new file mode 100644 index 0000000..e122f91 --- /dev/null +++ b/docs/authors.rst @@ -0,0 +1 @@ +.. include:: ../AUTHORS.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100755 index 0000000..ce2da86 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +import os +import sys +sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('../k2hr3_osnl')) + +import k2hr3_osnl + +# -- General configuration --------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'K2HR3 OpenStack Notification Listener' +copyright = u"2018, Yahoo Japan Corporation" +author = u"Hirotaka Wakabayashi" + +# The version info for the project you're documenting, acts as replacement +# for |version| and |release|, also used in various other places throughout +# the built documents. +# +# The short X.Y version. +version = k2hr3_osnl.__version__ +# The full version, including alpha/beta/rc tags. +release = k2hr3_osnl.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a +# theme further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output --------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'k2hr3_osnldoc' + + +# -- Options for LaTeX output ------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'k2hr3_osnl.tex', + u'K2HR3 OpenStack Notification Listener Documentation', + u'Hirotaka Wakabayashi', 'manual'), +] + + +# -- Options for manual page output ------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'k2hr3_osnl', + u'K2HR3 OpenStack Notification Listener Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'k2hr3_osnl', + u'K2HR3 OpenStack Notification Listener Documentation', + author, + 'k2hr3_osnl', + 'One line description of project.', + 'Miscellaneous'), +] + +# +# EOF +# diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..e582053 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 0000000..2506499 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1 @@ +.. include:: ../HISTORY.rst diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..140aac9 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +Welcome to K2HR3 OpenStack Notification Listener's documentation! +================================================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + readme + installation + usage + modules + contributing + authors + history + +Indices and tables +================== +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..d2c1454 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,174 @@ +.. highlight:: shell + +============ +Installation +============ + + +Stable release +-------------- + +To install K2HR3 OpenStack Notification Listener, run this command in your terminal: + +.. code-block:: console + + $ sudo pip install k2hr3_osnl + +This is the preferred method to install K2HR3 OpenStack Notification Listener, as it will always install the most recent stable release. + +If you don't have `pip`_ installed, this `Python installation guide`_ can guide +you through the process. + +.. _pip: https://pip.pypa.io +.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ + +Configuration +------------- + +There are two primary configurations in ``k2hr3_onsl.conf``: + +* ``transport_url`` + The message queue server location with an username and a password. +* ``api_url`` + The k2hr3 WebApi server location. + +The following figure can help you to understand the transport_url and api_url: + +.. image:: k2hr3_osnl_configure.png + +The ``k2hr3_osnl.conf`` path depends on the pip_ command shipped in your OS. You can get the path by two commands. + +.. _pip: https://pip.pypa.io + +.. code-block:: console + + $ sudo pip3 show -f k2hr3_osnl + Name: k2hr3-osnl + ... + Location: /usr/local/lib/python3.5/dist-packages + Requires: oslo.messaging, oslo.config + Files: + ../../../bin/k2hr3_osnl + ../../../etc/k2hr3/k2hr3_osnl.conf + ... + $ python -c " + > import os; + > print( + > os.path.abspath( + > '/usr/local/lib/python3.5/dist-packages/../../../etc/k2hr3/k2hr3_osnl.conf' + > ) + > ); + > " + /usr/local/etc/k2hr3/k2hr3_osnl.conf + +``/usr/local/etc/k2hr3/k2hr3_osnl.conf`` is it. You should change the ``transport_url`` and the ``api_url`` setting for your environment. + +.. code-block:: ini + + [DEFAULT] + debug_level = error + + [oslo_messaging_notifications] + event_type = ^port\.delete\.end$ + publisher_id = ^network.*$ + transport_url = rabbit://user:pass@127.0.0.1:5672/ + topic = notifications + exchange = neutron + + [k2hr3] + api_url = https://localhost/v1/role + allow_self_signed_cert = False + + +FYI: The `Usage` page describes every setting parameters. + +Start +----- + +This chapter instructs how to install the listener. + +.. code-block:: console + + $ sudo k2hr3_osnl -c /path/to/k2hr3_osnl.conf + +No error means the listener successfully starts to listen to the next notification message. + +Service Management +------------------ + +While you have already successfully started the listener, you would like to prepare for following troubles. + +* The listener process is dead after the OS rebooted. +* The listener is dead when you have stopped the terminal which started the listener. + +Most of modern OSs provide the way to register a process as a service to the service management system which launches them at boot time and stops them at shutdown. +systemd_ is one of such a service which is installed in Debian 9, Fedora 29, CentOS 7 and other recent Linux distributions. + +.. _systemd: https://freedesktop.org/wiki/Software/systemd/ + +An example of what systemd_ config file might look like is: + +.. _systemd: https://freedesktop.org/wiki/Software/systemd/ + +.. code-block:: ini + + [Unit] + Description=k2hr3_osnl + After=network-online.target + + [Service] + Type=simple + WorkingDirectory=/tmp + Environment=HOME=/tmp + User=nobody + Group=nobody + ExecStart=/usr/local/bin/k2hr3_osnl -c /usr/local/etc/k2hr3/k2hr3_osnl.conf + Restart=on-failure + PIDFile=/var/run/k2hr3_osnl.pid + + [Install] + WantedBy=multi-user.target + +FYI: systemd.unit_ and systemd.service_ page describe meaning of parameters. + +.. _systemd.service: https://www.freedesktop.org/software/systemd/man/systemd.service.html# +.. _systemd.unit: https://www.freedesktop.org/software/systemd/man/systemd.unit.html# + +The syntax is the ".INI" style. ``ExecStart`` specifies the absolute ``k2hr3_osnl`` path. +The path depends on your OS. I found it in ``/usr/local/bin/k2hr3_osnl`` in my environment. + +.. code-block:: console + + $ which k2hr3_osnl + /usr/local/bin/k2hr3_osnl + +After update the ``ExecStart``, save the configuration to the ``/lib/systemd/system/k2hr3_osnl.service`` and register it to systemd_. Please note the systemd configuration path depends on you OS. + +.. _systemd: https://freedesktop.org/wiki/Software/systemd/ + +.. code-block:: console + + $ sudo systemctl daemon-reload + $ sudo systemctl enable k2hr3_osnl.service + + +After that, you tell systemd_ to look for your service at the first command and you tell systemd_ to enable it at the second command, so that it will start every time the system boots. + +.. _systemd: https://freedesktop.org/wiki/Software/systemd/ + + +Then, you start the k2hr3_osnl as a service. + +.. code-block:: console + + $ sudo systemctl start k2hr3_osnl.service + +You can see the service status: + +.. code-block:: console + + $ sudo systemctl status k2hr3_osnl.service + +If you have got some errors, you should check logs put on stderr at first. Then please send a issue with it from issue_. + +.. _issue: https://github.com/yahoojapan/k2hr3_osnl/issues diff --git a/docs/k2hr3_osnl.rst b/docs/k2hr3_osnl.rst new file mode 100644 index 0000000..4ce72e3 --- /dev/null +++ b/docs/k2hr3_osnl.rst @@ -0,0 +1,53 @@ +k2hr3\_osnl package +=================== + +Submodules +---------- + +k2hr3\_osnl.cfg module +---------------------- + +.. automodule:: k2hr3_osnl.cfg + :members: + :undoc-members: + :show-inheritance: + +k2hr3\_osnl.endpoint module +--------------------------- + +.. automodule:: k2hr3_osnl.endpoint + :members: + :undoc-members: + :show-inheritance: + +k2hr3\_osnl.exceptions module +----------------------------- + +.. automodule:: k2hr3_osnl.exceptions + :members: + :undoc-members: + :show-inheritance: + +k2hr3\_osnl.httpresponse module +------------------------------- + +.. automodule:: k2hr3_osnl.httpresponse + :members: + :undoc-members: + :show-inheritance: + +k2hr3\_osnl.useragent module +---------------------------- + +.. automodule:: k2hr3_osnl.useragent + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: k2hr3_osnl + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/k2hr3_osnl_configure.png b/docs/k2hr3_osnl_configure.png new file mode 100644 index 0000000..38e2e83 Binary files /dev/null and b/docs/k2hr3_osnl_configure.png differ diff --git a/docs/k2hr3_osnl_overview.png b/docs/k2hr3_osnl_overview.png new file mode 100644 index 0000000..e983239 Binary files /dev/null and b/docs/k2hr3_osnl_overview.png differ diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..376a4bf --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=python -msphinx +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=k2hr3_osnl + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The Sphinx module was not found. Make sure you have Sphinx installed, + echo.then set the SPHINXBUILD environment variable to point to the full + echo.path of the 'sphinx-build' executable. Alternatively you may add the + echo.Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/modules.rst b/docs/modules.rst new file mode 100644 index 0000000..b489dad --- /dev/null +++ b/docs/modules.rst @@ -0,0 +1,7 @@ +k2hr3_osnl +========== + +.. toctree:: + :maxdepth: 4 + + k2hr3_osnl diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 0000000..72a3355 --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1 @@ +.. include:: ../README.rst diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000..312a6fa --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,358 @@ +===== +Usage +===== + +In this chapter I will explain how to configure the k2hr3_osnl. The k2hr3_osnl primarily consists of three parts of processing. + +1. Listening to OpenStack service backend +2. Parsing messages from OpenStack service backend to extract an instance-id +3. Calling the k2hr3 api with the instance-id + +``k2hr3_osnl.conf`` defines the k2hr3_osnl behavior. You can get the path by two commands. + +.. code-block:: console + + $ sudo pip3 show -f k2hr3_osnl + Name: k2hr3-osnl + ... + Location: /usr/local/lib/python3.5/dist-packages + Requires: oslo.messaging, oslo.config + Files: + ../../../bin/k2hr3_osnl + ../../../etc/k2hr3/k2hr3_osnl.conf + ... + $ python -c " + > import os; + > print(os.path.abspath('/usr/local/lib/python3.5/dist-packages/../../../etc/k2hr3/k2hr3_osnl.conf')); + > " + /usr/local/etc/k2hr3/k2hr3_osnl.conf + +``/usr/local/etc/k2hr3/k2hr3_osnl.conf`` is it in this case. + +Please note all information here I describe is based on `OpenStack Rocky`_. + +.. _`OpenStack Rocky`: https://releases.openstack.org/rocky/index.html + +Listening to OpenStack service backend +------------------------------------------ + +The following 3 parameters define the listener behavior: ``transport_url``, ``topic`` and ``exchange`` in k2hr3_osnl.conf. + +The ``transport_url`` specifies the address of OpenStack service backend and how to connect with it. oslo.messaging_ describes the syntax: + +.. _oslo.messaging: https://docs.openstack.org/oslo.messaging/latest/admin/AMQP1.0.html#transport-url-enable + + transport://user:pass@host1:port[,hostN:portN]/virtual_host + +The transport value specifies the notification backend as one of **amqp**, RabbitMQ, Apache Kafka and other backend. The default backend is RabbitMQ. For Instance, assuming the backend service is RabbitMQ , the file might contain: + +.. code-block:: INI + + [oslo_messaging_notifications] + transport_url = rabbit://guest:guestpass@127.0.0.1:5672/ + +The setting above means: + +* rabbitmq is a backend server. +* user name is guest. +* password is guestpass. +* address is localhost. +* port is 5672. + +Please note username and password is required for security reason. `RabbitMQ User Management`_ describes how to add a username and password. + +.. _`RabbitMQ User Management`: https://www.rabbitmq.com/rabbitmqctl.8.html#User_Management + +The ``topic`` parameter identifies where a message should be sent or what messages the k2hr3_osnl is listening for. The OpenStack services emit messages by the `oslo.messaging Notifier`_ which requires ``topic(s)``. A default value of ``topic(s)`` is ``notifications`` which is the same with the k2hr3_osnl's default ``topic`` value. An example of what the file might contain is: + +.. _`oslo.messaging Notifier`: https://docs.openstack.org/oslo.messaging/latest/reference/notifier.html + +.. code-block:: INI + + [oslo_messaging_notifications] + topic = notifications + +Please note the ``topic`` must be the same between OpenStack services and the k2hr3_osnl, because it is a part of subscriber queue name in OpenStack backend that the k2hr3_osnl listens to. So please remember you would need update it if OpenStack service administrators can change it the other value. + +The final parameter is ``exchange``. This parameter represents a container within which each service's topics are scoped. OpenStack services register the exchange when the send notifications by calling the set_transport_defaults_ function in oslo.messaging. The default value of ``exchange`` is 'openstack'. + +.. _set_transport_defaults: https://docs.openstack.org/oslo.messaging/latest/reference/transport.html#oslo_messaging.set_transport_defaults + +What I have explained in this chapter: + +* k2hr3_osnl connect with OpenStack service backend by transport_url. +* OpenStack services publish notifications to the configured exchange with a configured topic. +* The default topic name is ``notifications``. It can be changed. +* The exchange is almost same with the OpenStack service publishes the notification message. + +Parsing a message +--------------------- + +A message format depends on your OpenStack service settings. Currently the k2hr3_osnl can parse the following 3 kinds of message. + +1. a message from OpenStack neutron +2. a versioned message from OpenStack nova +3. a non-versioned message from OpenStack nova + +I will explain them one by one. + +Parsing a message from OpenStack neutron +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We assume the following message from `OpenStack neutron`_. + +.. _`OpenStack neutron`: https://docs.openstack.org/neutron/latest/ + +.. code-block:: javascript + + { + "event_type": "port.delete.end", + "message_id": "76c35877-9d0c-4faf-b4e5-7c51828f37a0", + "payload": { + ... + "device_id": "12345678-1234-5678-1234-567812345678", + "device_owner": "compute:nova", + "extra_dhcp_opts": [], + "fixed_ips": [ + { + "ip_address": "172.16.0.1", + "subnet_id": "subnet01-ffff-ffff-ffff-ffffffffffff" + }, + { + "ip_address": "2001:db8::6", + "subnet_id": "subnet02-ffff-ffff-ffff-ffffffffffff" + } + ], + ... + }, + "priority": "INFO", + "publisher_id": "network.hostname.domain_name", + "timestamp": "2018-11-25 09:00:06.842727" + } + +The ``event_type`` represents a event which OpenStack services send notification about and the ``publisher_id`` identifies who published the message. Let's see the 'oslo_messaging_notifications' group configuration to catch this message. + +.. code-block:: ini + + [oslo_messaging_notifications] + event_type = ^port\.delete\.end$ + publisher_id = ^network.*$ + transport_url = rabbit://user:pass@127.0.0.1:5672/ + topic = notifications + exchange = neutron + +The ``event_type`` and ``publisher_id`` define white rules that means k2hr3_osnl only parse the messages that match the filter’s rules. If your `Openstack neutron`_ emits a same message with this example, you can use the same configure with this example. + +.. _`OpenStack neutron`: https://docs.openstack.org/neutron/latest/ + +Please note we assume the `OpenStack neutron`_ use the **messagingv2** driver_. If you don't know much about the driver what your `OpenStack neutron`_ uses, Please ask your OpenStack system administrator or investigate your /etc/neutron/neutron.conf. Here is my neutron.conf setting. + +.. code-block:: ini + + [oslo_messaging_notifications] + # + # From oslo.messaging + # + # The Drivers(s) to handle sending notifications. Possible values are + # messaging, messagingv2, routing, log, test, noop (multi valued) + # Deprecated group/name - [DEFAULT]/notification_driver + driver = messagingv2 + +.. _`OpenStack neutron`: https://docs.openstack.org/neutron/latest/ +.. _driver: https://docs.openstack.org/neutron/latest/configuration/neutron.html#oslo-messaging-notifications + +What I have explained in this chapter: + +* k2hr3_osnl only listen to the message matches with defined in the configuration. +* Regular expression in filters is allowed. + + +Parsing versioned messages from OpenStack nova +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this chapter I will explain how to configure the k2hr3_osnl to parse messages from `OpenStack nova`_. When we tested the k2hr3_osnl with `OpenStack neutron`_ with the driver_ configuration was **not** messagingv2, the k2hr3_osnl could not get any notification messages we expected. If you met with same situation, please try the configuration in this chapter. + +.. _`OpenStack nova`: https://docs.openstack.org/nova/latest/ +.. _`OpenStack neutron`: https://docs.openstack.org/neutron/latest/ + +We assume the following message from `OpenStack nova`_. + +.. _`OpenStack nova`: https://docs.openstack.org/nova/latest/ + +.. code-block:: javascript + + { + "event_type" : "instance.delete.end", + "payload": { + "nova_object.data": { + "action_initiator_project": "project_string", + ... + "block_devices": [ + { + "nova_object.data": { + ... + "volume_id": "volumeid-ffff-ffff-ffff-ffffffffffff" + }, + "nova_object.name": "BlockDevicePayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.0" + } + ], + ... + "host": "node1.example.com", + ... + "uuid": "12345678-1234-5678-1234-567812345678" + }, + "priority": "INFO", + "publisher_id" : "nova-compute:node1.example.com", + } + +Here is a sample ``oslo_messaging_notifications`` group configuration k2hr3_osnl can read the message above. + +.. code-block:: ini + + [oslo_messaging_notifications] + event_type = ^instance\.delete\.end$ + publisher_id = ^nova-compute:.*$ + transport_url = rabbit://user:pass@127.0.0.1:5672/ + topic = versioned_notifications + exchange = nova + +You will recognize all items other than transport_url are different with neutron's case! Each OpenStack service defines its own event_type. FYI: `OpenStack nova`_ defines many event types. + +.. _`OpenStack nova`: https://docs.openstack.org/nova/latest/ + +https://docs.openstack.org/nova/latest/reference/notifications.html#existing-versioned-notifications + +What I have explained in this chapter: + +* `OpenStack nova`_ publishes different messages format from `OpenStack neutron`_. +* k2hr3_osnl can read messages from `OpenStack nova`_ too by changing the configuration. + +.. _`OpenStack nova`: https://docs.openstack.org/nova/latest/ +.. _`OpenStack neutron`: https://docs.openstack.org/neutron/latest/ + +List of configuration items +---------------------------- + +Settings in the configuration file define the k2hr3_osnl behavior except for loglevel. Loglevel augments override loglevel settings in configuration. If you want to change the loglevel only, you need not to change configuration file. use ``-d`` option. + +.. code-block:: console + + $ k2hr3_osnl --help + usage: k2hr3_osnl [-h] [-c CONFIG_FILE] [-d {debug,info,warn,error,critical}] + [-l {debug,info,warn,error,critical}] [-f LOG_FILE] [-v] + + An oslo.messaging notification listener for k2hr3. + + optional arguments: + -h, --help show this help message and exit + -c CONFIG_FILE, --config-file CONFIG_FILE + config file path + -d {debug,info,warn,error,critical} + debug level. default: defined in the config_file + -l {debug,info,warn,error,critical} + dependent libraries loglevel. default: defined in the + config_file + -f LOG_FILE log file path. default: defined in the config_file + -v show program's version number and exit + +The configuration file consists of 3 parts. + +* oslo_messaging_notifications + configurations for the oslo_messaging library. +* k2hr3 + configurations for the K2HR3 system. +* DEFAULT + the others. + +The configuration file syntax is the ".INI" format. Here is a default sample configuration. + +.. code-block:: ini + + [DEFAULT] + debug_level = error + log_file = sys.stderr + libs_debug_level = warning + + [oslo_messaging_notifications] + event_type = ^port\.delete\.end$ + publisher_id = ^network.*$ + transport_url = rabbit://user:pass@127.0.0.1:5672/ + topic = notifications + exchange = neutron + executor = threading + pool = k2hr3_osnl + allow_requeue = True + + [k2hr3] + api_url = https://localhost/v1/role + timeout_seconds = 30 + retries = 3 + retry_interval_seconds = 60 + allow_self_signed_cert = False + requeue_on_error = False + + +[DEFAULT] +~~~~~~~~~ + +debug_level + logging level. Each of debug, info, warning or error. (**default:** warning). + +log_file + destination of logging. (**default:** sys.stderr) + +libs_debug_level + log level. (**default:** warning) + + +[oslo_messaging_notifications] +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +event_type + event type of the notification message(**default:** ^port\.delete\.end$). + +publisher_id + publisher id of the notification message(**default:** ^network.*$) + +transport_url + transport url(**default:** rabbit://guest:guest@127.0.0.1:5672/) + +topic + topic of the notification message(**default:** notifications) + +exchange + exchange of the notification message(**default:** neutron) + +executor + executor of the listener(**default:** threading) + +pool +n pool identification of message queue(**default:** k2hr3_osnl) + +allow_requeue + allow requeue if error occurred(**default:** True) + +[k2hr3] +~~~~~~~~~~~~ + +api_url + K2HR3 WebAPI Url(**default:** https://localhost/v1/role). + +timeout_seconds + connection timeout in second(**default:** 30) + +retries + retries(**default:** 3) + +retry_interval_seconds + interval(**default:** 60) + +allow_self_signed_cert + certification(**default:** True) + +requeue_on_error + error(**default:** True) + + diff --git a/etc/k2hr3-osnl.conf b/etc/k2hr3-osnl.conf new file mode 100644 index 0000000..bbe9ccf --- /dev/null +++ b/etc/k2hr3-osnl.conf @@ -0,0 +1,47 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +[DEFAULT] +#log_file = sys.stderr +debug_level = error +#libs_debug_level = warning + +[oslo_messaging_notifications] +event_type = ^port\.delete\.end$ +publisher_id = ^network.*$ +transport_url = rabbit://guest:guest@127.0.0.1:5672/ +topic = notifications +exchange = neutron +#executor = threading +#pool = k2hr3_osnl +#allow_requeue = True + +[k2hr3] +api_url = https://localhost/v1/role +#timeout_seconds = 30 +#retries = 3 +#retry_interval_seconds = 60 +#allow_self_signed_cert = False +#requeue_on_error = False + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/etc/k2hr3-osnl.conf.compute.sample b/etc/k2hr3-osnl.conf.compute.sample new file mode 100644 index 0000000..fe2698e --- /dev/null +++ b/etc/k2hr3-osnl.conf.compute.sample @@ -0,0 +1,47 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +[DEFAULT] +#log_file = sys.stderr +debug_level = error +#libs_debug_level = warning + +[oslo_messaging_notifications] +event_type = ^compute\.instance\.delete\.end$ +publisher_id = ^compute.*$ +transport_url = rabbit://guest:guest@127.0.0.1:5672/ +topic = notifications +exchange = nova +#executor = threading +#pool = k2hr3_osnl +#allow_requeue = True + +[k2hr3] +api_url = https://localhost/v1/role +#timeout_seconds = 30 +#retries = 3 +#retry_interval_seconds = 60 +allow_self_signed_cert = True +#requeue_on_error = False + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/etc/k2hr3-osnl.conf.neutron.sample b/etc/k2hr3-osnl.conf.neutron.sample new file mode 100644 index 0000000..bbe9ccf --- /dev/null +++ b/etc/k2hr3-osnl.conf.neutron.sample @@ -0,0 +1,47 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +[DEFAULT] +#log_file = sys.stderr +debug_level = error +#libs_debug_level = warning + +[oslo_messaging_notifications] +event_type = ^port\.delete\.end$ +publisher_id = ^network.*$ +transport_url = rabbit://guest:guest@127.0.0.1:5672/ +topic = notifications +exchange = neutron +#executor = threading +#pool = k2hr3_osnl +#allow_requeue = True + +[k2hr3] +api_url = https://localhost/v1/role +#timeout_seconds = 30 +#retries = 3 +#retry_interval_seconds = 60 +#allow_self_signed_cert = False +#requeue_on_error = False + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/etc/k2hr3-osnl.conf.nova_compute.sample b/etc/k2hr3-osnl.conf.nova_compute.sample new file mode 100644 index 0000000..073ecff --- /dev/null +++ b/etc/k2hr3-osnl.conf.nova_compute.sample @@ -0,0 +1,47 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +[DEFAULT] +#log_file = sys.stderr +debug_level = error +#libs_debug_level = warning + +[oslo_messaging_notifications] +event_type = ^instance\.delete\.end$ +publisher_id = ^nova-compute:.*$ +transport_url = rabbit://guest:guest@127.0.0.1:5672/ +topic = versioned_notifications +exchange = nova +#executor = threading +#pool = k2hr3_osnl +#allow_requeue = True + +[k2hr3] +api_url = https://localhost/v1/role +#timeout_seconds = 30 +#retries = 3 +#retry_interval_seconds = 60 +allow_self_signed_cert = True +#requeue_on_error = False + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/etc/k2hr3_osnl.conf.sample b/etc/k2hr3_osnl.conf.sample new file mode 100644 index 0000000..0e86be3 --- /dev/null +++ b/etc/k2hr3_osnl.conf.sample @@ -0,0 +1,50 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +[DEFAULT] +debug = False +#log_file = sys.stdout +#log_dir = logs +#log_format = %(asctime)-15s %(levelname)s %(name)s %(message)s +#log_level = logging.INFO +#libraries_log_level = logging.WARNING + +[oslo_messaging_notifications] +event_type = ^port\.delete\.end$ +publisher_id = ^network.*$ +transport_url = rabbit://guest:guest@127.0.0.1:5672/ +topic = notifications +exchange = neutron +#executor = threading +#pool = k2hr3_osnl +#allow_requeue = True + +[k2hr3] +api_url = https://localhost/v1/role +#timeout_seconds = 30 +#retries = 3 +#retry_interval_seconds = 60 +#allow_self_signed_cert = False +#requeue_on_error = False + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/k2hr3-osnl.service b/k2hr3-osnl.service new file mode 100644 index 0000000..3f0e84f --- /dev/null +++ b/k2hr3-osnl.service @@ -0,0 +1,14 @@ +[Unit] +Description=K2HR3 OpenStack Notification Listener +After=network-online.target + +[Service] +Type=simple +User=k2hr3 +PermissionsStartOnly=true +ExecStart=/usr/bin/k2hr3-osnl -c /etc/k2hr3/k2hr3-osnl.conf +Restart=on-failure +PIDFile=/run/k2hr3-osnl.pid + +[Install] +WantedBy=multi-user.target diff --git a/k2hr3_osnl/__init__.py b/k2hr3_osnl/__init__.py new file mode 100644 index 0000000..b4903d6 --- /dev/null +++ b/k2hr3_osnl/__init__.py @@ -0,0 +1,265 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +"""K2hr3 OpenStack Notification message Listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +__all__ = [ + 'K2hr3Conf', + 'K2hr3ConfError', + 'K2hr3NotificationEndpoint', + 'K2hr3NotificationEndpointError', + 'listen', + 'main', + 'version', +] +__author__ = 'Hirotaka Wakabayashi ' +__version__ = '0.9.5' + +import argparse +import logging +from logging.handlers import TimedRotatingFileHandler +from logging import StreamHandler +from pathlib import Path +import sys +import time +from typing import List, Set, Dict, Tuple, Optional # noqa: pylint: disable=unused-import + +import oslo_config +import oslo_messaging + +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.exceptions import K2hr3Error, K2hr3ConfError, K2hr3NotificationEndpointError +from k2hr3_osnl.endpoint import K2hr3NotificationEndpoint + +LOG = logging.getLogger(__name__) + +if sys.platform.startswith('win'): + raise ImportError(r'Currently we do not test well on windows') + + +def version() -> str: + """Returns a version of k2hr3_osnl package. + + :returns: version + :rtype: str + """ + return __version__ + + +def main() -> int: + """Runs a oslo_messaging notification listener for k2hr3. + + You can configure the listener by the config file. + + Simple usage: + + $ k2hr3_osnl -c etc/k2hr3_osnl.config + + :returns: + 0 if success, otherwise 1. + :rtype: + int + """ + parser = argparse.ArgumentParser( + description='An oslo.messaging notification listener for k2hr3.') + parser.add_argument( + '-c', + '--config-file', + dest='config_file', + default='/etc/k2hr3/k2hr3_osnl.conf', + help='config file path') + parser.add_argument( + '-d', + dest='debug_level', + choices=('debug', 'info', 'warn', 'error', 'critical'), + help='debug level. default: defined in the config_file') + parser.add_argument( + '-l', + dest='libs_debug_level', + choices=('debug', 'info', 'warn', 'error', 'critical'), + help='dependent libraries loglevel. default: defined in the config_file' + ) + parser.add_argument( + '-f', + dest='log_file', + help='log file path. default: defined in the config_file') + parser.add_argument( + '-v', action='version', version='%(prog)s ' + __version__) + args = parser.parse_args() + + try: + conf = K2hr3Conf(Path(args.config_file)) + _configure_logger(args, conf) # logger configured by args and conf. + endpoints = [K2hr3NotificationEndpoint(conf)] + sys.exit(listen(endpoints)) + except K2hr3Error as error: + LOG.error('K2hr3Error error, %s', error) + raise K2hr3Error("K2hr3 RuntimeError") from error + except Exception as error: + LOG.error('Unknown error, %s', error) + raise RuntimeError("Unknown RuntimeError") from error + + +_nametolevel = { + 'error': logging.ERROR, + 'warn': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG, + 'notset': logging.NOTSET +} + + +def _configure_logger(args, conf) -> bool: + """Configures logger settings by args and conf. + + :param args: command line args + :type argparse: command line args + :param conf: configuration + :type K2hr3Conf: configuration + :returns: True if success, otherwise False + :rtype: bool + """ + # We prefer args than configuration file. + # 1. debug_level + debug_level = logging.WARNING + if args.debug_level is not None: + debug_level = _nametolevel.get(args.debug_level, logging.WARNING) + else: + debug_level = _nametolevel.get(conf.debug_level, logging.WARNING) + LOG.setLevel(debug_level) + + # 2. formatter + formatter = logging.Formatter( + '%(asctime)-15s %(levelname)s %(name)s:%(lineno)d %(message)s') # hardcoding + + # 3. log_file + if args.log_file is not None: + # check the permission of the destination file. + # if unable to open it, use default(stderr). + + # Add the log message handler to the logger + handler = TimedRotatingFileHandler( + args.log_file, when='midnight', encoding='UTF-8', backupCount=31) + handler.setFormatter(formatter) + LOG.addHandler(handler) + else: + if conf.log_file == 'sys.stderr': + stream_handler = StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + LOG.addHandler(stream_handler) + else: + # Add the log message handler to the logger + handler = TimedRotatingFileHandler( + conf.log_file, + when='midnight', + encoding='UTF-8', + backupCount=31) + handler.setFormatter(formatter) + LOG.addHandler(handler) + + # 3. libs_debug_level + libs_debug_level = logging.WARNING + if args.libs_debug_level is not None: + libs_debug_level = _nametolevel.get(args.libs_debug_level, + logging.WARNING) + else: + libs_debug_level = _nametolevel.get(conf.libs_debug_level, + logging.WARNING) + libs = [ + 'stevedore.extension', 'oslo.messaging._drivers.pool', + 'oslo.messaging._drivers.impl_rabbit', 'amqp' + ] + for i in libs: + logging.getLogger(i).setLevel(libs_debug_level) + + return True + + +def listen(endpoints: List[K2hr3NotificationEndpoint]) -> int: + """Runs a oslo_messaging notification listener for k2hr3. + + This function is a library endpoint to start a oslo_messaging notification + listener for k2hr3. + + :param endpoints: endpoint to be called by dispatcher when notification messages arrive. + :type endpoints: list of K2hr3NotificationEndpoint + :returns: 0 if success, otherwise 1. + :rtype: int + """ + # 1. validate endpoints + if not isinstance(endpoints, list) or len(endpoints) == 0: + LOG.error('invalid endpoints, %s', endpoints) + return 1 + + # 2. validate each endpoint + for endpoint in endpoints: + if not isinstance(endpoint, K2hr3NotificationEndpoint): + LOG.error('found an invalid endpoint, %s', endpoint) + return 1 + if not isinstance(endpoint.conf, K2hr3Conf): # this never happens. + LOG.error('found an invalid conf in an endpoint, %s', + endpoint.conf) + return 1 + + conf = endpoint.conf + assert isinstance(conf, K2hr3Conf) + + try: + # transport, targets + transport = oslo_messaging.get_notification_transport( + oslo_config.cfg.CONF, + url=conf.oslo_messaging_notifications.transport_url) + targets = [ + oslo_messaging.Target( + topic=conf.oslo_messaging_notifications.topic, + exchange=conf.oslo_messaging_notifications.exchange) + ] + listener = oslo_messaging.get_notification_listener( + transport, + targets, + endpoints, + pool=conf.oslo_messaging_notifications.pool, + executor=conf.oslo_messaging_notifications.executor, + allow_requeue=conf.oslo_messaging_notifications.allow_requeue) + listener.start() + LOG.info('Starting') + while True: + time.sleep(1) + except KeyboardInterrupt: + LOG.info('Stopping') + listener.stop() + listener.wait() + except NotImplementedError: + LOG.error('allow_requeue is not supported by driver') + return 1 + except oslo_messaging.ServerListenError as error: + LOG.error('listener error, %s', error.msg) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) + +# +# EOF +# diff --git a/k2hr3_osnl/cfg.py b/k2hr3_osnl/cfg.py new file mode 100644 index 0000000..316fb40 --- /dev/null +++ b/k2hr3_osnl/cfg.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Parses a config file and stores configurations.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +from pathlib import Path +from typing import List, Set, Dict, Tuple, Optional # noqa: pylint: disable=unused-import + +from oslo_config import cfg +from k2hr3_osnl.exceptions import K2hr3ConfError + +LOG = logging.getLogger(__name__) + + +class K2hr3Conf(cfg.ConfigOpts): # public class instantiated in __main__ + r"""Parses and stores configurations. + + This class is a wrapper of oslo_config.cfg.ConfigOpts class. + https://github.com/openstack/oslo.config/blob/master/oslo_config/cfg.py + + Simple usage: + + >>> from k2hr3_osnl.exceptions import K2hr3ConfError + >>> from k2hr3_osnl.cfg import K2hr3Conf + >>> from pathlib import Path + ... try: + ... conf = K2hr3Conf(Path('etc/k2hr3_osnl.conf')) + ... print(conf.oslo_messaging_notifications.event_type) + ... except K2hr3ConfError as error: + ... print('{}'.format(error)) + ... + ^port\.delete\.end$ + """ + + def __init__(self, path: Path) -> None: + """Initializes a K2hr3Conf object. + + :param path: configuration file path + :type path: Path + :raises K2hr3ConfError: if invalid augment or file parse error. + """ + if isinstance(path, Path) is False: + raise K2hr3ConfError('Path expected, not {}'.format( + type(path).__name__)) + if path.exists() is False: + raise K2hr3ConfError('path must exist, not {}'.format(path)) + if path.is_file() is False: + raise K2hr3ConfError( + 'path must be a regular file, not {}'.format(path)) + try: + with path.open(): # try reading + LOG.debug('successfully opened.') + except OSError as error: + raise K2hr3ConfError( + 'path must be a readable regular file, not {}'.format(error)) + + self._path = path # The path is valid and a Path object is immutable. + + try: + super(K2hr3Conf, self).__init__() # calls the oslo_config init. + except Exception as error: + raise K2hr3ConfError('initialization error') from error + + try: + self._parse_config() # can raise K2hr3ConfError if errors occur. + LOG.debug('cfg initialized, %s.', str(path)) + except K2hr3ConfError as error: + raise error + + def _parse_config(self) -> bool: + """Parses a configration file. + + A protected method called in the __init__(). + You can implement your own _parse_config in your own class which is + derived from K2hr3Conf class if you have own your configuration. + + We handle only exceptions we know. + + :returns: True if success. Otherwise an exception raises. + :raises K2hr3ConfError: if errors occur. + """ + assert isinstance(self._path, Path) + + oslo = cfg.OptGroup( + name='oslo_messaging_notifications', title='OsloGroupSettings') + self.register_group(oslo) + oslo_opts = [ + cfg.StrOpt( + 'event_type', + default=r'^port\.delete\.end$', + help='event_type'), + cfg.StrOpt( + 'publisher_id', default='^network.*$', help='publisher_id'), + cfg.DictOpt('context', default=None, help='context'), + cfg.DictOpt('metadata', default=None, help='metadata'), + cfg.DictOpt('payload', default=None, help='payload'), + cfg.StrOpt( + 'transport_url', + default='rabbit://guest:guest@127.0.0.1:5672/', + help='transport_url'), + cfg.StrOpt('topic', default='notifications', help='topic'), + cfg.StrOpt('exchange', default='neutron', help='exchange'), + cfg.StrOpt('executor', default='threading', help='executor'), + cfg.StrOpt('pool', default='k2hr3_osnl', help='pool'), + cfg.BoolOpt( + 'allow_requeue', + default=True, + help='requeue if listener fails to process a msg properly') + ] + self.register_opts(oslo_opts, group=oslo) + + k2hr3 = cfg.OptGroup(name='k2hr3', title='K2hr3GroupSettings') + self.register_group(k2hr3) + k2hr3_opts = [ + cfg.StrOpt( + 'api_url', + default='https//localhost/v1/role', + help='k2hr3 api Url'), + cfg.IntOpt( + 'timeout_seconds', + default=30, + help='connection and timeout in second'), + cfg.IntOpt('max_retries', default=5, help='max retry count'), + cfg.IntOpt( + 'retry_interval_seconds', + default=60, + help='interval seconds to wait until next retry'), + cfg.BoolOpt( + 'allow_self_signed_cert', + default=False, + help='allow self-signed certificate'), + cfg.BoolOpt( + 'requeue_on_error', + default=False, + help='requeue messages or not in case of errors in listener') + ] + self.register_opts(k2hr3_opts, group=k2hr3) + + self.register_opt( + cfg.StrOpt('log_file', default='sys.stderr', help='log file')) + self.register_opt( + cfg.StrOpt( + 'debug_level', + default='info', + choices=('debug', 'info', 'warn', 'error', 'notset'), + help='debug level')) + self.register_opt( + cfg.StrOpt( + 'libs_debug_level', + default='warn', + choices=('debug', 'info', 'warn', 'error'), + help='log level of dependent libs')) + + try: + # ConfigFileAction returns nothing. + # https://github.com/openstack/oslo.config/blob/master/oslo_config/cfg.py#L1311 + self(['--config-file', str(self._path)]) + LOG.debug('%s successfully parsed', str(self._path)) + except (cfg.ConfigFileParseError, cfg.ConfigFileValueError) as error: + raise K2hr3ConfError('parse error, {}'.format(error)) from error + + return True + +# +# EOF +# diff --git a/k2hr3_osnl/endpoint.py b/k2hr3_osnl/endpoint.py new file mode 100644 index 0000000..7027d41 --- /dev/null +++ b/k2hr3_osnl/endpoint.py @@ -0,0 +1,311 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""An endpoint for the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import json +import logging +import sys +import traceback +from typing import List, Set, Dict, Tuple, Optional, Any # noqa: pylint: disable=unused-import + +from oslo_messaging import NotificationFilter, NotificationResult + +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.useragent import _K2hr3UserAgent +from k2hr3_osnl.exceptions import K2hr3NotificationEndpointError, _K2hr3UserAgentError + +LOG = logging.getLogger(__name__) + + +class K2hr3NotificationEndpoint: # public class instantiated in main + """An endpoint called by a OpenStack dispatcher when a filtered notification message arrives. + + Simple usage: + + >>> from k2hr3_osnl.cfg import K2hr3Conf + >>> from k2hr3_osnl.exceptions import K2hr3Error + >>> from k2hr3_osnl.endpoint import K2hr3NotificationEndpoint + >>> import k2hr3_osnl + >>> from pathlib import Path + >>> try: + ... conf = K2hr3Conf(Path('etc/k2hr3_osnl.conf')) + ... endpoints = [K2hr3NotificationEndpoint(conf)] + ... k2hr3_osnl.listen(endpoints) + ... except K2hr3Error as error: + ... print(error) + """ + + def __init__(self, conf: K2hr3Conf) -> None: # public called in __main__ + """Initializes attributes. + + We instantiate the NotificationFilter instance as the 'filter_rule' + attribute here which is used to filter notifications that an + endpoint will received. + + Note: + The 'filter_rule' is a special attribute which is referred by + the oslo_messaging notify dispatcher. Don't change the name. + https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/notify/dispatcher.py#L48 + + :param conf: K2hr3Conf object + :type conf: K2hr3Conf + :raises K2hr3NotificationEndpointError: if invalid augment. + """ + if isinstance(conf, K2hr3Conf) is False: + raise K2hr3NotificationEndpointError( + 'conf is a K2hr3Conf instance, not {}'.format(type(conf))) + + context = conf.oslo_messaging_notifications.context + metadata = conf.oslo_messaging_notifications.metadata + payload = conf.oslo_messaging_notifications.payload + publisher_id = conf.oslo_messaging_notifications.publisher_id + event_type = conf.oslo_messaging_notifications.event_type + + # publisher_id and event_type are must + assert [ + isinstance(publisher_id, str), + isinstance(event_type, str), + ] + + self.filter_rule = NotificationFilter( + context=context, + # publiser_id + # ex) compute.hostname.domain_name + # ex) nova-compute:hostname.domain_name + # ex) network.hostname.domain_name + publisher_id=publisher_id, + # event_type + # ex) port.delete.end + # ex) instance.delete.end + # ex) compute.instance.delete.end + event_type=event_type, + metadata=metadata, + # payload contains the virtual machine instance id and the ips. + # ex) payload by neutron's port.delete.end event. + # "port": { + # ... + # "device_id": "deviceid-ffff-ffff-ffff-ffffffffffff", + # "fixed_ips": [ + # { + # "ip_address": "172.16.0.1", + # ... + payload=payload) + self._conf = conf + LOG.debug('endpoint initialized') + + @property + def conf(self) -> K2hr3Conf: + """Returns the K2hr3Conf object.""" + return self._conf + + def _payload_to_params(self, payload: Any) -> Dict[str, object]: # pylint: disable=no-self-use + """Parses a payload data. + + _payload_to_params is a protected method called in the info(). + You can implement your own _payload_to_params in your own class which is derived from + K2hr3NotificationEndpoint class if you want to parse your OpenStack notificatio messages + are different from us. + + :param payload: payload is a dict object. the format is not simple. + :type payload: dict + :returns: result of parsing payload if data found in the payload. + Note: + params['cuk'] is expected to be a str object. + params['ips'] is expedted to be a list object. + :rtype: dict + :raises K2hr3NotificationEndpointError: if the payload does not contain enough data. + """ + assert [ + isinstance(payload, dict), + isinstance(self._conf, K2hr3Conf), + ] + + params = {} # allocate new buffer. + + # 1. try parsing an expected neutron message. + if payload.get('port', None): + LOG.debug('expected neutron(port)') + if payload['port'].get('device_id', None): + LOG.debug('expected neutron(device_id)') + params['cuk'] = payload['port']['device_id'] + else: + LOG.warning('expected neutron but no device_id.') + if payload['port'].get('fixed_ips', None): + LOG.debug('expected neutron(fixed_ips)') + ips = [] # type: List[str] + ips.extend(v['ip_address'] + for v in payload['port']['fixed_ips'] + if v.get('ip_address', None)) + if ips: + params['ips'] = ips + else: + LOG.warning('expected neutron but ips is empty.') + else: + LOG.warning('expected neutron but no fixed_ips.') + + # 2. try parsing an expected compute message. + if payload.get('nova_object.data', None): + LOG.debug('expceted compute(nova_object.data)') + if payload['nova_object.data'].get('uuid', None): + LOG.debug('expected compute(uuid)') + params['cuk'] = payload['nova_object.data']['uuid'] + else: + LOG.warning('expected compute but uuid is empty') + + # 3. try parsing an expected nova message. + if payload.get('instance_id', None): + LOG.debug('expceted nova(instance_id)') + params['cuk'] = payload['instance_id'] + + # 4. finall check the params. + if params.get('ips', None) is None: + LOG.warning('ips is empty') + + if params.get('cuk', None) is None: + LOG.error('cuk is empty') + raise K2hr3NotificationEndpointError( + 'no cuk in params, {}'.format(params)) + + LOG.debug(json.dumps(params, indent=4, sort_keys=True)) + return params + + def __call_r3api(self, params: Dict[str, Any]) -> str: + """Calls the r3api. + + :returns: NotificationResult.REQUEUE if failed to call the r3api. + Otherwise NotificationResult.HANDLED. + :rtype: str + """ + assert [ + isinstance(params, dict), + isinstance(self._conf, K2hr3Conf), + ] + + try: + agent = _K2hr3UserAgent(self._conf) + agent.instance_id = params.get('cuk', None) + if params.get('ips', None): + agent.ips = params.get('ips', None) + if agent.send(): + LOG.debug('ok sent. %s code, %s', agent.instance_id, agent.code) + return NotificationResult.HANDLED # type: ignore + LOG.error('no sent. %s error %s', agent.instance_id, agent.error) + if self._conf.k2hr3.requeue_on_error is True: + LOG.warning('requeuing %s', agent.instance_id) + return NotificationResult.REQUEUE # type: ignore + LOG.warning('handled %s, even if an error occurred.', agent.instance_id) + return NotificationResult.HANDLED # type: ignore + except _K2hr3UserAgentError as error: + LOG.error('k2hr3 exception %s', error) + if self._conf.k2hr3.requeue_on_error is True: + LOG.warning('requeuing the msg') + return NotificationResult.REQUEUE # type: ignore + LOG.warning('handled the msg even if an error occurred.') + return NotificationResult.HANDLED # type: ignore + except Exception as error: + # Note: + # unknown exception should be handled by upstream caller. + LOG.error('unknown exception. upstream caller catch this %s', error) + raise + + def info( + self, + context: Dict[str, object], + publisher_id: str, # pylint: disable=too-many-arguments + event_type: str, + payload: Dict[str, object], + metadata: Dict[str, object]): + """Notification endpoint in info priority. + + Notification messages that match the filter’s rules will be passed + to the endpoint’s methods. The oslo_messaging's callback function + dispatcher calls when messages in 'info' priority have arrived. + + Reference: + + - https://docs.openstack.org/oslo.messaging/latest/reference/notification_listener.html + - https://github.com/openstack/oslo.messaging/blob/master/oslo_messaging/notify/dispatcher.py#L74 + + Note: + This function catches all exceptions to avoid an infinite loop. + If this function hasn't handled unexpected exceptions, the caller(dispatcher) + would have caught them and returned the NotificationResult.REQUEUE to + the message queue server which can cause infinite loop. To avoid the + posibility of inifinite loop, we catches standard exception in this function. + + :param context: Context of a notification for NotificationFilter. + :type context: dict + :param publisher_id: Publisher_id of a notification for NotificationFilter + :type publisher_id: str + :param event_type: Event_type of a notification for NotificationFilter + :type event_type: str + :param payload: Payload of a notification for NotificationFilter. + :type payload: dict + :param metadata: Metadata of a notification for NotificationFilter. + :type metadata: dict + :returns: NotificationResult.HANDLED or NotificationResult.REQUEUE + """ + assert [ + isinstance(payload, dict), # We are interested in payload only. + ] + + try: + LOG.debug('publisher_id %s event_type %s payload %s', + publisher_id, event_type, + json.dumps(payload, indent=4, sort_keys=True)) + params = self._payload_to_params(payload) + except K2hr3NotificationEndpointError as error: + # K2hr3NotificationEndpointError is a hard error. + # We don't raise an exception again since we should avoid infinite message parsing loop. + LOG.error('invalid payload %s', error) + return NotificationResult.HANDLED + except Exception: # pylint: disable=broad-except + # Unknown exception should be treat as a hard error. + # We don't raise an exception again since we should avoid infinite message parsing loop. + exc_type, exc_value, exc_traceback = sys.exc_info() + # Too much? https://docs.python.org/3/library/traceback.html + LOG.error('exec_type %s exec_value %s traceback %s', exc_type, + exc_value, repr(traceback.extract_tb(exc_traceback))) + return NotificationResult.HANDLED + + try: + # We calls the r3api. + if self.__call_r3api(params) == NotificationResult.HANDLED: + LOG.info('NotificationResult.HANDLED %s', params.get('cuk')) + return NotificationResult.HANDLED + LOG.info('NotificationResult.REQUEUE %s', params.get('cuk')) + return NotificationResult.REQUEUE + except Exception: # pylint: disable=broad-except + # we should handle exceptions to exit from here properly. + exc_type, exc_value, exc_traceback = sys.exc_info() + # Too much? https://docs.python.org/3/library/traceback.html + LOG.error('exec_type %s exec_value %s traceback %s', exc_type, + exc_value, repr(traceback.extract_tb(exc_traceback))) + # return HANDLED for avoiding infinite loop. + LOG.error( + 'got an exception in r3api. handled the msg even if an error occurred.' + ) + return NotificationResult.HANDLED + +# +# EOF +# diff --git a/k2hr3_osnl/exceptions.py b/k2hr3_osnl/exceptions.py new file mode 100644 index 0000000..3fb06be --- /dev/null +++ b/k2hr3_osnl/exceptions.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Exception classes for the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +from typing import List, Set, Dict, Tuple, Optional # noqa: pylint: disable=unused-import + + +class K2hr3Error(Exception): + """A base class of various exceptions from k2hr3_osnl package classes.""" + + pass + + +class K2hr3ConfError(K2hr3Error): + """Raised when failed to instantiate a k2hr3Conf class.""" + + def __init__(self, msg: str = None): + """Initializes members.""" + self.msg = msg + + +class K2hr3NotificationEndpointError(K2hr3Error): + """Raised when failed to instantiate a K2hr3NotificationEndpoint class.""" + + def __init__(self, msg: str = None): + """Initializes members.""" + self.msg = msg + + +class _K2hr3UserAgentError(K2hr3Error): + """Raised when failed to send request to K2hr3API in K2hr3Agent class.""" + + def __init__(self, msg: str = None): + """Initializes members.""" + self.msg = msg + +# +# EOF +# diff --git a/k2hr3_osnl/httpresponse.py b/k2hr3_osnl/httpresponse.py new file mode 100644 index 0000000..5cd44b0 --- /dev/null +++ b/k2hr3_osnl/httpresponse.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Sends http requests to the k2hr3 api. Classes in this module are not public.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from typing import List, Set, Dict, Tuple, Optional, Union # noqa: pylint: disable=unused-import + +from k2hr3_osnl.exceptions import _K2hr3UserAgentError + +LOG = logging.getLogger(__name__) + + +class _K2hr3HttpResponse: + def __init__(self): + self._code = -1 # http status code from api server + self._error = '' + + @property + def code(self) -> int: # public. + """Returns the HTTP status code. + + :returns: HTTP status code + :rtype: int + """ + return self._code + + @code.setter + def code(self, value: int) -> None: # public. + """Sets the HTTP status code. + + :param value: HTTP status code + :type value: int + """ + # Input validation in public method should be done first. + if isinstance(value, int) is True: + self._code = value + else: + raise _K2hr3UserAgentError('code should be int, not {}'.format( + type(value))) + + @property + def error(self) -> str: # public. + """Returns the HTTP error. + + :returns: HTTP error + :rtype: str + """ + return self._error + + @error.setter + def error(self, value: str) -> None: # public. + """Sets the HTTP error. + + :param value: HTTP error + :type value: str + """ + # Input validation in public method should be done first. + if isinstance(value, str) is True: + self._error = value + else: + raise _K2hr3UserAgentError( + 'error should be str, not {}'.format(value)) + + def __repr__(self): + attrs = [] + for attr in ['_error', '_code']: # should be hardcoded. + val = getattr(self, attr) + if val: + attrs.append((attr, repr(val))) + else: + attrs.append((attr, '')) + values = ', '.join(['%s=%s' % i for i in attrs]) + return '<_K2hr3HttpResponse ' + values + '>' + + def __str__(self): + attrs = {} + for attr in ['_error', '_code']: # should be hardcoded. + val = getattr(self, attr) + if val: + attrs[attr] = str(val) + else: + attrs[attr] = "" + values = '' + for key, value in attrs.items(): + values += '{}={} '.format(key, value) + return '<_K2hr3HttpResponse ' + values + '>' + +# +# EOF +# diff --git a/k2hr3_osnl/tests/test.py b/k2hr3_osnl/tests/test.py new file mode 100644 index 0000000..beaa603 --- /dev/null +++ b/k2hr3_osnl/tests/test.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Endpoint class for test the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +__author__ = 'Hirotaka Wakabayashi ' +__copyright__ = """ +Copyright (c) 2018 Yahoo Japan Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import logging +import os +import sys +import unittest + +import xmlrunner + +here = os.path.dirname(__file__) +src_dir = os.path.join(here, '../..') +if os.path.exists(src_dir): + sys.path.append(src_dir) + +loader = unittest.defaultTestLoader + + +def suite(): + """Loads each test case one by one instead of finding all the test modules by discovery. + + Simple Usage: + + 1. run all tests. + $ python3 -m k2hr3_osnl.tests + + 2. run each test. + $ python -m unittest k2hr3_osnl/tests/test_cfg.py + """ + suite = unittest.TestSuite() + for fn in os.listdir(here): + if fn.startswith("test") and fn.endswith(".py"): + modname = "k2hr3_osnl.tests." + fn[:-3] + print(modname) + __import__(modname) + module = sys.modules[modname] + suite.addTest(loader.loadTestsFromModule(module)) + suite.addTest(loader.loadTestsFromName('k2hr3_osnl.tests')) + return suite + + +def main(): + """Starts all the test modules by discovery. + + We sometimes want to debug while doing unittests. + If you need to debug test cases, set K2HR3_LOG_LEVEL to debug. + + Simple Usage: + $ export K2HR3_LOG_LEVEL=debug + $ python3 -m k2hr3_osnl.tests + """ + # 1. getenv + if os.getenv('K2HR3_LOG_LEVEL') == 'debug': + priority = logging.DEBUG + print('debug mode is on.') + else: + priority = logging.INFO + + # 2. setup logger + logging.basicConfig( + stream=sys.stdout, + level=priority, + format="%(asctime)-15s %(levelname)s %(name)s %(message)s") + + # 3. run unittest + unittest.main( + defaultTest="suite", + testRunner=xmlrunner.XMLTestRunner( + output='reports', outsuffix='unittest'), + failfast=False, + buffer=False, + catchbreak=False) + + +if __name__ == "__main__": + main() + +# +# EOF +# diff --git a/k2hr3_osnl/tests/test_cfg.py b/k2hr3_osnl/tests/test_cfg.py new file mode 100644 index 0000000..24925c7 --- /dev/null +++ b/k2hr3_osnl/tests/test_cfg.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Endpoint class for the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +__author__ = 'Hirotaka Wakabayashi ' +__copyright__ = """ +Copyright (c) 2018 Yahoo Japan Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import logging +import os +import unittest +from unittest.mock import patch + +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.exceptions import K2hr3ConfError + +test_path = os.path.abspath(os.path.dirname(__file__)) +src_dir = os.path.join(test_path, '../..') +LOG = logging.getLogger(__name__) + + +class TestK2hr3Conf(unittest.TestCase): + """Tests the K2hr3Conf class.""" + + def test_k2hr3_conf_construct(self): + """Creates a K2hr3Conf instance.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertIsInstance(conf, K2hr3Conf) + + def test_k2hr3_conf_construct_path_is_none(self): + """Checks if the __init__'s path is None.""" + with self.assertRaises(K2hr3ConfError) as cm: + K2hr3Conf(None) + + the_exception = cm.exception + self.assertEqual(the_exception.msg, 'str expected, not NoneType') + + def test_k2hr3_conf_construct_path_is_dir(self): + """Checks if the __init__'s path is a directory.""" + path = '/tmp' + with self.assertRaises(K2hr3ConfError) as cm: + K2hr3Conf(path) + + the_exception = cm.exception + self.assertEqual( + the_exception.msg, + 'path must be an existing regular file, not {}'.format(path)) + + def test_k2hr3_conf_construct_path_is_not_readable(self): + """Checks if the __init__'s path is not readable.""" + path = '/etc/sudoers' + with self.assertRaises(K2hr3ConfError) as cm: + K2hr3Conf(path) + + the_exception = cm.exception + self.assertEqual( + the_exception.msg, + 'path must be a readable regular file, not [Errno 13] Permission denied: \'{}\'' + .format(path)) + + def test_k2hr3_conf_construct_parse_error(self): + """Checks if the __init__'s path is not a config file.""" + path = os.path.abspath(__file__) + with self.assertRaises(K2hr3ConfError) as cm: + K2hr3Conf(path) + + the_exception = cm.exception + self.assertRegex(the_exception.msg, + '^parse error, Failed to parse {}.*'.format(path)) + + def test_k2hr3_conf_assert_parse_config_called(self): + """Asserts _parse_config method called from constructor.""" + path = src_dir + '/etc/k2hr3_osnl.conf.sample' + with patch.object( + K2hr3Conf, '_parse_config', return_value=True) as mock_method: + K2hr3Conf(path) + + mock_method.assert_called_once_with() + + def test_k2hr3_conf_oslo_messaging_notifications_event_type(self): + """Asserts event_type in oslo_messaging_notifications group. + + The publisher_id and the event_type are very important filter for this system. + """ + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(r'^port\.delete\.end$', + conf.oslo_messaging_notifications.event_type) + + def test_k2hr3_conf_oslo_messaging_notifications_publisher_id(self): + """Asserts publisher_id in oslo_messaging_notifications group. + + The publisher_id and the event_type are very important filter for this system. + """ + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(r'^network.*$', + conf.oslo_messaging_notifications.publisher_id) + + def test_k2hr3_conf_oslo_messaging_notifications_context(self): + """Asserts context in oslo_messaging_notifications group. + + The context is an optional NotificationFilter. + """ + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(None, conf.oslo_messaging_notifications.context) + + def test_k2hr3_conf_oslo_messaging_notifications_metadata(self): + """Asserts metadata in oslo_messaging_notifications group. + + The metadata is an optional NotificationFilter. + """ + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(None, conf.oslo_messaging_notifications.metadata) + + def test_k2hr3_conf_oslo_messaging_notifications_payload(self): + """Asserts payload in oslo_messaging_notifications group. + + The payload is an optional NotificationFilter. + """ + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(None, conf.oslo_messaging_notifications.payload) + + def test_k2hr3_conf_oslo_messaging_notifications_transport_url(self): + """Asserts transport_url in oslo_messaging_notifications group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('rabbit://guest:guest@127.0.0.1:5672/', + conf.oslo_messaging_notifications.transport_url) + + def test_k2hr3_conf_oslo_messaging_notifications_topic(self): + """Asserts topic in oslo_messaging_notifications group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('notifications', + conf.oslo_messaging_notifications.topic) + + def test_k2hr3_conf_oslo_messaging_notifications_exchange(self): + """Asserts exchange in oslo_messaging_notifications group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('neutron', conf.oslo_messaging_notifications.exchange) + + def test_k2hr3_conf_oslo_messaging_notifications_pool(self): + """Asserts pool in oslo_messaging_notifications group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('k2hr3_osnl', conf.oslo_messaging_notifications.pool) + + def test_k2hr3_conf_oslo_messaging_notifications_allow_requeue(self): + """Asserts allow_requeue in oslo_messaging_notifications group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(True, conf.oslo_messaging_notifications.allow_requeue) + + def test_k2hr3_conf_k2hr3_api_url(self): + """Asserts api_url in k2hr3 group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('https://localhost/v1/role', conf.k2hr3.api_url) + + def test_k2hr3_conf_k2hr3_timeout_seconds(self): + """Asserts timeout_second in k2hr3 group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(30, conf.k2hr3.timeout_seconds) + + def test_k2hr3_conf_k2hr3_max_retries(self): + """Asserts retries in k2hr3 group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(5, conf.k2hr3.max_retries) + + def test_k2hr3_conf_k2hr3_retry_interval_seconds(self): + """Asserts retry_interval_seconds in k2hr3 group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(60, conf.k2hr3.retry_interval_seconds) + + def test_k2hr3_conf_k2hr3_allow_self_signed_cert(self): + """Asserts allow_self_signed_cert in k2hr3 group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(False, conf.k2hr3.allow_self_signed_cert) + + def test_k2hr3_conf_k2hr3_requeue_on_error(self): + """Asserts requeue_on_error in k2hr3 group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(False, conf.k2hr3.requeue_on_error) + + def test_k2hr3_conf_default_debug(self): + """Asserts debug in default group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual(False, conf.debug) + + def test_k2hr3_conf_default_log_file(self): + """Asserts log_file in default group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('k2hr3_osnl.log', conf.log_file) + + def test_k2hr3_conf_default_log_dir(self): + """Asserts log_dir in default group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('/tmp', conf.log_dir) + + def test_k2hr3_conf_default_log_format(self): + """Asserts log_format in default group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('%(asctime)-15s %(levelname)s %(name)s %(message)s', + conf.log_format) + + def test_k2hr3_conf_default_log_level(self): + """Asserts log_level in default group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('logging.INFO', conf.log_level) + + def test_k2hr3_conf_default_libraries_log_level(self): + """Asserts libraries_log_level in default group.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + self.assertEqual('logging.WARNING', conf.libraries_log_level) + +# +# EOF +# diff --git a/k2hr3_osnl/tests/test_endpoint.py b/k2hr3_osnl/tests/test_endpoint.py new file mode 100644 index 0000000..0f0b3af --- /dev/null +++ b/k2hr3_osnl/tests/test_endpoint.py @@ -0,0 +1,390 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Endpoint class for the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +__author__ = 'Hirotaka Wakabayashi ' +__copyright__ = """ +Copyright (c) 2018 Yahoo Japan Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json +import logging +import os +import unittest +from unittest.mock import MagicMock, patch + +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.endpoint import K2hr3NotificationEndpoint +from k2hr3_osnl.exceptions import K2hr3NotificationEndpointError, _K2hr3UserAgentError +from k2hr3_osnl.useragent import _K2hr3UserAgent + +test_path = os.path.abspath(os.path.dirname(__file__)) +src_dir = os.path.join(test_path, '../..') +LOG = logging.getLogger(__name__) +HANDLED = 'handled' +REQUEUE = 'requeue' + + +class TestNotificationEndpoint(unittest.TestCase): + """Tests the NotificationEndpoint class.""" + + def setUp(self): + """Setup TestNotificationEndpoint.""" + self.patcher_call_r3api = patch.object( + K2hr3NotificationEndpoint, + '_K2hr3NotificationEndpoint__call_r3api', + result_value=HANDLED) + self.mock_method = self.patcher_call_r3api.start() + self.mock_method.return_value = HANDLED + self.addCleanup(self.patcher_call_r3api.stop) + + def test_notification_endpoint_construct(self): + """Creates a NotificationEndpoint instance.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + self.assertIsInstance(endpoint, K2hr3NotificationEndpoint) + + def test_notification_endpoint_construct_conf_is_str(self): + """Checks if the __init__'s conf is not K2hr3Conf object.""" + conf = 'hogehoge' + with self.assertRaises(K2hr3NotificationEndpointError) as cm: + K2hr3NotificationEndpoint(conf) + the_exception = cm.exception + self.assertEqual( + the_exception.msg, 'conf is a K2hr3Conf instance, not {}'.format( + type(conf))) + + def test_notification_endpoint_conf(self): + """Checks if conf is readable.""" + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + self.assertIsInstance(endpoint, K2hr3NotificationEndpoint) + self.assertEqual(endpoint.conf, conf) + + def test_notification_endpoint_readonly(self): + """Checks if conf is readonly.""" + with self.assertRaises(AttributeError) as cm: + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + new_conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint.conf = new_conf + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_neutron_payload_to_params(self): + """Checks if _payload_to_params() works correctly. + + The payload pattern is a neutron notification message. + """ + # input --- payload + payload = { + "port": { + "device_id": + "12345678-1234-5678-1234-567812345678", + "fixed_ips": [{ + "ip_address": "127.0.0.1", + }, { + "ip_address": "127.0.0.2", + }] + } + } + # output --- params + expect_params = { + 'cuk': '12345678-1234-5678-1234-567812345678', + 'ips': ['127.0.0.1', '127.0.0.2'] + } + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + # Ensure values are as expected at runtime. + self.mock_method.assert_called_once_with(expect_params) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_nova_compute_payload_to_params(self): + """Checks if _payload_to_params() works correctly. + + The payload pattern is a nova compute notification message. + """ + # input --- payload + payload = { + "nova_object.data": { + "uuid": "12345678-1234-5678-1234-567812345678" + }, + "nova_object.version": "1.7" + } + # output --- params + expect_params = {'cuk': '12345678-1234-5678-1234-567812345678'} + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + # Ensure values are as expected at runtime. + self.mock_method.assert_called_once_with(expect_params) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_compute_payload_to_params(self): + """Checks if _payload_to_params() works correctly. + + The payload pattern is a compute notification message. + """ + # input --- payload + payload = { + "instance_id": "12345678-1234-5678-1234-567812345678", + } + # output --- params + expect_params = {'cuk': '12345678-1234-5678-1234-567812345678'} + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + # Ensure values are as expected at runtime. + self.mock_method.assert_called_once_with(expect_params) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_payload_to_params_error_no_instance_id(self): + """Checks if _payload_to_params() works correctly. + + The payload has no instance_id. + """ + # input --- payload + payload = { + "invalid_named": "12345678-1234-5678-1234-567812345678", + } + # __call_r3api is mocked! no http request will send. + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + + # Ensure the__call_r3api is not called. + self.mock_method.mock_call_r3api.assert_not_called() + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_call_r3api_requeue_on_exception(self): + """Checks if call_r3api works correctly. + + __call_r3api calls the _K2hr3UserAgent::send() to call the R3 API. + We mock _K2hr3UserAgent::send() to throws a _K2hr3UserAgentError. + If requeue_on_error is true, then the function returns HANDLED. + """ + # Expected return_value is REQUEUE in this case. + self.mock_method.return_value = REQUEUE + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + conf.k2hr3.requeue_on_error = True + _K2hr3UserAgent.send = MagicMock( + side_effect=_K2hr3UserAgentError('error')) + endpoint = K2hr3NotificationEndpoint(conf) + with open(src_dir + '/tools/data/notifications_neutron.json') as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, REQUEUE) + + # Reset it. Default return_value is HANDLED. + self.mock_method.return_value = HANDLED + + def test_notification_endpoint_call_r3api_exception_raise(self): + """Checks if call_r3api works correctly. + + __call_r3api calls the _K2hr3UserAgent::send() to call the R3 API. + We mock _K2hr3UserAgent::send() to throws an unknown exception. + Then the function raises the exception again. + K2hr3NotificationEndpoint::info() method will catch all exceptions + then it returns HANDLED to the dispatcher. + """ + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + _K2hr3UserAgent.send = MagicMock(side_effect=Exception('error')) + endpoint = K2hr3NotificationEndpoint(conf) + with open(src_dir + '/tools/data/notifications_neutron.json') as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info_r3api_success(self): + """Checks if info works correctly. + + NotificationEndpoint::info returns HANDLED if __call_r3api returns HANDLED. + __call_r3api internally callses _K2hr3UserAgent::send() to call the R3 API. + We mock the method to return True without having access to the API. + """ + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + _K2hr3UserAgent.send = MagicMock(return_value=True) + with open(src_dir + '/tools/data/notifications_neutron.json') as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info_r3api_failed(self): + """Checks if info works correctly. + + NotificationEndpoint::info returns HANDLED if __call_r3api returns HANDLED. + __call_r3api internally callses _K2hr3UserAgent::send() to call the R3 API. + We mock the method to return False without having access to the API. + """ + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + _K2hr3UserAgent.send = MagicMock(return_value=False) + with open(src_dir + '/tools/data/notifications_neutron.json') as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info_r3api_failed_requeue(self): + """Checks if info works correctly. + + NotificationEndpoint::info returns HANDLED if __call_r3api returns HANDLED. + __call_r3api internally callses _K2hr3UserAgent::send() to call the R3 API. + We mock the method to return False without having access to the API. + """ + # Expected return_value is REQUEUE in this case. + self.mock_method.return_value = REQUEUE + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + conf.k2hr3.requeue_on_error = True + endpoint = K2hr3NotificationEndpoint(conf) + _K2hr3UserAgent.send = MagicMock(return_value=False) + + with open(src_dir + '/tools/data/notifications_neutron.json') as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, REQUEUE) + # Reset it. Default return_value is HANDLED. + self.mock_method.return_value = HANDLED + + def test_notification_endpoint_info_r3api_failed_by_exception(self): + """Checks if info works correctly. + + NotificationEndpoint::info returns HANDLED if __call_r3api returns HANDLED. + __call_r3api internally callses _K2hr3UserAgent::send() to call the R3 API. + We mock the method to return False without having access to the API. + """ + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + _K2hr3UserAgent.send = MagicMock( + side_effect=_K2hr3UserAgentError('send error')) + with open(src_dir + '/tools/data/notifications_neutron.json') as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + # Ensure the__call_r3api is not called. + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info__payload_to_params_exception(self): + """Checks if info works correctly. + + NotificationEndpoint::info returns HANDLED if the _payload_to_params + method throws other than K2hr3NotificationEndpointError excetion. + """ + payload = { + "instance_id": "12345678-1234-5678-1234-567812345678", + } + with patch.object( + K2hr3NotificationEndpoint, + '_payload_to_params', + side_effect=Exception('_payload_to_params error')): + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info_call_r3api_exception(self): + """Checks if info works correctly. + + NotificationEndpoint::info returns HANDLED if the __call_r3api + method throws other than K2hr3NotificationEndpointError excetion. + """ + payload = { + "instance_id": "12345678-1234-5678-1234-567812345678", + } + with patch.object( + K2hr3NotificationEndpoint, + '_K2hr3NotificationEndpoint__call_r3api', + side_effect=Exception('__call_r3api error')): + conf = K2hr3Conf(src_dir + '/etc/k2hr3_osnl.conf.sample') + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + +# +# EOF +# diff --git a/k2hr3_osnl/tests/test_useragent.py b/k2hr3_osnl/tests/test_useragent.py new file mode 100644 index 0000000..b42f78d --- /dev/null +++ b/k2hr3_osnl/tests/test_useragent.py @@ -0,0 +1,369 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Endpoint class for the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +__author__ = 'Hirotaka Wakabayashi ' +__copyright__ = """ +Copyright (c) 2018 Yahoo Japan Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json +import logging +import os +import sys +import unittest +from unittest.mock import patch + +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.exceptions import _K2hr3UserAgentError +from k2hr3_osnl.useragent import _K2hr3UserAgent + +test_path = os.path.abspath(os.path.dirname(__file__)) +src_dir = os.path.join(test_path, '../..') +LOG = logging.getLogger(__name__) + + +class TestK2hr3UserAgent(unittest.TestCase): + """Tests the K2hr3UserAgent class.""" + + def setUp(self): + """Sets up a test case.""" + path = src_dir + '/etc/k2hr3_osnl.conf.sample' + self._conf = K2hr3Conf(path) + + def tearDown(self): + """Tears down a test case.""" + self._conf = None + + def test_k2hr3useragent_construct(self): + """Creates a K2hr3UserAgent instance.""" + agent = _K2hr3UserAgent(self._conf) + self.assertIsInstance(agent, _K2hr3UserAgent) + self.assertEqual(agent._url, 'https://localhost/v1/role') + + def test_k2hr3useragent_construct_conf_is_str(self): + """Checks if the __init__'s conf is not K2hr3Conf object.""" + conf = 'invalid_param' + with self.assertRaises(_K2hr3UserAgentError) as cm: + _K2hr3UserAgent(conf) + + the_exception = cm.exception + self.assertEqual( + the_exception.msg, 'conf is a K2hr3Conf instance, not {}'.format( + type(conf))) + + def test_k2hr3useragent_headers(self): + """Checks if headers.""" + agent = _K2hr3UserAgent(self._conf) + headers = { + 'User-Agent': + 'Python-k2hr3_ua/{}.{}'.format(sys.version_info[0], + sys.version_info[1]) + } + self.assertEqual(agent.headers, headers) + + def test_k2hr3useragent_headers_readonly(self): + """Checks if headers is readonly.""" + with self.assertRaises(AttributeError) as cm: + agent = _K2hr3UserAgent(self._conf) + new_headers = { + 'User-Agent': + 'Python-k2hr3_ua/{}.{}'.format(sys.version_info[0], + sys.version_info[1]) + } + agent.headers = new_headers + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_k2hr3useragent_params(self): + """Checks if params.""" + agent = _K2hr3UserAgent(self._conf) + params = {'extra': '"openstack-auto-v1"'} + self.assertEqual(agent.params, params) + + def test_k2hr3useragent_params_readonly(self): + """Checks if params is readonly.""" + with self.assertRaises(AttributeError) as cm: + agent = _K2hr3UserAgent(self._conf) + new_params = {'newkey': 'value'} + agent.params = new_params + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_k2hr3useragent_code(self): + """Checks if the code.""" + agent = _K2hr3UserAgent(self._conf) + self.assertEqual(agent.code, -1) + + def test_k2hr3useragent_code_readonly(self): + """Checks if code is readonly.""" + with self.assertRaises(AttributeError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.code = 204 + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_k2hr3useragent_error(self): + """Checks if errors.""" + agent = _K2hr3UserAgent(self._conf) + self.assertEqual(agent.error, '') + + def test_k2hr3useragent_error_readonly(self): + """Checks if error is readonly.""" + with self.assertRaises(AttributeError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.error = 'i am broken' + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_k2hr3useragent_method(self): + """Checks if method is valid.""" + agent = _K2hr3UserAgent(self._conf) + method = 'GET' + agent._method = method + self.assertEqual(agent.method, agent.method) + + def test_k2hr3useragent_method_setter(self): + """Checks if the method setter works.""" + agent = _K2hr3UserAgent(self._conf) + method = 'GET' + agent.method = method + self.assertEqual(method, agent.method) + + def test_k2hr3useragent_method_setter_value_error_1(self): + """Checks if the method setter works.""" + invalid_method = [] + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.method = invalid_method + the_exception = cm.exception + self.assertEqual( + 'method should be string, not {}'.format(invalid_method), + '{}'.format(the_exception)) + + def test_k2hr3useragent_url(self): + """Checks if url is valid.""" + agent = _K2hr3UserAgent(self._conf) + url = 'https://localhost/v1/role' + agent._url = url + self.assertEqual(agent.url, agent.url) + + def test_k2hr3useragent_url_setter(self): + """Checks if the url setter works.""" + agent = _K2hr3UserAgent(self._conf) + url = 'https://localhost/v1/role' + agent.url = url + self.assertEqual(url, agent.url) + + def test_k2hr3useragent_url_setter_value_error_1(self): + """Checks if the url setter works.""" + invalid_url = 'http:/localhost/v1/role/' + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.url = invalid_url + the_exception = cm.exception + self.assertEqual( + 'scheme should contain ://, not {}'.format(invalid_url), + '{}'.format(the_exception)) + + def test_k2hr3useragent_url_setter_value_error_2(self): + """Checks if the url setter works.""" + invalid_scheme = 'httq' + invalid_url = '{}://localhost/v1/role/'.format(invalid_scheme) + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.url = invalid_url + the_exception = cm.exception + self.assertEqual( + 'scheme should be http or http, not {}'.format(invalid_scheme), + '{}'.format(the_exception)) + + def test_k2hr3useragent_url_setter_value_error_3(self): + """Checks if the url setter works.""" + invalid_domain = 'example.comm' + invalid_url = 'http://{}/v1/role/'.format(invalid_domain) + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.url = invalid_url + the_exception = cm.exception + self.assertRegex('{}'.format(the_exception), '^unresolved domain, {}.*$'.format(invalid_domain)) + + def test_k2hr3useragent_ips_setter(self): + """Checks if the ips setter works.""" + agent = _K2hr3UserAgent(self._conf) + ips = '127.0.0.1' + agent.ips = ips + self.assertEqual([ips], agent.ips) + + def test_k2hr3useragent_ips_setter_list(self): + """Checks if the ips setter works.""" + agent = _K2hr3UserAgent(self._conf) + ips = ['127.0.0.1'] + agent.ips = ips + self.assertEqual(ips, agent.ips) + + def test_k2hr3useragent_ips_setter_value_error_1(self): + """Checks if the ips setter works.""" + invalid_ip = None + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.ips = invalid_ip + the_exception = cm.exception + self.assertEqual('ips must be list or str, not {}'.format(invalid_ip), + '{}'.format(the_exception)) + + def test_k2hr3useragent_ips_setter_value_error_2(self): + """Checks if the ips setter works.""" + invalid_ip = '127.0.0' + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.ips = invalid_ip + the_exception = cm.exception + msg = 'illegal IP address string passed to inet_pton' + self.assertEqual( + 'ip must be valid string, not {} {}'.format(invalid_ip, msg), + '{}'.format(the_exception)) + + def test_k2hr3useragent_ips_setter_value_error_3(self): + """Checks if the ips setter works.""" + invalid_ip = ':::' + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.ips = invalid_ip + the_exception = cm.exception + msg = 'illegal IP address string passed to inet_pton' + self.assertEqual( + 'ip must be valid string, not {} {}'.format(invalid_ip, msg), + '{}'.format(the_exception)) + + def test_k2hr3useragent_ips_setter_value_error_4(self): + """Checks if the ips setter works.""" + invalid_ips = [{}, ['1']] + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.ips = invalid_ips + the_exception = cm.exception + msg = 'ip must be str, not {}'.format(invalid_ips[0]) + self.assertEqual(msg, '{}'.format(the_exception)) + + def test_k2hr3useragent_instance_id_setter(self): + """Checks if the ips setter works.""" + agent = _K2hr3UserAgent(self._conf) + instance_id = '12345678-1234-5678-1234-567812345678' + agent.instance_id = instance_id + self.assertEqual(instance_id, agent.instance_id) + + def test_k2hr3useragent_instance_id_setter_value_error_1(self): + """Checks if the uuid setter works.""" + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + invalid_id = None + agent.instance_id = invalid_id + the_exception = cm.exception + self.assertEqual( + 'Please pass UUID as a string, not {}'.format(invalid_id), + '{}'.format(the_exception)) + + def test_k2hr3useragent_instance_id_setter_value_error_2(self): + """Checks if the uuid setter works.""" + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + invalid_id = '12345678-1234-5678-1234-56781234567' # drops the last byte. + agent.instance_id = invalid_id + the_exception = cm.exception + self.assertEqual( + 'Invalid UUID, {} badly formed hexadecimal UUID string'.format( + invalid_id), '{}'.format(the_exception)) + + def test_k2hr3useragent_allow_self_signed_cert(self): + """Checks if the allow_self_signed_cert setter works.""" + agent = _K2hr3UserAgent(self._conf) + allow_self_signed_cert = True + agent.allow_self_signed_cert = allow_self_signed_cert + self.assertEqual(True, agent.allow_self_signed_cert) + # make sure boolean type object is immutable. + allow_self_signed_cert = False + self.assertEqual(True, agent.allow_self_signed_cert) + + def test_k2hr3useragent_allow_self_signed_cert_error_1(self): + """Checks if the allow_self_signed_cert setter works.""" + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + allow_self_signed_cert = None + agent.allow_self_signed_cert = allow_self_signed_cert + the_exception = cm.exception + self.assertEqual( + 'Boolean value expected, not {}'.format(allow_self_signed_cert), + '{}'.format(the_exception)) + + @unittest.skip('This testcase is failed now. We need to fix untile first release.') + def test_k2hr3useragent_send(self): + """Checks if send() works correctly.""" + url = 'https://localhost/v1/role' + instance_id = '12345678-1234-5678-1234-567812345678' + ips = ['127.0.0.1', '127.0.0.2'] + # params + params = {'extra': json.dumps('openstack-auto-v1')} + params['host'] = json.dumps(ips) + params['cuk'] = json.dumps(instance_id) + headers = { + 'User-Agent': + 'Python-k2hr3_ua/{}.{}'.format(sys.version_info[0], + sys.version_info[1]) + } + method = 'GET' + + # Patch to _K2hr3UserAgent._send() method which is expected to return True if success. + with patch.object( + _K2hr3UserAgent, '_send', + return_value=True) as mock_send_method: + agent = _K2hr3UserAgent(self._conf) + agent.url = url + agent.instance_id = instance_id + agent.ips = ips + result = agent.send() + + self.assertEqual(result, True) + # Ensure values are as expected at runtime. + mock_send_method.assert_called_once_with(url, params, headers, method) + +# +# EOF +# diff --git a/k2hr3_osnl/useragent.py b/k2hr3_osnl/useragent.py new file mode 100644 index 0000000..9ae3323 --- /dev/null +++ b/k2hr3_osnl/useragent.py @@ -0,0 +1,442 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Sends http requests to the k2hr3 api. Classes in this module are not public.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +from enum import Enum +import json +import logging +import re +import socket +import ssl +import sys +import time +import urllib +import urllib.parse +import urllib.request +from urllib.error import ContentTooShortError, HTTPError, URLError +import uuid + +from typing import List, Set, Dict, Tuple, Optional, Union # noqa: pylint: disable=unused-import + +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.exceptions import _K2hr3UserAgentError +from k2hr3_osnl.httpresponse import _K2hr3HttpResponse + +LOG = logging.getLogger(__name__) + + +class _AgentError(Enum): + NONE = 1 + TEMP = 2 + FATAL = 3 + + +class _K2hr3UserAgent: + """Send a http/https request to the K2hr3 WebAPI.""" + + def __init__(self, conf: K2hr3Conf) -> None: + """Initializes attributes. + + :param conf: K2hr3Conf object. + :type K2hr3Conf: K2hr3Conf + :raises K2hr3UserAgentError: api_url validation error. + """ + # api_url validated for myself. + if isinstance(conf, K2hr3Conf) is False: + raise _K2hr3UserAgentError( + 'conf is a K2hr3Conf instance, not {}'.format(type(conf))) + try: + _K2hr3UserAgent.validate_url(conf.k2hr3.api_url) + except _K2hr3UserAgentError as error: + raise _K2hr3UserAgentError( + 'a valid url is expected, not {}'.format( + conf.k2hr3.api_url)) from error + self._conf = conf + self._url = conf.k2hr3.api_url + # other params validated in oslo_config. + self._retries = conf.k2hr3.max_retries + self._allow_self_signed_cert = conf.k2hr3.allow_self_signed_cert + # init the others. + self._ips = [] # type: List[str] + self._instance_id = '' + self._method = 'DELETE' + self._params = {'extra': 'openstack-auto-v1'} + self._headers = { + 'User-Agent': + 'Python-k2hr3_ua/{}.{}'.format(sys.version_info[0], + sys.version_info[1]) + } + self._response = _K2hr3HttpResponse() + LOG.debug('useragent initialized.') + + @property + def headers(self) -> Dict[str, str]: + """Returns the headers. + + :returns: Request headers + :rtype: Dict + """ + return self._headers + + @property + def params(self) -> Dict[str, str]: + """Returns the url params. + + :returns: Url params + :rtype: Dict + """ + return self._params + + @property + def code(self) -> int: + """Returns the HTTP status code. + + :returns: HTTP status code + :rtype: int + """ + return self._response.code + + @property + def error(self) -> str: + """Returns the error string. + + :returns: error string + :rtype: str + """ + return self._response.error + + @property + def method(self) -> str: + """Returns the http request method string. + + :returns: url string + :rtype: str + """ + return self._method + + @method.setter + def method(self, value: str) -> None: + """Sets the http request method string. + + :param value: http request method string + :type value: str + """ + if isinstance(value, str) is True: + LOG.debug('http request method is %s', value) + self._method = value + else: + raise _K2hr3UserAgentError( + 'method should be string, not {}'.format(value)) + + @property + def url(self) -> str: # public. + """Returns the url string. + + :returns: url string + :rtype: str + """ + return self._url + + @url.setter + def url(self, value: str) -> None: # public. + """Sets the url string. + + :param value: url string + :type value: str + """ + try: + if _K2hr3UserAgent.validate_url(value): + self._url = value + except _K2hr3UserAgentError: + raise + + @staticmethod + def validate_url(value): + """Returns True if given string is a url. + + :param value: a url like string + :type value: str + :returns: True if given string is a url. + :rtype: bool + """ + # scheme + try: + scheme, url_string = value.split('://', maxsplit=2) + except ValueError as error: + raise _K2hr3UserAgentError( + 'scheme should contain ://, not {}'.format(value)) from error + if scheme not in ('http', 'https'): + raise _K2hr3UserAgentError( + 'scheme should be http or http, not {}'.format(scheme)) + else: + LOG.debug('scheme is %s', scheme) + + matches = re.match( + r'(?P[\w|\.]+)?(?P:\d{2,5})?(?P[\w|/]*)?', + url_string) + if matches is None: + raise _K2hr3UserAgentError( + 'the argument seems not to be a url string, {}'.format(value)) + + # domain must be resolved. + domain = matches.group('domain') + if domain is None: + raise _K2hr3UserAgentError( + 'url contains no domain, {}'.format(value)) + try: + # https://github.com/python/cpython/blob/master/Modules/socketmodule.c#L5729 + ipaddress = socket.gethostbyname(domain) + except OSError as error: # resolve failed + raise _K2hr3UserAgentError('unresolved domain, {} {}'.format( + domain, error)) + else: + LOG.debug('%s resolved %s', domain, ipaddress) + + # path(optional) + if matches.group('path') is None: + raise _K2hr3UserAgentError( + 'url contains no path, {}'.format(value)) + path = matches.group('path') + # port(optional) + port = matches.group('port') + LOG.debug('url=%s domain=%s port=%s path=%s', value, domain, port, + path) + return True + + @property + def ips(self) -> List[str]: # public. + """Gets the ipaddress list. + + :returns: url string + :rtype: str + """ + return self._ips + + @ips.setter + def ips(self, value: str) -> None: # public. + """Sets ip or ips to the ipaddress list. + + :param value: ipaddress(str or list) + :type value: object + """ + ips = [] # type: List[str] + if isinstance(value, list): + ips += value + elif isinstance(value, str): + ips = [value] + else: + raise _K2hr3UserAgentError( + 'ips must be list or str, not {}'.format(value)) + for ipaddress in ips: + if isinstance(ipaddress, str) is False: + raise _K2hr3UserAgentError( + 'ip must be str, not {}'.format(ipaddress)) + try: + # https://github.com/python/cpython/blob/master/Modules/socketmodule.c#L6172 + socket.inet_pton(socket.AF_INET, ipaddress) + self._ips += [ipaddress] + except OSError: + LOG.debug('not ip version4 string %s', ipaddress) + try: + socket.inet_pton(socket.AF_INET6, ipaddress) + self._ips += [ipaddress] + except OSError as error: + LOG.error('neither ip version4 nor version6 string %s %s', + ipaddress, error) + raise _K2hr3UserAgentError( + 'ip must be valid string, not {} {}'.format( + ipaddress, error)) + self._ips = ips # overwrite + LOG.debug('ips=%s', ips) + # Note: + # parameter name is 'host' when calling r3api. + self._params['host'] = json.dumps(self._ips) + + @property + def instance_id(self) -> str: # public. + """Gets the instance id. + + :returns: instance id + :rtype: str + """ + return self._instance_id + + @instance_id.setter + def instance_id(self, value: str) -> None: # publc. + """Sets instance id. + + :param value: instance id + :type value: str + """ + if isinstance(value, str) is False: + raise _K2hr3UserAgentError( + 'Please pass UUID as a string, not {}'.format(value)) + try: + if value: + uuid.UUID(value) + self._instance_id = value + except ValueError as error: + raise _K2hr3UserAgentError('Invalid UUID, {} {}'.format( + value, error)) + # Note: + # parameter name is 'cuk' when calling r3api. + self._params['cuk'] = self._instance_id + + @property + def allow_self_signed_cert(self) -> bool: # public. + """Gets the flag of self signed certificate or not. + + :returns: True if allow self signed certificate to use. + :rtype: bool + """ + return self._allow_self_signed_cert + + @allow_self_signed_cert.setter + def allow_self_signed_cert(self, value: bool) -> None: # public. + """Sets the flag of self signed certificate or not. + + :param value: True if allow self signed certificate to use. + :type value: bool + """ + if isinstance(value, bool): + self._allow_self_signed_cert = value + else: + raise _K2hr3UserAgentError( + 'Boolean value expected, not {}'.format(value)) + + def _send_internal(self, url: str, params: Dict[str, str], + headers: Dict[str, str], + method: str) -> bool: # non-public. + """Sends a http request. + + :returns: True if success, otherwise False + :rtype: bool + """ + assert [ + isinstance(url, str), + isinstance(params, dict), + isinstance(headers, dict), + isinstance(method, str), + ] + + LOG.debug('_send called by url %s params %s headers %s method %s', url, + params, headers, method) + + qstring = urllib.parse.urlencode(params, quote_via=urllib.parse.quote) # type: ignore + req = urllib.request.Request( + '?'.join([url, qstring]), headers=headers, method=method) + if req.type not in ('http', 'https'): + self._response.error = 'http or https, not {}'.format(req.type) + LOG.error(self._response) + return False + agent_error = _AgentError.NONE + try: + ctx = None + if req.type == 'https': + # https://docs.python.jp/3/library/ssl.html#ssl.create_default_context + ctx = ssl.create_default_context() + if self._allow_self_signed_cert: + # https://github.com/python/cpython/blob/master/Lib/ssl.py#L567 + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with urllib.request.urlopen( + req, timeout=self._conf.k2hr3.timeout_seconds, + context=ctx) as res: + self._response.code = res.getcode() + LOG.debug('code=[%s]\nurl=[%s]\nbody=[%s]\ninfo=[%s]\n', + res.getcode(), res.geturl(), res.read(), res.info()) + except HTTPError as error: + LOG.error( + 'Could not complete the request. code %s reason %s headers %s', + error.code, error.reason, error.headers) + agent_error = _AgentError.FATAL + except (ContentTooShortError, URLError) as error: + # https://github.com/python/cpython/blob/master/Lib/urllib/error.py#L73 + LOG.error('Could not read the server. reason %s', error.reason) + agent_error = _AgentError.FATAL + except (socket.timeout, OSError) as error: # temporary error + LOG.error('error(OSError, socket) %s', error) + agent_error = _AgentError.TEMP + finally: + if agent_error == _AgentError.TEMP: + self._retries -= 1 # decrement the retries value. + if self._retries >= 0: + LOG.warning('sleeping for %s. remaining retries=%s', + self._conf.k2hr3.retry_interval_seconds, + self._retries) + time.sleep(self._conf.k2hr3.retry_interval_seconds) + self._send_internal(url, params, headers, method) + else: + self._response.error = 'reached the max retry count.' + LOG.error(self._response.error) + agent_error = _AgentError.FATAL + + if agent_error == _AgentError.NONE: + LOG.debug('no problem.') + return True + LOG.debug('problem %s', self._response) + return False + + def send(self) -> bool: # public. + """Sends a http request. + + :returns: True if success, otherwise False + :rtype: bool + """ + assert [ + isinstance(self._url, str), + isinstance(self._params, dict), + self._params.get('host', None) is not None, + isinstance(self._params, dict), + self._params.get('cuk', None) is not None, + isinstance(self._params, dict), + self._params.get('extra', None) is not None, + ] + + return self._send_internal(self._url, self._params, self._headers, + self._method) + + def __repr__(self): + attrs = [] + for attr in ['_url', '_params', '_headers', '_method']: + val = getattr(self, attr) + if val: + attrs.append((attr, repr(val))) + values = ', '.join(['%s=%s' % i for i in attrs]) + return '<_K2hr3UserAgent ' + values + '>' + + def __str__(self): + attrs = {} + for attr in ['_url', '_params', '_headers', '_method']: + val = getattr(self, attr) + if val: + attrs[attr] = str(val) + else: + LOG.debug('%s empty', attr) + values = '' + for key, value in attrs.items(): + values += '{}={} '.format(key, value) + return '<_K2hr3UserAgent ' + values + '>' + +# +# EOF +# diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..cb146cb --- /dev/null +++ b/pylintrc @@ -0,0 +1,432 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. This option is deprecated +# and it will be removed in Pylint 2.0. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=oct-method,xrange-builtin,range-builtin-not-iterating,cmp-method,round-builtin,dict-iter-method,unicode-builtin,useless-suppression,input-builtin,old-octal-literal,dict-view-method,nonzero-method,old-division,delslice-method,apply-builtin,no-absolute-import,basestring-builtin,coerce-method,zip-builtin-not-iterating,reload-builtin,next-method-called,cmp-builtin,filter-builtin-not-iterating,import-star-module-level,backtick,long-suffix,execfile-builtin,coerce-builtin,indexing-exception,raw_input-builtin,suppressed-message,standarderror-builtin,raising-string,hex-method,old-raise-syntax,reduce-builtin,unpacking-in-except,old-ne-operator,unichr-builtin,map-builtin-not-iterating,metaclass-assignment,getslice-method,long-builtin,intern-builtin,setslice-method,parameter-unpacking,buffer-builtin,print-statement,file-builtin,using-cmp-argument,raise-missing-from + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=10 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/python-k2hr3-osnl.spec b/python-k2hr3-osnl.spec new file mode 100644 index 0000000..2280032 --- /dev/null +++ b/python-k2hr3-osnl.spec @@ -0,0 +1,110 @@ +%global srcname k2hr3_osnl +%global pypi_name k2hr3-osnl +Name: python-k2hr3-osnl +Version: 0.9.5 +Release: 1%{?dist} +Summary: An OpenStack notification listener for K2HR3 + +License: MIT +URL: https://github.com/yahoojapan/%{srcname} +Source0: https://github.com/yahoojapan/%{srcname}/archive/v%{version}/%{srcname}-%{version}.tar.gz + +BuildArch: noarch + +%description +k2hr3_osnl is an OpenStack Notification Listener, which listens to +notifications from OpenStack services. When catching a notification, it +sends the notification payload to K2HR3, the OpenStack role-based ACL +system developed by Yahoo Japan Corporation. + +%package -n python3-%{pypi_name} +Summary: %{summary} +BuildRequires: git +BuildRequires: python3-devel +BuildRequires: python3-oslo-config +BuildRequires: python3-oslo-messaging +BuildRequires: help2man +%if 0%{?fedora} >= 30 +BuildRequires: systemd-rpm-macros +%else +BuildRequires: systemd +%endif +%{?systemd_requires} +%{?python_provide:%python_provide python3-%{pypi_name}} + +# python3-oslo-messaging found. python3x-oslo-messaging not found. +# Tested with OpenStack rocky on fc29 +Requires: python3-oslo-messaging >= 5.35.1 +Requires: python3-oslo-config >= 5.2.0 + +%description -n python3-%{pypi_name} +k2hr3_osnl is an OpenStack Notification Listener, which listens to +notifications from OpenStack services. When catching a notification, it +sends the notification payload to K2HR3, the OpenStack role-based ACL +system developed by Yahoo Japan Corporation. + +%prep +%autosetup -n %{srcname}-%{version} -S git + +%post +%systemd_post k2hr3-osnl.service + +%preun +%systemd_preun k2hr3-osnl.service + +%postun +%systemd_postun_with_restart k2hr3-osnl.service + +%build +%py3_build + +%install +%py3_install +mkdir -p -m755 %{buildroot}%{_sysconfdir}/k2hr3 +mkdir -p -m755 %{buildroot}%{_unitdir} +mkdir -p -m755 %{buildroot}%{_mandir}/man1 +install -pm 644 etc/k2hr3-osnl.conf %{buildroot}%{_sysconfdir}/k2hr3/k2hr3-osnl.conf +install -pm 644 k2hr3-osnl.service %{buildroot}%{_unitdir}/k2hr3-osnl.service +help2man --no-discard-stderr --version-string=%{version} %{buildroot}%{_bindir}/k2hr3-osnl > %{buildroot}%{_mandir}/man1/k2hr3-osnl.1 +rm -rf %{buildroot}/usr/etc/k2hr3/k2hr3-osnl.conf + +%check +%{__python3} -m unittest + +%files -n python3-%{pypi_name} +%dir %{_sysconfdir}/k2hr3/ +%config(noreplace) %{_sysconfdir}/k2hr3/k2hr3-osnl.conf +%doc README.rst +%license LICENSE +%{_bindir}/k2hr3-osnl +%{python3_sitelib}/%{srcname} +%{python3_sitelib}/*.egg-info +%{_unitdir}/k2hr3-osnl.service +%{_mandir}/man1/k2hr3-osnl.1* + +%changelog +* Tue Dec 01 2020 Hirotaka Wakabayashi 0.9.5-1 +- Update for Release Version 0.9.5 + +* Tue Dec 01 2020 Hirotaka Wakabayashi 0.9.4-1 +- Update for Release Version 0.9.4 + +* Mon Nov 30 2020 Hirotaka Wakabayashi 0.9.3-1 +- Update for Release Version 0.9.3 + +* Tue Mar 26 2019 Hirotaka Wakabayashi 0.9.2-1 +- Update for Release Version 0.9.2 + +* Wed Mar 20 2019 Hirotaka Wakabayashi 0.9.1-2 +- Removed redundant lines + +* Tue Mar 19 2019 Hirotaka Wakabayashi 0.9.1-1 +- Fixed systemd unitfile + +* Tue Mar 19 2019 Hirotaka Wakabayashi 0.9.0-2 +- Used the config(noreplace) to preserve the previous config file +- Tested on fc29 and updated oslo library versions + +* Wed Mar 6 2019 Hirotaka Wakabayashi 0.9.0-1 +- Initial Version + diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..4a28bd3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,60 @@ +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +[metadata] +# This includes the license file in the wheel. +license_file = LICENSE + +[flake8] +# pycodestyle +max-line-length = 119 +# mccabe +max_complexity = 10 +# pycodelint +# it's not a bug that we aren't using all of hacking +ignore = + # D401: First line should be in imperative mood + D401, + # E402 module level import not at top of file + E402 +exclude = + # No need to traverse our git directory + .git, + # There's no value in checking cache directories + __pycache__, + # The conf file is mostly autogenerated, ignore it + docs/source/conf.py, + # The old directory contains Flake8 2.0 + old, + # This contains our built documentation + build, + # This contains builds of flake8 that we don't want to check + dist + +[mypy-oslo_config] +ignore_missing_imports = True + +[mypy-oslo_messaging] +ignore_missing_imports = True + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..443b392 --- /dev/null +++ b/setup.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +"""K2HR3 OpenStack Notification message Listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +__author__ = 'Hirotaka Wakabayashi ' +__copyright__ = """ +Copyright (c) 2018 Yahoo Japan Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +import sys + +PKG_NAME = 'k2hr3_osnl' + +with open('README.rst') as readme_file: + readme = readme_file.read() + +with open('HISTORY.rst') as history_file: + history = history_file.read() + +def get_version(pkg=PKG_NAME): + """Returns the package version from __ini__.py.""" + from pathlib import Path + from os import path, sep + import re + + here = path.abspath(path.dirname(__file__)) + init_py = Path(sep.join([here, pkg, '__init__.py'])).resolve() + + with init_py.open() as fp: + for line in fp: + version_match = re.search( + r"^__version__ = ['\"]([^'\"]*)['\"]", line.strip(), re.M) + if version_match: + return version_match.group(1) + raise RuntimeError('version expected, but no version found.') + +setup( + author="Hirotaka Wakabayashi", + author_email='hiwakaba@yahoo-corp.jp', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Environment :: OpenStack', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'License :: OSI Approved :: MIT License', + "Operating System :: POSIX :: Linux", + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + ], + data_files=[('etc/k2hr3',['etc/k2hr3-osnl.conf'])], + description="An OpenStack notification listener for the K2HR3 role-based ACL system", + entry_points={ + 'console_scripts': [ + 'k2hr3-osnl=k2hr3_osnl:main', + ], + }, + install_requires=[ + 'oslo.config>=5.2.0', + 'oslo.messaging>=5.17.1', + ], + include_package_data=True, + keywords='AntPickax IAM OpenStack', + license="MIT license", + long_description=readme + '\n\n' + history, + name=PKG_NAME, + packages=find_packages(include=['k2hr3_osnl']), + project_urls={ + 'Bugs': 'https://github.com/yahoojapan/k2hr3_osnl/issues', + 'Docs': 'https://k2hr3-osnl.readthedocs.io/en/latest/', + 'Source': 'https://github.com/yahoojapan/k2hr3_osnl', + }, + python_requires='>=3.5', + url='https://github.com/yahoojapan/k2hr3_osnl', + version=get_version(), + zip_safe=False, +) + +# +# EOF +# diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..f23db9a --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Test Package for K2hr3 OpenStack Notification message Listener. + +This file is needed to run tests simply like: +$ python -m unittest discover + +All of the test files must be a package importable from the top-level directory of the project. +https://docs.python.org/3.6/library/unittest.html#test-discovery +""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) +__author__ = 'Hirotaka Wakabayashi ' +__version__ = '0.0.1' + +# Disables the k2hr3_osnl library logs by failure assetion tests. +import logging +logging.getLogger('k2hr3_osnl').addHandler(logging.NullHandler()) + +# +# EOF +# diff --git a/tests/k2hr3-osnl.conf_broken b/tests/k2hr3-osnl.conf_broken new file mode 100644 index 0000000..3eba2f6 --- /dev/null +++ b/tests/k2hr3-osnl.conf_broken @@ -0,0 +1,2 @@ +value = value +value = value diff --git a/tests/test_cfg.py b/tests/test_cfg.py new file mode 100644 index 0000000..3e3b284 --- /dev/null +++ b/tests/test_cfg.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Endpoint class for the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +from pathlib import Path +from os import path, sep +import unittest +from unittest.mock import patch + +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.exceptions import K2hr3ConfError + +here = path.abspath(path.dirname(__file__)) +conf_file_path = Path(sep.join([here, '..', 'etc', + 'k2hr3-osnl.conf'])).resolve() +broken_conf_file_path = Path( + sep.join([here, 'k2hr3-osnl.conf_broken'])).resolve() + +LOG = logging.getLogger(__name__) + + +class TestK2hr3Conf(unittest.TestCase): + """Tests the K2hr3Conf class. + + Simple usage(this class only): + $ python -m unittest tests/test_cfg.py + + Simple usage(all): + $ python -m unittest tests + """ + + def test_k2hr3_conf_construct(self): + """Creates a K2hr3Conf instance.""" + conf = K2hr3Conf(conf_file_path) + self.assertIsInstance(conf, K2hr3Conf) + + def test_k2hr3_conf_construct_init_error(self): + """Initialization error.""" + with self.assertRaises(K2hr3ConfError) as cm: + K2hr3Conf(broken_conf_file_path) + + the_exception = cm.exception + self.assertRegex(the_exception.msg, '^parse error,.*') + + def test_k2hr3_conf_construct_path_is_none(self): + """Checks if the __init__'s path is None.""" + with self.assertRaises(K2hr3ConfError) as cm: + K2hr3Conf(None) + + the_exception = cm.exception + self.assertEqual(the_exception.msg, 'Path expected, not NoneType') + + def test_k2hr3_conf_construct_path_is_dir(self): + """Checks if the __init__'s path is a directory.""" + path = Path('/tmp') + with self.assertRaises(K2hr3ConfError) as cm: + K2hr3Conf(path) + + the_exception = cm.exception + self.assertEqual(the_exception.msg, + 'path must be a regular file, not {}'.format(path)) + + def test_k2hr3_conf_construct_path_is_not_readable(self): + """Checks if the __init__'s path is not readable.""" + path = Path('/etc/sudoers') + with self.assertRaises(K2hr3ConfError): + K2hr3Conf(path) + + def test_k2hr3_conf_construct_parse_error(self): + """Checks if the __init__'s path is not a config file.""" + path = Path(here) + with self.assertRaises(K2hr3ConfError) as cm: + K2hr3Conf(path) + + the_exception = cm.exception + self.assertRegex(the_exception.msg, + '^path must be a regular file, not {}.*'.format(path)) + + def test_k2hr3_conf_assert_parse_config_called(self): + """Asserts _parse_config method called from constructor.""" + with patch.object( + K2hr3Conf, '_parse_config', return_value=True) as mock_method: + K2hr3Conf(conf_file_path) + + mock_method.assert_called_once_with() + + def test_k2hr3_conf_oslo_messaging_notifications_event_type(self): + """Asserts event_type in oslo_messaging_notifications group. + + The publisher_id and the event_type are very important filter for this system. + """ + conf = K2hr3Conf(conf_file_path) + self.assertEqual(r'^port\.delete\.end$', + conf.oslo_messaging_notifications.event_type) + + def test_k2hr3_conf_oslo_messaging_notifications_publisher_id(self): + """Asserts publisher_id in oslo_messaging_notifications group. + + The publisher_id and the event_type are very important filter for this system. + """ + conf = K2hr3Conf(conf_file_path) + self.assertEqual(r'^network.*$', + conf.oslo_messaging_notifications.publisher_id) + + def test_k2hr3_conf_oslo_messaging_notifications_context(self): + """Asserts context in oslo_messaging_notifications group. + + The context is an optional NotificationFilter. + """ + conf = K2hr3Conf(conf_file_path) + self.assertEqual(None, conf.oslo_messaging_notifications.context) + + def test_k2hr3_conf_oslo_messaging_notifications_metadata(self): + """Asserts metadata in oslo_messaging_notifications group. + + The metadata is an optional NotificationFilter. + """ + conf = K2hr3Conf(conf_file_path) + self.assertEqual(None, conf.oslo_messaging_notifications.metadata) + + def test_k2hr3_conf_oslo_messaging_notifications_payload(self): + """Asserts payload in oslo_messaging_notifications group. + + The payload is an optional NotificationFilter. + """ + conf = K2hr3Conf(conf_file_path) + self.assertEqual(None, conf.oslo_messaging_notifications.payload) + + def test_k2hr3_conf_oslo_messaging_notifications_transport_url(self): + """Asserts transport_url in oslo_messaging_notifications group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual('rabbit://guest:guest@127.0.0.1:5672/', + conf.oslo_messaging_notifications.transport_url) + + def test_k2hr3_conf_oslo_messaging_notifications_topic(self): + """Asserts topic in oslo_messaging_notifications group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual('notifications', + conf.oslo_messaging_notifications.topic) + + def test_k2hr3_conf_oslo_messaging_notifications_exchange(self): + """Asserts exchange in oslo_messaging_notifications group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual('neutron', conf.oslo_messaging_notifications.exchange) + + def test_k2hr3_conf_oslo_messaging_notifications_pool(self): + """Asserts pool in oslo_messaging_notifications group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual('k2hr3_osnl', conf.oslo_messaging_notifications.pool) + + def test_k2hr3_conf_oslo_messaging_notifications_allow_requeue(self): + """Asserts allow_requeue in oslo_messaging_notifications group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual(True, conf.oslo_messaging_notifications.allow_requeue) + + def test_k2hr3_conf_k2hr3_api_url(self): + """Asserts api_url in k2hr3 group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual('https://localhost/v1/role', conf.k2hr3.api_url) + + def test_k2hr3_conf_k2hr3_timeout_seconds(self): + """Asserts timeout_second in k2hr3 group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual(30, conf.k2hr3.timeout_seconds) + + def test_k2hr3_conf_k2hr3_max_retries(self): + """Asserts retries in k2hr3 group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual(5, conf.k2hr3.max_retries) + + def test_k2hr3_conf_k2hr3_retry_interval_seconds(self): + """Asserts retry_interval_seconds in k2hr3 group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual(60, conf.k2hr3.retry_interval_seconds) + + def test_k2hr3_conf_k2hr3_allow_self_signed_cert(self): + """Asserts allow_self_signed_cert in k2hr3 group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual(False, conf.k2hr3.allow_self_signed_cert) + + def test_k2hr3_conf_k2hr3_requeue_on_error(self): + """Asserts requeue_on_error in k2hr3 group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual(False, conf.k2hr3.requeue_on_error) + + def test_k2hr3_conf_default_log_file(self): + """Asserts log_file in default group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual('sys.stderr', conf.log_file) + + def test_k2hr3_conf_default_debug_level(self): + """Asserts debug_level in default group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual('error', conf.debug_level) + + def test_k2hr3_conf_default_libs_debug_level(self): + """Asserts libs_debug_level in default group.""" + conf = K2hr3Conf(conf_file_path) + self.assertEqual('warn', conf.libs_debug_level) + +# +# EOF +# diff --git a/tests/test_useragent.py b/tests/test_useragent.py new file mode 100644 index 0000000..138d044 --- /dev/null +++ b/tests/test_useragent.py @@ -0,0 +1,380 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""UserAgent class for the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import json +import logging +import os +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.exceptions import _K2hr3UserAgentError +from k2hr3_osnl.useragent import _K2hr3UserAgent + +here = os.path.abspath(os.path.dirname(__file__)) +conf_file_path = Path( + os.sep.join([here, '..', 'etc', 'k2hr3-osnl.conf'])).resolve() + +LOG = logging.getLogger(__name__) + + +class TestK2hr3UserAgent(unittest.TestCase): + """Tests the K2hr3UserAgent class. + + Simple usage(this class only): + $ python -m unittest tests/test_useragent.py + + Simple usage(all): + $ python -m unittest tests + """ + + def setUp(self): + """Sets up a test case.""" + self._conf = K2hr3Conf(conf_file_path) + self.patcher_call_send_agent = patch.object(_K2hr3UserAgent, + '_send_internal') + self.mock_method_agent = self.patcher_call_send_agent.start() + self.mock_method_agent.return_value = True + self.addCleanup(self.patcher_call_send_agent.stop) + + def tearDown(self): + """Tears down a test case.""" + self._conf = None + self.patcher_call_send_agent = None + + def test_k2hr3useragent_construct(self): + """Creates a K2hr3UserAgent instance.""" + agent = _K2hr3UserAgent(self._conf) + self.assertIsInstance(agent, _K2hr3UserAgent) + self.assertEqual(agent._url, 'https://localhost/v1/role') + + def test_k2hr3useragent_construct_conf_is_str(self): + """Checks if the __init__'s conf is not K2hr3Conf object.""" + conf = 'invalid_param' + with self.assertRaises(_K2hr3UserAgentError) as cm: + _K2hr3UserAgent(conf) + + the_exception = cm.exception + self.assertEqual( + the_exception.msg, 'conf is a K2hr3Conf instance, not {}'.format( + type(conf))) + + def test_k2hr3useragent_construct_conf_validation_error(self): + """Checks if the url in conf is invalid.""" + self._conf.k2hr3.api_url = '' # self.conf instantiates in every setUp. + with self.assertRaises(_K2hr3UserAgentError) as cm: + _K2hr3UserAgent(self._conf) + the_exception = cm.exception + self.assertEqual('a valid url is expected, not ', + '{}'.format(the_exception)) + + def test_k2hr3useragent_repr(self): + """Represent a _K2hr3UserAgent instance.""" + agent = _K2hr3UserAgent(self._conf) + # Note: The order of _error and _code is unknown! + self.assertRegex(repr(agent), '<_K2hr3UserAgent _.*') + + def test_k2hr3httpresponse_str(self): + """Stringfy a _K2hr3UserAgent instance.""" + agent = _K2hr3UserAgent(self._conf) + # Note: The order of _error and _code is unknown! + self.assertRegex(str(agent), '<_K2hr3UserAgent _.*') + + def test_k2hr3useragent_headers(self): + """Checks if headers.""" + agent = _K2hr3UserAgent(self._conf) + headers = { + 'User-Agent': + 'Python-k2hr3_ua/{}.{}'.format(sys.version_info[0], + sys.version_info[1]) + } + self.assertEqual(agent.headers, headers) + + def test_k2hr3useragent_headers_readonly(self): + """Checks if headers is readonly.""" + with self.assertRaises(AttributeError) as cm: + agent = _K2hr3UserAgent(self._conf) + new_headers = { + 'User-Agent': + 'Python-k2hr3_ua/{}.{}'.format(sys.version_info[0], + sys.version_info[1]) + } + agent.headers = new_headers + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_k2hr3useragent_params(self): + """Checks if params.""" + agent = _K2hr3UserAgent(self._conf) + params = {'extra': 'openstack-auto-v1'} + self.assertEqual(agent.params, params) + + def test_k2hr3useragent_params_readonly(self): + """Checks if params is readonly.""" + with self.assertRaises(AttributeError) as cm: + agent = _K2hr3UserAgent(self._conf) + new_params = {'newkey': 'value'} + agent.params = new_params + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_k2hr3useragent_code(self): + """Checks if the code.""" + agent = _K2hr3UserAgent(self._conf) + self.assertEqual(agent.code, -1) + + def test_k2hr3useragent_code_readonly(self): + """Checks if code is readonly.""" + with self.assertRaises(AttributeError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.code = 204 + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_k2hr3useragent_error(self): + """Checks if errors.""" + agent = _K2hr3UserAgent(self._conf) + self.assertEqual(agent.error, '') + + def test_k2hr3useragent_error_readonly(self): + """Checks if error is readonly.""" + with self.assertRaises(AttributeError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.error = 'i am broken' + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_k2hr3useragent_method(self): + """Checks if method is valid.""" + agent = _K2hr3UserAgent(self._conf) + method = 'GET' + agent._method = method + self.assertEqual(agent.method, agent.method) + + def test_k2hr3useragent_method_setter(self): + """Checks if the method setter works.""" + agent = _K2hr3UserAgent(self._conf) + method = 'GET' + agent.method = method + self.assertEqual(method, agent.method) + + def test_k2hr3useragent_method_setter_value_error_1(self): + """Checks if the method setter works.""" + invalid_method = [] + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.method = invalid_method + the_exception = cm.exception + self.assertEqual( + 'method should be string, not {}'.format(invalid_method), + '{}'.format(the_exception)) + + def test_k2hr3useragent_url(self): + """Checks if url is valid.""" + agent = _K2hr3UserAgent(self._conf) + url = 'https://localhost/v1/role' + agent._url = url + self.assertEqual(agent.url, agent.url) + + def test_k2hr3useragent_url_setter(self): + """Checks if the url setter works.""" + agent = _K2hr3UserAgent(self._conf) + url = 'https://localhost/v1/role' + agent.url = url + self.assertEqual(url, agent.url) + + def test_k2hr3useragent_url_setter_value_error_1(self): + """Checks if the url setter works.""" + invalid_url = 'http:/localhost/v1/role/' + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.url = invalid_url + the_exception = cm.exception + self.assertEqual( + 'scheme should contain ://, not {}'.format(invalid_url), + '{}'.format(the_exception)) + + def test_k2hr3useragent_url_setter_value_error_2(self): + """Checks if the url setter works.""" + invalid_scheme = 'httq' + invalid_url = '{}://localhost/v1/role/'.format(invalid_scheme) + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.url = invalid_url + the_exception = cm.exception + self.assertEqual( + 'scheme should be http or http, not {}'.format(invalid_scheme), + '{}'.format(the_exception)) + + def test_k2hr3useragent_url_setter_value_error_3(self): + """Checks if the url setter works.""" + invalid_domain = 'example.comm' + invalid_url = 'http://{}/v1/role/'.format(invalid_domain) + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.url = invalid_url + the_exception = cm.exception + self.assertRegex('{}'.format(the_exception), + '^unresolved domain, {}.*$'.format(invalid_domain)) + + def test_k2hr3useragent_ips_setter(self): + """Checks if the ips setter works.""" + agent = _K2hr3UserAgent(self._conf) + ips = '127.0.0.1' + agent.ips = ips + self.assertEqual([ips], agent.ips) + + def test_k2hr3useragent_ips_setter_list(self): + """Checks if the ips setter works.""" + agent = _K2hr3UserAgent(self._conf) + ips = ['127.0.0.1'] + agent.ips = ips + self.assertEqual(ips, agent.ips) + + def test_k2hr3useragent_ips_setter_value_error_1(self): + """Checks if the ips setter works.""" + invalid_ip = None + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.ips = invalid_ip + the_exception = cm.exception + self.assertEqual('ips must be list or str, not {}'.format(invalid_ip), + '{}'.format(the_exception)) + + def test_k2hr3useragent_ips_setter_value_error_2(self): + """Checks if the ips setter works.""" + invalid_ip = '127.0.0' + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.ips = invalid_ip + the_exception = cm.exception + msg = 'illegal IP address string passed to inet_pton' + self.assertEqual( + 'ip must be valid string, not {} {}'.format(invalid_ip, msg), + '{}'.format(the_exception)) + + def test_k2hr3useragent_ips_setter_value_error_3(self): + """Checks if the ips setter works.""" + invalid_ip = ':::' + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.ips = invalid_ip + the_exception = cm.exception + msg = 'illegal IP address string passed to inet_pton' + self.assertEqual( + 'ip must be valid string, not {} {}'.format(invalid_ip, msg), + '{}'.format(the_exception)) + + def test_k2hr3useragent_ips_setter_value_error_4(self): + """Checks if the ips setter works.""" + invalid_ips = [{}, ['1']] + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + agent.ips = invalid_ips + the_exception = cm.exception + msg = 'ip must be str, not {}'.format(invalid_ips[0]) + self.assertEqual(msg, '{}'.format(the_exception)) + + def test_k2hr3useragent_instance_id_setter(self): + """Checks if the ips setter works.""" + agent = _K2hr3UserAgent(self._conf) + instance_id = '12345678-1234-5678-1234-567812345678' + agent.instance_id = instance_id + self.assertEqual(instance_id, agent.instance_id) + + def test_k2hr3useragent_instance_id_setter_value_error_1(self): + """Checks if the uuid setter works.""" + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + invalid_id = None + agent.instance_id = invalid_id + the_exception = cm.exception + self.assertEqual( + 'Please pass UUID as a string, not {}'.format(invalid_id), + '{}'.format(the_exception)) + + def test_k2hr3useragent_instance_id_setter_value_error_2(self): + """Checks if the uuid setter works.""" + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + invalid_id = '12345678-1234-5678-1234-56781234567' # drops the last byte. + agent.instance_id = invalid_id + the_exception = cm.exception + self.assertEqual( + 'Invalid UUID, {} badly formed hexadecimal UUID string'.format( + invalid_id), '{}'.format(the_exception)) + + def test_k2hr3useragent_allow_self_signed_cert(self): + """Checks if the allow_self_signed_cert setter works.""" + agent = _K2hr3UserAgent(self._conf) + allow_self_signed_cert = True + agent.allow_self_signed_cert = allow_self_signed_cert + self.assertEqual(True, agent.allow_self_signed_cert) + # make sure boolean type object is immutable. + allow_self_signed_cert = False + self.assertEqual(True, agent.allow_self_signed_cert) + + def test_k2hr3useragent_allow_self_signed_cert_error_1(self): + """Checks if the allow_self_signed_cert setter works.""" + with self.assertRaises(_K2hr3UserAgentError) as cm: + agent = _K2hr3UserAgent(self._conf) + allow_self_signed_cert = None + agent.allow_self_signed_cert = allow_self_signed_cert + the_exception = cm.exception + self.assertEqual( + 'Boolean value expected, not {}'.format(allow_self_signed_cert), + '{}'.format(the_exception)) + + def test_k2hr3useragent_send(self): + """Checks if send() works correctly.""" + url = 'https://localhost/v1/role' + instance_id = '12345678-1234-5678-1234-567812345678' + ips = ['127.0.0.1', '127.0.0.2'] + # params + params = {'extra': 'openstack-auto-v1'} + params['host'] = json.dumps(ips) + params['cuk'] = instance_id + headers = { + 'User-Agent': + 'Python-k2hr3_ua/{}.{}'.format(sys.version_info[0], + sys.version_info[1]) + } + method = 'DELETE' + + # Patch to _K2hr3UserAgent._send() method which is expected to return True if success. + agent = _K2hr3UserAgent(self._conf) + agent.url = url + agent.instance_id = instance_id + agent.ips = ips + result = agent.send() + + self.assertEqual(result, True) + # Ensure values are as expected at runtime. + self.mock_method_agent.assert_called_once_with(url, params, headers, + method) + +# +# EOF +# diff --git a/tests/test_useragent_response.py b/tests/test_useragent_response.py new file mode 100644 index 0000000..6b7a7fc --- /dev/null +++ b/tests/test_useragent_response.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""HTTPResponse class for the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +import os +from pathlib import Path +import unittest + +from k2hr3_osnl.exceptions import _K2hr3UserAgentError +from k2hr3_osnl.httpresponse import _K2hr3HttpResponse + +here = os.path.abspath(os.path.dirname(__file__)) +conf_file_path = Path( + os.sep.join([here, '..', 'etc', 'k2hr3-osnl.conf'])).resolve() + +LOG = logging.getLogger(__name__) + + +class TestK2hr3UserAgentResponse(unittest.TestCase): + """Tests the K2hr3UserAgentResponse class. + + Simple usage(this class only): + $ python -m unittest tests/test_useragent_response.py + + Simple usage(all): + $ python -m unittest tests + """ + + def setUp(self): + """Sets up a test case.""" + + def tearDown(self): + """Tears down a test case.""" + + def test_k2hr3httpresponse_construct(self): + """Creates a _K2hr3HttpResponse instance.""" + response = _K2hr3HttpResponse() + self.assertIsInstance(response, _K2hr3HttpResponse) + + def test_k2hr3httpresponse_repr(self): + """Represent a _K2hr3HttpResponse instance.""" + response = _K2hr3HttpResponse() + # Note: The order of _error and _code is unknown! + self.assertRegex(repr(response), '<_K2hr3HttpResponse _.*') + + def test_k2hr3httpresponse_str(self): + """Stringfy a _K2hr3HttpResponse instance.""" + response = _K2hr3HttpResponse() + # Note: The order of _error and _code is unknown! + self.assertRegex(str(response), '<_K2hr3HttpResponse _.*') + + def test_k2hr3httpresponse_code(self): + """Checks if the code is 204.""" + response = _K2hr3HttpResponse() + response.code = 204 + self.assertEqual(response._code, response.code) + + def test_k2hr3httpresponse_code_type_is_str(self): + """Checks if setting an invalid code raise an exception.""" + with self.assertRaises(_K2hr3UserAgentError) as cm: + response = _K2hr3HttpResponse() + response.code = '204' + the_exception = cm.exception + self.assertEqual('code should be int, not {}'.format(type('204')), + '{}'.format(the_exception)) + + def test_k2hr3httpresponse_error(self): + """Checks if the error is 'OSO'.""" + response = _K2hr3HttpResponse() + response.error = 'OSO' + self.assertEqual(response._error, response.error) + + def test_k2hr3httpresponse_error_type_is_str(self): + """Checks if setting an invalid error raise an exception.""" + with self.assertRaises(_K2hr3UserAgentError) as cm: + response = _K2hr3HttpResponse() + response.error = 111000111 + the_exception = cm.exception + self.assertEqual('error should be str, not {}'.format(111000111), + '{}'.format(the_exception)) + +# +# EOF +# diff --git a/tests/test_zendpoint.py b/tests/test_zendpoint.py new file mode 100644 index 0000000..b56e13c --- /dev/null +++ b/tests/test_zendpoint.py @@ -0,0 +1,460 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Endpoint class for the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import json +import logging +import os +from pathlib import Path +import unittest +from unittest.mock import MagicMock, patch + +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.endpoint import K2hr3NotificationEndpoint +from k2hr3_osnl.exceptions import K2hr3NotificationEndpointError, _K2hr3UserAgentError +from k2hr3_osnl.useragent import _K2hr3UserAgent + +here = os.path.abspath(os.path.dirname(__file__)) +conf_file_path = Path( + os.sep.join([here, '..', 'etc', 'k2hr3-osnl.conf'])).resolve() + +LOG = logging.getLogger(__name__) +HANDLED = 'handled' +REQUEUE = 'requeue' + + +class TestNotificationEndpoint(unittest.TestCase): + """Tests the NotificationEndpoint class. + + Simple usage(this class only): + $ python -m unittest tests/test_endpoint.py + + Simple usage(all): + $ python -m unittest tests + """ + + def setUp(self): + """Setups TestNotificationEndpoint.""" + + def tearDown(self): + """Tears Down TestNotificationEndpoint.""" + + def test_notification_endpoint_construct(self): + """Creates a NotificationEndpoint instance.""" + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + self.assertIsInstance(endpoint, K2hr3NotificationEndpoint) + + def test_notification_endpoint_construct_conf_is_str(self): + """Checks if the __init__'s conf is not K2hr3Conf object.""" + conf = 'hogehoge' + with self.assertRaises(K2hr3NotificationEndpointError) as cm: + K2hr3NotificationEndpoint(conf) + the_exception = cm.exception + self.assertEqual( + the_exception.msg, 'conf is a K2hr3Conf instance, not {}'.format( + type(conf))) + + def test_notification_endpoint_conf(self): + """Checks if conf is readable.""" + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + self.assertIsInstance(endpoint, K2hr3NotificationEndpoint) + self.assertEqual(endpoint.conf, conf) + + def test_notification_endpoint_readonly(self): + """Checks if conf is readonly.""" + with self.assertRaises(AttributeError) as cm: + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + new_conf = K2hr3Conf(conf_file_path) + endpoint.conf = new_conf + the_exception = cm.exception + self.assertEqual("can't set attribute", '{}'.format(the_exception)) + + def test_neutron_payload_to_params(self): + """Checks if _payload_to_params() works correctly. + + The payload pattern is a neutron notification message. + """ + # Ensure values are as expected at runtime. + self.patcher_call_r3api = patch.object( + K2hr3NotificationEndpoint, + '_K2hr3NotificationEndpoint__call_r3api') + self.mock_method = self.patcher_call_r3api.start() + self.mock_method.return_value = HANDLED + self.addCleanup(self.patcher_call_r3api.stop) + # input --- payload + payload = { + "port": { + "device_id": + "12345678-1234-5678-1234-567812345678", + "fixed_ips": [{ + "ip_address": "127.0.0.1", + }, { + "ip_address": "127.0.0.2", + }] + } + } + # output --- params + expect_params = { + 'cuk': '12345678-1234-5678-1234-567812345678', + 'ips': ['127.0.0.1', '127.0.0.2'] + } + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + self.mock_method.assert_called_once_with(expect_params) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_nova_compute_payload_to_params(self): + """Checks if _payload_to_params() works correctly. + + The payload pattern is a nova compute notification message. + """ + # Ensure values are as expected at runtime. + self.patcher_call_r3api = patch.object( + K2hr3NotificationEndpoint, + '_K2hr3NotificationEndpoint__call_r3api') + self.mock_method = self.patcher_call_r3api.start() + self.mock_method.return_value = HANDLED + self.addCleanup(self.patcher_call_r3api.stop) + # input --- payload + payload = { + "nova_object.data": { + "uuid": "12345678-1234-5678-1234-567812345678" + }, + "nova_object.version": "1.7" + } + # output --- params + expect_params = {'cuk': '12345678-1234-5678-1234-567812345678'} + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + self.mock_method.assert_called_once_with(expect_params) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_compute_payload_to_params(self): + """Checks if _payload_to_params() works correctly. + + The payload pattern is a compute notification message. + """ + # Ensure values are as expected at runtime. + self.patcher_call_r3api = patch.object( + K2hr3NotificationEndpoint, + '_K2hr3NotificationEndpoint__call_r3api') + self.mock_method = self.patcher_call_r3api.start() + self.mock_method.return_value = HANDLED + self.addCleanup(self.patcher_call_r3api.stop) + + # input --- payload + payload = { + "instance_id": "12345678-1234-5678-1234-567812345678", + } + # output --- params + expect_params = {'cuk': '12345678-1234-5678-1234-567812345678'} + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + self.mock_method.assert_called_once_with(expect_params) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_payload_to_params_error_no_instance_id(self): + """Checks if _payload_to_params() works correctly. + + The payload has no instance_id. + """ + # Ensure the __call_r3api is not called. + self.patcher_call_r3api = patch.object( + K2hr3NotificationEndpoint, + '_K2hr3NotificationEndpoint__call_r3api') + self.mock_method = self.patcher_call_r3api.start() + self.mock_method.return_value = HANDLED + self.addCleanup(self.patcher_call_r3api.stop) + + # input --- payload + payload = { + "invalid_named": "12345678-1234-5678-1234-567812345678", + } + # __call_r3api is mocked! no http request will send. + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + self.mock_method.mock_call_r3api.assert_not_called() + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_call_r3api_requeue_on_exception(self): + """Checks if call_r3api works correctly. + + __call_r3api calls the _K2hr3UserAgent::send() to call the R3 API. + We mock _K2hr3UserAgent::send() to throws a _K2hr3UserAgentError. + If requeue_on_error is true, then the function returns HANDLED. + """ + # Expected return_value is REQUEUE in this case. + _K2hr3UserAgent.send = MagicMock( + side_effect=_K2hr3UserAgentError('error')) + + conf = K2hr3Conf(conf_file_path) + conf.k2hr3.requeue_on_error = True # overwrites it True. + endpoint = K2hr3NotificationEndpoint(conf) + json_file_path = Path( + os.sep.join( + [here, '..', 'tools', 'data', + 'notifications_neutron.json'])).resolve() + with json_file_path.open() as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, REQUEUE) + + def test_notification_endpoint_call_r3api_exception_raise(self): + """Checks if call_r3api works correctly. + + __call_r3api calls the _K2hr3UserAgent::send() to call the R3 API. + We mock _K2hr3UserAgent::send() to throws an unknown exception. + Then the function raises the exception again. + K2hr3NotificationEndpoint::info() method will catch all exceptions + then it returns HANDLED to the dispatcher. + """ + _K2hr3UserAgent.send = MagicMock(side_effect=Exception('error')) + + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + json_file_path = Path( + os.sep.join( + [here, '..', 'tools', 'data', + 'notifications_neutron.json'])).resolve() + with json_file_path.open() as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info_r3api_success(self): + """Checks if info works correctly. + + NotificationEndpoint::__call_r3api returns HANDLED in this case. + + NotificationEndpoint::info() + --> NotificationEndpoint::__call_r3api() + --> _K2hr3UserAgent::send() # we mock this method. + """ + # Expected return_value of _K2hr3UserAgent::send() is True + _K2hr3UserAgent.send = MagicMock(return_value=True) + + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + json_file_path = Path( + os.sep.join( + [here, '..', 'tools', 'data', + 'notifications_neutron.json'])).resolve() + with json_file_path.open() as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info_r3api_failed(self): + """Checks if info works correctly. + + NotificationEndpoint::__call_r3api returns HANDLED in this case. + + NotificationEndpoint::info() + --> NotificationEndpoint::__call_r3api() + --> _K2hr3UserAgent::send() # we mock this method. + """ + # Expected return_value of _K2hr3UserAgent::send() is False + _K2hr3UserAgent.send = MagicMock(return_value=False) + + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + json_file_path = Path( + os.sep.join( + [here, '..', 'tools', 'data', + 'notifications_neutron.json'])).resolve() + with json_file_path.open() as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info_r3api_failed_requeue(self): + """Checks if info works correctly. + + NotificationEndpoint::__call_r3api returns REQUEUE in this case. + + NotificationEndpoint::info() + --> NotificationEndpoint::__call_r3api() + --> _K2hr3UserAgent::send() # we mock this method. + """ + # Expected return_value of _K2hr3UserAgent::send() is False + _K2hr3UserAgent.send = MagicMock(return_value=False) + + conf = K2hr3Conf(conf_file_path) + conf.k2hr3.requeue_on_error = True # overwrites it True. + endpoint = K2hr3NotificationEndpoint(conf) + + json_file_path = Path( + os.sep.join( + [here, '..', 'tools', 'data', + 'notifications_neutron.json'])).resolve() + with json_file_path.open() as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + self.assertEqual(result, REQUEUE) + + def test_notification_endpoint_info_r3api_failed_by_exception(self): + """Checks if info works correctly. + + NotificationEndpoint::__call_r3api returns REQUEUE in this case. + + NotificationEndpoint::info() + --> NotificationEndpoint::__call_r3api() + --> _K2hr3UserAgent::send() # we mock this method. + """ + # _K2hr3UserAgent::send() is expected to raise _K2hr3UserAgentError. + _K2hr3UserAgent.send = MagicMock( + side_effect=_K2hr3UserAgentError('send error')) + + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + json_file_path = Path( + os.sep.join( + [here, '..', 'tools', 'data', + 'notifications_neutron.json'])).resolve() + with json_file_path.open() as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + # Ensure the__call_r3api is not called. + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info_requeue_test_on_exception(self): + """Checks if info works correctly. + + NotificationEndpoint::__call_r3api returns REQUEUE in this case. + + NotificationEndpoint::info() + --> NotificationEndpoint::__call_r3api() + --> _K2hr3UserAgent::send() # we mock this method. + """ + # _K2hr3UserAgent::send() is expected to raise _K2hr3UserAgentError. + _K2hr3UserAgent.send = MagicMock( + side_effect=_K2hr3UserAgentError('send error')) + + conf = K2hr3Conf(conf_file_path) + conf.k2hr3.requeue_on_error = True # overwrites it True. + endpoint = K2hr3NotificationEndpoint(conf) + json_file_path = Path( + os.sep.join( + [here, '..', 'tools', 'data', + 'notifications_neutron.json'])).resolve() + with json_file_path.open() as fp: + data = json.load(fp) + result = endpoint.info(data['ctxt'], data['publisher_id'], + data['event_type'], data['payload'], + data['metadata']) + # Ensure the__call_r3api is not called. + self.assertEqual(result, REQUEUE) + + def test_notification_endpoint_info__payload_to_params_exception(self): + """Checks if info works correctly. + + NotificationEndpoint::info returns HANDLED if the _payload_to_params + method throws other than K2hr3NotificationEndpointError excetion. + """ + payload = { + "instance_id": "12345678-1234-5678-1234-567812345678", + } + with patch.object( + K2hr3NotificationEndpoint, + '_payload_to_params', + side_effect=Exception('_payload_to_params error')): + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + + def test_notification_endpoint_info_call_r3api_exception(self): + """Checks if info works correctly. + + NotificationEndpoint::info returns HANDLED if the __call_r3api + method throws other than K2hr3NotificationEndpointError excetion. + """ + payload = { + "instance_id": "12345678-1234-5678-1234-567812345678", + } + with patch.object( + K2hr3NotificationEndpoint, + '_K2hr3NotificationEndpoint__call_r3api', + side_effect=Exception('__call_r3api error')): + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + result = endpoint.info( + context={}, + publisher_id='', + event_type='', + payload=payload, + metadata={}) + # Ensucre the result of info is HANDLED. + self.assertEqual(result, HANDLED) + +# +# EOF +# diff --git a/tests/test_zinit.py b/tests/test_zinit.py new file mode 100644 index 0000000..df4b6f8 --- /dev/null +++ b/tests/test_zinit.py @@ -0,0 +1,347 @@ +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Test init of the oslo_messaging notification message listener.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import sys +import unittest +from pathlib import Path +from os import path, sep +from unittest.mock import patch +import oslo_messaging + +import k2hr3_osnl +from k2hr3_osnl.cfg import K2hr3Conf +from k2hr3_osnl.endpoint import K2hr3NotificationEndpoint +from k2hr3_osnl.exceptions import K2hr3Error + +here = path.abspath(path.dirname(__file__)) +conf_file_path = Path(sep.join([here, '..', 'etc', + 'k2hr3-osnl.conf'])).resolve() + +OLD_ARGV = sys.argv.copy() + + +class TestK2hr3Init(unittest.TestCase): + """Tests the k2hr3_osnl init functions. + + Simple usage(this class only): + $ python -m unittest tests/test_init.py + + Simple usage(all): + $ python -m unittest tests + """ + + def setUp(self): + """Sets up a test case.""" + sys.argv = OLD_ARGV.copy() + + def tearDown(self): + """Tears down a test case.""" + + def test_k2hr3_osnl_version(self): + """Gets the k2hr3_osnl version.""" + self.assertEqual(k2hr3_osnl.version(), k2hr3_osnl.__version__) + + # + # Note: + # We replace the argv to simulate call from command line. + # + # Case1: + # Before + # len(argv) == 2 + # ['python -m unittest', 'tests.test_init'] + # After + # len(argv) == 3 + # ['dummy_main', '-c', '/home/vagrant/work/qa/k2hr3_osnl/etc/k2hr3-osnl.conf'] + # + # Case2: + # Before + # len(argv) == 1 + # [python3 -m unittest] + # After + # len(argv) == 3 + # ['dummy_main', '-c', '/home/vagrant/work/qa/k2hr3_osnl/etc/k2hr3-osnl.conf'] + # + def test_k2hr3_osnl_main(self): + """Executes the k2hr3_osnl main function.""" + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + if len(sys.argv) == 1: + sys.argv.append('-c') + sys.argv.append(str(conf_file_path)) + elif len(sys.argv) == 2: + sys.argv[0] = 'dummy_main' + sys.argv[1] = '-c' + sys.argv.append(str(conf_file_path)) + else: + self.assertTrue(False, 'unknown case') + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + + # Ensure k2hr3_osnl.listen called. + with patch('k2hr3_osnl.listen', return_value=0) as cm1: + with self.assertRaises(SystemExit) as cm2: + k2hr3_osnl.main() + self.assertEqual(cm2.exception.code, 0) + # print(dir(cm1)) + self.assertTrue(cm1.called) + self.assertEqual(cm1.return_value, 0) + + def test_k2hr3_osnl_main_with_l(self): + """Executes the k2hr3_osnl main function.""" + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + if len(sys.argv) == 1: + sys.argv[0] = 'dummy_main' + sys.argv.append('-c') + sys.argv.append(str(conf_file_path)) + sys.argv.append('-l') + sys.argv.append('debug') + elif len(sys.argv) == 2: + sys.argv[0] = 'dummy_main' + sys.argv[1] = '-c' + sys.argv.append(str(conf_file_path)) + sys.argv.append('-l') + sys.argv.append('debug') + else: + self.assertTrue(False, 'unknown case') + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + + # Ensure k2hr3_osnl.listen called. + with patch('k2hr3_osnl.listen', return_value=0) as cm1: + with self.assertRaises(SystemExit) as cm2: + k2hr3_osnl.main() + self.assertEqual(cm2.exception.code, 0) + # print(dir(cm1)) + self.assertTrue(cm1.called) + self.assertEqual(cm1.return_value, 0) + + def test_k2hr3_osnl_main_with_d(self): + """Executes the k2hr3_osnl main function.""" + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + if len(sys.argv) == 1: + sys.argv[0] = 'dummy_main' + sys.argv.append('-c') + sys.argv.append(str(conf_file_path)) + sys.argv.append('-d') + sys.argv.append('critical') + elif len(sys.argv) == 2: + sys.argv[0] = 'dummy_main' + sys.argv[1] = '-c' + sys.argv.append(str(conf_file_path)) + sys.argv.append('-d') + sys.argv.append('critical') + else: + self.assertTrue(False, 'unknown case') + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + + # Ensure k2hr3_osnl.listen called. + with patch('k2hr3_osnl.listen', return_value=0) as cm1: + with self.assertRaises(SystemExit) as cm2: + k2hr3_osnl.main() + self.assertEqual(cm2.exception.code, 0) + # print(dir(cm1)) + self.assertTrue(cm1.called) + self.assertEqual(cm1.return_value, 0) + + def test_k2hr3_osnl_main_with_f(self): + """Executes the k2hr3_osnl main function.""" + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + if len(sys.argv) == 1: + sys.argv[0] = 'dummy_main' + sys.argv.append('-c') + sys.argv.append(str(conf_file_path)) + sys.argv.append('-f') + sys.argv.append('/tmp/k2hr3_osnl.log') + elif len(sys.argv) == 2: + sys.argv[0] = 'dummy_main' + sys.argv[1] = '-c' + sys.argv.append(str(conf_file_path)) + sys.argv.append('-f') + sys.argv.append('/tmp/k2hr3_osnl.log') + else: + self.assertTrue(False, 'unknown case') + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + + # Ensure k2hr3_osnl.listen called. + with patch('k2hr3_osnl.listen', return_value=0) as cm1: + with self.assertRaises(SystemExit) as cm2: + k2hr3_osnl.main() + self.assertEqual(cm2.exception.code, 0) + # print(dir(cm1)) + self.assertTrue(cm1.called) + self.assertEqual(cm1.return_value, 0) + + def test_k2hr3_osnl_main_error(self): + """Executes the k2hr3_osnl main function.""" + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + if len(sys.argv) == 1: + sys.argv[0] = 'dummy_main' + sys.argv.append('-c') + sys.argv.append(str('')) # set empty confi file. + sys.argv.append('-f') + sys.argv.append('/tmp/k2hr3_osnl.log') + elif len(sys.argv) == 2: + sys.argv[0] = 'dummy_main' + sys.argv[1] = '-c' + sys.argv.append(str('')) + sys.argv.append('-f') + sys.argv.append('/tmp/k2hr3_osnl.log') + else: + self.assertTrue(False, 'unknown case') + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + + # Ensure k2hr3_osnl.listen called. + if len(sys.argv) == 1: + with self.assertRaises(SystemExit) as cm: + k2hr3_osnl.main() + elif len(sys.argv) == 2: + with self.assertRaises(RuntimeError) as cm: + k2hr3_osnl.main() + the_exception = cm.exception + self.assertEqual('K2hr3 RuntimeError', '{}'.format(the_exception)) + + def test_k2hr3_osnl_main_k2hr3error(self): + """Executes the k2hr3_osnl main function.""" + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + if len(sys.argv) == 1: + sys.argv[0] = 'dummy_main' + sys.argv.append('-c') + sys.argv.append(str(conf_file_path)) + sys.argv.append('-f') + sys.argv.append('/tmp/k2hr3_osnl.log') + sys.argv.append('-d') + sys.argv.append('critical') + elif len(sys.argv) == 2: + sys.argv[0] = 'dummy_main' + sys.argv[1] = '-c' + sys.argv.append(str(conf_file_path)) + sys.argv.append('-f') + sys.argv.append('/tmp/k2hr3_osnl.log') + sys.argv.append('-d') + sys.argv.append('critical') + else: + self.assertTrue(False, 'unknown case') + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + + # Ensure k2hr3_osnl.listen called. + # Note: + # The error messages will display because the default log level is logging.WARN. + # Raising exceptions, k2hr3_osnl/__init__.py logs messages on logging.ERROR level. + with patch( + 'k2hr3_osnl.listen', + return_value=0, + side_effect=K2hr3Error('A failure proof message. Don\'t worry about this!')): + with self.assertRaises(K2hr3Error) as cm1: + k2hr3_osnl.main() + this_exception = cm1.exception + self.assertEqual('K2hr3 RuntimeError', '{}'.format(this_exception)) + + def test_k2hr3_osnl_main_error_2(self): + """Executes the k2hr3_osnl main function.""" + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + if len(sys.argv) == 1: + sys.argv[0] = 'dummy_main' + sys.argv.append('-c') + sys.argv.append(str(conf_file_path)) + sys.argv.append('-f') + sys.argv.append('/tmp/k2hr3_osnl.log') + elif len(sys.argv) == 2: + sys.argv[0] = 'dummy_main' + sys.argv[1] = '-c' + sys.argv.append(str(conf_file_path)) + sys.argv.append('-f') + sys.argv.append('/tmp/k2hr3_osnl.log') + else: + self.assertTrue(False, 'unknown case') + # print('len = {}'.format(len(sys.argv))) + # print(sys.argv) + + # Ensure k2hr3_osnl.listen called. + # Note: + # The error messages will display because the default log level is logging.WARN. + # Raising exceptions, k2hr3_osnl/__init__.py logs messages on logging.ERROR level. + with patch( + 'k2hr3_osnl.listen', + return_value=0, + side_effect=Exception('A failure proof message. Don\'t worry about this!')) as cm1: + with self.assertRaises(RuntimeError) as cm2: + k2hr3_osnl.main() + the_exception = cm2.exception + self.assertEqual('Unknown RuntimeError', + '{}'.format(the_exception)) + # print(dir(cm1)) + self.assertTrue(cm1.called) + self.assertEqual(cm1.return_value, 0) + + def test_k2hr3_osnl_listen_args_len_is_zero(self): + """Executes the k2hr3_osnl listen function.""" + with self.assertRaises(Exception) as cm: + k2hr3_osnl.listen() + the_exception = cm.exception + self.assertEqual( + "listen() missing 1 required positional argument: 'endpoints'", + '{}'.format(the_exception)) + + def test_k2hr3_osnl_listen_args_type_is_str(self): + """Executes the k2hr3_osnl listen function.""" + result = k2hr3_osnl.listen('invalid') + self.assertEqual(result, 1) + + def test_k2hr3_osnl_listen_args_endpoint_is_invalid(self): + """Executes the k2hr3_osnl listen function.""" + result = k2hr3_osnl.listen(['invalid']) + self.assertEqual(result, 1) + + @unittest.skip( + "function get_notification_listener at 0x7fa9ecad1620> does not have the attribute 'start'" + ) + def test_k2hr3_osnl_listen(self): + """Executes the k2hr3_osnl main function.""" + sys.argv[0] = 'me' + sys.argv[1] = '-c' + sys.argv.append(str(conf_file_path)) + + # Ensure k2hr3_osnl.listen called. + with patch.object( + oslo_messaging.get_notification_listener, + 'start', + return_value=None, + side_effect=Exception('skip listening')): + conf = K2hr3Conf(conf_file_path) + endpoint = K2hr3NotificationEndpoint(conf) + with self.assertRaises(Exception): + k2hr3_osnl.listen(endpoint) + +# +# EOF +# diff --git a/tools/data/notifications_neutron.json b/tools/data/notifications_neutron.json new file mode 100644 index 0000000..6596da3 --- /dev/null +++ b/tools/data/notifications_neutron.json @@ -0,0 +1,85 @@ +{ + "ctxt" : { + "auth_token": "auth_token_string", + "client_timeout": null, + "domain": null, + "global_request_id": null, + "is_admin": true, + "is_admin_project": true, + "project": "project_string", + "project_domain": "default", + "project_id": "project_string", + "project_name": "admin", + "read_only": false, + "request_id": "req-ffffffff-ffff-ffff-ffff-ffffffffffff", + "resource_uuid": null, + "roles": [ + "reader", + "member", + "admin" + ], + "show_deleted": false, + "system_scope": null, + "tenant": "project_string", + "tenant_id": "project_string", + "tenant_name": "admin", + "timestamp": "2018-10-26 23:59:59.999999", + "user": "userffffffffffffffffffffffffffff", + "user_domain": "default", + "user_id": "userffffffffffffffffffffffffffff", + "user_identity": "userffffffffffffffffffffffffffff project_string - default default", + "user_name": "admin" + }, + "publisher_id" : "network.node1.example.com", + "event_type" : "port.delete.end", + "metadata" : { + "message_id": "msgidfff-ffff-ffff-ffff-ffffffffffff", + "timestamp": "2018-10-26 23:59:59.999999" + }, + "payload" : { + "port": { + "admin_state_up": true, + "allowed_address_pairs": [], + "binding:host_id": "node1.example.com", + "binding:profile": {}, + "binding:vif_details": { + "bridge_name": "br-int", + "datapath_type": "system", + "ovs_hybrid_plug": false, + "port_filter": true + }, + "binding:vif_type": "ovs", + "binding:vnic_type": "normal", + "created_at": "2018-10-26T23:59:59Z", + "description": "", + "device_id": "12345678-1234-5678-1234-567812345678", + "device_owner": "compute:nova", + "extra_dhcp_opts": [], + "fixed_ips": [ + { + "ip_address": "172.16.0.1", + "subnet_id": "subnet01-ffff-ffff-ffff-ffffffffffff" + }, + { + "ip_address": "2001:db8::6", + "subnet_id": "subnet02-ffff-ffff-ffff-ffffffffffff" + } + ], + "id": "portidff-ffff-ffff-ffff-ffffffffffff", + "mac_address": "12:34:56:78:9a:bc", + "name": "", + "network_id": "netwrkid-ffff-ffff-ffff-ffffffffffff", + "port_security_enabled": true, + "project_id": "project_string", + "revision_number": 4, + "security_groups": [ + "security-ffff-ffff-ffff-ffffffffffff" + ], + "status": "ACTIVE", + "tags": [], + "tenant_id": "project_string", + "updated_at": "2018-10-26T23:59:59Z" + }, + "port_id": "portidff-ffff-ffff-ffff-ffffffffffff" + } +} diff --git a/tools/data/notifications_nova.json b/tools/data/notifications_nova.json new file mode 100644 index 0000000..ee67254 --- /dev/null +++ b/tools/data/notifications_nova.json @@ -0,0 +1,134 @@ +{ + "ctxt" : { + "auth_token": "auth_token_string", + "client_timeout": null, + "domain": null, + "global_request_id": null, + "is_admin": true, + "is_admin_project": true, + "project": "project_string", + "project_domain": "default", + "project_id": "project_string", + "project_name": "admin", + "quota_class": null, + "read_deleted": "no", + "read_only": false, + "remote_address": "172.16.0.0.1", + "request_id": "req-ffffffff-ffff-ffff-ffff-ffffffffffff", + "resource_uuid": null, + "roles": [ + "reader", + "member", + "admin" + ], + "service_catalog": [ + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1/placement", + "region": "RegionOne" + } + ], + "name": "placement", + "type": "placement" + }, + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1:9696/", + "region": "RegionOne" + } + ], + "name": "neutron", + "type": "network" + }, + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1/volume/v3/project_string", + "region": "RegionOne" + } + ], + "name": "cinderv3", + "type": "volumev3" + }, + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1/image", + "region": "RegionOne" + } + ], + "name": "glance", + "type": "image" + }, + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1/volume/v3/project_string", + "region": "RegionOne" + } + ], + "name": "cinder", + "type": "block-storage" + } + ], + "show_deleted": false, + "system_scope": null, + "tenant": "project_string", + "timestamp": "2018-10-26T23:59:49.739983", + "user": "user_fffffffffffffffffffffffffff", + "user_domain": "default", + "user_id": "user_fffffffffffffffffffffffffff", + "user_identity": "user_fffffffffffffffffffffffffff project_string - default default", + "user_name": "admin" + }, + "publisher_id" : "compute.node1.example.com", + "event_type" : "compute.instance.delete.end", + "metadata" : { + "message_id": "mssageid-ffff-ffff-ffff-ffffffffffff", + "timestamp": "2018-10-26 23:59:56.792042" + }, + "payload" : { + "access_ip_v4": null, + "access_ip_v6": null, + "architecture": null, + "availability_zone": "nova", + "cell_name": "", + "created_at": "2018-10-26 23:59:10+00:00", + "deleted_at": "2018-10-26T23:59:56.000000", + "disk_gb": 0, + "display_name": "coffee1426.example.com", + "ephemeral_gb": 0, + "host": "node1.example.com", + "hostname": "coffee1426.example.com", + "image_meta": { + "base_image_ref": "", + "container_format": "bare", + "disk_format": "qcow2", + "min_disk": "0", + "min_ram": "0" + }, + "image_ref_url": "http://172.16.0.0.1/image/images/", + "instance_flavor_id": "42", + "instance_id": "12345678-1234-5678-1234-567812345678", + "instance_type": "m1.nano", + "instance_type_id": 11, + "kernel_id": "", + "launched_at": "2018-10-26T23:59:22.000000", + "memory_mb": 64, + "metadata": {}, + "node": "node1.example.com", + "os_type": null, + "progress": "", + "ramdisk_id": "", + "reservation_id": "r-ffffffff", + "root_gb": 0, + "state": "deleted", + "state_description": "", + "tenant_id": "project_string", + "terminated_at": "2018-10-26T23:59:56.500934", + "user_id": "user_fffffffffffffffffffffffffff", + "vcpus": 1 + } +} diff --git a/tools/data/versioned_notifications_nova.json b/tools/data/versioned_notifications_nova.json new file mode 100644 index 0000000..12c06fb --- /dev/null +++ b/tools/data/versioned_notifications_nova.json @@ -0,0 +1,167 @@ +{ + "ctxt" : { + "auth_token": "auth_token_string", + "client_timeout": null, + "domain": null, + "global_request_id": null, + "is_admin": true, + "is_admin_project": true, + "project": "project_string", + "project_domain": "default", + "project_id": "project_string", + "project_name": "admin", + "quota_class": null, + "read_deleted": "no", + "read_only": false, + "remote_address": "172.16.0.0.1", + "request_id": "req-ffffffff-ffff-ffff-ffff-ffffffffffff", + "resource_uuid": null, + "roles": [ + "reader", + "member", + "admin" + ], + "service_catalog": [ + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1/placement", + "region": "RegionOne" + } + ], + "name": "placement", + "type": "placement" + }, + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1:9696/", + "region": "RegionOne" + } + ], + "name": "neutron", + "type": "network" + }, + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1/volume/v3/project_string", + "region": "RegionOne" + } + ], + "name": "cinderv3", + "type": "volumev3" + }, + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1/image", + "region": "RegionOne" + } + ], + "name": "glance", + "type": "image" + }, + { + "endpoints": [ + { + "publicURL": "http://172.16.0.0.1/volume/v3/project_string", + "region": "RegionOne" + } + ], + "name": "cinder", + "type": "block-storage" + } + ], + "show_deleted": false, + "system_scope": null, + "tenant": "project_string", + "timestamp": "2018-10-26T23:59:59.999999", + "user": "user_fffffffffffffffffffffffffff", + "user_domain": "default", + "user_id": "user_fffffffffffffffffffffffffff", + "user_identity": "user_fffffffffffffffffffffffffff project_string - default default", + "user_name": "admin" + }, + "publisher_id" : "nova-compute:node1.example.com", + "event_type" : "instance.delete.end", + "metadata" : { + "message_id": "mssageid-ffff-ffff-ffff-ffffffffffff", + "timestamp": "2018-10-26 23:59:59.999999" + }, + "payload" : { + "nova_object.data": { + "action_initiator_project": "project_string", + "action_initiator_user": "user_fffffffffffffffffffffffffff", + "architecture": null, + "auto_disk_config": "AUTO", + "availability_zone": "nova", + "block_devices": [ + { + "nova_object.data": { + "boot_index": 0, + "delete_on_termination": true, + "device_name": "/dev/vda", + "tag": null, + "volume_id": "volumeid-ffff-ffff-ffff-ffffffffffff" + }, + "nova_object.name": "BlockDevicePayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.0" + } + ], + "created_at": "2018-10-26T23:59:99Z", + "deleted_at": "2018-10-26T23:59:45Z", + "display_description": null, + "display_name": "coffee1421.example.com", + "fault": null, + "flavor": { + "nova_object.data": { + "description": null, + "disabled": false, + "ephemeral_gb": 0, + "extra_specs": {}, + "flavorid": "42", + "is_public": true, + "memory_mb": 64, + "name": "m1.nano", + "projects": null, + "root_gb": 0, + "rxtx_factor": 1.0, + "swap": 0, + "vcpu_weight": 0, + "vcpus": 1 + }, + "nova_object.name": "FlavorPayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.4" + }, + "host": "node1.example.com", + "host_name": "coffee1421.example.com", + "image_uuid": "", + "ip_addresses": [], + "kernel_id": "", + "key_name": null, + "launched_at": "2018-10-26T23:59:00Z", + "locked": false, + "metadata": {}, + "node": "node1.example.com", + "os_type": null, + "power_state": "pending", + "progress": 0, + "ramdisk_id": "", + "request_id": "req-ffffffff-ffff-ffff-ffff-ffffffffffff", + "reservation_id": "r-ffffffff", + "state": "deleted", + "task_state": null, + "tenant_id": "project_string", + "terminated_at": "2018-10-26T23:59:44Z", + "updated_at": "2018-10-26T23:59:44Z", + "user_id": "user_fffffffffffffffffffffffffff", + "uuid": "12345678-1234-5678-1234-567812345678" + }, + "nova_object.name": "InstanceActionPayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.7" + } +} diff --git a/tools/k2hr3_osnl_dump.py b/tools/k2hr3_osnl_dump.py new file mode 100755 index 0000000..85283a9 --- /dev/null +++ b/tools/k2hr3_osnl_dump.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# + +# +# Import +# +import sys +import logging as log +from kombu import BrokerConnection +from kombu import Exchange +from kombu import Queue +from kombu.mixins import ConsumerMixin + +# +# Symbols +# +# [NOTE] +# This code is implemented for RabbitMQ on DevStack, then user is +# default for DevStack. +# +EXCHANGE_NAME="neutron" +ROUTING_KEY="notifications.info" +QUEUE_NAME="test_nova_dump_queue" +BROKER_URI="amqp://guest:guest@localhost:5672//" + +log.basicConfig(stream=sys.stdout, level=log.DEBUG) + +class NotificationsDump(ConsumerMixin): + + def __init__(self, connection): + self.connection = connection + return + + def get_consumers(self, consumer, channel): + exchange = Exchange(EXCHANGE_NAME, type="topic", durable=False) + queue = Queue(QUEUE_NAME, exchange, routing_key = ROUTING_KEY, durable=False, auto_delete=True, no_ack=True) + return [ consumer(queue, callbacks = [ self.on_message ]) ] + + def on_message(self, body, message): + log.info('Body: %r' % body) + log.info('---------------') + +if __name__ == "__main__": + log.info("Connecting to broker {}".format(BROKER_URI)) + with BrokerConnection(BROKER_URI) as connection: + NotificationsDump(connection).run() + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/tools/notification_filter.py b/tools/notification_filter.py new file mode 100755 index 0000000..c1c7355 --- /dev/null +++ b/tools/notification_filter.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Main program for testing the oslo_messaging notification message filter.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +__author__ = "Hirotaka Wakabayashi " +__version__ = "1.0.0" + +import argparse +import json +import logging +import sys +import time + +from oslo_config import cfg +import oslo_messaging +from oslo_messaging import NotificationFilter +from oslo_messaging.exceptions import MessageDeliveryFailure + +class NotificationEndpoint(object): + r"""Dumps filtered messages on stdout. + """ + def __init__(self, context=None, publisher_id=None, event_type=None, metadata=None, payload=None): + self.filter_rule = NotificationFilter( + context=context, + publisher_id=publisher_id, + event_type=event_type, + metadata=metadata, + payload=payload) + + @staticmethod + def print_json_dumps(ctxt, publisher_id, event_type, payload, metadata, level): + r"""Dumps messages which is destinated for $topic.warn. + """ + print('--------------------ctxt--------------------------') + print(json.dumps(ctxt, indent=4, sort_keys=True)) + print('--------------------publisher_id--------------------------') + print(publisher_id) + print('--------------------event_type--------------------------') + print(event_type) + print('--------------------metadata--------------------------') + print(json.dumps(metadata, indent=4, sort_keys=True)) + print('--------------------payload--------------------------') + print(json.dumps(payload, indent=4, sort_keys=True)) + print('--------------------%s--------------------------' % level) + + def warn(self, ctxt, publisher_id, event_type, payload, metadata): + r"""Dumps messages which is destinaged for $topic.warn. + """ + self.print_json_dumps(ctxt, publisher_id, event_type, payload, metadata, 'warn') + return oslo_messaging.NotificationResult.HANDLED + + def error(self, ctxt, publisher_id, event_type, payload, metadata): + r"""Dumps messages which is destinaged for $topic.error. + """ + self.print_json_dumps(ctxt, publisher_id, event_type, payload, metadata, 'error') + return oslo_messaging.NotificationResult.HANDLED + + def info(self, ctxt, publisher_id, event_type, payload, metadata): + r"""Dumps messages which is destinaged for $topic.info. + """ + self.print_json_dumps(ctxt, publisher_id, event_type, payload, metadata, 'info') + return oslo_messaging.NotificationResult.HANDLED + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='An oslo.messaging notification listener.') + parser.add_argument('--driver', dest='driver', default='rabbit', help='oslo messaging driver') + parser.add_argument('--user', dest='user', default='guest', help='rabbitmq user') + parser.add_argument('--password', dest='password', default='guest', help='rabbitmq password') + parser.add_argument('--host', dest='host', default='127.0.0.1', help='rabbitmq host') + parser.add_argument('--port', dest='port', default='5672', help='rabbitmq port') + parser.add_argument('--vhost', dest='vhost', default='/', help='rabbitmq vhost') + parser.add_argument('--topic', dest='topic', default='versioned_notifications', help='topic of the destination of messages') + parser.add_argument('--exchange', dest='exchange', default='nova', help='exchange of the destination of messages') + parser.add_argument('--debug', dest='debug', action='store_true', help='debug is true') + parser.add_argument('--pool', dest='pool', default='k2hr3_osnl', help='pool name for listener') + parser.add_argument('--executor', dest='executor', default='threading', help='listener executor') + parser.add_argument('--allow_requeue', dest='allow_requeue', action='store_true', help='requeue a message if failed') + args = parser.parse_args() + + transport_url = '%s://%s:%s@%s:%s%s' % (args.driver, args.user, args.password, args.host, args.port, args.vhost) + transport = oslo_messaging.get_notification_transport(cfg.CONF, url=transport_url) + targets = [ + oslo_messaging.Target(topic=args.topic, exchange=args.exchange) + ] + + # Filters are different by the message topic and the exchange. + endpoints = [] + if args.topic == 'versioned_notifications' and args.exchange == 'nova': + endpoints += [ + NotificationEndpoint(publisher_id='^compute.*$',event_type='^compute\.instance\.delete\.end$') + ] + elif args.topic == 'notifications' and args.exchange == 'nova': + endpoints += [ + NotificationEndpoint(publisher_id='^nova-compute:.*$',event_type='^instance\.delete\.end$'), + ] + elif args.topic == 'notifications' and args.exchange == 'neutron': + endpoints += [ + NotificationEndpoint(publisher_id='^network.*$',event_type='^port\.delete\.end$') + ] + else: + raise NotImplementedError('Unsupported Notification Patterns') + + # dumps all underlying library loggers to stdout now. + logger = logging.getLogger('notification_filter') + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(asctime)-15s %(levelname)s %(name)s %(message)s") + for i in ['stevedore.extension', 'oslo.messaging._drivers.pool', 'oslo.messaging._drivers.impl_rabbit', 'amqp']: + logging.getLogger(i).setLevel(logging.WARN) + + # starts a listener thread. + try: + server = oslo_messaging.get_notification_listener(transport, targets, endpoints, pool=args.pool, executor=args.executor) #, allow_requeue=args.allow_requeue) + logger.info('Go!') + server.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + logger.info('Stopping') + server.stop() + server.wait() + except NotImplementedError: + logger.info('allow_requeue is not supported by driver') + sys.exit(1) + sys.exit(0) + +# +# EOF +# diff --git a/tools/sample_listener.py b/tools/sample_listener.py new file mode 100755 index 0000000..5407c60 --- /dev/null +++ b/tools/sample_listener.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Main program for testing the oslo_messaging notification message filter.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +__author__ = "Hirotaka Wakabayashi " +__version__ = "1.0.0" + +import argparse +import json +import logging +import sys +import time +import ssl +import urllib.parse +from urllib.request import Request, urlopen +from urllib.error import URLError, HTTPError + +from oslo_config import cfg +import oslo_messaging +from oslo_messaging import NotificationFilter + +def print_json_dumps(ctxt, publisher_id, event_type, payload, metadata, level): + r"""Dumps messages which is destinated for $topic.warn. + """ + print('--------------------ctxt--------------------------') + print(json.dumps(ctxt, indent=4, sort_keys=True)) + print('--------------------publisher_id--------------------------') + print(publisher_id) + print('--------------------event_type--------------------------') + print(event_type) + print('--------------------metadata--------------------------') + print(json.dumps(metadata, indent=4, sort_keys=True)) + print('--------------------payload--------------------------') + print(json.dumps(payload, indent=4, sort_keys=True)) + print('--------------------%s--------------------------' % level) + +class SampleEndpoint(object): + r"""Dumps filtered messages on stdout and call k2hr3 api. + """ + def __init__(self, context=None, publisher_id=None, event_type=None, metadata=None, payload=None, r3api_url=None, sss=False, byebye=False): + logging.info('publisher_id=' + publisher_id + ' event_type=' + event_type + ' r3api_url=' + r3api_url) + self.filter_rule = NotificationFilter( + context=context, + publisher_id=publisher_id, + event_type=event_type, + metadata=metadata, + payload=payload) + self.r3api_url = r3api_url + self.sss = sss + self.byebye = byebye + + @staticmethod + def r3api(url, params, timeout, retries): + """Calls the r3api recursively. + + :param url: + Request url. + + :param params: + Request data. + + :param timeout: + Request timeout seconds. The value is passed to urlopen(). + + :param retries: + Retry count which is decremented recursively.　 + + :raise Exception: + Reaches max retry count. + """ + ua_version = 'Python-k2hr3_ua/%d.%d' % sys.version_info[:2] + params['extra'] = 'openstack-auto-v1' # internal extra data. overwrite if exists. + urllib_exception = True + try: + qstring = urllib.parse.urlencode(params, quote_via=urllib.parse.quote) + req = urllib.request.Request('%s?%s' % (url, qstring)) + ctx = None + if req.type == 'https': + # https://docs.python.jp/3/library/ssl.html#ssl.create_default_context + ctx = ssl.create_default_context() + if self.sss: + # https://github.com/python/cpython/blob/master/Lib/ssl.py#L567 + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with urllib.request.urlopen( + req, timeout=self._conf.k2hr3.timeout_seconds, + context=ctx) as res: + req.add_header('User-Agent', ua_version) + req.add_header('Content-Type', 'application/json') + req.add_header('Retry-Count', retries) # for internal metrics + #print('url %s info %s getcode %s body' % (res.geturl(), res.info(), res.getcode(), res.read())) + logger.debug(res.getcode()) + #print('body=%s' % (res.read())) + #print('info=%s' % (res.info().as_string())) + urllib_exception = False + except HTTPError as e: + # https://docs.python.jp/3/library/urllib.error.html + print('Could not complete the request. code %s reason %s headers %s' % (e.code, e.reason, e.headers)) + except URLError as e: + # https://docs.python.jp/3/library/urllib.error.html + print('Could not read the server. reason %s' % (e.reason)) + except urllib.error.ContentTooShortError(msg, content): + # https://docs.python.jp/3/library/urllib.error.html + print('Could not get all contents. msg %s content %s' % (msg, content)) + finally: + if urllib_exception: + logger.error('urllib_exception is true') + retries -= 1 # decrement the retries value. + if retries >= 0: + logger.info('sleeping 1 minute. retries=%d' % retries) + time.sleep(60) # hard coding + SampleEndpoint.r3api(url=url, params=params, timeout=30, retries=retries) + else: + logger.error('reached max retry count. I raise Exception()') + raise Exception('reached max retry count') + + def info(self, ctxt, publisher_id, event_type, payload, metadata): + try: + #print_json_dumps(ctxt, publisher_id, event_type, payload, metadata, __name__) + params = {} + if payload.get('port', None): + if payload['port'].get('device_id', None): + params['cuk'] = payload['port']['device_id'] + if payload['port'].get('fixed_ips', None): + ips = [] + ips.extend(v['ip_address'] for v in payload['port']['fixed_ips'] if v.get('ip_address', None)) + if len(ips) != 0: + params['ips'] = ips + elif payload.get('nova_object.data', None): + if payload['nova_object.data'].get('uuid', None): + params['cuk'] = payload['nova_object.data']['uuid'] + elif payload.get('instance_id', None): + params['cuk'] = payload['instance_id'] + print(json.dumps(params, indent=4, sort_keys=True)) + # r3api + print('calling ' + self.r3api_url) + SampleEndpoint.r3api(url=self.r3api_url, params=params, timeout=30, retries=5) + if self.byebye: + print('Bye-Bye') + os._exit(0) + except: + # we should handle any unknown exceptions to exit this function properly. + logger.error(sys.exc_info()) + finally: + # return HANDLED for avoiding infinite loop. + return oslo_messaging.NotificationResult.HANDLED + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='An oslo.messaging notification listener.') + parser.add_argument('--pattern', choices=['neutron', 'nova-compute', 'compute'], dest='pattern', default='neutron', help='select the test pattern. default is "neutron"') + parser.add_argument('--url', dest='url', default='rabbit://guest:guest@127.0.0.1:5672/', help='select the url. default is "rabbit://guest:guest@127.0.0.1:5672/"') + parser.add_argument('--r3api_url', dest='r3api_url', default='https://127.0.0.1/v1/role', help='select the url. default is "https://127.0.0.1/v1/role"') + parser.add_argument('--sss', dest='sss', default=False, help='accepts Self-Signed SSL certificate') + parser.add_argument('--byebye', dest='byebye', default=False, help='Say byebye after calling r3api') + args = parser.parse_args() + + patterns = { + 'neutron' : { + 'data' : ['^network.*$','^port\.delete\.end$','notifications'], + 'exchange' : 'neutron' + }, + 'nova-compute' : { + 'data' : ['^nova-compute:.*$','^instance\.delete\.end$','versioned_notifications'], + 'exchange' : 'nova' + }, + 'compute' : { + 'data' : ['^compute.*$','^compute\.instance\.delete\.end$','notifications'], + 'exchange' : 'nova' + } + } + + # dumps all underlying library loggers to stdout now. + logger = logging.getLogger('sample_listener') + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(asctime)-15s %(levelname)s %(name)s %(message)s") + for i in ['stevedore.extension', 'oslo.messaging._drivers.pool', 'oslo.messaging._drivers.impl_rabbit', 'amqp']: + logging.getLogger(i).setLevel(logging.WARN) + + tmp = patterns.get(args.pattern, None) + if tmp is None: + logger.error('No such a test pattern exists.') + sys.exit(1) + logger.info('topic=' + tmp['data'][2] + ' exchange=' + tmp['exchange']) + + transport = oslo_messaging.get_notification_transport(cfg.CONF, url=args.url) + targets = [ + oslo_messaging.Target(topic=tmp['data'][2], exchange=tmp['exchange']) + ] + endpoints = [ + SampleEndpoint(publisher_id=tmp['data'][0], event_type=tmp['data'][1], r3api_url=args.r3api_url, sss=args.sss, byebye=args.byebye) + ] + + # starts a listener thread. + try: + listener = oslo_messaging.get_notification_listener(transport, targets, endpoints, pool='sample_listener', executor='threading', allow_requeue=True) + #logger.info('topic=' + data[2] + ' exchange=' + exchange) + listener.start() + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + logger.info('Stopping') + listener.stop() + listener.wait() + except NotImplementedError: + logger.info('allow_requeue is not supported by driver') + sys.exit(1) + sys.exit(0) + +# +# EOF +# diff --git a/tools/sample_notifier.py b/tools/sample_notifier.py new file mode 100755 index 0000000..6827aa5 --- /dev/null +++ b/tools/sample_notifier.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# K2HR3 OpenStack Notification Listener +# +# Copyright 2018 Yahoo! Japan Corporation. +# +# K2HR3 is K2hdkc based Resource and Roles and policy Rules, gathers +# common management information for the cloud. +# K2HR3 can dynamically manage information as "who", "what", "operate". +# These are stored as roles, resources, policies in K2hdkc, and the +# client system can dynamically read and modify these information. +# +# For the full copyright and license information, please view +# the licenses file that was distributed with this source code. +# +# AUTHOR: Hirotaka Wakabayashi +# CREATE: Tue Sep 11 2018 +# REVISION: +# +"""Main program for testing the oslo_messaging notification message filter.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +__author__ = "Hirotaka Wakabayashi " +__version__ = "1.0.0" + +import argparse +import logging +import sys +from datetime import datetime + +from oslo_config import cfg +from oslo_messaging.notify import notifier +from oslo_messaging.exceptions import MessageDeliveryFailure +from oslo_messaging.transport import set_transport_defaults + +import json + +# test data name and index +PUBLISHER_ID = 0 +EVENT_TYPE = 1 +TOPICS = 2 +MESSAGE_FILE = 3 + +def notify_msg(transport,item): + """Opens a message file and calls the notifier.Notifier.info(). + """ + try: + with open(item[MESSAGE_FILE]) as fp: + logger.info('publisher_id=' + item[PUBLISHER_ID] + ' event_type=' + item[EVENT_TYPE] + ' topics=' + ''.join(item[TOPICS])) + data = json.load(fp) + notifier_log = notifier.Notifier(transport, publisher_id=item[PUBLISHER_ID], driver='messagingv2', topics=item[TOPICS], retry=10) + notifier_log.info(ctxt=data['ctxt'], event_type=item[EVENT_TYPE], payload=data['payload']) + return True + except OSError as err: # open raises OSError + logger.error(err) + except MessageDeliveryFailure as err: # info raises MessageDeliveryFailure + logger.error(err) + +if __name__ == "__main__": + # dumps all underlying library loggers to stdout now. + logger = logging.getLogger('sample_notifier') + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format="%(asctime)-15s %(levelname)s %(name)s %(message)s") + + parser = argparse.ArgumentParser(description='An oslo.messaging notifier.') + parser.add_argument('--pattern', choices=['neutron', 'nova-compute', 'compute'], dest='pattern', default='neutron', help='select the test pattern. default is "neutron"') + parser.add_argument('--url', dest='url', default='rabbit://guest:guest@127.0.0.1:5672/', help='select the url. default is "rabbit://guest:guest@127.0.0.1:5672/"') + args = parser.parse_args() + patterns = { + 'neutron' : { + 'data' : ['network.hostname.domain_name','port.delete.end',['notifications'],'data/notifications_neutron.json'], + 'exchange' : 'neutron' + }, + 'nova-compute' : { + 'data' : ['nova-compute:node1.example.com','instance.delete.end',['versioned_notifications'],'data/versioned_notifications_nova.json'], + 'exchange' : 'nova' + }, + 'compute' : { + 'data' : ['compute.node1.example.com','compute.instance.delete.end',['notifications'],'data/notifications_nova.json'], + 'exchange' : 'nova' + } + } + tmp = patterns.get(args.pattern, None) + if tmp is None: + logger.error('No such a test pattern exists.') + sys.exit(1) + logger.info('exchange=' + tmp['exchange'] + ' url = ' + args.url) + + cfg.CONF.control_exchange = tmp['exchange'] # default exchange is openstack. + cfg.CONF.transport_url = args.url + transport = notifier.get_notification_transport(cfg.CONF) + if notify_msg(transport, tmp['data']): + logger.info('success!') + sys.exit(0) + logger.info('error.') + sys.exit(1) + +# +# EOF +#