Merge branch 'feature/docker-secrets' of github.com:andrasmaroy/pconf

.travis.yml

@@ -1,29 +1,27 @@
 language: python
-matrix:
-    include:
-        - python: 2.7
-          dist: trusty
-          sudo: false
-        - python: 3.4
-          dist: trusty
-          sudo: false
-        - python: 3.5
-          dist: trusty
-          sudo: false
-        - python: 3.6
-          dist: trusty
-          sudo: false
-        - python: 3.7
-          dist: xenial
-          sudo: true
+jobs:
+  include:
+    - python: 2.7
+      env: BLACK=false
+    - python: 3.4
+      env: BLACK=false
+    - python: 3.5
+      env: BLACK=false
+    - python: 3.6
+      env: BLACK=true
+    - python: 3.7
+      env: BLACK=true
+    - python: 3.8
+      env: BLACK=true
 # command to install dependencies
 install:
   - python setup.py -q install
   - pip install -U -r requirements-dev.txt
 # command to run tests
 script:
   - py.test --cov=./pconf/ ./tests/
-  - flake8 --max-line-length=140 pconf tests
+  - flake8 --max-line-length=88 pconf tests
+  - if [ $BLACK = true ]; then black --check pconf tests; fi
 # submit coverage
 after_success: codecov
 deploy:

README.md

@@ -7,6 +7,7 @@
 [![Py Versions](https://img.shields.io/pypi/pyversions/pconf.svg?style=flat)](https://pypi.python.org/pypi/pconf)
 [![Downloads](https://pepy.tech/badge/pconf)](https://pepy.tech/project/pconf)
 [![Known Vulnerabilities](https://img.shields.io/snyk/vulnerabilities/github/andrasmaroy/pconf/requirements.txt.svg)](https://snyk.io/test/github/andrasmaroy/pconf?targetFile=requirements.txt)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=flat)](https://github.com/psf/black)
 
 Hierarchical python configuration with files, environment variables, command-line arguments.
 
@@ -136,6 +137,9 @@ Pconf.env(to_lower=True)
 # Convert all underscores in the name to dashes, this takes place after separation via the separator option.
 Pconf.env(convert_underscores=True)
 
+# Handle docker secrets, described in detail below
+Pconf.env(docker_secrets=['MYSQL_ROOT_PASSWORD_FILE', 'MYSQL_PASSWORD_FILE']
+
 # Use all at once
 Pconf.env(separator='__',
           match='whatever_matches_this_will_be_whitelisted',
@@ -144,6 +148,26 @@ Pconf.env(separator='__',
           to_lower=True,
           convert_underscores=True)
 ```
+#### Docker secret handling
+
+As described in https://docs.docker.com/v17.12/engine/swarm/secrets/#build-support-for-docker-secrets-into-your-images, when using the `docker_secrets` parameter as above, given the following:
+* A docker secret called `db_root_password` containing the value `secret-password`
+* The secret passed to the environment in which Pconf is running
+* The `MYSQL_ROOT_PASSWORD_FILE` environment variable set to `/run/secrets/db_root_password`
+The result of
+```python
+Pconf.env(docker_secrets=['MYSQL_ROOT_PASSWORD_FILE'])
+Pconf.get()
+```
+will be
+```python
+{'MYSQL_ROOT_PASSWORD': 'secret-password'}
+```
+Notice that the environment name passed was stripped from the `_FILE` postfix.
+
+**Note:** this behaviour is not tied specifically to docker secrets, it will read files specified in the variable without regard to its path, will simply drop the value if the file is not found or the variable name does not end with `_FILE`.
+
+Combining this with the other parameters: the files are read **after** regex matching and whitelisting, but **before** parsing and conversions.
 
 ### File
 Responsible for loading values parsed from a given file into the configuration hierarchy. If the file does not exist the result will be empty and no error is thrown.

pconf/pconf.py

@@ -15,13 +15,10 @@ class Pconf(object):
 
     After sources are set the values can be accessed by calling get()
     """
+
     __hierarchy = []
 
-    merger = Merger(
-        [(dict, ["merge"])],
-        ["override"],
-        ["override"]
-    )
+    merger = Merger([(dict, ["merge"])], ["override"], ["override"])
 
     @classmethod
     def get(cls):
@@ -83,7 +80,16 @@ def argv(cls, name, short_name=None, type=None, help=None):
         cls.__hierarchy.append(argv.Argv(name, short_name, type, help))
 
     @classmethod
-    def env(cls, separator=None, match=None, whitelist=None, parse_values=None, to_lower=None, convert_underscores=None):
+    def env(
+        cls,
+        separator=None,
+        match=None,
+        whitelist=None,
+        parse_values=None,
+        to_lower=None,
+        convert_underscores=None,
+        docker_secrets=None,
+    ):
         """Set environment variables as a source.
 
         By default all environment variables available to the process are used.
@@ -100,8 +106,21 @@ def env(cls, separator=None, match=None, whitelist=None, parse_values=None, to_l
             to_lower: Convert all variable names to lower case.
             convert_underscores: Convert all underscores in the name to dashes,
                 this takes place after separation via the separator option.
+            docker_secrets: A list of names with _FILE postfix, which will have
+                their postfix removed and the content of the file pointed by
+                their original value.
         """
-        cls.__hierarchy.append(env.Env(separator, match, whitelist, parse_values, to_lower, convert_underscores))
+        cls.__hierarchy.append(
+            env.Env(
+                separator,
+                match,
+                whitelist,
+                parse_values,
+                to_lower,
+                convert_underscores,
+                docker_secrets,
+            )
+        )
 
     @classmethod
     def file(cls, path, encoding=None, parser=None):

pconf/store/argv.py

@@ -6,7 +6,7 @@ class Argv(object):
     parser = None
 
     def __init__(self, name, short_name=None, type=None, help=None):
-        if not name.startswith('-') or name.count(' ') != 0:
+        if not name.startswith("-") or name.count(" ") != 0:
             raise ValueError
 
         if Argv.parser is None:
@@ -15,22 +15,24 @@ def __init__(self, name, short_name=None, type=None, help=None):
         self.results = {}
 
         args = {}
-        args['help'] = help
+        args["help"] = help
         if type == bool:
-            args['action'] = 'store_true'
+            args["action"] = "store_true"
         # types supported by literal_eval
         elif type in [dict, list, tuple]:
-            args['type'] = literal_eval
-        elif type == 'repeated_list':
-            args['action'] = 'append'
+            args["type"] = literal_eval
+        elif type == "repeated_list":
+            args["action"] = "append"
         else:
-            args['type'] = type
+            args["type"] = type
 
-        if name.lstrip('-').count('-') != 0:
-            args['dest'] = name.lstrip('-')
+        if name.lstrip("-").count("-") != 0:
+            args["dest"] = name.lstrip("-")
 
         if short_name is not None:
-            Argv.parser.add_argument(name, short_name, default=argparse.SUPPRESS, **args)
+            Argv.parser.add_argument(
+                name, short_name, default=argparse.SUPPRESS, **args
+            )
         else:
             Argv.parser.add_argument(name, default=argparse.SUPPRESS, **args)
 

pconf/store/env.py

@@ -1,17 +1,28 @@
 import os
 import re
-from six import iteritems
 from ast import literal_eval
+from six import iteritems
+from warnings import warn
 
 
 class Env(object):
-    def __init__(self, separator=None, match=None, whitelist=None, parse_values=False, to_lower=False, convert_underscores=False):
+    def __init__(
+        self,
+        separator=None,
+        match=None,
+        whitelist=None,
+        parse_values=False,
+        to_lower=False,
+        convert_underscores=False,
+        docker_secrets=None,
+    ):
         self.separator = separator
         self.match = match
         self.whitelist = whitelist
         self.parse_values = parse_values
         self.to_lower = to_lower
         self.convert_underscores = convert_underscores
+        self.docker_secrets = docker_secrets
 
         if self.match is not None:
             self.re = re.compile(self.match)
@@ -75,22 +86,34 @@ def __merge_split(self, split, env_vars):
     def __try_parse(self, env_vars):
         for key, value in iteritems(env_vars):
             try:
-                if value.lower() == 'true':
+                if value.lower() == "true":
                     env_vars[key] = True
-                elif value.lower() == 'false':
+                elif value.lower() == "false":
                     env_vars[key] = False
                 else:
                     env_vars[key] = literal_eval(value)
             except (ValueError, SyntaxError):
                 pass
 
+    def __handle_docker_secret(self, key, value):
+        postfix = "_FILE"
+        if key.endswith(postfix):
+            try:
+                with open(value, "r") as f:
+                    self.vars[key[0 : -len(postfix)]] = f.read().strip()  # noqa: E203
+            except IOError:
+                warn("IOError when opening {}".format(value), UserWarning)
+
     def __gather_vars(self):
         self.vars = {}
         env_vars = os.environ
 
         for key in env_vars.keys():
             if self.__valid_key(key):
-                self.vars[key] = env_vars[key]
+                if self.docker_secrets is not None and key in self.docker_secrets:
+                    self.__handle_docker_secret(key, env_vars[key])
+                else:
+                    self.vars[key] = env_vars[key]
 
         if self.parse_values:
             self.__try_parse(self.vars)
@@ -102,7 +125,7 @@ def __to_lower(self, key):
         return key.lower()
 
     def __convert_underscores(self, key):
-        return key.replace('_', '-')
+        return key.replace("_", "-")
 
     def __change_keys(self, env_vars, operation):
         new_dict = {}

pconf/store/file.py

@@ -5,7 +5,8 @@
 import yaml
 
 from sys import version_info
-if (version_info.major < 3):
+
+if version_info.major < 3:
     import ConfigParser
     from StringIO import StringIO
 else:
@@ -14,27 +15,27 @@
 
 def parse_ini(content):
     config = ConfigParser.ConfigParser(allow_no_value=True)
-    if (version_info.major < 3):
+    if version_info.major < 3:
         config.readfp(StringIO(content))
     else:
         config.read_string(content)
     if len(config.sections()) == 0:
-        return dict(config.items('DEFAULT'))
+        return dict(config.items("DEFAULT"))
     result = {}
     for section in config.sections():
         result[section] = dict(config.items(section))
     return result
 
 
-class File():
+class File:
     ENCODINGS = {
-        'ini': parse_ini,
-        'json': json.loads,
-        'raw': literal_eval,
-        'yaml': yaml.safe_load
+        "ini": parse_ini,
+        "json": json.loads,
+        "raw": literal_eval,
+        "yaml": yaml.safe_load,
     }
 
-    def __init__(self, path, encoding='raw', parser=None):
+    def __init__(self, path, encoding="raw", parser=None):
         self.__read_file(path)
         self.__set_encoding(encoding, parser)
         self.__parse_content()
@@ -45,10 +46,10 @@ def get(self):
 
     def __read_file(self, path):
         try:
-            with open(path, 'r') as f:
+            with open(path, "r") as f:
                 self.content = f.read()
         except IOError:
-            warn('IOError when opening {}'.format(path), UserWarning)
+            warn("IOError when opening {}".format(path), UserWarning)
             self.content = {}
 
     def __set_encoding(self, encoding, parser=None):

requirements-dev.txt

@@ -1,5 +1,6 @@
-codecov==2.0.15
-flake8==3.7.9
-mock==3.0.5
-pytest # pyup: ignore
-pytest-cov # pyup: ignore
+black; python_version > '3.5'
+codecov
+flake8
+mock
+pytest
+pytest-cov

setup.py

@@ -12,7 +12,7 @@ def read(fname):
 
 setup(
     name="pconf",
-    version="1.6.2",
+    version="1.7.0",
     author="Andras Maroy",
     author_email="[email protected]",
     description=("Hierarchical python configuration with files, environment variables, command-line arguments."),
@@ -32,7 +32,8 @@ def read(fname):
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7'
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8'
     ],
     install_requires=['pyyaml', 'six', 'deepmerge'],
     extras_require={

tests/integration/example_secret

@@ -0,0 +1 @@
+secret

tests/integration/integration_base.py

@@ -8,14 +8,15 @@ class IntegrationBase(TestCase):
     def setUp(self):
         Pconf.clear()
         IntegrationBase.result = {
-                'tuple': (123, 'string'),
-                'int': 123,
-                'float': 1.23,
-                'list': ['list1', 'list2', {'dict-in-list': 'value'}],
-                'complex': (1+2j),
-                'bool': True,
-                'key': 'value',
-                'boolstring': 'false',
-                'string-with-specials': 'Test!@#$%^&*()-_=+[]{};:,<.>/?\\\'"`~',
-                'dict': {'dict': 'value', 'list-in-dict': ['nested-list1', 'nested-list2']}
-                }
+            "tuple": (123, "string"),
+            "int": 123,
+            "float": 1.23,
+            "list": ["list1", "list2", {"dict-in-list": "value"}],
+            "complex": (1 + 2j),
+            "bool": True,
+            "key": "value",
+            "boolstring": "false",
+            "string-with-specials": "Test!@#$%^&*()-_=+[]{};:,<.>/?\\'\"`~",
+            "dict": {"dict": "value", "list-in-dict": ["nested-list1", "nested-list2"]},
+            "secret": "secret",
+        }