diff --git a/continuous-delivery/pip-install-with-retry.py b/continuous-delivery/pip-install-with-retry.py new file mode 100644 index 0000000..347e0dc --- /dev/null +++ b/continuous-delivery/pip-install-with-retry.py @@ -0,0 +1,39 @@ +import time +import sys +import subprocess + +DOCS = """Given cmdline args, executes: python3 -m pip install [args...] +Keeps retrying until the new version becomes available in pypi (or we time out)""" +if len(sys.argv) < 2: + sys.exit(DOCS) + +RETRY_INTERVAL_SECS = 10 +GIVE_UP_AFTER_SECS = 60 * 15 + +pip_install_args = [sys.executable, '-m', 'pip', 'install'] + sys.argv[1:] + +start_time = time.time() +while True: + print(subprocess.list2cmdline(pip_install_args)) + result = subprocess.run(pip_install_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + stdout = result.stdout.decode().strip() + if stdout: + print(stdout) + + if result.returncode == 0: + # success + sys.exit(0) + + if "could not find a version" in stdout.lower(): + elapsed_secs = time.time() - start_time + if elapsed_secs < GIVE_UP_AFTER_SECS: + # try again + print("Retrying in", RETRY_INTERVAL_SECS, "secs...") + time.sleep(RETRY_INTERVAL_SECS) + continue + else: + print("Giving up on retries after", int(elapsed_secs), "total secs.") + + # fail + sys.exit(result.returncode) diff --git a/continuous-delivery/publish_to_prod_pypi.yml b/continuous-delivery/publish_to_prod_pypi.yml new file mode 100644 index 0000000..d911029 --- /dev/null +++ b/continuous-delivery/publish_to_prod_pypi.yml @@ -0,0 +1,25 @@ +version: 0.2 +# this image assumes Ubuntu 14.04 base image +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - export PATH=$PATH:$HOME/.local/bin + - python3 -m pip install --user --upgrade pip + - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six + pre_build: + commands: + - cd aws-iot-device-sdk-python + - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc + - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) + - echo "Updating package version to ${PKG_VERSION}" + - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py + build: + commands: + - echo Build started on `date` + - python3 setup.py sdist bdist_wheel --universal + - python3 -m twine upload -r pypi dist/* + post_build: + commands: + - echo Build completed on `date` diff --git a/continuous-delivery/publish_to_test_pypi.yml b/continuous-delivery/publish_to_test_pypi.yml new file mode 100644 index 0000000..49a69c9 --- /dev/null +++ b/continuous-delivery/publish_to_test_pypi.yml @@ -0,0 +1,25 @@ +version: 0.2 +# this image assumes Ubuntu 14.04 base image +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - export PATH=$PATH:$HOME/.local/bin + - python3 -m pip install --user --upgrade pip + - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six + pre_build: + commands: + - pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc + - cd aws-iot-device-sdk-python + - export PKG_VERSION=$(git describe --tags | cut -f2 -dv) + - echo "Updating package version to ${PKG_VERSION}" + - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py + build: + commands: + - echo Build started on `date` + - python3 setup_test.py sdist bdist_wheel --universal + - python3 -m twine upload -r testpypi dist/* --verbose + post_build: + commands: + - echo Build completed on `date` diff --git a/continuous-delivery/test_prod_pypi.yml b/continuous-delivery/test_prod_pypi.yml new file mode 100644 index 0000000..4575306 --- /dev/null +++ b/continuous-delivery/test_prod_pypi.yml @@ -0,0 +1,28 @@ +version: 0.2 +# this image assumes Ubuntu 14.04 base image +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - python3 -m pip install --upgrade pip + - python3 -m pip install --upgrade setuptools + + pre_build: + commands: + - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem + - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem + - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem + - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') + build: + commands: + - echo Build started on `date` + - cd aws-iot-device-sdk-python + - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) + - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user AWSIoTPythonSDK==$CURRENT_TAG_VERSION + - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only + + post_build: + commands: + - echo Build completed on `date` + diff --git a/continuous-delivery/test_test_pypi.yml b/continuous-delivery/test_test_pypi.yml new file mode 100644 index 0000000..c3aa47d --- /dev/null +++ b/continuous-delivery/test_test_pypi.yml @@ -0,0 +1,30 @@ +version: 0.2 +# this image assumes Ubuntu 14.04 base image +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - python3 -m pip install --upgrade pip + - python3 -m pip install --upgrade setuptools + + pre_build: + commands: + - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem + - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem + - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem + - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') + build: + commands: + - echo Build started on `date` + - cd aws-iot-device-sdk-python + - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) + # this is here because typing isn't in testpypi, so pull it from prod instead + - python3 -m pip install typing + - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION + - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only + + post_build: + commands: + - echo Build completed on `date` + diff --git a/continuous-delivery/test_version_exists b/continuous-delivery/test_version_exists new file mode 100644 index 0000000..3579dbc --- /dev/null +++ b/continuous-delivery/test_version_exists @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e +set -x +# force a failure if there's no tag +git describe --tags +# now get the tag +CURRENT_TAG=$(git describe --tags | cut -f2 -dv) +# convert v0.2.12-2-g50254a9 to 0.2.12 +CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv) +# if there's a hash on the tag, then this is not a release tagged commit +if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then + echo "Current tag version is not a release tag, cut a new release if you want to publish." + exit 1 +fi + +if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then + echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version." + exit 1 +fi + +echo "$CURRENT_TAG_VERSION currently does not exist in pypi, allowing pipeline to continue." +exit 0 diff --git a/continuous-delivery/test_version_exists.yml b/continuous-delivery/test_version_exists.yml new file mode 100644 index 0000000..2704ba7 --- /dev/null +++ b/continuous-delivery/test_version_exists.yml @@ -0,0 +1,21 @@ +version: 0.2 +#this build spec assumes the ubuntu 14.04 trusty image +#this build run simply verifies we haven't published something at this tag yet. +#if we have we fail the build and stop the pipeline, if we haven't we allow the pipeline to run. +phases: + install: + commands: + - sudo apt-get update -y + - sudo apt-get install python3 python3-pip -y + - pip3 install --upgrade setuptools + pre_build: + commands: + - echo Build start on `date` + build: + commands: + - cd aws-iot-device-sdk-python + - bash ./continuous-delivery/test_version_exists + post_build: + commands: + - echo Build completed on `date` + diff --git a/samples/greengrass/basicDiscovery.py b/samples/greengrass/basicDiscovery.py index cd73e43..a6fcd61 100644 --- a/samples/greengrass/basicDiscovery.py +++ b/samples/greengrass/basicDiscovery.py @@ -47,6 +47,8 @@ def customOnMessage(message): help="Operation modes: %s"%str(AllowedActions)) parser.add_argument("-M", "--message", action="store", dest="message", default="Hello World!", help="Message to publish") +#--print_discover_resp_only used for delopyment testing. The test run will return 0 as long as the SDK installed correctly. +parser.add_argument("-p", "--print_discover_resp_only", action="store_true", dest="print_only", default=False) args = parser.parse_args() host = args.host @@ -56,6 +58,7 @@ def customOnMessage(message): clientId = args.thingName thingName = args.thingName topic = args.topic +print_only = args.print_only if args.mode not in AllowedActions: parser.error("Unknown --mode option %s. Must be one of %s" % (args.mode, str(AllowedActions))) @@ -94,7 +97,7 @@ def customOnMessage(message): discoveryInfoProvider.configureCredentials(rootCAPath, certificatePath, privateKeyPath) discoveryInfoProvider.configureTimeout(10) # 10 sec -retryCount = MAX_DISCOVERY_RETRIES +retryCount = MAX_DISCOVERY_RETRIES if not print_only else 1 discovered = False groupCA = None coreInfo = None @@ -136,6 +139,9 @@ def customOnMessage(message): backOffCore.backOff() if not discovered: + # With print_discover_resp_only flag, we only woud like to check if the API get called correctly. + if print_only: + sys.exit(0) print("Discovery failed after %d retries. Exiting...\n" % (MAX_DISCOVERY_RETRIES)) sys.exit(-1) diff --git a/setup_test.py b/setup_test.py new file mode 100644 index 0000000..820d715 --- /dev/null +++ b/setup_test.py @@ -0,0 +1,38 @@ +# For test deployment with package AWSIoTPythonSDK. The package name has already taken. Therefore we used an +# alternative name for test pypi. +# prod_pypi : AWSIoTPythonSDK +# test_pypi : AWSIoTPythonSDK-V1 +import sys +sys.path.insert(0, 'AWSIoTPythonSDK') +import AWSIoTPythonSDK +currentVersion = AWSIoTPythonSDK.__version__ + +from distutils.core import setup +setup( + name = 'AWSIoTPythonSDK-V1', + packages=['AWSIoTPythonSDK', 'AWSIoTPythonSDK.core', + 'AWSIoTPythonSDK.core.util', 'AWSIoTPythonSDK.core.shadow', 'AWSIoTPythonSDK.core.protocol', + 'AWSIoTPythonSDK.core.jobs', + 'AWSIoTPythonSDK.core.protocol.paho', 'AWSIoTPythonSDK.core.protocol.internal', + 'AWSIoTPythonSDK.core.protocol.connection', 'AWSIoTPythonSDK.core.greengrass', + 'AWSIoTPythonSDK.core.greengrass.discovery', 'AWSIoTPythonSDK.exception'], + version = currentVersion, + description = 'SDK for connecting to AWS IoT using Python.', + author = 'Amazon Web Service', + author_email = '', + url = 'https://github.com/aws/aws-iot-device-sdk-python.git', + download_url = 'https://s3.amazonaws.com/aws-iot-device-sdk-python/aws-iot-device-sdk-python-latest.zip', + keywords = ['aws', 'iot', 'mqtt'], + classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5" + ] +)