From 9dd0a86e6d14593afb546bb3ddb0c8445e110825 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Wed, 4 May 2022 17:28:19 +0200 Subject: [PATCH 1/3] Add in-toto layout creation script Add a script to create two basic in-toto layouts, one for each build target (wheel + sdist) to verify their supply chains independently. See usage description in document header. Signed-off-by: Lukas Puehringer --- .gitignore | 4 ++ .in_toto/create_layout.py | 79 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 .in_toto/create_layout.py diff --git a/.gitignore b/.gitignore index ff032a6f68..8352ac824f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,10 @@ tests/htmlcov/ .pre-commit-config.yaml .vscode +# Ignore in-toto metadata +.in_toto/* +!.in_toto/create_layout.py + # Debian generated files debian/.debhelper/ debian/*-stamp diff --git a/.in_toto/create_layout.py b/.in_toto/create_layout.py new file mode 100644 index 0000000000..3a52f585df --- /dev/null +++ b/.in_toto/create_layout.py @@ -0,0 +1,79 @@ +"""Script to generate two generic in-toto layouts to verify wheel & sdist independently. + +The layouts define two steps and an inspection: +- step 1 (tag): used as initial reference for the source code +- step 2 (build): requires its inputs to match the tagged sources +- inspect: requires the actual final product available to the verifier, i.e. sdist or + wheel, to match the outputs of the build step (this is, where the two layouts differ). + +In addition the layouts define, which keys are authorized to provide attestations, and +how many are required: +- tag: any one maintainer +- build: at least two of maintainers and online build job + + +Usage: + # Create signing key pair for CD ('filepath' must be CD_KEY_PATH defined below) + python -c 'import securesystemslib.interface as i;\ + i.generate_and_write_ed25519_keypair_with_prompt(filepath="cd_key")' + + # Create unsigned layout files 'wheel.layout' and 'sdist.layout' + python create_layout.py + + # Sign layout with maintainer key + in-toto-sign --gpg -f wheel.layout + in-toto-sign --gpg -f sdist.layout + +""" +from in_toto.models.layout import Inspection, Layout, Step +from in_toto.models.metadata import Metablock +from securesystemslib.interface import import_ed25519_publickey_from_file + +MAINTAINER_KEYIDS = [ + "e9c059ec0d3264fab35f94ad465bf9f6f8eb475a", # Justin Cappos + "1343c98fab84859fe5ec9e370527d8a37f521a2f", # Jussi Kukkonen + "f3ff39b659ed00e877084a18b4934539a71e38cd", # Trishank Karthik Kuppusamy + "08f3409fcf71d87e30fbd3c21671f65cb74832a4", # Joshua Lock + "8ba69b87d43be294f23e812089a2ad3c07d962e8", # Lukas Puehringer +] +CD_KEY_PATH = "cd_key" +CD_KEY = import_ed25519_publickey_from_file(f"{CD_KEY_PATH}.pub") + +for build in ["sdist", "wheel"]: + layout = Layout() + # FIXME: What is a good expiration period? + layout.set_relative_expiration(months=12) + + # Add public keys for verifying in-toto attestion signatures to layout + # Requires 'MAINTAINER_KEYIDS' in your local keychain + layout.add_functionary_key(CD_KEY) + layout.add_functionary_keys_from_gpg_keyids(MAINTAINER_KEYIDS) + + # Define tag step, used as initial reference, to be signed by any maintainer. + tag_step = Step(name="tag") + tag_step.pubkeys = MAINTAINER_KEYIDS + tag_step.threshold = 1 + + # Define build step and require materials to match the sources recorded in tag step. + # Moreover, a threshold of 2 requires there to be at least 2 agreeing build + # attestations, e.g. from cd and from a maintainer. + build_step = Step(name=build) + build_step.pubkeys = [CD_KEY["keyid"]] + MAINTAINER_KEYIDS + build_step.threshold = 2 + build_step.add_material_rule_from_string("MATCH * WITH MATERIALS FROM tag") + build_step.add_material_rule_from_string("DISALLOW *") + + # Define inspection and require the actual final product available to the verifier, + # i.e. sdist or wheel, to match the product recorded by the build step. + # (see in-toto/docs#27 for a discussion about dummy inspections) + dummy_inspection = Inspection(name="final-product") + dummy_inspection.set_run_from_string("true") + dummy_inspection.add_material_rule_from_string( + f"MATCH * WITH PRODUCTS IN dist FROM {build}" + ) + dummy_inspection.add_material_rule_from_string("DISALLOW *") + + layout.steps = [tag_step, build_step] + layout.inspect = [dummy_inspection] + metablock = Metablock(signed=layout) + metablock.dump(f"{build}.layout") From 9fc615460b811273aad39c6d67b70ab908ebeeb6 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Wed, 4 May 2022 17:14:22 +0200 Subject: [PATCH 2/3] Generate in-toto build attestations in cd Use in-toto cli to generate signed attestations for each build target (sdist and wheel) and publish on the GH release page along with the build artifacts. Signed-off-by: Lukas Puehringer --- .github/workflows/cd.yml | 53 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index ccc67798ad..d1615951c9 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -31,10 +31,55 @@ jobs: python-version: '3.x' - name: Install build dependency - run: python3 -m pip install --upgrade pip build + run: python3 -m pip install --upgrade pip build in-toto[pynacl] - name: Build binary wheel and source tarball - run: python3 -m build --sdist --wheel --outdir dist/ . + env: + IN_TOTO_KEY: ${{ secrets.IN_TOTO_KEY }} + IN_TOTO_KEY_PW: ${{ secrets.IN_TOTO_KEY_PW }} + run: | + ####################################################### + # Build and generate signed attestions with in-toto CLI + + # Make signing key available to in-toto commands + echo -n "$IN_TOTO_KEY" > .in_toto/key + + # Define patterns for files that need not be recorded as materials below + exclude=('__pycache__' 'build' 'htmlcov' '.?*' '*~' '*.egg-info' '*.pyc') + + # Grab TUF version to construct build artifact names for product recording + version=$(python3 -c 'import tuf; print(tuf.__version__)') + + # Build sdist and record all files in CWD as materials and the build artifact + # as product in a signed attestation 'sdist..link'. + in-toto-run \ + --step-name sdist \ + --key .in_toto/key \ + --key-type ed25519 \ + --password "$IN_TOTO_KEY_PW" \ + --materials . \ + --products dist/tuf-${version}.tar.gz \ + --exclude ${exclude[@]} \ + --metadata-directory .in_toto \ + --verbose \ + -- python3 -m build --sdist --outdir dist/ . + + # Build wheel and record all files in CWD as materials and the build artifact + # as product in a signed attestation 'wheel..link'. + in-toto-run \ + --step-name wheel \ + --key .in_toto/key \ + --key-type ed25519 \ + --password "$IN_TOTO_KEY_PW" \ + --materials . \ + --products dist/tuf-${version}-py3-none-any.whl \ + --exclude ${exclude[@]} dist/tuf-${version}.tar.gz \ + --metadata-directory .in_toto \ + --verbose \ + -- python3 -m build --wheel --outdir dist/ . + + # Remove signing key file + rm .in_toto/key - id: gh-release name: Publish GitHub release candiate @@ -43,7 +88,9 @@ jobs: name: ${{ github.ref_name }}-rc tag_name: ${{ github.ref }} body: "Release waiting for review..." - files: dist/* + files: | + dist/* + .in_toto/*.link - name: Store build artifacts uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 From 017031b761fe08e6e037c89fad7e6c42ab5d1eb0 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Wed, 4 May 2022 17:29:50 +0200 Subject: [PATCH 3/3] Add 'release with in-toto' documentation Add supplementatry documentation for RELEASE.md that describes how to create local maintainer attestations for 'tag' and 'build' steps of the release process, and how to verify them together with attestations from the online CD build job against an in-toto supply chain layout. Signed-off-by: Lukas Puehringer --- docs/RELEASE_with_in-toto.md | 162 +++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 docs/RELEASE_with_in-toto.md diff --git a/docs/RELEASE_with_in-toto.md b/docs/RELEASE_with_in-toto.md new file mode 100644 index 0000000000..aeda547d0e --- /dev/null +++ b/docs/RELEASE_with_in-toto.md @@ -0,0 +1,162 @@ +# Release with in-toto attestations + +This document describes how to create local maintainer attestations for the 'tag' and +'build' steps of the release process, and how to verify them together with attestations +from the online CD build job against an in-toto supply chain layout. + +The instructions are based on RELEASE.md and require the GitHub release environment to +be configured as described. You can follow below instructions in addition to those in +RELEASE.md, except that `git tag ...` must be called with `in-toto-run` as described +below. + +**Prerequisites (one-time setup)** +- Install `in-toto` with *ed25519* support (e.g. `pip install in-toto[pynacl]`) +- Create CD build job signing key, and signed in-toto layouts (see + `.in_toto/create_layout.py` module docstring for instructions) +- Configure a GitHub secret `IN_TOTO_KEY` pasting the contents from the encrypted + private key created above, and a GitHub secret `IN_TOTO_KEY_PW` for the decryption + password (see `cd.yml` for how the secrets are used). + +**Define vars used by the CLI below** + +```bash +# Attestations are signed using the gpg key identified by `signing_key`, which means the +# corresponding **private** key must be in your local gpg keychain. +signing_key="****** REPLACE WITH YOUR GPG KEYID ******" + +# The fingerprints in `verification_keys` are used to verify the signatures on the +# layouts created and signed above, which requires the corresponding **public** +# keys to be in your local gpg keychain. +verification_keys=("****** REPLACE WITH YOUR GPG KEYID ******") + +# Define GitHub repo name to fetch CD build job attestations +github_repo=theupdateframework/python-tuf # <- CHANGE TO EXPERIMENT IN YOUR FORK!! + +# Grab tuf version string to infer tag name and build artifact names needed below +version=$(python3 -c 'import tuf; print(tuf.__version__)') + +# Define patterns to exclude files we from attestations created below +exclude=('__pycache__' 'build' 'htmlcov' '.?*' '*~' '*.egg-info' '*.pyc') + +# Make sure that neither builds nor attestations include unwanted files +# CAUTION: This deletes all untracked files (except above created layouts) +git clean -xf -e ".in_toto/*.layout" +``` + +## Tag + +Call `git tag ...` with `in-toto` as shown to create a release tag along with a signed +attestation. The attestation records the names and hashes of files in cwd as +*materials*. The attestation is written to `.in_toto/tag..link`. + +```bash +in-toto-run \ + --step-name tag \ + --gpg ${signing_key} \ + --materials . \ + --exclude ${exclude[@]} \ + --metadata-directory .in_toto \ + -- git tag --sign v${version} -m "v${version}" +``` + +**--> push tag to GitHub to trigger CD build job as described in RELEASE.md** + +## Build + +Call `python3 -m build --sdist ...` and `python3 -m build --wheel ...` with `in-toto` as +shown to create two signed attestations, recording the names and hashes of files in cwd +as *materials*, and the name and hash of each respective build artifact as product. The +attestations are written to `.in_toto/sdist..link` and +`.in_toto/wheel..link`. + +```bash +in-toto-run \ + --step-name sdist \ + --gpg ${signing_key} \ + --materials . \ + --products dist/tuf-${version}.tar.gz \ + --exclude ${exclude[@]} \ + --metadata-directory .in_toto \ + -- python3 -m build --sdist --outdir dist/ . +``` + +```bash +in-toto-run \ + --step-name wheel \ + --gpg ${signing_key} \ + --materials . \ + --products dist/tuf-${version}-py3-none-any.whl \ + --exclude ${exclude[@]} dist/tuf-${version}.tar.gz \ + --metadata-directory .in_toto \ + -- python3 -m build --wheel --outdir dist/ . +``` + +## Verify + +Use `in-toto` as shown to verify the supply chain of each build artifact. This means: +- Check layout signatures and layout expiration. *(Note: in-toto requires a valid layout + signature for every key passed to the verify command, and at least one)* + +- Check that there is a threshold of attestations per step, each signed with an + authorized key, both as defined in the layout. *(Note: the attestation signature + verification keys are included in the layout)* + + i.e.: + - one 'tag' attestation signed by any maintainer (we will take the one created above) + - two 'build' attestations per build artifact signed by any maintainer or the CD build + job (we will take the one created above and by the CD build job, which we will + download below) + +- Check that each build artifact matches the product listed in the corresponding 'build' + attestation, and the materials of the 'build' attestations align with the materials in + the 'tag' attestation. + + +**Download CD build job attestations** +```bash +# Workaround to glob download '{wheel, sdist}.*.link' files from release page +cd_keyid=$(wget -q -O - https://github.com/${github_repo}/releases/tag/v${version} | \ + grep -o "sdist.*.link" | head -1 | cut -d "." -f 2) + +wget -P .in_toto https://github.com/${github_repo}/releases/download/v${version}/sdist.${cd_keyid}.link +wget -P .in_toto https://github.com/${github_repo}/releases/download/v${version}/wheel.${cd_keyid}.link +``` + +**Verify 'tuf-${version}.tar.gz' against policies in 'sdist.layout'** +```bash +mkdir empty && cp dist/tuf-${version}.tar.gz empty/ && cd empty +in-toto-verify \ + --link-dir ../.in_toto \ + --layout ../.in_toto/sdist.layout \ + --gpg ${verification_keys[@]} \ + --verbose +cd .. && rm -rf empty +``` + +**Verify 'tuf-${version}-py3-none-any.whl' against policies in 'wheel.layout'** +```bash +mkdir empty && cp dist/tuf-${version}-py3-none-any.whl empty/ && cd empty +in-toto-verify \ + --link-dir ../.in_toto \ + --layout ../.in_toto/wheel.layout \ + --gpg ${verification_keys[@]} \ + --verbose +cd .. && rm -rf empty +``` + +*Note about mkdir/cp/cd/rm: `in-toto-verify` requires a directory that contains nothing +but the final product, i.e. the corresponding build artifact (see in-toto/docs#27 for +details).* + +## User verification (TODO) + +The verification instructions above assume that the maintainer tag and build +attestations are available to the verifier, and that the verifier knows the keys to +verify the layout root signatures. For user verification the following items need to be +resolved: + +- publish maintainer public keys to establish trust root (preferably out-of-band) +- sign metadata with multiple maintainer keys +- publish layout and maintainer attestations in canonical place (e.g. GitHub release) +- provide maintainer tools + docs for easy threshold layout signing and metadata upload +- provide user tools + docs for easy verification (w/o wget, mkdir, cp, ...)