Merge pull request #370 from clearmatics/develop

v0.7 - Daric

.github/workflows/code-scanning.yml

@@ -46,6 +46,11 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: '3.x'
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: codeql-py-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
       - name: Custom dependencies installation
         run: |
           cd client
@@ -63,4 +68,3 @@ jobs:
           setup-python-dependencies: false
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v1
-

.github/workflows/dev-docker-apps.yml

@@ -14,40 +14,39 @@ env:
 jobs:
 
   docker-build:
-    name: "Docker build"
+    name: Docker build
     runs-on: ubuntu-20.04
-    timeout-minutes: 360
+    timeout-minutes: 180
 
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         app: ['mpc', 'client', 'prover']
       max-parallel: 3
 
     steps:
-      - name: "Checkout"
+      - name: Checkout
         uses: actions/checkout@v2
 
-      - name: "Get vars from git"
+      - name: Get vars from git
         id: git_vars
         run: echo ::set-output name=COMMIT_HASH::$(git rev-parse --short HEAD)
 
-      - name: "Login to DockerHub"
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: "Docker build"
-        run: docker build -f ./Dockerfile-${{ matrix.app }} -t $REPO_OWNER/zeth-${{ matrix.app }}:git-${{ steps.git_vars.outputs.COMMIT_HASH }} .
+      - name: Configure Docker
+        run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
 
-      - name: "Docker push"
-        run: docker push $REPO_OWNER/zeth-${{ matrix.app }}:git-${{ steps.git_vars.outputs.COMMIT_HASH }}
+      - name: Set up Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: latest
 
-      - name: "Docker Info"
+      - name: Build and push image git-SHA
         run: |
-          docker inspect $REPO_OWNER/zeth-${{ matrix.app }}:git-${{ steps.git_vars.outputs.COMMIT_HASH }}
-          docker history $REPO_OWNER/zeth-${{ matrix.app }}:git-${{ steps.git_vars.outputs.COMMIT_HASH }} --no-trunc
-          docker version
-          docker info
-          uname -a
+          docker buildx build \
+            --tag ghcr.io/$GITHUB_REPOSITORY:git-${{ steps.git_vars.outputs.COMMIT_HASH }}-${{ matrix.app }} \
+            --cache-from=type=registry,ref=ghcr.io/$GITHUB_REPOSITORY:cache-${{ matrix.app }} \
+            --cache-to=type=registry,ref=ghcr.io/$GITHUB_REPOSITORY:cache-${{ matrix.app }} \
+            --platform linux/amd64 \
+            --output "type=image,push=true" \
+            --file ./Dockerfile-${{ matrix.app }} ./

.github/workflows/onpullrequest-build-ubuntu.yml

@@ -1,19 +1,130 @@
 # Actions for pull requests only
-name: zeth-ci-pull-request
+name: zeth-ci-onpullrequest-build-ubuntu
 
 on:
   pull_request:
 
 jobs:
 
-  build-linux-full:
+  # Job to build the grpc libraries. The grpc build directory is populated and
+  # cached so that all other jobs can mark this job as a prerequisite and just
+  # run `make install`.
+  #
+  # NOTE: the version number here (in `key` and in script arguments) must be
+  # kept in sync with the key used by the jobs.
+  onpr-build-grpc:
     runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Build grpc
+      run: if ! [ -d depends/grpc ] ; then scripts/install_grpc /usr v1.31.x depends/grpc ; fi
+
+  # Extract the commits of submodules for use by cache steps
+  onpr-submodules:
+    runs-on: ubuntu-20.04
+    outputs:
+      commits: ${{ steps.get-commits.outputs.commits }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Get Submodule Commits
+      id: get-commits
+      run: |
+        git submodule sync
+        echo "::set-output name=commits::"`git submodule status depends/libsodium | grep -oe '[0-9a-fA-F]\+' | head -c 8`-`git submodule status depends/libsnark | grep -oe '[0-9a-fA-F]\+' | head -c 8`
+
+  # Run the prover testing python scripts
+  prover-tests-linux:
+    runs-on: ubuntu-20.04
+    needs: [onpr-build-grpc, onpr-submodules]
     strategy:
       matrix:
         curve: [ BLS12_377, ALT_BN128 ]
     steps:
     - uses: actions/checkout@v1
       with:
         submodules: recursive
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 10
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    # - name: Cache ccache
+    #   uses: actions/cache@v2
+    #   with:
+    #     key: prover-tests-ccache-${{ needs.onpr-submodules.outputs.commits }}-${{ matrix.curve }}-${{ runner.os }}
+    #     path: ~/.ccache
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/pip
+          ~/Library/Caches/pip
+        key: prover-tests-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
+    - name: Cache npm
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.npm
+          depends/ganache-cli/node_modules
+        key: prover-tests-npm-${{ hashFiles('**/package-lock.json') }}-${{ runner.os }}
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
+    - name: Execute
+      run: CI_CONFIG=Release CI_CURVE=${{ matrix.curve }} CI_PROVER_TESTS=1 scripts/ci build
+
+  # Run all unit tests and integration tests
+  integration-tests-linux:
+    runs-on: ubuntu-20.04
+    needs: [onpr-build-grpc, onpr-submodules]
+    strategy:
+      matrix:
+        curve: [ BLS12_377, ALT_BN128 ]
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: recursive
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 10
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    # ccache in this job seems to make the tests crash occasionally. Disabling
+    # until the cause is understood.
+    # - name: Cache ccache
+    #   uses: actions/cache@v2
+    #   with:
+    #     key: integration-tests-ccache-${{ needs.onpr-submodules.outputs.commits }}-${{ matrix.curve }}-${{ runner.os }}
+    #     path: ~/.ccache
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/pip
+          ~/Library/Caches/pip
+        key: integration-tests-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
+    - name: Cache npm
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.npm
+          depends/ganache-cli/node_modules
+        key: integration-tests-npm-${{ hashFiles('**/package-lock.json') }}-${{ runner.os }}
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
     - name: Execute
-      run: CI_EVENT_NAME=pull_request CI_USE_DOCKER=1 CI_CONFIG=Release CI_CURVE=${{ matrix.curve }} scripts/ci build
+      run: CI_CONFIG=Release CI_CURVE=${{ matrix.curve }} CI_FULL_TESTS=1 CI_INTEGRATION_TESTS=1 scripts/ci build

.github/workflows/onpush-build-macos.yml

@@ -5,21 +5,45 @@ on:
   push:
 
 env:
-  MACOS_BREW_PACKAGES: "pkg-config libomp"
+  MACOS_BREW_PACKAGES: "pkg-config libomp ccache"
 
 jobs:
 
+  # Extract the commits of submodules for use by cache steps
+  submodules:
+    runs-on: ubuntu-20.04
+    outputs:
+      commits: ${{ steps.get-commits.outputs.commits }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Get Submodule Commits
+      id: get-commits
+      run: |
+        git submodule sync
+        echo "::set-output name=commits::"`git submodule status depends/libsodium | grep -oe '[0-9a-fA-F]\+' | head -c 8`-`git submodule status depends/libsnark | grep -oe '[0-9a-fA-F]\+' | head -c 8`
+
+  # Main build
   build-macos:
     runs-on: macos-10.15
+    needs: submodules
     strategy:
       matrix:
         config: [ Debug, Release ]
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
       with:
         submodules: recursive
+    - name: Cache ccache
+      uses: actions/cache@v2
+      with:
+        key: submodules-${{ needs.submodules.outputs.commits }}-${{ runner.os }}-${{ matrix.config }}
+        path: ~/Library/Caches/ccache
+    - name: Cache pip (for mpc tests)
+      uses: actions/cache@v2
+      with:
+        path: ~/Library/Caches/pip
+        key: build-macos-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
     - name: Install Dependencies
       run: brew install ${MACOS_BREW_PACKAGES}
     - name: Execute
-      run: CI_CONFIG=${{ matrix.config }} scripts/ci build
-
+      run: CI_MPC_TESTS=1 CI_CONFIG=${{ matrix.config }} scripts/ci build

.github/workflows/onpush-build-ubuntu.yml

@@ -6,33 +6,85 @@ on:
 
 jobs:
 
+  # Job to build and cache the grpc libraries. The grpc build directory is
+  # populated and cached so that all other jobs can mark this job as a
+  # prerequisite and just run `make install`.
+  #
+  # NOTE: the version number here (in `key` and in script arguments) must be
+  # kept in sync with the key used by the jobs.
+  build-grpc:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Build grpc
+      run: if ! [ -d depends/grpc ] ; then scripts/install_grpc /usr v1.31.x depends/grpc ; fi
+
   build-linux:
     runs-on: ubuntu-20.04
+    needs: build-grpc
     strategy:
       matrix:
         config: [ Debug, Release ]
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
       with:
         submodules: recursive
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Cache pip (for mpc tests)
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: build-linux-pip-${{ hashFiles('**/setup.py') }}-${{ runner.os }}
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
     - name: Execute
-      run: CI_CHECK_FORMAT=1 CI_USE_DOCKER=1 CI_CONFIG=${{ matrix.config }} scripts/ci build
+      run: CI_CHECK_FORMAT=1 CI_MPC_TESTS=1 CI_CONFIG=${{ matrix.config }} scripts/ci build
 
   build-linux-pghr13:
     runs-on: ubuntu-20.04
+    needs: build-grpc
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
       with:
         submodules: recursive
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
     - name: Execute
-      run: CI_USE_DOCKER=1 CI_CONFIG=Release CI_ZKSNARK=PGHR13 scripts/ci build
+      run: CI_CONFIG=Release CI_ZKSNARK=PGHR13 scripts/ci build
 
   build-linux-bls12-377:
     runs-on: ubuntu-20.04
+    needs: build-grpc
     steps:
     - uses: actions/checkout@v1
       with:
         submodules: recursive
+    - name: Cache grpc
+      uses: actions/cache@v2
+      with:
+        key: grpc-1.31.x-${{ runner.os }}
+        path: depends/grpc
+    - name: Install dependencies
+      run: |
+        scripts/install_grpc /usr v1.31.x depends/grpc
+        sudo apt install -y ccache
     - name: Execute
-      run: CI_USE_DOCKER=1 CI_CONFIG=Release CI_CURVE=BLS12_377 scripts/ci build
-
+      run: CI_CONFIG=Release CI_CURVE=BLS12_377 scripts/ci build