From fc0d6715ed797c7e0316f23aeb3635edbae03769 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Mon, 24 Jul 2023 09:37:14 -0400 Subject: [PATCH 1/6] Upgrade to latest cxx-common version 0.4.1 --- .github/workflows/ci.yml | 6 +++++- .github/workflows/diff_tests.yml | 4 ++-- Dockerfile | 4 +++- scripts/build.sh | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d1759f5..abab22d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,10 @@ on: branches: - '*' +env: + CC: clang + CXX: clang++ + jobs: build_linux: strategy: @@ -30,7 +34,7 @@ jobs: name: Rellic CI runs-on: "gha-ubuntu-32" container: - image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }} + image: ghcr.io/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}-v2:${{ matrix.image.tag }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/diff_tests.yml b/.github/workflows/diff_tests.yml index 895339bf..916cda2c 100644 --- a/.github/workflows/diff_tests.yml +++ b/.github/workflows/diff_tests.yml @@ -13,7 +13,7 @@ jobs: image: - { name: 'ubuntu', tag: '20.04', codename: 'focal' } llvm: [ '16' ] - common_base: [ 'https://github.com/lifting-bits/cxx-common/releases/latest/download' ] + common_base: [ 'https://github.com/lifting-bits/cxx-common/releases/download/v0.4.1' ] env: CC: clang-${{ matrix.llvm }} @@ -22,7 +22,7 @@ jobs: name: Diff in ouput between old and new rellic runs-on: gha-ubuntu-32 container: - image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }} + image: ghcr.io/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}-v2:${{ matrix.image.tag }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 2fa7eb61..1b40a102 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ FROM ${BUILD_BASE} as base # Build-time dependencies go here # See here for full list of those dependencies # https://github.com/lifting-bits/cxx-common/blob/master/docker/Dockerfile.ubuntu.vcpkg -FROM trailofbits/cxx-common-vcpkg-builder-ubuntu:${UBUNTU_VERSION} as deps +FROM ghcr.io/lifting-bits/cxx-common/vcpkg-builder-ubuntu-v2:${UBUNTU_VERSION} as deps ARG UBUNTU_VERSION ARG ARCH ARG LLVM_VERSION @@ -30,6 +30,8 @@ ARG LLVM_VERSION ARG LIBRARIES ENV TRAILOFBITS_LIBRARIES="${LIBRARIES}" ENV PATH="${LIBRARIES}/llvm/bin/:${LIBRARIES}/cmake/bin:${PATH}" +ENV CC=clang +ENV CXX=clang++ WORKDIR /rellic COPY ./ ./ diff --git a/scripts/build.sh b/scripts/build.sh index b205e86a..4d551304 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -19,7 +19,7 @@ CURR_DIR=$( pwd ) BUILD_DIR="${CURR_DIR}/rellic-build" INSTALL_DIR=/usr/local LLVM_VERSION=llvm-16 -CXX_COMMON_VERSION=v0.3.2 +CXX_COMMON_VERSION=v0.4.1 OS_VERSION=unknown ARCH_VERSION=unknown BUILD_FLAGS= From 34be68dc4bb420ec51225f2ba064ad1baf576379 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:58:46 -0400 Subject: [PATCH 2/6] install only in docker container to avoid running out of space in CI --- Dockerfile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1b40a102..90e1450a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,16 +35,11 @@ ENV CXX=clang++ WORKDIR /rellic COPY ./ ./ -# The reason we don't use --install -# is so that container has the same exact code as the packages RUN ./scripts/build.sh \ --llvm-version ${LLVM_VERSION} \ --prefix /opt/trailofbits \ - --extra-cmake-args "-DCMAKE_BUILD_TYPE=Release" - -RUN cd rellic-build && \ - CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --verbose --target test && \ - cmake --build . --target install + --extra-cmake-args "-DCMAKE_BUILD_TYPE=Release" \ + --install # Small installation image FROM base as install From 037dd30ae639d0fb2ad55a7eb2b3d762819eeaec Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:08:34 -0400 Subject: [PATCH 3/6] use ubuntu 22.04 --- .github/workflows/ci.yml | 16 ++++++++-------- README.md | 12 ++++++------ scripts/build.sh | 8 ++++++-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abab22d2..ba5cd5c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: fail-fast: false matrix: image: - - { name: 'ubuntu', tag: '20.04' } + - { name: 'ubuntu', tag: '22.04' } llvm: [ '16' ] @@ -177,7 +177,7 @@ jobs: release_packages: # Do not run the release procedure if any of the builds has failed needs: [ build_linux, build_mac ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') steps: @@ -216,13 +216,13 @@ jobs: - name: Group the packages by platform run: | - zip -r9 rellic_ubuntu-20.04_packages.zip \ - ubuntu-20.04* + zip -r9 rellic_ubuntu-22.04_packages.zip \ + ubuntu-22.04* zip -r9 rellic_macos-12_packages.zip \ macos-12* - - name: Upload the Ubuntu 20.04 packages + - name: Upload the Ubuntu 22.04 packages uses: actions/upload-release-asset@v1 env: @@ -230,8 +230,8 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rellic_ubuntu-20.04_packages.zip - asset_name: rellic_ubuntu-20.04_packages.zip + asset_path: rellic_ubuntu-22.04_packages.zip + asset_name: rellic_ubuntu-22.04_packages.zip asset_content_type: application/gzip - name: Upload the macOS 12 packages @@ -251,7 +251,7 @@ jobs: strategy: matrix: llvm: ["16"] - ubuntu: ["20.04"] + ubuntu: ["22.04"] steps: - uses: actions/checkout@v2 with: diff --git a/README.md b/README.md index 08f055dd..f4a5dc63 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ If you are experiencing undocumented problems with Rellic then ask for help in t ## Supported Platforms -Rellic is supported on Linux platforms and has been tested on Ubuntu 20.04. +Rellic is supported on Linux platforms and has been tested on Ubuntu 22.04. ## Dependencies @@ -236,7 +236,7 @@ Finally, we build and package Rellic. This script will create another directory, ```shell cd rellic -./scripts/build.sh --llvm-version 14 +./scripts/build.sh --llvm-version 16 # to install the deb package, then do: sudo dpkg -i rellic-build/*.deb ``` @@ -271,10 +271,10 @@ make -j8 The Docker image should provide an environment which can set-up, build, and run rellic. The Docker images are parameterized by Ubuntu verison, LLVM version, and architecture. -To build the docker image using LLVM 16 for Ubuntu 20.04 on amd64 you can run the following command: +To build the docker image using LLVM 16 for Ubuntu 22.04 on amd64 you can run the following command: ```sh -ARCH=amd64; UBUNTU=20.04; LLVM=16; docker build . \ +ARCH=amd64; UBUNTU=22.04; LLVM=16; docker build . \ -t rellic:llvm${LLVM}-ubuntu${UBUNTU}-${ARCH} \ -f Dockerfile \ --build-arg UBUNTU_VERSION=${UBUNTU} \ @@ -286,13 +286,13 @@ To run the decompiler, the entrypoint has already been set, but make sure the bi ```sh # Get the bc file -clang-14 -emit-llvm -c ./tests/tools/decomp/issue_4.c -o ./tests/tools/decomp/issue_4.bc +clang-16 -emit-llvm -c ./tests/tools/decomp/issue_4.c -o ./tests/tools/decomp/issue_4.bc # Decompile docker run --rm -t -i \ -v $(pwd):/test -w /test \ -u $(id -u):$(id -g) \ - rellic:llvm14-ubuntu20.04-amd64 --input ./tests/tools/decomp/issue_4.bc --output /dev/stdout + rellic:llvm16-ubuntu22.04-amd64 --input ./tests/tools/decomp/issue_4.bc --output /dev/stdout ``` To explain the above command more: diff --git a/scripts/build.sh b/scripts/build.sh index 4d551304..2377dcf8 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -38,6 +38,10 @@ function GetUbuntuOSVersion OS_VERSION=ubuntu-20.04 return 0 ;; + jammy) + OS_VERSION=ubuntu-22.04 + return 0 + ;; focal) OS_VERSION=ubuntu-20.04 return 0 @@ -62,7 +66,7 @@ function GetUbuntuOSVersion return 0 ;; *) - echo "[x] Ubuntu ${DISTRIB_CODENAME} is not supported. Only focal (20.04) and bionic (18.04) are pre-compiled." + echo "[x] Ubuntu ${DISTRIB_CODENAME} is not supported. Only jammy (22.04) are pre-compiled." echo "[x] Please see https://github.com/lifting-bits/cxx-common to build dependencies from source." return 1 ;; @@ -153,7 +157,7 @@ function GetOSVersion ;; *debian*) - OS_VERSION=ubuntu-20.04 + OS_VERSION=ubuntu-22.04 return 0; ;; From 7075dd3902b273debe89c9349519925d4dd41676 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:11:23 -0400 Subject: [PATCH 4/6] remove llvm apt repo --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba5cd5c2..c5be0d64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,9 +51,6 @@ jobs: shell: bash run: | # TODO some of these should probably live in the Docker build image - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - - echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-${{ matrix.llvm }} main" >> /etc/apt/sources.list - echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-${{ matrix.llvm }} main" >> /etc/apt/sources.list apt-get update apt-get install -y ninja-build pixz xz-utils make rpm python3.8 update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 100 From 9655b5e649e9e279664ca6ff75c806332383522c Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:20:58 -0400 Subject: [PATCH 5/6] 22.04 should have a sufficiently new python --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5be0d64..676fff78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,8 +52,7 @@ jobs: run: | # TODO some of these should probably live in the Docker build image apt-get update - apt-get install -y ninja-build pixz xz-utils make rpm python3.8 - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 100 + apt-get install -y ninja-build pixz xz-utils make rpm python3 - name: Build with build script shell: bash From bfd55adafd4b0d74b680afd79b0336c8be19f3a8 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:32:56 -0400 Subject: [PATCH 6/6] bump to llvm-16 --- .github/workflows/anghabench-after-build.yml | 2 +- .github/workflows/anghabench-cron-jobs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/anghabench-after-build.yml b/.github/workflows/anghabench-after-build.yml index 441267fa..23c5b761 100644 --- a/.github/workflows/anghabench-after-build.yml +++ b/.github/workflows/anghabench-after-build.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - llvm: [ '15' ] + llvm: [ '16' ] run_size: [ '1k' ] name: Run AnghaBench CI (AMD64) diff --git a/.github/workflows/anghabench-cron-jobs.yml b/.github/workflows/anghabench-cron-jobs.yml index ef7d958f..8de5fab3 100644 --- a/.github/workflows/anghabench-cron-jobs.yml +++ b/.github/workflows/anghabench-cron-jobs.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - llvm: [ '15' ] + llvm: [ '16' ] run_size: [ '1k' ] name: Run AnghaBench CI (AMD64)