PMM-12899 Use module and build cache

.github/workflows/main.yml

@@ -142,8 +142,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#00FF00"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"
@@ -160,8 +160,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#FF0000"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "Workflow failed: ${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"

build/docker/client/Dockerfile

@@ -5,19 +5,13 @@ RUN microdnf install shadow-utils jq -y
 RUN groupadd -g 1002 pmm-agent && \
     useradd -u 1002 -r -g pmm-agent -s /sbin/nologin \
             -d /usr/local/percona/pmm \
-            -c "PMM 2.X Client User" pmm-agent
+            -c "PMM 3.x Client User" pmm-agent
 
 FROM redhat/ubi9-micro
 
 ARG VERSION
 ARG BUILD_DATE
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses Apache-2.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management Client
-LABEL org.opencontainers.image.vendor Percona
-LABEL org.opencontainers.image.version ${VERSION}
-
 COPY LICENSE /licenses/
 COPY --from=users --chown=0:0 /etc/passwd /etc/passwd
 COPY --from=users --chown=0:0 /etc/group /etc/group
@@ -32,10 +26,18 @@ RUN install -d -o pmm-agent -g pmm-agent -m 0775 /usr/local/percona/pmm
 ADD pmm-client.tar.gz /tmp/
 
 RUN cd /tmp/pmm-client-* \
-    && env PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
+    && PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
     && cd /tmp \
     && rm -rf /tmp/pmm-client*
 
+LABEL org.opencontainers.image.created ${BUILD_DATE}
+LABEL org.opencontainers.image.licenses Apache-2.0
+LABEL org.opencontainers.image.title Percona Monitoring and Management Client
+LABEL org.opencontainers.image.vendor Percona
+LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/client/Dockerfile
+LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#docker
+
 USER pmm-agent
 WORKDIR /usr/local/percona/pmm/
 ENV PATH /usr/local/percona/pmm/bin/:$PATH

build/docker/server/Dockerfile.el9

@@ -8,12 +8,6 @@ ENV LC_ALL=en_US.utf8
 ENV GF_PLUGIN_DIR=/srv/grafana/plugins
 ENV PS1="[\u@\h \W] # "
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
-
 EXPOSE 8080 8443
 
 WORKDIR /opt
@@ -40,6 +34,14 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm-docker/update.yml && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml
 
+LABEL org.opencontainers.image.created ${BUILD_DATE}
+LABEL org.opencontainers.image.licenses AGPL-3.0
+LABEL org.opencontainers.image.title Percona Monitoring and Management
+LABEL org.opencontainers.image.vendor Percona LLC
+LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
+LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
+
 USER pmm
 
 HEALTHCHECK --interval=3s --timeout=2s --start-period=10s --retries=3 CMD curl -sf http://127.0.0.1:8080/v1/readyz

build/local/README.md

@@ -0,0 +1,34 @@
+# Local builds
+
+This directory contains a set of scripts aimed at providing a simple way to build PMM locally.
+
+## Background
+
+Historically, PMM used to be built using Jenkins. This worked well for the team, but not for the community. The learning curve was, and still is, rather steep, and it is hard for folks, even internally, to contribute to.
+
+Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to making it easier to build PMM locally.
+
+The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on to make the build process easy to understand and contribute to. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
+
+The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
+
+## Prerequisites
+
+Below is a list of prerequisites that are required to build PMM locally.
+
+- OS: Linux (tested on Oracle Linux 9.3, CentOS 7, Ubuntu 22.04.3 LTS)
+- Docker: 25.0.2+ (tested on 25.0.2)
+- Docker Compose Plugin: 2.24.7+ (tested on 2.24.7)
+- make: 4.3+ (tested on 4.3)
+- bash (GNU): 5.1+ (tested on 5.1)
+- git: 2.34.1+ (tested on 2.34.1)
+- curl: 7.81.0+ (tested on 7.81.0)
+- yq: 4.42.0+ (tested on 4.42.1)
+- jq: 1.6+ (tested on 1.6)
+
+## Build Steps
+
+1. Install the prerequisites
+2. Clone the PMM repository
+3. Change to the `build/local` directory
+4. Run the `build.sh` script

build/local/build.sh

@@ -0,0 +1,110 @@
+#!/bin/bash -e
+
+# Update submodules and PR branches
+/bin/bash $(dirname $0)/update.sh
+
+get_branch_name() {
+  local module="${1:-}"
+  local branch_name
+  local path
+
+  path=$(git config -f .gitmodules submodule.${module}.path)
+  cd "$path" || exit 1
+  branch_name=$(git branch --show-current)
+  cd - > /dev/null
+  echo $branch_name
+}
+
+# Define global variables
+pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+echo $pmm_commit > apiCommitSha
+pmm_branch=$(get_branch_name pmm)
+echo $pmm_branch > apiBranch
+pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+echo $pmm_url > apiURL
+pmm_qa_branch=$(get_branch_name pmm-qa)
+echo $pmm_qa_branch > pmmQABranch
+pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+echo $pmm_qa_commit > pmmQACommitSha
+pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
+echo $pmm_ui_tests_branch > pmmUITestBranch
+pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+fb_commit_sha=$(git rev-parse HEAD)
+echo $fb_commit_sha > fbCommitSha
+
+# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:ol9`.
+# Important: the docker container's user need to be able to write to these directories.
+# The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
+# that the directories we create on the host are owned by a user with same uid and gid.
+
+# Create cache directories. 
+test -d "${root_dir}/go-path" || mkdir -p "go-path"
+test -d "${root_dir}/go-build" || mkdir -p "go-build"
+
+PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9 
+
+# Local reference test environment
+# CPU: 4 cores
+# RAM: 16GB
+# OS: Ubuntu 22.04.1 LTS
+
+# Build client source: 4m39s from scratch, 0m27s using cache
+"$PATH_TO_SCRIPTS/build-client-source"
+
+# Build client binary: ??? from scratch, 0m20s using cache
+"$PATH_TO_SCRIPTS/build-client-binary"
+
+# Building client source rpm takes 13s (caching does not apply)
+"$PATH_TO_SCRIPTS/build-client-srpm"
+
+# Building client rpm takes 1m40s
+"$PATH_TO_SCRIPTS/build-client-rpm"
+
+# Building client docker image takes 17s
+GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
+"$PATH_TO_SCRIPTS/build-client-docker"
+
+# Building PMM CLient locally (non-CI, i.e. non-Jenkins)
+# total time: 6m26s - build from scratch, no initial cache
+# total time: 2m49s - subsequent build, using cache from prior builds
+
+
+# Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
+# total time: 8m45s - build from scratch, no initial cache
+# total time: ??? - subsequent build, using cache from prior builds
+
+export RPM_EPOCH=1
+export RPMBUILD_DIST="el9"
+"$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-controller
+"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-tools
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+"$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
+"$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+
+# 3rd-party
+"$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+"$PATH_TO_SCRIPTS/build-server-rpm" alertmanager
+"$PATH_TO_SCRIPTS/build-server-rpm" grafana
+
+export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9
+export RPMBUILD_DIST=el9
+export DOCKERFILE=Dockerfile.el9.local
+${PATH_TO_SCRIPTS}/build-server-docker
+
+# Clean up temporary files
+rm -f apiBranch \
+  apiCommitSha \
+	apiURL \
+	fbCommitSha \
+	pmmQABranch \
+	pmmQACommitSha \
+	pmmUITestBranch \
+	pmmUITestsCommitSha

build/local/update.sh

@@ -0,0 +1,101 @@
+#!/bin/bash -e
+
+needs-to-pull() {
+  local UPSTREAM=${1:-'@{u}'}
+  local LOCAL=$(git rev-parse @)
+  local BASE=$(git merge-base @ "$UPSTREAM")
+  local REMOTE=$(git rev-parse "$UPSTREAM")
+
+  if [ "$LOCAL" = "$REMOTE" ]; then
+    return 1 # false, we are up-to-date
+  fi
+
+  if [ "$LOCAL" = "$BASE" ]; then
+    return 0 # true, we are behind upstream
+  fi
+}
+
+rewind() {
+  local DIR="$1"
+  local BRANCH="$2"
+
+  cd "$DIR"
+  CURRENT=$(git branch --show-current)
+  git fetch
+
+  if [ "$CURRENT" != "$BRANCH" ]; then
+    echo "Currently on $CURRENT, checking out $BRANCH"
+    git checkout "$BRANCH"
+  fi
+
+  if needs-to-pull; then
+    git pull origin
+    echo "Submodule has pulled from upstream"
+    git logs -n 2
+    cd - >/dev/null
+    git add "$DIR"
+  else
+    cd - >/dev/null
+    echo "Submodule is up-to-date with upstream"
+  fi
+}
+
+check-files() {
+  local DIR="$1"
+
+  test -z "DIR" && exit 1
+
+  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
+    return 0
+  fi
+
+  return 1
+}
+
+main() {
+  local DEPS=
+  local CURDIR="$PWD"
+  local DIR=pmm-submodules
+
+  # Thouroughly verify the presence of known files, otherwise bail out
+  if check-files "."; then # pwd is pmm-submodules
+    DIR="."
+  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$DIR"; then
+      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      exit 1
+    fi
+  else
+    echo "FATAL: could not locate known files in $PWD"
+    exit 1
+  fi
+
+  cd "$DIR"
+
+  # Join the dependencies from ci-default.yml and ci.yml
+  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
+
+  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
+
+  echo "$DEPS" | jq -c '.[]' | while read -r item; do
+    branch=$(echo "$item" | jq -r '.branch')
+    path=$(echo "$item" | jq -r '.path')
+    name=$(echo "$item" | jq -r '.name')
+    echo
+    echo "Rewinding submodule '$name' ..."
+    echo "path: ${path}, branch: ${branch}"
+
+    rewind "$path" "$branch"
+  done
+
+  echo
+  echo "Printing git status..."
+  git status --short
+  echo
+  echo "Printing git submodule status..."
+  git submodule status
+
+  cd "$CURDIR" > /dev/null
+}
+
+main