Merge pull request docker#2376 from crazy-max/ci-test-experimental

tests: test with buildx experimental

.github/workflows/build.yml

@@ -73,11 +73,14 @@ jobs:
           - remote
         pkg:
           - ./tests
+        experimental:
+          - ""
+          - 1
     steps:
       -
         name: Prepare
         run: |
-          echo "TESTREPORTS_NAME=${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+          echo "TESTREPORTS_NAME=${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.worker }}-${{ matrix.experimental }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
       -
         name: Checkout
         uses: actions/checkout@v4
@@ -107,6 +110,7 @@ jobs:
           ./hack/test
         env:
           TEST_REPORT_SUFFIX: "-${{ env.TESTREPORTS_NAME }}"
+          TEST_BUILDX_EXPERIMENTAL: ${{ matrix.experimental }}
           TEST_DOCKERD: "${{ startsWith(matrix.worker, 'docker') && '1' || '0' }}"
           TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker\\+containerd') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
           TESTPKGS: "${{ matrix.pkg }}"

hack/test

@@ -11,6 +11,7 @@ set -eu -o pipefail
 : "${TEST_KEEP_CACHE=}"
 : "${TEST_DOCKERD=}"
 : "${TEST_BUILDKIT_IMAGE=}"
+: "${TEST_BUILDX_EXPERIMENTAL=}"
 
 if [ "$TEST_IMAGE_BUILD" = "1" ]; then
   ${BUILDX_CMD} bake integration-test --set "*.output=type=docker,name=$TEST_IMAGE_ID"
@@ -30,5 +31,5 @@ if [ "$TEST_KEEP_CACHE" != "1" ]; then
   trap 'docker rm -v $cacheVolume' EXIT
 fi
 
-cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
+cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e TEST_BUILDX_EXPERIMENTAL -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
 docker start -a -i $cid

tests/build.go

@@ -68,6 +68,11 @@ func testBuild(t *testing.T, sb integration.Sandbox) {
 }
 
 func testBuildStdin(t *testing.T, sb integration.Sandbox) {
+	if isExperimental() {
+		// FIXME: https://github.com/docker/buildx/issues/2368
+		t.Skip("build from stdin hangs in experimental mode: https://github.com/docker/buildx/issues/2368")
+	}
+
 	dockerfile := []byte(`
 FROM busybox:latest AS base
 COPY foo /etc/foo
@@ -300,6 +305,11 @@ RUN echo foo > /bar`)
 	require.NoError(t, err, string(out))
 	require.True(t, buildDetailsPattern.MatchString(string(out)), fmt.Sprintf("expected build details link in output, got %q", out))
 
+	if isExperimental() {
+		// FIXME: https://github.com/docker/buildx/issues/2382
+		t.Skip("build details link not displayed in experimental mode when build fails: https://github.com/docker/buildx/issues/2382")
+	}
+
 	// build erroneous dockerfile
 	dockerfile = []byte(`FROM busybox:latest
 RUN exit 1`)

tests/integration.go

@@ -3,6 +3,7 @@ package tests
 import (
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -55,6 +56,9 @@ func buildxCmd(sb integration.Sandbox, opts ...cmdOpt) *exec.Cmd {
 	if context := sb.DockerAddress(); context != "" {
 		cmd.Env = append(cmd.Env, "DOCKER_CONTEXT="+context)
 	}
+	if isExperimental() {
+		cmd.Env = append(cmd.Env, "BUILDX_EXPERIMENTAL=1")
+	}
 
 	return cmd
 }
@@ -95,3 +99,11 @@ func driverName(sbName string) (string, bool) {
 	}
 	return name, hasFeature
 }
+
+func isExperimental() bool {
+	if v, ok := os.LookupEnv("TEST_BUILDX_EXPERIMENTAL"); ok {
+		vv, _ := strconv.ParseBool(v)
+		return vv
+	}
+	return false
+}