Add support for converting nydus images to nerdctl image convert

Signed-off-by: Nan Li <[email protected]>

Dockerfile

@@ -26,6 +26,8 @@ ARG CNI_PLUGINS_VERSION=v1.1.1
 ARG BUILDKIT_VERSION=v0.10.5
 # Extra deps: Lazy-pulling
 ARG STARGZ_SNAPSHOTTER_VERSION=v0.12.1
+# Extra deps: Nydus Lazy-pulling
+ARG NYDUS_VERSION=v2.1.0
 # Extra deps: Encryption
 ARG IMGCRYPT_VERSION=v1.1.7
 # Extra deps: Rootless
@@ -285,6 +287,12 @@ RUN systemctl enable test-integration-ipfs-offline test-integration-buildkit-ner
     ipfs init && \
     ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5888" && \
     ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/5889"
+# install nydus components
+ARG NYDUS_VERSION
+RUN curl -L -o nydus-static.tgz "https://github.com/dragonflyoss/image-service/releases/download/${NYDUS_VERSION}/nydus-static-${NYDUS_VERSION}-linux-${TARGETARCH}.tgz" && \
+    tar xzf nydus-static.tgz && \
+    mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \
+    rm nydus-static.tgz
 CMD ["go", "test", "-v", "-timeout=20m", "./cmd/nerdctl/...", "-args", "-test.kill-daemon"]
 
 FROM test-integration AS test-integration-rootless

cmd/nerdctl/image_convert.go

@@ -27,6 +27,7 @@ import (
 	"github.com/containerd/containerd/images/converter/uncompress"
 	"github.com/containerd/nerdctl/pkg/platformutil"
 	"github.com/containerd/nerdctl/pkg/referenceutil"
+	nydusconvert "github.com/containerd/nydus-snapshotter/pkg/converter"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
 	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
@@ -67,6 +68,14 @@ func newImageConvertCommand() *cobra.Command {
 	imageConvertCommand.Flags().Bool("zstdchunked", false, "Use zstd compression instead of gzip (a.k.a zstd:chunked). Should be used in conjunction with '--oci'")
 	// #endregion
 
+	// #region nydus flags
+	imageConvertCommand.Flags().Bool("nydus", false, "Convert an OCI image to Nydus image. Should be used in conjunction with '--oci'")
+	imageConvertCommand.Flags().String("nydus-builder-path", "nydus-image", "The nydus-image binary path, if unset, search in PATH environment")
+	imageConvertCommand.Flags().String("nydus-work-dir", "", "Work directory path for image conversion, default is the nerdctl data root directory")
+	imageConvertCommand.Flags().String("nydus-prefetch-patterns", "", "The file path pattern list want to prefetch")
+	imageConvertCommand.Flags().String("nydus-compressor", "lz4_block", "Nydus blob compression algorithm, possible values: `none`, `lz4_block`, `zstd`, default is `lz4_block`")
+	// #endregion
+
 	// #region generic flags
 	imageConvertCommand.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers")
 	imageConvertCommand.Flags().Bool("oci", false, "Convert Docker media types to OCI media types")
@@ -126,6 +135,10 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+	nydus, err := cmd.Flags().GetBool("nydus")
+	if err != nil {
+		return err
+	}
 	oci, err := cmd.Flags().GetBool("oci")
 	if err != nil {
 		return err
@@ -165,6 +178,41 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	if nydus {
+		if estargz {
+			return errors.New("option --nydus conflicts with --estargz")
+		}
+		if zstdchunked {
+			return errors.New("option --nydus conflicts with --zstdchunked")
+		}
+		if !oci {
+			logrus.Warnln("option --nydus should be used in conjunction with '--oci', forcibly enabling on oci mediatype for nydus conversion")
+		}
+
+		nydusOpts, err := getNydusConvertOpts(cmd)
+		if err != nil {
+			return err
+		}
+		convertFunc := nydusconvert.LayerConvertFunc(*nydusOpts)
+		convertHooks := converter.ConvertHooks{
+			PostConvertHook: nydusconvert.ConvertHookFunc(nydusconvert.MergeOption{
+				WorkDir:          nydusOpts.WorkDir,
+				BuilderPath:      nydusOpts.BuilderPath,
+				FsVersion:        nydusOpts.FsVersion,
+				ChunkDictPath:    nydusOpts.ChunkDictPath,
+				PrefetchPatterns: nydusOpts.PrefetchPatterns,
+			}),
+		}
+		convertOpts = append(convertOpts, converter.WithIndexConvertFunc(
+			converter.IndexConvertFuncWithHook(
+				convertFunc,
+				true,
+				platMC,
+				convertHooks,
+			)),
+		)
+	}
+
 	if uncompressValue {
 		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(uncompress.LayerConvertFunc))
 	}
@@ -229,6 +277,41 @@ func getESGZConvertOpts(cmd *cobra.Command) ([]estargz.Option, error) {
 	return esgzOpts, nil
 }
 
+func getNydusConvertOpts(cmd *cobra.Command) (*nydusconvert.PackOption, error) {
+	builderPath, err := cmd.Flags().GetString("nydus-builder-path")
+	if err != nil {
+		return nil, err
+	}
+	workDir, err := cmd.Flags().GetString("nydus-work-dir")
+	if err != nil {
+		return nil, err
+	}
+	if workDir == "" {
+		workDir, err = getDataStore(cmd)
+		if err != nil {
+			return nil, err
+		}
+	}
+	prefetchPatterns, err := cmd.Flags().GetString("nydus-prefetch-patterns")
+	if err != nil {
+		return nil, err
+	}
+	compressor, err := cmd.Flags().GetString("nydus-compressor")
+	if err != nil {
+		return nil, err
+	}
+	return &nydusconvert.PackOption{
+		BuilderPath: builderPath,
+		// the path will finally be used is <NERDCTL_DATA_ROOT>/nydus-converter-<hash>,
+		// for example: /var/lib/nerdctl/1935db59/nydus-converter-3269662176/,
+		// and it will be deleted after the conversion
+		WorkDir:          workDir,
+		PrefetchPatterns: prefetchPatterns,
+		Compressor:       compressor,
+		FsVersion:        "6",
+	}, nil
+}
+
 func readPathsFromRecordFile(filename string) ([]string, error) {
 	r, err := os.Open(filename)
 	if err != nil {

cmd/nerdctl/image_convert_linux_test.go

@@ -0,0 +1,80 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"runtime"
+	"testing"
+
+	"github.com/containerd/nerdctl/pkg/rootlessutil"
+	"github.com/containerd/nerdctl/pkg/testutil"
+	"github.com/containerd/nerdctl/pkg/testutil/testregistry"
+	"gotest.tools/v3/icmd"
+)
+
+func TestImageConvertNydus(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("no windows support yet")
+	}
+
+	testutil.DockerIncompatible(t)
+	base := testutil.NewBase(t)
+	convertedImage := testutil.Identifier(t) + ":nydus"
+	base.Cmd("rmi", convertedImage).Run()
+	base.Cmd("pull", testutil.CommonImage).AssertOK()
+	base.Cmd("image", "convert", "--nydus", "--oci",
+		testutil.CommonImage, convertedImage).AssertOK()
+	defer base.Cmd("rmi", convertedImage).Run()
+
+	// use `nydusify` check whether the convertd nydus image is valid
+
+	// skip if rootless
+	if rootlessutil.IsRootless() {
+		t.Skip("Nydusify check is not supported rootless mode.")
+	}
+
+	// skip if nydusify is not installed
+	if _, err := exec.LookPath("nydusify"); err != nil {
+		t.Skip("Nydusify is not installed")
+	}
+
+	// setup local docker registry
+	registryPort := 15000
+	registry := testregistry.NewPlainHTTP(base, registryPort)
+	defer registry.Cleanup()
+
+	remoteImage := fmt.Sprintf("%s:%d/nydusd-image:test", registry.IP.String(), registryPort)
+	base.Cmd("tag", convertedImage, remoteImage).AssertOK()
+	defer base.Cmd("rmi", remoteImage).Run()
+	base.Cmd("push", "--insecure-registry", remoteImage).AssertOK()
+	nydusifyCmd := testutil.Cmd{
+		Cmd: icmd.Command(
+			"nydusify",
+			"check",
+			"--source",
+			testutil.CommonImage,
+			"--target",
+			remoteImage,
+			"--source-insecure",
+			"--target-insecure",
+		),
+		Base: base,
+	}
+	nydusifyCmd.AssertOK()
+}

docs/nydus.md

@@ -26,4 +26,12 @@ Nydus snapshotter is a remote snapshotter plugin of containerd for [Nydus](https
 
 For the list of pre-converted Nydus images, see https://github.com/orgs/dragonflyoss/packages?page=1&repo_name=image-service
 
-For more details about how to build Nydus image, please refer to [nydusify](https://github.com/dragonflyoss/image-service/blob/master/docs/nydusify.md) conversion tool and [acceld](https://github.com/goharbor/acceleration-service).
+## Build Nydus image using `nerdctl image convert`
+
+Nerdctl supports to convert an OCI image or docker format v2 image to Nydus image by using the `nerdctl image convert` command.
+
+Before the conversion, you should have the `nydus-image` binary installed, which is contained in the ["nydus static package"](https://github.com/dragonflyoss/image-service/releases). You can run the command like `nerdctl image convert --nydus --oci --nydus-image <the_path_of_nydus_image> <source_image> <target_image>` to convert the `<source_image>` to a Nydus image whose tag is `<target_image>`.
+
+By now, the converted Nydus image cannot be run directly. It shoud be unpacked to nydus snapshotter before `nerdctl run`, which is a part of the processing flow of `nerdctl image pull`. So you need to push the converted image to a registry after the conversion and use `nerdctl --snapshotter nydus image pull` to unpack it to the nydus snapshotter before running the image.
+
+Optionally, you can use the nydusify conversion tool to check if the format of the converted Nydus image is valid. For more details about the Nydus image validation and how to build Nydus image, please refer to [nydusify](https://github.com/dragonflyoss/image-service/blob/master/docs/nydusify.md) and [acceld](https://github.com/goharbor/acceleration-service).

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/containerd/continuity v0.3.0
 	github.com/containerd/go-cni v1.1.7
 	github.com/containerd/imgcrypt v1.1.7
-	github.com/containerd/nydus-snapshotter v0.3.0
+	github.com/containerd/nydus-snapshotter v0.3.1
 	github.com/containerd/stargz-snapshotter v0.12.1
 	github.com/containerd/stargz-snapshotter/estargz v0.12.1
 	github.com/containerd/stargz-snapshotter/ipfs v0.12.1

go.sum

@@ -221,8 +221,8 @@ github.com/containerd/imgcrypt v1.1.5-0.20220421044638-8ba028dca028/go.mod h1:Lo
 github.com/containerd/imgcrypt v1.1.7 h1:WSf9o9EQ0KGHiUx2ESFZ+PKf4nxK9BcvV/nJDX8RkB4=
 github.com/containerd/imgcrypt v1.1.7/go.mod h1:FD8gqIcX5aTotCtOmjeCsi3A1dHmTZpnMISGKSczt4k=
 github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
-github.com/containerd/nydus-snapshotter v0.3.0 h1:15z2Bslu1A7oi++vByV6cTIFzoSjvOGScVCU2y6bRdA=
-github.com/containerd/nydus-snapshotter v0.3.0/go.mod h1:j/4mx893AxVibOoAAvdzhPZmL1ynPZaKR7rhQPOqHuU=
+github.com/containerd/nydus-snapshotter v0.3.1 h1:b8WahTrPkt3XsabjG2o/leN4fw3HWZYr+qxo/Z8Mfzk=
+github.com/containerd/nydus-snapshotter v0.3.1/go.mod h1:+8R7NX7vrjlxAgtidnsstwIhpzyTlriYPssTxH++uiM=
 github.com/containerd/stargz-snapshotter v0.12.1 h1:kkt4LUsZafCr1JVsB6/8xQl6KsLt6mlTZEzrAlWpPrk=
 github.com/containerd/stargz-snapshotter v0.12.1/go.mod h1:tlYP/UpT5bAJhdAaN3YN6UP5SE8kbW7nNrUTbvtpx1c=
 github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM=