Skip to content

Commit

Permalink
feat: add support for copying certs to dockerd's registry directory (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
sreya authored Sep 5, 2024
1 parent 7f3cd37 commit 569d5de
Show file tree
Hide file tree
Showing 10 changed files with 412 additions and 39 deletions.
2 changes: 1 addition & 1 deletion cli/clitest/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func New(t *testing.T, cmd string, args ...string) (context.Context, *cobra.Comm

var (
execer = NewFakeExecer()
fs = NewMemFS()
fs = xunixfake.NewMemFS()
mnt = &mount.FakeMounter{}
client = NewFakeDockerClient()
iface = GetNetLink(t)
Expand Down
8 changes: 0 additions & 8 deletions cli/clitest/fake.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,13 @@ import (
"strings"

dockertypes "github.com/docker/docker/api/types"
"github.com/spf13/afero"
testingexec "k8s.io/utils/exec/testing"

"github.com/coder/envbox/dockerutil"
"github.com/coder/envbox/dockerutil/dockerfake"
"github.com/coder/envbox/xunix/xunixfake"
)

func NewMemFS() *xunixfake.MemFS {
return &xunixfake.MemFS{
MemMapFs: &afero.MemMapFs{},
Owner: map[string]xunixfake.FileOwner{},
}
}

func NewFakeExecer() *xunixfake.FakeExec {
return &xunixfake.FakeExec{
Commands: map[string]*xunixfake.FakeCmd{},
Expand Down
37 changes: 29 additions & 8 deletions cli/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,20 +157,20 @@ func dockerCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) (err error) {
var (
ctx = cmd.Context()
log = slog.Make(slogjson.Sink(io.Discard))
blog buildlog.Logger = buildlog.NopLogger{}
log = slog.Make(slogjson.Sink(cmd.ErrOrStderr()), slogkubeterminate.Make()).Leveled(slog.LevelDebug)
blog buildlog.Logger = buildlog.JSONLogger{Encoder: json.NewEncoder(os.Stderr)}
)

if flags.noStartupLogs {
log = slog.Make(slogjson.Sink(io.Discard))
blog = buildlog.NopLogger{}
}

httpClient, err := xhttp.Client(log, flags.extraCertsPath)
if err != nil {
return xerrors.Errorf("http client: %w", err)
}

if !flags.noStartupLogs {
log = slog.Make(slogjson.Sink(cmd.ErrOrStderr()), slogkubeterminate.Make()).Leveled(slog.LevelDebug)
blog = buildlog.JSONLogger{Encoder: json.NewEncoder(os.Stderr)}
}

if !flags.noStartupLogs && flags.agentToken != "" && flags.coderURL != "" {
coderURL, err := url.Parse(flags.coderURL)
if err != nil {
Expand All @@ -197,7 +197,6 @@ func dockerCmd() *cobra.Command {
}
}(&err)

blog.Info("Waiting for dockerd to startup...")
sysboxArgs := []string{}
if flags.disableIDMappedMount {
sysboxArgs = append(sysboxArgs, "--disable-idmapped-mount")
Expand Down Expand Up @@ -231,6 +230,7 @@ func dockerCmd() *cobra.Command {

log.Debug(ctx, "starting dockerd", slog.F("args", args))

blog.Info("Waiting for sysbox processes to startup...")
dockerd := background.New(ctx, log, "dockerd", dargs...)
err = dockerd.Start()
if err != nil {
Expand Down Expand Up @@ -289,11 +289,32 @@ func dockerCmd() *cobra.Command {
// We wait for the daemon after spawning the goroutine in case
// startup causes the daemon to encounter encounter a 'no space left
// on device' error.
blog.Info("Waiting for dockerd to startup...")
err = dockerutil.WaitForDaemon(ctx, client)
if err != nil {
return xerrors.Errorf("wait for dockerd: %w", err)
}

if flags.extraCertsPath != "" {
// Parse the registry from the inner image
registry, err := name.ParseReference(flags.innerImage)
if err != nil {
return xerrors.Errorf("invalid image: %w", err)
}
registryName := registry.Context().RegistryStr()

// Write certificates for the registry
err = dockerutil.WriteCertsForRegistry(ctx, registryName, flags.extraCertsPath)
if err != nil {
return xerrors.Errorf("write certs for registry: %w", err)
}

blog.Infof("Successfully copied certificates from %q to %q", flags.extraCertsPath, filepath.Join("/etc/docker/certs.d", registryName))
log.Debug(ctx, "wrote certificates for registry", slog.F("registry", registryName),
slog.F("extra_certs_path", flags.extraCertsPath),
)
}

err = runDockerCVM(ctx, log, client, blog, flags)
if err != nil {
// It's possible we failed because we ran out of disk while
Expand Down
13 changes: 13 additions & 0 deletions dockerutil/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"io"
"strings"
"time"

dockertypes "github.com/docker/docker/api/types"
Expand Down Expand Up @@ -65,6 +66,14 @@ func PullImage(ctx context.Context, config *PullImageConfig) error {
}

err = pullImageFn()
// We should bail early if we're going to fail due to a
// certificate error. We can't xerrors.As here since this is
// returned from the daemon so the client is reporting
// essentially an unwrapped error.
if isTLSVerificationErr(err) {
return err
}

if err == nil {
return nil
}
Expand Down Expand Up @@ -253,3 +262,7 @@ func DefaultLogImagePullFn(log buildlog.Logger) func(ImagePullEvent) error {
return nil
}
}

func isTLSVerificationErr(err error) bool {
return err != nil && strings.Contains(err.Error(), "tls: failed to verify certificate: x509: certificate signed by unknown authority")
}
92 changes: 92 additions & 0 deletions dockerutil/registry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package dockerutil

import (
"context"
"io"
"path/filepath"

"github.com/spf13/afero"
"golang.org/x/xerrors"

"github.com/coder/envbox/xunix"
)

// WriteCertsForRegistry writes the certificates found in the provided directory
// to the correct subdirectory that the Docker daemon uses when pulling images
// from the specified private registry.
func WriteCertsForRegistry(ctx context.Context, registryName, certsDir string) error {
fs := xunix.GetFS(ctx)

// Docker certs directory.
registryCertsDir := filepath.Join("/etc/docker/certs.d", registryName)

// If the directory already exists it means someone
// has either wrapped the image or has mounted in certs
// manually. We should assume the user knows what they're
// doing and avoid mucking with their solution.
if _, err := fs.Stat(registryCertsDir); err == nil {
return nil
}

// Ensure the registry certs directory exists.
err := fs.MkdirAll(registryCertsDir, 0o755)
if err != nil {
return xerrors.Errorf("create registry certs directory: %w", err)
}

// Check if certsDir is a file.
fileInfo, err := fs.Stat(certsDir)
if err != nil {
return xerrors.Errorf("stat certs directory/file: %w", err)
}

if !fileInfo.IsDir() {
// If it's a file, copy it directly
err = copyCertFile(fs, certsDir, filepath.Join(registryCertsDir, "ca.crt"))
if err != nil {
return xerrors.Errorf("copy cert file: %w", err)
}
return nil
}

// If it's a directory, copy all cert files in the root of the directory
entries, err := afero.ReadDir(fs, certsDir)
if err != nil {
return xerrors.Errorf("read certs directory: %w", err)
}

for _, entry := range entries {
if entry.IsDir() {
continue
}
srcPath := filepath.Join(certsDir, entry.Name())
dstPath := filepath.Join(registryCertsDir, entry.Name())
err = copyCertFile(fs, srcPath, dstPath)
if err != nil {
return xerrors.Errorf("copy cert file %s: %w", entry.Name(), err)
}
}

return nil
}

func copyCertFile(fs xunix.FS, src, dst string) error {
srcFile, err := fs.Open(src)
if err != nil {
return xerrors.Errorf("open source file: %w", err)
}
defer srcFile.Close()

dstFile, err := fs.Create(dst)
if err != nil {
return xerrors.Errorf("create destination file: %w", err)
}
defer dstFile.Close()

_, err = io.Copy(dstFile, srcFile)
if err != nil {
return xerrors.Errorf("copy file contents: %w", err)
}

return nil
}
100 changes: 100 additions & 0 deletions dockerutil/registry_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package dockerutil_test

import (
"context"
"os"
"path/filepath"
"testing"

"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/coder/envbox/dockerutil"
"github.com/coder/envbox/xunix"
"github.com/coder/envbox/xunix/xunixfake"
)

func TestWriteCertsForRegistry(t *testing.T) {
t.Parallel()

t.Run("SingleCertFile", func(t *testing.T) {
t.Parallel()
// Test setup
fs := xunixfake.NewMemFS()
ctx := xunix.WithFS(context.Background(), fs)

// Create a test certificate file
certContent := []byte("test certificate content")
err := afero.WriteFile(fs, "/certs/ca.crt", certContent, 0o644)
require.NoError(t, err)

// Run the function
err = dockerutil.WriteCertsForRegistry(ctx, "test.registry.com", "/certs/ca.crt")
require.NoError(t, err)

// Check the result
copiedContent, err := afero.ReadFile(fs, "/etc/docker/certs.d/test.registry.com/ca.crt")
require.NoError(t, err)
assert.Equal(t, certContent, copiedContent)
})

t.Run("MultipleCertFiles", func(t *testing.T) {
t.Parallel()
// Test setup
fs := xunixfake.NewMemFS()
ctx := xunix.WithFS(context.Background(), fs)

// Create test certificate files
certFiles := []string{"ca.crt", "client.cert", "client.key"}
for _, file := range certFiles {
err := afero.WriteFile(fs, filepath.Join("/certs", file), []byte("content of "+file), 0o644)
require.NoError(t, err)
}

// Run the function
err := dockerutil.WriteCertsForRegistry(ctx, "test.registry.com", "/certs")
require.NoError(t, err)

// Check the results
for _, file := range certFiles {
copiedContent, err := afero.ReadFile(fs, filepath.Join("/etc/docker/certs.d/test.registry.com", file))
require.NoError(t, err)
assert.Equal(t, []byte("content of "+file), copiedContent)
}
})
t.Run("ExistingRegistryCertsDir", func(t *testing.T) {
t.Parallel()
// Test setup
fs := xunixfake.NewMemFS()
ctx := xunix.WithFS(context.Background(), fs)

// Create an existing registry certs directory
registryCertsDir := "/etc/docker/certs.d/test.registry.com"
err := fs.MkdirAll(registryCertsDir, 0o755)
require.NoError(t, err)

// Create a file in the existing directory
existingContent := []byte("existing certificate content")
err = afero.WriteFile(fs, filepath.Join(registryCertsDir, "existing.crt"), existingContent, 0o644)
require.NoError(t, err)

// Create a test certificate file in the source directory
certContent := []byte("new certificate content")
err = afero.WriteFile(fs, "/certs/ca.crt", certContent, 0o644)
require.NoError(t, err)

// Run the function
err = dockerutil.WriteCertsForRegistry(ctx, "test.registry.com", "/certs")
require.NoError(t, err)

// Check that the existing file was not modified
existingFileContent, err := afero.ReadFile(fs, filepath.Join(registryCertsDir, "existing.crt"))
require.NoError(t, err)
assert.Equal(t, existingContent, existingFileContent)

// Check that the new file was not copied
_, err = fs.Stat(filepath.Join(registryCertsDir, "ca.crt"))
assert.True(t, os.IsNotExist(err), "New certificate file should not have been copied")
})
}
Loading

0 comments on commit 569d5de

Please sign in to comment.