diff --git a/pkg/checker/checker.go b/pkg/checker/checker.go index 54c5921c..879504ce 100644 --- a/pkg/checker/checker.go +++ b/pkg/checker/checker.go @@ -1,30 +1,57 @@ package checker import ( + "encoding/json" "fmt" "regexp" "github.com/hashicorp/go-version" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + remoteauth "oras.land/oras-go/v2/registry/remote/auth" + + "kcl-lang.io/kpm/pkg/constants" + "kcl-lang.io/kpm/pkg/downloader" + "kcl-lang.io/kpm/pkg/oci" + "kcl-lang.io/kpm/pkg/opt" pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" + "kcl-lang.io/kpm/pkg/settings" + "kcl-lang.io/kpm/pkg/utils" ) -// Checker defines an interface for KclPkg dependencies checkers. +// Checker defines an interface for checking KclPkg dependencies. type Checker interface { Check(pkg.KclPkg) error } +// DepChecker is responsible for running multiple checkers on a package's dependencies. type DepChecker struct { checkers []Checker } -// NewDepChecker creates a new DepChecker with provided checkers. -func NewDepChecker(checkers ...Checker) *DepChecker { - return &DepChecker{checkers: checkers} +// DepCheckerOption configures how we set up DepChecker. +type DepCheckerOption func(*DepChecker) + +// NewDepChecker creates a new DepChecker with options. +func NewDepChecker(options ...DepCheckerOption) *DepChecker { + depChecker := &DepChecker{} + for _, opt := range options { + opt(depChecker) + } + return depChecker +} + +// WithChecker adds a new Checker to DepChecker. +func WithChecker(checker Checker) DepCheckerOption { + return func(c *DepChecker) { + c.checkers = append(c.checkers, checker) + } } // Check runs all individual checks for a kclPkg. -func (c *DepChecker) Check(kclPkg pkg.KclPkg) error { - for _, checker := range c.checkers { +func (dc *DepChecker) Check(kclPkg pkg.KclPkg) error { + for _, checker := range dc.checkers { if err := checker.Check(kclPkg); err != nil { return err } @@ -35,7 +62,12 @@ func (c *DepChecker) Check(kclPkg pkg.KclPkg) error { // IdentChecker validates the dependencies name in kclPkg. type IdentChecker struct{} -func (c *IdentChecker) Check(kclPkg pkg.KclPkg) error { +// NewIdentChecker creates a new IdentChecker. +func NewIdentChecker() *IdentChecker { + return &IdentChecker{} +} + +func (ic *IdentChecker) Check(kclPkg pkg.KclPkg) error { for _, key := range kclPkg.Dependencies.Deps.Keys() { dep, _ := kclPkg.Dependencies.Deps.Get(key) if !isValidDependencyName(dep.Name) { @@ -48,7 +80,12 @@ func (c *IdentChecker) Check(kclPkg pkg.KclPkg) error { // VersionChecker validates the dependencies version in kclPkg. type VersionChecker struct{} -func (c *VersionChecker) Check(kclPkg pkg.KclPkg) error { +// NewVersionChecker creates a new VersionChecker. +func NewVersionChecker() *VersionChecker { + return &VersionChecker{} +} + +func (vc *VersionChecker) Check(kclPkg pkg.KclPkg) error { for _, key := range kclPkg.Dependencies.Deps.Keys() { dep, _ := kclPkg.Dependencies.Deps.Get(key) if !isValidDependencyVersion(dep.Version) { @@ -58,16 +95,39 @@ func (c *VersionChecker) Check(kclPkg pkg.KclPkg) error { return nil } -// SumChecker validates the dependencies checksum in kclPkg. -type SumChecker struct{} +// SumChecker validates the dependencies sum in kclPkg. +type SumChecker struct { + settings settings.Settings +} + +// SumCheckerOption configures how we set up SumChecker. +type SumCheckerOption func(*SumChecker) + +// NewSumChecker creates a new SumChecker with options. +func NewSumChecker(options ...SumCheckerOption) *SumChecker { + sumChecker := &SumChecker{} + for _, opt := range options { + opt(sumChecker) + } + return sumChecker +} + +// WithSettings sets the settings for SumChecker. +func WithSettings(settings settings.Settings) SumCheckerOption { + return func(s *SumChecker) { + s.settings = settings + } +} -func (c *SumChecker) Check(kclPkg pkg.KclPkg) error { +// Check verifies the checksums of the dependencies in the KclPkg. +func (sc *SumChecker) Check(kclPkg pkg.KclPkg) error { if kclPkg.NoSumCheck { return nil } + for _, key := range kclPkg.Dependencies.Deps.Keys() { dep, _ := kclPkg.Dependencies.Deps.Get(key) - trustedSum, err := getTrustedSum(dep) + trustedSum, err := sc.getTrustedSum(dep) if err != nil { return fmt.Errorf("failed to get checksum from trusted source: %w", err) } @@ -78,21 +138,112 @@ func (c *SumChecker) Check(kclPkg pkg.KclPkg) error { return nil } -// isValidDependencyName reports whether the name of the dependency is appropriate. +// isValidDependencyName checks whether the given dependency name is valid. func isValidDependencyName(name string) bool { validNamePattern := `^[a-zA-Z][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_]$` regex := regexp.MustCompile(validNamePattern) return regex.MatchString(name) } -// isValidDependencyVersion reports whether v is a valid semantic version string. +// isValidDependencyVersion checks whether the given version is a valid semantic version string. func isValidDependencyVersion(v string) bool { _, err := version.NewVersion(v) return err == nil } -// Placeholder for getTrustedSum function -func getTrustedSum(dep pkg.Dependency) (string, error) { - // Need to be implemented to get the trusted checksum. - return "", nil +// getTrustedSum retrieves the trusted checksum for the given dependency. +func (sc *SumChecker) getTrustedSum(dep pkg.Dependency) (string, error) { + if dep.Source.Oci == nil { + return "", fmt.Errorf("dependency is not from OCI") + } + + sc.populateOciFields(dep) + + manifest, err := sc.fetchOciManifest(dep) + if err != nil { + return "", err + } + + return sc.extractChecksumFromManifest(manifest) +} + +// populateOciFields fills in missing OCI fields with default values from settings. +func (sc *SumChecker) populateOciFields(dep pkg.Dependency) { + if len(dep.Source.Oci.Reg) == 0 { + dep.Source.Oci.Reg = sc.settings.DefaultOciRegistry() + } + + if len(dep.Source.Oci.Repo) == 0 { + dep.Source.Oci.Repo = utils.JoinPath(sc.settings.DefaultOciRepo(), dep.Name) + } +} + +// fetchOciManifest retrieves the OCI manifest for the given dependency. +func (sc *SumChecker) fetchOciManifest(dep pkg.Dependency) (ocispec.Manifest, error) { + manifest := ocispec.Manifest{} + jsonDesc, err := sc.FetchOciManifestIntoJsonStr(opt.OciFetchOptions{ + FetchBytesOptions: oras.DefaultFetchBytesOptions, + OciOptions: opt.OciOptions{ + Reg: dep.Source.Oci.Reg, + Repo: dep.Source.Oci.Repo, + Tag: dep.Source.Oci.Tag, + }, + }) + if err != nil { + return manifest, reporter.NewErrorEvent(reporter.FailedFetchOciManifest, err, fmt.Sprintf("failed to fetch the manifest of '%s'", dep.Name)) + } + + err = json.Unmarshal([]byte(jsonDesc), &manifest) + if err != nil { + return manifest, fmt.Errorf("failed to unmarshal manifest: %w", err) + } + + return manifest, nil +} + +// FetchOciManifestIntoJsonStr fetches the OCI manifest and returns it as a JSON string. +func (sc *SumChecker) FetchOciManifestIntoJsonStr(opts opt.OciFetchOptions) (string, error) { + repoPath := utils.JoinPath(opts.Reg, opts.Repo) + cred, err := sc.GetCredentials(opts.Reg) + if err != nil { + return "", err + } + + ociCli, err := oci.NewOciClientWithOpts( + oci.WithCredential(cred), + oci.WithRepoPath(repoPath), + oci.WithSettings(&sc.settings), + ) + if err != nil { + return "", err + } + + manifestJson, err := ociCli.FetchManifestIntoJsonStr(opts) + if err != nil { + return "", err + } + return manifestJson, nil +} + +// GetCredentials retrieves the OCI credentials for the given hostname. +func (sc *SumChecker) GetCredentials(hostName string) (*remoteauth.Credential, error) { + credCli, err := downloader.LoadCredentialFile(sc.settings.CredentialsFile) + if err != nil { + return nil, err + } + + creds, err := credCli.Credential(hostName) + if err != nil { + return nil, err + } + + return creds, nil +} + +// extractChecksumFromManifest extracts the checksum from the OCI manifest. +func (sc *SumChecker) extractChecksumFromManifest(manifest ocispec.Manifest) (string, error) { + if value, ok := manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]; ok { + return value, nil + } + return "", fmt.Errorf("checksum annotation not found in manifest") } diff --git a/pkg/checker/checker_test.go b/pkg/checker/checker_test.go index 58d28df8..5b752a68 100644 --- a/pkg/checker/checker_test.go +++ b/pkg/checker/checker_test.go @@ -1,90 +1,61 @@ package checker import ( + "runtime" "testing" "github.com/elliotchance/orderedmap/v2" "gotest.tools/v3/assert" + + "kcl-lang.io/kpm/pkg/downloader" + "kcl-lang.io/kpm/pkg/mock" pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" + "kcl-lang.io/kpm/pkg/settings" ) func TestDepCheckerCheck(t *testing.T) { depChecker := NewDepChecker( - &IdentChecker{}, - &VersionChecker{}, - &SumChecker{}, + WithChecker(NewIdentChecker()), + WithChecker(NewVersionChecker()), + WithChecker(NewSumChecker()), ) + deps1 := orderedmap.NewOrderedMap[string, pkg.Dependency]() deps1.Set("kcl1", pkg.Dependency{ Name: "kcl1", FullName: "kcl1", Version: "0.0.1", - Sum: "", + Sum: "no-sum-check-enabled", }) deps1.Set("kcl2", pkg.Dependency{ Name: "kcl2", FullName: "kcl2", Version: "0.2.1", - Sum: "", + Sum: "no-sum-check-enabled", }) deps2 := orderedmap.NewOrderedMap[string, pkg.Dependency]() deps2.Set("kcl1", pkg.Dependency{ - Name: "kcl1", - FullName: "kcl1", - Version: "0.0.1", - Sum: "no-sum-check-enabled", - }) - deps2.Set("kcl2", pkg.Dependency{ - Name: "kcl2", - FullName: "kcl2", - Version: "0.2.1", - Sum: "no-sum-check-enabled", - }) - - deps3 := orderedmap.NewOrderedMap[string, pkg.Dependency]() - deps3.Set("kcl1", pkg.Dependency{ Name: ".kcl1", FullName: "kcl1", Version: "0.0.1", Sum: "", }) - deps4 := orderedmap.NewOrderedMap[string, pkg.Dependency]() - deps4.Set("kcl1", pkg.Dependency{ + deps3 := orderedmap.NewOrderedMap[string, pkg.Dependency]() + deps3.Set("kcl1", pkg.Dependency{ Name: "kcl1", FullName: "kcl1", Version: "1.0.0-alpha#", Sum: "", }) - deps5 := orderedmap.NewOrderedMap[string, pkg.Dependency]() - deps5.Set("kcl1", pkg.Dependency{ - Name: "kcl1", - FullName: "kcl1", - Version: "0.0.1", - Sum: "invalid-no-sum-check-disabled", - }) - tests := []struct { name string KclPkg pkg.KclPkg wantErr bool }{ - { - name: "valid kcl package - with sum check", - KclPkg: pkg.KclPkg{ - ModFile: pkg.ModFile{ - HomePath: "path/to/modfile", - }, - HomePath: "path/to/kcl/pkg", - Dependencies: pkg.Dependencies{ - Deps: deps1, - }, - NoSumCheck: false, - }, - wantErr: false, - }, { name: "valid kcl package - with no sum check enabled", KclPkg: pkg.KclPkg{ @@ -93,7 +64,7 @@ func TestDepCheckerCheck(t *testing.T) { }, HomePath: "path/to/kcl/pkg", Dependencies: pkg.Dependencies{ - Deps: deps2, + Deps: deps1, }, NoSumCheck: true, }, @@ -107,7 +78,7 @@ func TestDepCheckerCheck(t *testing.T) { }, HomePath: "path/to/kcl/pkg", Dependencies: pkg.Dependencies{ - Deps: deps3, + Deps: deps2, }, NoSumCheck: false, }, @@ -121,21 +92,7 @@ func TestDepCheckerCheck(t *testing.T) { }, HomePath: "path/to/kcl/pkg", Dependencies: pkg.Dependencies{ - Deps: deps4, - }, - NoSumCheck: false, - }, - wantErr: true, - }, - { - name: "Invalid kcl package - with no sum check disabled - checksum mismatches", - KclPkg: pkg.KclPkg{ - ModFile: pkg.ModFile{ - HomePath: "path/to/modfile", - }, - HomePath: "path/to/kcl/pkg", - Dependencies: pkg.Dependencies{ - Deps: deps5, + Deps: deps3, }, NoSumCheck: false, }, @@ -221,3 +178,129 @@ func TestIsValidDependencyVersion(t *testing.T) { }) } } + +func getTestSettings() (*settings.Settings, error) { + settings := settings.GetSettings() + + if settings.ErrorEvent != (*reporter.KpmEvent)(nil) { + return nil, settings.ErrorEvent + } + return settings, nil +} + +func TestDepCheckerCheck_WithTrustedSum(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping TestDepCheckerCheck_WithTrustedSum test on Windows") + } + + // Start the local Docker registry required for testing + err := mock.StartDockerRegistry() + assert.Equal(t, err, nil) + + // Push the test package to the local OCI registry + err = mock.PushTestPkgToRegistry() + assert.Equal(t, err, nil) + + // Initialize settings for use with the DepChecker + settings, err := getTestSettings() + assert.Equal(t, err, nil) + + // Initialize the DepChecker with required checkers + depChecker := NewDepChecker( + WithChecker(NewIdentChecker()), + WithChecker(NewVersionChecker()), + WithChecker(NewSumChecker(WithSettings(*settings))), + ) + + deps1 := orderedmap.NewOrderedMap[string, pkg.Dependency]() + deps1.Set("kcl1", pkg.Dependency{ + Name: "test_data", + FullName: "test_data", + Version: "0.0.1", + Sum: "RpZZIvrXwfn5dpt6LqBR8+FlPE9Y+BEou47L3qaCCqk=", + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test", + Tag: "0.0.1", + }, + }, + }) + + deps2 := orderedmap.NewOrderedMap[string, pkg.Dependency]() + deps2.Set("kcl1", pkg.Dependency{ + Name: "test_data", + FullName: "test_data", + Version: "0.0.1", + Sum: "Invalid-sum", + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test", + Tag: "0.0.1", + }, + }, + }) + + tests := []struct { + name string + KclPkg pkg.KclPkg + wantErr bool + }{ + { + name: "valid kcl package - with sum check", + KclPkg: pkg.KclPkg{ + ModFile: pkg.ModFile{ + HomePath: "path/to/modfile", + }, + HomePath: "path/to/kcl/pkg", + Dependencies: pkg.Dependencies{ + Deps: deps1, + }, + NoSumCheck: false, + }, + wantErr: false, + }, + { + name: "valid kcl package - with no sum check enabled", + KclPkg: pkg.KclPkg{ + ModFile: pkg.ModFile{ + HomePath: "path/to/modfile", + }, + HomePath: "path/to/kcl/pkg", + Dependencies: pkg.Dependencies{ + Deps: deps2, + }, + NoSumCheck: true, + }, + wantErr: false, + }, + { + name: "Invalid kcl package - with no sum check disabled - checksum mismatches", + KclPkg: pkg.KclPkg{ + ModFile: pkg.ModFile{ + HomePath: "path/to/modfile", + }, + HomePath: "path/to/kcl/pkg", + Dependencies: pkg.Dependencies{ + Deps: deps2, + }, + NoSumCheck: false, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotErr := depChecker.Check(tt.KclPkg) + if (gotErr != nil) != tt.wantErr { + t.Errorf("depChecker.Check(%v) = %v, want error %v", tt.KclPkg, gotErr, tt.wantErr) + } + }) + } + + // Clean the environment after all tests have been run + err = mock.CleanTestEnv() + assert.Equal(t, err, nil) +} diff --git a/pkg/mock/oci_env_mock.go b/pkg/mock/oci_env_mock.go new file mode 100644 index 00000000..65840207 --- /dev/null +++ b/pkg/mock/oci_env_mock.go @@ -0,0 +1,23 @@ +package mock + +import ( + "os/exec" +) + +// StartDockerRegistry starts a local Docker registry by executing a shell script. +func StartDockerRegistry() error { + cmd := exec.Command("../../scripts/reg.sh") + return cmd.Run() +} + +// PushTestPkgToRegistry pushes the test package to the local Docker registry. +func PushTestPkgToRegistry() error { + cmd := exec.Command("../mock/test_script/push_pkg.sh") + return cmd.Run() +} + +// CleanTestEnv cleans up the test environment by executing a cleanup script. +func CleanTestEnv() error { + cmd := exec.Command("../mock/test_script/cleanup_test_environment.sh") + return cmd.Run() +} diff --git a/pkg/mock/test_data/kcl.mod b/pkg/mock/test_data/kcl.mod new file mode 100644 index 00000000..6c01364d --- /dev/null +++ b/pkg/mock/test_data/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "test_data" +edition = "v0.9.0" +version = "0.0.1" + +[dependencies] +k8s = "1.31" + diff --git a/pkg/mock/test_data/kcl.mod.lock b/pkg/mock/test_data/kcl.mod.lock new file mode 100644 index 00000000..41b4867f --- /dev/null +++ b/pkg/mock/test_data/kcl.mod.lock @@ -0,0 +1,5 @@ +[dependencies] + [dependencies.k8s] + name = "k8s" + full_name = "k8s_1.31" + version = "1.31" diff --git a/pkg/mock/test_data/main.k b/pkg/mock/test_data/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/mock/test_data/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/mock/test_script/cleanup_test_environment.sh b/pkg/mock/test_script/cleanup_test_environment.sh new file mode 100755 index 00000000..9d2d7fa6 --- /dev/null +++ b/pkg/mock/test_script/cleanup_test_environment.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Determine the directory where this script is located +SCRIPT_DIR="$(dirname "$(realpath "$0")")" + +# Stop and remove the Docker container then remove the Docker image +docker stop kcl-registry +docker rm kcl-registry +docker rmi registry + +# Delete all data stored in the Docker registry volume +rm -rf /var/lib/registry/* + +# Remove the directory that contains Docker authentication and related scripts +current_dir=$(pwd) +rm -rf "$current_dir/scripts/" + +# Delete the 'kpm' binary +cd "$SCRIPT_DIR/../../../" +rm -rf ./bin/ diff --git a/pkg/mock/test_script/push_pkg.sh b/pkg/mock/test_script/push_pkg.sh new file mode 100755 index 00000000..64a41530 --- /dev/null +++ b/pkg/mock/test_script/push_pkg.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Get the directory of the script +SCRIPT_DIR="$(dirname "$(realpath "$0")")" + +# Move to the root directory +cd "$SCRIPT_DIR/../../../" + +# Build kpm binary +LDFLAGS="-X kcl-lang.io/kpm/pkg/version.version=test_version" +go build -ldflags "$LDFLAGS" -o ./bin/kpm + +# Check kpm version +version=$(./bin/kpm --version) +if ! echo "$version" | grep -q "kpm version test_version"; then + echo "Incorrect version: '$version'." + exit 1 +fi + +export KPM_REG="localhost:5001" +export KPM_REPO="test" + +# Prepare the package on the registry +current_dir=$(pwd) +echo $current_dir + +# Log in to the local registry +"$current_dir/bin/kpm" login -u test -p 1234 localhost:5001 + +# Push the test_data package to the registry +cd "$SCRIPT_DIR/../test_data" +"$current_dir/bin/kpm" push oci://$KPM_REG/$KPM_REPO diff --git a/pkg/settings/settings.go b/pkg/settings/settings.go index 8d2940f6..6a8046eb 100644 --- a/pkg/settings/settings.go +++ b/pkg/settings/settings.go @@ -121,7 +121,7 @@ func (settings *Settings) ReleasePackageCacheLock() error { return nil } -// DefaultOciRepo return the default OCI registry 'ghcr.io'. +// DefaultOciRegistry return the default OCI registry 'ghcr.io'. func (settings *Settings) DefaultOciRegistry() string { return settings.Conf.DefaultOciRegistry }