From e8adf4fc19e4aeac3a8c805eeddc38536177afb9 Mon Sep 17 00:00:00 2001 From: Christoph Fiehe Date: Thu, 26 Sep 2024 17:58:18 +0200 Subject: [PATCH 1/2] Allow to remove components from a published repository. This commit introduces the new 'remove' command for a published repository. It allows to remove one or multiple components from a published repository without needing to recreate it. Signed-off-by: Christoph Fiehe --- api/publish.go | 84 +++++++++++++++ api/router.go | 1 + cmd/publish.go | 1 + cmd/publish_remove.go | 119 +++++++++++++++++++++ deb/publish.go | 8 ++ system/t06_publish/PublishRemove1Test_gold | 7 ++ system/t06_publish/PublishRemove2Test_gold | 8 ++ system/t06_publish/PublishRemove3Test_gold | 1 + system/t06_publish/PublishRemove4Test_gold | 27 +++++ system/t06_publish/remove.py | 87 +++++++++++++++ system/t12_api/publish.py | 68 ++++++++++++ 11 files changed, 411 insertions(+) create mode 100644 cmd/publish_remove.go create mode 100644 system/t06_publish/PublishRemove1Test_gold create mode 100644 system/t06_publish/PublishRemove2Test_gold create mode 100644 system/t06_publish/PublishRemove3Test_gold create mode 100644 system/t06_publish/PublishRemove4Test_gold create mode 100644 system/t06_publish/remove.py diff --git a/api/publish.go b/api/publish.go index a520878a0..44839e21e 100644 --- a/api/publish.go +++ b/api/publish.go @@ -3,6 +3,7 @@ package api import ( "fmt" "net/http" + "path/filepath" "strings" "github.com/aptly-dev/aptly/aptly" @@ -395,3 +396,86 @@ func apiPublishDrop(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil }) } + +// POST /publish/:prefix/:distribution/remove +func apiPublishRemove(c *gin.Context) { + param := parseEscapedPath(c.Params.ByName("prefix")) + storage, prefix := deb.ParsePrefix(param) + distribution := c.Params.ByName("distribution") + + var b struct { + ForceOverwrite bool + Signing SigningOptions + SkipCleanup *bool + Components []string `binding:"required"` + MultiDist bool + } + + if c.Bind(&b) != nil { + return + } + + signer, err := getSigner(&b.Signing) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to initialize GPG signer: %s", err)) + return + } + + collectionFactory := context.NewCollectionFactory() + collection := collectionFactory.PublishedRepoCollection() + + published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err)) + return + } + err = collection.LoadComplete(published, collectionFactory) + if err != nil { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err)) + return + } + + components := b.Components + + for _, component := range components { + _, exists := published.Sources[component] + if !exists { + AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: '%s' is not a published component", component)) + return + } + published.DropComponent(component) + } + + resources := []string{string(published.Key())} + + taskName := fmt.Sprintf("Remove components '%s' from publish %s (%s)", strings.Join(components, ","), prefix, distribution) + maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, b.MultiDist) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + err = collection.Update(published) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) + } + + if b.SkipCleanup == nil || !*b.SkipCleanup { + publishedStorage := context.GetPublishedStorage(storage) + + err = collection.CleanupPrefixComponentFiles(published.Prefix, components, publishedStorage, collectionFactory, out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + for _, component := range components { + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + } + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil + }) +} diff --git a/api/router.go b/api/router.go index d5b6e6123..cb6350700 100644 --- a/api/router.go +++ b/api/router.go @@ -191,6 +191,7 @@ func Router(c *ctx.AptlyContext) http.Handler { api.POST("/publish/:prefix", apiPublishRepoOrSnapshot) api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch) api.DELETE("/publish/:prefix/:distribution", apiPublishDrop) + api.POST("/publish/:prefix/:distribution/remove", apiPublishRemove) } { diff --git a/cmd/publish.go b/cmd/publish.go index d74384e06..9ab944afb 100644 --- a/cmd/publish.go +++ b/cmd/publish.go @@ -38,6 +38,7 @@ func makeCmdPublish() *commander.Command { makeCmdPublishSwitch(), makeCmdPublishUpdate(), makeCmdPublishShow(), + makeCmdPublishRemove(), }, } } diff --git a/cmd/publish_remove.go b/cmd/publish_remove.go new file mode 100644 index 000000000..5a664107a --- /dev/null +++ b/cmd/publish_remove.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/aptly-dev/aptly/deb" + "github.com/smira/commander" + "github.com/smira/flag" +) + +func aptlyPublishRemove(cmd *commander.Command, args []string) error { + var err error + + if len(args) < 2 { + cmd.Usage() + return commander.ErrCommandError + } + + distribution := args[0] + components := args[1:] + + param := context.Flags().Lookup("prefix").Value.String() + if param == "" { + param = "." + } + storage, prefix := deb.ParsePrefix(param) + + collectionFactory := context.NewCollectionFactory() + published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + + err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + + multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool) + + for _, component := range components { + _, exists := published.Sources[component] + if !exists { + return fmt.Errorf("unable to update: '%s' is not a published component", component) + } + published.DropComponent(component) + } + + signer, err := getSigner(context.Flags()) + if err != nil { + return fmt.Errorf("unable to initialize GPG signer: %s", err) + } + + forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool) + if forceOverwrite { + context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " + + "the same package pool.\n") + } + + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist) + if err != nil { + return fmt.Errorf("unable to publish: %s", err) + } + + err = collectionFactory.PublishedRepoCollection().Update(published) + if err != nil { + return fmt.Errorf("unable to save to DB: %s", err) + } + + skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool) + if !skipCleanup { + publishedStorage := context.GetPublishedStorage(storage) + + err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components, + publishedStorage, collectionFactory, context.Progress()) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + + for _, component := range components { + err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", published.Distribution, component), context.Progress()) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + } + } + + return err +} + +func makeCmdPublishRemove() *commander.Command { + cmd := &commander.Command{ + Run: aptlyPublishRemove, + UsageLine: "remove ...", + Short: "remove component from published repository", + Long: ` +Command removes one or multiple components from a published repository. + +Example: + + $ aptly publish remove -prefix=filesystem:symlink:/debian wheezy contrib non-free +`, + Flag: *flag.NewFlagSet("aptly-publish-remove", flag.ExitOnError), + } + cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release") + cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") + cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") + cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") + cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)") + cmd.Flag.Bool("batch", false, "run GPG with detached tty") + cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG") + cmd.Flag.String("prefix", "", "publishing prefix in the form of [:]") + cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") + cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component") + cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") + + return cmd +} diff --git a/deb/publish.go b/deb/publish.go index 2bdd608de..78faaf2d4 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -439,6 +439,14 @@ func (p *PublishedRepo) SourceNames() []string { return sources } +// DropComponent removes component from published repo +func (p *PublishedRepo) DropComponent(component string) { + delete(p.Sources, component) + delete(p.sourceItems, component) + + p.rePublishing = true +} + // UpdateLocalRepo updates content from local repo in component func (p *PublishedRepo) UpdateLocalRepo(component string) { if p.SourceKind != SourceLocalRepo { diff --git a/system/t06_publish/PublishRemove1Test_gold b/system/t06_publish/PublishRemove1Test_gold new file mode 100644 index 000000000..301aceecb --- /dev/null +++ b/system/t06_publish/PublishRemove1Test_gold @@ -0,0 +1,7 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components c... +Removing ${HOME}/.aptly/public/dists/maverick/c... diff --git a/system/t06_publish/PublishRemove2Test_gold b/system/t06_publish/PublishRemove2Test_gold new file mode 100644 index 000000000..0207b726c --- /dev/null +++ b/system/t06_publish/PublishRemove2Test_gold @@ -0,0 +1,8 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: +Cleaning up prefix "." components b, c... +Removing ${HOME}/.aptly/public/dists/maverick/b... +Removing ${HOME}/.aptly/public/dists/maverick/c... diff --git a/system/t06_publish/PublishRemove3Test_gold b/system/t06_publish/PublishRemove3Test_gold new file mode 100644 index 000000000..9cd8bc8bf --- /dev/null +++ b/system/t06_publish/PublishRemove3Test_gold @@ -0,0 +1 @@ +ERROR: unable to update: 'not-existent' is not a published component diff --git a/system/t06_publish/PublishRemove4Test_gold b/system/t06_publish/PublishRemove4Test_gold new file mode 100644 index 000000000..61f950296 --- /dev/null +++ b/system/t06_publish/PublishRemove4Test_gold @@ -0,0 +1,27 @@ +Usage: aptly publish remove ... + +aptly publish remove - remove component from published repository + + +Options: + -architectures="": list of architectures to consider during (comma-separated), default to all available + -batch: run GPG with detached tty + -config="": location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf) + -db-open-attempts=10: number of attempts to open DB if it's locked by other instance + -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b' + -dep-follow-recommends: when processing dependencies, follow Recommends + -dep-follow-source: when processing dependencies, follow from binary to Source packages + -dep-follow-suggests: when processing dependencies, follow Suggests + -dep-verbose-resolve: when processing dependencies, print detailed logs + -force-overwrite: overwrite files in package pool in case of mismatch + -gpg-key="": GPG key ID to use when signing the release + -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation) + -keyring=: GPG keyring to use (instead of default) + -multi-dist: enable multiple packages with the same filename in different distributions + -passphrase="": GPG passphrase for the key (warning: could be insecure) + -passphrase-file="": GPG passphrase-file for the key (warning: could be insecure) + -prefix="": publishing prefix in the form of [:] + -secret-keyring="": GPG secret keyring to use (instead of default) + -skip-cleanup: don't remove unreferenced files in prefix/component + -skip-signing: don't sign Release files with GPG +ERROR: unable to parse command diff --git a/system/t06_publish/remove.py b/system/t06_publish/remove.py new file mode 100644 index 000000000..88291ed22 --- /dev/null +++ b/system/t06_publish/remove.py @@ -0,0 +1,87 @@ +from lib import BaseTest + + +class PublishRemove1Test(BaseTest): + """ + publish remove: remove single component from published repository + """ + fixtureCmds = [ + "aptly snapshot create snap1 empty", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b,c snap1 snap2 snap3" + ] + runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick c" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishRemove1Test, self).check() + + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/a/binary-i386/Packages') + self.check_exists('public/dists/maverick/b/binary-i386/Packages') + self.check_not_exists('public/dists/maverick/c/binary-i386/Packages') + + release = self.read_file('public/dists/maverick/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + + if ['a', 'b'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'a b')) + + +class PublishRemove2Test(BaseTest): + """ + publish remove: remove multiple components from published repository + """ + fixtureCmds = [ + "aptly snapshot create snap1 empty", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b,c snap1 snap2 snap3" + ] + runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick b c" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishRemove2Test, self).check() + + self.check_exists('public/dists/maverick/Release') + self.check_exists('public/dists/maverick/a/binary-i386/Packages') + self.check_not_exists('public/dists/maverick/b/binary-i386/Packages') + self.check_not_exists('public/dists/maverick/c/binary-i386/Packages') + + release = self.read_file('public/dists/maverick/Release').split('\n') + components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None) + components = sorted(components.split(' ')) + + if ['a'] != components: + raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'a')) + + +class PublishRemove3Test(BaseTest): + """ + publish remove: remove not existing component from published repository + """ + fixtureCmds = [ + "aptly snapshot create snap1 empty", + "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a snap1" + ] + runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick not-existent" + expectedCode = 1 + gold_processor = BaseTest.expand_environ + + +class PublishRemove4Test(BaseTest): + """ + publish remove: unspecified components + """ + fixtureCmds = [ + "aptly snapshot create snap1 empty", + "aptly snapshot create snap2 empty", + "aptly snapshot create snap3 empty", + "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b,c snap1 snap2 snap3" + ] + runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" + expectedCode = 2 + gold_processor = BaseTest.expand_environ diff --git a/system/t12_api/publish.py b/system/t12_api/publish.py index 4ecffbc1c..b06aa6c1b 100644 --- a/system/t12_api/publish.py +++ b/system/t12_api/publish.py @@ -873,3 +873,71 @@ def check(self): get = self.get("/repos/apiandserve/pool/main/b/boost-defaults/i-dont-exist") if get.status_code != 404: raise Exception(f"Expected status 404 != {get.status_code}") + + +class PublishRemoveAPITestRepo(APITest): + """ + POST /publish/:prefix/:distribution/remove + """ + fixtureGpg = True + + def check(self): + snapshot1_name = self.random_name() + snapshot2_name = self.random_name() + snapshot3_name = self.random_name() + + self.check_equal( + self.post("/api/snapshots", json={"Name": snapshot1_name}).status_code, 201) + self.check_equal( + self.post("/api/snapshots", json={"Name": snapshot2_name}).status_code, 201) + self.check_equal( + self.post("/api/snapshots", json={"Name": snapshot3_name}).status_code, 201) + + prefix = self.random_name() + distribution = "stable" + task = self.post_task( + "/api/publish/" + prefix, + json={ + "AcquireByHash": True, + "Architectures": ["i386"], + "SourceKind": "snapshot", + "Sources": [{"Component": "A", "Name": snapshot1_name}, {"Component": "B", "Name": snapshot2_name}, {"Component": "C", "Name": snapshot3_name}], + "Signing": DefaultSigningOptions, + "Distribution": distribution, + "NotAutomatic": "yes", + "ButAutomaticUpgrades": "yes", + "Origin": "earth", + "Label": "fun", + "SkipContents": True, + } + ) + + task = self.post_task("/api/publish/" + prefix + '/' + distribution + '/remove', json={"Components": ["B", "C"]}) + self.check_task(task) + + repo_expected = { + "AcquireByHash": True, + "Architectures": ["i386"], + "Codename": "", + "Distribution": distribution, + "Label": "fun", + "Origin": "earth", + "NotAutomatic": "yes", + "ButAutomaticUpgrades": "yes", + "Path": prefix + '/' + distribution, + "Prefix": prefix, + "SkipContents": True, + "SourceKind": "snapshot", + "Sources": [{"Component": "A", "Name": snapshot1_name}], + "Storage": "", + "Suite": "", + } + + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) + + self.check_exists("public/" + prefix + "/dists/" + distribution + "/Release") + self.check_exists("public/" + prefix + "/dists/" + distribution + "/A/binary-i386/Packages") + self.check_not_exists("public/" + prefix + "/dists/" + distribution + "/B/binary-i386/Packages") + self.check_not_exists("public/" + prefix + "/dists/" + distribution + "/C/binary-i386/Packages") From c0b096eab34605e94bc1e0da525a438173d6a769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 4 Oct 2024 18:13:47 +0200 Subject: [PATCH 2/2] document apiPublishRemove --- api/publish.go | 57 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/api/publish.go b/api/publish.go index 44839e21e..2ae4c9b18 100644 --- a/api/publish.go +++ b/api/publish.go @@ -15,12 +15,18 @@ import ( // SigningOptions is a shared between publish API GPG options structure type SigningOptions struct { - Skip bool - GpgKey string - Keyring string - SecretKeyring string - Passphrase string - PassphraseFile string + // Set "true" to skip signing published repository + Skip bool ` json:"Skip" example:"false"` + // key name (local to aptly service user) + GpgKey string ` json:"GpgKey" example:"reposign@aptly.info"` + // keyring filename (local to aptly service user) + Keyring string ` json:"Keyring" example:"trustedkeys.gpg"` + // OBSOLETE (gpg1): secret keyring filename (local to aptly server user) + SecretKeyring string ` json:"SecretKeyring" example:"aptly.sec"` + // key passphrase (if using over http, would be transmitted in clear text!) + Passphrase string ` json:"Passphrase"` + // passphrase file (local to aptly service user) + PassphraseFile string ` json:"PassphraseFile" example:"/etc/aptly.pass"` } func getSigner(options *SigningOptions) (pgp.Signer, error) { @@ -397,20 +403,41 @@ func apiPublishDrop(c *gin.Context) { }) } -// POST /publish/:prefix/:distribution/remove +type publishRemoveParams struct { + // Signing parameters + Signing SigningOptions ` json:"Signing"` + // Do not remove unreferenced files in prefix/component + SkipCleanup *bool ` json:"SkipCleanup" example:"false"` + // List of Components to remove + Components []string `binding:"required" json:"Components" example:"main"` + // Enable multiple packages with the same filename in different distributions + MultiDist bool ` json:"MultiDist" example:"false"` + // Overwrite existing files in pool/ directory + ForceOverwrite bool ` json:"ForceOverwrite" example:"false"` +} + +// @Summary Remove sources +// @Description **Remove sources (snapshots / local repos) from publish** +// @Description +// @Description This removes specifies sources from a snapshot. +// @Tags Publish +// @Param prefix path string true "Publishing prefix, `.` for root" +// @Param distribution path string true "Distribution name" +// @Consume json +// @Param request body publishRemoveParams true "Parameters" +// @Produce json +// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully" +// @Success 202 {object} task.Task "Mirror is being updated" +// @Failure 400 {object} Error "Unable to determine list of architectures" +// @Failure 404 {object} Error "Mirror not found" +// @Failure 500 {object} Error "Internal Error" +// @Router /api/publish/{prefix}/{distribution}/remove [post] func apiPublishRemove(c *gin.Context) { param := parseEscapedPath(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := c.Params.ByName("distribution") - var b struct { - ForceOverwrite bool - Signing SigningOptions - SkipCleanup *bool - Components []string `binding:"required"` - MultiDist bool - } - + var b publishRemoveParams if c.Bind(&b) != nil { return }