Allow to remove components from a published repository.

This commit introduces the new 'remove' command for a published repository. It allows to remove one or multiple components from a published repository without needing to recreate it.

Signed-off-by: Christoph Fiehe <[email protected]>

api/publish.go

@@ -3,6 +3,7 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"path/filepath"
 	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -395,3 +396,86 @@ func apiPublishDrop(c *gin.Context) {
 		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
 	})
 }
+
+// POST /publish/:prefix/:distribution/remove
+func apiPublishRemove(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	var b struct {
+		ForceOverwrite bool
+		Signing        SigningOptions
+		SkipCleanup    *bool
+		Components     []string `binding:"required"`
+		MultiDist      bool
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+	err = collection.LoadComplete(published, collectionFactory)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	components := b.Components
+
+	for _, component := range components {
+		_, exists := published.Sources[component]
+		if !exists {
+			AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: '%s' is not a published component", component))
+			return
+		}
+		published.DropComponent(component)
+	}
+
+	resources := []string{string(published.Key())}
+
+	taskName := fmt.Sprintf("Remove components '%s' from publish %s (%s)", strings.Join(components, ","), prefix, distribution)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, b.MultiDist)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = collection.Update(published)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
+		}
+
+		if b.SkipCleanup == nil || !*b.SkipCleanup {
+			publishedStorage := context.GetPublishedStorage(storage)
+
+			err = collection.CleanupPrefixComponentFiles(published.Prefix, components, publishedStorage, collectionFactory, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+
+			for _, component := range components {
+				err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+				}
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
+	})
+}

api/router.go

@@ -191,6 +191,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
 		api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
 		api.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.POST("/publish/:prefix/:distribution/remove", apiPublishRemove)
 	}
 
 	{

cmd/publish.go

@@ -38,6 +38,7 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
 			makeCmdPublishShow(),
+			makeCmdPublishRemove(),
 		},
 	}
 }

cmd/publish_remove.go

@@ -0,0 +1,119 @@
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishRemove(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	components := args[1:]
+
+	param := context.Flags().Lookup("prefix").Value.String()
+	if param == "" {
+		param = "."
+	}
+	storage, prefix := deb.ParsePrefix(param)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
+
+	for _, component := range components {
+		_, exists := published.Sources[component]
+		if !exists {
+			return fmt.Errorf("unable to update: '%s' is not a published component", component)
+		}
+		published.DropComponent(component)
+	}
+
+	signer, err := getSigner(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		publishedStorage := context.GetPublishedStorage(storage)
+
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			publishedStorage, collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+
+		for _, component := range components {
+			err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", published.Distribution, component), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to update: %s", err)
+			}
+		}
+	}
+
+	return err
+}
+
+func makeCmdPublishRemove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishRemove,
+		UsageLine: "remove <distribution> <component>...",
+		Short:     "remove component from published repository",
+		Long: `
+Command removes one or multiple components from a published repository.
+
+Example:
+
+  $ aptly publish remove -prefix=filesystem:symlink:/debian wheezy contrib non-free
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-remove", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("prefix", "", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
+
+	return cmd
+}

deb/publish.go

@@ -439,6 +439,14 @@ func (p *PublishedRepo) SourceNames() []string {
 	return sources
 }
 
+// DropComponent removes component from published repo
+func (p *PublishedRepo) DropComponent(component string) {
+	delete(p.Sources, component)
+	delete(p.sourceItems, component)
+
+	p.rePublishing = true
+}
+
 // UpdateLocalRepo updates content from local repo in component
 func (p *PublishedRepo) UpdateLocalRepo(component string) {
 	if p.SourceKind != SourceLocalRepo {

system/t06_publish/PublishRemove1Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." components c...
+Removing ${HOME}/.aptly/public/dists/maverick/c...

system/t06_publish/PublishRemove2Test_gold

@@ -0,0 +1,8 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." components b, c...
+Removing ${HOME}/.aptly/public/dists/maverick/b...
+Removing ${HOME}/.aptly/public/dists/maverick/c...

system/t06_publish/PublishRemove3Test_gold

@@ -0,0 +1 @@
+ERROR: unable to update: 'not-existent' is not a published component

system/t06_publish/PublishRemove4Test_gold

@@ -0,0 +1,27 @@
+Usage: aptly publish remove <distribution> <component>...
+
+aptly publish remove - remove component from published repository
+
+
+Options:
+  -architectures="": list of architectures to consider during (comma-separated), default to all available
+  -batch: run GPG with detached tty
+  -config="": location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf)
+  -db-open-attempts=10: number of attempts to open DB if it's locked by other instance
+  -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b'
+  -dep-follow-recommends: when processing dependencies, follow Recommends
+  -dep-follow-source: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests: when processing dependencies, follow Suggests
+  -dep-verbose-resolve: when processing dependencies, print detailed logs
+  -force-overwrite: overwrite files in package pool in case of mismatch
+  -gpg-key="": GPG key ID to use when signing the release
+  -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation)
+  -keyring=: GPG keyring to use (instead of default)
+  -multi-dist: enable multiple packages with the same filename in different distributions
+  -passphrase="": GPG passphrase for the key (warning: could be insecure)
+  -passphrase-file="": GPG passphrase-file for the key (warning: could be insecure)
+  -prefix="": publishing prefix in the form of [<endpoint>:]<prefix>
+  -secret-keyring="": GPG secret keyring to use (instead of default)
+  -skip-cleanup: don't remove unreferenced files in prefix/component
+  -skip-signing: don't sign Release files with GPG
+ERROR: unable to parse command

system/t06_publish/remove.py

@@ -0,0 +1,87 @@
+from lib import BaseTest
+
+
+class PublishRemove1Test(BaseTest):
+    """
+    publish remove: remove single component from published repository
+    """
+    fixtureCmds = [
+        "aptly snapshot create snap1 empty",
+        "aptly snapshot create snap2 empty",
+        "aptly snapshot create snap3 empty",
+        "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b,c snap1 snap2 snap3"
+    ]
+    runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick c"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishRemove1Test, self).check()
+
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/a/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/b/binary-i386/Packages')
+        self.check_not_exists('public/dists/maverick/c/binary-i386/Packages')
+
+        release = self.read_file('public/dists/maverick/Release').split('\n')
+        components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None)
+        components = sorted(components.split(' '))
+
+        if ['a', 'b'] != components:
+            raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'a b'))
+
+
+class PublishRemove2Test(BaseTest):
+    """
+    publish remove: remove multiple components from published repository
+    """
+    fixtureCmds = [
+        "aptly snapshot create snap1 empty",
+        "aptly snapshot create snap2 empty",
+        "aptly snapshot create snap3 empty",
+        "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b,c snap1 snap2 snap3"
+    ]
+    runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick b c"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishRemove2Test, self).check()
+
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/a/binary-i386/Packages')
+        self.check_not_exists('public/dists/maverick/b/binary-i386/Packages')
+        self.check_not_exists('public/dists/maverick/c/binary-i386/Packages')
+
+        release = self.read_file('public/dists/maverick/Release').split('\n')
+        components = next((e.split(': ')[1] for e in release if e.startswith('Components')), None)
+        components = sorted(components.split(' '))
+
+        if ['a'] != components:
+            raise Exception("value of 'Components' in release file is '%s' and does not match '%s'." % (' '.join(components), 'a'))
+
+
+class PublishRemove3Test(BaseTest):
+    """
+    publish remove: remove not existing component from published repository
+    """
+    fixtureCmds = [
+        "aptly snapshot create snap1 empty",
+        "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a snap1"
+    ]
+    runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick not-existent"
+    expectedCode = 1
+    gold_processor = BaseTest.expand_environ
+
+
+class PublishRemove4Test(BaseTest):
+    """
+    publish remove: unspecified components
+    """
+    fixtureCmds = [
+        "aptly snapshot create snap1 empty",
+        "aptly snapshot create snap2 empty",
+        "aptly snapshot create snap3 empty",
+        "aptly publish snapshot -architectures=i386 -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -component=a,b,c snap1 snap2 snap3"
+    ]
+    runCmd = "aptly publish remove -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick"
+    expectedCode = 2
+    gold_processor = BaseTest.expand_environ