feature(publish): Allow to remove components from a published repository

api/publish.go

@@ -3,6 +3,7 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"path/filepath"
 	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -14,12 +15,18 @@ import (
 
 // SigningOptions is a shared between publish API GPG options structure
 type SigningOptions struct {
-	Skip           bool
-	GpgKey         string
-	Keyring        string
-	SecretKeyring  string
-	Passphrase     string
-	PassphraseFile string
+	// Set "true" to skip signing published repository
+	Skip bool `             json:"Skip"           example:"false"`
+	// key name (local to aptly service user)
+	GpgKey string `         json:"GpgKey"         example:"[email protected]"`
+	// keyring filename (local to aptly service user)
+	Keyring string `        json:"Keyring"        example:"trustedkeys.gpg"`
+	// OBSOLETE (gpg1): secret keyring filename (local to aptly server user)
+	SecretKeyring string `  json:"SecretKeyring"  example:"aptly.sec"`
+	// key passphrase (if using over http, would be transmitted in clear text!)
+	Passphrase string `     json:"Passphrase"`
+	// passphrase file (local to aptly service user)
+	PassphraseFile string ` json:"PassphraseFile" example:"/etc/aptly.pass"`
 }
 
 func getSigner(options *SigningOptions) (pgp.Signer, error) {
@@ -395,3 +402,107 @@ func apiPublishDrop(c *gin.Context) {
 		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
 	})
 }
+
+type publishRemoveParams struct {
+	// Signing parameters
+	Signing SigningOptions `                 json:"Signing"`
+	// Do not remove unreferenced files in prefix/component
+	SkipCleanup *bool `                      json:"SkipCleanup"     example:"false"`
+	// List of Components to remove
+	Components []string `binding:"required"  json:"Components"      example:"main"`
+	// Enable multiple packages with the same filename in different distributions
+	MultiDist bool `                         json:"MultiDist"       example:"false"`
+	// Overwrite existing files in pool/ directory
+	ForceOverwrite bool `                    json:"ForceOverwrite"  example:"false"`
+}
+
+// @Summary Remove sources
+// @Description **Remove sources (snapshots / local repos) from publish**
+// @Description
+// @Description This removes specifies sources from a snapshot.
+// @Tags Publish
+// @Param prefix path string true "Publishing prefix, `.` for root"
+// @Param distribution path string true "Distribution name"
+// @Consume json
+// @Param request body publishRemoveParams true "Parameters"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
+// @Success 202 {object} task.Task "Mirror is being updated"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/publish/{prefix}/{distribution}/remove [post]
+func apiPublishRemove(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	var b publishRemoveParams
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+	err = collection.LoadComplete(published, collectionFactory)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	components := b.Components
+
+	for _, component := range components {
+		_, exists := published.Sources[component]
+		if !exists {
+			AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: '%s' is not a published component", component))
+			return
+		}
+		published.DropComponent(component)
+	}
+
+	resources := []string{string(published.Key())}
+
+	taskName := fmt.Sprintf("Remove components '%s' from publish %s (%s)", strings.Join(components, ","), prefix, distribution)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, b.MultiDist)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = collection.Update(published)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
+		}
+
+		if b.SkipCleanup == nil || !*b.SkipCleanup {
+			publishedStorage := context.GetPublishedStorage(storage)
+
+			err = collection.CleanupPrefixComponentFiles(published.Prefix, components, publishedStorage, collectionFactory, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+
+			for _, component := range components {
+				err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), out)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+				}
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
+	})
+}

api/router.go

@@ -191,6 +191,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
 		api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
 		api.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.POST("/publish/:prefix/:distribution/remove", apiPublishRemove)
 	}
 
 	{

cmd/publish.go

@@ -38,6 +38,7 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
 			makeCmdPublishShow(),
+			makeCmdPublishRemove(),
 		},
 	}
 }

cmd/publish_remove.go

@@ -0,0 +1,119 @@
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishRemove(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	components := args[1:]
+
+	param := context.Flags().Lookup("prefix").Value.String()
+	if param == "" {
+		param = "."
+	}
+	storage, prefix := deb.ParsePrefix(param)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
+
+	for _, component := range components {
+		_, exists := published.Sources[component]
+		if !exists {
+			return fmt.Errorf("unable to update: '%s' is not a published component", component)
+		}
+		published.DropComponent(component)
+	}
+
+	signer, err := getSigner(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, multiDist)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		publishedStorage := context.GetPublishedStorage(storage)
+
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			publishedStorage, collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+
+		for _, component := range components {
+			err = publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", published.Distribution, component), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to update: %s", err)
+			}
+		}
+	}
+
+	return err
+}
+
+func makeCmdPublishRemove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishRemove,
+		UsageLine: "remove <distribution> <component>...",
+		Short:     "remove component from published repository",
+		Long: `
+Command removes one or multiple components from a published repository.
+
+Example:
+
+  $ aptly publish remove -prefix=filesystem:symlink:/debian wheezy contrib non-free
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-remove", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("prefix", "", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
+
+	return cmd
+}

deb/publish.go

@@ -439,6 +439,14 @@ func (p *PublishedRepo) SourceNames() []string {
 	return sources
 }
 
+// DropComponent removes component from published repo
+func (p *PublishedRepo) DropComponent(component string) {
+	delete(p.Sources, component)
+	delete(p.sourceItems, component)
+
+	p.rePublishing = true
+}
+
 // UpdateLocalRepo updates content from local repo in component
 func (p *PublishedRepo) UpdateLocalRepo(component string) {
 	if p.SourceKind != SourceLocalRepo {

system/t06_publish/PublishRemove1Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." components c...
+Removing ${HOME}/.aptly/public/dists/maverick/c...

system/t06_publish/PublishRemove2Test_gold

@@ -0,0 +1,8 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." components b, c...
+Removing ${HOME}/.aptly/public/dists/maverick/b...
+Removing ${HOME}/.aptly/public/dists/maverick/c...

system/t06_publish/PublishRemove3Test_gold

@@ -0,0 +1 @@
+ERROR: unable to update: 'not-existent' is not a published component

system/t06_publish/PublishRemove4Test_gold

@@ -0,0 +1,27 @@
+Usage: aptly publish remove <distribution> <component>...
+
+aptly publish remove - remove component from published repository
+
+
+Options:
+  -architectures="": list of architectures to consider during (comma-separated), default to all available
+  -batch: run GPG with detached tty
+  -config="": location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf)
+  -db-open-attempts=10: number of attempts to open DB if it's locked by other instance
+  -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b'
+  -dep-follow-recommends: when processing dependencies, follow Recommends
+  -dep-follow-source: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests: when processing dependencies, follow Suggests
+  -dep-verbose-resolve: when processing dependencies, print detailed logs
+  -force-overwrite: overwrite files in package pool in case of mismatch
+  -gpg-key="": GPG key ID to use when signing the release
+  -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation)
+  -keyring=: GPG keyring to use (instead of default)
+  -multi-dist: enable multiple packages with the same filename in different distributions
+  -passphrase="": GPG passphrase for the key (warning: could be insecure)
+  -passphrase-file="": GPG passphrase-file for the key (warning: could be insecure)
+  -prefix="": publishing prefix in the form of [<endpoint>:]<prefix>
+  -secret-keyring="": GPG secret keyring to use (instead of default)
+  -skip-cleanup: don't remove unreferenced files in prefix/component
+  -skip-signing: don't sign Release files with GPG
+ERROR: unable to parse command