Skip to content

Commit

Permalink
Merge pull request #1162 from aptly-dev/feature/176-snapshot-pull-api
Browse files Browse the repository at this point in the history 
Snapshot Pull API
  • Loading branch information
@neolynx
neolynx authored Oct 3, 2024
2 parents 880f487 + bce54d5 commit d6a156b
Show file tree
Hide file tree
Showing 6 changed files with 484 additions and 105 deletions.
17 changes: 9 additions & 8 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,8 @@ ifeq ($(RUN_LONG_TESTS), yes)
endif

install:
@echo Building aptly ...
@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
go generate
@echo go install -v
@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi

system/env: system/requirements.txt
Expand All @@ -66,32 +65,33 @@ ifeq ($(RUN_LONG_TESTS), yes)
endif

docker-test: ## Run system tests
@echo Building aptly.test ...
@echo "\e[33m\e[1mBuilding aptly.test ...\e[0m"
@rm -f aptly.test
go generate
# install and initialize swagger
test -f $(BINPATH)/swag || go install github.com/swaggo/swag/cmd/swag@latest
PATH=$(BINPATH)/:$(PATH) swag init -q
# build coverage binary
go test -v -coverpkg="./..." -c -tags testruncli
@echo Running python tests ...
@echo "\e[33m\e[1mRunning python tests ...\e[0m"
@test -e aws.creds && . ./aws.creds; \
export PATH=$(BINPATH)/:$(PATH); \
export APTLY_VERSION=$(VERSION); \
$(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)

test: prepare ## Run unit tests
@test -d /srv/etcd || system/t13_etcd/install-etcd.sh
@echo "\nStarting etcd ..."
@echo "\e[33m\e[1mStarting etcd ...\e[0m"
@mkdir -p /tmp/etcd-data; system/t13_etcd/start-etcd.sh > /tmp/etcd-data/etcd.log 2>&1 &
@echo "\nRunning go test ..."
@echo "\e[33m\e[1mRunning go test ...\e[0m"
go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
@echo "\nStopping etcd ..."
@echo "\e[33m\e[1mStopping etcd ...\e[0m"
@pid=`cat /tmp/etcd.pid`; kill $$pid
@rm -f /tmp/etcd-data/etcd.log
@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret

bench:
@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
go test -v ./deb -run=nothing -bench=. -benchmem

mem.png: mem.dat mem.gp
Expand Down Expand Up @@ -157,7 +157,7 @@ dpkg: ## Build debian packages
if [ "$(DEBARCH)" = "amd64" ]; then \
buildtype="any,all" ; \
fi ; \
echo Building: $$buildtype ; \
echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)
# cleanup
@test -f debian/changelog.dpkg-bak && mv debian/changelog.dpkg-bak debian/changelog || true ; \
Expand Down Expand Up @@ -221,5 +221,6 @@ flake8: ## run flake8 on system tests
clean: ## remove local build and module cache
test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true
rm -rf build/ docs/ obj-*-linux-gnu*
rm -f unit.out aptly.test

.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-shell clean releasetype dpkg dev-server docker-dev-server
161 changes: 83 additions & 78 deletions api/mirror.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@ func getVerifier(keyRings []string) (pgp.Verifier, error) {
}

// @Summary Get mirrors
// @Description Show list of currently available mirrors. Each mirror is returned as in “show” API.
// @Description **Show list of currently available mirrors**
// @Description Each mirror is returned as in “show” API.
// @Tags Mirrors
// @Produce json
// @Produce json
// @Success 200 {array} deb.RemoteRepo
// @Router /api/mirrors [get]
func apiMirrorsList(c *gin.Context) {
Expand All @@ -50,45 +51,49 @@ func apiMirrorsList(c *gin.Context) {
c.JSON(200, result)
}

type mirrorCreateParams struct {
// Name of mirror to be created
Name string `binding:"required" json:"Name" example:"mirror2"`
// Url of the archive to mirror
ArchiveURL string `binding:"required" json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Distribution name to mirror
Distribution string ` json:"Distribution" example:"'buster', for flat repositories use './'"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Gpg keyring(s) for verifying Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to mirror installer files
DownloadInstaller bool ` json:"DownloadInstaller"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to skip if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip the verification of architectures
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures bool ` json:"IgnoreSignatures"`
}

// @Summary Create mirror
// @Description Create empty mirror with specified parameters.
// @Description **Create a mirror**
// @Tags Mirrors
// @Accept json
// @Produce json
// @Param Name query string true "mirror name"
// @Param ArchiveURL query string true "url of the archive to mirror e.g. http://deb.debian.org/debian/"
// @Param Distribution query string false "distribution name to mirror e.g. `buster`, for flat repositories use `./` instead of distribution name"
// @Param Filter query string false "package query that is applied to packages in the mirror"
// @Param Components query []string false "components to mirror, if not specified aptly would fetch all components"
// @Param Architectures query []string false "limit mirror to those architectures, if not specified aptly would fetch all architectures"
// @Param Keyrings query []string false "gpg keyring(s) to use when verifying `Release` file"
// @Param DownloadSources query bool false "whether to mirror sources"
// @Param DownloadUdebs query bool false "whether to mirror `.udeb` packages (Debian installer support)"
// @Param DownloadInstaller query bool false "whether to download additional not packaged installer files"
// @Param FilterWithDeps query bool false "when filtering, include dependencies of matching packages as well"
// @Param SkipComponentCheck query bool false "whether to skip if the given components are in the `Release` file"
// @Param IgnoreSignatures query bool false "whether to skip the verification of `Release` file signatures"
// @Consume json
// @Param request body mirrorCreateParams true "Parameters"
// @Produce json
// @Success 200 {object} deb.RemoteRepo
// @Failure 400 {object} Error "Bad Request"
// @Router /api/mirrors [post]
func apiMirrorsCreate(c *gin.Context) {
var err error
var b struct {
Name string `binding:"required"`
ArchiveURL string `binding:"required"`
Distribution string
Filter string
Components []string
Architectures []string
Keyrings []string
DownloadSources bool
DownloadUdebs bool
DownloadInstaller bool
FilterWithDeps bool
SkipComponentCheck bool
SkipArchitectureCheck bool
IgnoreSignatures bool
}
var b mirrorCreateParams

b.DownloadSources = context.Config().DownloadSourcePackages
b.IgnoreSignatures = context.Config().GpgDisableVerify
Expand Down Expand Up @@ -155,12 +160,11 @@ func apiMirrorsCreate(c *gin.Context) {
}

// @Summary Delete Mirror
// @Description Delete a mirror
// @Description **Delete a mirror**
// @Tags Mirrors
// @Consume json
// @Produce json
// @Param name path string true "mirror name"
// @Param force query int true "force: 1 to enable"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue
// @Failure 404 {object} Error "Mirror not found"
// @Failure 403 {object} Error "Unable to delete mirror with snapshots"
Expand Down Expand Up @@ -205,11 +209,10 @@ func apiMirrorsDrop(c *gin.Context) {
}

// @Summary Show Mirror
// @Description Get mirror information by name
// @Description **Get mirror information by name**
// @Tags Mirrors
// @Consume json
// @Produce json
// @Param name path string true "mirror name"
// @Produce json
// @Success 200 {object} deb.RemoteRepo
// @Failure 404 {object} Error "Mirror not found"
// @Failure 500 {object} Error "Internal Error"
Expand All @@ -234,13 +237,12 @@ func apiMirrorsShow(c *gin.Context) {
}

// @Summary List Mirror Packages
// @Description Get a list of packages from a mirror
// @Description **Get a list of packages from a mirror**
// @Tags Mirrors
// @Consume json
// @Produce json
// @Param name path string true "mirror name"
// @Param q query string false "search query"
// @Param format query string false "format: `details` for more detailed information"
// @Produce json
// @Success 200 {array} deb.Package "List of Packages"
// @Failure 400 {object} Error "Unable to determine list of architectures"
// @Failure 404 {object} Error "Mirror not found"
Expand Down Expand Up @@ -323,26 +325,46 @@ func apiMirrorsPackages(c *gin.Context) {
}
}

type mirrorUpdateParams struct {
// Change mirror name to `Name`
Name string ` json:"Name" example:"mirror1"`
// Url of the archive to mirror
ArchiveURL string ` json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Gpg keyring(s) for verifing Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to skip checking if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip checking if the given architectures are in the Release file
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to ignore checksum errors
IgnoreChecksums bool ` json:"IgnoreChecksums"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures bool ` json:"IgnoreSignatures"`
// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
ForceUpdate bool ` json:"ForceUpdate"`
// Set "true" to skip downloading already downloaded packages
SkipExistingPackages bool ` json:"SkipExistingPackages"`
}

// @Summary Update Mirror
// @Description Update Mirror and download packages
// @Description **Update Mirror and download packages**
// @Tags Mirrors
// @Consume json
// @Produce json
// @Param name path string true "mirror name to update"
// @Param Name query string false "change mirror name"
// @Param ArchiveURL query string false "ArchiveURL"
// @Param Filter query string false "Filter"
// @Param Architectures query []string false "Architectures"
// @Param Components query []string false "Components"
// @Param Keyrings query []string false "Keyrings"
// @Param FilterWithDeps query bool false "FilterWithDeps"
// @Param DownloadSources query bool false "DownloadSources"
// @Param DownloadUdebs query bool false "DownloadUdebs"
// @Param SkipComponentCheck query bool false "SkipComponentCheck"
// @Param IgnoreChecksums query bool false "IgnoreChecksums"
// @Param IgnoreSignatures query bool false "IgnoreSignatures"
// @Param ForceUpdate query bool false "ForceUpdate"
// @Param SkipExistingPackages query bool false "SkipExistingPackages"
// @Consume json
// @Param request body mirrorUpdateParams true "Parameters"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
// @Success 202 {object} task.Task "Mirror is being updated"
// @Failure 400 {object} Error "Unable to determine list of architectures"
Expand All @@ -353,26 +375,9 @@ func apiMirrorsUpdate(c *gin.Context) {
var (
err error
remote *deb.RemoteRepo
b mirrorUpdateParams
)

var b struct {
Name string
ArchiveURL string
Filter string
Architectures []string
Components []string
Keyrings []string
FilterWithDeps bool
DownloadSources bool
DownloadUdebs bool
SkipComponentCheck bool
SkipArchitectureCheck bool
IgnoreChecksums bool
IgnoreSignatures bool
ForceUpdate bool
SkipExistingPackages bool
}

collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()

Expand Down
4 changes: 3 additions & 1 deletion api/router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
}

{

api.GET("/publish", apiPublishList)
api.POST("/publish", apiPublishRepoOrSnapshot)
api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
Expand All @@ -200,7 +201,8 @@ func Router(c *ctx.AptlyContext) http.Handler {
api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
api.DELETE("/snapshots/:name", apiSnapshotsDrop)
api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
api.POST("/snapshots/merge", apiSnapshotsMerge)
api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
api.POST("/snapshots/:name/pull", apiSnapshotsPull)
}

{
Expand Down
Loading

0 comments on commit d6a156b

Please sign in to comment.