Skip to content

Commit

Permalink
Merge pull request #1352 from aptly-dev/feature/storage-api
Browse files Browse the repository at this point in the history
Feature/storage api
  • Loading branch information
neolynx authored Oct 3, 2024
2 parents aff7b0d + dbc336f commit 880f487
Show file tree
Hide file tree
Showing 17 changed files with 128 additions and 42 deletions.
13 changes: 7 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ jobs:
with:
directory: ${{ runner.temp }}

- name: Make
- name: "Run System Tests"
env:
RUN_LONG_TESTS: 'yes'
AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
Expand All @@ -82,12 +82,12 @@ jobs:
sudo mkdir -p /srv ; sudo chown runner /srv
COVERAGE_DIR=${{ runner.temp }} make all
- name: Merge code coverage
- name: "Merge code coverage"
run: |
go install github.com/wadey/gocovmerge@latest
~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
- name: Upload code coverage
- name: "Upload code coverage"
uses: codecov/codecov-action@v2
with:
token: ${{ secrets.CODECOV_TOKEN }}
Expand Down Expand Up @@ -185,15 +185,15 @@ jobs:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
./upload-artifacts.sh ci ${{ matrix.suite }}
.github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}
- name: "Publish release to aptly"
if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
./upload-artifacts.sh release ${{ matrix.suite }}
.github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
ci-binary-build:
name: "Build"
Expand Down Expand Up @@ -247,7 +247,8 @@ jobs:
run: |
make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}
- uses: actions/upload-artifact@v4
- name: "Upload Artifacts"
uses: actions/upload-artifact@v4
if: startsWith(github.event.ref, 'refs/tags')
with:
name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
Expand Down
File renamed without changes.
18 changes: 9 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ prepare: ## Install go module dependencies
go mod download
# install and initialize swagger
go install github.com/swaggo/swag/cmd/swag@latest
PATH=$(BINPATH)/:$(PATH) swag init
PATH=$(BINPATH)/:$(PATH) swag init -q
go mod verify
go mod tidy -v
go generate
Expand Down Expand Up @@ -70,8 +70,8 @@ docker-test: ## Run system tests
@rm -f aptly.test
go generate
# install and initialize swagger
go install github.com/swaggo/swag/cmd/swag@latest
PATH=$(BINPATH)/:$(PATH) swag init
test -f $(BINPATH)/swag || go install github.com/swaggo/swag/cmd/swag@latest
PATH=$(BINPATH)/:$(PATH) swag init -q
# build coverage binary
go test -v -coverpkg="./..." -c -tags testruncli
@echo Running python tests ...
Expand Down Expand Up @@ -126,7 +126,7 @@ releasetype: # Print release type (ci/release)
build: ## Build aptly
# install and initialize swagger
unset GOBIN; go install github.com/swaggo/swag/cmd/swag@latest
PATH=$(BINPATH)/:$(PATH) swag init
PATH=$(BINPATH)/:$(PATH) swag init -q
# prepare
go mod tidy
go generate
Expand All @@ -137,15 +137,15 @@ dev-server: prepare ## Run dev-server
go install github.com/air-verse/[email protected]
cp debian/aptly.conf /var/lib/aptly/.aptly.conf
sed -i /enableSwaggerEndpoint/s/false/true/ /var/lib/aptly/.aptly.conf
PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142
PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142

dpkg: ## Build debian packages
@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
# go generate
GOPATH=$$PWD/.go go generate -v
# install and initialize swagger
go install github.com/swaggo/swag/cmd/swag@latest
PATH=$(BINPATH)/:$(PATH) swag init
PATH=$(BINPATH)/:$(PATH) swag init -q
# set debian version
@if [ "`make -s releasetype`" = "ci" ]; then \
echo CI Build, setting version... ; \
Expand All @@ -169,7 +169,7 @@ binaries: ## Build binary releases (FreeBSD, MacOS, Linux tar)
@make version > VERSION
# install and initialize swagger
GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest
PATH=$(BINPATH)/:$(PATH) swag init
PATH=$(BINPATH)/:$(PATH) swag init -q
# build aptly
GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
# install
Expand All @@ -194,7 +194,7 @@ docker-image: ## Build aptly-dev docker image
docker-build: ## Build aptly in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd make build

docker-aptly: ## Build and run aptly commands in docker container
docker-shell: ## Run aptly and other commands in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd

docker-deb: ## Build debian packages in docker container
Expand Down Expand Up @@ -222,4 +222,4 @@ clean: ## remove local build and module cache
test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true
rm -rf build/ docs/ obj-*-linux-gnu*

.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-aptly clean releasetype dpkg dev-server docker-dev-server
.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-shell clean releasetype dpkg dev-server docker-dev-server
1 change: 1 addition & 0 deletions api/publish.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ func parseEscapedPath(path string) string {
// @Tags Publish
// @Produce json
// @Success 200 {array} deb.PublishedRepo
// @Failure 500 {object} Error "Internal Error"
// @Router /api/publish [get]
func apiPublishList(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
Expand Down
1 change: 1 addition & 0 deletions api/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
api.GET("/metrics", apiMetricsGet())
}
api.GET("/version", apiVersion)
api.GET("/storage", apiDiskFree)

isReady := &atomic.Value{}
isReady.Store(false)
Expand Down
43 changes: 43 additions & 0 deletions api/storage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package api

import (
"fmt"
"syscall"

"github.com/gin-gonic/gin"
)

type diskFree struct {
// Storage size [MiB]
Total uint64
// Available Storage [MiB]
Free uint64
// Percentage Full
PercentFull float32
}

// @Summary Get Storage Utilization
// @Description **Get disk free information of aptly storage**
// @Tags Status
// @Produce json
// @Success 200 {object} diskFree "Storage information"
// @Failure 400 {object} Error "Internal Error"
// @Router /api/storage [get]
func apiDiskFree(c *gin.Context) {
var df diskFree

fs := context.Config().GetRootDir()

var stat syscall.Statfs_t
err := syscall.Statfs(fs, &stat)
if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
return
}

df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0

c.JSON(200, df)
}
10 changes: 6 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -101,16 +101,18 @@ require (
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
go.etcd.io/etcd/api/v3 v3.5.0-rc.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.15 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect
go.uber.org/multierr v1.10.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/grpc v1.64.1 // indirect
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand All @@ -127,5 +129,5 @@ require (
github.com/swaggo/files v1.0.1
github.com/swaggo/gin-swagger v1.6.0
github.com/swaggo/swag v1.16.3
go.etcd.io/etcd/client/v3 v3.5.0-rc.0
go.etcd.io/etcd/client/v3 v3.5.15
)
12 changes: 12 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -365,10 +365,16 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/etcd/api/v3 v3.5.0-rc.0 h1:oPIG9qBFXiGJFM1StoFd5EVllfVIhd7pPjB/mcMCzCg=
go.etcd.io/etcd/api/v3 v3.5.0-rc.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk=
go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM=
go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0 h1:7mW2vCG+tthhWwORSD/G0eesRZ6ZkOh19P7CyrT9xfA=
go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA=
go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU=
go.etcd.io/etcd/client/v3 v3.5.0-rc.0 h1:W2gUtoUho3+X5tCffYA3VfA7fH9DokrhGMscvZskuXc=
go.etcd.io/etcd/client/v3 v3.5.0-rc.0/go.mod h1:5+cKiK2IEMa230cLFrLL3+3sKlPPKASjiLOmG8tZ8vY=
go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4=
go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
Expand Down Expand Up @@ -529,6 +535,12 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4=
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
Expand Down
1 change: 1 addition & 0 deletions system/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg a
apt-get clean && rm -rf /var/lib/apt/lists/*

RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly
RUN sed -i 's/#force_color_prompt=yes/force_color_prompt=yes/' /var/lib/aptly/.bashrc

RUN mkdir /work
WORKDIR /work
Expand Down
3 changes: 3 additions & 0 deletions system/build-deb
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ chown -R `stat -c %u /work/src` /var/lib/aptly

su aptly -c 'set -e; cd /work/src;
GOPATH=$PWD/.go go generate -v
# install and initialize swagger
GOPATH=$PWD/.go go install github.com/swaggo/swag/cmd/swag@latest
PATH=$PWD/.go/bin:$PATH swag init
git checkout debian/changelog
DEBEMAIL="CI <runner@github>" dch -v `make version` "CI build"
dpkg-buildpackage -us -uc -b -d
Expand Down
40 changes: 31 additions & 9 deletions system/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import traceback
import random
import subprocess
import time

from lib import BaseTest
from s3_lib import S3Test
Expand Down Expand Up @@ -76,15 +77,15 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
if name in testignore:
continue

testout.clear()

o = getattr(testModule, name)

if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest and
o is not SwiftTest and o is not S3Test and o is not AzureTest and
o is not APITest and o is not FileSystemEndpointTest):
continue

testout.clear()
start_time = time.time()

newBase = o.__bases__[0]
if lastBase is not None and lastBase is not newBase:
lastBase.shutdown_class()
Expand All @@ -102,7 +103,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
if not matches:
continue

orig_stdout.write("%s: %s ... " % (test, colored(o.__name__, color="yellow", attrs=["bold"])))
orig_stdout.write("· %-13s ➔ %-48s ... " % (test, colored(o.__name__, color="yellow", attrs=["bold"])))
orig_stdout.flush()

t = o()
Expand All @@ -118,20 +119,41 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non

numTests += 1

failed = False
t.captureResults = capture_results
t.coverage_dir = coverage_dir
typ = None
val = None
tb = None
try:
t.captureResults = capture_results
t.coverage_dir = coverage_dir
t.test()
except Exception:
numFailed += 1
typ, val, tb = sys.exc_info()
failed = True

end_time = time.time()
execution_time = int(end_time - start_time) + 1
minutes = execution_time // 60
seconds = execution_time % 60
if minutes > 0:
minutes = f"{minutes}m"
if seconds < 10:
seconds = f"0{seconds}"
else:
minutes = " "
if seconds < 10:
seconds = f" {seconds}"
duration = f"{minutes}{seconds}s"

if failed:
numFailed += 1
fails.append((test, t, typ, val, tb, testModule))
orig_stdout.write(colored("\b\b\b\bFAIL\n", color="red", attrs=["bold"]))
orig_stdout.write(colored("\b\b\b\bFAIL", color="red", attrs=["bold"]) + f" {duration}\n")

orig_stdout.write(testout.get_contents())
traceback.print_exception(typ, val, tb, file=orig_stdout)
else:
orig_stdout.write(colored("\b\b\b\bOK \n", color="green", attrs=["bold"]))
orig_stdout.write(colored("\b\b\b\bOK", color="green", attrs=["bold"]) + f" {duration}\n")

t.shutdown()

Expand Down
File renamed without changes.
1 change: 0 additions & 1 deletion system/t04_mirror/edit.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,5 @@ class EditMirror10Test(BaseTest):
"""
edit mirror: change archive url
"""
requiresFTP = True
fixtureCmds = ["aptly mirror create -ignore-signatures mirror10 http://repo.aptly.info/system-tests/ftp.ru.debian.org/debian bookworm main"]
runCmd = "aptly mirror edit -ignore-signatures -archive-url http://repo.aptly.info/system-tests/ftp.ch.debian.org/debian mirror10"
2 changes: 1 addition & 1 deletion system/t04_mirror/update.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ class UpdateMirror10Test(BaseTest):
outputMatchPrepare = filterOutSignature


class UpdateMirror11Test(BaseTest):
class UpdateMirror11FTPTest(BaseTest):
"""
update mirrors: update over FTP
"""
Expand Down
4 changes: 2 additions & 2 deletions system/t06_publish/repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -671,12 +671,12 @@ class PublishRepo26Test(BaseTest):
"""
publish repo: sign with passphrase
"""
skipTest = "Failing on CI"
fixtureCmds = [
"aptly repo create local-repo",
"aptly repo add local-repo ${files}",
"gpg --import --batch --passphrase verysecret ${files}/aptly_passphrase.sec"
]
runCmd = "aptly publish repo -keyring=${files}/aptly_passphrase.pub -secret-keyring=${files}/aptly_passphrase.sec -passphrase=verysecret -distribution=maverick local-repo"
runCmd = "aptly publish repo -batch -keyring=${files}/aptly_passphrase.pub -passphrase=verysecret -distribution=maverick local-repo"
gold_processor = BaseTest.expand_environ

def outputMatchPrepare(_, s):
Expand Down
11 changes: 11 additions & 0 deletions system/t12_api/storage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
from api_lib import APITest


class TaskAPITestSwaggerDocs(APITest):
"""
GET /docs
"""

def check(self):
resp = self.get("/api/storage")
self.check_equal(resp.status_code, 200)
Loading

0 comments on commit 880f487

Please sign in to comment.