From 92f1d2d3cdb05bcfb2587e05c15e08651c017425 Mon Sep 17 00:00:00 2001 From: oklopfer <104327997+oklopfer@users.noreply.github.com> Date: Thu, 9 Jan 2025 01:56:07 -0500 Subject: [PATCH] ref: consolidate files (#13) Co-authored-by: ook37 --- .github/workflows/nushell-bin.yml | 2 +- .github/workflows/pacstall.yml | 2 +- .github/workflows/ppr-trigger.yml | 11 +++- .github/workflows/ppr-watchdog.yml | 8 ++- README.md | 8 +-- manager.py | 26 ++++++--- packages.json | 40 ++++++++------ scripts/checker.sh | 55 +++++++++++++++++-- scripts/overflow.sh | 41 -------------- {apache2 => server/apache2}/aptly.conf | 0 .../ppr-public-key.asc | 0 .../systemd/aptly-api.service | 0 updates.json | 4 -- 13 files changed, 110 insertions(+), 87 deletions(-) delete mode 100755 scripts/overflow.sh rename {apache2 => server/apache2}/aptly.conf (100%) rename ppr-public-key.asc => server/ppr-public-key.asc (100%) rename aptly-api.service => server/systemd/aptly-api.service (100%) delete mode 100644 updates.json diff --git a/.github/workflows/nushell-bin.yml b/.github/workflows/nushell-bin.yml index 4230daf..7a1ab52 100644 --- a/.github/workflows/nushell-bin.yml +++ b/.github/workflows/nushell-bin.yml @@ -46,7 +46,7 @@ jobs: REMOTE_PORT=8088 REPO_URL="http://localhost:${LOCAL_PORT}/api/repos/ppr-${{ matrix.distro }}/packages" ssh -i ~/.ssh/id_ed25519 -fN -L ${LOCAL_PORT}:localhost:${REMOTE_PORT} "${LOCATION}" - rm_str="$(./scripts/overflow.sh nushell-bin ${{ matrix.distro }} ${{ matrix.architecture }} 5 ${REPO_URL})" + rm_str="$(./scripts/checker.sh overflow nushell-bin ${{ matrix.distro }} ${{ matrix.architecture }} 5 ${REPO_URL})" if [ -n "${rm_str}" ]; then echo "Removing ${rm_str}..." curl -X DELETE -H 'Content-Type: application/json' --data "{\"PackageRefs\": [${rm_str}]}" "${REPO_URL}" | jq diff --git a/.github/workflows/pacstall.yml b/.github/workflows/pacstall.yml index 6be8d8c..5c468be 100644 --- a/.github/workflows/pacstall.yml +++ b/.github/workflows/pacstall.yml @@ -44,7 +44,7 @@ jobs: REMOTE_PORT=8088 REPO_URL="http://localhost:${LOCAL_PORT}/api/repos/ppr-${{ matrix.distro }}/packages" ssh -i ~/.ssh/id_ed25519 -fN -L ${LOCAL_PORT}:localhost:${REMOTE_PORT} "${LOCATION}" - rm_str="$(./scripts/overflow.sh pacstall ${{ matrix.distro }} ${{ matrix.architecture }} 5 ${REPO_URL})" + rm_str="$(./scripts/checker.sh overflow pacstall ${{ matrix.distro }} ${{ matrix.architecture }} 5 ${REPO_URL})" if [ -n "${rm_str}" ]; then echo "Removing ${rm_str}..." curl -X DELETE -H 'Content-Type: application/json' --data "{\"PackageRefs\": [${rm_str}]}" "${REPO_URL}" | jq diff --git a/.github/workflows/ppr-trigger.yml b/.github/workflows/ppr-trigger.yml index 19b23ae..7a6f4f3 100644 --- a/.github/workflows/ppr-trigger.yml +++ b/.github/workflows/ppr-trigger.yml @@ -16,12 +16,19 @@ jobs: - name: Build Matrix id: parse-log run: | + needed=0 updates_matrix="$(git log -1 --pretty=%b | awk '/^Update/,0 { if ($0 ~ /^ /) { sub(/:$/, "", $1); print $1 } }' | jq -R -s -c 'split("\n") | map(select(. != ""))')" echo "updated='${updates_matrix}'" >> $GITHUB_ENV - echo "${updates_matrix}" + if [[ ${updates_matrix} == "[]" ]]; then + echo "needed=0" >> $GITHUB_ENV + exit 0 + else + echo "needed=1" >> $GITHUB_ENV + echo "${updates_matrix}" + fi - name: Trigger Workflows - if: env.updated != '[]' + if: env.needed == '1' run: | for package in $(echo ${{ env.updated }} | jq -r '.[]'); do curl -X POST \ diff --git a/.github/workflows/ppr-watchdog.yml b/.github/workflows/ppr-watchdog.yml index b988d39..1e7d814 100644 --- a/.github/workflows/ppr-watchdog.yml +++ b/.github/workflows/ppr-watchdog.yml @@ -8,6 +8,9 @@ on: schedule: - cron: "0 */6 * * *" workflow_dispatch: + push: + branches: + - master jobs: update-check: @@ -20,10 +23,9 @@ jobs: ref: master - name: Check for updates from API - id: update-json run: | needed=0 - updates="$(./scripts/checker.sh packages.json updates.json)" + updates="$(./scripts/checker.sh updates packages.json)" if [[ "${updates}" == "No PPR updates found." ]]; then echo "No PPR updates found." echo "needed=0" >> $GITHUB_ENV @@ -40,7 +42,7 @@ jobs: uses: peter-evans/create-pull-request@v7 with: token: ${{ secrets.GITHUB_TOKEN }} - add-paths: updates.json + add-paths: packages.json base: master branch: watchdog title: "Bump timestamps from API" diff --git a/README.md b/README.md index 0775c5a..ae66a0a 100644 --- a/README.md +++ b/README.md @@ -13,18 +13,18 @@ cd chaotic-PPR # set up the user for hosting mkdir -p ~/.aptly/public -sed -i "s/\${USER}/${USER}/g" aptly-api.service apache2/aptly.conf +sed -i "s/\${USER}/${USER}/g" server/systemd/aptly-api.service server/apache2/aptly.conf chmod -R o+r ~/.aptly/public chmod o+x ~/ chmod o+x ~/.aptly -cp ppr-public-key.asc ~/.aptly/public +cp server/ppr-public-key.asc ~/.aptly/public # enable and start aptly api -sudo cp aptly-api.service /etc/systemd/system/ +sudo cp server/systemd/aptly-api.service /etc/systemd/system/ sudo systemctl enable --now aptly-api # enable and start apache forwarding -sudo cp apache2/aptly.conf /etc/apache2/sites-available/aptly.conf +sudo cp server/apache2/aptly.conf /etc/apache2/sites-available/aptly.conf sudo a2ensite aptly.conf sudo systemctl reload apache2 sudo systemctl enable apache2 --now diff --git a/manager.py b/manager.py index d12a793..12d09c5 100755 --- a/manager.py +++ b/manager.py @@ -18,7 +18,8 @@ def load_database(): def save_database(data): with open(DATABASE_FILE, "w") as f: - json.dump(data, f, indent=4) + json.dump(data, f, indent=2) + f.write("\n") def remove_package(name): data = load_database() @@ -60,6 +61,7 @@ class LiteralString(str): pass for package_name, package_data in packages.items(): distros = package_data["distros"] architectures = adjust_architectures(package_data["architectures"]) + overflow = package_data["maxOverflow"] matrix_combinations = [ {"distro": distro, "architecture": arch} @@ -130,7 +132,7 @@ class LiteralString(str): pass f"REMOTE_PORT=8088\n" f"REPO_URL=\"http://localhost:${{LOCAL_PORT}}/api/repos/ppr-${{{{ matrix.distro }}}}/packages\"\n" f"ssh -i ~/.ssh/id_ed25519 -fN -L ${{LOCAL_PORT}}:localhost:${{REMOTE_PORT}} \"${{LOCATION}}\"\n" - f"rm_str=\"$(./scripts/overflow.sh {package_name} ${{{{ matrix.distro }}}} ${{{{ matrix.architecture }}}} 5 ${{REPO_URL}})\"\n" + f"rm_str=\"$(./scripts/checker.sh overflow {package_name} ${{{{ matrix.distro }}}} ${{{{ matrix.architecture }}}} {overflow} ${{REPO_URL}})\"\n" f"if [ -n \"${{rm_str}}\" ]; then\n echo \"Removing ${{rm_str}}...\"\n" f" curl -X DELETE -H 'Content-Type: application/json' --data \"{{\\\"PackageRefs\\\": [${{rm_str}}]}}\" \"${{REPO_URL}}\" | jq\nfi\n" f"curl -X POST -F file=@out/${{{{ env.DEBNAME }}}} \"http://localhost:${{LOCAL_PORT}}/api/files/${{{{ matrix.distro }}}}\" | jq\n" @@ -152,12 +154,14 @@ class LiteralString(str): pass print(f"Generated {output_file}") -def get_architectures_from_api(pkg_name): +def get_api_data(pkg_name): response = requests.get(f"https://pacstall.dev/api/packages/{pkg_name}") if response.status_code != 200: raise ValueError(f"Failed to fetch data for package '{pkg_name}'") - archopts = response.json().get("architectures", []) + api_data = response.json() + last_updated = api_data.get("lastUpdatedAt") + archopts = api_data.get("architectures") archarr = [] if "arm64" in archopts or "aarch64" in archopts: @@ -169,12 +173,12 @@ def get_architectures_from_api(pkg_name): elif "any" in archopts: archarr = ["amd64", "arm64"] - return archarr + return last_updated, archarr -def add_or_update_package(name, distros, architectures): +def add_or_update_package(name, distros, architectures, overflow): data = load_database() try: - available_architectures = get_architectures_from_api(name) + last_updated, available_architectures = get_api_data(name) except ValueError as e: print(e) return @@ -188,7 +192,9 @@ def add_or_update_package(name, distros, architectures): data[name] = { "distros": distros, - "architectures": architectures + "architectures": architectures, + "lastUpdatedAt": last_updated, + "maxOverflow": overflow, } save_database(data) @@ -211,6 +217,8 @@ def main(): help="Comma-separated list of distros (e.g., ubuntu-latest,debian-stable)") add_parser.add_argument("-a", "--architectures", required=True, help="Comma-separated list of architectures (e.g., amd64,arm64)") + add_parser.add_argument("-o", "--overflow", required=False, type=int, default=5, + help="Integer value of the overflow limit (default 5)") remove_parser = subparsers.add_parser("remove", help="Remove a package") remove_parser.add_argument("name", help="Package name") @@ -236,7 +244,7 @@ def main(): print(f"Valid architectures are: {', '.join(valid_architectures)}") return - add_or_update_package(args.name, distros, architectures) + add_or_update_package(args.name, distros, architectures, args.overflow) elif args.command == "remove": remove_package(args.name) diff --git a/packages.json b/packages.json index d756cb4..204c131 100644 --- a/packages.json +++ b/packages.json @@ -1,19 +1,23 @@ { - "nushell-bin": { - "distros": [ - "main" - ], - "architectures": [ - "amd64", - "arm64" - ] - }, - "pacstall": { - "distros": [ - "main" - ], - "architectures": [ - "all" - ] - } -} \ No newline at end of file + "nushell-bin": { + "distros": [ + "main" + ], + "architectures": [ + "amd64", + "arm64" + ], + "lastUpdatedAt": "2025-01-01T23:11:21Z", + "maxOverflow": 5 + }, + "pacstall": { + "distros": [ + "main" + ], + "architectures": [ + "all" + ], + "lastUpdatedAt": "2024-12-30T06:51:48Z", + "maxOverflow": 5 + } +} diff --git a/scripts/checker.sh b/scripts/checker.sh index ec05451..404c18f 100755 --- a/scripts/checker.sh +++ b/scripts/checker.sh @@ -21,14 +21,25 @@ # You should have received a copy of the GNU General Public License # along with Pacstall. If not, see . +# @description +# Checks for updates to packages from the Pacsite API. +# If updates are found, returns a git description to be +# logged, and updates the database file with the new data. +# @ci +# +# @example +# check_updates packages.json +# @args +# $1 string The package database file. function check_updates() { - local pkgfile="${1}" updfile="${2}" packages details updated stored + local pkgfile="${1}" packages details updated stored mapfile -t packages < <(jq -r 'keys[]' "${pkgfile}") for i in "${packages[@]}"; do updated="$(curl -fsSL https://pacstall.dev/api/packages/${i} | jq -r '.lastUpdatedAt')" - stored="$(jq -r --arg pkg ${i} '.[$pkg] // "1970-01-01T00:00:00Z"' ${updfile})" + stored="$(jq -r --arg pkg ${i} '.[$pkg].lastUpdatedAt' ${pkgfile})" if [[ "${updated}" > "${stored}" ]]; then - jq --arg pkg "${i}" --arg time "${updated}" '.[$pkg] = $time' "${updfile}" > tmp.json && mv tmp.json "${updfile}" + jq --arg pkg "${i}" --arg time "${updated}" \ + '.[$pkg].lastUpdatedAt = $time' "${pkgfile}" > tmp.json && mv tmp.json "${pkgfile}" details+=("${i}: ${stored} -> ${updated}") fi done @@ -40,4 +51,40 @@ function check_updates() { fi } -check_updates "${1}" "${2}" +# @description +# Checks a bound local aptly repo API for overflow +# versions of a package. Returns any packages to be +# removed to make space for an incoming build. +# @ci +# +# @example +# check_overflow foobar main amd64 5 \ +# http://localhost:1234/api/repos/ppr-main/packages +# @args +# $1 string The package to check overflow of. +# $2 string The component repo to check. +# $3 string The architecture to check. +# $4 int The maximum number of packages allowed. +# $5 string The local API url. +function check_overflow() { + local package="${1}" repo="${2}" architecture="${3}" max="${4}" url="${5}" responses matches removes remove_ref + mapfile -t responses < <(curl -s "${url}?q=${package}" | jq -r ".[]" | sort -r) + for i in "${responses[@]}"; do + [[ "${i}" == "P${architecture}"* ]] && matches+=("${i}") + done + if ((${#matches[@]}>=max)); then + for i in "${!matches[@]}"; do + ((i>=max-1)) && removes+=("${matches[${i}]}") + done + fi + if [[ -n ${removes[*]} ]]; then + printf -v remove_ref '\"%s\",' "${removes[@]}" + echo "${remove_ref%,}" + fi +} + +case ${1} in + updates) shift 1; check_updates "${1:?PKGFILE required}" ;; + overflow) shift 1; check_overflow "${1:?need PKG}" "${2:?need REPO}" "${3:?need ARCH}" "${4:?need MAX}" "${5:?need URL}" ;; + *) echo "Usage: ${0} updates|overflow"; exit 1 ;; +esac diff --git a/scripts/overflow.sh b/scripts/overflow.sh deleted file mode 100755 index bbbf2a9..0000000 --- a/scripts/overflow.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# ____ __ ____ -# / __ \____ ___________/ /_____ _/ / / -# / /_/ / __ `/ ___/ ___/ __/ __ `/ / / -# / ____/ /_/ / /__(__ ) /_/ /_/ / / / -# /_/ \__,_/\___/____/\__/\__,_/_/_/ -# -# Copyright (C) 2020-present -# -# This file is part of Pacstall -# -# Pacstall is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 3 of the License -# -# Pacstall is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Pacstall. If not, see . - -function check_overflow() { - local package="${1}" repo="${2}" architecture="${3}" max="${4}" url="${5}" responses matches removes remove_ref - mapfile -t responses < <(curl -s "${url}?q=${package}" | jq -r ".[]" | sort -r) - for i in "${responses[@]}"; do - [[ "${i}" == "P${architecture}"* ]] && matches+=("${i}") - done - if ((${#matches[@]}>=max)); then - for i in "${!matches[@]}"; do - ((i>=max-1)) && removes+=("${matches[${i}]}") - done - fi - if [[ -n ${removes[*]} ]]; then - printf -v remove_ref '\"%s\",' "${removes[@]}" - echo "${remove_ref%,}" - fi -} - -check_overflow "${1}" "${2}" "${3}" "${4}" "${5}" diff --git a/apache2/aptly.conf b/server/apache2/aptly.conf similarity index 100% rename from apache2/aptly.conf rename to server/apache2/aptly.conf diff --git a/ppr-public-key.asc b/server/ppr-public-key.asc similarity index 100% rename from ppr-public-key.asc rename to server/ppr-public-key.asc diff --git a/aptly-api.service b/server/systemd/aptly-api.service similarity index 100% rename from aptly-api.service rename to server/systemd/aptly-api.service diff --git a/updates.json b/updates.json deleted file mode 100644 index 9c658d0..0000000 --- a/updates.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "nushell-bin": "2025-01-01T23:11:21Z", - "pacstall": "2024-12-30T06:51:48Z" -}