ref: consolidate files (#13)

Co-authored-by: ook37 <[email protected]>

.github/workflows/nushell-bin.yml

@@ -46,7 +46,7 @@ jobs:
         REMOTE_PORT=8088
         REPO_URL="http://localhost:${LOCAL_PORT}/api/repos/ppr-${{ matrix.distro }}/packages"
         ssh -i ~/.ssh/id_ed25519 -fN -L ${LOCAL_PORT}:localhost:${REMOTE_PORT} "${LOCATION}"
-        rm_str="$(./scripts/overflow.sh nushell-bin ${{ matrix.distro }} ${{ matrix.architecture }} 5 ${REPO_URL})"
+        rm_str="$(./scripts/checker.sh overflow nushell-bin ${{ matrix.distro }} ${{ matrix.architecture }} 5 ${REPO_URL})"
         if [ -n "${rm_str}" ]; then
           echo "Removing ${rm_str}..."
           curl -X DELETE -H 'Content-Type: application/json' --data "{\"PackageRefs\": [${rm_str}]}" "${REPO_URL}" | jq

.github/workflows/pacstall.yml

@@ -44,7 +44,7 @@ jobs:
         REMOTE_PORT=8088
         REPO_URL="http://localhost:${LOCAL_PORT}/api/repos/ppr-${{ matrix.distro }}/packages"
         ssh -i ~/.ssh/id_ed25519 -fN -L ${LOCAL_PORT}:localhost:${REMOTE_PORT} "${LOCATION}"
-        rm_str="$(./scripts/overflow.sh pacstall ${{ matrix.distro }} ${{ matrix.architecture }} 5 ${REPO_URL})"
+        rm_str="$(./scripts/checker.sh overflow pacstall ${{ matrix.distro }} ${{ matrix.architecture }} 5 ${REPO_URL})"
         if [ -n "${rm_str}" ]; then
           echo "Removing ${rm_str}..."
           curl -X DELETE -H 'Content-Type: application/json' --data "{\"PackageRefs\": [${rm_str}]}" "${REPO_URL}" | jq

.github/workflows/ppr-trigger.yml

@@ -16,12 +16,19 @@ jobs:
       - name: Build Matrix
         id: parse-log
         run: |
+          needed=0
           updates_matrix="$(git log -1 --pretty=%b | awk '/^Update/,0 { if ($0 ~ /^  /) { sub(/:$/, "", $1); print $1 } }' | jq -R -s -c 'split("\n") | map(select(. != ""))')"
           echo "updated='${updates_matrix}'" >> $GITHUB_ENV
-          echo "${updates_matrix}"
+          if [[ ${updates_matrix} == "[]" ]]; then
+            echo "needed=0" >> $GITHUB_ENV
+            exit 0
+          else
+            echo "needed=1" >> $GITHUB_ENV
+            echo "${updates_matrix}"
+          fi
 
       - name: Trigger Workflows
-        if: env.updated != '[]'
+        if: env.needed == '1'
         run: |
           for package in $(echo ${{ env.updated }} | jq -r '.[]'); do
             curl -X POST \

.github/workflows/ppr-watchdog.yml

@@ -8,6 +8,9 @@ on:
   schedule:
     - cron: "0 */6 * * *"
   workflow_dispatch:
+  push:
+    branches:
+      - master
 
 jobs:
   update-check:
@@ -20,10 +23,9 @@ jobs:
           ref: master
 
       - name: Check for updates from API
-        id: update-json
         run: |
           needed=0
-          updates="$(./scripts/checker.sh packages.json updates.json)"
+          updates="$(./scripts/checker.sh updates packages.json)"
           if [[ "${updates}" == "No PPR updates found." ]];  then
             echo "No PPR updates found."
             echo "needed=0" >> $GITHUB_ENV
@@ -40,7 +42,7 @@ jobs:
         uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          add-paths: updates.json
+          add-paths: packages.json
           base: master
           branch: watchdog
           title: "Bump timestamps from API"

README.md

@@ -13,18 +13,18 @@ cd chaotic-PPR
 
 # set up the user for hosting
 mkdir -p ~/.aptly/public
-sed -i "s/\${USER}/${USER}/g" aptly-api.service apache2/aptly.conf
+sed -i "s/\${USER}/${USER}/g" server/systemd/aptly-api.service server/apache2/aptly.conf
 chmod -R o+r ~/.aptly/public
 chmod o+x ~/
 chmod o+x ~/.aptly
-cp ppr-public-key.asc ~/.aptly/public
+cp server/ppr-public-key.asc ~/.aptly/public
 
 # enable and start aptly api
-sudo cp aptly-api.service /etc/systemd/system/
+sudo cp server/systemd/aptly-api.service /etc/systemd/system/
 sudo systemctl enable --now aptly-api
 
 # enable and start apache forwarding
-sudo cp apache2/aptly.conf /etc/apache2/sites-available/aptly.conf
+sudo cp server/apache2/aptly.conf /etc/apache2/sites-available/aptly.conf
 sudo a2ensite aptly.conf
 sudo systemctl reload apache2
 sudo systemctl enable apache2 --now

manager.py

@@ -18,7 +18,8 @@ def load_database():
 
 def save_database(data):
     with open(DATABASE_FILE, "w") as f:
-        json.dump(data, f, indent=4)
+        json.dump(data, f, indent=2)
+        f.write("\n")
 
 def remove_package(name):
     data = load_database()
@@ -60,6 +61,7 @@ class LiteralString(str): pass
     for package_name, package_data in packages.items():
         distros = package_data["distros"]
         architectures = adjust_architectures(package_data["architectures"])
+        overflow = package_data["maxOverflow"]
 
         matrix_combinations = [
             {"distro": distro, "architecture": arch}
@@ -130,7 +132,7 @@ class LiteralString(str): pass
                                 f"REMOTE_PORT=8088\n"
                                 f"REPO_URL=\"http://localhost:${{LOCAL_PORT}}/api/repos/ppr-${{{{ matrix.distro }}}}/packages\"\n"
                                 f"ssh -i ~/.ssh/id_ed25519 -fN -L ${{LOCAL_PORT}}:localhost:${{REMOTE_PORT}} \"${{LOCATION}}\"\n"
-                                f"rm_str=\"$(./scripts/overflow.sh {package_name} ${{{{ matrix.distro }}}} ${{{{ matrix.architecture }}}} 5 ${{REPO_URL}})\"\n"
+                                f"rm_str=\"$(./scripts/checker.sh overflow {package_name} ${{{{ matrix.distro }}}} ${{{{ matrix.architecture }}}} {overflow} ${{REPO_URL}})\"\n"
                                 f"if [ -n \"${{rm_str}}\" ]; then\n  echo \"Removing ${{rm_str}}...\"\n"
                                 f"  curl -X DELETE -H 'Content-Type: application/json' --data \"{{\\\"PackageRefs\\\": [${{rm_str}}]}}\" \"${{REPO_URL}}\" | jq\nfi\n"
                                 f"curl -X POST -F file=@out/${{{{ env.DEBNAME }}}} \"http://localhost:${{LOCAL_PORT}}/api/files/${{{{ matrix.distro }}}}\" | jq\n"
@@ -152,12 +154,14 @@ class LiteralString(str): pass
 
         print(f"Generated {output_file}")
 
-def get_architectures_from_api(pkg_name):
+def get_api_data(pkg_name):
     response = requests.get(f"https://pacstall.dev/api/packages/{pkg_name}")
     if response.status_code != 200:
         raise ValueError(f"Failed to fetch data for package '{pkg_name}'")
 
-    archopts = response.json().get("architectures", [])
+    api_data = response.json()
+    last_updated = api_data.get("lastUpdatedAt")
+    archopts = api_data.get("architectures")
 
     archarr = []
     if "arm64" in archopts or "aarch64" in archopts:
@@ -169,12 +173,12 @@ def get_architectures_from_api(pkg_name):
     elif "any" in archopts:
         archarr = ["amd64", "arm64"]
 
-    return archarr
+    return last_updated, archarr
 
-def add_or_update_package(name, distros, architectures):
+def add_or_update_package(name, distros, architectures, overflow):
     data = load_database()
     try:
-        available_architectures = get_architectures_from_api(name)
+        last_updated, available_architectures = get_api_data(name)
     except ValueError as e:
         print(e)
         return
@@ -188,7 +192,9 @@ def add_or_update_package(name, distros, architectures):
 
     data[name] = {
         "distros": distros,
-        "architectures": architectures
+        "architectures": architectures,
+        "lastUpdatedAt": last_updated,
+        "maxOverflow": overflow,
     }
 
     save_database(data)
@@ -211,6 +217,8 @@ def main():
                             help="Comma-separated list of distros (e.g., ubuntu-latest,debian-stable)")
     add_parser.add_argument("-a", "--architectures", required=True,
                             help="Comma-separated list of architectures (e.g., amd64,arm64)")
+    add_parser.add_argument("-o", "--overflow", required=False, type=int, default=5,
+                            help="Integer value of the overflow limit (default 5)")
 
     remove_parser = subparsers.add_parser("remove", help="Remove a package")
     remove_parser.add_argument("name", help="Package name")
@@ -236,7 +244,7 @@ def main():
             print(f"Valid architectures are: {', '.join(valid_architectures)}")
             return
 
-        add_or_update_package(args.name, distros, architectures)
+        add_or_update_package(args.name, distros, architectures, args.overflow)
 
     elif args.command == "remove":
         remove_package(args.name)

packages.json

@@ -1,19 +1,23 @@
 {
-    "nushell-bin": {
-        "distros": [
-            "main"
-        ],
-        "architectures": [
-            "amd64",
-            "arm64"
-        ]
-    },
-    "pacstall": {
-        "distros": [
-            "main"
-        ],
-        "architectures": [
-            "all"
-        ]
-    }
-}
+  "nushell-bin": {
+    "distros": [
+      "main"
+    ],
+    "architectures": [
+      "amd64",
+      "arm64"
+    ],
+    "lastUpdatedAt": "2025-01-01T23:11:21Z",
+    "maxOverflow": 5
+  },
+  "pacstall": {
+    "distros": [
+      "main"
+    ],
+    "architectures": [
+      "all"
+    ],
+    "lastUpdatedAt": "2024-12-30T06:51:48Z",
+    "maxOverflow": 5
+  }
+}

scripts/checker.sh

@@ -21,14 +21,25 @@
 # You should have received a copy of the GNU General Public License
 # along with Pacstall. If not, see <https://www.gnu.org/licenses/>.
 
+# @description
+#   Checks for updates to packages from the Pacsite API.
+#   If updates are found, returns a git description to be
+#   logged, and updates the database file with the new data.
+# @ci
+#
+# @example
+#   check_updates packages.json
+# @args
+#    $1 string The package database file.
 function check_updates() {
-  local pkgfile="${1}" updfile="${2}" packages details updated stored
+  local pkgfile="${1}" packages details updated stored
   mapfile -t packages < <(jq -r 'keys[]' "${pkgfile}")
   for i in "${packages[@]}"; do
     updated="$(curl -fsSL https://pacstall.dev/api/packages/${i} | jq -r '.lastUpdatedAt')"
-    stored="$(jq -r --arg pkg ${i} '.[$pkg] // "1970-01-01T00:00:00Z"' ${updfile})"
+    stored="$(jq -r --arg pkg ${i} '.[$pkg].lastUpdatedAt' ${pkgfile})"
     if [[ "${updated}" > "${stored}" ]]; then
-      jq --arg pkg "${i}" --arg time "${updated}" '.[$pkg] = $time' "${updfile}" > tmp.json && mv tmp.json "${updfile}"
+      jq --arg pkg "${i}" --arg time "${updated}" \
+        '.[$pkg].lastUpdatedAt = $time' "${pkgfile}" > tmp.json && mv tmp.json "${pkgfile}"
       details+=("${i}: ${stored} -> ${updated}")
     fi
   done
@@ -40,4 +51,40 @@ function check_updates() {
   fi
 }
 
-check_updates "${1}" "${2}"
+# @description
+#   Checks a bound local aptly repo API for overflow
+#   versions of a package. Returns any packages to be
+#   removed to make space for an incoming build.
+# @ci
+#
+# @example
+#   check_overflow foobar main amd64 5 \
+#     http://localhost:1234/api/repos/ppr-main/packages
+# @args
+#    $1 string The package to check overflow of.
+#    $2 string The component repo to check.
+#    $3 string The architecture to check.
+#    $4 int    The maximum number of packages allowed.
+#    $5 string The local API url.
+function check_overflow() {
+  local package="${1}" repo="${2}" architecture="${3}" max="${4}" url="${5}" responses matches removes remove_ref
+  mapfile -t responses < <(curl -s "${url}?q=${package}" | jq -r ".[]" | sort -r)
+  for i in "${responses[@]}"; do
+    [[ "${i}" == "P${architecture}"* ]] && matches+=("${i}")
+  done
+  if ((${#matches[@]}>=max)); then
+    for i in "${!matches[@]}"; do
+      ((i>=max-1)) && removes+=("${matches[${i}]}")
+    done
+  fi
+  if [[ -n ${removes[*]} ]]; then
+    printf -v remove_ref '\"%s\",' "${removes[@]}"
+    echo "${remove_ref%,}"
+  fi
+}
+
+case ${1} in
+  updates) shift 1; check_updates "${1:?PKGFILE required}" ;;
+  overflow) shift 1; check_overflow "${1:?need PKG}" "${2:?need REPO}" "${3:?need ARCH}" "${4:?need MAX}" "${5:?need URL}" ;;
+  *) echo "Usage: ${0} updates|overflow"; exit 1 ;;
+esac