diff --git a/.github/workflows/nushell-bin.yml b/.github/workflows/packages/nushell-bin.yml similarity index 100% rename from .github/workflows/nushell-bin.yml rename to .github/workflows/packages/nushell-bin.yml diff --git a/.github/workflows/pacstall.yml b/.github/workflows/packages/pacstall.yml similarity index 100% rename from .github/workflows/pacstall.yml rename to .github/workflows/packages/pacstall.yml diff --git a/manager.py b/manager.py index 6cb7e3e..cf3426b 100755 --- a/manager.py +++ b/manager.py @@ -8,8 +8,24 @@ import requests import argparse +class LiteralString(str): pass +yaml.add_representer(LiteralString, lambda dumper, data: dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")) + DATABASE_FILE = "packages.json" +valid_distros = [ + "main", + "ubuntu-latest", "ubuntu-rolling", "ubuntu-devel", + "debian-stable", "debian-testing", "debian-unstable" +] + +valid_architectures = ["all", "any", "amd64", "arm64"] + +def adjust_architectures(architectures): + if "any" in architectures: + return ["amd64", "arm64"] + return architectures + def load_database(): if os.path.exists(DATABASE_FILE): with open(DATABASE_FILE, "r") as f: @@ -26,10 +42,20 @@ def remove_package(name): if name in data: del data[name] save_database(data) + workflow_file = f"workflows/packages/{name}.yml" + if os.path.exists(workflow_file): + os.remove(workflow_file) print(f"Package '{name}' removed successfully.") else: print(f"Package '{name}' not found in {DATABASE_FILE}.") +def list_package(name, details): + print(f"Name: {name}") + print(f" Distros: {', '.join(details['distros'])}") + print(f" Architectures: {', '.join(details['architectures'])}") + print(f" Max Overflow: {details['maxOverflow']}") + print(f" Last Updated: {details['lastUpdatedAt']}") + def list_packages(): data = load_database() if not data: @@ -37,122 +63,109 @@ def list_packages(): sys.exit(1) for name, details in data.items(): - print(f"Name: {name}") - print(f" Distros: {', '.join(details['distros'])}") - print(f" Architectures: {', '.join(details['architectures'])}") - -def literal_representer(dumper, data): - return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") - -def adjust_architectures(architectures): - if architectures[0] == "any": - return ["amd64", "arm64"] - return architectures - -def gen_workflows(): - with open(DATABASE_FILE, "r") as f: - packages = json.load(f) - - os.makedirs("workflows", exist_ok=True) - class LiteralString(str): pass - - yaml.add_representer(LiteralString, literal_representer) + list_package(name, details) + +def gen_workflow(package_name, package_data): + os.makedirs("workflows/packages", exist_ok=True) + distros = package_data["distros"] + architectures = adjust_architectures(package_data["architectures"]) + overflow = package_data["maxOverflow"] + matrix_combinations = [ + {"distro": distro, "architecture": arch} + for distro, arch in itertools.product(distros, architectures) + ] - for package_name, package_data in packages.items(): - distros = package_data["distros"] - architectures = adjust_architectures(package_data["architectures"]) - overflow = package_data["maxOverflow"] - - matrix_combinations = [ - {"distro": distro, "architecture": arch} - for distro, arch in itertools.product(distros, architectures) - ] - - workflow_template = { - "name": f"{package_name}", - "on": { - "repository_dispatch": { - "types": [ - f"{package_name}" - ] - }, - "workflow_dispatch": {} + workflow_template = { + "name": f"{package_name}", + "on": { + "repository_dispatch": { + "types": [ + f"{package_name}" + ] }, - "jobs": { - "build": { - "runs-on": "ubuntu-latest", - "strategy": { - "matrix": { - "include": matrix_combinations - } + "workflow_dispatch": {} + }, + "jobs": { + "build": { + "runs-on": "ubuntu-latest", + "strategy": { + "matrix": { + "include": matrix_combinations + } + }, + "steps": [ + { + "name": "Init", + "uses": "actions/checkout@v4" + }, + { + "name": "Set up QEMU", + "uses": "docker/setup-qemu-action@v3" }, - "steps": [ - { - "name": "Init", - "uses": "actions/checkout@v4" - }, - { - "name": "Set up QEMU", - "uses": "docker/setup-qemu-action@v3" - }, - { - "name": "Set up SSH key", - "run": LiteralString( - f"mkdir -p ~/.ssh\n" - f"echo \"${{{{ secrets.SSH_KEY }}}}\" > ~/.ssh/id_ed25519\n" - f"chmod 600 ~/.ssh/id_ed25519\n" - f"ssh-keyscan -H \"${{{{ secrets.SSH_IP }}}}\" >> ~/.ssh/known_hosts" - ) - }, - { - "name": "Package", - "run": LiteralString( - f"mkdir -p out && cd out\n" - f"m_name=\"{package_name}\"\n" - f"m_dist=\"${{{{ matrix.distro }}}}\"\n" - f"m_arch=\"${{{{ matrix.architecture }}}}\"\n" - f"../scripts/packer.sh \"${{m_name}}\" \"${{m_dist}}\" \"${{m_arch}}\"\n" - "debfile=(*${m_arch}.deb)\n" - "echo \"DEBNAME=${debfile}\" >> $GITHUB_ENV" - ) - }, - { - "name": "Upload .deb files", - "uses": "actions/upload-artifact@v4", - "with": { - "name": "${{ env.DEBNAME }}@${{ matrix.distro }}", - "path": "out/${{ env.DEBNAME }}" - } - }, - { - "name": "Upload to server", - "run": LiteralString( - f"LOCATION=\"${{{{ secrets.SSH_USER }}}}@${{{{ secrets.SSH_IP }}}}\"\n" - f"LOCAL_PORT=8080\n" - f"REMOTE_PORT=${{{{ secrets.APTLY_PORT }}}}\n" - f"REPO_URL=\"http://localhost:${{LOCAL_PORT}}/api/repos/ppr-${{{{ matrix.distro }}}}/packages\"\n" - f"ssh -i ~/.ssh/id_ed25519 -fN -L ${{LOCAL_PORT}}:localhost:${{REMOTE_PORT}} \"${{LOCATION}}\"\n" - f"rm_str=\"$(./scripts/checker.sh overflow {package_name} ${{{{ matrix.distro }}}} ${{{{ matrix.architecture }}}} {overflow} ${{REPO_URL}})\"\n" - f"if [ -n \"${{rm_str}}\" ]; then\n echo \"Removing ${{rm_str}}...\"\n" - f" curl -X DELETE -H 'Content-Type: application/json' --data \"{{\\\"PackageRefs\\\": [${{rm_str}}]}}\" \"${{REPO_URL}}\" | jq\nfi\n" - f"curl -X POST -F file=@out/${{{{ env.DEBNAME }}}} \"http://localhost:${{LOCAL_PORT}}/api/files/${{{{ matrix.distro }}}}\" | jq\n" - f"curl -s -X POST -H 'Content-Type: application/json' \\\n" - f" \"http://localhost:${{LOCAL_PORT}}/api/repos/ppr-${{{{ matrix.distro }}}}/file/${{{{ matrix.distro }}}}?forceReplace=1\" | jq\n" - f"curl -X PUT -H 'Content-Type: application/json' --data '{{\"Signing\": {{\"Skip\": false, \"GpgKey\": \"${{{{ secrets.GPG_KEY }}}}\"}}, \"MultiDist\": true, \"ForceOverwrite\": true}}' \"http://localhost:${{LOCAL_PORT}}/api/publish/pacstall/pacstall\" | jq\n" - ) + { + "name": "Set up SSH key", + "run": LiteralString( + f"mkdir -p ~/.ssh\n" + f"echo \"${{{{ secrets.SSH_KEY }}}}\" > ~/.ssh/id_ed25519\n" + f"chmod 600 ~/.ssh/id_ed25519\n" + f"ssh-keyscan -H \"${{{{ secrets.SSH_IP }}}}\" >> ~/.ssh/known_hosts" + ) + }, + { + "name": "Package", + "run": LiteralString( + f"mkdir -p out && cd out\n" + f"m_name=\"{package_name}\"\n" + f"m_dist=\"${{{{ matrix.distro }}}}\"\n" + f"m_arch=\"${{{{ matrix.architecture }}}}\"\n" + f"../scripts/packer.sh \"${{m_name}}\" \"${{m_dist}}\" \"${{m_arch}}\"\n" + "debfile=(*${m_arch}.deb)\n" + "echo \"DEBNAME=${debfile}\" >> $GITHUB_ENV" + ) + }, + { + "name": "Upload .deb files", + "uses": "actions/upload-artifact@v4", + "with": { + "name": "${{ env.DEBNAME }}@${{ matrix.distro }}", + "path": "out/${{ env.DEBNAME }}" } - ] - } + }, + { + "name": "Upload to server", + "run": LiteralString( + f"LOCATION=\"${{{{ secrets.SSH_USER }}}}@${{{{ secrets.SSH_IP }}}}\"\n" + f"LOCAL_PORT=8080\n" + f"REMOTE_PORT=${{{{ secrets.APTLY_PORT }}}}\n" + f"REPO_URL=\"http://localhost:${{LOCAL_PORT}}/api/repos/ppr-${{{{ matrix.distro }}}}/packages\"\n" + f"ssh -i ~/.ssh/id_ed25519 -fN -L ${{LOCAL_PORT}}:localhost:${{REMOTE_PORT}} \"${{LOCATION}}\"\n" + f"rm_str=\"$(./scripts/checker.sh overflow {package_name} ${{{{ matrix.distro }}}} ${{{{ matrix.architecture }}}} {overflow} ${{REPO_URL}})\"\n" + f"if [ -n \"${{rm_str}}\" ]; then\n echo \"Removing ${{rm_str}}...\"\n" + f" curl -X DELETE -H 'Content-Type: application/json' --data \"{{\\\"PackageRefs\\\": [${{rm_str}}]}}\" \"${{REPO_URL}}\" | jq\nfi\n" + f"curl -X POST -F file=@out/${{{{ env.DEBNAME }}}} \"http://localhost:${{LOCAL_PORT}}/api/files/${{{{ matrix.distro }}}}\" | jq\n" + f"curl -s -X POST -H 'Content-Type: application/json' \\\n" + f" \"http://localhost:${{LOCAL_PORT}}/api/repos/ppr-${{{{ matrix.distro }}}}/file/${{{{ matrix.distro }}}}?forceReplace=1\" | jq\n" + f"curl -X PUT -H 'Content-Type: application/json' --data '{{\"Signing\": {{\"Skip\": false, \"GpgKey\": \"${{{{ secrets.GPG_KEY }}}}\"}}, \"MultiDist\": true, \"ForceOverwrite\": true}}' \"http://localhost:${{LOCAL_PORT}}/api/publish/pacstall/pacstall\" | jq\n" + ) + } + ] } } + } - yaml_str = yaml.dump(workflow_template, sort_keys=False, default_flow_style=False) - yaml_str = yaml_str.replace("run: |-", "run: |") - output_file = f"workflows/{package_name}.yml" - with open(output_file, "w") as f: - f.write(yaml_str) + yaml_str = yaml.dump(workflow_template, sort_keys=False, default_flow_style=False) + yaml_str = yaml_str.replace("run: |-", "run: |") + output_file = f"workflows/packages/{package_name}.yml" + with open(output_file, "w") as f: + f.write(yaml_str) - print(f"Generated {output_file}") + print(f"Generated: {output_file}") + +def gen_workflows(): + packages = load_database() + os.makedirs("workflows", exist_ok=True) + for package_name, package_data in packages.items(): + gen_workflow(package_name, package_data) def get_api_data(pkg_name): response = requests.get(f"https://pacstall.dev/api/packages/{pkg_name}") @@ -171,12 +184,33 @@ def get_api_data(pkg_name): if "all" in archopts: archarr = ["all"] elif "any" in archopts: - archarr = ["amd64", "arm64"] + archarr = ["any"] + adjust_architectures(archopts) return last_updated, archarr -def add_or_update_package(name, distros, architectures, overflow): +def alter_package(name, distros, architectures, overflow=5): data = load_database() + package_exists = name in data + if not package_exists: + missing = 0 + if distros is None: + missing = 1 + print(f"Error: missing 'distros'") + if architectures is None: + missing = 1 + print(f"Error: missing 'architectures'") + if (missing == 1): + return + if (overflow < 1): + print(f"Error: 'overflow' must be 1 or greater") + return + + if package_exists: + current_data = data[name] + distros = distros or current_data["distros"] + architectures = architectures or current_data["architectures"] + overflow = overflow or current_data["maxOverflow"] + try: last_updated, available_architectures = get_api_data(name) except ValueError as e: @@ -184,7 +218,7 @@ def add_or_update_package(name, distros, architectures, overflow): return for arch in architectures: - if arch not in available_architectures and "all" not in available_architectures: + if arch not in available_architectures: print(f"Error: '{arch}' is not supported by package '{name}'\nSupported architectures: {', '.join(available_architectures)}") if arch == 'any': print("Note: 'any' packages must specify each arch to build") @@ -192,70 +226,131 @@ def add_or_update_package(name, distros, architectures, overflow): data[name] = { "distros": distros, - "architectures": architectures, - "lastUpdatedAt": last_updated, + "architectures": adjust_architectures(architectures), "maxOverflow": overflow, + "lastUpdatedAt": last_updated, } save_database(data) - print(f"Package '{name}' has been added/updated successfully.") - -def main(): - valid_distros = [ - "main", "ubuntu-latest", "ubuntu-rolling", "ubuntu-devel", - "debian-stable", "debian-testing", "debian-unstable" - ] - - valid_architectures = ["all", "any", "amd64", "arm64"] - - parser = argparse.ArgumentParser(description="PPR Manager") - subparsers = parser.add_subparsers(dest="command", required=True) - - add_parser = subparsers.add_parser("add", help="Add or edit a package") - add_parser.add_argument("name", help="Package name") - add_parser.add_argument("-d", "--distros", required=True, - help="Comma-separated list of distros (e.g., ubuntu-latest,debian-stable)") - add_parser.add_argument("-a", "--architectures", required=True, - help="Comma-separated list of architectures (e.g., amd64,arm64)") - add_parser.add_argument("-o", "--overflow", required=False, type=int, default=5, - help="Integer value of the overflow limit (default 5)") - - remove_parser = subparsers.add_parser("remove", help="Remove a package") - remove_parser.add_argument("name", help="Package name") - - subparsers.add_parser("list", help="List all packages") - - subparsers.add_parser("generate", help="Generate workflows for all packages") - - args = parser.parse_args() - - if args.command == "add": + list_package(name, data[name]) + gen_workflow(name, data[name]) + action = "updated" if package_exists else "added" + print(f"Package has been {action} successfully.") + +def add_command(subparsers, name, aliases, help_text, arguments): + handler = globals().get(f"handle_{name}") + if handler is None: + print(f"Error: No handler defined for command '{name}'") + return + parser = subparsers.add_parser(name, help=f"{help_text} {{aliases: {'|'.join(aliases)}}}") + for arg_name, arg_kwargs in arguments.items(): + parser.add_argument(arg_name, **arg_kwargs) + parser.set_defaults(func=handler) + + for alias in aliases: + parser = subparsers.add_parser(alias) + for arg_name, arg_kwargs in arguments.items(): + parser.add_argument(arg_name, **arg_kwargs) + parser.set_defaults(func=handler) + +def handle_add(args): + if args.distros: distros = [d.strip() for d in args.distros.split(",")] invalid_distros = [d for d in distros if d not in valid_distros] if invalid_distros: print(f"Error: Invalid distros given: {', '.join(invalid_distros)}") print(f"Valid distros are: {', '.join(valid_distros)}") return + else: + distros = None + if args.architectures: architectures = [a.strip() for a in args.architectures.split(",")] invalid_architectures = [a for a in architectures if a not in valid_architectures] if invalid_architectures: print(f"Error: Invalid architectures given: {', '.join(invalid_architectures)}") print(f"Valid architectures are: {', '.join(valid_architectures)}") return + if 'any' in architectures and 'all' in architectures: + print(f"Error: 'any' and 'all' are mutually exclusive.") + return + architectures = adjust_architectures(architectures) + else: + architectures = None + + alter_package(args.name, distros, architectures, args.overflow) + +def handle_remove(args): + remove_package(args.name) - add_or_update_package(args.name, distros, architectures, args.overflow) +def handle_list(args): + list_packages() - elif args.command == "remove": - remove_package(args.name) +def handle_generate(args): + print("Working...") + gen_workflows() + print("Done.") - elif args.command == "list": - list_packages() +def main(): + parser = argparse.ArgumentParser(description="PPR Manager") + subparsers = parser.add_subparsers(dest="command", required=True, metavar='{add|remove|list|generate}') + add_command( + subparsers, + name="add", + aliases=["a", "e", "edit"], + help_text="Add or edit a package", + arguments={ + "name": { + "help": "Package name" + }, + "-d": { + "help": "Comma-separated list of distros (e.g., ubuntu-latest,debian-stable)", + "default": None, + "dest": "distros" + }, + "-a": { + "help": "Comma-separated list of architectures (e.g., amd64,arm64)", + "default": None, + "dest": "architectures" + }, + "-o": { + "help": "Integer value of the overflow limit (default 5)", + "type": int, + "default": 5, + "dest": "overflow" + }, + }, + ) + add_command( + subparsers, + name="remove", + aliases=["r", "rm"], + help_text="Remove a package", + arguments={ + "name": { + "help": "Package name" + }, + }, + ) + + add_command( + subparsers, + name="list", + aliases=["l"], + help_text="List all packages", + arguments={}, + ) + + add_command( + subparsers, + name="generate", + aliases=["g", "gen"], + help_text="Generate workflows for all packages", + arguments={}, + ) - elif args.command == "generate": - print("Working...") - gen_workflows() - print("Done.") + args = parser.parse_args() + args.func(args) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/packages.json b/packages.json index 204c131..9e3ce41 100644 --- a/packages.json +++ b/packages.json @@ -7,8 +7,8 @@ "amd64", "arm64" ], - "lastUpdatedAt": "2025-01-01T23:11:21Z", - "maxOverflow": 5 + "maxOverflow": 5, + "lastUpdatedAt": "2025-01-01T23:11:21Z" }, "pacstall": { "distros": [ @@ -17,7 +17,7 @@ "architectures": [ "all" ], - "lastUpdatedAt": "2024-12-30T06:51:48Z", - "maxOverflow": 5 + "maxOverflow": 5, + "lastUpdatedAt": "2024-12-30T06:51:48Z" } }