diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b86cd54..48ae120 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 - hooks: - - id: check-yaml - - id: check-toml - - id: mixed-line-ending - args: [ --fix=lf ] - - id: end-of-file-fixer - - id: trailing-whitespace +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: check-toml + - id: mixed-line-ending + args: [ --fix=lf ] + - id: end-of-file-fixer + - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.6 hooks: diff --git a/pyproject.toml b/pyproject.toml index eeb1883..04ae597 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,8 @@ ignore_missing_imports = true [tool.poetry.scripts] check-stl-corruption = "voron_ci.tools.stl_corruption_checker:main" check-stl-rotation = "voron_ci.tools.stl_rotation_checker:main" -check-files = "voron_ci.tools.file_checker:main" +check-mod-structure = "voron_ci.tools.mod_structure_checker:main" +check-whitespace = "voron_ci.tools.whitespace_checker:main" upload-images = "voron_ci.tools.imagekit_uploader:main" generate-readme = "voron_ci.tools.readme_generator:main" debug-container = "voron_ci.utils.debug_container:print_container_info" diff --git a/voron_ci/tools/file_checker.py b/voron_ci/tools/mod_structure_checker.py similarity index 90% rename from voron_ci/tools/file_checker.py rename to voron_ci/tools/mod_structure_checker.py index e120c68..4b97db6 100644 --- a/voron_ci/tools/file_checker.py +++ b/voron_ci/tools/mod_structure_checker.py @@ -14,12 +14,6 @@ logger = init_logging(__name__) -STEP_SUMMARY_PREAMBLE = """ -## Folder check - -""" - - class FileErrors(StrEnum): file_outside_mod_folder = "The file '{}' is located outside the expected folder structure of `printer_mods/user/mod`" mod_missing_metadata = "The mod '{}' does not have a metadata.yml file" @@ -30,7 +24,7 @@ class FileErrors(StrEnum): MOD_DEPTH = 2 -class FileChecker: +class ModStructureChecker: def __init__(self: Self, args: argparse.Namespace) -> None: self.input_dir: Path = Path(Path.cwd(), args.input_dir) self.verbosity: bool = args.verbose @@ -84,14 +78,16 @@ def run(self: Self) -> None: if self.print_gh_step_summary: self.check_summary = [(path.relative_to(self.input_dir).as_posix(), reason) for path, reason in self.errors.items()] - GithubActionHelper.print_summary_table( - preamble=STEP_SUMMARY_PREAMBLE, - columns=[ - "File/Folder", - "Reason", - ], - rows=self.check_summary, - ) + with GithubActionHelper.expandable_section( + title=f"Mod structure check (errors: {len(self.errors)})", default_open=self.return_status == ReturnStatus.SUCCESS + ): + GithubActionHelper.print_summary_table( + columns=[ + "File/Folder", + "Reason", + ], + rows=self.check_summary, + ) GithubActionHelper.write_output(output={"extended-outcome": EXTENDED_OUTCOME[self.return_status]}) @@ -138,7 +134,7 @@ def main() -> None: default=False, ) args: argparse.Namespace = parser.parse_args() - FileChecker(args=args).run() + ModStructureChecker(args=args).run() if __name__ == "__main__": diff --git a/voron_ci/tools/readme_generator.py b/voron_ci/tools/readme_generator.py index b1d15e8..2b28e97 100644 --- a/voron_ci/tools/readme_generator.py +++ b/voron_ci/tools/readme_generator.py @@ -70,10 +70,8 @@ def run(self: Self) -> None: ) ) prev_username = mod["creator"] - - GithubActionHelper.print_summary_table( - preamble="# Printer Readme Preview", columns=["Creator", "Mod title", "Description", "Printer compatibility", "Last Changed"], rows=readme_rows - ) + with GithubActionHelper.expandable_section(title="README.md preview", default_open=True): + GithubActionHelper.print_summary_table(columns=["Creator", "Mod title", "Description", "Printer compatibility", "Last Changed"], rows=readme_rows) if self.json_path: logger.info("Writing json file to '%s'", self.json_path) diff --git a/voron_ci/tools/stl_corruption_checker.py b/voron_ci/tools/stl_corruption_checker.py index 78d3ae4..1f7b326 100644 --- a/voron_ci/tools/stl_corruption_checker.py +++ b/voron_ci/tools/stl_corruption_checker.py @@ -29,6 +29,7 @@ def __init__(self: Self, args: argparse.Namespace) -> None: self.print_gh_step_summary: bool = args.github_step_summary self.return_status: ReturnStatus = ReturnStatus.SUCCESS self.check_summary: list[tuple[str, ...]] = [] + self.error_count: int = 0 def run(self: Self) -> None: if self.verbosity: @@ -46,11 +47,13 @@ def run(self: Self) -> None: self.return_status = ReturnStatus.SUCCESS if self.print_gh_step_summary: - GithubActionHelper.print_summary_table( - preamble=STEP_SUMMARY_PREAMBLE, - columns=["Filename", "Result", "Edges Fixed", "Backwards Edges", "Degenerate Facets", "Facets Removed", "Facets Added", "Facets Reversed"], - rows=self.check_summary, - ) + with GithubActionHelper.expandable_section( + title=f"STL corruption check (errors: {self.error_count})", default_open=self.return_status == ReturnStatus.SUCCESS + ): + GithubActionHelper.print_summary_table( + columns=["Filename", "Result", "Edges Fixed", "Backwards Edges", "Degenerate Facets", "Facets Removed", "Facets Added", "Facets Reversed"], + rows=self.check_summary, + ) GithubActionHelper.write_output(output={"extended-outcome": EXTENDED_OUTCOME[self.return_status]}) @@ -94,6 +97,7 @@ def _check_stl(self: Self, stl_file_path: Path) -> ReturnStatus: stl=stl, path=Path(self.output_dir, stl_file_path.relative_to(self.input_dir)), ) + self.error_count += 1 return ReturnStatus.FAILURE logger.info("STL '%s' does not contain any errors!", stl_file_path.relative_to(self.input_dir).as_posix()) self.check_summary.append( @@ -105,6 +109,7 @@ def _check_stl(self: Self, stl_file_path: Path) -> ReturnStatus: self.check_summary.append( (stl_file_path.name, SummaryStatus.EXCEPTION, "0", "0", "0", "0", "0", "0"), ) + self.error_count += 1 return ReturnStatus.EXCEPTION diff --git a/voron_ci/tools/stl_rotation_checker.py b/voron_ci/tools/stl_rotation_checker.py index 648d297..27f4c53 100644 --- a/voron_ci/tools/stl_rotation_checker.py +++ b/voron_ci/tools/stl_rotation_checker.py @@ -18,12 +18,6 @@ logger = init_logging(__name__) - -STEP_SUMMARY_PREAMBLE = """ -## STL rotation check summary - -""" - TWEAK_THRESHOLD = 0.1 @@ -39,6 +33,7 @@ def __init__(self: Self, args: argparse.Namespace) -> None: self.file_handler: FileHandler.FileHandler = FileHandler.FileHandler() self.return_status: ReturnStatus = ReturnStatus.SUCCESS self.check_summary: list[tuple[str, ...]] = [] + self.error_count: int = 0 @staticmethod def get_random_string(length: int) -> str: @@ -78,17 +73,19 @@ def run(self: Self) -> None: with ThreadPoolExecutor() as pool: return_statuses: list[ReturnStatus] = list(pool.map(self._check_stl, stl_paths)) + if return_statuses: self.return_status = max(*return_statuses, self.return_status) else: self.return_status = ReturnStatus.SUCCESS - if self.print_gh_step_summary: - GithubActionHelper.print_summary_table( - preamble=STEP_SUMMARY_PREAMBLE, - columns=["Filename", "Result", "Current orientation", "Suggested orientation"], - rows=self.check_summary, - ) + with GithubActionHelper.expandable_section( + title=f"STL rotation check (errors: {self.error_count})", default_open=self.return_status == ReturnStatus.SUCCESS + ): + GithubActionHelper.print_summary_table( + columns=["Filename", "Result", "Current orientation", "Suggested orientation"], + rows=self.check_summary, + ) GithubActionHelper.write_output(output={"extended-outcome": EXTENDED_OUTCOME[self.return_status]}) @@ -112,6 +109,7 @@ def _check_stl(self: Self, stl_file_path: Path) -> ReturnStatus: if len(mesh_objects.items()) > 1: logger.warning("File '%s' contains multiple objects and is therefore skipped!", stl_file_path.relative_to(self.input_dir).as_posix()) self.check_summary.append((stl_file_path.name, SummaryStatus.WARNING, "", "")) + self.error_count += 1 return ReturnStatus.WARNING rotated_mesh: Tweak = Tweak(mesh_objects[0]["mesh"], extended_mode=True, verbose=False, min_volume=True) original_image_url: str = self.make_markdown_image(base_dir=self.input_dir, stl_file_path=stl_file_path.relative_to(self.input_dir)) @@ -131,12 +129,14 @@ def _check_stl(self: Self, stl_file_path: Path) -> ReturnStatus: rotated_image_url, ), ) + self.error_count += 1 return ReturnStatus.WARNING self.check_summary.append((stl_file_path.name, SummaryStatus.SUCCESS, original_image_url, "")) return ReturnStatus.SUCCESS except Exception as e: logger.exception("A fatal error occurred during rotation checking", exc_info=e) self.check_summary.append((stl_file_path.name, SummaryStatus.EXCEPTION, "", "")) + self.error_count += 1 return ReturnStatus.EXCEPTION diff --git a/voron_ci/tools/whitespace_checker.py b/voron_ci/tools/whitespace_checker.py new file mode 100644 index 0000000..8ffd6f0 --- /dev/null +++ b/voron_ci/tools/whitespace_checker.py @@ -0,0 +1,133 @@ +import argparse +import os +import string +import sys +from typing import Self + +from voron_ci.contants import EXTENDED_OUTCOME, ReturnStatus +from voron_ci.utils.github_action_helper import GithubActionHelper +from voron_ci.utils.logging import init_logging + +logger = init_logging(__name__) + +STEP_SUMMARY_PREAMBLE = """ +## Whitespace check errors + +""" + + +class WhitespaceChecker: + def __init__(self: Self, args: argparse.Namespace) -> None: + self.input_env_var: str = args.input_env_var + self.output_gh_var: str = args.output_gh_var + self.verbosity: bool = args.verbose + self.fail_on_error: bool = args.fail_on_error + self.print_gh_step_summary: bool = args.github_step_summary + self.return_status: ReturnStatus = ReturnStatus.SUCCESS + self.check_summary: list[tuple[str, ...]] = [] + self.error_count: int = 0 + + def _check_for_whitespace(self: Self) -> None: + input_file_list = os.environ.get(self.input_env_var, "").splitlines() + + for input_file in input_file_list: + if not input_file: + continue + logger.info("Checking file '%s' for whitespace!", input_file) + for c in input_file: + if c in string.whitespace: + logger.error("File '%s' contains whitespace!", input_file) + self.check_summary.append((input_file, "This file contains whitespace!")) + self.error_count += 1 + self.return_status = ReturnStatus.FAILURE + break + + def _write_sanitized_output(self: Self) -> None: + input_file_list = os.environ.get(self.input_env_var, "").splitlines() + + output_file_list: list[str] = [input_file.replace("[", "\\[").replace("]", "\\]") for input_file in input_file_list] + + GithubActionHelper.write_output_multiline(output={self.output_gh_var: output_file_list}) + + def run(self: Self) -> None: + if self.verbosity: + logger.setLevel("INFO") + + logger.info("Starting files check from env var '%s'", self.input_env_var) + + self._check_for_whitespace() + + if self.output_gh_var: + self._write_sanitized_output() + + if self.print_gh_step_summary: + with GithubActionHelper.expandable_section( + title=f"Whitespace checks (errors: {self.error_count})", default_open=self.return_status != ReturnStatus.SUCCESS + ): + GithubActionHelper.print_summary_table( + columns=[ + "File/Folder", + "Reason", + ], + rows=self.check_summary, + ) + + GithubActionHelper.write_output(output={"extended-outcome": EXTENDED_OUTCOME[self.return_status]}) + + if self.return_status > ReturnStatus.SUCCESS and self.fail_on_error: + logger.error("Error detected during whitespace checking!") + sys.exit(255) + + +def main() -> None: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + prog="VoronDesign VoronUsers whitespace checker", + description="This tool is used to check changed files inside an env var for whitespace. The list is also prepared for sparse-checkout", + ) + parser.add_argument( + "-i", + "--input_env_var", + required=True, + action="store", + type=str, + help="Environment variable name containing a newline separated list of files to be checked", + ) + parser.add_argument( + "-v", + "--verbose", + required=False, + action="store_true", + help="Print debug output to stdout", + default=False, + ) + parser.add_argument( + "-f", + "--fail_on_error", + required=False, + action="store_true", + help="Whether to return an error exit code if one of the files contains whitespace", + default=False, + ) + parser.add_argument( + "-o", + "--output_gh_var", + required=False, + action="store", + type=str, + help="Github output variable to store a sanitized list of files into", + default="", + ) + parser.add_argument( + "-g", + "--github_step_summary", + required=False, + action="store_true", + help="Whether to output a step summary when running inside a github action", + default=False, + ) + args: argparse.Namespace = parser.parse_args() + WhitespaceChecker(args=args).run() + + +if __name__ == "__main__": + main() diff --git a/voron_ci/utils/github_action_helper.py b/voron_ci/utils/github_action_helper.py index 4644ffa..ed50473 100644 --- a/voron_ci/utils/github_action_helper.py +++ b/voron_ci/utils/github_action_helper.py @@ -1,7 +1,9 @@ +import contextlib import datetime import logging import os import zipfile +from collections.abc import Generator from http import HTTPStatus from io import BytesIO from pathlib import Path @@ -18,23 +20,35 @@ class GithubActionHelper: @classmethod - def create_table_header(cls: type[Self], columns: list[str]) -> str: + @contextlib.contextmanager + def expandable_section(cls: type[Self], title: str, *, default_open: bool) -> Generator[None, None, None]: + with Path(os.environ[STEP_SUMMARY_ENV_VAR]).open(mode="a") as gh_step_summary: + try: + gh_step_summary.write(f"\n") + gh_step_summary.write(f"{title}\n\n") + gh_step_summary.flush() + yield + finally: + gh_step_summary.write("\n") + + @classmethod + def _create_table_header(cls: type[Self], columns: list[str]) -> str: column_names = "| " + " | ".join(columns) + " |" dividers = "| " + " | ".join(["---"] * len(columns)) + " |" return f"{column_names}\n{dividers}" @classmethod - def create_markdown_table_rows(cls: type[Self], rows: list[tuple[str, ...]]) -> str: + def _create_markdown_table_rows(cls: type[Self], rows: list[tuple[str, ...]]) -> str: return "\n".join(["| " + " | ".join(row_elements) + " |" for row_elements in rows]) @classmethod def create_markdown_table(cls: type[Self], preamble: str, columns: list[str], rows: list[tuple[str, ...]]) -> str: - return "\n".join([preamble, GithubActionHelper.create_table_header(columns=columns), GithubActionHelper.create_markdown_table_rows(rows=rows)]) + return "\n".join([preamble, GithubActionHelper._create_table_header(columns=columns), GithubActionHelper._create_markdown_table_rows(rows=rows)]) @classmethod - def print_summary_table(cls: type[Self], preamble: str, columns: list[str], rows: list[tuple[str, ...]]) -> None: + def print_summary_table(cls: type[Self], columns: list[str], rows: list[tuple[str, ...]]) -> None: with Path(os.environ[STEP_SUMMARY_ENV_VAR]).open(mode="a") as gh_step_summary: - gh_step_summary.write(GithubActionHelper.create_markdown_table(preamble=preamble, columns=columns, rows=rows)) + gh_step_summary.write(f"{GithubActionHelper._create_table_header(columns=columns)}\n{GithubActionHelper._create_markdown_table_rows(rows=rows)}\n") @classmethod def get_job_id(cls: type[Self], github_repository: str, github_run_id: str, job_name: str) -> str: @@ -65,6 +79,15 @@ def write_output(cls: type[Self], output: dict[str, str]) -> None: for key, value in output.items(): gh_output.write(f"{key}={value}\n") + @classmethod + def write_output_multiline(cls: type[Self], output: dict[str, list[str]]) -> None: + with Path(os.environ[OUTPUT_ENV_VAR]).open(mode="a") as gh_output: + for key, value in output.items(): + gh_output.write(f"{key}< str: try: