From 1913d84e7ebe3a07c987dbed2abba5af7d974145 Mon Sep 17 00:00:00 2001 From: David Duarte Date: Tue, 14 May 2024 20:45:12 +0000 Subject: [PATCH 1/2] python_generator_test: Remove usage of typing_extensions All the used methods are available in the native typing module since python 3.8 --- pdl-compiler/tests/python_generator_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pdl-compiler/tests/python_generator_test.py b/pdl-compiler/tests/python_generator_test.py index 97e4968..21ab14b 100644 --- a/pdl-compiler/tests/python_generator_test.py +++ b/pdl-compiler/tests/python_generator_test.py @@ -21,7 +21,6 @@ import enum import json import typing -import typing_extensions import unittest from importlib import resources @@ -57,12 +56,12 @@ def create_object(typ, value): field_type = field_types[f] values[f] = create_object(field_type, v) return typ(**values) - elif typing_extensions.get_origin(typ) is list: - typ = typing_extensions.get_args(typ)[0] + elif typing.get_origin(typ) is list: + typ = typing.get_args(typ)[0] return [create_object(typ, v) for v in value] - elif typing_extensions.get_origin(typ) is typing.Union: + elif typing.get_origin(typ) is typing.Union: # typing.Optional[int] expands to typing.Union[int, None] - typ = typing_extensions.get_args(typ)[0] + typ = typing.get_args(typ)[0] return create_object(typ, value) if value is not None else None elif typ is bytes: return bytes(value) From 6531d9bb6c14d2592d896620fbe482e3123d032a Mon Sep 17 00:00:00 2001 From: David Duarte Date: Tue, 14 May 2024 20:36:44 +0000 Subject: [PATCH 2/2] Add support for --exclude-declaration to python and cxx generator --- pdl-compiler/scripts/generate_cxx_backend.py | 31 ++++------------ .../scripts/generate_python_backend.py | 13 ++++++- pdl-compiler/tests/run_cxx_generator_tests.sh | 36 +++++++++++++++++++ 3 files changed, 55 insertions(+), 25 deletions(-) diff --git a/pdl-compiler/scripts/generate_cxx_backend.py b/pdl-compiler/scripts/generate_cxx_backend.py index 0bb0d9c..d3abb39 100755 --- a/pdl-compiler/scripts/generate_cxx_backend.py +++ b/pdl-compiler/scripts/generate_cxx_backend.py @@ -1658,7 +1658,7 @@ class {struct_name} : public pdl::packet::Builder {{ def run(input: argparse.FileType, output: argparse.FileType, namespace: Optional[str], include_header: List[str], - using_namespace: List[str]): + using_namespace: List[str], exclude_declaration: List[str]): file = ast.File.from_json(json.load(input)) core.desugar(file) @@ -1668,28 +1668,6 @@ def run(input: argparse.FileType, output: argparse.FileType, namespace: Optional open_namespace = f"namespace {namespace} {{" if namespace else "" close_namespace = f"}} // {namespace}" if namespace else "" - # Disable unsupported features in the canonical test suite. - skipped_decls = [ - 'Packet_Custom_Field_ConstantSize', - 'Packet_Custom_Field_VariableSize', - 'Packet_Checksum_Field_FromStart', - 'Packet_Checksum_Field_FromEnd', - 'Struct_Custom_Field_ConstantSize', - 'Struct_Custom_Field_VariableSize', - 'Struct_Checksum_Field_FromStart', - 'Struct_Checksum_Field_FromEnd', - 'Struct_Custom_Field_ConstantSize_', - 'Struct_Custom_Field_VariableSize_', - 'Struct_Checksum_Field_FromStart_', - 'Struct_Checksum_Field_FromEnd_', - 'PartialParent5', - 'PartialChild5_A', - 'PartialChild5_B', - 'PartialParent12', - 'PartialChild12_A', - 'PartialChild12_B', - ] - output.write( dedent("""\ // File generated from {input_name}, with the command: @@ -1732,7 +1710,7 @@ def run(input: argparse.FileType, output: argparse.FileType, namespace: Optional output.write(f"class {d.id}View;\n") for d in file.declarations: - if d.id in skipped_decls: + if d.id in exclude_declaration: continue if isinstance(d, ast.EnumDeclaration): @@ -1759,6 +1737,11 @@ def main() -> int: default=[], action='append', help='Added using namespace statements') + parser.add_argument('--exclude-declaration', + type=str, + default=[], + action='append', + help='Exclude declaration from the generated output') return run(**vars(parser.parse_args())) diff --git a/pdl-compiler/scripts/generate_python_backend.py b/pdl-compiler/scripts/generate_python_backend.py index 7bd9106..a3a06de 100755 --- a/pdl-compiler/scripts/generate_python_backend.py +++ b/pdl-compiler/scripts/generate_python_backend.py @@ -1157,13 +1157,16 @@ def generate_checksum_declaration_check(decl: ast.ChecksumDeclaration) -> str: """).format(checksum_name=decl.id) -def run(input: argparse.FileType, output: argparse.FileType, custom_type_location: Optional[str]): +def run(input: argparse.FileType, output: argparse.FileType, custom_type_location: Optional[str], exclude_declaration: List[str]): file = ast.File.from_json(json.load(input)) core.desugar(file) custom_types = [] custom_type_checks = "" for d in file.declarations: + if d.id in exclude_declaration: + continue + if isinstance(d, ast.CustomFieldDeclaration): custom_types.append(d.id) custom_type_checks += generate_custom_field_declaration_check(d) @@ -1180,6 +1183,9 @@ def run(input: argparse.FileType, output: argparse.FileType, custom_type_locatio output.write(custom_type_checks) for d in file.declarations: + if d.id in exclude_declaration: + continue + if isinstance(d, ast.EnumDeclaration): output.write(generate_enum_declaration(d)) elif isinstance(d, (ast.PacketDeclaration, ast.StructDeclaration)): @@ -1195,6 +1201,11 @@ def main() -> int: type=str, required=False, help='Module of declaration of custom types') + parser.add_argument('--exclude-declaration', + type=str, + default=[], + action='append', + help='Exclude declaration from the generated output') return run(**vars(parser.parse_args())) diff --git a/pdl-compiler/tests/run_cxx_generator_tests.sh b/pdl-compiler/tests/run_cxx_generator_tests.sh index fa90323..b54b311 100755 --- a/pdl-compiler/tests/run_cxx_generator_tests.sh +++ b/pdl-compiler/tests/run_cxx_generator_tests.sh @@ -18,10 +18,46 @@ pdlc "$OUT_DIR"/be_test_file.pdl > "$OUT_DIR"/be_test_file.json python3 scripts/generate_cxx_backend.py \ --input "$OUT_DIR"/le_test_file.json \ --output "$OUT_DIR"/le_backend.h \ + --exclude-declaration Packet_Custom_Field_ConstantSize \ + --exclude-declaration Packet_Custom_Field_VariableSize \ + --exclude-declaration Packet_Checksum_Field_FromStart \ + --exclude-declaration Packet_Checksum_Field_FromEnd \ + --exclude-declaration Struct_Custom_Field_ConstantSize \ + --exclude-declaration Struct_Custom_Field_VariableSize \ + --exclude-declaration Struct_Checksum_Field_FromStart \ + --exclude-declaration Struct_Checksum_Field_FromEnd \ + --exclude-declaration Struct_Custom_Field_ConstantSize_ \ + --exclude-declaration Struct_Custom_Field_VariableSize_ \ + --exclude-declaration Struct_Checksum_Field_FromStart_ \ + --exclude-declaration Struct_Checksum_Field_FromEnd_ \ + --exclude-declaration PartialParent5 \ + --exclude-declaration PartialChild5_A \ + --exclude-declaration PartialChild5_B \ + --exclude-declaration PartialParent12 \ + --exclude-declaration PartialChild12_A \ + --exclude-declaration PartialChild12_B \ --namespace le_backend python3 scripts/generate_cxx_backend.py \ --input "$OUT_DIR"/be_test_file.json \ --output "$OUT_DIR"/be_backend.h \ + --exclude-declaration Packet_Custom_Field_ConstantSize \ + --exclude-declaration Packet_Custom_Field_VariableSize \ + --exclude-declaration Packet_Checksum_Field_FromStart \ + --exclude-declaration Packet_Checksum_Field_FromEnd \ + --exclude-declaration Struct_Custom_Field_ConstantSize \ + --exclude-declaration Struct_Custom_Field_VariableSize \ + --exclude-declaration Struct_Checksum_Field_FromStart \ + --exclude-declaration Struct_Checksum_Field_FromEnd \ + --exclude-declaration Struct_Custom_Field_ConstantSize_ \ + --exclude-declaration Struct_Custom_Field_VariableSize_ \ + --exclude-declaration Struct_Checksum_Field_FromStart_ \ + --exclude-declaration Struct_Checksum_Field_FromEnd_ \ + --exclude-declaration PartialParent5 \ + --exclude-declaration PartialChild5_A \ + --exclude-declaration PartialChild5_B \ + --exclude-declaration PartialParent12 \ + --exclude-declaration PartialChild12_A \ + --exclude-declaration PartialChild12_B \ --namespace be_backend python3 scripts/generate_cxx_backend_tests.py \