From c89c284463e5ab7df2539a4172c3a82f74b791bd Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Thu, 3 Feb 2022 14:57:08 +0100 Subject: [PATCH] python: simplify memory management scripts - merge functions to simplify control flow - clarify variable names - add comments - Improve code readability Signed-off-by: Axel Heider --- tools/hardware/outputs/c_header.py | 129 ++++++++++------------- tools/hardware/outputs/yaml.py | 24 ++--- tools/hardware/utils/memory.py | 164 +++++++++++++++++------------ 3 files changed, 158 insertions(+), 159 deletions(-) diff --git a/tools/hardware/outputs/c_header.py b/tools/hardware/outputs/c_header.py index 63fb990c4d..8addc7ec64 100644 --- a/tools/hardware/outputs/c_header.py +++ b/tools/hardware/outputs/c_header.py @@ -12,7 +12,8 @@ import hardware from hardware.config import Config from hardware.fdt import FdtParser -from hardware.utils.rule import HardwareYaml +from hardware.memory import Region +from hardware.utils.rule import HardwareYaml, KernelInterrupt HEADER_TEMPLATE = '''/* @@ -48,7 +49,7 @@ #define {{ irq.label }} {{ irq.irq }} {% if irq.has_sel() %} #else -#define {{ irq.label }} {{ irq.false_irq }} +#define {{ irq.label }} {{ irq.false_irq }} /* dummy value */ {{ irq.get_sel_endif() }} {% endif %} {% if irq.has_enable() %} @@ -57,7 +58,7 @@ {% endfor -%} /* KERNEL DEVICES */ -{% for (addr, macro) in sorted(kernel_macros.items()) %} +{% for (macro, addr) in kernel_dev_addr_macros %} #define {{ macro }} (KDEV_BASE + {{ "0x{:x}".format(addr) }}) {% endfor %} @@ -67,17 +68,13 @@ {% if group.has_macro() %} {{ group.get_macro() }} {% endif %} - /* {{ group.get_desc() }} */ + /* {{ group.get_desc() }} + * contains {{ ', '.join(group.labels.keys()) }} + */ {% for reg in group.regions %} { .paddr = {{ "0x{:x}".format(reg.base) }}, - {% set map_addr = group.get_map_offset(reg) %} - {% if map_addr in kernel_macros %} - .pptr = {{ kernel_macros[map_addr] }}, - {% else %} - /* contains {{ ', '.join(group.labels.keys()) }} */ - .pptr = KDEV_BASE + {{ "0x{:x}".format(map_addr) }}, - {% endif %} + .pptr = KDEV_BASE + {{ "0x{:x}".format(group.get_map_offset(reg)) }}, {% if config.arch == 'arm' %} .armExecuteNever = true, {% endif %} @@ -122,64 +119,10 @@ ''' -def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml) -> (List, Dict): - ''' - Given a device tree and a set of rules, returns a tuple (groups, offsets). - - Groups is a list of 'KernelRegionGroups', each of which represents a single - contiguous region of memory that is associated with a device. - Offsets is a dict of offset -> label, where label is the name given to the - kernel for that address (e.g. SERIAL_PPTR) and offset is the offset from - KDEV_BASE at which it's mapped. - ''' - kernel_devices = tree.get_kernel_devices() - - kernel_offset = 0 - groups = [] - for dev in kernel_devices: - dev_rule = hw_yaml.get_rule(dev) - new_regions = dev_rule.get_regions(dev) - for reg in new_regions: - if reg in groups: - other = groups[groups.index(reg)] - other.take_labels(reg) - else: - groups.append(reg) - - offsets = {} - for group in groups: - kernel_offset = group.set_kernel_offset(kernel_offset) - offsets.update(group.get_labelled_addresses()) - return (groups, offsets) - - -def get_interrupts(tree: FdtParser, hw_yaml: HardwareYaml) -> List: - ''' Get dict of interrupts, {label: KernelInterrupt} from the DT and hardware rules. ''' - kernel_devices = tree.get_kernel_devices() - - irqs = [] - for dev in kernel_devices: - dev_rule = hw_yaml.get_rule(dev) - if len(dev_rule.interrupts.items()) > 0: - irqs += dev_rule.get_interrupts(tree, dev) - - ret = {} - for irq in irqs: - if irq.label in ret: - if irq.prio > ret[irq.label].prio: - ret[irq.label] = irq - else: - ret[irq.label] = irq - - ret = list(ret.values()) - ret.sort(key=lambda a: a.label) - return ret - - -def create_c_header_file(config, kernel_irqs: List, kernel_macros: Dict, - kernel_regions: List, physBase: int, physical_memory, - outputStream): - +def create_c_header_file(config, kernel_irqs: List[KernelInterrupt], + kernel_dev_addr_macros: Dict[str, int], + kernel_regions: List[Region], physBase: int, + physical_memory: List[Region], outputStream): jinja_env = jinja2.Environment(loader=jinja2.BaseLoader, trim_blocks=True, lstrip_blocks=True) @@ -189,7 +132,7 @@ def create_c_header_file(config, kernel_irqs: List, kernel_macros: Dict, **{ 'config': config, 'kernel_irqs': kernel_irqs, - 'kernel_macros': kernel_macros, + 'kernel_dev_addr_macros': kernel_dev_addr_macros, 'kernel_regions': kernel_regions, 'physBase': physBase, 'physical_memory': physical_memory}) @@ -203,13 +146,51 @@ def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, args: argparse.N if not args.header_out: raise ValueError('You need to specify a header-out to use c header output') - physical_memory, reserved, physBase = hardware.utils.memory.get_physical_memory(tree, config) - kernel_regions, kernel_macros = get_kernel_devices(tree, hw_yaml) + # We only care about the available physical memory and the kernel's phys + # base. The device memory regions are not relevant here. + physical_memory, _, physBase = hardware.utils.memory.get_phys_mem_regions(tree, + config, + hw_yaml) + + # Collect the interrupts and kernel regions for the devices. + kernel_irq_dict = {} # dict of 'label:irq_obj' + kernel_regions = [] # list of Regions. + for dev in tree.get_kernel_devices(): + dev_rule = hw_yaml.get_rule(dev) + + if len(dev_rule.interrupts.items()) > 0: + for irq in dev_rule.get_interrupts(tree, dev): + # Add the interrupt if it does not exists or overwrite an + # existing entry if the priority for this device is higher + if (not irq.label in kernel_irq_dict) or \ + (irq.prio > kernel_irq_dict[irq.label].prio): + kernel_irq_dict[irq.label] = irq + + for reg in dev_rule.get_regions(dev): + existing_reg = next((r for r in kernel_regions if r == reg), None) + if existing_reg: + existing_reg.take_labels(reg) + else: + kernel_regions.append(reg) + + # Build a dict of 'label: offset' entries, where label is the name given to + # the kernel for that address (e.g. SERIAL_PPTR) and offset is the offset + # from KDEV_BASE at which it's mapped. + kernel_dev_addr_macros = {} + kernel_offset = 0 + for group in kernel_regions: + kernel_offset = group.set_kernel_offset(kernel_offset) + for (offset, label) in group.get_labelled_addresses().items(): + if label in kernel_dev_addr_macros: + raise ValueError( + '"{} = 0x{:x}" already exists, cannot change to 0x{:x}'.format( + label, kernel_dev_addr_macros[label], offset)) + kernel_dev_addr_macros[label] = offset create_c_header_file( config, - get_interrupts(tree, hw_yaml), - kernel_macros, + sorted(kernel_irq_dict.values(), key=lambda irq: irq.label), + sorted(kernel_dev_addr_macros.items(), key=lambda tupel: tupel[1]), kernel_regions, physBase, physical_memory, diff --git a/tools/hardware/outputs/yaml.py b/tools/hardware/outputs/yaml.py index da410303b1..b5b81247f1 100644 --- a/tools/hardware/outputs/yaml.py +++ b/tools/hardware/outputs/yaml.py @@ -13,7 +13,7 @@ from hardware.config import Config from hardware.fdt import FdtParser from hardware.memory import Region -from hardware.utils.rule import HardwareYaml +from hardware.utils.rule import HardwareYaml, KernelInterrupt def make_yaml_list_of_regions(regions: List[Region]) -> List: @@ -41,26 +41,16 @@ def create_yaml_file(regions_dict: Dict[str, List[Region]], outputStream): outputStream) -def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml): - kernel_devices = tree.get_kernel_devices() +def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, args: argparse.Namespace): - groups = [] - for dev in kernel_devices: - rule = hw_yaml.get_rule(dev) - groups += rule.get_regions(dev) - - return groups - - -def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, - args: argparse.Namespace): if not args.yaml_out: raise ValueError('you need to provide a yaml-out to use the yaml output method') - phys_mem, reserved, _ = hardware.utils.memory.get_physical_memory(tree, config) - kernel_devs = get_kernel_devices(tree, hw_yaml) - dev_mem = hardware.utils.memory.get_addrspace_exclude( - list(reserved) + phys_mem + kernel_devs, config) + # Get the physical memory and device regions, we don't care about the kernel + # phy_base address here. + phys_mem, dev_mem, _ = hardware.utils.memory.get_phys_mem_regions(tree, + config, + hw_yaml) create_yaml_file( { diff --git a/tools/hardware/utils/memory.py b/tools/hardware/utils/memory.py index fa145dd704..175bcc8363 100644 --- a/tools/hardware/utils/memory.py +++ b/tools/hardware/utils/memory.py @@ -11,21 +11,10 @@ from hardware.device import WrappedNode from hardware.fdt import FdtParser from hardware.memory import Region -from hardware.utils.rule import KernelRegionGroup +from hardware.utils.rule import HardwareYaml, KernelRegionGroup -def get_memory_regions(tree: FdtParser): - ''' Get all regions with device_type = memory in the tree ''' - regions = set() - - def visitor(node: WrappedNode): - if node.has_prop('device_type') and node.get_prop('device_type').strings[0] == 'memory': - regions.update(node.get_regions()) - tree.visit(visitor) - return regions - - -def merge_memory_regions(regions: Set[Region]) -> Set[Region]: +def merge_regions(regions: Set[Region]) -> Set[Region]: ''' Check all region and merge adjacent ones ''' all_regions = [dict(idx=idx, region=region, right_adj=None, left_adj=None) for (idx, region) in enumerate(regions)] @@ -55,67 +44,106 @@ def merge_memory_regions(regions: Set[Region]) -> Set[Region]: return contiguous_regions -def parse_reserved_regions(node: WrappedNode) -> Set[Region]: - ''' Parse a reserved-memory node, looking for regions that are - unusable by OS (e.g. reserved for firmware/bootloader) ''' - if node is None: - return set() +def carve_out_region(regions: Set[Region], reserved_reg: Region) -> Set[Region]: + ''' + Returns a set of regions with the reserved area carved out. None of the + regions in the returned set will overlap with the reserved area. + ''' + ret_regions = set() + + for r in regions: + reg_list = r.reserve(reserved_reg) + ret_regions.update(reg_list) - ret = set() - for child in node: - if child.has_prop('reg') and child.has_prop('no-map'): - ret.update(child.get_regions()) - return ret + return ret_regions -def reserve_regions(regions: Set[Region], reserved: Set[Region]) -> Set[Region]: - ''' Given a set of regions, and a set of reserved regions, - return a new set that is the first set of regions minus the second set. ''' - ret = set(regions) +def get_phys_mem_regions(tree: FdtParser, config: Config, hw_yaml: HardwareYaml) \ + -> (List[Region], List[Region], int): + ''' + Returns a list of regions representing physical memory as used by the kernel + ''' - while len(reserved) > 0: - reserve = reserved.pop() - new_ret = set() - for el in ret: - r = el.reserve(reserve) - new_ret.update(r) - ret = new_ret - return ret + # Using a set instead of a list as the advantage that duplicate regions are + # ignored automatically when adding them + mem_regions = set() + reserved_regions = set() + # Get all regions with 'device_type = memory' in the tree. + def visitor(node: WrappedNode): + if node.has_prop('device_type') and node.get_prop('device_type').strings[0] == 'memory': + # ToDo: Check if any regions are beyond the physical memory range + # we support (config.addrspace_max) and drop these ranges. + mem_regions.update(node.get_regions()) -def get_physical_memory(tree: FdtParser, config: Config) -> List[Region]: - ''' returns a list of regions representing physical memory as used by the kernel ''' - regions = merge_memory_regions(get_memory_regions(tree)) - reserved = parse_reserved_regions(tree.get_path('/reserved-memory')) - regions = sorted(reserve_regions(regions, reserved)) - # Align the first so that the ELF loader will be able to load the kernel - # into it. Will return the aligned memory region list, a set of any regions - # of memory that were + tree.visit(visitor) + + # Check device tree for reserved memory that we can't use. + node = tree.get_path('/reserved-memory') + if node: + for child in node: + if not child.has_prop('reg') or not child.has_prop('no-map'): + continue + tree_reserved_regs = child.get_regions() + if tree_reserved_regs: + reserved_regions.update(tree_reserved_regs) + # carve out all reserved regions from the memory + for r in tree_reserved_regs: + mem_regions = carve_out_region(mem_regions, r) + + # Merge adjacent regions and create a properly ordered list from the + # available memory regions set + mem_region_list = sorted(merge_regions(mem_regions), + key=lambda r: r.base) + + # Check config for memory alignment and reservation needs. Needs to be done + # after we have removed the reserved region from the device tree, because we + # also get 'kernel_phys_base' here. And once we have that, the memory + # regions can't the modified any longer. kernel_phys_align = config.get_kernel_phys_align() if kernel_phys_align != 0: - new = regions[0].align_base(kernel_phys_align) - reserved.add(Region(regions[0].base, new.base - regions[0].base)) - regions[0] = new - return regions, reserved, regions[0].base - - -def get_addrspace_exclude(regions: List[Region], config: Config): - ''' Returns a list of regions that represents the inverse of the given region list. ''' - ret = set() - # We can't create untypeds that exceed the addrspace_max, so we round down to the smallest - # untyped size alignment so that the kernel will be able to turn the entire range into untypeds. - as_max = hardware.utils.align_down(config.addrspace_max, - config.get_smallest_kernel_object_alignment()) - ret.add(Region(0, as_max)) - - for reg in regions: - if type(reg) == KernelRegionGroup: - if reg.user_ok: - continue - new_ret = set() - for el in ret: - new_ret.update(el.reserve(reg)) - - ret = new_ret - return sorted(ret, key=lambda a: a.base) + # Align the first so that the ELF loader will be able to load the kernel + # into it. Will return the aligned memory region list, a set of any + # regions of memory that were + new_first_reg = mem_region_list[0].align_base(kernel_phys_align) + reserved_regions.add( + Region(mem_region_list[0].base, + new_first_reg.base - mem_region_list[0].base)) + mem_region_list[0] = new_first_reg + + kernel_phys_base = mem_region_list[0].base + + # Merge adjacent regions and create a properly ordered list from the + # reserved memory regions set + reserved_region_list = sorted(merge_regions(reserved_regions), + key=lambda r: r.base) + + + # Build the device regions by starting with a set with one region for the + # whole address space and then remove know memory, reserved regions and + # kernel devices. Since we can't create untypeds that exceed the + # addrspace_max, we round down to the smallest untyped size alignment so + # that the kernel will be able to turn the entire range into untypeds. + all_regions = { + Region( + 0, + hardware.utils.align_down( + config.addrspace_max, + config.get_smallest_kernel_object_alignment())) + } + + for reg in mem_region_list + reserved_region_list: + all_regions = carve_out_region(all_regions, reg) + # if there are hardware rules, process them + if hw_yaml: + for dev in tree.get_kernel_devices(): + hw_regions = hw_yaml.get_rule(dev).get_regions(dev) + for reg in hw_regions: + if (type(reg) != KernelRegionGroup or not reg.user_ok): + all_regions = carve_out_region(all_regions, reg) + + # create a properly ordered list from the set or regions + dev_region_list = sorted(all_regions, key=lambda r: r.base) + + return mem_region_list, dev_region_list, kernel_phys_base