[patch-axel-81] python improvements

tools/hardware/config.py

@@ -3,7 +3,9 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
-from typing import List, Set
+
+from __future__ import annotations
+import hardware
 from hardware.memory import Region
 
 
@@ -15,15 +17,6 @@ def __init__(self, sel4arch, addrspace_max):
         self.sel4arch = sel4arch
         self.addrspace_max = addrspace_max
 
-    def get_kernel_phys_align(self) -> int:
-        ''' Used to align the base of physical memory. Returns alignment size in bits. '''
-        return 0
-
-    def get_bootloader_reserve(self) -> int:
-        ''' Used to reserve a fixed amount of memory for the bootloader. Offsets
-            the kernel load address by the amount returned in bytes. '''
-        return 0
-
     def get_page_bits(self) -> int:
         ''' Get page size in bits for this arch '''
         return 12  # 4096-byte pages
@@ -35,68 +28,75 @@ def get_device_page_bits(self) -> int:
         ''' Get page size in bits for mapping devices for this arch '''
         return self.get_page_bits()
 
-    def align_memory(self, regions: Set[Region]) -> List[Region]:
+    def align_memory(self, regions: Set[Region]) -> (List[Region], Set[Region], int):
         ''' Given a set of regions, sort them and align the first so that the
         ELF loader will be able to load the kernel into it. Will return the
         aligned memory region list, a set of any regions of memory that were
         aligned out and the physBase value that the kernel will use. memory
         region list, a set of any regions of memory that were aligned out and
         the physBase value that the kernel will use. '''
-        pass
+        regions = sorted(regions)
+        return regions, set(), region[0].base
 
 
 class ARMConfig(Config):
     ''' Config class for ARM '''
     arch = 'arm'
     SUPERSECTION_BITS = 24  # 2^24 = 16 MiByte
 
-    def get_kernel_phys_align(self) -> int:
-        ''' on ARM the ELF loader expects to be able to map a supersection page to load the kernel. '''
-        return self.SUPERSECTION_BITS
-
-    def align_memory(self, regions: Set[Region]) -> List[Region]:
-        ''' Arm wants physBase to be the physical load address of the kernel. '''
-        ret = sorted(regions)
+    def align_memory(self, regions: Set[Region]) -> (List[Region], Set[Region], int):
+        '''On ARM the ELF loader expects to be able to map a supersection page
+        to load the kernel and 'physBase' must be the physical load address of
+        the kernel. '''
+        regions = sorted(regions)
         extra_reserved = set()
 
-        new = ret[0].align_base(self.get_kernel_phys_align())
-        resv = Region(ret[0].base, new.base - ret[0].base)
-        extra_reserved.add(resv)
-        ret[0] = new
+        # kernel is in the first region
+        reg = regions[0]
+        physBase = hardware.utils.align_up(reg.base, self.SUPERSECTION_BITS)
+        diff = physBase - reg.base
+        if diff > reg.size:
+            raise ValueError(
+                'can\'t cut off {} from start, region size is only {}'.format(
+                    diff, reg.size))
+        if (diff > 0):
+            extra_reserved.add(Region(reg.base, diff))
+            reg.base = physBase
+            reg.size -= diff
 
-        physBase = ret[0].base
-
-        return ret, extra_reserved, physBase
+        return regions, extra_reserved, physBase
 
 
 class RISCVConfig(Config):
     ''' Config class for RISCV '''
     arch = 'riscv'
     MEGAPAGE_BITS_RV32 = 22  # 2^22 = 4 MiByte
     MEGAPAGE_BITS_RV64 = 21  # 2^21 = 2 MiByte
-    MEGA_PAGE_SIZE_RV64 = 2**MEGAPAGE_BITS_RV64
-
-    def get_bootloader_reserve(self) -> int:
-        ''' OpenSBI reserved the first 2 MiByte of physical memory on rv64,
-        which is exactly a megapage. For rv32 we use the same value for now, as
-        this seems to work nicely - even if this is just half of the 4 MiByte
-        magepages that exist there. '''
-        return self.MEGA_PAGE_SIZE_RV64
-
-    def align_memory(self, regions: Set[Region]) -> List[Region]:
-        ''' Currently the RISC-V port expects physBase to be the address that the
-        bootloader is loaded at. To be generalised in the future. '''
-        ret = sorted(regions)
-        extra_reserved = set()
 
-        physBase = ret[0].base
-
-        resv = Region(ret[0].base, self.get_bootloader_reserve())
-        extra_reserved.add(resv)
-        ret[0].base += self.get_bootloader_reserve()
-        ret[0].size -= self.get_bootloader_reserve()
+    def align_memory(self, regions: Set[Region]) -> (List[Region], Set[Region], int):
+        ''' The RISC-V port expects physBase to be the address that the OpenSBI
+        bootloader is loaded at, which is the start of the physical memory.
+        OpenSBI reserved the first 2 MiByte of physical memory on rv64, which
+        is exactly a megapage. On rv32 we use the same value for now, as this
+        seems to work nicely - even if this is just half of the 4 MiByte
+        magepages that exist there.'''
+        regions = sorted(regions)
+        extra_reserved = set()
 
-        return ret, extra_reserved, physBase
+        # kernel is in the first region
+        reg = regions[0]
+        physBase = reg.base
+        # reserve space for bootloader in the region
+        len_bootloader_reserved = 1 << self.MEGAPAGE_BITS_RV64
+        if len_bootloader_reserved > reg.size:
+            raise ValueError(
+                'can\'t cut off {} from start, region size is only {}'.format(
+                    len_bootloader_reserved, reg.size))
+        extra_reserved.add(Region(reg.base, len_bootloader_reserved))
+        reg.base += len_bootloader_reserved
+        reg.size -= len_bootloader_reserved
+
+        return regions, extra_reserved, physBase
 
     def get_device_page_bits(self) -> int:
         ''' Get page size in bits for mapping devices for this arch '''

tools/hardware/device.py

@@ -4,26 +4,22 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+from __future__ import annotations
 from collections import OrderedDict
-from typing import Any, Dict, Generator, List, Tuple, cast
-
 import logging
 import pyfdt.pyfdt
-
 from hardware.memory import Region
 
 
 class WrappedNode:
     ''' A wrapper around an underlying pyfdt node '''
 
-    # TODO: Python 3.7 with 'from __future__ import annotations' will remove the need
-    # to put 'WrappedNode' in a string.
-    def __init__(self, node: pyfdt.pyfdt.FdtNode, parent: 'WrappedNode', path: str):
+    def __init__(self, node: pyfdt.pyfdt.FdtNode, parent: WrappedNode, path: str):
         self.node = node
         self.parent = parent
         self.depth = 0
         self.path = path
-        self.children: Dict[str, 'WrappedNode'] = OrderedDict()
+        self.children: Dict[str, WrappedNode] = OrderedDict()
         self.props: Dict[str, pyfdt.pyfdt.FdtProperty] = {}
         for prop in node:
             if not isinstance(prop, pyfdt.pyfdt.FdtProperty):
@@ -38,7 +34,7 @@ def __init__(self, node: pyfdt.pyfdt.FdtNode, parent: 'WrappedNode', path: str):
         else:
             self.is_cpu_addressable = True  # root node is always cpu addressable
 
-    def add_child(self, child: 'WrappedNode'):
+    def add_child(self, child: WrappedNode):
         ''' Add a child to this node '''
         self.children[child.node.get_name()] = child
 
@@ -142,7 +138,7 @@ def visit(self, visitor: Any):
             ret += child.visit(visitor)
         return ret
 
-    def __iter__(self) -> Generator['WrappedNode', None, None]:
+    def __iter__(self) -> Generator[WrappedNode, None, None]:
         ''' Iterate over all immediate children of this node '''
         for child in self.children.values():
             yield child

tools/hardware/fdt.py

@@ -4,9 +4,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from typing import Any, Dict, IO, List
+from __future__ import annotations
 import pyfdt.pyfdt
-
 from hardware.device import WrappedNode
 from hardware.irq import create_irq_controller, IrqController
 

tools/hardware/memory.py

@@ -4,6 +4,10 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+# Circular type dependency support has been postponed to Python 4, thus we
+# cannot simply use "from hardware.device import WrappedNode", but have to use
+# the fully qualified name in the type annotations.
+from __future__ import annotations
 import functools
 import hardware
 
@@ -12,56 +16,56 @@
 class Region:
     ''' Represents a region of memory. '''
 
-    def __init__(self, base: int, size: int, owner: 'WrappedNode' = None):
+    def __init__(self, base: int, size: int, owner: hardware.device.WrappedNode = None):
         self.base = base
         self.size = size
         self.owner = owner
 
     @staticmethod
-    def clone(other):
+    def clone(other) -> Region:
         return Region(other.base, other.size)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         ''' Returns a string representation that is a valid Python expression
         that eval() can parse. '''
         return 'Region(base=0x{:x},size=0x{:x})'.format(self.base, self.size)
 
-    def __str__(self):
+    def __str__(self) -> str:
         ''' Returns a string representation. '''
         return 'Region [0x{:x}..0x{:x}] ({:d} bytes)'.format(
             self.base,
             self.base + self.size - (1 if self.size > 0 else 0),
             self.size)
 
-    def __eq__(self, other):
+    def __eq__(self, other: Region) -> bool:
         return self.base == other.base and self.size == other.size
 
-    def __ne__(self, other):
+    def __ne__(self, other: Region) -> bool:
         # Needed only for py2.
         return not self.__eq__(other)
 
-    def __gt__(self, other):
+    def __gt__(self, other: Region) -> bool:
         return self.base > other.base
 
-    def __hash__(self):
+    def __hash__(self) -> bool:
         return hash((self.base, self.size))
 
     @staticmethod
-    def from_range(start, end, owner=None):
+    def from_range(start: int, end: int, owner: hardware.device.WrappedNode = None) -> Region:
         ''' create a region from a start/end rather than start/size '''
         if start > end:
             raise ValueError(
                 'invalid rage start (0x{:x}) > end (0x{:x})'.format(start > end))
         return Region(start, end - start, owner)
 
-    def overlaps(self, other):
+    def overlaps(self, other: Region) -> bool:
         ''' returns True if this region overlaps the given region '''
         # Either our base is first, and to overlap our end must be > other.base
         # or other.base is first, and to overlap other's end must be > self.base
         return (self.base <= other.base and (self.base + self.size) > other.base) \
             or (other.base <= self.base and (other.base + other.size) > self.base)
 
-    def reserve(self, excluded):
+    def reserve(self, excluded: Region) -> List[Region]:
         ''' returns an array of regions that represent this region
         minus the excluded range '''
         if not self.overlaps(excluded):
@@ -83,33 +87,3 @@ def reserve(self, excluded):
                 ret.append(Region.from_range(excluded.base + excluded.size,
                                              self.base + self.size, self.owner))
         return ret
-
-    def align_base(self, align_bits):
-        ''' align this region up to a given number of bits '''
-        new_base = hardware.utils.align_up(self.base, align_bits)
-        diff = new_base - self.base
-        if self.size < diff:
-            raise ValueError(
-                'can''t align region base to {} bits, {} too small'.format(
-                    align_bits, self))
-        # This could become an empty region now. We don't care, the caller has
-        # to check if this region still fits its needs.
-        return Region(new_base, self.size - diff, self.owner)
-
-    def align_size(self, align_bits):
-        ''' align this region's size to a given number of bits.
-         will move the base address down and the region's size
-         up '''
-        new_base = hardware.utils.align_down(self.base, align_bits)
-        new_size = hardware.utils.align_up(self.size, align_bits)
-        return Region(new_base, new_size, self.owner)
-
-    def make_chunks(self, chunksz):
-        base = self.base
-        size = self.size
-        ret = []
-        while size > 0:
-            ret.append(Region(base, min(size, chunksz), self.owner))
-            base += chunksz
-            size -= chunksz
-        return ret

tools/hardware/outputs/c_header.py

@@ -5,10 +5,11 @@
 #
 
 ''' generate a c header file from the device tree '''
+
+from __future__ import annotations
 import argparse
 import builtins
 import jinja2
-from typing import Dict, List
 import hardware
 from hardware.config import Config
 from hardware.fdt import FdtParser
@@ -225,6 +226,6 @@ def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, args: argparse.N
         args.header_out)
 
 
-def add_args(parser):
+def add_args(parser: argparse.ArgumentParser):
     parser.add_argument('--header-out', help='output file for c header',
                         type=argparse.FileType('w'))

tools/hardware/outputs/compat_strings.py

@@ -25,6 +25,6 @@ def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config,
     args.compat_strings_out.close()
 
 
-def add_args(parser):
+def add_args(parser: argparse.ArgumentParser):
     parser.add_argument('--compat-strings-out',
                         help='output file for compat strings list', type=argparse.FileType('w'))

tools/hardware/outputs/elfloader.py

@@ -6,14 +6,12 @@
 
 ''' generate a header file for the elfloader from a device tree '''
 
+from __future__ import annotations
 import argparse
 import builtins
 import logging
 import pyfdt.pyfdt
-
 from jinja2 import Environment, BaseLoader
-from typing import List
-
 from hardware import config, device, fdt
 from hardware.utils import cpu, memory, rule
 
@@ -173,6 +171,6 @@ def run(tree: fdt.FdtParser, hardware: rule.HardwareYaml, config: config.Config,
     args.elfloader_out.close()
 
 
-def add_args(parser):
+def add_args(parser: argparse.ArgumentParser):
     parser.add_argument('--elfloader-out', help='output file for elfloader header',
                         type=argparse.FileType('w'))