refactor(memmap): fix typing-related issues
This change introduces Pyright - a static type checker for Python - and
resolves a large number of the errors that it reports. Some errors have
not been resolved as they require larger refactors, so these will be
done in later patches.
Change-Id: I2ed9b1cc729bf44a1fa8e69882af34657cc30b38
Signed-off-by: Chris Kay <chris.kay@arm.com>
diff --git a/tools/memory/poetry.lock b/tools/memory/poetry.lock
index 99a6ab1..99fbfac 100644
--- a/tools/memory/poetry.lock
+++ b/tools/memory/poetry.lock
@@ -40,6 +40,17 @@
]
[[package]]
+name = "nodeenv"
+version = "1.9.1"
+description = "Node.js virtual environment builder"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+files = [
+ {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"},
+ {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"},
+]
+
+[[package]]
name = "prettytable"
version = "3.11.0"
description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format"
@@ -68,6 +79,26 @@
]
[[package]]
+name = "pyright"
+version = "1.1.399"
+description = "Command line wrapper for pyright"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "pyright-1.1.399-py3-none-any.whl", hash = "sha256:55f9a875ddf23c9698f24208c764465ffdfd38be6265f7faf9a176e1dc549f3b"},
+ {file = "pyright-1.1.399.tar.gz", hash = "sha256:439035d707a36c3d1b443aec980bc37053fbda88158eded24b8eedcf1c7b7a1b"},
+]
+
+[package.dependencies]
+nodeenv = ">=1.6.0"
+typing-extensions = ">=4.1"
+
+[package.extras]
+all = ["nodejs-wheel-binaries", "twine (>=3.4.1)"]
+dev = ["twine (>=3.4.1)"]
+nodejs = ["nodejs-wheel-binaries"]
+
+[[package]]
name = "ruff"
version = "0.11.2"
description = "An extremely fast Python linter and code formatter, written in Rust."
@@ -106,6 +137,17 @@
]
[[package]]
+name = "typing-extensions"
+version = "4.13.2"
+description = "Backported and Experimental Type Hints for Python 3.8+"
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"},
+ {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"},
+]
+
+[[package]]
name = "wcwidth"
version = "0.2.13"
description = "Measures the displayed width of unicode strings in a terminal"
@@ -119,4 +161,4 @@
[metadata]
lock-version = "2.0"
python-versions = "^3.8.0"
-content-hash = "9d94ba1676988aa442e1f795038b55da2bbb774f461f539f58abeb96887ecf05"
+content-hash = "91360b55f18ea83b2b9143a9c7662fae1a4570956b07bce96c635327ce3cc1da"
diff --git a/tools/memory/pyproject.toml b/tools/memory/pyproject.toml
index 477962f..f50c26a 100644
--- a/tools/memory/pyproject.toml
+++ b/tools/memory/pyproject.toml
@@ -21,6 +21,7 @@
[tool.poetry.group.dev.dependencies]
ruff = "^0.11.2"
+pyright = "^1.1.399"
[build-system]
requires = ["poetry-core"]
diff --git a/tools/memory/src/memory/buildparser.py b/tools/memory/src/memory/buildparser.py
index eaccd19..3c73ea4 100755
--- a/tools/memory/src/memory/buildparser.py
+++ b/tools/memory/src/memory/buildparser.py
@@ -37,7 +37,7 @@
files = list(self._path.glob("**/*.elf"))
io_perms = "rb"
- if self.map_backend or len(files) == 0:
+ if self.map_backend or not files:
backend = TfaMapParser
files = self._path.glob("**/*.map")
io_perms = "r"
@@ -47,7 +47,7 @@
with open(file, io_perms) as f:
self._modules[module_name] = backend(f)
- if not len(self._modules):
+ if not self._modules:
raise FileNotFoundError(
f"failed to find files to analyse in path {self._path}!"
)
@@ -82,7 +82,7 @@
k: {
"name": k,
**v.get_mod_mem_usage_dict(),
- **{"children": v.get_seg_map_as_dict()},
+ "children": v.get_seg_map_as_dict(),
}
for k, v in self._modules.items()
}
diff --git a/tools/memory/src/memory/elfparser.py b/tools/memory/src/memory/elfparser.py
index 33f36e3..019d6da 100644
--- a/tools/memory/src/memory/elfparser.py
+++ b/tools/memory/src/memory/elfparser.py
@@ -7,6 +7,7 @@
import re
from dataclasses import asdict, dataclass
from typing import (
+ Any,
BinaryIO,
Dict,
Iterable,
@@ -17,7 +18,7 @@
)
from elftools.elf.elffile import ELFFile
-from elftools.elf.sections import Section
+from elftools.elf.sections import Section, SymbolTableSection
from elftools.elf.segments import Segment
from memory.image import Image, Region
@@ -46,9 +47,11 @@
elf = ELFFile(elf_file)
+ symtab = elf.get_section_by_name(".symtab")
+ assert isinstance(symtab, SymbolTableSection)
+
self._symbols: Dict[str, int] = {
- sym.name: sym.entry["st_value"]
- for sym in elf.get_section_by_name(".symtab").iter_symbols()
+ sym.name: sym.entry["st_value"] for sym in symtab.iter_symbols()
}
self.set_segment_section_map(elf.iter_segments(), elf.iter_sections())
@@ -85,20 +88,20 @@
"Attempting to make segment without a name"
)
- if children is None:
- children = list()
-
# Segment and sections header keys have different prefixes.
vaddr = "p_vaddr" if isinstance(elf_obj, Segment) else "sh_addr"
size = "p_memsz" if isinstance(elf_obj, Segment) else "sh_size"
+ name = name if isinstance(elf_obj, Segment) else elf_obj.name
+ assert name is not None
+
# TODO figure out how to handle free space for sections and segments
return TfaMemObject(
- name if isinstance(elf_obj, Segment) else elf_obj.name,
+ name,
elf_obj[vaddr],
elf_obj[vaddr] + elf_obj[size],
elf_obj[size],
- [] if not children else children,
+ children or [],
)
def _get_mem_usage(self) -> Tuple[int, int]:
@@ -122,12 +125,13 @@
sections: Iterable[Section],
) -> None:
"""Set segment to section mappings."""
- segments = list(filter(lambda seg: seg["p_type"] == "PT_LOAD", segments))
+ segments = filter(lambda seg: seg["p_type"] == "PT_LOAD", segments)
+ segments_list = list(segments)
for sec in sections:
- for n, seg in enumerate(segments):
+ for n, seg in enumerate(segments_list):
if seg.section_in_segment(sec):
- if n not in self._segments.keys():
+ if n not in self._segments:
self._segments[n] = self.tfa_mem_obj_factory(
seg, name=f"{n:#02}"
)
@@ -138,14 +142,14 @@
"""Retrieve information about the memory configuration from the symbol
table.
"""
- assert len(self._symbols), "Symbol table is empty!"
+ assert self._symbols, "Symbol table is empty!"
expr = r".*(.?R.M)_REGION.*(START|END|LENGTH)"
region_symbols = filter(lambda s: re.match(expr, s), self._symbols)
memory_layout: Dict[str, Dict[str, int]] = {}
for symbol in region_symbols:
- region, _, attr = tuple(symbol.lower().strip("__").split("_"))
+ region, _, attr = symbol.lower().strip("__").split("_")
if region not in memory_layout:
memory_layout[region] = {}
@@ -154,9 +158,9 @@
return memory_layout
- def get_seg_map_as_dict(self) -> List[Dict[str, int]]:
+ def get_seg_map_as_dict(self) -> List[Dict[str, Any]]:
"""Get a dictionary of segments and their section mappings."""
- return [asdict(v) for k, v in self._segments.items()]
+ return [asdict(segment) for segment in self._segments.values()]
def get_memory_layout(self) -> Dict[str, Region]:
"""Get the total memory consumed by this module from the memory
diff --git a/tools/memory/src/memory/memmap.py b/tools/memory/src/memory/memmap.py
index 553e51f..10087f2 100755
--- a/tools/memory/src/memory/memmap.py
+++ b/tools/memory/src/memory/memmap.py
@@ -6,6 +6,7 @@
import shutil
from pathlib import Path
+from typing import Optional
import click
@@ -51,6 +52,7 @@
@click.option(
"--depth",
default=3,
+ show_default=True,
help="Generate a virtual address map of important TF symbols.",
)
@click.option(
@@ -78,7 +80,7 @@
help="Analyse the build's map files instead of ELF images.",
)
def main(
- root: Path,
+ root: Optional[Path],
platform: str,
build_type: str,
footprint: bool,
@@ -88,28 +90,31 @@
width: int,
d: bool,
no_elf_images: bool,
-) -> None:
- build_path = root if root else Path("build/", platform, build_type)
+):
+ build_path: Path = root if root is not None else Path("build", platform, build_type)
click.echo(f"build-path: {build_path.resolve()}")
- parser = TfaBuildParser(build_path, map_backend=no_elf_images)
- printer = TfaPrettyPrinter(columns=width, as_decimal=d)
+ parser: TfaBuildParser = TfaBuildParser(build_path, map_backend=no_elf_images)
+ printer: TfaPrettyPrinter = TfaPrettyPrinter(columns=width, as_decimal=d)
if footprint or not (tree or symbols):
printer.print_footprint(parser.get_mem_usage_dict())
if tree:
printer.print_mem_tree(
- parser.get_mem_tree_as_dict(), parser.module_names, depth=depth
+ parser.get_mem_tree_as_dict(),
+ parser.module_names,
+ depth=depth,
)
if symbols:
- expr = (
+ expr: str = (
r"(.*)(TEXT|BSS|RO|RODATA|STACKS|_OPS|PMF|XLAT|GOT|FCONF|RELA"
r"|R.M)(.*)(START|UNALIGNED|END)__$"
)
printer.print_symbol_table(
- parser.filter_symbols(parser.symbols, expr), parser.module_names
+ parser.filter_symbols(parser.symbols, expr),
+ parser.module_names,
)
diff --git a/tools/memory/src/memory/printer.py b/tools/memory/src/memory/printer.py
index bb9cad4..1a8c1c3 100755
--- a/tools/memory/src/memory/printer.py
+++ b/tools/memory/src/memory/printer.py
@@ -32,10 +32,10 @@
*args: Any,
width: int = 10,
fmt: Optional[str] = None,
- ) -> List[Any]:
- if not fmt and type(args[0]) is int:
+ ) -> List[str]:
+ if not fmt and isinstance(args[0], int):
fmt = f">{width}x" if not self.as_decimal else f">{width}"
- return [f"{arg:{fmt}}" if fmt else arg for arg in args]
+ return [f"{arg:{fmt}}" if fmt else str(arg) for arg in args]
def format_row(
self,
@@ -76,14 +76,14 @@
self,
app_mem_usage: Dict[str, Dict[str, Region]],
):
- assert len(app_mem_usage), "Empty memory layout dictionary!"
+ assert app_mem_usage, "Empty memory layout dictionary!"
fields = ["Component", "Start", "Limit", "Size", "Free", "Total"]
sort_key = fields[0]
# Iterate through all the memory types, create a table for each
# type, rows represent a single module.
- for mem in sorted(set(k for _, v in app_mem_usage.items() for k in v)):
+ for mem in sorted({k for v in app_mem_usage.values() for k in v}):
table = PrettyTable(
sortby=sort_key,
title=f"Memory Usage (bytes) [{mem.upper()}]",
@@ -91,7 +91,7 @@
)
for mod, vals in app_mem_usage.items():
- if mem in vals.keys():
+ if mem in vals:
val = vals[mem]
table.add_row(
[
@@ -117,7 +117,7 @@
) -> None:
assert len(symbol_table), "Empty symbol list!"
modules = sorted(modules)
- col_width = int((self.term_size - start) / len(modules))
+ col_width = (self.term_size - start) // len(modules)
address_fixed_width = 11
num_fmt = f"0=#0{address_fixed_width}x" if not self.as_decimal else ">10"
@@ -147,14 +147,13 @@
modules.index(mod),
len(modules),
col_width,
- is_edge=(not i or i == len(symbols_list) - 1),
+ is_edge=(i == 0 or i == len(symbols_list) - 1),
)
)
last_addr = addr
- self._symbol_map = ["Memory Layout:"]
- self._symbol_map += list(reversed(_symbol_map))
+ self._symbol_map = ["Memory Layout:"] + list(reversed(_symbol_map))
print("\n".join(self._symbol_map))
def print_mem_tree(
@@ -170,9 +169,7 @@
anchor = min_pad + node_right_pad * (depth - 1)
headers = ["start", "end", "size"]
- self._tree = [
- (f"{'name':<{anchor}}" + " ".join(f"{arg:>10}" for arg in headers))
- ]
+ self._tree = [f"{'name':<{anchor}}" + " ".join(f"{arg:>10}" for arg in headers)]
for mod in sorted(modules):
root = DictImporter().import_(mem_map_dict[mod])