| #!/usr/bin/env python3 | |
| # pylint: disable=R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,C0302 | |
| # Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. | |
| # SPDX-License-Identifier: GPL-2.0 | |
| """ | |
| Parse ABI documentation and produce results from it. | |
| """ | |
| from argparse import Namespace | |
| import logging | |
| import os | |
| import re | |
| from pprint import pformat | |
| from random import randrange, seed | |
| # Import Python modules | |
| from abi.helpers import AbiDebug, ABI_DIR | |
| class AbiParser: | |
| """Main class to parse ABI files.""" | |
| #: Valid tags at Documentation/ABI. | |
| TAGS = r"(what|where|date|kernelversion|contact|description|users)" | |
| #: ABI elements that will auto-generate cross-references. | |
| XREF = r"(?:^|\s|\()(\/(?:sys|config|proc|dev|kvd)\/[^,.:;\)\s]+)(?:[,.:;\)\s]|\Z)" | |
| def __init__(self, directory, logger=None, | |
| enable_lineno=False, show_warnings=True, debug=0): | |
| """Stores arguments for the class and initialize class vars.""" | |
| self.directory = directory | |
| self.enable_lineno = enable_lineno | |
| self.show_warnings = show_warnings | |
| self.debug = debug | |
| if not logger: | |
| self.log = logging.getLogger("get_abi") | |
| else: | |
| self.log = logger | |
| self.data = {} | |
| self.what_symbols = {} | |
| self.file_refs = {} | |
| self.what_refs = {} | |
| # Ignore files that contain such suffixes | |
| self.ignore_suffixes = (".rej", ".org", ".orig", ".bak", "~") | |
| # Regular expressions used on parser | |
| self.re_abi_dir = re.compile(r"(.*)" + ABI_DIR) | |
| self.re_tag = re.compile(r"(\S+)(:\s*)(.*)", re.I) | |
| self.re_valid = re.compile(self.TAGS) | |
| self.re_start_spc = re.compile(r"(\s*)(\S.*)") | |
| self.re_whitespace = re.compile(r"^\s+") | |
| # Regular used on print | |
| self.re_what = re.compile(r"(\/?(?:[\w\-]+\/?){1,2})") | |
| self.re_escape = re.compile(r"([\.\x01-\x08\x0e-\x1f\x21-\x2f\x3a-\x40\x7b-\xff])") | |
| self.re_unprintable = re.compile(r"([\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\xff]+)") | |
| self.re_title_mark = re.compile(r"\n[\-\*\=\^\~]+\n") | |
| self.re_doc = re.compile(r"Documentation/(?!devicetree)(\S+)\.rst") | |
| self.re_abi = re.compile(r"(Documentation/ABI/)([\w\/\-]+)") | |
| self.re_xref_node = re.compile(self.XREF) | |
| def warn(self, fdata, msg, extra=None): | |
| """Displays a parse error if warning is enabled.""" | |
| if not self.show_warnings: | |
| return | |
| msg = f"{fdata.fname}:{fdata.ln}: {msg}" | |
| if extra: | |
| msg += "\n\t\t" + extra | |
| self.log.warning(msg) | |
| def add_symbol(self, what, fname, ln=None, xref=None): | |
| """Create a reference table describing where each 'what' is located.""" | |
| if what not in self.what_symbols: | |
| self.what_symbols[what] = {"file": {}} | |
| if fname not in self.what_symbols[what]["file"]: | |
| self.what_symbols[what]["file"][fname] = [] | |
| if ln and ln not in self.what_symbols[what]["file"][fname]: | |
| self.what_symbols[what]["file"][fname].append(ln) | |
| if xref: | |
| self.what_symbols[what]["xref"] = xref | |
| def _parse_line(self, fdata, line): | |
| """Parse a single line of an ABI file.""" | |
| new_what = False | |
| new_tag = False | |
| content = None | |
| match = self.re_tag.match(line) | |
| if match: | |
| new = match.group(1).lower() | |
| sep = match.group(2) | |
| content = match.group(3) | |
| match = self.re_valid.search(new) | |
| if match: | |
| new_tag = match.group(1) | |
| else: | |
| if fdata.tag == "description": | |
| # New "tag" is actually part of description. | |
| # Don't consider it a tag | |
| new_tag = False | |
| elif fdata.tag != "": | |
| self.warn(fdata, f"tag '{fdata.tag}' is invalid", line) | |
| if new_tag: | |
| # "where" is Invalid, but was a common mistake. Warn if found | |
| if new_tag == "where": | |
| self.warn(fdata, "tag 'Where' is invalid. Should be 'What:' instead") | |
| new_tag = "what" | |
| if new_tag == "what": | |
| fdata.space = None | |
| if content not in self.what_symbols: | |
| self.add_symbol(what=content, fname=fdata.fname, ln=fdata.ln) | |
| if fdata.tag == "what": | |
| fdata.what.append(content.strip("\n")) | |
| else: | |
| if fdata.key: | |
| if "description" not in self.data.get(fdata.key, {}): | |
| self.warn(fdata, f"{fdata.key} doesn't have a description") | |
| for w in fdata.what: | |
| self.add_symbol(what=w, fname=fdata.fname, | |
| ln=fdata.what_ln, xref=fdata.key) | |
| fdata.label = content | |
| new_what = True | |
| key = "abi_" + content.lower() | |
| fdata.key = self.re_unprintable.sub("_", key).strip("_") | |
| # Avoid duplicated keys but using a defined seed, to make | |
| # the namespace identical if there aren't changes at the | |
| # ABI symbols | |
| seed(42) | |
| while fdata.key in self.data: | |
| char = randrange(0, 51) + ord("A") | |
| if char > ord("Z"): | |
| char += ord("a") - ord("Z") - 1 | |
| fdata.key += chr(char) | |
| if fdata.key and fdata.key not in self.data: | |
| self.data[fdata.key] = { | |
| "what": [content], | |
| "file": [fdata.file_ref], | |
| "path": fdata.ftype, | |
| "line_no": fdata.ln, | |
| } | |
| fdata.what = self.data[fdata.key]["what"] | |
| self.what_refs[content] = fdata.key | |
| fdata.tag = new_tag | |
| fdata.what_ln = fdata.ln | |
| if fdata.nametag["what"]: | |
| t = (content, fdata.key) | |
| if t not in fdata.nametag["symbols"]: | |
| fdata.nametag["symbols"].append(t) | |
| return | |
| if fdata.tag and new_tag: | |
| fdata.tag = new_tag | |
| if new_what: | |
| fdata.label = "" | |
| if "description" in self.data[fdata.key]: | |
| self.data[fdata.key]["description"] += "\n\n" | |
| if fdata.file_ref not in self.data[fdata.key]["file"]: | |
| self.data[fdata.key]["file"].append(fdata.file_ref) | |
| if self.debug == AbiDebug.WHAT_PARSING: | |
| self.log.debug("what: %s", fdata.what) | |
| if not fdata.what: | |
| self.warn(fdata, "'What:' should come first:", line) | |
| return | |
| if new_tag == "description": | |
| fdata.space = None | |
| if content: | |
| sep = sep.replace(":", " ") | |
| c = " " * len(new_tag) + sep + content | |
| c = c.expandtabs() | |
| match = self.re_start_spc.match(c) | |
| if match: | |
| # Preserve initial spaces for the first line | |
| fdata.space = match.group(1) | |
| content = match.group(2) + "\n" | |
| self.data[fdata.key][fdata.tag] = content | |
| return | |
| # Store any contents before tags at the database | |
| if not fdata.tag and "what" in fdata.nametag: | |
| fdata.nametag["description"] += line | |
| return | |
| if fdata.tag == "description": | |
| content = line.expandtabs() | |
| if self.re_whitespace.sub("", content) == "": | |
| self.data[fdata.key][fdata.tag] += "\n" | |
| return | |
| if fdata.space is None: | |
| match = self.re_start_spc.match(content) | |
| if match: | |
| # Preserve initial spaces for the first line | |
| fdata.space = match.group(1) | |
| content = match.group(2) + "\n" | |
| else: | |
| if content.startswith(fdata.space): | |
| content = content[len(fdata.space):] | |
| else: | |
| fdata.space = "" | |
| if fdata.tag == "what": | |
| w = content.strip("\n") | |
| if w: | |
| self.data[fdata.key][fdata.tag].append(w) | |
| else: | |
| self.data[fdata.key][fdata.tag] += content | |
| return | |
| content = line.strip() | |
| if fdata.tag: | |
| if fdata.tag == "what": | |
| w = content.strip("\n") | |
| if w: | |
| self.data[fdata.key][fdata.tag].append(w) | |
| else: | |
| self.data[fdata.key][fdata.tag] += "\n" + content.rstrip("\n") | |
| return | |
| # Everything else is error | |
| if content: | |
| self.warn(fdata, "Unexpected content", line) | |
| def parse_readme(self, nametag, fname): | |
| """Parse ABI README file.""" | |
| nametag["what"] = ["Introduction"] | |
| nametag["path"] = "README" | |
| with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: | |
| for line in fp: | |
| match = self.re_tag.match(line) | |
| if match: | |
| new = match.group(1).lower() | |
| match = self.re_valid.search(new) | |
| if match: | |
| nametag["description"] += "\n:" + line | |
| continue | |
| nametag["description"] += line | |
| def parse_file(self, fname, path, basename): | |
| """Parse a single file.""" | |
| ref = f"abi_file_{path}_{basename}" | |
| ref = self.re_unprintable.sub("_", ref).strip("_") | |
| # Store per-file state into a namespace variable. This will be used | |
| # by the per-line parser state machine and by the warning function. | |
| fdata = Namespace | |
| fdata.fname = fname | |
| fdata.name = basename | |
| pos = fname.find(ABI_DIR) | |
| if pos > 0: | |
| f = fname[pos:] | |
| else: | |
| f = fname | |
| fdata.file_ref = (f, ref) | |
| self.file_refs[f] = ref | |
| fdata.ln = 0 | |
| fdata.what_ln = 0 | |
| fdata.tag = "" | |
| fdata.label = "" | |
| fdata.what = [] | |
| fdata.key = None | |
| fdata.xrefs = None | |
| fdata.space = None | |
| fdata.ftype = path.split("/")[0] | |
| fdata.nametag = {} | |
| fdata.nametag["what"] = [f"ABI file {path}/{basename}"] | |
| fdata.nametag["type"] = "File" | |
| fdata.nametag["path"] = fdata.ftype | |
| fdata.nametag["file"] = [fdata.file_ref] | |
| fdata.nametag["line_no"] = 1 | |
| fdata.nametag["description"] = "" | |
| fdata.nametag["symbols"] = [] | |
| self.data[ref] = fdata.nametag | |
| if self.debug & AbiDebug.WHAT_OPEN: | |
| self.log.debug("Opening file %s", fname) | |
| if basename == "README": | |
| self.parse_readme(fdata.nametag, fname) | |
| return | |
| with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: | |
| for line in fp: | |
| fdata.ln += 1 | |
| self._parse_line(fdata, line) | |
| if "description" in fdata.nametag: | |
| fdata.nametag["description"] = fdata.nametag["description"].lstrip("\n") | |
| if fdata.key: | |
| if "description" not in self.data.get(fdata.key, {}): | |
| self.warn(fdata, f"{fdata.key} doesn't have a description") | |
| for w in fdata.what: | |
| self.add_symbol(what=w, fname=fname, xref=fdata.key) | |
| def _parse_abi(self, root=None): | |
| """Internal function to parse documentation ABI recursively.""" | |
| if not root: | |
| root = self.directory | |
| with os.scandir(root) as obj: | |
| for entry in obj: | |
| name = os.path.join(root, entry.name) | |
| if entry.is_dir(): | |
| self._parse_abi(name) | |
| continue | |
| if not entry.is_file(): | |
| continue | |
| basename = os.path.basename(name) | |
| if basename.startswith("."): | |
| continue | |
| if basename.endswith(self.ignore_suffixes): | |
| continue | |
| path = self.re_abi_dir.sub("", os.path.dirname(name)) | |
| self.parse_file(name, path, basename) | |
| def parse_abi(self, root=None): | |
| """Parse documentation ABI.""" | |
| self._parse_abi(root) | |
| if self.debug & AbiDebug.DUMP_ABI_STRUCTS: | |
| self.log.debug(pformat(self.data)) | |
| def desc_txt(self, desc): | |
| """Print description as found inside ABI files.""" | |
| desc = desc.strip(" \t\n") | |
| return desc + "\n\n" | |
| def xref(self, fname): | |
| """ | |
| Converts a Documentation/ABI + basename into a ReST cross-reference. | |
| """ | |
| xref = self.file_refs.get(fname) | |
| if not xref: | |
| return None | |
| else: | |
| return xref | |
| def desc_rst(self, desc): | |
| """Enrich ReST output by creating cross-references.""" | |
| # Remove title markups from the description | |
| # Having titles inside ABI files will only work if extra | |
| # care would be taken in order to strictly follow the same | |
| # level order for each markup. | |
| desc = self.re_title_mark.sub("\n\n", "\n" + desc) | |
| desc = desc.rstrip(" \t\n").lstrip("\n") | |
| # Python's regex performance for non-compiled expressions is a lot | |
| # than Perl, as Perl automatically caches them at their | |
| # first usage. Here, we'll need to do the same, as otherwise the | |
| # performance penalty is be high | |
| new_desc = "" | |
| for d in desc.split("\n"): | |
| if d == "": | |
| new_desc += "\n" | |
| continue | |
| # Use cross-references for doc files where needed | |
| d = self.re_doc.sub(r":doc:`/\1`", d) | |
| # Use cross-references for ABI generated docs where needed | |
| matches = self.re_abi.findall(d) | |
| for m in matches: | |
| abi = m[0] + m[1] | |
| xref = self.file_refs.get(abi) | |
| if not xref: | |
| # This may happen if ABI is on a separate directory, | |
| # like parsing ABI testing and symbol is at stable. | |
| # The proper solution is to move this part of the code | |
| # for it to be inside sphinx/kernel_abi.py | |
| self.log.info("Didn't find ABI reference for '%s'", abi) | |
| else: | |
| new = self.re_escape.sub(r"\\\1", m[1]) | |
| d = re.sub(fr"\b{abi}\b", f":ref:`{new} <{xref}>`", d) | |
| # Seek for cross reference symbols like /sys/... | |
| # Need to be careful to avoid doing it on a code block | |
| if d[0] not in [" ", "\t"]: | |
| matches = self.re_xref_node.findall(d) | |
| for m in matches: | |
| # Finding ABI here is more complex due to wildcards | |
| xref = self.what_refs.get(m) | |
| if xref: | |
| new = self.re_escape.sub(r"\\\1", m) | |
| d = re.sub(fr"\b{m}\b", f":ref:`{new} <{xref}>`", d) | |
| new_desc += d + "\n" | |
| return new_desc + "\n\n" | |
| def doc(self, output_in_txt=False, show_symbols=True, show_file=True, | |
| filter_path=None): | |
| """Print ABI at stdout.""" | |
| part = None | |
| for key, v in sorted(self.data.items(), | |
| key=lambda x: (x[1].get("type", ""), | |
| x[1].get("what"))): | |
| wtype = v.get("type", "Symbol") | |
| file_ref = v.get("file") | |
| names = v.get("what", [""]) | |
| if wtype == "File": | |
| if not show_file: | |
| continue | |
| else: | |
| if not show_symbols: | |
| continue | |
| if filter_path: | |
| if v.get("path") != filter_path: | |
| continue | |
| msg = "" | |
| if wtype != "File": | |
| cur_part = names[0] | |
| if cur_part.find("/") >= 0: | |
| match = self.re_what.match(cur_part) | |
| if match: | |
| symbol = match.group(1).rstrip("/") | |
| cur_part = "Symbols under " + symbol | |
| if cur_part and cur_part != part: | |
| part = cur_part | |
| msg += part + "\n"+ "-" * len(part) +"\n\n" | |
| msg += f".. _{key}:\n\n" | |
| max_len = 0 | |
| for i in range(0, len(names)): # pylint: disable=C0200 | |
| names[i] = "**" + self.re_escape.sub(r"\\\1", names[i]) + "**" | |
| max_len = max(max_len, len(names[i])) | |
| msg += "+-" + "-" * max_len + "-+\n" | |
| for name in names: | |
| msg += f"| {name}" + " " * (max_len - len(name)) + " |\n" | |
| msg += "+-" + "-" * max_len + "-+\n" | |
| msg += "\n" | |
| for ref in file_ref: | |
| if wtype == "File": | |
| msg += f".. _{ref[1]}:\n\n" | |
| else: | |
| base = os.path.basename(ref[0]) | |
| msg += f"Defined on file :ref:`{base} <{ref[1]}>`\n\n" | |
| if wtype == "File": | |
| msg += names[0] +"\n" + "-" * len(names[0]) +"\n\n" | |
| desc = v.get("description") | |
| if not desc and wtype != "File": | |
| msg += f"DESCRIPTION MISSING for {names[0]}\n\n" | |
| if desc: | |
| if output_in_txt: | |
| msg += self.desc_txt(desc) | |
| else: | |
| msg += self.desc_rst(desc) | |
| symbols = v.get("symbols") | |
| if symbols: | |
| msg += "Has the following ABI:\n\n" | |
| for w, label in symbols: | |
| # Escape special chars from content | |
| content = self.re_escape.sub(r"\\\1", w) | |
| msg += f"- :ref:`{content} <{label}>`\n\n" | |
| users = v.get("users") | |
| if users and users.strip(" \t\n"): | |
| users = users.strip("\n").replace('\n', '\n\t') | |
| msg += f"Users:\n\t{users}\n\n" | |
| ln = v.get("line_no", 1) | |
| yield (msg, file_ref[0][0], ln) | |
| def check_issues(self): | |
| """Warn about duplicated ABI entries.""" | |
| for what, v in self.what_symbols.items(): | |
| files = v.get("file") | |
| if not files: | |
| # Should never happen if the parser works properly | |
| self.log.warning("%s doesn't have a file associated", what) | |
| continue | |
| if len(files) == 1: | |
| continue | |
| f = [] | |
| for fname, lines in sorted(files.items()): | |
| if not lines: | |
| f.append(f"{fname}") | |
| elif len(lines) == 1: | |
| f.append(f"{fname}:{lines[0]}") | |
| else: | |
| m = fname + "lines " | |
| m += ", ".join(str(x) for x in lines) | |
| f.append(m) | |
| self.log.warning("%s is defined %d times: %s", what, len(f), "; ".join(f)) | |
| def search_symbols(self, expr): | |
| """ Searches for ABI symbols.""" | |
| regex = re.compile(expr, re.I) | |
| found_keys = 0 | |
| for t in sorted(self.data.items(), key=lambda x: [0]): | |
| v = t[1] | |
| wtype = v.get("type", "") | |
| if wtype == "File": | |
| continue | |
| for what in v.get("what", [""]): | |
| if regex.search(what): | |
| found_keys += 1 | |
| kernelversion = v.get("kernelversion", "").strip(" \t\n") | |
| date = v.get("date", "").strip(" \t\n") | |
| contact = v.get("contact", "").strip(" \t\n") | |
| users = v.get("users", "").strip(" \t\n") | |
| desc = v.get("description", "").strip(" \t\n") | |
| files = [] | |
| for f in v.get("file", ()): | |
| files.append(f[0]) | |
| what = str(found_keys) + ". " + what | |
| title_tag = "-" * len(what) | |
| print(f"\n{what}\n{title_tag}\n") | |
| if kernelversion: | |
| print(f"Kernel version:\t\t{kernelversion}") | |
| if date: | |
| print(f"Date:\t\t\t{date}") | |
| if contact: | |
| print(f"Contact:\t\t{contact}") | |
| if users: | |
| print(f"Users:\t\t\t{users}") | |
| print("Defined on file(s):\t" + ", ".join(files)) | |
| if desc: | |
| desc = desc.strip("\n") | |
| print(f"\n{desc}\n") | |
| if not found_keys: | |
| print(f"Regular expression /{expr}/ not found.") |