blob: 927407b3efb3d7645c13157fe2d4c8b8f486d0a5 [file] [log] [blame]
Breno Leitaof061c9f2023-11-21 03:48:31 -08001#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0
3# -*- coding: utf-8; mode: python -*-
4
5"""
6 Script to auto generate the documentation for Netlink specifications.
7
8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org>
9 :license: GPL Version 2, June 1991 see linux/COPYING for details.
10
11 This script performs extensive parsing to the Linux kernel's netlink YAML
12 spec files, in an effort to avoid needing to heavily mark up the original
13 YAML file.
14
15 This code is split in three big parts:
16 1) RST formatters: Use to convert a string to a RST output
17 2) Parser helpers: Functions to parse the YAML data structure
18 3) Main function and small helpers
19"""
20
21from typing import Any, Dict, List
22import os.path
23import sys
24import argparse
25import logging
26import yaml
27
28
29SPACE_PER_LEVEL = 4
30
31
32# RST Formatters
33# ==============
34def headroom(level: int) -> str:
35 """Return space to format"""
36 return " " * (level * SPACE_PER_LEVEL)
37
38
39def bold(text: str) -> str:
40 """Format bold text"""
41 return f"**{text}**"
42
43
44def inline(text: str) -> str:
45 """Format inline text"""
46 return f"``{text}``"
47
48
49def sanitize(text: str) -> str:
50 """Remove newlines and multiple spaces"""
51 # This is useful for some fields that are spread across multiple lines
52 return str(text).replace("\n", "").strip()
53
54
55def rst_fields(key: str, value: str, level: int = 0) -> str:
56 """Return a RST formatted field"""
57 return headroom(level) + f":{key}: {value}"
58
59
60def rst_definition(key: str, value: Any, level: int = 0) -> str:
61 """Format a single rst definition"""
62 return headroom(level) + key + "\n" + headroom(level + 1) + str(value)
63
64
65def rst_paragraph(paragraph: str, level: int = 0) -> str:
66 """Return a formatted paragraph"""
67 return headroom(level) + paragraph
68
69
70def rst_bullet(item: str, level: int = 0) -> str:
71 """Return a formatted a bullet"""
Donald Hunter9b0aa222023-12-15 09:37:20 +000072 return headroom(level) + f"- {item}"
Breno Leitaof061c9f2023-11-21 03:48:31 -080073
74
75def rst_subsection(title: str) -> str:
76 """Add a sub-section to the document"""
77 return f"{title}\n" + "-" * len(title)
78
79
80def rst_subsubsection(title: str) -> str:
81 """Add a sub-sub-section to the document"""
82 return f"{title}\n" + "~" * len(title)
83
84
85def rst_section(title: str) -> str:
86 """Add a section to the document"""
87 return f"\n{title}\n" + "=" * len(title)
88
89
90def rst_subtitle(title: str) -> str:
91 """Add a subtitle to the document"""
92 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n"
93
94
95def rst_title(title: str) -> str:
96 """Add a title to the document"""
97 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n"
98
99
100def rst_list_inline(list_: List[str], level: int = 0) -> str:
101 """Format a list using inlines"""
102 return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]"
103
104
105def rst_header() -> str:
106 """The headers for all the auto generated RST files"""
107 lines = []
108
109 lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0"))
110 lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n"))
111
112 return "\n".join(lines)
113
114
115def rst_toctree(maxdepth: int = 2) -> str:
116 """Generate a toctree RST primitive"""
117 lines = []
118
119 lines.append(".. toctree::")
120 lines.append(f" :maxdepth: {maxdepth}\n\n")
121
122 return "\n".join(lines)
123
124
Jakub Kicinskie8c780a2023-11-28 20:14:27 -0800125def rst_label(title: str) -> str:
126 """Return a formatted label"""
127 return f".. _{title}:\n\n"
128
129
Breno Leitaof061c9f2023-11-21 03:48:31 -0800130# Parsers
131# =======
132
133
134def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str:
135 """Parse 'multicast' group list and return a formatted string"""
136 lines = []
137 for group in mcast_group:
138 lines.append(rst_bullet(group["name"]))
139
140 return "\n".join(lines)
141
142
143def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str:
144 """Parse 'do' section and return a formatted string"""
145 lines = []
146 for key in do_dict.keys():
147 lines.append(rst_paragraph(bold(key), level + 1))
148 lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n")
149
150 return "\n".join(lines)
151
152
153def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str:
154 """Parse 'attributes' section"""
155 if "attributes" not in attrs:
156 return ""
157 lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)]
158
159 return "\n".join(lines)
160
161
162def parse_operations(operations: List[Dict[str, Any]]) -> str:
163 """Parse operations block"""
164 preprocessed = ["name", "doc", "title", "do", "dump"]
165 lines = []
166
167 for operation in operations:
168 lines.append(rst_section(operation["name"]))
169 lines.append(rst_paragraph(sanitize(operation["doc"])) + "\n")
170
171 for key in operation.keys():
172 if key in preprocessed:
173 # Skip the special fields
174 continue
175 lines.append(rst_fields(key, operation[key], 0))
176
177 if "do" in operation:
178 lines.append(rst_paragraph(":do:", 0))
179 lines.append(parse_do(operation["do"], 0))
180 if "dump" in operation:
181 lines.append(rst_paragraph(":dump:", 0))
182 lines.append(parse_do(operation["dump"], 0))
183
184 # New line after fields
185 lines.append("\n")
186
187 return "\n".join(lines)
188
189
190def parse_entries(entries: List[Dict[str, Any]], level: int) -> str:
191 """Parse a list of entries"""
Donald Hunterfe09ae52024-01-29 22:34:57 +0000192 ignored = ["pad"]
Breno Leitaof061c9f2023-11-21 03:48:31 -0800193 lines = []
194 for entry in entries:
195 if isinstance(entry, dict):
196 # entries could be a list or a dictionary
Donald Hunterfe09ae52024-01-29 22:34:57 +0000197 field_name = entry.get("name", "")
198 if field_name in ignored:
199 continue
200 type_ = entry.get("type")
201 if type_:
202 field_name += f" ({inline(type_)})"
Breno Leitaof061c9f2023-11-21 03:48:31 -0800203 lines.append(
Donald Hunterfe09ae52024-01-29 22:34:57 +0000204 rst_fields(field_name, sanitize(entry.get("doc", "")), level)
Breno Leitaof061c9f2023-11-21 03:48:31 -0800205 )
206 elif isinstance(entry, list):
207 lines.append(rst_list_inline(entry, level))
208 else:
209 lines.append(rst_bullet(inline(sanitize(entry)), level))
210
211 lines.append("\n")
212 return "\n".join(lines)
213
214
215def parse_definitions(defs: Dict[str, Any]) -> str:
216 """Parse definitions section"""
217 preprocessed = ["name", "entries", "members"]
218 ignored = ["render-max"] # This is not printed
219 lines = []
220
221 for definition in defs:
222 lines.append(rst_section(definition["name"]))
223 for k in definition.keys():
224 if k in preprocessed + ignored:
225 continue
226 lines.append(rst_fields(k, sanitize(definition[k]), 0))
227
228 # Field list needs to finish with a new line
229 lines.append("\n")
230 if "entries" in definition:
231 lines.append(rst_paragraph(":entries:", 0))
232 lines.append(parse_entries(definition["entries"], 1))
233 if "members" in definition:
234 lines.append(rst_paragraph(":members:", 0))
235 lines.append(parse_entries(definition["members"], 1))
236
237 return "\n".join(lines)
238
239
240def parse_attr_sets(entries: List[Dict[str, Any]]) -> str:
241 """Parse attribute from attribute-set"""
242 preprocessed = ["name", "type"]
243 ignored = ["checks"]
244 lines = []
245
246 for entry in entries:
247 lines.append(rst_section(entry["name"]))
248 for attr in entry["attributes"]:
249 type_ = attr.get("type")
Donald Huntere9d7c5922023-12-15 09:37:19 +0000250 attr_line = attr["name"]
Breno Leitaof061c9f2023-11-21 03:48:31 -0800251 if type_:
252 # Add the attribute type in the same line
253 attr_line += f" ({inline(type_)})"
254
255 lines.append(rst_subsubsection(attr_line))
256
257 for k in attr.keys():
258 if k in preprocessed + ignored:
259 continue
Donald Hunter9b0aa222023-12-15 09:37:20 +0000260 lines.append(rst_fields(k, sanitize(attr[k]), 0))
Breno Leitaof061c9f2023-11-21 03:48:31 -0800261 lines.append("\n")
262
263 return "\n".join(lines)
264
265
Donald Hunter6235b3d82023-12-15 09:37:17 +0000266def parse_sub_messages(entries: List[Dict[str, Any]]) -> str:
267 """Parse sub-message definitions"""
268 lines = []
269
270 for entry in entries:
271 lines.append(rst_section(entry["name"]))
272 for fmt in entry["formats"]:
273 value = fmt["value"]
274
275 lines.append(rst_bullet(bold(value)))
276 for attr in ['fixed-header', 'attribute-set']:
277 if attr in fmt:
Donald Hunter9b0aa222023-12-15 09:37:20 +0000278 lines.append(rst_fields(attr, fmt[attr], 1))
Donald Hunter6235b3d82023-12-15 09:37:17 +0000279 lines.append("\n")
280
281 return "\n".join(lines)
282
283
Breno Leitaof061c9f2023-11-21 03:48:31 -0800284def parse_yaml(obj: Dict[str, Any]) -> str:
285 """Format the whole YAML into a RST string"""
286 lines = []
287
288 # Main header
289
290 lines.append(rst_header())
291
292 title = f"Family ``{obj['name']}`` netlink specification"
293 lines.append(rst_title(title))
294 lines.append(rst_paragraph(".. contents::\n"))
295
296 if "doc" in obj:
297 lines.append(rst_subtitle("Summary"))
298 lines.append(rst_paragraph(obj["doc"], 0))
299
300 # Operations
301 if "operations" in obj:
302 lines.append(rst_subtitle("Operations"))
303 lines.append(parse_operations(obj["operations"]["list"]))
304
305 # Multicast groups
306 if "mcast-groups" in obj:
307 lines.append(rst_subtitle("Multicast groups"))
308 lines.append(parse_mcast_group(obj["mcast-groups"]["list"]))
309
310 # Definitions
311 if "definitions" in obj:
312 lines.append(rst_subtitle("Definitions"))
313 lines.append(parse_definitions(obj["definitions"]))
314
315 # Attributes set
316 if "attribute-sets" in obj:
317 lines.append(rst_subtitle("Attribute sets"))
318 lines.append(parse_attr_sets(obj["attribute-sets"]))
319
Donald Hunter6235b3d82023-12-15 09:37:17 +0000320 # Sub-messages
321 if "sub-messages" in obj:
322 lines.append(rst_subtitle("Sub-messages"))
323 lines.append(parse_sub_messages(obj["sub-messages"]))
324
Breno Leitaof061c9f2023-11-21 03:48:31 -0800325 return "\n".join(lines)
326
327
328# Main functions
329# ==============
330
331
332def parse_arguments() -> argparse.Namespace:
333 """Parse arguments from user"""
334 parser = argparse.ArgumentParser(description="Netlink RST generator")
335
336 parser.add_argument("-v", "--verbose", action="store_true")
337 parser.add_argument("-o", "--output", help="Output file name")
338
339 # Index and input are mutually exclusive
340 group = parser.add_mutually_exclusive_group()
341 group.add_argument(
342 "-x", "--index", action="store_true", help="Generate the index page"
343 )
344 group.add_argument("-i", "--input", help="YAML file name")
345
346 args = parser.parse_args()
347
348 if args.verbose:
349 logging.basicConfig(level=logging.DEBUG)
350
351 if args.input and not os.path.isfile(args.input):
352 logging.warning("%s is not a valid file.", args.input)
353 sys.exit(-1)
354
355 if not args.output:
356 logging.error("No output file specified.")
357 sys.exit(-1)
358
359 if os.path.isfile(args.output):
360 logging.debug("%s already exists. Overwriting it.", args.output)
361
362 return args
363
364
365def parse_yaml_file(filename: str) -> str:
366 """Transform the YAML specified by filename into a rst-formmated string"""
367 with open(filename, "r", encoding="utf-8") as spec_file:
368 yaml_data = yaml.safe_load(spec_file)
369 content = parse_yaml(yaml_data)
370
371 return content
372
373
374def write_to_rstfile(content: str, filename: str) -> None:
375 """Write the generated content into an RST file"""
376 logging.debug("Saving RST file to %s", filename)
377
378 with open(filename, "w", encoding="utf-8") as rst_file:
379 rst_file.write(content)
380
381
382def generate_main_index_rst(output: str) -> None:
383 """Generate the `networking_spec/index` content and write to the file"""
384 lines = []
385
386 lines.append(rst_header())
Jakub Kicinskie8c780a2023-11-28 20:14:27 -0800387 lines.append(rst_label("specs"))
388 lines.append(rst_title("Netlink Family Specifications"))
Breno Leitaof061c9f2023-11-21 03:48:31 -0800389 lines.append(rst_toctree(1))
390
391 index_dir = os.path.dirname(output)
392 logging.debug("Looking for .rst files in %s", index_dir)
Donald Huntere8c32332023-12-15 09:37:18 +0000393 for filename in sorted(os.listdir(index_dir)):
Breno Leitaof061c9f2023-11-21 03:48:31 -0800394 if not filename.endswith(".rst") or filename == "index.rst":
395 continue
396 lines.append(f" {filename.replace('.rst', '')}\n")
397
398 logging.debug("Writing an index file at %s", output)
399 write_to_rstfile("".join(lines), output)
400
401
402def main() -> None:
403 """Main function that reads the YAML files and generates the RST files"""
404
405 args = parse_arguments()
406
407 if args.input:
408 logging.debug("Parsing %s", args.input)
409 try:
410 content = parse_yaml_file(os.path.join(args.input))
411 except Exception as exception:
412 logging.warning("Failed to parse %s.", args.input)
413 logging.warning(exception)
414 sys.exit(-1)
415
416 write_to_rstfile(content, args.output)
417
418 if args.index:
419 # Generate the index RST file
420 generate_main_index_rst(args.output)
421
422
423if __name__ == "__main__":
424 main()