| #!/usr/bin/env python3 | |
| # SPDX-License-Identifier: GPL-2.0 | |
| # Copyright(c) 2025: Mauro Carvalho Chehab <mchehab+huawei@kernel.org> | |
| # | |
| # pylint: disable=R0903,R0912,R0913,R0914,R0917,C0301 | |
| """ | |
| Install minimal supported requirements for different Sphinx versions | |
| and optionally test the build. | |
| """ | |
| import argparse | |
| import asyncio | |
| import os.path | |
| import shutil | |
| import sys | |
| import time | |
| import subprocess | |
| # Minimal python version supported by the building system. | |
| PYTHON = os.path.basename(sys.executable) | |
| min_python_bin = None | |
| for i in range(9, 13): | |
| p = f"python3.{i}" | |
| if shutil.which(p): | |
| min_python_bin = p | |
| break | |
| if not min_python_bin: | |
| min_python_bin = PYTHON | |
| # Starting from 8.0, Python 3.9 is not supported anymore. | |
| PYTHON_VER_CHANGES = {(8, 0, 0): PYTHON} | |
| DEFAULT_VERSIONS_TO_TEST = [ | |
| (3, 4, 3), # Minimal supported version | |
| (5, 3, 0), # CentOS Stream 9 / AlmaLinux 9 | |
| (6, 1, 1), # Debian 12 | |
| (7, 2, 1), # openSUSE Leap 15.6 | |
| (7, 2, 6), # Ubuntu 24.04 LTS | |
| (7, 4, 7), # Ubuntu 24.10 | |
| (7, 3, 0), # openSUSE Tumbleweed | |
| (8, 1, 3), # Fedora 42 | |
| (8, 2, 3) # Latest version - covers rolling distros | |
| ] | |
| # Sphinx versions to be installed and their incremental requirements | |
| SPHINX_REQUIREMENTS = { | |
| # Oldest versions we support for each package required by Sphinx 3.4.3 | |
| (3, 4, 3): { | |
| "docutils": "0.16", | |
| "alabaster": "0.7.12", | |
| "babel": "2.8.0", | |
| "certifi": "2020.6.20", | |
| "docutils": "0.16", | |
| "idna": "2.10", | |
| "imagesize": "1.2.0", | |
| "Jinja2": "2.11.2", | |
| "MarkupSafe": "1.1.1", | |
| "packaging": "20.4", | |
| "Pygments": "2.6.1", | |
| "PyYAML": "5.1", | |
| "requests": "2.24.0", | |
| "snowballstemmer": "2.0.0", | |
| "sphinxcontrib-applehelp": "1.0.2", | |
| "sphinxcontrib-devhelp": "1.0.2", | |
| "sphinxcontrib-htmlhelp": "1.0.3", | |
| "sphinxcontrib-jsmath": "1.0.1", | |
| "sphinxcontrib-qthelp": "1.0.3", | |
| "sphinxcontrib-serializinghtml": "1.1.4", | |
| "urllib3": "1.25.9", | |
| }, | |
| # Update package dependencies to a more modern base. The goal here | |
| # is to avoid to many incremental changes for the next entries | |
| (3, 5, 0): { | |
| "alabaster": "0.7.13", | |
| "babel": "2.17.0", | |
| "certifi": "2025.6.15", | |
| "idna": "3.10", | |
| "imagesize": "1.4.1", | |
| "packaging": "25.0", | |
| "Pygments": "2.8.1", | |
| "requests": "2.32.4", | |
| "snowballstemmer": "3.0.1", | |
| "sphinxcontrib-applehelp": "1.0.4", | |
| "sphinxcontrib-htmlhelp": "2.0.1", | |
| "sphinxcontrib-serializinghtml": "1.1.5", | |
| "urllib3": "2.0.0", | |
| }, | |
| # Starting from here, ensure all docutils versions are covered with | |
| # supported Sphinx versions. Other packages are upgraded only when | |
| # required by pip | |
| (4, 0, 0): { | |
| "PyYAML": "5.1", | |
| }, | |
| (4, 1, 0): { | |
| "docutils": "0.17", | |
| "Pygments": "2.19.1", | |
| "Jinja2": "3.0.3", | |
| "MarkupSafe": "2.0", | |
| }, | |
| (4, 3, 0): {}, | |
| (4, 4, 0): {}, | |
| (4, 5, 0): { | |
| "docutils": "0.17.1", | |
| }, | |
| (5, 0, 0): {}, | |
| (5, 1, 0): {}, | |
| (5, 2, 0): { | |
| "docutils": "0.18", | |
| "Jinja2": "3.1.2", | |
| "MarkupSafe": "2.0", | |
| "PyYAML": "5.3.1", | |
| }, | |
| (5, 3, 0): { | |
| "docutils": "0.18.1", | |
| }, | |
| (6, 0, 0): {}, | |
| (6, 1, 0): {}, | |
| (6, 2, 0): { | |
| "PyYAML": "5.4.1", | |
| }, | |
| (7, 0, 0): {}, | |
| (7, 1, 0): {}, | |
| (7, 2, 0): { | |
| "docutils": "0.19", | |
| "PyYAML": "6.0.1", | |
| "sphinxcontrib-serializinghtml": "1.1.9", | |
| }, | |
| (7, 2, 6): { | |
| "docutils": "0.20", | |
| }, | |
| (7, 3, 0): { | |
| "alabaster": "0.7.14", | |
| "PyYAML": "6.0.1", | |
| "tomli": "2.0.1", | |
| }, | |
| (7, 4, 0): { | |
| "docutils": "0.20.1", | |
| "PyYAML": "6.0.1", | |
| }, | |
| (8, 0, 0): { | |
| "docutils": "0.21", | |
| }, | |
| (8, 1, 0): { | |
| "docutils": "0.21.1", | |
| "PyYAML": "6.0.1", | |
| "sphinxcontrib-applehelp": "1.0.7", | |
| "sphinxcontrib-devhelp": "1.0.6", | |
| "sphinxcontrib-htmlhelp": "2.0.6", | |
| "sphinxcontrib-qthelp": "1.0.6", | |
| }, | |
| (8, 2, 0): { | |
| "docutils": "0.21.2", | |
| "PyYAML": "6.0.1", | |
| "sphinxcontrib-serializinghtml": "1.1.9", | |
| }, | |
| } | |
| class AsyncCommands: | |
| """Excecute command synchronously""" | |
| def __init__(self, fp=None): | |
| self.stdout = None | |
| self.stderr = None | |
| self.output = None | |
| self.fp = fp | |
| def log(self, out, verbose, is_info=True): | |
| out = out.removesuffix('\n') | |
| if verbose: | |
| if is_info: | |
| print(out) | |
| else: | |
| print(out, file=sys.stderr) | |
| if self.fp: | |
| self.fp.write(out + "\n") | |
| async def _read(self, stream, verbose, is_info): | |
| """Ancillary routine to capture while displaying""" | |
| while stream is not None: | |
| line = await stream.readline() | |
| if line: | |
| out = line.decode("utf-8", errors="backslashreplace") | |
| self.log(out, verbose, is_info) | |
| if is_info: | |
| self.stdout += out | |
| else: | |
| self.stderr += out | |
| else: | |
| break | |
| async def run(self, cmd, capture_output=False, check=False, | |
| env=None, verbose=True): | |
| """ | |
| Execute an arbitrary command, handling errors. | |
| Please notice that this class is not thread safe | |
| """ | |
| self.stdout = "" | |
| self.stderr = "" | |
| self.log("$ " + " ".join(cmd), verbose) | |
| proc = await asyncio.create_subprocess_exec(cmd[0], | |
| *cmd[1:], | |
| env=env, | |
| stdout=asyncio.subprocess.PIPE, | |
| stderr=asyncio.subprocess.PIPE) | |
| # Handle input and output in realtime | |
| await asyncio.gather( | |
| self._read(proc.stdout, verbose, True), | |
| self._read(proc.stderr, verbose, False), | |
| ) | |
| await proc.wait() | |
| if check and proc.returncode > 0: | |
| raise subprocess.CalledProcessError(returncode=proc.returncode, | |
| cmd=" ".join(cmd), | |
| output=self.stdout, | |
| stderr=self.stderr) | |
| if capture_output: | |
| if proc.returncode > 0: | |
| self.log(f"Error {proc.returncode}", verbose=True, is_info=False) | |
| return "" | |
| return self.output | |
| ret = subprocess.CompletedProcess(args=cmd, | |
| returncode=proc.returncode, | |
| stdout=self.stdout, | |
| stderr=self.stderr) | |
| return ret | |
| class SphinxVenv: | |
| """ | |
| Installs Sphinx on one virtual env per Sphinx version with a minimal | |
| set of dependencies, adjusting them to each specific version. | |
| """ | |
| def __init__(self): | |
| """Initialize instance variables""" | |
| self.built_time = {} | |
| self.first_run = True | |
| async def _handle_version(self, args, fp, | |
| cur_ver, cur_requirements, python_bin): | |
| """Handle a single Sphinx version""" | |
| cmd = AsyncCommands(fp) | |
| ver = ".".join(map(str, cur_ver)) | |
| if not self.first_run and args.wait_input and args.build: | |
| ret = input("Press Enter to continue or 'a' to abort: ").strip().lower() | |
| if ret == "a": | |
| print("Aborted.") | |
| sys.exit() | |
| else: | |
| self.first_run = False | |
| venv_dir = f"Sphinx_{ver}" | |
| req_file = f"requirements_{ver}.txt" | |
| cmd.log(f"\nSphinx {ver} with {python_bin}", verbose=True) | |
| # Create venv | |
| await cmd.run([python_bin, "-m", "venv", venv_dir], | |
| verbose=args.verbose, check=True) | |
| pip = os.path.join(venv_dir, "bin/pip") | |
| # Create install list | |
| reqs = [] | |
| for pkg, verstr in cur_requirements.items(): | |
| reqs.append(f"{pkg}=={verstr}") | |
| reqs.append(f"Sphinx=={ver}") | |
| await cmd.run([pip, "install"] + reqs, check=True, verbose=args.verbose) | |
| # Freeze environment | |
| result = await cmd.run([pip, "freeze"], verbose=False, check=True) | |
| # Pip install succeeded. Write requirements file | |
| if args.req_file: | |
| with open(req_file, "w", encoding="utf-8") as fp: | |
| fp.write(result.stdout) | |
| if args.build: | |
| start_time = time.time() | |
| # Prepare a venv environment | |
| env = os.environ.copy() | |
| bin_dir = os.path.join(venv_dir, "bin") | |
| env["PATH"] = bin_dir + ":" + env["PATH"] | |
| env["VIRTUAL_ENV"] = venv_dir | |
| if "PYTHONHOME" in env: | |
| del env["PYTHONHOME"] | |
| # Test doc build | |
| await cmd.run(["make", "cleandocs"], env=env, check=True) | |
| make = ["make"] | |
| if args.output: | |
| sphinx_build = os.path.realpath(f"{bin_dir}/sphinx-build") | |
| make += [f"O={args.output}", f"SPHINXBUILD={sphinx_build}"] | |
| if args.make_args: | |
| make += args.make_args | |
| make += args.targets | |
| if args.verbose: | |
| cmd.log(f". {bin_dir}/activate", verbose=True) | |
| await cmd.run(make, env=env, check=True, verbose=True) | |
| if args.verbose: | |
| cmd.log("deactivate", verbose=True) | |
| end_time = time.time() | |
| elapsed_time = end_time - start_time | |
| hours, minutes = divmod(elapsed_time, 3600) | |
| minutes, seconds = divmod(minutes, 60) | |
| hours = int(hours) | |
| minutes = int(minutes) | |
| seconds = int(seconds) | |
| self.built_time[ver] = f"{hours:02d}:{minutes:02d}:{seconds:02d}" | |
| cmd.log(f"Finished doc build for Sphinx {ver}. Elapsed time: {self.built_time[ver]}", verbose=True) | |
| async def run(self, args): | |
| """ | |
| Navigate though multiple Sphinx versions, handling each of them | |
| on a loop. | |
| """ | |
| if args.log: | |
| fp = open(args.log, "w", encoding="utf-8") | |
| if not args.verbose: | |
| args.verbose = False | |
| else: | |
| fp = None | |
| if not args.verbose: | |
| args.verbose = True | |
| cur_requirements = {} | |
| python_bin = min_python_bin | |
| vers = set(SPHINX_REQUIREMENTS.keys()) | set(args.versions) | |
| for cur_ver in sorted(vers): | |
| if cur_ver in SPHINX_REQUIREMENTS: | |
| new_reqs = SPHINX_REQUIREMENTS[cur_ver] | |
| cur_requirements.update(new_reqs) | |
| if cur_ver in PYTHON_VER_CHANGES: # pylint: disable=R1715 | |
| python_bin = PYTHON_VER_CHANGES[cur_ver] | |
| if cur_ver not in args.versions: | |
| continue | |
| if args.min_version: | |
| if cur_ver < args.min_version: | |
| continue | |
| if args.max_version: | |
| if cur_ver > args.max_version: | |
| break | |
| await self._handle_version(args, fp, cur_ver, cur_requirements, | |
| python_bin) | |
| if args.build: | |
| cmd = AsyncCommands(fp) | |
| cmd.log("\nSummary:", verbose=True) | |
| for ver, elapsed_time in sorted(self.built_time.items()): | |
| cmd.log(f"\tSphinx {ver} elapsed time: {elapsed_time}", | |
| verbose=True) | |
| if fp: | |
| fp.close() | |
| def parse_version(ver_str): | |
| """Convert a version string into a tuple.""" | |
| return tuple(map(int, ver_str.split("."))) | |
| DEFAULT_VERS = " - " | |
| DEFAULT_VERS += "\n - ".join(map(lambda v: f"{v[0]}.{v[1]}.{v[2]}", | |
| DEFAULT_VERSIONS_TO_TEST)) | |
| SCRIPT = os.path.relpath(__file__) | |
| DESCRIPTION = f""" | |
| This tool allows creating Python virtual environments for different | |
| Sphinx versions that are supported by the Linux Kernel build system. | |
| Besides creating the virtual environment, it can also test building | |
| the documentation using "make htmldocs" (and/or other doc targets). | |
| If called without "--versions" argument, it covers the versions shipped | |
| on major distros, plus the lowest supported version: | |
| {DEFAULT_VERS} | |
| A typical usage is to run: | |
| {SCRIPT} -m -l sphinx_builds.log | |
| This will create one virtual env for the default version set and run | |
| "make htmldocs" for each version, creating a log file with the | |
| excecuted commands on it. | |
| NOTE: The build time can be very long, specially on old versions. Also, there | |
| is a known bug with Sphinx version 6.0.x: each subprocess uses a lot of | |
| memory. That, together with "-jauto" may cause OOM killer to cause | |
| failures at the doc generation. To minimize the risk, you may use the | |
| "-a" command line parameter to constrain the built directories and/or | |
| reduce the number of threads from "-jauto" to, for instance, "-j4": | |
| {SCRIPT} -m -V 6.0.1 -a "SPHINXDIRS=process" "SPHINXOPTS='-j4'" | |
| """ | |
| MAKE_TARGETS = [ | |
| "htmldocs", | |
| "texinfodocs", | |
| "infodocs", | |
| "latexdocs", | |
| "pdfdocs", | |
| "epubdocs", | |
| "xmldocs", | |
| ] | |
| async def main(): | |
| """Main program""" | |
| parser = argparse.ArgumentParser(description=DESCRIPTION, | |
| formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ver_group = parser.add_argument_group("Version range options") | |
| ver_group.add_argument('-V', '--versions', nargs="*", | |
| default=DEFAULT_VERSIONS_TO_TEST,type=parse_version, | |
| help='Sphinx versions to test') | |
| ver_group.add_argument('--min-version', "--min", type=parse_version, | |
| help='Sphinx minimal version') | |
| ver_group.add_argument('--max-version', "--max", type=parse_version, | |
| help='Sphinx maximum version') | |
| ver_group.add_argument('-f', '--full', action='store_true', | |
| help='Add all Sphinx (major,minor) supported versions to the version range') | |
| build_group = parser.add_argument_group("Build options") | |
| build_group.add_argument('-b', '--build', action='store_true', | |
| help='Build documentation') | |
| build_group.add_argument('-a', '--make-args', nargs="*", | |
| help='extra arguments for make, like SPHINXDIRS=netlink/specs', | |
| ) | |
| build_group.add_argument('-t', '--targets', nargs="+", choices=MAKE_TARGETS, | |
| default=[MAKE_TARGETS[0]], | |
| help="make build targets. Default: htmldocs.") | |
| build_group.add_argument("-o", '--output', | |
| help="output directory for the make O=OUTPUT") | |
| other_group = parser.add_argument_group("Other options") | |
| other_group.add_argument('-r', '--req-file', action='store_true', | |
| help='write a requirements.txt file') | |
| other_group.add_argument('-l', '--log', | |
| help='Log command output on a file') | |
| other_group.add_argument('-v', '--verbose', action='store_true', | |
| help='Verbose all commands') | |
| other_group.add_argument('-i', '--wait-input', action='store_true', | |
| help='Wait for an enter before going to the next version') | |
| args = parser.parse_args() | |
| if not args.make_args: | |
| args.make_args = [] | |
| sphinx_versions = sorted(list(SPHINX_REQUIREMENTS.keys())) | |
| if args.full: | |
| args.versions += list(SPHINX_REQUIREMENTS.keys()) | |
| venv = SphinxVenv() | |
| await venv.run(args) | |
| # Call main method | |
| if __name__ == "__main__": | |
| asyncio.run(main()) |