| #!/usr/bin/env python3 |
| |
| import os.path |
| import re |
| import requests |
| import textwrap |
| |
| BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains" |
| |
| AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains |
| # Do not edit |
| """ |
| |
| # In the below dict: |
| |
| # - 'conditions' indicate the cumulative conditions under which the |
| # toolchain will be made available. In several situations, a given |
| # toolchain is usable on several architectures variants (for |
| # example, an ARMv6 toolchain can be used on ARMv7) |
| # - 'test_options' indicate one specific configuration where the |
| # toolchain can be used. It is used to create the runtime test |
| # cases. If 'test_options' does not exist, the code assumes it can |
| # be made equal to 'conditions' |
| # - 'prefix' is the prefix of the cross-compilation toolchain tools |
| |
| arches = { |
| 'aarch64': { |
| 'conditions': ['BR2_aarch64'], |
| 'prefix': 'aarch64', |
| }, |
| 'aarch64be': { |
| 'conditions': ['BR2_aarch64_be'], |
| 'prefix': 'aarch64_be', |
| }, |
| 'arcle-750d': { |
| 'conditions': ['BR2_arcle', 'BR2_arc750d'], |
| 'prefix': 'arc', |
| }, |
| 'arcle-hs38': { |
| 'conditions': ['BR2_arcle', 'BR2_archs38'], |
| 'prefix': 'arc', |
| }, |
| 'armv5-eabi': { |
| 'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'], |
| 'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'], |
| 'prefix': 'arm', |
| }, |
| 'armv6-eabihf': { |
| 'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'], |
| 'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'], |
| 'prefix': 'arm', |
| }, |
| 'armv7-eabihf': { |
| 'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'], |
| 'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'], |
| 'prefix': 'arm', |
| }, |
| 'armv7m': { |
| 'conditions': ['BR2_ARM_CPU_ARMV7M'], |
| 'test_options': ['BR2_arm', 'BR2_cortex_m4'], |
| 'prefix': 'arm', |
| }, |
| 'm68k-68xxx': { |
| 'conditions': ['BR2_m68k_m68k'], |
| 'test_options': ['BR2_m68k', 'BR2_m68k_68040'], |
| 'prefix': 'm68k', |
| }, |
| 'm68k-coldfire': { |
| 'conditions': ['BR2_m68k_cf'], |
| 'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'], |
| 'prefix': 'm68k', |
| }, |
| 'microblazebe': { |
| 'conditions': ['BR2_microblazebe'], |
| 'prefix': 'microblaze', |
| }, |
| 'microblazeel': { |
| 'conditions': ['BR2_microblazeel'], |
| 'prefix': 'microblazeel', |
| }, |
| 'mips32': { |
| # Not sure it could be used by other mips32 variants? |
| 'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'], |
| 'prefix': 'mips', |
| }, |
| 'mips32el': { |
| # Not sure it could be used by other mips32el variants? |
| 'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'], |
| 'prefix': 'mipsel', |
| }, |
| 'mips32r5el': { |
| 'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'], |
| 'prefix': 'mipsel', |
| }, |
| 'mips32r6el': { |
| 'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'], |
| 'prefix': 'mipsel', |
| }, |
| 'mips64': { |
| # Not sure it could be used by other mips64 variants? |
| 'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'], |
| 'prefix': 'mips64', |
| }, |
| 'mips64-n32': { |
| # Not sure it could be used by other mips64 variants? |
| 'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], |
| 'prefix': 'mips64', |
| }, |
| 'mips64el-n32': { |
| # Not sure it could be used by other mips64el variants? |
| 'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], |
| 'prefix': 'mips64el', |
| }, |
| 'mips64r6el-n32': { |
| 'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], |
| 'prefix': 'mips64el', |
| }, |
| 'nios2': { |
| 'conditions': ['BR2_nios2'], |
| 'prefix': 'nios2', |
| }, |
| 'openrisc': { |
| 'conditions': ['BR2_or1k'], |
| 'prefix': 'or1k', |
| }, |
| 'powerpc-440fp': { |
| # Not sure it could be used by other powerpc variants? |
| 'conditions': ['BR2_powerpc', 'BR2_powerpc_440fp'], |
| 'prefix': 'powerpc', |
| }, |
| 'powerpc-e300c3': { |
| # Not sure it could be used by other powerpc variants? |
| 'conditions': ['BR2_powerpc', 'BR2_powerpc_e300c3'], |
| 'prefix': 'powerpc', |
| }, |
| 'powerpc-e500mc': { |
| # Not sure it could be used by other powerpc variants? |
| 'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'], |
| 'prefix': 'powerpc', |
| }, |
| 'powerpc64-e5500': { |
| 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'], |
| 'prefix': 'powerpc64', |
| }, |
| 'powerpc64-e6500': { |
| 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e6500'], |
| 'prefix': 'powerpc64', |
| }, |
| 'powerpc64-power8': { |
| 'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'], |
| 'prefix': 'powerpc64', |
| }, |
| 'powerpc64le-power8': { |
| 'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'], |
| 'prefix': 'powerpc64le', |
| }, |
| 'riscv32-ilp32d': { |
| 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'], |
| 'prefix': 'riscv32', |
| }, |
| 'riscv64': { |
| 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'], |
| 'prefix': 'riscv64', |
| }, |
| 'sh-sh4': { |
| 'conditions': ['BR2_sh', 'BR2_sh4'], |
| 'prefix': 'sh4', |
| }, |
| 'sh-sh4aeb': { |
| 'conditions': ['BR2_sh', 'BR2_sh4aeb'], |
| 'prefix': 'sh4aeb', |
| }, |
| 'sparc64': { |
| 'conditions': ['BR2_sparc64', 'BR2_sparc_v9'], |
| 'prefix': 'sparc64', |
| }, |
| 'sparcv8': { |
| 'conditions': ['BR2_sparc', 'BR2_sparc_v8'], |
| 'prefix': 'sparc', |
| }, |
| 'x86-64-core-i7': { |
| 'conditions': ['BR2_x86_64', |
| 'BR2_X86_CPU_HAS_MMX', |
| 'BR2_X86_CPU_HAS_SSE', |
| 'BR2_X86_CPU_HAS_SSE2', |
| 'BR2_X86_CPU_HAS_SSE3', |
| 'BR2_X86_CPU_HAS_SSSE3', |
| 'BR2_X86_CPU_HAS_SSE4', |
| 'BR2_X86_CPU_HAS_SSE42'], |
| 'test_options': ['BR2_x86_64', 'BR2_x86_corei7'], |
| 'prefix': 'x86_64', |
| }, |
| 'x86-core2': { |
| 'conditions': ['BR2_i386', |
| 'BR2_X86_CPU_HAS_MMX', |
| 'BR2_X86_CPU_HAS_SSE', |
| 'BR2_X86_CPU_HAS_SSE2', |
| 'BR2_X86_CPU_HAS_SSE3', |
| 'BR2_X86_CPU_HAS_SSSE3'], |
| 'test_options': ['BR2_i386', 'BR2_x86_core2'], |
| 'prefix': 'i686', |
| }, |
| 'x86-i686': { |
| 'conditions': ['BR2_i386', |
| '!BR2_x86_i486', |
| '!BR2_x86_i586', |
| '!BR2_x86_x1000'], |
| 'test_options': ['BR2_i386', |
| 'BR2_x86_i686'], |
| 'prefix': 'i686', |
| }, |
| 'xtensa-lx60': { |
| 'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'], |
| 'prefix': 'xtensa', |
| }, |
| } |
| |
| |
| class Toolchain: |
| def __init__(self, arch, libc, variant, version): |
| self.arch = arch |
| self.libc = libc |
| self.variant = variant |
| self.version = version |
| self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version) |
| self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \ |
| (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper()) |
| self.fragment = requests.get(self.fragment_url).text.split("\n") |
| self.sha256 = requests.get(self.hash_url).text.split(" ")[0] |
| |
| @property |
| def tarball_url(self): |
| return os.path.join(BASE_URL, self.arch, "tarballs", |
| self.fname_prefix + ".tar.bz2") |
| |
| @property |
| def hash_url(self): |
| return os.path.join(BASE_URL, self.arch, "tarballs", |
| self.fname_prefix + ".sha256") |
| |
| @property |
| def fragment_url(self): |
| return os.path.join(BASE_URL, self.arch, "fragments", |
| self.fname_prefix + ".frag") |
| |
| def gen_config_in_options(self, f): |
| f.write("config %s\n" % self.option_name) |
| f.write("\tbool \"%s %s %s %s\"\n" % |
| (self.arch, self.libc, self.variant, self.version)) |
| depends = [] |
| selects = [] |
| |
| for c in arches[self.arch]['conditions']: |
| depends.append(c) |
| |
| for frag in self.fragment: |
| # libc type |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"): |
| selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC") |
| elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"): |
| # glibc needs mmu support |
| depends.append("BR2_USE_MMU") |
| # glibc doesn't support static only configuration |
| depends.append("!BR2_STATIC_LIBS") |
| selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC") |
| elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"): |
| # musl needs mmu support |
| depends.append("BR2_USE_MMU") |
| selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL") |
| |
| # gcc version |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"): |
| m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag) |
| assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix |
| selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1]) |
| |
| # kernel headers version |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"): |
| m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag) |
| assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix |
| selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1]) |
| |
| # C++ |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"): |
| selects.append("BR2_INSTALL_LIBSTDCPP") |
| |
| # SSP |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"): |
| selects.append("BR2_TOOLCHAIN_HAS_SSP") |
| |
| # wchar |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"): |
| selects.append("BR2_USE_WCHAR") |
| |
| # locale |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"): |
| # locale implies the availability of wchar |
| selects.append("BR2_USE_WCHAR") |
| selects.append("BR2_ENABLE_LOCALE") |
| |
| # thread support |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"): |
| selects.append("BR2_TOOLCHAIN_HAS_THREADS") |
| |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"): |
| selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG") |
| |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"): |
| selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL") |
| |
| # RPC |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"): |
| selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC") |
| |
| # D language |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"): |
| selects.append("BR2_TOOLCHAIN_HAS_DLANG") |
| |
| # fortran |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"): |
| selects.append("BR2_TOOLCHAIN_HAS_FORTRAN") |
| |
| # OpenMP |
| if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"): |
| selects.append("BR2_TOOLCHAIN_HAS_OPENMP") |
| |
| for depend in depends: |
| f.write("\tdepends on %s\n" % depend) |
| |
| for select in selects: |
| f.write("\tselect %s\n" % select) |
| |
| f.write("\thelp\n") |
| |
| desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \ |
| (self.arch, self.libc) |
| |
| if self.variant == "stable": |
| desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils." |
| else: |
| desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils." |
| |
| f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n") |
| f.write("\n") |
| f.write("\t https://toolchains.bootlin.com/\n") |
| |
| f.write("\n") |
| |
| def gen_mk(self, f): |
| f.write("ifeq ($(%s),y)\n" % self.option_name) |
| f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version) |
| f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" % |
| (self.arch, self.libc, self.variant)) |
| f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" % |
| os.path.join(BASE_URL, self.arch, "tarballs")) |
| f.write("endif\n\n") |
| pass |
| |
| def gen_hash(self, f): |
| f.write("# From %s\n" % self.hash_url) |
| f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url))) |
| |
| def gen_test(self, f): |
| if self.variant == "stable": |
| variant = "Stable" |
| else: |
| variant = "BleedingEdge" |
| testname = "TestExternalToolchainBootlin" + \ |
| self.arch.replace("-", "").capitalize() + \ |
| self.libc.capitalize() + variant |
| f.write("\n\n") |
| f.write("class %s(TestExternalToolchain):\n" % testname) |
| f.write(" config = \"\"\"\n") |
| if 'test_options' in arches[self.arch]: |
| test_options = arches[self.arch]['test_options'] |
| else: |
| test_options = arches[self.arch]['conditions'] |
| for opt in test_options: |
| if opt.startswith("!"): |
| f.write(" # %s is not set\n" % opt[1:]) |
| else: |
| f.write(" %s=y\n" % opt) |
| f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n") |
| f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n") |
| f.write(" %s=y\n" % self.option_name) |
| f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n") |
| f.write(" \"\"\"\n") |
| f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix']) |
| f.write("\n") |
| f.write(" def test_run(self):\n") |
| f.write(" TestExternalToolchain.common_check(self)\n") |
| |
| def __repr__(self): |
| return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \ |
| (self.arch, self.libc, self.variant, self.version, self.option_name) |
| |
| |
| def get_toolchains(): |
| toolchains = list() |
| for arch, details in arches.items(): |
| print(arch) |
| url = os.path.join(BASE_URL, arch, "available_toolchains") |
| page = requests.get(url).text |
| fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page)) |
| # This dict will allow us to keep only the latest version for |
| # each toolchain. |
| tmp = dict() |
| for fname in fnames: |
| parts = fname.split('--') |
| assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch) |
| libc = parts[1] |
| if parts[2].startswith("stable-"): |
| variant = "stable" |
| version = parts[2][len("stable-"):] |
| elif parts[2].startswith("bleeding-edge-"): |
| variant = "bleeding-edge" |
| version = parts[2][len("bleeding-edge-"):] |
| tmp[(arch, libc, variant)] = version |
| |
| toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()] |
| |
| return toolchains |
| |
| |
| def gen_config_in_options(toolchains, fpath): |
| with open(fpath, "w") as f: |
| f.write(AUTOGENERATED_COMMENT) |
| |
| f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n") |
| f.write("\tbool\n") |
| for arch, details in arches.items(): |
| f.write("\tdefault y if %s\n" % " && ".join(details['conditions'])) |
| f.write("\n") |
| |
| f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n") |
| |
| f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n") |
| f.write("\tdefault \"$(ARCH)-linux\"\n") |
| |
| f.write("\n") |
| |
| f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n") |
| f.write("\tdefault \"toolchain-external-bootlin\"\n") |
| |
| f.write("\n") |
| |
| f.write("choice\n") |
| f.write("\tprompt \"Bootlin toolchain variant\"\n") |
| |
| for toolchain in toolchains: |
| toolchain.gen_config_in_options(f) |
| |
| f.write("endchoice\n") |
| f.write("endif\n") |
| |
| |
| def gen_mk(toolchains, fpath): |
| with open(fpath, "w") as f: |
| f.write("#" * 80 + "\n") |
| f.write("#\n") |
| f.write("# toolchain-external-bootlin\n") |
| f.write("#\n") |
| f.write("#" * 80 + "\n") |
| f.write("\n") |
| f.write(AUTOGENERATED_COMMENT) |
| for toolchain in toolchains: |
| toolchain.gen_mk(f) |
| f.write("$(eval $(toolchain-external-package))\n") |
| |
| |
| def gen_hash(toolchains, fpath): |
| with open(fpath, "w") as f: |
| f.write(AUTOGENERATED_COMMENT) |
| for toolchain in toolchains: |
| toolchain.gen_hash(f) |
| |
| |
| def gen_runtime_test(toolchains, fpath): |
| with open(fpath, "w") as f: |
| f.write(AUTOGENERATED_COMMENT) |
| f.write("from tests.toolchain.test_external import TestExternalToolchain\n") |
| for toolchain in toolchains: |
| toolchain.gen_test(f) |
| |
| |
| def gen_toolchains(toolchains): |
| maindir = "toolchain/toolchain-external/toolchain-external-bootlin" |
| gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options")) |
| gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk")) |
| gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash")) |
| gen_runtime_test(toolchains, |
| os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py")) |
| |
| |
| toolchains = get_toolchains() |
| gen_toolchains(toolchains) |
