| # Copyright 2020 The Android KVM Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| SELF := $(abspath $(lastword $(MAKEFILE_LIST))) |
| |
| CPUS ?= $(shell nproc) |
| MAKEFLAGS += --jobs=$(CPUS) |
| |
| ## |
| ## Folders |
| ## |
| ROOT_DIR := $(CURDIR) |
| |
| # DIST_DIR might be defined by the CI build (pKVM Busytown) |
| ifdef DIST_DIR |
| OUT_DIR := $(DIST_DIR) |
| else |
| OUT_DIR := $(ROOT_DIR)/out |
| endif |
| |
| LINUX_OUT := $(OUT_DIR)/linux |
| SCRIPTS_DIR := $(ROOT_DIR)/build |
| |
| ## |
| ## Common options |
| ## |
| VERBOSE ?= 0 |
| CUSTOM_KERNEL_IMAGE = 0 |
| ARCH = arm64 |
| |
| # KERNEL_IMAGE can be set to use own custom kernel image. |
| ifdef KERNEL_IMAGE |
| CUSTOM_KERNEL_IMAGE = 1 |
| else |
| KERNEL_IMAGE := $(LINUX_OUT)/arch/arm64/boot/Image.gz |
| endif |
| |
| ## |
| ## Toolchain options and variables |
| ## |
| |
| # Specifies which toolchain of the prebuilt toolchains to use. |
| # Can be either clang, gcc-4.9 (deprecated), gcc-5.1, or gcc-9.2. |
| TOOLCHAIN ?= clang |
| |
| # Find the toolchain path for the current OS. |
| UNNAME_S := $(shell uname -s | tr '[:upper:]' '[:lower:]') |
| TOOLCHAIN_CLANG := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/clang/clang-r383902/ |
| TOOLCHAIN_GCC_49 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-4.9/ |
| TOOLCHAIN_GCC_51 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-5.1/ |
| TOOLCHAIN_GCC_92 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-9.2/ |
| |
| # Set the toolchain build binary paths and prefixes. |
| TARGET = aarch64-linux-gnu |
| TARGET_GCC_92 = aarch64-none-linux-gnu |
| |
| ifeq ($(TOOLCHAIN), gcc-4.9) |
| TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_49)/bin |
| $(warning "The gcc-4.9 toolchain is deprecated and will be removed soon.") |
| $(warning "The gcc-5.1 toolchain is the oldest version Linux needs to support.") |
| else ifeq ($(TOOLCHAIN), gcc-5.1) |
| TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_51)/bin |
| else ifeq ($(TOOLCHAIN), gcc-9.2) |
| TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_92)/bin |
| TARGET = $(TARGET_GCC_92) |
| else ifeq ($(TOOLCHAIN), clang) |
| TOOLCHAIN_BIN := $(TOOLCHAIN_CLANG)/bin |
| CC := $(TOOLCHAIN_BIN)/clang |
| LD := $(TOOLCHAIN_BIN)/ld.lld |
| OBJCOPY := $(TOOLCHAIN_BIN)/llvm-objcopy |
| OBJDUMP := $(TOOLCHAIN_BIN)/llvm-objdump |
| # The GCC toolchain is needed when building kvm-unit-tests as well as Linux |
| # with Clang, because of lack of support for the LLVM integrated assembler. |
| GCC_TOOLCHAIN_DIR = $(TOOLCHAIN_GCC_51) |
| GCC_TOOLCHAIN_BIN = $(GCC_TOOLCHAIN_DIR)/bin/ |
| LLVM = "LLVM=1" |
| # Some targets require additional flags to build with Clang. |
| COMMON_CFLAGS := -Qunused-arguments --target=$(TARGET) -fno-integrated-as -Wno-asm-operand-widths -fpic --gcc-toolchain=$(GCC_TOOLCHAIN_DIR) --sysroot=$(GCC_TOOLCHAIN_DIR)/$(TARGET)/libc --prefix=$(GCC_TOOLCHAIN_BIN)/$(TARGET)- |
| else |
| $(error Unrecognized toolchain: TOOLCHAIN=$(TOOLCHAIN)) |
| endif |
| |
| # common to all GCC versions |
| ifneq ($(TOOLCHAIN), clang) |
| CC := $(TOOLCHAIN_BIN)/$(TARGET)-gcc |
| LD := $(TOOLCHAIN_BIN)/$(TARGET)-ld |
| OBJCOPY := $(TOOLCHAIN_BIN)/$(TARGET)-objcopy |
| OBJDUMP := $(TOOLCHAIN_BIN)/$(TARGET)-objdump |
| GCC_TOOLCHAIN_BIN := $(TOOLCHAIN_BIN) |
| endif |
| |
| ## |
| ## Common targets |
| ## |
| |
| .DEFAULT_GOAL := default |
| |
| .PHONY: default |
| default: kvm-unit-tests linux |
| |
| .PHONY: clean |
| clean: buildroot_clean kvm-unit-tests_clean linux_clean u-boot_clean |
| |
| .PHONY: distclean |
| distclean: |
| - rm -rf $(OUT_DIR) |
| |
| .PHONY: all |
| all: buildroot kvm-unit-tests linux u-boot |
| |
| .PHONY: help |
| help: |
| @echo 'Generic Targets:' |
| @echo ' default - builds all generic targets except for buildroot' |
| @echo ' all - builds all generic targets' |
| @echo ' buildroot - the buildroot rootfs image as well as the host qemu' |
| @echo ' kvm-unit-tests - the kvm-unit-tests' |
| @echo ' linux - the linux kernel' |
| @echo ' u-boot - the u-boot' |
| @echo '' |
| @echo 'Clean Targets:' |
| @echo ' clean - cleans all, but keeps the prebuilts' |
| @echo ' distclean - distclean for all targets' |
| @echo ' buildroot_clean - the buildroot rootfs image as well as the host qemu' |
| @echo ' kvm-unit-tests_clean - the kvm-unit-tests' |
| @echo ' linux_clean - the linux kernel' |
| @echo ' u-boot_clean - the u-boot' |
| @echo '' |
| @echo 'Test Targets:' |
| @echo ' test - runs all tests in qemu-aarch64 simulated environment' |
| @echo ' test-kut - runs all kvm-unit-tests in qemu-aarch64 simulated environment' |
| @echo ' test-list - lists all test targets' |
| @echo '' |
| @echo 'Misc Targets:' |
| @echo ' update-prebuilts - generates/update the prebuilt rootfs image (aarch64) and the qemu host (aarch64)' |
| |
| |
| ## |
| ## Buildroot |
| ## |
| |
| BR_DEFCONFIG ?= qemu_aarch64_virt_kvmunittests_defconfig |
| BR_VERBOSE ?= $(VERBOSE) |
| |
| BR_SRC := $(ROOT_DIR)/buildroot |
| BR_OUT := $(OUT_DIR)/buildroot |
| BR_MAKE := $(MAKE) -C $(BR_SRC) V=$(BR_VERBOSE) O=$(BR_OUT) |
| |
| .PHONY: buildroot |
| buildroot: |
| + $(BR_MAKE) $(BR_DEFCONFIG) |
| + $(BR_MAKE) all |
| |
| .PHONY: buildroot_clean |
| buildroot_clean: |
| + $(BR_MAKE) clean |
| |
| |
| ## |
| ## kvm-unit-tests |
| ## |
| KUT_SRC := $(ROOT_DIR)/kvm-unit-tests |
| KUT_OUT := $(OUT_DIR)/kvm-unit-tests |
| KUT_STAMP := $(KUT_OUT)/kvm-unit-tests.stamp |
| |
| |
| .PHONY: kvm-unit-tests |
| kvm-unit-tests $(KUT_STAMP): |
| mkdir -p $(KUT_OUT) |
| cd $(KUT_OUT) && \ |
| $(KUT_SRC)/configure \ |
| --erratatxt=$(KUT_SRC)/errata.txt \ |
| --prefix=$(KUT_OUT) --arch=$(ARCH) \ |
| --cc=$(CC) --ld=$(LD) \ |
| --objcopy=$(OBJCOPY) --objdump=$(OBJDUMP) |
| COMMON_CFLAGS="$(COMMON_CFLAGS)" \ |
| $(MAKE) -C $(KUT_OUT) standalone |
| touch $(KUT_STAMP) |
| |
| .PHONY: kvm-unit-tests_clean |
| kvm-unit-tests_clean: |
| - rm -f $(KUT_STAMP) |
| - $(MAKE) -C $(KUT_OUT) clean |
| |
| |
| ## |
| ## Linux kernel |
| ## |
| LINUX_DEFCONFIG ?= defconfig |
| LINUX_VERBOSE ?= $(VERBOSE) |
| LINUX_SRC ?= $(ROOT_DIR)/linux |
| |
| LINUX_MAKE := \ |
| PATH=$(TOOLCHAIN_BIN):$(PATH) \ |
| ARCH=$(ARCH) \ |
| CROSS_COMPILE="$(TARGET)-" \ |
| $(MAKE) \ |
| $(LLVM) \ |
| GCC_TOOLCHAIN_DIR="$(GCC_TOOLCHAIN_BIN)" \ |
| -C $(LINUX_SRC) \ |
| V=$(LINUX_VERBOSE) \ |
| O=$(LINUX_OUT) |
| |
| LINUX_CONFIG := $(LINUX_OUT)/.config |
| |
| .PHONY: linux_defconfig |
| linux_defconfig $(LINUX_CONFIG): |
| + $(LINUX_MAKE) $(LINUX_DEFCONFIG) |
| |
| .PHONY: linux_menuconfig |
| linux_menuconfig: $(LINUX_CONFIG) |
| + $(LINUX_MAKE) menuconfig |
| |
| .PHONY: linux |
| linux: $(LINUX_CONFIG) |
| + $(LINUX_MAKE) |
| |
| .PHONY: linux_image |
| linux_image: $(LINUX_CONFIG) |
| + $(LINUX_MAKE) Image.gz |
| |
| # If using own kernel image (KERNEL_IMAGE is set in the environment), then skip. |
| ifeq ($(CUSTOM_KERNEL_IMAGE), 0) |
| $(KERNEL_IMAGE): linux_image |
| endif |
| |
| .PHONY: linux_clean |
| linux_clean: |
| + $(LINUX_MAKE) mrproper |
| |
| |
| ## |
| ## u-boot |
| ## |
| UBOOT_DEFCONFIG ?= pvmfw_arm64_defconfig |
| UBOOT_VERBOSE ?= $(VERBOSE) |
| UBOOT_SRC ?= $(ROOT_DIR)/u-boot |
| |
| # U-boot doesn't support gcc < 6.0. To avoid compilation failure, |
| # we upgrade the active toolchain version for the u-boot build. |
| ifeq ($(TOOLCHAIN), gcc-5.1) |
| $(warning "gcc below 6.0 is not supported by u-boot, switching to 9.2") |
| UBOOT_TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_92)/bin |
| UBOOT_TARGET = $(TARGET_GCC_92) |
| UBOOT_CC := $(TOOLCHAIN_BIN)/$(TARGET)-gcc |
| UBOOT_LD := $(TOOLCHAIN_BIN)/$(TARGET)-ld |
| UBOOT_GCC_TOOLCHAIN_BIN := $(TOOLCHAIN_BIN) |
| else |
| UBOOT_TOOLCHAIN_BIN := $(TOOLCHAIN_BIN) |
| UBOOT_TARGET = $(TARGET) |
| UBOOT_CC := $(CC) |
| UBOOT_LD := $(LD) |
| UBOOT_GCC_TOOLCHAIN_BIN := $(GCC_TOOLCHAIN_BIN) |
| endif |
| |
| UBOOT_OUT := $(OUT_DIR)/u-boot |
| UBOOT_MAKE := \ |
| PATH=$(UBOOT_TOOLCHAIN_BIN):$(UBOOT_GCC_TOOLCHAIN_BIN):$(PATH) \ |
| ARCH=$(ARCH) \ |
| CROSS_COMPILE="$(UBOOT_TARGET)-" \ |
| $(MAKE) \ |
| CC=$(UBOOT_CC) \ |
| LD=$(UBOOT_LD) \ |
| -C $(UBOOT_SRC) \ |
| V=$(UBOOT_VERBOSE) \ |
| O=$(UBOOT_OUT) |
| |
| UBOOT_CONFIG := $(UBOOT_OUT)/.config |
| |
| .PHONY: u-boot_defconfig |
| u-boot_defconfig $(UBOOT_CONFIG): |
| + $(UBOOT_MAKE) $(UBOOT_DEFCONFIG) |
| |
| .PHONY: u-boot_menuconfig |
| u-boot_menuconfig: $(UBOOT_CONFIG) |
| + $(UBOOT_MAKE) menuconfig |
| |
| .PHONY: u-boot |
| u-boot: $(UBOOT_CONFIG) |
| + $(UBOOT_MAKE) |
| |
| .PHONY: u-boot_clean |
| u-boot_clean: |
| + $(UBOOT_MAKE) mrproper |
| |
| |
| ## |
| ## Generating/Updating prebuilts |
| ## |
| |
| PREBUILTS_CROSVM_EXT4 := prebuilts/linux-aarch64/images/crosvm.ext4 |
| PREBUILTS_ROOTFS_EXT4 := prebuilts/linux-aarch64/images/rootfs.ext4 |
| PREBUILTS_QEMU_ROM := prebuilts/linux-aarch64/images/efi-virtio.rom |
| PREBUILTS_QEMU_BIN := prebuilts/linux-x86/qemu |
| |
| .PHONY:update-prebuilts |
| update-prebuilts: \ |
| $(PREBUILTS_ROOTFS_EXT4) \ |
| $(PREBUILTS_CROSVM_EXT4) \ |
| $(PREBUILTS_QEMU_ROM) \ |
| $(PREBUILTS_QEMU_BIN) |
| |
| $(PREBUILTS_ROOTFS_EXT4): buildroot |
| mkdir -p $(shell dirname $@) |
| cp $(BR_OUT)/images/rootfs.ext4 $@ |
| |
| $(PREBUILTS_QEMU_ROM): buildroot |
| mkdir -p $(shell dirname $@) |
| cp $(BR_OUT)/per-package/host-qemu/host/share/qemu/efi-virtio.rom $@ |
| |
| CF_DOWNLOAD_AOSP_SH := $(ROOT_DIR)/build/cuttlefish/download-aosp.sh |
| CROSVM_IMG_SIZE_MB := 1024 |
| |
| $(PREBUILTS_CROSVM_EXT4): TMP_DIR := $(shell mktemp -d) |
| $(PREBUILTS_CROSVM_EXT4): $(CF_DOWNLOAD_AOSP_SH) |
| mkdir -p $(shell dirname $@) |
| cd $(TMP_DIR) && $(CF_DOWNLOAD_AOSP_SH) -a arm64 -C |
| dd if=/dev/zero of=$@.tmp bs=1M count=$(CROSVM_IMG_SIZE_MB) |
| mkfs.ext4 -d $(TMP_DIR) $@.tmp |
| rm -rf $(TMP_DIR) |
| mv $@.tmp $@ |
| |
| # The qemu binary |
| .PHONY:$(PREBUILTS_QEMU_BIN) |
| $(PREBUILTS_QEMU_BIN): |
| - rm -rf $@ |
| ${SCRIPTS_DIR}/aarch64/build_qemu.sh $(OUT_DIR)/qemu ${TOOLCHAIN_BIN} |
| mkdir -p $@/bin |
| cp $(OUT_DIR)/qemu/qemu-system-aarch64 $@/bin |
| |
| ## |
| ## Run tests |
| ## |
| |
| # Root test targets. Dynamically generated per-test targets |
| # will add themselves as dependencies of these. |
| .PHONY: test test-list |
| test: |
| test-list: |
| |
| # |
| # kvm-unit-tests |
| # |
| |
| KUT_RUN_TEST := $(SCRIPTS_DIR)/kvm-unit-tests/run_test.sh |
| KUT_GEN_MAKEFILE := $(SCRIPTS_DIR)/kvm-unit-tests/gen_makefile.sh |
| KUT_MAKEFILE := $(OUT_DIR)/Makefile.kvm-unit-tests |
| KUT_TEST_DIR := $(KUT_OUT)/tests |
| KUT_LOG_DIR := $(OUT_DIR)/test/kvm-unit-tests/ |
| |
| # Exclude tests that require user interaction or are known to fail. |
| KUT_EXCLUDE := (.+migrat.+)|(gicv2-.+)|(pmu-event-introspection)|(micro-bench) |
| |
| # Generate a Makefile with targets per test and configuration. |
| # Pass in variable/target names from this Makefile that the |
| # generated Makefile will refer to. |
| $(KUT_MAKEFILE): $(SELF) $(KUT_GEN_MAKEFILE) $(KUT_STAMP) |
| @ mkdir -p $(shell dirname $@) |
| @ $(KUT_GEN_MAKEFILE) \ |
| $(KUT_TEST_DIR) "$(KUT_EXCLUDE)" \ |
| KERNEL_IMAGE KUT_LOG_DIR test test-list \ |
| > $@.tmp |
| @ mv $@.tmp $@ |
| include $(KUT_MAKEFILE) |
| |
| # |
| # CrosVM Hello World test |
| # |
| |
| CROSVM_HELLOWORLD_SCRIPT_DIR := $(SCRIPTS_DIR)/crosvm/helloworld/ |
| CROSVM_HELLOWORLD_LOG_DIR := $(OUT_DIR)/test/crosvm/helloworld/ |
| CROSVM_HELLOWORLD_TESTS := test-crosvm-vhe-helloworld \ |
| test-crosvm-nvhe-helloworld \ |
| test-crosvm-pkvm-helloworld |
| |
| define gen_crosvm_helloworld_test |
| $(KUT_RUN_TEST) -d $(1) $(2) -t 300s \ |
| -k $(KERNEL_IMAGE) \ |
| -o $(CROSVM_HELLOWORLD_LOG_DIR)/$(1).log \ |
| $(CROSVM_HELLOWORLD_SCRIPT_DIR)/host.sh -- \ |
| -R $(CROSVM_HELLOWORLD_SCRIPT_DIR)/guest.sh \ |
| -R $(KERNEL_IMAGE) \ |
| -R $(PREBUILTS_ROOTFS_EXT4) \ |
| -R $(PREBUILTS_CROSVM_EXT4) |
| endef |
| |
| test: test-crosvm-helloworld |
| test-list: test-list-crosvm-helloworld |
| |
| .PHONY: test-crosvm-helloworld |
| test-crosvm-helloworld: $(CROSVM_HELLOWORLD_TESTS) |
| |
| .PHONY: test-list-crosvm-helloworld |
| test-list-crosvm-helloworld: |
| @ for x in $(CROSVM_HELLOWORLD_TESTS); do echo $x; done |
| |
| .PHONY: test-crosvm-vhe-helloworld |
| test-crosvm-vhe-helloworld: $(KERNEL_IMAGE) |
| @ $(call gen_crosvm_helloworld_test,$@,-V) |
| |
| .PHONY: test-crosvm-nvhe-helloworld |
| test-crosvm-nvhe-helloworld: $(KERNEL_IMAGE) |
| @ $(call gen_crosvm_helloworld_test,$@,-N) |
| |
| .PHONY: test-crosvm-pkvm-helloworld |
| test-crosvm-pkvm-helloworld: $(KERNEL_IMAGE) |
| @ $(call gen_crosvm_helloworld_test,$@,) |