blob: dd03eabadf98dc2229d7107731a02fefee458fb6 [file] [log] [blame]
# Copyright 2020 The Android KVM Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
SELF := $(abspath $(lastword $(MAKEFILE_LIST)))
CPUS ?= $(shell nproc)
MAKEFLAGS += --jobs=$(CPUS)
##
## Folders
##
ROOT_DIR := $(CURDIR)
OUT_DIR := $(ROOT_DIR)/out
# DIST_DIR: target directory for artifacts and logs.
# The CI build (pKVM Busytown) uses it to separate artifacts from other
# temporary build files, which are placed in in OUT_DIR.
# For other users, OUT_DIR and DIST_DIR are typically the same.
DIST_DIR ?= $(OUT_DIR)
# Copy specified file or directory only if OUT_DIR and DIST_DIR aren't the same.
#
# Parameters:
# $(1): Source files or directories (to be copied recursively)
# $(2): Destination directory (is created if it doesn't exist)
ifneq ($(abspath $(OUT_DIR)), $(abspath $(DIST_DIR)))
define copy_to_dist
mkdir -p $(2) && cp -r $(1) $(2)
endef
endif
LINUX_OUT := $(OUT_DIR)/linux
LINUX_DIST := $(DIST_DIR)/linux
SCRIPTS_DIR := $(ROOT_DIR)/build
##
## Common options
##
VERBOSE ?= 0
CUSTOM_KERNEL_IMAGE = 0
ARCH = arm64
# KERNEL_IMAGE can be set to use own custom kernel image.
ifdef KERNEL_IMAGE
CUSTOM_KERNEL_IMAGE = 1
else
KERNEL_IMAGE := $(LINUX_OUT)/arch/arm64/boot/Image
endif
##
## Toolchain options and variables
##
# Specifies which toolchain of the prebuilt toolchains to use.
# Can be either clang, gcc-4.9 (deprecated), gcc-5.1, or gcc-9.2.
TOOLCHAIN ?= clang
# Find the toolchain path for the current OS.
UNNAME_S := $(shell uname -s | tr '[:upper:]' '[:lower:]')
TOOLCHAIN_CLANG := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/clang/clang-r412851/
TOOLCHAIN_GCC_49 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-4.9/
TOOLCHAIN_GCC_51 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-5.1/
TOOLCHAIN_GCC_92 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-9.2/
# Set the toolchain build binary paths and prefixes.
TARGET = aarch64-linux-gnu
TARGET_GCC_92 = aarch64-none-linux-gnu
ifeq ($(TOOLCHAIN), gcc-4.9)
TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_49)/bin
$(warning "The gcc-4.9 toolchain is deprecated and will be removed soon.")
$(warning "The gcc-5.1 toolchain is the oldest version Linux needs to support.")
else ifeq ($(TOOLCHAIN), gcc-5.1)
TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_51)/bin
else ifeq ($(TOOLCHAIN), gcc-9.2)
TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_92)/bin
TARGET = $(TARGET_GCC_92)
else ifeq ($(TOOLCHAIN), clang)
TOOLCHAIN_BIN := $(TOOLCHAIN_CLANG)/bin
CC := $(TOOLCHAIN_BIN)/clang
LD := $(TOOLCHAIN_BIN)/ld.lld
OBJCOPY := $(TOOLCHAIN_BIN)/llvm-objcopy
OBJDUMP := $(TOOLCHAIN_BIN)/llvm-objdump
# The GCC toolchain is needed when building kvm-unit-tests as well as Linux
# with Clang, because of lack of support for the LLVM integrated assembler.
GCC_TOOLCHAIN_DIR = $(TOOLCHAIN_GCC_49)
GCC_TOOLCHAIN_BIN = $(GCC_TOOLCHAIN_DIR)/bin/
LLVM = LLVM=1 LLVM_IAS=1
# Some targets require additional flags to build with Clang.
COMMON_CFLAGS := -Qunused-arguments --target=$(TARGET) -fno-integrated-as -Wno-asm-operand-widths -fpic --gcc-toolchain=$(GCC_TOOLCHAIN_DIR) --sysroot=$(GCC_TOOLCHAIN_DIR)/$(TARGET)/libc --prefix=$(GCC_TOOLCHAIN_BIN)/$(TARGET)-
else
$(error Unrecognized toolchain: TOOLCHAIN=$(TOOLCHAIN))
endif
# common to all GCC versions
ifneq ($(TOOLCHAIN), clang)
CC := $(TOOLCHAIN_BIN)/$(TARGET)-gcc
LD := $(TOOLCHAIN_BIN)/$(TARGET)-ld
OBJCOPY := $(TOOLCHAIN_BIN)/$(TARGET)-objcopy
OBJDUMP := $(TOOLCHAIN_BIN)/$(TARGET)-objdump
GCC_TOOLCHAIN_BIN := $(TOOLCHAIN_BIN)
endif
##
## Common targets
##
.DEFAULT_GOAL := default
.PHONY: default
default: kvm-unit-tests linux
.PHONY: clean
clean: buildroot_clean kvm-unit-tests_clean linux_clean u-boot_clean
.PHONY: distclean
distclean:
- rm -rf $(OUT_DIR) $(DIST_DIR)
.PHONY: all
all: buildroot kvm-unit-tests linux u-boot
.PHONY: help
help:
@echo 'Generic Targets:'
@echo ' default - builds all generic targets except for buildroot'
@echo ' all - builds all generic targets'
@echo ' buildroot - the buildroot rootfs image as well as the host qemu'
@echo ' kvm-unit-tests - the kvm-unit-tests'
@echo ' linux - the linux kernel'
@echo ' u-boot - the u-boot'
@echo ''
@echo 'Clean Targets:'
@echo ' clean - cleans all, but keeps the prebuilts'
@echo ' distclean - distclean for all targets'
@echo ' buildroot_clean - the buildroot rootfs image as well as the host qemu'
@echo ' kvm-unit-tests_clean - the kvm-unit-tests'
@echo ' linux_clean - the linux kernel'
@echo ' u-boot_clean - the u-boot'
@echo ''
@echo 'Test Targets:'
@echo ' test - runs all tests in qemu-aarch64 simulated environment'
@echo ' test-kut - runs all kvm-unit-tests in qemu-aarch64 simulated environment'
@echo ' test-list - lists all test targets'
@echo ''
@echo 'Misc Targets:'
@echo ' update-prebuilts - generates/update the prebuilt rootfs image (aarch64) and the qemu host (aarch64)'
##
## Buildroot
##
BR_DEFCONFIG ?= qemu_aarch64_virt_kvmunittests_defconfig
BR_VERBOSE ?= $(VERBOSE)
BR_SRC := $(ROOT_DIR)/buildroot
BR_OUT := $(OUT_DIR)/buildroot
BR_MAKE := $(MAKE) -C $(BR_SRC) V=$(BR_VERBOSE) O=$(BR_OUT)
.PHONY: buildroot
buildroot:
+ $(BR_MAKE) $(BR_DEFCONFIG)
+ $(BR_MAKE) all
.PHONY: buildroot_clean
buildroot_clean:
+ $(BR_MAKE) clean
##
## kvm-unit-tests
##
KUT_SRC := $(ROOT_DIR)/kvm-unit-tests
KUT_OUT := $(OUT_DIR)/kvm-unit-tests
KUT_DIST := $(DIST_DIR)/kvm-unit-tests
KUT_STAMP := $(KUT_OUT)/kvm-unit-tests.stamp
.PHONY: kvm-unit-tests
kvm-unit-tests $(KUT_STAMP):
mkdir -p $(KUT_OUT)
cd $(KUT_OUT) && \
$(KUT_SRC)/configure \
--erratatxt=$(KUT_SRC)/errata.txt \
--prefix=$(KUT_OUT) --arch=$(ARCH) \
--cc=$(CC) --ld=$(LD) \
--objcopy=$(OBJCOPY) --objdump=$(OBJDUMP)
COMMON_CFLAGS="$(COMMON_CFLAGS)" \
$(MAKE) -C $(KUT_OUT) standalone
$(call copy_to_dist, $(KUT_OUT)/tests, $(KUT_DIST))
touch $(KUT_STAMP)
.PHONY: kvm-unit-tests_clean
kvm-unit-tests_clean:
- rm -f $(KUT_STAMP)
- $(MAKE) -C $(KUT_OUT) clean
##
## Linux kernel
##
LINUX_DEFCONFIG ?= defconfig
LINUX_VERBOSE ?= $(VERBOSE)
LINUX_SRC ?= $(ROOT_DIR)/linux
LINUX_VMLINUX = $(LINUX_OUT)/vmlinux
LINUX_DIST_FILES = $(LINUX_CONFIG) $(LINUX_VMLINUX) $(KERNEL_IMAGE)
LINUX_MAKE := \
PATH=$(TOOLCHAIN_BIN):$(PATH) \
ARCH=$(ARCH) \
CROSS_COMPILE="$(TARGET)-" \
$(MAKE) \
$(LLVM) \
GCC_TOOLCHAIN_DIR="$(GCC_TOOLCHAIN_BIN)" \
-C $(LINUX_SRC) \
V=$(LINUX_VERBOSE) \
O=$(LINUX_OUT)
LINUX_CONFIG := $(LINUX_OUT)/.config
.PHONY: linux_defconfig
linux_defconfig $(LINUX_CONFIG):
+ $(LINUX_MAKE) $(LINUX_DEFCONFIG)
.PHONY: linux_menuconfig
linux_menuconfig: $(LINUX_CONFIG)
+ $(LINUX_MAKE) menuconfig
.PHONY: linux
linux: $(LINUX_CONFIG)
+ $(LINUX_MAKE)
$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))
.PHONY: linux_image
linux_image: $(LINUX_CONFIG)
+ $(LINUX_MAKE) Image.gz
$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))
# If using own kernel image (KERNEL_IMAGE is set in the environment), then skip.
ifeq ($(CUSTOM_KERNEL_IMAGE), 0)
$(KERNEL_IMAGE): linux_image
endif
.PHONY: linux_clean
linux_clean:
+ $(LINUX_MAKE) mrproper
##
## u-boot
##
UBOOT_DEFCONFIG ?= pvmfw_arm64_defconfig
UBOOT_VERBOSE ?= $(VERBOSE)
UBOOT_SRC ?= $(ROOT_DIR)/u-boot
# U-boot doesn't support gcc < 6.0. To avoid compilation failure,
# we upgrade the active toolchain version for the u-boot build.
ifeq ($(TOOLCHAIN), gcc-5.1)
$(warning "gcc below 6.0 is not supported by u-boot, switching to 9.2")
UBOOT_TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_92)/bin
UBOOT_TARGET = $(TARGET_GCC_92)
UBOOT_CC := $(UBOOT_TOOLCHAIN_BIN)/$(UBOOT_TARGET)-gcc
UBOOT_LD := $(UBOOT_TOOLCHAIN_BIN)/$(UBOOT_TARGET)-ld
UBOOT_GCC_TOOLCHAIN_BIN := $(UBOOT_TOOLCHAIN_BIN)
else
UBOOT_TOOLCHAIN_BIN := $(TOOLCHAIN_BIN)
UBOOT_TARGET = $(TARGET)
UBOOT_CC := $(CC)
UBOOT_LD := $(LD)
UBOOT_GCC_TOOLCHAIN_BIN := $(GCC_TOOLCHAIN_BIN)
endif
UBOOT_OUT := $(OUT_DIR)/u-boot
UBOOT_MAKE := \
PATH=$(UBOOT_TOOLCHAIN_BIN):$(UBOOT_GCC_TOOLCHAIN_BIN):$(PATH) \
ARCH=$(ARCH) \
CROSS_COMPILE="$(UBOOT_TARGET)-" \
$(MAKE) \
CC=$(UBOOT_CC) \
LD=$(UBOOT_LD) \
-C $(UBOOT_SRC) \
V=$(UBOOT_VERBOSE) \
O=$(UBOOT_OUT)
UBOOT_CONFIG := $(UBOOT_OUT)/.config
UBOOT_BIN := $(UBOOT_OUT)/u-boot.bin
.PHONY: u-boot_defconfig
u-boot_defconfig $(UBOOT_CONFIG):
+ $(UBOOT_MAKE) $(UBOOT_DEFCONFIG)
.PHONY: u-boot_menuconfig
u-boot_menuconfig: $(UBOOT_CONFIG)
+ $(UBOOT_MAKE) menuconfig
.PHONY: u-boot
u-boot $(UBOOT_BIN): $(UBOOT_CONFIG)
+ $(UBOOT_MAKE)
.PHONY: u-boot_clean
u-boot_clean:
+ $(UBOOT_MAKE) mrproper
##
## Generating/Updating prebuilts
##
PREBUILTS_CROSVM_EXT4 := prebuilts/linux-aarch64/images/crosvm.ext4
PREBUILTS_ROOTFS_EXT4 := prebuilts/linux-aarch64/images/rootfs.ext4
PREBUILTS_QEMU_ROM := prebuilts/linux-aarch64/images/efi-virtio.rom
PREBUILTS_QEMU_BIN := prebuilts/linux-x86/qemu
.PHONY:update-prebuilts
update-prebuilts: \
$(PREBUILTS_ROOTFS_EXT4) \
$(PREBUILTS_CROSVM_EXT4) \
$(PREBUILTS_QEMU_ROM) \
$(PREBUILTS_QEMU_BIN)
$(PREBUILTS_ROOTFS_EXT4): buildroot
mkdir -p $(shell dirname $@)
cp $(BR_OUT)/images/rootfs.ext4 $@
$(PREBUILTS_QEMU_ROM): buildroot
mkdir -p $(shell dirname $@)
cp $(BR_OUT)/per-package/host-qemu/host/share/qemu/efi-virtio.rom $@
CF_DOWNLOAD_AOSP_SH := $(ROOT_DIR)/build/cuttlefish/download-aosp.sh
CROSVM_IMG_SIZE_MB := 1024
$(PREBUILTS_CROSVM_EXT4): TMP_DIR := $(shell mktemp -d)
$(PREBUILTS_CROSVM_EXT4): $(CF_DOWNLOAD_AOSP_SH)
mkdir -p $(shell dirname $@)
cd $(TMP_DIR) && $(CF_DOWNLOAD_AOSP_SH) -a arm64 -C
dd if=/dev/zero of=$@.tmp bs=1M count=$(CROSVM_IMG_SIZE_MB)
mkfs.ext4 -d $(TMP_DIR) $@.tmp
rm -rf $(TMP_DIR)
mv $@.tmp $@
# The qemu binary
.PHONY:$(PREBUILTS_QEMU_BIN)
$(PREBUILTS_QEMU_BIN):
- rm -rf $@
${SCRIPTS_DIR}/aarch64/build_qemu.sh $(OUT_DIR)/qemu ${TOOLCHAIN_BIN}
mkdir -p $@/bin
cp $(OUT_DIR)/qemu/qemu-system-aarch64 $@/bin
##
## Run tests
##
# Root test targets. Dynamically generated per-test targets
# will add themselves as dependencies of these.
.PHONY: test test-list
test:
test-list:
#
# kvm-unit-tests
#
KUT_RUN_TEST := $(SCRIPTS_DIR)/kvm-unit-tests/run_test.sh
KUT_GEN_MAKEFILE := $(SCRIPTS_DIR)/kvm-unit-tests/gen_makefile.sh
KUT_MAKEFILE := $(OUT_DIR)/Makefile.kvm-unit-tests
KUT_TEST_DIR := $(KUT_OUT)/tests
KUT_LOG_DIR := $(DIST_DIR)/logs/kvm-unit-tests/
# Exclude tests that require user interaction or are known to fail.
KUT_EXCLUDE := (.+migrat.+)|(gicv2-.+)|(pmu-event-introspection)|(micro-bench)
# Generate a Makefile with targets per test and configuration.
# Pass in variable/target names from this Makefile that the
# generated Makefile will refer to.
$(KUT_MAKEFILE): $(SELF) $(KUT_GEN_MAKEFILE) $(KUT_STAMP)
@ mkdir -p $(shell dirname $@)
@ $(KUT_GEN_MAKEFILE) \
$(KUT_TEST_DIR) "$(KUT_EXCLUDE)" \
KERNEL_IMAGE KUT_LOG_DIR test test-list \
> $@.tmp
@ mv $@.tmp $@
include $(KUT_MAKEFILE)
#
# Basic boot tests in the various arm64 KVM modes
#
BOOT_TEST_SCRIPT_DIR := $(SCRIPTS_DIR)/aarch64/
BOOT_TEST_LOG_DIR := $(DIST_DIR)/logs/boot/
BOOT_TESTS := test-boot-vhe test-boot-nvhe test-boot-pkvm
define gen_boot_test
$(KUT_RUN_TEST) -d $(1) -M $(2) \
-k $(KERNEL_IMAGE) \
-o $(BOOT_TEST_LOG_DIR)/$(1).log \
$(BOOT_TEST_SCRIPT_DIR)/test_boot.sh -- \
-a "boot_test_mode=$(2)"
endef
test: test-boot
test-list: test-list-boot
.PHONY: test-boot
test-boot: $(BOOT_TESTS)
.PHONY: test-list-boot
test-list-boot:
@ for x in $(BOOT_TESTS); do echo $x; done
.PHONY: test-boot-vhe
test-boot-vhe: $(KERNEL_IMAGE)
@ $(call gen_boot_test,$@,vhe)
.PHONY: test-boot-nvhe
test-boot-nvhe: $(KERNEL_IMAGE)
@ $(call gen_boot_test,$@,nvhe)
.PHONY: test-boot-pkvm
test-boot-pkvm: $(KERNEL_IMAGE)
@ $(call gen_boot_test,$@,pkvm)
#
# CrosVM Hello World test
#
CROSVM_HELLOWORLD_SCRIPT_DIR := $(SCRIPTS_DIR)/crosvm/helloworld/
CROSVM_HELLOWORLD_LOG_DIR := $(DIST_DIR)/logs/crosvm/helloworld/
CROSVM_HELLOWORLD_TESTS := test-crosvm-vhe-helloworld \
test-crosvm-nvhe-helloworld \
test-crosvm-pkvm-helloworld
define gen_crosvm_helloworld_test
$(KUT_RUN_TEST) -d $(1) -M $(2) -t 300s \
-k $(KERNEL_IMAGE) \
-F $(UBOOT_BIN) \
-o $(CROSVM_HELLOWORLD_LOG_DIR)/$(1).log \
$(CROSVM_HELLOWORLD_SCRIPT_DIR)/host.sh -- \
-s 4 \
-R $(CROSVM_HELLOWORLD_SCRIPT_DIR)/guest.sh \
-R $(KERNEL_IMAGE) \
-R $(PREBUILTS_ROOTFS_EXT4) \
-R $(PREBUILTS_CROSVM_EXT4)
endef
test: test-crosvm-helloworld
test-list: test-list-crosvm-helloworld
.PHONY: test-crosvm-helloworld
test-crosvm-helloworld: $(CROSVM_HELLOWORLD_TESTS)
.PHONY: test-list-crosvm-helloworld
test-list-crosvm-helloworld:
@ for x in $(CROSVM_HELLOWORLD_TESTS); do echo $x; done
.PHONY: test-crosvm-vhe-helloworld
test-crosvm-vhe-helloworld: $(KERNEL_IMAGE) $(UBOOT_BIN)
@ $(call gen_crosvm_helloworld_test,$@,vhe)
.PHONY: test-crosvm-nvhe-helloworld
test-crosvm-nvhe-helloworld: $(KERNEL_IMAGE) $(UBOOT_BIN)
@ $(call gen_crosvm_helloworld_test,$@,nvhe)
.PHONY: test-crosvm-pkvm-helloworld
test-crosvm-pkvm-helloworld: $(KERNEL_IMAGE) $(UBOOT_BIN)
@ $(call gen_crosvm_helloworld_test,$@,pkvm)