Makefile - build - Git at Google

 # Copyright 2020 The Android KVM Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 SELF := $(abspath $(lastword $(MAKEFILE_LIST)))

 CPUS ?= $(shell nproc)
 MAKEFLAGS += --jobs=$(CPUS)

 ##
 ## Folders
 ##
 ROOT_DIR := $(CURDIR)

 OUT_DIR := $(ROOT_DIR)/out

 # DIST_DIR: target directory for artifacts and logs.
 # The CI build (pKVM Busytown) uses it to separate artifacts from other
 # temporary build files, which are placed in in OUT_DIR.
 # For other users, OUT_DIR and DIST_DIR are typically the same.
 DIST_DIR ?= $(OUT_DIR)

 # Copy specified file or directory only if OUT_DIR and DIST_DIR aren't the same.
 #
 # Parameters:
 # $(1): Source files or directories (to be copied recursively)
 # $(2): Destination directory (is created if it doesn't exist)
 ifneq ($(abspath $(OUT_DIR)), $(abspath $(DIST_DIR)))
 define copy_to_dist
 	mkdir -p $(2) && cp -r $(1) $(2)
 endef
 endif

 LINUX_OUT := $(OUT_DIR)/linux
 LINUX_DIST := $(DIST_DIR)/linux
 SCRIPTS_DIR := $(ROOT_DIR)/build

 ##
 ## Common options
 ##
 VERBOSE ?= 0
 CUSTOM_KERNEL_IMAGE = 0
 ARCH = arm64

 # KERNEL_IMAGE can be set to use own custom kernel image.
 ifdef KERNEL_IMAGE
 CUSTOM_KERNEL_IMAGE = 1
 else
 KERNEL_IMAGE := $(LINUX_OUT)/arch/arm64/boot/Image
 endif

 ##
 ## Toolchain options and variables
 ##

 # Specifies which toolchain of the prebuilt toolchains to use.
 # Can be either clang, gcc-4.9 (deprecated), gcc-5.1, or gcc-9.2.
 TOOLCHAIN ?= clang

 # Find the toolchain path for the current OS.
 UNNAME_S := $(shell uname -s | tr '[:upper:]' '[:lower:]')
 TOOLCHAIN_CLANG := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/clang/clang-r412851/
 TOOLCHAIN_GCC_49 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-4.9/
 TOOLCHAIN_GCC_51 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-5.1/
 TOOLCHAIN_GCC_92 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-9.2/

 # Set the toolchain build binary paths and prefixes.
 TARGET = aarch64-linux-gnu
 TARGET_GCC_92 = aarch64-none-linux-gnu

 ifeq ($(TOOLCHAIN), gcc-4.9)
 TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_49)/bin
 $(warning "The gcc-4.9 toolchain is deprecated and will be removed soon.")
 $(warning "The gcc-5.1 toolchain is the oldest version Linux needs to support.")
 else ifeq ($(TOOLCHAIN), gcc-5.1)
 TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_51)/bin
 else ifeq ($(TOOLCHAIN), gcc-9.2)
 TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_92)/bin
 TARGET = $(TARGET_GCC_92)
 else ifeq ($(TOOLCHAIN), clang)
 TOOLCHAIN_BIN := $(TOOLCHAIN_CLANG)/bin
 CC := $(TOOLCHAIN_BIN)/clang
 LD := $(TOOLCHAIN_BIN)/ld.lld
 OBJCOPY := $(TOOLCHAIN_BIN)/llvm-objcopy
 OBJDUMP := $(TOOLCHAIN_BIN)/llvm-objdump
 # The GCC toolchain is needed when building kvm-unit-tests as well as Linux
 # with Clang, because of lack of support for the LLVM integrated assembler.
 GCC_TOOLCHAIN_DIR = $(TOOLCHAIN_GCC_49)
 GCC_TOOLCHAIN_BIN = $(GCC_TOOLCHAIN_DIR)/bin/
 LLVM = LLVM=1 LLVM_IAS=1
 # Some targets require additional flags to build with Clang.
 COMMON_CFLAGS := -Qunused-arguments --target=$(TARGET) -fno-integrated-as -Wno-asm-operand-widths -fpic --gcc-toolchain=$(GCC_TOOLCHAIN_DIR) --sysroot=$(GCC_TOOLCHAIN_DIR)/$(TARGET)/libc --prefix=$(GCC_TOOLCHAIN_BIN)/$(TARGET)-
 else
 $(error Unrecognized toolchain: TOOLCHAIN=$(TOOLCHAIN))
 endif

 # common to all GCC versions
 ifneq ($(TOOLCHAIN), clang)
 CC := $(TOOLCHAIN_BIN)/$(TARGET)-gcc
 LD := $(TOOLCHAIN_BIN)/$(TARGET)-ld
 OBJCOPY := $(TOOLCHAIN_BIN)/$(TARGET)-objcopy
 OBJDUMP := $(TOOLCHAIN_BIN)/$(TARGET)-objdump
 GCC_TOOLCHAIN_BIN := $(TOOLCHAIN_BIN)
 endif

 ##
 ## Common targets
 ##

 .DEFAULT_GOAL := default

 .PHONY: default
 default: kvm-unit-tests linux

 .PHONY: clean
 clean: buildroot_clean kvm-unit-tests_clean linux_clean u-boot_clean

 .PHONY: distclean
 distclean:
 	- rm -rf $(OUT_DIR) $(DIST_DIR)

 .PHONY: all
 all: buildroot kvm-unit-tests linux u-boot

 .PHONY: help
 help:
 	@echo 'Generic Targets:'
 	@echo '   default              - builds all generic targets except for buildroot'
 	@echo '   all                  - builds all generic targets'
 	@echo '   buildroot            - the buildroot rootfs image as well as the host qemu'
 	@echo '   kvm-unit-tests       - the kvm-unit-tests'
 	@echo '   linux                - the linux kernel'
 	@echo '   u-boot               - the u-boot'
 	@echo ''
 	@echo 'Clean Targets:'
 	@echo '   clean                - cleans all, but keeps the prebuilts'
 	@echo '   distclean            - distclean for all targets'
 	@echo '   buildroot_clean      - the buildroot rootfs image as well as the host qemu'
 	@echo '   kvm-unit-tests_clean - the kvm-unit-tests'
 	@echo '   linux_clean          - the linux kernel'
 	@echo '   u-boot_clean         - the u-boot'
 	@echo ''
 	@echo 'Test Targets:'
 	@echo '   test                 - runs all tests in qemu-aarch64 simulated environment'
 	@echo '   test-kut             - runs all kvm-unit-tests in qemu-aarch64 simulated environment'
 	@echo '   test-list            - lists all test targets'
 	@echo ''
 	@echo 'Misc Targets:'
 	@echo '   update-prebuilts     - generates/update the prebuilt rootfs image (aarch64) and the qemu host (aarch64)'


 ##
 ## Buildroot
 ##

 BR_DEFCONFIG ?= qemu_aarch64_virt_kvmunittests_defconfig
 BR_VERBOSE ?= $(VERBOSE)

 BR_SRC := $(ROOT_DIR)/buildroot
 BR_OUT := $(OUT_DIR)/buildroot
 BR_MAKE := $(MAKE) -C $(BR_SRC) V=$(BR_VERBOSE) O=$(BR_OUT)

 .PHONY: buildroot
 buildroot:
 	+ $(BR_MAKE) $(BR_DEFCONFIG)
 	+ $(BR_MAKE) all

 .PHONY: buildroot_clean
 buildroot_clean:
 	+ $(BR_MAKE) clean


 ##
 ## kvm-unit-tests
 ##
 KUT_SRC := $(ROOT_DIR)/kvm-unit-tests
 KUT_OUT := $(OUT_DIR)/kvm-unit-tests
 KUT_DIST := $(DIST_DIR)/kvm-unit-tests
 KUT_STAMP := $(KUT_OUT)/kvm-unit-tests.stamp


 .PHONY: kvm-unit-tests
 kvm-unit-tests $(KUT_STAMP):
 	mkdir -p $(KUT_OUT)
 	cd $(KUT_OUT) && \
 		$(KUT_SRC)/configure \
 			--erratatxt=$(KUT_SRC)/errata.txt \
 			--prefix=$(KUT_OUT) --arch=$(ARCH) \
 			--cc=$(CC) --ld=$(LD) \
 			--objcopy=$(OBJCOPY) --objdump=$(OBJDUMP)
 	COMMON_CFLAGS="$(COMMON_CFLAGS)" \
 		$(MAKE) -C $(KUT_OUT) standalone
 	$(call copy_to_dist, $(KUT_OUT)/tests, $(KUT_DIST))
 	touch $(KUT_STAMP)

 .PHONY: kvm-unit-tests_clean
 kvm-unit-tests_clean:
 	- rm -f $(KUT_STAMP)
 	- $(MAKE) -C $(KUT_OUT) clean


 ##
 ## Linux kernel
 ##
 LINUX_DEFCONFIG ?= defconfig
 LINUX_VERBOSE ?= $(VERBOSE)
 LINUX_SRC ?= $(ROOT_DIR)/linux
 LINUX_VMLINUX = $(LINUX_OUT)/vmlinux
 LINUX_DIST_FILES = $(LINUX_CONFIG) $(LINUX_VMLINUX) $(KERNEL_IMAGE)

 LINUX_MAKE := \
 	PATH=$(TOOLCHAIN_BIN):$(PATH) \
 	ARCH=$(ARCH) \
 	CROSS_COMPILE="$(TARGET)-" \
 	$(MAKE) \
 	$(LLVM) \
 	GCC_TOOLCHAIN_DIR="$(GCC_TOOLCHAIN_BIN)" \
 	-C $(LINUX_SRC) \
 	V=$(LINUX_VERBOSE) \
 	O=$(LINUX_OUT)

 LINUX_CONFIG := $(LINUX_OUT)/.config

 .PHONY: linux_defconfig
 linux_defconfig $(LINUX_CONFIG):
 	+ $(LINUX_MAKE) $(LINUX_DEFCONFIG)

 .PHONY: linux_menuconfig
 linux_menuconfig: $(LINUX_CONFIG)
 	+ $(LINUX_MAKE) menuconfig

 .PHONY: linux
 linux: $(LINUX_CONFIG)
 	+ $(LINUX_MAKE)
 	$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))

 .PHONY: linux_image
 linux_image: $(LINUX_CONFIG)
 	+ $(LINUX_MAKE) Image.gz
 	$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))

 # If using own kernel image (KERNEL_IMAGE is set in the environment), then skip.
 ifeq ($(CUSTOM_KERNEL_IMAGE), 0)
 $(KERNEL_IMAGE): linux_image
 endif

 .PHONY: linux_clean
 linux_clean:
 	+ $(LINUX_MAKE) mrproper


 ##
 ## u-boot
 ##
 UBOOT_DEFCONFIG ?= pvmfw_arm64_defconfig
 UBOOT_VERBOSE ?= $(VERBOSE)
 UBOOT_SRC ?= $(ROOT_DIR)/u-boot

 # U-boot doesn't support gcc < 6.0. To avoid compilation failure,
 # we upgrade the active toolchain version for the u-boot build.
 ifeq ($(TOOLCHAIN), gcc-5.1)
 $(warning "gcc below 6.0 is not supported by u-boot, switching to 9.2")
 UBOOT_TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_92)/bin
 UBOOT_TARGET = $(TARGET_GCC_92)
 UBOOT_CC := $(UBOOT_TOOLCHAIN_BIN)/$(UBOOT_TARGET)-gcc
 UBOOT_LD := $(UBOOT_TOOLCHAIN_BIN)/$(UBOOT_TARGET)-ld
 UBOOT_GCC_TOOLCHAIN_BIN := $(UBOOT_TOOLCHAIN_BIN)
 else
 UBOOT_TOOLCHAIN_BIN := $(TOOLCHAIN_BIN)
 UBOOT_TARGET = $(TARGET)
 UBOOT_CC := $(CC)
 UBOOT_LD := $(LD)
 UBOOT_GCC_TOOLCHAIN_BIN := $(GCC_TOOLCHAIN_BIN)
 endif

 UBOOT_OUT := $(OUT_DIR)/u-boot
 UBOOT_MAKE := \
 	PATH=$(UBOOT_TOOLCHAIN_BIN):$(UBOOT_GCC_TOOLCHAIN_BIN):$(PATH) \
 	ARCH=$(ARCH) \
 	CROSS_COMPILE="$(UBOOT_TARGET)-" \
 	$(MAKE) \
 	CC=$(UBOOT_CC) \
 	LD=$(UBOOT_LD) \
 	-C $(UBOOT_SRC) \
 	V=$(UBOOT_VERBOSE) \
 	O=$(UBOOT_OUT)

 UBOOT_CONFIG := $(UBOOT_OUT)/.config
 UBOOT_BIN := $(UBOOT_OUT)/u-boot.bin

 .PHONY: u-boot_defconfig
 u-boot_defconfig $(UBOOT_CONFIG):
 	+ $(UBOOT_MAKE) $(UBOOT_DEFCONFIG)

 .PHONY: u-boot_menuconfig
 u-boot_menuconfig: $(UBOOT_CONFIG)
 	+ $(UBOOT_MAKE) menuconfig

 .PHONY: u-boot
 u-boot $(UBOOT_BIN): $(UBOOT_CONFIG)
 	+ $(UBOOT_MAKE)

 .PHONY: u-boot_clean
 u-boot_clean:
 	+ $(UBOOT_MAKE) mrproper


 ##
 ## Generating/Updating prebuilts
 ##

 PREBUILTS_CROSVM_EXT4 := prebuilts/linux-aarch64/images/crosvm.ext4
 PREBUILTS_ROOTFS_EXT4 := prebuilts/linux-aarch64/images/rootfs.ext4
 PREBUILTS_QEMU_ROM := prebuilts/linux-aarch64/images/efi-virtio.rom
 PREBUILTS_QEMU_BIN := prebuilts/linux-x86/qemu

 .PHONY:update-prebuilts
 update-prebuilts: \
 	$(PREBUILTS_ROOTFS_EXT4) \
 	$(PREBUILTS_CROSVM_EXT4) \
 	$(PREBUILTS_QEMU_ROM) \
 	$(PREBUILTS_QEMU_BIN)

 $(PREBUILTS_ROOTFS_EXT4): buildroot
 	mkdir -p $(shell dirname $@)
 	cp $(BR_OUT)/images/rootfs.ext4 $@

 $(PREBUILTS_QEMU_ROM): buildroot
 	mkdir -p $(shell dirname $@)
 	cp $(BR_OUT)/per-package/host-qemu/host/share/qemu/efi-virtio.rom $@

 CF_DOWNLOAD_AOSP_SH := $(ROOT_DIR)/build/cuttlefish/download-aosp.sh
 CROSVM_IMG_SIZE_MB := 1024

 $(PREBUILTS_CROSVM_EXT4): TMP_DIR := $(shell mktemp -d)
 $(PREBUILTS_CROSVM_EXT4): $(CF_DOWNLOAD_AOSP_SH)
 	mkdir -p $(shell dirname $@)
 	cd $(TMP_DIR) && $(CF_DOWNLOAD_AOSP_SH) -a arm64 -C
 	dd if=/dev/zero of=$@.tmp bs=1M count=$(CROSVM_IMG_SIZE_MB)
 	mkfs.ext4 -d $(TMP_DIR) $@.tmp
 	rm -rf $(TMP_DIR)
 	mv $@.tmp $@

 # The qemu binary
 .PHONY:$(PREBUILTS_QEMU_BIN)
 $(PREBUILTS_QEMU_BIN):
 	- rm -rf $@
 	${SCRIPTS_DIR}/aarch64/build_qemu.sh $(OUT_DIR)/qemu ${TOOLCHAIN_BIN}
 	mkdir -p $@/bin
 	cp $(OUT_DIR)/qemu/qemu-system-aarch64 $@/bin

 ##
 ## Run tests
 ##

 # Root test targets. Dynamically generated per-test targets
 # will add themselves as dependencies of these.
 .PHONY: test test-list
 test:
 test-list:

 #
 # kvm-unit-tests
 #

 KUT_RUN_TEST := $(SCRIPTS_DIR)/kvm-unit-tests/run_test.sh
 KUT_GEN_MAKEFILE := $(SCRIPTS_DIR)/kvm-unit-tests/gen_makefile.sh
 KUT_MAKEFILE := $(OUT_DIR)/Makefile.kvm-unit-tests
 KUT_TEST_DIR := $(KUT_OUT)/tests
 KUT_LOG_DIR := $(DIST_DIR)/logs/kvm-unit-tests/

 # Exclude tests that require user interaction or are known to fail.
 KUT_EXCLUDE := (.+migrat.+)|(gicv2-.+)|(pmu-event-introspection)|(micro-bench)

 # Generate a Makefile with targets per test and configuration.
 # Pass in variable/target names from this Makefile that the
 # generated Makefile will refer to.
 $(KUT_MAKEFILE): $(SELF) $(KUT_GEN_MAKEFILE) $(KUT_STAMP)
 	@ mkdir -p $(shell dirname $@)
 	@ $(KUT_GEN_MAKEFILE)				\
 		$(KUT_TEST_DIR) "$(KUT_EXCLUDE)"	\
 		KERNEL_IMAGE KUT_LOG_DIR test test-list	\
 		> $@.tmp
 	@ mv $@.tmp $@
 include $(KUT_MAKEFILE)

 #
 # Basic boot tests in the various arm64 KVM modes
 #

 BOOT_TEST_SCRIPT_DIR := $(SCRIPTS_DIR)/aarch64/
 BOOT_TEST_LOG_DIR := $(DIST_DIR)/logs/boot/
 BOOT_TESTS := test-boot-vhe test-boot-nvhe test-boot-pkvm

 define gen_boot_test
 	$(KUT_RUN_TEST) -d $(1) -M $(2) \
 		-k $(KERNEL_IMAGE) \
 		-o $(BOOT_TEST_LOG_DIR)/$(1).log \
 		$(BOOT_TEST_SCRIPT_DIR)/test_boot.sh -- \
 		-a "boot_test_mode=$(2)"
 endef

 test: test-boot
 test-list: test-list-boot

 .PHONY: test-boot
 test-boot: $(BOOT_TESTS)

 .PHONY: test-list-boot
 test-list-boot:
 	@ for x in $(BOOT_TESTS); do echo $x; done

 .PHONY: test-boot-vhe
 test-boot-vhe: $(KERNEL_IMAGE)
 	@ $(call gen_boot_test,$@,vhe)

 .PHONY: test-boot-nvhe
 test-boot-nvhe: $(KERNEL_IMAGE)
 	@ $(call gen_boot_test,$@,nvhe)

 .PHONY: test-boot-pkvm
 test-boot-pkvm: $(KERNEL_IMAGE)
 	@ $(call gen_boot_test,$@,pkvm)

 #
 # CrosVM Hello World test
 #

 CROSVM_HELLOWORLD_SCRIPT_DIR := $(SCRIPTS_DIR)/crosvm/helloworld/
 CROSVM_HELLOWORLD_LOG_DIR := $(DIST_DIR)/logs/crosvm/helloworld/
 CROSVM_HELLOWORLD_TESTS := test-crosvm-vhe-helloworld \
 			   test-crosvm-nvhe-helloworld \
 			   test-crosvm-pkvm-helloworld

 define gen_crosvm_helloworld_test
 	$(KUT_RUN_TEST) -d $(1) -M $(2) -t 300s \
 		-k $(KERNEL_IMAGE) \
 		-F $(UBOOT_BIN) \
 		-o $(CROSVM_HELLOWORLD_LOG_DIR)/$(1).log \
 		$(CROSVM_HELLOWORLD_SCRIPT_DIR)/host.sh -- \
 		-R $(CROSVM_HELLOWORLD_SCRIPT_DIR)/guest.sh \
 		-R $(KERNEL_IMAGE) \
 		-R $(PREBUILTS_ROOTFS_EXT4) \
 		-R $(PREBUILTS_CROSVM_EXT4)
 endef

 test: test-crosvm-helloworld
 test-list: test-list-crosvm-helloworld

 .PHONY: test-crosvm-helloworld
 test-crosvm-helloworld: $(CROSVM_HELLOWORLD_TESTS)

 .PHONY: test-list-crosvm-helloworld
 test-list-crosvm-helloworld:
 	@ for x in $(CROSVM_HELLOWORLD_TESTS); do echo $x; done

 .PHONY: test-crosvm-vhe-helloworld
 test-crosvm-vhe-helloworld: $(KERNEL_IMAGE) $(UBOOT_BIN)
 	@ $(call gen_crosvm_helloworld_test,$@,vhe)

 .PHONY: test-crosvm-nvhe-helloworld
 test-crosvm-nvhe-helloworld: $(KERNEL_IMAGE) $(UBOOT_BIN)
 	@ $(call gen_crosvm_helloworld_test,$@,nvhe)

 .PHONY: test-crosvm-pkvm-helloworld
 test-crosvm-pkvm-helloworld: $(KERNEL_IMAGE) $(UBOOT_BIN)
 	@ $(call gen_crosvm_helloworld_test,$@,pkvm)
	# Copyright 2020 The Android KVM Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	SELF := $(abspath $(lastword $(MAKEFILE_LIST)))

	CPUS ?= $(shell nproc)
	MAKEFLAGS += --jobs=$(CPUS)

	##
	## Folders
	##
	ROOT_DIR := $(CURDIR)

	OUT_DIR := $(ROOT_DIR)/out

	# DIST_DIR: target directory for artifacts and logs.
	# The CI build (pKVM Busytown) uses it to separate artifacts from other
	# temporary build files, which are placed in in OUT_DIR.
	# For other users, OUT_DIR and DIST_DIR are typically the same.
	DIST_DIR ?= $(OUT_DIR)

	# Copy specified file or directory only if OUT_DIR and DIST_DIR aren't the same.
	#
	# Parameters:
	# $(1): Source files or directories (to be copied recursively)
	# $(2): Destination directory (is created if it doesn't exist)
	ifneq ($(abspath $(OUT_DIR)), $(abspath $(DIST_DIR)))
	define copy_to_dist
	mkdir -p $(2) && cp -r $(1) $(2)
	endef
	endif

	LINUX_OUT := $(OUT_DIR)/linux
	LINUX_DIST := $(DIST_DIR)/linux
	SCRIPTS_DIR := $(ROOT_DIR)/build

	##
	## Common options
	##
	VERBOSE ?= 0
	CUSTOM_KERNEL_IMAGE = 0
	ARCH = arm64

	# KERNEL_IMAGE can be set to use own custom kernel image.
	ifdef KERNEL_IMAGE
	CUSTOM_KERNEL_IMAGE = 1
	else
	KERNEL_IMAGE := $(LINUX_OUT)/arch/arm64/boot/Image
	endif

	##
	## Toolchain options and variables
	##

	# Specifies which toolchain of the prebuilt toolchains to use.
	# Can be either clang, gcc-4.9 (deprecated), gcc-5.1, or gcc-9.2.
	TOOLCHAIN ?= clang

	# Find the toolchain path for the current OS.
	UNNAME_S := $(shell uname -s \| tr '[:upper:]' '[:lower:]')
	TOOLCHAIN_CLANG := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/clang/clang-r412851/
	TOOLCHAIN_GCC_49 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-4.9/
	TOOLCHAIN_GCC_51 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-5.1/
	TOOLCHAIN_GCC_92 := $(ROOT_DIR)/toolchains/$(UNNAME_S)-x86/gcc/aarch64-linux-gnu-9.2/

	# Set the toolchain build binary paths and prefixes.
	TARGET = aarch64-linux-gnu
	TARGET_GCC_92 = aarch64-none-linux-gnu

	ifeq ($(TOOLCHAIN), gcc-4.9)
	TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_49)/bin
	$(warning "The gcc-4.9 toolchain is deprecated and will be removed soon.")
	$(warning "The gcc-5.1 toolchain is the oldest version Linux needs to support.")
	else ifeq ($(TOOLCHAIN), gcc-5.1)
	TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_51)/bin
	else ifeq ($(TOOLCHAIN), gcc-9.2)
	TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_92)/bin
	TARGET = $(TARGET_GCC_92)
	else ifeq ($(TOOLCHAIN), clang)
	TOOLCHAIN_BIN := $(TOOLCHAIN_CLANG)/bin
	CC := $(TOOLCHAIN_BIN)/clang
	LD := $(TOOLCHAIN_BIN)/ld.lld
	OBJCOPY := $(TOOLCHAIN_BIN)/llvm-objcopy
	OBJDUMP := $(TOOLCHAIN_BIN)/llvm-objdump
	# The GCC toolchain is needed when building kvm-unit-tests as well as Linux
	# with Clang, because of lack of support for the LLVM integrated assembler.
	GCC_TOOLCHAIN_DIR = $(TOOLCHAIN_GCC_49)
	GCC_TOOLCHAIN_BIN = $(GCC_TOOLCHAIN_DIR)/bin/
	LLVM = LLVM=1 LLVM_IAS=1
	# Some targets require additional flags to build with Clang.
	COMMON_CFLAGS := -Qunused-arguments --target=$(TARGET) -fno-integrated-as -Wno-asm-operand-widths -fpic --gcc-toolchain=$(GCC_TOOLCHAIN_DIR) --sysroot=$(GCC_TOOLCHAIN_DIR)/$(TARGET)/libc --prefix=$(GCC_TOOLCHAIN_BIN)/$(TARGET)-
	else
	$(error Unrecognized toolchain: TOOLCHAIN=$(TOOLCHAIN))
	endif

	# common to all GCC versions
	ifneq ($(TOOLCHAIN), clang)
	CC := $(TOOLCHAIN_BIN)/$(TARGET)-gcc
	LD := $(TOOLCHAIN_BIN)/$(TARGET)-ld
	OBJCOPY := $(TOOLCHAIN_BIN)/$(TARGET)-objcopy
	OBJDUMP := $(TOOLCHAIN_BIN)/$(TARGET)-objdump
	GCC_TOOLCHAIN_BIN := $(TOOLCHAIN_BIN)
	endif

	##
	## Common targets
	##

	.DEFAULT_GOAL := default

	.PHONY: default
	default: kvm-unit-tests linux

	.PHONY: clean
	clean: buildroot_clean kvm-unit-tests_clean linux_clean u-boot_clean

	.PHONY: distclean
	distclean:
	- rm -rf $(OUT_DIR) $(DIST_DIR)

	.PHONY: all
	all: buildroot kvm-unit-tests linux u-boot

	.PHONY: help
	help:
	@echo 'Generic Targets:'
	@echo ' default - builds all generic targets except for buildroot'
	@echo ' all - builds all generic targets'
	@echo ' buildroot - the buildroot rootfs image as well as the host qemu'
	@echo ' kvm-unit-tests - the kvm-unit-tests'
	@echo ' linux - the linux kernel'
	@echo ' u-boot - the u-boot'
	@echo ''
	@echo 'Clean Targets:'
	@echo ' clean - cleans all, but keeps the prebuilts'
	@echo ' distclean - distclean for all targets'
	@echo ' buildroot_clean - the buildroot rootfs image as well as the host qemu'
	@echo ' kvm-unit-tests_clean - the kvm-unit-tests'
	@echo ' linux_clean - the linux kernel'
	@echo ' u-boot_clean - the u-boot'
	@echo ''
	@echo 'Test Targets:'
	@echo ' test - runs all tests in qemu-aarch64 simulated environment'
	@echo ' test-kut - runs all kvm-unit-tests in qemu-aarch64 simulated environment'
	@echo ' test-list - lists all test targets'
	@echo ''
	@echo 'Misc Targets:'
	@echo ' update-prebuilts - generates/update the prebuilt rootfs image (aarch64) and the qemu host (aarch64)'


	##
	## Buildroot
	##

	BR_DEFCONFIG ?= qemu_aarch64_virt_kvmunittests_defconfig
	BR_VERBOSE ?= $(VERBOSE)

	BR_SRC := $(ROOT_DIR)/buildroot
	BR_OUT := $(OUT_DIR)/buildroot
	BR_MAKE := $(MAKE) -C $(BR_SRC) V=$(BR_VERBOSE) O=$(BR_OUT)

	.PHONY: buildroot
	buildroot:
	+ $(BR_MAKE) $(BR_DEFCONFIG)
	+ $(BR_MAKE) all

	.PHONY: buildroot_clean
	buildroot_clean:
	+ $(BR_MAKE) clean


	##
	## kvm-unit-tests
	##
	KUT_SRC := $(ROOT_DIR)/kvm-unit-tests
	KUT_OUT := $(OUT_DIR)/kvm-unit-tests
	KUT_DIST := $(DIST_DIR)/kvm-unit-tests
	KUT_STAMP := $(KUT_OUT)/kvm-unit-tests.stamp


	.PHONY: kvm-unit-tests
	kvm-unit-tests $(KUT_STAMP):
	mkdir -p $(KUT_OUT)
	cd $(KUT_OUT) && \
	$(KUT_SRC)/configure \
	--erratatxt=$(KUT_SRC)/errata.txt \
	--prefix=$(KUT_OUT) --arch=$(ARCH) \
	--cc=$(CC) --ld=$(LD) \
	--objcopy=$(OBJCOPY) --objdump=$(OBJDUMP)
	COMMON_CFLAGS="$(COMMON_CFLAGS)" \
	$(MAKE) -C $(KUT_OUT) standalone
	$(call copy_to_dist, $(KUT_OUT)/tests, $(KUT_DIST))
	touch $(KUT_STAMP)

	.PHONY: kvm-unit-tests_clean
	kvm-unit-tests_clean:
	- rm -f $(KUT_STAMP)
	- $(MAKE) -C $(KUT_OUT) clean


	##
	## Linux kernel
	##
	LINUX_DEFCONFIG ?= defconfig
	LINUX_VERBOSE ?= $(VERBOSE)
	LINUX_SRC ?= $(ROOT_DIR)/linux
	LINUX_VMLINUX = $(LINUX_OUT)/vmlinux
	LINUX_DIST_FILES = $(LINUX_CONFIG) $(LINUX_VMLINUX) $(KERNEL_IMAGE)

	LINUX_MAKE := \
	PATH=$(TOOLCHAIN_BIN):$(PATH) \
	ARCH=$(ARCH) \
	CROSS_COMPILE="$(TARGET)-" \
	$(MAKE) \
	$(LLVM) \
	GCC_TOOLCHAIN_DIR="$(GCC_TOOLCHAIN_BIN)" \
	-C $(LINUX_SRC) \
	V=$(LINUX_VERBOSE) \
	O=$(LINUX_OUT)

	LINUX_CONFIG := $(LINUX_OUT)/.config

	.PHONY: linux_defconfig
	linux_defconfig $(LINUX_CONFIG):
	+ $(LINUX_MAKE) $(LINUX_DEFCONFIG)

	.PHONY: linux_menuconfig
	linux_menuconfig: $(LINUX_CONFIG)
	+ $(LINUX_MAKE) menuconfig

	.PHONY: linux
	linux: $(LINUX_CONFIG)
	+ $(LINUX_MAKE)
	$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))

	.PHONY: linux_image
	linux_image: $(LINUX_CONFIG)
	+ $(LINUX_MAKE) Image.gz
	$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))

	# If using own kernel image (KERNEL_IMAGE is set in the environment), then skip.
	ifeq ($(CUSTOM_KERNEL_IMAGE), 0)
	$(KERNEL_IMAGE): linux_image
	endif

	.PHONY: linux_clean
	linux_clean:
	+ $(LINUX_MAKE) mrproper


	##
	## u-boot
	##
	UBOOT_DEFCONFIG ?= pvmfw_arm64_defconfig
	UBOOT_VERBOSE ?= $(VERBOSE)
	UBOOT_SRC ?= $(ROOT_DIR)/u-boot

	# U-boot doesn't support gcc < 6.0. To avoid compilation failure,
	# we upgrade the active toolchain version for the u-boot build.
	ifeq ($(TOOLCHAIN), gcc-5.1)
	$(warning "gcc below 6.0 is not supported by u-boot, switching to 9.2")
	UBOOT_TOOLCHAIN_BIN := $(TOOLCHAIN_GCC_92)/bin
	UBOOT_TARGET = $(TARGET_GCC_92)
	UBOOT_CC := $(UBOOT_TOOLCHAIN_BIN)/$(UBOOT_TARGET)-gcc
	UBOOT_LD := $(UBOOT_TOOLCHAIN_BIN)/$(UBOOT_TARGET)-ld
	UBOOT_GCC_TOOLCHAIN_BIN := $(UBOOT_TOOLCHAIN_BIN)
	else
	UBOOT_TOOLCHAIN_BIN := $(TOOLCHAIN_BIN)
	UBOOT_TARGET = $(TARGET)
	UBOOT_CC := $(CC)
	UBOOT_LD := $(LD)
	UBOOT_GCC_TOOLCHAIN_BIN := $(GCC_TOOLCHAIN_BIN)
	endif

	UBOOT_OUT := $(OUT_DIR)/u-boot
	UBOOT_MAKE := \
	PATH=$(UBOOT_TOOLCHAIN_BIN):$(UBOOT_GCC_TOOLCHAIN_BIN):$(PATH) \
	ARCH=$(ARCH) \
	CROSS_COMPILE="$(UBOOT_TARGET)-" \
	$(MAKE) \
	CC=$(UBOOT_CC) \
	LD=$(UBOOT_LD) \
	-C $(UBOOT_SRC) \
	V=$(UBOOT_VERBOSE) \
	O=$(UBOOT_OUT)

	UBOOT_CONFIG := $(UBOOT_OUT)/.config
	UBOOT_BIN := $(UBOOT_OUT)/u-boot.bin

	.PHONY: u-boot_defconfig
	u-boot_defconfig $(UBOOT_CONFIG):
	+ $(UBOOT_MAKE) $(UBOOT_DEFCONFIG)

	.PHONY: u-boot_menuconfig
	u-boot_menuconfig: $(UBOOT_CONFIG)
	+ $(UBOOT_MAKE) menuconfig

	.PHONY: u-boot
	u-boot $(UBOOT_BIN): $(UBOOT_CONFIG)
	+ $(UBOOT_MAKE)

	.PHONY: u-boot_clean
	u-boot_clean:
	+ $(UBOOT_MAKE) mrproper


	##
	## Generating/Updating prebuilts
	##

	PREBUILTS_CROSVM_EXT4 := prebuilts/linux-aarch64/images/crosvm.ext4
	PREBUILTS_ROOTFS_EXT4 := prebuilts/linux-aarch64/images/rootfs.ext4
	PREBUILTS_QEMU_ROM := prebuilts/linux-aarch64/images/efi-virtio.rom
	PREBUILTS_QEMU_BIN := prebuilts/linux-x86/qemu

	.PHONY:update-prebuilts
	update-prebuilts: \
	$(PREBUILTS_ROOTFS_EXT4) \
	$(PREBUILTS_CROSVM_EXT4) \
	$(PREBUILTS_QEMU_ROM) \
	$(PREBUILTS_QEMU_BIN)

	$(PREBUILTS_ROOTFS_EXT4): buildroot
	mkdir -p $(shell dirname $@)
	cp $(BR_OUT)/images/rootfs.ext4 $@

	$(PREBUILTS_QEMU_ROM): buildroot
	mkdir -p $(shell dirname $@)
	cp $(BR_OUT)/per-package/host-qemu/host/share/qemu/efi-virtio.rom $@

	CF_DOWNLOAD_AOSP_SH := $(ROOT_DIR)/build/cuttlefish/download-aosp.sh
	CROSVM_IMG_SIZE_MB := 1024

	$(PREBUILTS_CROSVM_EXT4): TMP_DIR := $(shell mktemp -d)
	$(PREBUILTS_CROSVM_EXT4): $(CF_DOWNLOAD_AOSP_SH)
	mkdir -p $(shell dirname $@)
	cd $(TMP_DIR) && $(CF_DOWNLOAD_AOSP_SH) -a arm64 -C
	dd if=/dev/zero of=$@.tmp bs=1M count=$(CROSVM_IMG_SIZE_MB)
	mkfs.ext4 -d $(TMP_DIR) $@.tmp
	rm -rf $(TMP_DIR)
	mv $@.tmp $@

	# The qemu binary
	.PHONY:$(PREBUILTS_QEMU_BIN)
	$(PREBUILTS_QEMU_BIN):
	- rm -rf $@
	${SCRIPTS_DIR}/aarch64/build_qemu.sh $(OUT_DIR)/qemu ${TOOLCHAIN_BIN}
	mkdir -p $@/bin
	cp $(OUT_DIR)/qemu/qemu-system-aarch64 $@/bin

	##
	## Run tests
	##

	# Root test targets. Dynamically generated per-test targets
	# will add themselves as dependencies of these.
	.PHONY: test test-list
	test:
	test-list:

	#
	# kvm-unit-tests
	#

	KUT_RUN_TEST := $(SCRIPTS_DIR)/kvm-unit-tests/run_test.sh
	KUT_GEN_MAKEFILE := $(SCRIPTS_DIR)/kvm-unit-tests/gen_makefile.sh
	KUT_MAKEFILE := $(OUT_DIR)/Makefile.kvm-unit-tests
	KUT_TEST_DIR := $(KUT_OUT)/tests
	KUT_LOG_DIR := $(DIST_DIR)/logs/kvm-unit-tests/

	# Exclude tests that require user interaction or are known to fail.
	KUT_EXCLUDE := (.+migrat.+)\|(gicv2-.+)\|(pmu-event-introspection)\|(micro-bench)

	# Generate a Makefile with targets per test and configuration.
	# Pass in variable/target names from this Makefile that the
	# generated Makefile will refer to.
	$(KUT_MAKEFILE): $(SELF) $(KUT_GEN_MAKEFILE) $(KUT_STAMP)
	@ mkdir -p $(shell dirname $@)
	@ $(KUT_GEN_MAKEFILE) \
	$(KUT_TEST_DIR) "$(KUT_EXCLUDE)" \
	KERNEL_IMAGE KUT_LOG_DIR test test-list \
	> $@.tmp
	@ mv $@.tmp $@
	include $(KUT_MAKEFILE)

	#
	# Basic boot tests in the various arm64 KVM modes
	#

	BOOT_TEST_SCRIPT_DIR := $(SCRIPTS_DIR)/aarch64/
	BOOT_TEST_LOG_DIR := $(DIST_DIR)/logs/boot/
	BOOT_TESTS := test-boot-vhe test-boot-nvhe test-boot-pkvm

	define gen_boot_test
	$(KUT_RUN_TEST) -d $(1) -M $(2) \
	-k $(KERNEL_IMAGE) \
	-o $(BOOT_TEST_LOG_DIR)/$(1).log \
	$(BOOT_TEST_SCRIPT_DIR)/test_boot.sh -- \
	-a "boot_test_mode=$(2)"
	endef

	test: test-boot
	test-list: test-list-boot

	.PHONY: test-boot
	test-boot: $(BOOT_TESTS)

	.PHONY: test-list-boot
	test-list-boot:
	@ for x in $(BOOT_TESTS); do echo $x; done

	.PHONY: test-boot-vhe
	test-boot-vhe: $(KERNEL_IMAGE)
	@ $(call gen_boot_test,$@,vhe)

	.PHONY: test-boot-nvhe
	test-boot-nvhe: $(KERNEL_IMAGE)
	@ $(call gen_boot_test,$@,nvhe)

	.PHONY: test-boot-pkvm
	test-boot-pkvm: $(KERNEL_IMAGE)
	@ $(call gen_boot_test,$@,pkvm)

	#
	# CrosVM Hello World test
	#

	CROSVM_HELLOWORLD_SCRIPT_DIR := $(SCRIPTS_DIR)/crosvm/helloworld/
	CROSVM_HELLOWORLD_LOG_DIR := $(DIST_DIR)/logs/crosvm/helloworld/
	CROSVM_HELLOWORLD_TESTS := test-crosvm-vhe-helloworld \
	test-crosvm-nvhe-helloworld \
	test-crosvm-pkvm-helloworld

	define gen_crosvm_helloworld_test
	$(KUT_RUN_TEST) -d $(1) -M $(2) -t 300s \
	-k $(KERNEL_IMAGE) \
	-F $(UBOOT_BIN) \
	-o $(CROSVM_HELLOWORLD_LOG_DIR)/$(1).log \
	$(CROSVM_HELLOWORLD_SCRIPT_DIR)/host.sh -- \
	-R $(CROSVM_HELLOWORLD_SCRIPT_DIR)/guest.sh \
	-R $(KERNEL_IMAGE) \
	-R $(PREBUILTS_ROOTFS_EXT4) \
	-R $(PREBUILTS_CROSVM_EXT4)
	endef

	test: test-crosvm-helloworld
	test-list: test-list-crosvm-helloworld

	.PHONY: test-crosvm-helloworld
	test-crosvm-helloworld: $(CROSVM_HELLOWORLD_TESTS)

	.PHONY: test-list-crosvm-helloworld
	test-list-crosvm-helloworld:
	@ for x in $(CROSVM_HELLOWORLD_TESTS); do echo $x; done

	.PHONY: test-crosvm-vhe-helloworld
	test-crosvm-vhe-helloworld: $(KERNEL_IMAGE) $(UBOOT_BIN)
	@ $(call gen_crosvm_helloworld_test,$@,vhe)

	.PHONY: test-crosvm-nvhe-helloworld
	test-crosvm-nvhe-helloworld: $(KERNEL_IMAGE) $(UBOOT_BIN)
	@ $(call gen_crosvm_helloworld_test,$@,nvhe)

	.PHONY: test-crosvm-pkvm-helloworld
	test-crosvm-pkvm-helloworld: $(KERNEL_IMAGE) $(UBOOT_BIN)
	@ $(call gen_crosvm_helloworld_test,$@,pkvm)