Makefile.root - build - Git at Google

 # Copyright 2020 The Android KVM Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 SELF := $(abspath $(lastword $(MAKEFILE_LIST)))

 CPUS ?= $(shell nproc)
 MAKEFLAGS += --jobs=$(CPUS)
 SHELL ?= $(shell which bash)

 ##
 ## Folders
 ##
 ROOT_DIR := $(CURDIR)

 OUT_DIR := $(ROOT_DIR)/out

 # DIST_DIR: target directory for artifacts and logs.
 # The CI build (pKVM Busytown) uses it to separate artifacts from other
 # temporary build files, which are placed in in OUT_DIR.
 # For other users, OUT_DIR and DIST_DIR are typically the same.
 DIST_DIR ?= $(OUT_DIR)

 # Copy specified file or directory only if OUT_DIR and DIST_DIR aren't the same.
 #
 # Parameters:
 # $(1): Source files or directories (to be copied recursively)
 # $(2): Destination directory (is created if it doesn't exist)
 ifneq ($(abspath $(OUT_DIR)), $(abspath $(DIST_DIR)))
 define copy_to_dist
 	mkdir -p $(2) && cp -r $(1) $(2)
 endef
 endif

 LINUX_OUT := $(OUT_DIR)/linux
 LINUX_DIST := $(DIST_DIR)/linux
 SCRIPTS_DIR := $(ROOT_DIR)/build

 include $(SCRIPTS_DIR)/toolchain.mk

 ##
 ## Common options
 ##
 VERBOSE ?= 0
 ARCH = arm64

 LINUX_OUT_IMAGE := $(LINUX_OUT)/arch/arm64/boot/Image
 LINUX_OUT_MODULES := $(LINUX_OUT)/modules_install
 LINUX_OUT_MODULES_DEP := $(LINUX_OUT)/modules_install.stamp

 # KERNEL_IMAGE/MODULES can be set to use own custom kernel image.
 KERNEL_IMAGE ?= $(LINUX_OUT_IMAGE)
 KERNEL_MODULES ?= $(LINUX_OUT_MODULES)
 KERNEL_MODULES_DEP ?= $(LINUX_OUT_MODULES_DEP)

 ##
 ## Toolchain options and variables
 ##

 # Specifies which toolchain of the prebuilt toolchains to use.
 # Can be either clang, gcc-5.1, or gcc-9.2.
 # GCC will always use the latest version for building everything else other
 # than the kernel.
 TOOLCHAIN ?= clang
 $(eval $(call define_toolchain,$(TOOLCHAIN)))

 ##
 ## Common targets
 ##

 .DEFAULT_GOAL := default

 .PHONY: default
 default: kvm-unit-tests linux

 .PHONY: clean
 clean: bl_clean bl33_clean buildroot_clean kvm-unit-tests_clean linux_clean kselftest_clean tfa_clean

 .PHONY: distclean
 distclean:
 	- rm -rf $(OUT_DIR) $(DIST_DIR)

 FORCE:

 .PHONY: all
 all: buildroot kvm-unit-tests linux

 .PHONY: help
 help:
 	@echo 'Generic Targets:'
 	@echo '   default              - builds all generic targets except for buildroot'
 	@echo '   all                  - builds all generic targets'
 	@echo '   bl                   - builds U-Boot and TF-A and copies them to the bl output directory'
 	@echo '   bl33                 - U-Boot as BL33'
 	@echo '   buildroot            - the buildroot rootfs image as well as the host qemu'
 	@echo '   kvm-unit-tests       - the kvm-unit-tests'
 	@echo '   linux                - the Linux kernel'
 	@echo '   kselftest            - the Linux kernel selftests'
 	@echo '   tfa                  - Arm Trusted Firmware-A'
 	@echo ''
 	@echo 'Clean Targets:'
 	@echo '   clean                - cleans all, but keeps the prebuilts'
 	@echo '   distclean            - distclean for all targets'
 	@echo '   bl_clean             - TF-A, U-Boot and the bl output directory'
 	@echo '   bl33_clean           - the U-Boot BL33'
 	@echo '   buildroot_clean      - the buildroot rootfs image as well as the host qemu'
 	@echo '   kvm-unit-tests_clean - the kvm-unit-tests'
 	@echo '   linux_clean          - the Linux kernel'
 	@echo '   kselftest_clean      - the Linux kernel selftests'
 	@echo '   tfa_clean            - Arm Trusted Firmware-A'
 	@echo ''
 	@echo 'Test Targets:'
 	@echo '   test                 - runs all tests in qemu-aarch64 simulated environment'
 	@echo '   test-kut             - runs all kvm-unit-tests in qemu-aarch64 simulated environment'
 	@echo '   test-ksft            - runs all kselftests in qemu-aarch64 simulated environment'
 	@echo '   test-list            - lists all test targets'
 	@echo ''
 	@echo 'Test Parameters:'
 	@echo '   CROSVM_TAR_GZ        - path to a custom cvd-host_package.tar.gz'
 	@echo '                          (default: uses a pre-built version)'
 	@echo '   VM_IMAGE             - path to an image to be used by the VMM to boot the VM'
 	@echo '                          (default: uses the host image)'
 	@echo ''
 	@echo 'Test Flags:'
 	@echo '   VERBOSE=[0|1]        - prints the verbose output of the scripts running the test'
 	@echo '                          (default: 0)'
 	@echo '   QUIET=[0|1]          - redirects the output to a file instead of stdout'
 	@echo '                          (default: 1)'
 	@echo '   GDB=[0|1]            - makes the started test wait for a GDB connection'
 	@echo '                          (default: 0)'
 	@echo ''
 	@echo 'Misc Targets:'
 	@echo '   update-prebuilts     - generates/update the prebuilt rootfs image (aarch64) and the qemu host (aarch64)'


 ##
 ## Buildroot
 ##

 BR_DEFCONFIG ?= qemu_aarch64_virt_kvmunittests_defconfig
 BR_VERBOSE ?= $(VERBOSE)

 BR_SRC := $(ROOT_DIR)/buildroot
 BR_OUT := $(OUT_DIR)/buildroot
 BR_MAKE := $(MAKE) -C $(BR_SRC) V=$(BR_VERBOSE) O=$(BR_OUT)

 .PHONY: buildroot
 buildroot:
 	+ $(BR_MAKE) $(BR_DEFCONFIG)
 	+ $(BR_MAKE) all

 .PHONY: buildroot_clean
 buildroot_clean:
 	+ $(BR_MAKE) clean

 # Bootloader stages come from TF-A, U-Boot, or just an empty file.
 BL_DIR := $(OUT_DIR)/bl
 BL1_BIN := $(BL_DIR)/bl1.bin
 BL2_BIN := $(BL_DIR)/bl2.bin
 BL31_BIN := $(BL_DIR)/bl31.bin
 BL32_BIN := $(BL_DIR)/bl32.bin
 BL33_BIN := $(BL_DIR)/bl33.bin
 BL_BINS := $(BL1_BIN) $(BL2_BIN) $(BL31_BIN) $(BL32_BIN) $(BL33_BIN)

 ##
 ## U-Boot
 ##
 UBOOT_SRC := $(ROOT_DIR)/u-boot
 UBOOT_OUT := $(OUT_DIR)/u-boot
 UBOOT_DEFCONFIG ?= qemu_arm64_defconfig
 UBOOT_CONFIG := $(UBOOT_OUT)/.config
 UBOOT_TOOLCHAIN ?= gcc-9.2
 UBOOT_VERBOSE ?= $(VERBOSE)
 $(eval $(call define_toolchain,$(UBOOT_TOOLCHAIN),UBOOT_))
 UBOOT_MAKE := \
 	PATH=$(UBOOT_TOOLCHAIN_BIN):$(PATH) \
 	CROSS_COMPILE=$(UBOOT_TARGET)- \
 	$(MAKE) \
 	V=$(UBOOT_VERBOSE) \
 	-C $(UBOOT_SRC) \
 	O=$(UBOOT_OUT)

 .PHONY: bl33
 bl33: $(UBOOT_OUT)/u-boot.bin

 $(UBOOT_CONFIG): $(UBOOT_SRC)/configs/$(UBOOT_DEFCONFIG)
 	+ $(UBOOT_MAKE) $(UBOOT_DEFCONFIG)
 	$(LINUX_SRC)/scripts/config --file $(UBOOT_CONFIG) --set-val CONFIG_BOOTDELAY 0
 	$(LINUX_SRC)/scripts/config --file $(UBOOT_CONFIG) --set-str CONFIG_BOOTCOMMAND \
 		"qfw load 0x40000000 0x50000000 ; booti 0x40000000 - $$\{fdtcontroladdr\}"

 $(UBOOT_OUT)/u-boot.bin: $(UBOOT_CONFIG) FORCE
 	+ $(UBOOT_MAKE)

 .PHONY: bl33_clean
 bl33_clean:
 	+ $(UBOOT_MAKE) distclean

 ##
 ## TF-A
 ##
 TFA_SRC := $(ROOT_DIR)/trusted-firmware-a
 TFA_OUT := $(OUT_DIR)/trusted-firmware-a
 TFA_TOOLCHAIN ?= gcc-9.2
 TFA_VERBOSE ?= $(VERBOSE)
 $(eval $(call define_toolchain,$(TFA_TOOLCHAIN),TFA_))
 TFA_MAKE := \
 	PATH=$(TFA_TOOLCHAIN_BIN):$(PATH) \
 	$(MAKE) \
 	CROSS_COMPILE="$(TFA_TARGET)-" \
 	PLAT=qemu \
 	ENABLE_SVE_FOR_NS=1 \
 	QEMU_USE_GIC_DRIVER=QEMU_GICV3 \
 	ARM_LINUX_KERNEL_AS_BL33=1 \
 	SPD=trusty \
 	TRUSTY_SPD_WITH_GENERIC_SERVICES=1 \
 	V=$(TFA_VERBOSE) \
 	-C $(TFA_SRC) \
 	BUILD_BASE=$(TFA_OUT)

 .PHONY: tfa
 tfa: $(TFA_OUT)/qemu/release/bl1.bin $(TFA_OUT)/qemu/release/bl2.bin $(TFA_OUT)/qemu/release/bl31.bin

 $(TFA_OUT)/qemu/release/bl1.bin $(TFA_OUT)/qemu/release/bl2.bin $(TFA_OUT)/qemu/release/bl31.bin &: FORCE
 	+ $(TFA_MAKE) all

 .PHONY: tfa_clean
 tfa_clean:
 	rm -rf $(TFA_OUT)

 ##
 ## Copy bootloader stages to BL_DIR
 ##

 $(BL_DIR):
 	mkdir -p $@

 $(BL1_BIN) $(BL2_BIN) $(BL31_BIN): $(BL_DIR)/bl%.bin: $(TFA_OUT)/qemu/release/bl%.bin | $(BL_DIR)
 	cp $< $@

 # An empty BL32 is enough for us, it just needs to exist.
 $(BL32_BIN): | $(BL_DIR)
 	touch $@

 $(BL33_BIN): $(UBOOT_OUT)/u-boot.bin | $(BL_DIR)
 	cp $< $@

 .PHONY: bl
 bl: $(BL_BINS)

 .PHONY: bl_clean
 bl_clean: tfa_clean bl33_clean
 	rm -rf $(BL_DIR)

 ##
 ## kvm-unit-tests
 ##
 KUT_SRC := $(ROOT_DIR)/kvm-unit-tests
 KUT_OUT := $(OUT_DIR)/kvm-unit-tests
 KUT_DIST := $(DIST_DIR)/kvm-unit-tests
 KUT_STAMP := $(KUT_OUT)/kvm-unit-tests.stamp

 ifneq ($(TOOLCHAIN_IS_LEGACY),1)
 KUT_TOOLCHAIN ?= $(TOOLCHAIN)
 else
 KUT_TOOLCHAIN ?= gcc-9.2
 endif
 $(eval $(call define_toolchain,$(KUT_TOOLCHAIN),KUT_))

 ifeq ($(KUT_TOOLCHAIN_IS_LLVM),1)
 KUT_CFLAGS += --target=$(KUT_TARGET) -fpic
 endif

 .PHONY: kvm-unit-tests
 kvm-unit-tests $(KUT_STAMP):
 	mkdir -p $(KUT_OUT)
 	cd $(KUT_OUT) && \
 		$(KUT_SRC)/configure \
 			--erratatxt=$(KUT_SRC)/errata.txt \
 			--prefix=$(KUT_OUT) --arch=$(ARCH) \
 			--cc=$(KUT_CC) --ld=$(KUT_LD) \
 			--objcopy=$(KUT_OBJCOPY) --objdump=$(KUT_OBJDUMP)
 	COMMON_CFLAGS="$(KUT_CFLAGS)" \
 		$(MAKE) -C $(KUT_OUT) standalone
 	$(call copy_to_dist, $(KUT_OUT)/tests, $(KUT_DIST))
 	touch $(KUT_STAMP)

 .PHONY: kvm-unit-tests_clean
 kvm-unit-tests_clean:
 	- rm -f $(KUT_STAMP)
 	- $(MAKE) -C $(KUT_OUT) clean


 ##
 ## Linux kernel
 ##
 LINUX_DEFCONFIG ?= defconfig
 LINUX_VERBOSE ?= $(VERBOSE)
 LINUX_SRC ?= $(ROOT_DIR)/linux
 LINUX_VMLINUX = $(LINUX_OUT)/vmlinux
 LINUX_CONFIG := $(LINUX_OUT)/.config
 LINUX_DIST_FILES = $(LINUX_CONFIG) $(LINUX_VMLINUX) $(KERNEL_IMAGE)
 LINUX_LSMOD_TXT := $(LINUX_OUT)/lsmod.txt

 LINUX_TOOLCHAIN ?= $(TOOLCHAIN)
 $(eval $(call define_toolchain,$(LINUX_TOOLCHAIN),LINUX_))

 ifeq ($(LINUX_TOOLCHAIN_IS_LLVM),1)
 LINUX_EXTRA += LLVM=1 LLVM_IAS=1
 endif

 LINUX_MAKE := \
 	PATH=$(LINUX_TOOLCHAIN_BIN):$(PATH) \
 	ARCH=$(ARCH) \
 	CROSS_COMPILE="$(LINUX_TARGET)-" \
 	$(MAKE) \
 	$(LINUX_EXTRA) \
 	CC=$(LINUX_CC) \
 	LD=$(LINUX_LD) \
 	OBJCOPY=$(LINUX_OBJCOPY) \
 	OBJDUMP=$(LINUX_OBJDUMP) \
 	-C $(LINUX_SRC) \
 	V=$(LINUX_VERBOSE) \
 	O=$(LINUX_OUT)

 LINUX_MODULES := \
 	TEST_PRINTF

 .PHONY: linux_defconfig
 linux_defconfig $(LINUX_CONFIG):
 	+ $(LINUX_MAKE) $(LINUX_DEFCONFIG)
 	# Disable all modules, then selectively enable those in LINUX_MODULES.
 	@ echo > $(LINUX_LSMOD_TXT)
 	+ $(LINUX_MAKE) LSMOD=$(LINUX_LSMOD_TXT) localmodconfig
 	$(LINUX_SRC)/scripts/config --file $(LINUX_CONFIG) $(addprefix -m CONFIG_,$(LINUX_MODULES))

 .PHONY: linux_menuconfig
 linux_menuconfig: $(LINUX_CONFIG)
 	+ $(LINUX_MAKE) menuconfig

 .PHONY: linux
 linux: $(LINUX_CONFIG)
 	+ $(LINUX_MAKE)
 	$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))

 .PHONY: linux_image
 linux_image $(LINUX_OUT_IMAGE): $(LINUX_CONFIG)
 	+ $(LINUX_MAKE) Image.gz
 	$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))

 .PHONY: linux_modules
 linux_modules $(LINUX_OUT_MODULES_DEP): $(LINUX_OUT_IMAGE)
 	@ rm -rf $(LINUX_OUT_MODULES)
 	+ $(LINUX_MAKE) modules
 	+ $(LINUX_MAKE) INSTALL_MOD_PATH=$(LINUX_OUT_MODULES) modules_install
 	@ find $(LINUX_OUT_MODULES) -type f | sort | \
 		xargs sha1sum > $(LINUX_OUT_MODULES_DEP).tmp
 	@ cmp $(LINUX_OUT_MODULES_DEP).tmp $(LINUX_OUT_MODULES_DEP) || \
 		mv $(LINUX_OUT_MODULES_DEP).tmp $(LINUX_OUT_MODULES_DEP)

 .PHONY: linux_clean
 linux_clean:
 	+ $(LINUX_MAKE) mrproper


 ##
 ## Linux kselftest
 ##
 KSFT_VERBOSE ?= $(VERBOSE)
 KSFT_SRC ?= $(LINUX_SRC)
 KSFT_SRC_STAMP ?= $(KSFT_OUT)/ksefltest_src.stamp
 KSFT_OUT ?= $(OUT_DIR)/kselftest
 KSFT_OUT_CONFIG ?= $(KSFT_OUT)/.config
 KSFT_INSTALL_DIR ?= $(KSFT_OUT)/tools/testing/selftests/kselftest_install
 KSFT_STAMP ?= $(KSFT_INSTALL_DIR)/kselftest-list.txt

 KSFT_TESTS := \
 	lib:printf.sh \
 	kvm:kvm_create_max_vcpus \
 	kvm:set_memory_region_test \
 	kvm:steal_time

 ifneq ($(TOOLCHAIN_IS_LLVM),1)
 KSFT_TOOLCHAIN ?= $(TOOLCHAIN)
 else
 KSFT_TOOLCHAIN ?= gcc-9.2
 endif
 $(eval $(call define_toolchain,$(KSFT_TOOLCHAIN),KSFT_))

 KSFT_MAKE := \
 	PATH=$(KSFT_TOOLCHAIN_BIN):$(PATH) \
 	ARCH=$(ARCH) \
 	CROSS_COMPILE="$(KSFT_TARGET)-" \
 	$(MAKE) \
 	-C $(KSFT_OUT) \
 	V=$(KSFT_VERBOSE)

 KSFT_MAKE_TARGETS := $(sort $(foreach t,$(KSFT_TESTS),$(firstword $(subst :, ,$(t)))))

 # Copy the source to another folder because 'kselftest-install' clobbers it.
 # Stamp is touched if any files (not directories) were synced.
 $(KSFT_SRC_STAMP): FORCE
 	@ mkdir -p $(KSFT_OUT) $(@D)
 	@ LIST="$$(rsync -a --info=name $(KSFT_SRC)/ $(KSFT_OUT) | grep -vE '/$$')"; \
 	  [ -f $@ -a -z "$$LIST" ] || touch $@

 .PHONY: kselftest
 kselftest $(KSFT_STAMP): $(KSFT_SRC_STAMP) $(LINUX_CONFIG)
 	@ cp $(LINUX_CONFIG) $(KSFT_OUT_CONFIG)
 	+ $(KSFT_MAKE) olddefconfig prepare
 	+ $(KSFT_MAKE) TARGETS="$(KSFT_MAKE_TARGETS)" kselftest-install

 .PHONY: kselftest_clean
 kselftest_clean:
 	@ rm -rf $(KSFT_OUT)


 ##
 ## Generating/Updating prebuilts
 ##

 PREBUILTS_CROSVM_EXT4 := $(ROOT_DIR)/prebuilts/linux-aarch64/images/crosvm.ext4
 PREBUILTS_ROOTFS_EXT4 := $(ROOT_DIR)/prebuilts/linux-aarch64/images/rootfs.ext4
 PREBUILTS_QEMU_ROM := $(ROOT_DIR)/prebuilts/linux-aarch64/images/efi-virtio.rom
 PREBUILTS_QEMU_BIN := $(ROOT_DIR)/prebuilts/linux-x86/qemu
 PREBUILTS_MAKE_BIN := $(ROOT_DIR)/prebuilts/linux-x86/make

 CUSTOM_CROSVM_EXT4 := $(OUT_DIR)/images/crosvm.ext4

 ifeq ($(CROSVM_TAR_GZ),)
 CROSVM_EXT4 := $(PREBUILTS_CROSVM_EXT4)
 else
 CROSVM_EXT4 := $(CUSTOM_CROSVM_EXT4)
 endif

 .PHONY:update-prebuilts
 update-prebuilts: \
 	$(PREBUILTS_ROOTFS_EXT4) \
 	update-prebuilts-crosvm \
 	$(PREBUILTS_QEMU_ROM) \
 	$(PREBUILTS_QEMU_BIN) \
 	$(PREBUILTS_MAKE_BIN)

 $(PREBUILTS_ROOTFS_EXT4): buildroot
 	mkdir -p $(shell dirname $@)
 	cp $(BR_OUT)/images/rootfs.ext4 $@

 $(PREBUILTS_QEMU_ROM): buildroot
 	mkdir -p $(shell dirname $@)
 	cp $(BR_OUT)/per-package/host-qemu/host/share/qemu/efi-virtio.rom $@

 CF_DOWNLOAD_AOSP_SH := $(ROOT_DIR)/build/cuttlefish/download-aosp.sh

 # A PHONY target is necessary for allowing the user to update the archive
 # because, as we are downloading it from a remote server, make dependencies are
 # unable to express the local-->remote dependency:
 .PHONY: update-prebuilts-crosvm
 update-prebuilts-crosvm $(PREBUILTS_CROSVM_TAR_GZ):
 	@rm -f $(PREBUILTS_CROSVM_TAR_GZ)  # else, wget will create "$@.1"
 	cd $(dir $(PREBUILTS_CROSVM_TAR_GZ)) && $(CF_DOWNLOAD_AOSP_SH) -XCa arm64
 	@touch -c $(PREBUILTS_CROSVM_TAR_GZ)  # wget doesn't set timestamp to now

 CROSVM_IMG_SIZE_MB := 1024

 $(CUSTOM_CROSVM_EXT4): TMP_DIR := $(shell mktemp -d)
 $(CUSTOM_CROSVM_EXT4): $(CROSVM_TAR_GZ)
 	mkdir -p $(@D)
 	cd $(TMP_DIR) && tar xzvf $(abspath $<)
 	dd if=/dev/zero of=$@.tmp bs=1M count=$(CROSVM_IMG_SIZE_MB)
 	mkfs.ext4 -d $(TMP_DIR) $@.tmp
 	rm -rf $(TMP_DIR)
 	mv $@.tmp $@

 # The qemu binary
 .PHONY:$(PREBUILTS_QEMU_BIN)
 $(PREBUILTS_QEMU_BIN):
 	- rm -rf $@
 	${SCRIPTS_DIR}/aarch64/build_qemu.sh $(OUT_DIR)/qemu ${TOOLCHAIN_BIN}
 	mkdir -p $@/bin
 	cp $(OUT_DIR)/qemu/qemu-system-aarch64 $@/bin

 .PHONY: $(PREBUILTS_MAKE_BIN)
 $(PREBUILTS_MAKE_BIN):
 	- rm -rf $(OUT_DIR)/make
 	mkdir -p $(OUT_DIR)/make $@/bin
 	wget 'https://ftp.gnu.org/gnu/make/make-4.3.tar.gz' -O - | \
 		tar -xzf - -C $(OUT_DIR)/make --strip 1
 	(cd $(OUT_DIR)/make && ./configure LDFLAGS="-static" && $(MAKE))
 	cp $(OUT_DIR)/make/make $@/bin/

 ##
 ## Run tests
 ##

 KVM_MODES := vhe nvhe pkvm

 hyphensuffix = $(lastword $(subst -, ,$(1)))
 space := $(subst ,, )
 # Like wordlist but takes negative upper bounds (-2 means remove last two words)
 revwordlist = $(wordlist $(1),$(shell echo '(($(2) - 1 + $(words $(3))) % $(words $(3))) + 1' | bc),$(3))
 hyphenlist = $(subst $(space),-,$(call revwordlist,$(1),$(2),$(subst -, ,$(3))))

 RUN_TEST_SH := $(SCRIPTS_DIR)/kvm-unit-tests/run_test.sh
 RUN_TEST = $(RUN_TEST_SH) \
 		-d $@ \
 		-M $(call hyphensuffix,$@) \
 		-k "$(KERNEL_IMAGE)" \
 		$(RUN_TEST_ARGS) "$(RUN_TEST_HOST_SH)" \
 		-- \
 		-D modules:${KERNEL_MODULES}/lib/modules \
 		$(RUN_TEST_QEMU_ARGS)

 ifeq ($(VERBOSE),1)
 RUN_TEST_ARGS += -v
 endif

 ifneq ($(QUIET),0)
 RUN_TEST_ARGS += -o "$(RUN_TEST_LOG_DIR)/$@.log"
 endif

 ifeq ($(GDB),1)
 # RUN_TEST_SH defaults to no-timeout when -G is passed but -t forces one:
 RUN_TEST_ARGS += -G
 else
 # Without -G, RUN_TEST_SH defaults to a non-zero timeout duration.
 # RUN_TEST_TIMEOUT should be use if a specific value is wanted (see 'man timeout').
 # In particular, RUN_TEST_TIMEOUT=0 prevents the test from timing out.
 RUN_TEST_ARGS += $(if $(RUN_TEST_TIMEOUT),-t ,)$(RUN_TEST_TIMEOUT)
 endif

 # Root test targets. Dynamically generated per-test targets
 # will add themselves as dependencies of these.
 .PHONY: test test-list
 test:
 test-list:

 #
 # kvm-unit-tests
 #

 KUT_RUN_TEST = $(RUN_TEST_SH)
 KUT_GEN_MAKEFILE := $(SCRIPTS_DIR)/kvm-unit-tests/gen_makefile.sh
 KUT_MAKEFILE := $(OUT_DIR)/Makefile.kvm-unit-tests
 KUT_TEST_DIR := $(KUT_OUT)/tests
 KUT_LOG_DIR := $(DIST_DIR)/logs/kvm-unit-tests/

 # Exclude tests that require user interaction or are known to fail.
 KUT_EXCLUDE := (.+migrat.+)|(gicv2-.+)|(pmu-event-introspection)|(micro-bench)

 # Generate a Makefile with targets per test and configuration.
 # Pass in variable/target names from this Makefile that the
 # generated Makefile will refer to.
 $(KUT_MAKEFILE): $(SELF) $(KUT_GEN_MAKEFILE) $(KUT_STAMP)
 	@ mkdir -p $(shell dirname $@)
 	@ $(KUT_GEN_MAKEFILE)				\
 		$(KUT_TEST_DIR) "$(KUT_EXCLUDE)"	\
 		KERNEL_IMAGE KUT_LOG_DIR test test-list	\
 		> $@.tmp
 	@ mv $@.tmp $@
 include $(KUT_MAKEFILE)

 #
 # Basic boot tests in the various arm64 KVM modes
 #

 BOOT_TESTS := $(addprefix test-boot-,$(KVM_MODES))

 .PHONY: test-boot
 test-boot: $(BOOT_TESTS)

 TEST_LISTS += test-list-boot
 test-list-boot: LISTED_TESTS = $(BOOT_TESTS)

 TESTS += $(BOOT_TESTS)
 $(BOOT_TESTS): RUN_TEST_LOG_DIR = $(DIST_DIR)/logs/boot
 $(BOOT_TESTS): RUN_TEST_HOST_SH = $(SCRIPTS_DIR)/aarch64/test_boot.sh
 $(BOOT_TESTS): RUN_TEST_QEMU_ARGS = -a "boot_test_mode=$(call hyphensuffix,$@)"

 #
 # CrosVM Hello World test
 #

 CROSVM_HELLOWORLD_TESTS := $(addprefix test-crosvm-helloworld-,$(KVM_MODES))

 VM_IMAGE ?= $(KERNEL_IMAGE)

 .PHONY: test-crosvm-helloworld
 test-crosvm-helloworld: $(CROSVM_HELLOWORLD_TESTS)

 TEST_LISTS += test-list-crosvm-helloworld
 test-list-crosvm-helloworld: LISTED_TESTS = $(CROSVM_HELLOWORLD_TESTS)

 TESTS += $(CROSVM_HELLOWORLD_TESTS)
 $(CROSVM_HELLOWORLD_TESTS): $(CROSVM_EXT4)
 $(CROSVM_HELLOWORLD_TESTS): RUN_TEST_LOG_DIR = $(DIST_DIR)/logs/crosvm/helloworld
 $(CROSVM_HELLOWORLD_TESTS): RUN_TEST_HOST_SH = $(SCRIPTS_DIR)/crosvm/helloworld/host.sh
 $(CROSVM_HELLOWORLD_TESTS): RUN_TEST_QEMU_ARGS = \
 		-s 4 \
 		-R $(SCRIPTS_DIR)/crosvm/helloworld/guest.sh \
 		-R $(VM_IMAGE) \
 		-R $(PREBUILTS_ROOTFS_EXT4) \
 		-R $(CROSVM_EXT4)
 $(CROSVM_HELLOWORLD_TESTS): RUN_TEST_TIMEOUT := 300s

 #
 # Linux kselftests
 #

 KSFT_TEST_TARGETS := $(foreach t,$(KSFT_TESTS),$(foreach m,$(KVM_MODES),\
 	test-ksft-$(subst :,-,$(t))-$(m)))

 .PHONY: test-ksft
 test-ksft: $(KSFT_TEST_TARGETS)


 TEST_LISTS += test-list-ksft
 test-list-ksft: LISTED_TESTS = $(KSFT_TEST_TARGETS)

 TESTS += $(KSFT_TEST_TARGETS)
 $(KSFT_TEST_TARGETS): $(KSFT_STAMP)
 $(KSFT_TEST_TARGETS): RUN_TEST_LOG_DIR = $(DIST_DIR)/logs/kselftest
 $(KSFT_TEST_TARGETS): RUN_TEST_HOST_SH = $(SCRIPTS_DIR)/kselftest/host.sh
 $(KSFT_TEST_TARGETS): RUN_TEST_QEMU_ARGS = \
 	-D kselftest:$(KSFT_INSTALL_DIR) \
 	-a 'ksft=$(call hyphenlist,3,3,$@):$(call hyphenlist,4,-1,$@)'

 .PHONY test: $(TESTS)
 $(TESTS): $(KERNEL_IMAGE) $(KERNEL_MODULES_DEP) $(BL_BINS)
 	@ $(RUN_TEST)
 .PHONY test-list: $(TEST_LISTS)
 $(TEST_LISTS):
 	@ for x in $(LISTED_TESTS); do echo $$x; done
	# Copyright 2020 The Android KVM Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	SELF := $(abspath $(lastword $(MAKEFILE_LIST)))

	CPUS ?= $(shell nproc)
	MAKEFLAGS += --jobs=$(CPUS)
	SHELL ?= $(shell which bash)

	##
	## Folders
	##
	ROOT_DIR := $(CURDIR)

	OUT_DIR := $(ROOT_DIR)/out

	# DIST_DIR: target directory for artifacts and logs.
	# The CI build (pKVM Busytown) uses it to separate artifacts from other
	# temporary build files, which are placed in in OUT_DIR.
	# For other users, OUT_DIR and DIST_DIR are typically the same.
	DIST_DIR ?= $(OUT_DIR)

	# Copy specified file or directory only if OUT_DIR and DIST_DIR aren't the same.
	#
	# Parameters:
	# $(1): Source files or directories (to be copied recursively)
	# $(2): Destination directory (is created if it doesn't exist)
	ifneq ($(abspath $(OUT_DIR)), $(abspath $(DIST_DIR)))
	define copy_to_dist
	mkdir -p $(2) && cp -r $(1) $(2)
	endef
	endif

	LINUX_OUT := $(OUT_DIR)/linux
	LINUX_DIST := $(DIST_DIR)/linux
	SCRIPTS_DIR := $(ROOT_DIR)/build

	include $(SCRIPTS_DIR)/toolchain.mk

	##
	## Common options
	##
	VERBOSE ?= 0
	ARCH = arm64

	LINUX_OUT_IMAGE := $(LINUX_OUT)/arch/arm64/boot/Image
	LINUX_OUT_MODULES := $(LINUX_OUT)/modules_install
	LINUX_OUT_MODULES_DEP := $(LINUX_OUT)/modules_install.stamp

	# KERNEL_IMAGE/MODULES can be set to use own custom kernel image.
	KERNEL_IMAGE ?= $(LINUX_OUT_IMAGE)
	KERNEL_MODULES ?= $(LINUX_OUT_MODULES)
	KERNEL_MODULES_DEP ?= $(LINUX_OUT_MODULES_DEP)

	##
	## Toolchain options and variables
	##

	# Specifies which toolchain of the prebuilt toolchains to use.
	# Can be either clang, gcc-5.1, or gcc-9.2.
	# GCC will always use the latest version for building everything else other
	# than the kernel.
	TOOLCHAIN ?= clang
	$(eval $(call define_toolchain,$(TOOLCHAIN)))

	##
	## Common targets
	##

	.DEFAULT_GOAL := default

	.PHONY: default
	default: kvm-unit-tests linux

	.PHONY: clean
	clean: bl_clean bl33_clean buildroot_clean kvm-unit-tests_clean linux_clean kselftest_clean tfa_clean

	.PHONY: distclean
	distclean:
	- rm -rf $(OUT_DIR) $(DIST_DIR)

	FORCE:

	.PHONY: all
	all: buildroot kvm-unit-tests linux

	.PHONY: help
	help:
	@echo 'Generic Targets:'
	@echo ' default - builds all generic targets except for buildroot'
	@echo ' all - builds all generic targets'
	@echo ' bl - builds U-Boot and TF-A and copies them to the bl output directory'
	@echo ' bl33 - U-Boot as BL33'
	@echo ' buildroot - the buildroot rootfs image as well as the host qemu'
	@echo ' kvm-unit-tests - the kvm-unit-tests'
	@echo ' linux - the Linux kernel'
	@echo ' kselftest - the Linux kernel selftests'
	@echo ' tfa - Arm Trusted Firmware-A'
	@echo ''
	@echo 'Clean Targets:'
	@echo ' clean - cleans all, but keeps the prebuilts'
	@echo ' distclean - distclean for all targets'
	@echo ' bl_clean - TF-A, U-Boot and the bl output directory'
	@echo ' bl33_clean - the U-Boot BL33'
	@echo ' buildroot_clean - the buildroot rootfs image as well as the host qemu'
	@echo ' kvm-unit-tests_clean - the kvm-unit-tests'
	@echo ' linux_clean - the Linux kernel'
	@echo ' kselftest_clean - the Linux kernel selftests'
	@echo ' tfa_clean - Arm Trusted Firmware-A'
	@echo ''
	@echo 'Test Targets:'
	@echo ' test - runs all tests in qemu-aarch64 simulated environment'
	@echo ' test-kut - runs all kvm-unit-tests in qemu-aarch64 simulated environment'
	@echo ' test-ksft - runs all kselftests in qemu-aarch64 simulated environment'
	@echo ' test-list - lists all test targets'
	@echo ''
	@echo 'Test Parameters:'
	@echo ' CROSVM_TAR_GZ - path to a custom cvd-host_package.tar.gz'
	@echo ' (default: uses a pre-built version)'
	@echo ' VM_IMAGE - path to an image to be used by the VMM to boot the VM'
	@echo ' (default: uses the host image)'
	@echo ''
	@echo 'Test Flags:'
	@echo ' VERBOSE=[0\|1] - prints the verbose output of the scripts running the test'
	@echo ' (default: 0)'
	@echo ' QUIET=[0\|1] - redirects the output to a file instead of stdout'
	@echo ' (default: 1)'
	@echo ' GDB=[0\|1] - makes the started test wait for a GDB connection'
	@echo ' (default: 0)'
	@echo ''
	@echo 'Misc Targets:'
	@echo ' update-prebuilts - generates/update the prebuilt rootfs image (aarch64) and the qemu host (aarch64)'


	##
	## Buildroot
	##

	BR_DEFCONFIG ?= qemu_aarch64_virt_kvmunittests_defconfig
	BR_VERBOSE ?= $(VERBOSE)

	BR_SRC := $(ROOT_DIR)/buildroot
	BR_OUT := $(OUT_DIR)/buildroot
	BR_MAKE := $(MAKE) -C $(BR_SRC) V=$(BR_VERBOSE) O=$(BR_OUT)

	.PHONY: buildroot
	buildroot:
	+ $(BR_MAKE) $(BR_DEFCONFIG)
	+ $(BR_MAKE) all

	.PHONY: buildroot_clean
	buildroot_clean:
	+ $(BR_MAKE) clean

	# Bootloader stages come from TF-A, U-Boot, or just an empty file.
	BL_DIR := $(OUT_DIR)/bl
	BL1_BIN := $(BL_DIR)/bl1.bin
	BL2_BIN := $(BL_DIR)/bl2.bin
	BL31_BIN := $(BL_DIR)/bl31.bin
	BL32_BIN := $(BL_DIR)/bl32.bin
	BL33_BIN := $(BL_DIR)/bl33.bin
	BL_BINS := $(BL1_BIN) $(BL2_BIN) $(BL31_BIN) $(BL32_BIN) $(BL33_BIN)

	##
	## U-Boot
	##
	UBOOT_SRC := $(ROOT_DIR)/u-boot
	UBOOT_OUT := $(OUT_DIR)/u-boot
	UBOOT_DEFCONFIG ?= qemu_arm64_defconfig
	UBOOT_CONFIG := $(UBOOT_OUT)/.config
	UBOOT_TOOLCHAIN ?= gcc-9.2
	UBOOT_VERBOSE ?= $(VERBOSE)
	$(eval $(call define_toolchain,$(UBOOT_TOOLCHAIN),UBOOT_))
	UBOOT_MAKE := \
	PATH=$(UBOOT_TOOLCHAIN_BIN):$(PATH) \
	CROSS_COMPILE=$(UBOOT_TARGET)- \
	$(MAKE) \
	V=$(UBOOT_VERBOSE) \
	-C $(UBOOT_SRC) \
	O=$(UBOOT_OUT)

	.PHONY: bl33
	bl33: $(UBOOT_OUT)/u-boot.bin

	$(UBOOT_CONFIG): $(UBOOT_SRC)/configs/$(UBOOT_DEFCONFIG)
	+ $(UBOOT_MAKE) $(UBOOT_DEFCONFIG)
	$(LINUX_SRC)/scripts/config --file $(UBOOT_CONFIG) --set-val CONFIG_BOOTDELAY 0
	$(LINUX_SRC)/scripts/config --file $(UBOOT_CONFIG) --set-str CONFIG_BOOTCOMMAND \
	"qfw load 0x40000000 0x50000000 ; booti 0x40000000 - $$\{fdtcontroladdr\}"

	$(UBOOT_OUT)/u-boot.bin: $(UBOOT_CONFIG) FORCE
	+ $(UBOOT_MAKE)

	.PHONY: bl33_clean
	bl33_clean:
	+ $(UBOOT_MAKE) distclean

	##
	## TF-A
	##
	TFA_SRC := $(ROOT_DIR)/trusted-firmware-a
	TFA_OUT := $(OUT_DIR)/trusted-firmware-a
	TFA_TOOLCHAIN ?= gcc-9.2
	TFA_VERBOSE ?= $(VERBOSE)
	$(eval $(call define_toolchain,$(TFA_TOOLCHAIN),TFA_))
	TFA_MAKE := \
	PATH=$(TFA_TOOLCHAIN_BIN):$(PATH) \
	$(MAKE) \
	CROSS_COMPILE="$(TFA_TARGET)-" \
	PLAT=qemu \
	ENABLE_SVE_FOR_NS=1 \
	QEMU_USE_GIC_DRIVER=QEMU_GICV3 \
	ARM_LINUX_KERNEL_AS_BL33=1 \
	SPD=trusty \
	TRUSTY_SPD_WITH_GENERIC_SERVICES=1 \
	V=$(TFA_VERBOSE) \
	-C $(TFA_SRC) \
	BUILD_BASE=$(TFA_OUT)

	.PHONY: tfa
	tfa: $(TFA_OUT)/qemu/release/bl1.bin $(TFA_OUT)/qemu/release/bl2.bin $(TFA_OUT)/qemu/release/bl31.bin

	$(TFA_OUT)/qemu/release/bl1.bin $(TFA_OUT)/qemu/release/bl2.bin $(TFA_OUT)/qemu/release/bl31.bin &: FORCE
	+ $(TFA_MAKE) all

	.PHONY: tfa_clean
	tfa_clean:
	rm -rf $(TFA_OUT)

	##
	## Copy bootloader stages to BL_DIR
	##

	$(BL_DIR):
	mkdir -p $@

	$(BL1_BIN) $(BL2_BIN) $(BL31_BIN): $(BL_DIR)/bl%.bin: $(TFA_OUT)/qemu/release/bl%.bin \| $(BL_DIR)
	cp $< $@

	# An empty BL32 is enough for us, it just needs to exist.
	$(BL32_BIN): \| $(BL_DIR)
	touch $@

	$(BL33_BIN): $(UBOOT_OUT)/u-boot.bin \| $(BL_DIR)
	cp $< $@

	.PHONY: bl
	bl: $(BL_BINS)

	.PHONY: bl_clean
	bl_clean: tfa_clean bl33_clean
	rm -rf $(BL_DIR)

	##
	## kvm-unit-tests
	##
	KUT_SRC := $(ROOT_DIR)/kvm-unit-tests
	KUT_OUT := $(OUT_DIR)/kvm-unit-tests
	KUT_DIST := $(DIST_DIR)/kvm-unit-tests
	KUT_STAMP := $(KUT_OUT)/kvm-unit-tests.stamp

	ifneq ($(TOOLCHAIN_IS_LEGACY),1)
	KUT_TOOLCHAIN ?= $(TOOLCHAIN)
	else
	KUT_TOOLCHAIN ?= gcc-9.2
	endif
	$(eval $(call define_toolchain,$(KUT_TOOLCHAIN),KUT_))

	ifeq ($(KUT_TOOLCHAIN_IS_LLVM),1)
	KUT_CFLAGS += --target=$(KUT_TARGET) -fpic
	endif

	.PHONY: kvm-unit-tests
	kvm-unit-tests $(KUT_STAMP):
	mkdir -p $(KUT_OUT)
	cd $(KUT_OUT) && \
	$(KUT_SRC)/configure \
	--erratatxt=$(KUT_SRC)/errata.txt \
	--prefix=$(KUT_OUT) --arch=$(ARCH) \
	--cc=$(KUT_CC) --ld=$(KUT_LD) \
	--objcopy=$(KUT_OBJCOPY) --objdump=$(KUT_OBJDUMP)
	COMMON_CFLAGS="$(KUT_CFLAGS)" \
	$(MAKE) -C $(KUT_OUT) standalone
	$(call copy_to_dist, $(KUT_OUT)/tests, $(KUT_DIST))
	touch $(KUT_STAMP)

	.PHONY: kvm-unit-tests_clean
	kvm-unit-tests_clean:
	- rm -f $(KUT_STAMP)
	- $(MAKE) -C $(KUT_OUT) clean


	##
	## Linux kernel
	##
	LINUX_DEFCONFIG ?= defconfig
	LINUX_VERBOSE ?= $(VERBOSE)
	LINUX_SRC ?= $(ROOT_DIR)/linux
	LINUX_VMLINUX = $(LINUX_OUT)/vmlinux
	LINUX_CONFIG := $(LINUX_OUT)/.config
	LINUX_DIST_FILES = $(LINUX_CONFIG) $(LINUX_VMLINUX) $(KERNEL_IMAGE)
	LINUX_LSMOD_TXT := $(LINUX_OUT)/lsmod.txt

	LINUX_TOOLCHAIN ?= $(TOOLCHAIN)
	$(eval $(call define_toolchain,$(LINUX_TOOLCHAIN),LINUX_))

	ifeq ($(LINUX_TOOLCHAIN_IS_LLVM),1)
	LINUX_EXTRA += LLVM=1 LLVM_IAS=1
	endif

	LINUX_MAKE := \
	PATH=$(LINUX_TOOLCHAIN_BIN):$(PATH) \
	ARCH=$(ARCH) \
	CROSS_COMPILE="$(LINUX_TARGET)-" \
	$(MAKE) \
	$(LINUX_EXTRA) \
	CC=$(LINUX_CC) \
	LD=$(LINUX_LD) \
	OBJCOPY=$(LINUX_OBJCOPY) \
	OBJDUMP=$(LINUX_OBJDUMP) \
	-C $(LINUX_SRC) \
	V=$(LINUX_VERBOSE) \
	O=$(LINUX_OUT)

	LINUX_MODULES := \
	TEST_PRINTF

	.PHONY: linux_defconfig
	linux_defconfig $(LINUX_CONFIG):
	+ $(LINUX_MAKE) $(LINUX_DEFCONFIG)
	# Disable all modules, then selectively enable those in LINUX_MODULES.
	@ echo > $(LINUX_LSMOD_TXT)
	+ $(LINUX_MAKE) LSMOD=$(LINUX_LSMOD_TXT) localmodconfig
	$(LINUX_SRC)/scripts/config --file $(LINUX_CONFIG) $(addprefix -m CONFIG_,$(LINUX_MODULES))

	.PHONY: linux_menuconfig
	linux_menuconfig: $(LINUX_CONFIG)
	+ $(LINUX_MAKE) menuconfig

	.PHONY: linux
	linux: $(LINUX_CONFIG)
	+ $(LINUX_MAKE)
	$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))

	.PHONY: linux_image
	linux_image $(LINUX_OUT_IMAGE): $(LINUX_CONFIG)
	+ $(LINUX_MAKE) Image.gz
	$(call copy_to_dist, $(LINUX_DIST_FILES), $(LINUX_DIST))

	.PHONY: linux_modules
	linux_modules $(LINUX_OUT_MODULES_DEP): $(LINUX_OUT_IMAGE)
	@ rm -rf $(LINUX_OUT_MODULES)
	+ $(LINUX_MAKE) modules
	+ $(LINUX_MAKE) INSTALL_MOD_PATH=$(LINUX_OUT_MODULES) modules_install
	@ find $(LINUX_OUT_MODULES) -type f \| sort \| \
	xargs sha1sum > $(LINUX_OUT_MODULES_DEP).tmp
	@ cmp $(LINUX_OUT_MODULES_DEP).tmp $(LINUX_OUT_MODULES_DEP) \|\| \
	mv $(LINUX_OUT_MODULES_DEP).tmp $(LINUX_OUT_MODULES_DEP)

	.PHONY: linux_clean
	linux_clean:
	+ $(LINUX_MAKE) mrproper


	##
	## Linux kselftest
	##
	KSFT_VERBOSE ?= $(VERBOSE)
	KSFT_SRC ?= $(LINUX_SRC)
	KSFT_SRC_STAMP ?= $(KSFT_OUT)/ksefltest_src.stamp
	KSFT_OUT ?= $(OUT_DIR)/kselftest
	KSFT_OUT_CONFIG ?= $(KSFT_OUT)/.config
	KSFT_INSTALL_DIR ?= $(KSFT_OUT)/tools/testing/selftests/kselftest_install
	KSFT_STAMP ?= $(KSFT_INSTALL_DIR)/kselftest-list.txt

	KSFT_TESTS := \
	lib:printf.sh \
	kvm:kvm_create_max_vcpus \
	kvm:set_memory_region_test \
	kvm:steal_time

	ifneq ($(TOOLCHAIN_IS_LLVM),1)
	KSFT_TOOLCHAIN ?= $(TOOLCHAIN)
	else
	KSFT_TOOLCHAIN ?= gcc-9.2
	endif
	$(eval $(call define_toolchain,$(KSFT_TOOLCHAIN),KSFT_))

	KSFT_MAKE := \
	PATH=$(KSFT_TOOLCHAIN_BIN):$(PATH) \
	ARCH=$(ARCH) \
	CROSS_COMPILE="$(KSFT_TARGET)-" \
	$(MAKE) \
	-C $(KSFT_OUT) \
	V=$(KSFT_VERBOSE)

	KSFT_MAKE_TARGETS := $(sort $(foreach t,$(KSFT_TESTS),$(firstword $(subst :, ,$(t)))))

	# Copy the source to another folder because 'kselftest-install' clobbers it.
	# Stamp is touched if any files (not directories) were synced.
	$(KSFT_SRC_STAMP): FORCE
	@ mkdir -p $(KSFT_OUT) $(@D)
	@ LIST="$$(rsync -a --info=name $(KSFT_SRC)/ $(KSFT_OUT) \| grep -vE '/$$')"; \
	[ -f $@ -a -z "$$LIST" ] \|\| touch $@

	.PHONY: kselftest
	kselftest $(KSFT_STAMP): $(KSFT_SRC_STAMP) $(LINUX_CONFIG)
	@ cp $(LINUX_CONFIG) $(KSFT_OUT_CONFIG)
	+ $(KSFT_MAKE) olddefconfig prepare
	+ $(KSFT_MAKE) TARGETS="$(KSFT_MAKE_TARGETS)" kselftest-install

	.PHONY: kselftest_clean
	kselftest_clean:
	@ rm -rf $(KSFT_OUT)


	##
	## Generating/Updating prebuilts
	##

	PREBUILTS_CROSVM_EXT4 := $(ROOT_DIR)/prebuilts/linux-aarch64/images/crosvm.ext4
	PREBUILTS_ROOTFS_EXT4 := $(ROOT_DIR)/prebuilts/linux-aarch64/images/rootfs.ext4
	PREBUILTS_QEMU_ROM := $(ROOT_DIR)/prebuilts/linux-aarch64/images/efi-virtio.rom
	PREBUILTS_QEMU_BIN := $(ROOT_DIR)/prebuilts/linux-x86/qemu
	PREBUILTS_MAKE_BIN := $(ROOT_DIR)/prebuilts/linux-x86/make

	CUSTOM_CROSVM_EXT4 := $(OUT_DIR)/images/crosvm.ext4

	ifeq ($(CROSVM_TAR_GZ),)
	CROSVM_EXT4 := $(PREBUILTS_CROSVM_EXT4)
	else
	CROSVM_EXT4 := $(CUSTOM_CROSVM_EXT4)
	endif

	.PHONY:update-prebuilts
	update-prebuilts: \
	$(PREBUILTS_ROOTFS_EXT4) \
	update-prebuilts-crosvm \
	$(PREBUILTS_QEMU_ROM) \
	$(PREBUILTS_QEMU_BIN) \
	$(PREBUILTS_MAKE_BIN)

	$(PREBUILTS_ROOTFS_EXT4): buildroot
	mkdir -p $(shell dirname $@)
	cp $(BR_OUT)/images/rootfs.ext4 $@

	$(PREBUILTS_QEMU_ROM): buildroot
	mkdir -p $(shell dirname $@)
	cp $(BR_OUT)/per-package/host-qemu/host/share/qemu/efi-virtio.rom $@

	CF_DOWNLOAD_AOSP_SH := $(ROOT_DIR)/build/cuttlefish/download-aosp.sh

	# A PHONY target is necessary for allowing the user to update the archive
	# because, as we are downloading it from a remote server, make dependencies are
	# unable to express the local-->remote dependency:
	.PHONY: update-prebuilts-crosvm
	update-prebuilts-crosvm $(PREBUILTS_CROSVM_TAR_GZ):
	@rm -f $(PREBUILTS_CROSVM_TAR_GZ) # else, wget will create "$@.1"
	cd $(dir $(PREBUILTS_CROSVM_TAR_GZ)) && $(CF_DOWNLOAD_AOSP_SH) -XCa arm64
	@touch -c $(PREBUILTS_CROSVM_TAR_GZ) # wget doesn't set timestamp to now

	CROSVM_IMG_SIZE_MB := 1024

	$(CUSTOM_CROSVM_EXT4): TMP_DIR := $(shell mktemp -d)
	$(CUSTOM_CROSVM_EXT4): $(CROSVM_TAR_GZ)
	mkdir -p $(@D)
	cd $(TMP_DIR) && tar xzvf $(abspath $<)
	dd if=/dev/zero of=$@.tmp bs=1M count=$(CROSVM_IMG_SIZE_MB)
	mkfs.ext4 -d $(TMP_DIR) $@.tmp
	rm -rf $(TMP_DIR)
	mv $@.tmp $@

	# The qemu binary
	.PHONY:$(PREBUILTS_QEMU_BIN)
	$(PREBUILTS_QEMU_BIN):
	- rm -rf $@
	${SCRIPTS_DIR}/aarch64/build_qemu.sh $(OUT_DIR)/qemu ${TOOLCHAIN_BIN}
	mkdir -p $@/bin
	cp $(OUT_DIR)/qemu/qemu-system-aarch64 $@/bin

	.PHONY: $(PREBUILTS_MAKE_BIN)
	$(PREBUILTS_MAKE_BIN):
	- rm -rf $(OUT_DIR)/make
	mkdir -p $(OUT_DIR)/make $@/bin
	wget 'https://ftp.gnu.org/gnu/make/make-4.3.tar.gz' -O - \| \
	tar -xzf - -C $(OUT_DIR)/make --strip 1
	(cd $(OUT_DIR)/make && ./configure LDFLAGS="-static" && $(MAKE))
	cp $(OUT_DIR)/make/make $@/bin/

	##
	## Run tests
	##

	KVM_MODES := vhe nvhe pkvm

	hyphensuffix = $(lastword $(subst -, ,$(1)))
	space := $(subst ,, )
	# Like wordlist but takes negative upper bounds (-2 means remove last two words)
	revwordlist = $(wordlist $(1),$(shell echo '(($(2) - 1 + $(words $(3))) % $(words $(3))) + 1' \| bc),$(3))
	hyphenlist = $(subst $(space),-,$(call revwordlist,$(1),$(2),$(subst -, ,$(3))))

	RUN_TEST_SH := $(SCRIPTS_DIR)/kvm-unit-tests/run_test.sh
	RUN_TEST = $(RUN_TEST_SH) \
	-d $@ \
	-M $(call hyphensuffix,$@) \
	-k "$(KERNEL_IMAGE)" \
	$(RUN_TEST_ARGS) "$(RUN_TEST_HOST_SH)" \
	-- \
	-D modules:${KERNEL_MODULES}/lib/modules \
	$(RUN_TEST_QEMU_ARGS)

	ifeq ($(VERBOSE),1)
	RUN_TEST_ARGS += -v
	endif

	ifneq ($(QUIET),0)
	RUN_TEST_ARGS += -o "$(RUN_TEST_LOG_DIR)/$@.log"
	endif

	ifeq ($(GDB),1)
	# RUN_TEST_SH defaults to no-timeout when -G is passed but -t forces one:
	RUN_TEST_ARGS += -G
	else
	# Without -G, RUN_TEST_SH defaults to a non-zero timeout duration.
	# RUN_TEST_TIMEOUT should be use if a specific value is wanted (see 'man timeout').
	# In particular, RUN_TEST_TIMEOUT=0 prevents the test from timing out.
	RUN_TEST_ARGS += $(if $(RUN_TEST_TIMEOUT),-t ,)$(RUN_TEST_TIMEOUT)
	endif

	# Root test targets. Dynamically generated per-test targets
	# will add themselves as dependencies of these.
	.PHONY: test test-list
	test:
	test-list:

	#
	# kvm-unit-tests
	#

	KUT_RUN_TEST = $(RUN_TEST_SH)
	KUT_GEN_MAKEFILE := $(SCRIPTS_DIR)/kvm-unit-tests/gen_makefile.sh
	KUT_MAKEFILE := $(OUT_DIR)/Makefile.kvm-unit-tests
	KUT_TEST_DIR := $(KUT_OUT)/tests
	KUT_LOG_DIR := $(DIST_DIR)/logs/kvm-unit-tests/

	# Exclude tests that require user interaction or are known to fail.
	KUT_EXCLUDE := (.+migrat.+)\|(gicv2-.+)\|(pmu-event-introspection)\|(micro-bench)

	# Generate a Makefile with targets per test and configuration.
	# Pass in variable/target names from this Makefile that the
	# generated Makefile will refer to.
	$(KUT_MAKEFILE): $(SELF) $(KUT_GEN_MAKEFILE) $(KUT_STAMP)
	@ mkdir -p $(shell dirname $@)
	@ $(KUT_GEN_MAKEFILE) \
	$(KUT_TEST_DIR) "$(KUT_EXCLUDE)" \
	KERNEL_IMAGE KUT_LOG_DIR test test-list \
	> $@.tmp
	@ mv $@.tmp $@
	include $(KUT_MAKEFILE)

	#
	# Basic boot tests in the various arm64 KVM modes
	#

	BOOT_TESTS := $(addprefix test-boot-,$(KVM_MODES))

	.PHONY: test-boot
	test-boot: $(BOOT_TESTS)

	TEST_LISTS += test-list-boot
	test-list-boot: LISTED_TESTS = $(BOOT_TESTS)

	TESTS += $(BOOT_TESTS)
	$(BOOT_TESTS): RUN_TEST_LOG_DIR = $(DIST_DIR)/logs/boot
	$(BOOT_TESTS): RUN_TEST_HOST_SH = $(SCRIPTS_DIR)/aarch64/test_boot.sh
	$(BOOT_TESTS): RUN_TEST_QEMU_ARGS = -a "boot_test_mode=$(call hyphensuffix,$@)"

	#
	# CrosVM Hello World test
	#

	CROSVM_HELLOWORLD_TESTS := $(addprefix test-crosvm-helloworld-,$(KVM_MODES))

	VM_IMAGE ?= $(KERNEL_IMAGE)

	.PHONY: test-crosvm-helloworld
	test-crosvm-helloworld: $(CROSVM_HELLOWORLD_TESTS)

	TEST_LISTS += test-list-crosvm-helloworld
	test-list-crosvm-helloworld: LISTED_TESTS = $(CROSVM_HELLOWORLD_TESTS)

	TESTS += $(CROSVM_HELLOWORLD_TESTS)
	$(CROSVM_HELLOWORLD_TESTS): $(CROSVM_EXT4)
	$(CROSVM_HELLOWORLD_TESTS): RUN_TEST_LOG_DIR = $(DIST_DIR)/logs/crosvm/helloworld
	$(CROSVM_HELLOWORLD_TESTS): RUN_TEST_HOST_SH = $(SCRIPTS_DIR)/crosvm/helloworld/host.sh
	$(CROSVM_HELLOWORLD_TESTS): RUN_TEST_QEMU_ARGS = \
	-s 4 \
	-R $(SCRIPTS_DIR)/crosvm/helloworld/guest.sh \
	-R $(VM_IMAGE) \
	-R $(PREBUILTS_ROOTFS_EXT4) \
	-R $(CROSVM_EXT4)
	$(CROSVM_HELLOWORLD_TESTS): RUN_TEST_TIMEOUT := 300s

	#
	# Linux kselftests
	#

	KSFT_TEST_TARGETS := $(foreach t,$(KSFT_TESTS),$(foreach m,$(KVM_MODES),\
	test-ksft-$(subst :,-,$(t))-$(m)))

	.PHONY: test-ksft
	test-ksft: $(KSFT_TEST_TARGETS)


	TEST_LISTS += test-list-ksft
	test-list-ksft: LISTED_TESTS = $(KSFT_TEST_TARGETS)

	TESTS += $(KSFT_TEST_TARGETS)
	$(KSFT_TEST_TARGETS): $(KSFT_STAMP)
	$(KSFT_TEST_TARGETS): RUN_TEST_LOG_DIR = $(DIST_DIR)/logs/kselftest
	$(KSFT_TEST_TARGETS): RUN_TEST_HOST_SH = $(SCRIPTS_DIR)/kselftest/host.sh
	$(KSFT_TEST_TARGETS): RUN_TEST_QEMU_ARGS = \
	-D kselftest:$(KSFT_INSTALL_DIR) \
	-a 'ksft=$(call hyphenlist,3,3,$@):$(call hyphenlist,4,-1,$@)'

	.PHONY test: $(TESTS)
	$(TESTS): $(KERNEL_IMAGE) $(KERNEL_MODULES_DEP) $(BL_BINS)
	@ $(RUN_TEST)
	.PHONY test-list: $(TEST_LISTS)
	$(TEST_LISTS):
	@ for x in $(LISTED_TESTS); do echo $$x; done