Replace run_tests.sh with a dynamically-generated Makefile
The script's main purpose is to spawn a parallel job per test case
and configuration. Since that is what Make does for compilation jobs,
this patch turns test jobs into Make targets.
Replace run_tests.sh with gen_makefile.sh, which enumerates tests in
the kvm-unit-tests output folder and generates one Make target per
test case and configuration. Rebuild the list whenever kvm-unit-tests
are recompiled.
Example usage:
$ make test -j20 # run all tests, 20 parallel jobs
$ make test-kut-nvhe # run all nVHE tests
$ make test-kut-nvhe-psci # run one specific test
Change-Id: Ic1930c752814e68f8437ceb08e9b8c975ceca047
diff --git a/Makefile b/Makefile
index ee8665a..8c71dd1 100644
--- a/Makefile
+++ b/Makefile
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+SELF := $(abspath $(lastword $(MAKEFILE_LIST)))
+
##
## Folders
##
@@ -19,7 +21,7 @@
OUT_DIR := $(ROOT_DIR)/out
LINUX_OUT := $(OUT_DIR)/linux
CCACHE_DIR := $(OUT_DIR)/.ccache
-TEST_SCRIPTS_DIR := $(ROOT_DIR)/build/aarch64-unit-tests
+SCRIPTS_DIR := $(ROOT_DIR)/build
##
## Common options
@@ -76,8 +78,12 @@
@echo ' kvm-unit-tests_clean - the kvm-unit-tests'
@echo ' linux_clean - the linux kernel'
@echo ''
+ @echo 'Test Targets:'
+ @echo ' test - runs all tests in qemu-aarch64 simulated environment'
+ @echo ' test-kut - runs all kvm-unit-tests in qemu-aarch64 simulated environment'
+ @echo ' test-list - lists all test targets'
+ @echo ''
@echo 'Misc Targets:'
- @echo ' test - runs the kvm unit tests in the qemu-aarch64 simulated environment'
@echo ' update-prebuilts - generates/update the prebuilt rootfs image (aarch64) and the qemu host (aarch64)'
@@ -110,6 +116,7 @@
KUT_ARCH := "arm64"
KUT_SRC := $(ROOT_DIR)/kvm-unit-tests
KUT_OUT := $(OUT_DIR)/kvm-unit-tests
+KUT_STAMP := $(KUT_OUT)/kvm-unit-tests.stamp
KUT_CC := $(TOOLCHAIN_CLANG)/bin/clang
KUT_LD := $(TOOLCHAIN_CLANG)/bin/ld.lld
KUT_OBJCOPY := $(TOOLCHAIN_CLANG)/bin/llvm-objcopy
@@ -119,7 +126,7 @@
.PHONY: kvm-unit-tests
-kvm-unit-tests:
+kvm-unit-tests $(KUT_STAMP):
mkdir -p $(KUT_OUT)
cd $(KUT_OUT) && \
$(KUT_SRC)/configure \
@@ -129,9 +136,11 @@
--objcopy=$(KUT_OBJCOPY) --objdump=$(KUT_OBJDUMP)
PATH=$(KUT_PATH) COMMON_CFLAGS=$(KUT_COMMON_CFLAGS) \
$(MAKE) -C $(KUT_OUT) standalone
+ touch $(KUT_STAMP)
.PHONY: kvm-unit-tests_clean
kvm-unit-tests_clean:
+ - rm -f $(KUT_STAMP)
- $(MAKE) -C $(KUT_OUT) clean
@@ -156,18 +165,23 @@
V=$(LINUX_VERBOSE) \
O=$(LINUX_OUT)
-.PHONY: linux
-linux:
+LINUX_CONFIG := $(LINUX_OUT)/.config
+
+.PHONY: linux_defconfig
+linux_defconfig $(LINUX_CONFIG):
+ $(LINUX_MAKE) $(LINUX_DEFCONFIG)
+
+.PHONY: linux
+linux: $(LINUX_CONFIG)
+ $(LINUX_MAKE)
.PHONY: linux_image
-linux_image:
+linux_image: $(LINUX_CONFIG)
+ + $(LINUX_MAKE) Image.gz
+
# If using own kernel image (KERNEL_IMAGE is set in the environment), then skip.
ifeq ($(CUSTOM_KERNEL_IMAGE), 0)
- echo $(CUSTOM_KERNEL_IMAGE)
- + $(LINUX_MAKE) $(LINUX_DEFCONFIG)
- + $(LINUX_MAKE) Image.gz
+$(KERNEL_IMAGE): linux_image
endif
.PHONY: linux_clean
@@ -200,29 +214,35 @@
##
-## Run unit tests
+## Run tests
##
-MAKE_PID := $(shell echo $$PPID)
-JOB_FLAG := $(filter -j%, $(subst -j ,-j,$(shell ps T | grep "^\s*$(MAKE_PID).*$(MAKE)")))
-JOBS := $(subst -j,,$(JOB_FLAG))
-ifeq ($(JOBS),)
- JOBS := "1"
-endif
+# Root test targets. Dynamically generated per-test targets
+# will add themselves as dependencies of these.
+.PHONY: test test-list
+test:
+test-list:
-KUT_QEMU_BIN := $(ROOT_DIR)/prebuilts/linux-x86/qemu/bin/qemu-system-aarch64.sh
-KUT_ROOTFS_IMAGE := $(ROOT_DIR)/prebuilts/linux-aarch64/images/rootfs.ext4
+#
+# kvm-unit-tests
+#
+
+KUT_GEN_MAKEFILE := $(SCRIPTS_DIR)/kvm-unit-tests/gen_makefile.sh
+KUT_MAKEFILE := $(OUT_DIR)/Makefile.kvm-unit-tests
+KUT_TEST_DIR := $(KUT_OUT)/tests
+KUT_LOG_DIR := $(OUT_DIR)/test/kvm-unit-tests/
# Exclude tests that require user interaction or are known to fail.
-KUT_EXCLUDE := "(.+migrat.+)|(gicv2-.+)|(pmu-event-introspection)|(micro-bench)"
+KUT_EXCLUDE := (.+migrat.+)|(gicv2-.+)|(pmu-event-introspection)|(micro-bench)
-.PHONY: test
-test: kvm-unit-tests linux_image
- @VERBOSE=$(VERBOSE) \
- $(TEST_SCRIPTS_DIR)/run_tests.sh \
- -j $(JOBS) \
- -t $(KUT_OUT)/tests/ \
- -x $(KUT_EXCLUDE) \
- -l $(KERNEL_IMAGE) \
- -r $(KUT_ROOTFS_IMAGE) \
- -e $(KUT_QEMU_BIN)
+# Generate a Makefile with targets per test and configuration.
+# Pass in variable/target names from this Makefile that the
+# generated Makefile will refer to.
+$(KUT_MAKEFILE): $(SELF) $(KUT_GEN_MAKEFILE) $(KUT_STAMP)
+ @ mkdir -p $(shell dirname $@)
+ @ $(KUT_GEN_MAKEFILE) \
+ $(KUT_TEST_DIR) "$(KUT_EXCLUDE)" \
+ KERNEL_IMAGE KUT_LOG_DIR test test-list \
+ > $@.tmp
+ @ mv $@.tmp $@
+include $(KUT_MAKEFILE)
diff --git a/aarch64-unit-tests/run_tests.sh b/aarch64-unit-tests/run_tests.sh
deleted file mode 100755
index c292d98..0000000
--- a/aarch64-unit-tests/run_tests.sh
+++ /dev/null
@@ -1,181 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2020 The Android KVM Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-exit_status=0
-script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-test_dir=""
-test_script="${script_dir}/../kvm-unit-tests/run_test.sh"
-log_dir=${LOG_DIR:-out/test/kvm-unit-tests}
-exclude_regex=""
-j=1
-linux=""
-rootfs=""
-verbose=${VERBOSE:-0}
-qemu=${QEMU:-qemu-system-aarch64}
-
-function get_outcome()
-{
- grep -E -h --color=never "[^ ](PASS|FAIL|SKIP)[^:]"
-}
-
-function usage()
-{
-cat <<EOF
-
-Usage: $0 [-h] [-t TEST-FILES] [-j NUM-TASKS] [-l KERNEL] [-e QEMU] [-r ROOT]
-
- -h Output this help text
- -t kvm-unit-tests standalone directory
- -x tests to exclude (regex), can be specified more than once
- -j Execute tests in parallel
- -l Linux kernel image to use
- -e QEMU emulator binary
- -r Root file system image to use
-
-EOF
-}
-
-function usage_abort()
-{
- usage
- exit 1
-}
-
-function wait_free_job_slots()
-{
- while (( $(jobs | wc -l) == j )); do
- # Wait for enough jobs to finish
- wait -n || exit_status=1
- done
-}
-
-function wait_background_jobs()
-{
- while (( $(jobs | wc -l) > 0 )); do
- # Wait for background jobs
- wait -n || exit_status=1
- done
-}
-
-function run_job()
-{
- wait_free_job_slots
-
- if [ "$j" = 1 ]; then
- bash -c "$@" || exit_status=1
- else
- bash -c "$@" &
- fi
-}
-
-while getopts ":t:x:j:l:r:e:hv" opt; do
- case "${opt}" in
- t)
- test_dir=${OPTARG}
- ;;
- x)
- # OR with the previous regex if specified
- if [ -n "${exclude_regex}" ]; then
- exclude_regex+="|"
- fi
- exclude_regex+="(^${OPTARG}$)"
- ;;
- j)
- j=${OPTARG}
- ;;
- l)
- linux=${OPTARG}
- ;;
- r)
- rootfs=${OPTARG}
- ;;
- e)
- qemu=${OPTARG}
- ;;
- h)
- usage
- exit 0
- ;;
- v)
- verbose=1
- ;;
- *)
- usage_abort
- ;;
- esac
-done
-if ((OPTIND == 1)); then
- usage_abort
-fi
-shift $((OPTIND-1))
-
-if [[ ! -d ${test_dir} ]]; then
- echo "No test directory specified."
- usage_abort
-fi
-
-if [[ ! -f ${linux} ]]; then
- echo "Linux kernel image file not found."
- usage_abort
-fi
-
-if [[ ! -f ${rootfs} ]]; then
- echo "Root filesystem image file not found."
- usage_abort
-fi
-
-if [[ ! -x ${qemu} ]]; then
- echo "QEMU executable not found."
- usage_abort
-fi
-
-rm -rf "$log_dir"/logs.old
-if [ -d "$log_dir"/logs ]; then
- mv "$log_dir"/logs "$log_dir"/logs.old
-fi
-mkdir -p "$log_dir"/logs/vhe || exit 2
-mkdir -p "$log_dir"/logs/nvhe || exit 2
-
-for test in "${test_dir}"/*; do
- test_name=$(basename "${test}")
-
- if [[ -n "${exclude_regex}" && "${test_name}" =~ ${exclude_regex} ]]; then
- if [ "$verbose" != "0" ]; then
- echo "Excluding ${test_name}"
- fi
- continue
- fi
-
- if [ "$verbose" != "0" ]; then
- echo "Running ${test_name}"
- fi
-
- # Redirect output instead of pipe to preserve exit status.
- run_job "${test_script} -V -k ${linux} \
- -o ${log_dir}/logs/vhe/${test_name}.log \
- -d \"${test_name} (vhe)\" ${test} \
- -- -e ${qemu} -r ${rootfs} \
- &> >(tail -1)"
- run_job "${test_script} -k ${linux} \
- -o ${log_dir}/logs/nvhe/${test_name}.log \
- -d \"${test_name} (nvhe)\" ${test} \
- -- -e ${qemu} -r ${rootfs} \
- &> >(tail -1)"
-done
-
-wait_background_jobs
-
-exit $exit_status
diff --git a/common.inc b/common.inc
index 29dfaa1..c2443bb 100644
--- a/common.inc
+++ b/common.inc
@@ -20,6 +20,7 @@
ROOT_DIR="$(dirname "${SCRIPT_DIR}")"
SCRIPT_RUN_QEMU="${SCRIPT_DIR}/aarch64/run_qemu.sh"
+SCRIPT_RUN_KUT="${SCRIPT_DIR}/kvm-unit-tests/run_test.sh"
PREBUILTS_DIR="${ROOT_DIR}/prebuilts"
PREBUILTS_IMG_DIR="${PREBUILTS_DIR}/linux-aarch64/images"
diff --git a/kvm-unit-tests/gen_makefile.sh b/kvm-unit-tests/gen_makefile.sh
new file mode 100755
index 0000000..aafc2db
--- /dev/null
+++ b/kvm-unit-tests/gen_makefile.sh
@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Android KVM Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source "$(dirname "${BASH_SOURCE[0]}")/../common.inc"
+
+# Directory containing kvm-unit-tests' binaries
+TEST_DIR="$1"
+
+# Regex for excluding known-to-fail or irrelevant tests.
+EXCLUDE="$2"
+
+# Variable/target names from the root Makefile.
+VAR_KERNEL_IMAGE="$3"
+VAR_LOG_DIR="$4"
+GLOBAL_TEST_TARGET="$5"
+GLOBAL_TEST_LIST_TARGET="$6"
+
+# Generates a Make target name for a test case.
+# Args:
+# 1) name of test case
+# 2) target name of the test's group.
+function target_name {
+ local target="$1"
+ local parent="$2"
+ echo "${parent}-${target}"
+}
+
+# Generates target name for a group of given parameters.
+# Args:
+# 1) Root target name
+# 2) is VHE (0/1)
+function group_target_name {
+ local group="$1"
+ local vhe="$2"
+ group="$(target_name kut "${group}")"
+ if [ "${vhe}" -eq 1 ]; then
+ group="$(target_name vhe "${group}")"
+ else
+ group="$(target_name nvhe "${group}")"
+ fi
+ echo "${group}"
+}
+
+# Generates target name for a target printing list of all test targets.
+# Args:
+# 1) Root target name
+function list_target_name {
+ local name="$1"
+ name="$(target_name kut "${name}")"
+ echo "${name}"
+}
+
+# Emits a Make target and a dependency of its parent on the target.
+# Args:
+# 1) Target name
+# 2) Parent target name
+function dependency_target {
+ local target="$1"
+ local parent="$2"
+ cat <<EOF
+.PHONY: ${target}
+${target}:
+${parent}: ${target}
+EOF
+}
+
+# Emits a Make target for a test case.
+# Args:
+# 1) Target name
+# 2) Test binary path
+# 3) is VHE (0/1)
+function test_target {
+ local target="$1"
+ local binary="$2"
+ local vhe="$3"
+
+ local extra_args=()
+ if [ "${vhe}" -eq 1 ]; then
+ extra_args+=(-V)
+ fi
+
+ cat <<EOF
+.PHONY: ${target}
+${target}: \$(${VAR_KERNEL_IMAGE}) ${binary}
+ @ mkdir -p "\$(${VAR_LOG_DIR})"
+ @ "${SCRIPT_RUN_KUT}" \
+ -k \$(${VAR_KERNEL_IMAGE}) \
+ -d "${target}" \
+ -o \$(${VAR_LOG_DIR})/${target}.log \
+ ${extra_args[@]} \
+ "${binary}"
+EOF
+}
+
+# Emits a Make target which prints all provided target names,
+# one per line.
+# Args:
+# 1) target name
+# ...) list of target names to print
+function test_list_target {
+ local target="$1"
+ shift 1
+ cat <<EOF
+.PHONY: ${target}
+${target}:
+ @ for T in $@; do echo "\$\$T"; done
+EOF
+}
+
+if [ ! -d "${TEST_DIR}" ]; then
+ echo "ERROR: Test directory does not exist" 1>&2
+ exit 1
+fi
+
+TARGET_LIST=()
+for VHE in 0 1; do
+ # Emit a target for this test group.
+ ROOT="${GLOBAL_TEST_TARGET}"
+ GROUP="$(group_target_name "${ROOT}" "${VHE}")"
+ dependency_target "${GROUP}" "${ROOT}"
+
+ for TEST_PATH in "${TEST_DIR}"/*; do
+ NAME=$(basename "${TEST_PATH}")
+ if [[ ${NAME} =~ ${EXCLUDE} ]]; then
+ continue
+ fi
+
+ # Emit a target for this test case.
+ TARGET="$(target_name ${NAME} ${GROUP})"
+ test_target "${TARGET}" "${TEST_PATH}" "${VHE}"
+ dependency_target "${TARGET}" "${GROUP}"
+ TARGET_LIST+=("${TARGET}")
+ done
+done
+
+# Emit a target which prints all test targets from the generated Makefile.
+ROOT="${GLOBAL_TEST_LIST_TARGET}"
+TARGET="$(list_target_name "${ROOT}")"
+test_list_target "${TARGET}" "${TARGET_LIST[@]}"
+dependency_target "${TARGET}" "${ROOT}"
