Add host.test.mk for DIRDEPS_BUILD

If we are doing DIRDEPS_BUILD and MK_RUN_TESTS is yes
we can hook tests associated with dirs to be built into the graph
such that we avoid circular dependencies but the build will fail
should any test fail.

Make MK_RUN_TESTS independent of MK_TESTS
also for non-DIRDEPS_BUILD ensure we have RELDIR and check it
against patterns in HOST_RUN_TESTS.

Reviewed by:	stevek
Sponsored by:	Juniper Networks, Inc.
Differential Revision:	https://reviews.freebsd.org/D50387

share/mk/Makefile

@@ -62,6 +62,7 @@ FILES=	\
 	dirdeps-targets.mk \
 	gendirdeps.mk \
 	host-target.mk \
+	host.test.mk \
 	install-new.mk \
 	jobs.mk \
 	meta.autodep.mk \

share/mk/bsd.opts.mk

@@ -79,6 +79,7 @@ __DEFAULT_NO_OPTIONS = \
     CTF \
     INSTALL_AS_USER \
     RETPOLINE \
+    RUN_TESTS \
     STALE_STAGED \
     UBSAN \
     UNDEFINED_VERSION \

share/mk/bsd.test.mk

@@ -91,6 +91,11 @@ MAN=
 .include <suite.test.mk>
 .endif
 
+.if ${MK_RUN_TESTS} == "yes"
+# Run tests as part of the build
+.include <host.test.mk>
+.endif
+
 .if !target(realcheck)
 realcheck: .PHONY
 	@echo "$@ not defined; skipping"

share/mk/host.test.mk

@@ -0,0 +1,89 @@
+#
+# If MK_RUN_TESTS is "yes"; we want to run tests for "host"
+# during the build - and fail the build if any test fails.
+#
+# HOST_RUN_TESTS can be used to filter which tests should be built and
+# run.  It should be a list of glob patterns to apply to RELDIR,
+# so "*" would mean all.
+# 
+# For the DIRDEPS_BUILD, HOST_RUN_TESTS defaults to "*" as the
+# selection of tests to build and run is taken controlled at level 0
+# in local.dirdeps.mk
+#
+
+.if !target(__<bsd.test.mk>__)
+.error ${.PARSEFILE} cannot be included directly.
+.endif
+
+all:
+
+.if ${.MAKE.LEVEL} > 0 && !empty(_TESTS) && ${MACHINE:Nhost*} == ""
+
+# allow for customization
+.-include <local.host.test.mk>
+
+.if ${MK_DIRDEPS_BUILD} == "yes"
+# orchestration choices are already made 
+HOST_RUN_TESTS ?= *
+.else
+.if empty(RELDIR)
+RELDIR:= ${.CURDIR:S,${SRCTOP}/,,:S,${OBJTOP}/,,}
+.endif
+.endif
+
+.if ${HOST_RUN_TESTS:Uno:@x@${RELDIR:M$x}@} != ""
+all: run-tests
+.endif
+
+KYUA?= kyua
+
+# we need to make sure kyua-report can find the results
+KYUA_RESULTS?= ${.OBJDIR}/kyua.results
+KYUA_ARGS?= --results-file=${KYUA_RESULTS}
+KYUA_ENV?= HOME=${KYUA_HOME} TMPDIR=${.OBJDIR}
+KYUA_FLAGS?= --config none --loglevel=${KYUA_LOGLEVEL:Uinfo}
+KYUA_HOME?= ${OBJTOP}
+
+.if make(debug*)
+KYUA_LOGLEVEL?= debug
+.endif
+
+# some tests have files they need
+.if ${${PACKAGE}FILES:U:NKyuafile} != ""
+run-tests run-tests.log: link-test-files
+link-test-files: ${${PACKAGE}FILES:NKyuafile}
+	@for f in ${.ALLSRC:N*Kyuafile:M*/*}; do \
+	    ln -sf $$f .; \
+	done
+.endif
+
+# we do not want to stage any of this
+RUN_TESTS_LOG= run-tests.log
+MK_STAGING= no
+META_XTRAS+= ${RUN_TESTS_LOG}
+
+run-tests: ${RUN_TESTS_LOG}
+
+# This is the main event.
+# Run kyua-test followed by kyua-report.
+# If we have any test failues we want to run kyua-report --verbose
+# Also on fail, we rename run-tests.log to run-tests.err so we save the
+# output but the target will be out-of-date.
+# We prepend ${.OBJDIR}:${.OBJDIR:H}: to PATH seen by kyua
+# so tests for things like cat, cp, cmp etc can find the one we just built
+# rather than the one from the host.
+${RUN_TESTS_LOG}: ${_TESTS} Kyuafile
+	@( export PATH=${.OBJDIR}:${.OBJDIR:H}:${PATH}; \
+	rm -f ${KYUA_RESULTS}; \
+	${KYUA_ENV} ${KYUA} ${KYUA_FLAGS} test ${KYUA_ARGS} -k ${.OBJDIR}/Kyuafile --build-root=${.OBJDIR} && \
+	${KYUA_ENV} ${KYUA} ${KYUA_FLAGS} report ${KYUA_ARGS} ) > ${.TARGET} || \
+	{ mv ${.TARGET} ${.TARGET:R}.err; \
+	${KYUA_ENV} ${KYUA} ${KYUA_FLAGS} report ${KYUA_ARGS} --verbose --results-filter broken,failed; echo See ${.TARGET:R:tA}.err; \
+	exit 1; }
+
+# make kyua-debug KYUA_DEBUG_ARGS=app:test
+kyua-debug:
+	@(export PATH=${.OBJDIR}:${.OBJDIR:H}:${PATH}; \
+	${KYUA_ENV} ${KYUA} ${KYUA_FLAGS} debug ${KYUA_DEBUG_ARGS}) || true
+
+.endif

share/mk/local.dirdeps.mk

@@ -250,3 +250,59 @@ BOOT_MACHINE_DIR:= ${BOOT_MACHINE_DIR.${DEP_MACHINE}}
 KERNEL_NAME:= ${KERNEL_NAME.${DEP_MACHINE}}
 
 .-include <site.dirdeps.mk>
+
+.if ${MK_RUN_TESTS} == "yes"
+# some the tests below will not run correctly on the host
+# as they require support files not present in .CURDIR
+BROKEN_HOST_TESTS += \
+	${BROKEN_HOST_DEP_TESTS} \
+	bin/cp/tests \
+	lib/atf/libatf-c++/tests \
+	lib/atf/libatf-c/tests \
+	lib/libarchive/tests \
+	lib/libnv/tests \
+	libexec/rtld-elf/tests \
+	sbin/devd/tests \
+	usr.bin/comm/tests \
+	usr.bin/du/tests \
+	usr.bin/procstat/tests \
+	usr.bin/sed/tests \
+	usr.bin/tar/tests \
+	usr.bin/tr/tests \
+	usr.bin/xargs/tests \
+	usr.bin/yacc/tests \
+	usr.sbin/makefs/tests \
+	usr.sbin/rpcbind/tests \
+
+# these all have broken dependencies which we choose not to fix for host
+BROKEN_HOST_DEP_TESTS += \
+	lib/msun/tests \
+	usr.bin/cut/tests \
+	usr.bin/diff/tests \
+	usr.bin/grep/tests \
+	usr.bin/gzip/tests \
+	usr.bin/printf/tests \
+
+.endif
+
+TESTS_DIR_LIST += tests
+# most of the tree only has Makefile.depend
+# by default we only want to run host tests for Makefile.depend.host
+TESTS_DEPENDFILE_PREFERENCE ?= Makefile.depend.host
+EXCLUDE_TESTS_FILTER += . \
+	lib/libcasper/services/cap_* \
+
+.for t in ${TESTS_DIR_LIST:O:u}
+.if !target(${DEP_RELDIR}/$t.test)
+.if ${BROKEN_HOST_TESTS:U:M${DEP_RELDIR}/$t} == "" && \
+	${RUN_TESTS_FILTER:U*:@m@${DEP_RELDIR:M$m}@} != "" && \
+	${DEP_RELDIR:${EXCLUDE_TESTS_FILTER:Uno:${M_ListToSkip}}} != "" && \
+	${TESTS_DEPENDFILE_PREFERENCE:@m@${exists(${SRCTOP}/${DEP_RELDIR}/$t/$m):?$m:}@} != ""
+# build but not via DIRDEPS to avoid circular dependencies
+_build_xtra_dirs += ${SRCTOP}/${DEP_RELDIR}/$t.host
+# mark it done
+${DEP_RELDIR}/$t.test:
+.endif
+.endif
+.endfor
+.endif