From 787302bde4d89279180303b753eb73b9bc6820b9 Mon Sep 17 00:00:00 2001 From: "Simon J. Gerraty" Date: Tue, 5 Aug 2025 15:31:51 -0700 Subject: [PATCH] Import bmake-20250804 Intersting/relevant changes since bmake-20250707 ChangeLog since bmake-20250707 2025-08-04 Simon J Gerraty * VERSION (_MAKE_VERSION): 20250804 Merge with NetBSD make, pick up o meta.c: allow printing only partial string to meta file in jobs mode, otherwise we end up with duplicated output when buffer does not end in newline. Add a suitable unit test. mk/ChangeLog since bmake-20250707 2025-07-24 Simon J Gerraty * install-mk (MK_VERSION): 20250724 * meta2deps: Allow X record to have 3 or 4 args. V4 filemon on Linux produces 3 V5 filemon on FreeBSD produces 4 2025-07-22 Simon J Gerraty * install-mk (MK_VERSION): 20250721 * meta2deps.{py,sh}: detect corrupted filemon output (an issue on Linux) by checking each record type has the correct number of words. Throw an error if necessary so that gendirdeps.mk will not update Makefile.depend --- ChangeLog | 9 +++++++++ FILES | 2 ++ VERSION | 2 +- job.c | 6 +++--- meta.c | 8 ++++---- meta.h | 4 ++-- mk/ChangeLog | 17 +++++++++++++++++ mk/install-mk | 4 ++-- mk/meta2deps.py | 28 +++++++++++++++++++++++----- mk/meta2deps.sh | 14 ++++++++++++-- unit-tests/Makefile | 7 +++++-- unit-tests/meta-output.exp | 11 +++++++++++ unit-tests/meta-output.mk | 30 ++++++++++++++++++++++++++++++ 13 files changed, 121 insertions(+), 21 deletions(-) mode change 100644 => 100755 mk/install-mk create mode 100644 unit-tests/meta-output.exp create mode 100644 unit-tests/meta-output.mk diff --git a/ChangeLog b/ChangeLog index 5a1c30a9575..e8f1567733e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +2025-08-04 Simon J Gerraty + + * VERSION (_MAKE_VERSION): 20250804 + Merge with NetBSD make, pick up + o meta.c: allow printing only partial string to meta file + in jobs mode, otherwise we end up with duplicated output when + buffer does not end in newline. + Add a suitable unit test. + 2025-07-07 Simon J Gerraty * VERSION (_MAKE_VERSION): 20250707 diff --git a/FILES b/FILES index 1cec16b73ef..28465b6a8c3 100644 --- a/FILES +++ b/FILES @@ -431,6 +431,8 @@ unit-tests/make-exported.exp unit-tests/make-exported.mk unit-tests/meta-cmd-cmp.exp unit-tests/meta-cmd-cmp.mk +unit-tests/meta-output.exp +unit-tests/meta-output.mk unit-tests/meta-ignore.inc unit-tests/moderrs.exp unit-tests/moderrs.mk diff --git a/VERSION b/VERSION index eef1ef4b8ba..e301cfafab7 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ # keep this compatible with sh and make -_MAKE_VERSION=20250707 +_MAKE_VERSION=20250804 diff --git a/job.c b/job.c index f15dcf21ef6..59b27b93041 100644 --- a/job.c +++ b/job.c @@ -1,4 +1,4 @@ -/* $NetBSD: job.c,v 1.517 2025/07/06 07:11:31 rillig Exp $ */ +/* $NetBSD: job.c,v 1.519 2025/08/04 15:40:39 sjg Exp $ */ /* * Copyright (c) 1988, 1989, 1990 The Regents of the University of California. @@ -137,7 +137,7 @@ #include "trace.h" /* "@(#)job.c 8.2 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: job.c,v 1.517 2025/07/06 07:11:31 rillig Exp $"); +MAKE_RCSID("$NetBSD: job.c,v 1.519 2025/08/04 15:40:39 sjg Exp $"); #ifdef USE_SELECT @@ -1867,7 +1867,7 @@ CollectOutput(Job *job, bool finish) SwitchOutputTo(job->node); #ifdef USE_META if (useMeta) - meta_job_output(job, p); + meta_job_output(job, p, (i < max) ? i : max); #endif (void)fwrite(p, 1, (size_t)(job->outBuf + i - p), stdout); (void)fflush(stdout); diff --git a/meta.c b/meta.c index 5d8d17f1970..f87f51c3b0a 100644 --- a/meta.c +++ b/meta.c @@ -1,4 +1,4 @@ -/* $NetBSD: meta.c,v 1.215 2025/06/13 06:13:19 rillig Exp $ */ +/* $NetBSD: meta.c,v 1.219 2025/08/04 18:57:20 rillig Exp $ */ /* * Implement 'meta' mode. @@ -767,7 +767,7 @@ meta_job_error(Job *job, GNode *gn, bool ignerr, int status) } void -meta_job_output(Job *job, const char *cp) +meta_job_output(Job *job, const char *cp, size_t len) { BuildMon *pbm; @@ -790,7 +790,7 @@ meta_job_output(Job *job, const char *cp) cp++; } } - fprintf(pbm->mfp, "%s", cp); + fprintf(pbm->mfp, "%.*s", (int)len, cp); } } @@ -1703,7 +1703,7 @@ meta_compat_parent(pid_t child) fwrite(buf, 1, (size_t)nread, stdout); fflush(stdout); buf[nread] = '\0'; - meta_job_output(NULL, buf); + meta_job_output(NULL, buf, (size_t)nread); } while (false); if (metafd != -1 && FD_ISSET(metafd, &readfds) != 0) { if (meta_job_event(NULL) <= 0) diff --git a/meta.h b/meta.h index a7478cbc3e5..4796cdeb3c0 100644 --- a/meta.h +++ b/meta.h @@ -1,4 +1,4 @@ -/* $NetBSD: meta.h,v 1.13 2025/06/13 06:13:19 rillig Exp $ */ +/* $NetBSD: meta.h,v 1.15 2025/08/04 03:47:26 sjg Exp $ */ /* * Things needed for 'meta' mode. @@ -49,7 +49,7 @@ void meta_job_parent(struct Job *, pid_t); int meta_job_fd(struct Job *) MAKE_ATTR_USE; int meta_job_event(struct Job *) MAKE_ATTR_USE; void meta_job_error(struct Job *, GNode *, bool, int); -void meta_job_output(struct Job *, const char *); +void meta_job_output(struct Job *, const char *, size_t); int meta_cmd_finish(void *); int meta_job_finish(struct Job *); bool meta_oodate(GNode *, bool) MAKE_ATTR_USE; diff --git a/mk/ChangeLog b/mk/ChangeLog index db524d2343b..fda6d8b0215 100644 --- a/mk/ChangeLog +++ b/mk/ChangeLog @@ -1,3 +1,20 @@ +2025-07-24 Simon J Gerraty + + * install-mk (MK_VERSION): 20250724 + + * meta2deps: Allow X record to have 3 or 4 args. + V4 filemon on Linux produces 3 + V5 filemon on FreeBSD produces 4 + +2025-07-22 Simon J Gerraty + + * install-mk (MK_VERSION): 20250721 + + * meta2deps.{py,sh}: detect corrupted filemon output (an issue on + Linux) by checking each record type has the correct number of + words. Throw an error if necessary so that gendirdeps.mk will not + update Makefile.depend + 2025-07-04 Simon J Gerraty * prog.mk: .MADE is a special source not a target! diff --git a/mk/install-mk b/mk/install-mk old mode 100644 new mode 100755 index 8d354de17cb..3ed5fd63ee5 --- a/mk/install-mk +++ b/mk/install-mk @@ -59,7 +59,7 @@ # Simon J. Gerraty # RCSid: -# $Id: install-mk,v 1.266 2025/05/29 01:48:06 sjg Exp $ +# $Id: install-mk,v 1.268 2025/07/24 15:55:48 sjg Exp $ # # @(#) Copyright (c) 1994-2025 Simon J. Gerraty # @@ -74,7 +74,7 @@ # sjg@crufty.net # -MK_VERSION=20250528 +MK_VERSION=20250724 OWNER= GROUP= MODE=444 diff --git a/mk/meta2deps.py b/mk/meta2deps.py index 70b12100398..77ed86397a0 100755 --- a/mk/meta2deps.py +++ b/mk/meta2deps.py @@ -39,7 +39,7 @@ SPDX-License-Identifier: BSD-2-Clause RCSid: - $Id: meta2deps.py,v 1.51 2025/05/16 20:03:43 sjg Exp $ + $Id: meta2deps.py,v 1.54 2025/07/24 16:05:48 sjg Exp $ Copyright (c) 2011-2025, Simon J. Gerraty Copyright (c) 2011-2017, Juniper Networks, Inc. @@ -441,7 +441,7 @@ def parse(self, name=None, file=None): # Bye bye We go to some effort to avoid processing a dependency more than once. - Of the above record types only C,E,F,L,R,V and W are of interest. + Of the above record types only C,E,F,L,M,R,V,W and X are of interest. """ version = 0 # unknown @@ -465,8 +465,8 @@ def parse(self, name=None, file=None): if self.sb and self.name.startswith(self.sb): error_name = self.name.replace(self.sb+'/','') else: - error_name = self.name - interesting = '#CEFLRVX' + error_name = self.name + interesting = '#CEFLMRVX' for line in f: self.line += 1 # ignore anything we don't care about @@ -475,6 +475,7 @@ def parse(self, name=None, file=None): if self.debug > 2: print("input:", line, end=' ', file=self.debug_out) w = line.split() + wlen = len(w) if skip: if w[0] == 'V': @@ -498,6 +499,23 @@ def parse(self, name=None, file=None): if line.find('Bye') > 0: eof_token = True continue + else: + # before we go further check we have a sane number of args + # the Linux filemon module is rather unreliable. + if w[0] in 'LM': + elen = 4 + elif w[0] == 'X': + # at least V4 on Linux does 3 args + if wlen == 3: + elen = 3 + else: + elen = 4 + else: + elen = 3 + if self.debug > 2: + print('op={} elen={} wlen={} line="{}"'.format(w[0], elen, wlen, line.strip()), file=self.debug_out) + if wlen != elen: + raise AssertionError('corrupted filemon data: wrong number of words: expected {} got {} in: {}'.format(elen, wlen, line)) pid = int(w[1]) if pid != last_pid: @@ -540,7 +558,7 @@ def parse(self, name=None, file=None): print("seen:", w[2], file=self.debug_out) continue # file operations - if w[0] in 'ML': + if w[0] in 'LM': # these are special, tread src as read and # target as write self.parse_path(w[3].strip("'"), cwd, 'W', w) diff --git a/mk/meta2deps.sh b/mk/meta2deps.sh index 4af15971b84..83478ae5b87 100755 --- a/mk/meta2deps.sh +++ b/mk/meta2deps.sh @@ -77,7 +77,7 @@ # RCSid: -# $Id: meta2deps.sh,v 1.22 2025/05/16 20:03:43 sjg Exp $ +# $Id: meta2deps.sh,v 1.24 2025/07/24 15:55:48 sjg Exp $ # SPDX-License-Identifier: BSD-2-Clause # @@ -251,11 +251,21 @@ meta2deps() { ;; *) cat /dev/null "$@";; esac 2> /dev/null | - sed -e 's,^CWD,C C,;/^[#CREFLMVX] /!d' -e "s,',,g" | + sed -e 's,^CWD,C C,;/^[#CREFLMVWX] /!d' -e "s,',,g" | $_excludes | ( version=no epids= xpids= eof_token=no while read op pid path path2 do : op=$op pid=$pid path=$path path2=$path2 + # first a sanity check - filemon on Linux is not very reliable + # path2 should only be non-empty for op L or M + # and it should not contain spaces. + case "$op,$path2" in + \#*) ;; # ok + [LM],) error "missing path2 in: '$op $pid $path'";; + [LMX],*" "*) error "wrong number of words in: '$op $pid $path $path2'";; + *,|[LMX],*) ;; # ok + *) error "wrong number of words in: '$op $pid $path $path2'";; + esac # we track cwd and ldir (of interest) per pid # CWD is bmake's cwd case "$lpid,$pid" in diff --git a/unit-tests/Makefile b/unit-tests/Makefile index 4e639056815..f6c298679a6 100644 --- a/unit-tests/Makefile +++ b/unit-tests/Makefile @@ -1,6 +1,6 @@ -# $Id: Makefile,v 1.240 2025/06/30 18:40:54 sjg Exp $ +# $Id: Makefile,v 1.245 2025/08/05 16:18:07 sjg Exp $ # -# $NetBSD: Makefile,v 1.369 2025/06/29 09:40:13 rillig Exp $ +# $NetBSD: Makefile,v 1.372 2025/08/04 22:44:49 sjg Exp $ # # Unit tests for make(1) # @@ -234,6 +234,7 @@ TESTS+= jobs-error-nested-make TESTS+= lint TESTS+= make-exported TESTS+= meta-cmd-cmp +TESTS+= meta-output TESTS+= moderrs TESTS+= modmisc .if ${.MAKE.UID} > 0 @@ -592,6 +593,7 @@ SED_CMDS.directive-include-guard= \ -e '/^ParseDependency/d' \ -e '/^ParseEOF:/d' SED_CMDS.export= -e '/^[^=_A-Za-z0-9]*=/d' +SED_CMDS.export+= -e '/^DIFF/d' .if ${.MAKE.OS:NCygwin} == "" SED_CMDS.export+= -e '/^WINDIR=/d' -e '/^SYSTEMROOT=/d' .endif @@ -802,6 +804,7 @@ EGREP= grep -E EGREP?= egrep MAKE_TEST_ENV= EGREP="${EGREP}" +MAKE_TEST_ENV+= DIFF="${TOOL_DIFF}" DIFF_FLAGS="${DIFF_FLAGS}" MAKE_TEST_ENV+= MALLOC_OPTIONS="JA" # for jemalloc 100 MAKE_TEST_ENV+= MALLOC_CONF="junk:true" # for jemalloc 510 MAKE_TEST_ENV+= TMPDIR=${TMPDIR} diff --git a/unit-tests/meta-output.exp b/unit-tests/meta-output.exp new file mode 100644 index 00000000000..37ef54b4e49 --- /dev/null +++ b/unit-tests/meta-output.exp @@ -0,0 +1,11 @@ +Test -B output +test1: Done +test2: Done +test3: Done + +Test -j1 output +test1: Done +test2: Done +test3: Done + +exit status 0 diff --git a/unit-tests/meta-output.mk b/unit-tests/meta-output.mk new file mode 100644 index 00000000000..104091df5d0 --- /dev/null +++ b/unit-tests/meta-output.mk @@ -0,0 +1,30 @@ +# + +.MAIN: all + +.if make(output) +.MAKE.MODE= meta curDirOk=true nofilemon +.else +.MAKE.MODE= compat +.endif + +all: output.-B output.-j1 + +_mf := ${.PARSEDIR}/${.PARSEFILE} + +# This output should be accurately reflected in the .meta file. +# We append an extra newline to ${.TARGET} (after it has been +# written to stdout) to match what meta_cmd_finish() will do. +output: .NOPATH + @{ echo Test ${tag} output; \ + for i in 1 2 3; do \ + printf "test$$i: "; sleep 0; echo " Done"; \ + done; echo; } | tee ${.TARGET}; echo >> ${.TARGET} + +# The diff at the end should produce nothing. +output.-B output.-j1: + @{ rm -f ${TMPDIR}/output; mkdir -p ${TMPDIR}/obj; \ + MAKEFLAGS= ${.MAKE} -r -C ${TMPDIR} ${.TARGET:E} tag=${.TARGET:E} -f ${_mf} output; \ + sed '1,/command output/d' ${TMPDIR}/obj/output.meta > ${TMPDIR}/obj/output-meta; \ + ${DIFF:Udiff} ${TMPDIR}/obj/output ${TMPDIR}/obj/output-meta; } +