Import bmake-20250804

Intersting/relevant changes since bmake-20250707

ChangeLog since bmake-20250707

2025-08-04  Simon J Gerraty  <sjg@beast.crufty.net>

	* VERSION (_MAKE_VERSION): 20250804
	Merge with NetBSD make, pick up
	o meta.c: allow printing only partial string to meta file
	in jobs mode, otherwise we end up with duplicated output when
	buffer does not end in newline.
	Add a suitable unit test.

mk/ChangeLog since bmake-20250707

2025-07-24  Simon J Gerraty  <sjg@beast.crufty.net>

	* install-mk (MK_VERSION): 20250724

	* meta2deps: Allow X record to have 3 or 4 args.
	V4 filemon on Linux produces 3
	V5 filemon on FreeBSD produces 4

2025-07-22  Simon J Gerraty  <sjg@beast.crufty.net>

	* install-mk (MK_VERSION): 20250721

	* meta2deps.{py,sh}: detect corrupted filemon output (an issue on
	Linux) by checking each record type has the correct number of
	words.  Throw an error if necessary so that gendirdeps.mk will not
	update Makefile.depend

ChangeLog

@@ -1,3 +1,12 @@
+2025-08-04  Simon J Gerraty  <sjg@beast.crufty.net>
+
+	* VERSION (_MAKE_VERSION): 20250804
+	Merge with NetBSD make, pick up
+	o meta.c: allow printing only partial string to meta file
+	in jobs mode, otherwise we end up with duplicated output when
+	buffer does not end in newline.
+	Add a suitable unit test.
+
 2025-07-07  Simon J Gerraty  <sjg@beast.crufty.net>
 
 	* VERSION (_MAKE_VERSION): 20250707

FILES

@@ -431,6 +431,8 @@ unit-tests/make-exported.exp
 unit-tests/make-exported.mk
 unit-tests/meta-cmd-cmp.exp
 unit-tests/meta-cmd-cmp.mk
+unit-tests/meta-output.exp
+unit-tests/meta-output.mk
 unit-tests/meta-ignore.inc
 unit-tests/moderrs.exp
 unit-tests/moderrs.mk

VERSION

@@ -1,2 +1,2 @@
 # keep this compatible with sh and make
-_MAKE_VERSION=20250707
+_MAKE_VERSION=20250804

job.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: job.c,v 1.517 2025/07/06 07:11:31 rillig Exp $	*/
+/*	$NetBSD: job.c,v 1.519 2025/08/04 15:40:39 sjg Exp $	*/
 
 /*
  * Copyright (c) 1988, 1989, 1990 The Regents of the University of California.
@@ -137,7 +137,7 @@
 #include "trace.h"
 
 /*	"@(#)job.c	8.2 (Berkeley) 3/19/94"	*/
-MAKE_RCSID("$NetBSD: job.c,v 1.517 2025/07/06 07:11:31 rillig Exp $");
+MAKE_RCSID("$NetBSD: job.c,v 1.519 2025/08/04 15:40:39 sjg Exp $");
 
 
 #ifdef USE_SELECT
@@ -1867,7 +1867,7 @@ CollectOutput(Job *job, bool finish)
 			SwitchOutputTo(job->node);
 #ifdef USE_META
 		if (useMeta)
-			meta_job_output(job, p);
+			meta_job_output(job, p, (i < max) ? i : max);
 #endif
 		(void)fwrite(p, 1, (size_t)(job->outBuf + i - p), stdout);
 		(void)fflush(stdout);

meta.c

@@ -1,4 +1,4 @@
-/*      $NetBSD: meta.c,v 1.215 2025/06/13 06:13:19 rillig Exp $ */
+/*      $NetBSD: meta.c,v 1.219 2025/08/04 18:57:20 rillig Exp $ */
 
 /*
  * Implement 'meta' mode.
@@ -767,7 +767,7 @@ meta_job_error(Job *job, GNode *gn, bool ignerr, int status)
 }
 
 void
-meta_job_output(Job *job, const char *cp)
+meta_job_output(Job *job, const char *cp, size_t len)
 {
     BuildMon *pbm;
 
@@ -790,7 +790,7 @@ meta_job_output(Job *job, const char *cp)
 		cp++;
 	    }
 	}
-	fprintf(pbm->mfp, "%s", cp);
+	fprintf(pbm->mfp, "%.*s", (int)len, cp);
     }
 }
 
@@ -1703,7 +1703,7 @@ meta_compat_parent(pid_t child)
 	    fwrite(buf, 1, (size_t)nread, stdout);
 	    fflush(stdout);
 	    buf[nread] = '\0';
-	    meta_job_output(NULL, buf);
+	    meta_job_output(NULL, buf, (size_t)nread);
 	} while (false);
 	if (metafd != -1 && FD_ISSET(metafd, &readfds) != 0) {
 	    if (meta_job_event(NULL) <= 0)

meta.h

@@ -1,4 +1,4 @@
-/*      $NetBSD: meta.h,v 1.13 2025/06/13 06:13:19 rillig Exp $ */
+/*      $NetBSD: meta.h,v 1.15 2025/08/04 03:47:26 sjg Exp $ */
 
 /*
  * Things needed for 'meta' mode.
@@ -49,7 +49,7 @@ void meta_job_parent(struct Job *, pid_t);
 int  meta_job_fd(struct Job *) MAKE_ATTR_USE;
 int  meta_job_event(struct Job *) MAKE_ATTR_USE;
 void meta_job_error(struct Job *, GNode *, bool, int);
-void meta_job_output(struct Job *, const char *);
+void meta_job_output(struct Job *, const char *, size_t);
 int  meta_cmd_finish(void *);
 int  meta_job_finish(struct Job *);
 bool meta_oodate(GNode *, bool) MAKE_ATTR_USE;

mk/ChangeLog

@@ -1,3 +1,20 @@
+2025-07-24  Simon J Gerraty  <sjg@beast.crufty.net>
+
+	* install-mk (MK_VERSION): 20250724
+
+	* meta2deps: Allow X record to have 3 or 4 args.
+	V4 filemon on Linux produces 3
+	V5 filemon on FreeBSD produces 4
+
+2025-07-22  Simon J Gerraty  <sjg@beast.crufty.net>
+
+	* install-mk (MK_VERSION): 20250721
+
+	* meta2deps.{py,sh}: detect corrupted filemon output (an issue on
+	Linux) by checking each record type has the correct number of
+	words.  Throw an error if necessary so that gendirdeps.mk will not
+	update Makefile.depend
+
 2025-07-04  Simon J Gerraty  <sjg@beast.crufty.net>
 
 	* prog.mk: .MADE is a special source not a target!

mk/install-mk

@@ -59,7 +59,7 @@
 #       Simon J. Gerraty <sjg@crufty.net>
 
 # RCSid:
-#	$Id: install-mk,v 1.266 2025/05/29 01:48:06 sjg Exp $
+#	$Id: install-mk,v 1.268 2025/07/24 15:55:48 sjg Exp $
 #
 #	@(#) Copyright (c) 1994-2025 Simon J. Gerraty
 #
@@ -74,7 +74,7 @@
 #	sjg@crufty.net
 #
 
-MK_VERSION=20250528
+MK_VERSION=20250724
 OWNER=
 GROUP=
 MODE=444

mk/meta2deps.py

@@ -39,7 +39,7 @@
 SPDX-License-Identifier: BSD-2-Clause
 
 RCSid:
-	$Id: meta2deps.py,v 1.51 2025/05/16 20:03:43 sjg Exp $
+	$Id: meta2deps.py,v 1.54 2025/07/24 16:05:48 sjg Exp $
 
 	Copyright (c) 2011-2025, Simon J. Gerraty
 	Copyright (c) 2011-2017, Juniper Networks, Inc.
@@ -441,7 +441,7 @@ def parse(self, name=None, file=None):
         # Bye bye
 
         We go to some effort to avoid processing a dependency more than once.
-        Of the above record types only C,E,F,L,R,V and W are of interest.
+        Of the above record types only C,E,F,L,M,R,V,W and X are of interest.
         """
 
         version = 0                     # unknown
@@ -465,8 +465,8 @@ def parse(self, name=None, file=None):
         if self.sb and self.name.startswith(self.sb):
             error_name = self.name.replace(self.sb+'/','')
         else:
-            error_name = self.name 
-        interesting = '#CEFLRVX'
+            error_name = self.name
+        interesting = '#CEFLMRVX'
         for line in f:
             self.line += 1
             # ignore anything we don't care about
@@ -475,6 +475,7 @@ def parse(self, name=None, file=None):
             if self.debug > 2:
                 print("input:", line, end=' ', file=self.debug_out)
             w = line.split()
+            wlen = len(w)
 
             if skip:
                 if w[0] == 'V':
@@ -498,6 +499,23 @@ def parse(self, name=None, file=None):
                 if line.find('Bye') > 0:
                     eof_token = True
                 continue
+            else:
+                # before we go further check we have a sane number of args
+                # the Linux filemon module is rather unreliable.
+                if w[0] in 'LM':
+                    elen = 4
+                elif w[0] == 'X':
+                    # at least V4 on Linux does 3 args
+                    if wlen == 3:
+                        elen = 3
+                    else:
+                        elen = 4
+                else:
+                    elen = 3
+                if self.debug > 2:
+                    print('op={} elen={} wlen={} line="{}"'.format(w[0], elen, wlen, line.strip()), file=self.debug_out)
+                if wlen != elen:
+                    raise AssertionError('corrupted filemon data: wrong number of words: expected {} got {} in: {}'.format(elen, wlen, line))
 
             pid = int(w[1])
             if pid != last_pid:
@@ -540,7 +558,7 @@ def parse(self, name=None, file=None):
                     print("seen:", w[2], file=self.debug_out)
                 continue
             # file operations
-            if w[0] in 'ML':
+            if w[0] in 'LM':
                 # these are special, tread src as read and
                 # target as write
                 self.parse_path(w[3].strip("'"), cwd, 'W', w)

mk/meta2deps.sh

@@ -77,7 +77,7 @@
 
 
 # RCSid:
-#	$Id: meta2deps.sh,v 1.22 2025/05/16 20:03:43 sjg Exp $
+#	$Id: meta2deps.sh,v 1.24 2025/07/24 15:55:48 sjg Exp $
 
 # SPDX-License-Identifier: BSD-2-Clause
 #
@@ -251,11 +251,21 @@ meta2deps() {
 	;;
     *) cat /dev/null "$@";;
     esac 2> /dev/null |
-    sed -e 's,^CWD,C C,;/^[#CREFLMVX] /!d' -e "s,',,g" |
+    sed -e 's,^CWD,C C,;/^[#CREFLMVWX] /!d' -e "s,',,g" |
     $_excludes | ( version=no epids= xpids= eof_token=no
     while read op pid path path2
     do
 	: op=$op pid=$pid path=$path path2=$path2
+	# first a sanity check - filemon on Linux is not very reliable
+	# path2 should only be non-empty for op L or M
+	# and it should not contain spaces.
+	case "$op,$path2" in
+	\#*) ;;			# ok
+	[LM],) error "missing path2 in: '$op $pid $path'";;
+	[LMX],*" "*) error "wrong number of words in: '$op $pid $path $path2'";;
+	*,|[LMX],*) ;;		# ok
+	*) error "wrong number of words in: '$op $pid $path $path2'";;
+	esac
 	# we track cwd and ldir (of interest) per pid
 	# CWD is bmake's cwd
 	case "$lpid,$pid" in

unit-tests/Makefile

@@ -1,6 +1,6 @@
-# $Id: Makefile,v 1.240 2025/06/30 18:40:54 sjg Exp $
+# $Id: Makefile,v 1.245 2025/08/05 16:18:07 sjg Exp $
 #
-# $NetBSD: Makefile,v 1.369 2025/06/29 09:40:13 rillig Exp $
+# $NetBSD: Makefile,v 1.372 2025/08/04 22:44:49 sjg Exp $
 #
 # Unit tests for make(1)
 #
@@ -234,6 +234,7 @@ TESTS+=		jobs-error-nested-make
 TESTS+=		lint
 TESTS+=		make-exported
 TESTS+=		meta-cmd-cmp
+TESTS+=		meta-output
 TESTS+=		moderrs
 TESTS+=		modmisc
 .if ${.MAKE.UID} > 0
@@ -592,6 +593,7 @@ SED_CMDS.directive-include-guard= \
 	-e '/^ParseDependency/d' \
 	-e '/^ParseEOF:/d'
 SED_CMDS.export=	-e '/^[^=_A-Za-z0-9]*=/d'
+SED_CMDS.export+=	-e '/^DIFF/d'
 .if ${.MAKE.OS:NCygwin} == ""
 SED_CMDS.export+=	-e '/^WINDIR=/d' -e '/^SYSTEMROOT=/d'
 .endif
@@ -802,6 +804,7 @@ EGREP= grep -E
 EGREP?= egrep
 
 MAKE_TEST_ENV=  EGREP="${EGREP}"
+MAKE_TEST_ENV+= DIFF="${TOOL_DIFF}" DIFF_FLAGS="${DIFF_FLAGS}"
 MAKE_TEST_ENV+=	MALLOC_OPTIONS="JA"	# for jemalloc 100
 MAKE_TEST_ENV+=	MALLOC_CONF="junk:true"	# for jemalloc 510
 MAKE_TEST_ENV+= TMPDIR=${TMPDIR}

unit-tests/meta-output.exp

@@ -0,0 +1,11 @@
+Test -B output
+test1:   Done
+test2:   Done
+test3:   Done
+
+Test -j1 output
+test1:   Done
+test2:   Done
+test3:   Done
+
+exit status 0

unit-tests/meta-output.mk

@@ -0,0 +1,30 @@
+#
+
+.MAIN: all
+
+.if make(output)
+.MAKE.MODE= meta curDirOk=true nofilemon
+.else
+.MAKE.MODE= compat
+.endif
+
+all: output.-B output.-j1
+
+_mf := ${.PARSEDIR}/${.PARSEFILE}
+
+# This output should be accurately reflected in the .meta file.
+# We append an extra newline to ${.TARGET} (after it has been
+# written to stdout) to match what meta_cmd_finish() will do.
+output: .NOPATH
+	@{ echo Test ${tag} output; \
+	for i in 1 2 3; do \
+	printf "test$$i:  "; sleep 0; echo " Done"; \
+	done; echo; } | tee ${.TARGET}; echo >> ${.TARGET}
+
+# The diff at the end should produce nothing.
+output.-B output.-j1:
+	@{ rm -f ${TMPDIR}/output; mkdir -p ${TMPDIR}/obj; \
+	MAKEFLAGS= ${.MAKE} -r -C ${TMPDIR} ${.TARGET:E} tag=${.TARGET:E} -f ${_mf} output; \
+	sed '1,/command output/d' ${TMPDIR}/obj/output.meta > ${TMPDIR}/obj/output-meta; \
+	${DIFF:Udiff} ${TMPDIR}/obj/output ${TMPDIR}/obj/output-meta; }
+