From ab041f713aeccdf23b4805ffb71815c8d4aa9c88 Mon Sep 17 00:00:00 2001 From: D Scott Phillips Date: Mon, 21 Sep 2020 22:20:37 +0000 Subject: [PATCH] Move vm_page_dump bitset array definition to MI code These definitions were repeated by all architectures, with small variations. Consolidate the common definitons in machine independent code and use bitset(9) macros for manipulation. Many opportunities for deduplication remain in the machine dependent minidump logic. The only intended functional change is increasing the bit index type to vm_pindex_t, allowing the indexing of pages with address of 8 TiB and greater. Reviewed by: kib, markj Approved by: scottl (implicit) MFC after: 1 week Sponsored by: Ampere Computing, Inc. Differential Revision: https://reviews.freebsd.org/D26129 --- .clang-format | 1 + sys/amd64/amd64/minidump_machdep.c | 68 +++++--------------- sys/amd64/include/md_var.h | 1 - sys/amd64/include/vmparam.h | 5 ++ sys/arm/arm/minidump_machdep.c | 88 +++++++------------------- sys/arm/include/md_var.h | 4 -- sys/arm/include/vmparam.h | 5 ++ sys/arm64/arm64/minidump_machdep.c | 69 ++++---------------- sys/arm64/include/md_var.h | 4 -- sys/arm64/include/vmparam.h | 5 ++ sys/i386/i386/minidump_machdep.c | 27 -------- sys/i386/i386/minidump_machdep_base.c | 44 +++++-------- sys/i386/include/md_var.h | 1 - sys/i386/include/vmparam.h | 5 ++ sys/mips/include/md_var.h | 4 -- sys/mips/include/vmparam.h | 5 ++ sys/mips/mips/minidump_machdep.c | 74 +++++----------------- sys/powerpc/include/md_var.h | 5 -- sys/powerpc/include/vmparam.h | 12 ++++ sys/powerpc/powerpc/minidump_machdep.c | 86 +++++-------------------- sys/riscv/include/md_var.h | 4 -- sys/riscv/include/vmparam.h | 5 ++ sys/riscv/riscv/minidump_machdep.c | 75 ++++------------------ sys/vm/vm_page.c | 14 ++-- sys/vm/vm_page.h | 22 +++++++ sys/x86/include/x86_var.h | 3 - 26 files changed, 185 insertions(+), 451 deletions(-) diff --git a/.clang-format b/.clang-format index 480affab572..78e005ff7d3 100644 --- a/.clang-format +++ b/.clang-format @@ -76,6 +76,7 @@ ForEachMacros: - TAILQ_FOREACH_REVERSE_SAFE - TAILQ_FOREACH_SAFE - VM_MAP_ENTRY_FOREACH + - VM_PAGE_DUMP_FOREACH IndentCaseLabels: false IndentPPDirectives: None Language: Cpp diff --git a/sys/amd64/amd64/minidump_machdep.c b/sys/amd64/amd64/minidump_machdep.c index 48a427869ba..69d12672e2f 100644 --- a/sys/amd64/amd64/minidump_machdep.c +++ b/sys/amd64/amd64/minidump_machdep.c @@ -54,9 +54,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint64_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -64,7 +61,6 @@ static size_t fragsz; static void *dump_va; static size_t counter, progress, dumpsize, wdog_next; -CTASSERT(sizeof(*vm_page_dump) == 8); static int dump_retry_count = 5; SYSCTL_INT(_machdep, OID_AUTO, dump_retry_count, CTLFLAG_RWTUN, &dump_retry_count, 0, "Number of times dump has to retry before bailing out"); @@ -223,9 +219,8 @@ minidumpsys(struct dumperinfo *di) uint32_t pmapsize; vm_offset_t va; int error; - uint64_t bits; uint64_t *pml4, *pdp, *pd, *pt, pa; - int i, ii, j, k, n, bit; + int i, ii, j, k, n; int retry_count; struct minidumphdr mdhdr; @@ -304,19 +299,13 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfq(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) { - dumpsize += PAGE_SIZE; - } else { - dump_drop_page(pa); - } - bits &= ~(1ul << bit); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) { + dumpsize += PAGE_SIZE; + } else { + dump_drop_page(pa); } } dumpsize += PAGE_SIZE; @@ -328,7 +317,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.dmapbase = DMAP_MIN_ADDRESS; @@ -357,7 +346,8 @@ minidumpsys(struct dumperinfo *di) goto fail; /* Dump bitmap */ - error = blk_write(di, (char *)vm_page_dump, 0, round_page(vm_page_dump_size)); + error = blk_write(di, (char *)vm_page_dump, 0, + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -409,16 +399,10 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfq(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -454,25 +438,3 @@ minidumpsys(struct dumperinfo *di) printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h index 9a550d7024f..7169474b31d 100644 --- a/sys/amd64/include/md_var.h +++ b/sys/amd64/include/md_var.h @@ -36,7 +36,6 @@ #include -extern uint64_t *vm_page_dump; extern int hw_lower_amd64_sharedpage; extern int hw_ibrs_disable; extern int hw_ssb_disable; diff --git a/sys/amd64/include/vmparam.h b/sys/amd64/include/vmparam.h index 0117b94b2f3..38074a647b7 100644 --- a/sys/amd64/include/vmparam.h +++ b/sys/amd64/include/vmparam.h @@ -253,4 +253,9 @@ */ #define VM_BATCHQUEUE_SIZE 31 +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* _MACHINE_VMPARAM_H_ */ diff --git a/sys/arm/arm/minidump_machdep.c b/sys/arm/arm/minidump_machdep.c index c8d55118ec7..4d4d565885b 100644 --- a/sys/arm/arm/minidump_machdep.c +++ b/sys/arm/arm/minidump_machdep.c @@ -57,9 +57,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -67,8 +64,6 @@ static size_t fragsz; static void *dump_va; static uint64_t counter, progress; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -182,10 +177,9 @@ minidumpsys(struct dumperinfo *di) struct minidumphdr mdhdr; uint64_t dumpsize; uint32_t ptesize; - uint32_t bits; uint32_t pa, prev_pa = 0, count = 0; vm_offset_t va; - int i, bit, error; + int error; char *addr; /* @@ -212,20 +206,13 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -236,7 +223,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.ptesize = ptesize; mdhdr.kernbase = KERNBASE; mdhdr.arch = __ARM_ARCH; @@ -270,7 +257,7 @@ minidumpsys(struct dumperinfo *di) /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -293,28 +280,21 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - if (!count) { - prev_pa = pa; + VM_PAGE_DUMP_FOREACH(pa) { + if (!count) { + prev_pa = pa; + count++; + } else { + if (pa == (prev_pa + count * PAGE_SIZE)) count++; - } else { - if (pa == (prev_pa + count * PAGE_SIZE)) - count++; - else { - error = blk_write(di, NULL, prev_pa, - count * PAGE_SIZE); - if (error) - goto fail; - count = 1; - prev_pa = pa; - } + else { + error = blk_write(di, NULL, prev_pa, + count * PAGE_SIZE); + if (error) + goto fail; + count = 1; + prev_pa = pa; } - bits &= ~(1ul << bit); } } if (count) { @@ -349,25 +329,3 @@ minidumpsys(struct dumperinfo *di) printf("\n** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} diff --git a/sys/arm/include/md_var.h b/sys/arm/include/md_var.h index 6b58c96c1b8..eb6e5979462 100644 --- a/sys/arm/include/md_var.h +++ b/sys/arm/include/md_var.h @@ -38,8 +38,6 @@ extern long Maxmem; extern char sigcode[]; extern int szsigcode; -extern uint32_t *vm_page_dump; -extern int vm_page_dump_size; extern u_long elf_hwcap; extern u_long elf_hwcap2; extern vm_paddr_t arm_physmem_kernaddr; @@ -72,8 +70,6 @@ extern enum cpu_class cpu_class; struct dumperinfo; extern int busdma_swi_pending; void busdma_swi(void); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); extern uint32_t initial_fpscr; diff --git a/sys/arm/include/vmparam.h b/sys/arm/include/vmparam.h index b9b9163d8af..28269074d13 100644 --- a/sys/arm/include/vmparam.h +++ b/sys/arm/include/vmparam.h @@ -193,4 +193,9 @@ extern vm_offset_t vm_max_kernel_address; #define DEVMAP_MAX_VADDR ARM_VECTORS_HIGH +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* _MACHINE_VMPARAM_H_ */ diff --git a/sys/arm64/arm64/minidump_machdep.c b/sys/arm64/arm64/minidump_machdep.c index ba22f7dfc16..758b7849a6b 100644 --- a/sys/arm64/arm64/minidump_machdep.c +++ b/sys/arm64/arm64/minidump_machdep.c @@ -56,9 +56,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint64_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -68,8 +65,6 @@ static size_t counter, progress, dumpsize; static uint64_t tmpbuffer[Ln_ENTRIES]; -CTASSERT(sizeof(*vm_page_dump) == 8); - static int is_dumpable(vm_paddr_t pa) { @@ -213,9 +208,8 @@ minidumpsys(struct dumperinfo *di) pt_entry_t *l3; vm_offset_t va; vm_paddr_t pa; - uint64_t bits; uint32_t pmapsize; - int bit, error, i, j, retry_count; + int error, i, j, retry_count; retry_count = 0; retry: @@ -255,20 +249,12 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -279,7 +265,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.dmapphys = DMAP_MIN_PHYSADDR; @@ -311,7 +297,7 @@ minidumpsys(struct dumperinfo *di) /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -378,17 +364,10 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -424,25 +403,3 @@ minidumpsys(struct dumperinfo *di) printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} diff --git a/sys/arm64/include/md_var.h b/sys/arm64/include/md_var.h index cc790d98471..0132ab0dd8f 100644 --- a/sys/arm64/include/md_var.h +++ b/sys/arm64/include/md_var.h @@ -36,8 +36,6 @@ extern long Maxmem; extern char sigcode[]; extern int szsigcode; -extern uint64_t *vm_page_dump; -extern int vm_page_dump_size; extern u_long elf_hwcap; extern u_long elf_hwcap2; @@ -45,8 +43,6 @@ struct dumperinfo; extern int busdma_swi_pending; void busdma_swi(void); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); void generic_bs_fault(void) __asm(__STRING(generic_bs_fault)); void generic_bs_peek_1(void) __asm(__STRING(generic_bs_peek_1)); diff --git a/sys/arm64/include/vmparam.h b/sys/arm64/include/vmparam.h index 96b46e57548..aa5b68bd6d8 100644 --- a/sys/arm64/include/vmparam.h +++ b/sys/arm64/include/vmparam.h @@ -243,4 +243,9 @@ extern vm_offset_t init_pt_va; #define DEVMAP_MAX_VADDR VM_MAX_KERNEL_ADDRESS +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* !_MACHINE_VMPARAM_H_ */ diff --git a/sys/i386/i386/minidump_machdep.c b/sys/i386/i386/minidump_machdep.c index 5ef23cbc34e..d2a7999cb2c 100644 --- a/sys/i386/i386/minidump_machdep.c +++ b/sys/i386/i386/minidump_machdep.c @@ -49,33 +49,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - -CTASSERT(sizeof(*vm_page_dump) == 4); - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} - int minidumpsys(struct dumperinfo *di) { diff --git a/sys/i386/i386/minidump_machdep_base.c b/sys/i386/i386/minidump_machdep_base.c index 49400b0cc2a..1c963907a43 100644 --- a/sys/i386/i386/minidump_machdep_base.c +++ b/sys/i386/i386/minidump_machdep_base.c @@ -62,8 +62,6 @@ static size_t fragsz; static void *dump_va; static uint64_t counter, progress; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -181,11 +179,10 @@ minidumpsys(struct dumperinfo *di) uint32_t ptesize; vm_offset_t va; int error; - uint32_t bits; uint64_t pa; pd_entry_t *pd; pt_entry_t *pt; - int i, j, k, bit; + int j, k; struct minidumphdr mdhdr; counter = 0; @@ -227,19 +224,13 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfl(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) { - dumpsize += PAGE_SIZE; - } else { - dump_drop_page(pa); - } - bits &= ~(1ul << bit); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) { + dumpsize += PAGE_SIZE; + } else { + dump_drop_page(pa); } } dumpsize += PAGE_SIZE; @@ -251,7 +242,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.ptesize = ptesize; mdhdr.kernbase = KERNBASE; mdhdr.paemode = pae_mode; @@ -279,7 +270,8 @@ minidumpsys(struct dumperinfo *di) goto fail; /* Dump bitmap */ - error = blk_write(di, (char *)vm_page_dump, 0, round_page(vm_page_dump_size)); + error = blk_write(di, (char *)vm_page_dump, 0, + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -321,16 +313,10 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfl(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); diff --git a/sys/i386/include/md_var.h b/sys/i386/include/md_var.h index b20446dce12..c41de85b9bc 100644 --- a/sys/i386/include/md_var.h +++ b/sys/i386/include/md_var.h @@ -47,7 +47,6 @@ extern int szfreebsd4_sigcode; extern int szosigcode; extern int sz_lcall_tramp; #endif -extern uint32_t *vm_page_dump; extern vm_offset_t proc0kstack; extern uintptr_t setidt_disp; diff --git a/sys/i386/include/vmparam.h b/sys/i386/include/vmparam.h index c7cf4de8cf7..212ddf758d1 100644 --- a/sys/i386/include/vmparam.h +++ b/sys/i386/include/vmparam.h @@ -240,4 +240,9 @@ #define PHYS_TO_DMAP(x) ({ panic("No direct map exists"); 0; }) #define DMAP_TO_PHYS(x) ({ panic("No direct map exists"); 0; }) +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* _MACHINE_VMPARAM_H_ */ diff --git a/sys/mips/include/md_var.h b/sys/mips/include/md_var.h index 4bf9c64a8f5..8462b1beb22 100644 --- a/sys/mips/include/md_var.h +++ b/sys/mips/include/md_var.h @@ -50,8 +50,6 @@ extern int szsigcode; extern char sigcode32[]; extern int szsigcode32; #endif -extern uint32_t *vm_page_dump; -extern int vm_page_dump_size; extern vm_offset_t kstack0; extern vm_offset_t kernel_kseg0_end; @@ -84,8 +82,6 @@ extern int busdma_swi_pending; void busdma_swi(void); struct dumperinfo; -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); #endif /* !_MACHINE_MD_VAR_H_ */ diff --git a/sys/mips/include/vmparam.h b/sys/mips/include/vmparam.h index 305224dc7b8..183efe1a7c0 100644 --- a/sys/mips/include/vmparam.h +++ b/sys/mips/include/vmparam.h @@ -197,4 +197,9 @@ #define PHYS_TO_DMAP(x) MIPS_PHYS_TO_DIRECT(x) #define DMAP_TO_PHYS(x) MIPS_DIRECT_TO_PHYS(x) +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* !_MACHINE_VMPARAM_H_ */ diff --git a/sys/mips/mips/minidump_machdep.c b/sys/mips/mips/minidump_machdep.c index c44ee8a7778..164e7a04204 100644 --- a/sys/mips/mips/minidump_machdep.c +++ b/sys/mips/mips/minidump_machdep.c @@ -54,9 +54,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -66,8 +63,6 @@ static char tmpbuffer[PAGE_SIZE]; extern pd_entry_t *kernel_segmap; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -83,28 +78,6 @@ is_dumpable(vm_paddr_t pa) return (0); } -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} - static struct { int min_per; int max_per; @@ -192,13 +165,12 @@ minidumpsys(struct dumperinfo *di) { struct minidumphdr mdhdr; uint32_t ptesize; - uint32_t bits; vm_paddr_t pa; vm_offset_t prev_pte = 0; uint32_t count = 0; vm_offset_t va; pt_entry_t *pte; - int i, bit, error; + int i, error; void *dump_va; /* Flush cache */ @@ -233,20 +205,13 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -257,7 +222,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.ptesize = ptesize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; @@ -286,7 +251,7 @@ minidumpsys(struct dumperinfo *di) /* Dump bitmap */ error = write_buffer(di, (char *)vm_page_dump, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -320,19 +285,12 @@ minidumpsys(struct dumperinfo *di) } /* Dump memory chunks page by page*/ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - dump_va = pmap_kenter_temporary(pa, 0); - error = write_buffer(di, dump_va, PAGE_SIZE); - if (error) - goto fail; - pmap_kenter_temporary_free(pa); - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + dump_va = pmap_kenter_temporary(pa, 0); + error = write_buffer(di, dump_va, PAGE_SIZE); + if (error) + goto fail; + pmap_kenter_temporary_free(pa); } error = dump_finish(di, &kdh); diff --git a/sys/powerpc/include/md_var.h b/sys/powerpc/include/md_var.h index 7db9d3d3bf3..da9232313ff 100644 --- a/sys/powerpc/include/md_var.h +++ b/sys/powerpc/include/md_var.h @@ -42,14 +42,9 @@ extern int szsigcode32; extern char sigcode64[], sigcode64_elfv2[]; extern int szsigcode64, szsigcode64_elfv2; -extern uint64_t *vm_page_dump; -extern int vm_page_dump_size; - struct dumperinfo; int minidumpsys(struct dumperinfo *); int is_dumpable(vm_paddr_t); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); #endif extern long Maxmem; diff --git a/sys/powerpc/include/vmparam.h b/sys/powerpc/include/vmparam.h index f46b058e726..03ea95c86d0 100644 --- a/sys/powerpc/include/vmparam.h +++ b/sys/powerpc/include/vmparam.h @@ -307,6 +307,18 @@ struct pmap_physseg { #define PMAP_HAS_PAGE_ARRAY 1 #endif +#if defined(__powerpc64__) +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 +#else +/* + * No minidump with 32-bit powerpc. + */ +#define MINIDUMP_PAGE_TRACKING 0 +#endif + #define PMAP_HAS_DMAP (hw_direct_map) #define PHYS_TO_DMAP(x) ({ \ KASSERT(hw_direct_map, ("Direct map not provided by PMAP")); \ diff --git a/sys/powerpc/powerpc/minidump_machdep.c b/sys/powerpc/powerpc/minidump_machdep.c index e93b11379db..74b790537e3 100644 --- a/sys/powerpc/powerpc/minidump_machdep.c +++ b/sys/powerpc/powerpc/minidump_machdep.c @@ -45,16 +45,6 @@ #include #include -/* - * bit to physical address - * - * bm - bitmap - * i - bitmap entry index - * bit - bit number - */ -#define BTOP(bm, i, bit) \ - (((uint64_t)(i) * sizeof(*(bm)) * NBBY + (bit)) * PAGE_SIZE) - /* Debugging stuff */ #define MINIDUMP_DEBUG 0 #if MINIDUMP_DEBUG @@ -70,9 +60,6 @@ static void dump_total(const char *id, size_t sz); extern vm_offset_t __startkernel, __endkernel; -int vm_page_dump_size; -uint64_t *vm_page_dump; - static int dump_retry_count = 5; SYSCTL_INT(_machdep, OID_AUTO, dump_retry_count, CTLFLAG_RWTUN, &dump_retry_count, 0, @@ -103,28 +90,6 @@ static size_t counter, dumpsize, progress; /* Handle chunked writes. */ static size_t fragsz; -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} - int is_dumpable(vm_paddr_t pa) { @@ -280,9 +245,8 @@ int minidumpsys(struct dumperinfo *di) { vm_paddr_t pa; - int bit, error, i, retry_count; + int error, i, retry_count; uint32_t pmapsize; - uint64_t bits; struct minidumphdr mdhdr; retry_count = 0; @@ -306,24 +270,14 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size */ dumpsize = PAGE_SIZE; /* header */ dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); dumpsize += pmapsize; - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - /* TODO optimize with bit manipulation instructions */ - if (bits == 0) - continue; - for (bit = 0; bit < 64; bit++) { - if ((bits & (1ul<msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.kernend = VM_MAX_SAFE_KERNEL_ADDRESS; @@ -368,10 +322,10 @@ minidumpsys(struct dumperinfo *di) /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; - dump_total("bitmap", round_page(vm_page_dump_size)); + dump_total("bitmap", round_page(BITSET_SIZE(vm_page_dump_pages))); /* Dump kernel page directory pages */ error = dump_pmap(di); @@ -380,20 +334,10 @@ minidumpsys(struct dumperinfo *di) dump_total("pmap", pmapsize); /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - /* TODO optimize with bit manipulation instructions */ - if (bits == 0) - continue; - for (bit = 0; bit < 64; bit++) { - if ((bits & (1ul< CTASSERT(sizeof(struct kerneldumpheader) == 512); -CTASSERT(sizeof(*vm_page_dump) == 8); - -uint64_t *vm_page_dump; -int vm_page_dump_size; static struct kerneldumpheader kdh; @@ -221,8 +217,7 @@ minidumpsys(struct dumperinfo *di) vm_offset_t va; vm_paddr_t pa; int error; - uint64_t bits; - int i, bit; + int i; int retry_count; retry_count = 0; @@ -262,20 +257,13 @@ minidumpsys(struct dumperinfo *di) /* Calculate dump size */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -286,7 +274,7 @@ minidumpsys(struct dumperinfo *di) strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = KERNBASE; mdhdr.dmapphys = DMAP_MIN_PHYSADDR; @@ -318,7 +306,7 @@ minidumpsys(struct dumperinfo *di) /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -360,17 +348,10 @@ minidumpsys(struct dumperinfo *di) /* Dump memory chunks */ /* XXX cluster it up and use blk_dump() */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -406,31 +387,3 @@ minidumpsys(struct dumperinfo *di) printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -/* - * Add a page to the minidump bitmap. - */ -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -/* - * Remove page from the minidump bitmap. - */ -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index e120c0d2909..140711a0022 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -155,6 +155,9 @@ vm_page_t vm_page_array; long vm_page_array_size; long first_page; +struct bitset *vm_page_dump; +long vm_page_dump_pages; + static TAILQ_HEAD(, vm_page) blacklist_head; static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | @@ -554,6 +557,9 @@ vm_page_startup(vm_offset_t vaddr) vm_paddr_t page_range __unused; vm_paddr_t last_pa, pa; u_long pagecount; +#if MINIDUMP_PAGE_TRACKING + u_long vm_page_dump_size; +#endif int biggestone, i, segind; #ifdef WITNESS vm_offset_t mapped; @@ -588,9 +594,7 @@ vm_page_startup(vm_offset_t vaddr) witness_startup((void *)mapped); #endif -#if defined(__aarch64__) || defined(__amd64__) || defined(__arm__) || \ - defined(__i386__) || defined(__mips__) || defined(__riscv) || \ - defined(__powerpc64__) +#if MINIDUMP_PAGE_TRACKING /* * Allocate a bitmap to indicate that a random physical page * needs to be included in a minidump. @@ -606,8 +610,8 @@ vm_page_startup(vm_offset_t vaddr) for (i = 0; dump_avail[i + 1] != 0; i += 2) if (dump_avail[i + 1] > last_pa) last_pa = dump_avail[i + 1]; - page_range = last_pa / PAGE_SIZE; - vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY); + vm_page_dump_pages = last_pa / PAGE_SIZE; + vm_page_dump_size = round_page(BITSET_SIZE(vm_page_dump_pages)); new_end -= vm_page_dump_size; vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end, new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE); diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index a44e31f506d..ec9a539b8f0 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -69,6 +69,8 @@ #ifndef _VM_PAGE_ #define _VM_PAGE_ +#include +#include #include /* @@ -585,6 +587,26 @@ malloc2vm_flags(int malloc_flags) #define PS_ALL_VALID 0x2 #define PS_NONE_BUSY 0x4 +extern struct bitset *vm_page_dump; +extern long vm_page_dump_pages; + +static inline void +dump_add_page(vm_paddr_t pa) +{ + BIT_SET_ATOMIC(vm_page_dump_pages, pa >> PAGE_SHIFT, vm_page_dump); +} + +static inline void +dump_drop_page(vm_paddr_t pa) +{ + BIT_CLR_ATOMIC(vm_page_dump_pages, pa >> PAGE_SHIFT, vm_page_dump); +} + +#define VM_PAGE_DUMP_FOREACH(pa) \ + for (vm_pindex_t __b = BIT_FFS(vm_page_dump_pages, vm_page_dump); \ + (pa) = (__b - 1) * PAGE_SIZE, __b != 0; \ + __b = BIT_FFS_AT(vm_page_dump_pages, vm_page_dump, __b)) + bool vm_page_busy_acquire(vm_page_t m, int allocflags); void vm_page_busy_downgrade(vm_page_t m); int vm_page_busy_tryupgrade(vm_page_t m); diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h index 789b8a2cbc4..a1ce8f9e311 100644 --- a/sys/x86/include/x86_var.h +++ b/sys/x86/include/x86_var.h @@ -78,7 +78,6 @@ extern char hv_vendor[]; extern char kstack[]; extern char sigcode[]; extern int szsigcode; -extern int vm_page_dump_size; extern int workaround_erratum383; extern int _udatasel; extern int _ucodesel; @@ -122,8 +121,6 @@ void cpu_probe_amdc1e(void); void cpu_setregs(void); bool disable_wp(void); void restore_wp(bool old_wp); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); void finishidentcpu(void); void identify_cpu1(void); void identify_cpu2(void);