Call acpi_pxm_set_proximity_info() slightly earlier on x86.

This function is responsible for setting pc_domain in each pcpu
structure.  Call it from the main function that starts APs, rather than
a separate SYSINIT.  This makes it easier to close the window where
UMA's per-CPU slab allocator may be called while pc_domain is
uninitialized.  In particular, the allocator uses pc_domain to allocate
domain-local pages, so allocations before this point end up using domain
0 for everything.

Reviewed by:	kib
MFC after:	1 week
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D24757

sys/amd64/amd64/mp_machdep.c

@@ -264,8 +264,9 @@ cpu_mp_start(void)
 	init_ops.start_all_aps();
 
 	set_interrupt_apic_ids();
-}
 
+	acpi_pxm_set_cpu_locality();
+}
 
 /*
  * AP CPU's call this to initialize themselves.

sys/arm64/arm64/mp_machdep.c

@@ -535,9 +535,7 @@ cpu_init_acpi(void)
 	acpi_unmap_table(madt);
 
 #if MAXMEMDOM > 1
-	/* set proximity info */
 	acpi_pxm_set_cpu_locality();
-	acpi_pxm_free();
 #endif
 }
 #endif

sys/dev/acpica/acpi_pxm.c

@@ -628,7 +628,8 @@ srat_walk_table(acpi_subtable_handler *handler, void *arg)
 }
 
 /*
- * Setup per-CPU domain IDs from information saved in 'cpus'.
+ * Set up per-CPU domain IDs from information saved in 'cpus' and tear down data
+ * structures allocated by acpi_pxm_init().
  */
 void
 acpi_pxm_set_cpu_locality(void)
@@ -651,6 +652,10 @@ acpi_pxm_set_cpu_locality(void)
 			printf("SRAT: CPU %u has memory domain %d\n", i,
 			    pc->pc_domain);
 	}
+	/* XXXMJ the page is leaked. */
+	pmap_unmapbios((vm_offset_t)cpus, sizeof(*cpus) * max_cpus);
+	srat_physaddr = 0;
+	cpus = NULL;
 }
 
 int
@@ -664,20 +669,6 @@ acpi_pxm_get_cpu_locality(int apic_id)
 	return (cpu->domain);
 }
 
-/*
- * Free data structures allocated during acpi_pxm_init.
- */
-void
-acpi_pxm_free(void)
-{
-
-	if (srat_physaddr == 0)
-		return;
-	pmap_unmapbios((vm_offset_t)cpus, sizeof(*cpus) * max_cpus);
-	srat_physaddr = 0;
-	cpus = NULL;
-}
-
 /*
  * Map a _PXM value to a VM domain ID.
  *

sys/dev/acpica/acpivar.h

@@ -540,7 +540,6 @@ void		acpi_pxm_parse_tables(void);
 void		acpi_pxm_set_mem_locality(void);
 void		acpi_pxm_set_cpu_locality(void);
 int		acpi_pxm_get_cpu_locality(int apic_id);
-void		acpi_pxm_free(void);
 
 /*
  * Map a PXM to a VM domain.

sys/i386/i386/mp_machdep.c

@@ -198,6 +198,8 @@ cpu_mp_start(void)
 	start_all_aps();
 
 	set_interrupt_apic_ids();
+
+	acpi_pxm_set_cpu_locality();
 }
 
 /*

sys/x86/acpica/srat.c

@@ -56,13 +56,4 @@ parse_acpi_tables(void *dummy)
 SYSINIT(parse_acpi_tables, SI_SUB_VM - 1, SI_ORDER_FIRST, parse_acpi_tables,
     NULL);
 
-static void
-srat_set_cpus(void *dummy)
-{
-
-	acpi_pxm_set_cpu_locality();
-	acpi_pxm_free();
-}
-SYSINIT(srat_set_cpus, SI_SUB_CPU, SI_ORDER_ANY, srat_set_cpus, NULL);
-
 #endif /* MAXMEMDOM > 1 */