Support emulated NUMA for BCM2711 and BCM2712

arch/arm/boot/dts/broadcom/bcm2711-rpi-4-b.dts

@@ -300,10 +300,6 @@
 #include "bcm283x-rpi-i2c0mux_0_44.dtsi"
 
 / {
-	chosen {
-		bootargs = "coherent_pool=1M 8250.nr_uarts=1 snd_bcm2835.enable_headphones=0";
-	};
-
 	/delete-node/ wifi-pwrseq;
 };
 

arch/arm/boot/dts/broadcom/bcm2711-rpi-cm4.dts

@@ -274,10 +274,6 @@
 #include "bcm283x-rpi-i2c0mux_0_44.dtsi"
 
 / {
-	chosen {
-		bootargs = "coherent_pool=1M 8250.nr_uarts=1 snd_bcm2835.enable_headphones=0";
-	};
-
 	/delete-node/ wifi-pwrseq;
 };
 

arch/arm/boot/dts/broadcom/bcm2711-rpi-cm4s.dts

@@ -148,7 +148,7 @@
 
 / {
 	chosen {
-		bootargs = "coherent_pool=1M snd_bcm2835.enable_headphones=0";
+		bootargs = "coherent_pool=1M snd_bcm2835.enable_headphones=0 numa_policy=interleave";
 	};
 
 	aliases {

arch/arm/boot/dts/broadcom/bcm2711-rpi-ds.dtsi

@@ -3,6 +3,7 @@
 
 / {
 	chosen: chosen {
+		bootargs = "coherent_pool=1M 8250.nr_uarts=1 snd_bcm2835.enable_headphones=0 numa_policy=interleave";
 	};
 
 	__overrides__ {

arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts

@@ -389,11 +389,6 @@ dpi_16bit_gpio2:        &rp1_dpi_16bit_gpio2        { };
 };
 
 / {
-	chosen: chosen {
-		bootargs = "reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe";
-		stdout-path = "serial10:115200n8";
-	};
-
 	fan: cooling_fan {
 		status = "disabled";
 		compatible = "pwm-fan";

arch/arm64/boot/dts/broadcom/bcm2712-rpi-cm5.dtsi

@@ -379,11 +379,6 @@ dpi_16bit_gpio2:        &rp1_dpi_16bit_gpio2        { };
 };
 
 / {
-	chosen: chosen {
-		bootargs = "reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe";
-		stdout-path = "serial10:115200n8";
-	};
-
 	fan: cooling_fan {
 		status = "disabled";
 		compatible = "pwm-fan";

arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi

@@ -98,6 +98,11 @@
 };
 
 / {
+	chosen: chosen {
+		bootargs = "reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe numa_policy=interleave iommu_dma_numa_policy=interleave system_heap.max_order=0";
+		stdout-path = "serial10:115200n8";
+	};
+
 	aliases: aliases {
 		blconfig = &blconfig;
 		blpubkey = &blpubkey;

arch/arm64/configs/bcm2711_defconfig

@@ -50,6 +50,7 @@ CONFIG_ARM64_ERRATUM_2441009=y
 CONFIG_ARM64_VA_BITS_39=y
 CONFIG_NR_CPUS=4
 CONFIG_HOTPLUG_CPU=y
+CONFIG_NUMA=y
 CONFIG_COMPAT=y
 CONFIG_ARMV8_DEPRECATED=y
 CONFIG_SWP_EMULATION=y
@@ -90,6 +91,7 @@ CONFIG_CMA=y
 CONFIG_CMA_AREAS=7
 CONFIG_LRU_GEN=y
 CONFIG_LRU_GEN_ENABLED=y
+CONFIG_NUMA_EMU=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_XFRM_USER=m

arch/arm64/configs/bcm2712_defconfig

@@ -51,6 +51,7 @@ CONFIG_ARM64_16K_PAGES=y
 CONFIG_ARM64_VA_BITS_47=y
 CONFIG_NR_CPUS=4
 CONFIG_HOTPLUG_CPU=y
+CONFIG_NUMA=y
 CONFIG_COMPAT=y
 CONFIG_ARMV8_DEPRECATED=y
 CONFIG_SWP_EMULATION=y
@@ -93,6 +94,7 @@ CONFIG_CMA=y
 CONFIG_CMA_AREAS=7
 CONFIG_LRU_GEN=y
 CONFIG_LRU_GEN_ENABLED=y
+CONFIG_NUMA_EMU=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_XFRM_USER=m

drivers/dma-buf/heaps/system_heap.c

@@ -54,6 +54,11 @@ static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP};
 static const unsigned int orders[] = {8, 4, 0};
 #define NUM_ORDERS ARRAY_SIZE(orders)
 
+static unsigned int module_max_order = orders[0];
+
+module_param_named(max_order, module_max_order, uint, 0400);
+MODULE_PARM_DESC(max_order, "Maximum allocation order override.");
+
 static struct sg_table *dup_sg_table(struct sg_table *table)
 {
 	struct sg_table *new_table;
@@ -339,7 +344,7 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
 	struct system_heap_buffer *buffer;
 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 	unsigned long size_remaining = len;
-	unsigned int max_order = orders[0];
+	unsigned int max_order = module_max_order;
 	struct dma_buf *dmabuf;
 	struct sg_table *table;
 	struct scatterlist *sg;
@@ -433,6 +438,9 @@ static int system_heap_create(void)
 	if (IS_ERR(sys_heap))
 		return PTR_ERR(sys_heap);
 
+	if (module_max_order > orders[0])
+		module_max_order = orders[0];
+
 	return 0;
 }
 module_init(system_heap_create);

include/linux/cma.h

@@ -56,6 +56,7 @@ extern void cma_reserve_pages_on_error(struct cma *cma);
 #ifdef CONFIG_CMA
 struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp);
 bool cma_free_folio(struct cma *cma, const struct folio *folio);
+int cma_check_range(u64 *start, u64 *end);
 #else
 static inline struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp)
 {
@@ -66,6 +67,11 @@ static inline bool cma_free_folio(struct cma *cma, const struct folio *folio)
 {
 	return false;
 }
+
+static inline int cma_check_range(u64 *start, u64 *end)
+{
+	return 0;
+}
 #endif
 
 #endif

mm/cma.c

@@ -602,3 +602,39 @@ int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data)
 
 	return 0;
 }
+
+struct cma_check_range_data {
+	u64 start, end;
+};
+
+static int check_range(struct cma *cma_, void *data)
+{
+	struct cma_check_range_data *range = data;
+	struct cma_check_range_data cma;
+	bool starts_in_range;
+	bool ends_in_range;
+
+	cma.start = cma_get_base(cma_);
+	cma.end = cma.start + cma_get_size(cma_) - 1;
+
+	starts_in_range = cma.start >= range->start && cma.start <= range->end;
+	ends_in_range = cma.end >= range->start && cma.end <= range->end;
+
+	if (starts_in_range == ends_in_range)
+		return 0;
+
+	pr_notice("CMA %s [%llx-%llx] straddles range [%llx-%llx]\n",
+		  cma_->name, cma.start, cma.end, range->start, range->end);
+
+	return -EINVAL;
+}
+
+int cma_check_range(u64 *start, u64 *end)
+{
+	struct cma_check_range_data range = {
+		.start = *start,
+		.end = *end,
+	};
+
+	return cma_for_each_area(check_range, &range);
+}

mm/mempolicy.c

@@ -3142,7 +3142,9 @@ void __init numa_policy_init(void)
 /* Reset policy of current process to default */
 void numa_default_policy(void)
 {
-	do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
+	struct mempolicy *pol = &default_policy;
+
+	do_set_mempolicy(pol->mode, pol->flags, &pol->nodes);
 }
 
 /*
@@ -3159,7 +3161,6 @@ static const char * const policy_modes[] =
 	[MPOL_PREFERRED_MANY]  = "prefer (many)",
 };
 
-#ifdef CONFIG_TMPFS
 /**
  * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
  * @str:  string containing mempolicy to parse
@@ -3172,13 +3173,18 @@ static const char * const policy_modes[] =
  */
 int mpol_parse_str(char *str, struct mempolicy **mpol)
 {
-	struct mempolicy *new = NULL;
+	struct mempolicy *new;
 	unsigned short mode_flags;
 	nodemask_t nodes;
 	char *nodelist = strchr(str, ':');
 	char *flags = strchr(str, '=');
 	int err = 1, mode;
 
+	if (*mpol)
+		new = *mpol;
+	else
+		new = NULL;
+
 	if (flags)
 		*flags++ = '\0';	/* terminate mode string */
 
@@ -3258,9 +3264,16 @@ int mpol_parse_str(char *str, struct mempolicy **mpol)
 			goto out;
 	}
 
-	new = mpol_new(mode, mode_flags, &nodes);
-	if (IS_ERR(new))
-		goto out;
+	if (!new) {
+		new = mpol_new(mode, mode_flags, &nodes);
+		if (IS_ERR(new))
+			goto out;
+	} else {
+		atomic_set(&new->refcnt, 1);
+		new->mode = mode;
+		new->flags = mode_flags;
+		new->home_node = NUMA_NO_NODE;
+	}
 
 	/*
 	 * Save nodes for mpol_to_str() to show the tmpfs mount options
@@ -3293,7 +3306,29 @@ int mpol_parse_str(char *str, struct mempolicy **mpol)
 		*mpol = new;
 	return err;
 }
-#endif /* CONFIG_TMPFS */
+
+static int __init setup_numapolicy(char *str)
+{
+	struct mempolicy pol = { }, *ppol = &pol;
+	char buf[128];
+	int ret;
+
+	if (str)
+		ret = mpol_parse_str(str, &ppol);
+	else
+		ret = -EINVAL;
+
+	if (!ret) {
+		default_policy = pol;
+		mpol_to_str(buf, sizeof(buf), &pol);
+		pr_info("NUMA default policy overridden to '%s'\n", buf);
+	} else {
+		pr_warn("Unable to parse numa_policy=\n");
+	}
+
+	return ret == 0;
+}
+__setup("numa_policy=", setup_numapolicy);
 
 /**
  * mpol_to_str - format a mempolicy structure for printing

mm/numa_emulation.c

@@ -7,6 +7,7 @@
 #include <linux/topology.h>
 #include <linux/memblock.h>
 #include <linux/numa_memblks.h>
+#include <linux/cma.h>
 #include <asm/numa.h>
 
 #define FAKE_NODE_MIN_SIZE	((u64)32 << 20)
@@ -51,6 +52,7 @@ static int __init emu_setup_memblk(struct numa_meminfo *ei,
 {
 	struct numa_memblk *eb = &ei->blk[ei->nr_blks];
 	struct numa_memblk *pb = &pi->blk[phys_blk];
+	int ret;
 
 	if (ei->nr_blks >= NR_NODE_MEMBLKS) {
 		pr_err("NUMA: Too many emulated memblks, failing emulation\n");
@@ -62,6 +64,10 @@ static int __init emu_setup_memblk(struct numa_meminfo *ei,
 	eb->end = pb->start + size;
 	eb->nid = nid;
 
+	ret = cma_check_range(&eb->start, &eb->end);
+	if (ret)
+		return ret;
+
 	if (emu_nid_to_phys[nid] == NUMA_NO_NODE)
 		emu_nid_to_phys[nid] = pb->nid;