ARM64: Round-Robin dispatch IRQs between CPUs.

IRQ-CPU mapping is round robined on ARM64 to increase
concurrency and allow multiple interrupts to be serviced
at a time.  This reduces the need for FIQ.

Signed-off-by: Michael Zoran <[email protected]>

drivers: irqchip: irq-bcm2835: Concurrency fix

The commit shown in Fixes: aims to improve interrupt throughput by
getting the handlers invoked on different CPU cores. It does so (*) by
using an irq_ack hook to change the interrupt routing.

Unfortunately, the IRQ status bits must be cleared at source, which only
happens once the interrupt handler has run - there is no easy way for
one core to claim one of the IRQs before sending the remainder to the
next core on the list, so waking another core immediately results in a
race with a chance of both cores handling the same IRQ. It is probably
for this reason that the routing change is deferred to irq_ack, but that
doesn't guarantee no clashes - after irq_ack is called, control returns
to bcm2836_chained_handler_irq which proceeds to check for other pending
IRQs at a time when the next core is probably doing the same thing.

Since the whole point of the original commit is to distribute the IRQ
handling, there is no reason to attempt to handle multiple IRQs in one
interrupt callback, so the problem can be solved (or at least made much
harder to reproduce) by changing a "while" into an "if", so that each
invocation only handles one IRQ.

(*) I'm not convinced it's as effective as claimed since irq_ack is
called _after_ the interrupt handler, but the author thought it made a
difference.

See: #5214
     #1794

Fixes: fd4c978 ("ARM64: Round-Robin dispatch IRQs between CPUs.")
Signed-off-by: Phil Elwell <[email protected]>

irqchip: irq-bcm2836: Avoid prototype warning

Declare bcm2836_arm_irqchip_spin_gpu_irq in irq-bcm2836.h to avoid a
compiler warning about a missing prototype.

Signed-off-by: Phil Elwell <[email protected]>

drivers/irqchip/irq-bcm2835.c

@@ -40,6 +40,7 @@
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/irqchip.h>
+#include <linux/irqchip/irq-bcm2836.h>
 #include <linux/irqdomain.h>
 
 #include <asm/exception.h>
@@ -154,10 +155,22 @@ static void armctrl_unmask_irq(struct irq_data *d)
 	}
 }
 
+#ifdef CONFIG_ARM64
+
+static void armctrl_ack_irq(struct irq_data *d)
+{
+	bcm2836_arm_irqchip_spin_gpu_irq();
+}
+
+#endif
+
 static struct irq_chip armctrl_chip = {
 	.name = "ARMCTRL-level",
 	.irq_mask = armctrl_mask_irq,
-	.irq_unmask = armctrl_unmask_irq
+	.irq_unmask = armctrl_unmask_irq,
+#ifdef CONFIG_ARM64
+	.irq_ack    = armctrl_ack_irq
+#endif
 };
 
 static int armctrl_xlate(struct irq_domain *d, struct device_node *ctrlr,
@@ -330,7 +343,8 @@ static void bcm2836_chained_handle_irq(struct irq_desc *desc)
 {
 	u32 hwirq;
 
-	while ((hwirq = get_next_armctrl_hwirq()) != ~0)
+	hwirq = get_next_armctrl_hwirq();
+	if (hwirq != ~0)
 		generic_handle_domain_irq(intc.domain, hwirq);
 }
 

drivers/irqchip/irq-bcm2836.c

@@ -87,6 +87,27 @@ static void bcm2836_arm_irqchip_unmask_gpu_irq(struct irq_data *d)
 {
 }
 
+#ifdef CONFIG_ARM64
+
+void bcm2836_arm_irqchip_spin_gpu_irq(void)
+{
+	u32 i;
+	void __iomem *gpurouting = (intc.base + LOCAL_GPU_ROUTING);
+	u32 routing_val = readl(gpurouting);
+
+	for (i = 1; i <= 3; i++) {
+		u32 new_routing_val = (routing_val + i) & 3;
+
+		if (cpu_active(new_routing_val)) {
+			writel(new_routing_val, gpurouting);
+			return;
+		}
+	}
+}
+EXPORT_SYMBOL(bcm2836_arm_irqchip_spin_gpu_irq);
+
+#endif
+
 static struct irq_chip bcm2836_arm_irqchip_gpu = {
 	.name		= "bcm2836-gpu",
 	.irq_mask	= bcm2836_arm_irqchip_mask_gpu_irq,

include/linux/irqchip/irq-bcm2836.h

@@ -59,3 +59,5 @@
 #define LOCAL_IRQ_GPU_FAST	8
 #define LOCAL_IRQ_PMU_FAST	9
 #define LAST_IRQ		LOCAL_IRQ_PMU_FAST
+
+void bcm2836_arm_irqchip_spin_gpu_irq(void);