[SPARC64]: Make IVEC pointers 64-bit.

Currently we chain IVEC entries using 32-bit "pointers"
because we know that the ivector_table is in the main
kernel image, thus below 4GB.

This uses proper 64-bit pointers instead.

Whilst this bloats up the kernel image size, this sets
the infrastructure necessary to significantly shrink the
kernel size by using physical addresses and dynamically
allocating the ivector table.

Signed-off-by: David S. Miller <davem@davemloft.net>

arch/sparc64/kernel/entry.S

@@ -430,15 +430,15 @@ do_ivec:
 	membar		#Sync
 
 	sethi		%hi(ivector_table), %g2
-	sllx		%g3, 3, %g3
+	sllx		%g3, 4, %g3
 	or		%g2, %lo(ivector_table), %g2
 	add		%g2, %g3, %g3
 
 	TRAP_LOAD_IRQ_WORK(%g6, %g1)
 
-	lduw		[%g6], %g5		/* g5 = irq_work(cpu) */
-	stw		%g5, [%g3 + 0x00]	/* bucket->irq_chain = g5 */
-	stw		%g3, [%g6]		/* irq_work(cpu) = bucket */
+	ldx		[%g6], %g5		/* g5 = irq_work(cpu) */
+	stx		%g5, [%g3 + 0x00]	/* bucket->irq_chain = g5 */
+	stx		%g3, [%g6]		/* irq_work(cpu) = bucket */
 	wr		%g0, 1 << PIL_DEVICE_IRQ, %set_softint
 	retry
 do_ivec_xcall:

arch/sparc64/kernel/irq.c

@@ -59,23 +59,20 @@
  * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S
  */
 struct ino_bucket {
-	/* Next handler in per-CPU IRQ worklist.  We know that
-	 * bucket pointers have the high 32-bits clear, so to
-	 * save space we only store the bits we need.
-	 */
-/*0x00*/unsigned int irq_chain;
+/*0x00*/unsigned long irq_chain;
 
 	/* Virtual interrupt number assigned to this INO.  */
-/*0x04*/unsigned int virt_irq;
+/*0x08*/unsigned int virt_irq;
+/*0x0c*/unsigned int __pad;
 };
 
 #define NUM_IVECS	(IMAP_INR + 1)
 struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES)));
 
 #define __irq_ino(irq) \
-        (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0])
-#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq))
-#define __irq(bucket) ((unsigned int)(unsigned long)(bucket))
+        (((struct ino_bucket *)(irq)) - &ivector_table[0])
+#define __bucket(irq) ((struct ino_bucket *)(irq))
+#define __irq(bucket) ((unsigned long)(bucket))
 
 /* This has to be in the main kernel image, it cannot be
  * turned into per-cpu data.  The reason is that the main
@@ -87,13 +84,13 @@ struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BY
 #define irq_work(__cpu)	&(trap_block[(__cpu)].irq_worklist)
 
 static struct {
-	unsigned int irq;
+	unsigned long irq;
 	unsigned int dev_handle;
 	unsigned int dev_ino;
 } virt_to_real_irq_table[NR_IRQS];
 static DEFINE_SPINLOCK(virt_irq_alloc_lock);
 
-unsigned char virt_irq_alloc(unsigned int real_irq)
+unsigned char virt_irq_alloc(unsigned long real_irq)
 {
 	unsigned long flags;
 	unsigned char ent;
@@ -134,7 +131,7 @@ void virt_irq_free(unsigned int virt_irq)
 }
 #endif
 
-static unsigned int virt_to_real_irq(unsigned char virt_irq)
+static unsigned long virt_to_real_irq(unsigned char virt_irq)
 {
 	return virt_to_real_irq_table[virt_irq].irq;
 }
@@ -227,7 +224,7 @@ struct irq_handler_data {
 
 static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq)
 {
-	unsigned int real_irq = virt_to_real_irq(virt_irq);
+	unsigned long real_irq = virt_to_real_irq(virt_irq);
 	struct ino_bucket *bucket = NULL;
 
 	if (likely(real_irq))
@@ -694,18 +691,28 @@ void handler_irq(int irq, struct pt_regs *regs)
 {
 	struct ino_bucket *bucket;
 	struct pt_regs *old_regs;
+	unsigned long pstate;
 
 	clear_softint(1 << irq);
 
 	old_regs = set_irq_regs(regs);
 	irq_enter();
 
-	/* Sliiiick... */
-	bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0));
+	/* Grab an atomic snapshot of the pending IVECs.  */
+	__asm__ __volatile__("rdpr	%%pstate, %0\n\t"
+			     "wrpr	%0, %3, %%pstate\n\t"
+			     "ldx	[%2], %1\n\t"
+			     "stx	%%g0, [%2]\n\t"
+			     "wrpr	%0, 0x0, %%pstate\n\t"
+			     : "=&r" (pstate), "=&r" (bucket)
+			     : "r" (irq_work(smp_processor_id())),
+			       "i" (PSTATE_IE)
+			     : "memory");
+
 	while (bucket) {
 		struct ino_bucket *next = __bucket(bucket->irq_chain);
 
-		bucket->irq_chain = 0;
+		bucket->irq_chain = 0UL;
 		__do_IRQ(bucket->virt_irq);
 
 		bucket = next;
@@ -808,7 +815,7 @@ void init_irqwork_curcpu(void)
 {
 	int cpu = hard_smp_processor_id();
 
-	trap_block[cpu].irq_worklist = 0;
+	trap_block[cpu].irq_worklist = 0UL;
 }
 
 /* Please be very careful with register_one_mondo() and

arch/sparc64/kernel/sun4v_ivec.S

@@ -101,14 +101,14 @@ sun4v_dev_mondo:
 
 	/* Get &ivector_table[IVEC] into %g4.  */
 	sethi	%hi(ivector_table), %g4
-	sllx	%g3, 3, %g3
+	sllx	%g3, 4, %g3
 	or	%g4, %lo(ivector_table), %g4
 	add	%g4, %g3, %g4
 
 	/* Insert ivector_table[] entry into __irq_work[] queue.  */
-	lduw	[%g1], %g2		/* g2 = irq_work(cpu) */
-	stw	%g2, [%g4 + 0x00]	/* bucket->irq_chain = g2 */
-	stw	%g4, [%g1]		/* irq_work(cpu) = bucket */
+	ldx	[%g1], %g2		/* g2 = irq_work(cpu) */
+	stx	%g2, [%g4 + 0x00]	/* bucket->irq_chain = g2 */
+	stx	%g4, [%g1]		/* irq_work(cpu) = bucket */
 
 	/* Signal the interrupt by setting (1 << pil) in %softint.  */
 	wr	%g0, 1 << PIL_DEVICE_IRQ, %set_softint

include/asm-sparc64/cpudata.h

@@ -75,12 +75,11 @@ struct trap_per_cpu {
 	unsigned long		tsb_huge_temp;
 
 /* Dcache line 8: IRQ work list, and keep trap_block a power-of-2 in size.  */
-	unsigned int		irq_worklist;
+	unsigned long		irq_worklist;
 	unsigned int		cpu_mondo_qmask;
 	unsigned int		dev_mondo_qmask;
 	unsigned int		resum_qmask;
 	unsigned int		nonresum_qmask;
-	unsigned int		__pad2[1];
 	void			*hdesc;
 } __attribute__((aligned(64)));
 extern struct trap_per_cpu trap_block[NR_CPUS];
@@ -129,10 +128,10 @@ extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch,
 #define TRAP_PER_CPU_TSB_HUGE		0xd0
 #define TRAP_PER_CPU_TSB_HUGE_TEMP	0xd8
 #define TRAP_PER_CPU_IRQ_WORKLIST	0xe0
-#define TRAP_PER_CPU_CPU_MONDO_QMASK	0xe4
-#define TRAP_PER_CPU_DEV_MONDO_QMASK	0xe8
-#define TRAP_PER_CPU_RESUM_QMASK	0xec
-#define TRAP_PER_CPU_NONRESUM_QMASK	0xf0
+#define TRAP_PER_CPU_CPU_MONDO_QMASK	0xe8
+#define TRAP_PER_CPU_DEV_MONDO_QMASK	0xec
+#define TRAP_PER_CPU_RESUM_QMASK	0xf0
+#define TRAP_PER_CPU_NONRESUM_QMASK	0xf4
 
 #define TRAP_BLOCK_SZ_SHIFT		8
 

include/asm-sparc64/irq.h

@@ -59,7 +59,7 @@ extern unsigned int sun4u_build_msi(u32 portid, unsigned int *virt_irq_p,
 extern void sun4u_destroy_msi(unsigned int virt_irq);
 extern unsigned int sbus_build_irq(void *sbus, unsigned int ino);
 
-extern unsigned char virt_irq_alloc(unsigned int real_irq);
+extern unsigned char virt_irq_alloc(unsigned long real_irq);
 #ifdef CONFIG_PCI_MSI
 extern void virt_irq_free(unsigned int virt_irq);
 #endif