[SPARC64]: More TLB/TSB handling fixes.

The SUN4V convention with non-shared TSBs is that the context
bit of the TAG is clear.  So we have to choose an "invalid"
bit and initialize new TSBs appropriately.  Otherwise a zero
TAG looks "valid".

Make sure, for the window fixup cases, that we use the right
global registers and that we don't potentially trample on
the live global registers in etrap/rtrap handling (%g2 and
%g6) and that we put the missing virtual address properly
in %g5.

Signed-off-by: David S. Miller <davem@davemloft.net>

arch/sparc64/kernel/dtlb_miss.S

@@ -2,10 +2,10 @@
 	ldxa	[%g0] ASI_DMMU_TSB_8KB_PTR, %g1	! Get TSB 8K pointer
 	ldxa	[%g0] ASI_DMMU, %g6		! Get TAG TARGET
 	srlx	%g6, 48, %g5			! Get context
+	sllx	%g6, 22, %g6			! Zero out context
 	brz,pn	%g5, kvmap_dtlb			! Context 0 processing
-	 nop					! Delay slot (fill me)
+	 srlx	%g6, 22, %g6			! Delay slot
 	TSB_LOAD_QUAD(%g1, %g4)			! Load TSB entry
-	nop					! Push branch to next I$ line
 	cmp	%g4, %g6			! Compare TAG
 
 /* DTLB ** ICACHE line 2: TSB compare and TLB load	*/

arch/sparc64/kernel/itlb_miss.S

@@ -2,25 +2,25 @@
 	ldxa	[%g0] ASI_IMMU_TSB_8KB_PTR, %g1	! Get TSB 8K pointer
 	ldxa	[%g0] ASI_IMMU, %g6		! Get TAG TARGET
 	srlx	%g6, 48, %g5			! Get context
+	sllx	%g6, 22, %g6			! Zero out context
 	brz,pn	%g5, kvmap_itlb			! Context 0 processing
-	 nop					! Delay slot (fill me)
+	 srlx	%g6, 22, %g6			! Delay slot
 	TSB_LOAD_QUAD(%g1, %g4)			! Load TSB entry
 	cmp	%g4, %g6			! Compare TAG
-	sethi	%hi(PAGE_EXEC), %g4		! Setup exec check
 
 /* ITLB ** ICACHE line 2: TSB compare and TLB load	*/
+	sethi	%hi(PAGE_EXEC), %g4		! Setup exec check
 	ldx	[%g4 + %lo(PAGE_EXEC)], %g4
 	bne,pn	%xcc, tsb_miss_itlb		! Miss
 	 mov	FAULT_CODE_ITLB, %g3
 	andcc	%g5, %g4, %g0			! Executable?
 	be,pn	%xcc, tsb_do_fault
 	 nop					! Delay slot, fill me
-	stxa	%g5, [%g0] ASI_ITLB_DATA_IN	! Load TLB
-	retry					! Trap done
+	nop
 
 /* ITLB ** ICACHE line 3: 				*/
-	nop
-	nop
+	stxa	%g5, [%g0] ASI_ITLB_DATA_IN	! Load TLB
+	retry					! Trap done
 	nop
 	nop
 	nop

arch/sparc64/kernel/ktlb.S

@@ -52,8 +52,10 @@ kvmap_itlb_vmalloc_addr:
 
 	/* Load and check PTE.  */
 	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
+	mov		1, %g7
+	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
 	brgez,a,pn	%g5, kvmap_itlb_longpath
-	 KTSB_STORE(%g1, %g0)
+	 KTSB_STORE(%g1, %g7)
 
 	KTSB_WRITE(%g1, %g5, %g6)
 
@@ -146,8 +148,10 @@ kvmap_dtlb_vmalloc_addr:
 
 	/* Load and check PTE.  */
 	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
+	mov		1, %g7
+	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
 	brgez,a,pn	%g5, kvmap_dtlb_longpath
-	 KTSB_STORE(%g1, %g0)
+	 KTSB_STORE(%g1, %g7)
 
 	KTSB_WRITE(%g1, %g5, %g6)
 
@@ -215,8 +219,8 @@ kvmap_dtlb_longpath:
 	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
 	.section .sun4v_2insn_patch, "ax"
 	.word	661b
-	nop
-	nop
+	SET_GL(1)
+	ldxa		[%g0] ASI_SCRATCHPAD, %g5
 	.previous
 
 	rdpr	%tl, %g3
@@ -226,7 +230,7 @@ kvmap_dtlb_longpath:
 	ldxa	[%g4] ASI_DMMU, %g5
 	.section .sun4v_2insn_patch, "ax"
 	.word	661b
-	mov	%g4, %g5
+	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
 	nop
 	.previous
 

arch/sparc64/kernel/sun4v_tlb_miss.S

@@ -16,15 +16,14 @@
 	ldx	[BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \
 	ldx	[BASE + HV_FAULT_D_CTX_OFFSET], CTX;
 
-	/* DEST = (CTX << 48) | (VADDR >> 22)
+	/* DEST = (VADDR >> 22)
 	 *
 	 * Branch to ZERO_CTX_LABEL is context is zero.
 	 */
-#define	COMPUTE_TAG_TARGET(DEST, VADDR, CTX, TMP, ZERO_CTX_LABEL) \
-	srlx	VADDR, 22, TMP; \
-	sllx	CTX, 48, DEST; \
+#define	COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \
+	srlx	VADDR, 22, DEST; \
 	brz,pn	CTX, ZERO_CTX_LABEL; \
-	 or	DEST, TMP, DEST;
+	 nop;
 
 	/* Create TSB pointer.  This is something like:
 	 *
@@ -53,7 +52,7 @@ sun4v_itlb_miss:
 	ldxa	[%g1] ASI_SCRATCHPAD, %g1
 
 	LOAD_ITLB_INFO(%g2, %g4, %g5)
-	COMPUTE_TAG_TARGET(%g6, %g4, %g5, %g3, kvmap_itlb_4v)
+	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
 	COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7)
 
 	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
@@ -72,15 +71,15 @@ sun4v_itlb_miss:
 	 *
 	 * %g3:	PTE
 	 * %g4:	vaddr
-	 * %g6:	TAG TARGET (only "CTX << 48" part matters)
 	 */
 sun4v_itlb_load:
+	ldxa	[%g0] ASI_SCRATCHPAD, %g6
 	mov	%o0, %g1		! save %o0
 	mov	%o1, %g2		! save %o1
 	mov	%o2, %g5		! save %o2
 	mov	%o3, %g7		! save %o3
 	mov	%g4, %o0		! vaddr
-	srlx	%g6, 48, %o1		! ctx
+	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1	! ctx
 	mov	%g3, %o2		! PTE
 	mov	HV_MMU_IMMU, %o3	! flags
 	ta	HV_MMU_MAP_ADDR_TRAP
@@ -101,7 +100,7 @@ sun4v_dtlb_miss:
 	ldxa	[%g1] ASI_SCRATCHPAD, %g1
 
 	LOAD_DTLB_INFO(%g2, %g4, %g5)
-	COMPUTE_TAG_TARGET(%g6, %g4, %g5, %g3, kvmap_dtlb_4v)
+	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
 	COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7)
 
 	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
@@ -115,15 +114,15 @@ sun4v_dtlb_miss:
 	 *
 	 * %g3:	PTE
 	 * %g4:	vaddr
-	 * %g6:	TAG TARGET (only "CTX << 48" part matters)
 	 */
 sun4v_dtlb_load:
+	ldxa	[%g0] ASI_SCRATCHPAD, %g6
 	mov	%o0, %g1		! save %o0
 	mov	%o1, %g2		! save %o1
 	mov	%o2, %g5		! save %o2
 	mov	%o3, %g7		! save %o3
 	mov	%g4, %o0		! vaddr
-	srlx	%g6, 48, %o1		! ctx
+	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1	! ctx
 	mov	%g3, %o2		! PTE
 	mov	HV_MMU_DMMU, %o3	! flags
 	ta	HV_MMU_MAP_ADDR_TRAP
@@ -136,16 +135,18 @@ sun4v_dtlb_load:
 	retry
 
 sun4v_dtlb_prot:
+	SET_GL(1)
+
 	/* Load MMU Miss base into %g2.  */
-	ldxa	[%g0] ASI_SCRATCHPAD, %g2
+	ldxa	[%g0] ASI_SCRATCHPAD, %g5
 
-	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
+	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
 	rdpr	%tl, %g1
 	cmp	%g1, 1
-	bgu,pn		%xcc, winfix_trampoline
+	bgu,pn	%xcc, winfix_trampoline
 	 nop
-	ba,pt		%xcc, sparc64_realfault_common
-	 mov		FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
+	ba,pt	%xcc, sparc64_realfault_common
+	 mov	FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
 
 	/* Called from trap table with TAG TARGET placed into
 	 * %g6, SCRATCHPAD_UTSBREG1 contents in %g1, and
@@ -189,7 +190,8 @@ sun4v_itlb_error:
 	sethi	%hi(sun4v_err_itlb_vaddr), %g1
 	stx	%g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
 	sethi	%hi(sun4v_err_itlb_ctx), %g1
-	srlx	%g6, 48, %o1		! ctx
+	ldxa	[%g0] ASI_SCRATCHPAD, %g6
+	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1
 	stx	%o1, [%g1 + %lo(sun4v_err_itlb_ctx)]
 	sethi	%hi(sun4v_err_itlb_pte), %g1
 	stx	%g3, [%g1 + %lo(sun4v_err_itlb_pte)]
@@ -214,7 +216,8 @@ sun4v_dtlb_error:
 	sethi	%hi(sun4v_err_dtlb_vaddr), %g1
 	stx	%g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
 	sethi	%hi(sun4v_err_dtlb_ctx), %g1
-	srlx	%g6, 48, %o1		! ctx
+	ldxa	[%g0] ASI_SCRATCHPAD, %g6
+	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1
 	stx	%o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]
 	sethi	%hi(sun4v_err_dtlb_pte), %g1
 	stx	%g3, [%g1 + %lo(sun4v_err_dtlb_pte)]

arch/sparc64/kernel/tsb.S

@@ -36,7 +36,7 @@ tsb_miss_itlb:
 	/* At this point we have:
 	 * %g4 --	missing virtual address
 	 * %g1 --	TSB entry address
-	 * %g6 --	TAG TARGET ((vaddr >> 22) | (ctx << 48))
+	 * %g6 --	TAG TARGET (vaddr >> 22)
 	 */
 tsb_miss_page_table_walk:
 	TRAP_LOAD_PGD_PHYS(%g7, %g5)
@@ -50,8 +50,10 @@ tsb_reload:
 
 	/* Load and check PTE.  */
 	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
+	mov		1, %g7
+	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
 	brgez,a,pn	%g5, tsb_do_fault
-	 TSB_STORE(%g1, %g0)
+	 TSB_STORE(%g1, %g7)
 
 	/* If it is larger than the base page size, don't
 	 * bother putting it into the TSB.
@@ -62,8 +64,10 @@ tsb_reload:
 	sethi		%hi(_PAGE_SZBITS), %g7
 	ldx		[%g7 + %lo(_PAGE_SZBITS)], %g7
 	cmp		%g2, %g7
+	mov		1, %g7
+	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
 	bne,a,pn	%xcc, tsb_tlb_reload
-	 TSB_STORE(%g1, %g0)
+	 TSB_STORE(%g1, %g7)
 
 	TSB_WRITE(%g1, %g5, %g6)
 
@@ -136,7 +140,7 @@ tsb_do_fault:
 	.section	.sun4v_2insn_patch, "ax"
 	.word		661b
 	SET_GL(1)
-	ldxa		[%g0] ASI_SCRATCHPAD, %g2
+	ldxa		[%g0] ASI_SCRATCHPAD, %g4
 	.previous
 
 	bne,pn		%xcc, tsb_do_itlb_fault
@@ -150,7 +154,7 @@ tsb_do_dtlb_fault:
 	ldxa	[%g4] ASI_DMMU, %g5
 	.section .sun4v_2insn_patch, "ax"
 	.word	661b
-	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
+	ldx	[%g4 + HV_FAULT_D_ADDR_OFFSET], %g5
 	nop
 	.previous
 
@@ -217,8 +221,9 @@ tsb_flush:
 	bne,pn	%icc, 1b
 	 membar	#LoadLoad
 	cmp	%g1, %o1
+	mov	1, %o3
 	bne,pt	%xcc, 2f
-	 clr	%o3
+	 sllx	%o3, TSB_TAG_INVALID_BIT, %o3
 	TSB_CAS_TAG(%o0, %g1, %o3)
 	cmp	%g1, %o3
 	bne,pn	%xcc, 1b

arch/sparc64/mm/init.c

@@ -296,7 +296,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t p
 
 		tsb = &mm->context.tsb[(address >> PAGE_SHIFT) &
 				       (mm->context.tsb_nentries - 1UL)];
-		tag = (address >> 22UL) | CTX_HWBITS(mm->context) << 48UL;
+		tag = (address >> 22UL);
 		tsb_insert(tsb, tag, pte_val(pte));
 	}
 }
@@ -1110,6 +1110,8 @@ void __init paging_init(void)
 	kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
 	kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
 
+	memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
+
 	if (tlb_type == hypervisor)
 		sun4v_pgprot_init();
 	else

arch/sparc64/mm/tsb.c

@@ -20,9 +20,9 @@ static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long nentries
 	return vaddr & (nentries - 1);
 }
 
-static inline int tag_compare(unsigned long tag, unsigned long vaddr, unsigned long context)
+static inline int tag_compare(unsigned long tag, unsigned long vaddr)
 {
-	return (tag == ((vaddr >> 22) | (context << 48)));
+	return (tag == (vaddr >> 22));
 }
 
 /* TSB flushes need only occur on the processor initiating the address
@@ -38,8 +38,8 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
 		unsigned long hash = tsb_hash(v, KERNEL_TSB_NENTRIES);
 		struct tsb *ent = &swapper_tsb[hash];
 
-		if (tag_compare(ent->tag, v, 0)) {
-			ent->tag = 0UL;
+		if (tag_compare(ent->tag, v)) {
+			ent->tag = (1UL << TSB_TAG_INVALID_BIT);
 			membar_storeload_storestore();
 		}
 	}
@@ -50,14 +50,9 @@ void flush_tsb_user(struct mmu_gather *mp)
 	struct mm_struct *mm = mp->mm;
 	struct tsb *tsb = mm->context.tsb;
 	unsigned long nentries = mm->context.tsb_nentries;
-	unsigned long ctx, base;
+	unsigned long base;
 	int i;
 
-	if (unlikely(!CTX_VALID(mm->context)))
-		return;
-
-	ctx = CTX_HWBITS(mm->context);
-
 	if (tlb_type == cheetah_plus || tlb_type == hypervisor)
 		base = __pa(tsb);
 	else
@@ -71,7 +66,7 @@ void flush_tsb_user(struct mmu_gather *mp)
 
 		hash = tsb_hash(v, nentries);
 		ent = base + (hash * sizeof(struct tsb));
-		tag = (v >> 22UL) | (ctx << 48UL);
+		tag = (v >> 22UL);
 
 		tsb_flush(ent, tag);
 	}
@@ -243,7 +238,8 @@ static void copy_tsb(struct tsb *old_tsb, unsigned long old_size,
 				  "i" (ASI_NUCLEUS_QUAD_LDD));
 		}
 
-		if (!tag || (tag & (1UL << TSB_TAG_LOCK_BIT)))
+		if (tag & ((1UL << TSB_TAG_LOCK_BIT) |
+			   (1UL << TSB_TAG_INVALID_BIT)))
 			continue;
 
 		/* We only put base page size PTEs into the TSB,
@@ -315,10 +311,13 @@ void tsb_grow(struct mm_struct *mm, unsigned long rss, gfp_t gfp_flags)
 			break;
 	}
 
-	page = alloc_pages(gfp_flags | __GFP_ZERO, get_order(size));
+	page = alloc_pages(gfp_flags, get_order(size));
 	if (unlikely(!page))
 		return;
 
+	/* Mark all tags as invalid.  */
+	memset(page_address(page), 0x40, size);
+
 	if (size == max_tsb_size)
 		mm->context.tsb_rss_limit = ~0UL;
 	else

include/asm-sparc64/tsb.h

@@ -12,6 +12,8 @@
  *
  * 	ldxa		[%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1
  * 	ldxa		[%g0] ASI_{D,I}MMU, %g6
+ *	sllx		%g6, 22, %g6
+ *	srlx		%g6, 22, %g6
  * 	ldda		[%g1] ASI_NUCLEUS_QUAD_LDD, %g4
  * 	cmp		%g4, %g6
  * 	bne,pn	%xcc, tsb_miss_{d,i}tlb
@@ -29,6 +31,9 @@
  * -------------------------------------------------
  *  63 61 60      48 47 42 41                     0
  *
+ * But actually, since we use per-mm TSB's, we zero out the CONTEXT
+ * field.
+ *
  * Like the powerpc hashtables we need to use locking in order to
  * synchronize while we update the entries.  PTE updates need locking
  * as well.
@@ -42,6 +47,9 @@
 #define TSB_TAG_LOCK_BIT	47
 #define TSB_TAG_LOCK_HIGH	(1 << (TSB_TAG_LOCK_BIT - 32))
 
+#define TSB_TAG_INVALID_BIT	46
+#define TSB_TAG_INVALID_HIGH	(1 << (TSB_TAG_INVALID_BIT - 32))
+
 #define TSB_MEMBAR	membar	#StoreStore
 
 /* Some cpus support physical address quad loads.  We want to use