diff --git a/mm/memory.c b/mm/memory.c
index 6c85cba021132..086cb3dd86081 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1465,8 +1465,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
 	/* Do the actual TLB flush before dropping ptl */
 	if (force_flush) {
 		tlb_flush_mmu_tlbonly(tlb);
-		if (tlb->delayed_rmap)
-			tlb_flush_rmaps(tlb, vma);
+		tlb_flush_rmaps(tlb, vma);
 	}
 	pte_unmap_unlock(start_pte, ptl);
 
diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c
index 1de1cf9ba581d..dd1f8ca40cb53 100644
--- a/mm/mmu_gather.c
+++ b/mm/mmu_gather.c
@@ -61,6 +61,9 @@ void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
 	struct mmu_gather_batch *batch;
 
+	if (!tlb->delayed_rmap)
+		return;
+
 	batch = tlb->active;
 	for (int i = 0; i < batch->nr; i++) {
 		struct encoded_page *enc = batch->encoded_pages[i];