Skip to content

Commit

Permalink
iommu/vt-d: Cleanup use of iommu_flush_iotlb_psi()
Browse files Browse the repository at this point in the history 
Use cache_tag_flush_range() in switch_to_super_page() to invalidate the
necessary caches when switching mappings from normal to super pages. The
iommu_flush_iotlb_psi() call in intel_iommu_memory_notifier() is
unnecessary since there should be no cache invalidation for the identity
domain.

Clean up iommu_flush_iotlb_psi() after the last call site is removed.

Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Link: https://lore.kernel.org/r/20240416080656.60968-8-baolu.lu@linux.intel.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>
  • Loading branch information
Lu Baolu authored and Joerg Roedel committed Apr 26, 2024
1 parent 129dab6 commit 06792d0
Showing 1 changed file with 2 additions and 169 deletions.
171 changes: 2 additions & 169 deletions drivers/iommu/intel/iommu.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1390,157 +1390,6 @@ static void __iommu_flush_dev_iotlb(struct device_domain_info *info,
quirk_extra_dev_tlb_flush(info, addr, mask, IOMMU_NO_PASID, qdep);
}

static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
u64 addr, unsigned mask)
{
struct dev_pasid_info *dev_pasid;
struct device_domain_info *info;
unsigned long flags;

if (!domain->has_iotlb_device)
return;

spin_lock_irqsave(&domain->lock, flags);
list_for_each_entry(info, &domain->devices, link)
__iommu_flush_dev_iotlb(info, addr, mask);

list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) {
info = dev_iommu_priv_get(dev_pasid->dev);

if (!info->ats_enabled)
continue;

qi_flush_dev_iotlb_pasid(info->iommu,
PCI_DEVID(info->bus, info->devfn),
info->pfsid, dev_pasid->pasid,
info->ats_qdep, addr,
mask);
}
spin_unlock_irqrestore(&domain->lock, flags);
}

static void domain_flush_pasid_iotlb(struct intel_iommu *iommu,
struct dmar_domain *domain, u64 addr,
unsigned long npages, bool ih)
{
u16 did = domain_id_iommu(domain, iommu);
struct dev_pasid_info *dev_pasid;
unsigned long flags;

spin_lock_irqsave(&domain->lock, flags);
list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain)
qi_flush_piotlb(iommu, did, dev_pasid->pasid, addr, npages, ih);

if (!list_empty(&domain->devices))
qi_flush_piotlb(iommu, did, IOMMU_NO_PASID, addr, npages, ih);
spin_unlock_irqrestore(&domain->lock, flags);
}

static void __iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
unsigned long pfn, unsigned int pages,
int ih)
{
unsigned int aligned_pages = __roundup_pow_of_two(pages);
unsigned long bitmask = aligned_pages - 1;
unsigned int mask = ilog2(aligned_pages);
u64 addr = (u64)pfn << VTD_PAGE_SHIFT;

/*
* PSI masks the low order bits of the base address. If the
* address isn't aligned to the mask, then compute a mask value
* needed to ensure the target range is flushed.
*/
if (unlikely(bitmask & pfn)) {
unsigned long end_pfn = pfn + pages - 1, shared_bits;

/*
* Since end_pfn <= pfn + bitmask, the only way bits
* higher than bitmask can differ in pfn and end_pfn is
* by carrying. This means after masking out bitmask,
* high bits starting with the first set bit in
* shared_bits are all equal in both pfn and end_pfn.
*/
shared_bits = ~(pfn ^ end_pfn) & ~bitmask;
mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG;
}

/*
* Fallback to domain selective flush if no PSI support or
* the size is too big.
*/
if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
iommu->flush.flush_iotlb(iommu, did, 0, 0,
DMA_TLB_DSI_FLUSH);
else
iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
DMA_TLB_PSI_FLUSH);
}

static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
struct dmar_domain *domain,
unsigned long pfn, unsigned int pages,
int ih, int map)
{
unsigned int aligned_pages = __roundup_pow_of_two(pages);
unsigned int mask = ilog2(aligned_pages);
uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
u16 did = domain_id_iommu(domain, iommu);

if (WARN_ON(!pages))
return;

if (ih)
ih = 1 << 6;

if (domain->use_first_level)
domain_flush_pasid_iotlb(iommu, domain, addr, pages, ih);
else
__iommu_flush_iotlb_psi(iommu, did, pfn, pages, ih);

if (!map)
iommu_flush_dev_iotlb(domain, addr, mask);
}

/*
* Flush the relevant caches in nested translation if the domain
* also serves as a parent
*/
static void parent_domain_flush(struct dmar_domain *domain,
unsigned long pfn,
unsigned long pages, int ih)
{
struct dmar_domain *s1_domain;

spin_lock(&domain->s1_lock);
list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
struct device_domain_info *device_info;
struct iommu_domain_info *info;
unsigned long flags;
unsigned long i;

xa_for_each(&s1_domain->iommu_array, i, info)
__iommu_flush_iotlb_psi(info->iommu, info->did,
pfn, pages, ih);

if (!s1_domain->has_iotlb_device)
continue;

spin_lock_irqsave(&s1_domain->lock, flags);
list_for_each_entry(device_info, &s1_domain->devices, link)
/*
* Address translation cache in device side caches the
* result of nested translation. There is no easy way
* to identify the exact set of nested translations
* affected by a change in S2. So just flush the entire
* device cache.
*/
__iommu_flush_dev_iotlb(device_info, 0,
MAX_AGAW_PFN_WIDTH);
spin_unlock_irqrestore(&s1_domain->lock, flags);
}
spin_unlock(&domain->s1_lock);
}

static void intel_flush_iotlb_all(struct iommu_domain *domain)
{
cache_tag_flush_all(to_dmar_domain(domain));
Expand Down Expand Up @@ -1991,9 +1840,7 @@ static void switch_to_super_page(struct dmar_domain *domain,
unsigned long end_pfn, int level)
{
unsigned long lvl_pages = lvl_to_nr_pages(level);
struct iommu_domain_info *info;
struct dma_pte *pte = NULL;
unsigned long i;

while (start_pfn <= end_pfn) {
if (!pte)
Expand All @@ -2005,13 +1852,8 @@ static void switch_to_super_page(struct dmar_domain *domain,
start_pfn + lvl_pages - 1,
level + 1);

xa_for_each(&domain->iommu_array, i, info)
iommu_flush_iotlb_psi(info->iommu, domain,
start_pfn, lvl_pages,
0, 0);
if (domain->nested_parent)
parent_domain_flush(domain, start_pfn,
lvl_pages, 0);
cache_tag_flush_range(domain, start_pfn << VTD_PAGE_SHIFT,
end_pfn << VTD_PAGE_SHIFT, 0);
}

pte++;
Expand Down Expand Up @@ -3381,18 +3223,9 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
case MEM_OFFLINE:
case MEM_CANCEL_ONLINE:
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
LIST_HEAD(freelist);

domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist);

rcu_read_lock();
for_each_active_iommu(iommu, drhd)
iommu_flush_iotlb_psi(iommu, si_domain,
start_vpfn, mhp->nr_pages,
list_empty(&freelist), 0);
rcu_read_unlock();
put_pages_list(&freelist);
}
break;
Expand Down

0 comments on commit 06792d0

Please sign in to comment.