Skip to content

Commit

Permalink
iommu/vt-d: Use iommu_sva_alloc(free)_pasid() helpers
Browse files Browse the repository at this point in the history 
Align the pasid alloc/free code with the generic helpers defined in the
iommu core. This also refactored the SVA binding code to improve the
readability.

Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Link: https://lore.kernel.org/r/20210520031531.712333-1-baolu.lu@linux.intel.com
Link: https://lore.kernel.org/r/20210610020115.1637656-8-baolu.lu@linux.intel.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>
  • Loading branch information
Lu Baolu authored and Joerg Roedel committed Jun 10, 2021
1 parent 100b8a1 commit 4048377
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 163 deletions.
1 change: 1 addition & 0 deletions drivers/iommu/intel/Kconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ config INTEL_IOMMU_SVM
select PCI_PRI
select MMU_NOTIFIER
select IOASID
select IOMMU_SVA_LIB
help
Shared Virtual Memory (SVM) provides a facility for devices
to access DMA resources through process address space by
Expand Down
3 changes: 3 additions & 0 deletions drivers/iommu/intel/iommu.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5411,6 +5411,9 @@ intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
if (!info)
return -EINVAL;

if (intel_iommu_enable_pasid(info->iommu, dev))
return -ENODEV;

if (!info->pasid_enabled || !info->pri_enabled || !info->ats_enabled)
return -EINVAL;

Expand Down
278 changes: 116 additions & 162 deletions drivers/iommu/intel/svm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@
#include <asm/fpu/api.h>

#include "pasid.h"
#include "../iommu-sva-lib.h"

static irqreturn_t prq_event_thread(int irq, void *d);
static void intel_svm_drain_prq(struct device *dev, u32 pasid);
#define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva)

#define PRQ_ORDER 0

Expand Down Expand Up @@ -222,7 +224,6 @@ static const struct mmu_notifier_ops intel_mmuops = {
};

static DEFINE_MUTEX(pasid_mutex);
static LIST_HEAD(global_svm_list);

#define for_each_svm_dev(sdev, svm, d) \
list_for_each_entry((sdev), &(svm)->devs, list) \
Expand Down Expand Up @@ -477,176 +478,118 @@ static void load_pasid(struct mm_struct *mm, u32 pasid)
mutex_unlock(&mm->context.lock);
}

/* Caller must hold pasid_mutex, mm reference */
static int
intel_svm_bind_mm(struct device *dev, unsigned int flags,
struct mm_struct *mm, struct intel_svm_dev **sd)
static int intel_svm_alloc_pasid(struct device *dev, struct mm_struct *mm,
unsigned int flags)
{
struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
struct intel_svm *svm = NULL, *t;
struct device_domain_info *info;
struct intel_svm_dev *sdev;
unsigned long iflags;
int pasid_max;
int ret;
ioasid_t max_pasid = dev_is_pci(dev) ?
pci_max_pasids(to_pci_dev(dev)) : intel_pasid_max_id;

if (!iommu || dmar_disabled)
return -EINVAL;
return iommu_sva_alloc_pasid(mm, PASID_MIN, max_pasid - 1);
}

if (!intel_svm_capable(iommu))
return -ENOTSUPP;
static void intel_svm_free_pasid(struct mm_struct *mm)
{
iommu_sva_free_pasid(mm);
}

if (dev_is_pci(dev)) {
pasid_max = pci_max_pasids(to_pci_dev(dev));
if (pasid_max < 0)
return -EINVAL;
} else
pasid_max = 1 << 20;
static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu,
struct device *dev,
struct mm_struct *mm,
unsigned int flags)
{
struct device_domain_info *info = get_domain_info(dev);
unsigned long iflags, sflags;
struct intel_svm_dev *sdev;
struct intel_svm *svm;
int ret = 0;

/* Bind supervisor PASID shuld have mm = NULL */
if (flags & SVM_FLAG_SUPERVISOR_MODE) {
if (!ecap_srs(iommu->ecap) || mm) {
pr_err("Supervisor PASID with user provided mm.\n");
return -EINVAL;
}
}
svm = pasid_private_find(mm->pasid);
if (!svm) {
svm = kzalloc(sizeof(*svm), GFP_KERNEL);
if (!svm)
return ERR_PTR(-ENOMEM);

list_for_each_entry(t, &global_svm_list, list) {
if (t->mm != mm)
continue;
svm->pasid = mm->pasid;
svm->mm = mm;
svm->flags = flags;
INIT_LIST_HEAD_RCU(&svm->devs);

svm = t;
if (svm->pasid >= pasid_max) {
dev_warn(dev,
"Limited PASID width. Cannot use existing PASID %d\n",
svm->pasid);
ret = -ENOSPC;
goto out;
if (!(flags & SVM_FLAG_SUPERVISOR_MODE)) {
svm->notifier.ops = &intel_mmuops;
ret = mmu_notifier_register(&svm->notifier, mm);
if (ret) {
kfree(svm);
return ERR_PTR(ret);
}
}

/* Find the matching device in svm list */
for_each_svm_dev(sdev, svm, dev) {
sdev->users++;
goto success;
ret = pasid_private_add(svm->pasid, svm);
if (ret) {
if (svm->notifier.ops)
mmu_notifier_unregister(&svm->notifier, mm);
kfree(svm);
return ERR_PTR(ret);
}
}

break;
/* Find the matching device in svm list */
for_each_svm_dev(sdev, svm, dev) {
sdev->users++;
goto success;
}

sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
if (!sdev) {
ret = -ENOMEM;
goto out;
goto free_svm;
}

sdev->dev = dev;
sdev->iommu = iommu;

ret = intel_iommu_enable_pasid(iommu, dev);
if (ret) {
kfree(sdev);
goto out;
}

info = get_domain_info(dev);
sdev->did = FLPT_DEFAULT_DID;
sdev->sid = PCI_DEVID(info->bus, info->devfn);
sdev->users = 1;
sdev->pasid = svm->pasid;
sdev->sva.dev = dev;
init_rcu_head(&sdev->rcu);
if (info->ats_enabled) {
sdev->dev_iotlb = 1;
sdev->qdep = info->ats_qdep;
if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
sdev->qdep = 0;
}

/* Finish the setup now we know we're keeping it */
sdev->users = 1;
init_rcu_head(&sdev->rcu);

if (!svm) {
svm = kzalloc(sizeof(*svm), GFP_KERNEL);
if (!svm) {
ret = -ENOMEM;
goto sdev_err;
}

if (pasid_max > intel_pasid_max_id)
pasid_max = intel_pasid_max_id;

/* Do not use PASID 0, reserved for RID to PASID */
svm->pasid = ioasid_alloc(NULL, PASID_MIN,
pasid_max - 1, NULL);
if (svm->pasid == INVALID_IOASID) {
ret = -ENOSPC;
goto svm_err;
}

ret = pasid_private_add(svm->pasid, svm);
if (ret)
goto pasid_err;
/* Setup the pasid table: */
sflags = (flags & SVM_FLAG_SUPERVISOR_MODE) ?
PASID_FLAG_SUPERVISOR_MODE : 0;
sflags |= cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
spin_lock_irqsave(&iommu->lock, iflags);
ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, mm->pasid,
FLPT_DEFAULT_DID, sflags);
spin_unlock_irqrestore(&iommu->lock, iflags);

svm->notifier.ops = &intel_mmuops;
svm->mm = mm;
svm->flags = flags;
INIT_LIST_HEAD_RCU(&svm->devs);
INIT_LIST_HEAD(&svm->list);
ret = -ENOMEM;
if (mm) {
ret = mmu_notifier_register(&svm->notifier, mm);
if (ret)
goto priv_err;
}
if (ret)
goto free_sdev;

spin_lock_irqsave(&iommu->lock, iflags);
ret = intel_pasid_setup_first_level(iommu, dev,
mm ? mm->pgd : init_mm.pgd,
svm->pasid, FLPT_DEFAULT_DID,
(mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) |
(cpu_feature_enabled(X86_FEATURE_LA57) ?
PASID_FLAG_FL5LP : 0));
spin_unlock_irqrestore(&iommu->lock, iflags);
if (ret) {
if (mm)
mmu_notifier_unregister(&svm->notifier, mm);
priv_err:
pasid_private_remove(svm->pasid);
pasid_err:
ioasid_put(svm->pasid);
svm_err:
kfree(svm);
sdev_err:
kfree(sdev);
goto out;
}
/* The newly allocated pasid is loaded to the mm. */
if (!(flags & SVM_FLAG_SUPERVISOR_MODE) && list_empty(&svm->devs))
load_pasid(mm, svm->pasid);

list_add_tail(&svm->list, &global_svm_list);
if (mm) {
/* The newly allocated pasid is loaded to the mm. */
load_pasid(mm, svm->pasid);
}
} else {
/*
* Binding a new device with existing PASID, need to setup
* the PASID entry.
*/
spin_lock_irqsave(&iommu->lock, iflags);
ret = intel_pasid_setup_first_level(iommu, dev,
mm ? mm->pgd : init_mm.pgd,
svm->pasid, FLPT_DEFAULT_DID,
(mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) |
(cpu_feature_enabled(X86_FEATURE_LA57) ?
PASID_FLAG_FL5LP : 0));
spin_unlock_irqrestore(&iommu->lock, iflags);
if (ret)
goto sdev_err;
}
list_add_rcu(&sdev->list, &svm->devs);
success:
sdev->pasid = svm->pasid;
sdev->sva.dev = dev;
if (sd)
*sd = sdev;
ret = 0;
out:
return ret;
return &sdev->sva;

free_sdev:
kfree(sdev);
free_svm:
if (list_empty(&svm->devs)) {
if (svm->notifier.ops)
mmu_notifier_unregister(&svm->notifier, mm);
pasid_private_remove(mm->pasid);
kfree(svm);
}

return ERR_PTR(ret);
}

/* Caller must hold pasid_mutex */
Expand All @@ -655,6 +598,7 @@ static int intel_svm_unbind_mm(struct device *dev, u32 pasid)
struct intel_svm_dev *sdev;
struct intel_iommu *iommu;
struct intel_svm *svm;
struct mm_struct *mm;
int ret = -EINVAL;

iommu = device_to_iommu(dev, NULL, NULL);
Expand All @@ -664,6 +608,7 @@ static int intel_svm_unbind_mm(struct device *dev, u32 pasid)
ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);
if (ret)
goto out;
mm = svm->mm;

if (sdev) {
sdev->users--;
Expand All @@ -682,13 +627,12 @@ static int intel_svm_unbind_mm(struct device *dev, u32 pasid)
kfree_rcu(sdev, rcu);

if (list_empty(&svm->devs)) {
ioasid_put(svm->pasid);
if (svm->mm) {
mmu_notifier_unregister(&svm->notifier, svm->mm);
intel_svm_free_pasid(mm);
if (svm->notifier.ops) {
mmu_notifier_unregister(&svm->notifier, mm);
/* Clear mm's pasid. */
load_pasid(svm->mm, PASID_DISABLED);
load_pasid(mm, PASID_DISABLED);
}
list_del(&svm->list);
pasid_private_remove(svm->pasid);
/* We mandate that no page faults may be outstanding
* for the PASID when intel_svm_unbind_mm() is called.
Expand Down Expand Up @@ -1073,42 +1017,52 @@ static irqreturn_t prq_event_thread(int irq, void *d)
return IRQ_RETVAL(handled);
}

#define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva)
struct iommu_sva *
intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
{
struct iommu_sva *sva = ERR_PTR(-EINVAL);
struct intel_svm_dev *sdev = NULL;
struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
unsigned int flags = 0;
struct iommu_sva *sva;
int ret;

/*
* TODO: Consolidate with generic iommu-sva bind after it is merged.
* It will require shared SVM data structures, i.e. combine io_mm
* and intel_svm etc.
*/
if (drvdata)
flags = *(unsigned int *)drvdata;

if (flags & SVM_FLAG_SUPERVISOR_MODE) {
if (!ecap_srs(iommu->ecap)) {
dev_err(dev, "%s: Supervisor PASID not supported\n",
iommu->name);
return ERR_PTR(-EOPNOTSUPP);
}

if (mm) {
dev_err(dev, "%s: Supervisor PASID with user provided mm\n",
iommu->name);
return ERR_PTR(-EINVAL);
}

mm = &init_mm;
}

mutex_lock(&pasid_mutex);
ret = intel_svm_bind_mm(dev, flags, mm, &sdev);
if (ret)
sva = ERR_PTR(ret);
else if (sdev)
sva = &sdev->sva;
else
WARN(!sdev, "SVM bind succeeded with no sdev!\n");
ret = intel_svm_alloc_pasid(dev, mm, flags);
if (ret) {
mutex_unlock(&pasid_mutex);
return ERR_PTR(ret);
}

sva = intel_svm_bind_mm(iommu, dev, mm, flags);
if (IS_ERR_OR_NULL(sva))
intel_svm_free_pasid(mm);
mutex_unlock(&pasid_mutex);

return sva;
}

void intel_svm_unbind(struct iommu_sva *sva)
{
struct intel_svm_dev *sdev;
struct intel_svm_dev *sdev = to_intel_svm_dev(sva);

mutex_lock(&pasid_mutex);
sdev = to_intel_svm_dev(sva);
intel_svm_unbind_mm(sdev->dev, sdev->pasid);
mutex_unlock(&pasid_mutex);
}
Expand Down
Loading

0 comments on commit 4048377

Please sign in to comment.