Skip to content

Commit

Permalink
Merge tag 'for-joerg' of git://git.kernel.org/pub/scm/linux/kernel/gi…
Browse files Browse the repository at this point in the history 
…t/jgg/iommufd into core

iommu shared branch with iommufd

The three dependent series on a shared branch:

- Change the iommufd fault handle into an always present hwpt handle in
  the domain

- Give iommufd its own SW_MSI implementation along with some IRQ layer
  rework

- Improvements to the handle attach API
  • Loading branch information
Joerg Roedel committed Feb 28, 2025
2 parents 0ad2507 + 5e9f822 commit 59b6c34
Show file tree
Hide file tree
Showing 17 changed files with 501 additions and 357 deletions.
1 change: 0 additions & 1 deletion drivers/iommu/Kconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,6 @@ config IOMMU_DMA
select DMA_OPS_HELPERS
select IOMMU_API
select IOMMU_IOVA
select IRQ_MSI_IOMMU
select NEED_SG_DMA_LENGTH
select NEED_SG_DMA_FLAGS if SWIOTLB

Expand Down
65 changes: 21 additions & 44 deletions drivers/iommu/dma-iommu.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <linux/memremap.h>
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/msi.h>
#include <linux/of_iommu.h>
#include <linux/pci.h>
#include <linux/scatterlist.h>
Expand Down Expand Up @@ -102,6 +103,9 @@ static int __init iommu_dma_forcedac_setup(char *str)
}
early_param("iommu.forcedac", iommu_dma_forcedac_setup);

static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr);

/* Number of entries per flush queue */
#define IOVA_DEFAULT_FQ_SIZE 256
#define IOVA_SINGLE_FQ_SIZE 32768
Expand Down Expand Up @@ -398,6 +402,7 @@ int iommu_get_dma_cookie(struct iommu_domain *domain)
return -ENOMEM;

mutex_init(&domain->iova_cookie->mutex);
iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);
return 0;
}

Expand Down Expand Up @@ -429,6 +434,7 @@ int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)

cookie->msi_iova = base;
domain->iova_cookie = cookie;
iommu_domain_set_sw_msi(domain, iommu_dma_sw_msi);
return 0;
}
EXPORT_SYMBOL(iommu_get_msi_cookie);
Expand All @@ -443,6 +449,11 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)
struct iommu_dma_cookie *cookie = domain->iova_cookie;
struct iommu_dma_msi_page *msi, *tmp;

#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
if (domain->sw_msi != iommu_dma_sw_msi)
return;
#endif

if (!cookie)
return;

Expand Down Expand Up @@ -1800,60 +1811,26 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
return NULL;
}

/**
* iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain
* @desc: MSI descriptor, will store the MSI page
* @msi_addr: MSI target address to be mapped
*
* Return: 0 on success or negative error code if the mapping failed.
*/
int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
static int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
phys_addr_t msi_addr)
{
struct device *dev = msi_desc_to_dev(desc);
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct iommu_dma_msi_page *msi_page;
static DEFINE_MUTEX(msi_prepare_lock); /* see below */
const struct iommu_dma_msi_page *msi_page;

if (!domain || !domain->iova_cookie) {
desc->iommu_cookie = NULL;
if (!domain->iova_cookie) {
msi_desc_set_iommu_msi_iova(desc, 0, 0);
return 0;
}

/*
* In fact the whole prepare operation should already be serialised by
* irq_domain_mutex further up the callchain, but that's pretty subtle
* on its own, so consider this locking as failsafe documentation...
*/
mutex_lock(&msi_prepare_lock);
iommu_group_mutex_assert(dev);
msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain);
mutex_unlock(&msi_prepare_lock);

msi_desc_set_iommu_cookie(desc, msi_page);

if (!msi_page)
return -ENOMEM;
return 0;
}

/**
* iommu_dma_compose_msi_msg() - Apply translation to an MSI message
* @desc: MSI descriptor prepared by iommu_dma_prepare_msi()
* @msg: MSI message containing target physical address
*/
void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
{
struct device *dev = msi_desc_to_dev(desc);
const struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
const struct iommu_dma_msi_page *msi_page;

msi_page = msi_desc_get_iommu_cookie(desc);

if (!domain || !domain->iova_cookie || WARN_ON(!msi_page))
return;

msg->address_hi = upper_32_bits(msi_page->iova);
msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1;
msg->address_lo += lower_32_bits(msi_page->iova);
msi_desc_set_iommu_msi_iova(
desc, msi_page->iova,
ilog2(cookie_msi_granule(domain->iova_cookie)));
return 0;
}

static int iommu_dma_init(void)
Expand Down
3 changes: 0 additions & 3 deletions drivers/iommu/iommu-priv.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@ static inline const struct iommu_ops *iommu_fwspec_ops(struct iommu_fwspec *fwsp
return iommu_ops_from_fwnode(fwspec ? fwspec->iommu_fwnode : NULL);
}

int iommu_group_replace_domain(struct iommu_group *group,
struct iommu_domain *new_domain);

int iommu_device_register_bus(struct iommu_device *iommu,
const struct iommu_ops *ops,
const struct bus_type *bus,
Expand Down
164 changes: 106 additions & 58 deletions drivers/iommu/iommu.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ static unsigned int iommu_def_domain_type __read_mostly;
static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_DMA_STRICT);
static u32 iommu_cmd_line __read_mostly;

/* Tags used with xa_tag_pointer() in group->pasid_array */
enum { IOMMU_PASID_ARRAY_DOMAIN = 0, IOMMU_PASID_ARRAY_HANDLE = 1 };

struct iommu_group {
struct kobject kobj;
struct kobject *devices_kobj;
Expand Down Expand Up @@ -2147,6 +2150,17 @@ struct iommu_domain *iommu_get_dma_domain(struct device *dev)
return dev->iommu_group->default_domain;
}

static void *iommu_make_pasid_array_entry(struct iommu_domain *domain,
struct iommu_attach_handle *handle)
{
if (handle) {
handle->domain = domain;
return xa_tag_pointer(handle, IOMMU_PASID_ARRAY_HANDLE);
}

return xa_tag_pointer(domain, IOMMU_PASID_ARRAY_DOMAIN);
}

static int __iommu_attach_group(struct iommu_domain *domain,
struct iommu_group *group)
{
Expand Down Expand Up @@ -2187,32 +2201,6 @@ int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
}
EXPORT_SYMBOL_GPL(iommu_attach_group);

/**
* iommu_group_replace_domain - replace the domain that a group is attached to
* @group: IOMMU group that will be attached to the new domain
* @new_domain: new IOMMU domain to replace with
*
* This API allows the group to switch domains without being forced to go to
* the blocking domain in-between.
*
* If the currently attached domain is a core domain (e.g. a default_domain),
* it will act just like the iommu_attach_group().
*/
int iommu_group_replace_domain(struct iommu_group *group,
struct iommu_domain *new_domain)
{
int ret;

if (!new_domain)
return -EINVAL;

mutex_lock(&group->mutex);
ret = __iommu_group_set_domain(group, new_domain);
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_NS_GPL(iommu_group_replace_domain, "IOMMUFD_INTERNAL");

static int __iommu_device_set_domain(struct iommu_group *group,
struct device *dev,
struct iommu_domain *new_domain,
Expand Down Expand Up @@ -3374,6 +3362,7 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
struct iommu_group *group = dev->iommu_group;
struct group_device *device;
const struct iommu_ops *ops;
void *entry;
int ret;

if (!group)
Expand All @@ -3397,16 +3386,31 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
}
}

if (handle)
handle->domain = domain;
entry = iommu_make_pasid_array_entry(domain, handle);

ret = xa_insert(&group->pasid_array, pasid, handle, GFP_KERNEL);
/*
* Entry present is a failure case. Use xa_insert() instead of
* xa_reserve().
*/
ret = xa_insert(&group->pasid_array, pasid, XA_ZERO_ENTRY, GFP_KERNEL);
if (ret)
goto out_unlock;

ret = __iommu_set_group_pasid(domain, group, pasid);
if (ret)
xa_erase(&group->pasid_array, pasid);
if (ret) {
xa_release(&group->pasid_array, pasid);
goto out_unlock;
}

/*
* The xa_insert() above reserved the memory, and the group->mutex is
* held, this cannot fail. The new domain cannot be visible until the
* operation succeeds as we cannot tolerate PRIs becoming concurrently
* queued and then failing attach.
*/
WARN_ON(xa_is_err(xa_store(&group->pasid_array,
pasid, entry, GFP_KERNEL)));

out_unlock:
mutex_unlock(&group->mutex);
return ret;
Expand Down Expand Up @@ -3480,13 +3484,17 @@ struct iommu_attach_handle *
iommu_attach_handle_get(struct iommu_group *group, ioasid_t pasid, unsigned int type)
{
struct iommu_attach_handle *handle;
void *entry;

xa_lock(&group->pasid_array);
handle = xa_load(&group->pasid_array, pasid);
if (!handle)
entry = xa_load(&group->pasid_array, pasid);
if (!entry || xa_pointer_tag(entry) != IOMMU_PASID_ARRAY_HANDLE) {
handle = ERR_PTR(-ENOENT);
else if (type && handle->domain->type != type)
handle = ERR_PTR(-EBUSY);
} else {
handle = xa_untag_pointer(entry);
if (type && handle->domain->type != type)
handle = ERR_PTR(-EBUSY);
}
xa_unlock(&group->pasid_array);

return handle;
Expand All @@ -3509,25 +3517,35 @@ int iommu_attach_group_handle(struct iommu_domain *domain,
struct iommu_group *group,
struct iommu_attach_handle *handle)
{
void *entry;
int ret;

if (handle)
handle->domain = domain;
if (!handle)
return -EINVAL;

mutex_lock(&group->mutex);
ret = xa_insert(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
entry = iommu_make_pasid_array_entry(domain, handle);
ret = xa_insert(&group->pasid_array,
IOMMU_NO_PASID, XA_ZERO_ENTRY, GFP_KERNEL);
if (ret)
goto err_unlock;
goto out_unlock;

ret = __iommu_attach_group(domain, group);
if (ret)
goto err_erase;
mutex_unlock(&group->mutex);
if (ret) {
xa_release(&group->pasid_array, IOMMU_NO_PASID);
goto out_unlock;
}

return 0;
err_erase:
xa_erase(&group->pasid_array, IOMMU_NO_PASID);
err_unlock:
/*
* The xa_insert() above reserved the memory, and the group->mutex is
* held, this cannot fail. The new domain cannot be visible until the
* operation succeeds as we cannot tolerate PRIs becoming concurrently
* queued and then failing attach.
*/
WARN_ON(xa_is_err(xa_store(&group->pasid_array,
IOMMU_NO_PASID, entry, GFP_KERNEL)));

out_unlock:
mutex_unlock(&group->mutex);
return ret;
}
Expand Down Expand Up @@ -3557,33 +3575,34 @@ EXPORT_SYMBOL_NS_GPL(iommu_detach_group_handle, "IOMMUFD_INTERNAL");
* @new_domain: new IOMMU domain to replace with
* @handle: attach handle
*
* This is a variant of iommu_group_replace_domain(). It allows the caller to
* provide an attach handle for the new domain and use it when the domain is
* attached.
* This API allows the group to switch domains without being forced to go to
* the blocking domain in-between. It allows the caller to provide an attach
* handle for the new domain and use it when the domain is attached.
*
* If the currently attached domain is a core domain (e.g. a default_domain),
* it will act just like the iommu_attach_group_handle().
*/
int iommu_replace_group_handle(struct iommu_group *group,
struct iommu_domain *new_domain,
struct iommu_attach_handle *handle)
{
void *curr;
void *curr, *entry;
int ret;

if (!new_domain)
if (!new_domain || !handle)
return -EINVAL;

mutex_lock(&group->mutex);
if (handle) {
ret = xa_reserve(&group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);
if (ret)
goto err_unlock;
handle->domain = new_domain;
}
entry = iommu_make_pasid_array_entry(new_domain, handle);
ret = xa_reserve(&group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);
if (ret)
goto err_unlock;

ret = __iommu_group_set_domain(group, new_domain);
if (ret)
goto err_release;

curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, handle, GFP_KERNEL);
curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, entry, GFP_KERNEL);
WARN_ON(xa_is_err(curr));

mutex_unlock(&group->mutex);
Expand All @@ -3596,3 +3615,32 @@ int iommu_replace_group_handle(struct iommu_group *group,
return ret;
}
EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL");

#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
/**
* iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain
* @desc: MSI descriptor, will store the MSI page
* @msi_addr: MSI target address to be mapped
*
* The implementation of sw_msi() should take msi_addr and map it to
* an IOVA in the domain and call msi_desc_set_iommu_msi_iova() with the
* mapping information.
*
* Return: 0 on success or negative error code if the mapping failed.
*/
int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
{
struct device *dev = msi_desc_to_dev(desc);
struct iommu_group *group = dev->iommu_group;
int ret = 0;

if (!group)
return 0;

mutex_lock(&group->mutex);
if (group->domain && group->domain->sw_msi)
ret = group->domain->sw_msi(group->domain, desc, msi_addr);
mutex_unlock(&group->mutex);
return ret;
}
#endif /* CONFIG_IRQ_MSI_IOMMU */
Loading

0 comments on commit 59b6c34

Please sign in to comment.