Skip to content

Commit

Permalink
iommu/amd: Modify set_dte_entry() to use 256-bit DTE helpers
Browse files Browse the repository at this point in the history 
Also, the set_dte_entry() is used to program several DTE fields (e.g.
stage1 table, stage2 table, domain id, and etc.), which is difficult
to keep track with current implementation.

Therefore, separate logic for clearing DTE (i.e. make_clear_dte) and
another function for setting up the GCR3 Table Root Pointer, GIOV, GV,
GLX, and GuestPagingMode into another function set_dte_gcr3_table().

Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Link: https://lore.kernel.org/r/20241118054937.5203-6-suravee.suthikulpanit@amd.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>
  • Loading branch information
Suravee Suthikulpanit authored and Joerg Roedel committed Dec 18, 2024
1 parent 8b3f787 commit fd5dff9
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 68 deletions.
2 changes: 2 additions & 0 deletions drivers/iommu/amd/amd_iommu.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,3 +184,5 @@ void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
struct dev_table_entry *get_dev_table(struct amd_iommu *iommu);

#endif

struct dev_table_entry *amd_iommu_get_ivhd_dte_flags(u16 segid, u16 devid);
13 changes: 4 additions & 9 deletions drivers/iommu/amd/amd_iommu_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,22 +409,17 @@
#define DTE_FLAG_HAD (3ULL << 7)
#define DTE_FLAG_GIOV BIT_ULL(54)
#define DTE_FLAG_GV BIT_ULL(55)
#define DTE_GLX_SHIFT (56)
#define DTE_GLX_MASK (3)
#define DTE_GLX GENMASK_ULL(57, 56)
#define DTE_FLAG_IR BIT_ULL(61)
#define DTE_FLAG_IW BIT_ULL(62)

#define DTE_FLAG_IOTLB BIT_ULL(32)
#define DTE_FLAG_MASK (0x3ffULL << 32)
#define DEV_DOMID_MASK 0xffffULL

#define DTE_GCR3_VAL_A(x) (((x) >> 12) & 0x00007ULL)
#define DTE_GCR3_VAL_B(x) (((x) >> 15) & 0x0ffffULL)
#define DTE_GCR3_VAL_C(x) (((x) >> 31) & 0x1fffffULL)

#define DTE_GCR3_SHIFT_A 58
#define DTE_GCR3_SHIFT_B 16
#define DTE_GCR3_SHIFT_C 43
#define DTE_GCR3_14_12 GENMASK_ULL(60, 58)
#define DTE_GCR3_30_15 GENMASK_ULL(31, 16)
#define DTE_GCR3_51_31 GENMASK_ULL(63, 43)

#define DTE_GPT_LEVEL_SHIFT 54
#define DTE_GPT_LEVEL_MASK GENMASK_ULL(55, 54)
Expand Down
30 changes: 26 additions & 4 deletions drivers/iommu/amd/init.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1089,11 +1089,9 @@ static bool __copy_device_table(struct amd_iommu *iommu)
}
/* If gcr3 table existed, mask it out */
if (old_devtb[devid].data[0] & DTE_FLAG_GV) {
tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
tmp = (DTE_GCR3_30_15 | DTE_GCR3_51_31);
pci_seg->old_dev_tbl_cpy[devid].data[1] &= ~tmp;
tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A;
tmp |= DTE_FLAG_GV;
tmp = (DTE_GCR3_14_12 | DTE_FLAG_GV);
pci_seg->old_dev_tbl_cpy[devid].data[0] &= ~tmp;
}
}
Expand Down Expand Up @@ -1144,6 +1142,30 @@ static bool copy_device_table(void)
return true;
}

struct dev_table_entry *amd_iommu_get_ivhd_dte_flags(u16 segid, u16 devid)
{
struct ivhd_dte_flags *e;
unsigned int best_len = UINT_MAX;
struct dev_table_entry *dte = NULL;

for_each_ivhd_dte_flags(e) {
/*
* Need to go through the whole list to find the smallest range,
* which contains the devid.
*/
if ((e->segid == segid) &&
(e->devid_first <= devid) && (devid <= e->devid_last)) {
unsigned int len = e->devid_last - e->devid_first;

if (len < best_len) {
dte = &(e->dte);
best_len = len;
}
}
}
return dte;
}

static bool search_ivhd_dte_flags(u16 segid, u16 first, u16 last)
{
struct ivhd_dte_flags *e;
Expand Down
129 changes: 74 additions & 55 deletions drivers/iommu/amd/iommu.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1949,90 +1949,109 @@ int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid)
return ret;
}

static void make_clear_dte(struct iommu_dev_data *dev_data, struct dev_table_entry *ptr,
struct dev_table_entry *new)
{
/* All existing DTE must have V bit set */
new->data128[0] = DTE_FLAG_V;
new->data128[1] = 0;
}

/*
* Note:
* The old value for GCR3 table and GPT have been cleared from caller.
*/
static void set_dte_gcr3_table(struct amd_iommu *iommu,
struct iommu_dev_data *dev_data,
struct dev_table_entry *target)
{
struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
u64 gcr3;

if (!gcr3_info->gcr3_tbl)
return;

pr_debug("%s: devid=%#x, glx=%#x, gcr3_tbl=%#llx\n",
__func__, dev_data->devid, gcr3_info->glx,
(unsigned long long)gcr3_info->gcr3_tbl);

gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl);

target->data[0] |= DTE_FLAG_GV |
FIELD_PREP(DTE_GLX, gcr3_info->glx) |
FIELD_PREP(DTE_GCR3_14_12, gcr3 >> 12);
if (pdom_is_v2_pgtbl_mode(dev_data->domain))
target->data[0] |= DTE_FLAG_GIOV;

target->data[1] |= FIELD_PREP(DTE_GCR3_30_15, gcr3 >> 15) |
FIELD_PREP(DTE_GCR3_51_31, gcr3 >> 31);

/* Guest page table can only support 4 and 5 levels */
if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL)
target->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_5_LEVEL);
else
target->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_4_LEVEL);
}

static void set_dte_entry(struct amd_iommu *iommu,
struct iommu_dev_data *dev_data)
{
u64 pte_root = 0;
u64 flags = 0;
u32 old_domid;
u16 devid = dev_data->devid;
u16 domid;
u32 old_domid;
struct dev_table_entry *initial_dte;
struct dev_table_entry new = {};
struct protection_domain *domain = dev_data->domain;
struct dev_table_entry *dev_table = get_dev_table(iommu);
struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];

if (gcr3_info && gcr3_info->gcr3_tbl)
domid = dev_data->gcr3_info.domid;
else
domid = domain->id;

make_clear_dte(dev_data, dte, &new);

if (domain->iop.mode != PAGE_MODE_NONE)
pte_root = iommu_virt_to_phys(domain->iop.root);
new.data[0] = iommu_virt_to_phys(domain->iop.root);

pte_root |= (domain->iop.mode & DEV_ENTRY_MODE_MASK)
new.data[0] |= (domain->iop.mode & DEV_ENTRY_MODE_MASK)
<< DEV_ENTRY_MODE_SHIFT;

pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V;
new.data[0] |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V;

/*
* When SNP is enabled, Only set TV bit when IOMMU
* page translation is in use.
* When SNP is enabled, we can only support TV=1 with non-zero domain ID.
* This is prevented by the SNP-enable and IOMMU_DOMAIN_IDENTITY check in
* do_iommu_domain_alloc().
*/
if (!amd_iommu_snp_en || (domid != 0))
pte_root |= DTE_FLAG_TV;

flags = dev_table[devid].data[1];

if (dev_data->ats_enabled)
flags |= DTE_FLAG_IOTLB;
WARN_ON(amd_iommu_snp_en && (domid == 0));
new.data[0] |= DTE_FLAG_TV;

if (dev_data->ppr)
pte_root |= 1ULL << DEV_ENTRY_PPR;
new.data[0] |= 1ULL << DEV_ENTRY_PPR;

if (domain->dirty_tracking)
pte_root |= DTE_FLAG_HAD;

if (gcr3_info && gcr3_info->gcr3_tbl) {
u64 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl);
u64 glx = gcr3_info->glx;
u64 tmp;

pte_root |= DTE_FLAG_GV;
pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
new.data[0] |= DTE_FLAG_HAD;

/* First mask out possible old values for GCR3 table */
tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
flags &= ~tmp;

tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
flags &= ~tmp;

/* Encode GCR3 table into DTE */
tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
pte_root |= tmp;

tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
flags |= tmp;

tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
flags |= tmp;
if (dev_data->ats_enabled)
new.data[1] |= DTE_FLAG_IOTLB;

if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) {
dev_table[devid].data[2] |=
((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT);
}
old_domid = READ_ONCE(dte->data[1]) & DEV_DOMID_MASK;
new.data[1] |= domid;

/* GIOV is supported with V2 page table mode only */
if (pdom_is_v2_pgtbl_mode(domain))
pte_root |= DTE_FLAG_GIOV;
/*
* Restore cached persistent DTE bits, which can be set by information
* in IVRS table. See set_dev_entry_from_acpi().
*/
initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid);
if (initial_dte) {
new.data128[0] |= initial_dte->data128[0];
new.data128[1] |= initial_dte->data128[1];
}

flags &= ~DEV_DOMID_MASK;
flags |= domid;
set_dte_gcr3_table(iommu, dev_data, &new);

old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK;
dev_table[devid].data[1] = flags;
dev_table[devid].data[0] = pte_root;
update_dte256(iommu, dev_data, &new);

/*
* A kdump kernel might be replacing a domain ID that was copied from
Expand Down

0 comments on commit fd5dff9

Please sign in to comment.