Skip to content

Commit

Permalink
VT-d: add device IOTLB invalidation support
Browse files Browse the repository at this point in the history 
Support device IOTLB invalidation to flush the translation cached
in the Endpoint.

Signed-off-by: Yu Zhao <yu.zhao@intel.com>
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
  • Loading branch information
Yu Zhao authored and David Woodhouse committed May 18, 2009
1 parent aa5d2b5 commit 6ba6c3a
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 9 deletions.
77 changes: 69 additions & 8 deletions drivers/pci/dmar.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -699,7 +699,8 @@ void free_iommu(struct intel_iommu *iommu)
*/
static inline void reclaim_free_desc(struct q_inval *qi)
{
while (qi->desc_status[qi->free_tail] == QI_DONE) {
while (qi->desc_status[qi->free_tail] == QI_DONE ||
qi->desc_status[qi->free_tail] == QI_ABORT) {
qi->desc_status[qi->free_tail] = QI_FREE;
qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
qi->free_cnt++;
Expand All @@ -709,10 +710,13 @@ static inline void reclaim_free_desc(struct q_inval *qi)
static int qi_check_fault(struct intel_iommu *iommu, int index)
{
u32 fault;
int head;
int head, tail;
struct q_inval *qi = iommu->qi;
int wait_index = (index + 1) % QI_LENGTH;

if (qi->desc_status[wait_index] == QI_ABORT)
return -EAGAIN;

fault = readl(iommu->reg + DMAR_FSTS_REG);

/*
Expand All @@ -722,7 +726,11 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
*/
if (fault & DMA_FSTS_IQE) {
head = readl(iommu->reg + DMAR_IQH_REG);
if ((head >> 4) == index) {
if ((head >> DMAR_IQ_SHIFT) == index) {
printk(KERN_ERR "VT-d detected invalid descriptor: "
"low=%llx, high=%llx\n",
(unsigned long long)qi->desc[index].low,
(unsigned long long)qi->desc[index].high);
memcpy(&qi->desc[index], &qi->desc[wait_index],
sizeof(struct qi_desc));
__iommu_flush_cache(iommu, &qi->desc[index],
Expand All @@ -732,6 +740,32 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
}
}

/*
* If ITE happens, all pending wait_desc commands are aborted.
* No new descriptors are fetched until the ITE is cleared.
*/
if (fault & DMA_FSTS_ITE) {
head = readl(iommu->reg + DMAR_IQH_REG);
head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
head |= 1;
tail = readl(iommu->reg + DMAR_IQT_REG);
tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;

writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);

do {
if (qi->desc_status[head] == QI_IN_USE)
qi->desc_status[head] = QI_ABORT;
head = (head - 2 + QI_LENGTH) % QI_LENGTH;
} while (head != tail);

if (qi->desc_status[wait_index] == QI_ABORT)
return -EAGAIN;
}

if (fault & DMA_FSTS_ICE)
writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);

return 0;
}

Expand All @@ -741,7 +775,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
*/
int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
{
int rc = 0;
int rc;
struct q_inval *qi = iommu->qi;
struct qi_desc *hw, wait_desc;
int wait_index, index;
Expand All @@ -752,6 +786,9 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)

hw = qi->desc;

restart:
rc = 0;

spin_lock_irqsave(&qi->q_lock, flags);
while (qi->free_cnt < 3) {
spin_unlock_irqrestore(&qi->q_lock, flags);
Expand Down Expand Up @@ -782,7 +819,7 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
* update the HW tail register indicating the presence of
* new descriptors.
*/
writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);

while (qi->desc_status[wait_index] != QI_DONE) {
/*
Expand All @@ -794,18 +831,21 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
*/
rc = qi_check_fault(iommu, index);
if (rc)
goto out;
break;

spin_unlock(&qi->q_lock);
cpu_relax();
spin_lock(&qi->q_lock);
}
out:
qi->desc_status[index] = qi->desc_status[wait_index] = QI_DONE;

qi->desc_status[index] = QI_DONE;

reclaim_free_desc(qi);
spin_unlock_irqrestore(&qi->q_lock, flags);

if (rc == -EAGAIN)
goto restart;

return rc;
}

Expand Down Expand Up @@ -857,6 +897,27 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
qi_submit_sync(&desc, iommu);
}

void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
u64 addr, unsigned mask)
{
struct qi_desc desc;

if (mask) {
BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
} else
desc.high = QI_DEV_IOTLB_ADDR(addr);

if (qdep >= QI_DEV_IOTLB_MAX_INVS)
qdep = 0;

desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
QI_DIOTLB_TYPE;

qi_submit_sync(&desc, iommu);
}

/*
* Disable Queued Invalidation interface.
*/
Expand Down
14 changes: 13 additions & 1 deletion include/linux/intel-iommu.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
#define DMAR_PHMLIMIT_REG 0x78 /* pmrr high limit */
#define DMAR_IQH_REG 0x80 /* Invalidation queue head register */
#define DMAR_IQT_REG 0x88 /* Invalidation queue tail register */
#define DMAR_IQ_SHIFT 4 /* Invalidation queue head/tail shift */
#define DMAR_IQA_REG 0x90 /* Invalidation queue addr register */
#define DMAR_ICS_REG 0x98 /* Invalidation complete status register */
#define DMAR_IRTA_REG 0xb8 /* Interrupt remapping table addr register */
Expand Down Expand Up @@ -198,6 +199,8 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
#define DMA_FSTS_PPF ((u32)2)
#define DMA_FSTS_PFO ((u32)1)
#define DMA_FSTS_IQE (1 << 4)
#define DMA_FSTS_ICE (1 << 5)
#define DMA_FSTS_ITE (1 << 6)
#define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff)

/* FRCD_REG, 32 bits access */
Expand Down Expand Up @@ -226,7 +229,8 @@ do { \
enum {
QI_FREE,
QI_IN_USE,
QI_DONE
QI_DONE,
QI_ABORT
};

#define QI_CC_TYPE 0x1
Expand Down Expand Up @@ -255,6 +259,12 @@ enum {
#define QI_CC_DID(did) (((u64)did) << 16)
#define QI_CC_GRAN(gran) (((u64)gran) >> (DMA_CCMD_INVL_GRANU_OFFSET-4))

#define QI_DEV_IOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32)
#define QI_DEV_IOTLB_QDEP(qdep) (((qdep) & 0x1f) << 16)
#define QI_DEV_IOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK)
#define QI_DEV_IOTLB_SIZE 1
#define QI_DEV_IOTLB_MAX_INVS 32

struct qi_desc {
u64 low, high;
};
Expand Down Expand Up @@ -344,6 +354,8 @@ extern void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid,
u8 fm, u64 type);
extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
unsigned int size_order, u64 type);
extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
u64 addr, unsigned mask);

extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu);

Expand Down

0 comments on commit 6ba6c3a

Please sign in to comment.