From d79e8cab32371b0f08ae0ed6f1b2b9bd8b7a2873 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Mon, 27 May 2024 13:20:15 +0200 Subject: [PATCH] drm/xe/vf: Use only assigned GGTT region MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each VF is assigned a limited range of the GGTT address space. To ensure that the VF driver does not use GGTT allocations outside of the assigned region, explicitly reserve GGTT space below and above this region when initializing GGTT. Signed-off-by: Michal Wajdeczko Reviewed-by: Michał Winiarski Link: https://patchwork.freedesktop.org/patch/msgid/20240527112015.1020-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_device_types.h | 3 + drivers/gpu/drm/xe/xe_ggtt.c | 14 ++++- drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 92 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_gt_sriov_vf.h | 1 + 4 files changed, 109 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 03bedc33b21ad..3ff60da7d1cdb 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -196,6 +196,9 @@ struct xe_tile { struct { /** @sriov.vf.memirq: Memory Based Interrupts. */ struct xe_memirq memirq; + + /** @sriov.vf.ggtt_balloon: GGTT regions excluded from use. */ + struct drm_mm_node ggtt_balloon[2]; } vf; } sriov; diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index 17e5066763dbd..b01a670fecb82 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -20,6 +20,7 @@ #include "xe_device.h" #include "xe_gt.h" #include "xe_gt_printk.h" +#include "xe_gt_sriov_vf.h" #include "xe_gt_tlb_invalidation.h" #include "xe_map.h" #include "xe_pm.h" @@ -141,6 +142,7 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt) struct xe_device *xe = tile_to_xe(ggtt->tile); struct pci_dev *pdev = to_pci_dev(xe->drm.dev); unsigned int gsm_size; + int err; if (IS_SRIOV_VF(xe)) gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */ @@ -194,7 +196,17 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt) mutex_init(&ggtt->lock); primelockdep(ggtt); - return drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt); + err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt); + if (err) + return err; + + if (IS_SRIOV_VF(xe)) { + err = xe_gt_sriov_vf_prepare_ggtt(xe_tile_get_gt(ggtt->tile, 0)); + if (err) + return err; + } + + return 0; } static void xe_ggtt_invalidate(struct xe_ggtt *ggtt); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 66069eb8c5be5..da7c64f6285bd 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -14,9 +14,11 @@ #include "abi/guc_klvs_abi.h" #include "abi/guc_relay_actions_abi.h" #include "regs/xe_gt_regs.h" +#include "regs/xe_gtt_defs.h" #include "xe_assert.h" #include "xe_device.h" +#include "xe_ggtt.h" #include "xe_gt_sriov_printk.h" #include "xe_gt_sriov_vf.h" #include "xe_gt_sriov_vf_types.h" @@ -25,6 +27,8 @@ #include "xe_guc_relay.h" #include "xe_mmio.h" #include "xe_sriov.h" +#include "xe_uc_fw.h" +#include "xe_wopcm.h" #define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo))) @@ -474,6 +478,94 @@ u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt) return gt->sriov.vf.self_config.num_ctxs; } +static int vf_balloon_ggtt(struct xe_gt *gt) +{ + struct xe_gt_sriov_vf_selfconfig *config = >->sriov.vf.self_config; + struct xe_tile *tile = gt_to_tile(gt); + struct xe_ggtt *ggtt = tile->mem.ggtt; + struct xe_device *xe = gt_to_xe(gt); + u64 start, end; + int err; + + xe_gt_assert(gt, IS_SRIOV_VF(xe)); + xe_gt_assert(gt, !xe_gt_is_media_type(gt)); + + if (!config->ggtt_size) + return -ENODATA; + + /* + * VF can only use part of the GGTT as allocated by the PF: + * + * WOPCM GUC_GGTT_TOP + * |<------------ Total GGTT size ------------------>| + * + * VF GGTT base -->|<- size ->| + * + * +--------------------+----------+-----------------+ + * |////////////////////| block |\\\\\\\\\\\\\\\\\| + * +--------------------+----------+-----------------+ + * + * |<--- balloon[0] --->|<-- VF -->|<-- balloon[1] ->| + */ + + start = xe_wopcm_size(xe); + end = config->ggtt_base; + if (end != start) { + err = xe_ggtt_balloon(ggtt, start, end, &tile->sriov.vf.ggtt_balloon[0]); + if (err) + goto failed; + } + + start = config->ggtt_base + config->ggtt_size; + end = GUC_GGTT_TOP; + if (end != start) { + err = xe_ggtt_balloon(ggtt, start, end, &tile->sriov.vf.ggtt_balloon[1]); + if (err) + goto deballoon; + } + + return 0; + +deballoon: + xe_ggtt_deballoon(ggtt, &tile->sriov.vf.ggtt_balloon[0]); +failed: + return err; +} + +static void deballoon_ggtt(struct drm_device *drm, void *arg) +{ + struct xe_tile *tile = arg; + struct xe_ggtt *ggtt = tile->mem.ggtt; + + xe_tile_assert(tile, IS_SRIOV_VF(tile_to_xe(tile))); + xe_ggtt_deballoon(ggtt, &tile->sriov.vf.ggtt_balloon[1]); + xe_ggtt_deballoon(ggtt, &tile->sriov.vf.ggtt_balloon[0]); +} + +/** + * xe_gt_sriov_vf_prepare_ggtt - Prepare a VF's GGTT configuration. + * @gt: the &xe_gt + * + * This function is for VF use only. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_vf_prepare_ggtt(struct xe_gt *gt) +{ + struct xe_tile *tile = gt_to_tile(gt); + struct xe_device *xe = tile_to_xe(tile); + int err; + + if (xe_gt_is_media_type(gt)) + return 0; + + err = vf_balloon_ggtt(gt); + if (err) + return err; + + return drmm_add_action_or_reset(&xe->drm, deballoon_ggtt, tile); +} + static int relay_action_handshake(struct xe_gt *gt, u32 *major, u32 *minor) { u32 request[VF2PF_HANDSHAKE_REQUEST_MSG_LEN] = { diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h index 0391ed72a5705..7a04bcaffe9fc 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h @@ -16,6 +16,7 @@ int xe_gt_sriov_vf_bootstrap(struct xe_gt *gt); int xe_gt_sriov_vf_query_config(struct xe_gt *gt); int xe_gt_sriov_vf_connect(struct xe_gt *gt); int xe_gt_sriov_vf_query_runtime(struct xe_gt *gt); +int xe_gt_sriov_vf_prepare_ggtt(struct xe_gt *gt); u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt); u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt);