From 13c96ac9a3f0f1c7ba1ff0656ea508e7fa065e7e Mon Sep 17 00:00:00 2001 From: Xiaolei Wang Date: Wed, 2 Oct 2024 07:34:30 +0800 Subject: [PATCH 01/17] drm/etnaviv: Request pages from DMA32 zone on addressing_limited Remove __GFP_HIGHMEM when requesting a page from DMA32 zone, and since all vivante GPUs in the system will share the same DMA constraints, move the check of whether to get a page from DMA32 to etnaviv_bind(). Fixes: b72af445cd38 ("drm/etnaviv: request pages from DMA32 zone when needed") Suggested-by: Sui Jingfeng Signed-off-by: Xiaolei Wang Reviewed-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_drv.c | 10 ++++++++++ drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 8 -------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c index 6500f3999c5fa..19ec67a5a918e 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c @@ -538,6 +538,16 @@ static int etnaviv_bind(struct device *dev) priv->num_gpus = 0; priv->shm_gfp_mask = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN; + /* + * If the GPU is part of a system with DMA addressing limitations, + * request pages for our SHM backend buffers from the DMA32 zone to + * hopefully avoid performance killing SWIOTLB bounce buffering. + */ + if (dma_addressing_limited(dev)) { + priv->shm_gfp_mask |= GFP_DMA32; + priv->shm_gfp_mask &= ~__GFP_HIGHMEM; + } + priv->cmdbuf_suballoc = etnaviv_cmdbuf_suballoc_new(drm->dev); if (IS_ERR(priv->cmdbuf_suballoc)) { dev_err(drm->dev, "Failed to create cmdbuf suballocator\n"); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 7c7f97793ddd0..5e753dd42f721 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -839,14 +839,6 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu) if (ret) goto fail; - /* - * If the GPU is part of a system with DMA addressing limitations, - * request pages for our SHM backend buffers from the DMA32 zone to - * hopefully avoid performance killing SWIOTLB bounce buffering. - */ - if (dma_addressing_limited(gpu->dev)) - priv->shm_gfp_mask |= GFP_DMA32; - /* Create buffer: */ ret = etnaviv_cmdbuf_init(priv->cmdbuf_suballoc, &gpu->buffer, PAGE_SIZE); From 56257d4c422d0f45b1367145ffe1a1350d4338b6 Mon Sep 17 00:00:00 2001 From: Sui Jingfeng Date: Sun, 8 Sep 2024 20:11:03 +0800 Subject: [PATCH 02/17] drm/etnaviv: Use unsigned type to count the number of pages The drm_prime_pages_to_sg() function takes an 'unsigned int' argument to store the length of the page vector. The size of the object in number of CPU pages can not be negative, hence, use 'unsigned' variable to store the number of pages, instead of the 'signed' one. Reviewed-by: Christian Gmeiner Signed-off-by: Sui Jingfeng Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c index 3524b5811682a..6b98200068e45 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c @@ -17,7 +17,7 @@ static struct lock_class_key etnaviv_prime_lock_class; struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj) { struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); - int npages = obj->size >> PAGE_SHIFT; + unsigned int npages = obj->size >> PAGE_SHIFT; if (WARN_ON(!etnaviv_obj->pages)) /* should have already pinned! */ return ERR_PTR(-EINVAL); From c82e8b292c2a2a4e780677af895c6233ef5538b2 Mon Sep 17 00:00:00 2001 From: Sui Jingfeng Date: Sun, 8 Sep 2024 20:11:04 +0800 Subject: [PATCH 03/17] drm/etnaviv: Use 'unsigned' type to count the number of pages The unpin_user_pages() function takes an 'unsigned long' argument to store the number of userspace pages, and the struct drm_gem_object::size is a size_t type. The number of pages can not be negative, hence, use 'unsigned' variable to count the number of pages. Reviewed-by: Christian Gmeiner Signed-off-by: Sui Jingfeng Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index 5c0c9d4e3be18..63b7e143c273e 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -686,7 +686,7 @@ static void etnaviv_gem_userptr_release(struct etnaviv_gem_object *etnaviv_obj) kfree(etnaviv_obj->sgt); } if (etnaviv_obj->pages) { - int npages = etnaviv_obj->base.size >> PAGE_SHIFT; + unsigned int npages = etnaviv_obj->base.size >> PAGE_SHIFT; unpin_user_pages(etnaviv_obj->pages, npages); kvfree(etnaviv_obj->pages); From b09ccba779c8dd241c9254a4b5c062c51e6fa80d Mon Sep 17 00:00:00 2001 From: Sui Jingfeng Date: Sun, 8 Sep 2024 20:11:05 +0800 Subject: [PATCH 04/17] drm/etnaviv: Drop the header Currently, the etnaviv_gem_submit.c isn't call any runtime power management functions. So drop this unused header, we can include it back when it really get used though. Reviewed-by: Christian Gmeiner Signed-off-by: Sui Jingfeng Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c index 3d0f8d182506e..3c0a5c3e0e3d2 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include From deadf1ef4adce096352489d1ef3721e43fa188bf Mon Sep 17 00:00:00 2001 From: Sui Jingfeng Date: Sun, 8 Sep 2024 20:11:06 +0800 Subject: [PATCH 05/17] drm/etnaviv: Fix missing mutex_destroy() Currently, the calling of mutex_destroy() is ignored on error handling code path. It is safe for now, since mutex_destroy() actually does nothing in non-debug builds. But the mutex_destroy() is used to mark the mutex uninitialized on debug builds, and any subsequent use of the mutex is forbidden. It also could lead to problems if mutex_destroy() gets extended, add missing mutex_destroy() to eliminate potential concerns. Reviewed-by: Christian Gmeiner Signed-off-by: Sui Jingfeng Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c | 2 ++ drivers/gpu/drm/etnaviv/etnaviv_drv.c | 1 + drivers/gpu/drm/etnaviv/etnaviv_gem.c | 1 + drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 5 +++++ drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 2 +- 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c index 721d633aece9d..66a407f1b3ee7 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c @@ -55,6 +55,7 @@ etnaviv_cmdbuf_suballoc_new(struct device *dev) return suballoc; free_suballoc: + mutex_destroy(&suballoc->lock); kfree(suballoc); return ERR_PTR(ret); @@ -79,6 +80,7 @@ void etnaviv_cmdbuf_suballoc_destroy(struct etnaviv_cmdbuf_suballoc *suballoc) { dma_free_wc(suballoc->dev, SUBALLOC_SIZE, suballoc->vaddr, suballoc->paddr); + mutex_destroy(&suballoc->lock); kfree(suballoc); } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c index 19ec67a5a918e..aeacf5863fc86 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c @@ -574,6 +574,7 @@ static int etnaviv_bind(struct device *dev) out_destroy_suballoc: etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc); out_free_priv: + mutex_destroy(&priv->gem_lock); kfree(priv); out_put: drm_dev_put(drm); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index 63b7e143c273e..592a566ea747c 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -514,6 +514,7 @@ void etnaviv_gem_free_object(struct drm_gem_object *obj) etnaviv_obj->ops->release(etnaviv_obj); drm_gem_object_release(obj); + mutex_destroy(&etnaviv_obj->lock); kfree(etnaviv_obj); } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 5e753dd42f721..93aa6f762cd25 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -1921,8 +1921,13 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev) static void etnaviv_gpu_platform_remove(struct platform_device *pdev) { + struct etnaviv_gpu *gpu = dev_get_drvdata(&pdev->dev); + component_del(&pdev->dev, &gpu_ops); pm_runtime_disable(&pdev->dev); + + mutex_destroy(&gpu->lock); + mutex_destroy(&gpu->sched_lock); } static int etnaviv_gpu_rpm_suspend(struct device *dev) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c index 1661d589bf3e7..02d9408d41bcf 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c @@ -358,7 +358,7 @@ static void etnaviv_iommu_context_free(struct kref *kref) container_of(kref, struct etnaviv_iommu_context, refcount); etnaviv_cmdbuf_suballoc_unmap(context, &context->cmdbuf_mapping); - + mutex_destroy(&context->lock); context->global->ops->free(context); } void etnaviv_iommu_context_put(struct etnaviv_iommu_context *context) From 72dc70a062f9faea178d674416d72a2596b69633 Mon Sep 17 00:00:00 2001 From: Sui Jingfeng Date: Sun, 8 Sep 2024 20:11:07 +0800 Subject: [PATCH 06/17] drm/etnaviv: Replace the '&pdev->dev' with 'dev' In the etnaviv_pdev_probe() and etnaviv_gpu_platform_probe() function, the value of '&pdev->dev' has been cached to the local auto variable 'dev'. But some callers use 'dev', while the rest use '&pdev->dev'. To keep it consistent, use 'dev' uniformly. Tested-by: Christian Gmeiner Signed-off-by: Sui Jingfeng Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_drv.c | 10 +++++----- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c index aeacf5863fc86..9b4e2f4b1bc71 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c @@ -619,7 +619,7 @@ static int etnaviv_pdev_probe(struct platform_device *pdev) if (!of_device_is_available(core_node)) continue; - drm_of_component_match_add(&pdev->dev, &match, + drm_of_component_match_add(dev, &match, component_compare_of, core_node); } } else { @@ -642,9 +642,9 @@ static int etnaviv_pdev_probe(struct platform_device *pdev) * bit to make sure we are allocating the command buffers and * TLBs in the lower 4 GiB address space. */ - if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(40)) || - dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32))) { - dev_dbg(&pdev->dev, "No suitable DMA available\n"); + if (dma_set_mask(dev, DMA_BIT_MASK(40)) || + dma_set_coherent_mask(dev, DMA_BIT_MASK(32))) { + dev_dbg(dev, "No suitable DMA available\n"); return -ENODEV; } @@ -655,7 +655,7 @@ static int etnaviv_pdev_probe(struct platform_device *pdev) */ first_node = etnaviv_of_first_available_node(); if (first_node) { - of_dma_configure(&pdev->dev, first_node, true); + of_dma_configure(dev, first_node, true); of_node_put(first_node); } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 93aa6f762cd25..5a8110586d58b 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -1854,7 +1854,7 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev) if (!gpu) return -ENOMEM; - gpu->dev = &pdev->dev; + gpu->dev = dev; mutex_init(&gpu->lock); mutex_init(&gpu->sched_lock); @@ -1868,8 +1868,8 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev) if (gpu->irq < 0) return gpu->irq; - err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0, - dev_name(gpu->dev), gpu); + err = devm_request_irq(dev, gpu->irq, irq_handler, 0, + dev_name(dev), gpu); if (err) { dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err); return err; @@ -1906,13 +1906,13 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev) * autosuspend delay is rather arbitary: no measurements have * yet been performed to determine an appropriate value. */ - pm_runtime_use_autosuspend(gpu->dev); - pm_runtime_set_autosuspend_delay(gpu->dev, 200); - pm_runtime_enable(gpu->dev); + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, 200); + pm_runtime_enable(dev); - err = component_add(&pdev->dev, &gpu_ops); + err = component_add(dev, &gpu_ops); if (err < 0) { - dev_err(&pdev->dev, "failed to register component: %d\n", err); + dev_err(dev, "failed to register component: %d\n", err); return err; } From 37dc4737447a7667f8e9ec790dac251da057eb27 Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 5 Jul 2024 22:00:09 +0200 Subject: [PATCH 07/17] drm/etnaviv: hold GPU lock across perfmon sampling The perfmon sampling mutates shared GPU state (e.g. VIVS_HI_CLOCK_CONTROL to select the pipe for the perf counter reads). To avoid clashing with other functions mutating the same state (e.g. etnaviv_gpu_update_clock) the perfmon sampling needs to hold the GPU lock. Fixes: 68dc0b295dcb ("drm/etnaviv: use 'sync points' for performance monitor requests") Reviewed-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 5a8110586d58b..b43c6f3416a60 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -1322,6 +1322,8 @@ static void sync_point_perfmon_sample_pre(struct etnaviv_gpu *gpu, { u32 val; + mutex_lock(&gpu->lock); + /* disable clock gating */ val = gpu_read_power(gpu, VIVS_PM_POWER_CONTROLS); val &= ~VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING; @@ -1333,6 +1335,8 @@ static void sync_point_perfmon_sample_pre(struct etnaviv_gpu *gpu, gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, val); sync_point_perfmon_sample(gpu, event, ETNA_PM_PROCESS_PRE); + + mutex_unlock(&gpu->lock); } static void sync_point_perfmon_sample_post(struct etnaviv_gpu *gpu, @@ -1342,13 +1346,9 @@ static void sync_point_perfmon_sample_post(struct etnaviv_gpu *gpu, unsigned int i; u32 val; - sync_point_perfmon_sample(gpu, event, ETNA_PM_PROCESS_POST); - - for (i = 0; i < submit->nr_pmrs; i++) { - const struct etnaviv_perfmon_request *pmr = submit->pmrs + i; + mutex_lock(&gpu->lock); - *pmr->bo_vma = pmr->sequence; - } + sync_point_perfmon_sample(gpu, event, ETNA_PM_PROCESS_POST); /* disable debug register */ val = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); @@ -1359,6 +1359,14 @@ static void sync_point_perfmon_sample_post(struct etnaviv_gpu *gpu, val = gpu_read_power(gpu, VIVS_PM_POWER_CONTROLS); val |= VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING; gpu_write_power(gpu, VIVS_PM_POWER_CONTROLS, val); + + mutex_unlock(&gpu->lock); + + for (i = 0; i < submit->nr_pmrs; i++) { + const struct etnaviv_perfmon_request *pmr = submit->pmrs + i; + + *pmr->bo_vma = pmr->sequence; + } } From 67cb86039f8e2e29164b317d16756804e0deaa2a Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 5 Jul 2024 22:00:10 +0200 Subject: [PATCH 08/17] drm/etnaviv: assert GPU lock held in perfmon pipe_*_read functions The perf counter read functions don't just read registers, but they also mutate state to direct the reads towards the correct pipe and engine. Assert that the GPU mutex is held at this point, so that those state changes don't interfere with others. Reviewed-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_perfmon.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c index dc9dea664a28d..d53a5c2933739 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c @@ -62,6 +62,8 @@ static u32 pipe_perf_reg_read(struct etnaviv_gpu *gpu, u32 value = 0; unsigned i; + lockdep_assert_held(&gpu->lock); + for (i = 0; i < gpu->identity.pixel_pipes; i++) { pipe_select(gpu, clock, i); value += perf_reg_read(gpu, domain, signal); @@ -81,6 +83,8 @@ static u32 pipe_reg_read(struct etnaviv_gpu *gpu, u32 value = 0; unsigned i; + lockdep_assert_held(&gpu->lock); + for (i = 0; i < gpu->identity.pixel_pipes; i++) { pipe_select(gpu, clock, i); value += gpu_read(gpu, signal->data); From ca0593a29e9ea2df17bf222a649a41bc16936255 Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 5 Jul 2024 22:00:11 +0200 Subject: [PATCH 09/17] drm/etnaviv: unconditionally enable debug registers A later change will use the FE debug registers to improve GPU progress monitoring. Instead of having to keep track of the usage state of the debug registers and lock access to the VIVS_HI_CLOCK_CONTROL register, statically enable debug register access during GPU init. The Vivante downstream driver seems to do the same thing since a while, so it should be okay to keep access enabled. (See gckHARDWARE_InitializeHardware in 6.4.11 downstream driver). Many debug registers contain bogus data if clock gating is enabled, so even if they are always accessible performance profiling still needs to manage some prerequisites. Reviewed-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index b43c6f3416a60..8c1ef86e901c5 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -574,8 +574,8 @@ static int etnaviv_hw_reset(struct etnaviv_gpu *gpu) continue; } - /* disable debug registers, as they are not normally needed */ - control |= VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS; + /* enable debug register access */ + control &= ~VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS; gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); failed = false; @@ -1329,11 +1329,6 @@ static void sync_point_perfmon_sample_pre(struct etnaviv_gpu *gpu, val &= ~VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING; gpu_write_power(gpu, VIVS_PM_POWER_CONTROLS, val); - /* enable debug register */ - val = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); - val &= ~VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS; - gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, val); - sync_point_perfmon_sample(gpu, event, ETNA_PM_PROCESS_PRE); mutex_unlock(&gpu->lock); @@ -1350,11 +1345,6 @@ static void sync_point_perfmon_sample_post(struct etnaviv_gpu *gpu, sync_point_perfmon_sample(gpu, event, ETNA_PM_PROCESS_POST); - /* disable debug register */ - val = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); - val |= VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS; - gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, val); - /* enable clock gating */ val = gpu_read_power(gpu, VIVS_PM_POWER_CONTROLS); val |= VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING; From 46864a690b92464063947420ad1a55e295b2004d Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 5 Jul 2024 22:00:12 +0200 Subject: [PATCH 10/17] drm/etnaviv: update hardware headers from rnndb Update state_hi.xml.h header from etna_viv commit 8f43a34fd9cd ("rndb: document FE current primitve debug reg") Reviewed-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/state_hi.xml.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/state_hi.xml.h b/drivers/gpu/drm/etnaviv/state_hi.xml.h index 829bc528e6180..f7bc5f6e20ff2 100644 --- a/drivers/gpu/drm/etnaviv/state_hi.xml.h +++ b/drivers/gpu/drm/etnaviv/state_hi.xml.h @@ -8,17 +8,17 @@ This file was generated by the rules-ng-ng headergen tool in this git repository git clone git://0x04.net/rules-ng-ng The rules-ng-ng source files this header was generated from are: -- state.xml ( 29355 bytes, from 2024-01-19 10:18:54) -- common.xml ( 35664 bytes, from 2023-12-06 10:55:32) -- common_3d.xml ( 15069 bytes, from 2023-11-22 10:05:24) -- state_hi.xml ( 35854 bytes, from 2023-12-11 15:50:17) -- copyright.xml ( 1597 bytes, from 2016-11-10 13:58:32) -- state_2d.xml ( 52271 bytes, from 2023-06-02 12:35:03) -- state_3d.xml ( 89522 bytes, from 2024-01-19 10:18:54) -- state_blt.xml ( 14592 bytes, from 2023-11-22 10:05:09) -- state_vg.xml ( 5975 bytes, from 2016-11-10 13:58:32) - -Copyright (C) 2012-2023 by the following authors: +- state.xml ( 30729 bytes, from 2024-06-21 11:31:54) +- common.xml ( 35664 bytes, from 2023-12-13 09:33:18) +- common_3d.xml ( 15069 bytes, from 2023-12-13 09:33:18) +- state_hi.xml ( 35909 bytes, from 2024-06-21 11:31:54) +- copyright.xml ( 1597 bytes, from 2020-10-28 12:56:03) +- state_2d.xml ( 52271 bytes, from 2023-05-30 20:50:02) +- state_3d.xml ( 89626 bytes, from 2024-06-21 11:32:57) +- state_blt.xml ( 14592 bytes, from 2023-12-13 09:33:18) +- state_vg.xml ( 5975 bytes, from 2020-10-28 12:56:03) + +Copyright (C) 2012-2024 by the following authors: - Wladimir J. van der Laan - Christian Gmeiner - Lucas Stach @@ -467,6 +467,7 @@ DEALINGS IN THE SOFTWARE. #define VIVS_MC_PROFILE_CONFIG0 0x00000470 #define VIVS_MC_PROFILE_CONFIG0_FE__MASK 0x000000ff #define VIVS_MC_PROFILE_CONFIG0_FE__SHIFT 0 +#define VIVS_MC_PROFILE_CONFIG0_FE_CURRENT_PRIM 0x00000009 #define VIVS_MC_PROFILE_CONFIG0_FE_DRAW_COUNT 0x0000000a #define VIVS_MC_PROFILE_CONFIG0_FE_OUT_VERTEX_COUNT 0x0000000b #define VIVS_MC_PROFILE_CONFIG0_FE_CACHE_MISS_COUNT 0x0000000c From e1f3220b957a55a875d668f3e2fadd6315c8bbf4 Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 5 Jul 2024 22:00:13 +0200 Subject: [PATCH 11/17] drm/etnaviv: take current primitive into account when checking for hung GPU Large draws can make the GPU appear to be stuck to the current hangcheck logic as the FE address will not move until the draw is finished. However, the FE has a debug register, which records the current primitive ID within a draw. Using this debug register we can extend the timeout as long as the draw progresses. Reviewed-by: Christian Gmeiner Reviewed-by: Philipp Zabel Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gpu.h | 1 + drivers/gpu/drm/etnaviv/etnaviv_sched.c | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h index 31322195b9e41..4d8a7d48ade31 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h @@ -144,6 +144,7 @@ struct etnaviv_gpu { /* hang detection */ u32 hangcheck_dma_addr; + u32 hangcheck_primid; u32 hangcheck_fence; void __iomem *mmio; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index ab9ca4824b62e..c5c146eef78a5 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -11,6 +11,7 @@ #include "etnaviv_gpu.h" #include "etnaviv_sched.h" #include "state.xml.h" +#include "state_hi.xml.h" static int etnaviv_job_hang_limit = 0; module_param_named(job_hang_limit, etnaviv_job_hang_limit, int , 0444); @@ -35,7 +36,7 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job { struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job); struct etnaviv_gpu *gpu = submit->gpu; - u32 dma_addr; + u32 dma_addr, primid = 0; int change; /* @@ -52,10 +53,22 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job */ dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); change = dma_addr - gpu->hangcheck_dma_addr; + if (submit->exec_state == ETNA_PIPE_3D) { + /* guard against concurrent usage from perfmon_sample */ + mutex_lock(&gpu->lock); + gpu_write(gpu, VIVS_MC_PROFILE_CONFIG0, + VIVS_MC_PROFILE_CONFIG0_FE_CURRENT_PRIM << + VIVS_MC_PROFILE_CONFIG0_FE__SHIFT); + primid = gpu_read(gpu, VIVS_MC_PROFILE_FE_READ); + mutex_unlock(&gpu->lock); + } if (gpu->state == ETNA_GPU_STATE_RUNNING && (gpu->completed_fence != gpu->hangcheck_fence || - change < 0 || change > 16)) { + change < 0 || change > 16 || + (submit->exec_state == ETNA_PIPE_3D && + gpu->hangcheck_primid != primid))) { gpu->hangcheck_dma_addr = dma_addr; + gpu->hangcheck_primid = primid; gpu->hangcheck_fence = gpu->completed_fence; goto out_no_timeout; } From b5f1eed853c6ea6a99149fd97fe179f3ebd96a02 Mon Sep 17 00:00:00 2001 From: Sui Jingfeng Date: Sat, 26 Oct 2024 04:43:54 +0800 Subject: [PATCH 12/17] drm/etnaviv: Record GPU visible size of GEM BO separately The GPU visible size of a GEM BO is not necessarily PAGE_SIZE aligned, which happens when CPU page size is not equal to GPU page size. Extra precious resources such as GPU page tables and GPU TLBs may being paid because of this but never get used. Track the size of GPU visible part of GEM BO separately, ensure no GPUVA range wasting by aligning that size to GPU page size. Signed-off-by: Sui Jingfeng Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gem.c | 11 +++++------ drivers/gpu/drm/etnaviv/etnaviv_gem.h | 5 +++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index 592a566ea747c..16473c371444c 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -544,7 +544,7 @@ static const struct drm_gem_object_funcs etnaviv_gem_object_funcs = { .vm_ops = &vm_ops, }; -static int etnaviv_gem_new_impl(struct drm_device *dev, u32 flags, +static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags, const struct etnaviv_gem_ops *ops, struct drm_gem_object **obj) { struct etnaviv_gem_object *etnaviv_obj; @@ -571,6 +571,7 @@ static int etnaviv_gem_new_impl(struct drm_device *dev, u32 flags, if (!etnaviv_obj) return -ENOMEM; + etnaviv_obj->size = ALIGN(size, SZ_4K); etnaviv_obj->flags = flags; etnaviv_obj->ops = ops; @@ -591,15 +592,13 @@ int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file, struct drm_gem_object *obj = NULL; int ret; - size = PAGE_ALIGN(size); - - ret = etnaviv_gem_new_impl(dev, flags, &etnaviv_gem_shmem_ops, &obj); + ret = etnaviv_gem_new_impl(dev, size, flags, &etnaviv_gem_shmem_ops, &obj); if (ret) goto fail; lockdep_set_class(&to_etnaviv_bo(obj)->lock, &etnaviv_shm_lock_class); - ret = drm_gem_object_init(dev, obj, size); + ret = drm_gem_object_init(dev, obj, PAGE_ALIGN(size)); if (ret) goto fail; @@ -628,7 +627,7 @@ int etnaviv_gem_new_private(struct drm_device *dev, size_t size, u32 flags, struct drm_gem_object *obj; int ret; - ret = etnaviv_gem_new_impl(dev, flags, ops, &obj); + ret = etnaviv_gem_new_impl(dev, size, flags, ops, &obj); if (ret) return ret; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h index a42d260cac2cf..687555aae8079 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h @@ -36,6 +36,11 @@ struct etnaviv_gem_object { const struct etnaviv_gem_ops *ops; struct mutex lock; + /* + * The actual size that is visible to the GPU, not necessarily + * PAGE_SIZE aligned, but should be aligned to GPU page size. + */ + u32 size; u32 flags; struct list_head gem_node; From 68786b7f49873c69ec332a045a9bf4337d71ec20 Mon Sep 17 00:00:00 2001 From: Sui Jingfeng Date: Sat, 26 Oct 2024 04:43:55 +0800 Subject: [PATCH 13/17] drm/etnaviv: Map and unmap GPUVA range with respect to the GPUVA size Etnaviv assumes that GPU page size is 4KiB, however, GPUVA ranges collision when using softpin capable GPUs on a non 4KiB CPU page size configuration. The root cause is that kernel side BO takes up bigger address space than userspace expect, the size of backing memory of GEM buffer objects are required to align to the CPU PAGE_SIZE. Therefore, results in userspace allocated GPUVA range fails to be inserted to the specified hole exactly. To solve this problem, record the GPU visiable size of a BO firstly, then map and unmap the SG entry strictly with respect to the total GPUVA size. Signed-off-by: Sui Jingfeng Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 38 +++++++++------------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c index 02d9408d41bcf..7e065b3723cff 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c @@ -69,9 +69,11 @@ static int etnaviv_context_map(struct etnaviv_iommu_context *context, return ret; } -static int etnaviv_iommu_map(struct etnaviv_iommu_context *context, u32 iova, +static int etnaviv_iommu_map(struct etnaviv_iommu_context *context, + u32 iova, unsigned int va_len, struct sg_table *sgt, int prot) -{ struct scatterlist *sg; +{ + struct scatterlist *sg; unsigned int da = iova; unsigned int i; int ret; @@ -81,14 +83,16 @@ static int etnaviv_iommu_map(struct etnaviv_iommu_context *context, u32 iova, for_each_sgtable_dma_sg(sgt, sg, i) { phys_addr_t pa = sg_dma_address(sg) - sg->offset; - size_t bytes = sg_dma_len(sg) + sg->offset; + unsigned int da_len = sg_dma_len(sg) + sg->offset; + unsigned int bytes = min_t(unsigned int, da_len, va_len); - VERB("map[%d]: %08x %pap(%zx)", i, iova, &pa, bytes); + VERB("map[%d]: %08x %pap(%x)", i, iova, &pa, bytes); ret = etnaviv_context_map(context, da, pa, bytes, prot); if (ret) goto fail; + va_len -= bytes; da += bytes; } @@ -104,21 +108,7 @@ static int etnaviv_iommu_map(struct etnaviv_iommu_context *context, u32 iova, static void etnaviv_iommu_unmap(struct etnaviv_iommu_context *context, u32 iova, struct sg_table *sgt, unsigned len) { - struct scatterlist *sg; - unsigned int da = iova; - int i; - - for_each_sgtable_dma_sg(sgt, sg, i) { - size_t bytes = sg_dma_len(sg) + sg->offset; - - etnaviv_context_unmap(context, da, bytes); - - VERB("unmap[%d]: %08x(%zx)", i, iova, bytes); - - BUG_ON(!PAGE_ALIGNED(bytes)); - - da += bytes; - } + etnaviv_context_unmap(context, iova, len); context->flush_seq++; } @@ -131,7 +121,7 @@ static void etnaviv_iommu_remove_mapping(struct etnaviv_iommu_context *context, lockdep_assert_held(&context->lock); etnaviv_iommu_unmap(context, mapping->vram_node.start, - etnaviv_obj->sgt, etnaviv_obj->base.size); + etnaviv_obj->sgt, etnaviv_obj->size); drm_mm_remove_node(&mapping->vram_node); } @@ -305,16 +295,14 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu_context *context, node = &mapping->vram_node; if (va) - ret = etnaviv_iommu_insert_exact(context, node, - etnaviv_obj->base.size, va); + ret = etnaviv_iommu_insert_exact(context, node, etnaviv_obj->size, va); else - ret = etnaviv_iommu_find_iova(context, node, - etnaviv_obj->base.size); + ret = etnaviv_iommu_find_iova(context, node, etnaviv_obj->size); if (ret < 0) goto unlock; mapping->iova = node->start; - ret = etnaviv_iommu_map(context, node->start, sgt, + ret = etnaviv_iommu_map(context, node->start, etnaviv_obj->size, sgt, ETNAVIV_PROT_READ | ETNAVIV_PROT_WRITE); if (ret < 0) { From 5a85670cfaaca6a88eb6ee23777e45de444e5967 Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 25 Oct 2024 17:14:17 +0200 Subject: [PATCH 14/17] drm/etnaviv: always allocate 4K for kernel ringbuffers Since the kernel ringbuffers are allocated from a larger suballocated area, same as the user commandbufs, they don't need to be CPU page sized. Allocate 4KB for the kernel ring buffers, as we never use more than that. Reviewed-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 8c1ef86e901c5..c7d59c06ccd18 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -840,8 +840,7 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu) goto fail; /* Create buffer: */ - ret = etnaviv_cmdbuf_init(priv->cmdbuf_suballoc, &gpu->buffer, - PAGE_SIZE); + ret = etnaviv_cmdbuf_init(priv->cmdbuf_suballoc, &gpu->buffer, SZ_4K); if (ret) { dev_err(gpu->dev, "could not create command buffer\n"); goto fail; From 4f8dbadef085ab447a01a8d4806a3f629fea05ed Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 25 Oct 2024 17:14:46 +0200 Subject: [PATCH 15/17] drm/etnaviv: flush shader L1 cache after user commandstream The shader L1 cache is a writeback cache for shader loads/stores and thus must be flushed before any BOs backing the shader buffers are potentially freed. Cc: stable@vger.kernel.org Reviewed-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_buffer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c index 384df1659be60..b13a17276d07c 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c @@ -482,7 +482,8 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state, } else { CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_DEPTH | - VIVS_GL_FLUSH_CACHE_COLOR); + VIVS_GL_FLUSH_CACHE_COLOR | + VIVS_GL_FLUSH_CACHE_SHADER_L1); if (has_blt) { CMD_LOAD_STATE(buffer, VIVS_BLT_ENABLE, 0x1); CMD_LOAD_STATE(buffer, VIVS_BLT_SET_COMMAND, 0x1); From 6fbd70e1706147955602e312e1bbecd9ee22b794 Mon Sep 17 00:00:00 2001 From: Sui Jingfeng Date: Sat, 26 Oct 2024 01:51:36 +0800 Subject: [PATCH 16/17] drm/etnaviv: Drop the 'struct etnaviv_iommu_global::pta_lock' data member Because it is not get used, drop it. Signed-off-by: Sui Jingfeng Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_mmu.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h index c01a147f0dfdd..7f8ac01785474 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h @@ -61,7 +61,6 @@ struct etnaviv_iommu_global { /* P(age) T(able) A(rray) */ u64 *pta_cpu; dma_addr_t pta_dma; - struct spinlock pta_lock; DECLARE_BITMAP(pta_alloc, ETNAVIV_PTA_ENTRIES); } v2; }; From 2c7ac7dd1b6c295636849c242685b0dd15beb093 Mon Sep 17 00:00:00 2001 From: Sui Jingfeng Date: Sat, 26 Oct 2024 01:56:20 +0800 Subject: [PATCH 17/17] drm/etnaviv: etnaviv_cmdbuf.c: Drop the unneeded include of drm_mm.h The etnaviv_cmdbuf.c doesn't reference any functions or data members defined in drm_mm.h, remove unneeded headers may reduce kernel compile times. Signed-off-by: Sui Jingfeng Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c index 66a407f1b3ee7..7aa5f14d0c873 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c @@ -5,8 +5,6 @@ #include -#include - #include "etnaviv_cmdbuf.h" #include "etnaviv_gem.h" #include "etnaviv_gpu.h"