-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
drm/i915: Perform object clflushing asynchronously
Flushing the cachelines for an object is slow, can be as much as 100ms for a large framebuffer. We currently do this under the struct_mutex BKL on execution or on pageflip. But now with the ability to add fences to obj->resv for both flips and execbuf (and we naturally wait on the fence before CPU access), we can move the clflush operation to a workqueue and signal a fence for completion, thereby doing the work asynchronously and not blocking the driver or its clients. v2: Introduce i915_gem_clflush.h and use a new name, split out some extras into separate patches. Suggested-by: Akash Goel <akash.goel@intel.com> Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com> Cc: Matthew Auld <matthew.auld@intel.com> Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com> Link: http://patchwork.freedesktop.org/patch/msgid/20170222114049.28456-5-chris@chris-wilson.co.uk
- Loading branch information
Chris Wilson
committed
Feb 22, 2017
1 parent
f6aaba4
commit 57822dc
Showing
7 changed files
with
264 additions
and
72 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,189 @@ | ||
| /* | ||
| * Copyright © 2016 Intel Corporation | ||
| * | ||
| * Permission is hereby granted, free of charge, to any person obtaining a | ||
| * copy of this software and associated documentation files (the "Software"), | ||
| * to deal in the Software without restriction, including without limitation | ||
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| * and/or sell copies of the Software, and to permit persons to whom the | ||
| * Software is furnished to do so, subject to the following conditions: | ||
| * | ||
| * The above copyright notice and this permission notice (including the next | ||
| * paragraph) shall be included in all copies or substantial portions of the | ||
| * Software. | ||
| * | ||
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | ||
| * IN THE SOFTWARE. | ||
| * | ||
| */ | ||
|
|
||
| #include "i915_drv.h" | ||
| #include "intel_frontbuffer.h" | ||
| #include "i915_gem_clflush.h" | ||
|
|
||
| static DEFINE_SPINLOCK(clflush_lock); | ||
| static u64 clflush_context; | ||
|
|
||
| struct clflush { | ||
| struct dma_fence dma; /* Must be first for dma_fence_free() */ | ||
| struct i915_sw_fence wait; | ||
| struct work_struct work; | ||
| struct drm_i915_gem_object *obj; | ||
| }; | ||
|
|
||
| static const char *i915_clflush_get_driver_name(struct dma_fence *fence) | ||
| { | ||
| return DRIVER_NAME; | ||
| } | ||
|
|
||
| static const char *i915_clflush_get_timeline_name(struct dma_fence *fence) | ||
| { | ||
| return "clflush"; | ||
| } | ||
|
|
||
| static bool i915_clflush_enable_signaling(struct dma_fence *fence) | ||
| { | ||
| return true; | ||
| } | ||
|
|
||
| static void i915_clflush_release(struct dma_fence *fence) | ||
| { | ||
| struct clflush *clflush = container_of(fence, typeof(*clflush), dma); | ||
|
|
||
| i915_sw_fence_fini(&clflush->wait); | ||
|
|
||
| BUILD_BUG_ON(offsetof(typeof(*clflush), dma)); | ||
| dma_fence_free(&clflush->dma); | ||
| } | ||
|
|
||
| static const struct dma_fence_ops i915_clflush_ops = { | ||
| .get_driver_name = i915_clflush_get_driver_name, | ||
| .get_timeline_name = i915_clflush_get_timeline_name, | ||
| .enable_signaling = i915_clflush_enable_signaling, | ||
| .wait = dma_fence_default_wait, | ||
| .release = i915_clflush_release, | ||
| }; | ||
|
|
||
| static void __i915_do_clflush(struct drm_i915_gem_object *obj) | ||
| { | ||
| drm_clflush_sg(obj->mm.pages); | ||
| obj->cache_dirty = false; | ||
|
|
||
| intel_fb_obj_flush(obj, false, ORIGIN_CPU); | ||
| } | ||
|
|
||
| static void i915_clflush_work(struct work_struct *work) | ||
| { | ||
| struct clflush *clflush = container_of(work, typeof(*clflush), work); | ||
| struct drm_i915_gem_object *obj = clflush->obj; | ||
|
|
||
| if (!obj->cache_dirty) | ||
| goto out; | ||
|
|
||
| if (i915_gem_object_pin_pages(obj)) { | ||
| DRM_ERROR("Failed to acquire obj->pages for clflushing\n"); | ||
| goto out; | ||
| } | ||
|
|
||
| __i915_do_clflush(obj); | ||
|
|
||
| i915_gem_object_unpin_pages(obj); | ||
|
|
||
| out: | ||
| i915_gem_object_put(obj); | ||
|
|
||
| dma_fence_signal(&clflush->dma); | ||
| dma_fence_put(&clflush->dma); | ||
| } | ||
|
|
||
| static int __i915_sw_fence_call | ||
| i915_clflush_notify(struct i915_sw_fence *fence, | ||
| enum i915_sw_fence_notify state) | ||
| { | ||
| struct clflush *clflush = container_of(fence, typeof(*clflush), wait); | ||
|
|
||
| switch (state) { | ||
| case FENCE_COMPLETE: | ||
| schedule_work(&clflush->work); | ||
| break; | ||
|
|
||
| case FENCE_FREE: | ||
| dma_fence_put(&clflush->dma); | ||
| break; | ||
| } | ||
|
|
||
| return NOTIFY_DONE; | ||
| } | ||
|
|
||
| void i915_gem_clflush_object(struct drm_i915_gem_object *obj, | ||
| unsigned int flags) | ||
| { | ||
| struct clflush *clflush; | ||
|
|
||
| /* | ||
| * Stolen memory is always coherent with the GPU as it is explicitly | ||
| * marked as wc by the system, or the system is cache-coherent. | ||
| * Similarly, we only access struct pages through the CPU cache, so | ||
| * anything not backed by physical memory we consider to be always | ||
| * coherent and not need clflushing. | ||
| */ | ||
| if (!i915_gem_object_has_struct_page(obj)) | ||
| return; | ||
|
|
||
| obj->cache_dirty = true; | ||
|
|
||
| /* If the GPU is snooping the contents of the CPU cache, | ||
| * we do not need to manually clear the CPU cache lines. However, | ||
| * the caches are only snooped when the render cache is | ||
| * flushed/invalidated. As we always have to emit invalidations | ||
| * and flushes when moving into and out of the RENDER domain, correct | ||
| * snooping behaviour occurs naturally as the result of our domain | ||
| * tracking. | ||
| */ | ||
| if (!(flags & I915_CLFLUSH_FORCE) && i915_gem_object_is_coherent(obj)) | ||
| return; | ||
|
|
||
| trace_i915_gem_object_clflush(obj); | ||
|
|
||
| clflush = NULL; | ||
| if (!(flags & I915_CLFLUSH_SYNC)) | ||
| clflush = kmalloc(sizeof(*clflush), GFP_KERNEL); | ||
| if (clflush) { | ||
| dma_fence_init(&clflush->dma, | ||
| &i915_clflush_ops, | ||
| &clflush_lock, | ||
| clflush_context, | ||
| 0); | ||
| i915_sw_fence_init(&clflush->wait, i915_clflush_notify); | ||
|
|
||
| clflush->obj = i915_gem_object_get(obj); | ||
| INIT_WORK(&clflush->work, i915_clflush_work); | ||
|
|
||
| dma_fence_get(&clflush->dma); | ||
|
|
||
| i915_sw_fence_await_reservation(&clflush->wait, | ||
| obj->resv, NULL, | ||
| false, I915_FENCE_TIMEOUT, | ||
| GFP_KERNEL); | ||
|
|
||
| reservation_object_lock(obj->resv, NULL); | ||
| reservation_object_add_excl_fence(obj->resv, &clflush->dma); | ||
| reservation_object_unlock(obj->resv); | ||
|
|
||
| i915_sw_fence_commit(&clflush->wait); | ||
| } else if (obj->mm.pages) { | ||
| __i915_do_clflush(obj); | ||
| } else { | ||
| GEM_BUG_ON(obj->base.write_domain != I915_GEM_DOMAIN_CPU); | ||
| } | ||
| } | ||
|
|
||
| void i915_gem_clflush_init(struct drm_i915_private *i915) | ||
| { | ||
| clflush_context = dma_fence_context_alloc(1); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| /* | ||
| * Copyright © 2016 Intel Corporation | ||
| * | ||
| * Permission is hereby granted, free of charge, to any person obtaining a | ||
| * copy of this software and associated documentation files (the "Software"), | ||
| * to deal in the Software without restriction, including without limitation | ||
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| * and/or sell copies of the Software, and to permit persons to whom the | ||
| * Software is furnished to do so, subject to the following conditions: | ||
| * | ||
| * The above copyright notice and this permission notice (including the next | ||
| * paragraph) shall be included in all copies or substantial portions of the | ||
| * Software. | ||
| * | ||
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | ||
| * IN THE SOFTWARE. | ||
| * | ||
| */ | ||
|
|
||
| #ifndef __I915_GEM_CLFLUSH_H__ | ||
| #define __I915_GEM_CLFLUSH_H__ | ||
|
|
||
| struct drm_i915_private; | ||
| struct drm_i915_gem_object; | ||
|
|
||
| void i915_gem_clflush_init(struct drm_i915_private *i915); | ||
| void i915_gem_clflush_object(struct drm_i915_gem_object *obj, | ||
| unsigned int flags); | ||
| #define I915_CLFLUSH_FORCE BIT(0) | ||
| #define I915_CLFLUSH_SYNC BIT(1) | ||
|
|
||
| #endif /* __I915_GEM_CLFLUSH_H__ */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.