Skip to content
Navigation Menu
Toggle navigation
Sign in
In this repository
All GitHub Enterprise
↵
Jump to
↵
No suggested jump to results
In this repository
All GitHub Enterprise
↵
Jump to
↵
In this organization
All GitHub Enterprise
↵
Jump to
↵
In this repository
All GitHub Enterprise
↵
Jump to
↵
Sign in
Reseting focus
You signed in with another tab or window.
Reload
to refresh your session.
You signed out in another tab or window.
Reload
to refresh your session.
You switched accounts on another tab or window.
Reload
to refresh your session.
Dismiss alert
{{ message }}
mariux64
/
linux
Public
Notifications
You must be signed in to change notification settings
Fork
0
Star
0
Code
Issues
2
Pull requests
0
Actions
Projects
0
Wiki
Security
Insights
Additional navigation options
Code
Issues
Pull requests
Actions
Projects
Wiki
Security
Insights
Files
e1b4ce2
Documentation
LICENSES
arch
block
certs
crypto
drivers
accessibility
acpi
amba
android
ata
atm
auxdisplay
base
bcma
block
bluetooth
bus
cdrom
char
clk
clocksource
connector
counter
cpufreq
cpuidle
crypto
dax
dca
devfreq
dio
dma-buf
dma
edac
eisa
extcon
firewire
firmware
fpga
fsi
gnss
gpio
gpu
drm
amd
arc
arm
armada
aspeed
ast
atmel-hlcdc
bochs
bridge
cirrus
etnaviv
exynos
fsl-dcu
gma500
hisilicon
i2c
i810
i915
imx
ingenic
lib
lima
mcde
mediatek
meson
mga
mgag200
msm
mxsfb
nouveau
omapdrm
panel
panfrost
pl111
qxl
r128
radeon
rcar-du
rockchip
savage
scheduler
Makefile
gpu_scheduler_trace.h
sched_entity.c
sched_fence.c
sched_main.c
selftests
shmobile
sis
sti
stm
sun4i
tdfx
tegra
tilcdc
tinydrm
ttm
tve200
udl
v3d
vboxvideo
vc4
vgem
via
virtio
vkms
vmwgfx
xen
zte
Kconfig
Makefile
ati_pcigart.c
drm_agpsupport.c
drm_atomic.c
drm_atomic_helper.c
drm_atomic_state_helper.c
drm_atomic_uapi.c
drm_auth.c
drm_blend.c
drm_bridge.c
drm_bufs.c
drm_cache.c
drm_client.c
drm_client_modeset.c
drm_color_mgmt.c
drm_connector.c
drm_context.c
drm_crtc.c
drm_crtc_helper.c
drm_crtc_helper_internal.h
drm_crtc_internal.h
drm_damage_helper.c
drm_debugfs.c
drm_debugfs_crc.c
drm_dma.c
drm_dp_aux_dev.c
drm_dp_cec.c
drm_dp_dual_mode_helper.c
drm_dp_helper.c
drm_dp_mst_topology.c
drm_drv.c
drm_dsc.c
drm_dumb_buffers.c
drm_edid.c
drm_edid_load.c
drm_encoder.c
drm_encoder_slave.c
drm_fb_cma_helper.c
drm_fb_helper.c
drm_file.c
drm_flip_work.c
drm_format_helper.c
drm_fourcc.c
drm_framebuffer.c
drm_gem.c
drm_gem_cma_helper.c
drm_gem_framebuffer_helper.c
drm_gem_shmem_helper.c
drm_gem_vram_helper.c
drm_hashtab.c
drm_hdcp.c
drm_internal.h
drm_ioc32.c
drm_ioctl.c
drm_irq.c
drm_kms_helper_common.c
drm_lease.c
drm_legacy.h
drm_legacy_misc.c
drm_lock.c
drm_memory.c
drm_mipi_dsi.c
drm_mm.c
drm_mode_config.c
drm_mode_object.c
drm_modes.c
drm_modeset_helper.c
drm_modeset_lock.c
drm_of.c
drm_panel.c
drm_panel_orientation_quirks.c
drm_pci.c
drm_plane.c
drm_plane_helper.c
drm_prime.c
drm_print.c
drm_probe_helper.c
drm_property.c
drm_rect.c
drm_scatter.c
drm_scdc_helper.c
drm_self_refresh_helper.c
drm_simple_kms_helper.c
drm_syncobj.c
drm_sysfs.c
drm_trace.h
drm_trace_points.c
drm_vblank.c
drm_vm.c
drm_vma_manager.c
drm_vram_helper_common.c
drm_vram_mm_helper.c
drm_writeback.c
host1x
ipu-v3
vga
Makefile
hid
hsi
hv
hwmon
hwspinlock
hwtracing
i2c
i3c
ide
idle
iio
infiniband
input
interconnect
iommu
ipack
irqchip
isdn
leds
lightnvm
macintosh
mailbox
mcb
md
media
memory
memstick
message
mfd
misc
mmc
mtd
mux
net
nfc
ntb
nubus
nvdimm
nvme
nvmem
of
opp
oprofile
parisc
parport
pci
pcmcia
perf
phy
pinctrl
platform
pnp
power
powercap
pps
ps3
ptp
pwm
rapidio
ras
regulator
remoteproc
reset
rpmsg
rtc
s390
sbus
scsi
sfi
sh
siox
slimbus
sn
soc
soundwire
spi
spmi
ssb
staging
target
tc
tee
thermal
thunderbolt
tty
uio
usb
uwb
vfio
vhost
video
virt
virtio
visorbus
vlynq
vme
w1
watchdog
xen
zorro
Kconfig
Makefile
fs
include
init
ipc
kernel
lib
mm
net
samples
scripts
security
sound
tools
usr
virt
.clang-format
.cocciconfig
.get_maintainer.ignore
.gitattributes
.gitignore
.mailmap
COPYING
CREDITS
Kbuild
Kconfig
MAINTAINERS
Makefile
README
Breadcrumbs
linux
/
drivers
/
gpu
/
drm
/
scheduler
/
sched_entity.c
Blame
Blame
Latest commit
Christian König
and
Alex Deucher
drm/scheduler: use job count instead of peek
Aug 14, 2019
e1b4ce2
·
Aug 14, 2019
History
History
540 lines (466 loc) · 14.4 KB
Breadcrumbs
linux
/
drivers
/
gpu
/
drm
/
scheduler
/
sched_entity.c
Top
File metadata and controls
Code
Blame
540 lines (466 loc) · 14.4 KB
Raw
/* * Copyright 2015 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <linux/kthread.h> #include <drm/gpu_scheduler.h> #include "gpu_scheduler_trace.h" #define to_drm_sched_job(sched_job) \ container_of((sched_job), struct drm_sched_job, queue_node) /** * drm_sched_entity_init - Init a context entity used by scheduler when * submit to HW ring. * * @entity: scheduler entity to init * @rq_list: the list of run queue on which jobs from this * entity can be submitted * @num_rq_list: number of run queue in rq_list * @guilty: atomic_t set to 1 when a job on this queue * is found to be guilty causing a timeout * * Note: the rq_list should have atleast one element to schedule * the entity * * Returns 0 on success or a negative error code on failure. */ int drm_sched_entity_init(struct drm_sched_entity *entity, struct drm_sched_rq **rq_list, unsigned int num_rq_list, atomic_t *guilty) { int i; if (!(entity && rq_list && (num_rq_list == 0 || rq_list[0]))) return -EINVAL; memset(entity, 0, sizeof(struct drm_sched_entity)); INIT_LIST_HEAD(&entity->list); entity->rq = NULL; entity->guilty = guilty; entity->num_rq_list = num_rq_list; entity->rq_list = kcalloc(num_rq_list, sizeof(struct drm_sched_rq *), GFP_KERNEL); if (!entity->rq_list) return -ENOMEM; for (i = 0; i < num_rq_list; ++i) entity->rq_list[i] = rq_list[i]; if (num_rq_list) entity->rq = rq_list[0]; entity->last_scheduled = NULL; spin_lock_init(&entity->rq_lock); spsc_queue_init(&entity->job_queue); atomic_set(&entity->fence_seq, 0); entity->fence_context = dma_fence_context_alloc(2); return 0; } EXPORT_SYMBOL(drm_sched_entity_init); /** * drm_sched_entity_is_idle - Check if entity is idle * * @entity: scheduler entity * * Returns true if the entity does not have any unscheduled jobs. */ static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity) { rmb(); /* for list_empty to work without lock */ if (list_empty(&entity->list) || spsc_queue_count(&entity->job_queue) == 0) return true; return false; } /** * drm_sched_entity_is_ready - Check if entity is ready * * @entity: scheduler entity * * Return true if entity could provide a job. */ bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) { if (spsc_queue_peek(&entity->job_queue) == NULL) return false; if (READ_ONCE(entity->dependency)) return false; return true; } /** * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load * * @entity: scheduler entity * * Return the pointer to the rq with least load. */ static struct drm_sched_rq * drm_sched_entity_get_free_sched(struct drm_sched_entity *entity) { struct drm_sched_rq *rq = NULL; unsigned int min_jobs = UINT_MAX, num_jobs; int i; for (i = 0; i < entity->num_rq_list; ++i) { struct drm_gpu_scheduler *sched = entity->rq_list[i]->sched; if (!entity->rq_list[i]->sched->ready) { DRM_WARN("sched%s is not ready, skipping", sched->name); continue; } num_jobs = atomic_read(&sched->num_jobs); if (num_jobs < min_jobs) { min_jobs = num_jobs; rq = entity->rq_list[i]; } } return rq; } /** * drm_sched_entity_flush - Flush a context entity * * @entity: scheduler entity * @timeout: time to wait in for Q to become empty in jiffies. * * Splitting drm_sched_entity_fini() into two functions, The first one does the * waiting, removes the entity from the runqueue and returns an error when the * process was killed. * * Returns the remaining time in jiffies left from the input timeout */ long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout) { struct drm_gpu_scheduler *sched; struct task_struct *last_user; long ret = timeout; if (!entity->rq) return 0; sched = entity->rq->sched; /** * The client will not queue more IBs during this fini, consume existing * queued IBs or discard them on SIGKILL */ if (current->flags & PF_EXITING) { if (timeout) ret = wait_event_timeout( sched->job_scheduled, drm_sched_entity_is_idle(entity), timeout); } else { wait_event_killable(sched->job_scheduled, drm_sched_entity_is_idle(entity)); } /* For killed process disable any more IBs enqueue right now */ last_user = cmpxchg(&entity->last_user, current->group_leader, NULL); if ((!last_user || last_user == current->group_leader) && (current->flags & PF_EXITING) && (current->exit_code == SIGKILL)) { spin_lock(&entity->rq_lock); entity->stopped = true; drm_sched_rq_remove_entity(entity->rq, entity); spin_unlock(&entity->rq_lock); } return ret; } EXPORT_SYMBOL(drm_sched_entity_flush); /** * drm_sched_entity_kill_jobs - helper for drm_sched_entity_kill_jobs * * @f: signaled fence * @cb: our callback structure * * Signal the scheduler finished fence when the entity in question is killed. */ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, struct dma_fence_cb *cb) { struct drm_sched_job *job = container_of(cb, struct drm_sched_job, finish_cb); drm_sched_fence_finished(job->s_fence); WARN_ON(job->s_fence->parent); job->sched->ops->free_job(job); } /** * drm_sched_entity_kill_jobs - Make sure all remaining jobs are killed * * @entity: entity which is cleaned up * * Makes sure that all remaining jobs in an entity are killed before it is * destroyed. */ static void drm_sched_entity_kill_jobs(struct drm_sched_entity *entity) { struct drm_sched_job *job; int r; while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) { struct drm_sched_fence *s_fence = job->s_fence; drm_sched_fence_scheduled(s_fence); dma_fence_set_error(&s_fence->finished, -ESRCH); /* * When pipe is hanged by older entity, new entity might * not even have chance to submit it's first job to HW * and so entity->last_scheduled will remain NULL */ if (!entity->last_scheduled) { drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); continue; } r = dma_fence_add_callback(entity->last_scheduled, &job->finish_cb, drm_sched_entity_kill_jobs_cb); if (r == -ENOENT) drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); } } /** * drm_sched_entity_cleanup - Destroy a context entity * * @entity: scheduler entity * * This should be called after @drm_sched_entity_do_release. It goes over the * entity and signals all jobs with an error code if the process was killed. * */ void drm_sched_entity_fini(struct drm_sched_entity *entity) { struct drm_gpu_scheduler *sched = NULL; if (entity->rq) { sched = entity->rq->sched; drm_sched_rq_remove_entity(entity->rq, entity); } /* Consumption of existing IBs wasn't completed. Forcefully * remove them here. */ if (spsc_queue_count(&entity->job_queue)) { if (sched) { /* Park the kernel for a moment to make sure it isn't processing * our enity. */ kthread_park(sched->thread); kthread_unpark(sched->thread); } if (entity->dependency) { dma_fence_remove_callback(entity->dependency, &entity->cb); dma_fence_put(entity->dependency); entity->dependency = NULL; } drm_sched_entity_kill_jobs(entity); } dma_fence_put(entity->last_scheduled); entity->last_scheduled = NULL; kfree(entity->rq_list); } EXPORT_SYMBOL(drm_sched_entity_fini); /** * drm_sched_entity_fini - Destroy a context entity * * @entity: scheduler entity * * Calls drm_sched_entity_do_release() and drm_sched_entity_cleanup() */ void drm_sched_entity_destroy(struct drm_sched_entity *entity) { drm_sched_entity_flush(entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY); drm_sched_entity_fini(entity); } EXPORT_SYMBOL(drm_sched_entity_destroy); /** * drm_sched_entity_clear_dep - callback to clear the entities dependency */ static void drm_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb) { struct drm_sched_entity *entity = container_of(cb, struct drm_sched_entity, cb); entity->dependency = NULL; dma_fence_put(f); } /** * drm_sched_entity_clear_dep - callback to clear the entities dependency and * wake up scheduler */ static void drm_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb) { struct drm_sched_entity *entity = container_of(cb, struct drm_sched_entity, cb); drm_sched_entity_clear_dep(f, cb); drm_sched_wakeup(entity->rq->sched); } /** * drm_sched_entity_set_rq_priority - helper for drm_sched_entity_set_priority */ static void drm_sched_entity_set_rq_priority(struct drm_sched_rq **rq, enum drm_sched_priority priority) { *rq = &(*rq)->sched->sched_rq[priority]; } /** * drm_sched_entity_set_priority - Sets priority of the entity * * @entity: scheduler entity * @priority: scheduler priority * * Update the priority of runqueus used for the entity. */ void drm_sched_entity_set_priority(struct drm_sched_entity *entity, enum drm_sched_priority priority) { unsigned int i; spin_lock(&entity->rq_lock); for (i = 0; i < entity->num_rq_list; ++i) drm_sched_entity_set_rq_priority(&entity->rq_list[i], priority); if (entity->rq) { drm_sched_rq_remove_entity(entity->rq, entity); drm_sched_entity_set_rq_priority(&entity->rq, priority); drm_sched_rq_add_entity(entity->rq, entity); } spin_unlock(&entity->rq_lock); } EXPORT_SYMBOL(drm_sched_entity_set_priority); /** * drm_sched_entity_add_dependency_cb - add callback for the entities dependency * * @entity: entity with dependency * * Add a callback to the current dependency of the entity to wake up the * scheduler when the entity becomes available. */ static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity) { struct drm_gpu_scheduler *sched = entity->rq->sched; struct dma_fence *fence = entity->dependency; struct drm_sched_fence *s_fence; if (fence->context == entity->fence_context || fence->context == entity->fence_context + 1) { /* * Fence is a scheduled/finished fence from a job * which belongs to the same entity, we can ignore * fences from ourself */ dma_fence_put(entity->dependency); return false; } s_fence = to_drm_sched_fence(fence); if (s_fence && s_fence->sched == sched) { /* * Fence is from the same scheduler, only need to wait for * it to be scheduled */ fence = dma_fence_get(&s_fence->scheduled); dma_fence_put(entity->dependency); entity->dependency = fence; if (!dma_fence_add_callback(fence, &entity->cb, drm_sched_entity_clear_dep)) return true; /* Ignore it when it is already scheduled */ dma_fence_put(fence); return false; } if (!dma_fence_add_callback(entity->dependency, &entity->cb, drm_sched_entity_wakeup)) return true; dma_fence_put(entity->dependency); return false; } /** * drm_sched_entity_pop_job - get a ready to be scheduled job from the entity * * @entity: entity to get the job from * * Process all dependencies and try to get one job from the entities queue. */ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) { struct drm_gpu_scheduler *sched = entity->rq->sched; struct drm_sched_job *sched_job; sched_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue)); if (!sched_job) return NULL; while ((entity->dependency = sched->ops->dependency(sched_job, entity))) { trace_drm_sched_job_wait_dep(sched_job, entity->dependency); if (drm_sched_entity_add_dependency_cb(entity)) return NULL; } /* skip jobs from entity that marked guilty */ if (entity->guilty && atomic_read(entity->guilty)) dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED); dma_fence_put(entity->last_scheduled); entity->last_scheduled = dma_fence_get(&sched_job->s_fence->finished); spsc_queue_pop(&entity->job_queue); return sched_job; } /** * drm_sched_entity_select_rq - select a new rq for the entity * * @entity: scheduler entity * * Check all prerequisites and select a new rq for the entity for load * balancing. */ void drm_sched_entity_select_rq(struct drm_sched_entity *entity) { struct dma_fence *fence; struct drm_sched_rq *rq; if (spsc_queue_count(&entity->job_queue) || entity->num_rq_list <= 1) return; fence = READ_ONCE(entity->last_scheduled); if (fence && !dma_fence_is_signaled(fence)) return; rq = drm_sched_entity_get_free_sched(entity); if (rq == entity->rq) return; spin_lock(&entity->rq_lock); drm_sched_rq_remove_entity(entity->rq, entity); entity->rq = rq; spin_unlock(&entity->rq_lock); } /** * drm_sched_entity_push_job - Submit a job to the entity's job queue * * @sched_job: job to submit * @entity: scheduler entity * * Note: To guarantee that the order of insertion to queue matches * the job's fence sequence number this function should be * called with drm_sched_job_init under common lock. * * Returns 0 for success, negative error code otherwise. */ void drm_sched_entity_push_job(struct drm_sched_job *sched_job, struct drm_sched_entity *entity) { bool first; trace_drm_sched_job(sched_job, entity); atomic_inc(&entity->rq->sched->num_jobs); WRITE_ONCE(entity->last_user, current->group_leader); first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node); /* first job wakes up scheduler */ if (first) { /* Add the entity to the run queue */ spin_lock(&entity->rq_lock); if (entity->stopped) { spin_unlock(&entity->rq_lock); DRM_ERROR("Trying to push to a killed entity\n"); return; } drm_sched_rq_add_entity(entity->rq, entity); spin_unlock(&entity->rq_lock); drm_sched_wakeup(entity->rq->sched); } } EXPORT_SYMBOL(drm_sched_entity_push_job);
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
You can’t perform that action at this time.