Skip to content

Commit

Permalink
drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of
Browse files Browse the repository at this point in the history
compute units that are in use.

[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.

[How]
Surface files in Sysfs that allow user to determine the number of compute
units that are in use for a given process. One Sysfs file is used per
device.

Change-Id: I99f45e7b60312ef9f3b6aab5458dc4475e69a204
Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
Reviewed-By: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
  • Loading branch information
Ramesh Errabolu authored and Harish Kasiviswanathan committed Oct 23, 2020
1 parent 99dc7b9 commit f6e990e
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 7 deletions.
25 changes: 25 additions & 0 deletions drivers/gpu/drm/amd/amdkfd/kfd_priv.h
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,31 @@ struct kfd_process_device {
struct attribute attr_evict;

struct kobject *kobj_stats;

/*
* @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
* that is associated with device encoded by "this" struct instance. The
* value reflects CU usage by all of the waves launched by this process
* on this device. A very important property of occupancy parameter is
* that its value is a snapshot of current use.
*
* Following is to be noted regarding how this parameter is reported:
*
* The number of waves that a CU can launch is limited by couple of
* parameters. These are encoded by struct amdgpu_cu_info instance
* that is part of every device definition. For GFX9 devices this
* translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
* do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
* when they do use scratch memory. This could change for future
* devices and therefore this example should be considered as a guide.
*
* All CU's of a device are available for the process. This may not be true
* under certain conditions - e.g. CU masking.
*
* Finally number of CU's that are occupied by a process is affected by both
* number of CU's a device has along with number of other competing processes
*/
struct attribute attr_cu_occupancy;
};

#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
Expand Down
76 changes: 69 additions & 7 deletions drivers/gpu/drm/amd/amdkfd/kfd_process.c
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,52 @@ static void kfd_sdma_activity_worker(struct work_struct *work)
}
}

/**
* @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
* by current process. Translates acquired wave count into number of compute units
* that are occupied.
*
* @atr: Handle of attribute that allows reporting of wave count. The attribute
* handle encapsulates GPU device it is associated with, thereby allowing collection
* of waves in flight, etc
*
* @buffer: Handle of user provided buffer updated with wave count
*
* Return: Number of bytes written to user buffer or an error value
*/
static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
{
int cu_cnt;
int wave_cnt;
int max_waves_per_cu;
struct kfd_dev *dev = NULL;
struct kfd_process *proc = NULL;
struct kfd_process_device *pdd = NULL;

pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
dev = pdd->dev;
if (dev->kfd2kgd->get_cu_occupancy == NULL)
return -EINVAL;

cu_cnt = 0;
proc = pdd->process;
if (pdd->qpd.queue_count == 0) {
pr_debug("Gpu-Id: %d has no active queues for process %d\n",
dev->id, proc->pasid);
return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
}

/* Collect wave count from device if it supports */
wave_cnt = 0;
max_waves_per_cu = 0;
dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
&max_waves_per_cu);

/* Translate wave count to number of compute units */
cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
}

static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
char *buffer)
{
Expand Down Expand Up @@ -359,6 +405,7 @@ static ssize_t kfd_procfs_queue_show(struct kobject *kobj,

return 0;
}

static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
struct attribute *attr, char *buffer)
{
Expand All @@ -374,8 +421,13 @@ static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
PAGE_SIZE,
"%llu\n",
jiffies64_to_msecs(evict_jiffies));
} else

/* Sysfs handle that gets CU occupancy is per device */
} else if (strcmp(attr->name, "cu_occupancy") == 0) {
return kfd_get_cu_occupancy(attr, buffer);
} else {
pr_err("Invalid attribute");
}

return 0;
}
Expand Down Expand Up @@ -481,6 +533,7 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
* Create sysfs files for each GPU:
* - proc/<pid>/stats_<gpuid>/
* - proc/<pid>/stats_<gpuid>/evicted_ms
* - proc/<pid>/stats_<gpuid>/cu_occupancy
*/
list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
struct kobject *kobj_stats;
Expand Down Expand Up @@ -511,6 +564,19 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
if (ret)
pr_warn("Creating eviction stats for gpuid %d failed",
(int)pdd->dev->id);

/* Add sysfs file to report compute unit occupancy */
if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
pdd->attr_cu_occupancy.name = "cu_occupancy";
pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE;
sysfs_attr_init(&pdd->attr_cu_occupancy);
ret = sysfs_create_file(kobj_stats,
&pdd->attr_cu_occupancy);
if (ret)
pr_warn("Creating %s failed for gpuid: %d",
pdd->attr_cu_occupancy.name,
(int)pdd->dev->id);
}
}
err:
return ret;
Expand Down Expand Up @@ -552,7 +618,6 @@ static int kfd_procfs_add_sysfs_files(struct kfd_process *p)
return ret;
}


void kfd_procfs_del_queue(struct queue *q)
{
if (!q)
Expand Down Expand Up @@ -770,11 +835,6 @@ struct kfd_process *kfd_create_process(struct task_struct *thread)
pr_warn("Creating sysfs stats dir for pid %d failed",
(int)process->lead_thread->pid);

ret = kfd_procfs_add_sysfs_stats(process);
if (ret)
pr_warn("Creating sysfs stats dir for pid %d failed",
(int)process->lead_thread->pid);

ret = kfd_procfs_add_sysfs_files(process);
if (ret)
pr_warn("Creating sysfs usage file for pid %d failed",
Expand Down Expand Up @@ -960,6 +1020,8 @@ static void kfd_process_wq_release(struct work_struct *work)
sysfs_remove_file(p->kobj, &pdd->attr_vram);
sysfs_remove_file(p->kobj, &pdd->attr_sdma);
sysfs_remove_file(p->kobj, &pdd->attr_evict);
if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL)
sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy);
kobject_del(pdd->kobj_stats);
kobject_put(pdd->kobj_stats);
pdd->kobj_stats = NULL;
Expand Down

0 comments on commit f6e990e

Please sign in to comment.