From 16ccbd87c2e5d95f40f3b7a90f40cc2b34b89a22 Mon Sep 17 00:00:00 2001 From: Jacob Pan <jacob.jun.pan@linux.intel.com> Date: Wed, 24 Feb 2016 13:31:35 -0800 Subject: [PATCH 1/4] cpumask: export cpumask_any_but Export cpumask_any_but() for module use. This will be used by drivers such as intel_rapl to locate an active cpu on a socket. Signed-off-by: Jacob Pan <jacob.jun.pan@linux.intel.com> Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com> --- lib/cpumask.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cpumask.c b/lib/cpumask.c index 5a70f6196f577..81dedaab36ccf 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -41,6 +41,7 @@ int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) break; return i; } +EXPORT_SYMBOL(cpumask_any_but); /* These are not inline because of header tangles. */ #ifdef CONFIG_CPUMASK_OFFSTACK From f14a1396d8f19b6c53593045eba86d10360a0cee Mon Sep 17 00:00:00 2001 From: Jacob Pan <jacob.jun.pan@linux.intel.com> Date: Wed, 24 Feb 2016 13:31:36 -0800 Subject: [PATCH 2/4] powercap/rapl: reduce ipi calls Reduce remote CPU calls for MSR access by combining read modify write into one function. Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Jacob Pan <jacob.jun.pan@linux.intel.com> Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com> --- drivers/powercap/intel_rapl.c | 119 ++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 34 deletions(-) diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c index 6c592dc71aeef..a50e644ae07a1 100644 --- a/drivers/powercap/intel_rapl.c +++ b/drivers/powercap/intel_rapl.c @@ -133,6 +133,12 @@ struct rapl_domain_data { unsigned long timestamp; }; +struct msrl_action { + u32 msr_no; + u64 clear_mask; + u64 set_mask; + int err; +}; #define DOMAIN_STATE_INACTIVE BIT(0) #define DOMAIN_STATE_POWER_LIMIT_SET BIT(1) @@ -800,35 +806,62 @@ static int rapl_read_data_raw(struct rapl_domain *rd, return 0; } + +static int msrl_update_safe(u32 msr_no, u64 clear_mask, u64 set_mask) +{ + int err; + u64 val; + + err = rdmsrl_safe(msr_no, &val); + if (err) + goto out; + + val &= ~clear_mask; + val |= set_mask; + + err = wrmsrl_safe(msr_no, val); + +out: + return err; +} + +static void msrl_update_func(void *info) +{ + struct msrl_action *ma = info; + + ma->err = msrl_update_safe(ma->msr_no, ma->clear_mask, ma->set_mask); +} + /* Similar use of primitive info in the read counterpart */ static int rapl_write_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, unsigned long long value) { - u64 msr_val; - u32 msr; struct rapl_primitive_info *rp = &rpi[prim]; int cpu; + u64 bits; + struct msrl_action ma; + int ret; cpu = find_active_cpu_on_package(rd->package_id); if (cpu < 0) return cpu; - msr = rd->msrs[rp->id]; - if (rdmsrl_safe_on_cpu(cpu, msr, &msr_val)) { - dev_dbg(&rd->power_zone.dev, - "failed to read msr 0x%x on cpu %d\n", msr, cpu); - return -EIO; - } - value = rapl_unit_xlate(rd, rd->package_id, rp->unit, value, 1); - msr_val &= ~rp->mask; - msr_val |= value << rp->shift; - if (wrmsrl_safe_on_cpu(cpu, msr, msr_val)) { - dev_dbg(&rd->power_zone.dev, - "failed to write msr 0x%x on cpu %d\n", msr, cpu); - return -EIO; - } - return 0; + bits = rapl_unit_xlate(rd, rd->package_id, rp->unit, value, 1); + bits |= bits << rp->shift; + memset(&ma, 0, sizeof(ma)); + + ma.msr_no = rd->msrs[rp->id]; + ma.clear_mask = rp->mask; + ma.set_mask = bits; + + ret = smp_call_function_single(cpu, msrl_update_func, &ma, 1); + if (ret) + WARN_ON_ONCE(ret); + else + ret = ma.err; + + return ret; } /* @@ -893,6 +926,21 @@ static int rapl_check_unit_atom(struct rapl_package *rp, int cpu) return 0; } +static void power_limit_irq_save_cpu(void *info) +{ + u32 l, h = 0; + struct rapl_package *rp = (struct rapl_package *)info; + + /* save the state of PLN irq mask bit before disabling it */ + rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h); + if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) { + rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE; + rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED; + } + l &= ~PACKAGE_THERM_INT_PLN_ENABLE; + wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); +} + /* REVISIT: * When package power limit is set artificially low by RAPL, LVT @@ -906,7 +954,6 @@ static int rapl_check_unit_atom(struct rapl_package *rp, int cpu) static void package_power_limit_irq_save(int package_id) { - u32 l, h = 0; int cpu; struct rapl_package *rp; @@ -920,20 +967,30 @@ static void package_power_limit_irq_save(int package_id) cpu = find_active_cpu_on_package(package_id); if (cpu < 0) return; - /* save the state of PLN irq mask bit before disabling it */ - rdmsr_safe_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h); - if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) { - rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE; - rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED; - } - l &= ~PACKAGE_THERM_INT_PLN_ENABLE; - wrmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); + if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) + return; + + smp_call_function_single(cpu, power_limit_irq_save_cpu, rp, 1); +} + +static void power_limit_irq_restore_cpu(void *info) +{ + u32 l, h = 0; + struct rapl_package *rp = (struct rapl_package *)info; + + rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h); + + if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE) + l |= PACKAGE_THERM_INT_PLN_ENABLE; + else + l &= ~PACKAGE_THERM_INT_PLN_ENABLE; + + wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); } /* restore per package power limit interrupt enable state */ static void package_power_limit_irq_restore(int package_id) { - u32 l, h; int cpu; struct rapl_package *rp; @@ -951,14 +1008,8 @@ static void package_power_limit_irq_restore(int package_id) /* irq enable state not saved, nothing to restore */ if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) return; - rdmsr_safe_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h); - - if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE) - l |= PACKAGE_THERM_INT_PLN_ENABLE; - else - l &= ~PACKAGE_THERM_INT_PLN_ENABLE; - wrmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); + smp_call_function_single(cpu, power_limit_irq_restore_cpu, rp, 1); } static void set_floor_freq_default(struct rapl_domain *rd, bool mode) From 309557f558a6f276e364b08d916c0f644b5bd2e1 Mon Sep 17 00:00:00 2001 From: Jacob Pan <jacob.jun.pan@linux.intel.com> Date: Wed, 24 Feb 2016 13:31:37 -0800 Subject: [PATCH 3/4] powercap/rapl: add package reference per domain This patch adds to each rapl domain a reference of the package it belongs to. At runtime, we can then avoid searching the package data for each access. It simplifies the domain level operations which depend on package level information. Signed-off-by: Jacob Pan <jacob.jun.pan@linux.intel.com> Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com> --- drivers/powercap/intel_rapl.c | 66 +++++++++++------------------------ 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c index a50e644ae07a1..3413692c4dde3 100644 --- a/drivers/powercap/intel_rapl.c +++ b/drivers/powercap/intel_rapl.c @@ -155,6 +155,7 @@ struct rapl_power_limit { static const char pl1_name[] = "long_term"; static const char pl2_name[] = "short_term"; +struct rapl_package; struct rapl_domain { const char *name; enum rapl_domain_type id; @@ -165,7 +166,7 @@ struct rapl_domain { u64 attr_map; /* track capabilities */ unsigned int state; unsigned int domain_energy_unit; - int package_id; + struct rapl_package *rp; }; #define power_zone_to_rapl_domain(_zone) \ container_of(_zone, struct rapl_domain, power_zone) @@ -237,10 +238,10 @@ static int rapl_read_data_raw(struct rapl_domain *rd, static int rapl_write_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, unsigned long long value); -static u64 rapl_unit_xlate(struct rapl_domain *rd, int package, +static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, u64 value, int to_raw); -static void package_power_limit_irq_save(int package_id); +static void package_power_limit_irq_save(struct rapl_package *rp); static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */ @@ -318,25 +319,19 @@ static int get_max_energy_counter(struct powercap_zone *pcd_dev, u64 *energy) { struct rapl_domain *rd = power_zone_to_rapl_domain(pcd_dev); - *energy = rapl_unit_xlate(rd, 0, ENERGY_UNIT, ENERGY_STATUS_MASK, 0); + *energy = rapl_unit_xlate(rd, ENERGY_UNIT, ENERGY_STATUS_MASK, 0); return 0; } static int release_zone(struct powercap_zone *power_zone) { struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone); - struct rapl_package *rp; + struct rapl_package *rp = rd->rp; /* package zone is the last zone of a package, we can free * memory here since all children has been unregistered. */ if (rd->id == RAPL_DOMAIN_PACKAGE) { - rp = find_package_by_id(rd->package_id); - if (!rp) { - dev_warn(&power_zone->dev, "no package id %s\n", - rd->name); - return -ENODEV; - } kfree(rd); rp->domains = NULL; } @@ -438,11 +433,7 @@ static int set_power_limit(struct powercap_zone *power_zone, int id, get_online_cpus(); rd = power_zone_to_rapl_domain(power_zone); - rp = find_package_by_id(rd->package_id); - if (!rp) { - ret = -ENODEV; - goto set_exit; - } + rp = rd->rp; if (rd->state & DOMAIN_STATE_BIOS_LOCKED) { dev_warn(&power_zone->dev, "%s locked by BIOS, monitoring only\n", @@ -462,7 +453,7 @@ static int set_power_limit(struct powercap_zone *power_zone, int id, ret = -EINVAL; } if (!ret) - package_power_limit_irq_save(rd->package_id); + package_power_limit_irq_save(rp); set_exit: put_online_cpus(); return ret; @@ -661,24 +652,19 @@ static void rapl_init_domains(struct rapl_package *rp) break; } if (mask) { - rd->package_id = rp->id; + rd->rp = rp; rd++; } } } -static u64 rapl_unit_xlate(struct rapl_domain *rd, int package, - enum unit_type type, u64 value, - int to_raw) +static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, + u64 value, int to_raw) { u64 units = 1; - struct rapl_package *rp; + struct rapl_package *rp = rd->rp; u64 scale = 1; - rp = find_package_by_id(package); - if (!rp) - return value; - switch (type) { case POWER_UNIT: units = rp->power_unit; @@ -776,7 +762,7 @@ static int rapl_read_data_raw(struct rapl_domain *rd, if (!msr) return -EINVAL; /* use physical package id to look up active cpus */ - cpu = find_active_cpu_on_package(rd->package_id); + cpu = find_active_cpu_on_package(rd->rp->id); if (cpu < 0) return cpu; @@ -799,7 +785,7 @@ static int rapl_read_data_raw(struct rapl_domain *rd, final = value & rp->mask; final = final >> rp->shift; if (xlate) - *data = rapl_unit_xlate(rd, rd->package_id, rp->unit, final, 0); + *data = rapl_unit_xlate(rd, rp->unit, final, 0); else *data = final; @@ -843,11 +829,11 @@ static int rapl_write_data_raw(struct rapl_domain *rd, struct msrl_action ma; int ret; - cpu = find_active_cpu_on_package(rd->package_id); + cpu = find_active_cpu_on_package(rd->rp->id); if (cpu < 0) return cpu; - bits = rapl_unit_xlate(rd, rd->package_id, rp->unit, value, 1); + bits = rapl_unit_xlate(rd, rp->unit, value, 1); bits |= bits << rp->shift; memset(&ma, 0, sizeof(ma)); @@ -952,19 +938,14 @@ static void power_limit_irq_save_cpu(void *info) * to do by adding an atomic notifier. */ -static void package_power_limit_irq_save(int package_id) +static void package_power_limit_irq_save(struct rapl_package *rp) { int cpu; - struct rapl_package *rp; - - rp = find_package_by_id(package_id); - if (!rp) - return; if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) return; - cpu = find_active_cpu_on_package(package_id); + cpu = find_active_cpu_on_package(rp->id); if (cpu < 0) return; if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) @@ -989,19 +970,14 @@ static void power_limit_irq_restore_cpu(void *info) } /* restore per package power limit interrupt enable state */ -static void package_power_limit_irq_restore(int package_id) +static void package_power_limit_irq_restore(struct rapl_package *rp) { int cpu; - struct rapl_package *rp; - - rp = find_package_by_id(package_id); - if (!rp) - return; if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) return; - cpu = find_active_cpu_on_package(package_id); + cpu = find_active_cpu_on_package(rp->id); if (cpu < 0) return; @@ -1192,7 +1168,7 @@ static int rapl_unregister_powercap(void) * hotplug lock held */ list_for_each_entry(rp, &rapl_packages, plist) { - package_power_limit_irq_restore(rp->id); + package_power_limit_irq_restore(rp); for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { From 323ee64aa175a67fbbe744e809777d17e6fb42d7 Mon Sep 17 00:00:00 2001 From: Jacob Pan <jacob.jun.pan@linux.intel.com> Date: Wed, 24 Feb 2016 13:31:38 -0800 Subject: [PATCH 4/4] powercap/rapl: track lead cpu per package RAPL driver operates on MSRs that are under package/socket scope instead of core scope. However, the current code does not keep track of which CPUs are available on each package for MSR access. Therefore it has to search for an active CPU on a given package each time. This patch optimizes the package level operations by tracking a per package lead CPU during initialization and CPU hotplug. The runtime search for active CPU is avoided. Suggested-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Jacob Pan <jacob.jun.pan@linux.intel.com> Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com> --- drivers/powercap/intel_rapl.c | 59 ++++++++++++----------------------- 1 file changed, 20 insertions(+), 39 deletions(-) diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c index 3413692c4dde3..cdfd01f0adb84 100644 --- a/drivers/powercap/intel_rapl.c +++ b/drivers/powercap/intel_rapl.c @@ -191,6 +191,7 @@ struct rapl_package { * notify interrupt enable status. */ struct list_head plist; + int lead_cpu; /* one active cpu per package for access */ }; struct rapl_defaults { @@ -267,20 +268,6 @@ static struct rapl_package *find_package_by_id(int id) return NULL; } -/* caller to ensure CPU hotplug lock is held */ -static int find_active_cpu_on_package(int package_id) -{ - int i; - - for_each_online_cpu(i) { - if (topology_physical_package_id(i) == package_id) - return i; - } - /* all CPUs on this package are offline */ - - return -ENODEV; -} - /* caller must hold cpu hotplug lock */ static void rapl_cleanup_data(void) { @@ -761,10 +748,8 @@ static int rapl_read_data_raw(struct rapl_domain *rd, msr = rd->msrs[rp->id]; if (!msr) return -EINVAL; - /* use physical package id to look up active cpus */ - cpu = find_active_cpu_on_package(rd->rp->id); - if (cpu < 0) - return cpu; + + cpu = rd->rp->lead_cpu; /* special-case package domain, which uses a different bit*/ if (prim == FW_LOCK && rd->id == RAPL_DOMAIN_PACKAGE) { @@ -829,10 +814,7 @@ static int rapl_write_data_raw(struct rapl_domain *rd, struct msrl_action ma; int ret; - cpu = find_active_cpu_on_package(rd->rp->id); - if (cpu < 0) - return cpu; - + cpu = rd->rp->lead_cpu; bits = rapl_unit_xlate(rd, rp->unit, value, 1); bits |= bits << rp->shift; memset(&ma, 0, sizeof(ma)); @@ -940,18 +922,10 @@ static void power_limit_irq_save_cpu(void *info) static void package_power_limit_irq_save(struct rapl_package *rp) { - int cpu; - - if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) - return; - - cpu = find_active_cpu_on_package(rp->id); - if (cpu < 0) - return; if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) return; - smp_call_function_single(cpu, power_limit_irq_save_cpu, rp, 1); + smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1); } static void power_limit_irq_restore_cpu(void *info) @@ -972,20 +946,14 @@ static void power_limit_irq_restore_cpu(void *info) /* restore per package power limit interrupt enable state */ static void package_power_limit_irq_restore(struct rapl_package *rp) { - int cpu; - if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) return; - cpu = find_active_cpu_on_package(rp->id); - if (cpu < 0) - return; - /* irq enable state not saved, nothing to restore */ if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) return; - smp_call_function_single(cpu, power_limit_irq_restore_cpu, rp, 1); + smp_call_function_single(rp->lead_cpu, power_limit_irq_restore_cpu, rp, 1); } static void set_floor_freq_default(struct rapl_domain *rd, bool mode) @@ -1419,7 +1387,8 @@ static int rapl_detect_topology(void) /* add the new package to the list */ new_package->id = phy_package_id; new_package->nr_cpus = 1; - + /* use the first active cpu of the package to access */ + new_package->lead_cpu = i; /* check if the package contains valid domains */ if (rapl_detect_domains(new_package, i) || rapl_defaults->check_unit(new_package, i)) { @@ -1475,6 +1444,8 @@ static int rapl_add_package(int cpu) /* add the new package to the list */ rp->id = phy_package_id; rp->nr_cpus = 1; + rp->lead_cpu = cpu; + /* check if the package contains valid domains */ if (rapl_detect_domains(rp, cpu) || rapl_defaults->check_unit(rp, cpu)) { @@ -1507,6 +1478,7 @@ static int rapl_cpu_callback(struct notifier_block *nfb, unsigned long cpu = (unsigned long)hcpu; int phy_package_id; struct rapl_package *rp; + int lead_cpu; phy_package_id = topology_physical_package_id(cpu); switch (action) { @@ -1527,6 +1499,15 @@ static int rapl_cpu_callback(struct notifier_block *nfb, break; if (--rp->nr_cpus == 0) rapl_remove_package(rp); + else if (cpu == rp->lead_cpu) { + /* choose another active cpu in the package */ + lead_cpu = cpumask_any_but(topology_core_cpumask(cpu), cpu); + if (lead_cpu < nr_cpu_ids) + rp->lead_cpu = lead_cpu; + else /* should never go here */ + pr_err("no active cpu available for package %d\n", + phy_package_id); + } } return NOTIFY_OK;