Skip to content

Commit

Permalink
xen: implement CPU hotplugging
Browse files Browse the repository at this point in the history
Note the changes from 2.6.18-xen CPU hotplugging:

A vcpu_down request from the remote admin via Xenbus both hotunplugs the
CPU, and disables it by removing it from the cpu_present map, and removing
its entry in /sys.

A vcpu_up request from the remote admin only re-enables the CPU, and does
not immediately bring the CPU up. A udev event is emitted, which can be
caught by the user if he wishes to automatically re-up CPUs when available,
or implement a more complex policy.

Signed-off-by: Alex Nixon <alex.nixon@citrix.com>
Acked-by: Jeremy Fitzhardinge <jeremy@goop.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
  • Loading branch information
Alex Nixon authored and Ingo Molnar committed Aug 25, 2008
1 parent 8227dce commit d68d82a
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 13 deletions.
60 changes: 48 additions & 12 deletions arch/x86/xen/smp.c
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@
* useful topology information for the kernel to make use of. As a
* result, all CPUs are treated as if they're single-core and
* single-threaded.
*
* This does not handle HOTPLUG_CPU yet.
*/
#include <linux/sched.h>
#include <linux/err.h>
Expand Down Expand Up @@ -61,11 +59,12 @@ static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
return IRQ_HANDLED;
}

static __cpuinit void cpu_bringup_and_idle(void)
static __cpuinit void cpu_bringup(void)
{
int cpu = smp_processor_id();

cpu_init();
touch_softlockup_watchdog();
preempt_disable();

xen_enable_sysenter();
Expand All @@ -86,6 +85,11 @@ static __cpuinit void cpu_bringup_and_idle(void)
local_irq_enable();

wmb(); /* make sure everything is out */
}

static __cpuinit void cpu_bringup_and_idle(void)
{
cpu_bringup();
cpu_idle();
}

Expand Down Expand Up @@ -209,8 +213,6 @@ static void __init xen_smp_prepare_cpus(unsigned int max_cpus)

cpu_set(cpu, cpu_present_map);
}

//init_xenbus_allowed_cpumask();
}

static __cpuinit int
Expand Down Expand Up @@ -278,12 +280,6 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)
struct task_struct *idle = idle_task(cpu);
int rc;

#if 0
rc = cpu_up_check(cpu);
if (rc)
return rc;
#endif

#ifdef CONFIG_X86_64
/* Allocate node local memory for AP pdas */
WARN_ON(cpu == 0);
Expand Down Expand Up @@ -336,6 +332,42 @@ static void xen_smp_cpus_done(unsigned int max_cpus)
{
}

int xen_cpu_disable(void)
{
unsigned int cpu = smp_processor_id();
if (cpu == 0)
return -EBUSY;

cpu_disable_common();

load_cr3(swapper_pg_dir);
return 0;
}

void xen_cpu_die(unsigned int cpu)
{
while (HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
current->state = TASK_UNINTERRUPTIBLE;
schedule_timeout(HZ/10);
}
unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);
xen_uninit_lock_cpu(cpu);
xen_teardown_timer(cpu);

if (num_online_cpus() == 1)
alternatives_smp_switch(0);
}

void xen_play_dead(void)
{
play_dead_common();
HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
cpu_bringup();
}

static void stop_self(void *v)
{
int cpu = smp_processor_id();
Expand Down Expand Up @@ -419,9 +451,13 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
static const struct smp_ops xen_smp_ops __initdata = {
.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
.smp_prepare_cpus = xen_smp_prepare_cpus,
.cpu_up = xen_cpu_up,
.smp_cpus_done = xen_smp_cpus_done,

.cpu_up = xen_cpu_up,
.cpu_die = xen_cpu_die,
.cpu_disable = xen_cpu_disable,
.play_dead = xen_play_dead,

.smp_send_stop = xen_smp_send_stop,
.smp_send_reschedule = xen_smp_send_reschedule,

Expand Down
5 changes: 5 additions & 0 deletions arch/x86/xen/spinlock.c
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,11 @@ void __cpuinit xen_init_lock_cpu(int cpu)
printk("cpu %d spinlock event irq %d\n", cpu, irq);
}

void xen_uninit_lock_cpu(int cpu)
{
unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL);
}

void __init xen_init_spinlocks(void)
{
pv_lock_ops.spin_is_locked = xen_spin_is_locked;
Expand Down
8 changes: 8 additions & 0 deletions arch/x86/xen/time.c
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,14 @@ void xen_setup_timer(int cpu)
setup_runstate_info(cpu);
}

void xen_teardown_timer(int cpu)
{
struct clock_event_device *evt;
BUG_ON(cpu == 0);
evt = &per_cpu(xen_clock_events, cpu);
unbind_from_irqhandler(evt->irq, NULL);
}

void xen_setup_cpu_clockevents(void)
{
BUG_ON(preemptible());
Expand Down
6 changes: 6 additions & 0 deletions arch/x86/xen/xen-ops.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ void __init xen_build_dynamic_phys_to_machine(void);

void xen_init_irq_ops(void);
void xen_setup_timer(int cpu);
void xen_teardown_timer(int cpu);
cycle_t xen_clocksource_read(void);
void xen_setup_cpu_clockevents(void);
unsigned long xen_tsc_khz(void);
Expand All @@ -50,11 +51,16 @@ void xen_mark_init_mm_pinned(void);

void __init xen_setup_vcpu_info_placement(void);

void xen_play_dead(void);
void xen_cpu_die(unsigned int cpu);
int xen_cpu_disable(void);

#ifdef CONFIG_SMP
void xen_smp_init(void);

void __init xen_init_spinlocks(void);
__cpuinit void xen_init_lock_cpu(int cpu);
void xen_uninit_lock_cpu(int cpu);

extern cpumask_t xen_cpu_initialized_map;
#else
Expand Down
2 changes: 1 addition & 1 deletion drivers/xen/Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
obj-y += grant-table.o features.o events.o manage.o
obj-y += grant-table.o features.o events.o manage.o cpu_hotplug.o
obj-y += xenbus/
obj-$(CONFIG_XEN_XENCOMM) += xencomm.o
obj-$(CONFIG_XEN_BALLOON) += balloon.o
90 changes: 90 additions & 0 deletions drivers/xen/cpu_hotplug.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
#include <linux/notifier.h>

#include <xen/xenbus.h>

#include <asm-x86/xen/hypervisor.h>
#include <asm/cpu.h>

static void enable_hotplug_cpu(int cpu)
{
if (!cpu_present(cpu))
arch_register_cpu(cpu);

cpu_set(cpu, cpu_present_map);
}

static void disable_hotplug_cpu(int cpu)
{
if (cpu_present(cpu))
arch_unregister_cpu(cpu);

cpu_clear(cpu, cpu_present_map);
}

static void vcpu_hotplug(unsigned int cpu)
{
int err;
char dir[32], state[32];

if (!cpu_possible(cpu))
return;

sprintf(dir, "cpu/%u", cpu);
err = xenbus_scanf(XBT_NIL, dir, "availability", "%s", state);
if (err != 1) {
printk(KERN_ERR "XENBUS: Unable to read cpu state\n");
return;
}

if (strcmp(state, "online") == 0) {
enable_hotplug_cpu(cpu);
} else if (strcmp(state, "offline") == 0) {
(void)cpu_down(cpu);
disable_hotplug_cpu(cpu);
} else {
printk(KERN_ERR "XENBUS: unknown state(%s) on CPU%d\n",
state, cpu);
}
}

static void handle_vcpu_hotplug_event(struct xenbus_watch *watch,
const char **vec, unsigned int len)
{
unsigned int cpu;
char *cpustr;
const char *node = vec[XS_WATCH_PATH];

cpustr = strstr(node, "cpu/");
if (cpustr != NULL) {
sscanf(cpustr, "cpu/%u", &cpu);
vcpu_hotplug(cpu);
}
}

static int setup_cpu_watcher(struct notifier_block *notifier,
unsigned long event, void *data)
{
static struct xenbus_watch cpu_watch = {
.node = "cpu",
.callback = handle_vcpu_hotplug_event};

(void)register_xenbus_watch(&cpu_watch);

return NOTIFY_DONE;
}

static int __init setup_vcpu_hotplug_event(void)
{
static struct notifier_block xsn_cpu = {
.notifier_call = setup_cpu_watcher };

if (!is_running_on_xen())
return -ENODEV;

register_xenstore_notifier(&xsn_cpu);

return 0;
}

arch_initcall(setup_vcpu_hotplug_event);

4 changes: 4 additions & 0 deletions drivers/xen/events.c
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,10 @@ static void unbind_from_irq(unsigned int irq)
per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
[index_from_irq(irq)] = -1;
break;
case IRQT_IPI:
per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))
[index_from_irq(irq)] = -1;
break;
default:
break;
}
Expand Down

0 comments on commit d68d82a

Please sign in to comment.