/****************************************************************************
* (C) 2002-2003 - Rolf Neugebauer - Intel Research Cambridge
* (C) 2002-2003 University of Cambridge
* (C) 2004 - Mark Williamson - Intel Research Cambridge
****************************************************************************
*
* File: common/schedule.c
* Author: Rolf Neugebauer & Keir Fraser
* Updated for generic API by Mark Williamson
*
* Description: Generic CPU scheduling code
* implements support functionality for the Xen scheduler API.
*
*/
#ifndef COMPAT
#include <xen/init.h>
#include <xen/lib.h>
#include <xen/sched.h>
#include <xen/domain.h>
#include <xen/delay.h>
#include <xen/event.h>
#include <xen/time.h>
#include <xen/timer.h>
#include <xen/perfc.h>
#include <xen/sched-if.h>
#include <xen/softirq.h>
#include <xen/trace.h>
#include <xen/mm.h>
#include <xen/err.h>
#include <xen/guest_access.h>
#include <xen/hypercall.h>
#include <xen/multicall.h>
#include <xen/cpu.h>
#include <xen/preempt.h>
#include <xen/event.h>
#include <public/sched.h>
#include <xsm/xsm.h>
#include <xen/err.h>
/* opt_sched: scheduler - default to configured value */
static char __initdata opt_sched[10] = CONFIG_SCHED_DEFAULT;
string_param("sched", opt_sched);
/* if sched_smt_power_savings is set,
* scheduler will give preferrence to partially idle package compared to
* the full idle package, when picking pCPU to schedule vCPU.
*/
bool_t sched_smt_power_savings = 0;
boolean_param("sched_smt_power_savings", sched_smt_power_savings);
/* Default scheduling rate limit: 1ms
* The behavior when sched_ratelimit_us is greater than sched_credit_tslice_ms is undefined
* */
int sched_ratelimit_us = SCHED_DEFAULT_RATELIMIT_US;
integer_param("sched_ratelimit_us", sched_ratelimit_us);
/* Various timer handlers. */
static void s_timer_fn(void *unused);
static void vcpu_periodic_timer_fn(void *data);
static void vcpu_singleshot_timer_fn(void *data);
static void poll_timer_fn(void *data);
/* This is global for now so that private implementations can reach it */
DEFINE_PER_CPU(struct schedule_data, schedule_data);
DEFINE_PER_CPU(struct scheduler *, scheduler);
/* Scratch space for cpumasks. */
DEFINE_PER_CPU(cpumask_t, cpumask_scratch);
extern const struct scheduler *__start_schedulers_array[], *__end_schedulers_array[];
#define NUM_SCHEDULERS (__end_schedulers_array - __start_schedulers_array)
#define schedulers __start_schedulers_array
static struct scheduler __read_mostly ops;
#define SCHED_OP(opsptr, fn, ...) \
(( (opsptr)->fn != NULL ) ? (opsptr)->fn(opsptr, ##__VA_ARGS__ ) \
: (typeof((opsptr)->fn(opsptr, ##__VA_ARGS__)))0 )
static inline struct scheduler *dom_scheduler(const struct domain *d)
{
if ( likely(d->cpupool != NULL) )
return d->cpupool->sched;
/*
* If d->cpupool is NULL, this is the idle domain. This is special
* because the idle domain does not really belong to any cpupool, and,
* hence, does not really have a scheduler.
*
* This is (should be!) only called like this for allocating the idle
* vCPUs for the first time, during boot, in which case what we want
* is the default scheduler that has been, choosen at boot.
*/
ASSERT(is_idle_domain(d));
return &ops;
}
static inline struct scheduler *vcpu_scheduler(const struct vcpu *v)
{
struct domain *d = v->domain;
if ( likely(d->cpupool != NULL) )
return d->cpupool->sched;
/*
* If d->cpupool is NULL, this is a vCPU of the idle domain. And this
* case is special because the idle domain does not really belong to
* a cpupool and, hence, doesn't really have a scheduler). In fact, its
* vCPUs (may) run on pCPUs which are in different pools, with different
* schedulers.
*
* What we want, in this case, is the scheduler of the pCPU where this
* particular idle vCPU is running. And, since v->processor never changes
* for idle vCPUs, it is safe to use it, with no locks, to figure that out.
*/
ASSERT(is_idle_domain(d));
return per_cpu(scheduler, v->processor);
}
#define VCPU2ONLINE(_v) cpupool_domain_cpumask((_v)->domain)
static inline void trace_runstate_change(struct vcpu *v, int new_state)
{
struct { uint32_t vcpu:16, domain:16; } d;
uint32_t event;
if ( likely(!tb_init_done) )
return;
d.vcpu = v->vcpu_id;
d.domain = v->domain->domain_id;
event = TRC_SCHED_RUNSTATE_CHANGE;
event |= ( v->runstate.state & 0x3 ) << 8;
event |= ( new_state & 0x3 ) << 4;
__trace_var(event, 1/*tsc*/, sizeof(d), &d);
}
static inline void trace_continue_running(struct vcpu *v)
{
struct { uint32_t vcpu:16, domain:16; } d;
if ( likely(!tb_init_done) )
return;
d.vcpu = v->vcpu_id;
d.domain = v->domain->domain_id;
__trace_var(TRC_SCHED_CONTINUE_RUNNING, 1/*tsc*/, sizeof(d), &d);
}
static inline void vcpu_urgent_count_update(struct vcpu *v)
{
if ( is_idle_vcpu(v) )
return;
if ( unlikely(v->is_urgent) )
{
if ( !(v->pause_flags & VPF_blocked) ||
!test_bit(v->vcpu_id, v->domain->poll_mask) )
{
v->is_urgent = 0;
atomic_dec(&per_cpu(schedule_data,v->processor).urgent_count);
}
}
else
{
if ( unlikely(v->pause_flags & VPF_blocked) &&
unlikely(test_bit(v->vcpu_id, v->domain->poll_mask)) )
{
v->is_urgent = 1;
atomic_inc(&per_cpu(schedule_data,v->processor).urgent_count);
}
}
}
static inline void vcpu_runstate_change(
struct vcpu *v, int new_state, s_time_t new_entry_time)
{
s_time_t delta;
ASSERT(v->runstate.state != new_state);
ASSERT(spin_is_locked(per_cpu(schedule_data,v->processor).schedule_lock));
vcpu_urgent_count_update(v);
trace_runstate_change(v, new_state);
delta = new_entry_time - v->runstate.state_entry_time;
if ( delta > 0 )
{
v->runstate.time[v->runstate.state] += delta;
v->runstate.state_entry_time = new_entry_time;
}
v->runstate.state = new_state;
}
void vcpu_runstate_get(struct vcpu *v, struct vcpu_runstate_info *runstate)
{
spinlock_t *lock = likely(v == current) ? NULL : vcpu_schedule_lock_irq(v);
s_time_t delta;
memcpy(runstate, &v->runstate, sizeof(*runstate));
delta = NOW() - runstate->state_entry_time;
if ( delta > 0 )
runstate->time[runstate->state] += delta;
if ( unlikely(lock != NULL) )
vcpu_schedule_unlock_irq(lock, v);
}
uint64_t get_cpu_idle_time(unsigned int cpu)
{
struct vcpu_runstate_info state = { 0 };
struct vcpu *v = idle_vcpu[cpu];
if ( cpu_online(cpu) && v )
vcpu_runstate_get(v, &state);
return state.time[RUNSTATE_running];
}
/*
* If locks are different, take the one with the lower address first.
* This avoids dead- or live-locks when this code is running on both
* cpus at the same time.
*/
static void sched_spin_lock_double(spinlock_t *lock1, spinlock_t *lock2,
unsigned long *flags)
{
if ( lock1 == lock2 )
{
spin_lock_irqsave(lock1, *flags);
}
else if ( lock1 < lock2 )
{
spin_lock_irqsave(lock1, *flags);
spin_lock(lock2);
}
else
{
spin_lock_irqsave(lock2, *flags);
spin_lock(lock1);
}
}
static void sched_spin_unlock_double(spinlock_t *lock1, spinlock_t *lock2,
unsigned long flags)
{
if ( lock1 != lock2 )
spin_unlock(lock2);
spin_unlock_irqrestore(lock1, flags);
}
int sched_init_vcpu(struct vcpu *v, unsigned int processor)
{
struct domain *d = v->domain;
/*
* Initialize processor and affinity settings. The idler, and potentially
* domain-0 VCPUs, are pinned onto their respective physical CPUs.
*/
v->processor = processor;
if ( is_idle_domain(d) || d->is_pinned )
cpumask_copy(v->cpu_hard_affinity, cpumask_of(processor));
else
cpumask_setall(v->cpu_hard_affinity);
cpumask_setall(v->cpu_soft_affinity);
/* Initialise the per-vcpu timers. */
init_timer(&v->periodic_timer, vcpu_periodic_timer_fn,
v, v->processor);
init_timer(&v->singleshot_timer, vcpu_singleshot_timer_fn,
v, v->processor);
init_timer(&v->poll_timer, poll_timer_fn,
v, v->processor);
v->sched_priv = SCHED_OP(dom_scheduler(d), alloc_vdata, v,
d->sched_priv);
if ( v->sched_priv == NULL )
return 1;
/* Idle VCPUs are scheduled immediately, so don't put them in runqueue. */
if ( is_idle_domain(d) )
{
per_cpu(schedule_data, v->processor).curr = v;
v->is_running = 1;
}
else
{
SCHED_OP(dom_scheduler(d), insert_vcpu, v);
}
return 0;
}
static void sched_move_irqs(struct vcpu *v)
{
arch_move_irqs(v);
evtchn_move_pirqs(v);
}
int sched_move_domain(struct domain *d, struct cpupool *c)
{
struct vcpu *v;
unsigned int new_p;
void **vcpu_priv;
void *domdata;
void *vcpudata;
struct scheduler *old_ops;
void *old_domdata;
for_each_vcpu ( d, v )
{
if ( v->affinity_broken )
return -EBUSY;
}
domdata = SCHED_OP(c->sched, alloc_domdata, d);
if ( domdata == NULL )
return -ENOMEM;
vcpu_priv = xzalloc_array(void *, d->max_vcpus);
if ( vcpu_priv == NULL )
{
SCHED_OP(c->sched, free_domdata, domdata);
return -ENOMEM;
}
for_each_vcpu ( d, v )
{
vcpu_priv[v->vcpu_id] = SCHED_OP(c->sched, alloc_vdata, v, domdata);
if ( vcpu_priv[v->vcpu_id] == NULL )
{
for_each_vcpu ( d, v )
xfree(vcpu_priv[v->vcpu_id]);
xfree(vcpu_priv);
SCHED_OP(c->sched, free_domdata, domdata);
return -ENOMEM;
}
}
domain_pause(d);
old_ops = dom_scheduler(d);
old_domdata = d->sched_priv;
for_each_vcpu ( d, v )
{
SCHED_OP(old_ops, remove_vcpu, v);
}
d->cpupool = c;
d->sched_priv = domdata;
new_p = cpumask_first(c->cpu_valid);
for_each_vcpu ( d, v )
{
spinlock_t *lock;
vcpudata = v->sched_priv;
migrate_timer(&v->periodic_timer, new_p);
migrate_timer(&v->singleshot_timer, new_p);
migrate_timer(&v->poll_timer, new_p);
cpumask_setall(v->cpu_hard_affinity);
cpumask_setall(v->cpu_soft_affinity);
lock = vcpu_schedule_lock_irq(v);
v->processor = new_p;
/*
* With v->processor modified we must not
* - make any further changes assuming we hold the scheduler lock,
* - use vcpu_schedule_unlock_irq().
*/
spin_unlock_irq(lock);
v->sched_priv = vcpu_priv[v->vcpu_id];
if ( !d->is_dying )
sched_move_irqs(v);
new_p = cpumask_cycle(new_p, c->cpu_valid);
SCHED_OP(c->sched, insert_vcpu, v);
SCHED_OP(old_ops, free_vdata, vcpudata);
}
domain_update_node_affinity(d);
domain_unpause(d);
SCHED_OP(old_ops, free_domdata, old_domdata);
xfree(vcpu_priv);
return 0;
}
void sched_destroy_vcpu(struct vcpu *v)
{
kill_timer(&v->periodic_timer);
kill_timer(&v->singleshot_timer);
kill_timer(&v->poll_timer);
if ( test_and_clear_bool(v->is_urgent) )
atomic_dec(&per_cpu(schedule_data, v->processor).urgent_count);
SCHED_OP(vcpu_scheduler(v), remove_vcpu, v);
SCHED_OP(vcpu_scheduler(v), free_vdata, v->sched_priv);
}
int sched_init_domain(struct domain *d, int poolid)
{
int ret;
ASSERT(d->cpupool == NULL);
if ( (ret = cpupool_add_domain(d, poolid)) )
return ret;
SCHED_STAT_CRANK(dom_init);
TRACE_1D(TRC_SCHED_DOM_ADD, d->domain_id);
return SCHED_OP(dom_scheduler(d), init_domain, d);
}
void sched_destroy_domain(struct domain *d)
{
ASSERT(d->cpupool != NULL || is_idle_domain(d));
SCHED_STAT_CRANK(dom_destroy);
TRACE_1D(TRC_SCHED_DOM_REM, d->domain_id);
SCHED_OP(dom_scheduler(d), destroy_domain, d);
cpupool_rm_domain(d);
}
void vcpu_sleep_nosync(struct vcpu *v)
{
unsigned long flags;
spinlock_t *lock;
TRACE_2D(TRC_SCHED_SLEEP, v->domain->domain_id, v->vcpu_id);
lock = vcpu_schedule_lock_irqsave(v, &flags);
if ( likely(!vcpu_runnable(v)) )
{
if ( v->runstate.state == RUNSTATE_runnable )
vcpu_runstate_change(v, RUNSTATE_offline, NOW());
SCHED_OP(vcpu_scheduler(v), sleep, v);
}
vcpu_schedule_unlock_irqrestore(lock, flags, v);
}
void vcpu_sleep_sync(struct vcpu *v)
{
vcpu_sleep_nosync(v);
while ( !vcpu_runnable(v) && v->is_running )
cpu_relax();
sync_vcpu_execstate(v);
}
void vcpu_wake(struct vcpu *v)
{
unsigned long flags;
spinlock_t *lock;
TRACE_2D(TRC_SCHED_WAKE, v->domain->domain_id, v->vcpu_id);
lock = vcpu_schedule_lock_irqsave(v, &flags);
if ( likely(vcpu_runnable(v)) )
{
if ( v->runstate.state >= RUNSTATE_blocked )
vcpu_runstate_change(v, RUNSTATE_runnable, NOW());
SCHED_OP(vcpu_scheduler(v), wake, v);
}
else if ( !(v->pause_flags & VPF_blocked) )
{
if ( v->runstate.state == RUNSTATE_blocked )
vcpu_runstate_change(v, RUNSTATE_offline, NOW());
}
vcpu_schedule_unlock_irqrestore(lock, flags, v);
}
void vcpu_unblock(struct vcpu *v)
{
if ( !test_and_clear_bit(_VPF_blocked, &v->pause_flags) )
return;
/* Polling period ends when a VCPU is unblocked. */
if ( unlikely(v->poll_evtchn != 0) )
{
v->poll_evtchn = 0;
/*
* We *must* re-clear _VPF_blocked to avoid racing other wakeups of
* this VCPU (and it then going back to sleep on poll_mask).
* Test-and-clear is idiomatic and ensures clear_bit not reordered.
*/
if ( test_and_clear_bit(v->vcpu_id, v->domain->poll_mask) )
clear_bit(_VPF_blocked, &v->pause_flags);
}
vcpu_wake(v);
}
/*
* Do the actual movement of a vcpu from old to new CPU. Locks for *both*
* CPUs needs to have been taken already when calling this!
*/
static void vcpu_move_locked(struct vcpu *v, unsigned int new_cpu)
{
unsigned int old_cpu = v->processor;
/*
* Transfer urgency status to new CPU before switching CPUs, as
* once the switch occurs, v->is_urgent is no longer protected by
* the per-CPU scheduler lock we are holding.
*/
if ( unlikely(v->is_urgent) && (old_cpu != new_cpu) )
{
atomic_inc(&per_cpu(schedule_data, new_cpu).urgent_count);
atomic_dec(&per_cpu(schedule_data, old_cpu).urgent_count);
}
/*
* Actual CPU switch to new CPU. This is safe because the lock
* pointer cant' change while the current lock is held.
*/
if ( vcpu_scheduler(v)->migrate )
SCHED_OP(vcpu_scheduler(v), migrate, v, new_cpu);
else
v->processor = new_cpu;
}
/*
* Move a vcpu from its current processor to a target new processor,
* without asking the scheduler to do any placement. This is intended
* for being called from special contexts, where things are quiet
* enough that no contention is supposed to happen (i.e., during
* shutdown or software suspend, like ACPI S3).
*/
static void vcpu_move_nosched(struct vcpu *v, unsigned int new_cpu)
{
unsigned long flags;
spinlock_t *lock, *new_lock;
ASSERT(system_state == SYS_STATE_suspend);
ASSERT(!vcpu_runnable(v) && (atomic_read(&v->pause_count) ||
atomic_read(&v->domain->pause_count)));
lock = per_cpu(schedule_data, v->processor).schedule_lock;
new_lock = per_cpu(schedule_data, new_cpu).schedule_lock;
sched_spin_lock_double(lock, new_lock, &flags);
ASSERT(new_cpu != v->processor);
vcpu_move_locked(v, new_cpu);
sched_spin_unlock_double(lock, new_lock, flags);
sched_move_irqs(v);
}
static void vcpu_migrate(struct vcpu *v)
{
unsigned long flags;
unsigned int old_cpu, new_cpu;
spinlock_t *old_lock, *new_lock;
bool_t pick_called = 0;
old_cpu = new_cpu = v->processor;
for ( ; ; )
{
/*
* We need another iteration if the pre-calculated lock addresses
* are not correct any longer after evaluating old and new cpu holding
* the locks.
*/
old_lock = per_cpu(schedule_data, old_cpu).schedule_lock;
new_lock = per_cpu(schedule_data, new_cpu).schedule_lock;
sched_spin_lock_double(old_lock, new_lock, &flags);
old_cpu = v->processor;
if ( old_lock == per_cpu(schedule_data, old_cpu).schedule_lock )
{
/*
* If we selected a CPU on the previosu iteration, check if it
* remains suitable for running this vCPU.
*/
if ( pick_called &&
(new_lock == per_cpu(schedule_data, new_cpu).schedule_lock) &&
cpumask_test_cpu(new_cpu, v->cpu_hard_affinity) &&
cpumask_test_cpu(new_cpu, v->domain->cpupool->cpu_valid) )
break;
/* Select a new CPU. */
new_cpu = SCHED_OP(vcpu_scheduler(v), pick_cpu, v);
if ( (new_lock == per_cpu(schedule_data, new_cpu).schedule_lock) &&
cpumask_test_cpu(new_cpu, v->domain->cpupool->cpu_valid) )
break;
pick_called = 1;
}
else
{
/*
* We do not hold the scheduler lock appropriate for this vCPU.
* Thus we cannot select a new CPU on this iteration. Try again.
*/
pick_called = 0;
}
sched_spin_unlock_double(old_lock, new_lock, flags);
}
/*
* NB. Check of v->running happens /after/ setting migration flag
* because they both happen in (different) spinlock regions, and those
* regions are strictly serialised.
*/
if ( v->is_running ||
!test_and_clear_bit(_VPF_migrating, &v->pause_flags) )
{
sched_spin_unlock_double(old_lock, new_lock, flags);
return;
}
vcpu_move_locked(v, new_cpu);
sched_spin_unlock_double(old_lock, new_lock, flags);
if ( old_cpu != new_cpu )
sched_move_irqs(v);
/* Wake on new CPU. */
vcpu_wake(v);
}
/*
* Force a VCPU through a deschedule/reschedule path.
* For example, using this when setting the periodic timer period means that
* most periodic-timer state need only be touched from within the scheduler
* which can thus be done without need for synchronisation.
*/
void vcpu_force_reschedule(struct vcpu *v)
{
spinlock_t *lock = vcpu_schedule_lock_irq(v);
if ( v->is_running )
set_bit(_VPF_migrating, &v->pause_flags);
vcpu_schedule_unlock_irq(lock, v);
if ( v->pause_flags & VPF_migrating )
{
vcpu_sleep_nosync(v);
vcpu_migrate(v);
}
}
void restore_vcpu_affinity(struct domain *d)
{
unsigned int cpu = smp_processor_id();
struct vcpu *v;
ASSERT(system_state == SYS_STATE_resume);
for_each_vcpu ( d, v )
{
spinlock_t *lock;
ASSERT(!vcpu_runnable(v));
lock = vcpu_schedule_lock_irq(v);
if ( v->affinity_broken )
{
cpumask_copy(v->cpu_hard_affinity, v->cpu_hard_affinity_saved);
v->affinity_broken = 0;
}
/*
* During suspend (in cpu_disable_scheduler()), we moved every vCPU
* to BSP (which, as of now, is pCPU 0), as a temporary measure to
* allow the nonboot processors to have their data structure freed
* and go to sleep. But nothing guardantees that the BSP is a valid
* pCPU for a particular domain.
*
* Therefore, here, before actually unpausing the domains, we should
* set v->processor of each of their vCPUs to something that will
* make sense for the scheduler of the cpupool in which they are in.
*/
cpumask_and(cpumask_scratch_cpu(cpu), v->cpu_hard_affinity,
cpupool_domain_cpumask(v->domain));
v->processor = cpumask_any(cpumask_scratch_cpu(cpu));
spin_unlock_irq(lock);
lock = vcpu_schedule_lock_irq(v);
v->processor = SCHED_OP(vcpu_scheduler(v), pick_cpu, v);
spin_unlock_irq(lock);
}
domain_update_node_affinity(d);
}
/*
* This function is used by cpu_hotplug code from stop_machine context
* and from cpupools to switch schedulers on a cpu.
*/
int cpu_disable_scheduler(unsigned int cpu)
{
struct domain *d;
struct vcpu *v;
struct cpupool *c;
cpumask_t online_affinity;
unsigned int new_cpu;
int ret = 0;
c = per_cpu(cpupool, cpu);
if ( c == NULL )
return ret;
/*
* We'd need the domain RCU lock, but:
* - when we are called from cpupool code, it's acquired there already;
* - when we are called for CPU teardown, we're in stop-machine context,
* so that's not be a problem.
*/
for_each_domain_in_cpupool ( d, c )
{
for_each_vcpu ( d, v )
{
unsigned long flags;
spinlock_t *lock = vcpu_schedule_lock_irqsave(v, &flags);
cpumask_and(&online_affinity, v->cpu_hard_affinity, c->cpu_valid);
if ( cpumask_empty(&online_affinity) &&
cpumask_test_cpu(cpu, v->cpu_hard_affinity) )
{
if ( v->affinity_broken )
{
/* The vcpu is temporarily pinned, can't move it. */
vcpu_schedule_unlock_irqrestore(lock, flags, v);
ret = -EADDRINUSE;
break;
}
if (system_state == SYS_STATE_suspend)
{
cpumask_copy(v->cpu_hard_affinity_saved,
v->cpu_hard_affinity);
v->affinity_broken = 1;
}
else
printk(XENLOG_DEBUG "Breaking affinity for %pv\n", v);
cpumask_setall(v->cpu_hard_affinity);
}
if ( v->processor != cpu )
{
/* The vcpu is not on this cpu, so we can move on. */
vcpu_schedule_unlock_irqrestore(lock, flags, v);
continue;
}
/* If it is on this cpu, we must send it away. */
if ( unlikely(system_state == SYS_STATE_suspend) )
{
vcpu_schedule_unlock_irqrestore(lock, flags, v);
/*
* If we are doing a shutdown/suspend, it is not necessary to
* ask the scheduler to chime in. In fact:
* * there is no reason for it: the end result we are after
* is just 'all the vcpus on the boot pcpu, and no vcpu
* anywhere else', so let's just go for it;
* * it's wrong, for cpupools with only non-boot pcpus, as
* the scheduler would always fail to send the vcpus away
* from the last online (non boot) pcpu!
*
* Therefore, in the shutdown/suspend case, we just pick up
* one (still) online pcpu. Note that, at this stage, all
* domains (including dom0) have been paused already, so we
* do not expect any vcpu activity at all.
*/
cpumask_andnot(&online_affinity, &cpu_online_map,
cpumask_of(cpu));
BUG_ON(cpumask_empty(&online_affinity));
/*
* As boot cpu is, usually, pcpu #0, using cpumask_first()
* will make us converge quicker.
*/
new_cpu = cpumask_first(&online_affinity);
vcpu_move_nosched(v, new_cpu);
}
else
{
/*
* OTOH, if the system is still live, and we are here because
* we are doing some cpupool manipulations:
* * we want to call the scheduler, and let it re-evaluation
* the placement of the vcpu, taking into account the new
* cpupool configuration;
* * the scheduler will always fine a suitable solution, or
* things would have failed before getting in here.
*/
set_bit(_VPF_migrating, &v->pause_flags);
vcpu_schedule_unlock_irqrestore(lock, flags, v);
vcpu_sleep_nosync(v);
vcpu_migrate(v);
/*
* The only caveat, in this case, is that if a vcpu active in
* the hypervisor isn't migratable. In this case, the caller
* should try again after releasing and reaquiring all locks.
*/
if ( v->processor == cpu )
ret = -EAGAIN;
}
}
}
return ret;
}
static int vcpu_set_affinity(
struct vcpu *v, const cpumask_t *affinity, cpumask_t *which)
{
spinlock_t *lock;
int ret = 0;
lock = vcpu_schedule_lock_irq(v);
if ( v->affinity_broken )
ret = -EBUSY;
else
{
cpumask_copy(which, affinity);
/*
* Always ask the scheduler to re-evaluate placement
* when changing the affinity.
*/
set_bit(_VPF_migrating, &v->pause_flags);
}
vcpu_schedule_unlock_irq(lock, v);
domain_update_node_affinity(v->domain);
if ( v->pause_flags & VPF_migrating )
{
vcpu_sleep_nosync(v);
vcpu_migrate(v);
}
return ret;
}
int vcpu_set_hard_affinity(struct vcpu *v, const cpumask_t *affinity)
{
cpumask_t online_affinity;
cpumask_t *online;
if ( v->domain->is_pinned )
return -EINVAL;
online = VCPU2ONLINE(v);
cpumask_and(&online_affinity, affinity, online);
if ( cpumask_empty(&online_affinity) )
return -EINVAL;
return vcpu_set_affinity(v, affinity, v->cpu_hard_affinity);
}
int vcpu_set_soft_affinity(struct vcpu *v, const cpumask_t *affinity)
{
return vcpu_set_affinity(v, affinity, v->cpu_soft_affinity);
}
/* Block the currently-executing domain until a pertinent event occurs. */
void vcpu_block(void)
{
struct vcpu *v = current;
set_bit(_VPF_blocked, &v->pause_flags);
arch_vcpu_block(v);
/* Check for events /after/ blocking: avoids wakeup waiting race. */
if ( local_events_need_delivery() )
{
clear_bit(_VPF_blocked, &v->pause_flags);
}
else
{
TRACE_2D(TRC_SCHED_BLOCK, v->domain->domain_id, v->vcpu_id);
raise_softirq(SCHEDULE_SOFTIRQ);
}
}
static void vcpu_block_enable_events(void)
{
local_event_delivery_enable();
vcpu_block();
}
static long do_poll(struct sched_poll *sched_poll)
{
struct vcpu *v = current;
struct domain *d = v->domain;
evtchn_port_t port;
long rc;
unsigned int i;
/* Fairly arbitrary limit. */
if ( sched_poll->nr_ports > 128 )
return -EINVAL;
if ( !guest_handle_okay(sched_poll->ports, sched_poll->nr_ports) )
return -EFAULT;
set_bit(_VPF_blocked, &v->pause_flags);
v->poll_evtchn = -1;
set_bit(v->vcpu_id, d->poll_mask);
arch_vcpu_block(v);
#ifndef CONFIG_X86 /* set_bit() implies mb() on x86 */
/* Check for events /after/ setting flags: avoids wakeup waiting race. */
smp_mb();
/*
* Someone may have seen we are blocked but not that we are polling, or
* vice versa. We are certainly being woken, so clean up and bail. Beyond
* this point others can be guaranteed to clean up for us if they wake us.
*/
rc = 0;
if ( (v->poll_evtchn == 0) ||
!test_bit(_VPF_blocked, &v->pause_flags) ||
!test_bit(v->vcpu_id, d->poll_mask) )
goto out;
#endif
rc = 0;
if ( local_events_need_delivery() )
goto out;
for ( i = 0; i < sched_poll->nr_ports; i++ )
{
rc = -EFAULT;
if ( __copy_from_guest_offset(&port, sched_poll->ports, i, 1) )
goto out;
rc = -EINVAL;
if ( port >= d->max_evtchns )
goto out;
rc = 0;
if ( evtchn_port_is_pending(d, port) )
goto out;
}
if ( sched_poll->nr_ports == 1 )
v->poll_evtchn = port;
if ( sched_poll->timeout != 0 )
set_timer(&v->poll_timer, sched_poll->timeout);
TRACE_2D(TRC_SCHED_BLOCK, d->domain_id, v->vcpu_id);
raise_softirq(SCHEDULE_SOFTIRQ);
return 0;
out:
v->poll_evtchn = 0;
clear_bit(v->vcpu_id, d->poll_mask);
clear_bit(_VPF_blocked, &v->pause_flags);
return rc;
}
/* Voluntarily yield the processor for this allocation. */
long vcpu_yield(void)
{
struct vcpu * v=current;
spinlock_t *lock = vcpu_schedule_lock_irq(v);
SCHED_OP(vcpu_scheduler(v), yield, v);
vcpu_schedule_unlock_irq(lock, v);
SCHED_STAT_CRANK(vcpu_yield);
TRACE_2D(TRC_SCHED_YIELD, current->domain->domain_id, current->vcpu_id);
raise_softirq(SCHEDULE_SOFTIRQ);
return 0;
}
static void domain_watchdog_timeout(void *data)
{
struct domain *d = data;
if ( d->is_shutting_down || d->is_dying )
return;
printk("Watchdog timer fired for domain %u\n", d->domain_id);
domain_shutdown(d, SHUTDOWN_watchdog);
}
static long domain_watchdog(struct domain *d, uint32_t id, uint32_t timeout)
{
if ( id > NR_DOMAIN_WATCHDOG_TIMERS )
return -EINVAL;
spin_lock(&d->watchdog_lock);
if ( id == 0 )
{
for ( id = 0; id < NR_DOMAIN_WATCHDOG_TIMERS; id++ )
{
if ( test_and_set_bit(id, &d->watchdog_inuse_map) )
continue;
set_timer(&d->watchdog_timer[id], NOW() + SECONDS(timeout));
break;
}
spin_unlock(&d->watchdog_lock);
return id == NR_DOMAIN_WATCHDOG_TIMERS ? -ENOSPC : id + 1;
}
id -= 1;
if ( !test_bit(id, &d->watchdog_inuse_map) )
{
spin_unlock(&d->watchdog_lock);
return -EINVAL;
}
if ( timeout == 0 )
{
stop_timer(&d->watchdog_timer[id]);
clear_bit(id, &d->watchdog_inuse_map);
}
else
{
set_timer(&d->watchdog_timer[id], NOW() + SECONDS(timeout));
}
spin_unlock(&d->watchdog_lock);
return 0;
}
void watchdog_domain_init(struct domain *d)
{
unsigned int i;
spin_lock_init(&d->watchdog_lock);
d->watchdog_inuse_map = 0;
for ( i = 0; i < NR_DOMAIN_WATCHDOG_TIMERS; i++ )
init_timer(&d->watchdog_timer[i], domain_watchdog_timeout, d, 0);
}
void watchdog_domain_destroy(struct domain *d)
{
unsigned int i;
for ( i = 0; i < NR_DOMAIN_WATCHDOG_TIMERS; i++ )
kill_timer(&d->watchdog_timer[i]);
}
int vcpu_pin_override(struct vcpu *v, int cpu)
{
spinlock_t *lock;
int ret = -EINVAL;
lock = vcpu_schedule_lock_irq(v);
if ( cpu < 0 )
{
if ( v->affinity_broken )
{
cpumask_copy(v->cpu_hard_affinity, v->cpu_hard_affinity_saved);
v->affinity_broken = 0;
set_bit(_VPF_migrating, &v->pause_flags);
ret = 0;
}
}
else if ( cpu < nr_cpu_ids )
{
if ( v->affinity_broken )
ret = -EBUSY;
else if ( cpumask_test_cpu(cpu, VCPU2ONLINE(v)) )
{
cpumask_copy(v->cpu_hard_affinity_saved, v->cpu_hard_affinity);
v->affinity_broken = 1;
cpumask_copy(v->cpu_hard_affinity, cpumask_of(cpu));
set_bit(_VPF_migrating, &v->pause_flags);
ret = 0;
}
}
vcpu_schedule_unlock_irq(lock, v);
domain_update_node_affinity(v->domain);
if ( v->pause_flags & VPF_migrating )
{
vcpu_sleep_nosync(v);
vcpu_migrate(v);
}
return ret;
}
typedef long ret_t;
#endif /* !COMPAT */
ret_t do_sched_op(int cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
{
ret_t ret = 0;
switch ( cmd )
{
case SCHEDOP_yield:
{
ret = vcpu_yield();
break;
}
case SCHEDOP_block:
{
vcpu_block_enable_events();
break;
}
case SCHEDOP_shutdown:
{
struct sched_shutdown sched_shutdown;
ret = -EFAULT;
if ( copy_from_guest(&sched_shutdown, arg, 1) )
break;
TRACE_3D(TRC_SCHED_SHUTDOWN,
current->domain->domain_id, current->vcpu_id,
sched_shutdown.reason);
ret = domain_shutdown(current->domain, (u8)sched_shutdown.reason);
break;
}
case SCHEDOP_shutdown_code:
{
struct sched_shutdown sched_shutdown;
struct domain *d = current->domain;
ret = -EFAULT;
if ( copy_from_guest(&sched_shutdown, arg, 1) )
break;
TRACE_3D(TRC_SCHED_SHUTDOWN_CODE,
d->domain_id, current->vcpu_id, sched_shutdown.reason);
spin_lock(&d->shutdown_lock);
if ( d->shutdown_code == SHUTDOWN_CODE_INVALID )
d->shutdown_code = (u8)sched_shutdown.reason;
spin_unlock(&d->shutdown_lock);
ret = 0;
break;
}
case SCHEDOP_poll:
{
struct sched_poll sched_poll;
ret = -EFAULT;
if ( copy_from_guest(&sched_poll, arg, 1) )
break;
ret = do_poll(&sched_poll);
break;
}
case SCHEDOP_remote_shutdown:
{
struct domain *d;
struct sched_remote_shutdown sched_remote_shutdown;
ret = -EFAULT;
if ( copy_from_guest(&sched_remote_shutdown, arg, 1) )
break;
ret = -ESRCH;
d = rcu_lock_domain_by_id(sched_remote_shutdown.domain_id);
if ( d == NULL )
break;
ret = xsm_schedop_shutdown(XSM_DM_PRIV, current->domain, d);
if ( likely(!ret) )
domain_shutdown(d, sched_remote_shutdown.reason);
rcu_unlock_domain(d);
break;
}
case SCHEDOP_watchdog:
{
struct sched_watchdog sched_watchdog;
ret = -EFAULT;
if ( copy_from_guest(&sched_watchdog, arg, 1) )
break;
ret = domain_watchdog(
current->domain, sched_watchdog.id, sched_watchdog.timeout);
break;
}
case SCHEDOP_pin_override:
{
struct sched_pin_override sched_pin_override;
ret = -EPERM;
if ( !is_hardware_domain(current->domain) )
break;
ret = -EFAULT;
if ( copy_from_guest(&sched_pin_override, arg, 1) )
break;
ret = vcpu_pin_override(current, sched_pin_override.pcpu);
break;
}
default:
ret = -ENOSYS;
}
return ret;
}
#ifndef COMPAT
/* Per-vcpu oneshot-timer hypercall. */
long do_set_timer_op(s_time_t timeout)
{
struct vcpu *v = current;
s_time_t offset = timeout - NOW();
if ( timeout == 0 )
{
stop_timer(&v->singleshot_timer);
}
else if ( unlikely(timeout < 0) || /* overflow into 64th bit? */
unlikely((offset > 0) && ((uint32_t)(offset >> 50) != 0)) )
{
/*
* Linux workaround: occasionally we will see timeouts a long way in
* the future due to wrapping in Linux's jiffy time handling. We check
* for timeouts wrapped negative, and for positive timeouts more than
* about 13 days in the future (2^50ns). The correct fix is to trigger
* an interrupt immediately (since Linux in fact has pending work to
* do in this situation). However, older guests also set a long timeout
* when they have *no* pending timers at all: setting an immediate
* timeout in this case can burn a lot of CPU. We therefore go for a
* reasonable middleground of triggering a timer event in 100ms.
*/
gdprintk(XENLOG_INFO, "Warning: huge timeout set: %"PRIx64"\n",
timeout);
set_timer(&v->singleshot_timer, NOW() + MILLISECS(100));
}
else
{
migrate_timer(&v->singleshot_timer, smp_processor_id());
set_timer(&v->singleshot_timer, timeout);
}
return 0;
}
/* sched_id - fetch ID of current scheduler */
int sched_id(void)
{
return ops.sched_id;
}
/* Adjust scheduling parameter for a given domain. */
long sched_adjust(struct domain *d, struct xen_domctl_scheduler_op *op)
{
long ret;
ret = xsm_domctl_scheduler_op(XSM_HOOK, d, op->cmd);
if ( ret )
return ret;
if ( op->sched_id != dom_scheduler(d)->sched_id )
return -EINVAL;
switch ( op->cmd )
{
case XEN_DOMCTL_SCHEDOP_putinfo:
case XEN_DOMCTL_SCHEDOP_getinfo:
case XEN_DOMCTL_SCHEDOP_putvcpuinfo:
case XEN_DOMCTL_SCHEDOP_getvcpuinfo:
break;
default:
return -EINVAL;
}
/* NB: the pluggable scheduler code needs to take care
* of locking by itself. */
if ( (ret = SCHED_OP(dom_scheduler(d), adjust, d, op)) == 0 )
TRACE_1D(TRC_SCHED_ADJDOM, d->domain_id);
return ret;
}
long sched_adjust_global(struct xen_sysctl_scheduler_op *op)
{
struct cpupool *pool;
int rc;
rc = xsm_sysctl_scheduler_op(XSM_HOOK, op->cmd);
if ( rc )
return rc;
if ( (op->cmd != XEN_SYSCTL_SCHEDOP_putinfo) &&
(op->cmd != XEN_SYSCTL_SCHEDOP_getinfo) )
return -EINVAL;
pool = cpupool_get_by_id(op->cpupool_id);
if ( pool == NULL )
return -ESRCH;
rc = ((op->sched_id == pool->sched->sched_id)
? SCHED_OP(pool->sched, adjust_global, op) : -EINVAL);
cpupool_put(pool);
return rc;
}
static void vcpu_periodic_timer_work(struct vcpu *v)
{
s_time_t now = NOW();
s_time_t periodic_next_event;
if ( v->periodic_period == 0 )
return;
periodic_next_event = v->periodic_last_event + v->periodic_period;
if ( now >= periodic_next_event )
{
send_timer_event(v);
v->periodic_last_event = now;
periodic_next_event = now + v->periodic_period;
}
migrate_timer(&v->periodic_timer, smp_processor_id());
set_timer(&v->periodic_timer, periodic_next_event);
}
/*
* The main function
* - deschedule the current domain (scheduler independent).
* - pick a new domain (scheduler dependent).
*/
static void schedule(void)
{
struct vcpu *prev = current, *next = NULL;
s_time_t now;
struct scheduler *sched;
unsigned long *tasklet_work = &this_cpu(tasklet_work_to_do);
bool_t tasklet_work_scheduled = 0;
struct schedule_data *sd;
spinlock_t *lock;
struct task_slice next_slice;
int cpu = smp_processor_id();
ASSERT_NOT_IN_ATOMIC();
SCHED_STAT_CRANK(sched_run);
sd = &this_cpu(schedule_data);
/* Update tasklet scheduling status. */
switch ( *tasklet_work )
{
case TASKLET_enqueued:
set_bit(_TASKLET_scheduled, tasklet_work);
/* fallthrough */
case TASKLET_enqueued|TASKLET_scheduled:
tasklet_work_scheduled = 1;
break;
case TASKLET_scheduled:
clear_bit(_TASKLET_scheduled, tasklet_work);
case 0:
/*tasklet_work_scheduled = 0;*/
break;
default:
BUG();
}
lock = pcpu_schedule_lock_irq(cpu);
now = NOW();
stop_timer(&sd->s_timer);
/* get policy-specific decision on scheduling... */
sched = this_cpu(scheduler);
next_slice = sched->do_schedule(sched, now, tasklet_work_scheduled);
next = next_slice.task;
sd->curr = next;
if ( next_slice.time >= 0 ) /* -ve means no limit */
set_timer(&sd->s_timer, now + next_slice.time);
if ( unlikely(prev == next) )
{
pcpu_schedule_unlock_irq(lock, cpu);
TRACE_4D(TRC_SCHED_SWITCH_INFCONT,
next->domain->domain_id, next->vcpu_id,
now - prev->runstate.state_entry_time,
next_slice.time);
trace_continue_running(next);
return continue_running(prev);
}
TRACE_3D(TRC_SCHED_SWITCH_INFPREV,
prev->domain->domain_id, prev->vcpu_id,
now - prev->runstate.state_entry_time);
TRACE_4D(TRC_SCHED_SWITCH_INFNEXT,
next->domain->domain_id, next->vcpu_id,
(next->runstate.state == RUNSTATE_runnable) ?
(now - next->runstate.state_entry_time) : 0,
next_slice.time);
ASSERT(prev->runstate.state == RUNSTATE_running);
TRACE_4D(TRC_SCHED_SWITCH,
prev->domain->domain_id, prev->vcpu_id,
next->domain->domain_id, next->vcpu_id);
vcpu_runstate_change(
prev,
((prev->pause_flags & VPF_blocked) ? RUNSTATE_blocked :
(vcpu_runnable(prev) ? RUNSTATE_runnable : RUNSTATE_offline)),
now);
prev->last_run_time = now;
ASSERT(next->runstate.state != RUNSTATE_running);
vcpu_runstate_change(next, RUNSTATE_running, now);
/*
* NB. Don't add any trace records from here until the actual context
* switch, else lost_records resume will not work properly.
*/
ASSERT(!next->is_running);
next->is_running = 1;
pcpu_schedule_unlock_irq(lock, cpu);
SCHED_STAT_CRANK(sched_ctx);
stop_timer(&prev->periodic_timer);
if ( next_slice.migrated )
sched_move_irqs(next);
vcpu_periodic_timer_work(next);
context_switch(prev, next);
}
void context_saved(struct vcpu *prev)
{
/* Clear running flag /after/ writing context to memory. */
smp_wmb();
prev->is_running = 0;
/* Check for migration request /after/ clearing running flag. */
smp_mb();
SCHED_OP(vcpu_scheduler(prev), context_saved, prev);
if ( unlikely(prev->pause_flags & VPF_migrating) )
vcpu_migrate(prev);
}
/* The scheduler timer: force a run through the scheduler */
static void s_timer_fn(void *unused)
{
raise_softirq(SCHEDULE_SOFTIRQ);
SCHED_STAT_CRANK(sched_irq);
}
/* Per-VCPU periodic timer function: sends a virtual timer interrupt. */
static void vcpu_periodic_timer_fn(void *data)
{
struct vcpu *v = data;
vcpu_periodic_timer_work(v);
}
/* Per-VCPU single-shot timer function: sends a virtual timer interrupt. */
static void vcpu_singleshot_timer_fn(void *data)
{
struct vcpu *v = data;
send_timer_event(v);
}
/* SCHEDOP_poll timeout callback. */
static void poll_timer_fn(void *data)
{
struct vcpu *v = data;
if ( test_and_clear_bit(v->vcpu_id, v->domain->poll_mask) )
vcpu_unblock(v);
}
static int cpu_schedule_up(unsigned int cpu)
{
struct schedule_data *sd = &per_cpu(schedule_data, cpu);
void *sched_priv;
per_cpu(scheduler, cpu) = &ops;
spin_lock_init(&sd->_lock);
sd->schedule_lock = &sd->_lock;
sd->curr = idle_vcpu[cpu];
init_timer(&sd->s_timer, s_timer_fn, NULL, cpu);
atomic_set(&sd->urgent_count, 0);
/* Boot CPU is dealt with later in schedule_init(). */
if ( cpu == 0 )
return 0;
if ( idle_vcpu[cpu] == NULL )
alloc_vcpu(idle_vcpu[0]->domain, cpu, cpu);
else
{
struct vcpu *idle = idle_vcpu[cpu];
/*
* During (ACPI?) suspend the idle vCPU for this pCPU is not freed,
* while its scheduler specific data (what is pointed by sched_priv)
* is. Also, at this stage of the resume path, we attach the pCPU
* to the default scheduler, no matter in what cpupool it was before
* suspend. To avoid inconsistency, let's allocate default scheduler
* data for the idle vCPU here. If the pCPU was in a different pool
* with a different scheduler, it is schedule_cpu_switch(), invoked
* later, that will set things up as appropriate.
*/
ASSERT(idle->sched_priv == NULL);
idle->sched_priv = SCHED_OP(&ops, alloc_vdata, idle,
idle->domain->sched_priv);
if ( idle->sched_priv == NULL )
return -ENOMEM;
}
if ( idle_vcpu[cpu] == NULL )
return -ENOMEM;
/*
* We don't want to risk calling xfree() on an sd->sched_priv
* (e.g., inside free_pdata, from cpu_schedule_down() called
* during CPU_UP_CANCELLED) that contains an IS_ERR value.
*/
sched_priv = SCHED_OP(&ops, alloc_pdata, cpu);
if ( IS_ERR(sched_priv) )
return PTR_ERR(sched_priv);
sd->sched_priv = sched_priv;
return 0;
}
static void cpu_schedule_down(unsigned int cpu)
{
struct schedule_data *sd = &per_cpu(schedule_data, cpu);
struct scheduler *sched = per_cpu(scheduler, cpu);
SCHED_OP(sched, free_pdata, sd->sched_priv, cpu);
SCHED_OP(sched, free_vdata, idle_vcpu[cpu]->sched_priv);
idle_vcpu[cpu]->sched_priv = NULL;
sd->sched_priv = NULL;
kill_timer(&sd->s_timer);
}
static int cpu_schedule_callback(
struct notifier_block *nfb, unsigned long action, void *hcpu)
{
unsigned int cpu = (unsigned long)hcpu;
struct scheduler *sched = per_cpu(scheduler, cpu);
struct schedule_data *sd = &per_cpu(schedule_data, cpu);
int rc = 0;
/*
* From the scheduler perspective, bringing up a pCPU requires
* allocating and initializing the per-pCPU scheduler specific data,
* as well as "registering" this pCPU to the scheduler (which may
* involve modifying some scheduler wide data structures).
* This happens by calling the alloc_pdata and init_pdata hooks, in
* this order. A scheduler that does not need to allocate any per-pCPU
* data can avoid implementing alloc_pdata. init_pdata may, however, be
* necessary/useful in this case too (e.g., it can contain the "register
* the pCPU to the scheduler" part). alloc_pdata (if present) is called
* during CPU_UP_PREPARE. init_pdata (if present) is called during
* CPU_STARTING.
*
* On the other hand, at teardown, we need to reverse what has been done
* during initialization, and then free the per-pCPU specific data. This
* happens by calling the deinit_pdata and free_pdata hooks, in this
* order. If no per-pCPU memory was allocated, there is no need to
* provide an implementation of free_pdata. deinit_pdata may, however,
* be necessary/useful in this case too (e.g., it can undo something done
* on scheduler wide data structure during init_pdata). Both deinit_pdata
* and free_pdata are called during CPU_DEAD.
*
* If someting goes wrong during bringup, we go to CPU_UP_CANCELLED
* *before* having called init_pdata. In this case, as there is no
* initialization needing undoing, only free_pdata should be called.
* This means it is possible to call free_pdata just after alloc_pdata,
* without a init_pdata/deinit_pdata "cycle" in between the two.
*
* So, in summary, the usage pattern should look either
* - alloc_pdata-->init_pdata-->deinit_pdata-->free_pdata, or
* - alloc_pdata-->free_pdata.
*/
switch ( action )
{
case CPU_STARTING:
SCHED_OP(sched, init_pdata, sd->sched_priv, cpu);
break;
case CPU_UP_PREPARE:
rc = cpu_schedule_up(cpu);
break;
case CPU_DEAD:
SCHED_OP(sched, deinit_pdata, sd->sched_priv, cpu);
/* Fallthrough */
case CPU_UP_CANCELED:
cpu_schedule_down(cpu);
break;
default:
break;
}
return !rc ? NOTIFY_DONE : notifier_from_errno(rc);
}
static struct notifier_block cpu_schedule_nfb = {
.notifier_call = cpu_schedule_callback
};
/* Initialise the data structures. */
void __init scheduler_init(void)
{
struct domain *idle_domain;
int i;
open_softirq(SCHEDULE_SOFTIRQ, schedule);
for ( i = 0; i < NUM_SCHEDULERS; i++)
{
if ( schedulers[i]->global_init && schedulers[i]->global_init() < 0 )
schedulers[i] = NULL;
else if ( !ops.name && !strcmp(schedulers[i]->opt_name, opt_sched) )
ops = *schedulers[i];
}
if ( !ops.name )
{
printk("Could not find scheduler: %s\n", opt_sched);
for ( i = 0; i < NUM_SCHEDULERS; i++ )
if ( schedulers[i] &&
!strcmp(schedulers[i]->opt_name, CONFIG_SCHED_DEFAULT) )
{
ops = *schedulers[i];
break;
}
BUG_ON(!ops.name);
printk("Using '%s' (%s)\n", ops.name, ops.opt_name);
}
if ( cpu_schedule_up(0) )
BUG();
register_cpu_notifier(&cpu_schedule_nfb);
printk("Using scheduler: %s (%s)\n", ops.name, ops.opt_name);
if ( SCHED_OP(&ops, init) )
panic("scheduler returned error on init");
if ( sched_ratelimit_us &&
(sched_ratelimit_us > XEN_SYSCTL_SCHED_RATELIMIT_MAX
|| sched_ratelimit_us < XEN_SYSCTL_SCHED_RATELIMIT_MIN) )
{
printk("WARNING: sched_ratelimit_us outside of valid range [%d,%d].\n"
" Resetting to default %u\n",
XEN_SYSCTL_SCHED_RATELIMIT_MIN,
XEN_SYSCTL_SCHED_RATELIMIT_MAX,
SCHED_DEFAULT_RATELIMIT_US);
sched_ratelimit_us = SCHED_DEFAULT_RATELIMIT_US;
}
idle_domain = domain_create(DOMID_IDLE, 0, 0, NULL);
BUG_ON(IS_ERR(idle_domain));
idle_domain->vcpu = idle_vcpu;
idle_domain->max_vcpus = nr_cpu_ids;
if ( alloc_vcpu(idle_domain, 0, 0) == NULL )
BUG();
this_cpu(schedule_data).sched_priv = SCHED_OP(&ops, alloc_pdata, 0);
BUG_ON(IS_ERR(this_cpu(schedule_data).sched_priv));
SCHED_OP(&ops, init_pdata, this_cpu(schedule_data).sched_priv, 0);
}
/*
* Move a pCPU outside of the influence of the scheduler of its current
* cpupool, or subject it to the scheduler of a new cpupool.
*
* For the pCPUs that are removed from their cpupool, their scheduler becomes
* &ops (the default scheduler, selected at boot, which also services the
* default cpupool). However, as these pCPUs are not really part of any pool,
* there won't be any scheduling event on them, not even from the default
* scheduler. Basically, they will just sit idle until they are explicitly
* added back to a cpupool.
*/
int schedule_cpu_switch(unsigned int cpu, struct cpupool *c)
{
struct vcpu *idle;
void *ppriv, *ppriv_old, *vpriv, *vpriv_old;
struct scheduler *old_ops = per_cpu(scheduler, cpu);
struct scheduler *new_ops = (c == NULL) ? &ops : c->sched;
struct cpupool *old_pool = per_cpu(cpupool, cpu);
spinlock_t * old_lock;
/*
* pCPUs only move from a valid cpupool to free (i.e., out of any pool),
* or from free to a valid cpupool. In the former case (which happens when
* c is NULL), we want the CPU to have been marked as free already, as
* well as to not be valid for the source pool any longer, when we get to
* here. In the latter case (which happens when c is a valid cpupool), we
* want the CPU to still be marked as free, as well as to not yet be valid
* for the destination pool.
*/
ASSERT(c != old_pool && (c != NULL || old_pool != NULL));
ASSERT(cpumask_test_cpu(cpu, &cpupool_free_cpus));
ASSERT((c == NULL && !cpumask_test_cpu(cpu, old_pool->cpu_valid)) ||
(c != NULL && !cpumask_test_cpu(cpu, c->cpu_valid)));
if ( old_ops == new_ops )
goto out;
/*
* To setup the cpu for the new scheduler we need:
* - a valid instance of per-CPU scheduler specific data, as it is
* allocated by SCHED_OP(alloc_pdata). Note that we do not want to
* initialize it yet (i.e., we are not calling SCHED_OP(init_pdata)).
* That will be done by the target scheduler, in SCHED_OP(switch_sched),
* in proper ordering and with locking.
* - a valid instance of per-vCPU scheduler specific data, for the idle
* vCPU of cpu. That is what the target scheduler will use for the
* sched_priv field of the per-vCPU info of the idle domain.
*/
idle = idle_vcpu[cpu];
ppriv = SCHED_OP(new_ops, alloc_pdata, cpu);
if ( IS_ERR(ppriv) )
return PTR_ERR(ppriv);
vpriv = SCHED_OP(new_ops, alloc_vdata, idle, idle->domain->sched_priv);
if ( vpriv == NULL )
{
SCHED_OP(new_ops, free_pdata, ppriv, cpu);
return -ENOMEM;
}
SCHED_OP(old_ops, tick_suspend, cpu);
/*
* The actual switch, including (if necessary) the rerouting of the
* scheduler lock to whatever new_ops prefers, needs to happen in one
* critical section, protected by old_ops' lock, or races are possible.
* It is, in fact, the lock of another scheduler that we are taking (the
* scheduler of the cpupool that cpu still belongs to). But that is ok
* as, anyone trying to schedule on this cpu will spin until when we
* release that lock (bottom of this function). When he'll get the lock
* --thanks to the loop inside *_schedule_lock() functions-- he'll notice
* that the lock itself changed, and retry acquiring the new one (which
* will be the correct, remapped one, at that point).
*/
old_lock = pcpu_schedule_lock_irq(cpu);
vpriv_old = idle->sched_priv;
ppriv_old = per_cpu(schedule_data, cpu).sched_priv;
SCHED_OP(new_ops, switch_sched, cpu, ppriv, vpriv);
/* _Not_ pcpu_schedule_unlock(): schedule_lock may have changed! */
spin_unlock_irq(old_lock);
SCHED_OP(new_ops, tick_resume, cpu);
SCHED_OP(old_ops, deinit_pdata, ppriv_old, cpu);
SCHED_OP(old_ops, free_vdata, vpriv_old);
SCHED_OP(old_ops, free_pdata, ppriv_old, cpu);
out:
per_cpu(cpupool, cpu) = c;
/* When a cpu is added to a pool, trigger it to go pick up some work */
if ( c != NULL )
cpu_raise_softirq(cpu, SCHEDULE_SOFTIRQ);
return 0;
}
struct scheduler *scheduler_get_default(void)
{
return &ops;
}
struct scheduler *scheduler_alloc(unsigned int sched_id, int *perr)
{
int i;
struct scheduler *sched;
for ( i = 0; i < NUM_SCHEDULERS; i++ )
if ( schedulers[i] && schedulers[i]->sched_id == sched_id )
goto found;
*perr = -ENOENT;
return NULL;
found:
*perr = -ENOMEM;
if ( (sched = xmalloc(struct scheduler)) == NULL )
return NULL;
memcpy(sched, schedulers[i], sizeof(*sched));
if ( (*perr = SCHED_OP(sched, init)) != 0 )
{
xfree(sched);
sched = NULL;
}
return sched;
}
void scheduler_free(struct scheduler *sched)
{
BUG_ON(sched == &ops);
SCHED_OP(sched, deinit);
xfree(sched);
}
void schedule_dump(struct cpupool *c)
{
unsigned int i;
struct scheduler *sched;
cpumask_t *cpus;
/* Locking, if necessary, must be handled withing each scheduler */
if ( c != NULL )
{
sched = c->sched;
cpus = c->cpu_valid;
printk("Scheduler: %s (%s)\n", sched->name, sched->opt_name);
SCHED_OP(sched, dump_settings);
}
else
{
sched = &ops;
cpus = &cpupool_free_cpus;
}
if ( sched->dump_cpu_state != NULL )
{
printk("CPUs info:\n");
for_each_cpu (i, cpus)
SCHED_OP(sched, dump_cpu_state, i);
}
}
void sched_tick_suspend(void)
{
struct scheduler *sched;
unsigned int cpu = smp_processor_id();
sched = per_cpu(scheduler, cpu);
SCHED_OP(sched, tick_suspend, cpu);
rcu_idle_enter(cpu);
rcu_idle_timer_start();
}
void sched_tick_resume(void)
{
struct scheduler *sched;
unsigned int cpu = smp_processor_id();
rcu_idle_timer_stop();
rcu_idle_exit(cpu);
sched = per_cpu(scheduler, cpu);
SCHED_OP(sched, tick_resume, cpu);
}
void wait(void)
{
schedule();
}
#ifdef CONFIG_COMPAT
#include "compat/schedule.c"
#endif
#endif /* !COMPAT */
/*
* Local variables:
* mode: C
* c-file-style: "BSD"
* c-basic-offset: 4
* tab-width: 4
* indent-tabs-mode: nil
* End:
*/