[thirdparty/linux.git] / arch / s390 / kernel / vtime.c

/*
 *    Virtual cpu timer based timer functions.
 *
 *    Copyright IBM Corp. 2004, 2012
 *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
 */

#include <linux/kernel_stat.h>
#include <linux/sched/cputime.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/timex.h>
#include <linux/types.h>
#include <linux/time.h>

#include <asm/vtimer.h>
#include <asm/vtime.h>
#include <asm/cpu_mf.h>
#include <asm/smp.h>

#include "entry.h"

static void virt_timer_expire(void);

static LIST_HEAD(virt_timer_list);
static DEFINE_SPINLOCK(virt_timer_lock);
static atomic64_t virt_timer_current;
static atomic64_t virt_timer_elapsed;

DEFINE_PER_CPU(u64, mt_cycles[8]);
static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
static DEFINE_PER_CPU(u64, mt_scaling_jiffies);

static inline u64 get_vtimer(void)
{
	u64 timer;

	asm volatile("stpt %0" : "=m" (timer));
	return timer;
}

static inline void set_vtimer(u64 expires)
{
	u64 timer;

	asm volatile(
		"	stpt	%0\n"	/* Store current cpu timer value */
		"	spt	%1"	/* Set new value imm. afterwards */
		: "=m" (timer) : "m" (expires));
	S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
	S390_lowcore.last_update_timer = expires;
}

static inline int virt_timer_forward(u64 elapsed)
{
	BUG_ON(!irqs_disabled());

	if (list_empty(&virt_timer_list))
		return 0;
	elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
	return elapsed >= atomic64_read(&virt_timer_current);
}

static void update_mt_scaling(void)
{
	u64 cycles_new[8], *cycles_old;
	u64 delta, fac, mult, div;
	int i;

	stcctm5(smp_cpu_mtid + 1, cycles_new);
	cycles_old = this_cpu_ptr(mt_cycles);
	fac = 1;
	mult = div = 0;
	for (i = 0; i <= smp_cpu_mtid; i++) {
		delta = cycles_new[i] - cycles_old[i];
		div += delta;
		mult *= i + 1;
		mult += delta * fac;
		fac *= i + 1;
	}
	div *= fac;
	if (div > 0) {
		/* Update scaling factor */
		__this_cpu_write(mt_scaling_mult, mult);
		__this_cpu_write(mt_scaling_div, div);
		memcpy(cycles_old, cycles_new,
		       sizeof(u64) * (smp_cpu_mtid + 1));
	}
	__this_cpu_write(mt_scaling_jiffies, jiffies_64);
}

static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
{
	u64 delta;

	delta = new - *tsk_vtime;
	*tsk_vtime = new;
	return delta;
}


static inline u64 scale_vtime(u64 vtime)
{
	u64 mult = __this_cpu_read(mt_scaling_mult);
	u64 div = __this_cpu_read(mt_scaling_div);

	if (smp_cpu_mtid)
		return vtime * mult / div;
	return vtime;
}

static void account_system_index_scaled(struct task_struct *p,
					u64 cputime, u64 scaled,
					enum cpu_usage_stat index)
{
	p->stimescaled += cputime_to_nsecs(scaled);
	account_system_index_time(p, cputime_to_nsecs(cputime), index);
}

/*
 * Update process times based on virtual cpu times stored by entry.S
 * to the lowcore fields user_timer, system_timer & steal_clock.
 */
static int do_account_vtime(struct task_struct *tsk)
{
	u64 timer, clock, user, guest, system, hardirq, softirq, steal;

	timer = S390_lowcore.last_update_timer;
	clock = S390_lowcore.last_update_clock;
	asm volatile(
		"	stpt	%0\n"	/* Store current cpu timer value */
#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
		"	stckf	%1"	/* Store current tod clock value */
#else
		"	stck	%1"	/* Store current tod clock value */
#endif
		: "=m" (S390_lowcore.last_update_timer),
		  "=m" (S390_lowcore.last_update_clock));
	clock = S390_lowcore.last_update_clock - clock;
	timer -= S390_lowcore.last_update_timer;

	if (hardirq_count())
		S390_lowcore.hardirq_timer += timer;
	else
		S390_lowcore.system_timer += timer;

	/* Update MT utilization calculation */
	if (smp_cpu_mtid &&
	    time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
		update_mt_scaling();

	/* Calculate cputime delta */
	user = update_tsk_timer(&tsk->thread.user_timer,
				READ_ONCE(S390_lowcore.user_timer));
	guest = update_tsk_timer(&tsk->thread.guest_timer,
				 READ_ONCE(S390_lowcore.guest_timer));
	system = update_tsk_timer(&tsk->thread.system_timer,
				  READ_ONCE(S390_lowcore.system_timer));
	hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
				   READ_ONCE(S390_lowcore.hardirq_timer));
	softirq = update_tsk_timer(&tsk->thread.softirq_timer,
				   READ_ONCE(S390_lowcore.softirq_timer));
	S390_lowcore.steal_timer +=
		clock - user - guest - system - hardirq - softirq;

	/* Push account value */
	if (user) {
		account_user_time(tsk, cputime_to_nsecs(user));
		tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
	}

	if (guest) {
		account_guest_time(tsk, cputime_to_nsecs(guest));
		tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
	}

	if (system)
		account_system_index_scaled(tsk, system, scale_vtime(system),
					    CPUTIME_SYSTEM);
	if (hardirq)
		account_system_index_scaled(tsk, hardirq, scale_vtime(hardirq),
					    CPUTIME_IRQ);
	if (softirq)
		account_system_index_scaled(tsk, softirq, scale_vtime(softirq),
					    CPUTIME_SOFTIRQ);

	steal = S390_lowcore.steal_timer;
	if ((s64) steal > 0) {
		S390_lowcore.steal_timer = 0;
		account_steal_time(cputime_to_nsecs(steal));
	}

	return virt_timer_forward(user + guest + system + hardirq + softirq);
}

void vtime_task_switch(struct task_struct *prev)
{
	do_account_vtime(prev);
	prev->thread.user_timer = S390_lowcore.user_timer;
	prev->thread.guest_timer = S390_lowcore.guest_timer;
	prev->thread.system_timer = S390_lowcore.system_timer;
	prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
	prev->thread.softirq_timer = S390_lowcore.softirq_timer;
	S390_lowcore.user_timer = current->thread.user_timer;
	S390_lowcore.guest_timer = current->thread.guest_timer;
	S390_lowcore.system_timer = current->thread.system_timer;
	S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
	S390_lowcore.softirq_timer = current->thread.softirq_timer;
}

/*
 * In s390, accounting pending user time also implies
 * accounting system time in order to correctly compute
 * the stolen time accounting.
 */
void vtime_flush(struct task_struct *tsk)
{
	if (do_account_vtime(tsk))
		virt_timer_expire();
}

/*
 * Update process times based on virtual cpu times stored by entry.S
 * to the lowcore fields user_timer, system_timer & steal_clock.
 */
void vtime_account_irq_enter(struct task_struct *tsk)
{
	u64 timer;

	timer = S390_lowcore.last_update_timer;
	S390_lowcore.last_update_timer = get_vtimer();
	timer -= S390_lowcore.last_update_timer;

	if ((tsk->flags & PF_VCPU) && (irq_count() == 0))
		S390_lowcore.guest_timer += timer;
	else if (hardirq_count())
		S390_lowcore.hardirq_timer += timer;
	else if (in_serving_softirq())
		S390_lowcore.softirq_timer += timer;
	else
		S390_lowcore.system_timer += timer;

	virt_timer_forward(timer);
}
EXPORT_SYMBOL_GPL(vtime_account_irq_enter);

void vtime_account_system(struct task_struct *tsk)
__attribute__((alias("vtime_account_irq_enter")));
EXPORT_SYMBOL_GPL(vtime_account_system);

/*
 * Sorted add to a list. List is linear searched until first bigger
 * element is found.
 */
static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
{
	struct vtimer_list *tmp;

	list_for_each_entry(tmp, head, entry) {
		if (tmp->expires > timer->expires) {
			list_add_tail(&timer->entry, &tmp->entry);
			return;
		}
	}
	list_add_tail(&timer->entry, head);
}

/*
 * Handler for expired virtual CPU timer.
 */
static void virt_timer_expire(void)
{
	struct vtimer_list *timer, *tmp;
	unsigned long elapsed;
	LIST_HEAD(cb_list);

	/* walk timer list, fire all expired timers */
	spin_lock(&virt_timer_lock);
	elapsed = atomic64_read(&virt_timer_elapsed);
	list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
		if (timer->expires < elapsed)
			/* move expired timer to the callback queue */
			list_move_tail(&timer->entry, &cb_list);
		else
			timer->expires -= elapsed;
	}
	if (!list_empty(&virt_timer_list)) {
		timer = list_first_entry(&virt_timer_list,
					 struct vtimer_list, entry);
		atomic64_set(&virt_timer_current, timer->expires);
	}
	atomic64_sub(elapsed, &virt_timer_elapsed);
	spin_unlock(&virt_timer_lock);

	/* Do callbacks and recharge periodic timers */
	list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
		list_del_init(&timer->entry);
		timer->function(timer->data);
		if (timer->interval) {
			/* Recharge interval timer */
			timer->expires = timer->interval +
				atomic64_read(&virt_timer_elapsed);
			spin_lock(&virt_timer_lock);
			list_add_sorted(timer, &virt_timer_list);
			spin_unlock(&virt_timer_lock);
		}
	}
}

void init_virt_timer(struct vtimer_list *timer)
{
	timer->function = NULL;
	INIT_LIST_HEAD(&timer->entry);
}
EXPORT_SYMBOL(init_virt_timer);

static inline int vtimer_pending(struct vtimer_list *timer)
{
	return !list_empty(&timer->entry);
}

static void internal_add_vtimer(struct vtimer_list *timer)
{
	if (list_empty(&virt_timer_list)) {
		/* First timer, just program it. */
		atomic64_set(&virt_timer_current, timer->expires);
		atomic64_set(&virt_timer_elapsed, 0);
		list_add(&timer->entry, &virt_timer_list);
	} else {
		/* Update timer against current base. */
		timer->expires += atomic64_read(&virt_timer_elapsed);
		if (likely((s64) timer->expires <
			   (s64) atomic64_read(&virt_timer_current)))
			/* The new timer expires before the current timer. */
			atomic64_set(&virt_timer_current, timer->expires);
		/* Insert new timer into the list. */
		list_add_sorted(timer, &virt_timer_list);
	}
}

static void __add_vtimer(struct vtimer_list *timer, int periodic)
{
	unsigned long flags;

	timer->interval = periodic ? timer->expires : 0;
	spin_lock_irqsave(&virt_timer_lock, flags);
	internal_add_vtimer(timer);
	spin_unlock_irqrestore(&virt_timer_lock, flags);
}

/*
 * add_virt_timer - add a oneshot virtual CPU timer
 */
void add_virt_timer(struct vtimer_list *timer)
{
	__add_vtimer(timer, 0);
}
EXPORT_SYMBOL(add_virt_timer);

/*
 * add_virt_timer_int - add an interval virtual CPU timer
 */
void add_virt_timer_periodic(struct vtimer_list *timer)
{
	__add_vtimer(timer, 1);
}
EXPORT_SYMBOL(add_virt_timer_periodic);

static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
{
	unsigned long flags;
	int rc;

	BUG_ON(!timer->function);

	if (timer->expires == expires && vtimer_pending(timer))
		return 1;
	spin_lock_irqsave(&virt_timer_lock, flags);
	rc = vtimer_pending(timer);
	if (rc)
		list_del_init(&timer->entry);
	timer->interval = periodic ? expires : 0;
	timer->expires = expires;
	internal_add_vtimer(timer);
	spin_unlock_irqrestore(&virt_timer_lock, flags);
	return rc;
}

/*
 * returns whether it has modified a pending timer (1) or not (0)
 */
int mod_virt_timer(struct vtimer_list *timer, u64 expires)
{
	return __mod_vtimer(timer, expires, 0);
}
EXPORT_SYMBOL(mod_virt_timer);

/*
 * returns whether it has modified a pending timer (1) or not (0)
 */
int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
{
	return __mod_vtimer(timer, expires, 1);
}
EXPORT_SYMBOL(mod_virt_timer_periodic);

/*
 * Delete a virtual timer.
 *
 * returns whether the deleted timer was pending (1) or not (0)
 */
int del_virt_timer(struct vtimer_list *timer)
{
	unsigned long flags;

	if (!vtimer_pending(timer))
		return 0;
	spin_lock_irqsave(&virt_timer_lock, flags);
	list_del_init(&timer->entry);
	spin_unlock_irqrestore(&virt_timer_lock, flags);
	return 1;
}
EXPORT_SYMBOL(del_virt_timer);

/*
 * Start the virtual CPU timer on the current CPU.
 */
void vtime_init(void)
{
	/* set initial cpu timer */
	set_vtimer(VTIMER_MAX_SLICE);
	/* Setup initial MT scaling values */
	if (smp_cpu_mtid) {
		__this_cpu_write(mt_scaling_jiffies, jiffies);
		__this_cpu_write(mt_scaling_mult, 1);
		__this_cpu_write(mt_scaling_div, 1);
		stcctm5(smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
	}
}
Commit	Line	Data
1da177e4	1	/*
1da177e4 LT	2	* Virtual cpu timer based timer functions.
1da177e4 LT	3	*
27f6b416	4	* Copyright IBM Corp. 2004, 2012
1da177e4 LT	5	* Author(s): Jan Glauber <jan.glauber@de.ibm.com>
	6	*/
	7
27f6b416	8	#include <linux/kernel_stat.h>
32ef5517	9	#include <linux/sched/cputime.h>
27f6b416	10	#include <linux/export.h>
1da177e4	11	#include <linux/kernel.h>
1da177e4	12	#include <linux/timex.h>
27f6b416 MS	13	#include <linux/types.h>
27f6b416 MS	14	#include <linux/time.h>
1da177e4	15
27f6b416	16	#include <asm/vtimer.h>
a5725ac2	17	#include <asm/vtime.h>
10ad34bc MS	18	#include <asm/cpu_mf.h>
10ad34bc MS	19	#include <asm/smp.h>
1da177e4	20
521b00cd HC	21	#include "entry.h"
521b00cd HC	22
27f6b416	23	static void virt_timer_expire(void);
1da177e4	24
27f6b416 MS	25	static LIST_HEAD(virt_timer_list);
	26	static DEFINE_SPINLOCK(virt_timer_lock);
	27	static atomic64_t virt_timer_current;
	28	static atomic64_t virt_timer_elapsed;
	29
72d38b19	30	DEFINE_PER_CPU(u64, mt_cycles[8]);
10ad34bc MS	31	static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
10ad34bc MS	32	static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
f341b8df	33	static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
10ad34bc	34
27f6b416	35	static inline u64 get_vtimer(void)
9cfb9b3c	36	{
27f6b416	37	u64 timer;
9cfb9b3c	38
27f6b416	39	asm volatile("stpt %0" : "=m" (timer));
9cfb9b3c MS	40	return timer;
	41	}
	42
27f6b416	43	static inline void set_vtimer(u64 expires)
9cfb9b3c	44	{
27f6b416	45	u64 timer;
9cfb9b3c	46
27f6b416 MS	47	asm volatile(
	48	" stpt %0\n" /* Store current cpu timer value */
	49	" spt %1" /* Set new value imm. afterwards */
	50	: "=m" (timer) : "m" (expires));
9cfb9b3c MS	51	S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
	52	S390_lowcore.last_update_timer = expires;
	53	}
	54
27f6b416 MS	55	static inline int virt_timer_forward(u64 elapsed)
	56	{
	57	BUG_ON(!irqs_disabled());
	58
	59	if (list_empty(&virt_timer_list))
	60	return 0;
	61	elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
	62	return elapsed >= atomic64_read(&virt_timer_current);
	63	}
	64
72d38b19 MS	65	static void update_mt_scaling(void)
	66	{
	67	u64 cycles_new[8], *cycles_old;
	68	u64 delta, fac, mult, div;
	69	int i;
	70
	71	stcctm5(smp_cpu_mtid + 1, cycles_new);
	72	cycles_old = this_cpu_ptr(mt_cycles);
	73	fac = 1;
	74	mult = div = 0;
	75	for (i = 0; i <= smp_cpu_mtid; i++) {
	76	delta = cycles_new[i] - cycles_old[i];
	77	div += delta;
	78	mult *= i + 1;
	79	mult += delta * fac;
	80	fac *= i + 1;
	81	}
	82	div *= fac;
	83	if (div > 0) {
	84	/* Update scaling factor */
	85	__this_cpu_write(mt_scaling_mult, mult);
	86	__this_cpu_write(mt_scaling_div, div);
	87	memcpy(cycles_old, cycles_new,
	88	sizeof(u64) * (smp_cpu_mtid + 1));
	89	}
	90	__this_cpu_write(mt_scaling_jiffies, jiffies_64);
	91	}
	92
b7394a5f MS	93	static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
	94	{
	95	u64 delta;
	96
	97	delta = new - *tsk_vtime;
	98	*tsk_vtime = new;
	99	return delta;
	100	}
	101
	102
	103	static inline u64 scale_vtime(u64 vtime)
	104	{
	105	u64 mult = __this_cpu_read(mt_scaling_mult);
	106	u64 div = __this_cpu_read(mt_scaling_div);
	107
	108	if (smp_cpu_mtid)
	109	return vtime * mult / div;
	110	return vtime;
	111	}
	112
	113	static void account_system_index_scaled(struct task_struct *p,
e69ca822	114	u64 cputime, u64 scaled,
b7394a5f MS	115	enum cpu_usage_stat index)
b7394a5f MS	116	{
5613fda9	117	p->stimescaled += cputime_to_nsecs(scaled);
fb8b049c	118	account_system_index_time(p, cputime_to_nsecs(cputime), index);
b7394a5f MS	119	}
b7394a5f MS	120
1da177e4 LT	121	/*
	122	* Update process times based on virtual cpu times stored by entry.S
	123	* to the lowcore fields user_timer, system_timer & steal_clock.
	124	*/
8f2b468a	125	static int do_account_vtime(struct task_struct *tsk)
1da177e4	126	{
b7394a5f	127	u64 timer, clock, user, guest, system, hardirq, softirq, steal;
1da177e4 LT	128
	129	timer = S390_lowcore.last_update_timer;
	130	clock = S390_lowcore.last_update_clock;
27f6b416 MS	131	asm volatile(
27f6b416 MS	132	" stpt %0\n" /* Store current cpu timer value */
1f759bb3 MS	133	#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
	134	" stckf %1" /* Store current tod clock value */
	135	#else
27f6b416	136	" stck %1" /* Store current tod clock value */
1f759bb3	137	#endif
27f6b416 MS	138	: "=m" (S390_lowcore.last_update_timer),
27f6b416 MS	139	"=m" (S390_lowcore.last_update_clock));
b7394a5f MS	140	clock = S390_lowcore.last_update_clock - clock;
	141	timer -= S390_lowcore.last_update_timer;
	142
	143	if (hardirq_count())
	144	S390_lowcore.hardirq_timer += timer;
	145	else
	146	S390_lowcore.system_timer += timer;
1da177e4	147
72d38b19	148	/* Update MT utilization calculation */
f341b8df	149	if (smp_cpu_mtid &&
72d38b19 MS	150	time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
72d38b19 MS	151	update_mt_scaling();
10ad34bc	152
b7394a5f MS	153	/* Calculate cputime delta */
	154	user = update_tsk_timer(&tsk->thread.user_timer,
	155	READ_ONCE(S390_lowcore.user_timer));
	156	guest = update_tsk_timer(&tsk->thread.guest_timer,
	157	READ_ONCE(S390_lowcore.guest_timer));
	158	system = update_tsk_timer(&tsk->thread.system_timer,
	159	READ_ONCE(S390_lowcore.system_timer));
	160	hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
	161	READ_ONCE(S390_lowcore.hardirq_timer));
	162	softirq = update_tsk_timer(&tsk->thread.softirq_timer,
	163	READ_ONCE(S390_lowcore.softirq_timer));
	164	S390_lowcore.steal_timer +=
	165	clock - user - guest - system - hardirq - softirq;
	166
	167	/* Push account value */
	168	if (user) {
23244a5c	169	account_user_time(tsk, cputime_to_nsecs(user));
5613fda9	170	tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
b7394a5f	171	}
10ad34bc	172
b7394a5f	173	if (guest) {
fb8b049c	174	account_guest_time(tsk, cputime_to_nsecs(guest));
5613fda9	175	tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
10ad34bc	176	}
b7394a5f MS	177
	178	if (system)
	179	account_system_index_scaled(tsk, system, scale_vtime(system),
	180	CPUTIME_SYSTEM);
	181	if (hardirq)
	182	account_system_index_scaled(tsk, hardirq, scale_vtime(hardirq),
	183	CPUTIME_IRQ);
	184	if (softirq)
	185	account_system_index_scaled(tsk, softirq, scale_vtime(softirq),
	186	CPUTIME_SOFTIRQ);
1da177e4	187
aa5e97ce MS	188	steal = S390_lowcore.steal_timer;
	189	if ((s64) steal > 0) {
	190	S390_lowcore.steal_timer = 0;
be9095ed	191	account_steal_time(cputime_to_nsecs(steal));
1da177e4	192	}
27f6b416	193
b7394a5f	194	return virt_timer_forward(user + guest + system + hardirq + softirq);
1da177e4 LT	195	}
1da177e4 LT	196
bf9fae9f	197	void vtime_task_switch(struct task_struct *prev)
1f1c12af	198	{
8f2b468a	199	do_account_vtime(prev);
90c53e65	200	prev->thread.user_timer = S390_lowcore.user_timer;
b7394a5f	201	prev->thread.guest_timer = S390_lowcore.guest_timer;
90c53e65	202	prev->thread.system_timer = S390_lowcore.system_timer;
b7394a5f MS	203	prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
b7394a5f MS	204	prev->thread.softirq_timer = S390_lowcore.softirq_timer;
90c53e65	205	S390_lowcore.user_timer = current->thread.user_timer;
b7394a5f	206	S390_lowcore.guest_timer = current->thread.guest_timer;
90c53e65	207	S390_lowcore.system_timer = current->thread.system_timer;
b7394a5f MS	208	S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
b7394a5f MS	209	S390_lowcore.softirq_timer = current->thread.softirq_timer;
aa5e97ce	210	}
1f1c12af	211
bcebdf84 FW	212	/*
	213	* In s390, accounting pending user time also implies
	214	* accounting system time in order to correctly compute
	215	* the stolen time accounting.
	216	*/
c8d7dabf	217	void vtime_flush(struct task_struct *tsk)
aa5e97ce	218	{
8f2b468a	219	if (do_account_vtime(tsk))
27f6b416	220	virt_timer_expire();
1f1c12af MS	221	}
1f1c12af MS	222
1da177e4 LT	223	/*
	224	* Update process times based on virtual cpu times stored by entry.S
	225	* to the lowcore fields user_timer, system_timer & steal_clock.
	226	*/
6a61671b	227	void vtime_account_irq_enter(struct task_struct *tsk)
1da177e4	228	{
b7394a5f	229	u64 timer;
1da177e4 LT	230
1da177e4 LT	231	timer = S390_lowcore.last_update_timer;
9cfb9b3c	232	S390_lowcore.last_update_timer = get_vtimer();
b7394a5f MS	233	timer -= S390_lowcore.last_update_timer;
	234
	235	if ((tsk->flags & PF_VCPU) && (irq_count() == 0))
	236	S390_lowcore.guest_timer += timer;
	237	else if (hardirq_count())
	238	S390_lowcore.hardirq_timer += timer;
	239	else if (in_serving_softirq())
	240	S390_lowcore.softirq_timer += timer;
	241	else
	242	S390_lowcore.system_timer += timer;
	243
	244	virt_timer_forward(timer);
1da177e4	245	}
6a61671b	246	EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
1da177e4	247
fd25b4c2	248	void vtime_account_system(struct task_struct *tsk)
6a61671b	249	__attribute__((alias("vtime_account_irq_enter")));
fd25b4c2	250	EXPORT_SYMBOL_GPL(vtime_account_system);
11113334	251
1da177e4 LT	252	/*
	253	* Sorted add to a list. List is linear searched until first bigger
	254	* element is found.
	255	*/
	256	static void list_add_sorted(struct vtimer_list timer, struct list_head head)
	257	{
27f6b416	258	struct vtimer_list *tmp;
1da177e4	259
27f6b416 MS	260	list_for_each_entry(tmp, head, entry) {
	261	if (tmp->expires > timer->expires) {
	262	list_add_tail(&timer->entry, &tmp->entry);
1da177e4 LT	263	return;
	264	}
	265	}
	266	list_add_tail(&timer->entry, head);
	267	}
	268
	269	/*
27f6b416	270	* Handler for expired virtual CPU timer.
1da177e4	271	*/
27f6b416	272	static void virt_timer_expire(void)
1da177e4	273	{
27f6b416 MS	274	struct vtimer_list timer, tmp;
	275	unsigned long elapsed;
	276	LIST_HEAD(cb_list);
	277
	278	/* walk timer list, fire all expired timers */
	279	spin_lock(&virt_timer_lock);
	280	elapsed = atomic64_read(&virt_timer_elapsed);
	281	list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
	282	if (timer->expires < elapsed)
9cfb9b3c	283	/* move expired timer to the callback queue */
27f6b416	284	list_move_tail(&timer->entry, &cb_list);
9cfb9b3c	285	else
27f6b416	286	timer->expires -= elapsed;
1da177e4	287	}
27f6b416 MS	288	if (!list_empty(&virt_timer_list)) {
	289	timer = list_first_entry(&virt_timer_list,
	290	struct vtimer_list, entry);
	291	atomic64_set(&virt_timer_current, timer->expires);
	292	}
	293	atomic64_sub(elapsed, &virt_timer_elapsed);
	294	spin_unlock(&virt_timer_lock);
	295
	296	/* Do callbacks and recharge periodic timers */
	297	list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
	298	list_del_init(&timer->entry);
	299	timer->function(timer->data);
	300	if (timer->interval) {
	301	/* Recharge interval timer */
	302	timer->expires = timer->interval +
	303	atomic64_read(&virt_timer_elapsed);
	304	spin_lock(&virt_timer_lock);
	305	list_add_sorted(timer, &virt_timer_list);
	306	spin_unlock(&virt_timer_lock);
	307	}
4c1051e3	308	}
1da177e4 LT	309	}
	310
	311	void init_virt_timer(struct vtimer_list *timer)
	312	{
1da177e4 LT	313	timer->function = NULL;
1da177e4 LT	314	INIT_LIST_HEAD(&timer->entry);
1da177e4 LT	315	}
	316	EXPORT_SYMBOL(init_virt_timer);
	317
1da177e4 LT	318	static inline int vtimer_pending(struct vtimer_list *timer)
1da177e4 LT	319	{
27f6b416	320	return !list_empty(&timer->entry);
1da177e4 LT	321	}
1da177e4 LT	322
1da177e4 LT	323	static void internal_add_vtimer(struct vtimer_list *timer)
1da177e4 LT	324	{
27f6b416 MS	325	if (list_empty(&virt_timer_list)) {
	326	/* First timer, just program it. */
	327	atomic64_set(&virt_timer_current, timer->expires);
	328	atomic64_set(&virt_timer_elapsed, 0);
	329	list_add(&timer->entry, &virt_timer_list);
9cfb9b3c	330	} else {
27f6b416 MS	331	/* Update timer against current base. */
	332	timer->expires += atomic64_read(&virt_timer_elapsed);
	333	if (likely((s64) timer->expires <
	334	(s64) atomic64_read(&virt_timer_current)))
9cfb9b3c	335	/* The new timer expires before the current timer. */
27f6b416 MS	336	atomic64_set(&virt_timer_current, timer->expires);
	337	/* Insert new timer into the list. */
	338	list_add_sorted(timer, &virt_timer_list);
1da177e4	339	}
1da177e4 LT	340	}
1da177e4 LT	341
27f6b416	342	static void __add_vtimer(struct vtimer_list *timer, int periodic)
1da177e4	343	{
27f6b416 MS	344	unsigned long flags;
	345
	346	timer->interval = periodic ? timer->expires : 0;
	347	spin_lock_irqsave(&virt_timer_lock, flags);
	348	internal_add_vtimer(timer);
	349	spin_unlock_irqrestore(&virt_timer_lock, flags);
1da177e4 LT	350	}
	351
	352	/*
0f5e1558	353	* add_virt_timer - add a oneshot virtual CPU timer
1da177e4	354	*/
27f6b416	355	void add_virt_timer(struct vtimer_list *timer)
1da177e4	356	{
27f6b416	357	__add_vtimer(timer, 0);
1da177e4 LT	358	}
	359	EXPORT_SYMBOL(add_virt_timer);
	360
	361	/*
	362	* add_virt_timer_int - add an interval virtual CPU timer
	363	*/
27f6b416	364	void add_virt_timer_periodic(struct vtimer_list *timer)
1da177e4	365	{
27f6b416	366	__add_vtimer(timer, 1);
1da177e4 LT	367	}
	368	EXPORT_SYMBOL(add_virt_timer_periodic);
	369
27f6b416	370	static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
1da177e4	371	{
1da177e4	372	unsigned long flags;
27f6b416	373	int rc;
1da177e4	374
ca366a32	375	BUG_ON(!timer->function);
1da177e4	376
1da177e4 LT	377	if (timer->expires == expires && vtimer_pending(timer))
1da177e4 LT	378	return 1;
27f6b416 MS	379	spin_lock_irqsave(&virt_timer_lock, flags);
	380	rc = vtimer_pending(timer);
	381	if (rc)
	382	list_del_init(&timer->entry);
	383	timer->interval = periodic ? expires : 0;
1da177e4	384	timer->expires = expires;
1da177e4	385	internal_add_vtimer(timer);
27f6b416 MS	386	spin_unlock_irqrestore(&virt_timer_lock, flags);
27f6b416 MS	387	return rc;
1da177e4	388	}
b6ecfa92 JG	389
b6ecfa92 JG	390	/*
b6ecfa92 JG	391	* returns whether it has modified a pending timer (1) or not (0)
b6ecfa92 JG	392	*/
27f6b416	393	int mod_virt_timer(struct vtimer_list *timer, u64 expires)
b6ecfa92 JG	394	{
	395	return __mod_vtimer(timer, expires, 0);
	396	}
1da177e4 LT	397	EXPORT_SYMBOL(mod_virt_timer);
1da177e4 LT	398
b6ecfa92	399	/*
b6ecfa92 JG	400	* returns whether it has modified a pending timer (1) or not (0)
b6ecfa92 JG	401	*/
27f6b416	402	int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
b6ecfa92 JG	403	{
	404	return __mod_vtimer(timer, expires, 1);
	405	}
	406	EXPORT_SYMBOL(mod_virt_timer_periodic);
	407
1da177e4	408	/*
27f6b416	409	* Delete a virtual timer.
1da177e4 LT	410	*
	411	* returns whether the deleted timer was pending (1) or not (0)
	412	*/
	413	int del_virt_timer(struct vtimer_list *timer)
	414	{
	415	unsigned long flags;
1da177e4	416
1da177e4 LT	417	if (!vtimer_pending(timer))
1da177e4 LT	418	return 0;
27f6b416	419	spin_lock_irqsave(&virt_timer_lock, flags);
1da177e4	420	list_del_init(&timer->entry);
27f6b416	421	spin_unlock_irqrestore(&virt_timer_lock, flags);
1da177e4 LT	422	return 1;
	423	}
	424	EXPORT_SYMBOL(del_virt_timer);
	425
	426	/*
	427	* Start the virtual CPU timer on the current CPU.
	428	*/
b5f87f15	429	void vtime_init(void)
1da177e4	430	{
8b646bd7	431	/* set initial cpu timer */
27f6b416	432	set_vtimer(VTIMER_MAX_SLICE);
f341b8df MS	433	/* Setup initial MT scaling values */
	434	if (smp_cpu_mtid) {
	435	__this_cpu_write(mt_scaling_jiffies, jiffies);
	436	__this_cpu_write(mt_scaling_mult, 1);
	437	__this_cpu_write(mt_scaling_div, 1);
	438	stcctm5(smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
	439	}
1da177e4	440	}