[thirdparty/linux.git] / lib / vdso / gettimeofday.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Generic userspace implementations of gettimeofday() and similar.
 */
#include <vdso/datapage.h>
#include <vdso/helpers.h>

#ifndef vdso_calc_delta
/*
 * Default implementation which works for all sane clocksources. That
 * obviously excludes x86/TSC.
 */
static __always_inline
u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
{
	return ((cycles - last) & mask) * mult;
}
#endif

#ifndef vdso_shift_ns
static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
{
	return ns >> shift;
}
#endif

#ifndef __arch_vdso_hres_capable
static inline bool __arch_vdso_hres_capable(void)
{
	return true;
}
#endif

#ifndef vdso_clocksource_ok
static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
{
	return vd->clock_mode != VDSO_CLOCKMODE_NONE;
}
#endif

#ifndef vdso_cycles_ok
static inline bool vdso_cycles_ok(u64 cycles)
{
	return true;
}
#endif

#ifdef CONFIG_TIME_NS
static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
					  struct __kernel_timespec *ts)
{
	const struct vdso_data *vd;
	const struct timens_offset *offs = &vdns->offset[clk];
	const struct vdso_timestamp *vdso_ts;
	u64 cycles, last, ns;
	u32 seq;
	s64 sec;

	vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);
	vd = __arch_get_timens_vdso_data(vd);
	if (clk != CLOCK_MONOTONIC_RAW)
		vd = &vd[CS_HRES_COARSE];
	else
		vd = &vd[CS_RAW];
	vdso_ts = &vd->basetime[clk];

	do {
		seq = vdso_read_begin(vd);

		if (unlikely(!vdso_clocksource_ok(vd)))
			return -1;

		cycles = __arch_get_hw_counter(vd->clock_mode, vd);
		if (unlikely(!vdso_cycles_ok(cycles)))
			return -1;
		ns = vdso_ts->nsec;
		last = vd->cycle_last;
		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
		ns = vdso_shift_ns(ns, vd->shift);
		sec = vdso_ts->sec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/* Add the namespace offset */
	sec += offs->sec;
	ns += offs->nsec;

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return 0;
}
#else
static __always_inline
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
{
	return NULL;
}

static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
					  struct __kernel_timespec *ts)
{
	return -EINVAL;
}
#endif

static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
				   struct __kernel_timespec *ts)
{
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	u64 cycles, last, sec, ns;
	u32 seq;

	/* Allows to compile the high resolution parts out */
	if (!__arch_vdso_hres_capable())
		return -1;

	do {
		/*
		 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
		 * enabled tasks have a special VVAR page installed which
		 * has vd->seq set to 1 and vd->clock_mode set to
		 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
		 * this does not affect performance because if vd->seq is
		 * odd, i.e. a concurrent update is in progress the extra
		 * check for vd->clock_mode is just a few extra
		 * instructions while spin waiting for vd->seq to become
		 * even again.
		 */
		while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
			if (IS_ENABLED(CONFIG_TIME_NS) &&
			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
				return do_hres_timens(vd, clk, ts);
			cpu_relax();
		}
		smp_rmb();

		if (unlikely(!vdso_clocksource_ok(vd)))
			return -1;

		cycles = __arch_get_hw_counter(vd->clock_mode, vd);
		if (unlikely(!vdso_cycles_ok(cycles)))
			return -1;
		ns = vdso_ts->nsec;
		last = vd->cycle_last;
		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
		ns = vdso_shift_ns(ns, vd->shift);
		sec = vdso_ts->sec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return 0;
}

#ifdef CONFIG_TIME_NS
static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
					    struct __kernel_timespec *ts)
{
	const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns);
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	const struct timens_offset *offs = &vdns->offset[clk];
	u64 nsec;
	s64 sec;
	s32 seq;

	do {
		seq = vdso_read_begin(vd);
		sec = vdso_ts->sec;
		nsec = vdso_ts->nsec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/* Add the namespace offset */
	sec += offs->sec;
	nsec += offs->nsec;

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
	ts->tv_nsec = nsec;
	return 0;
}
#else
static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
					    struct __kernel_timespec *ts)
{
	return -1;
}
#endif

static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
				     struct __kernel_timespec *ts)
{
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	u32 seq;

	do {
		/*
		 * Open coded to handle VDSO_CLOCK_TIMENS. See comment in
		 * do_hres().
		 */
		while ((seq = READ_ONCE(vd->seq)) & 1) {
			if (IS_ENABLED(CONFIG_TIME_NS) &&
			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
				return do_coarse_timens(vd, clk, ts);
			cpu_relax();
		}
		smp_rmb();

		ts->tv_sec = vdso_ts->sec;
		ts->tv_nsec = vdso_ts->nsec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	return 0;
}

static __always_inline int
__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
			     struct __kernel_timespec *ts)
{
	u32 msk;

	/* Check for negative values or invalid clocks */
	if (unlikely((u32) clock >= MAX_CLOCKS))
		return -1;

	/*
	 * Convert the clockid to a bitmask and use it to check which
	 * clocks are handled in the VDSO directly.
	 */
	msk = 1U << clock;
	if (likely(msk & VDSO_HRES))
		vd = &vd[CS_HRES_COARSE];
	else if (msk & VDSO_COARSE)
		return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
	else if (msk & VDSO_RAW)
		vd = &vd[CS_RAW];
	else
		return -1;

	return do_hres(vd, clock, ts);
}

static __maybe_unused int
__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
			   struct __kernel_timespec *ts)
{
	int ret = __cvdso_clock_gettime_common(vd, clock, ts);

	if (unlikely(ret))
		return clock_gettime_fallback(clock, ts);
	return 0;
}

static __maybe_unused int
__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
{
	return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
}

#ifdef BUILD_VDSO32
static __maybe_unused int
__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
			     struct old_timespec32 *res)
{
	struct __kernel_timespec ts;
	int ret;

	ret = __cvdso_clock_gettime_common(vd, clock, &ts);

	if (unlikely(ret))
		return clock_gettime32_fallback(clock, res);

	/* For ret == 0 */
	res->tv_sec = ts.tv_sec;
	res->tv_nsec = ts.tv_nsec;

	return ret;
}

static __maybe_unused int
__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
{
	return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
}
#endif /* BUILD_VDSO32 */

static __maybe_unused int
__cvdso_gettimeofday_data(const struct vdso_data *vd,
			  struct __kernel_old_timeval *tv, struct timezone *tz)
{

	if (likely(tv != NULL)) {
		struct __kernel_timespec ts;

		if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
			return gettimeofday_fallback(tv, tz);

		tv->tv_sec = ts.tv_sec;
		tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
	}

	if (unlikely(tz != NULL)) {
		if (IS_ENABLED(CONFIG_TIME_NS) &&
		    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
			vd = __arch_get_timens_vdso_data(vd);

		tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
		tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
	}

	return 0;
}

static __maybe_unused int
__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
	return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
}

#ifdef VDSO_HAS_TIME
static __maybe_unused __kernel_old_time_t
__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)
{
	__kernel_old_time_t t;

	if (IS_ENABLED(CONFIG_TIME_NS) &&
	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
		vd = __arch_get_timens_vdso_data(vd);

	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);

	if (time)
		*time = t;

	return t;
}

static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
{
	return __cvdso_time_data(__arch_get_vdso_data(), time);
}
#endif /* VDSO_HAS_TIME */

#ifdef VDSO_HAS_CLOCK_GETRES
static __maybe_unused
int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
				struct __kernel_timespec *res)
{
	u32 msk;
	u64 ns;

	/* Check for negative values or invalid clocks */
	if (unlikely((u32) clock >= MAX_CLOCKS))
		return -1;

	if (IS_ENABLED(CONFIG_TIME_NS) &&
	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
		vd = __arch_get_timens_vdso_data(vd);

	/*
	 * Convert the clockid to a bitmask and use it to check which
	 * clocks are handled in the VDSO directly.
	 */
	msk = 1U << clock;
	if (msk & (VDSO_HRES | VDSO_RAW)) {
		/*
		 * Preserves the behaviour of posix_get_hrtimer_res().
		 */
		ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
	} else if (msk & VDSO_COARSE) {
		/*
		 * Preserves the behaviour of posix_get_coarse_res().
		 */
		ns = LOW_RES_NSEC;
	} else {
		return -1;
	}

	if (likely(res)) {
		res->tv_sec = 0;
		res->tv_nsec = ns;
	}
	return 0;
}

static __maybe_unused
int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
			      struct __kernel_timespec *res)
{
	int ret = __cvdso_clock_getres_common(vd, clock, res);

	if (unlikely(ret))
		return clock_getres_fallback(clock, res);
	return 0;
}

static __maybe_unused
int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
{
	return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
}

#ifdef BUILD_VDSO32
static __maybe_unused int
__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
				 struct old_timespec32 *res)
{
	struct __kernel_timespec ts;
	int ret;

	ret = __cvdso_clock_getres_common(vd, clock, &ts);

	if (unlikely(ret))
		return clock_getres32_fallback(clock, res);

	if (likely(res)) {
		res->tv_sec = ts.tv_sec;
		res->tv_nsec = ts.tv_nsec;
	}
	return ret;
}

static __maybe_unused int
__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
{
	return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
						clock, res);
}
#endif /* BUILD_VDSO32 */
#endif /* VDSO_HAS_CLOCK_GETRES */
Commit	Line	Data
00b26474 VF	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Generic userspace implementations of gettimeofday() and similar.
	4	*/
00b26474 VF	5	#include <vdso/datapage.h>
	6	#include <vdso/helpers.h>
	7
9d90b93b TG	8	#ifndef vdso_calc_delta
	9	/*
	10	* Default implementation which works for all sane clocksources. That
	11	* obviously excludes x86/TSC.
	12	*/
	13	static __always_inline
	14	u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
	15	{
	16	return ((cycles - last) & mask) * mult;
	17	}
	18	#endif
	19
8345228c CL	20	#ifndef vdso_shift_ns
	21	static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
	22	{
	23	return ns >> shift;
	24	}
	25	#endif
	26
1dff4156 TG	27	#ifndef __arch_vdso_hres_capable
	28	static inline bool __arch_vdso_hres_capable(void)
	29	{
	30	return true;
	31	}
	32	#endif
	33
ae12e085 CL	34	#ifndef vdso_clocksource_ok
	35	static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
	36	{
	37	return vd->clock_mode != VDSO_CLOCKMODE_NONE;
	38	}
	39	#endif
	40
72ce7780 TG	41	#ifndef vdso_cycles_ok
	42	static inline bool vdso_cycles_ok(u64 cycles)
	43	{
	44	return true;
	45	}
	46	#endif
	47
660fd04f	48	#ifdef CONFIG_TIME_NS
58efe9f6 CL	49	static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
58efe9f6 CL	50	struct __kernel_timespec *ts)
660fd04f	51	{
808094fc	52	const struct vdso_data *vd;
660fd04f TG	53	const struct timens_offset *offs = &vdns->offset[clk];
	54	const struct vdso_timestamp *vdso_ts;
	55	u64 cycles, last, ns;
	56	u32 seq;
	57	s64 sec;
	58
808094fc CL	59	vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);
808094fc CL	60	vd = __arch_get_timens_vdso_data(vd);
660fd04f TG	61	if (clk != CLOCK_MONOTONIC_RAW)
	62	vd = &vd[CS_HRES_COARSE];
	63	else
	64	vd = &vd[CS_RAW];
	65	vdso_ts = &vd->basetime[clk];
	66
	67	do {
	68	seq = vdso_read_begin(vd);
f86fd32d	69
ae12e085	70	if (unlikely(!vdso_clocksource_ok(vd)))
5d51bee7	71	return -1;
f86fd32d	72
4c5a116a	73	cycles = __arch_get_hw_counter(vd->clock_mode, vd);
72ce7780 TG	74	if (unlikely(!vdso_cycles_ok(cycles)))
72ce7780 TG	75	return -1;
660fd04f TG	76	ns = vdso_ts->nsec;
660fd04f TG	77	last = vd->cycle_last;
660fd04f	78	ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
8345228c	79	ns = vdso_shift_ns(ns, vd->shift);
660fd04f TG	80	sec = vdso_ts->sec;
	81	} while (unlikely(vdso_read_retry(vd, seq)));
	82
	83	/* Add the namespace offset */
	84	sec += offs->sec;
	85	ns += offs->nsec;
	86
	87	/*
	88	* Do this outside the loop: a race inside the loop could result
	89	* in __iter_div_u64_rem() being extremely slow.
	90	*/
	91	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	92	ts->tv_nsec = ns;
	93
	94	return 0;
	95	}
	96	#else
808094fc CL	97	static __always_inline
808094fc CL	98	const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
660fd04f TG	99	{
	100	return NULL;
	101	}
	102
58efe9f6 CL	103	static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
58efe9f6 CL	104	struct __kernel_timespec *ts)
660fd04f TG	105	{
	106	return -EINVAL;
	107	}
	108	#endif
	109
c966533f	110	static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
660fd04f	111	struct __kernel_timespec *ts)
00b26474 VF	112	{
	113	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	114	u64 cycles, last, sec, ns;
	115	u32 seq;
	116
1dff4156 TG	117	/* Allows to compile the high resolution parts out */
	118	if (!__arch_vdso_hres_capable())
	119	return -1;
	120
00b26474	121	do {
660fd04f	122	/*
2d6b01bd	123	* Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
660fd04f TG	124	* enabled tasks have a special VVAR page installed which
660fd04f TG	125	* has vd->seq set to 1 and vd->clock_mode set to
2d6b01bd	126	* VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
660fd04f TG	127	* this does not affect performance because if vd->seq is
	128	* odd, i.e. a concurrent update is in progress the extra
	129	* check for vd->clock_mode is just a few extra
	130	* instructions while spin waiting for vd->seq to become
	131	* even again.
	132	*/
	133	while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
	134	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd	135	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	136	return do_hres_timens(vd, clk, ts);
	137	cpu_relax();
	138	}
	139	smp_rmb();
	140
ae12e085	141	if (unlikely(!vdso_clocksource_ok(vd)))
5d51bee7	142	return -1;
f86fd32d	143
4c5a116a	144	cycles = __arch_get_hw_counter(vd->clock_mode, vd);
72ce7780 TG	145	if (unlikely(!vdso_cycles_ok(cycles)))
72ce7780 TG	146	return -1;
00b26474 VF	147	ns = vdso_ts->nsec;
00b26474 VF	148	last = vd->cycle_last;
9d90b93b	149	ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
8345228c	150	ns = vdso_shift_ns(ns, vd->shift);
00b26474 VF	151	sec = vdso_ts->sec;
	152	} while (unlikely(vdso_read_retry(vd, seq)));
	153
	154	/*
	155	* Do this outside the loop: a race inside the loop could result
	156	* in __iter_div_u64_rem() being extremely slow.
	157	*/
	158	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	159	ts->tv_nsec = ns;
	160
	161	return 0;
	162	}
	163
660fd04f	164	#ifdef CONFIG_TIME_NS
58efe9f6 CL	165	static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
58efe9f6 CL	166	struct __kernel_timespec *ts)
660fd04f	167	{
808094fc	168	const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns);
660fd04f TG	169	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	170	const struct timens_offset *offs = &vdns->offset[clk];
	171	u64 nsec;
	172	s64 sec;
	173	s32 seq;
	174
	175	do {
	176	seq = vdso_read_begin(vd);
	177	sec = vdso_ts->sec;
	178	nsec = vdso_ts->nsec;
	179	} while (unlikely(vdso_read_retry(vd, seq)));
	180
	181	/* Add the namespace offset */
	182	sec += offs->sec;
	183	nsec += offs->nsec;
	184
	185	/*
	186	* Do this outside the loop: a race inside the loop could result
	187	* in __iter_div_u64_rem() being extremely slow.
	188	*/
	189	ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
	190	ts->tv_nsec = nsec;
	191	return 0;
	192	}
	193	#else
58efe9f6 CL	194	static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
58efe9f6 CL	195	struct __kernel_timespec *ts)
660fd04f TG	196	{
	197	return -1;
	198	}
	199	#endif
	200
c966533f AV	201	static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
c966533f AV	202	struct __kernel_timespec *ts)
00b26474 VF	203	{
	204	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	205	u32 seq;
	206
	207	do {
660fd04f	208	/*
2d6b01bd	209	* Open coded to handle VDSO_CLOCK_TIMENS. See comment in
660fd04f TG	210	* do_hres().
	211	*/
	212	while ((seq = READ_ONCE(vd->seq)) & 1) {
	213	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd	214	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	215	return do_coarse_timens(vd, clk, ts);
	216	cpu_relax();
	217	}
	218	smp_rmb();
	219
00b26474 VF	220	ts->tv_sec = vdso_ts->sec;
	221	ts->tv_nsec = vdso_ts->nsec;
	222	} while (unlikely(vdso_read_retry(vd, seq)));
8463cf80 CL	223
8463cf80 CL	224	return 0;
00b26474 VF	225	}
00b26474 VF	226
b91c8c42	227	static __always_inline int
e876f0b6 CL	228	__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	229	struct __kernel_timespec *ts)
00b26474	230	{
00b26474 VF	231	u32 msk;
	232
	233	/* Check for negative values or invalid clocks */
	234	if (unlikely((u32) clock >= MAX_CLOCKS))
502a590a	235	return -1;
00b26474 VF	236
	237	/*
	238	* Convert the clockid to a bitmask and use it to check which
	239	* clocks are handled in the VDSO directly.
	240	*/
	241	msk = 1U << clock;
8463cf80	242	if (likely(msk & VDSO_HRES))
c966533f	243	vd = &vd[CS_HRES_COARSE];
8463cf80 CL	244	else if (msk & VDSO_COARSE)
	245	return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
	246	else if (msk & VDSO_RAW)
c966533f AV	247	vd = &vd[CS_RAW];
	248	else
	249	return -1;
8463cf80	250
c966533f	251	return do_hres(vd, clock, ts);
502a590a	252	}
00b26474	253
502a590a	254	static __maybe_unused int
e876f0b6 CL	255	__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	256	struct __kernel_timespec *ts)
502a590a	257	{
e876f0b6	258	int ret = __cvdso_clock_gettime_common(vd, clock, ts);
502a590a TG	259
	260	if (unlikely(ret))
	261	return clock_gettime_fallback(clock, ts);
	262	return 0;
00b26474 VF	263	}
00b26474 VF	264
e876f0b6 CL	265	static __maybe_unused int
	266	__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
	267	{
	268	return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
	269	}
	270
bf279849	271	#ifdef BUILD_VDSO32
00b26474	272	static __maybe_unused int
e876f0b6 CL	273	__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	274	struct old_timespec32 *res)
00b26474 VF	275	{
	276	struct __kernel_timespec ts;
	277	int ret;
	278
e876f0b6	279	ret = __cvdso_clock_gettime_common(vd, clock, &ts);
00b26474	280
c60a32ea TG	281	if (unlikely(ret))
c60a32ea TG	282	return clock_gettime32_fallback(clock, res);
502a590a	283
a279235d VF	284	/* For ret == 0 */
	285	res->tv_sec = ts.tv_sec;
	286	res->tv_nsec = ts.tv_nsec;
	287
00b26474	288	return ret;
00b26474	289	}
e876f0b6 CL	290
	291	static __maybe_unused int
	292	__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
	293	{
	294	return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
	295	}
bf279849	296	#endif /* BUILD_VDSO32 */
00b26474 VF	297
00b26474 VF	298	static __maybe_unused int
e876f0b6 CL	299	__cvdso_gettimeofday_data(const struct vdso_data *vd,
e876f0b6 CL	300	struct __kernel_old_timeval tv, struct timezone tz)
00b26474	301	{
00b26474 VF	302
	303	if (likely(tv != NULL)) {
	304	struct __kernel_timespec ts;
	305
	306	if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
	307	return gettimeofday_fallback(tv, tz);
	308
	309	tv->tv_sec = ts.tv_sec;
	310	tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
	311	}
	312
	313	if (unlikely(tz != NULL)) {
660fd04f	314	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd	315	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
808094fc	316	vd = __arch_get_timens_vdso_data(vd);
660fd04f	317
00b26474 VF	318	tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
	319	tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
	320	}
	321
	322	return 0;
	323	}
	324
e876f0b6 CL	325	static __maybe_unused int
	326	__cvdso_gettimeofday(struct __kernel_old_timeval tv, struct timezone tz)
	327	{
	328	return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
	329	}
	330
00b26474	331	#ifdef VDSO_HAS_TIME
e876f0b6 CL	332	static __maybe_unused __kernel_old_time_t
e876f0b6 CL	333	__cvdso_time_data(const struct vdso_data vd, __kernel_old_time_t time)
00b26474	334	{
660fd04f TG	335	__kernel_old_time_t t;
660fd04f TG	336
2d6b01bd TG	337	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd TG	338	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
808094fc	339	vd = __arch_get_timens_vdso_data(vd);
660fd04f TG	340
660fd04f TG	341	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
00b26474 VF	342
	343	if (time)
	344	*time = t;
	345
	346	return t;
	347	}
e876f0b6 CL	348
	349	static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
	350	{
	351	return __cvdso_time_data(__arch_get_vdso_data(), time);
	352	}
00b26474 VF	353	#endif /* VDSO_HAS_TIME */
	354
	355	#ifdef VDSO_HAS_CLOCK_GETRES
	356	static __maybe_unused
e876f0b6 CL	357	int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	358	struct __kernel_timespec *res)
00b26474	359	{
00b26474	360	u32 msk;
502a590a	361	u64 ns;
00b26474 VF	362
	363	/* Check for negative values or invalid clocks */
	364	if (unlikely((u32) clock >= MAX_CLOCKS))
502a590a	365	return -1;
00b26474	366
2d6b01bd TG	367	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd TG	368	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
808094fc	369	vd = __arch_get_timens_vdso_data(vd);
660fd04f	370
00b26474 VF	371	/*
	372	* Convert the clockid to a bitmask and use it to check which
	373	* clocks are handled in the VDSO directly.
	374	*/
	375	msk = 1U << clock;
cdb7c5a9	376	if (msk & (VDSO_HRES \| VDSO_RAW)) {
00b26474 VF	377	/*
	378	* Preserves the behaviour of posix_get_hrtimer_res().
	379	*/
49a101d7	380	ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
00b26474 VF	381	} else if (msk & VDSO_COARSE) {
	382	/*
	383	* Preserves the behaviour of posix_get_coarse_res().
	384	*/
	385	ns = LOW_RES_NSEC;
00b26474	386	} else {
502a590a	387	return -1;
00b26474 VF	388	}
00b26474 VF	389
1638b8f0 TG	390	if (likely(res)) {
	391	res->tv_sec = 0;
	392	res->tv_nsec = ns;
	393	}
00b26474	394	return 0;
502a590a TG	395	}
502a590a TG	396
ffd08731	397	static __maybe_unused
e876f0b6 CL	398	int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	399	struct __kernel_timespec *res)
502a590a	400	{
e876f0b6	401	int ret = __cvdso_clock_getres_common(vd, clock, res);
00b26474	402
502a590a TG	403	if (unlikely(ret))
	404	return clock_getres_fallback(clock, res);
	405	return 0;
00b26474 VF	406	}
00b26474 VF	407
e876f0b6 CL	408	static __maybe_unused
	409	int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
	410	{
	411	return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
	412	}
	413
bf279849	414	#ifdef BUILD_VDSO32
00b26474	415	static __maybe_unused int
e876f0b6 CL	416	__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	417	struct old_timespec32 *res)
00b26474 VF	418	{
	419	struct __kernel_timespec ts;
	420	int ret;
	421
e876f0b6	422	ret = __cvdso_clock_getres_common(vd, clock, &ts);
c60a32ea	423
c60a32ea TG	424	if (unlikely(ret))
c60a32ea TG	425	return clock_getres32_fallback(clock, res);
00b26474	426
a279235d	427	if (likely(res)) {
00b26474 VF	428	res->tv_sec = ts.tv_sec;
	429	res->tv_nsec = ts.tv_nsec;
	430	}
00b26474	431	return ret;
00b26474	432	}
e876f0b6 CL	433
	434	static __maybe_unused int
	435	__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
	436	{
	437	return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
	438	clock, res);
	439	}
bf279849	440	#endif /* BUILD_VDSO32 */
00b26474	441	#endif /* VDSO_HAS_CLOCK_GETRES */