[thirdparty/linux.git] / lib / vdso / gettimeofday.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Generic userspace implementations of gettimeofday() and similar.
 */
#include <vdso/datapage.h>
#include <vdso/helpers.h>

#ifndef vdso_calc_delta
/*
 * Default implementation which works for all sane clocksources. That
 * obviously excludes x86/TSC.
 */
static __always_inline
u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
{
	return ((cycles - last) & mask) * mult;
}
#endif

#ifndef vdso_shift_ns
static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
{
	return ns >> shift;
}
#endif

#ifndef __arch_vdso_hres_capable
static inline bool __arch_vdso_hres_capable(void)
{
	return true;
}
#endif

#ifndef vdso_clocksource_ok
static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
{
	return vd->clock_mode != VDSO_CLOCKMODE_NONE;
}
#endif

#ifdef CONFIG_TIME_NS
static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
			  struct __kernel_timespec *ts)
{
	const struct vdso_data *vd = __arch_get_timens_vdso_data();
	const struct timens_offset *offs = &vdns->offset[clk];
	const struct vdso_timestamp *vdso_ts;
	u64 cycles, last, ns;
	u32 seq;
	s64 sec;

	if (clk != CLOCK_MONOTONIC_RAW)
		vd = &vd[CS_HRES_COARSE];
	else
		vd = &vd[CS_RAW];
	vdso_ts = &vd->basetime[clk];

	do {
		seq = vdso_read_begin(vd);

		if (unlikely(!vdso_clocksource_ok(vd)))
			return -1;

		cycles = __arch_get_hw_counter(vd->clock_mode);
		ns = vdso_ts->nsec;
		last = vd->cycle_last;
		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
		ns = vdso_shift_ns(ns, vd->shift);
		sec = vdso_ts->sec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/* Add the namespace offset */
	sec += offs->sec;
	ns += offs->nsec;

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return 0;
}
#else
static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
{
	return NULL;
}

static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
			  struct __kernel_timespec *ts)
{
	return -EINVAL;
}
#endif

static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
				   struct __kernel_timespec *ts)
{
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	u64 cycles, last, sec, ns;
	u32 seq;

	/* Allows to compile the high resolution parts out */
	if (!__arch_vdso_hres_capable())
		return -1;

	do {
		/*
		 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
		 * enabled tasks have a special VVAR page installed which
		 * has vd->seq set to 1 and vd->clock_mode set to
		 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
		 * this does not affect performance because if vd->seq is
		 * odd, i.e. a concurrent update is in progress the extra
		 * check for vd->clock_mode is just a few extra
		 * instructions while spin waiting for vd->seq to become
		 * even again.
		 */
		while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
			if (IS_ENABLED(CONFIG_TIME_NS) &&
			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
				return do_hres_timens(vd, clk, ts);
			cpu_relax();
		}
		smp_rmb();

		if (unlikely(!vdso_clocksource_ok(vd)))
			return -1;

		cycles = __arch_get_hw_counter(vd->clock_mode);
		ns = vdso_ts->nsec;
		last = vd->cycle_last;
		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
		ns = vdso_shift_ns(ns, vd->shift);
		sec = vdso_ts->sec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return 0;
}

#ifdef CONFIG_TIME_NS
static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
			    struct __kernel_timespec *ts)
{
	const struct vdso_data *vd = __arch_get_timens_vdso_data();
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	const struct timens_offset *offs = &vdns->offset[clk];
	u64 nsec;
	s64 sec;
	s32 seq;

	do {
		seq = vdso_read_begin(vd);
		sec = vdso_ts->sec;
		nsec = vdso_ts->nsec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/* Add the namespace offset */
	sec += offs->sec;
	nsec += offs->nsec;

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
	ts->tv_nsec = nsec;
	return 0;
}
#else
static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
			    struct __kernel_timespec *ts)
{
	return -1;
}
#endif

static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
				     struct __kernel_timespec *ts)
{
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	u32 seq;

	do {
		/*
		 * Open coded to handle VDSO_CLOCK_TIMENS. See comment in
		 * do_hres().
		 */
		while ((seq = READ_ONCE(vd->seq)) & 1) {
			if (IS_ENABLED(CONFIG_TIME_NS) &&
			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
				return do_coarse_timens(vd, clk, ts);
			cpu_relax();
		}
		smp_rmb();

		ts->tv_sec = vdso_ts->sec;
		ts->tv_nsec = vdso_ts->nsec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	return 0;
}

static __maybe_unused int
__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
			     struct __kernel_timespec *ts)
{
	u32 msk;

	/* Check for negative values or invalid clocks */
	if (unlikely((u32) clock >= MAX_CLOCKS))
		return -1;

	/*
	 * Convert the clockid to a bitmask and use it to check which
	 * clocks are handled in the VDSO directly.
	 */
	msk = 1U << clock;
	if (likely(msk & VDSO_HRES))
		vd = &vd[CS_HRES_COARSE];
	else if (msk & VDSO_COARSE)
		return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
	else if (msk & VDSO_RAW)
		vd = &vd[CS_RAW];
	else
		return -1;

	return do_hres(vd, clock, ts);
}

static __maybe_unused int
__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
			   struct __kernel_timespec *ts)
{
	int ret = __cvdso_clock_gettime_common(vd, clock, ts);

	if (unlikely(ret))
		return clock_gettime_fallback(clock, ts);
	return 0;
}

static __maybe_unused int
__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
{
	return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
}

#ifdef BUILD_VDSO32
static __maybe_unused int
__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
			     struct old_timespec32 *res)
{
	struct __kernel_timespec ts;
	int ret;

	ret = __cvdso_clock_gettime_common(vd, clock, &ts);

	if (unlikely(ret))
		return clock_gettime32_fallback(clock, res);

	/* For ret == 0 */
	res->tv_sec = ts.tv_sec;
	res->tv_nsec = ts.tv_nsec;

	return ret;
}

static __maybe_unused int
__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
{
	return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
}
#endif /* BUILD_VDSO32 */

static __maybe_unused int
__cvdso_gettimeofday_data(const struct vdso_data *vd,
			  struct __kernel_old_timeval *tv, struct timezone *tz)
{

	if (likely(tv != NULL)) {
		struct __kernel_timespec ts;

		if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
			return gettimeofday_fallback(tv, tz);

		tv->tv_sec = ts.tv_sec;
		tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
	}

	if (unlikely(tz != NULL)) {
		if (IS_ENABLED(CONFIG_TIME_NS) &&
		    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
			vd = __arch_get_timens_vdso_data();

		tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
		tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
	}

	return 0;
}

static __maybe_unused int
__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
	return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
}

#ifdef VDSO_HAS_TIME
static __maybe_unused __kernel_old_time_t
__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)
{
	__kernel_old_time_t t;

	if (IS_ENABLED(CONFIG_TIME_NS) &&
	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
		vd = __arch_get_timens_vdso_data();

	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);

	if (time)
		*time = t;

	return t;
}

static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
{
	return __cvdso_time_data(__arch_get_vdso_data(), time);
}
#endif /* VDSO_HAS_TIME */

#ifdef VDSO_HAS_CLOCK_GETRES
static __maybe_unused
int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
				struct __kernel_timespec *res)
{
	u32 msk;
	u64 ns;

	/* Check for negative values or invalid clocks */
	if (unlikely((u32) clock >= MAX_CLOCKS))
		return -1;

	if (IS_ENABLED(CONFIG_TIME_NS) &&
	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
		vd = __arch_get_timens_vdso_data();

	/*
	 * Convert the clockid to a bitmask and use it to check which
	 * clocks are handled in the VDSO directly.
	 */
	msk = 1U << clock;
	if (msk & (VDSO_HRES | VDSO_RAW)) {
		/*
		 * Preserves the behaviour of posix_get_hrtimer_res().
		 */
		ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
	} else if (msk & VDSO_COARSE) {
		/*
		 * Preserves the behaviour of posix_get_coarse_res().
		 */
		ns = LOW_RES_NSEC;
	} else {
		return -1;
	}

	if (likely(res)) {
		res->tv_sec = 0;
		res->tv_nsec = ns;
	}
	return 0;
}

static __maybe_unused
int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
			      struct __kernel_timespec *res)
{
	int ret = __cvdso_clock_getres_common(vd, clock, res);

	if (unlikely(ret))
		return clock_getres_fallback(clock, res);
	return 0;
}

static __maybe_unused
int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
{
	return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
}

#ifdef BUILD_VDSO32
static __maybe_unused int
__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
				 struct old_timespec32 *res)
{
	struct __kernel_timespec ts;
	int ret;

	ret = __cvdso_clock_getres_common(vd, clock, &ts);

	if (unlikely(ret))
		return clock_getres32_fallback(clock, res);

	if (likely(res)) {
		res->tv_sec = ts.tv_sec;
		res->tv_nsec = ts.tv_nsec;
	}
	return ret;
}

static __maybe_unused int
__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
{
	return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
						clock, res);
}
#endif /* BUILD_VDSO32 */
#endif /* VDSO_HAS_CLOCK_GETRES */
Commit	Line	Data
00b26474 VF	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Generic userspace implementations of gettimeofday() and similar.
	4	*/
00b26474 VF	5	#include <vdso/datapage.h>
	6	#include <vdso/helpers.h>
	7
9d90b93b TG	8	#ifndef vdso_calc_delta
	9	/*
	10	* Default implementation which works for all sane clocksources. That
	11	* obviously excludes x86/TSC.
	12	*/
	13	static __always_inline
	14	u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
	15	{
	16	return ((cycles - last) & mask) * mult;
	17	}
	18	#endif
	19
8345228c CL	20	#ifndef vdso_shift_ns
	21	static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
	22	{
	23	return ns >> shift;
	24	}
	25	#endif
	26
1dff4156 TG	27	#ifndef __arch_vdso_hres_capable
	28	static inline bool __arch_vdso_hres_capable(void)
	29	{
	30	return true;
	31	}
	32	#endif
	33
ae12e085 CL	34	#ifndef vdso_clocksource_ok
	35	static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
	36	{
	37	return vd->clock_mode != VDSO_CLOCKMODE_NONE;
	38	}
	39	#endif
	40
660fd04f TG	41	#ifdef CONFIG_TIME_NS
	42	static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
	43	struct __kernel_timespec *ts)
	44	{
	45	const struct vdso_data *vd = __arch_get_timens_vdso_data();
	46	const struct timens_offset *offs = &vdns->offset[clk];
	47	const struct vdso_timestamp *vdso_ts;
	48	u64 cycles, last, ns;
	49	u32 seq;
	50	s64 sec;
	51
	52	if (clk != CLOCK_MONOTONIC_RAW)
	53	vd = &vd[CS_HRES_COARSE];
	54	else
	55	vd = &vd[CS_RAW];
	56	vdso_ts = &vd->basetime[clk];
	57
	58	do {
	59	seq = vdso_read_begin(vd);
f86fd32d	60
ae12e085	61	if (unlikely(!vdso_clocksource_ok(vd)))
5d51bee7	62	return -1;
f86fd32d	63
660fd04f TG	64	cycles = __arch_get_hw_counter(vd->clock_mode);
	65	ns = vdso_ts->nsec;
	66	last = vd->cycle_last;
660fd04f	67	ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
8345228c	68	ns = vdso_shift_ns(ns, vd->shift);
660fd04f TG	69	sec = vdso_ts->sec;
	70	} while (unlikely(vdso_read_retry(vd, seq)));
	71
	72	/* Add the namespace offset */
	73	sec += offs->sec;
	74	ns += offs->nsec;
	75
	76	/*
	77	* Do this outside the loop: a race inside the loop could result
	78	* in __iter_div_u64_rem() being extremely slow.
	79	*/
	80	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	81	ts->tv_nsec = ns;
	82
	83	return 0;
	84	}
	85	#else
	86	static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
	87	{
	88	return NULL;
	89	}
	90
	91	static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
	92	struct __kernel_timespec *ts)
	93	{
	94	return -EINVAL;
	95	}
	96	#endif
	97
c966533f	98	static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
660fd04f	99	struct __kernel_timespec *ts)
00b26474 VF	100	{
	101	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	102	u64 cycles, last, sec, ns;
	103	u32 seq;
	104
1dff4156 TG	105	/* Allows to compile the high resolution parts out */
	106	if (!__arch_vdso_hres_capable())
	107	return -1;
	108
00b26474	109	do {
660fd04f	110	/*
2d6b01bd	111	* Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
660fd04f TG	112	* enabled tasks have a special VVAR page installed which
660fd04f TG	113	* has vd->seq set to 1 and vd->clock_mode set to
2d6b01bd	114	* VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
660fd04f TG	115	* this does not affect performance because if vd->seq is
	116	* odd, i.e. a concurrent update is in progress the extra
	117	* check for vd->clock_mode is just a few extra
	118	* instructions while spin waiting for vd->seq to become
	119	* even again.
	120	*/
	121	while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
	122	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd	123	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	124	return do_hres_timens(vd, clk, ts);
	125	cpu_relax();
	126	}
	127	smp_rmb();
	128
ae12e085	129	if (unlikely(!vdso_clocksource_ok(vd)))
5d51bee7	130	return -1;
f86fd32d	131
9d90b93b	132	cycles = __arch_get_hw_counter(vd->clock_mode);
00b26474 VF	133	ns = vdso_ts->nsec;
00b26474 VF	134	last = vd->cycle_last;
9d90b93b	135	ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
8345228c	136	ns = vdso_shift_ns(ns, vd->shift);
00b26474 VF	137	sec = vdso_ts->sec;
	138	} while (unlikely(vdso_read_retry(vd, seq)));
	139
	140	/*
	141	* Do this outside the loop: a race inside the loop could result
	142	* in __iter_div_u64_rem() being extremely slow.
	143	*/
	144	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	145	ts->tv_nsec = ns;
	146
	147	return 0;
	148	}
	149
660fd04f TG	150	#ifdef CONFIG_TIME_NS
	151	static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
	152	struct __kernel_timespec *ts)
	153	{
	154	const struct vdso_data *vd = __arch_get_timens_vdso_data();
	155	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	156	const struct timens_offset *offs = &vdns->offset[clk];
	157	u64 nsec;
	158	s64 sec;
	159	s32 seq;
	160
	161	do {
	162	seq = vdso_read_begin(vd);
	163	sec = vdso_ts->sec;
	164	nsec = vdso_ts->nsec;
	165	} while (unlikely(vdso_read_retry(vd, seq)));
	166
	167	/* Add the namespace offset */
	168	sec += offs->sec;
	169	nsec += offs->nsec;
	170
	171	/*
	172	* Do this outside the loop: a race inside the loop could result
	173	* in __iter_div_u64_rem() being extremely slow.
	174	*/
	175	ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
	176	ts->tv_nsec = nsec;
	177	return 0;
	178	}
	179	#else
	180	static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
	181	struct __kernel_timespec *ts)
	182	{
	183	return -1;
	184	}
	185	#endif
	186
c966533f AV	187	static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
c966533f AV	188	struct __kernel_timespec *ts)
00b26474 VF	189	{
	190	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	191	u32 seq;
	192
	193	do {
660fd04f	194	/*
2d6b01bd	195	* Open coded to handle VDSO_CLOCK_TIMENS. See comment in
660fd04f TG	196	* do_hres().
	197	*/
	198	while ((seq = READ_ONCE(vd->seq)) & 1) {
	199	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd	200	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	201	return do_coarse_timens(vd, clk, ts);
	202	cpu_relax();
	203	}
	204	smp_rmb();
	205
00b26474 VF	206	ts->tv_sec = vdso_ts->sec;
	207	ts->tv_nsec = vdso_ts->nsec;
	208	} while (unlikely(vdso_read_retry(vd, seq)));
8463cf80 CL	209
8463cf80 CL	210	return 0;
00b26474 VF	211	}
	212
	213	static __maybe_unused int
e876f0b6 CL	214	__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	215	struct __kernel_timespec *ts)
00b26474	216	{
00b26474 VF	217	u32 msk;
	218
	219	/* Check for negative values or invalid clocks */
	220	if (unlikely((u32) clock >= MAX_CLOCKS))
502a590a	221	return -1;
00b26474 VF	222
	223	/*
	224	* Convert the clockid to a bitmask and use it to check which
	225	* clocks are handled in the VDSO directly.
	226	*/
	227	msk = 1U << clock;
8463cf80	228	if (likely(msk & VDSO_HRES))
c966533f	229	vd = &vd[CS_HRES_COARSE];
8463cf80 CL	230	else if (msk & VDSO_COARSE)
	231	return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
	232	else if (msk & VDSO_RAW)
c966533f AV	233	vd = &vd[CS_RAW];
	234	else
	235	return -1;
8463cf80	236
c966533f	237	return do_hres(vd, clock, ts);
502a590a	238	}
00b26474	239
502a590a	240	static __maybe_unused int
e876f0b6 CL	241	__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	242	struct __kernel_timespec *ts)
502a590a	243	{
e876f0b6	244	int ret = __cvdso_clock_gettime_common(vd, clock, ts);
502a590a TG	245
	246	if (unlikely(ret))
	247	return clock_gettime_fallback(clock, ts);
	248	return 0;
00b26474 VF	249	}
00b26474 VF	250
e876f0b6 CL	251	static __maybe_unused int
	252	__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
	253	{
	254	return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
	255	}
	256
bf279849	257	#ifdef BUILD_VDSO32
00b26474	258	static __maybe_unused int
e876f0b6 CL	259	__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	260	struct old_timespec32 *res)
00b26474 VF	261	{
	262	struct __kernel_timespec ts;
	263	int ret;
	264
e876f0b6	265	ret = __cvdso_clock_gettime_common(vd, clock, &ts);
00b26474	266
c60a32ea TG	267	if (unlikely(ret))
c60a32ea TG	268	return clock_gettime32_fallback(clock, res);
502a590a	269
a279235d VF	270	/* For ret == 0 */
	271	res->tv_sec = ts.tv_sec;
	272	res->tv_nsec = ts.tv_nsec;
	273
00b26474	274	return ret;
00b26474	275	}
e876f0b6 CL	276
	277	static __maybe_unused int
	278	__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
	279	{
	280	return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
	281	}
bf279849	282	#endif /* BUILD_VDSO32 */
00b26474 VF	283
00b26474 VF	284	static __maybe_unused int
e876f0b6 CL	285	__cvdso_gettimeofday_data(const struct vdso_data *vd,
e876f0b6 CL	286	struct __kernel_old_timeval tv, struct timezone tz)
00b26474	287	{
00b26474 VF	288
	289	if (likely(tv != NULL)) {
	290	struct __kernel_timespec ts;
	291
	292	if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
	293	return gettimeofday_fallback(tv, tz);
	294
	295	tv->tv_sec = ts.tv_sec;
	296	tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
	297	}
	298
	299	if (unlikely(tz != NULL)) {
660fd04f	300	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd	301	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	302	vd = __arch_get_timens_vdso_data();
660fd04f TG	303
00b26474 VF	304	tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
	305	tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
	306	}
	307
	308	return 0;
	309	}
	310
e876f0b6 CL	311	static __maybe_unused int
	312	__cvdso_gettimeofday(struct __kernel_old_timeval tv, struct timezone tz)
	313	{
	314	return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
	315	}
	316
00b26474	317	#ifdef VDSO_HAS_TIME
e876f0b6 CL	318	static __maybe_unused __kernel_old_time_t
e876f0b6 CL	319	__cvdso_time_data(const struct vdso_data vd, __kernel_old_time_t time)
00b26474	320	{
660fd04f TG	321	__kernel_old_time_t t;
660fd04f TG	322
2d6b01bd TG	323	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd TG	324	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	325	vd = __arch_get_timens_vdso_data();
	326
	327	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
00b26474 VF	328
	329	if (time)
	330	*time = t;
	331
	332	return t;
	333	}
e876f0b6 CL	334
	335	static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
	336	{
	337	return __cvdso_time_data(__arch_get_vdso_data(), time);
	338	}
00b26474 VF	339	#endif /* VDSO_HAS_TIME */
	340
	341	#ifdef VDSO_HAS_CLOCK_GETRES
	342	static __maybe_unused
e876f0b6 CL	343	int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	344	struct __kernel_timespec *res)
00b26474	345	{
00b26474	346	u32 msk;
502a590a	347	u64 ns;
00b26474 VF	348
	349	/* Check for negative values or invalid clocks */
	350	if (unlikely((u32) clock >= MAX_CLOCKS))
502a590a	351	return -1;
00b26474	352
2d6b01bd TG	353	if (IS_ENABLED(CONFIG_TIME_NS) &&
2d6b01bd TG	354	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
660fd04f TG	355	vd = __arch_get_timens_vdso_data();
660fd04f TG	356
00b26474 VF	357	/*
	358	* Convert the clockid to a bitmask and use it to check which
	359	* clocks are handled in the VDSO directly.
	360	*/
	361	msk = 1U << clock;
cdb7c5a9	362	if (msk & (VDSO_HRES \| VDSO_RAW)) {
00b26474 VF	363	/*
	364	* Preserves the behaviour of posix_get_hrtimer_res().
	365	*/
49a101d7	366	ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
00b26474 VF	367	} else if (msk & VDSO_COARSE) {
	368	/*
	369	* Preserves the behaviour of posix_get_coarse_res().
	370	*/
	371	ns = LOW_RES_NSEC;
00b26474	372	} else {
502a590a	373	return -1;
00b26474 VF	374	}
00b26474 VF	375
1638b8f0 TG	376	if (likely(res)) {
	377	res->tv_sec = 0;
	378	res->tv_nsec = ns;
	379	}
00b26474	380	return 0;
502a590a TG	381	}
502a590a TG	382
ffd08731	383	static __maybe_unused
e876f0b6 CL	384	int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	385	struct __kernel_timespec *res)
502a590a	386	{
e876f0b6	387	int ret = __cvdso_clock_getres_common(vd, clock, res);
00b26474	388
502a590a TG	389	if (unlikely(ret))
	390	return clock_getres_fallback(clock, res);
	391	return 0;
00b26474 VF	392	}
00b26474 VF	393
e876f0b6 CL	394	static __maybe_unused
	395	int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
	396	{
	397	return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
	398	}
	399
bf279849	400	#ifdef BUILD_VDSO32
00b26474	401	static __maybe_unused int
e876f0b6 CL	402	__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
e876f0b6 CL	403	struct old_timespec32 *res)
00b26474 VF	404	{
	405	struct __kernel_timespec ts;
	406	int ret;
	407
e876f0b6	408	ret = __cvdso_clock_getres_common(vd, clock, &ts);
c60a32ea	409
c60a32ea TG	410	if (unlikely(ret))
c60a32ea TG	411	return clock_getres32_fallback(clock, res);
00b26474	412
a279235d	413	if (likely(res)) {
00b26474 VF	414	res->tv_sec = ts.tv_sec;
	415	res->tv_nsec = ts.tv_nsec;
	416	}
00b26474	417	return ret;
00b26474	418	}
e876f0b6 CL	419
	420	static __maybe_unused int
	421	__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
	422	{
	423	return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
	424	clock, res);
	425	}
bf279849	426	#endif /* BUILD_VDSO32 */
00b26474	427	#endif /* VDSO_HAS_CLOCK_GETRES */