]>
Commit | Line | Data |
---|---|---|
00b26474 VF |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Generic userspace implementations of gettimeofday() and similar. | |
4 | */ | |
5 | #include <linux/compiler.h> | |
6 | #include <linux/math64.h> | |
7 | #include <linux/time.h> | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/hrtimer_defs.h> | |
10 | #include <vdso/datapage.h> | |
11 | #include <vdso/helpers.h> | |
12 | ||
13 | /* | |
14 | * The generic vDSO implementation requires that gettimeofday.h | |
15 | * provides: | |
16 | * - __arch_get_vdso_data(): to get the vdso datapage. | |
17 | * - __arch_get_hw_counter(): to get the hw counter based on the | |
18 | * clock_mode. | |
19 | * - gettimeofday_fallback(): fallback for gettimeofday. | |
20 | * - clock_gettime_fallback(): fallback for clock_gettime. | |
21 | * - clock_getres_fallback(): fallback for clock_getres. | |
22 | */ | |
629fdf77 VF |
23 | #ifdef ENABLE_COMPAT_VDSO |
24 | #include <asm/vdso/compat_gettimeofday.h> | |
25 | #else | |
00b26474 | 26 | #include <asm/vdso/gettimeofday.h> |
629fdf77 | 27 | #endif /* ENABLE_COMPAT_VDSO */ |
00b26474 | 28 | |
9d90b93b TG |
29 | #ifndef vdso_calc_delta |
30 | /* | |
31 | * Default implementation which works for all sane clocksources. That | |
32 | * obviously excludes x86/TSC. | |
33 | */ | |
34 | static __always_inline | |
35 | u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) | |
36 | { | |
37 | return ((cycles - last) & mask) * mult; | |
38 | } | |
39 | #endif | |
40 | ||
00b26474 VF |
41 | static int do_hres(const struct vdso_data *vd, clockid_t clk, |
42 | struct __kernel_timespec *ts) | |
43 | { | |
44 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
45 | u64 cycles, last, sec, ns; | |
46 | u32 seq; | |
47 | ||
48 | do { | |
49 | seq = vdso_read_begin(vd); | |
9d90b93b | 50 | cycles = __arch_get_hw_counter(vd->clock_mode); |
00b26474 VF |
51 | ns = vdso_ts->nsec; |
52 | last = vd->cycle_last; | |
53 | if (unlikely((s64)cycles < 0)) | |
502a590a | 54 | return -1; |
9d90b93b TG |
55 | |
56 | ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); | |
00b26474 VF |
57 | ns >>= vd->shift; |
58 | sec = vdso_ts->sec; | |
59 | } while (unlikely(vdso_read_retry(vd, seq))); | |
60 | ||
61 | /* | |
62 | * Do this outside the loop: a race inside the loop could result | |
63 | * in __iter_div_u64_rem() being extremely slow. | |
64 | */ | |
65 | ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); | |
66 | ts->tv_nsec = ns; | |
67 | ||
68 | return 0; | |
69 | } | |
70 | ||
71 | static void do_coarse(const struct vdso_data *vd, clockid_t clk, | |
72 | struct __kernel_timespec *ts) | |
73 | { | |
74 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
75 | u32 seq; | |
76 | ||
77 | do { | |
78 | seq = vdso_read_begin(vd); | |
79 | ts->tv_sec = vdso_ts->sec; | |
80 | ts->tv_nsec = vdso_ts->nsec; | |
81 | } while (unlikely(vdso_read_retry(vd, seq))); | |
82 | } | |
83 | ||
84 | static __maybe_unused int | |
502a590a | 85 | __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) |
00b26474 VF |
86 | { |
87 | const struct vdso_data *vd = __arch_get_vdso_data(); | |
88 | u32 msk; | |
89 | ||
90 | /* Check for negative values or invalid clocks */ | |
91 | if (unlikely((u32) clock >= MAX_CLOCKS)) | |
502a590a | 92 | return -1; |
00b26474 VF |
93 | |
94 | /* | |
95 | * Convert the clockid to a bitmask and use it to check which | |
96 | * clocks are handled in the VDSO directly. | |
97 | */ | |
98 | msk = 1U << clock; | |
99 | if (likely(msk & VDSO_HRES)) { | |
100 | return do_hres(&vd[CS_HRES_COARSE], clock, ts); | |
101 | } else if (msk & VDSO_COARSE) { | |
102 | do_coarse(&vd[CS_HRES_COARSE], clock, ts); | |
103 | return 0; | |
104 | } else if (msk & VDSO_RAW) { | |
105 | return do_hres(&vd[CS_RAW], clock, ts); | |
106 | } | |
502a590a TG |
107 | return -1; |
108 | } | |
00b26474 | 109 | |
502a590a TG |
110 | static __maybe_unused int |
111 | __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) | |
112 | { | |
113 | int ret = __cvdso_clock_gettime_common(clock, ts); | |
114 | ||
115 | if (unlikely(ret)) | |
116 | return clock_gettime_fallback(clock, ts); | |
117 | return 0; | |
00b26474 VF |
118 | } |
119 | ||
120 | static __maybe_unused int | |
121 | __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) | |
122 | { | |
123 | struct __kernel_timespec ts; | |
124 | int ret; | |
125 | ||
502a590a | 126 | ret = __cvdso_clock_gettime_common(clock, &ts); |
00b26474 | 127 | |
c60a32ea TG |
128 | #ifdef VDSO_HAS_32BIT_FALLBACK |
129 | if (unlikely(ret)) | |
130 | return clock_gettime32_fallback(clock, res); | |
131 | #else | |
502a590a TG |
132 | if (unlikely(ret)) |
133 | ret = clock_gettime_fallback(clock, &ts); | |
c60a32ea | 134 | #endif |
502a590a TG |
135 | |
136 | if (likely(!ret)) { | |
00b26474 VF |
137 | res->tv_sec = ts.tv_sec; |
138 | res->tv_nsec = ts.tv_nsec; | |
139 | } | |
00b26474 | 140 | return ret; |
00b26474 VF |
141 | } |
142 | ||
143 | static __maybe_unused int | |
144 | __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) | |
145 | { | |
146 | const struct vdso_data *vd = __arch_get_vdso_data(); | |
147 | ||
148 | if (likely(tv != NULL)) { | |
149 | struct __kernel_timespec ts; | |
150 | ||
151 | if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) | |
152 | return gettimeofday_fallback(tv, tz); | |
153 | ||
154 | tv->tv_sec = ts.tv_sec; | |
155 | tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; | |
156 | } | |
157 | ||
158 | if (unlikely(tz != NULL)) { | |
159 | tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; | |
160 | tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; | |
161 | } | |
162 | ||
163 | return 0; | |
164 | } | |
165 | ||
166 | #ifdef VDSO_HAS_TIME | |
167 | static __maybe_unused time_t __cvdso_time(time_t *time) | |
168 | { | |
169 | const struct vdso_data *vd = __arch_get_vdso_data(); | |
170 | time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); | |
171 | ||
172 | if (time) | |
173 | *time = t; | |
174 | ||
175 | return t; | |
176 | } | |
177 | #endif /* VDSO_HAS_TIME */ | |
178 | ||
179 | #ifdef VDSO_HAS_CLOCK_GETRES | |
180 | static __maybe_unused | |
502a590a | 181 | int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) |
00b26474 VF |
182 | { |
183 | const struct vdso_data *vd = __arch_get_vdso_data(); | |
502a590a | 184 | u64 hrtimer_res; |
00b26474 | 185 | u32 msk; |
502a590a | 186 | u64 ns; |
00b26474 VF |
187 | |
188 | /* Check for negative values or invalid clocks */ | |
189 | if (unlikely((u32) clock >= MAX_CLOCKS)) | |
502a590a | 190 | return -1; |
00b26474 | 191 | |
502a590a | 192 | hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); |
00b26474 VF |
193 | /* |
194 | * Convert the clockid to a bitmask and use it to check which | |
195 | * clocks are handled in the VDSO directly. | |
196 | */ | |
197 | msk = 1U << clock; | |
198 | if (msk & VDSO_HRES) { | |
199 | /* | |
200 | * Preserves the behaviour of posix_get_hrtimer_res(). | |
201 | */ | |
202 | ns = hrtimer_res; | |
203 | } else if (msk & VDSO_COARSE) { | |
204 | /* | |
205 | * Preserves the behaviour of posix_get_coarse_res(). | |
206 | */ | |
207 | ns = LOW_RES_NSEC; | |
208 | } else if (msk & VDSO_RAW) { | |
209 | /* | |
210 | * Preserves the behaviour of posix_get_hrtimer_res(). | |
211 | */ | |
212 | ns = hrtimer_res; | |
213 | } else { | |
502a590a | 214 | return -1; |
00b26474 VF |
215 | } |
216 | ||
a9446a90 TG |
217 | res->tv_sec = 0; |
218 | res->tv_nsec = ns; | |
00b26474 VF |
219 | |
220 | return 0; | |
502a590a TG |
221 | } |
222 | ||
223 | int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) | |
224 | { | |
225 | int ret = __cvdso_clock_getres_common(clock, res); | |
00b26474 | 226 | |
502a590a TG |
227 | if (unlikely(ret)) |
228 | return clock_getres_fallback(clock, res); | |
229 | return 0; | |
00b26474 VF |
230 | } |
231 | ||
232 | static __maybe_unused int | |
233 | __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) | |
234 | { | |
235 | struct __kernel_timespec ts; | |
236 | int ret; | |
237 | ||
502a590a | 238 | ret = __cvdso_clock_getres_common(clock, &ts); |
c60a32ea TG |
239 | |
240 | #ifdef VDSO_HAS_32BIT_FALLBACK | |
241 | if (unlikely(ret)) | |
242 | return clock_getres32_fallback(clock, res); | |
243 | #else | |
502a590a TG |
244 | if (unlikely(ret)) |
245 | ret = clock_getres_fallback(clock, &ts); | |
c60a32ea | 246 | #endif |
00b26474 | 247 | |
502a590a | 248 | if (likely(!ret)) { |
00b26474 VF |
249 | res->tv_sec = ts.tv_sec; |
250 | res->tv_nsec = ts.tv_nsec; | |
251 | } | |
00b26474 | 252 | return ret; |
00b26474 VF |
253 | } |
254 | #endif /* VDSO_HAS_CLOCK_GETRES */ |