1 From: Tony Breeds <tony@bakeyournoodle.com>
2 Subject: [PATCH] powerpc: Improve resolution of VDSO clock_gettime
3 References: 439908 - LTC49499
5 Currently the clock_gettime implementation in the VDSO produces a
6 result with microsecond resolution for the cases that are handled
7 without a system call, i.e. CLOCK_REALTIME and CLOCK_MONOTONIC. The
8 nanoseconds field of the result is obtained by computing a
9 microseconds value and multiplying by 1000.
11 This changes the code in the VDSO to do the computation for
12 clock_gettime with nanosecond resolution. That means that the
13 resolution of the result will ultimately depend on the timebase
16 Because the timestamp in the VDSO datapage (stamp_xsec, the real time
17 corresponding to the timebase count in tb_orig_stamp) is in units of
18 2^-20 seconds, it doesn't have sufficient resolution for computing a
19 result with nanosecond resolution. Therefore this adds a copy of
20 xtime to the VDSO datapage and updates it in update_gtod() along with
21 the other time-related fields.
23 Signed-off-by: Paul Mackerras <paulus@samba.org>
24 Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
25 Signed-off-by: Olaf Hering <olh@suse.de>
27 arch/powerpc/include/asm/vdso_datapage.h | 3
28 arch/powerpc/kernel/asm-offsets.c | 1
29 arch/powerpc/kernel/time.c | 1
30 arch/powerpc/kernel/vdso32/gettimeofday.S | 196 ++++++++++++++++++------------
31 arch/powerpc/kernel/vdso64/gettimeofday.S | 143 +++++++++++----------
32 5 files changed, 205 insertions(+), 139 deletions(-)
34 --- a/arch/powerpc/include/asm/vdso_datapage.h
35 +++ b/arch/powerpc/include/asm/vdso_datapage.h
39 #include <linux/unistd.h>
40 +#include <linux/time.h>
42 #define SYSCALL_MAP_SIZE ((__NR_syscalls + 31) / 32)
44 @@ -83,6 +84,7 @@ struct vdso_data {
45 __u32 icache_log_block_size; /* L1 i-cache log block size */
46 __s32 wtom_clock_sec; /* Wall to monotonic clock */
47 __s32 wtom_clock_nsec;
48 + struct timespec stamp_xtime; /* xtime value for tb_orig_stamp */
49 __u32 syscall_map_64[SYSCALL_MAP_SIZE]; /* map of syscalls */
50 __u32 syscall_map_32[SYSCALL_MAP_SIZE]; /* map of syscalls */
52 @@ -102,6 +104,7 @@ struct vdso_data {
53 __u32 tz_dsttime; /* Type of dst correction 0x5C */
54 __s32 wtom_clock_sec; /* Wall to monotonic clock */
55 __s32 wtom_clock_nsec;
56 + struct timespec stamp_xtime; /* xtime value for tb_orig_stamp */
57 __u32 syscall_map_32[SYSCALL_MAP_SIZE]; /* map of syscalls */
58 __u32 dcache_block_size; /* L1 d-cache block size */
59 __u32 icache_block_size; /* L1 i-cache block size */
60 --- a/arch/powerpc/kernel/asm-offsets.c
61 +++ b/arch/powerpc/kernel/asm-offsets.c
62 @@ -304,6 +304,7 @@ int main(void)
63 DEFINE(CFG_SYSCALL_MAP32, offsetof(struct vdso_data, syscall_map_32));
64 DEFINE(WTOM_CLOCK_SEC, offsetof(struct vdso_data, wtom_clock_sec));
65 DEFINE(WTOM_CLOCK_NSEC, offsetof(struct vdso_data, wtom_clock_nsec));
66 + DEFINE(STAMP_XTIME, offsetof(struct vdso_data, stamp_xtime));
67 DEFINE(CFG_ICACHE_BLOCKSZ, offsetof(struct vdso_data, icache_block_size));
68 DEFINE(CFG_DCACHE_BLOCKSZ, offsetof(struct vdso_data, dcache_block_size));
69 DEFINE(CFG_ICACHE_LOGBLOCKSZ, offsetof(struct vdso_data, icache_log_block_size));
70 --- a/arch/powerpc/kernel/time.c
71 +++ b/arch/powerpc/kernel/time.c
72 @@ -456,6 +456,7 @@ static inline void update_gtod(u64 new_t
73 vdso_data->tb_to_xs = new_tb_to_xs;
74 vdso_data->wtom_clock_sec = wall_to_monotonic.tv_sec;
75 vdso_data->wtom_clock_nsec = wall_to_monotonic.tv_nsec;
76 + vdso_data->stamp_xtime = xtime;
78 ++(vdso_data->tb_update_count);
80 --- a/arch/powerpc/kernel/vdso32/gettimeofday.S
81 +++ b/arch/powerpc/kernel/vdso32/gettimeofday.S
83 #include <asm/asm-offsets.h>
84 #include <asm/unistd.h>
86 +/* Offset for the low 32-bit part of a field of long type */
95 * Exact prototype of gettimeofday
96 @@ -90,101 +97,53 @@ V_FUNCTION_BEGIN(__kernel_clock_gettime)
98 mflr r12 /* r12 saves lr */
100 - mr r10,r3 /* r10 saves id */
101 mr r11,r4 /* r11 saves tp */
102 bl __get_datapage@local /* get data page */
103 mr r9,r3 /* datapage ptr in r9 */
104 - beq cr1,50f /* if monotonic -> jump there */
110 - bl __do_get_xsec@local /* get xsec from tb & kernel */
111 - bne- 98f /* out of line -> do syscall */
113 - /* seconds are xsec >> 20 */
114 - rlwinm r5,r4,12,20,31
115 - rlwimi r5,r3,12,0,19
116 - stw r5,TSPC32_TV_SEC(r11)
118 - /* get remaining xsec and convert to nsec. we scale
119 - * up remaining xsec by 12 bits and get the top 32 bits
120 - * of the multiplication, then we multiply by 1000
122 - rlwinm r5,r4,12,0,19
124 - ori r6,r6,1000000@l
127 - stw r5,TSPC32_TV_NSEC(r11)
132 +50: bl __do_get_tspec@local /* get sec/nsec from tb & kernel */
133 + bne cr1,80f /* not monotonic -> all done */
139 -50: bl __do_get_xsec@local /* get xsec from tb & kernel */
140 - bne- 98f /* out of line -> do syscall */
142 - /* seconds are xsec >> 20 */
143 - rlwinm r6,r4,12,20,31
144 - rlwimi r6,r3,12,0,19
146 - /* get remaining xsec and convert to nsec. we scale
147 - * up remaining xsec by 12 bits and get the top 32 bits
148 - * of the multiplication, then we multiply by 1000
150 - rlwinm r7,r4,12,0,19
152 - ori r5,r5,1000000@l
156 /* now we must fixup using wall to monotonic. We need to snapshot
157 * that value and do the counter trick again. Fortunately, we still
158 * have the counter value in r8 that was returned by __do_get_xsec.
159 - * At this point, r6,r7 contain our sec/nsec values, r3,r4 and r5
161 + * At this point, r3,r4 contain our sec/nsec values, r5 and r6
162 + * can be used, r7 contains NSEC_PER_SEC.
165 - lwz r3,WTOM_CLOCK_SEC(r9)
166 - lwz r4,WTOM_CLOCK_NSEC(r9)
167 + lwz r5,WTOM_CLOCK_SEC(r9)
168 + lwz r6,WTOM_CLOCK_NSEC(r9)
170 - /* We now have our result in r3,r4. We create a fake dependency
171 - * on that result and re-check the counter
172 + /* We now have our offset in r5,r6. We create a fake dependency
173 + * on that value and re-check the counter
181 - lwz r0,(CFG_TB_UPDATE_COUNT+4)(r9)
183 - lwz r0,(CFG_TB_UPDATE_COUNT)(r9)
185 + lwz r0,(CFG_TB_UPDATE_COUNT+LOPART)(r9)
186 cmpl cr0,r8,r0 /* check if updated */
189 - /* Calculate and store result. Note that this mimmics the C code,
190 + /* Calculate and store result. Note that this mimics the C code,
191 * which may cause funny results if nsec goes negative... is that
196 - lis r5,NSEC_PER_SEC@h
197 - ori r5,r5,NSEC_PER_SEC@l
212 -1: stw r3,TSPC32_TV_SEC(r11)
215 +80: stw r3,TSPC32_TV_SEC(r11)
216 stw r4,TSPC32_TV_NSEC(r11)
219 @@ -195,10 +154,6 @@ V_FUNCTION_BEGIN(__kernel_clock_gettime)
228 li r0,__NR_clock_gettime
230 @@ -322,3 +277,98 @@ __do_get_xsec:
236 + * This is the core of clock_gettime(), it returns the current
237 + * time in seconds and nanoseconds in r3 and r4.
238 + * It expects the datapage ptr in r9 and doesn't clobber it.
239 + * It clobbers r0, r5, r6, r10 and returns NSEC_PER_SEC in r7.
240 + * On return, r8 contains the counter value that can be reused.
241 + * This clobbers cr0 but not any other cr field.
245 + /* Check for update count & load values. We use the low
246 + * order 32 bits of the update count
248 +1: lwz r8,(CFG_TB_UPDATE_COUNT+LOPART)(r9)
249 + andi. r0,r8,1 /* pending update ? loop */
251 + xor r0,r8,r8 /* create dependency */
254 + /* Load orig stamp (offset to TB) */
255 + lwz r5,CFG_TB_ORIG_STAMP(r9)
256 + lwz r6,(CFG_TB_ORIG_STAMP+4)(r9)
258 + /* Get a stable TB value */
265 + /* Subtract tb orig stamp and shift left 12 bits.
270 + rlwimi. r0,r7,12,20,31
273 + /* Load scale factor & do multiplication */
274 + lwz r5,CFG_TB_TO_XS(r9) /* load values */
275 + lwz r6,(CFG_TB_TO_XS+4)(r9)
282 + beq+ 4f /* skip high part computation if 0 */
293 +4: addze r4,r4 /* add in carry */
294 + lis r7,NSEC_PER_SEC@h
295 + ori r7,r7,NSEC_PER_SEC@l
296 + mulhwu r4,r4,r7 /* convert to nanoseconds */
298 + /* At this point, we have seconds & nanoseconds since the xtime
299 + * stamp in r3+CA and r4. Load & add the xtime stamp.
302 + lwz r5,STAMP_XTIME+TSPC64_TV_SEC+LOPART(r9)
303 + lwz r6,STAMP_XTIME+TSPC64_TV_NSEC+LOPART(r9)
305 + lwz r5,STAMP_XTIME+TSPC32_TV_SEC(r9)
306 + lwz r6,STAMP_XTIME+TSPC32_TV_NSEC(r9)
311 + /* We now have our result in r3,r4. We create a fake dependency
312 + * on that result and re-check the counter
317 + lwz r0,(CFG_TB_UPDATE_COUNT+LOPART)(r9)
318 + cmpl cr0,r8,r0 /* check if updated */
321 + /* check for nanosecond overflow and adjust if necessary */
323 + bltlr /* all done if no overflow */
324 + subf r4,r7,r4 /* adjust if overflow */
329 --- a/arch/powerpc/kernel/vdso64/gettimeofday.S
330 +++ b/arch/powerpc/kernel/vdso64/gettimeofday.S
331 @@ -75,90 +75,49 @@ V_FUNCTION_BEGIN(__kernel_clock_gettime)
333 mflr r12 /* r12 saves lr */
335 - mr r10,r3 /* r10 saves id */
336 mr r11,r4 /* r11 saves tp */
337 bl V_LOCAL_FUNC(__get_datapage) /* get data page */
338 - beq cr1,50f /* if monotonic -> jump there */
344 - bl V_LOCAL_FUNC(__do_get_xsec) /* get xsec from tb & kernel */
346 - lis r7,15 /* r7 = 1000000 = USEC_PER_SEC */
348 - rldicl r5,r4,44,20 /* r5 = sec = xsec / XSEC_PER_SEC */
349 - rldicr r6,r5,20,43 /* r6 = sec * XSEC_PER_SEC */
350 - std r5,TSPC64_TV_SEC(r11) /* store sec in tv */
351 - subf r0,r6,r4 /* r0 = xsec = (xsec - r6) */
352 - mulld r0,r0,r7 /* usec = (xsec * USEC_PER_SEC) /
356 - mulli r0,r0,1000 /* nsec = usec * 1000 */
357 - std r0,TSPC64_TV_NSEC(r11) /* store nsec in tp */
363 +50: bl V_LOCAL_FUNC(__do_get_tspec) /* get time from tb & kernel */
364 + bne cr1,80f /* if not monotonic, all done */
370 -50: bl V_LOCAL_FUNC(__do_get_xsec) /* get xsec from tb & kernel */
372 - lis r7,15 /* r7 = 1000000 = USEC_PER_SEC */
374 - rldicl r5,r4,44,20 /* r5 = sec = xsec / XSEC_PER_SEC */
375 - rldicr r6,r5,20,43 /* r6 = sec * XSEC_PER_SEC */
376 - subf r0,r6,r4 /* r0 = xsec = (xsec - r6) */
377 - mulld r0,r0,r7 /* usec = (xsec * USEC_PER_SEC) /
381 - mulli r6,r6,1000 /* nsec = usec * 1000 */
383 /* now we must fixup using wall to monotonic. We need to snapshot
384 * that value and do the counter trick again. Fortunately, we still
385 - * have the counter value in r8 that was returned by __do_get_xsec.
386 - * At this point, r5,r6 contain our sec/nsec values.
388 + * have the counter value in r8 that was returned by __do_get_tspec.
389 + * At this point, r4,r5 contain our sec/nsec values.
392 - lwa r4,WTOM_CLOCK_SEC(r3)
393 - lwa r7,WTOM_CLOCK_NSEC(r3)
394 + lwa r6,WTOM_CLOCK_SEC(r3)
395 + lwa r9,WTOM_CLOCK_NSEC(r3)
397 - /* We now have our result in r4,r7. We create a fake dependency
398 + /* We now have our result in r6,r9. We create a fake dependency
399 * on that result and re-check the counter
406 ld r0,CFG_TB_UPDATE_COUNT(r3)
407 cmpld cr0,r0,r8 /* check if updated */
410 - /* Calculate and store result. Note that this mimmics the C code,
411 - * which may cause funny results if nsec goes negative... is that
412 - * possible at all ?
416 - lis r9,NSEC_PER_SEC@h
417 - ori r9,r9,NSEC_PER_SEC@l
420 + /* Add wall->monotonic offset and check for overflow or underflow.
434 -1: std r4,TSPC64_TV_SEC(r11)
435 - std r7,TSPC64_TV_NSEC(r11)
438 +80: std r4,TSPC64_TV_SEC(r11)
439 + std r5,TSPC64_TV_NSEC(r11)
443 @@ -168,10 +127,6 @@ V_FUNCTION_BEGIN(__kernel_clock_gettime)
452 li r0,__NR_clock_gettime
454 @@ -253,3 +208,59 @@ V_FUNCTION_BEGIN(__do_get_xsec)
457 V_FUNCTION_END(__do_get_xsec)
460 + * This is the core of clock_gettime(), it returns the current
461 + * time in seconds and nanoseconds in r4 and r5.
462 + * It expects the datapage ptr in r3 and doesn't clobber it.
463 + * It clobbers r0 and r6 and returns NSEC_PER_SEC in r7.
464 + * On return, r8 contains the counter value that can be reused.
465 + * This clobbers cr0 but not any other cr field.
467 +V_FUNCTION_BEGIN(__do_get_tspec)
469 + /* check for update count & load values */
470 +1: ld r8,CFG_TB_UPDATE_COUNT(r3)
471 + andi. r0,r8,1 /* pending update ? loop */
473 + xor r0,r8,r8 /* create dependency */
476 + /* Get TB & offset it. We use the MFTB macro which will generate
477 + * workaround code for Cell.
480 + ld r9,CFG_TB_ORIG_STAMP(r3)
484 + ld r5,CFG_TB_TO_XS(r3)
485 + sldi r7,r7,12 /* compute time since stamp_xtime */
486 + mulhdu r6,r7,r5 /* in units of 2^-32 seconds */
488 + /* Add stamp since epoch */
489 + ld r4,STAMP_XTIME+TSPC64_TV_SEC(r3)
490 + ld r5,STAMP_XTIME+TSPC64_TV_NSEC(r3)
495 + ld r0,CFG_TB_UPDATE_COUNT(r3)
496 + cmpld r0,r8 /* check if updated */
497 + bne- 1b /* reload if so */
499 + /* convert to seconds & nanoseconds and add to stamp */
500 + lis r7,NSEC_PER_SEC@h
501 + ori r7,r7,NSEC_PER_SEC@l
502 + mulhwu r0,r6,r7 /* compute nanoseconds and */
503 + srdi r6,r6,32 /* seconds since stamp_xtime */
505 + add r5,r5,r0 /* add nanoseconds together */
506 + cmpd r5,r7 /* overflow? */
508 + bltlr /* all done if no overflow */
509 + subf r5,r7,r5 /* if overflow, adjust */
513 +V_FUNCTION_END(__do_get_tspec)