1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Go execution tracer.
6 // The tracer captures a wide range of execution events like goroutine
7 // creation/blocking/unblocking, syscall enter/exit/block, GC-related events,
8 // changes of heap size, processor start/stop, etc and writes them to a buffer
9 // in a compact form. A precise nanosecond-precision timestamp and a stack
10 // trace is captured for most events.
11 // See https://golang.org/s/go15trace for more info.
16 "runtime/internal/sys"
20 // Event types in the trace, args are given in square brackets.
22 traceEvNone = 0 // unused
23 traceEvBatch = 1 // start of per-P batch of events [pid, timestamp]
24 traceEvFrequency = 2 // contains tracer timer frequency [frequency (ticks per second)]
25 traceEvStack = 3 // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}]
26 traceEvGomaxprocs = 4 // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id]
27 traceEvProcStart = 5 // start of P [timestamp, thread id]
28 traceEvProcStop = 6 // stop of P [timestamp]
29 traceEvGCStart = 7 // GC start [timestamp, seq, stack id]
30 traceEvGCDone = 8 // GC done [timestamp]
31 traceEvGCSTWStart = 9 // GC STW start [timestamp, kind]
32 traceEvGCSTWDone = 10 // GC STW done [timestamp]
33 traceEvGCSweepStart = 11 // GC sweep start [timestamp, stack id]
34 traceEvGCSweepDone = 12 // GC sweep done [timestamp, swept, reclaimed]
35 traceEvGoCreate = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id]
36 traceEvGoStart = 14 // goroutine starts running [timestamp, goroutine id, seq]
37 traceEvGoEnd = 15 // goroutine ends [timestamp]
38 traceEvGoStop = 16 // goroutine stops (like in select{}) [timestamp, stack]
39 traceEvGoSched = 17 // goroutine calls Gosched [timestamp, stack]
40 traceEvGoPreempt = 18 // goroutine is preempted [timestamp, stack]
41 traceEvGoSleep = 19 // goroutine calls Sleep [timestamp, stack]
42 traceEvGoBlock = 20 // goroutine blocks [timestamp, stack]
43 traceEvGoUnblock = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack]
44 traceEvGoBlockSend = 22 // goroutine blocks on chan send [timestamp, stack]
45 traceEvGoBlockRecv = 23 // goroutine blocks on chan recv [timestamp, stack]
46 traceEvGoBlockSelect = 24 // goroutine blocks on select [timestamp, stack]
47 traceEvGoBlockSync = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack]
48 traceEvGoBlockCond = 26 // goroutine blocks on Cond [timestamp, stack]
49 traceEvGoBlockNet = 27 // goroutine blocks on network [timestamp, stack]
50 traceEvGoSysCall = 28 // syscall enter [timestamp, stack]
51 traceEvGoSysExit = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp]
52 traceEvGoSysBlock = 30 // syscall blocks [timestamp]
53 traceEvGoWaiting = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id]
54 traceEvGoInSyscall = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id]
55 traceEvHeapAlloc = 33 // memstats.heap_live change [timestamp, heap_alloc]
56 traceEvNextGC = 34 // memstats.next_gc change [timestamp, next_gc]
57 traceEvTimerGoroutine = 35 // not currently used; previously denoted timer goroutine [timer goroutine id]
58 traceEvFutileWakeup = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp]
59 traceEvString = 37 // string dictionary entry [ID, length, string]
60 traceEvGoStartLocal = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id]
61 traceEvGoUnblockLocal = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack]
62 traceEvGoSysExitLocal = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp]
63 traceEvGoStartLabel = 41 // goroutine starts running with label [timestamp, goroutine id, seq, label string id]
64 traceEvGoBlockGC = 42 // goroutine blocks on GC assist [timestamp, stack]
65 traceEvGCMarkAssistStart = 43 // GC mark assist start [timestamp, stack]
66 traceEvGCMarkAssistDone = 44 // GC mark assist done [timestamp]
67 traceEvUserTaskCreate = 45 // trace.NewContext [timestamp, internal task id, internal parent task id, stack, name string]
68 traceEvUserTaskEnd = 46 // end of a task [timestamp, internal task id, stack]
69 traceEvUserRegion = 47 // trace.WithRegion [timestamp, internal task id, mode(0:start, 1:end), stack, name string]
70 traceEvUserLog = 48 // trace.Log [timestamp, internal task id, key string id, stack, value string]
72 // Byte is used but only 6 bits are available for event type.
73 // The remaining 2 bits are used to specify the number of arguments.
74 // That means, the max event type value is 63.
78 // Timestamps in trace are cputicks/traceTickDiv.
79 // This makes absolute values of timestamp diffs smaller,
80 // and so they are encoded in less number of bytes.
81 // 64 on x86 is somewhat arbitrary (one tick is ~20ns on a 3GHz machine).
82 // The suggested increment frequency for PowerPC's time base register is
83 // 512 MHz according to Power ISA v2.07 section 6.2, so we use 16 on ppc64
85 // Tracing won't work reliably for architectures where cputicks is emulated
86 // by nanotime, so the value doesn't matter for those architectures.
87 traceTickDiv = 16 + 48*(sys.Goarch386|sys.GoarchAmd64)
88 // Maximum number of PCs in a single stack trace.
89 // Since events contain only stack id rather than whole stack trace,
90 // we can allow quite large values here.
92 // Identifier of a fake P that is used when we trace without a real P.
94 // Maximum number of bytes to encode uint64 in base-128.
95 traceBytesPerNumber = 10
96 // Shift of the number of arguments in the first event byte.
97 traceArgCountShift = 6
98 // Flag passed to traceGoPark to denote that the previous wakeup of this
99 // goroutine was futile. For example, a goroutine was unblocked on a mutex,
100 // but another goroutine got ahead and acquired the mutex before the first
101 // goroutine is scheduled, so the first goroutine has to block again.
102 // Such wakeups happen on buffered channels and sync.Mutex,
103 // but are generally not interesting for end user.
104 traceFutileWakeup byte = 128
107 // trace is global tracing context.
109 lock mutex // protects the following members
110 lockOwner *g // to avoid deadlocks during recursive lock locks
111 enabled bool // when set runtime traces events
112 shutdown bool // set when we are waiting for trace reader to finish after setting enabled to false
113 headerWritten bool // whether ReadTrace has emitted trace header
114 footerWritten bool // whether ReadTrace has emitted trace footer
115 shutdownSema uint32 // used to wait for ReadTrace completion
116 seqStart uint64 // sequence number when tracing was started
117 ticksStart int64 // cputicks when tracing was started
118 ticksEnd int64 // cputicks when tracing was stopped
119 timeStart int64 // nanotime when tracing was started
120 timeEnd int64 // nanotime when tracing was stopped
121 seqGC uint64 // GC start/done sequencer
122 reading traceBufPtr // buffer currently handed off to user
123 empty traceBufPtr // stack of empty buffers
124 fullHead traceBufPtr // queue of full buffers
126 reader guintptr // goroutine that called ReadTrace, or nil
127 stackTab traceStackTable // maps stack traces to unique ids
129 // Dictionary for traceEvString.
131 // TODO: central lock to access the map is not ideal.
132 // option: pre-assign ids to all user annotation region names and tags
133 // option: per-P cache
134 // option: sync.Map like data structure
136 strings map[string]uint64
139 // markWorkerLabels maps gcMarkWorkerMode to string ID.
140 markWorkerLabels [len(gcMarkWorkerModeStrings)]uint64
142 bufLock mutex // protects buf
143 buf traceBufPtr // global trace buffer, used when running without a p
146 // traceBufHeader is per-P tracing buffer.
148 type traceBufHeader struct {
149 link traceBufPtr // in trace.empty/full
150 lastTicks uint64 // when we wrote the last event
151 pos int // next write offset in arr
152 stk [traceStackSize]location // scratch buffer for traceback
155 // traceBuf is per-P tracing buffer.
158 type traceBuf struct {
160 arr [64<<10 - unsafe.Sizeof(traceBufHeader{})]byte // underlying buffer for traceBufHeader.buf
163 // traceBufPtr is a *traceBuf that is not traced by the garbage
164 // collector and doesn't have write barriers. traceBufs are not
165 // allocated from the GC'd heap, so this is safe, and are often
166 // manipulated in contexts where write barriers are not allowed, so
167 // this is necessary.
169 // TODO: Since traceBuf is now go:notinheap, this isn't necessary.
170 type traceBufPtr uintptr
172 func (tp traceBufPtr) ptr() *traceBuf { return (*traceBuf)(unsafe.Pointer(tp)) }
173 func (tp *traceBufPtr) set(b *traceBuf) { *tp = traceBufPtr(unsafe.Pointer(b)) }
174 func traceBufPtrOf(b *traceBuf) traceBufPtr {
175 return traceBufPtr(unsafe.Pointer(b))
178 // StartTrace enables tracing for the current process.
179 // While tracing, the data will be buffered and available via ReadTrace.
180 // StartTrace returns an error if tracing is already enabled.
181 // Most clients should use the runtime/trace package or the testing package's
182 // -test.trace flag instead of calling StartTrace directly.
183 func StartTrace() error {
184 // Stop the world so that we can take a consistent snapshot
185 // of all goroutines at the beginning of the trace.
186 // Do not stop the world during GC so we ensure we always see
187 // a consistent view of GC-related events (e.g. a start is always
188 // paired with an end).
189 stopTheWorldGC("start tracing")
191 // We are in stop-the-world, but syscalls can finish and write to trace concurrently.
192 // Exitsyscall could check trace.enabled long before and then suddenly wake up
193 // and decide to write to trace at a random point in time.
194 // However, such syscall will use the global trace.buf buffer, because we've
195 // acquired all p's by doing stop-the-world. So this protects us from such races.
198 if trace.enabled || trace.shutdown {
199 unlock(&trace.bufLock)
201 return errorString("tracing is already enabled")
204 // Can't set trace.enabled yet. While the world is stopped, exitsyscall could
205 // already emit a delayed event (see exitTicks in exitsyscall) if we set trace.enabled here.
206 // That would lead to an inconsistent trace:
207 // - either GoSysExit appears before EvGoInSyscall,
208 // - or GoSysExit appears for a goroutine for which we don't emit EvGoInSyscall below.
209 // To instruct traceEvent that it must not ignore events below, we set startingtrace.
210 // trace.enabled is set afterwards once we have emitted all preliminary events.
212 _g_.m.startingtrace = true
214 // Obtain current stack ID to use in all traceEvGoCreate events below.
216 stkBuf := make([]location, traceStackSize)
217 stackID := traceStackID(mp, stkBuf, 2)
220 for _, gp := range allgs {
221 status := readgstatus(gp)
222 if status != _Gdead {
224 gp.tracelastp = getg().m.p
225 // +PCQuantum because traceFrameForPC expects return PCs and subtracts PCQuantum.
226 id := trace.stackTab.put([]location{location{pc: gp.startpc + sys.PCQuantum}})
227 traceEvent(traceEvGoCreate, -1, uint64(gp.goid), uint64(id), stackID)
229 if status == _Gwaiting {
230 // traceEvGoWaiting is implied to have seq=1.
232 traceEvent(traceEvGoWaiting, -1, uint64(gp.goid))
234 if status == _Gsyscall {
236 traceEvent(traceEvGoInSyscall, -1, uint64(gp.goid))
238 gp.sysblocktraced = false
243 // Note: ticksStart needs to be set after we emit traceEvGoInSyscall events.
244 // If we do it the other way around, it is possible that exitsyscall will
245 // query sysexitticks after ticksStart but before traceEvGoInSyscall timestamp.
246 // It will lead to a false conclusion that cputicks is broken.
247 trace.ticksStart = cputicks()
248 trace.timeStart = nanotime()
249 trace.headerWritten = false
250 trace.footerWritten = false
252 // string to id mapping
253 // 0 : reserved for an empty string
254 // remaining: other strings registered by traceString
256 trace.strings = make(map[string]uint64)
259 _g_.m.startingtrace = false
262 // Register runtime goroutine labels.
263 _, pid, bufp := traceAcquireBuffer()
264 for i, label := range gcMarkWorkerModeStrings[:] {
265 trace.markWorkerLabels[i], bufp = traceString(bufp, pid, label)
267 traceReleaseBuffer(pid)
269 unlock(&trace.bufLock)
275 // StopTrace stops tracing, if it was previously enabled.
276 // StopTrace only returns after all the reads for the trace have completed.
278 // Stop the world so that we can collect the trace buffers from all p's below,
279 // and also to avoid races with traceEvent.
280 stopTheWorldGC("stop tracing")
282 // See the comment in StartTrace.
286 unlock(&trace.bufLock)
293 // Loop over all allocated Ps because dead Ps may still have
295 for _, p := range allp[:cap(allp)] {
305 if buf.ptr().pos != 0 {
311 trace.ticksEnd = cputicks()
312 trace.timeEnd = nanotime()
313 // Windows time can tick only every 15ms, wait for at least one tick.
314 if trace.timeEnd != trace.timeStart {
320 trace.enabled = false
321 trace.shutdown = true
322 unlock(&trace.bufLock)
326 // The world is started but we've set trace.shutdown, so new tracing can't start.
327 // Wait for the trace reader to flush pending buffers and stop.
328 semacquire(&trace.shutdownSema)
330 raceacquire(unsafe.Pointer(&trace.shutdownSema))
333 // The lock protects us from races with StartTrace/StopTrace because they do stop-the-world.
335 for _, p := range allp[:cap(allp)] {
337 throw("trace: non-empty trace buffer in proc")
341 throw("trace: non-empty global trace buffer")
343 if trace.fullHead != 0 || trace.fullTail != 0 {
344 throw("trace: non-empty full trace buffer")
346 if trace.reading != 0 || trace.reader != 0 {
347 throw("trace: reading after shutdown")
349 for trace.empty != 0 {
351 trace.empty = buf.ptr().link
352 sysFree(unsafe.Pointer(buf), unsafe.Sizeof(*buf.ptr()), &memstats.other_sys)
355 trace.shutdown = false
359 // ReadTrace returns the next chunk of binary tracing data, blocking until data
360 // is available. If tracing is turned off and all the data accumulated while it
361 // was on has been returned, ReadTrace returns nil. The caller must copy the
362 // returned data before calling ReadTrace again.
363 // ReadTrace must be called from one goroutine at a time.
364 func ReadTrace() []byte {
365 // This function may need to lock trace.lock recursively
366 // (goparkunlock -> traceGoPark -> traceEvent -> traceFlush).
367 // To allow this we use trace.lockOwner.
368 // Also this function must not allocate while holding trace.lock:
369 // allocation can call heap allocate, which will try to emit a trace
370 // event while holding heap lock.
372 trace.lockOwner = getg()
374 if trace.reader != 0 {
375 // More than one goroutine reads trace. This is bad.
376 // But we rather do not crash the program because of tracing,
377 // because tracing can be enabled at runtime on prod servers.
378 trace.lockOwner = nil
380 println("runtime: ReadTrace called from multiple goroutines simultaneously")
383 // Recycle the old buffer.
384 if buf := trace.reading; buf != 0 {
385 buf.ptr().link = trace.empty
389 // Write trace header.
390 if !trace.headerWritten {
391 trace.headerWritten = true
392 trace.lockOwner = nil
394 return []byte("go 1.11 trace\x00\x00\x00")
396 // Wait for new data.
397 if trace.fullHead == 0 && !trace.shutdown {
398 trace.reader.set(getg())
399 goparkunlock(&trace.lock, waitReasonTraceReaderBlocked, traceEvGoBlock, 2)
403 if trace.fullHead != 0 {
404 buf := traceFullDequeue()
406 trace.lockOwner = nil
408 return buf.ptr().arr[:buf.ptr().pos]
410 // Write footer with timer frequency.
411 if !trace.footerWritten {
412 trace.footerWritten = true
413 // Use float64 because (trace.ticksEnd - trace.ticksStart) * 1e9 can overflow int64.
414 freq := float64(trace.ticksEnd-trace.ticksStart) * 1e9 / float64(trace.timeEnd-trace.timeStart) / traceTickDiv
415 trace.lockOwner = nil
418 data = append(data, traceEvFrequency|0<<traceArgCountShift)
419 data = traceAppend(data, uint64(freq))
420 // This will emit a bunch of full buffers, we will pick them up
421 // on the next iteration.
422 trace.stackTab.dump()
427 trace.lockOwner = nil
430 // Model synchronization on trace.shutdownSema, which race
431 // detector does not see. This is required to avoid false
432 // race reports on writer passed to trace.Start.
433 racerelease(unsafe.Pointer(&trace.shutdownSema))
435 // trace.enabled is already reset, so can call traceable functions.
436 semrelease(&trace.shutdownSema)
439 // Also bad, but see the comment above.
440 trace.lockOwner = nil
442 println("runtime: spurious wakeup of trace reader")
446 // traceReader returns the trace reader that should be woken up, if any.
447 func traceReader() *g {
448 if trace.reader == 0 || (trace.fullHead == 0 && !trace.shutdown) {
452 if trace.reader == 0 || (trace.fullHead == 0 && !trace.shutdown) {
456 gp := trace.reader.ptr()
457 trace.reader.set(nil)
462 // traceProcFree frees trace buffer associated with pp.
463 func traceProcFree(pp *p) {
474 // traceFullQueue queues buf into queue of full buffers.
475 func traceFullQueue(buf traceBufPtr) {
477 if trace.fullHead == 0 {
480 trace.fullTail.ptr().link = buf
485 // traceFullDequeue dequeues from queue of full buffers.
486 func traceFullDequeue() traceBufPtr {
487 buf := trace.fullHead
491 trace.fullHead = buf.ptr().link
492 if trace.fullHead == 0 {
499 // traceEvent writes a single event to trace buffer, flushing the buffer if necessary.
501 // If skip > 0, write current stack id as the last argument (skipping skip top frames).
502 // If skip = 0, this event type should contain a stack, but we don't want
503 // to collect and remember it for this particular call.
504 func traceEvent(ev byte, skip int, args ...uint64) {
505 mp, pid, bufp := traceAcquireBuffer()
506 // Double-check trace.enabled now that we've done m.locks++ and acquired bufLock.
507 // This protects from races between traceEvent and StartTrace/StopTrace.
509 // The caller checked that trace.enabled == true, but trace.enabled might have been
510 // turned off between the check and now. Check again. traceLockBuffer did mp.locks++,
511 // StopTrace does stopTheWorld, and stopTheWorld waits for mp.locks to go back to zero,
512 // so if we see trace.enabled == true now, we know it's true for the rest of the function.
513 // Exitsyscall can run even during stopTheWorld. The race with StartTrace/StopTrace
514 // during tracing in exitsyscall is resolved by locking trace.bufLock in traceLockBuffer.
516 // Note trace_userTaskCreate runs the same check.
517 if !trace.enabled && !mp.startingtrace {
518 traceReleaseBuffer(pid)
523 if getg() == mp.curg {
524 skip++ // +1 because stack is captured in traceEventLocked.
527 traceEventLocked(0, mp, pid, bufp, ev, skip, args...)
528 traceReleaseBuffer(pid)
531 func traceEventLocked(extraBytes int, mp *m, pid int32, bufp *traceBufPtr, ev byte, skip int, args ...uint64) {
533 // TODO: test on non-zero extraBytes param.
534 maxSize := 2 + 5*traceBytesPerNumber + extraBytes // event type, length, sequence, timestamp, stack id and two add params
535 if buf == nil || len(buf.arr)-buf.pos < maxSize {
536 buf = traceFlush(traceBufPtrOf(buf), pid).ptr()
540 ticks := uint64(cputicks()) / traceTickDiv
541 tickDiff := ticks - buf.lastTicks
542 buf.lastTicks = ticks
543 narg := byte(len(args))
547 // We have only 2 bits for number of arguments.
548 // If number is >= 3, then the event type is followed by event length in bytes.
553 buf.byte(ev | narg<<traceArgCountShift)
556 // Reserve the byte for length assuming that length < 128.
558 lenp = &buf.arr[buf.pos-1]
561 for _, a := range args {
567 buf.varint(traceStackID(mp, buf.stk[:], skip))
569 evSize := buf.pos - startPos
570 if evSize > maxSize {
571 throw("invalid length of trace event")
574 // Fill in actual length.
575 *lenp = byte(evSize - 2)
579 func traceStackID(mp *m, buf []location, skip int) uint64 {
584 nstk = callers(skip+1, buf)
585 } else if gp != nil {
586 // FIXME: get stack trace of different goroutine.
589 nstk-- // skip runtime.goexit
591 if nstk > 0 && gp.goid == 1 {
592 nstk-- // skip runtime.main
594 id := trace.stackTab.put(buf[:nstk])
598 // traceAcquireBuffer returns trace buffer to use and, if necessary, locks it.
599 func traceAcquireBuffer() (mp *m, pid int32, bufp *traceBufPtr) {
601 if p := mp.p.ptr(); p != nil {
602 return mp, p.id, &p.tracebuf
605 return mp, traceGlobProc, &trace.buf
608 // traceReleaseBuffer releases a buffer previously acquired with traceAcquireBuffer.
609 func traceReleaseBuffer(pid int32) {
610 if pid == traceGlobProc {
611 unlock(&trace.bufLock)
616 // traceFlush puts buf onto stack of full buffers and returns an empty buffer.
617 func traceFlush(buf traceBufPtr, pid int32) traceBufPtr {
618 owner := trace.lockOwner
619 dolock := owner == nil || owner != getg().m.curg
626 if trace.empty != 0 {
628 trace.empty = buf.ptr().link
630 buf = traceBufPtr(sysAlloc(unsafe.Sizeof(traceBuf{}), &memstats.other_sys))
632 throw("trace: out of memory")
639 // initialize the buffer for a new batch
640 ticks := uint64(cputicks()) / traceTickDiv
641 bufp.lastTicks = ticks
642 bufp.byte(traceEvBatch | 1<<traceArgCountShift)
643 bufp.varint(uint64(pid))
652 // traceString adds a string to the trace.strings and returns the id.
653 func traceString(bufp *traceBufPtr, pid int32, s string) (uint64, *traceBufPtr) {
658 lock(&trace.stringsLock)
660 // raceacquire is necessary because the map access
661 // below is race annotated.
662 raceacquire(unsafe.Pointer(&trace.stringsLock))
665 if id, ok := trace.strings[s]; ok {
667 racerelease(unsafe.Pointer(&trace.stringsLock))
669 unlock(&trace.stringsLock)
675 id := trace.stringSeq
676 trace.strings[s] = id
679 racerelease(unsafe.Pointer(&trace.stringsLock))
681 unlock(&trace.stringsLock)
683 // memory allocation in above may trigger tracing and
684 // cause *bufp changes. Following code now works with *bufp,
685 // so there must be no memory allocation or any activities
686 // that causes tracing after this point.
689 size := 1 + 2*traceBytesPerNumber + len(s)
690 if buf == nil || len(buf.arr)-buf.pos < size {
691 buf = traceFlush(traceBufPtrOf(buf), pid).ptr()
694 buf.byte(traceEvString)
697 // double-check the string and the length can fit.
698 // Otherwise, truncate the string.
700 if room := len(buf.arr) - buf.pos; room < slen+traceBytesPerNumber {
704 buf.varint(uint64(slen))
705 buf.pos += copy(buf.arr[buf.pos:], s[:slen])
711 // traceAppend appends v to buf in little-endian-base-128 encoding.
712 func traceAppend(buf []byte, v uint64) []byte {
713 for ; v >= 0x80; v >>= 7 {
714 buf = append(buf, 0x80|byte(v))
716 buf = append(buf, byte(v))
720 // varint appends v to buf in little-endian-base-128 encoding.
721 func (buf *traceBuf) varint(v uint64) {
723 for ; v >= 0x80; v >>= 7 {
724 buf.arr[pos] = 0x80 | byte(v)
727 buf.arr[pos] = byte(v)
732 // byte appends v to buf.
733 func (buf *traceBuf) byte(v byte) {
738 // traceStackTable maps stack traces (arrays of PC's) to unique uint32 ids.
739 // It is lock-free for reading.
740 type traceStackTable struct {
744 tab [1 << 13]traceStackPtr
747 // traceStack is a single stack in traceStackTable.
748 type traceStack struct {
753 stk [0]location // real type [n]location
756 type traceStackPtr uintptr
758 func (tp traceStackPtr) ptr() *traceStack { return (*traceStack)(unsafe.Pointer(tp)) }
760 // stack returns slice of PCs.
761 func (ts *traceStack) stack() []location {
762 return (*[traceStackSize]location)(unsafe.Pointer(&ts.stk))[:ts.n]
765 // put returns a unique id for the stack trace pcs and caches it in the table,
766 // if it sees the trace for the first time.
767 func (tab *traceStackTable) put(pcs []location) uint32 {
772 for _, loc := range pcs {
777 // First, search the hashtable w/o the mutex.
778 if id := tab.find(pcs, hash); id != 0 {
781 // Now, double check under the mutex.
783 if id := tab.find(pcs, hash); id != 0 {
787 // Create new record.
789 stk := tab.newStack(len(pcs))
794 for i, pc := range pcs {
795 // Use memmove to avoid write barrier.
796 memmove(unsafe.Pointer(&stkpc[i]), unsafe.Pointer(&pc), unsafe.Sizeof(pc))
798 part := int(hash % uintptr(len(tab.tab)))
799 stk.link = tab.tab[part]
800 atomicstorep(unsafe.Pointer(&tab.tab[part]), unsafe.Pointer(stk))
805 // find checks if the stack trace pcs is already present in the table.
806 func (tab *traceStackTable) find(pcs []location, hash uintptr) uint32 {
807 part := int(hash % uintptr(len(tab.tab)))
809 for stk := tab.tab[part].ptr(); stk != nil; stk = stk.link.ptr() {
810 if stk.hash == hash && stk.n == len(pcs) {
811 for i, stkpc := range stk.stack() {
822 // newStack allocates a new stack of size n.
823 func (tab *traceStackTable) newStack(n int) *traceStack {
824 return (*traceStack)(tab.mem.alloc(unsafe.Sizeof(traceStack{}) + uintptr(n)*unsafe.Sizeof(location{})))
827 // dump writes all previously cached stacks to trace buffers,
828 // releases all memory and resets state.
829 func (tab *traceStackTable) dump() {
830 var tmp [(2 + 4*traceStackSize) * traceBytesPerNumber]byte
831 bufp := traceFlush(0, 0)
832 for _, stk := range tab.tab {
834 for ; stk != nil; stk = stk.link.ptr() {
836 tmpbuf = traceAppend(tmpbuf, uint64(stk.id))
837 frames := stk.stack()
838 tmpbuf = traceAppend(tmpbuf, uint64(len(frames)))
839 for _, f := range frames {
841 frame, bufp = traceFrameForPC(bufp, 0, f)
842 tmpbuf = traceAppend(tmpbuf, uint64(f.pc))
843 tmpbuf = traceAppend(tmpbuf, uint64(frame.funcID))
844 tmpbuf = traceAppend(tmpbuf, uint64(frame.fileID))
845 tmpbuf = traceAppend(tmpbuf, uint64(frame.line))
847 // Now copy to the buffer.
848 size := 1 + traceBytesPerNumber + len(tmpbuf)
849 if buf := bufp.ptr(); len(buf.arr)-buf.pos < size {
850 bufp = traceFlush(bufp, 0)
853 buf.byte(traceEvStack | 3<<traceArgCountShift)
854 buf.varint(uint64(len(tmpbuf)))
855 buf.pos += copy(buf.arr[buf.pos:], tmpbuf)
864 *tab = traceStackTable{}
867 type traceFrame struct {
873 // traceFrameForPC records the frame information.
874 // It may allocate memory.
875 func traceFrameForPC(buf traceBufPtr, pid int32, f location) (traceFrame, traceBufPtr) {
880 const maxLen = 1 << 10
881 if len(fn) > maxLen {
882 fn = fn[len(fn)-maxLen:]
884 frame.funcID, bufp = traceString(bufp, pid, fn)
885 frame.line = uint64(f.lineno)
887 if len(file) > maxLen {
888 file = file[len(file)-maxLen:]
890 frame.fileID, bufp = traceString(bufp, pid, file)
891 return frame, (*bufp)
894 // traceAlloc is a non-thread-safe region allocator.
895 // It holds a linked list of traceAllocBlock.
896 type traceAlloc struct {
897 head traceAllocBlockPtr
901 // traceAllocBlock is a block in traceAlloc.
903 // traceAllocBlock is allocated from non-GC'd memory, so it must not
904 // contain heap pointers. Writes to pointers to traceAllocBlocks do
905 // not need write barriers.
908 type traceAllocBlock struct {
909 next traceAllocBlockPtr
910 data [64<<10 - sys.PtrSize]byte
913 // TODO: Since traceAllocBlock is now go:notinheap, this isn't necessary.
914 type traceAllocBlockPtr uintptr
916 func (p traceAllocBlockPtr) ptr() *traceAllocBlock { return (*traceAllocBlock)(unsafe.Pointer(p)) }
917 func (p *traceAllocBlockPtr) set(x *traceAllocBlock) { *p = traceAllocBlockPtr(unsafe.Pointer(x)) }
919 // alloc allocates n-byte block.
920 func (a *traceAlloc) alloc(n uintptr) unsafe.Pointer {
921 n = alignUp(n, sys.PtrSize)
922 if a.head == 0 || a.off+n > uintptr(len(a.head.ptr().data)) {
923 if n > uintptr(len(a.head.ptr().data)) {
924 throw("trace: alloc too large")
926 // This is only safe because the strings returned by callers
927 // are stored in a location that is not in the Go heap.
928 block := (*traceAllocBlock)(sysAlloc(unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys))
930 throw("trace: out of memory")
932 block.next.set(a.head.ptr())
936 p := &a.head.ptr().data[a.off]
938 return unsafe.Pointer(p)
941 // drop frees all previously allocated memory and resets the allocator.
942 func (a *traceAlloc) drop() {
944 block := a.head.ptr()
945 a.head.set(block.next.ptr())
946 sysFree(unsafe.Pointer(block), unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys)
950 // The following functions write specific events to trace.
952 func traceGomaxprocs(procs int32) {
953 traceEvent(traceEvGomaxprocs, 1, uint64(procs))
956 func traceProcStart() {
957 traceEvent(traceEvProcStart, -1, uint64(getg().m.id))
960 func traceProcStop(pp *p) {
961 // Sysmon and stopTheWorld can stop Ps blocked in syscalls,
962 // to handle this we temporary employ the P.
966 traceEvent(traceEvProcStop, -1)
971 func traceGCStart() {
972 traceEvent(traceEvGCStart, 3, trace.seqGC)
977 traceEvent(traceEvGCDone, -1)
980 func traceGCSTWStart(kind int) {
981 traceEvent(traceEvGCSTWStart, -1, uint64(kind))
984 func traceGCSTWDone() {
985 traceEvent(traceEvGCSTWDone, -1)
988 // traceGCSweepStart prepares to trace a sweep loop. This does not
989 // emit any events until traceGCSweepSpan is called.
991 // traceGCSweepStart must be paired with traceGCSweepDone and there
992 // must be no preemption points between these two calls.
993 func traceGCSweepStart() {
994 // Delay the actual GCSweepStart event until the first span
995 // sweep. If we don't sweep anything, don't emit any events.
996 _p_ := getg().m.p.ptr()
998 throw("double traceGCSweepStart")
1000 _p_.traceSweep, _p_.traceSwept, _p_.traceReclaimed = true, 0, 0
1003 // traceGCSweepSpan traces the sweep of a single page.
1005 // This may be called outside a traceGCSweepStart/traceGCSweepDone
1006 // pair; however, it will not emit any trace events in this case.
1007 func traceGCSweepSpan(bytesSwept uintptr) {
1008 _p_ := getg().m.p.ptr()
1010 if _p_.traceSwept == 0 {
1011 traceEvent(traceEvGCSweepStart, 1)
1013 _p_.traceSwept += bytesSwept
1017 func traceGCSweepDone() {
1018 _p_ := getg().m.p.ptr()
1019 if !_p_.traceSweep {
1020 throw("missing traceGCSweepStart")
1022 if _p_.traceSwept != 0 {
1023 traceEvent(traceEvGCSweepDone, -1, uint64(_p_.traceSwept), uint64(_p_.traceReclaimed))
1025 _p_.traceSweep = false
1028 func traceGCMarkAssistStart() {
1029 traceEvent(traceEvGCMarkAssistStart, 1)
1032 func traceGCMarkAssistDone() {
1033 traceEvent(traceEvGCMarkAssistDone, -1)
1036 func traceGoCreate(newg *g, pc uintptr) {
1038 newg.tracelastp = getg().m.p
1039 // +PCQuantum because traceFrameForPC expects return PCs and subtracts PCQuantum.
1040 id := trace.stackTab.put([]location{location{pc: pc + sys.PCQuantum}})
1041 traceEvent(traceEvGoCreate, 2, uint64(newg.goid), uint64(id))
1044 func traceGoStart() {
1045 _g_ := getg().m.curg
1048 if _g_ == _p_.ptr().gcBgMarkWorker.ptr() {
1049 traceEvent(traceEvGoStartLabel, -1, uint64(_g_.goid), _g_.traceseq, trace.markWorkerLabels[_p_.ptr().gcMarkWorkerMode])
1050 } else if _g_.tracelastp == _p_ {
1051 traceEvent(traceEvGoStartLocal, -1, uint64(_g_.goid))
1053 _g_.tracelastp = _p_
1054 traceEvent(traceEvGoStart, -1, uint64(_g_.goid), _g_.traceseq)
1059 traceEvent(traceEvGoEnd, -1)
1062 func traceGoSched() {
1064 _g_.tracelastp = _g_.m.p
1065 traceEvent(traceEvGoSched, 1)
1068 func traceGoPreempt() {
1070 _g_.tracelastp = _g_.m.p
1071 traceEvent(traceEvGoPreempt, 1)
1074 func traceGoPark(traceEv byte, skip int) {
1075 if traceEv&traceFutileWakeup != 0 {
1076 traceEvent(traceEvFutileWakeup, -1)
1078 traceEvent(traceEv & ^traceFutileWakeup, skip)
1081 func traceGoUnpark(gp *g, skip int) {
1084 if gp.tracelastp == _p_ {
1085 traceEvent(traceEvGoUnblockLocal, skip, uint64(gp.goid))
1088 traceEvent(traceEvGoUnblock, skip, uint64(gp.goid), gp.traceseq)
1092 func traceGoSysCall() {
1093 traceEvent(traceEvGoSysCall, 1)
1096 func traceGoSysExit(ts int64) {
1097 if ts != 0 && ts < trace.ticksStart {
1098 // There is a race between the code that initializes sysexitticks
1099 // (in exitsyscall, which runs without a P, and therefore is not
1100 // stopped with the rest of the world) and the code that initializes
1101 // a new trace. The recorded sysexitticks must therefore be treated
1102 // as "best effort". If they are valid for this trace, then great,
1103 // use them for greater accuracy. But if they're not valid for this
1104 // trace, assume that the trace was started after the actual syscall
1105 // exit (but before we actually managed to start the goroutine,
1106 // aka right now), and assign a fresh time stamp to keep the log consistent.
1109 _g_ := getg().m.curg
1111 _g_.tracelastp = _g_.m.p
1112 traceEvent(traceEvGoSysExit, -1, uint64(_g_.goid), _g_.traceseq, uint64(ts)/traceTickDiv)
1115 func traceGoSysBlock(pp *p) {
1116 // Sysmon and stopTheWorld can declare syscalls running on remote Ps as blocked,
1117 // to handle this we temporary employ the P.
1121 traceEvent(traceEvGoSysBlock, -1)
1126 func traceHeapAlloc() {
1127 traceEvent(traceEvHeapAlloc, -1, memstats.heap_live)
1130 func traceNextGC() {
1131 if memstats.next_gc == ^uint64(0) {
1132 // Heap-based triggering is disabled.
1133 traceEvent(traceEvNextGC, -1, 0)
1135 traceEvent(traceEvNextGC, -1, memstats.next_gc)
1139 // To access runtime functions from runtime/trace.
1140 // See runtime/trace/annotation.go
1142 //go:linkname trace_userTaskCreate runtime..z2ftrace.userTaskCreate
1143 func trace_userTaskCreate(id, parentID uint64, taskType string) {
1148 // Same as in traceEvent.
1149 mp, pid, bufp := traceAcquireBuffer()
1150 if !trace.enabled && !mp.startingtrace {
1151 traceReleaseBuffer(pid)
1155 typeStringID, bufp := traceString(bufp, pid, taskType)
1156 traceEventLocked(0, mp, pid, bufp, traceEvUserTaskCreate, 3, id, parentID, typeStringID)
1157 traceReleaseBuffer(pid)
1160 //go:linkname trace_userTaskEnd runtime..z2ftrace.userTaskEnd
1161 func trace_userTaskEnd(id uint64) {
1162 traceEvent(traceEvUserTaskEnd, 2, id)
1165 //go:linkname trace_userRegion runtime..z2ftrace.userRegion
1166 func trace_userRegion(id, mode uint64, name string) {
1171 mp, pid, bufp := traceAcquireBuffer()
1172 if !trace.enabled && !mp.startingtrace {
1173 traceReleaseBuffer(pid)
1177 nameStringID, bufp := traceString(bufp, pid, name)
1178 traceEventLocked(0, mp, pid, bufp, traceEvUserRegion, 3, id, mode, nameStringID)
1179 traceReleaseBuffer(pid)
1182 //go:linkname trace_userLog runtime..z2ftrace.userLog
1183 func trace_userLog(id uint64, category, message string) {
1188 mp, pid, bufp := traceAcquireBuffer()
1189 if !trace.enabled && !mp.startingtrace {
1190 traceReleaseBuffer(pid)
1194 categoryID, bufp := traceString(bufp, pid, category)
1196 extraSpace := traceBytesPerNumber + len(message) // extraSpace for the value string
1197 traceEventLocked(extraSpace, mp, pid, bufp, traceEvUserLog, 3, id, categoryID)
1198 // traceEventLocked reserved extra space for val and len(val)
1199 // in buf, so buf now has room for the following.
1202 // double-check the message and its length can fit.
1203 // Otherwise, truncate the message.
1204 slen := len(message)
1205 if room := len(buf.arr) - buf.pos; room < slen+traceBytesPerNumber {
1208 buf.varint(uint64(slen))
1209 buf.pos += copy(buf.arr[buf.pos:], message[:slen])
1211 traceReleaseBuffer(pid)