]>
Commit | Line | Data |
---|---|---|
77c8fdae | 1 | /* SPDX-License-Identifier: MIT */ |
8187a2b7 ZN |
2 | #ifndef _INTEL_RINGBUFFER_H_ |
3 | #define _INTEL_RINGBUFFER_H_ | |
4 | ||
d78aa650 SV |
5 | #include <drm/drm_util.h> |
6 | ||
44e895a8 | 7 | #include <linux/hashtable.h> |
52c0fdb2 | 8 | #include <linux/irq_work.h> |
89531e7d | 9 | #include <linux/random.h> |
741258cd | 10 | #include <linux/seqlock.h> |
e61e0f51 | 11 | |
b46a33e2 | 12 | #include "i915_pmu.h" |
8a68d464 | 13 | #include "i915_reg.h" |
e61e0f51 | 14 | #include "i915_request.h" |
f97fbf96 | 15 | #include "i915_selftest.h" |
f0c02c1b | 16 | #include "gt/intel_timeline.h" |
39e2f501 | 17 | #include "intel_engine_types.h" |
c080363f | 18 | #include "intel_gpu_commands.h" |
4a15c75c | 19 | #include "intel_workarounds.h" |
44e895a8 | 20 | |
f636edb2 | 21 | struct drm_printer; |
adcb5264 TU |
22 | struct intel_gt; |
23 | ||
4712274c OM |
24 | /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill, |
25 | * but keeps the logic simple. Indeed, the whole purpose of this macro is just | |
26 | * to give some inclination as to some of the magic values used in the various | |
27 | * workarounds! | |
28 | */ | |
29 | #define CACHELINE_BYTES 64 | |
739f3abd | 30 | #define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(u32)) |
4712274c | 31 | |
639f2f24 VSD |
32 | #define ENGINE_TRACE(e, fmt, ...) do { \ |
33 | const struct intel_engine_cs *e__ __maybe_unused = (e); \ | |
34 | GEM_TRACE("%s %s: " fmt, \ | |
35 | dev_name(e__->i915->drm.dev), e__->name, \ | |
36 | ##__VA_ARGS__); \ | |
37 | } while (0) | |
38 | ||
baba6e57 DCS |
39 | /* |
40 | * The register defines to be used with the following macros need to accept a | |
41 | * base param, e.g: | |
42 | * | |
43 | * REG_FOO(base) _MMIO((base) + <relative offset>) | |
44 | * ENGINE_READ(engine, REG_FOO); | |
45 | * | |
46 | * register arrays are to be defined and accessed as follows: | |
47 | * | |
48 | * REG_BAR(base, i) _MMIO((base) + <relative offset> + (i) * <shift>) | |
49 | * ENGINE_READ_IDX(engine, REG_BAR, i) | |
50 | */ | |
51 | ||
52 | #define __ENGINE_REG_OP(op__, engine__, ...) \ | |
53 | intel_uncore_##op__((engine__)->uncore, __VA_ARGS__) | |
54 | ||
55 | #define __ENGINE_READ_OP(op__, engine__, reg__) \ | |
56 | __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base)) | |
cae5852d | 57 | |
baba6e57 DCS |
58 | #define ENGINE_READ16(...) __ENGINE_READ_OP(read16, __VA_ARGS__) |
59 | #define ENGINE_READ(...) __ENGINE_READ_OP(read, __VA_ARGS__) | |
60 | #define ENGINE_READ_FW(...) __ENGINE_READ_OP(read_fw, __VA_ARGS__) | |
21de5a9e | 61 | #define ENGINE_POSTING_READ(...) __ENGINE_READ_OP(posting_read_fw, __VA_ARGS__) |
e44d62d1 | 62 | #define ENGINE_POSTING_READ16(...) __ENGINE_READ_OP(posting_read16, __VA_ARGS__) |
cae5852d | 63 | |
baba6e57 DCS |
64 | #define ENGINE_READ64(engine__, lower_reg__, upper_reg__) \ |
65 | __ENGINE_REG_OP(read64_2x32, (engine__), \ | |
66 | lower_reg__((engine__)->mmio_base), \ | |
67 | upper_reg__((engine__)->mmio_base)) | |
cae5852d | 68 | |
baba6e57 DCS |
69 | #define ENGINE_READ_IDX(engine__, reg__, idx__) \ |
70 | __ENGINE_REG_OP(read, (engine__), reg__((engine__)->mmio_base, (idx__))) | |
cae5852d | 71 | |
baba6e57 DCS |
72 | #define __ENGINE_WRITE_OP(op__, engine__, reg__, val__) \ |
73 | __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base), (val__)) | |
870e86dd | 74 | |
baba6e57 DCS |
75 | #define ENGINE_WRITE16(...) __ENGINE_WRITE_OP(write16, __VA_ARGS__) |
76 | #define ENGINE_WRITE(...) __ENGINE_WRITE_OP(write, __VA_ARGS__) | |
77 | #define ENGINE_WRITE_FW(...) __ENGINE_WRITE_OP(write_fw, __VA_ARGS__) | |
e9fea574 | 78 | |
77a302e0 TU |
79 | #define GEN6_RING_FAULT_REG_READ(engine__) \ |
80 | intel_uncore_read((engine__)->uncore, RING_FAULT_REG(engine__)) | |
81 | ||
82 | #define GEN6_RING_FAULT_REG_POSTING_READ(engine__) \ | |
83 | intel_uncore_posting_read((engine__)->uncore, RING_FAULT_REG(engine__)) | |
84 | ||
85 | #define GEN6_RING_FAULT_REG_RMW(engine__, clear__, set__) \ | |
86 | ({ \ | |
87 | u32 __val; \ | |
88 | \ | |
89 | __val = intel_uncore_read((engine__)->uncore, \ | |
90 | RING_FAULT_REG(engine__)); \ | |
91 | __val &= ~(clear__); \ | |
92 | __val |= (set__); \ | |
93 | intel_uncore_write((engine__)->uncore, RING_FAULT_REG(engine__), \ | |
94 | __val); \ | |
95 | }) | |
96 | ||
3e78998a BW |
97 | /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to |
98 | * do the writes, and that must have qw aligned offsets, simply pretend it's 8b. | |
99 | */ | |
b6b0fac0 | 100 | |
22b7a426 CW |
101 | static inline unsigned int |
102 | execlists_num_ports(const struct intel_engine_execlists * const execlists) | |
0051163a | 103 | { |
22b7a426 | 104 | return execlists->port_mask + 1; |
0051163a CW |
105 | } |
106 | ||
22b7a426 CW |
107 | static inline struct i915_request * |
108 | execlists_active(const struct intel_engine_execlists *execlists) | |
4a118ecb | 109 | { |
f494960d CW |
110 | struct i915_request * const *cur, * const *old, *active; |
111 | ||
112 | cur = READ_ONCE(execlists->active); | |
113 | smp_rmb(); /* pairs with overwrite protection in process_csb() */ | |
114 | do { | |
115 | old = cur; | |
116 | ||
117 | active = READ_ONCE(*cur); | |
118 | cur = READ_ONCE(execlists->active); | |
119 | ||
120 | smp_rmb(); /* and complete the seqlock retry */ | |
121 | } while (unlikely(cur != old)); | |
122 | ||
123 | return active; | |
4a118ecb CW |
124 | } |
125 | ||
c36eebd9 CW |
126 | static inline void |
127 | execlists_active_lock_bh(struct intel_engine_execlists *execlists) | |
128 | { | |
129 | local_bh_disable(); /* prevent local softirq and lock recursion */ | |
130 | tasklet_lock(&execlists->tasklet); | |
131 | } | |
132 | ||
133 | static inline void | |
134 | execlists_active_unlock_bh(struct intel_engine_execlists *execlists) | |
135 | { | |
136 | tasklet_unlock(&execlists->tasklet); | |
137 | local_bh_enable(); /* restore softirq, and kick ksoftirqd! */ | |
138 | } | |
139 | ||
292ad25c | 140 | struct i915_request * |
c41937fd MW |
141 | execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists); |
142 | ||
8187a2b7 | 143 | static inline u32 |
3ceda3a4 | 144 | intel_read_status_page(const struct intel_engine_cs *engine, int reg) |
8187a2b7 | 145 | { |
4225d0f2 | 146 | /* Ensure that the compiler doesn't optimize away the load. */ |
0ca88ba0 | 147 | return READ_ONCE(engine->status_page.addr[reg]); |
8187a2b7 ZN |
148 | } |
149 | ||
b70ec5bf | 150 | static inline void |
9a29dd85 | 151 | intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value) |
b70ec5bf | 152 | { |
9a29dd85 CW |
153 | /* Writing into the status page should be done sparingly. Since |
154 | * we do when we are uncertain of the device state, we take a bit | |
155 | * of extra paranoia to try and ensure that the HWS takes the value | |
156 | * we give and that it doesn't end up trapped inside the CPU! | |
157 | */ | |
158 | if (static_cpu_has(X86_FEATURE_CLFLUSH)) { | |
159 | mb(); | |
0ca88ba0 CW |
160 | clflush(&engine->status_page.addr[reg]); |
161 | engine->status_page.addr[reg] = value; | |
162 | clflush(&engine->status_page.addr[reg]); | |
9a29dd85 CW |
163 | mb(); |
164 | } else { | |
0ca88ba0 | 165 | WRITE_ONCE(engine->status_page.addr[reg], value); |
9a29dd85 | 166 | } |
b70ec5bf MK |
167 | } |
168 | ||
e2828914 | 169 | /* |
311bd68e CW |
170 | * Reads a dword out of the status page, which is written to from the command |
171 | * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or | |
172 | * MI_STORE_DATA_IMM. | |
173 | * | |
174 | * The following dwords have a reserved meaning: | |
175 | * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes. | |
176 | * 0x04: ring 0 head pointer | |
177 | * 0x05: ring 1 head pointer (915-class) | |
178 | * 0x06: ring 2 head pointer (915-class) | |
179 | * 0x10-0x1b: Context status DWords (GM45) | |
180 | * 0x1f: Last written status offset. (GM45) | |
b07da53c | 181 | * 0x20-0x2f: Reserved (Gen6+) |
311bd68e | 182 | * |
b07da53c | 183 | * The area from dword 0x30 to 0x3ff is available for driver usage. |
311bd68e | 184 | */ |
832a67bd CW |
185 | #define I915_GEM_HWS_PREEMPT 0x32 |
186 | #define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT * sizeof(u32)) | |
52954edd CW |
187 | #define I915_GEM_HWS_SEQNO 0x40 |
188 | #define I915_GEM_HWS_SEQNO_ADDR (I915_GEM_HWS_SEQNO * sizeof(u32)) | |
189 | #define I915_GEM_HWS_SCRATCH 0x80 | |
832a67bd | 190 | #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH * sizeof(u32)) |
311bd68e | 191 | |
6d2cb5aa | 192 | #define I915_HWS_CSB_BUF0_INDEX 0x10 |
767a983a CW |
193 | #define I915_HWS_CSB_WRITE_INDEX 0x1f |
194 | #define CNL_HWS_CSB_WRITE_INDEX 0x2f | |
6d2cb5aa | 195 | |
7e37f889 CW |
196 | void intel_engine_stop(struct intel_engine_cs *engine); |
197 | void intel_engine_cleanup(struct intel_engine_cs *engine); | |
96f298aa | 198 | |
adcb5264 | 199 | int intel_engines_init_mmio(struct intel_gt *gt); |
7841fcbd | 200 | int intel_engines_init(struct intel_gt *gt); |
e26b6d43 CW |
201 | |
202 | void intel_engines_release(struct intel_gt *gt); | |
203 | void intel_engines_free(struct intel_gt *gt); | |
45b9c968 | 204 | |
019bf277 | 205 | int intel_engine_init_common(struct intel_engine_cs *engine); |
96a945aa | 206 | void intel_engine_cleanup_common(struct intel_engine_cs *engine); |
019bf277 | 207 | |
faea1792 DCS |
208 | int intel_engine_resume(struct intel_engine_cs *engine); |
209 | ||
11334c6a | 210 | int intel_ring_submission_setup(struct intel_engine_cs *engine); |
8187a2b7 | 211 | |
3f6e9822 | 212 | int intel_engine_stop_cs(struct intel_engine_cs *engine); |
a99b32a6 | 213 | void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine); |
3f6e9822 | 214 | |
060f2322 CW |
215 | void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask); |
216 | ||
3ceda3a4 CW |
217 | u64 intel_engine_get_active_head(const struct intel_engine_cs *engine); |
218 | u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine); | |
1b36595f | 219 | |
742379c0 | 220 | void intel_engine_get_instdone(const struct intel_engine_cs *engine, |
0e704476 CW |
221 | struct intel_instdone *instdone); |
222 | ||
79ffac85 CW |
223 | void intel_engine_init_execlists(struct intel_engine_cs *engine); |
224 | ||
52c0fdb2 CW |
225 | void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine); |
226 | void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); | |
688e6c72 | 227 | |
52c0fdb2 | 228 | void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine); |
688e6c72 | 229 | |
52c0fdb2 | 230 | static inline void |
54400257 | 231 | intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine) |
688e6c72 | 232 | { |
52c0fdb2 | 233 | irq_work_queue(&engine->breadcrumbs.irq_work); |
688e6c72 CW |
234 | } |
235 | ||
ad07dfcd | 236 | void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine); |
688e6c72 | 237 | void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); |
688e6c72 | 238 | |
52c0fdb2 CW |
239 | void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, |
240 | struct drm_printer *p); | |
241 | ||
9f235dfa TU |
242 | static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset) |
243 | { | |
244 | memset(batch, 0, 6 * sizeof(u32)); | |
245 | ||
246 | batch[0] = GFX_OP_PIPE_CONTROL(6); | |
247 | batch[1] = flags; | |
248 | batch[2] = offset; | |
249 | ||
250 | return batch + 6; | |
251 | } | |
252 | ||
df77cd83 | 253 | static inline u32 * |
6a623729 | 254 | gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset, u32 flags) |
df77cd83 MW |
255 | { |
256 | /* We're using qword write, offset should be aligned to 8 bytes. */ | |
257 | GEM_BUG_ON(!IS_ALIGNED(gtt_offset, 8)); | |
258 | ||
259 | /* w/a for post sync ops following a GPGPU operation we | |
260 | * need a prior CS_STALL, which is emitted by the flush | |
261 | * following the batch. | |
262 | */ | |
263 | *cs++ = GFX_OP_PIPE_CONTROL(6); | |
6a623729 | 264 | *cs++ = flags | PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_GLOBAL_GTT_IVB; |
df77cd83 MW |
265 | *cs++ = gtt_offset; |
266 | *cs++ = 0; | |
267 | *cs++ = value; | |
268 | /* We're thrashing one dword of HWS. */ | |
269 | *cs++ = 0; | |
270 | ||
271 | return cs; | |
272 | } | |
273 | ||
274 | static inline u32 * | |
54939ea0 | 275 | gen8_emit_ggtt_write(u32 *cs, u32 value, u32 gtt_offset, u32 flags) |
df77cd83 MW |
276 | { |
277 | /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */ | |
278 | GEM_BUG_ON(gtt_offset & (1 << 5)); | |
279 | /* Offset should be aligned to 8 bytes for both (QW/DW) write types */ | |
280 | GEM_BUG_ON(!IS_ALIGNED(gtt_offset, 8)); | |
281 | ||
54939ea0 | 282 | *cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW | flags; |
df77cd83 MW |
283 | *cs++ = gtt_offset | MI_FLUSH_DW_USE_GTT; |
284 | *cs++ = 0; | |
285 | *cs++ = value; | |
286 | ||
287 | return cs; | |
288 | } | |
289 | ||
cb823ed9 CW |
290 | static inline void __intel_engine_reset(struct intel_engine_cs *engine, |
291 | bool stalled) | |
eb8d0f5a | 292 | { |
e26b6d43 CW |
293 | if (engine->reset.rewind) |
294 | engine->reset.rewind(engine, stalled); | |
79ffac85 | 295 | engine->serial++; /* contexts lost */ |
eb8d0f5a CW |
296 | } |
297 | ||
cb823ed9 | 298 | bool intel_engines_are_idle(struct intel_gt *gt); |
d99f7b07 | 299 | bool intel_engine_is_idle(struct intel_engine_cs *engine); |
30084b14 | 300 | void intel_engine_flush_submission(struct intel_engine_cs *engine); |
5400367a | 301 | |
cb823ed9 | 302 | void intel_engines_reset_default_submission(struct intel_gt *gt); |
ff44ad51 | 303 | |
90cad095 | 304 | bool intel_engine_can_store_dword(struct intel_engine_cs *engine); |
f2f5c061 | 305 | |
0db18b17 CW |
306 | __printf(3, 4) |
307 | void intel_engine_dump(struct intel_engine_cs *engine, | |
308 | struct drm_printer *m, | |
309 | const char *header, ...); | |
f636edb2 | 310 | |
30e17b78 TU |
311 | int intel_enable_engine_stats(struct intel_engine_cs *engine); |
312 | void intel_disable_engine_stats(struct intel_engine_cs *engine); | |
313 | ||
314 | ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine); | |
315 | ||
cf4331dd CW |
316 | struct i915_request * |
317 | intel_engine_find_active_request(struct intel_engine_cs *engine); | |
318 | ||
92c964ca | 319 | u32 intel_engine_context_size(struct intel_gt *gt, u8 class); |
ffd5ce22 | 320 | |
422d7df4 CW |
321 | void intel_engine_init_active(struct intel_engine_cs *engine, |
322 | unsigned int subclass); | |
323 | #define ENGINE_PHYSICAL 0 | |
324 | #define ENGINE_MOCK 1 | |
325 | #define ENGINE_VIRTUAL 2 | |
326 | ||
3a7a92ab CW |
327 | static inline bool |
328 | intel_engine_has_preempt_reset(const struct intel_engine_cs *engine) | |
329 | { | |
b79029b2 CW |
330 | if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT)) |
331 | return false; | |
3a7a92ab CW |
332 | |
333 | return intel_engine_has_preemption(engine); | |
334 | } | |
335 | ||
8187a2b7 | 336 | #endif /* _INTEL_RINGBUFFER_H_ */ |