drivers/gpu/drm/i915/display/intel_dsb.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  *
   5  */
   6
   7 #include "gem/i915_gem_internal.h"
   8 #include "gem/i915_gem_lmem.h"
   9
  10 #include "i915_drv.h"
  11 #include "i915_irq.h"
  12 #include "i915_reg.h"
  13 #include "intel_crtc.h"
  14 #include "intel_de.h"
  15 #include "intel_display_types.h"
  16 #include "intel_dsb.h"
  17 #include "intel_dsb_regs.h"
  18 #include "intel_vblank.h"
  19 #include "intel_vrr.h"
  20 #include "skl_watermark.h"
  21
  22 struct i915_vma;
  23
  24 enum dsb_id {
  25         INVALID_DSB = -1,
  26         DSB1,
  27         DSB2,
  28         DSB3,
  29         MAX_DSB_PER_PIPE
  30 };
  31
  32 struct intel_dsb {
  33         enum dsb_id id;
  34
  35         u32 *cmd_buf;
  36         struct i915_vma *vma;
  37         struct intel_crtc *crtc;
  38
  39         /*
  40          * maximum number of dwords the buffer will hold.
  41          */
  42         unsigned int size;
  43
  44         /*
  45          * free_pos will point the first free dword and
  46          * help in calculating tail of command buffer.
  47          */
  48         unsigned int free_pos;
  49
  50         /*
  51          * ins_start_offset will help to store start dword of the dsb
  52          * instuction and help in identifying the batch of auto-increment
  53          * register.
  54          */
  55         unsigned int ins_start_offset;
  56
  57         int dewake_scanline;
  58 };
  59
  60 /**
  61  * DOC: DSB
  62  *
  63  * A DSB (Display State Buffer) is a queue of MMIO instructions in the memory
  64  * which can be offloaded to DSB HW in Display Controller. DSB HW is a DMA
  65  * engine that can be programmed to download the DSB from memory.
  66  * It allows driver to batch submit display HW programming. This helps to
  67  * reduce loading time and CPU activity, thereby making the context switch
  68  * faster. DSB Support added from Gen12 Intel graphics based platform.
  69  *
  70  * DSB's can access only the pipe, plane, and transcoder Data Island Packet
  71  * registers.
  72  *
  73  * DSB HW can support only register writes (both indexed and direct MMIO
  74  * writes). There are no registers reads possible with DSB HW engine.
  75  */
  76
  77 /* DSB opcodes. */
  78 #define DSB_OPCODE_SHIFT                24
  79 #define DSB_OPCODE_NOOP                 0x0
  80 #define DSB_OPCODE_MMIO_WRITE           0x1
  81 #define   DSB_BYTE_EN                   0xf
  82 #define   DSB_BYTE_EN_SHIFT             20
  83 #define   DSB_REG_VALUE_MASK            0xfffff
  84 #define DSB_OPCODE_WAIT_USEC            0x2
  85 #define DSB_OPCODE_WAIT_SCANLINE        0x3
  86 #define DSB_OPCODE_WAIT_VBLANKS         0x4
  87 #define DSB_OPCODE_WAIT_DSL_IN          0x5
  88 #define DSB_OPCODE_WAIT_DSL_OUT         0x6
  89 #define   DSB_SCANLINE_UPPER_SHIFT      20
  90 #define   DSB_SCANLINE_LOWER_SHIFT      0
  91 #define DSB_OPCODE_INTERRUPT            0x7
  92 #define DSB_OPCODE_INDEXED_WRITE        0x9
  93 /* see DSB_REG_VALUE_MASK */
  94 #define DSB_OPCODE_POLL                 0xA
  95 /* see DSB_REG_VALUE_MASK */
  96
  97 static bool assert_dsb_has_room(struct intel_dsb *dsb)
  98 {
  99         struct intel_crtc *crtc = dsb->crtc;
 100         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 101
 102         /* each instruction is 2 dwords */
 103         return !drm_WARN(&i915->drm, dsb->free_pos > dsb->size - 2,
 104                          "[CRTC:%d:%s] DSB %d buffer overflow\n",
 105                          crtc->base.base.id, crtc->base.name, dsb->id);
 106 }
 107
 108 static void intel_dsb_dump(struct intel_dsb *dsb)
 109 {
 110         struct intel_crtc *crtc = dsb->crtc;
 111         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 112         const u32 *buf = dsb->cmd_buf;
 113         int i;
 114
 115         drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] DSB %d commands {\n",
 116                     crtc->base.base.id, crtc->base.name, dsb->id);
 117         for (i = 0; i < ALIGN(dsb->free_pos, 64 / 4); i += 4)
 118                 drm_dbg_kms(&i915->drm,
 119                             " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
 120                             i * 4, buf[i], buf[i+1], buf[i+2], buf[i+3]);
 121         drm_dbg_kms(&i915->drm, "}\n");
 122 }
 123
 124 static bool is_dsb_busy(struct drm_i915_private *i915, enum pipe pipe,
 125                         enum dsb_id id)
 126 {
 127         return intel_de_read_fw(i915, DSB_CTRL(pipe, id)) & DSB_STATUS_BUSY;
 128 }
 129
 130 static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
 131 {
 132         u32 *buf = dsb->cmd_buf;
 133
 134         if (!assert_dsb_has_room(dsb))
 135                 return;
 136
 137         /* Every instruction should be 8 byte aligned. */
 138         dsb->free_pos = ALIGN(dsb->free_pos, 2);
 139
 140         dsb->ins_start_offset = dsb->free_pos;
 141
 142         buf[dsb->free_pos++] = ldw;
 143         buf[dsb->free_pos++] = udw;
 144 }
 145
 146 static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
 147                                         u32 opcode, i915_reg_t reg)
 148 {
 149         const u32 *buf = dsb->cmd_buf;
 150         u32 prev_opcode, prev_reg;
 151
 152         /*
 153          * Nothing emitted yet? Must check before looking
 154          * at the actual data since i915_gem_object_create_internal()
 155          * does *not* give you zeroed memory!
 156          */
 157         if (dsb->free_pos == 0)
 158                 return false;
 159
 160         prev_opcode = buf[dsb->ins_start_offset + 1] & ~DSB_REG_VALUE_MASK;
 161         prev_reg = buf[dsb->ins_start_offset + 1] & DSB_REG_VALUE_MASK;
 162
 163         return prev_opcode == opcode && prev_reg == i915_mmio_reg_offset(reg);
 164 }
 165
 166 static bool intel_dsb_prev_ins_is_mmio_write(struct intel_dsb *dsb, i915_reg_t reg)
 167 {
 168         /* only full byte-enables can be converted to indexed writes */
 169         return intel_dsb_prev_ins_is_write(dsb,
 170                                            DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT |
 171                                            DSB_BYTE_EN << DSB_BYTE_EN_SHIFT,
 172                                            reg);
 173 }
 174
 175 static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_t reg)
 176 {
 177         return intel_dsb_prev_ins_is_write(dsb,
 178                                            DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT,
 179                                            reg);
 180 }
 181
 182 /**
 183  * intel_dsb_reg_write() - Emit register wriite to the DSB context
 184  * @dsb: DSB context
 185  * @reg: register address.
 186  * @val: value.
 187  *
 188  * This function is used for writing register-value pair in command
 189  * buffer of DSB.
 190  */
 191 void intel_dsb_reg_write(struct intel_dsb *dsb,
 192                          i915_reg_t reg, u32 val)
 193 {
 194         /*
 195          * For example the buffer will look like below for 3 dwords for auto
 196          * increment register:
 197          * +--------------------------------------------------------+
 198          * | size = 3 | offset &| value1 | value2 | value3 | zero   |
 199          * |          | opcode  |        |        |        |        |
 200          * +--------------------------------------------------------+
 201          * +          +         +        +        +        +        +
 202          * 0          4         8        12       16       20       24
 203          * Byte
 204          *
 205          * As every instruction is 8 byte aligned the index of dsb instruction
 206          * will start always from even number while dealing with u32 array. If
 207          * we are writing odd no of dwords, Zeros will be added in the end for
 208          * padding.
 209          */
 210         if (!intel_dsb_prev_ins_is_mmio_write(dsb, reg) &&
 211             !intel_dsb_prev_ins_is_indexed_write(dsb, reg)) {
 212                 intel_dsb_emit(dsb, val,
 213                                (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
 214                                (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) |
 215                                i915_mmio_reg_offset(reg));
 216         } else {
 217                 u32 *buf = dsb->cmd_buf;
 218
 219                 if (!assert_dsb_has_room(dsb))
 220                         return;
 221
 222                 /* convert to indexed write? */
 223                 if (intel_dsb_prev_ins_is_mmio_write(dsb, reg)) {
 224                         u32 prev_val = buf[dsb->ins_start_offset + 0];
 225
 226                         buf[dsb->ins_start_offset + 0] = 1; /* count */
 227                         buf[dsb->ins_start_offset + 1] =
 228                                 (DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
 229                                 i915_mmio_reg_offset(reg);
 230                         buf[dsb->ins_start_offset + 2] = prev_val;
 231
 232                         dsb->free_pos++;
 233                 }
 234
 235                 buf[dsb->free_pos++] = val;
 236                 /* Update the count */
 237                 buf[dsb->ins_start_offset]++;
 238
 239                 /* if number of data words is odd, then the last dword should be 0.*/
 240                 if (dsb->free_pos & 0x1)
 241                         buf[dsb->free_pos] = 0;
 242         }
 243 }
 244
 245 static u32 intel_dsb_mask_to_byte_en(u32 mask)
 246 {
 247         return (!!(mask & 0xff000000) << 3 |
 248                 !!(mask & 0x00ff0000) << 2 |
 249                 !!(mask & 0x0000ff00) << 1 |
 250                 !!(mask & 0x000000ff) << 0);
 251 }
 252
 253 /* Note: mask implemented via byte enables! */
 254 void intel_dsb_reg_write_masked(struct intel_dsb *dsb,
 255                                 i915_reg_t reg, u32 mask, u32 val)
 256 {
 257         intel_dsb_emit(dsb, val,
 258                        (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
 259                        (intel_dsb_mask_to_byte_en(mask) << DSB_BYTE_EN_SHIFT) |
 260                        i915_mmio_reg_offset(reg));
 261 }
 262
 263 void intel_dsb_noop(struct intel_dsb *dsb, int count)
 264 {
 265         int i;
 266
 267         for (i = 0; i < count; i++)
 268                 intel_dsb_emit(dsb, 0,
 269                                DSB_OPCODE_NOOP << DSB_OPCODE_SHIFT);
 270 }
 271
 272 void intel_dsb_nonpost_start(struct intel_dsb *dsb)
 273 {
 274         struct intel_crtc *crtc = dsb->crtc;
 275         enum pipe pipe = crtc->pipe;
 276
 277         intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id),
 278                                    DSB_NON_POSTED, DSB_NON_POSTED);
 279         intel_dsb_noop(dsb, 4);
 280 }
 281
 282 void intel_dsb_nonpost_end(struct intel_dsb *dsb)
 283 {
 284         struct intel_crtc *crtc = dsb->crtc;
 285         enum pipe pipe = crtc->pipe;
 286
 287         intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id),
 288                                    DSB_NON_POSTED, 0);
 289         intel_dsb_noop(dsb, 4);
 290 }
 291
 292 static void intel_dsb_align_tail(struct intel_dsb *dsb)
 293 {
 294         u32 aligned_tail, tail;
 295
 296         tail = dsb->free_pos * 4;
 297         aligned_tail = ALIGN(tail, CACHELINE_BYTES);
 298
 299         if (aligned_tail > tail)
 300                 memset(&dsb->cmd_buf[dsb->free_pos], 0,
 301                        aligned_tail - tail);
 302
 303         dsb->free_pos = aligned_tail / 4;
 304 }
 305
 306 void intel_dsb_finish(struct intel_dsb *dsb)
 307 {
 308         struct intel_crtc *crtc = dsb->crtc;
 309
 310         /*
 311          * DSB_FORCE_DEWAKE remains active even after DSB is
 312          * disabled, so make sure to clear it (if set during
 313          * intel_dsb_commit()).
 314          */
 315         intel_dsb_reg_write_masked(dsb, DSB_PMCTRL_2(crtc->pipe, dsb->id),
 316                                    DSB_FORCE_DEWAKE, 0);
 317
 318         intel_dsb_align_tail(dsb);
 319
 320         i915_gem_object_flush_map(dsb->vma->obj);
 321 }
 322
 323 static int intel_dsb_dewake_scanline(const struct intel_crtc_state *crtc_state)
 324 {
 325         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 326         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 327         unsigned int latency = skl_watermark_max_latency(i915);
 328         int vblank_start;
 329
 330         if (crtc_state->vrr.enable) {
 331                 vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
 332         } else {
 333                 vblank_start = adjusted_mode->crtc_vblank_start;
 334
 335                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
 336                         vblank_start = DIV_ROUND_UP(vblank_start, 2);
 337         }
 338
 339         return max(0, vblank_start - intel_usecs_to_scanlines(adjusted_mode, latency));
 340 }
 341
 342 static void _intel_dsb_commit(struct intel_dsb *dsb, u32 ctrl,
 343                               int dewake_scanline)
 344 {
 345         struct intel_crtc *crtc = dsb->crtc;
 346         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 347         enum pipe pipe = crtc->pipe;
 348         u32 tail;
 349
 350         tail = dsb->free_pos * 4;
 351         if (drm_WARN_ON(&dev_priv->drm, !IS_ALIGNED(tail, CACHELINE_BYTES)))
 352                 return;
 353
 354         if (is_dsb_busy(dev_priv, pipe, dsb->id)) {
 355                 drm_err(&dev_priv->drm, "[CRTC:%d:%s] DSB %d is busy\n",
 356                         crtc->base.base.id, crtc->base.name, dsb->id);
 357                 return;
 358         }
 359
 360         intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id),
 361                           ctrl | DSB_ENABLE);
 362
 363         intel_de_write_fw(dev_priv, DSB_HEAD(pipe, dsb->id),
 364                           i915_ggtt_offset(dsb->vma));
 365
 366         if (dewake_scanline >= 0) {
 367                 int diff, hw_dewake_scanline;
 368
 369                 hw_dewake_scanline = intel_crtc_scanline_to_hw(crtc, dewake_scanline);
 370
 371                 intel_de_write_fw(dev_priv, DSB_PMCTRL(pipe, dsb->id),
 372                                   DSB_ENABLE_DEWAKE |
 373                                   DSB_SCANLINE_FOR_DEWAKE(hw_dewake_scanline));
 374
 375                 /*
 376                  * Force DEwake immediately if we're already past
 377                  * or close to racing past the target scanline.
 378                  */
 379                 diff = dewake_scanline - intel_get_crtc_scanline(crtc);
 380                 intel_de_write_fw(dev_priv, DSB_PMCTRL_2(pipe, dsb->id),
 381                                   (diff >= 0 && diff < 5 ? DSB_FORCE_DEWAKE : 0) |
 382                                   DSB_BLOCK_DEWAKE_EXTENSION);
 383         }
 384
 385         intel_de_write_fw(dev_priv, DSB_TAIL(pipe, dsb->id),
 386                           i915_ggtt_offset(dsb->vma) + tail);
 387 }
 388
 389 /**
 390  * intel_dsb_commit() - Trigger workload execution of DSB.
 391  * @dsb: DSB context
 392  * @wait_for_vblank: wait for vblank before executing
 393  *
 394  * This function is used to do actual write to hardware using DSB.
 395  */
 396 void intel_dsb_commit(struct intel_dsb *dsb,
 397                       bool wait_for_vblank)
 398 {
 399         _intel_dsb_commit(dsb,
 400                           wait_for_vblank ? DSB_WAIT_FOR_VBLANK : 0,
 401                           wait_for_vblank ? dsb->dewake_scanline : -1);
 402 }
 403
 404 void intel_dsb_wait(struct intel_dsb *dsb)
 405 {
 406         struct intel_crtc *crtc = dsb->crtc;
 407         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 408         enum pipe pipe = crtc->pipe;
 409
 410         if (wait_for(!is_dsb_busy(dev_priv, pipe, dsb->id), 1)) {
 411                 u32 offset = i915_ggtt_offset(dsb->vma);
 412
 413                 intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id),
 414                                   DSB_ENABLE | DSB_HALT);
 415
 416                 drm_err(&dev_priv->drm,
 417                         "[CRTC:%d:%s] DSB %d timed out waiting for idle (current head=0x%x, head=0x%x, tail=0x%x)\n",
 418                         crtc->base.base.id, crtc->base.name, dsb->id,
 419                         intel_de_read_fw(dev_priv, DSB_CURRENT_HEAD(pipe, dsb->id)) - offset,
 420                         intel_de_read_fw(dev_priv, DSB_HEAD(pipe, dsb->id)) - offset,
 421                         intel_de_read_fw(dev_priv, DSB_TAIL(pipe, dsb->id)) - offset);
 422
 423                 intel_dsb_dump(dsb);
 424         }
 425
 426         /* Attempt to reset it */
 427         dsb->free_pos = 0;
 428         dsb->ins_start_offset = 0;
 429         intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id), 0);
 430 }
 431
 432 /**
 433  * intel_dsb_prepare() - Allocate, pin and map the DSB command buffer.
 434  * @crtc_state: the CRTC state
 435  * @max_cmds: number of commands we need to fit into command buffer
 436  *
 437  * This function prepare the command buffer which is used to store dsb
 438  * instructions with data.
 439  *
 440  * Returns:
 441  * DSB context, NULL on failure
 442  */
 443 struct intel_dsb *intel_dsb_prepare(const struct intel_crtc_state *crtc_state,
 444                                     unsigned int max_cmds)
 445 {
 446         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 447         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 448         struct drm_i915_gem_object *obj;
 449         intel_wakeref_t wakeref;
 450         struct intel_dsb *dsb;
 451         struct i915_vma *vma;
 452         unsigned int size;
 453         u32 *buf;
 454
 455         if (!HAS_DSB(i915))
 456                 return NULL;
 457
 458         dsb = kzalloc(sizeof(*dsb), GFP_KERNEL);
 459         if (!dsb)
 460                 goto out;
 461
 462         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 463
 464         /* ~1 qword per instruction, full cachelines */
 465         size = ALIGN(max_cmds * 8, CACHELINE_BYTES);
 466
 467         if (HAS_LMEM(i915)) {
 468                 obj = i915_gem_object_create_lmem(i915, PAGE_ALIGN(size),
 469                                                   I915_BO_ALLOC_CONTIGUOUS);
 470                 if (IS_ERR(obj))
 471                         goto out_put_rpm;
 472         } else {
 473                 obj = i915_gem_object_create_internal(i915, PAGE_ALIGN(size));
 474                 if (IS_ERR(obj))
 475                         goto out_put_rpm;
 476
 477                 i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
 478         }
 479
 480         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
 481         if (IS_ERR(vma)) {
 482                 i915_gem_object_put(obj);
 483                 goto out_put_rpm;
 484         }
 485
 486         buf = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
 487         if (IS_ERR(buf)) {
 488                 i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);
 489                 goto out_put_rpm;
 490         }
 491
 492         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 493
 494         dsb->id = DSB1;
 495         dsb->vma = vma;
 496         dsb->crtc = crtc;
 497         dsb->cmd_buf = buf;
 498         dsb->size = size / 4; /* in dwords */
 499         dsb->free_pos = 0;
 500         dsb->ins_start_offset = 0;
 501         dsb->dewake_scanline = intel_dsb_dewake_scanline(crtc_state);
 502
 503         return dsb;
 504
 505 out_put_rpm:
 506         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 507         kfree(dsb);
 508 out:
 509         drm_info_once(&i915->drm,
 510                       "[CRTC:%d:%s] DSB %d queue setup failed, will fallback to MMIO for display HW programming\n",
 511                       crtc->base.base.id, crtc->base.name, DSB1);
 512
 513         return NULL;
 514 }
 515
 516 /**
 517  * intel_dsb_cleanup() - To cleanup DSB context.
 518  * @dsb: DSB context
 519  *
 520  * This function cleanup the DSB context by unpinning and releasing
 521  * the VMA object associated with it.
 522  */
 523 void intel_dsb_cleanup(struct intel_dsb *dsb)
 524 {
 525         i915_vma_unpin_and_release(&dsb->vma, I915_VMA_RELEASE_MAP);
 526         kfree(dsb);
 527 }