drivers/gpu/drm/i915/gt/uc/intel_uc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2016-2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7
   8 #include "gt/intel_gt.h"
   9 #include "gt/intel_gt_print.h"
  10 #include "gt/intel_reset.h"
  11 #include "intel_gsc_fw.h"
  12 #include "intel_gsc_uc.h"
  13 #include "intel_guc.h"
  14 #include "intel_guc_ads.h"
  15 #include "intel_guc_print.h"
  16 #include "intel_guc_submission.h"
  17 #include "gt/intel_rps.h"
  18 #include "intel_uc.h"
  19
  20 #include "i915_drv.h"
  21 #include "i915_hwmon.h"
  22
  23 static const struct intel_uc_ops uc_ops_off;
  24 static const struct intel_uc_ops uc_ops_on;
  25
  26 static void uc_expand_default_options(struct intel_uc *uc)
  27 {
  28         struct drm_i915_private *i915 = uc_to_gt(uc)->i915;
  29
  30         if (i915->params.enable_guc != -1)
  31                 return;
  32
  33         /* Don't enable GuC/HuC on pre-Gen12 */
  34         if (GRAPHICS_VER(i915) < 12) {
  35                 i915->params.enable_guc = 0;
  36                 return;
  37         }
  38
  39         /* Don't enable GuC/HuC on older Gen12 platforms */
  40         if (IS_TIGERLAKE(i915) || IS_ROCKETLAKE(i915)) {
  41                 i915->params.enable_guc = 0;
  42                 return;
  43         }
  44
  45         /* Intermediate platforms are HuC authentication only */
  46         if (IS_ALDERLAKE_S(i915) && !IS_RAPTORLAKE_S(i915)) {
  47                 i915->params.enable_guc = ENABLE_GUC_LOAD_HUC;
  48                 return;
  49         }
  50
  51         /* Default: enable HuC authentication and GuC submission */
  52         i915->params.enable_guc = ENABLE_GUC_LOAD_HUC | ENABLE_GUC_SUBMISSION;
  53
  54         /* XEHPSDV and PVC do not use HuC */
  55         if (IS_XEHPSDV(i915) || IS_PONTEVECCHIO(i915))
  56                 i915->params.enable_guc &= ~ENABLE_GUC_LOAD_HUC;
  57 }
  58
  59 /* Reset GuC providing us with fresh state for both GuC and HuC.
  60  */
  61 static int __intel_uc_reset_hw(struct intel_uc *uc)
  62 {
  63         struct intel_gt *gt = uc_to_gt(uc);
  64         int ret;
  65         u32 guc_status;
  66
  67         ret = i915_inject_probe_error(gt->i915, -ENXIO);
  68         if (ret)
  69                 return ret;
  70
  71         ret = intel_reset_guc(gt);
  72         if (ret) {
  73                 gt_err(gt, "Failed to reset GuC, ret = %d\n", ret);
  74                 return ret;
  75         }
  76
  77         guc_status = intel_uncore_read(gt->uncore, GUC_STATUS);
  78         gt_WARN(gt, !(guc_status & GS_MIA_IN_RESET),
  79                 "GuC status: 0x%x, MIA core expected to be in reset\n",
  80                 guc_status);
  81
  82         return ret;
  83 }
  84
  85 static void __confirm_options(struct intel_uc *uc)
  86 {
  87         struct intel_gt *gt = uc_to_gt(uc);
  88         struct drm_i915_private *i915 = gt->i915;
  89
  90         gt_dbg(gt, "enable_guc=%d (guc:%s submission:%s huc:%s slpc:%s)\n",
  91                i915->params.enable_guc,
  92                str_yes_no(intel_uc_wants_guc(uc)),
  93                str_yes_no(intel_uc_wants_guc_submission(uc)),
  94                str_yes_no(intel_uc_wants_huc(uc)),
  95                str_yes_no(intel_uc_wants_guc_slpc(uc)));
  96
  97         if (i915->params.enable_guc == 0) {
  98                 GEM_BUG_ON(intel_uc_wants_guc(uc));
  99                 GEM_BUG_ON(intel_uc_wants_guc_submission(uc));
 100                 GEM_BUG_ON(intel_uc_wants_huc(uc));
 101                 GEM_BUG_ON(intel_uc_wants_guc_slpc(uc));
 102                 return;
 103         }
 104
 105         if (!intel_uc_supports_guc(uc))
 106                 gt_info(gt,  "Incompatible option enable_guc=%d - %s\n",
 107                         i915->params.enable_guc, "GuC is not supported!");
 108
 109         if (i915->params.enable_guc & ENABLE_GUC_SUBMISSION &&
 110             !intel_uc_supports_guc_submission(uc))
 111                 gt_info(gt, "Incompatible option enable_guc=%d - %s\n",
 112                         i915->params.enable_guc, "GuC submission is N/A");
 113
 114         if (i915->params.enable_guc & ~ENABLE_GUC_MASK)
 115                 gt_info(gt, "Incompatible option enable_guc=%d - %s\n",
 116                         i915->params.enable_guc, "undocumented flag");
 117 }
 118
 119 void intel_uc_init_early(struct intel_uc *uc)
 120 {
 121         uc_expand_default_options(uc);
 122
 123         intel_guc_init_early(&uc->guc);
 124         intel_huc_init_early(&uc->huc);
 125         intel_gsc_uc_init_early(&uc->gsc);
 126
 127         __confirm_options(uc);
 128
 129         if (intel_uc_wants_guc(uc))
 130                 uc->ops = &uc_ops_on;
 131         else
 132                 uc->ops = &uc_ops_off;
 133 }
 134
 135 void intel_uc_init_late(struct intel_uc *uc)
 136 {
 137         intel_guc_init_late(&uc->guc);
 138         intel_gsc_uc_load_start(&uc->gsc);
 139 }
 140
 141 void intel_uc_driver_late_release(struct intel_uc *uc)
 142 {
 143 }
 144
 145 /**
 146  * intel_uc_init_mmio - setup uC MMIO access
 147  * @uc: the intel_uc structure
 148  *
 149  * Setup minimal state necessary for MMIO accesses later in the
 150  * initialization sequence.
 151  */
 152 void intel_uc_init_mmio(struct intel_uc *uc)
 153 {
 154         intel_guc_init_send_regs(&uc->guc);
 155 }
 156
 157 static void __uc_capture_load_err_log(struct intel_uc *uc)
 158 {
 159         struct intel_guc *guc = &uc->guc;
 160
 161         if (guc->log.vma && !uc->load_err_log)
 162                 uc->load_err_log = i915_gem_object_get(guc->log.vma->obj);
 163 }
 164
 165 static void __uc_free_load_err_log(struct intel_uc *uc)
 166 {
 167         struct drm_i915_gem_object *log = fetch_and_zero(&uc->load_err_log);
 168
 169         if (log)
 170                 i915_gem_object_put(log);
 171 }
 172
 173 void intel_uc_driver_remove(struct intel_uc *uc)
 174 {
 175         intel_uc_fini_hw(uc);
 176         intel_uc_fini(uc);
 177         __uc_free_load_err_log(uc);
 178 }
 179
 180 /*
 181  * Events triggered while CT buffers are disabled are logged in the SCRATCH_15
 182  * register using the same bits used in the CT message payload. Since our
 183  * communication channel with guc is turned off at this point, we can save the
 184  * message and handle it after we turn it back on.
 185  */
 186 static void guc_clear_mmio_msg(struct intel_guc *guc)
 187 {
 188         intel_uncore_write(guc_to_gt(guc)->uncore, SOFT_SCRATCH(15), 0);
 189 }
 190
 191 static void guc_get_mmio_msg(struct intel_guc *guc)
 192 {
 193         u32 val;
 194
 195         spin_lock_irq(&guc->irq_lock);
 196
 197         val = intel_uncore_read(guc_to_gt(guc)->uncore, SOFT_SCRATCH(15));
 198         guc->mmio_msg |= val & guc->msg_enabled_mask;
 199
 200         /*
 201          * clear all events, including the ones we're not currently servicing,
 202          * to make sure we don't try to process a stale message if we enable
 203          * handling of more events later.
 204          */
 205         guc_clear_mmio_msg(guc);
 206
 207         spin_unlock_irq(&guc->irq_lock);
 208 }
 209
 210 static void guc_handle_mmio_msg(struct intel_guc *guc)
 211 {
 212         /* we need communication to be enabled to reply to GuC */
 213         GEM_BUG_ON(!intel_guc_ct_enabled(&guc->ct));
 214
 215         spin_lock_irq(&guc->irq_lock);
 216         if (guc->mmio_msg) {
 217                 intel_guc_to_host_process_recv_msg(guc, &guc->mmio_msg, 1);
 218                 guc->mmio_msg = 0;
 219         }
 220         spin_unlock_irq(&guc->irq_lock);
 221 }
 222
 223 static int guc_enable_communication(struct intel_guc *guc)
 224 {
 225         struct intel_gt *gt = guc_to_gt(guc);
 226         struct drm_i915_private *i915 = gt->i915;
 227         int ret;
 228
 229         GEM_BUG_ON(intel_guc_ct_enabled(&guc->ct));
 230
 231         ret = i915_inject_probe_error(i915, -ENXIO);
 232         if (ret)
 233                 return ret;
 234
 235         ret = intel_guc_ct_enable(&guc->ct);
 236         if (ret)
 237                 return ret;
 238
 239         /* check for mmio messages received before/during the CT enable */
 240         guc_get_mmio_msg(guc);
 241         guc_handle_mmio_msg(guc);
 242
 243         intel_guc_enable_interrupts(guc);
 244
 245         /* check for CT messages received before we enabled interrupts */
 246         spin_lock_irq(gt->irq_lock);
 247         intel_guc_ct_event_handler(&guc->ct);
 248         spin_unlock_irq(gt->irq_lock);
 249
 250         guc_dbg(guc, "communication enabled\n");
 251
 252         return 0;
 253 }
 254
 255 static void guc_disable_communication(struct intel_guc *guc)
 256 {
 257         /*
 258          * Events generated during or after CT disable are logged by guc in
 259          * via mmio. Make sure the register is clear before disabling CT since
 260          * all events we cared about have already been processed via CT.
 261          */
 262         guc_clear_mmio_msg(guc);
 263
 264         intel_guc_disable_interrupts(guc);
 265
 266         intel_guc_ct_disable(&guc->ct);
 267
 268         /*
 269          * Check for messages received during/after the CT disable. We do not
 270          * expect any messages to have arrived via CT between the interrupt
 271          * disable and the CT disable because GuC should've been idle until we
 272          * triggered the CT disable protocol.
 273          */
 274         guc_get_mmio_msg(guc);
 275
 276         guc_dbg(guc, "communication disabled\n");
 277 }
 278
 279 static void __uc_fetch_firmwares(struct intel_uc *uc)
 280 {
 281         struct intel_gt *gt = uc_to_gt(uc);
 282         int err;
 283
 284         GEM_BUG_ON(!intel_uc_wants_guc(uc));
 285
 286         err = intel_uc_fw_fetch(&uc->guc.fw);
 287         if (err) {
 288                 /* Make sure we transition out of transient "SELECTED" state */
 289                 if (intel_uc_wants_huc(uc)) {
 290                         gt_dbg(gt, "Failed to fetch GuC fw (%pe) disabling HuC\n", ERR_PTR(err));
 291                         intel_uc_fw_change_status(&uc->huc.fw,
 292                                                   INTEL_UC_FIRMWARE_ERROR);
 293                 }
 294
 295                 if (intel_uc_wants_gsc_uc(uc)) {
 296                         gt_dbg(gt, "Failed to fetch GuC fw (%pe) disabling GSC\n", ERR_PTR(err));
 297                         intel_uc_fw_change_status(&uc->gsc.fw,
 298                                                   INTEL_UC_FIRMWARE_ERROR);
 299                 }
 300
 301                 return;
 302         }
 303
 304         if (intel_uc_wants_huc(uc))
 305                 intel_uc_fw_fetch(&uc->huc.fw);
 306
 307         if (intel_uc_wants_gsc_uc(uc))
 308                 intel_uc_fw_fetch(&uc->gsc.fw);
 309 }
 310
 311 static void __uc_cleanup_firmwares(struct intel_uc *uc)
 312 {
 313         intel_uc_fw_cleanup_fetch(&uc->gsc.fw);
 314         intel_uc_fw_cleanup_fetch(&uc->huc.fw);
 315         intel_uc_fw_cleanup_fetch(&uc->guc.fw);
 316 }
 317
 318 static int __uc_init(struct intel_uc *uc)
 319 {
 320         struct intel_guc *guc = &uc->guc;
 321         struct intel_huc *huc = &uc->huc;
 322         int ret;
 323
 324         GEM_BUG_ON(!intel_uc_wants_guc(uc));
 325
 326         if (!intel_uc_uses_guc(uc))
 327                 return 0;
 328
 329         if (i915_inject_probe_failure(uc_to_gt(uc)->i915))
 330                 return -ENOMEM;
 331
 332         ret = intel_guc_init(guc);
 333         if (ret)
 334                 return ret;
 335
 336         if (intel_uc_uses_huc(uc))
 337                 intel_huc_init(huc);
 338
 339         if (intel_uc_uses_gsc_uc(uc))
 340                 intel_gsc_uc_init(&uc->gsc);
 341
 342         return 0;
 343 }
 344
 345 static void __uc_fini(struct intel_uc *uc)
 346 {
 347         intel_gsc_uc_fini(&uc->gsc);
 348         intel_huc_fini(&uc->huc);
 349         intel_guc_fini(&uc->guc);
 350 }
 351
 352 static int __uc_sanitize(struct intel_uc *uc)
 353 {
 354         struct intel_guc *guc = &uc->guc;
 355         struct intel_huc *huc = &uc->huc;
 356
 357         GEM_BUG_ON(!intel_uc_supports_guc(uc));
 358
 359         intel_huc_sanitize(huc);
 360         intel_guc_sanitize(guc);
 361
 362         return __intel_uc_reset_hw(uc);
 363 }
 364
 365 /* Initialize and verify the uC regs related to uC positioning in WOPCM */
 366 static int uc_init_wopcm(struct intel_uc *uc)
 367 {
 368         struct intel_gt *gt = uc_to_gt(uc);
 369         struct intel_uncore *uncore = gt->uncore;
 370         u32 base = intel_wopcm_guc_base(&gt->wopcm);
 371         u32 size = intel_wopcm_guc_size(&gt->wopcm);
 372         u32 huc_agent = intel_uc_uses_huc(uc) ? HUC_LOADING_AGENT_GUC : 0;
 373         u32 mask;
 374         int err;
 375
 376         if (unlikely(!base || !size)) {
 377                 gt_probe_error(gt, "Unsuccessful WOPCM partitioning\n");
 378                 return -E2BIG;
 379         }
 380
 381         GEM_BUG_ON(!intel_uc_supports_guc(uc));
 382         GEM_BUG_ON(!(base & GUC_WOPCM_OFFSET_MASK));
 383         GEM_BUG_ON(base & ~GUC_WOPCM_OFFSET_MASK);
 384         GEM_BUG_ON(!(size & GUC_WOPCM_SIZE_MASK));
 385         GEM_BUG_ON(size & ~GUC_WOPCM_SIZE_MASK);
 386
 387         err = i915_inject_probe_error(gt->i915, -ENXIO);
 388         if (err)
 389                 return err;
 390
 391         mask = GUC_WOPCM_SIZE_MASK | GUC_WOPCM_SIZE_LOCKED;
 392         err = intel_uncore_write_and_verify(uncore, GUC_WOPCM_SIZE, size, mask,
 393                                             size | GUC_WOPCM_SIZE_LOCKED);
 394         if (err)
 395                 goto err_out;
 396
 397         mask = GUC_WOPCM_OFFSET_MASK | GUC_WOPCM_OFFSET_VALID | huc_agent;
 398         err = intel_uncore_write_and_verify(uncore, DMA_GUC_WOPCM_OFFSET,
 399                                             base | huc_agent, mask,
 400                                             base | huc_agent |
 401                                             GUC_WOPCM_OFFSET_VALID);
 402         if (err)
 403                 goto err_out;
 404
 405         return 0;
 406
 407 err_out:
 408         gt_probe_error(gt, "Failed to init uC WOPCM registers!\n");
 409         gt_probe_error(gt, "%s(%#x)=%#x\n", "DMA_GUC_WOPCM_OFFSET",
 410                        i915_mmio_reg_offset(DMA_GUC_WOPCM_OFFSET),
 411                        intel_uncore_read(uncore, DMA_GUC_WOPCM_OFFSET));
 412         gt_probe_error(gt, "%s(%#x)=%#x\n", "GUC_WOPCM_SIZE",
 413                        i915_mmio_reg_offset(GUC_WOPCM_SIZE),
 414                        intel_uncore_read(uncore, GUC_WOPCM_SIZE));
 415
 416         return err;
 417 }
 418
 419 static bool uc_is_wopcm_locked(struct intel_uc *uc)
 420 {
 421         struct intel_gt *gt = uc_to_gt(uc);
 422         struct intel_uncore *uncore = gt->uncore;
 423
 424         return (intel_uncore_read(uncore, GUC_WOPCM_SIZE) & GUC_WOPCM_SIZE_LOCKED) ||
 425                (intel_uncore_read(uncore, DMA_GUC_WOPCM_OFFSET) & GUC_WOPCM_OFFSET_VALID);
 426 }
 427
 428 static int __uc_check_hw(struct intel_uc *uc)
 429 {
 430         if (uc->fw_table_invalid)
 431                 return -EIO;
 432
 433         if (!intel_uc_supports_guc(uc))
 434                 return 0;
 435
 436         /*
 437          * We can silently continue without GuC only if it was never enabled
 438          * before on this system after reboot, otherwise we risk GPU hangs.
 439          * To check if GuC was loaded before we look at WOPCM registers.
 440          */
 441         if (uc_is_wopcm_locked(uc))
 442                 return -EIO;
 443
 444         return 0;
 445 }
 446
 447 static void print_fw_ver(struct intel_gt *gt, struct intel_uc_fw *fw)
 448 {
 449         gt_info(gt, "%s firmware %s version %u.%u.%u\n",
 450                 intel_uc_fw_type_repr(fw->type), fw->file_selected.path,
 451                 fw->file_selected.ver.major,
 452                 fw->file_selected.ver.minor,
 453                 fw->file_selected.ver.patch);
 454 }
 455
 456 static int __uc_init_hw(struct intel_uc *uc)
 457 {
 458         struct intel_gt *gt = uc_to_gt(uc);
 459         struct drm_i915_private *i915 = gt->i915;
 460         struct intel_guc *guc = &uc->guc;
 461         struct intel_huc *huc = &uc->huc;
 462         int ret, attempts;
 463         bool pl1en = false;
 464
 465         GEM_BUG_ON(!intel_uc_supports_guc(uc));
 466         GEM_BUG_ON(!intel_uc_wants_guc(uc));
 467
 468         print_fw_ver(gt, &guc->fw);
 469
 470         if (intel_uc_uses_huc(uc))
 471                 print_fw_ver(gt, &huc->fw);
 472
 473         if (!intel_uc_fw_is_loadable(&guc->fw)) {
 474                 ret = __uc_check_hw(uc) ||
 475                       intel_uc_fw_is_overridden(&guc->fw) ||
 476                       intel_uc_wants_guc_submission(uc) ?
 477                       intel_uc_fw_status_to_error(guc->fw.status) : 0;
 478                 goto err_out;
 479         }
 480
 481         ret = uc_init_wopcm(uc);
 482         if (ret)
 483                 goto err_out;
 484
 485         intel_guc_reset_interrupts(guc);
 486
 487         /* WaEnableuKernelHeaderValidFix:skl */
 488         /* WaEnableGuCBootHashCheckNotSet:skl,bxt,kbl */
 489         if (GRAPHICS_VER(i915) == 9)
 490                 attempts = 3;
 491         else
 492                 attempts = 1;
 493
 494         /* Disable a potentially low PL1 power limit to allow freq to be raised */
 495         i915_hwmon_power_max_disable(gt->i915, &pl1en);
 496
 497         intel_rps_raise_unslice(&uc_to_gt(uc)->rps);
 498
 499         while (attempts--) {
 500                 /*
 501                  * Always reset the GuC just before (re)loading, so
 502                  * that the state and timing are fairly predictable
 503                  */
 504                 ret = __uc_sanitize(uc);
 505                 if (ret)
 506                         goto err_rps;
 507
 508                 intel_huc_fw_upload(huc);
 509                 intel_guc_ads_reset(guc);
 510                 intel_guc_write_params(guc);
 511                 ret = intel_guc_fw_upload(guc);
 512                 if (ret == 0)
 513                         break;
 514
 515                 gt_dbg(gt, "GuC fw load failed (%pe) will reset and retry %d more time(s)\n",
 516                        ERR_PTR(ret), attempts);
 517         }
 518
 519         /* Did we succeded or run out of retries? */
 520         if (ret)
 521                 goto err_log_capture;
 522
 523         ret = guc_enable_communication(guc);
 524         if (ret)
 525                 goto err_log_capture;
 526
 527         /*
 528          * GSC-loaded HuC is authenticated by the GSC, so we don't need to
 529          * trigger the auth here. However, given that the HuC loaded this way
 530          * survive GT reset, we still need to update our SW bookkeeping to make
 531          * sure it reflects the correct HW status.
 532          */
 533         if (intel_huc_is_loaded_by_gsc(huc))
 534                 intel_huc_update_auth_status(huc);
 535         else
 536                 intel_huc_auth(huc, INTEL_HUC_AUTH_BY_GUC);
 537
 538         if (intel_uc_uses_guc_submission(uc)) {
 539                 ret = intel_guc_submission_enable(guc);
 540                 if (ret)
 541                         goto err_log_capture;
 542         }
 543
 544         if (intel_uc_uses_guc_slpc(uc)) {
 545                 ret = intel_guc_slpc_enable(&guc->slpc);
 546                 if (ret)
 547                         goto err_submission;
 548         } else {
 549                 /* Restore GT back to RPn for non-SLPC path */
 550                 intel_rps_lower_unslice(&uc_to_gt(uc)->rps);
 551         }
 552
 553         i915_hwmon_power_max_restore(gt->i915, pl1en);
 554
 555         guc_info(guc, "submission %s\n", str_enabled_disabled(intel_uc_uses_guc_submission(uc)));
 556         guc_info(guc, "SLPC %s\n", str_enabled_disabled(intel_uc_uses_guc_slpc(uc)));
 557
 558         return 0;
 559
 560         /*
 561          * We've failed to load the firmware :(
 562          */
 563 err_submission:
 564         intel_guc_submission_disable(guc);
 565 err_log_capture:
 566         __uc_capture_load_err_log(uc);
 567 err_rps:
 568         /* Return GT back to RPn */
 569         intel_rps_lower_unslice(&uc_to_gt(uc)->rps);
 570
 571         i915_hwmon_power_max_restore(gt->i915, pl1en);
 572 err_out:
 573         __uc_sanitize(uc);
 574
 575         if (!ret) {
 576                 gt_notice(gt, "GuC is uninitialized\n");
 577                 /* We want to run without GuC submission */
 578                 return 0;
 579         }
 580
 581         gt_probe_error(gt, "GuC initialization failed %pe\n", ERR_PTR(ret));
 582
 583         /* We want to keep KMS alive */
 584         return -EIO;
 585 }
 586
 587 static void __uc_fini_hw(struct intel_uc *uc)
 588 {
 589         struct intel_guc *guc = &uc->guc;
 590
 591         if (!intel_guc_is_fw_running(guc))
 592                 return;
 593
 594         if (intel_uc_uses_guc_submission(uc))
 595                 intel_guc_submission_disable(guc);
 596
 597         __uc_sanitize(uc);
 598 }
 599
 600 /**
 601  * intel_uc_reset_prepare - Prepare for reset
 602  * @uc: the intel_uc structure
 603  *
 604  * Preparing for full gpu reset.
 605  */
 606 void intel_uc_reset_prepare(struct intel_uc *uc)
 607 {
 608         struct intel_guc *guc = &uc->guc;
 609
 610         uc->reset_in_progress = true;
 611
 612         /* Nothing to do if GuC isn't supported */
 613         if (!intel_uc_supports_guc(uc))
 614                 return;
 615
 616         /* Firmware expected to be running when this function is called */
 617         if (!intel_guc_is_ready(guc))
 618                 goto sanitize;
 619
 620         if (intel_uc_uses_guc_submission(uc))
 621                 intel_guc_submission_reset_prepare(guc);
 622
 623 sanitize:
 624         __uc_sanitize(uc);
 625 }
 626
 627 void intel_uc_reset(struct intel_uc *uc, intel_engine_mask_t stalled)
 628 {
 629         struct intel_guc *guc = &uc->guc;
 630
 631         /* Firmware can not be running when this function is called  */
 632         if (intel_uc_uses_guc_submission(uc))
 633                 intel_guc_submission_reset(guc, stalled);
 634 }
 635
 636 void intel_uc_reset_finish(struct intel_uc *uc)
 637 {
 638         struct intel_guc *guc = &uc->guc;
 639
 640         uc->reset_in_progress = false;
 641
 642         /* Firmware expected to be running when this function is called */
 643         if (intel_uc_uses_guc_submission(uc))
 644                 intel_guc_submission_reset_finish(guc);
 645 }
 646
 647 void intel_uc_cancel_requests(struct intel_uc *uc)
 648 {
 649         struct intel_guc *guc = &uc->guc;
 650
 651         /* Firmware can not be running when this function is called  */
 652         if (intel_uc_uses_guc_submission(uc))
 653                 intel_guc_submission_cancel_requests(guc);
 654 }
 655
 656 void intel_uc_runtime_suspend(struct intel_uc *uc)
 657 {
 658         struct intel_guc *guc = &uc->guc;
 659
 660         if (!intel_guc_is_ready(guc)) {
 661                 guc->interrupts.enabled = false;
 662                 return;
 663         }
 664
 665         /*
 666          * Wait for any outstanding CTB before tearing down communication /w the
 667          * GuC.
 668          */
 669 #define OUTSTANDING_CTB_TIMEOUT_PERIOD  (HZ / 5)
 670         intel_guc_wait_for_pending_msg(guc, &guc->outstanding_submission_g2h,
 671                                        false, OUTSTANDING_CTB_TIMEOUT_PERIOD);
 672         GEM_WARN_ON(atomic_read(&guc->outstanding_submission_g2h));
 673
 674         guc_disable_communication(guc);
 675 }
 676
 677 void intel_uc_suspend(struct intel_uc *uc)
 678 {
 679         struct intel_guc *guc = &uc->guc;
 680         intel_wakeref_t wakeref;
 681         int err;
 682
 683         /* flush the GSC worker */
 684         intel_gsc_uc_flush_work(&uc->gsc);
 685
 686         wake_up_all_tlb_invalidate(guc);
 687
 688         if (!intel_guc_is_ready(guc)) {
 689                 guc->interrupts.enabled = false;
 690                 return;
 691         }
 692
 693         intel_guc_submission_flush_work(guc);
 694
 695         with_intel_runtime_pm(&uc_to_gt(uc)->i915->runtime_pm, wakeref) {
 696                 err = intel_guc_suspend(guc);
 697                 if (err)
 698                         guc_dbg(guc, "Failed to suspend, %pe", ERR_PTR(err));
 699         }
 700 }
 701
 702 static void __uc_resume_mappings(struct intel_uc *uc)
 703 {
 704         intel_uc_fw_resume_mapping(&uc->guc.fw);
 705         intel_uc_fw_resume_mapping(&uc->huc.fw);
 706 }
 707
 708 static int __uc_resume(struct intel_uc *uc, bool enable_communication)
 709 {
 710         struct intel_guc *guc = &uc->guc;
 711         struct intel_gt *gt = guc_to_gt(guc);
 712         int err;
 713
 714         if (!intel_guc_is_fw_running(guc))
 715                 return 0;
 716
 717         /* Make sure we enable communication if and only if it's disabled */
 718         GEM_BUG_ON(enable_communication == intel_guc_ct_enabled(&guc->ct));
 719
 720         if (enable_communication)
 721                 guc_enable_communication(guc);
 722
 723         /* If we are only resuming GuC communication but not reloading
 724          * GuC, we need to ensure the ARAT timer interrupt is enabled
 725          * again. In case of GuC reload, it is enabled during SLPC enable.
 726          */
 727         if (enable_communication && intel_uc_uses_guc_slpc(uc))
 728                 intel_guc_pm_intrmsk_enable(gt);
 729
 730         err = intel_guc_resume(guc);
 731         if (err) {
 732                 guc_dbg(guc, "Failed to resume, %pe", ERR_PTR(err));
 733                 return err;
 734         }
 735
 736         intel_gsc_uc_resume(&uc->gsc);
 737
 738         if (intel_guc_tlb_invalidation_is_available(guc)) {
 739                 intel_guc_invalidate_tlb_engines(guc);
 740                 intel_guc_invalidate_tlb_guc(guc);
 741         }
 742
 743         return 0;
 744 }
 745
 746 int intel_uc_resume(struct intel_uc *uc)
 747 {
 748         /*
 749          * When coming out of S3/S4 we sanitize and re-init the HW, so
 750          * communication is already re-enabled at this point.
 751          */
 752         return __uc_resume(uc, false);
 753 }
 754
 755 int intel_uc_runtime_resume(struct intel_uc *uc)
 756 {
 757         /*
 758          * During runtime resume we don't sanitize, so we need to re-init
 759          * communication as well.
 760          */
 761         return __uc_resume(uc, true);
 762 }
 763
 764 static const struct intel_uc_ops uc_ops_off = {
 765         .init_hw = __uc_check_hw,
 766         .fini = __uc_fini, /* to clean-up the init_early initialization */
 767 };
 768
 769 static const struct intel_uc_ops uc_ops_on = {
 770         .sanitize = __uc_sanitize,
 771
 772         .init_fw = __uc_fetch_firmwares,
 773         .fini_fw = __uc_cleanup_firmwares,
 774
 775         .init = __uc_init,
 776         .fini = __uc_fini,
 777
 778         .init_hw = __uc_init_hw,
 779         .fini_hw = __uc_fini_hw,
 780
 781         .resume_mappings = __uc_resume_mappings,
 782 };