drivers/pci/controller/dwc/pcie-qcom.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Qualcomm PCIe root complex driver
   4  *
   5  * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
   6  * Copyright 2015 Linaro Limited.
   7  *
   8  * Author: Stanimir Varbanov <svarbanov@mm-sol.com>
   9  */
  10
  11 #include <linux/clk.h>
  12 #include <linux/crc8.h>
  13 #include <linux/debugfs.h>
  14 #include <linux/delay.h>
  15 #include <linux/gpio/consumer.h>
  16 #include <linux/interconnect.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/iopoll.h>
  20 #include <linux/kernel.h>
  21 #include <linux/init.h>
  22 #include <linux/of.h>
  23 #include <linux/of_gpio.h>
  24 #include <linux/pci.h>
  25 #include <linux/pm_runtime.h>
  26 #include <linux/platform_device.h>
  27 #include <linux/phy/pcie.h>
  28 #include <linux/phy/phy.h>
  29 #include <linux/regulator/consumer.h>
  30 #include <linux/reset.h>
  31 #include <linux/slab.h>
  32 #include <linux/types.h>
  33
  34 #include "../../pci.h"
  35 #include "pcie-designware.h"
  36
  37 /* PARF registers */
  38 #define PARF_SYS_CTRL                           0x00
  39 #define PARF_PM_CTRL                            0x20
  40 #define PARF_PCS_DEEMPH                         0x34
  41 #define PARF_PCS_SWING                          0x38
  42 #define PARF_PHY_CTRL                           0x40
  43 #define PARF_PHY_REFCLK                         0x4c
  44 #define PARF_CONFIG_BITS                        0x50
  45 #define PARF_DBI_BASE_ADDR                      0x168
  46 #define PARF_MHI_CLOCK_RESET_CTRL               0x174
  47 #define PARF_AXI_MSTR_WR_ADDR_HALT              0x178
  48 #define PARF_AXI_MSTR_WR_ADDR_HALT_V2           0x1a8
  49 #define PARF_Q2A_FLUSH                          0x1ac
  50 #define PARF_LTSSM                              0x1b0
  51 #define PARF_SID_OFFSET                         0x234
  52 #define PARF_BDF_TRANSLATE_CFG                  0x24c
  53 #define PARF_SLV_ADDR_SPACE_SIZE                0x358
  54 #define PARF_DEVICE_TYPE                        0x1000
  55 #define PARF_BDF_TO_SID_TABLE_N                 0x2000
  56
  57 /* ELBI registers */
  58 #define ELBI_SYS_CTRL                           0x04
  59
  60 /* DBI registers */
  61 #define AXI_MSTR_RESP_COMP_CTRL0                0x818
  62 #define AXI_MSTR_RESP_COMP_CTRL1                0x81c
  63
  64 /* MHI registers */
  65 #define PARF_DEBUG_CNT_PM_LINKST_IN_L2          0xc04
  66 #define PARF_DEBUG_CNT_PM_LINKST_IN_L1          0xc0c
  67 #define PARF_DEBUG_CNT_PM_LINKST_IN_L0S         0xc10
  68 #define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1      0xc84
  69 #define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2      0xc88
  70
  71 /* PARF_SYS_CTRL register fields */
  72 #define MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN        BIT(29)
  73 #define MST_WAKEUP_EN                           BIT(13)
  74 #define SLV_WAKEUP_EN                           BIT(12)
  75 #define MSTR_ACLK_CGC_DIS                       BIT(10)
  76 #define SLV_ACLK_CGC_DIS                        BIT(9)
  77 #define CORE_CLK_CGC_DIS                        BIT(6)
  78 #define AUX_PWR_DET                             BIT(4)
  79 #define L23_CLK_RMV_DIS                         BIT(2)
  80 #define L1_CLK_RMV_DIS                          BIT(1)
  81
  82 /* PARF_PM_CTRL register fields */
  83 #define REQ_NOT_ENTR_L1                         BIT(5)
  84
  85 /* PARF_PCS_DEEMPH register fields */
  86 #define PCS_DEEMPH_TX_DEEMPH_GEN1(x)            FIELD_PREP(GENMASK(21, 16), x)
  87 #define PCS_DEEMPH_TX_DEEMPH_GEN2_3_5DB(x)      FIELD_PREP(GENMASK(13, 8), x)
  88 #define PCS_DEEMPH_TX_DEEMPH_GEN2_6DB(x)        FIELD_PREP(GENMASK(5, 0), x)
  89
  90 /* PARF_PCS_SWING register fields */
  91 #define PCS_SWING_TX_SWING_FULL(x)              FIELD_PREP(GENMASK(14, 8), x)
  92 #define PCS_SWING_TX_SWING_LOW(x)               FIELD_PREP(GENMASK(6, 0), x)
  93
  94 /* PARF_PHY_CTRL register fields */
  95 #define PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK       GENMASK(20, 16)
  96 #define PHY_CTRL_PHY_TX0_TERM_OFFSET(x)         FIELD_PREP(PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK, x)
  97 #define PHY_TEST_PWR_DOWN                       BIT(0)
  98
  99 /* PARF_PHY_REFCLK register fields */
 100 #define PHY_REFCLK_SSP_EN                       BIT(16)
 101 #define PHY_REFCLK_USE_PAD                      BIT(12)
 102
 103 /* PARF_CONFIG_BITS register fields */
 104 #define PHY_RX0_EQ(x)                           FIELD_PREP(GENMASK(26, 24), x)
 105
 106 /* PARF_SLV_ADDR_SPACE_SIZE register value */
 107 #define SLV_ADDR_SPACE_SZ                       0x10000000
 108
 109 /* PARF_MHI_CLOCK_RESET_CTRL register fields */
 110 #define AHB_CLK_EN                              BIT(0)
 111 #define MSTR_AXI_CLK_EN                         BIT(1)
 112 #define BYPASS                                  BIT(4)
 113
 114 /* PARF_AXI_MSTR_WR_ADDR_HALT register fields */
 115 #define EN                                      BIT(31)
 116
 117 /* PARF_LTSSM register fields */
 118 #define LTSSM_EN                                BIT(8)
 119
 120 /* PARF_DEVICE_TYPE register fields */
 121 #define DEVICE_TYPE_RC                          0x4
 122
 123 /* ELBI_SYS_CTRL register fields */
 124 #define ELBI_SYS_CTRL_LT_ENABLE                 BIT(0)
 125
 126 /* AXI_MSTR_RESP_COMP_CTRL0 register fields */
 127 #define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K        0x4
 128 #define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_4K        0x5
 129
 130 /* AXI_MSTR_RESP_COMP_CTRL1 register fields */
 131 #define CFG_BRIDGE_SB_INIT                      BIT(0)
 132
 133 /* PCI_EXP_SLTCAP register fields */
 134 #define PCIE_CAP_SLOT_POWER_LIMIT_VAL           FIELD_PREP(PCI_EXP_SLTCAP_SPLV, 250)
 135 #define PCIE_CAP_SLOT_POWER_LIMIT_SCALE         FIELD_PREP(PCI_EXP_SLTCAP_SPLS, 1)
 136 #define PCIE_CAP_SLOT_VAL                       (PCI_EXP_SLTCAP_ABP | \
 137                                                 PCI_EXP_SLTCAP_PCP | \
 138                                                 PCI_EXP_SLTCAP_MRLSP | \
 139                                                 PCI_EXP_SLTCAP_AIP | \
 140                                                 PCI_EXP_SLTCAP_PIP | \
 141                                                 PCI_EXP_SLTCAP_HPS | \
 142                                                 PCI_EXP_SLTCAP_EIP | \
 143                                                 PCIE_CAP_SLOT_POWER_LIMIT_VAL | \
 144                                                 PCIE_CAP_SLOT_POWER_LIMIT_SCALE)
 145
 146 #define PERST_DELAY_US                          1000
 147
 148 #define QCOM_PCIE_CRC8_POLYNOMIAL               (BIT(2) | BIT(1) | BIT(0))
 149
 150 #define QCOM_PCIE_LINK_SPEED_TO_BW(speed) \
 151                 Mbps_to_icc(PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]))
 152
 153 #define QCOM_PCIE_1_0_0_MAX_CLOCKS              4
 154 struct qcom_pcie_resources_1_0_0 {
 155         struct clk_bulk_data clks[QCOM_PCIE_1_0_0_MAX_CLOCKS];
 156         struct reset_control *core;
 157         struct regulator *vdda;
 158 };
 159
 160 #define QCOM_PCIE_2_1_0_MAX_CLOCKS              5
 161 #define QCOM_PCIE_2_1_0_MAX_RESETS              6
 162 #define QCOM_PCIE_2_1_0_MAX_SUPPLY              3
 163 struct qcom_pcie_resources_2_1_0 {
 164         struct clk_bulk_data clks[QCOM_PCIE_2_1_0_MAX_CLOCKS];
 165         struct reset_control_bulk_data resets[QCOM_PCIE_2_1_0_MAX_RESETS];
 166         int num_resets;
 167         struct regulator_bulk_data supplies[QCOM_PCIE_2_1_0_MAX_SUPPLY];
 168 };
 169
 170 #define QCOM_PCIE_2_3_2_MAX_CLOCKS              4
 171 #define QCOM_PCIE_2_3_2_MAX_SUPPLY              2
 172 struct qcom_pcie_resources_2_3_2 {
 173         struct clk_bulk_data clks[QCOM_PCIE_2_3_2_MAX_CLOCKS];
 174         struct regulator_bulk_data supplies[QCOM_PCIE_2_3_2_MAX_SUPPLY];
 175 };
 176
 177 #define QCOM_PCIE_2_3_3_MAX_CLOCKS              5
 178 #define QCOM_PCIE_2_3_3_MAX_RESETS              7
 179 struct qcom_pcie_resources_2_3_3 {
 180         struct clk_bulk_data clks[QCOM_PCIE_2_3_3_MAX_CLOCKS];
 181         struct reset_control_bulk_data rst[QCOM_PCIE_2_3_3_MAX_RESETS];
 182 };
 183
 184 #define QCOM_PCIE_2_4_0_MAX_CLOCKS              4
 185 #define QCOM_PCIE_2_4_0_MAX_RESETS              12
 186 struct qcom_pcie_resources_2_4_0 {
 187         struct clk_bulk_data clks[QCOM_PCIE_2_4_0_MAX_CLOCKS];
 188         int num_clks;
 189         struct reset_control_bulk_data resets[QCOM_PCIE_2_4_0_MAX_RESETS];
 190         int num_resets;
 191 };
 192
 193 #define QCOM_PCIE_2_7_0_MAX_CLOCKS              15
 194 #define QCOM_PCIE_2_7_0_MAX_SUPPLIES            2
 195 struct qcom_pcie_resources_2_7_0 {
 196         struct clk_bulk_data clks[QCOM_PCIE_2_7_0_MAX_CLOCKS];
 197         int num_clks;
 198         struct regulator_bulk_data supplies[QCOM_PCIE_2_7_0_MAX_SUPPLIES];
 199         struct reset_control *rst;
 200 };
 201
 202 #define QCOM_PCIE_2_9_0_MAX_CLOCKS              5
 203 struct qcom_pcie_resources_2_9_0 {
 204         struct clk_bulk_data clks[QCOM_PCIE_2_9_0_MAX_CLOCKS];
 205         struct reset_control *rst;
 206 };
 207
 208 union qcom_pcie_resources {
 209         struct qcom_pcie_resources_1_0_0 v1_0_0;
 210         struct qcom_pcie_resources_2_1_0 v2_1_0;
 211         struct qcom_pcie_resources_2_3_2 v2_3_2;
 212         struct qcom_pcie_resources_2_3_3 v2_3_3;
 213         struct qcom_pcie_resources_2_4_0 v2_4_0;
 214         struct qcom_pcie_resources_2_7_0 v2_7_0;
 215         struct qcom_pcie_resources_2_9_0 v2_9_0;
 216 };
 217
 218 struct qcom_pcie;
 219
 220 struct qcom_pcie_ops {
 221         int (*get_resources)(struct qcom_pcie *pcie);
 222         int (*init)(struct qcom_pcie *pcie);
 223         int (*post_init)(struct qcom_pcie *pcie);
 224         void (*host_post_init)(struct qcom_pcie *pcie);
 225         void (*deinit)(struct qcom_pcie *pcie);
 226         void (*ltssm_enable)(struct qcom_pcie *pcie);
 227         int (*config_sid)(struct qcom_pcie *pcie);
 228 };
 229
 230 struct qcom_pcie_cfg {
 231         const struct qcom_pcie_ops *ops;
 232 };
 233
 234 struct qcom_pcie {
 235         struct dw_pcie *pci;
 236         void __iomem *parf;                     /* DT parf */
 237         void __iomem *elbi;                     /* DT elbi */
 238         void __iomem *mhi;
 239         union qcom_pcie_resources res;
 240         struct phy *phy;
 241         struct gpio_desc *reset;
 242         struct icc_path *icc_mem;
 243         const struct qcom_pcie_cfg *cfg;
 244         struct dentry *debugfs;
 245         bool suspended;
 246 };
 247
 248 #define to_qcom_pcie(x)         dev_get_drvdata((x)->dev)
 249
 250 static void qcom_ep_reset_assert(struct qcom_pcie *pcie)
 251 {
 252         gpiod_set_value_cansleep(pcie->reset, 1);
 253         usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500);
 254 }
 255
 256 static void qcom_ep_reset_deassert(struct qcom_pcie *pcie)
 257 {
 258         /* Ensure that PERST has been asserted for at least 100 ms */
 259         msleep(100);
 260         gpiod_set_value_cansleep(pcie->reset, 0);
 261         usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500);
 262 }
 263
 264 static int qcom_pcie_start_link(struct dw_pcie *pci)
 265 {
 266         struct qcom_pcie *pcie = to_qcom_pcie(pci);
 267
 268         /* Enable Link Training state machine */
 269         if (pcie->cfg->ops->ltssm_enable)
 270                 pcie->cfg->ops->ltssm_enable(pcie);
 271
 272         return 0;
 273 }
 274
 275 static void qcom_pcie_clear_hpc(struct dw_pcie *pci)
 276 {
 277         u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
 278         u32 val;
 279
 280         dw_pcie_dbi_ro_wr_en(pci);
 281
 282         val = readl(pci->dbi_base + offset + PCI_EXP_SLTCAP);
 283         val &= ~PCI_EXP_SLTCAP_HPC;
 284         writel(val, pci->dbi_base + offset + PCI_EXP_SLTCAP);
 285
 286         dw_pcie_dbi_ro_wr_dis(pci);
 287 }
 288
 289 static void qcom_pcie_2_1_0_ltssm_enable(struct qcom_pcie *pcie)
 290 {
 291         u32 val;
 292
 293         /* enable link training */
 294         val = readl(pcie->elbi + ELBI_SYS_CTRL);
 295         val |= ELBI_SYS_CTRL_LT_ENABLE;
 296         writel(val, pcie->elbi + ELBI_SYS_CTRL);
 297 }
 298
 299 static int qcom_pcie_get_resources_2_1_0(struct qcom_pcie *pcie)
 300 {
 301         struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
 302         struct dw_pcie *pci = pcie->pci;
 303         struct device *dev = pci->dev;
 304         bool is_apq = of_device_is_compatible(dev->of_node, "qcom,pcie-apq8064");
 305         int ret;
 306
 307         res->supplies[0].supply = "vdda";
 308         res->supplies[1].supply = "vdda_phy";
 309         res->supplies[2].supply = "vdda_refclk";
 310         ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies),
 311                                       res->supplies);
 312         if (ret)
 313                 return ret;
 314
 315         res->clks[0].id = "iface";
 316         res->clks[1].id = "core";
 317         res->clks[2].id = "phy";
 318         res->clks[3].id = "aux";
 319         res->clks[4].id = "ref";
 320
 321         /* iface, core, phy are required */
 322         ret = devm_clk_bulk_get(dev, 3, res->clks);
 323         if (ret < 0)
 324                 return ret;
 325
 326         /* aux, ref are optional */
 327         ret = devm_clk_bulk_get_optional(dev, 2, res->clks + 3);
 328         if (ret < 0)
 329                 return ret;
 330
 331         res->resets[0].id = "pci";
 332         res->resets[1].id = "axi";
 333         res->resets[2].id = "ahb";
 334         res->resets[3].id = "por";
 335         res->resets[4].id = "phy";
 336         res->resets[5].id = "ext";
 337
 338         /* ext is optional on APQ8016 */
 339         res->num_resets = is_apq ? 5 : 6;
 340         ret = devm_reset_control_bulk_get_exclusive(dev, res->num_resets, res->resets);
 341         if (ret < 0)
 342                 return ret;
 343
 344         return 0;
 345 }
 346
 347 static void qcom_pcie_deinit_2_1_0(struct qcom_pcie *pcie)
 348 {
 349         struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
 350
 351         clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
 352         reset_control_bulk_assert(res->num_resets, res->resets);
 353
 354         writel(1, pcie->parf + PARF_PHY_CTRL);
 355
 356         regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
 357 }
 358
 359 static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie)
 360 {
 361         struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
 362         struct dw_pcie *pci = pcie->pci;
 363         struct device *dev = pci->dev;
 364         int ret;
 365
 366         /* reset the PCIe interface as uboot can leave it undefined state */
 367         ret = reset_control_bulk_assert(res->num_resets, res->resets);
 368         if (ret < 0) {
 369                 dev_err(dev, "cannot assert resets\n");
 370                 return ret;
 371         }
 372
 373         ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies);
 374         if (ret < 0) {
 375                 dev_err(dev, "cannot enable regulators\n");
 376                 return ret;
 377         }
 378
 379         ret = reset_control_bulk_deassert(res->num_resets, res->resets);
 380         if (ret < 0) {
 381                 dev_err(dev, "cannot deassert resets\n");
 382                 regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
 383                 return ret;
 384         }
 385
 386         return 0;
 387 }
 388
 389 static int qcom_pcie_post_init_2_1_0(struct qcom_pcie *pcie)
 390 {
 391         struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
 392         struct dw_pcie *pci = pcie->pci;
 393         struct device *dev = pci->dev;
 394         struct device_node *node = dev->of_node;
 395         u32 val;
 396         int ret;
 397
 398         /* enable PCIe clocks and resets */
 399         val = readl(pcie->parf + PARF_PHY_CTRL);
 400         val &= ~PHY_TEST_PWR_DOWN;
 401         writel(val, pcie->parf + PARF_PHY_CTRL);
 402
 403         ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks);
 404         if (ret)
 405                 return ret;
 406
 407         if (of_device_is_compatible(node, "qcom,pcie-ipq8064") ||
 408             of_device_is_compatible(node, "qcom,pcie-ipq8064-v2")) {
 409                 writel(PCS_DEEMPH_TX_DEEMPH_GEN1(24) |
 410                                PCS_DEEMPH_TX_DEEMPH_GEN2_3_5DB(24) |
 411                                PCS_DEEMPH_TX_DEEMPH_GEN2_6DB(34),
 412                        pcie->parf + PARF_PCS_DEEMPH);
 413                 writel(PCS_SWING_TX_SWING_FULL(120) |
 414                                PCS_SWING_TX_SWING_LOW(120),
 415                        pcie->parf + PARF_PCS_SWING);
 416                 writel(PHY_RX0_EQ(4), pcie->parf + PARF_CONFIG_BITS);
 417         }
 418
 419         if (of_device_is_compatible(node, "qcom,pcie-ipq8064")) {
 420                 /* set TX termination offset */
 421                 val = readl(pcie->parf + PARF_PHY_CTRL);
 422                 val &= ~PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK;
 423                 val |= PHY_CTRL_PHY_TX0_TERM_OFFSET(7);
 424                 writel(val, pcie->parf + PARF_PHY_CTRL);
 425         }
 426
 427         /* enable external reference clock */
 428         val = readl(pcie->parf + PARF_PHY_REFCLK);
 429         /* USE_PAD is required only for ipq806x */
 430         if (!of_device_is_compatible(node, "qcom,pcie-apq8064"))
 431                 val &= ~PHY_REFCLK_USE_PAD;
 432         val |= PHY_REFCLK_SSP_EN;
 433         writel(val, pcie->parf + PARF_PHY_REFCLK);
 434
 435         /* wait for clock acquisition */
 436         usleep_range(1000, 1500);
 437
 438         /* Set the Max TLP size to 2K, instead of using default of 4K */
 439         writel(CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K,
 440                pci->dbi_base + AXI_MSTR_RESP_COMP_CTRL0);
 441         writel(CFG_BRIDGE_SB_INIT,
 442                pci->dbi_base + AXI_MSTR_RESP_COMP_CTRL1);
 443
 444         qcom_pcie_clear_hpc(pcie->pci);
 445
 446         return 0;
 447 }
 448
 449 static int qcom_pcie_get_resources_1_0_0(struct qcom_pcie *pcie)
 450 {
 451         struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0;
 452         struct dw_pcie *pci = pcie->pci;
 453         struct device *dev = pci->dev;
 454         int ret;
 455
 456         res->vdda = devm_regulator_get(dev, "vdda");
 457         if (IS_ERR(res->vdda))
 458                 return PTR_ERR(res->vdda);
 459
 460         res->clks[0].id = "iface";
 461         res->clks[1].id = "aux";
 462         res->clks[2].id = "master_bus";
 463         res->clks[3].id = "slave_bus";
 464
 465         ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks);
 466         if (ret < 0)
 467                 return ret;
 468
 469         res->core = devm_reset_control_get_exclusive(dev, "core");
 470         return PTR_ERR_OR_ZERO(res->core);
 471 }
 472
 473 static void qcom_pcie_deinit_1_0_0(struct qcom_pcie *pcie)
 474 {
 475         struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0;
 476
 477         reset_control_assert(res->core);
 478         clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
 479         regulator_disable(res->vdda);
 480 }
 481
 482 static int qcom_pcie_init_1_0_0(struct qcom_pcie *pcie)
 483 {
 484         struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0;
 485         struct dw_pcie *pci = pcie->pci;
 486         struct device *dev = pci->dev;
 487         int ret;
 488
 489         ret = reset_control_deassert(res->core);
 490         if (ret) {
 491                 dev_err(dev, "cannot deassert core reset\n");
 492                 return ret;
 493         }
 494
 495         ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks);
 496         if (ret) {
 497                 dev_err(dev, "cannot prepare/enable clocks\n");
 498                 goto err_assert_reset;
 499         }
 500
 501         ret = regulator_enable(res->vdda);
 502         if (ret) {
 503                 dev_err(dev, "cannot enable vdda regulator\n");
 504                 goto err_disable_clks;
 505         }
 506
 507         return 0;
 508
 509 err_disable_clks:
 510         clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
 511 err_assert_reset:
 512         reset_control_assert(res->core);
 513
 514         return ret;
 515 }
 516
 517 static int qcom_pcie_post_init_1_0_0(struct qcom_pcie *pcie)
 518 {
 519         /* change DBI base address */
 520         writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 521
 522         if (IS_ENABLED(CONFIG_PCI_MSI)) {
 523                 u32 val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT);
 524
 525                 val |= EN;
 526                 writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT);
 527         }
 528
 529         qcom_pcie_clear_hpc(pcie->pci);
 530
 531         return 0;
 532 }
 533
 534 static void qcom_pcie_2_3_2_ltssm_enable(struct qcom_pcie *pcie)
 535 {
 536         u32 val;
 537
 538         /* enable link training */
 539         val = readl(pcie->parf + PARF_LTSSM);
 540         val |= LTSSM_EN;
 541         writel(val, pcie->parf + PARF_LTSSM);
 542 }
 543
 544 static int qcom_pcie_get_resources_2_3_2(struct qcom_pcie *pcie)
 545 {
 546         struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
 547         struct dw_pcie *pci = pcie->pci;
 548         struct device *dev = pci->dev;
 549         int ret;
 550
 551         res->supplies[0].supply = "vdda";
 552         res->supplies[1].supply = "vddpe-3v3";
 553         ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies),
 554                                       res->supplies);
 555         if (ret)
 556                 return ret;
 557
 558         res->clks[0].id = "aux";
 559         res->clks[1].id = "cfg";
 560         res->clks[2].id = "bus_master";
 561         res->clks[3].id = "bus_slave";
 562
 563         ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks);
 564         if (ret < 0)
 565                 return ret;
 566
 567         return 0;
 568 }
 569
 570 static void qcom_pcie_deinit_2_3_2(struct qcom_pcie *pcie)
 571 {
 572         struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
 573
 574         clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
 575         regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
 576 }
 577
 578 static int qcom_pcie_init_2_3_2(struct qcom_pcie *pcie)
 579 {
 580         struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
 581         struct dw_pcie *pci = pcie->pci;
 582         struct device *dev = pci->dev;
 583         int ret;
 584
 585         ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies);
 586         if (ret < 0) {
 587                 dev_err(dev, "cannot enable regulators\n");
 588                 return ret;
 589         }
 590
 591         ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks);
 592         if (ret) {
 593                 dev_err(dev, "cannot prepare/enable clocks\n");
 594                 regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
 595                 return ret;
 596         }
 597
 598         return 0;
 599 }
 600
 601 static int qcom_pcie_post_init_2_3_2(struct qcom_pcie *pcie)
 602 {
 603         u32 val;
 604
 605         /* enable PCIe clocks and resets */
 606         val = readl(pcie->parf + PARF_PHY_CTRL);
 607         val &= ~PHY_TEST_PWR_DOWN;
 608         writel(val, pcie->parf + PARF_PHY_CTRL);
 609
 610         /* change DBI base address */
 611         writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 612
 613         /* MAC PHY_POWERDOWN MUX DISABLE  */
 614         val = readl(pcie->parf + PARF_SYS_CTRL);
 615         val &= ~MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN;
 616         writel(val, pcie->parf + PARF_SYS_CTRL);
 617
 618         val = readl(pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
 619         val |= BYPASS;
 620         writel(val, pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
 621
 622         val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
 623         val |= EN;
 624         writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
 625
 626         qcom_pcie_clear_hpc(pcie->pci);
 627
 628         return 0;
 629 }
 630
 631 static int qcom_pcie_get_resources_2_4_0(struct qcom_pcie *pcie)
 632 {
 633         struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0;
 634         struct dw_pcie *pci = pcie->pci;
 635         struct device *dev = pci->dev;
 636         bool is_ipq = of_device_is_compatible(dev->of_node, "qcom,pcie-ipq4019");
 637         int ret;
 638
 639         res->clks[0].id = "aux";
 640         res->clks[1].id = "master_bus";
 641         res->clks[2].id = "slave_bus";
 642         res->clks[3].id = "iface";
 643
 644         /* qcom,pcie-ipq4019 is defined without "iface" */
 645         res->num_clks = is_ipq ? 3 : 4;
 646
 647         ret = devm_clk_bulk_get(dev, res->num_clks, res->clks);
 648         if (ret < 0)
 649                 return ret;
 650
 651         res->resets[0].id = "axi_m";
 652         res->resets[1].id = "axi_s";
 653         res->resets[2].id = "axi_m_sticky";
 654         res->resets[3].id = "pipe_sticky";
 655         res->resets[4].id = "pwr";
 656         res->resets[5].id = "ahb";
 657         res->resets[6].id = "pipe";
 658         res->resets[7].id = "axi_m_vmid";
 659         res->resets[8].id = "axi_s_xpu";
 660         res->resets[9].id = "parf";
 661         res->resets[10].id = "phy";
 662         res->resets[11].id = "phy_ahb";
 663
 664         res->num_resets = is_ipq ? 12 : 6;
 665
 666         ret = devm_reset_control_bulk_get_exclusive(dev, res->num_resets, res->resets);
 667         if (ret < 0)
 668                 return ret;
 669
 670         return 0;
 671 }
 672
 673 static void qcom_pcie_deinit_2_4_0(struct qcom_pcie *pcie)
 674 {
 675         struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0;
 676
 677         reset_control_bulk_assert(res->num_resets, res->resets);
 678         clk_bulk_disable_unprepare(res->num_clks, res->clks);
 679 }
 680
 681 static int qcom_pcie_init_2_4_0(struct qcom_pcie *pcie)
 682 {
 683         struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0;
 684         struct dw_pcie *pci = pcie->pci;
 685         struct device *dev = pci->dev;
 686         int ret;
 687
 688         ret = reset_control_bulk_assert(res->num_resets, res->resets);
 689         if (ret < 0) {
 690                 dev_err(dev, "cannot assert resets\n");
 691                 return ret;
 692         }
 693
 694         usleep_range(10000, 12000);
 695
 696         ret = reset_control_bulk_deassert(res->num_resets, res->resets);
 697         if (ret < 0) {
 698                 dev_err(dev, "cannot deassert resets\n");
 699                 return ret;
 700         }
 701
 702         usleep_range(10000, 12000);
 703
 704         ret = clk_bulk_prepare_enable(res->num_clks, res->clks);
 705         if (ret) {
 706                 reset_control_bulk_assert(res->num_resets, res->resets);
 707                 return ret;
 708         }
 709
 710         return 0;
 711 }
 712
 713 static int qcom_pcie_get_resources_2_3_3(struct qcom_pcie *pcie)
 714 {
 715         struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
 716         struct dw_pcie *pci = pcie->pci;
 717         struct device *dev = pci->dev;
 718         int ret;
 719
 720         res->clks[0].id = "iface";
 721         res->clks[1].id = "axi_m";
 722         res->clks[2].id = "axi_s";
 723         res->clks[3].id = "ahb";
 724         res->clks[4].id = "aux";
 725
 726         ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks);
 727         if (ret < 0)
 728                 return ret;
 729
 730         res->rst[0].id = "axi_m";
 731         res->rst[1].id = "axi_s";
 732         res->rst[2].id = "pipe";
 733         res->rst[3].id = "axi_m_sticky";
 734         res->rst[4].id = "sticky";
 735         res->rst[5].id = "ahb";
 736         res->rst[6].id = "sleep";
 737
 738         ret = devm_reset_control_bulk_get_exclusive(dev, ARRAY_SIZE(res->rst), res->rst);
 739         if (ret < 0)
 740                 return ret;
 741
 742         return 0;
 743 }
 744
 745 static void qcom_pcie_deinit_2_3_3(struct qcom_pcie *pcie)
 746 {
 747         struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
 748
 749         clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
 750 }
 751
 752 static int qcom_pcie_init_2_3_3(struct qcom_pcie *pcie)
 753 {
 754         struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
 755         struct dw_pcie *pci = pcie->pci;
 756         struct device *dev = pci->dev;
 757         int ret;
 758
 759         ret = reset_control_bulk_assert(ARRAY_SIZE(res->rst), res->rst);
 760         if (ret < 0) {
 761                 dev_err(dev, "cannot assert resets\n");
 762                 return ret;
 763         }
 764
 765         usleep_range(2000, 2500);
 766
 767         ret = reset_control_bulk_deassert(ARRAY_SIZE(res->rst), res->rst);
 768         if (ret < 0) {
 769                 dev_err(dev, "cannot deassert resets\n");
 770                 return ret;
 771         }
 772
 773         /*
 774          * Don't have a way to see if the reset has completed.
 775          * Wait for some time.
 776          */
 777         usleep_range(2000, 2500);
 778
 779         ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks);
 780         if (ret) {
 781                 dev_err(dev, "cannot prepare/enable clocks\n");
 782                 goto err_assert_resets;
 783         }
 784
 785         return 0;
 786
 787 err_assert_resets:
 788         /*
 789          * Not checking for failure, will anyway return
 790          * the original failure in 'ret'.
 791          */
 792         reset_control_bulk_assert(ARRAY_SIZE(res->rst), res->rst);
 793
 794         return ret;
 795 }
 796
 797 static int qcom_pcie_post_init_2_3_3(struct qcom_pcie *pcie)
 798 {
 799         struct dw_pcie *pci = pcie->pci;
 800         u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
 801         u32 val;
 802
 803         writel(SLV_ADDR_SPACE_SZ, pcie->parf + PARF_SLV_ADDR_SPACE_SIZE);
 804
 805         val = readl(pcie->parf + PARF_PHY_CTRL);
 806         val &= ~PHY_TEST_PWR_DOWN;
 807         writel(val, pcie->parf + PARF_PHY_CTRL);
 808
 809         writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 810
 811         writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS
 812                 | SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS |
 813                 AUX_PWR_DET | L23_CLK_RMV_DIS | L1_CLK_RMV_DIS,
 814                 pcie->parf + PARF_SYS_CTRL);
 815         writel(0, pcie->parf + PARF_Q2A_FLUSH);
 816
 817         writel(PCI_COMMAND_MASTER, pci->dbi_base + PCI_COMMAND);
 818
 819         dw_pcie_dbi_ro_wr_en(pci);
 820
 821         writel(PCIE_CAP_SLOT_VAL, pci->dbi_base + offset + PCI_EXP_SLTCAP);
 822
 823         val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP);
 824         val &= ~PCI_EXP_LNKCAP_ASPMS;
 825         writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP);
 826
 827         writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset +
 828                 PCI_EXP_DEVCTL2);
 829
 830         dw_pcie_dbi_ro_wr_dis(pci);
 831
 832         return 0;
 833 }
 834
 835 static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie)
 836 {
 837         struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0;
 838         struct dw_pcie *pci = pcie->pci;
 839         struct device *dev = pci->dev;
 840         unsigned int num_clks, num_opt_clks;
 841         unsigned int idx;
 842         int ret;
 843
 844         res->rst = devm_reset_control_array_get_exclusive(dev);
 845         if (IS_ERR(res->rst))
 846                 return PTR_ERR(res->rst);
 847
 848         res->supplies[0].supply = "vdda";
 849         res->supplies[1].supply = "vddpe-3v3";
 850         ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies),
 851                                       res->supplies);
 852         if (ret)
 853                 return ret;
 854
 855         idx = 0;
 856         res->clks[idx++].id = "aux";
 857         res->clks[idx++].id = "cfg";
 858         res->clks[idx++].id = "bus_master";
 859         res->clks[idx++].id = "bus_slave";
 860         res->clks[idx++].id = "slave_q2a";
 861
 862         num_clks = idx;
 863
 864         ret = devm_clk_bulk_get(dev, num_clks, res->clks);
 865         if (ret < 0)
 866                 return ret;
 867
 868         res->clks[idx++].id = "tbu";
 869         res->clks[idx++].id = "ddrss_sf_tbu";
 870         res->clks[idx++].id = "aggre0";
 871         res->clks[idx++].id = "aggre1";
 872         res->clks[idx++].id = "noc_aggr";
 873         res->clks[idx++].id = "noc_aggr_4";
 874         res->clks[idx++].id = "noc_aggr_south_sf";
 875         res->clks[idx++].id = "cnoc_qx";
 876         res->clks[idx++].id = "sleep";
 877         res->clks[idx++].id = "cnoc_sf_axi";
 878
 879         num_opt_clks = idx - num_clks;
 880         res->num_clks = idx;
 881
 882         ret = devm_clk_bulk_get_optional(dev, num_opt_clks, res->clks + num_clks);
 883         if (ret < 0)
 884                 return ret;
 885
 886         return 0;
 887 }
 888
 889 static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie)
 890 {
 891         struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0;
 892         struct dw_pcie *pci = pcie->pci;
 893         struct device *dev = pci->dev;
 894         u32 val;
 895         int ret;
 896
 897         ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies);
 898         if (ret < 0) {
 899                 dev_err(dev, "cannot enable regulators\n");
 900                 return ret;
 901         }
 902
 903         ret = clk_bulk_prepare_enable(res->num_clks, res->clks);
 904         if (ret < 0)
 905                 goto err_disable_regulators;
 906
 907         ret = reset_control_assert(res->rst);
 908         if (ret) {
 909                 dev_err(dev, "reset assert failed (%d)\n", ret);
 910                 goto err_disable_clocks;
 911         }
 912
 913         usleep_range(1000, 1500);
 914
 915         ret = reset_control_deassert(res->rst);
 916         if (ret) {
 917                 dev_err(dev, "reset deassert failed (%d)\n", ret);
 918                 goto err_disable_clocks;
 919         }
 920
 921         /* Wait for reset to complete, required on SM8450 */
 922         usleep_range(1000, 1500);
 923
 924         /* configure PCIe to RC mode */
 925         writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE);
 926
 927         /* enable PCIe clocks and resets */
 928         val = readl(pcie->parf + PARF_PHY_CTRL);
 929         val &= ~PHY_TEST_PWR_DOWN;
 930         writel(val, pcie->parf + PARF_PHY_CTRL);
 931
 932         /* change DBI base address */
 933         writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
 934
 935         /* MAC PHY_POWERDOWN MUX DISABLE  */
 936         val = readl(pcie->parf + PARF_SYS_CTRL);
 937         val &= ~MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN;
 938         writel(val, pcie->parf + PARF_SYS_CTRL);
 939
 940         val = readl(pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
 941         val |= BYPASS;
 942         writel(val, pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
 943
 944         /* Enable L1 and L1SS */
 945         val = readl(pcie->parf + PARF_PM_CTRL);
 946         val &= ~REQ_NOT_ENTR_L1;
 947         writel(val, pcie->parf + PARF_PM_CTRL);
 948
 949         val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
 950         val |= EN;
 951         writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2);
 952
 953         return 0;
 954 err_disable_clocks:
 955         clk_bulk_disable_unprepare(res->num_clks, res->clks);
 956 err_disable_regulators:
 957         regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
 958
 959         return ret;
 960 }
 961
 962 static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie)
 963 {
 964         qcom_pcie_clear_hpc(pcie->pci);
 965
 966         return 0;
 967 }
 968
 969 static int qcom_pcie_enable_aspm(struct pci_dev *pdev, void *userdata)
 970 {
 971         /* Downstream devices need to be in D0 state before enabling PCI PM substates */
 972         pci_set_power_state(pdev, PCI_D0);
 973         pci_enable_link_state(pdev, PCIE_LINK_STATE_ALL);
 974
 975         return 0;
 976 }
 977
 978 static void qcom_pcie_host_post_init_2_7_0(struct qcom_pcie *pcie)
 979 {
 980         struct dw_pcie_rp *pp = &pcie->pci->pp;
 981
 982         pci_walk_bus(pp->bridge->bus, qcom_pcie_enable_aspm, NULL);
 983 }
 984
 985 static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie)
 986 {
 987         struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0;
 988
 989         clk_bulk_disable_unprepare(res->num_clks, res->clks);
 990
 991         regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
 992 }
 993
 994 static int qcom_pcie_config_sid_1_9_0(struct qcom_pcie *pcie)
 995 {
 996         /* iommu map structure */
 997         struct {
 998                 u32 bdf;
 999                 u32 phandle;
1000                 u32 smmu_sid;
1001                 u32 smmu_sid_len;
1002         } *map;
1003         void __iomem *bdf_to_sid_base = pcie->parf + PARF_BDF_TO_SID_TABLE_N;
1004         struct device *dev = pcie->pci->dev;
1005         u8 qcom_pcie_crc8_table[CRC8_TABLE_SIZE];
1006         int i, nr_map, size = 0;
1007         u32 smmu_sid_base;
1008
1009         of_get_property(dev->of_node, "iommu-map", &size);
1010         if (!size)
1011                 return 0;
1012
1013         map = kzalloc(size, GFP_KERNEL);
1014         if (!map)
1015                 return -ENOMEM;
1016
1017         of_property_read_u32_array(dev->of_node, "iommu-map", (u32 *)map,
1018                                    size / sizeof(u32));
1019
1020         nr_map = size / (sizeof(*map));
1021
1022         crc8_populate_msb(qcom_pcie_crc8_table, QCOM_PCIE_CRC8_POLYNOMIAL);
1023
1024         /* Registers need to be zero out first */
1025         memset_io(bdf_to_sid_base, 0, CRC8_TABLE_SIZE * sizeof(u32));
1026
1027         /* Extract the SMMU SID base from the first entry of iommu-map */
1028         smmu_sid_base = map[0].smmu_sid;
1029
1030         /* Look for an available entry to hold the mapping */
1031         for (i = 0; i < nr_map; i++) {
1032                 __be16 bdf_be = cpu_to_be16(map[i].bdf);
1033                 u32 val;
1034                 u8 hash;
1035
1036                 hash = crc8(qcom_pcie_crc8_table, (u8 *)&bdf_be, sizeof(bdf_be), 0);
1037
1038                 val = readl(bdf_to_sid_base + hash * sizeof(u32));
1039
1040                 /* If the register is already populated, look for next available entry */
1041                 while (val) {
1042                         u8 current_hash = hash++;
1043                         u8 next_mask = 0xff;
1044
1045                         /* If NEXT field is NULL then update it with next hash */
1046                         if (!(val & next_mask)) {
1047                                 val |= (u32)hash;
1048                                 writel(val, bdf_to_sid_base + current_hash * sizeof(u32));
1049                         }
1050
1051                         val = readl(bdf_to_sid_base + hash * sizeof(u32));
1052                 }
1053
1054                 /* BDF [31:16] | SID [15:8] | NEXT [7:0] */
1055                 val = map[i].bdf << 16 | (map[i].smmu_sid - smmu_sid_base) << 8 | 0;
1056                 writel(val, bdf_to_sid_base + hash * sizeof(u32));
1057         }
1058
1059         kfree(map);
1060
1061         return 0;
1062 }
1063
1064 static int qcom_pcie_get_resources_2_9_0(struct qcom_pcie *pcie)
1065 {
1066         struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0;
1067         struct dw_pcie *pci = pcie->pci;
1068         struct device *dev = pci->dev;
1069         int ret;
1070
1071         res->clks[0].id = "iface";
1072         res->clks[1].id = "axi_m";
1073         res->clks[2].id = "axi_s";
1074         res->clks[3].id = "axi_bridge";
1075         res->clks[4].id = "rchng";
1076
1077         ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks);
1078         if (ret < 0)
1079                 return ret;
1080
1081         res->rst = devm_reset_control_array_get_exclusive(dev);
1082         if (IS_ERR(res->rst))
1083                 return PTR_ERR(res->rst);
1084
1085         return 0;
1086 }
1087
1088 static void qcom_pcie_deinit_2_9_0(struct qcom_pcie *pcie)
1089 {
1090         struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0;
1091
1092         clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks);
1093 }
1094
1095 static int qcom_pcie_init_2_9_0(struct qcom_pcie *pcie)
1096 {
1097         struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0;
1098         struct device *dev = pcie->pci->dev;
1099         int ret;
1100
1101         ret = reset_control_assert(res->rst);
1102         if (ret) {
1103                 dev_err(dev, "reset assert failed (%d)\n", ret);
1104                 return ret;
1105         }
1106
1107         /*
1108          * Delay periods before and after reset deassert are working values
1109          * from downstream Codeaurora kernel
1110          */
1111         usleep_range(2000, 2500);
1112
1113         ret = reset_control_deassert(res->rst);
1114         if (ret) {
1115                 dev_err(dev, "reset deassert failed (%d)\n", ret);
1116                 return ret;
1117         }
1118
1119         usleep_range(2000, 2500);
1120
1121         return clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks);
1122 }
1123
1124 static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie)
1125 {
1126         struct dw_pcie *pci = pcie->pci;
1127         u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
1128         u32 val;
1129         int i;
1130
1131         writel(SLV_ADDR_SPACE_SZ,
1132                 pcie->parf + PARF_SLV_ADDR_SPACE_SIZE);
1133
1134         val = readl(pcie->parf + PARF_PHY_CTRL);
1135         val &= ~PHY_TEST_PWR_DOWN;
1136         writel(val, pcie->parf + PARF_PHY_CTRL);
1137
1138         writel(0, pcie->parf + PARF_DBI_BASE_ADDR);
1139
1140         writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE);
1141         writel(BYPASS | MSTR_AXI_CLK_EN | AHB_CLK_EN,
1142                 pcie->parf + PARF_MHI_CLOCK_RESET_CTRL);
1143         writel(GEN3_RELATED_OFF_RXEQ_RGRDLESS_RXTS |
1144                 GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL,
1145                 pci->dbi_base + GEN3_RELATED_OFF);
1146
1147         writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS |
1148                 SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS |
1149                 AUX_PWR_DET | L23_CLK_RMV_DIS | L1_CLK_RMV_DIS,
1150                 pcie->parf + PARF_SYS_CTRL);
1151
1152         writel(0, pcie->parf + PARF_Q2A_FLUSH);
1153
1154         dw_pcie_dbi_ro_wr_en(pci);
1155
1156         writel(PCIE_CAP_SLOT_VAL, pci->dbi_base + offset + PCI_EXP_SLTCAP);
1157
1158         val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP);
1159         val &= ~PCI_EXP_LNKCAP_ASPMS;
1160         writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP);
1161
1162         writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset +
1163                         PCI_EXP_DEVCTL2);
1164
1165         dw_pcie_dbi_ro_wr_dis(pci);
1166
1167         for (i = 0; i < 256; i++)
1168                 writel(0, pcie->parf + PARF_BDF_TO_SID_TABLE_N + (4 * i));
1169
1170         return 0;
1171 }
1172
1173 static int qcom_pcie_link_up(struct dw_pcie *pci)
1174 {
1175         u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
1176         u16 val = readw(pci->dbi_base + offset + PCI_EXP_LNKSTA);
1177
1178         return !!(val & PCI_EXP_LNKSTA_DLLLA);
1179 }
1180
1181 static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
1182 {
1183         struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1184         struct qcom_pcie *pcie = to_qcom_pcie(pci);
1185         int ret;
1186
1187         qcom_ep_reset_assert(pcie);
1188
1189         ret = pcie->cfg->ops->init(pcie);
1190         if (ret)
1191                 return ret;
1192
1193         ret = phy_set_mode_ext(pcie->phy, PHY_MODE_PCIE, PHY_MODE_PCIE_RC);
1194         if (ret)
1195                 goto err_deinit;
1196
1197         ret = phy_power_on(pcie->phy);
1198         if (ret)
1199                 goto err_deinit;
1200
1201         if (pcie->cfg->ops->post_init) {
1202                 ret = pcie->cfg->ops->post_init(pcie);
1203                 if (ret)
1204                         goto err_disable_phy;
1205         }
1206
1207         qcom_ep_reset_deassert(pcie);
1208
1209         if (pcie->cfg->ops->config_sid) {
1210                 ret = pcie->cfg->ops->config_sid(pcie);
1211                 if (ret)
1212                         goto err_assert_reset;
1213         }
1214
1215         return 0;
1216
1217 err_assert_reset:
1218         qcom_ep_reset_assert(pcie);
1219 err_disable_phy:
1220         phy_power_off(pcie->phy);
1221 err_deinit:
1222         pcie->cfg->ops->deinit(pcie);
1223
1224         return ret;
1225 }
1226
1227 static void qcom_pcie_host_deinit(struct dw_pcie_rp *pp)
1228 {
1229         struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1230         struct qcom_pcie *pcie = to_qcom_pcie(pci);
1231
1232         qcom_ep_reset_assert(pcie);
1233         phy_power_off(pcie->phy);
1234         pcie->cfg->ops->deinit(pcie);
1235 }
1236
1237 static void qcom_pcie_host_post_init(struct dw_pcie_rp *pp)
1238 {
1239         struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1240         struct qcom_pcie *pcie = to_qcom_pcie(pci);
1241
1242         if (pcie->cfg->ops->host_post_init)
1243                 pcie->cfg->ops->host_post_init(pcie);
1244 }
1245
1246 static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
1247         .host_init      = qcom_pcie_host_init,
1248         .host_deinit    = qcom_pcie_host_deinit,
1249         .host_post_init = qcom_pcie_host_post_init,
1250 };
1251
1252 /* Qcom IP rev.: 2.1.0  Synopsys IP rev.: 4.01a */
1253 static const struct qcom_pcie_ops ops_2_1_0 = {
1254         .get_resources = qcom_pcie_get_resources_2_1_0,
1255         .init = qcom_pcie_init_2_1_0,
1256         .post_init = qcom_pcie_post_init_2_1_0,
1257         .deinit = qcom_pcie_deinit_2_1_0,
1258         .ltssm_enable = qcom_pcie_2_1_0_ltssm_enable,
1259 };
1260
1261 /* Qcom IP rev.: 1.0.0  Synopsys IP rev.: 4.11a */
1262 static const struct qcom_pcie_ops ops_1_0_0 = {
1263         .get_resources = qcom_pcie_get_resources_1_0_0,
1264         .init = qcom_pcie_init_1_0_0,
1265         .post_init = qcom_pcie_post_init_1_0_0,
1266         .deinit = qcom_pcie_deinit_1_0_0,
1267         .ltssm_enable = qcom_pcie_2_1_0_ltssm_enable,
1268 };
1269
1270 /* Qcom IP rev.: 2.3.2  Synopsys IP rev.: 4.21a */
1271 static const struct qcom_pcie_ops ops_2_3_2 = {
1272         .get_resources = qcom_pcie_get_resources_2_3_2,
1273         .init = qcom_pcie_init_2_3_2,
1274         .post_init = qcom_pcie_post_init_2_3_2,
1275         .deinit = qcom_pcie_deinit_2_3_2,
1276         .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
1277 };
1278
1279 /* Qcom IP rev.: 2.4.0  Synopsys IP rev.: 4.20a */
1280 static const struct qcom_pcie_ops ops_2_4_0 = {
1281         .get_resources = qcom_pcie_get_resources_2_4_0,
1282         .init = qcom_pcie_init_2_4_0,
1283         .post_init = qcom_pcie_post_init_2_3_2,
1284         .deinit = qcom_pcie_deinit_2_4_0,
1285         .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
1286 };
1287
1288 /* Qcom IP rev.: 2.3.3  Synopsys IP rev.: 4.30a */
1289 static const struct qcom_pcie_ops ops_2_3_3 = {
1290         .get_resources = qcom_pcie_get_resources_2_3_3,
1291         .init = qcom_pcie_init_2_3_3,
1292         .post_init = qcom_pcie_post_init_2_3_3,
1293         .deinit = qcom_pcie_deinit_2_3_3,
1294         .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
1295 };
1296
1297 /* Qcom IP rev.: 2.7.0  Synopsys IP rev.: 4.30a */
1298 static const struct qcom_pcie_ops ops_2_7_0 = {
1299         .get_resources = qcom_pcie_get_resources_2_7_0,
1300         .init = qcom_pcie_init_2_7_0,
1301         .post_init = qcom_pcie_post_init_2_7_0,
1302         .deinit = qcom_pcie_deinit_2_7_0,
1303         .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
1304 };
1305
1306 /* Qcom IP rev.: 1.9.0 */
1307 static const struct qcom_pcie_ops ops_1_9_0 = {
1308         .get_resources = qcom_pcie_get_resources_2_7_0,
1309         .init = qcom_pcie_init_2_7_0,
1310         .post_init = qcom_pcie_post_init_2_7_0,
1311         .host_post_init = qcom_pcie_host_post_init_2_7_0,
1312         .deinit = qcom_pcie_deinit_2_7_0,
1313         .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
1314         .config_sid = qcom_pcie_config_sid_1_9_0,
1315 };
1316
1317 /* Qcom IP rev.: 2.9.0  Synopsys IP rev.: 5.00a */
1318 static const struct qcom_pcie_ops ops_2_9_0 = {
1319         .get_resources = qcom_pcie_get_resources_2_9_0,
1320         .init = qcom_pcie_init_2_9_0,
1321         .post_init = qcom_pcie_post_init_2_9_0,
1322         .deinit = qcom_pcie_deinit_2_9_0,
1323         .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
1324 };
1325
1326 static const struct qcom_pcie_cfg cfg_1_0_0 = {
1327         .ops = &ops_1_0_0,
1328 };
1329
1330 static const struct qcom_pcie_cfg cfg_1_9_0 = {
1331         .ops = &ops_1_9_0,
1332 };
1333
1334 static const struct qcom_pcie_cfg cfg_2_1_0 = {
1335         .ops = &ops_2_1_0,
1336 };
1337
1338 static const struct qcom_pcie_cfg cfg_2_3_2 = {
1339         .ops = &ops_2_3_2,
1340 };
1341
1342 static const struct qcom_pcie_cfg cfg_2_3_3 = {
1343         .ops = &ops_2_3_3,
1344 };
1345
1346 static const struct qcom_pcie_cfg cfg_2_4_0 = {
1347         .ops = &ops_2_4_0,
1348 };
1349
1350 static const struct qcom_pcie_cfg cfg_2_7_0 = {
1351         .ops = &ops_2_7_0,
1352 };
1353
1354 static const struct qcom_pcie_cfg cfg_2_9_0 = {
1355         .ops = &ops_2_9_0,
1356 };
1357
1358 static const struct dw_pcie_ops dw_pcie_ops = {
1359         .link_up = qcom_pcie_link_up,
1360         .start_link = qcom_pcie_start_link,
1361 };
1362
1363 static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
1364 {
1365         struct dw_pcie *pci = pcie->pci;
1366         int ret;
1367
1368         pcie->icc_mem = devm_of_icc_get(pci->dev, "pcie-mem");
1369         if (IS_ERR(pcie->icc_mem))
1370                 return PTR_ERR(pcie->icc_mem);
1371
1372         /*
1373          * Some Qualcomm platforms require interconnect bandwidth constraints
1374          * to be set before enabling interconnect clocks.
1375          *
1376          * Set an initial peak bandwidth corresponding to single-lane Gen 1
1377          * for the pcie-mem path.
1378          */
1379         ret = icc_set_bw(pcie->icc_mem, 0, QCOM_PCIE_LINK_SPEED_TO_BW(1));
1380         if (ret) {
1381                 dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n",
1382                         ret);
1383                 return ret;
1384         }
1385
1386         return 0;
1387 }
1388
1389 static void qcom_pcie_icc_update(struct qcom_pcie *pcie)
1390 {
1391         struct dw_pcie *pci = pcie->pci;
1392         u32 offset, status;
1393         int speed, width;
1394         int ret;
1395
1396         if (!pcie->icc_mem)
1397                 return;
1398
1399         offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
1400         status = readw(pci->dbi_base + offset + PCI_EXP_LNKSTA);
1401
1402         /* Only update constraints if link is up. */
1403         if (!(status & PCI_EXP_LNKSTA_DLLLA))
1404                 return;
1405
1406         speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, status);
1407         width = FIELD_GET(PCI_EXP_LNKSTA_NLW, status);
1408
1409         ret = icc_set_bw(pcie->icc_mem, 0, width * QCOM_PCIE_LINK_SPEED_TO_BW(speed));
1410         if (ret) {
1411                 dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n",
1412                         ret);
1413         }
1414 }
1415
1416 static int qcom_pcie_link_transition_count(struct seq_file *s, void *data)
1417 {
1418         struct qcom_pcie *pcie = (struct qcom_pcie *)dev_get_drvdata(s->private);
1419
1420         seq_printf(s, "L0s transition count: %u\n",
1421                    readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L0S));
1422
1423         seq_printf(s, "L1 transition count: %u\n",
1424                    readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L1));
1425
1426         seq_printf(s, "L1.1 transition count: %u\n",
1427                    readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1));
1428
1429         seq_printf(s, "L1.2 transition count: %u\n",
1430                    readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2));
1431
1432         seq_printf(s, "L2 transition count: %u\n",
1433                    readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L2));
1434
1435         return 0;
1436 }
1437
1438 static void qcom_pcie_init_debugfs(struct qcom_pcie *pcie)
1439 {
1440         struct dw_pcie *pci = pcie->pci;
1441         struct device *dev = pci->dev;
1442         char *name;
1443
1444         name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1445         if (!name)
1446                 return;
1447
1448         pcie->debugfs = debugfs_create_dir(name, NULL);
1449         debugfs_create_devm_seqfile(dev, "link_transition_count", pcie->debugfs,
1450                                     qcom_pcie_link_transition_count);
1451 }
1452
1453 static int qcom_pcie_probe(struct platform_device *pdev)
1454 {
1455         const struct qcom_pcie_cfg *pcie_cfg;
1456         struct device *dev = &pdev->dev;
1457         struct qcom_pcie *pcie;
1458         struct dw_pcie_rp *pp;
1459         struct resource *res;
1460         struct dw_pcie *pci;
1461         int ret;
1462
1463         pcie_cfg = of_device_get_match_data(dev);
1464         if (!pcie_cfg || !pcie_cfg->ops) {
1465                 dev_err(dev, "Invalid platform data\n");
1466                 return -EINVAL;
1467         }
1468
1469         pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
1470         if (!pcie)
1471                 return -ENOMEM;
1472
1473         pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
1474         if (!pci)
1475                 return -ENOMEM;
1476
1477         pm_runtime_enable(dev);
1478         ret = pm_runtime_get_sync(dev);
1479         if (ret < 0)
1480                 goto err_pm_runtime_put;
1481
1482         pci->dev = dev;
1483         pci->ops = &dw_pcie_ops;
1484         pp = &pci->pp;
1485
1486         pcie->pci = pci;
1487
1488         pcie->cfg = pcie_cfg;
1489
1490         pcie->reset = devm_gpiod_get_optional(dev, "perst", GPIOD_OUT_HIGH);
1491         if (IS_ERR(pcie->reset)) {
1492                 ret = PTR_ERR(pcie->reset);
1493                 goto err_pm_runtime_put;
1494         }
1495
1496         pcie->parf = devm_platform_ioremap_resource_byname(pdev, "parf");
1497         if (IS_ERR(pcie->parf)) {
1498                 ret = PTR_ERR(pcie->parf);
1499                 goto err_pm_runtime_put;
1500         }
1501
1502         pcie->elbi = devm_platform_ioremap_resource_byname(pdev, "elbi");
1503         if (IS_ERR(pcie->elbi)) {
1504                 ret = PTR_ERR(pcie->elbi);
1505                 goto err_pm_runtime_put;
1506         }
1507
1508         /* MHI region is optional */
1509         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mhi");
1510         if (res) {
1511                 pcie->mhi = devm_ioremap_resource(dev, res);
1512                 if (IS_ERR(pcie->mhi)) {
1513                         ret = PTR_ERR(pcie->mhi);
1514                         goto err_pm_runtime_put;
1515                 }
1516         }
1517
1518         pcie->phy = devm_phy_optional_get(dev, "pciephy");
1519         if (IS_ERR(pcie->phy)) {
1520                 ret = PTR_ERR(pcie->phy);
1521                 goto err_pm_runtime_put;
1522         }
1523
1524         ret = qcom_pcie_icc_init(pcie);
1525         if (ret)
1526                 goto err_pm_runtime_put;
1527
1528         ret = pcie->cfg->ops->get_resources(pcie);
1529         if (ret)
1530                 goto err_pm_runtime_put;
1531
1532         pp->ops = &qcom_pcie_dw_ops;
1533
1534         ret = phy_init(pcie->phy);
1535         if (ret)
1536                 goto err_pm_runtime_put;
1537
1538         platform_set_drvdata(pdev, pcie);
1539
1540         ret = dw_pcie_host_init(pp);
1541         if (ret) {
1542                 dev_err(dev, "cannot initialize host\n");
1543                 goto err_phy_exit;
1544         }
1545
1546         qcom_pcie_icc_update(pcie);
1547
1548         if (pcie->mhi)
1549                 qcom_pcie_init_debugfs(pcie);
1550
1551         return 0;
1552
1553 err_phy_exit:
1554         phy_exit(pcie->phy);
1555 err_pm_runtime_put:
1556         pm_runtime_put(dev);
1557         pm_runtime_disable(dev);
1558
1559         return ret;
1560 }
1561
1562 static int qcom_pcie_suspend_noirq(struct device *dev)
1563 {
1564         struct qcom_pcie *pcie = dev_get_drvdata(dev);
1565         int ret;
1566
1567         /*
1568          * Set minimum bandwidth required to keep data path functional during
1569          * suspend.
1570          */
1571         ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1));
1572         if (ret) {
1573                 dev_err(dev, "Failed to set interconnect bandwidth: %d\n", ret);
1574                 return ret;
1575         }
1576
1577         /*
1578          * Turn OFF the resources only for controllers without active PCIe
1579          * devices. For controllers with active devices, the resources are kept
1580          * ON and the link is expected to be in L0/L1 (sub)states.
1581          *
1582          * Turning OFF the resources for controllers with active PCIe devices
1583          * will trigger access violation during the end of the suspend cycle,
1584          * as kernel tries to access the PCIe devices config space for masking
1585          * MSIs.
1586          *
1587          * Also, it is not desirable to put the link into L2/L3 state as that
1588          * implies VDD supply will be removed and the devices may go into
1589          * powerdown state. This will affect the lifetime of the storage devices
1590          * like NVMe.
1591          */
1592         if (!dw_pcie_link_up(pcie->pci)) {
1593                 qcom_pcie_host_deinit(&pcie->pci->pp);
1594                 pcie->suspended = true;
1595         }
1596
1597         return 0;
1598 }
1599
1600 static int qcom_pcie_resume_noirq(struct device *dev)
1601 {
1602         struct qcom_pcie *pcie = dev_get_drvdata(dev);
1603         int ret;
1604
1605         if (pcie->suspended) {
1606                 ret = qcom_pcie_host_init(&pcie->pci->pp);
1607                 if (ret)
1608                         return ret;
1609
1610                 pcie->suspended = false;
1611         }
1612
1613         qcom_pcie_icc_update(pcie);
1614
1615         return 0;
1616 }
1617
1618 static const struct of_device_id qcom_pcie_match[] = {
1619         { .compatible = "qcom,pcie-apq8064", .data = &cfg_2_1_0 },
1620         { .compatible = "qcom,pcie-apq8084", .data = &cfg_1_0_0 },
1621         { .compatible = "qcom,pcie-ipq4019", .data = &cfg_2_4_0 },
1622         { .compatible = "qcom,pcie-ipq6018", .data = &cfg_2_9_0 },
1623         { .compatible = "qcom,pcie-ipq8064", .data = &cfg_2_1_0 },
1624         { .compatible = "qcom,pcie-ipq8064-v2", .data = &cfg_2_1_0 },
1625         { .compatible = "qcom,pcie-ipq8074", .data = &cfg_2_3_3 },
1626         { .compatible = "qcom,pcie-ipq8074-gen3", .data = &cfg_2_9_0 },
1627         { .compatible = "qcom,pcie-msm8996", .data = &cfg_2_3_2 },
1628         { .compatible = "qcom,pcie-qcs404", .data = &cfg_2_4_0 },
1629         { .compatible = "qcom,pcie-sa8540p", .data = &cfg_1_9_0 },
1630         { .compatible = "qcom,pcie-sa8775p", .data = &cfg_1_9_0},
1631         { .compatible = "qcom,pcie-sc7280", .data = &cfg_1_9_0 },
1632         { .compatible = "qcom,pcie-sc8180x", .data = &cfg_1_9_0 },
1633         { .compatible = "qcom,pcie-sc8280xp", .data = &cfg_1_9_0 },
1634         { .compatible = "qcom,pcie-sdm845", .data = &cfg_2_7_0 },
1635         { .compatible = "qcom,pcie-sdx55", .data = &cfg_1_9_0 },
1636         { .compatible = "qcom,pcie-sm8150", .data = &cfg_1_9_0 },
1637         { .compatible = "qcom,pcie-sm8250", .data = &cfg_1_9_0 },
1638         { .compatible = "qcom,pcie-sm8350", .data = &cfg_1_9_0 },
1639         { .compatible = "qcom,pcie-sm8450-pcie0", .data = &cfg_1_9_0 },
1640         { .compatible = "qcom,pcie-sm8450-pcie1", .data = &cfg_1_9_0 },
1641         { .compatible = "qcom,pcie-sm8550", .data = &cfg_1_9_0 },
1642         { }
1643 };
1644
1645 static void qcom_fixup_class(struct pci_dev *dev)
1646 {
1647         dev->class = PCI_CLASS_BRIDGE_PCI_NORMAL;
1648 }
1649 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0101, qcom_fixup_class);
1650 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0104, qcom_fixup_class);
1651 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0106, qcom_fixup_class);
1652 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0107, qcom_fixup_class);
1653 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0302, qcom_fixup_class);
1654 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1000, qcom_fixup_class);
1655 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class);
1656
1657 static const struct dev_pm_ops qcom_pcie_pm_ops = {
1658         NOIRQ_SYSTEM_SLEEP_PM_OPS(qcom_pcie_suspend_noirq, qcom_pcie_resume_noirq)
1659 };
1660
1661 static struct platform_driver qcom_pcie_driver = {
1662         .probe = qcom_pcie_probe,
1663         .driver = {
1664                 .name = "qcom-pcie",
1665                 .suppress_bind_attrs = true,
1666                 .of_match_table = qcom_pcie_match,
1667                 .pm = &qcom_pcie_pm_ops,
1668                 .probe_type = PROBE_PREFER_ASYNCHRONOUS,
1669         },
1670 };
1671 builtin_platform_driver(qcom_pcie_driver);