drivers/staging/gasket/apex_driver.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Driver for the Apex chip.
   4  *
   5  * Copyright (C) 2018 Google, Inc.
   6  */
   7
   8 #include <linux/compiler.h>
   9 #include <linux/delay.h>
  10 #include <linux/device.h>
  11 #include <linux/fs.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/module.h>
  15 #include <linux/moduleparam.h>
  16 #include <linux/pci.h>
  17 #include <linux/printk.h>
  18 #include <linux/sched.h>
  19 #include <linux/uaccess.h>
  20
  21 #include "apex.h"
  22
  23 #include "gasket_core.h"
  24 #include "gasket_interrupt.h"
  25 #include "gasket_page_table.h"
  26 #include "gasket_sysfs.h"
  27
  28 /* Constants */
  29 #define APEX_DEVICE_NAME "Apex"
  30 #define APEX_DRIVER_VERSION "1.0"
  31
  32 /* CSRs are in BAR 2. */
  33 #define APEX_BAR_INDEX 2
  34
  35 #define APEX_PCI_VENDOR_ID 0x1ac1
  36 #define APEX_PCI_DEVICE_ID 0x089a
  37
  38 /* Bar Offsets. */
  39 #define APEX_BAR_OFFSET 0
  40 #define APEX_CM_OFFSET 0x1000000
  41
  42 /* The sizes of each Apex BAR 2. */
  43 #define APEX_BAR_BYTES 0x100000
  44 #define APEX_CH_MEM_BYTES (PAGE_SIZE * MAX_NUM_COHERENT_PAGES)
  45
  46 /* The number of user-mappable memory ranges in BAR2 of a Apex chip. */
  47 #define NUM_REGIONS 3
  48
  49 /* The number of nodes in a Apex chip. */
  50 #define NUM_NODES 1
  51
  52 /*
  53  * The total number of entries in the page table. Should match the value read
  54  * from the register APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE.
  55  */
  56 #define APEX_PAGE_TABLE_TOTAL_ENTRIES 8192
  57
  58 #define APEX_EXTENDED_SHIFT 63 /* Extended address bit position. */
  59
  60 /* Check reset 120 times */
  61 #define APEX_RESET_RETRY 120
  62 /* Wait 100 ms between checks. Total 12 sec wait maximum. */
  63 #define APEX_RESET_DELAY 100
  64
  65 /* Enumeration of the supported sysfs entries. */
  66 enum sysfs_attribute_type {
  67         ATTR_KERNEL_HIB_PAGE_TABLE_SIZE,
  68         ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE,
  69         ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES,
  70 };
  71
  72 /*
  73  * Register offsets into BAR2 memory.
  74  * Only values necessary for driver implementation are defined.
  75  */
  76 enum apex_bar2_regs {
  77         APEX_BAR2_REG_SCU_BASE = 0x1A300,
  78         APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE = 0x46000,
  79         APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE = 0x46008,
  80         APEX_BAR2_REG_KERNEL_HIB_TRANSLATION_ENABLE = 0x46010,
  81         APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL = 0x46018,
  82         APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL = 0x46020,
  83         APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL = 0x46028,
  84         APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL = 0x46030,
  85         APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL = 0x46038,
  86         APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL = 0x46040,
  87         APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL = 0x46048,
  88         APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE = 0x46050,
  89         APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE_MASK = 0x46058,
  90         APEX_BAR2_REG_KERNEL_HIB_STATUS_BLOCK_DELAY = 0x46060,
  91         APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY0 = 0x46068,
  92         APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY1 = 0x46070,
  93         APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT = 0x46078,
  94         APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT = 0x46080,
  95         APEX_BAR2_REG_KERNEL_WIRE_INT_PENDING_BIT_ARRAY = 0x48778,
  96         APEX_BAR2_REG_KERNEL_WIRE_INT_MASK_ARRAY = 0x48780,
  97         APEX_BAR2_REG_USER_HIB_DMA_PAUSE = 0x486D8,
  98         APEX_BAR2_REG_USER_HIB_DMA_PAUSED = 0x486E0,
  99         APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER = 0x4A000,
 100         APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE = 0x50000,
 101
 102         /* Error registers - Used mostly for debug */
 103         APEX_BAR2_REG_USER_HIB_ERROR_STATUS = 0x86f0,
 104         APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS = 0x41a0,
 105 };
 106
 107 /* Addresses for packed registers. */
 108 #define APEX_BAR2_REG_AXI_QUIESCE (APEX_BAR2_REG_SCU_BASE + 0x2C)
 109 #define APEX_BAR2_REG_GCB_CLOCK_GATE (APEX_BAR2_REG_SCU_BASE + 0x14)
 110 #define APEX_BAR2_REG_SCU_0 (APEX_BAR2_REG_SCU_BASE + 0xc)
 111 #define APEX_BAR2_REG_SCU_1 (APEX_BAR2_REG_SCU_BASE + 0x10)
 112 #define APEX_BAR2_REG_SCU_2 (APEX_BAR2_REG_SCU_BASE + 0x14)
 113 #define APEX_BAR2_REG_SCU_3 (APEX_BAR2_REG_SCU_BASE + 0x18)
 114 #define APEX_BAR2_REG_SCU_4 (APEX_BAR2_REG_SCU_BASE + 0x1c)
 115 #define APEX_BAR2_REG_SCU_5 (APEX_BAR2_REG_SCU_BASE + 0x20)
 116
 117 #define SCU3_RG_PWR_STATE_OVR_BIT_OFFSET 26
 118 #define SCU3_RG_PWR_STATE_OVR_MASK_WIDTH 2
 119 #define SCU3_CUR_RST_GCB_BIT_MASK 0x10
 120 #define SCU2_RG_RST_GCB_BIT_MASK 0xc
 121
 122 /* Configuration for page table. */
 123 static struct gasket_page_table_config apex_page_table_configs[NUM_NODES] = {
 124         {
 125                 .id = 0,
 126                 .mode = GASKET_PAGE_TABLE_MODE_NORMAL,
 127                 .total_entries = APEX_PAGE_TABLE_TOTAL_ENTRIES,
 128                 .base_reg = APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE,
 129                 .extended_reg = APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE,
 130                 .extended_bit = APEX_EXTENDED_SHIFT,
 131         },
 132 };
 133
 134 /* The regions in the BAR2 space that can be mapped into user space. */
 135 static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = {
 136         { 0x40000, 0x1000 },
 137         { 0x44000, 0x1000 },
 138         { 0x48000, 0x1000 },
 139 };
 140
 141 /* Gasket device interrupts enums must be dense (i.e., no empty slots). */
 142 enum apex_interrupt {
 143         APEX_INTERRUPT_INSTR_QUEUE = 0,
 144         APEX_INTERRUPT_INPUT_ACTV_QUEUE = 1,
 145         APEX_INTERRUPT_PARAM_QUEUE = 2,
 146         APEX_INTERRUPT_OUTPUT_ACTV_QUEUE = 3,
 147         APEX_INTERRUPT_SC_HOST_0 = 4,
 148         APEX_INTERRUPT_SC_HOST_1 = 5,
 149         APEX_INTERRUPT_SC_HOST_2 = 6,
 150         APEX_INTERRUPT_SC_HOST_3 = 7,
 151         APEX_INTERRUPT_TOP_LEVEL_0 = 8,
 152         APEX_INTERRUPT_TOP_LEVEL_1 = 9,
 153         APEX_INTERRUPT_TOP_LEVEL_2 = 10,
 154         APEX_INTERRUPT_TOP_LEVEL_3 = 11,
 155         APEX_INTERRUPT_FATAL_ERR = 12,
 156         APEX_INTERRUPT_COUNT = 13,
 157 };
 158
 159 /* Interrupt descriptors for Apex */
 160 static struct gasket_interrupt_desc apex_interrupts[] = {
 161         {
 162                 APEX_INTERRUPT_INSTR_QUEUE,
 163                 APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL,
 164                 UNPACKED,
 165         },
 166         {
 167                 APEX_INTERRUPT_INPUT_ACTV_QUEUE,
 168                 APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL,
 169                 UNPACKED
 170         },
 171         {
 172                 APEX_INTERRUPT_PARAM_QUEUE,
 173                 APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL,
 174                 UNPACKED
 175         },
 176         {
 177                 APEX_INTERRUPT_OUTPUT_ACTV_QUEUE,
 178                 APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL,
 179                 UNPACKED
 180         },
 181         {
 182                 APEX_INTERRUPT_SC_HOST_0,
 183                 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
 184                 PACK_0
 185         },
 186         {
 187                 APEX_INTERRUPT_SC_HOST_1,
 188                 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
 189                 PACK_1
 190         },
 191         {
 192                 APEX_INTERRUPT_SC_HOST_2,
 193                 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
 194                 PACK_2
 195         },
 196         {
 197                 APEX_INTERRUPT_SC_HOST_3,
 198                 APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
 199                 PACK_3
 200         },
 201         {
 202                 APEX_INTERRUPT_TOP_LEVEL_0,
 203                 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
 204                 PACK_0
 205         },
 206         {
 207                 APEX_INTERRUPT_TOP_LEVEL_1,
 208                 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
 209                 PACK_1
 210         },
 211         {
 212                 APEX_INTERRUPT_TOP_LEVEL_2,
 213                 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
 214                 PACK_2
 215         },
 216         {
 217                 APEX_INTERRUPT_TOP_LEVEL_3,
 218                 APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
 219                 PACK_3
 220         },
 221         {
 222                 APEX_INTERRUPT_FATAL_ERR,
 223                 APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL,
 224                 UNPACKED
 225         },
 226 };
 227
 228 /* Allows device to enter power save upon driver close(). */
 229 static int allow_power_save = 1;
 230
 231 /* Allows SW based clock gating. */
 232 static int allow_sw_clock_gating;
 233
 234 /* Allows HW based clock gating. */
 235 /* Note: this is not mutual exclusive with SW clock gating. */
 236 static int allow_hw_clock_gating = 1;
 237
 238 /* Act as if only GCB is instantiated. */
 239 static int bypass_top_level;
 240
 241 module_param(allow_power_save, int, 0644);
 242 module_param(allow_sw_clock_gating, int, 0644);
 243 module_param(allow_hw_clock_gating, int, 0644);
 244 module_param(bypass_top_level, int, 0644);
 245
 246 /* Check the device status registers and return device status ALIVE or DEAD. */
 247 static int apex_get_status(struct gasket_dev *gasket_dev)
 248 {
 249         /* TODO: Check device status. */
 250         return GASKET_STATUS_ALIVE;
 251 }
 252
 253 /* Enter GCB reset state. */
 254 static int apex_enter_reset(struct gasket_dev *gasket_dev)
 255 {
 256         if (bypass_top_level)
 257                 return 0;
 258
 259         /*
 260          * Software reset:
 261          * Enable sleep mode
 262          *  - Software force GCB idle
 263          *    - Enable GCB idle
 264          */
 265         gasket_read_modify_write_64(gasket_dev, APEX_BAR_INDEX,
 266                                     APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER,
 267                                     0x0, 1, 32);
 268
 269         /*    - Initiate DMA pause */
 270         gasket_dev_write_64(gasket_dev, 1, APEX_BAR_INDEX,
 271                             APEX_BAR2_REG_USER_HIB_DMA_PAUSE);
 272
 273         /*    - Wait for DMA pause complete. */
 274         if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
 275                                         APEX_BAR2_REG_USER_HIB_DMA_PAUSED, 1, 1,
 276                                         APEX_RESET_DELAY, APEX_RESET_RETRY)) {
 277                 dev_err(gasket_dev->dev,
 278                         "DMAs did not quiesce within timeout (%d ms)\n",
 279                         APEX_RESET_RETRY * APEX_RESET_DELAY);
 280                 return -ETIMEDOUT;
 281         }
 282
 283         /*  - Enable GCB reset (0x1 to rg_rst_gcb) */
 284         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 285                                     APEX_BAR2_REG_SCU_2, 0x1, 2, 2);
 286
 287         /*  - Enable GCB clock Gate (0x1 to rg_gated_gcb) */
 288         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 289                                     APEX_BAR2_REG_SCU_2, 0x1, 2, 18);
 290
 291         /*  - Enable GCB memory shut down (0x3 to rg_force_ram_sd) */
 292         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 293                                     APEX_BAR2_REG_SCU_3, 0x3, 2, 14);
 294
 295         /*    - Wait for RAM shutdown. */
 296         if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
 297                                         APEX_BAR2_REG_SCU_3, BIT(6), BIT(6),
 298                                         APEX_RESET_DELAY, APEX_RESET_RETRY)) {
 299                 dev_err(gasket_dev->dev,
 300                         "RAM did not shut down within timeout (%d ms)\n",
 301                         APEX_RESET_RETRY * APEX_RESET_DELAY);
 302                 return -ETIMEDOUT;
 303         }
 304
 305         return 0;
 306 }
 307
 308 /* Quit GCB reset state. */
 309 static int apex_quit_reset(struct gasket_dev *gasket_dev)
 310 {
 311         u32 val0, val1;
 312
 313         if (bypass_top_level)
 314                 return 0;
 315
 316         /*
 317          * Disable sleep mode:
 318          *  - Disable GCB memory shut down:
 319          *    - b00: Not forced (HW controlled)
 320          *    - b1x: Force disable
 321          */
 322         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 323                                     APEX_BAR2_REG_SCU_3, 0x0, 2, 14);
 324
 325         /*
 326          *  - Disable software clock gate:
 327          *    - b00: Not forced (HW controlled)
 328          *    - b1x: Force disable
 329          */
 330         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 331                                     APEX_BAR2_REG_SCU_2, 0x0, 2, 18);
 332
 333         /*
 334          *  - Disable GCB reset (rg_rst_gcb):
 335          *    - b00: Not forced (HW controlled)
 336          *    - b1x: Force disable = Force not Reset
 337          */
 338         gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 339                                     APEX_BAR2_REG_SCU_2, 0x2, 2, 2);
 340
 341         /*    - Wait for RAM enable. */
 342         if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
 343                                         APEX_BAR2_REG_SCU_3, BIT(6), 0,
 344                                         APEX_RESET_DELAY, APEX_RESET_RETRY)) {
 345                 dev_err(gasket_dev->dev,
 346                         "RAM did not enable within timeout (%d ms)\n",
 347                         APEX_RESET_RETRY * APEX_RESET_DELAY);
 348                 return -ETIMEDOUT;
 349         }
 350
 351         /*    - Wait for Reset complete. */
 352         if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
 353                                         APEX_BAR2_REG_SCU_3,
 354                                         SCU3_CUR_RST_GCB_BIT_MASK, 0,
 355                                         APEX_RESET_DELAY, APEX_RESET_RETRY)) {
 356                 dev_err(gasket_dev->dev,
 357                         "GCB did not leave reset within timeout (%d ms)\n",
 358                         APEX_RESET_RETRY * APEX_RESET_DELAY);
 359                 return -ETIMEDOUT;
 360         }
 361
 362         if (!allow_hw_clock_gating) {
 363                 val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 364                                           APEX_BAR2_REG_SCU_3);
 365                 /* Inactive and Sleep mode are disabled. */
 366                 gasket_read_modify_write_32(gasket_dev,
 367                                             APEX_BAR_INDEX,
 368                                             APEX_BAR2_REG_SCU_3, 0x3,
 369                                             SCU3_RG_PWR_STATE_OVR_MASK_WIDTH,
 370                                             SCU3_RG_PWR_STATE_OVR_BIT_OFFSET);
 371                 val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 372                                           APEX_BAR2_REG_SCU_3);
 373                 dev_dbg(gasket_dev->dev,
 374                         "Disallow HW clock gating 0x%x -> 0x%x\n", val0, val1);
 375         } else {
 376                 val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 377                                           APEX_BAR2_REG_SCU_3);
 378                 /* Inactive mode enabled - Sleep mode disabled. */
 379                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 380                                             APEX_BAR2_REG_SCU_3, 2,
 381                                             SCU3_RG_PWR_STATE_OVR_MASK_WIDTH,
 382                                             SCU3_RG_PWR_STATE_OVR_BIT_OFFSET);
 383                 val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 384                                           APEX_BAR2_REG_SCU_3);
 385                 dev_dbg(gasket_dev->dev, "Allow HW clock gating 0x%x -> 0x%x\n",
 386                         val0, val1);
 387         }
 388
 389         return 0;
 390 }
 391
 392 /* Reset the Apex hardware. Called on final close via device_close_cb. */
 393 static int apex_device_cleanup(struct gasket_dev *gasket_dev)
 394 {
 395         u64 scalar_error;
 396         u64 hib_error;
 397         int ret = 0;
 398
 399         hib_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
 400                                        APEX_BAR2_REG_USER_HIB_ERROR_STATUS);
 401         scalar_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
 402                                           APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS);
 403
 404         dev_dbg(gasket_dev->dev,
 405                 "%s 0x%p hib_error 0x%llx scalar_error 0x%llx\n",
 406                 __func__, gasket_dev, hib_error, scalar_error);
 407
 408         if (allow_power_save)
 409                 ret = apex_enter_reset(gasket_dev);
 410
 411         return ret;
 412 }
 413
 414 /* Determine if GCB is in reset state. */
 415 static bool is_gcb_in_reset(struct gasket_dev *gasket_dev)
 416 {
 417         u32 val = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX,
 418                                      APEX_BAR2_REG_SCU_3);
 419
 420         /* Masks rg_rst_gcb bit of SCU_CTRL_2 */
 421         return (val & SCU3_CUR_RST_GCB_BIT_MASK);
 422 }
 423
 424 /* Reset the hardware, then quit reset.  Called on device open. */
 425 static int apex_reset(struct gasket_dev *gasket_dev)
 426 {
 427         int ret;
 428
 429         if (bypass_top_level)
 430                 return 0;
 431
 432         if (!is_gcb_in_reset(gasket_dev)) {
 433                 /* We are not in reset - toggle the reset bit so as to force
 434                  * re-init of custom block
 435                  */
 436                 dev_dbg(gasket_dev->dev, "%s: toggle reset\n", __func__);
 437
 438                 ret = apex_enter_reset(gasket_dev);
 439                 if (ret)
 440                         return ret;
 441         }
 442         return apex_quit_reset(gasket_dev);
 443 }
 444
 445 /*
 446  * Check permissions for Apex ioctls.
 447  * Returns true if the current user may execute this ioctl, and false otherwise.
 448  */
 449 static bool apex_ioctl_check_permissions(struct file *filp, uint cmd)
 450 {
 451         return !!(filp->f_mode & FMODE_WRITE);
 452 }
 453
 454 /* Gates or un-gates Apex clock. */
 455 static long apex_clock_gating(struct gasket_dev *gasket_dev,
 456                               struct apex_gate_clock_ioctl __user *argp)
 457 {
 458         struct apex_gate_clock_ioctl ibuf;
 459
 460         if (bypass_top_level || !allow_sw_clock_gating)
 461                 return 0;
 462
 463         if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
 464                 return -EFAULT;
 465
 466         dev_dbg(gasket_dev->dev, "%s %llu\n", __func__, ibuf.enable);
 467
 468         if (ibuf.enable) {
 469                 /* Quiesce AXI, gate GCB clock. */
 470                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 471                                             APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1,
 472                                             16);
 473                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 474                                             APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1,
 475                                             2, 18);
 476         } else {
 477                 /* Un-gate GCB clock, un-quiesce AXI. */
 478                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 479                                             APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0,
 480                                             2, 18);
 481                 gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX,
 482                                             APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1,
 483                                             16);
 484         }
 485         return 0;
 486 }
 487
 488 /* Apex-specific ioctl handler. */
 489 static long apex_ioctl(struct file *filp, uint cmd, void __user *argp)
 490 {
 491         struct gasket_dev *gasket_dev = filp->private_data;
 492
 493         if (!apex_ioctl_check_permissions(filp, cmd))
 494                 return -EPERM;
 495
 496         switch (cmd) {
 497         case APEX_IOCTL_GATE_CLOCK:
 498                 return apex_clock_gating(gasket_dev, argp);
 499         default:
 500                 return -ENOTTY; /* unknown command */
 501         }
 502 }
 503
 504 /* Display driver sysfs entries. */
 505 static ssize_t sysfs_show(struct device *device, struct device_attribute *attr,
 506                           char *buf)
 507 {
 508         int ret;
 509         struct gasket_dev *gasket_dev;
 510         struct gasket_sysfs_attribute *gasket_attr;
 511         enum sysfs_attribute_type type;
 512         struct gasket_page_table *gpt;
 513         uint val;
 514
 515         gasket_dev = gasket_sysfs_get_device_data(device);
 516         if (!gasket_dev) {
 517                 dev_err(device, "No Apex device sysfs mapping found\n");
 518                 return -ENODEV;
 519         }
 520
 521         gasket_attr = gasket_sysfs_get_attr(device, attr);
 522         if (!gasket_attr) {
 523                 dev_err(device, "No Apex device sysfs attr data found\n");
 524                 gasket_sysfs_put_device_data(device, gasket_dev);
 525                 return -ENODEV;
 526         }
 527
 528         type = (enum sysfs_attribute_type)gasket_attr->data.attr_type;
 529         gpt = gasket_dev->page_table[0];
 530         switch (type) {
 531         case ATTR_KERNEL_HIB_PAGE_TABLE_SIZE:
 532                 val = gasket_page_table_num_entries(gpt);
 533                 break;
 534         case ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE:
 535                 val = gasket_page_table_num_simple_entries(gpt);
 536                 break;
 537         case ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES:
 538                 val = gasket_page_table_num_active_pages(gpt);
 539                 break;
 540         default:
 541                 dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
 542                         attr->attr.name);
 543                 ret = 0;
 544                 goto exit;
 545         }
 546         ret = scnprintf(buf, PAGE_SIZE, "%u\n", val);
 547 exit:
 548         gasket_sysfs_put_attr(device, gasket_attr);
 549         gasket_sysfs_put_device_data(device, gasket_dev);
 550         return ret;
 551 }
 552
 553 static struct gasket_sysfs_attribute apex_sysfs_attrs[] = {
 554         GASKET_SYSFS_RO(node_0_page_table_entries, sysfs_show,
 555                         ATTR_KERNEL_HIB_PAGE_TABLE_SIZE),
 556         GASKET_SYSFS_RO(node_0_simple_page_table_entries, sysfs_show,
 557                         ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE),
 558         GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show,
 559                         ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES),
 560         GASKET_END_OF_ATTR_ARRAY
 561 };
 562
 563 /* On device open, perform a core reinit reset. */
 564 static int apex_device_open_cb(struct gasket_dev *gasket_dev)
 565 {
 566         return gasket_reset_nolock(gasket_dev);
 567 }
 568
 569 static const struct pci_device_id apex_pci_ids[] = {
 570         { PCI_DEVICE(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID) }, { 0 }
 571 };
 572
 573 static int apex_pci_probe(struct pci_dev *pci_dev,
 574                           const struct pci_device_id *id)
 575 {
 576         int ret;
 577         ulong page_table_ready, msix_table_ready;
 578         int retries = 0;
 579         struct gasket_dev *gasket_dev;
 580
 581         ret = pci_enable_device(pci_dev);
 582         if (ret) {
 583                 dev_err(&pci_dev->dev, "error enabling PCI device\n");
 584                 return ret;
 585         }
 586
 587         pci_set_master(pci_dev);
 588
 589         ret = gasket_pci_add_device(pci_dev, &gasket_dev);
 590         if (ret) {
 591                 dev_err(&pci_dev->dev, "error adding gasket device\n");
 592                 pci_disable_device(pci_dev);
 593                 return ret;
 594         }
 595
 596         pci_set_drvdata(pci_dev, gasket_dev);
 597         apex_reset(gasket_dev);
 598
 599         while (retries < APEX_RESET_RETRY) {
 600                 page_table_ready =
 601                         gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
 602                                            APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT);
 603                 msix_table_ready =
 604                         gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX,
 605                                            APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT);
 606                 if (page_table_ready && msix_table_ready)
 607                         break;
 608                 schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY));
 609                 retries++;
 610         }
 611
 612         if (retries == APEX_RESET_RETRY) {
 613                 if (!page_table_ready)
 614                         dev_err(gasket_dev->dev, "Page table init timed out\n");
 615                 if (!msix_table_ready)
 616                         dev_err(gasket_dev->dev, "MSI-X table init timed out\n");
 617                 ret = -ETIMEDOUT;
 618                 goto remove_device;
 619         }
 620
 621         ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device,
 622                                           apex_sysfs_attrs);
 623         if (ret)
 624                 dev_err(&pci_dev->dev, "error creating device sysfs entries\n");
 625
 626         ret = gasket_enable_device(gasket_dev);
 627         if (ret) {
 628                 dev_err(&pci_dev->dev, "error enabling gasket device\n");
 629                 goto remove_device;
 630         }
 631
 632         /* Place device in low power mode until opened */
 633         if (allow_power_save)
 634                 apex_enter_reset(gasket_dev);
 635
 636         return 0;
 637
 638 remove_device:
 639         gasket_pci_remove_device(pci_dev);
 640         pci_disable_device(pci_dev);
 641         return ret;
 642 }
 643
 644 static void apex_pci_remove(struct pci_dev *pci_dev)
 645 {
 646         struct gasket_dev *gasket_dev = pci_get_drvdata(pci_dev);
 647
 648         gasket_disable_device(gasket_dev);
 649         gasket_pci_remove_device(pci_dev);
 650         pci_disable_device(pci_dev);
 651 }
 652
 653 static const struct gasket_driver_desc apex_desc = {
 654         .name = "apex",
 655         .driver_version = APEX_DRIVER_VERSION,
 656         .major = 120,
 657         .minor = 0,
 658         .module = THIS_MODULE,
 659         .pci_id_table = apex_pci_ids,
 660
 661         .num_page_tables = NUM_NODES,
 662         .page_table_bar_index = APEX_BAR_INDEX,
 663         .page_table_configs = apex_page_table_configs,
 664         .page_table_extended_bit = APEX_EXTENDED_SHIFT,
 665
 666         .bar_descriptions = {
 667                 GASKET_UNUSED_BAR,
 668                 GASKET_UNUSED_BAR,
 669                 { APEX_BAR_BYTES, (VM_WRITE | VM_READ), APEX_BAR_OFFSET,
 670                         NUM_REGIONS, mappable_regions, PCI_BAR },
 671                 GASKET_UNUSED_BAR,
 672                 GASKET_UNUSED_BAR,
 673                 GASKET_UNUSED_BAR,
 674         },
 675         .coherent_buffer_description = {
 676                 APEX_CH_MEM_BYTES,
 677                 (VM_WRITE | VM_READ),
 678                 APEX_CM_OFFSET,
 679         },
 680         .interrupt_type = PCI_MSIX,
 681         .interrupt_bar_index = APEX_BAR_INDEX,
 682         .num_interrupts = APEX_INTERRUPT_COUNT,
 683         .interrupts = apex_interrupts,
 684         .interrupt_pack_width = 7,
 685
 686         .device_open_cb = apex_device_open_cb,
 687         .device_close_cb = apex_device_cleanup,
 688
 689         .ioctl_handler_cb = apex_ioctl,
 690         .device_status_cb = apex_get_status,
 691         .hardware_revision_cb = NULL,
 692         .device_reset_cb = apex_reset,
 693 };
 694
 695 static struct pci_driver apex_pci_driver = {
 696         .name = "apex",
 697         .probe = apex_pci_probe,
 698         .remove = apex_pci_remove,
 699         .id_table = apex_pci_ids,
 700 };
 701
 702 static int __init apex_init(void)
 703 {
 704         int ret;
 705
 706         ret = gasket_register_device(&apex_desc);
 707         if (ret)
 708                 return ret;
 709         ret = pci_register_driver(&apex_pci_driver);
 710         if (ret)
 711                 gasket_unregister_device(&apex_desc);
 712         return ret;
 713 }
 714
 715 static void apex_exit(void)
 716 {
 717         pci_unregister_driver(&apex_pci_driver);
 718         gasket_unregister_device(&apex_desc);
 719 }
 720 MODULE_DESCRIPTION("Google Apex driver");
 721 MODULE_VERSION(APEX_DRIVER_VERSION);
 722 MODULE_LICENSE("GPL v2");
 723 MODULE_AUTHOR("John Joseph <jnjoseph@google.com>");
 724 MODULE_DEVICE_TABLE(pci, apex_pci_ids);
 725 module_init(apex_init);
 726 module_exit(apex_exit);