--- /dev/null
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <strings.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/malloc.h>
+#include <ipxe/umalloc.h>
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+#include <ipxe/profile.h>
+#include "xhci.h"
+
+/** @file
+ *
+ * USB eXtensible Host Controller Interface (xHCI) driver
+ *
+ */
+
+/** Message transfer profiler */
+static struct profiler xhci_message_profiler __profiler =
+ { .name = "xhci.message" };
+
+/** Stream transfer profiler */
+static struct profiler xhci_stream_profiler __profiler =
+ { .name = "xhci.stream" };
+
+/** Event ring profiler */
+static struct profiler xhci_event_profiler __profiler =
+ { .name = "xhci.event" };
+
+/** Transfer event profiler */
+static struct profiler xhci_transfer_profiler __profiler =
+ { .name = "xhci.transfer" };
+
+/* Disambiguate the various error causes */
+#define EIO_DATA \
+ __einfo_error ( EINFO_EIO_DATA )
+#define EINFO_EIO_DATA \
+ __einfo_uniqify ( EINFO_EIO, ( 2 - 0 ), \
+ "Data buffer error" )
+#define EIO_BABBLE \
+ __einfo_error ( EINFO_EIO_BABBLE )
+#define EINFO_EIO_BABBLE \
+ __einfo_uniqify ( EINFO_EIO, ( 3 - 0 ), \
+ "Babble detected" )
+#define EIO_USB \
+ __einfo_error ( EINFO_EIO_USB )
+#define EINFO_EIO_USB \
+ __einfo_uniqify ( EINFO_EIO, ( 4 - 0 ), \
+ "USB transaction error" )
+#define EIO_TRB \
+ __einfo_error ( EINFO_EIO_TRB )
+#define EINFO_EIO_TRB \
+ __einfo_uniqify ( EINFO_EIO, ( 5 - 0 ), \
+ "TRB error" )
+#define EIO_STALL \
+ __einfo_error ( EINFO_EIO_STALL )
+#define EINFO_EIO_STALL \
+ __einfo_uniqify ( EINFO_EIO, ( 6 - 0 ), \
+ "Stall error" )
+#define EIO_RESOURCE \
+ __einfo_error ( EINFO_EIO_RESOURCE )
+#define EINFO_EIO_RESOURCE \
+ __einfo_uniqify ( EINFO_EIO, ( 7 - 0 ), \
+ "Resource error" )
+#define EIO_BANDWIDTH \
+ __einfo_error ( EINFO_EIO_BANDWIDTH )
+#define EINFO_EIO_BANDWIDTH \
+ __einfo_uniqify ( EINFO_EIO, ( 8 - 0 ), \
+ "Bandwidth error" )
+#define EIO_NO_SLOTS \
+ __einfo_error ( EINFO_EIO_NO_SLOTS )
+#define EINFO_EIO_NO_SLOTS \
+ __einfo_uniqify ( EINFO_EIO, ( 9 - 0 ), \
+ "No slots available" )
+#define EIO_STREAM_TYPE \
+ __einfo_error ( EINFO_EIO_STREAM_TYPE )
+#define EINFO_EIO_STREAM_TYPE \
+ __einfo_uniqify ( EINFO_EIO, ( 10 - 0 ), \
+ "Invalid stream type" )
+#define EIO_SLOT \
+ __einfo_error ( EINFO_EIO_SLOT )
+#define EINFO_EIO_SLOT \
+ __einfo_uniqify ( EINFO_EIO, ( 11 - 0 ), \
+ "Slot not enabled" )
+#define EIO_ENDPOINT \
+ __einfo_error ( EINFO_EIO_ENDPOINT )
+#define EINFO_EIO_ENDPOINT \
+ __einfo_uniqify ( EINFO_EIO, ( 12 - 0 ), \
+ "Endpoint not enabled" )
+#define EIO_SHORT \
+ __einfo_error ( EINFO_EIO_SHORT )
+#define EINFO_EIO_SHORT \
+ __einfo_uniqify ( EINFO_EIO, ( 13 - 0 ), \
+ "Short packet" )
+#define EIO_UNDERRUN \
+ __einfo_error ( EINFO_EIO_UNDERRUN )
+#define EINFO_EIO_UNDERRUN \
+ __einfo_uniqify ( EINFO_EIO, ( 14 - 0 ), \
+ "Ring underrun" )
+#define EIO_OVERRUN \
+ __einfo_error ( EINFO_EIO_OVERRUN )
+#define EINFO_EIO_OVERRUN \
+ __einfo_uniqify ( EINFO_EIO, ( 15 - 0 ), \
+ "Ring overrun" )
+#define EIO_VF_RING_FULL \
+ __einfo_error ( EINFO_EIO_VF_RING_FULL )
+#define EINFO_EIO_VF_RING_FULL \
+ __einfo_uniqify ( EINFO_EIO, ( 16 - 0 ), \
+ "Virtual function event ring full" )
+#define EIO_PARAMETER \
+ __einfo_error ( EINFO_EIO_PARAMETER )
+#define EINFO_EIO_PARAMETER \
+ __einfo_uniqify ( EINFO_EIO, ( 17 - 0 ), \
+ "Parameter error" )
+#define EIO_BANDWIDTH_OVERRUN \
+ __einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN )
+#define EINFO_EIO_BANDWIDTH_OVERRUN \
+ __einfo_uniqify ( EINFO_EIO, ( 18 - 0 ), \
+ "Bandwidth overrun" )
+#define EIO_CONTEXT \
+ __einfo_error ( EINFO_EIO_CONTEXT )
+#define EINFO_EIO_CONTEXT \
+ __einfo_uniqify ( EINFO_EIO, ( 19 - 0 ), \
+ "Context state error" )
+#define EIO_NO_PING \
+ __einfo_error ( EINFO_EIO_NO_PING )
+#define EINFO_EIO_NO_PING \
+ __einfo_uniqify ( EINFO_EIO, ( 20 - 0 ), \
+ "No ping response" )
+#define EIO_RING_FULL \
+ __einfo_error ( EINFO_EIO_RING_FULL )
+#define EINFO_EIO_RING_FULL \
+ __einfo_uniqify ( EINFO_EIO, ( 21 - 0 ), \
+ "Event ring full" )
+#define EIO_INCOMPATIBLE \
+ __einfo_error ( EINFO_EIO_INCOMPATIBLE )
+#define EINFO_EIO_INCOMPATIBLE \
+ __einfo_uniqify ( EINFO_EIO, ( 22 - 0 ), \
+ "Incompatible device" )
+#define EIO_MISSED \
+ __einfo_error ( EINFO_EIO_MISSED )
+#define EINFO_EIO_MISSED \
+ __einfo_uniqify ( EINFO_EIO, ( 23 - 0 ), \
+ "Missed service error" )
+#define EIO_CMD_STOPPED \
+ __einfo_error ( EINFO_EIO_CMD_STOPPED )
+#define EINFO_EIO_CMD_STOPPED \
+ __einfo_uniqify ( EINFO_EIO, ( 24 - 0 ), \
+ "Command ring stopped" )
+#define EIO_CMD_ABORTED \
+ __einfo_error ( EINFO_EIO_CMD_ABORTED )
+#define EINFO_EIO_CMD_ABORTED \
+ __einfo_uniqify ( EINFO_EIO, ( 25 - 0 ), \
+ "Command aborted" )
+#define EIO_STOP \
+ __einfo_error ( EINFO_EIO_STOP )
+#define EINFO_EIO_STOP \
+ __einfo_uniqify ( EINFO_EIO, ( 26 - 0 ), \
+ "Stopped" )
+#define EIO_STOP_LEN \
+ __einfo_error ( EINFO_EIO_STOP_LEN )
+#define EINFO_EIO_STOP_LEN \
+ __einfo_uniqify ( EINFO_EIO, ( 27 - 0 ), \
+ "Stopped - length invalid" )
+#define EIO_STOP_SHORT \
+ __einfo_error ( EINFO_EIO_STOP_SHORT )
+#define EINFO_EIO_STOP_SHORT \
+ __einfo_uniqify ( EINFO_EIO, ( 28 - 0 ), \
+ "Stopped - short packet" )
+#define EIO_LATENCY \
+ __einfo_error ( EINFO_EIO_LATENCY )
+#define EINFO_EIO_LATENCY \
+ __einfo_uniqify ( EINFO_EIO, ( 29 - 0 ), \
+ "Maximum exit latency too large" )
+#define EIO_ISOCH \
+ __einfo_error ( EINFO_EIO_ISOCH )
+#define EINFO_EIO_ISOCH \
+ __einfo_uniqify ( EINFO_EIO, ( 31 - 0 ), \
+ "Isochronous buffer overrun" )
+#define EPROTO_LOST \
+ __einfo_error ( EINFO_EPROTO_LOST )
+#define EINFO_EPROTO_LOST \
+ __einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ), \
+ "Event lost" )
+#define EPROTO_UNDEFINED \
+ __einfo_error ( EINFO_EPROTO_UNDEFINED )
+#define EINFO_EPROTO_UNDEFINED \
+ __einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ), \
+ "Undefined error" )
+#define EPROTO_STREAM_ID \
+ __einfo_error ( EINFO_EPROTO_STREAM_ID )
+#define EINFO_EPROTO_STREAM_ID \
+ __einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ), \
+ "Invalid stream ID" )
+#define EPROTO_SECONDARY \
+ __einfo_error ( EINFO_EPROTO_SECONDARY )
+#define EINFO_EPROTO_SECONDARY \
+ __einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ), \
+ "Secondary bandwidth error" )
+#define EPROTO_SPLIT \
+ __einfo_error ( EINFO_EPROTO_SPLIT )
+#define EINFO_EPROTO_SPLIT \
+ __einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ), \
+ "Split transaction error" )
+#define ECODE(code) \
+ ( ( (code) < 32 ) ? \
+ EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE, \
+ EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE, \
+ EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE, \
+ EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN, \
+ EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER, \
+ EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING, \
+ EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED, \
+ EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP, \
+ EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY, \
+ EIO_ISOCH ) : \
+ ( (code) < 64 ) ? \
+ EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST, \
+ EPROTO_UNDEFINED, EPROTO_STREAM_ID, \
+ EPROTO_SECONDARY, EPROTO_SPLIT ) : \
+ EFAULT )
+
+/******************************************************************************
+ *
+ * Register access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Initialise device
+ *
+ * @v xhci xHCI device
+ * @v regs MMIO registers
+ */
+static void xhci_init ( struct xhci_device *xhci, void *regs ) {
+ uint32_t hcsparams1;
+ uint32_t hcsparams2;
+ uint32_t hccparams1;
+ uint32_t pagesize;
+ size_t caplength;
+ size_t rtsoff;
+ size_t dboff;
+
+ /* Locate capability, operational, runtime, and doorbell registers */
+ xhci->cap = regs;
+ caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH );
+ rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF );
+ dboff = readl ( xhci->cap + XHCI_CAP_DBOFF );
+ xhci->op = ( xhci->cap + caplength );
+ xhci->run = ( xhci->cap + rtsoff );
+ xhci->db = ( xhci->cap + dboff );
+ DBGC2 ( xhci, "XHCI %p cap %08lx op %08lx run %08lx db %08lx\n",
+ xhci, virt_to_phys ( xhci->cap ), virt_to_phys ( xhci->op ),
+ virt_to_phys ( xhci->run ), virt_to_phys ( xhci->db ) );
+
+ /* Read structural parameters 1 */
+ hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 );
+ xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 );
+ xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 );
+ xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 );
+ DBGC ( xhci, "XHCI %p has %d slots %d intrs %d ports\n",
+ xhci, xhci->slots, xhci->intrs, xhci->ports );
+
+ /* Read structural parameters 2 */
+ hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
+ xhci->scratchpads = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
+ DBGC2 ( xhci, "XHCI %p needs %d scratchpads\n",
+ xhci, xhci->scratchpads );
+
+ /* Read capability parameters 1 */
+ hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
+ xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 );
+ xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 );
+ xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 );
+
+ /* Read page size */
+ pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
+ xhci->pagesize = XHCI_PAGESIZE ( pagesize );
+ assert ( xhci->pagesize != 0 );
+ assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 );
+ DBGC2 ( xhci, "XHCI %p page size %zd bytes\n",
+ xhci, xhci->pagesize );
+}
+
+/**
+ * Find extended capability
+ *
+ * @v xhci xHCI device
+ * @v id Capability ID
+ * @v offset Offset to previous extended capability instance, or zero
+ * @ret offset Offset to extended capability, or zero if not found
+ */
+static unsigned int xhci_extended_capability ( struct xhci_device *xhci,
+ unsigned int id,
+ unsigned int offset ) {
+ uint32_t xecp;
+ unsigned int next;
+
+ /* Locate the extended capability */
+ while ( 1 ) {
+
+ /* Locate first or next capability as applicable */
+ if ( offset ) {
+ xecp = readl ( xhci->cap + offset );
+ next = XHCI_XECP_NEXT ( xecp );
+ } else {
+ next = xhci->xecp;
+ }
+ if ( ! next )
+ return 0;
+ offset += next;
+
+ /* Check if this is the requested capability */
+ xecp = readl ( xhci->cap + offset );
+ if ( XHCI_XECP_ID ( xecp ) == id )
+ return offset;
+ }
+}
+
+/**
+ * Write potentially 64-bit register
+ *
+ * @v xhci xHCI device
+ * @v value Value
+ * @v reg Register address
+ * @ret rc Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) {
+
+ /* If this is a 32-bit build, then this can never fail
+ * (allowing the compiler to optimise out the error path).
+ */
+ if ( sizeof ( value ) <= sizeof ( uint32_t ) ) {
+ writel ( value, reg );
+ writel ( 0, ( reg + sizeof ( uint32_t ) ) );
+ return 0;
+ }
+
+ /* If the device does not support 64-bit addresses and this
+ * address is outside the 32-bit address space, then fail.
+ */
+ if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) {
+ DBGC ( xhci, "XHCI %p cannot access address %lx\n",
+ xhci, value );
+ return -ENOTSUP;
+ }
+
+ /* If this is a 64-bit build, then writeq() is available */
+ writeq ( value, reg );
+ return 0;
+}
+
+/**
+ * Calculate buffer alignment
+ *
+ * @v len Length
+ * @ret align Buffer alignment
+ *
+ * Determine alignment required for a buffer which must be aligned to
+ * at least XHCI_MIN_ALIGN and which must not cross a page boundary.
+ */
+static inline size_t xhci_align ( size_t len ) {
+ size_t align;
+
+ /* Align to own length (rounded up to a power of two) */
+ align = ( 1 << fls ( len - 1 ) );
+
+ /* Round up to XHCI_MIN_ALIGN if needed */
+ if ( align < XHCI_MIN_ALIGN )
+ align = XHCI_MIN_ALIGN;
+
+ return align;
+}
+
+/**
+ * Calculate device context offset
+ *
+ * @v xhci xHCI device
+ * @v ctx Context index
+ */
+static inline size_t xhci_device_context_offset ( struct xhci_device *xhci,
+ unsigned int ctx ) {
+
+ return ( XHCI_DCI ( ctx ) << xhci->csz_shift );
+}
+
+/**
+ * Calculate input context offset
+ *
+ * @v xhci xHCI device
+ * @v ctx Context index
+ */
+static inline size_t xhci_input_context_offset ( struct xhci_device *xhci,
+ unsigned int ctx ) {
+
+ return ( XHCI_ICI ( ctx ) << xhci->csz_shift );
+}
+
+/******************************************************************************
+ *
+ * Diagnostics
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Dump host controller registers
+ *
+ * @v xhci xHCI device
+ */
+static inline void xhci_dump ( struct xhci_device *xhci ) {
+ uint32_t usbcmd;
+ uint32_t usbsts;
+ uint32_t pagesize;
+ uint32_t dnctrl;
+ uint32_t config;
+
+ /* Do nothing unless debugging is enabled */
+ if ( ! DBG_LOG )
+ return;
+
+ /* Dump USBCMD */
+ usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+ DBGC ( xhci, "XHCI %p USBCMD %08x%s%s\n", xhci, usbcmd,
+ ( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ),
+ ( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) );
+
+ /* Dump USBSTS */
+ usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
+ DBGC ( xhci, "XHCI %p USBSTS %08x%s\n", xhci, usbsts,
+ ( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) );
+
+ /* Dump PAGESIZE */
+ pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
+ DBGC ( xhci, "XHCI %p PAGESIZE %08x\n", xhci, pagesize );
+
+ /* Dump DNCTRL */
+ dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL );
+ DBGC ( xhci, "XHCI %p DNCTRL %08x\n", xhci, dnctrl );
+
+ /* Dump CONFIG */
+ config = readl ( xhci->op + XHCI_OP_CONFIG );
+ DBGC ( xhci, "XHCI %p CONFIG %08x\n", xhci, config );
+}
+
+/**
+ * Dump port registers
+ *
+ * @v xhci xHCI device
+ * @v port Port number
+ */
+static inline void xhci_dump_port ( struct xhci_device *xhci,
+ unsigned int port ) {
+ uint32_t portsc;
+ uint32_t portpmsc;
+ uint32_t portli;
+ uint32_t porthlpmc;
+
+ /* Do nothing unless debugging is enabled */
+ if ( ! DBG_LOG )
+ return;
+
+ /* Dump PORTSC */
+ portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) );
+ DBGC ( xhci, "XHCI %p port %d PORTSC %08x%s%s%s%s psiv=%d\n",
+ xhci, port, portsc,
+ ( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ),
+ ( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ),
+ ( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ),
+ ( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ),
+ XHCI_PORTSC_PSIV ( portsc ) );
+
+ /* Dump PORTPMSC */
+ portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) );
+ DBGC ( xhci, "XHCI %p port %d PORTPMSC %08x\n", xhci, port, portpmsc );
+
+ /* Dump PORTLI */
+ portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) );
+ DBGC ( xhci, "XHCI %p port %d PORTLI %08x\n", xhci, port, portli );
+
+ /* Dump PORTHLPMC */
+ porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) );
+ DBGC ( xhci, "XHCI %p port %d PORTHLPMC %08x\n",
+ xhci, port, porthlpmc );
+}
+
+/******************************************************************************
+ *
+ * USB legacy support
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Initialise USB legacy support
+ *
+ * @v xhci xHCI device
+ */
+static void xhci_legacy_init ( struct xhci_device *xhci ) {
+ unsigned int legacy;
+ uint8_t bios;
+
+ /* Locate USB legacy support capability (if present) */
+ legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 );
+ if ( ! legacy ) {
+ /* Not an error; capability may not be present */
+ DBGC ( xhci, "XHCI %p has no USB legacy support capability\n",
+ xhci );
+ return;
+ }
+
+ /* Check if legacy USB support is enabled */
+ bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS );
+ if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
+ /* Not an error; already owned by OS */
+ DBGC ( xhci, "XHCI %p USB legacy support already disabled\n",
+ xhci );
+ return;
+ }
+
+ /* Record presence of USB legacy support capability */
+ xhci->legacy = legacy;
+}
+
+/**
+ * Claim ownership from BIOS
+ *
+ * @v xhci xHCI device
+ * @ret rc Return status code
+ */
+static int xhci_legacy_claim ( struct xhci_device *xhci ) {
+ uint32_t ctlsts;
+ uint8_t bios;
+ unsigned int i;
+
+ /* Do nothing unless legacy support capability is present */
+ if ( ! xhci->legacy )
+ return 0;
+
+ /* Claim ownership */
+ writeb ( XHCI_USBLEGSUP_OS_OWNED,
+ xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
+
+ /* Wait for BIOS to release ownership */
+ for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
+
+ /* Check if BIOS has released ownership */
+ bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS );
+ if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
+ DBGC ( xhci, "XHCI %p claimed ownership from BIOS\n",
+ xhci );
+ ctlsts = readl ( xhci->cap + xhci->legacy +
+ XHCI_USBLEGSUP_CTLSTS );
+ if ( ctlsts ) {
+ DBGC ( xhci, "XHCI %p warning: BIOS retained "
+ "SMIs: %08x\n", xhci, ctlsts );
+ }
+ return 0;
+ }
+
+ /* Delay */
+ mdelay ( 1 );
+ }
+
+ DBGC ( xhci, "XHCI %p timed out waiting for BIOS to release "
+ "ownership\n", xhci );
+ return -ETIMEDOUT;
+}
+
+/**
+ * Release ownership back to BIOS
+ *
+ * @v xhci xHCI device
+ */
+static void xhci_legacy_release ( struct xhci_device *xhci ) {
+
+ /* Do nothing unless legacy support capability is present */
+ if ( ! xhci->legacy )
+ return;
+
+ /* Release ownership */
+ writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
+ DBGC ( xhci, "XHCI %p released ownership to BIOS\n", xhci );
+}
+
+/******************************************************************************
+ *
+ * Supported protocols
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Transcribe port speed (for debugging)
+ *
+ * @v psi Protocol speed ID
+ * @ret speed Transcribed speed
+ */
+static inline const char * xhci_speed_name ( uint32_t psi ) {
+ static const char *exponents[4] = { "", "k", "M", "G" };
+ static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
+ unsigned int mantissa;
+ unsigned int exponent;
+
+ /* Extract mantissa and exponent */
+ mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
+ exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
+
+ /* Transcribe speed */
+ snprintf ( buf, sizeof ( buf ), "%d%sbps",
+ mantissa, exponents[exponent] );
+ return buf;
+}
+
+/**
+ * Find supported protocol extended capability for a port
+ *
+ * @v xhci xHCI device
+ * @v port Port number
+ * @ret supported Offset to extended capability, or zero if not found
+ */
+static unsigned int xhci_supported_protocol ( struct xhci_device *xhci,
+ unsigned int port ) {
+ unsigned int supported = 0;
+ unsigned int offset;
+ unsigned int count;
+ uint32_t ports;
+
+ /* Iterate over all supported protocol structures */
+ while ( ( supported = xhci_extended_capability ( xhci,
+ XHCI_XECP_ID_SUPPORTED,
+ supported ) ) ) {
+
+ /* Determine port range */
+ ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+ offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports );
+ count = XHCI_SUPPORTED_PORTS_COUNT ( ports );
+
+ /* Check if port lies within this range */
+ if ( ( port - offset ) < count )
+ return supported;
+ }
+
+ DBGC ( xhci, "XHCI %p port %d has no supported protocol\n",
+ xhci, port );
+ return 0;
+}
+
+/**
+ * Find port protocol
+ *
+ * @v xhci xHCI device
+ * @v port Port number
+ * @ret protocol USB protocol, or zero if not found
+ */
+static unsigned int xhci_port_protocol ( struct xhci_device *xhci,
+ unsigned int port ) {
+ unsigned int supported = xhci_supported_protocol ( xhci, port );
+ union {
+ uint32_t raw;
+ char text[5];
+ } name;
+ unsigned int protocol;
+ unsigned int type;
+ unsigned int psic;
+ unsigned int psiv;
+ unsigned int i;
+ uint32_t revision;
+ uint32_t ports;
+ uint32_t slot;
+ uint32_t psi;
+
+ /* Fail if there is no supported protocol */
+ if ( ! supported )
+ return 0;
+
+ /* Determine protocol version */
+ revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION );
+ protocol = XHCI_SUPPORTED_REVISION_VER ( revision );
+
+ /* Describe port protocol */
+ if ( DBG_EXTRA ) {
+ name.raw = cpu_to_le32 ( readl ( xhci->cap + supported +
+ XHCI_SUPPORTED_NAME ) );
+ name.text[4] = '\0';
+ slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
+ type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
+ DBGC2 ( xhci, "XHCI %p port %d %sv%04x type %d",
+ xhci, port, name.text, protocol, type );
+ ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+ psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
+ if ( psic ) {
+ DBGC2 ( xhci, " speeds" );
+ for ( i = 0 ; i < psic ; i++ ) {
+ psi = readl ( xhci->cap + supported +
+ XHCI_SUPPORTED_PSI ( i ) );
+ psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
+ DBGC2 ( xhci, " %d:%s", psiv,
+ xhci_speed_name ( psi ) );
+ }
+ }
+ DBGC2 ( xhci, "\n" );
+ }
+
+ return protocol;
+}
+
+/**
+ * Find port slot type
+ *
+ * @v xhci xHCI device
+ * @v port Port number
+ * @ret type Slot type, or negative error
+ */
+static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) {
+ unsigned int supported = xhci_supported_protocol ( xhci, port );
+ unsigned int type;
+ uint32_t slot;
+
+ /* Fail if there is no supported protocol */
+ if ( ! supported )
+ return -ENOTSUP;
+
+ /* Get slot type */
+ slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
+ type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
+
+ return type;
+}
+
+/**
+ * Find port speed
+ *
+ * @v xhci xHCI device
+ * @v port Port number
+ * @v psiv Protocol speed ID value
+ * @ret speed Port speed, or negative error
+ */
+static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port,
+ unsigned int psiv ) {
+ unsigned int supported = xhci_supported_protocol ( xhci, port );
+ unsigned int psic;
+ unsigned int mantissa;
+ unsigned int exponent;
+ unsigned int speed;
+ unsigned int i;
+ uint32_t ports;
+ uint32_t psi;
+
+ /* Fail if there is no supported protocol */
+ if ( ! supported )
+ return -ENOTSUP;
+
+ /* Get protocol speed ID count */
+ ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+ psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
+
+ /* Use the default mappings if applicable */
+ if ( ! psic ) {
+ switch ( psiv ) {
+ case XHCI_SPEED_LOW : return USB_SPEED_LOW;
+ case XHCI_SPEED_FULL : return USB_SPEED_FULL;
+ case XHCI_SPEED_HIGH : return USB_SPEED_HIGH;
+ case XHCI_SPEED_SUPER : return USB_SPEED_SUPER;
+ default:
+ DBGC ( xhci, "XHCI %p port %d non-standard PSI value "
+ "%d\n", xhci, port, psiv );
+ return -ENOTSUP;
+ }
+ }
+
+ /* Iterate over PSI dwords looking for a match */
+ for ( i = 0 ; i < psic ; i++ ) {
+ psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
+ if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) {
+ mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
+ exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
+ speed = USB_SPEED ( mantissa, exponent );
+ return speed;
+ }
+ }
+
+ DBGC ( xhci, "XHCI %p port %d spurious PSI value %d\n",
+ xhci, port, psiv );
+ return -ENOENT;
+}
+
+/**
+ * Find protocol speed ID value
+ *
+ * @v xhci xHCI device
+ * @v port Port number
+ * @v speed USB speed
+ * @ret psiv Protocol speed ID value, or negative error
+ */
+static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port,
+ unsigned int speed ) {
+ unsigned int supported = xhci_supported_protocol ( xhci, port );
+ unsigned int psic;
+ unsigned int mantissa;
+ unsigned int exponent;
+ unsigned int psiv;
+ unsigned int i;
+ uint32_t ports;
+ uint32_t psi;
+
+ /* Fail if there is no supported protocol */
+ if ( ! supported )
+ return -ENOTSUP;
+
+ /* Get protocol speed ID count */
+ ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+ psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
+
+ /* Use the default mappings if applicable */
+ if ( ! psic ) {
+ switch ( speed ) {
+ case USB_SPEED_LOW : return XHCI_SPEED_LOW;
+ case USB_SPEED_FULL : return XHCI_SPEED_FULL;
+ case USB_SPEED_HIGH : return XHCI_SPEED_HIGH;
+ case USB_SPEED_SUPER : return XHCI_SPEED_SUPER;
+ default:
+ DBGC ( xhci, "XHCI %p port %d non-standad speed %d\n",
+ xhci, port, speed );
+ return -ENOTSUP;
+ }
+ }
+
+ /* Iterate over PSI dwords looking for a match */
+ for ( i = 0 ; i < psic ; i++ ) {
+ psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
+ mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
+ exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
+ if ( speed == USB_SPEED ( mantissa, exponent ) ) {
+ psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
+ return psiv;
+ }
+ }
+
+ DBGC ( xhci, "XHCI %p port %d unrepresentable speed %#x\n",
+ xhci, port, speed );
+ return -ENOENT;
+}
+
+/******************************************************************************
+ *
+ * Device context base address array
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate device context base address array
+ *
+ * @v xhci xHCI device
+ * @ret rc Return status code
+ */
+static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
+ size_t len;
+ physaddr_t dcbaap;
+ int rc;
+
+ /* Allocate and initialise structure. Must be at least
+ * 64-byte aligned and must not cross a page boundary, so
+ * align on its own size (rounded up to a power of two and
+ * with a minimum of 64 bytes).
+ */
+ len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
+ xhci->dcbaa = malloc_dma ( len, xhci_align ( len ) );
+ if ( ! xhci->dcbaa ) {
+ DBGC ( xhci, "XHCI %p could not allocate DCBAA\n", xhci );
+ rc = -ENOMEM;
+ goto err_alloc;
+ }
+ memset ( xhci->dcbaa, 0, len );
+
+ /* Program DCBAA pointer */
+ dcbaap = virt_to_phys ( xhci->dcbaa );
+ if ( ( rc = xhci_writeq ( xhci, dcbaap,
+ xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
+ goto err_writeq;
+
+ DBGC2 ( xhci, "XHCI %p DCBAA at [%08lx,%08lx)\n",
+ xhci, dcbaap, ( dcbaap + len ) );
+ return 0;
+
+ err_writeq:
+ free_dma ( xhci->dcbaa, len );
+ err_alloc:
+ return rc;
+}
+
+/**
+ * Free device context base address array
+ *
+ * @v xhci xHCI device
+ */
+static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
+ size_t len;
+ unsigned int i;
+
+ /* Sanity check */
+ for ( i = 0 ; i <= xhci->slots ; i++ )
+ assert ( xhci->dcbaa[i] == 0 );
+
+ /* Clear DCBAA pointer */
+ xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
+
+ /* Free DCBAA */
+ len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
+ free_dma ( xhci->dcbaa, len );
+}
+
+/******************************************************************************
+ *
+ * Scratchpad buffers
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate scratchpad buffers
+ *
+ * @v xhci xHCI device
+ * @ret rc Return status code
+ */
+static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
+ size_t array_len;
+ size_t len;
+ physaddr_t phys;
+ unsigned int i;
+ int rc;
+
+ /* Do nothing if no scratchpad buffers are used */
+ if ( ! xhci->scratchpads )
+ return 0;
+
+ /* Allocate scratchpads */
+ len = ( xhci->scratchpads * xhci->pagesize );
+ xhci->scratchpad = umalloc ( len );
+ if ( ! xhci->scratchpad ) {
+ DBGC ( xhci, "XHCI %p could not allocate scratchpad buffers\n",
+ xhci );
+ rc = -ENOMEM;
+ goto err_alloc;
+ }
+ memset_user ( xhci->scratchpad, 0, 0, len );
+
+ /* Allocate scratchpad array */
+ array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
+ xhci->scratchpad_array =
+ malloc_dma ( array_len, xhci_align ( array_len ) );
+ if ( ! xhci->scratchpad_array ) {
+ DBGC ( xhci, "XHCI %p could not allocate scratchpad buffer "
+ "array\n", xhci );
+ rc = -ENOMEM;
+ goto err_alloc_array;
+ }
+
+ /* Populate scratchpad array */
+ for ( i = 0 ; i < xhci->scratchpads ; i++ ) {
+ phys = user_to_phys ( xhci->scratchpad, ( i * xhci->pagesize ));
+ xhci->scratchpad_array[i] = phys;
+ }
+
+ /* Set scratchpad array pointer */
+ assert ( xhci->dcbaa != NULL );
+ xhci->dcbaa[0] = cpu_to_le64 ( virt_to_phys ( xhci->scratchpad_array ));
+
+ DBGC2 ( xhci, "XHCI %p scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
+ xhci, user_to_phys ( xhci->scratchpad, 0 ),
+ user_to_phys ( xhci->scratchpad, len ),
+ virt_to_phys ( xhci->scratchpad_array ),
+ ( virt_to_phys ( xhci->scratchpad_array ) + array_len ) );
+ return 0;
+
+ free_dma ( xhci->scratchpad_array, array_len );
+ err_alloc_array:
+ ufree ( xhci->scratchpad );
+ err_alloc:
+ return rc;
+}
+
+/**
+ * Free scratchpad buffers
+ *
+ * @v xhci xHCI device
+ */
+static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
+ size_t array_len;
+
+ /* Do nothing if no scratchpad buffers are used */
+ if ( ! xhci->scratchpads )
+ return;
+
+ /* Clear scratchpad array pointer */
+ assert ( xhci->dcbaa != NULL );
+ xhci->dcbaa[0] = 0;
+
+ /* Free scratchpad array */
+ array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
+ free_dma ( xhci->scratchpad_array, array_len );
+
+ /* Free scratchpads */
+ ufree ( xhci->scratchpad );
+}
+
+/******************************************************************************
+ *
+ * Run / stop / reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Start xHCI device
+ *
+ * @v xhci xHCI device
+ */
+static void xhci_run ( struct xhci_device *xhci ) {
+ uint32_t config;
+ uint32_t usbcmd;
+
+ /* Configure number of device slots */
+ config = readl ( xhci->op + XHCI_OP_CONFIG );
+ config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK;
+ config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots );
+ writel ( config, xhci->op + XHCI_OP_CONFIG );
+
+ /* Set run/stop bit */
+ usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+ usbcmd |= XHCI_USBCMD_RUN;
+ writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
+}
+
+/**
+ * Stop xHCI device
+ *
+ * @v xhci xHCI device
+ * @ret rc Return status code
+ */
+static int xhci_stop ( struct xhci_device *xhci ) {
+ uint32_t usbcmd;
+ uint32_t usbsts;
+ unsigned int i;
+
+ /* Clear run/stop bit */
+ usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+ usbcmd &= ~XHCI_USBCMD_RUN;
+ writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
+
+ /* Wait for device to stop */
+ for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) {
+
+ /* Check if device is stopped */
+ usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
+ if ( usbsts & XHCI_USBSTS_HCH )
+ return 0;
+
+ /* Delay */
+ mdelay ( 1 );
+ }
+
+ DBGC ( xhci, "XHCI %p timed out waiting for stop\n", xhci );
+ return -ETIMEDOUT;
+}
+
+/**
+ * Reset xHCI device
+ *
+ * @v xhci xHCI device
+ * @ret rc Return status code
+ */
+static int xhci_reset ( struct xhci_device *xhci ) {
+ uint32_t usbcmd;
+ unsigned int i;
+ int rc;
+
+ /* The xHCI specification states that resetting a running
+ * device may result in undefined behaviour, so try stopping
+ * it first.
+ */
+ if ( ( rc = xhci_stop ( xhci ) ) != 0 ) {
+ /* Ignore errors and attempt to reset the device anyway */
+ }
+
+ /* Reset device */
+ writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD );
+
+ /* Wait for reset to complete */
+ for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) {
+
+ /* Check if reset is complete */
+ usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+ if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) )
+ return 0;
+
+ /* Delay */
+ mdelay ( 1 );
+ }
+
+ DBGC ( xhci, "XHCI %p timed out waiting for reset\n", xhci );
+ return -ETIMEDOUT;
+}
+
+/******************************************************************************
+ *
+ * Transfer request blocks
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate transfer request block ring
+ *
+ * @v xhci xHCI device
+ * @v ring TRB ring
+ * @v shift Ring size (log2)
+ * @v slot Device slot
+ * @v target Doorbell target
+ * @v stream Doorbell stream ID
+ * @ret rc Return status code
+ */
+static int xhci_ring_alloc ( struct xhci_device *xhci,
+ struct xhci_trb_ring *ring,
+ unsigned int shift, unsigned int slot,
+ unsigned int target, unsigned int stream ) {
+ struct xhci_trb_link *link;
+ unsigned int count;
+ int rc;
+
+ /* Sanity check */
+ assert ( shift > 0 );
+
+ /* Initialise structure */
+ memset ( ring, 0, sizeof ( *ring ) );
+ ring->shift = shift;
+ count = ( 1U << shift );
+ ring->mask = ( count - 1 );
+ ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) );
+ ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) );
+ ring->dbval = XHCI_DBVAL ( target, stream );
+
+ /* Allocate I/O buffers */
+ ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) );
+ if ( ! ring->iobuf ) {
+ rc = -ENOMEM;
+ goto err_alloc_iobuf;
+ }
+
+ /* Allocate TRBs */
+ ring->trb = malloc_dma ( ring->len, xhci_align ( ring->len ) );
+ if ( ! ring->trb ) {
+ rc = -ENOMEM;
+ goto err_alloc_trb;
+ }
+ memset ( ring->trb, 0, ring->len );
+
+ /* Initialise Link TRB */
+ link = &ring->trb[count].link;
+ link->next = cpu_to_le64 ( virt_to_phys ( ring->trb ) );
+ link->flags = XHCI_TRB_TC;
+ link->type = XHCI_TRB_LINK;
+ ring->link = link;
+
+ return 0;
+
+ free_dma ( ring->trb, ring->len );
+ err_alloc_trb:
+ free ( ring->iobuf );
+ err_alloc_iobuf:
+ return rc;
+}
+
+/**
+ * Free transfer request block ring
+ *
+ * @v ring TRB ring
+ */
+static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
+ unsigned int count = ( 1U << ring->shift );
+ unsigned int i;
+
+ /* Sanity checks */
+ assert ( ring->cons == ring->prod );
+ for ( i = 0 ; i < count ; i++ )
+ assert ( ring->iobuf[i] == NULL );
+
+ /* Free TRBs */
+ free_dma ( ring->trb, ring->len );
+
+ /* Free I/O buffers */
+ free ( ring->iobuf );
+}
+
+/**
+ * Enqueue a transfer request block
+ *
+ * @v ring TRB ring
+ * @v iobuf I/O buffer (if any)
+ * @v trb Transfer request block (with empty Cycle flag)
+ * @ret rc Return status code
+ *
+ * This operation does not implicitly ring the doorbell register.
+ */
+static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf,
+ const union xhci_trb *trb ) {
+ union xhci_trb *dest;
+ unsigned int prod;
+ unsigned int mask;
+ unsigned int index;
+ unsigned int cycle;
+
+ /* Sanity check */
+ assert ( ! ( trb->common.flags & XHCI_TRB_C ) );
+
+ /* Fail if ring is full */
+ if ( ! xhci_ring_remaining ( ring ) )
+ return -ENOBUFS;
+
+ /* Update producer counter (and link TRB, if applicable) */
+ prod = ring->prod++;
+ mask = ring->mask;
+ cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C );
+ index = ( prod & mask );
+ if ( index == 0 )
+ ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) );
+
+ /* Record I/O buffer */
+ ring->iobuf[index] = iobuf;
+
+ /* Enqueue TRB */
+ dest = &ring->trb[index];
+ dest->template.parameter = trb->template.parameter;
+ dest->template.status = trb->template.status;
+ wmb();
+ dest->template.control = ( trb->template.control |
+ cpu_to_le32 ( cycle ) );
+
+ return 0;
+}
+
+/**
+ * Dequeue a transfer request block
+ *
+ * @v ring TRB ring
+ * @ret iobuf I/O buffer
+ */
+static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) {
+ struct io_buffer *iobuf;
+ unsigned int cons;
+ unsigned int mask;
+ unsigned int index;
+
+ /* Sanity check */
+ assert ( xhci_ring_fill ( ring ) != 0 );
+
+ /* Update consumer counter */
+ cons = ring->cons++;
+ mask = ring->mask;
+ index = ( cons & mask );
+
+ /* Retrieve I/O buffer */
+ iobuf = ring->iobuf[index];
+ ring->iobuf[index] = NULL;
+
+ return iobuf;
+}
+
+/**
+ * Enqueue multiple transfer request blocks
+ *
+ * @v ring TRB ring
+ * @v iobuf I/O buffer
+ * @v trbs Transfer request blocks (with empty Cycle flag)
+ * @v count Number of transfer request blocks
+ * @ret rc Return status code
+ *
+ * This operation does not implicitly ring the doorbell register.
+ */
+static int xhci_enqueue_multi ( struct xhci_trb_ring *ring,
+ struct io_buffer *iobuf,
+ const union xhci_trb *trbs,
+ unsigned int count ) {
+ const union xhci_trb *trb = trbs;
+ int rc;
+
+ /* Sanity check */
+ assert ( iobuf != NULL );
+
+ /* Fail if ring does not have sufficient space */
+ if ( xhci_ring_remaining ( ring ) < count )
+ return -ENOBUFS;
+
+ /* Enqueue each TRB, recording the I/O buffer with the final TRB */
+ while ( count-- ) {
+ rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ );
+ assert ( rc == 0 ); /* Should never be able to fail */
+ }
+
+ return 0;
+}
+
+/**
+ * Dequeue multiple transfer request blocks
+ *
+ * @v ring TRB ring
+ * @ret iobuf I/O buffer
+ */
+static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) {
+ struct io_buffer *iobuf;
+
+ /* Dequeue TRBs until we reach the final TRB for an I/O buffer */
+ do {
+ iobuf = xhci_dequeue ( ring );
+ } while ( iobuf == NULL );
+
+ return iobuf;
+}
+
+/**
+ * Ring doorbell register
+ *
+ * @v ring TRB ring
+ */
+static inline __attribute__ (( always_inline )) void
+xhci_doorbell ( struct xhci_trb_ring *ring ) {
+
+ wmb();
+ writel ( ring->dbval, ring->db );
+}
+
+/******************************************************************************
+ *
+ * Command and event rings
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate command ring
+ *
+ * @v xhci xHCI device
+ * @ret rc Return status code
+ */
+static int xhci_command_alloc ( struct xhci_device *xhci ) {
+ physaddr_t crp;
+ int rc;
+
+ /* Allocate TRB ring */
+ if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2,
+ 0, 0, 0 ) ) != 0 )
+ goto err_ring_alloc;
+
+ /* Program command ring control register */
+ crp = virt_to_phys ( xhci->command.trb );
+ if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
+ xhci->op + XHCI_OP_CRCR ) ) != 0 )
+ goto err_writeq;
+
+ DBGC2 ( xhci, "XHCI %p CRCR at [%08lx,%08lx)\n",
+ xhci, crp, ( crp + xhci->command.len ) );
+ return 0;
+
+ err_writeq:
+ xhci_ring_free ( &xhci->command );
+ err_ring_alloc:
+ return rc;
+}
+
+/**
+ * Free command ring
+ *
+ * @v xhci xHCI device
+ */
+static void xhci_command_free ( struct xhci_device *xhci ) {
+
+ /* Sanity check */
+ assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
+
+ /* Clear command ring control register */
+ xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR );
+
+ /* Free TRB ring */
+ xhci_ring_free ( &xhci->command );
+}
+
+/**
+ * Allocate event ring
+ *
+ * @v xhci xHCI device
+ * @ret rc Return status code
+ */
+static int xhci_event_alloc ( struct xhci_device *xhci ) {
+ struct xhci_event_ring *event = &xhci->event;
+ unsigned int count;
+ size_t len;
+ int rc;
+
+ /* Allocate event ring */
+ count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
+ len = ( count * sizeof ( event->trb[0] ) );
+ event->trb = malloc_dma ( len, xhci_align ( len ) );
+ if ( ! event->trb ) {
+ rc = -ENOMEM;
+ goto err_alloc_trb;
+ }
+ memset ( event->trb, 0, len );
+
+ /* Allocate event ring segment table */
+ event->segment = malloc_dma ( sizeof ( event->segment[0] ),
+ xhci_align ( sizeof (event->segment[0])));
+ if ( ! event->segment ) {
+ rc = -ENOMEM;
+ goto err_alloc_segment;
+ }
+ memset ( event->segment, 0, sizeof ( event->segment[0] ) );
+ event->segment[0].base = cpu_to_le64 ( virt_to_phys ( event->trb ) );
+ event->segment[0].count = cpu_to_le32 ( count );
+
+ /* Program event ring registers */
+ writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
+ if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->trb ),
+ xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
+ goto err_writeq_erdp;
+ if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->segment ),
+ xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
+ goto err_writeq_erstba;
+
+ DBGC2 ( xhci, "XHCI %p event ring [%08lx,%08lx) table [%08lx,%08lx)\n",
+ xhci, virt_to_phys ( event->trb ),
+ ( virt_to_phys ( event->trb ) + len ),
+ virt_to_phys ( event->segment ),
+ ( virt_to_phys ( event->segment ) +
+ sizeof (event->segment[0] ) ) );
+ return 0;
+
+ xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
+ err_writeq_erstba:
+ xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
+ err_writeq_erdp:
+ free_dma ( event->trb, len );
+ err_alloc_segment:
+ free_dma ( event->segment, sizeof ( event->segment[0] ) );
+ err_alloc_trb:
+ return rc;
+}
+
+/**
+ * Free event ring
+ *
+ * @v xhci xHCI device
+ */
+static void xhci_event_free ( struct xhci_device *xhci ) {
+ struct xhci_event_ring *event = &xhci->event;
+ unsigned int count;
+ size_t len;
+
+ /* Clear event ring registers */
+ writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
+ xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
+ xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
+
+ /* Free event ring segment table */
+ free_dma ( event->segment, sizeof ( event->segment[0] ) );
+
+ /* Free event ring */
+ count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
+ len = ( count * sizeof ( event->trb[0] ) );
+ free_dma ( event->trb, len );
+}
+
+/**
+ * Handle transfer event
+ *
+ * @v xhci xHCI device
+ * @v transfer Transfer event TRB
+ */
+static void xhci_transfer ( struct xhci_device *xhci,
+ struct xhci_trb_transfer *transfer ) {
+ struct xhci_slot *slot;
+ struct xhci_endpoint *endpoint;
+ struct io_buffer *iobuf;
+ int rc;
+
+ /* Profile transfer events */
+ profile_start ( &xhci_transfer_profiler );
+
+ /* Identify slot */
+ if ( ( transfer->slot > xhci->slots ) ||
+ ( ( slot = xhci->slot[transfer->slot] ) == NULL ) ) {
+ DBGC ( xhci, "XHCI %p transfer event invalid slot %d:\n",
+ xhci, transfer->slot );
+ DBGC_HDA ( xhci, 0, transfer, sizeof ( *transfer ) );
+ return;
+ }
+
+ /* Identify endpoint */
+ if ( ( transfer->endpoint > XHCI_CTX_END ) ||
+ ( ( endpoint = slot->endpoint[transfer->endpoint] ) == NULL ) ) {
+ DBGC ( xhci, "XHCI %p slot %d transfer event invalid epid "
+ "%d:\n", xhci, slot->id, transfer->endpoint );
+ DBGC_HDA ( xhci, 0, transfer, sizeof ( *transfer ) );
+ return;
+ }
+
+ /* Dequeue TRB(s) */
+ iobuf = xhci_dequeue_multi ( &endpoint->ring );
+ assert ( iobuf != NULL );
+
+ /* Check for errors */
+ if ( ! ( ( transfer->code == XHCI_CMPLT_SUCCESS ) ||
+ ( transfer->code == XHCI_CMPLT_SHORT ) ) ) {
+
+ /* Construct error */
+ rc = -ECODE ( transfer->code );
+ DBGC ( xhci, "XHCI %p slot %d ctx %d failed (code %d): %s\n",
+ xhci, slot->id, endpoint->ctx, transfer->code,
+ strerror ( rc ) );
+ DBGC_HDA ( xhci, 0, transfer, sizeof ( *transfer ) );
+
+ /* Sanity check */
+ assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK )
+ != XHCI_ENDPOINT_RUNNING );
+
+ /* Report failure to USB core */
+ usb_complete_err ( endpoint->ep, iobuf, rc );
+ return;
+ }
+
+ /* Record actual transfer size */
+ iob_unput ( iobuf, le16_to_cpu ( transfer->residual ) );
+
+ /* Sanity check (for successful completions only) */
+ assert ( xhci_ring_consumed ( &endpoint->ring ) ==
+ le64_to_cpu ( transfer->transfer ) );
+
+ /* Report completion to USB core */
+ usb_complete ( endpoint->ep, iobuf );
+ profile_stop ( &xhci_transfer_profiler );
+}
+
+/**
+ * Handle command completion event
+ *
+ * @v xhci xHCI device
+ * @v complete Command completion event
+ */
+static void xhci_complete ( struct xhci_device *xhci,
+ struct xhci_trb_complete *complete ) {
+
+ /* Dequeue command TRB */
+ xhci_dequeue ( &xhci->command );
+
+ /* Sanity check */
+ assert ( xhci_ring_consumed ( &xhci->command ) ==
+ le64_to_cpu ( complete->command ) );
+
+ /* Record completion if applicable */
+ if ( xhci->completion ) {
+ memcpy ( xhci->completion, complete,
+ sizeof ( *xhci->completion ) );
+ xhci->completion = NULL;
+ } else {
+ DBGC ( xhci, "XHCI %p unexpected completion:\n", xhci );
+ DBGC_HDA ( xhci, 0, complete, sizeof ( *complete ) );
+ }
+}
+
+/**
+ * Handle port status event
+ *
+ * @v xhci xHCI device
+ * @v port Port status event
+ */
+static void xhci_port_status ( struct xhci_device *xhci,
+ struct xhci_trb_port_status *port ) {
+ uint32_t portsc;
+
+ /* Sanity check */
+ assert ( ( port->port > 0 ) && ( port->port <= xhci->ports ) );
+
+ /* Clear port status change bits */
+ portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->port ) );
+ portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
+ writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->port ) );
+
+ /* Report port status change */
+ usb_port_changed ( usb_port ( xhci->bus->hub, port->port ) );
+}
+
+/**
+ * Poll event ring
+ *
+ * @v xhci xHCI device
+ */
+static void xhci_event_poll ( struct xhci_device *xhci ) {
+ struct xhci_event_ring *event = &xhci->event;
+ union xhci_trb *trb;
+ unsigned int shift = XHCI_EVENT_TRBS_LOG2;
+ unsigned int count = ( 1 << shift );
+ unsigned int mask = ( count - 1 );
+ unsigned int consumed;
+ unsigned int type;
+
+ /* Poll for events */
+ profile_start ( &xhci_event_profiler );
+ for ( consumed = 0 ; ; consumed++ ) {
+
+ /* Stop if we reach an empty TRB */
+ rmb();
+ trb = &event->trb[ event->cons & mask ];
+ if ( ! ( ( trb->common.flags ^
+ ( event->cons >> shift ) ) & XHCI_TRB_C ) )
+ break;
+
+ /* Handle TRB */
+ type = ( trb->common.type & XHCI_TRB_TYPE_MASK );
+ switch ( type ) {
+
+ case XHCI_TRB_TRANSFER :
+ xhci_transfer ( xhci, &trb->transfer );
+ break;
+
+ case XHCI_TRB_COMPLETE :
+ xhci_complete ( xhci, &trb->complete );
+ break;
+
+ case XHCI_TRB_PORT_STATUS:
+ xhci_port_status ( xhci, &trb->port );
+ break;
+
+ default:
+ DBGC ( xhci, "XHCI %p unrecognised event %#x\n:",
+ xhci, event->cons );
+ DBGC_HDA ( xhci, virt_to_phys ( trb ),
+ trb, sizeof ( *trb ) );
+ break;
+ }
+
+ /* Consume this TRB */
+ event->cons++;
+ }
+
+ /* Update dequeue pointer if applicable */
+ if ( consumed ) {
+ xhci_writeq ( xhci, virt_to_phys ( trb ),
+ xhci->run + XHCI_RUN_ERDP ( 0 ) );
+ profile_stop ( &xhci_event_profiler );
+ }
+}
+
+/**
+ * Issue command and wait for completion
+ *
+ * @v xhci xHCI device
+ * @v trb Transfer request block (with empty Cycle flag)
+ * @ret rc Return status code
+ *
+ * On a successful completion, the TRB will be overwritten with the
+ * completion.
+ */
+static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
+ struct xhci_trb_complete *complete = &trb->complete;
+ unsigned int i;
+ int rc;
+
+ /* Record the completion buffer */
+ xhci->completion = trb;
+
+ /* Enqueue the command */
+ if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 )
+ goto err_enqueue;
+
+ /* Ring the command doorbell */
+ xhci_doorbell ( &xhci->command );
+
+ /* Wait for the command to complete */
+ for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) {
+
+ /* Poll event ring */
+ xhci_event_poll ( xhci );
+
+ /* Check for completion */
+ if ( ! xhci->completion ) {
+ if ( complete->code != XHCI_CMPLT_SUCCESS ) {
+ rc = -ECODE ( complete->code );
+ DBGC ( xhci, "XHCI %p command failed (code "
+ "%d): %s\n", xhci, complete->code,
+ strerror ( rc ) );
+ DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+ return rc;
+ }
+ return 0;
+ }
+
+ /* Delay */
+ mdelay ( 1 );
+ }
+
+ /* Timeout */
+ DBGC ( xhci, "XHCI %p timed out waiting for completion\n", xhci );
+ rc = -ETIMEDOUT;
+
+ err_enqueue:
+ xhci->completion = NULL;
+ return rc;
+}
+
+/**
+ * Issue NOP and wait for completion
+ *
+ * @v xhci xHCI device
+ * @ret rc Return status code
+ */
+static inline int xhci_nop ( struct xhci_device *xhci ) {
+ union xhci_trb trb;
+ struct xhci_trb_common *nop = &trb.common;
+ int rc;
+
+ /* Construct command */
+ memset ( nop, 0, sizeof ( *nop ) );
+ nop->flags = XHCI_TRB_IOC;
+ nop->type = XHCI_TRB_NOP_CMD;
+
+ /* Issue command and wait for completion */
+ if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
+ return rc;
+
+ return 0;
+}
+
+/**
+ * Enable slot
+ *
+ * @v xhci xHCI device
+ * @v type Slot type
+ * @ret slot Device slot ID, or negative error
+ */
+static inline int xhci_enable_slot ( struct xhci_device *xhci,
+ unsigned int type ) {
+ union xhci_trb trb;
+ struct xhci_trb_enable_slot *enable = &trb.enable;
+ struct xhci_trb_complete *enabled = &trb.complete;
+ unsigned int slot;
+ int rc;
+
+ /* Construct command */
+ memset ( enable, 0, sizeof ( *enable ) );
+ enable->slot = type;
+ enable->type = XHCI_TRB_ENABLE_SLOT;
+
+ /* Issue command and wait for completion */
+ if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+ DBGC ( xhci, "XHCI %p could not enable new slot: %s\n",
+ xhci, strerror ( rc ) );
+ return rc;
+ }
+
+ /* Extract slot number */
+ slot = enabled->slot;
+
+ DBGC2 ( xhci, "XHCI %p slot %d enabled\n", xhci, slot );
+ return slot;
+}
+
+/**
+ * Disable slot
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @ret rc Return status code
+ */
+static inline int xhci_disable_slot ( struct xhci_device *xhci,
+ unsigned int slot ) {
+ union xhci_trb trb;
+ struct xhci_trb_disable_slot *disable = &trb.disable;
+ int rc;
+
+ /* Construct command */
+ memset ( disable, 0, sizeof ( *disable ) );
+ disable->type = XHCI_TRB_DISABLE_SLOT;
+ disable->slot = slot;
+
+ /* Issue command and wait for completion */
+ if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+ DBGC ( xhci, "XHCI %p could not disable slot %d: %s\n",
+ xhci, slot, strerror ( rc ) );
+ return rc;
+ }
+
+ DBGC2 ( xhci, "XHCI %p slot %d disabled\n", xhci, slot );
+ return 0;
+}
+
+/**
+ * Issue context-based command and wait for completion
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @v type TRB type
+ * @v populate Input context populater
+ * @ret rc Return status code
+ */
+static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
+ struct xhci_endpoint *endpoint, unsigned int type,
+ void ( * populate ) ( struct xhci_device *xhci,
+ struct xhci_slot *slot,
+ struct xhci_endpoint *endpoint,
+ void *input ) ) {
+ union xhci_trb trb;
+ struct xhci_trb_context *context = &trb.context;
+ size_t len;
+ void *input;
+ int rc;
+
+ /* Allocate an input context */
+ len = xhci_input_context_offset ( xhci, XHCI_CTX_END );
+ input = malloc_dma ( len, xhci_align ( len ) );
+ if ( ! input ) {
+ rc = -ENOMEM;
+ goto err_alloc;
+ }
+ memset ( input, 0, len );
+
+ /* Populate input context */
+ populate ( xhci, slot, endpoint, input );
+
+ /* Construct command */
+ memset ( context, 0, sizeof ( *context ) );
+ context->type = type;
+ context->input = cpu_to_le64 ( virt_to_phys ( input ) );
+ context->slot = slot->id;
+
+ /* Issue command and wait for completion */
+ if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
+ goto err_command;
+
+ err_command:
+ free_dma ( input, len );
+ err_alloc:
+ return rc;
+}
+
+/**
+ * Populate address device input context
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @v input Input context
+ */
+static void xhci_address_device_input ( struct xhci_device *xhci,
+ struct xhci_slot *slot,
+ struct xhci_endpoint *endpoint,
+ void *input ) {
+ struct xhci_control_context *control_ctx;
+ struct xhci_slot_context *slot_ctx;
+ struct xhci_endpoint_context *ep_ctx;
+
+ /* Sanity checks */
+ assert ( endpoint->ctx == XHCI_CTX_EP0 );
+
+ /* Populate control context */
+ control_ctx = input;
+ control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
+ ( 1 << XHCI_CTX_EP0 ) );
+
+ /* Populate slot context */
+ slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+ slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv,
+ slot->route ) );
+ slot_ctx->port = slot->port;
+
+ /* Populate control endpoint context */
+ ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) );
+ ep_ctx->type = XHCI_EP_TYPE_CONTROL;
+ ep_ctx->burst = endpoint->ep->burst;
+ ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
+ ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
+ XHCI_EP_DCS );
+ ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
+}
+
+/**
+ * Address device
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @ret rc Return status code
+ */
+static inline int xhci_address_device ( struct xhci_device *xhci,
+ struct xhci_slot *slot ) {
+ struct usb_device *usb = slot->usb;
+ struct xhci_slot_context *slot_ctx;
+ int rc;
+
+ /* Assign device address */
+ if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
+ XHCI_TRB_ADDRESS_DEVICE,
+ xhci_address_device_input ) ) != 0 )
+ return rc;
+
+ /* Get assigned address */
+ slot_ctx = ( slot->context +
+ xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) );
+ usb->address = slot_ctx->address;
+ DBGC2 ( xhci, "XHCI %p assigned address %d to %s\n",
+ xhci, usb->address, usb->name );
+
+ return 0;
+}
+
+/**
+ * Populate configure endpoint input context
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @v input Input context
+ */
+static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
+ struct xhci_slot *slot __unused,
+ struct xhci_endpoint *endpoint,
+ void *input ) {
+ struct xhci_control_context *control_ctx;
+ struct xhci_slot_context *slot_ctx;
+ struct xhci_endpoint_context *ep_ctx;
+
+ /* Populate control context */
+ control_ctx = input;
+ control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
+ ( 1 << endpoint->ctx ) );
+
+ /* Populate slot context */
+ slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+ slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
+ 0, 0, 0 ) );
+
+ /* Populate endpoint context */
+ ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
+ ep_ctx->type = endpoint->type;
+ ep_ctx->burst = endpoint->ep->burst;
+ ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
+ ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
+ XHCI_EP_DCS );
+ ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
+}
+
+/**
+ * Configure endpoint
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @ret rc Return status code
+ */
+static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
+ struct xhci_slot *slot,
+ struct xhci_endpoint *endpoint ) {
+ int rc;
+
+ /* Configure endpoint */
+ if ( ( rc = xhci_context ( xhci, slot, endpoint,
+ XHCI_TRB_CONFIGURE_ENDPOINT,
+ xhci_configure_endpoint_input ) ) != 0 )
+ return rc;
+
+ DBGC2 ( xhci, "XHCI %p slot %d ctx %d configured\n",
+ xhci, slot->id, endpoint->ctx );
+ return 0;
+}
+
+/**
+ * Populate deconfigure endpoint input context
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @v input Input context
+ */
+static void
+xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused,
+ struct xhci_slot *slot __unused,
+ struct xhci_endpoint *endpoint,
+ void *input ) {
+ struct xhci_control_context *control_ctx;
+ struct xhci_slot_context *slot_ctx;
+
+ /* Populate control context */
+ control_ctx = input;
+ control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT );
+ control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx );
+
+ /* Populate slot context */
+ slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+ slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
+ 0, 0, 0 ) );
+}
+
+/**
+ * Deconfigure endpoint
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @ret rc Return status code
+ */
+static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
+ struct xhci_slot *slot,
+ struct xhci_endpoint *endpoint ) {
+ int rc;
+
+ /* Deconfigure endpoint */
+ if ( ( rc = xhci_context ( xhci, slot, endpoint,
+ XHCI_TRB_CONFIGURE_ENDPOINT,
+ xhci_deconfigure_endpoint_input ) ) != 0 )
+ return rc;
+
+ DBGC2 ( xhci, "XHCI %p slot %d ctx %d deconfigured\n",
+ xhci, slot->id, endpoint->ctx );
+ return 0;
+}
+
+/**
+ * Populate evaluate context input context
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @v input Input context
+ */
+static void xhci_evaluate_context_input ( struct xhci_device *xhci,
+ struct xhci_slot *slot __unused,
+ struct xhci_endpoint *endpoint,
+ void *input ) {
+ struct xhci_control_context *control_ctx;
+ struct xhci_slot_context *slot_ctx;
+ struct xhci_endpoint_context *ep_ctx;
+
+ /* Populate control context */
+ control_ctx = input;
+ control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
+ ( 1 << endpoint->ctx ) );
+
+ /* Populate slot context */
+ slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+ slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
+ 0, 0, 0 ) );
+
+ /* Populate endpoint context */
+ ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
+ ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
+}
+
+/**
+ * Evaluate context
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @ret rc Return status code
+ */
+static inline int xhci_evaluate_context ( struct xhci_device *xhci,
+ struct xhci_slot *slot,
+ struct xhci_endpoint *endpoint ) {
+ int rc;
+
+ /* Configure endpoint */
+ if ( ( rc = xhci_context ( xhci, slot, endpoint,
+ XHCI_TRB_EVALUATE_CONTEXT,
+ xhci_evaluate_context_input ) ) != 0 )
+ return rc;
+
+ DBGC2 ( xhci, "XHCI %p slot %d ctx %d (re-)evaluated\n",
+ xhci, slot->id, endpoint->ctx );
+ return 0;
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @ret rc Return status code
+ */
+static inline int xhci_reset_endpoint ( struct xhci_device *xhci,
+ struct xhci_slot *slot,
+ struct xhci_endpoint *endpoint ) {
+ union xhci_trb trb;
+ struct xhci_trb_reset_endpoint *reset = &trb.reset;
+ int rc;
+
+ /* Construct command */
+ memset ( reset, 0, sizeof ( *reset ) );
+ reset->slot = slot->id;
+ reset->endpoint = endpoint->ctx;
+ reset->type = XHCI_TRB_RESET_ENDPOINT;
+
+ /* Issue command and wait for completion */
+ if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+ DBGC ( xhci, "XHCI %p slot %d ctx %d could not reset endpoint "
+ "in state %d: %s\n", xhci, slot->id, endpoint->ctx,
+ endpoint->context->state, strerror ( rc ) );
+ return rc;
+ }
+
+ return 0;
+}
+
+/**
+ * Stop endpoint
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @ret rc Return status code
+ */
+static inline int xhci_stop_endpoint ( struct xhci_device *xhci,
+ struct xhci_slot *slot,
+ struct xhci_endpoint *endpoint ) {
+ union xhci_trb trb;
+ struct xhci_trb_stop_endpoint *stop = &trb.stop;
+ int rc;
+
+ /* Construct command */
+ memset ( stop, 0, sizeof ( *stop ) );
+ stop->slot = slot->id;
+ stop->endpoint = endpoint->ctx;
+ stop->type = XHCI_TRB_STOP_ENDPOINT;
+
+ /* Issue command and wait for completion */
+ if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+ DBGC ( xhci, "XHCI %p slot %d ctx %d could not stop endpoint "
+ "in state %d: %s\n", xhci, slot->id, endpoint->ctx,
+ endpoint->context->state, strerror ( rc ) );
+ return rc;
+ }
+
+ return 0;
+}
+
+/**
+ * Set transfer ring dequeue pointer
+ *
+ * @v xhci xHCI device
+ * @v slot Device slot
+ * @v endpoint Endpoint
+ * @ret rc Return status code
+ */
+static inline int
+xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
+ struct xhci_slot *slot,
+ struct xhci_endpoint *endpoint ) {
+ union xhci_trb trb;
+ struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue;
+ struct xhci_trb_ring *ring = &endpoint->ring;
+ unsigned int cons;
+ unsigned int mask;
+ unsigned int index;
+ unsigned int dcs;
+ int rc;
+
+ /* Construct command */
+ memset ( dequeue, 0, sizeof ( *dequeue ) );
+ cons = ring->cons;
+ mask = ring->mask;
+ dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
+ index = ( cons & mask );
+ dequeue->dequeue =
+ cpu_to_le64 ( virt_to_phys ( &ring->trb[index] ) | dcs );
+ dequeue->slot = slot->id;
+ dequeue->endpoint = endpoint->ctx;
+ dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER;
+
+ /* Issue command and wait for completion */
+ if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+ DBGC ( xhci, "XHCI %p slot %d ctx %d could not set TR dequeue "
+ "pointer in state %d: %s\n", xhci, slot->id,
+ endpoint->ctx, endpoint->context->state, strerror ( rc));
+ return rc;
+ }
+
+ return 0;
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open endpoint
+ *
+ * @v ep USB endpoint
+ * @ret rc Return status code
+ */
+static int xhci_endpoint_open ( struct usb_endpoint *ep ) {
+ struct usb_device *usb = ep->usb;
+ struct xhci_slot *slot = usb_get_hostdata ( usb );
+ struct xhci_device *xhci = slot->xhci;
+ struct xhci_endpoint *endpoint;
+ unsigned int ctx;
+ unsigned int type;
+ int rc;
+
+ /* Calculate context index */
+ ctx = XHCI_CTX ( ep->address );
+ assert ( slot->endpoint[ctx] == NULL );
+
+ /* Calculate endpoint type */
+ type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+ if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) )
+ type = XHCI_EP_TYPE_CONTROL;
+ if ( ep->address & USB_DIR_IN )
+ type |= XHCI_EP_TYPE_IN;
+
+ /* Allocate and initialise structure */
+ endpoint = zalloc ( sizeof ( *endpoint ) );
+ if ( ! endpoint ) {
+ rc = -ENOMEM;
+ goto err_alloc;
+ }
+ usb_endpoint_set_hostdata ( ep, endpoint );
+ slot->endpoint[ctx] = endpoint;
+ endpoint->xhci = xhci;
+ endpoint->slot = slot;
+ endpoint->ep = ep;
+ endpoint->ctx = ctx;
+ endpoint->type = type;
+ endpoint->context = ( ( ( void * ) slot->context ) +
+ xhci_device_context_offset ( xhci, ctx ) );
+
+ /* Allocate transfer ring */
+ if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring,
+ XHCI_TRANSFER_TRBS_LOG2,
+ slot->id, ctx, 0 ) ) != 0 )
+ goto err_ring_alloc;
+
+ /* Configure endpoint, if applicable */
+ if ( ( ctx != XHCI_CTX_EP0 ) &&
+ ( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 ))
+ goto err_configure_endpoint;
+
+ DBGC2 ( xhci, "XHCI %p slot %d ctx %d ring [%08lx,%08lx)\n",
+ xhci, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ),
+ ( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) );
+ return 0;
+
+ xhci_deconfigure_endpoint ( xhci, slot, endpoint );
+ err_configure_endpoint:
+ xhci_ring_free ( &endpoint->ring );
+ err_ring_alloc:
+ slot->endpoint[ctx] = NULL;
+ free ( endpoint );
+ err_alloc:
+ return rc;
+}
+
+/**
+ * Close endpoint
+ *
+ * @v ep USB endpoint
+ */
+static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
+ struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+ struct xhci_slot *slot = endpoint->slot;
+ struct xhci_device *xhci = slot->xhci;
+ struct io_buffer *iobuf;
+ unsigned int ctx = endpoint->ctx;
+
+ /* Deconfigure endpoint, if applicable */
+ if ( ctx != XHCI_CTX_EP0 )
+ xhci_deconfigure_endpoint ( xhci, slot, endpoint );
+
+ /* Cancel any incomplete transfers */
+ while ( xhci_ring_fill ( &endpoint->ring ) ) {
+ iobuf = xhci_dequeue_multi ( &endpoint->ring );
+ usb_complete_err ( ep, iobuf, -ECANCELED );
+ }
+
+ /* Free endpoint */
+ xhci_ring_free ( &endpoint->ring );
+ slot->endpoint[ctx] = NULL;
+ free ( endpoint );
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v ep USB endpoint
+ * @ret rc Return status code
+ */
+static int xhci_endpoint_reset ( struct usb_endpoint *ep ) {
+ struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+ struct xhci_slot *slot = endpoint->slot;
+ struct xhci_device *xhci = slot->xhci;
+ int rc;
+
+ /* Reset endpoint context */
+ if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 )
+ return rc;
+
+ /* Set transfer ring dequeue pointer */
+ if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0)
+ return rc;
+
+ DBGC ( xhci, "XHCI %p slot %d ctx %d reset\n",
+ xhci, slot->id, endpoint->ctx );
+ return 0;
+}
+
+/**
+ * Update MTU
+ *
+ * @v ep USB endpoint
+ * @ret rc Return status code
+ */
+static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
+ struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+ struct xhci_slot *slot = endpoint->slot;
+ struct xhci_device *xhci = slot->xhci;
+ int rc;
+
+ /* Evalulate context */
+ if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 )
+ return rc;
+
+ return 0;
+}
+
+/**
+ * Enqueue message transfer
+ *
+ * @v ep USB endpoint
+ * @v packet Setup packet
+ * @v iobuf I/O buffer
+ * @ret rc Return status code
+ */
+static int xhci_endpoint_message ( struct usb_endpoint *ep,
+ struct usb_setup_packet *packet,
+ struct io_buffer *iobuf ) {
+ struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+ unsigned int input = ( le16_to_cpu ( packet->request ) & USB_DIR_IN );
+ size_t len = iob_len ( iobuf );
+ union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ +
+ 1 /* status */ ];
+ union xhci_trb *trb = trbs;
+ struct xhci_trb_setup *setup;
+ struct xhci_trb_data *data;
+ struct xhci_trb_status *status;
+ int rc;
+
+ /* Profile message transfers */
+ profile_start ( &xhci_message_profiler );
+
+ /* Construct setup stage TRB */
+ memset ( trbs, 0, sizeof ( trbs ) );
+ setup = &(trb++)->setup;
+ memcpy ( &setup->packet, packet, sizeof ( setup->packet ) );
+ setup->len = cpu_to_le32 ( sizeof ( *packet ) );
+ setup->flags = XHCI_TRB_IDT;
+ setup->type = XHCI_TRB_SETUP;
+ if ( len )
+ setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
+
+ /* Construct data stage TRB, if applicable */
+ if ( len ) {
+ data = &(trb++)->data;
+ data->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
+ data->len = cpu_to_le32 ( len );
+ data->type = XHCI_TRB_DATA;
+ data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
+ }
+
+ /* Construct status stage TRB */
+ status = &(trb++)->status;
+ status->flags = XHCI_TRB_IOC;
+ status->type = XHCI_TRB_STATUS;
+ status->direction =
+ ( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN );
+
+ /* Enqueue TRBs */
+ if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
+ ( trb - trbs ) ) ) != 0 )
+ return rc;
+
+ /* Ring the doorbell */
+ xhci_doorbell ( &endpoint->ring );
+
+ profile_stop ( &xhci_message_profiler );
+ return 0;
+}
+
+/**
+ * Enqueue stream transfer
+ *
+ * @v ep USB endpoint
+ * @v iobuf I/O buffer
+ * @ret rc Return status code
+ */
+static int xhci_endpoint_stream ( struct usb_endpoint *ep,
+ struct io_buffer *iobuf ) {
+ struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+ union xhci_trb trb;
+ struct xhci_trb_normal *normal;
+ int rc;
+
+ /* Profile stream transfers */
+ profile_start ( &xhci_stream_profiler );
+
+ /* Construct normal TRBs */
+ normal = &trb.normal;
+ normal->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
+ normal->len = cpu_to_le32 ( iob_len ( iobuf ) );
+ normal->flags = XHCI_TRB_IOC;
+ normal->type = XHCI_TRB_NORMAL;
+
+ /* Enqueue TRBs */
+ if ( ( rc = xhci_enqueue ( &endpoint->ring, iobuf, &trb ) ) != 0 )
+ return rc;
+
+ /* Ring the doorbell */
+ xhci_doorbell ( &endpoint->ring );
+
+ profile_stop ( &xhci_stream_profiler );
+ return 0;
+}
+
+/******************************************************************************
+ *
+ * Device operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open device
+ *
+ * @v usb USB device
+ * @ret rc Return status code
+ */
+static int xhci_device_open ( struct usb_device *usb ) {
+ struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus );
+ struct xhci_slot *slot;
+ size_t len;
+ int type;
+ int id;
+ int rc;
+
+ /* Determine applicable slot type */
+ type = xhci_port_slot_type ( xhci, usb->port->address );
+ if ( type < 0 ) {
+ rc = type;
+ DBGC ( xhci, "XHCI %p port %d has no slot type\n",
+ xhci, usb->port->address );
+ goto err_type;
+ }
+
+ /* Allocate a device slot number */
+ id = xhci_enable_slot ( xhci, type );
+ if ( id < 0 ) {
+ rc = id;
+ goto err_enable_slot;
+ }
+ assert ( xhci->slot[id] == NULL );
+
+ /* Allocate and initialise structure */
+ slot = zalloc ( sizeof ( *slot ) );
+ if ( ! slot ) {
+ rc = -ENOMEM;
+ goto err_alloc;
+ }
+ usb_set_hostdata ( usb, slot );
+ xhci->slot[id] = slot;
+ slot->xhci = xhci;
+ slot->usb = usb;
+ slot->id = id;
+
+ /* Allocate a device context */
+ len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
+ slot->context = malloc_dma ( len, xhci_align ( len ) );
+ if ( ! slot->context ) {
+ rc = -ENOMEM;
+ goto err_alloc_context;
+ }
+ memset ( slot->context, 0, len );
+
+ /* Set device context base address */
+ assert ( xhci->dcbaa[id] == 0 );
+ xhci->dcbaa[id] = cpu_to_le64 ( virt_to_phys ( slot->context ) );
+
+ DBGC2 ( xhci, "XHCI %p slot %d device context [%08lx,%08lx) for %s\n",
+ xhci, slot->id, virt_to_phys ( slot->context ),
+ ( virt_to_phys ( slot->context ) + len ), usb->name );
+ return 0;
+
+ xhci->dcbaa[id] = 0;
+ free_dma ( slot->context, len );
+ err_alloc_context:
+ xhci->slot[id] = NULL;
+ free ( slot );
+ err_alloc:
+ xhci_disable_slot ( xhci, id );
+ err_enable_slot:
+ err_type:
+ return rc;
+}
+
+/**
+ * Close device
+ *
+ * @v usb USB device
+ */
+static void xhci_device_close ( struct usb_device *usb ) {
+ struct xhci_slot *slot = usb_get_hostdata ( usb );
+ struct xhci_device *xhci = slot->xhci;
+ size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
+ unsigned int id = slot->id;
+
+ /* Disable slot */
+ xhci_disable_slot ( xhci, id );
+
+ /* Free slot */
+ free_dma ( slot->context, len );
+ xhci->dcbaa[id] = 0;
+ xhci->slot[id] = NULL;
+ free ( slot );
+}
+
+/**
+ * Assign device address
+ *
+ * @v usb USB device
+ * @ret rc Return status code
+ */
+static int xhci_device_address ( struct usb_device *usb ) {
+ struct xhci_slot *slot = usb_get_hostdata ( usb );
+ struct xhci_device *xhci = slot->xhci;
+ struct usb_port *port = usb->port;
+ struct usb_port *root_port;
+ int psiv;
+ int rc;
+
+ /* Calculate route string */
+ slot->route = usb_route_string ( usb );
+
+ /* Calculate root hub port number */
+ root_port = usb_root_hub_port ( usb );
+ slot->port = root_port->address;
+
+ /* Calculate protocol speed ID */
+ psiv = xhci_port_psiv ( xhci, slot->port, port->speed );
+ if ( psiv < 0 ) {
+ rc = psiv;
+ return rc;
+ }
+ slot->psiv = psiv;
+
+ /* Address device */
+ if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 )
+ return rc;
+
+ return 0;
+}
+
+/******************************************************************************
+ *
+ * Bus operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open USB bus
+ *
+ * @v bus USB bus
+ * @ret rc Return status code
+ */
+static int xhci_bus_open ( struct usb_bus *bus ) {
+ struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+ int rc;
+
+ /* Allocate device slot array */
+ xhci->slot = zalloc ( xhci->slots * sizeof ( xhci->slot[0] ) );
+ if ( ! xhci->slot ) {
+ rc = -ENOMEM;
+ goto err_slot_alloc;
+ }
+
+ /* Allocate device context base address array */
+ if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 )
+ goto err_dcbaa_alloc;
+
+ /* Allocate scratchpad buffers */
+ if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 )
+ goto err_scratchpad_alloc;
+
+ /* Allocate command ring */
+ if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 )
+ goto err_command_alloc;
+
+ /* Allocate event ring */
+ if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 )
+ goto err_event_alloc;
+
+ /* Start controller */
+ xhci_run ( xhci );
+
+ return 0;
+
+ xhci_stop ( xhci );
+ xhci_event_free ( xhci );
+ err_event_alloc:
+ xhci_command_free ( xhci );
+ err_command_alloc:
+ xhci_scratchpad_free ( xhci );
+ err_scratchpad_alloc:
+ xhci_dcbaa_free ( xhci );
+ err_dcbaa_alloc:
+ free ( xhci->slot );
+ err_slot_alloc:
+ return rc;
+}
+
+/**
+ * Close USB bus
+ *
+ * @v bus USB bus
+ */
+static void xhci_bus_close ( struct usb_bus *bus ) {
+ struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+ unsigned int i;
+
+ /* Sanity checks */
+ assert ( xhci->slot != NULL );
+ for ( i = 0 ; i < xhci->slots ; i++ )
+ assert ( xhci->slot[i] == NULL );
+
+ xhci_stop ( xhci );
+ xhci_event_free ( xhci );
+ xhci_command_free ( xhci );
+ xhci_scratchpad_free ( xhci );
+ xhci_dcbaa_free ( xhci );
+ free ( xhci->slot );
+}
+
+/**
+ * Poll USB bus
+ *
+ * @v bus USB bus
+ */
+static void xhci_bus_poll ( struct usb_bus *bus ) {
+ struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+
+ /* Poll event ring */
+ xhci_event_poll ( xhci );
+}
+
+/******************************************************************************
+ *
+ * Root hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open root hub
+ *
+ * @v hub USB hub
+ * @ret rc Return status code
+ */
+static int xhci_hub_open ( struct usb_hub *hub ) {
+ struct usb_bus *bus = hub->bus;
+ struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+ struct usb_port *port;
+ uint32_t portsc;
+ unsigned int i;
+
+ /* Enable power to all ports */
+ for ( i = 1 ; i <= xhci->ports ; i++ ) {
+ portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
+ portsc &= XHCI_PORTSC_PRESERVE;
+ portsc |= XHCI_PORTSC_PP;
+ writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
+ }
+
+ /* xHCI spec requires us to potentially wait 20ms after
+ * enabling power to a port.
+ */
+ mdelay ( XHCI_PORT_POWER_DELAY_MS );
+
+ /* USB3 ports may power up as Disabled */
+ for ( i = 1 ; i <= xhci->ports ; i++ ) {
+ portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
+ port = usb_port ( hub, i );
+ if ( ( port->protocol >= USB_PROTO_3_0 ) &&
+ ( ( portsc & XHCI_PORTSC_PLS_MASK ) ==
+ XHCI_PORTSC_PLS_DISABLED ) ) {
+ /* Force link state to RxDetect */
+ portsc &= XHCI_PORTSC_PRESERVE;
+ portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS);
+ writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
+ }
+ }
+
+ /* Record hub driver private data */
+ usb_hub_set_drvdata ( hub, xhci );
+
+ return 0;
+}
+
+/**
+ * Close root hub
+ *
+ * @v hub USB hub
+ */
+static void xhci_hub_close ( struct usb_hub *hub ) {
+
+ /* Clear hub driver private data */
+ usb_hub_set_drvdata ( hub, NULL );
+}
+
+/**
+ * Enable port
+ *
+ * @v hub USB hub
+ * @v port USB port
+ * @ret rc Return status code
+ */
+static int xhci_hub_enable ( struct usb_hub *hub, struct usb_port *port ) {
+ struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+ uint32_t portsc;
+ unsigned int i;
+
+ /* Reset port if applicable */
+ if ( port->protocol < USB_PROTO_3_0 ) {
+ portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+ portsc &= XHCI_PORTSC_PRESERVE;
+ portsc |= XHCI_PORTSC_PR;
+ writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
+ }
+
+ /* Wait for port to become enabled */
+ for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
+
+ /* Check port status */
+ portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+ if ( portsc & XHCI_PORTSC_PED )
+ return 0;
+
+ /* Delay */
+ mdelay ( 1 );
+ }
+
+ DBGC ( xhci, "XHCI %p timed out waiting for port %d to enable\n",
+ xhci, port->address );
+ return -ETIMEDOUT;
+}
+
+/**
+ * Disable port
+ *
+ * @v hub USB hub
+ * @v port USB port
+ * @ret rc Return status code
+ */
+static int xhci_hub_disable ( struct usb_hub *hub, struct usb_port *port ) {
+ struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+ uint32_t portsc;
+
+ /* Disable port */
+ portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+ portsc &= XHCI_PORTSC_PRESERVE;
+ portsc |= XHCI_PORTSC_PED;
+ writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
+
+ return 0;
+}
+
+/**
+ * Update root hub port speed
+ *
+ * @v hub USB hub
+ * @v port USB port
+ * @ret rc Return status code
+ */
+static int xhci_hub_speed ( struct usb_hub *hub, struct usb_port *port ) {
+ struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+ uint32_t portsc;
+ unsigned int psiv;
+ int ccs;
+ int ped;
+ int speed;
+ int rc;
+
+ /* Read port status */
+ portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+ DBGC2 ( xhci, "XHCI %p port %d status is %08x\n",
+ xhci, port->address, portsc );
+
+ /* Check whether or not port is connected */
+ ccs = ( portsc & XHCI_PORTSC_CCS );
+ if ( ! ccs ) {
+ port->speed = USB_SPEED_NONE;
+ return 0;
+ }
+
+ /* For USB2 ports, the PSIV field is not valid until the port
+ * completes reset and becomes enabled.
+ */
+ ped = ( portsc & XHCI_PORTSC_PED );
+ if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) {
+ port->speed = USB_SPEED_FULL;
+ return 0;
+ }
+
+ /* Get port speed and map to generic USB speed */
+ psiv = XHCI_PORTSC_PSIV ( portsc );
+ speed = xhci_port_speed ( xhci, port->address, psiv );
+ if ( speed < 0 ) {
+ rc = speed;
+ return rc;
+ }
+
+ port->speed = speed;
+ return 0;
+}
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/** USB host controller operations */
+static struct usb_host_operations xhci_operations = {
+ .endpoint = {
+ .open = xhci_endpoint_open,
+ .close = xhci_endpoint_close,
+ .reset = xhci_endpoint_reset,
+ .mtu = xhci_endpoint_mtu,
+ .message = xhci_endpoint_message,
+ .stream = xhci_endpoint_stream,
+ },
+ .device = {
+ .open = xhci_device_open,
+ .close = xhci_device_close,
+ .address = xhci_device_address,
+ },
+ .bus = {
+ .open = xhci_bus_open,
+ .close = xhci_bus_close,
+ .poll = xhci_bus_poll,
+ },
+ .hub = {
+ .open = xhci_hub_open,
+ .close = xhci_hub_close,
+ .enable = xhci_hub_enable,
+ .disable = xhci_hub_disable,
+ .speed = xhci_hub_speed,
+ },
+};
+
+/**
+ * Probe PCI device
+ *
+ * @v pci PCI device
+ * @ret rc Return status code
+ */
+static int xhci_probe ( struct pci_device *pci ) {
+ struct xhci_device *xhci;
+ struct usb_port *port;
+ unsigned long bar_start;
+ size_t bar_size;
+ unsigned int i;
+ int rc;
+
+ /* Allocate and initialise structure */
+ xhci = zalloc ( sizeof ( *xhci ) );
+ if ( ! xhci ) {
+ rc = -ENOMEM;
+ goto err_alloc;
+ }
+
+ /* Fix up PCI device */
+ adjust_pci_device ( pci );
+
+ /* Map registers */
+ bar_start = pci_bar_start ( pci, XHCI_BAR );
+ bar_size = pci_bar_size ( pci, XHCI_BAR );
+ xhci->regs = ioremap ( bar_start, bar_size );
+ if ( ! xhci->regs ) {
+ rc = -ENODEV;
+ goto err_ioremap;
+ }
+
+ /* Initialise xHCI device */
+ xhci_init ( xhci, xhci->regs );
+
+ /* Initialise USB legacy support and claim ownership */
+ xhci_legacy_init ( xhci );
+ if ( ( rc = xhci_legacy_claim ( xhci ) ) != 0 )
+ goto err_legacy_claim;
+
+ /* Reset device */
+ if ( ( rc = xhci_reset ( xhci ) ) != 0 )
+ goto err_reset;
+
+ /* Allocate USB bus */
+ xhci->bus = alloc_usb_bus ( &pci->dev, xhci->ports,
+ &xhci_operations );
+ if ( ! xhci->bus ) {
+ rc = -ENOMEM;
+ goto err_alloc_bus;
+ }
+ usb_bus_set_hostdata ( xhci->bus, xhci );
+ usb_hub_set_drvdata ( xhci->bus->hub, xhci );
+
+ /* Set port protocols */
+ for ( i = 1 ; i <= xhci->ports ; i++ ) {
+ port = usb_port ( xhci->bus->hub, i );
+ port->protocol = xhci_port_protocol ( xhci, i );
+ }
+
+ /* Register USB bus */
+ if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 )
+ goto err_register;
+
+ pci_set_drvdata ( pci, xhci );
+ return 0;
+
+ unregister_usb_bus ( xhci->bus );
+ err_register:
+ free_usb_bus ( xhci->bus );
+ err_alloc_bus:
+ xhci_reset ( xhci );
+ err_reset:
+ xhci_legacy_release ( xhci );
+ err_legacy_claim:
+ iounmap ( xhci->regs );
+ err_ioremap:
+ free ( xhci );
+ err_alloc:
+ return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci PCI device
+ */
+static void xhci_remove ( struct pci_device *pci ) {
+ struct xhci_device *xhci = pci_get_drvdata ( pci );
+ struct usb_bus *bus = xhci->bus;
+
+ unregister_usb_bus ( bus );
+ free_usb_bus ( bus );
+ xhci_reset ( xhci );
+ xhci_legacy_release ( xhci );
+ iounmap ( xhci->regs );
+ free ( xhci );
+}
+
+/** XHCI PCI device IDs */
+static struct pci_device_id xhci_ids[] = {
+ PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ),
+};
+
+/** XHCI PCI driver */
+struct pci_driver xhci_driver __pci_driver = {
+ .ids = xhci_ids,
+ .id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ),
+ .class = PCI_CLASS ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
+ PCI_CLASS_SERIAL_USB_XHCI ),
+ .probe = xhci_probe,
+ .remove = xhci_remove,
+};
--- /dev/null
+#ifndef _IPXE_XHCI_H
+#define _IPXE_XHCI_H
+
+/** @file
+ *
+ * USB eXtensible Host Controller Interface (xHCI) driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include <assert.h>
+#include <ipxe/pci.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/usb.h>
+
+/** Minimum alignment required for data structures
+ *
+ * With the exception of the scratchpad buffer pages (which are
+ * page-aligned), data structures used by xHCI generally require from
+ * 16 to 64 byte alignment and must not cross an (xHCI) page boundary.
+ * We simplify this requirement by aligning each structure on its own
+ * size, with a minimum of a 64 byte alignment.
+ */
+#define XHCI_MIN_ALIGN 64
+
+/** xHCI PCI BAR */
+#define XHCI_BAR PCI_BASE_ADDRESS_0
+
+/** Capability register length */
+#define XHCI_CAP_CAPLENGTH 0x00
+
+/** Host controller interface version number */
+#define XHCI_CAP_HCIVERSION 0x02
+
+/** Structural parameters 1 */
+#define XHCI_CAP_HCSPARAMS1 0x04
+
+/** Number of device slots */
+#define XHCI_HCSPARAMS1_SLOTS(params) ( ( (params) >> 0 ) & 0xff )
+
+/** Number of interrupters */
+#define XHCI_HCSPARAMS1_INTRS(params) ( ( (params) >> 8 ) & 0x3ff )
+
+/** Number of ports */
+#define XHCI_HCSPARAMS1_PORTS(params) ( ( (params) >> 24 ) & 0xff )
+
+/** Structural parameters 2 */
+#define XHCI_CAP_HCSPARAMS2 0x08
+
+/** Number of page-sized scratchpad buffers */
+#define XHCI_HCSPARAMS2_SCRATCHPADS(params) \
+ ( ( ( (params) >> 16 ) & 0x3e0 ) | ( ( (params) >> 27 ) & 0x1f ) )
+
+/** Capability parameters */
+#define XHCI_CAP_HCCPARAMS1 0x10
+
+/** 64-bit addressing capability */
+#define XHCI_HCCPARAMS1_ADDR64(params) ( ( (params) >> 0 ) & 0x1 )
+
+/** Context size shift */
+#define XHCI_HCCPARAMS1_CSZ_SHIFT(params) ( 5 + ( ( (params) >> 2 ) & 0x1 ) )
+
+/** xHCI extended capabilities pointer */
+#define XHCI_HCCPARAMS1_XECP(params) ( ( ( (params) >> 16 ) & 0xffff ) << 2 )
+
+/** Doorbell offset */
+#define XHCI_CAP_DBOFF 0x14
+
+/** Runtime register space offset */
+#define XHCI_CAP_RTSOFF 0x18
+
+/** xHCI extended capability ID */
+#define XHCI_XECP_ID(xecp) ( ( (xecp) >> 0 ) & 0xff )
+
+/** Next xHCI extended capability pointer */
+#define XHCI_XECP_NEXT(xecp) ( ( ( (xecp) >> 8 ) & 0xff ) << 2 )
+
+/** USB legacy support extended capability */
+#define XHCI_XECP_ID_LEGACY 1
+
+/** USB legacy support BIOS owned semaphore */
+#define XHCI_USBLEGSUP_BIOS 0x02
+
+/** USB legacy support BIOS ownership flag */
+#define XHCI_USBLEGSUP_BIOS_OWNED 0x01
+
+/** USB legacy support OS owned semaphore */
+#define XHCI_USBLEGSUP_OS 0x03
+
+/** USB legacy support OS ownership flag */
+#define XHCI_USBLEGSUP_OS_OWNED 0x01
+
+/** USB legacy support control/status */
+#define XHCI_USBLEGSUP_CTLSTS 0x04
+
+/** Supported protocol extended capability */
+#define XHCI_XECP_ID_SUPPORTED 2
+
+/** Supported protocol revision */
+#define XHCI_SUPPORTED_REVISION 0x00
+
+/** Supported protocol minor revision */
+#define XHCI_SUPPORTED_REVISION_VER(revision) ( ( (revision) >> 16 ) & 0xffff )
+
+/** Supported protocol name */
+#define XHCI_SUPPORTED_NAME 0x04
+
+/** Supported protocol ports */
+#define XHCI_SUPPORTED_PORTS 0x08
+
+/** Supported protocol port offset */
+#define XHCI_SUPPORTED_PORTS_OFFSET(ports) ( ( (ports) >> 0 ) & 0xff )
+
+/** Supported protocol port count */
+#define XHCI_SUPPORTED_PORTS_COUNT(ports) ( ( (ports) >> 8 ) & 0xff )
+
+/** Supported protocol PSI count */
+#define XHCI_SUPPORTED_PORTS_PSIC(ports) ( ( (ports) >> 28 ) & 0x0f )
+
+/** Supported protocol slot */
+#define XHCI_SUPPORTED_SLOT 0x0c
+
+/** Supported protocol slot type */
+#define XHCI_SUPPORTED_SLOT_TYPE(slot) ( ( (slot) >> 0 ) & 0x1f )
+
+/** Supported protocol PSI */
+#define XHCI_SUPPORTED_PSI(index) ( 0x10 + ( (index) * 4 ) )
+
+/** Supported protocol PSI value */
+#define XHCI_SUPPORTED_PSI_VALUE(psi) ( ( (psi) >> 0 ) & 0x0f )
+
+/** Supported protocol PSI mantissa */
+#define XHCI_SUPPORTED_PSI_MANTISSA(psi) ( ( (psi) >> 16 ) & 0xffff )
+
+/** Supported protocol PSI exponent */
+#define XHCI_SUPPORTED_PSI_EXPONENT(psi) ( ( (psi) >> 4 ) & 0x03 )
+
+/** Default PSI values */
+enum xhci_default_psi_value {
+ /** Full speed (12Mbps) */
+ XHCI_SPEED_FULL = 1,
+ /** Low speed (1.5Mbps) */
+ XHCI_SPEED_LOW = 2,
+ /** High speed (480Mbps) */
+ XHCI_SPEED_HIGH = 3,
+ /** Super speed */
+ XHCI_SPEED_SUPER = 4,
+};
+
+/** USB command register */
+#define XHCI_OP_USBCMD 0x00
+
+/** Run/stop */
+#define XHCI_USBCMD_RUN 0x00000001UL
+
+/** Host controller reset */
+#define XHCI_USBCMD_HCRST 0x00000002UL
+
+/** USB status register */
+#define XHCI_OP_USBSTS 0x04
+
+/** Host controller halted */
+#define XHCI_USBSTS_HCH 0x00000001UL
+
+/** Page size register */
+#define XHCI_OP_PAGESIZE 0x08
+
+/** Page size */
+#define XHCI_PAGESIZE(pagesize) ( (pagesize) << 12 )
+
+/** Device notifcation control register */
+#define XHCI_OP_DNCTRL 0x14
+
+/** Command ring control register */
+#define XHCI_OP_CRCR 0x18
+
+/** Command ring cycle state */
+#define XHCI_CRCR_RCS 0x00000001UL
+
+/** Command ring running */
+#define XHCI_CRCR_CRR 0x00000008UL
+
+/** Device context base address array pointer */
+#define XHCI_OP_DCBAAP 0x30
+
+/** Configure register */
+#define XHCI_OP_CONFIG 0x38
+
+/** Maximum device slots enabled */
+#define XHCI_CONFIG_MAX_SLOTS_EN(slots) ( (slots) << 0 )
+
+/** Maximum device slots enabled mask */
+#define XHCI_CONFIG_MAX_SLOTS_EN_MASK \
+ XHCI_CONFIG_MAX_SLOTS_EN ( 0xff )
+
+/** Port status and control register */
+#define XHCI_OP_PORTSC(port) ( 0x400 - 0x10 + ( (port) << 4 ) )
+
+/** Current connect status */
+#define XHCI_PORTSC_CCS 0x00000001UL
+
+/** Port enabled */
+#define XHCI_PORTSC_PED 0x00000002UL
+
+/** Port reset */
+#define XHCI_PORTSC_PR 0x00000010UL
+
+/** Port link state */
+#define XHCI_PORTSC_PLS(pls) ( (pls) << 5 )
+
+/** Disabled port link state */
+#define XHCI_PORTSC_PLS_DISABLED XHCI_PORTSC_PLS ( 4 )
+
+/** RxDetect port link state */
+#define XHCI_PORTSC_PLS_RXDETECT XHCI_PORTSC_PLS ( 5 )
+
+/** Port link state mask */
+#define XHCI_PORTSC_PLS_MASK XHCI_PORTSC_PLS ( 0xf )
+
+/** Port power */
+#define XHCI_PORTSC_PP 0x00000200UL
+
+/** Time to delay after enabling power to a port */
+#define XHCI_PORT_POWER_DELAY_MS 20
+
+/** Port speed ID value */
+#define XHCI_PORTSC_PSIV(portsc) ( ( (portsc) >> 10 ) & 0xf )
+
+/** Port indicator control */
+#define XHCI_PORTSC_PIC(indicators) ( (indicators) << 14 )
+
+/** Port indicator control mask */
+#define XHCI_PORTSC_PIC_MASK XHCI_PORTSC_PIC ( 3 )
+
+/** Port link state write strobe */
+#define XHCI_PORTSC_LWS 0x00010000UL
+
+/** Connect status change */
+#define XHCI_PORTSC_CSC 0x00020000UL
+
+/** Port enabled/disabled change */
+#define XHCI_PORTSC_PEC 0x00040000UL
+
+/** Warm port reset change */
+#define XHCI_PORTSC_WRC 0x00080000UL
+
+/** Over-current change */
+#define XHCI_PORTSC_OCC 0x00100000UL
+
+/** Port reset change */
+#define XHCI_PORTSC_PRC 0x00200000UL
+
+/** Port link state change */
+#define XHCI_PORTSC_PLC 0x00400000UL
+
+/** Port config error change */
+#define XHCI_PORTSC_CEC 0x00800000UL
+
+/** Port status change mask */
+#define XHCI_PORTSC_CHANGE \
+ ( XHCI_PORTSC_CSC | XHCI_PORTSC_PEC | XHCI_PORTSC_WRC | \
+ XHCI_PORTSC_OCC | XHCI_PORTSC_PRC | XHCI_PORTSC_PLC | \
+ XHCI_PORTSC_CEC )
+
+/** Port status and control bits which should be preserved
+ *
+ * The port status and control register is a horrendous mix of
+ * differing semantics. Some bits are written to only when a separate
+ * write strobe bit is set. Some bits should be preserved when
+ * modifying other bits. Some bits will be cleared if written back as
+ * a one. Most excitingly, the "port enabled" bit has the semantics
+ * that 1=enabled, 0=disabled, yet writing a 1 will disable the port.
+ */
+#define XHCI_PORTSC_PRESERVE ( XHCI_PORTSC_PP | XHCI_PORTSC_PIC_MASK )
+
+/** Port power management status and control register */
+#define XHCI_OP_PORTPMSC(port) ( 0x404 - 0x10 + ( (port) << 4 ) )
+
+/** Port link info register */
+#define XHCI_OP_PORTLI(port) ( 0x408 - 0x10 + ( (port) << 4 ) )
+
+/** Port hardware link power management control register */
+#define XHCI_OP_PORTHLPMC(port) ( 0x40c - 0x10 + ( (port) << 4 ) )
+
+/** Event ring segment table size register */
+#define XHCI_RUN_ERSTSZ(intr) ( 0x28 + ( (intr) << 5 ) )
+
+/** Event ring segment table base address register */
+#define XHCI_RUN_ERSTBA(intr) ( 0x30 + ( (intr) << 5 ) )
+
+/** Event ring dequeue pointer register */
+#define XHCI_RUN_ERDP(intr) ( 0x38 + ( (intr) << 5 ) )
+
+/** A transfer request block template */
+struct xhci_trb_template {
+ /** Parameter */
+ uint64_t parameter;
+ /** Status */
+ uint32_t status;
+ /** Control */
+ uint32_t control;
+};
+
+/** A transfer request block */
+struct xhci_trb_common {
+ /** Reserved */
+ uint64_t reserved_a;
+ /** Reserved */
+ uint32_t reserved_b;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Reserved */
+ uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** Transfer request block cycle bit flag */
+#define XHCI_TRB_C 0x01
+
+/** Transfer request block toggle cycle bit flag */
+#define XHCI_TRB_TC 0x02
+
+/** Transfer request block chain flag */
+#define XHCI_TRB_CH 0x10
+
+/** Transfer request block interrupt on completion flag */
+#define XHCI_TRB_IOC 0x20
+
+/** Transfer request block immediate data flag */
+#define XHCI_TRB_IDT 0x40
+
+/** Transfer request block type */
+#define XHCI_TRB_TYPE(type) ( (type) << 2 )
+
+/** Transfer request block type mask */
+#define XHCI_TRB_TYPE_MASK XHCI_TRB_TYPE ( 0x3f )
+
+/** A normal transfer request block */
+struct xhci_trb_normal {
+ /** Data buffer */
+ uint64_t data;
+ /** Length */
+ uint32_t len;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Reserved */
+ uint16_t reserved;
+} __attribute__ (( packed ));
+
+/** A normal transfer request block */
+#define XHCI_TRB_NORMAL XHCI_TRB_TYPE ( 1 )
+
+/** Construct TD size field */
+#define XHCI_TD_SIZE(remaining) \
+ ( ( ( (remaining) <= 0xf ) ? remaining : 0xf ) << 17 )
+
+/** A setup stage transfer request block */
+struct xhci_trb_setup {
+ /** Setup packet */
+ struct usb_setup_packet packet;
+ /** Length */
+ uint32_t len;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Transfer direction */
+ uint8_t direction;
+ /** Reserved */
+ uint8_t reserved;
+} __attribute__ (( packed ));
+
+/** A setup stage transfer request block */
+#define XHCI_TRB_SETUP XHCI_TRB_TYPE ( 2 )
+
+/** Setup stage input data direction */
+#define XHCI_SETUP_IN 3
+
+/** Setup stage output data direction */
+#define XHCI_SETUP_OUT 2
+
+/** A data stage transfer request block */
+struct xhci_trb_data {
+ /** Data buffer */
+ uint64_t data;
+ /** Length */
+ uint32_t len;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Transfer direction */
+ uint8_t direction;
+ /** Reserved */
+ uint8_t reserved;
+} __attribute__ (( packed ));
+
+/** A data stage transfer request block */
+#define XHCI_TRB_DATA XHCI_TRB_TYPE ( 3 )
+
+/** Input data direction */
+#define XHCI_DATA_IN 0x01
+
+/** Output data direction */
+#define XHCI_DATA_OUT 0x00
+
+/** A status stage transfer request block */
+struct xhci_trb_status {
+ /** Reserved */
+ uint64_t reserved_a;
+ /** Reserved */
+ uint32_t reserved_b;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Direction */
+ uint8_t direction;
+ /** Reserved */
+ uint8_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A status stage transfer request block */
+#define XHCI_TRB_STATUS XHCI_TRB_TYPE ( 4 )
+
+/** Input status direction */
+#define XHCI_STATUS_IN 0x01
+
+/** Output status direction */
+#define XHCI_STATUS_OUT 0x00
+
+/** A link transfer request block */
+struct xhci_trb_link {
+ /** Next ring segment */
+ uint64_t next;
+ /** Reserved */
+ uint32_t reserved_a;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Reserved */
+ uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A link transfer request block */
+#define XHCI_TRB_LINK XHCI_TRB_TYPE ( 6 )
+
+/** A no-op transfer request block */
+#define XHCI_TRB_NOP XHCI_TRB_TYPE ( 8 )
+
+/** An enable slot transfer request block */
+struct xhci_trb_enable_slot {
+ /** Reserved */
+ uint64_t reserved_a;
+ /** Reserved */
+ uint32_t reserved_b;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Slot type */
+ uint8_t slot;
+ /** Reserved */
+ uint8_t reserved_c;
+} __attribute__ (( packed ));
+
+/** An enable slot transfer request block */
+#define XHCI_TRB_ENABLE_SLOT XHCI_TRB_TYPE ( 9 )
+
+/** A disable slot transfer request block */
+struct xhci_trb_disable_slot {
+ /** Reserved */
+ uint64_t reserved_a;
+ /** Reserved */
+ uint32_t reserved_b;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Reserved */
+ uint8_t reserved_c;
+ /** Slot ID */
+ uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A disable slot transfer request block */
+#define XHCI_TRB_DISABLE_SLOT XHCI_TRB_TYPE ( 10 )
+
+/** A context transfer request block */
+struct xhci_trb_context {
+ /** Input context */
+ uint64_t input;
+ /** Reserved */
+ uint32_t reserved_a;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Reserved */
+ uint8_t reserved_b;
+ /** Slot ID */
+ uint8_t slot;
+} __attribute__ (( packed ));
+
+/** An address device transfer request block */
+#define XHCI_TRB_ADDRESS_DEVICE XHCI_TRB_TYPE ( 11 )
+
+/** A configure endpoint transfer request block */
+#define XHCI_TRB_CONFIGURE_ENDPOINT XHCI_TRB_TYPE ( 12 )
+
+/** An evaluate context transfer request block */
+#define XHCI_TRB_EVALUATE_CONTEXT XHCI_TRB_TYPE ( 13 )
+
+/** A reset endpoint transfer request block */
+struct xhci_trb_reset_endpoint {
+ /** Reserved */
+ uint64_t reserved_a;
+ /** Reserved */
+ uint32_t reserved_b;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Endpoint ID */
+ uint8_t endpoint;
+ /** Slot ID */
+ uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A reset endpoint transfer request block */
+#define XHCI_TRB_RESET_ENDPOINT XHCI_TRB_TYPE ( 14 )
+
+/** A stop endpoint transfer request block */
+struct xhci_trb_stop_endpoint {
+ /** Reserved */
+ uint64_t reserved_a;
+ /** Reserved */
+ uint32_t reserved_b;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Endpoint ID */
+ uint8_t endpoint;
+ /** Slot ID */
+ uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A stop endpoint transfer request block */
+#define XHCI_TRB_STOP_ENDPOINT XHCI_TRB_TYPE ( 15 )
+
+/** A set transfer ring dequeue pointer transfer request block */
+struct xhci_trb_set_tr_dequeue_pointer {
+ /** Dequeue pointer */
+ uint64_t dequeue;
+ /** Reserved */
+ uint32_t reserved;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Endpoint ID */
+ uint8_t endpoint;
+ /** Slot ID */
+ uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A set transfer ring dequeue pointer transfer request block */
+#define XHCI_TRB_SET_TR_DEQUEUE_POINTER XHCI_TRB_TYPE ( 16 )
+
+/** A no-op command transfer request block */
+#define XHCI_TRB_NOP_CMD XHCI_TRB_TYPE ( 23 )
+
+/** A transfer event transfer request block */
+struct xhci_trb_transfer {
+ /** Transfer TRB pointer */
+ uint64_t transfer;
+ /** Residual transfer length */
+ uint16_t residual;
+ /** Reserved */
+ uint8_t reserved;
+ /** Completion code */
+ uint8_t code;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Endpoint ID */
+ uint8_t endpoint;
+ /** Slot ID */
+ uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A transfer event transfer request block */
+#define XHCI_TRB_TRANSFER XHCI_TRB_TYPE ( 32 )
+
+/** A command completion event transfer request block */
+struct xhci_trb_complete {
+ /** Command TRB pointer */
+ uint64_t command;
+ /** Parameter */
+ uint8_t parameter[3];
+ /** Completion code */
+ uint8_t code;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Virtual function ID */
+ uint8_t vf;
+ /** Slot ID */
+ uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A command completion event transfer request block */
+#define XHCI_TRB_COMPLETE XHCI_TRB_TYPE ( 33 )
+
+/** xHCI completion codes */
+enum xhci_completion_code {
+ /** Success */
+ XHCI_CMPLT_SUCCESS = 1,
+ /** Short packet */
+ XHCI_CMPLT_SHORT = 13,
+};
+
+/** A port status change transfer request block */
+struct xhci_trb_port_status {
+ /** Reserved */
+ uint8_t reserved_a[3];
+ /** Port ID */
+ uint8_t port;
+ /** Reserved */
+ uint8_t reserved_b[7];
+ /** Completion code */
+ uint8_t code;
+ /** Flags */
+ uint8_t flags;
+ /** Type */
+ uint8_t type;
+ /** Reserved */
+ uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A port status change transfer request block */
+#define XHCI_TRB_PORT_STATUS XHCI_TRB_TYPE ( 34 )
+
+/** A transfer request block */
+union xhci_trb {
+ /** Template */
+ struct xhci_trb_template template;
+ /** Common fields */
+ struct xhci_trb_common common;
+ /** Normal TRB */
+ struct xhci_trb_normal normal;
+ /** Setup stage TRB */
+ struct xhci_trb_setup setup;
+ /** Data stage TRB */
+ struct xhci_trb_data data;
+ /** Status stage TRB */
+ struct xhci_trb_status status;
+ /** Link TRB */
+ struct xhci_trb_link link;
+ /** Enable slot TRB */
+ struct xhci_trb_enable_slot enable;
+ /** Disable slot TRB */
+ struct xhci_trb_disable_slot disable;
+ /** Input context TRB */
+ struct xhci_trb_context context;
+ /** Reset endpoint TRB */
+ struct xhci_trb_reset_endpoint reset;
+ /** Stop endpoint TRB */
+ struct xhci_trb_stop_endpoint stop;
+ /** Set transfer ring dequeue pointer TRB */
+ struct xhci_trb_set_tr_dequeue_pointer dequeue;
+ /** Transfer event */
+ struct xhci_trb_transfer transfer;
+ /** Command completion event */
+ struct xhci_trb_complete complete;
+ /** Port status changed event */
+ struct xhci_trb_port_status port;
+} __attribute__ (( packed ));
+
+/** An input control context */
+struct xhci_control_context {
+ /** Drop context flags */
+ uint32_t drop;
+ /** Add context flags */
+ uint32_t add;
+ /** Reserved */
+ uint32_t reserved_a[5];
+ /** Configuration value */
+ uint8_t config;
+ /** Interface number */
+ uint8_t intf;
+ /** Alternate setting */
+ uint8_t alt;
+ /** Reserved */
+ uint8_t reserved_b;
+} __attribute__ (( packed ));
+
+/** A slot context */
+struct xhci_slot_context {
+ /** Device info */
+ uint32_t info;
+ /** Maximum exit latency */
+ uint16_t latency;
+ /** Root hub port number */
+ uint8_t port;
+ /** Number of downstream ports */
+ uint8_t ports;
+ /** TT hub slot ID */
+ uint8_t tt_id;
+ /** TT port number */
+ uint8_t tt_port;
+ /** Interrupter target */
+ uint16_t intr;
+ /** USB address */
+ uint8_t address;
+ /** Reserved */
+ uint16_t reserved_a;
+ /** Slot state */
+ uint8_t state;
+ /** Reserved */
+ uint32_t reserved_b[4];
+} __attribute__ (( packed ));
+
+/** Construct slot context device info */
+#define XHCI_SLOT_INFO( entries, hub, speed, route ) \
+ ( ( (entries) << 27 ) | ( (hub) << 26 ) | ( (speed) << 20 ) | (route) )
+
+/** An endpoint context */
+struct xhci_endpoint_context {
+ /** Endpoint state */
+ uint8_t state;
+ /** Stream configuration */
+ uint8_t stream;
+ /** Polling interval */
+ uint8_t interval;
+ /** Max ESIT payload high */
+ uint8_t esit_high;
+ /** Endpoint type */
+ uint8_t type;
+ /** Maximum burst size */
+ uint8_t burst;
+ /** Maximum packet size */
+ uint16_t mtu;
+ /** Transfer ring dequeue pointer */
+ uint64_t dequeue;
+ /** Average TRB length */
+ uint16_t trb_len;
+ /** Max ESIT payload low */
+ uint16_t esit_low;
+ /** Reserved */
+ uint32_t reserved[3];
+} __attribute__ (( packed ));
+
+/** Endpoint states */
+enum xhci_endpoint_state {
+ /** Endpoint is disabled */
+ XHCI_ENDPOINT_DISABLED = 0,
+ /** Endpoint is running */
+ XHCI_ENDPOINT_RUNNING = 1,
+ /** Endpoint is halted due to a USB Halt condition */
+ XHCI_ENDPOINT_HALTED = 2,
+ /** Endpoint is stopped */
+ XHCI_ENDPOINT_STOPPED = 3,
+ /** Endpoint is halted due to a TRB error */
+ XHCI_ENDPOINT_ERROR = 4,
+};
+
+/** Endpoint state mask */
+#define XHCI_ENDPOINT_STATE_MASK 0x07
+
+/** Endpoint type */
+#define XHCI_EP_TYPE(type) ( (type) << 3 )
+
+/** Control endpoint type */
+#define XHCI_EP_TYPE_CONTROL XHCI_EP_TYPE ( 4 )
+
+/** Input endpoint type */
+#define XHCI_EP_TYPE_IN XHCI_EP_TYPE ( 4 )
+
+/** Endpoint dequeue cycle state */
+#define XHCI_EP_DCS 0x00000001UL
+
+/** Control endpoint average TRB length */
+#define XHCI_EP0_TRB_LEN 8
+
+/** An event ring segment */
+struct xhci_event_ring_segment {
+ /** Base address */
+ uint64_t base;
+ /** Number of TRBs */
+ uint32_t count;
+ /** Reserved */
+ uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** A transfer request block command/transfer ring */
+struct xhci_trb_ring {
+ /** Producer counter */
+ unsigned int prod;
+ /** Consumer counter */
+ unsigned int cons;
+ /** Ring size (log2) */
+ unsigned int shift;
+ /** Ring counter mask */
+ unsigned int mask;
+
+ /** I/O buffers */
+ struct io_buffer **iobuf;
+
+ /** Transfer request blocks */
+ union xhci_trb *trb;
+ /** Length of transfer request blocks */
+ size_t len;
+ /** Link TRB (if applicable) */
+ struct xhci_trb_link *link;
+
+ /** Doorbell register */
+ void *db;
+ /** Doorbell register value */
+ uint32_t dbval;
+};
+
+/** An event ring */
+struct xhci_event_ring {
+ /** Consumer counter */
+ unsigned int cons;
+ /** Event ring segment table */
+ struct xhci_event_ring_segment *segment;
+ /** Transfer request blocks */
+ union xhci_trb *trb;
+};
+
+/**
+ * Calculate doorbell register value
+ *
+ * @v target Doorbell target
+ * @v stream Doorbell stream ID
+ * @ret dbval Doorbell register value
+ */
+#define XHCI_DBVAL( target, stream ) ( (target) | ( (stream) << 16 ) )
+
+/**
+ * Calculate space used in TRB ring
+ *
+ * @v ring TRB ring
+ * @ret fill Number of entries used
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+xhci_ring_fill ( struct xhci_trb_ring *ring ) {
+
+ return ( ring->prod - ring->cons );
+}
+
+/**
+ * Calculate space remaining in TRB ring
+ *
+ * @v ring TRB ring
+ * @ret remaining Number of entries remaining
+ *
+ * xHCI does not allow us to completely fill a ring; there must be at
+ * least one free entry (excluding the Link TRB).
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+xhci_ring_remaining ( struct xhci_trb_ring *ring ) {
+ unsigned int fill = xhci_ring_fill ( ring );
+
+ /* We choose to utilise rings with ( 2^n + 1 ) entries, with
+ * the final entry being a Link TRB. The maximum fill level
+ * is therefore
+ *
+ * ( ( 2^n + 1 ) - 1 (Link TRB) - 1 (one slot always empty)
+ * == ( 2^n - 1 )
+ *
+ * which is therefore equal to the ring mask.
+ */
+ assert ( fill <= ring->mask );
+ return ( ring->mask - fill );
+}
+
+/**
+ * Calculate physical address of most recently consumed TRB
+ *
+ * @v ring TRB ring
+ * @ret trb TRB physical address
+ */
+static inline __attribute__ (( always_inline )) physaddr_t
+xhci_ring_consumed ( struct xhci_trb_ring *ring ) {
+ unsigned int index = ( ( ring->cons - 1 ) & ring->mask );
+
+ return virt_to_phys ( &ring->trb[index] );
+}
+
+/** Slot context index */
+#define XHCI_CTX_SLOT 0
+
+/** Calculate context index from USB endpoint address */
+#define XHCI_CTX(address) \
+ ( (address) ? ( ( ( (address) & 0x0f ) << 1 ) | \
+ ( ( (address) & 0x80 ) >> 7 ) ) : 1 )
+
+/** Endpoint zero context index */
+#define XHCI_CTX_EP0 XHCI_CTX ( 0x00 )
+
+/** End of contexts */
+#define XHCI_CTX_END 32
+
+/** Device context index */
+#define XHCI_DCI(ctx) ( (ctx) + 0 )
+
+/** Input context index */
+#define XHCI_ICI(ctx) ( (ctx) + 1 )
+
+/** Number of TRBs (excluding Link TRB) in the command ring
+ *
+ * This is a policy decision.
+ */
+#define XHCI_CMD_TRBS_LOG2 2
+
+/** Number of TRBs in the event ring
+ *
+ * This is a policy decision.
+ */
+#define XHCI_EVENT_TRBS_LOG2 6
+
+/** Number of TRBs in a transfer ring
+ *
+ * This is a policy decision.
+ */
+#define XHCI_TRANSFER_TRBS_LOG2 6
+
+/** Maximum time to wait for BIOS to release ownership
+ *
+ * This is a policy decision.
+ */
+#define XHCI_USBLEGSUP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for host controller to stop
+ *
+ * This is a policy decision.
+ */
+#define XHCI_STOP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for reset to complete
+ *
+ * This is a policy decision.
+ */
+#define XHCI_RESET_MAX_WAIT_MS 500
+
+/** Maximum time to wait for a command to complete
+ *
+ * This is a policy decision.
+ */
+#define XHCI_COMMAND_MAX_WAIT_MS 500
+
+/** Maximum time to wait for a port reset to complete
+ *
+ * This is a policy decision.
+ */
+#define XHCI_PORT_RESET_MAX_WAIT_MS 500
+
+/** An xHCI device */
+struct xhci_device {
+ /** Registers */
+ void *regs;
+
+ /** Capability registers */
+ void *cap;
+ /** Operational registers */
+ void *op;
+ /** Runtime registers */
+ void *run;
+ /** Doorbell registers */
+ void *db;
+
+ /** Number of device slots */
+ unsigned int slots;
+ /** Number of interrupters */
+ unsigned int intrs;
+ /** Number of ports */
+ unsigned int ports;
+
+ /** Number of page-sized scratchpad buffers */
+ unsigned int scratchpads;
+
+ /** 64-bit addressing capability */
+ int addr64;
+ /** Context size shift */
+ unsigned int csz_shift;
+ /** xHCI extended capabilities offset */
+ unsigned int xecp;
+
+ /** Page size */
+ size_t pagesize;
+
+ /** USB legacy support capability (if present and enabled) */
+ unsigned int legacy;
+
+ /** Device context base address array */
+ uint64_t *dcbaa;
+
+ /** Scratchpad buffer area */
+ userptr_t scratchpad;
+ /** Scratchpad buffer array */
+ uint64_t *scratchpad_array;
+
+ /** Command ring */
+ struct xhci_trb_ring command;
+ /** Event ring */
+ struct xhci_event_ring event;
+ /** Current command completion buffer (if any) */
+ union xhci_trb *completion;
+
+ /** Device slots, indexed by slot ID */
+ struct xhci_slot **slot;
+
+ /** USB bus */
+ struct usb_bus *bus;
+};
+
+/** An xHCI device slot */
+struct xhci_slot {
+ /** xHCI device */
+ struct xhci_device *xhci;
+ /** USB device */
+ struct usb_device *usb;
+ /** Slot ID */
+ unsigned int id;
+ /** Slot context */
+ struct xhci_slot_context *context;
+ /** Route string */
+ unsigned int route;
+ /** Root hub port number */
+ unsigned int port;
+ /** Protocol speed ID */
+ unsigned int psiv;
+ /** Endpoints, indexed by context ID */
+ struct xhci_endpoint *endpoint[XHCI_CTX_END];
+};
+
+/** An xHCI endpoint */
+struct xhci_endpoint {
+ /** xHCI device */
+ struct xhci_device *xhci;
+ /** xHCI slot */
+ struct xhci_slot *slot;
+ /** USB endpoint */
+ struct usb_endpoint *ep;
+ /** Context index */
+ unsigned int ctx;
+ /** Endpoint type */
+ unsigned int type;
+ /** Endpoint context */
+ struct xhci_endpoint_context *context;
+ /** Transfer ring */
+ struct xhci_trb_ring ring;
+};
+
+#endif /* _IPXE_XHCI_H */
projects / thirdparty / ipxe.git / commitdiff
