1 // SPDX-License-Identifier: GPL-2.0
3 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2018-2020 Linaro Ltd.
7 #include <linux/types.h>
8 #include <linux/bits.h>
9 #include <linux/bitfield.h>
10 #include <linux/mutex.h>
11 #include <linux/completion.h>
13 #include <linux/bug.h>
14 #include <linux/interrupt.h>
15 #include <linux/platform_device.h>
16 #include <linux/netdevice.h>
20 #include "gsi_private.h"
21 #include "gsi_trans.h"
26 * DOC: The IPA Generic Software Interface
28 * The generic software interface (GSI) is an integral component of the IPA,
29 * providing a well-defined communication layer between the AP subsystem
30 * and the IPA core. The modem uses the GSI layer as well.
34 * | AP +<---. .----+ Modem |
37 * -------- | | | | ---------
47 * In the above diagram, the AP and Modem represent "execution environments"
48 * (EEs), which are independent operating environments that use the IPA for
51 * Each EE uses a set of unidirectional GSI "channels," which allow transfer
52 * of data to or from the IPA. A channel is implemented as a ring buffer,
53 * with a DRAM-resident array of "transfer elements" (TREs) available to
54 * describe transfers to or from other EEs through the IPA. A transfer
55 * element can also contain an immediate command, requesting the IPA perform
56 * actions other than data transfer.
58 * Each TRE refers to a block of data--also located DRAM. After writing one
59 * or more TREs to a channel, the writer (either the IPA or an EE) writes a
60 * doorbell register to inform the receiving side how many elements have
63 * Each channel has a GSI "event ring" associated with it. An event ring
64 * is implemented very much like a channel ring, but is always directed from
65 * the IPA to an EE. The IPA notifies an EE (such as the AP) about channel
66 * events by adding an entry to the event ring associated with the channel.
67 * The GSI then writes its doorbell for the event ring, causing the target
68 * EE to be interrupted. Each entry in an event ring contains a pointer
69 * to the channel TRE whose completion the event represents.
71 * Each TRE in a channel ring has a set of flags. One flag indicates whether
72 * the completion of the transfer operation generates an entry (and possibly
73 * an interrupt) in the channel's event ring. Other flags allow transfer
74 * elements to be chained together, forming a single logical transaction.
75 * TRE flags are used to control whether and when interrupts are generated
76 * to signal completion of channel transfers.
78 * Elements in channel and event rings are completed (or consumed) strictly
79 * in order. Completion of one entry implies the completion of all preceding
80 * entries. A single completion interrupt can therefore communicate the
81 * completion of many transfers.
83 * Note that all GSI registers are little-endian, which is the assumed
84 * endianness of I/O space accesses. The accessor functions perform byte
85 * swapping if needed (i.e., for a big endian CPU).
88 /* Delay period for interrupt moderation (in 32KHz IPA internal timer ticks) */
89 #define GSI_EVT_RING_INT_MODT (32 * 1) /* 1ms under 32KHz clock */
91 #define GSI_CMD_TIMEOUT 5 /* seconds */
93 #define GSI_CHANNEL_STOP_RX_RETRIES 10
95 #define GSI_MHI_EVENT_ID_START 10 /* 1st reserved event id */
96 #define GSI_MHI_EVENT_ID_END 16 /* Last reserved event id */
98 #define GSI_ISR_MAX_ITER 50 /* Detect interrupt storms */
100 /* An entry in an event ring */
111 /* Hardware values from the error log register error code field */
113 GSI_INVALID_TRE_ERR
= 0x1,
114 GSI_OUT_OF_BUFFERS_ERR
= 0x2,
115 GSI_OUT_OF_RESOURCES_ERR
= 0x3,
116 GSI_UNSUPPORTED_INTER_EE_OP_ERR
= 0x4,
117 GSI_EVT_RING_EMPTY_ERR
= 0x5,
118 GSI_NON_ALLOCATED_EVT_ACCESS_ERR
= 0x6,
122 /* Hardware values from the error log register error type field */
124 GSI_ERR_TYPE_GLOB
= 0x1,
125 GSI_ERR_TYPE_CHAN
= 0x2,
126 GSI_ERR_TYPE_EVT
= 0x3,
129 /* Hardware values used when programming an event ring */
130 enum gsi_evt_chtype
{
131 GSI_EVT_CHTYPE_MHI_EV
= 0x0,
132 GSI_EVT_CHTYPE_XHCI_EV
= 0x1,
133 GSI_EVT_CHTYPE_GPI_EV
= 0x2,
134 GSI_EVT_CHTYPE_XDCI_EV
= 0x3,
137 /* Hardware values used when programming a channel */
138 enum gsi_channel_protocol
{
139 GSI_CHANNEL_PROTOCOL_MHI
= 0x0,
140 GSI_CHANNEL_PROTOCOL_XHCI
= 0x1,
141 GSI_CHANNEL_PROTOCOL_GPI
= 0x2,
142 GSI_CHANNEL_PROTOCOL_XDCI
= 0x3,
145 /* Hardware values representing an event ring immediate command opcode */
146 enum gsi_evt_cmd_opcode
{
147 GSI_EVT_ALLOCATE
= 0x0,
149 GSI_EVT_DE_ALLOC
= 0xa,
152 /* Hardware values representing a generic immediate command opcode */
153 enum gsi_generic_cmd_opcode
{
154 GSI_GENERIC_HALT_CHANNEL
= 0x1,
155 GSI_GENERIC_ALLOCATE_CHANNEL
= 0x2,
158 /* Hardware values representing a channel immediate command opcode */
159 enum gsi_ch_cmd_opcode
{
160 GSI_CH_ALLOCATE
= 0x0,
164 GSI_CH_DE_ALLOC
= 0xa,
167 /** gsi_channel_scratch_gpi - GPI protocol scratch register
168 * @max_outstanding_tre:
169 * Defines the maximum number of TREs allowed in a single transaction
170 * on a channel (in bytes). This determines the amount of prefetch
171 * performed by the hardware. We configure this to equal the size of
172 * the TLV FIFO for the channel.
173 * @outstanding_threshold:
174 * Defines the threshold (in bytes) determining when the sequencer
175 * should update the channel doorbell. We configure this to equal
176 * the size of two TREs.
178 struct gsi_channel_scratch_gpi
{
181 u16 max_outstanding_tre
;
183 u16 outstanding_threshold
;
186 /** gsi_channel_scratch - channel scratch configuration area
188 * The exact interpretation of this register is protocol-specific.
189 * We only use GPI channels; see struct gsi_channel_scratch_gpi, above.
191 union gsi_channel_scratch
{
192 struct gsi_channel_scratch_gpi gpi
;
201 /* Check things that can be validated at build time. */
202 static void gsi_validate_build(void)
204 /* This is used as a divisor */
205 BUILD_BUG_ON(!GSI_RING_ELEMENT_SIZE
);
207 /* Code assumes the size of channel and event ring element are
208 * the same (and fixed). Make sure the size of an event ring
209 * element is what's expected.
211 BUILD_BUG_ON(sizeof(struct gsi_event
) != GSI_RING_ELEMENT_SIZE
);
213 /* Hardware requires a 2^n ring size. We ensure the number of
214 * elements in an event ring is a power of 2 elsewhere; this
215 * ensure the elements themselves meet the requirement.
217 BUILD_BUG_ON(!is_power_of_2(GSI_RING_ELEMENT_SIZE
));
219 /* The channel element size must fit in this field */
220 BUILD_BUG_ON(GSI_RING_ELEMENT_SIZE
> field_max(ELEMENT_SIZE_FMASK
));
222 /* The event ring element size must fit in this field */
223 BUILD_BUG_ON(GSI_RING_ELEMENT_SIZE
> field_max(EV_ELEMENT_SIZE_FMASK
));
226 /* Return the channel id associated with a given channel */
227 static u32
gsi_channel_id(struct gsi_channel
*channel
)
229 return channel
- &channel
->gsi
->channel
[0];
232 static void gsi_irq_ieob_enable(struct gsi
*gsi
, u32 evt_ring_id
)
236 gsi
->event_enable_bitmap
|= BIT(evt_ring_id
);
237 val
= gsi
->event_enable_bitmap
;
238 iowrite32(val
, gsi
->virt
+ GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET
);
241 static void gsi_isr_ieob_clear(struct gsi
*gsi
, u32 mask
)
243 iowrite32(mask
, gsi
->virt
+ GSI_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET
);
246 static void gsi_irq_ieob_disable(struct gsi
*gsi
, u32 evt_ring_id
)
250 gsi
->event_enable_bitmap
&= ~BIT(evt_ring_id
);
251 val
= gsi
->event_enable_bitmap
;
252 iowrite32(val
, gsi
->virt
+ GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET
);
255 /* Enable all GSI_interrupt types */
256 static void gsi_irq_enable(struct gsi
*gsi
)
260 /* We don't use inter-EE channel or event interrupts */
261 val
= GSI_CNTXT_TYPE_IRQ_MSK_ALL
;
262 val
&= ~MSK_INTER_EE_CH_CTRL_FMASK
;
263 val
&= ~MSK_INTER_EE_EV_CTRL_FMASK
;
264 iowrite32(val
, gsi
->virt
+ GSI_CNTXT_TYPE_IRQ_MSK_OFFSET
);
266 val
= GENMASK(gsi
->channel_count
- 1, 0);
267 iowrite32(val
, gsi
->virt
+ GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET
);
269 val
= GENMASK(gsi
->evt_ring_count
- 1, 0);
270 iowrite32(val
, gsi
->virt
+ GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET
);
272 /* Each IEOB interrupt is enabled (later) as needed by channels */
273 iowrite32(0, gsi
->virt
+ GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET
);
275 val
= GSI_CNTXT_GLOB_IRQ_ALL
;
276 iowrite32(val
, gsi
->virt
+ GSI_CNTXT_GLOB_IRQ_EN_OFFSET
);
278 /* Never enable GSI_BREAK_POINT */
279 val
= GSI_CNTXT_GSI_IRQ_ALL
& ~EN_BREAK_POINT_FMASK
;
280 iowrite32(val
, gsi
->virt
+ GSI_CNTXT_GSI_IRQ_EN_OFFSET
);
283 /* Disable all GSI_interrupt types */
284 static void gsi_irq_disable(struct gsi
*gsi
)
286 iowrite32(0, gsi
->virt
+ GSI_CNTXT_GSI_IRQ_EN_OFFSET
);
287 iowrite32(0, gsi
->virt
+ GSI_CNTXT_GLOB_IRQ_EN_OFFSET
);
288 iowrite32(0, gsi
->virt
+ GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET
);
289 iowrite32(0, gsi
->virt
+ GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET
);
290 iowrite32(0, gsi
->virt
+ GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET
);
291 iowrite32(0, gsi
->virt
+ GSI_CNTXT_TYPE_IRQ_MSK_OFFSET
);
294 /* Return the virtual address associated with a ring index */
295 void *gsi_ring_virt(struct gsi_ring
*ring
, u32 index
)
297 /* Note: index *must* be used modulo the ring count here */
298 return ring
->virt
+ (index
% ring
->count
) * GSI_RING_ELEMENT_SIZE
;
301 /* Return the 32-bit DMA address associated with a ring index */
302 static u32
gsi_ring_addr(struct gsi_ring
*ring
, u32 index
)
304 return (ring
->addr
& GENMASK(31, 0)) + index
* GSI_RING_ELEMENT_SIZE
;
307 /* Return the ring index of a 32-bit ring offset */
308 static u32
gsi_ring_index(struct gsi_ring
*ring
, u32 offset
)
310 return (offset
- gsi_ring_addr(ring
, 0)) / GSI_RING_ELEMENT_SIZE
;
313 /* Issue a GSI command by writing a value to a register, then wait for
314 * completion to be signaled. Returns true if the command completes
315 * or false if it times out.
318 gsi_command(struct gsi
*gsi
, u32 reg
, u32 val
, struct completion
*completion
)
320 reinit_completion(completion
);
322 iowrite32(val
, gsi
->virt
+ reg
);
324 return !!wait_for_completion_timeout(completion
, GSI_CMD_TIMEOUT
* HZ
);
327 /* Return the hardware's notion of the current state of an event ring */
328 static enum gsi_evt_ring_state
329 gsi_evt_ring_state(struct gsi
*gsi
, u32 evt_ring_id
)
333 val
= ioread32(gsi
->virt
+ GSI_EV_CH_E_CNTXT_0_OFFSET(evt_ring_id
));
335 return u32_get_bits(val
, EV_CHSTATE_FMASK
);
338 /* Issue an event ring command and wait for it to complete */
339 static int evt_ring_command(struct gsi
*gsi
, u32 evt_ring_id
,
340 enum gsi_evt_cmd_opcode opcode
)
342 struct gsi_evt_ring
*evt_ring
= &gsi
->evt_ring
[evt_ring_id
];
343 struct completion
*completion
= &evt_ring
->completion
;
346 val
= u32_encode_bits(evt_ring_id
, EV_CHID_FMASK
);
347 val
|= u32_encode_bits(opcode
, EV_OPCODE_FMASK
);
349 if (gsi_command(gsi
, GSI_EV_CH_CMD_OFFSET
, val
, completion
))
350 return 0; /* Success! */
352 dev_err(gsi
->dev
, "GSI command %u to event ring %u timed out "
353 "(state is %u)\n", opcode
, evt_ring_id
, evt_ring
->state
);
358 /* Allocate an event ring in NOT_ALLOCATED state */
359 static int gsi_evt_ring_alloc_command(struct gsi
*gsi
, u32 evt_ring_id
)
361 struct gsi_evt_ring
*evt_ring
= &gsi
->evt_ring
[evt_ring_id
];
364 /* Get initial event ring state */
365 evt_ring
->state
= gsi_evt_ring_state(gsi
, evt_ring_id
);
367 if (evt_ring
->state
!= GSI_EVT_RING_STATE_NOT_ALLOCATED
)
370 ret
= evt_ring_command(gsi
, evt_ring_id
, GSI_EVT_ALLOCATE
);
371 if (!ret
&& evt_ring
->state
!= GSI_EVT_RING_STATE_ALLOCATED
) {
372 dev_err(gsi
->dev
, "bad event ring state (%u) after alloc\n",
380 /* Reset a GSI event ring in ALLOCATED or ERROR state. */
381 static void gsi_evt_ring_reset_command(struct gsi
*gsi
, u32 evt_ring_id
)
383 struct gsi_evt_ring
*evt_ring
= &gsi
->evt_ring
[evt_ring_id
];
384 enum gsi_evt_ring_state state
= evt_ring
->state
;
387 if (state
!= GSI_EVT_RING_STATE_ALLOCATED
&&
388 state
!= GSI_EVT_RING_STATE_ERROR
) {
389 dev_err(gsi
->dev
, "bad event ring state (%u) before reset\n",
394 ret
= evt_ring_command(gsi
, evt_ring_id
, GSI_EVT_RESET
);
395 if (!ret
&& evt_ring
->state
!= GSI_EVT_RING_STATE_ALLOCATED
)
396 dev_err(gsi
->dev
, "bad event ring state (%u) after reset\n",
400 /* Issue a hardware de-allocation request for an allocated event ring */
401 static void gsi_evt_ring_de_alloc_command(struct gsi
*gsi
, u32 evt_ring_id
)
403 struct gsi_evt_ring
*evt_ring
= &gsi
->evt_ring
[evt_ring_id
];
406 if (evt_ring
->state
!= GSI_EVT_RING_STATE_ALLOCATED
) {
407 dev_err(gsi
->dev
, "bad event ring state (%u) before dealloc\n",
412 ret
= evt_ring_command(gsi
, evt_ring_id
, GSI_EVT_DE_ALLOC
);
413 if (!ret
&& evt_ring
->state
!= GSI_EVT_RING_STATE_NOT_ALLOCATED
)
414 dev_err(gsi
->dev
, "bad event ring state (%u) after dealloc\n",
418 /* Return the hardware's notion of the current state of a channel */
419 static enum gsi_channel_state
420 gsi_channel_state(struct gsi
*gsi
, u32 channel_id
)
424 val
= ioread32(gsi
->virt
+ GSI_CH_C_CNTXT_0_OFFSET(channel_id
));
426 return u32_get_bits(val
, CHSTATE_FMASK
);
429 /* Issue a channel command and wait for it to complete */
431 gsi_channel_command(struct gsi_channel
*channel
, enum gsi_ch_cmd_opcode opcode
)
433 struct completion
*completion
= &channel
->completion
;
434 u32 channel_id
= gsi_channel_id(channel
);
437 val
= u32_encode_bits(channel_id
, CH_CHID_FMASK
);
438 val
|= u32_encode_bits(opcode
, CH_OPCODE_FMASK
);
440 if (gsi_command(channel
->gsi
, GSI_CH_CMD_OFFSET
, val
, completion
))
441 return 0; /* Success! */
443 dev_err(channel
->gsi
->dev
, "GSI command %u to channel %u timed out "
444 "(state is %u)\n", opcode
, channel_id
, channel
->state
);
449 /* Allocate GSI channel in NOT_ALLOCATED state */
450 static int gsi_channel_alloc_command(struct gsi
*gsi
, u32 channel_id
)
452 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
455 /* Get initial channel state */
456 channel
->state
= gsi_channel_state(gsi
, channel_id
);
458 if (channel
->state
!= GSI_CHANNEL_STATE_NOT_ALLOCATED
)
461 ret
= gsi_channel_command(channel
, GSI_CH_ALLOCATE
);
462 if (!ret
&& channel
->state
!= GSI_CHANNEL_STATE_ALLOCATED
) {
463 dev_err(gsi
->dev
, "bad channel state (%u) after alloc\n",
471 /* Start an ALLOCATED channel */
472 static int gsi_channel_start_command(struct gsi_channel
*channel
)
474 enum gsi_channel_state state
= channel
->state
;
477 if (state
!= GSI_CHANNEL_STATE_ALLOCATED
&&
478 state
!= GSI_CHANNEL_STATE_STOPPED
)
481 ret
= gsi_channel_command(channel
, GSI_CH_START
);
482 if (!ret
&& channel
->state
!= GSI_CHANNEL_STATE_STARTED
) {
483 dev_err(channel
->gsi
->dev
,
484 "bad channel state (%u) after start\n",
492 /* Stop a GSI channel in STARTED state */
493 static int gsi_channel_stop_command(struct gsi_channel
*channel
)
495 enum gsi_channel_state state
= channel
->state
;
498 if (state
!= GSI_CHANNEL_STATE_STARTED
&&
499 state
!= GSI_CHANNEL_STATE_STOP_IN_PROC
)
502 ret
= gsi_channel_command(channel
, GSI_CH_STOP
);
503 if (ret
|| channel
->state
== GSI_CHANNEL_STATE_STOPPED
)
506 /* We may have to try again if stop is in progress */
507 if (channel
->state
== GSI_CHANNEL_STATE_STOP_IN_PROC
)
510 dev_err(channel
->gsi
->dev
, "bad channel state (%u) after stop\n",
516 /* Reset a GSI channel in ALLOCATED or ERROR state. */
517 static void gsi_channel_reset_command(struct gsi_channel
*channel
)
521 msleep(1); /* A short delay is required before a RESET command */
523 if (channel
->state
!= GSI_CHANNEL_STATE_STOPPED
&&
524 channel
->state
!= GSI_CHANNEL_STATE_ERROR
) {
525 dev_err(channel
->gsi
->dev
,
526 "bad channel state (%u) before reset\n",
531 ret
= gsi_channel_command(channel
, GSI_CH_RESET
);
532 if (!ret
&& channel
->state
!= GSI_CHANNEL_STATE_ALLOCATED
)
533 dev_err(channel
->gsi
->dev
,
534 "bad channel state (%u) after reset\n",
538 /* Deallocate an ALLOCATED GSI channel */
539 static void gsi_channel_de_alloc_command(struct gsi
*gsi
, u32 channel_id
)
541 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
544 if (channel
->state
!= GSI_CHANNEL_STATE_ALLOCATED
) {
545 dev_err(gsi
->dev
, "bad channel state (%u) before dealloc\n",
550 ret
= gsi_channel_command(channel
, GSI_CH_DE_ALLOC
);
551 if (!ret
&& channel
->state
!= GSI_CHANNEL_STATE_NOT_ALLOCATED
)
552 dev_err(gsi
->dev
, "bad channel state (%u) after dealloc\n",
556 /* Ring an event ring doorbell, reporting the last entry processed by the AP.
557 * The index argument (modulo the ring count) is the first unfilled entry, so
558 * we supply one less than that with the doorbell. Update the event ring
559 * index field with the value provided.
561 static void gsi_evt_ring_doorbell(struct gsi
*gsi
, u32 evt_ring_id
, u32 index
)
563 struct gsi_ring
*ring
= &gsi
->evt_ring
[evt_ring_id
].ring
;
566 ring
->index
= index
; /* Next unused entry */
568 /* Note: index *must* be used modulo the ring count here */
569 val
= gsi_ring_addr(ring
, (index
- 1) % ring
->count
);
570 iowrite32(val
, gsi
->virt
+ GSI_EV_CH_E_DOORBELL_0_OFFSET(evt_ring_id
));
573 /* Program an event ring for use */
574 static void gsi_evt_ring_program(struct gsi
*gsi
, u32 evt_ring_id
)
576 struct gsi_evt_ring
*evt_ring
= &gsi
->evt_ring
[evt_ring_id
];
577 size_t size
= evt_ring
->ring
.count
* GSI_RING_ELEMENT_SIZE
;
580 val
= u32_encode_bits(GSI_EVT_CHTYPE_GPI_EV
, EV_CHTYPE_FMASK
);
581 val
|= EV_INTYPE_FMASK
;
582 val
|= u32_encode_bits(GSI_RING_ELEMENT_SIZE
, EV_ELEMENT_SIZE_FMASK
);
583 iowrite32(val
, gsi
->virt
+ GSI_EV_CH_E_CNTXT_0_OFFSET(evt_ring_id
));
585 val
= u32_encode_bits(size
, EV_R_LENGTH_FMASK
);
586 iowrite32(val
, gsi
->virt
+ GSI_EV_CH_E_CNTXT_1_OFFSET(evt_ring_id
));
588 /* The context 2 and 3 registers store the low-order and
589 * high-order 32 bits of the address of the event ring,
592 val
= evt_ring
->ring
.addr
& GENMASK(31, 0);
593 iowrite32(val
, gsi
->virt
+ GSI_EV_CH_E_CNTXT_2_OFFSET(evt_ring_id
));
595 val
= evt_ring
->ring
.addr
>> 32;
596 iowrite32(val
, gsi
->virt
+ GSI_EV_CH_E_CNTXT_3_OFFSET(evt_ring_id
));
598 /* Enable interrupt moderation by setting the moderation delay */
599 val
= u32_encode_bits(GSI_EVT_RING_INT_MODT
, MODT_FMASK
);
600 val
|= u32_encode_bits(1, MODC_FMASK
); /* comes from channel */
601 iowrite32(val
, gsi
->virt
+ GSI_EV_CH_E_CNTXT_8_OFFSET(evt_ring_id
));
603 /* No MSI write data, and MSI address high and low address is 0 */
604 iowrite32(0, gsi
->virt
+ GSI_EV_CH_E_CNTXT_9_OFFSET(evt_ring_id
));
605 iowrite32(0, gsi
->virt
+ GSI_EV_CH_E_CNTXT_10_OFFSET(evt_ring_id
));
606 iowrite32(0, gsi
->virt
+ GSI_EV_CH_E_CNTXT_11_OFFSET(evt_ring_id
));
608 /* We don't need to get event read pointer updates */
609 iowrite32(0, gsi
->virt
+ GSI_EV_CH_E_CNTXT_12_OFFSET(evt_ring_id
));
610 iowrite32(0, gsi
->virt
+ GSI_EV_CH_E_CNTXT_13_OFFSET(evt_ring_id
));
612 /* Finally, tell the hardware we've completed event 0 (arbitrary) */
613 gsi_evt_ring_doorbell(gsi
, evt_ring_id
, 0);
616 /* Return the last (most recent) transaction completed on a channel. */
617 static struct gsi_trans
*gsi_channel_trans_last(struct gsi_channel
*channel
)
619 struct gsi_trans_info
*trans_info
= &channel
->trans_info
;
620 struct gsi_trans
*trans
;
622 spin_lock_bh(&trans_info
->spinlock
);
624 if (!list_empty(&trans_info
->complete
))
625 trans
= list_last_entry(&trans_info
->complete
,
626 struct gsi_trans
, links
);
627 else if (!list_empty(&trans_info
->polled
))
628 trans
= list_last_entry(&trans_info
->polled
,
629 struct gsi_trans
, links
);
633 /* Caller will wait for this, so take a reference */
635 refcount_inc(&trans
->refcount
);
637 spin_unlock_bh(&trans_info
->spinlock
);
642 /* Wait for transaction activity on a channel to complete */
643 static void gsi_channel_trans_quiesce(struct gsi_channel
*channel
)
645 struct gsi_trans
*trans
;
647 /* Get the last transaction, and wait for it to complete */
648 trans
= gsi_channel_trans_last(channel
);
650 wait_for_completion(&trans
->completion
);
651 gsi_trans_free(trans
);
655 /* Stop channel activity. Transactions may not be allocated until thawed. */
656 static void gsi_channel_freeze(struct gsi_channel
*channel
)
658 gsi_channel_trans_quiesce(channel
);
660 napi_disable(&channel
->napi
);
662 gsi_irq_ieob_disable(channel
->gsi
, channel
->evt_ring_id
);
665 /* Allow transactions to be used on the channel again. */
666 static void gsi_channel_thaw(struct gsi_channel
*channel
)
668 gsi_irq_ieob_enable(channel
->gsi
, channel
->evt_ring_id
);
670 napi_enable(&channel
->napi
);
673 /* Program a channel for use */
674 static void gsi_channel_program(struct gsi_channel
*channel
, bool doorbell
)
676 size_t size
= channel
->tre_ring
.count
* GSI_RING_ELEMENT_SIZE
;
677 u32 channel_id
= gsi_channel_id(channel
);
678 union gsi_channel_scratch scr
= { };
679 struct gsi_channel_scratch_gpi
*gpi
;
680 struct gsi
*gsi
= channel
->gsi
;
684 /* Arbitrarily pick TRE 0 as the first channel element to use */
685 channel
->tre_ring
.index
= 0;
687 /* We program all channels to use GPI protocol */
688 val
= u32_encode_bits(GSI_CHANNEL_PROTOCOL_GPI
, CHTYPE_PROTOCOL_FMASK
);
689 if (channel
->toward_ipa
)
690 val
|= CHTYPE_DIR_FMASK
;
691 val
|= u32_encode_bits(channel
->evt_ring_id
, ERINDEX_FMASK
);
692 val
|= u32_encode_bits(GSI_RING_ELEMENT_SIZE
, ELEMENT_SIZE_FMASK
);
693 iowrite32(val
, gsi
->virt
+ GSI_CH_C_CNTXT_0_OFFSET(channel_id
));
695 val
= u32_encode_bits(size
, R_LENGTH_FMASK
);
696 iowrite32(val
, gsi
->virt
+ GSI_CH_C_CNTXT_1_OFFSET(channel_id
));
698 /* The context 2 and 3 registers store the low-order and
699 * high-order 32 bits of the address of the channel ring,
702 val
= channel
->tre_ring
.addr
& GENMASK(31, 0);
703 iowrite32(val
, gsi
->virt
+ GSI_CH_C_CNTXT_2_OFFSET(channel_id
));
705 val
= channel
->tre_ring
.addr
>> 32;
706 iowrite32(val
, gsi
->virt
+ GSI_CH_C_CNTXT_3_OFFSET(channel_id
));
708 /* Command channel gets low weighted round-robin priority */
709 if (channel
->command
)
710 wrr_weight
= field_max(WRR_WEIGHT_FMASK
);
711 val
= u32_encode_bits(wrr_weight
, WRR_WEIGHT_FMASK
);
713 /* Max prefetch is 1 segment (do not set MAX_PREFETCH_FMASK) */
715 /* Enable the doorbell engine if requested */
717 val
|= USE_DB_ENG_FMASK
;
719 if (!channel
->use_prefetch
)
720 val
|= USE_ESCAPE_BUF_ONLY_FMASK
;
722 iowrite32(val
, gsi
->virt
+ GSI_CH_C_QOS_OFFSET(channel_id
));
724 /* Now update the scratch registers for GPI protocol */
726 gpi
->max_outstanding_tre
= gsi_channel_trans_tre_max(gsi
, channel_id
) *
727 GSI_RING_ELEMENT_SIZE
;
728 gpi
->outstanding_threshold
= 2 * GSI_RING_ELEMENT_SIZE
;
730 val
= scr
.data
.word1
;
731 iowrite32(val
, gsi
->virt
+ GSI_CH_C_SCRATCH_0_OFFSET(channel_id
));
733 val
= scr
.data
.word2
;
734 iowrite32(val
, gsi
->virt
+ GSI_CH_C_SCRATCH_1_OFFSET(channel_id
));
736 val
= scr
.data
.word3
;
737 iowrite32(val
, gsi
->virt
+ GSI_CH_C_SCRATCH_2_OFFSET(channel_id
));
739 /* We must preserve the upper 16 bits of the last scratch register.
740 * The next sequence assumes those bits remain unchanged between the
741 * read and the write.
743 val
= ioread32(gsi
->virt
+ GSI_CH_C_SCRATCH_3_OFFSET(channel_id
));
744 val
= (scr
.data
.word4
& GENMASK(31, 16)) | (val
& GENMASK(15, 0));
745 iowrite32(val
, gsi
->virt
+ GSI_CH_C_SCRATCH_3_OFFSET(channel_id
));
750 static void gsi_channel_deprogram(struct gsi_channel
*channel
)
755 /* Start an allocated GSI channel */
756 int gsi_channel_start(struct gsi
*gsi
, u32 channel_id
)
758 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
759 u32 evt_ring_id
= channel
->evt_ring_id
;
762 mutex_lock(&gsi
->mutex
);
764 ret
= gsi_channel_start_command(channel
);
766 mutex_unlock(&gsi
->mutex
);
768 /* Clear the channel's event ring interrupt in case it's pending */
769 gsi_isr_ieob_clear(gsi
, BIT(evt_ring_id
));
771 gsi_channel_thaw(channel
);
776 /* Stop a started channel */
777 int gsi_channel_stop(struct gsi
*gsi
, u32 channel_id
)
779 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
783 gsi_channel_freeze(channel
);
785 /* Channel could have entered STOPPED state since last call if the
786 * STOP command timed out. We won't stop a channel if stopping it
787 * was successful previously (so we still want the freeze above).
789 if (channel
->state
== GSI_CHANNEL_STATE_STOPPED
)
792 /* RX channels might require a little time to enter STOPPED state */
793 retries
= channel
->toward_ipa
? 0 : GSI_CHANNEL_STOP_RX_RETRIES
;
795 mutex_lock(&gsi
->mutex
);
798 ret
= gsi_channel_stop_command(channel
);
804 mutex_unlock(&gsi
->mutex
);
806 /* Thaw the channel if we need to retry (or on error) */
808 gsi_channel_thaw(channel
);
813 /* Reset and reconfigure a channel (possibly leaving doorbell disabled) */
814 void gsi_channel_reset(struct gsi
*gsi
, u32 channel_id
, bool db_enable
)
816 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
818 mutex_lock(&gsi
->mutex
);
820 /* Due to a hardware quirk we need to reset RX channels twice. */
821 gsi_channel_reset_command(channel
);
822 if (!channel
->toward_ipa
)
823 gsi_channel_reset_command(channel
);
825 gsi_channel_program(channel
, db_enable
);
826 gsi_channel_trans_cancel_pending(channel
);
828 mutex_unlock(&gsi
->mutex
);
831 /* Stop a STARTED channel for suspend (using stop if requested) */
832 int gsi_channel_suspend(struct gsi
*gsi
, u32 channel_id
, bool stop
)
834 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
837 return gsi_channel_stop(gsi
, channel_id
);
839 gsi_channel_freeze(channel
);
844 /* Resume a suspended channel (starting will be requested if STOPPED) */
845 int gsi_channel_resume(struct gsi
*gsi
, u32 channel_id
, bool start
)
847 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
850 return gsi_channel_start(gsi
, channel_id
);
852 gsi_channel_thaw(channel
);
858 * gsi_channel_tx_queued() - Report queued TX transfers for a channel
859 * @channel: Channel for which to report
861 * Report to the network stack the number of bytes and transactions that
862 * have been queued to hardware since last call. This and the next function
863 * supply information used by the network stack for throttling.
865 * For each channel we track the number of transactions used and bytes of
866 * data those transactions represent. We also track what those values are
867 * each time this function is called. Subtracting the two tells us
868 * the number of bytes and transactions that have been added between
871 * Calling this each time we ring the channel doorbell allows us to
872 * provide accurate information to the network stack about how much
873 * work we've given the hardware at any point in time.
875 void gsi_channel_tx_queued(struct gsi_channel
*channel
)
880 byte_count
= channel
->byte_count
- channel
->queued_byte_count
;
881 trans_count
= channel
->trans_count
- channel
->queued_trans_count
;
882 channel
->queued_byte_count
= channel
->byte_count
;
883 channel
->queued_trans_count
= channel
->trans_count
;
885 ipa_gsi_channel_tx_queued(channel
->gsi
, gsi_channel_id(channel
),
886 trans_count
, byte_count
);
890 * gsi_channel_tx_update() - Report completed TX transfers
891 * @channel: Channel that has completed transmitting packets
892 * @trans: Last transation known to be complete
894 * Compute the number of transactions and bytes that have been transferred
895 * over a TX channel since the given transaction was committed. Report this
896 * information to the network stack.
898 * At the time a transaction is committed, we record its channel's
899 * committed transaction and byte counts *in the transaction*.
900 * Completions are signaled by the hardware with an interrupt, and
901 * we can determine the latest completed transaction at that time.
903 * The difference between the byte/transaction count recorded in
904 * the transaction and the count last time we recorded a completion
905 * tells us exactly how much data has been transferred between
908 * Calling this each time we learn of a newly-completed transaction
909 * allows us to provide accurate information to the network stack
910 * about how much work has been completed by the hardware at a given
914 gsi_channel_tx_update(struct gsi_channel
*channel
, struct gsi_trans
*trans
)
916 u64 byte_count
= trans
->byte_count
+ trans
->len
;
917 u64 trans_count
= trans
->trans_count
+ 1;
919 byte_count
-= channel
->compl_byte_count
;
920 channel
->compl_byte_count
+= byte_count
;
921 trans_count
-= channel
->compl_trans_count
;
922 channel
->compl_trans_count
+= trans_count
;
924 ipa_gsi_channel_tx_completed(channel
->gsi
, gsi_channel_id(channel
),
925 trans_count
, byte_count
);
928 /* Channel control interrupt handler */
929 static void gsi_isr_chan_ctrl(struct gsi
*gsi
)
933 channel_mask
= ioread32(gsi
->virt
+ GSI_CNTXT_SRC_CH_IRQ_OFFSET
);
934 iowrite32(channel_mask
, gsi
->virt
+ GSI_CNTXT_SRC_CH_IRQ_CLR_OFFSET
);
936 while (channel_mask
) {
937 u32 channel_id
= __ffs(channel_mask
);
938 struct gsi_channel
*channel
;
940 channel_mask
^= BIT(channel_id
);
942 channel
= &gsi
->channel
[channel_id
];
943 channel
->state
= gsi_channel_state(gsi
, channel_id
);
945 complete(&channel
->completion
);
949 /* Event ring control interrupt handler */
950 static void gsi_isr_evt_ctrl(struct gsi
*gsi
)
954 event_mask
= ioread32(gsi
->virt
+ GSI_CNTXT_SRC_EV_CH_IRQ_OFFSET
);
955 iowrite32(event_mask
, gsi
->virt
+ GSI_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET
);
958 u32 evt_ring_id
= __ffs(event_mask
);
959 struct gsi_evt_ring
*evt_ring
;
961 event_mask
^= BIT(evt_ring_id
);
963 evt_ring
= &gsi
->evt_ring
[evt_ring_id
];
964 evt_ring
->state
= gsi_evt_ring_state(gsi
, evt_ring_id
);
966 complete(&evt_ring
->completion
);
970 /* Global channel error interrupt handler */
972 gsi_isr_glob_chan_err(struct gsi
*gsi
, u32 err_ee
, u32 channel_id
, u32 code
)
974 if (code
== GSI_OUT_OF_RESOURCES_ERR
) {
975 dev_err(gsi
->dev
, "channel %u out of resources\n", channel_id
);
976 complete(&gsi
->channel
[channel_id
].completion
);
980 /* Report, but otherwise ignore all other error codes */
981 dev_err(gsi
->dev
, "channel %u global error ee 0x%08x code 0x%08x\n",
982 channel_id
, err_ee
, code
);
985 /* Global event error interrupt handler */
987 gsi_isr_glob_evt_err(struct gsi
*gsi
, u32 err_ee
, u32 evt_ring_id
, u32 code
)
989 if (code
== GSI_OUT_OF_RESOURCES_ERR
) {
990 struct gsi_evt_ring
*evt_ring
= &gsi
->evt_ring
[evt_ring_id
];
991 u32 channel_id
= gsi_channel_id(evt_ring
->channel
);
993 complete(&evt_ring
->completion
);
994 dev_err(gsi
->dev
, "evt_ring for channel %u out of resources\n",
999 /* Report, but otherwise ignore all other error codes */
1000 dev_err(gsi
->dev
, "event ring %u global error ee %u code 0x%08x\n",
1001 evt_ring_id
, err_ee
, code
);
1004 /* Global error interrupt handler */
1005 static void gsi_isr_glob_err(struct gsi
*gsi
)
1007 enum gsi_err_type type
;
1008 enum gsi_err_code code
;
1013 /* Get the logged error, then reinitialize the log */
1014 val
= ioread32(gsi
->virt
+ GSI_ERROR_LOG_OFFSET
);
1015 iowrite32(0, gsi
->virt
+ GSI_ERROR_LOG_OFFSET
);
1016 iowrite32(~0, gsi
->virt
+ GSI_ERROR_LOG_CLR_OFFSET
);
1018 ee
= u32_get_bits(val
, ERR_EE_FMASK
);
1019 which
= u32_get_bits(val
, ERR_VIRT_IDX_FMASK
);
1020 type
= u32_get_bits(val
, ERR_TYPE_FMASK
);
1021 code
= u32_get_bits(val
, ERR_CODE_FMASK
);
1023 if (type
== GSI_ERR_TYPE_CHAN
)
1024 gsi_isr_glob_chan_err(gsi
, ee
, which
, code
);
1025 else if (type
== GSI_ERR_TYPE_EVT
)
1026 gsi_isr_glob_evt_err(gsi
, ee
, which
, code
);
1027 else /* type GSI_ERR_TYPE_GLOB should be fatal */
1028 dev_err(gsi
->dev
, "unexpected global error 0x%08x\n", type
);
1031 /* Generic EE interrupt handler */
1032 static void gsi_isr_gp_int1(struct gsi
*gsi
)
1037 val
= ioread32(gsi
->virt
+ GSI_CNTXT_SCRATCH_0_OFFSET
);
1038 result
= u32_get_bits(val
, GENERIC_EE_RESULT_FMASK
);
1039 if (result
!= GENERIC_EE_SUCCESS_FVAL
)
1040 dev_err(gsi
->dev
, "global INT1 generic result %u\n", result
);
1042 complete(&gsi
->completion
);
1044 /* Inter-EE interrupt handler */
1045 static void gsi_isr_glob_ee(struct gsi
*gsi
)
1049 val
= ioread32(gsi
->virt
+ GSI_CNTXT_GLOB_IRQ_STTS_OFFSET
);
1051 if (val
& ERROR_INT_FMASK
)
1052 gsi_isr_glob_err(gsi
);
1054 iowrite32(val
, gsi
->virt
+ GSI_CNTXT_GLOB_IRQ_CLR_OFFSET
);
1056 val
&= ~ERROR_INT_FMASK
;
1058 if (val
& EN_GP_INT1_FMASK
) {
1059 val
^= EN_GP_INT1_FMASK
;
1060 gsi_isr_gp_int1(gsi
);
1064 dev_err(gsi
->dev
, "unexpected global interrupt 0x%08x\n", val
);
1067 /* I/O completion interrupt event */
1068 static void gsi_isr_ieob(struct gsi
*gsi
)
1072 event_mask
= ioread32(gsi
->virt
+ GSI_CNTXT_SRC_IEOB_IRQ_OFFSET
);
1073 gsi_isr_ieob_clear(gsi
, event_mask
);
1075 while (event_mask
) {
1076 u32 evt_ring_id
= __ffs(event_mask
);
1078 event_mask
^= BIT(evt_ring_id
);
1080 gsi_irq_ieob_disable(gsi
, evt_ring_id
);
1081 napi_schedule(&gsi
->evt_ring
[evt_ring_id
].channel
->napi
);
1085 /* General event interrupts represent serious problems, so report them */
1086 static void gsi_isr_general(struct gsi
*gsi
)
1088 struct device
*dev
= gsi
->dev
;
1091 val
= ioread32(gsi
->virt
+ GSI_CNTXT_GSI_IRQ_STTS_OFFSET
);
1092 iowrite32(val
, gsi
->virt
+ GSI_CNTXT_GSI_IRQ_CLR_OFFSET
);
1095 dev_err(dev
, "unexpected general interrupt 0x%08x\n", val
);
1099 * gsi_isr() - Top level GSI interrupt service routine
1100 * @irq: Interrupt number (ignored)
1101 * @dev_id: GSI pointer supplied to request_irq()
1103 * This is the main handler function registered for the GSI IRQ. Each type
1104 * of interrupt has a separate handler function that is called from here.
1106 static irqreturn_t
gsi_isr(int irq
, void *dev_id
)
1108 struct gsi
*gsi
= dev_id
;
1112 while ((intr_mask
= ioread32(gsi
->virt
+ GSI_CNTXT_TYPE_IRQ_OFFSET
))) {
1113 /* intr_mask contains bitmask of pending GSI interrupts */
1115 u32 gsi_intr
= BIT(__ffs(intr_mask
));
1117 intr_mask
^= gsi_intr
;
1121 gsi_isr_chan_ctrl(gsi
);
1124 gsi_isr_evt_ctrl(gsi
);
1127 gsi_isr_glob_ee(gsi
);
1133 gsi_isr_general(gsi
);
1137 "%s: unrecognized type 0x%08x\n",
1138 __func__
, gsi_intr
);
1141 } while (intr_mask
);
1143 if (++cnt
> GSI_ISR_MAX_ITER
) {
1144 dev_err(gsi
->dev
, "interrupt flood\n");
1152 /* Return the transaction associated with a transfer completion event */
1153 static struct gsi_trans
*gsi_event_trans(struct gsi_channel
*channel
,
1154 struct gsi_event
*event
)
1159 /* Event xfer_ptr records the TRE it's associated with */
1160 tre_offset
= le64_to_cpu(event
->xfer_ptr
) & GENMASK(31, 0);
1161 tre_index
= gsi_ring_index(&channel
->tre_ring
, tre_offset
);
1163 return gsi_channel_trans_mapped(channel
, tre_index
);
1167 * gsi_evt_ring_rx_update() - Record lengths of received data
1168 * @evt_ring: Event ring associated with channel that received packets
1169 * @index: Event index in ring reported by hardware
1171 * Events for RX channels contain the actual number of bytes received into
1172 * the buffer. Every event has a transaction associated with it, and here
1173 * we update transactions to record their actual received lengths.
1175 * This function is called whenever we learn that the GSI hardware has filled
1176 * new events since the last time we checked. The ring's index field tells
1177 * the first entry in need of processing. The index provided is the
1178 * first *unfilled* event in the ring (following the last filled one).
1180 * Events are sequential within the event ring, and transactions are
1181 * sequential within the transaction pool.
1183 * Note that @index always refers to an element *within* the event ring.
1185 static void gsi_evt_ring_rx_update(struct gsi_evt_ring
*evt_ring
, u32 index
)
1187 struct gsi_channel
*channel
= evt_ring
->channel
;
1188 struct gsi_ring
*ring
= &evt_ring
->ring
;
1189 struct gsi_trans_info
*trans_info
;
1190 struct gsi_event
*event_done
;
1191 struct gsi_event
*event
;
1192 struct gsi_trans
*trans
;
1197 trans_info
= &channel
->trans_info
;
1199 /* We'll start with the oldest un-processed event. RX channels
1200 * replenish receive buffers in single-TRE transactions, so we
1201 * can just map that event to its transaction. Transactions
1202 * associated with completion events are consecutive.
1204 old_index
= ring
->index
;
1205 event
= gsi_ring_virt(ring
, old_index
);
1206 trans
= gsi_event_trans(channel
, event
);
1208 /* Compute the number of events to process before we wrap,
1209 * and determine when we'll be done processing events.
1211 event_avail
= ring
->count
- old_index
% ring
->count
;
1212 event_done
= gsi_ring_virt(ring
, index
);
1214 trans
->len
= __le16_to_cpu(event
->len
);
1215 byte_count
+= trans
->len
;
1217 /* Move on to the next event and transaction */
1221 event
= gsi_ring_virt(ring
, 0);
1222 trans
= gsi_trans_pool_next(&trans_info
->pool
, trans
);
1223 } while (event
!= event_done
);
1225 /* We record RX bytes when they are received */
1226 channel
->byte_count
+= byte_count
;
1227 channel
->trans_count
++;
1230 /* Initialize a ring, including allocating DMA memory for its entries */
1231 static int gsi_ring_alloc(struct gsi
*gsi
, struct gsi_ring
*ring
, u32 count
)
1233 size_t size
= count
* GSI_RING_ELEMENT_SIZE
;
1234 struct device
*dev
= gsi
->dev
;
1237 /* Hardware requires a 2^n ring size, with alignment equal to size */
1238 ring
->virt
= dma_alloc_coherent(dev
, size
, &addr
, GFP_KERNEL
);
1239 if (ring
->virt
&& addr
% size
) {
1240 dma_free_coherent(dev
, size
, ring
->virt
, ring
->addr
);
1241 dev_err(dev
, "unable to alloc 0x%zx-aligned ring buffer\n",
1243 return -EINVAL
; /* Not a good error value, but distinct */
1244 } else if (!ring
->virt
) {
1248 ring
->count
= count
;
1253 /* Free a previously-allocated ring */
1254 static void gsi_ring_free(struct gsi
*gsi
, struct gsi_ring
*ring
)
1256 size_t size
= ring
->count
* GSI_RING_ELEMENT_SIZE
;
1258 dma_free_coherent(gsi
->dev
, size
, ring
->virt
, ring
->addr
);
1261 /* Allocate an available event ring id */
1262 static int gsi_evt_ring_id_alloc(struct gsi
*gsi
)
1266 if (gsi
->event_bitmap
== ~0U) {
1267 dev_err(gsi
->dev
, "event rings exhausted\n");
1271 evt_ring_id
= ffz(gsi
->event_bitmap
);
1272 gsi
->event_bitmap
|= BIT(evt_ring_id
);
1274 return (int)evt_ring_id
;
1277 /* Free a previously-allocated event ring id */
1278 static void gsi_evt_ring_id_free(struct gsi
*gsi
, u32 evt_ring_id
)
1280 gsi
->event_bitmap
&= ~BIT(evt_ring_id
);
1283 /* Ring a channel doorbell, reporting the first un-filled entry */
1284 void gsi_channel_doorbell(struct gsi_channel
*channel
)
1286 struct gsi_ring
*tre_ring
= &channel
->tre_ring
;
1287 u32 channel_id
= gsi_channel_id(channel
);
1288 struct gsi
*gsi
= channel
->gsi
;
1291 /* Note: index *must* be used modulo the ring count here */
1292 val
= gsi_ring_addr(tre_ring
, tre_ring
->index
% tre_ring
->count
);
1293 iowrite32(val
, gsi
->virt
+ GSI_CH_C_DOORBELL_0_OFFSET(channel_id
));
1296 /* Consult hardware, move any newly completed transactions to completed list */
1297 static void gsi_channel_update(struct gsi_channel
*channel
)
1299 u32 evt_ring_id
= channel
->evt_ring_id
;
1300 struct gsi
*gsi
= channel
->gsi
;
1301 struct gsi_evt_ring
*evt_ring
;
1302 struct gsi_trans
*trans
;
1303 struct gsi_ring
*ring
;
1307 evt_ring
= &gsi
->evt_ring
[evt_ring_id
];
1308 ring
= &evt_ring
->ring
;
1310 /* See if there's anything new to process; if not, we're done. Note
1311 * that index always refers to an entry *within* the event ring.
1313 offset
= GSI_EV_CH_E_CNTXT_4_OFFSET(evt_ring_id
);
1314 index
= gsi_ring_index(ring
, ioread32(gsi
->virt
+ offset
));
1315 if (index
== ring
->index
% ring
->count
)
1318 /* Get the transaction for the latest completed event. Take a
1319 * reference to keep it from completing before we give the events
1320 * for this and previous transactions back to the hardware.
1322 trans
= gsi_event_trans(channel
, gsi_ring_virt(ring
, index
- 1));
1323 refcount_inc(&trans
->refcount
);
1325 /* For RX channels, update each completed transaction with the number
1326 * of bytes that were actually received. For TX channels, report
1327 * the number of transactions and bytes this completion represents
1328 * up the network stack.
1330 if (channel
->toward_ipa
)
1331 gsi_channel_tx_update(channel
, trans
);
1333 gsi_evt_ring_rx_update(evt_ring
, index
);
1335 gsi_trans_move_complete(trans
);
1337 /* Tell the hardware we've handled these events */
1338 gsi_evt_ring_doorbell(channel
->gsi
, channel
->evt_ring_id
, index
);
1340 gsi_trans_free(trans
);
1344 * gsi_channel_poll_one() - Return a single completed transaction on a channel
1345 * @channel: Channel to be polled
1347 * @Return: Transaction pointer, or null if none are available
1349 * This function returns the first entry on a channel's completed transaction
1350 * list. If that list is empty, the hardware is consulted to determine
1351 * whether any new transactions have completed. If so, they're moved to the
1352 * completed list and the new first entry is returned. If there are no more
1353 * completed transactions, a null pointer is returned.
1355 static struct gsi_trans
*gsi_channel_poll_one(struct gsi_channel
*channel
)
1357 struct gsi_trans
*trans
;
1359 /* Get the first transaction from the completed list */
1360 trans
= gsi_channel_trans_complete(channel
);
1362 /* List is empty; see if there's more to do */
1363 gsi_channel_update(channel
);
1364 trans
= gsi_channel_trans_complete(channel
);
1368 gsi_trans_move_polled(trans
);
1374 * gsi_channel_poll() - NAPI poll function for a channel
1375 * @napi: NAPI structure for the channel
1376 * @budget: Budget supplied by NAPI core
1378 * @Return: Number of items polled (<= budget)
1380 * Single transactions completed by hardware are polled until either
1381 * the budget is exhausted, or there are no more. Each transaction
1382 * polled is passed to gsi_trans_complete(), to perform remaining
1383 * completion processing and retire/free the transaction.
1385 static int gsi_channel_poll(struct napi_struct
*napi
, int budget
)
1387 struct gsi_channel
*channel
;
1390 channel
= container_of(napi
, struct gsi_channel
, napi
);
1391 while (count
< budget
) {
1392 struct gsi_trans
*trans
;
1394 trans
= gsi_channel_poll_one(channel
);
1397 gsi_trans_complete(trans
);
1400 if (count
< budget
) {
1401 napi_complete(&channel
->napi
);
1402 gsi_irq_ieob_enable(channel
->gsi
, channel
->evt_ring_id
);
1408 /* The event bitmap represents which event ids are available for allocation.
1409 * Set bits are not available, clear bits can be used. This function
1410 * initializes the map so all events supported by the hardware are available,
1411 * then precludes any reserved events from being allocated.
1413 static u32
gsi_event_bitmap_init(u32 evt_ring_max
)
1415 u32 event_bitmap
= GENMASK(BITS_PER_LONG
- 1, evt_ring_max
);
1417 event_bitmap
|= GENMASK(GSI_MHI_EVENT_ID_END
, GSI_MHI_EVENT_ID_START
);
1419 return event_bitmap
;
1422 /* Setup function for event rings */
1423 static void gsi_evt_ring_setup(struct gsi
*gsi
)
1428 /* Inverse of gsi_evt_ring_setup() */
1429 static void gsi_evt_ring_teardown(struct gsi
*gsi
)
1434 /* Setup function for a single channel */
1435 static int gsi_channel_setup_one(struct gsi
*gsi
, u32 channel_id
,
1438 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
1439 u32 evt_ring_id
= channel
->evt_ring_id
;
1443 return 0; /* Ignore uninitialized channels */
1445 ret
= gsi_evt_ring_alloc_command(gsi
, evt_ring_id
);
1449 gsi_evt_ring_program(gsi
, evt_ring_id
);
1451 ret
= gsi_channel_alloc_command(gsi
, channel_id
);
1453 goto err_evt_ring_de_alloc
;
1455 gsi_channel_program(channel
, db_enable
);
1457 if (channel
->toward_ipa
)
1458 netif_tx_napi_add(&gsi
->dummy_dev
, &channel
->napi
,
1459 gsi_channel_poll
, NAPI_POLL_WEIGHT
);
1461 netif_napi_add(&gsi
->dummy_dev
, &channel
->napi
,
1462 gsi_channel_poll
, NAPI_POLL_WEIGHT
);
1466 err_evt_ring_de_alloc
:
1467 /* We've done nothing with the event ring yet so don't reset */
1468 gsi_evt_ring_de_alloc_command(gsi
, evt_ring_id
);
1473 /* Inverse of gsi_channel_setup_one() */
1474 static void gsi_channel_teardown_one(struct gsi
*gsi
, u32 channel_id
)
1476 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
1477 u32 evt_ring_id
= channel
->evt_ring_id
;
1480 return; /* Ignore uninitialized channels */
1482 netif_napi_del(&channel
->napi
);
1484 gsi_channel_deprogram(channel
);
1485 gsi_channel_de_alloc_command(gsi
, channel_id
);
1486 gsi_evt_ring_reset_command(gsi
, evt_ring_id
);
1487 gsi_evt_ring_de_alloc_command(gsi
, evt_ring_id
);
1490 static int gsi_generic_command(struct gsi
*gsi
, u32 channel_id
,
1491 enum gsi_generic_cmd_opcode opcode
)
1493 struct completion
*completion
= &gsi
->completion
;
1496 val
= u32_encode_bits(opcode
, GENERIC_OPCODE_FMASK
);
1497 val
|= u32_encode_bits(channel_id
, GENERIC_CHID_FMASK
);
1498 val
|= u32_encode_bits(GSI_EE_MODEM
, GENERIC_EE_FMASK
);
1500 if (gsi_command(gsi
, GSI_GENERIC_CMD_OFFSET
, val
, completion
))
1501 return 0; /* Success! */
1503 dev_err(gsi
->dev
, "GSI generic command %u to channel %u timed out\n",
1504 opcode
, channel_id
);
1509 static int gsi_modem_channel_alloc(struct gsi
*gsi
, u32 channel_id
)
1511 return gsi_generic_command(gsi
, channel_id
,
1512 GSI_GENERIC_ALLOCATE_CHANNEL
);
1515 static void gsi_modem_channel_halt(struct gsi
*gsi
, u32 channel_id
)
1519 ret
= gsi_generic_command(gsi
, channel_id
, GSI_GENERIC_HALT_CHANNEL
);
1521 dev_err(gsi
->dev
, "error %d halting modem channel %u\n",
1525 /* Setup function for channels */
1526 static int gsi_channel_setup(struct gsi
*gsi
, bool db_enable
)
1532 gsi_evt_ring_setup(gsi
);
1533 gsi_irq_enable(gsi
);
1535 mutex_lock(&gsi
->mutex
);
1538 ret
= gsi_channel_setup_one(gsi
, channel_id
, db_enable
);
1541 } while (++channel_id
< gsi
->channel_count
);
1543 /* Make sure no channels were defined that hardware does not support */
1544 while (channel_id
< GSI_CHANNEL_COUNT_MAX
) {
1545 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
++];
1548 continue; /* Ignore uninitialized channels */
1550 dev_err(gsi
->dev
, "channel %u not supported by hardware\n",
1552 channel_id
= gsi
->channel_count
;
1556 /* Allocate modem channels if necessary */
1557 mask
= gsi
->modem_channel_bitmap
;
1559 u32 modem_channel_id
= __ffs(mask
);
1561 ret
= gsi_modem_channel_alloc(gsi
, modem_channel_id
);
1563 goto err_unwind_modem
;
1565 /* Clear bit from mask only after success (for unwind) */
1566 mask
^= BIT(modem_channel_id
);
1569 mutex_unlock(&gsi
->mutex
);
1574 /* Compute which modem channels need to be deallocated */
1575 mask
^= gsi
->modem_channel_bitmap
;
1577 u32 channel_id
= __fls(mask
);
1579 mask
^= BIT(channel_id
);
1581 gsi_modem_channel_halt(gsi
, channel_id
);
1585 while (channel_id
--)
1586 gsi_channel_teardown_one(gsi
, channel_id
);
1588 mutex_unlock(&gsi
->mutex
);
1590 gsi_irq_disable(gsi
);
1591 gsi_evt_ring_teardown(gsi
);
1596 /* Inverse of gsi_channel_setup() */
1597 static void gsi_channel_teardown(struct gsi
*gsi
)
1599 u32 mask
= gsi
->modem_channel_bitmap
;
1602 mutex_lock(&gsi
->mutex
);
1605 u32 channel_id
= __fls(mask
);
1607 mask
^= BIT(channel_id
);
1609 gsi_modem_channel_halt(gsi
, channel_id
);
1612 channel_id
= gsi
->channel_count
- 1;
1614 gsi_channel_teardown_one(gsi
, channel_id
);
1615 while (channel_id
--);
1617 mutex_unlock(&gsi
->mutex
);
1619 gsi_irq_disable(gsi
);
1620 gsi_evt_ring_teardown(gsi
);
1623 /* Setup function for GSI. GSI firmware must be loaded and initialized */
1624 int gsi_setup(struct gsi
*gsi
, bool db_enable
)
1628 /* Here is where we first touch the GSI hardware */
1629 val
= ioread32(gsi
->virt
+ GSI_GSI_STATUS_OFFSET
);
1630 if (!(val
& ENABLED_FMASK
)) {
1631 dev_err(gsi
->dev
, "GSI has not been enabled\n");
1635 val
= ioread32(gsi
->virt
+ GSI_GSI_HW_PARAM_2_OFFSET
);
1637 gsi
->channel_count
= u32_get_bits(val
, NUM_CH_PER_EE_FMASK
);
1638 if (!gsi
->channel_count
) {
1639 dev_err(gsi
->dev
, "GSI reports zero channels supported\n");
1642 if (gsi
->channel_count
> GSI_CHANNEL_COUNT_MAX
) {
1644 "limiting to %u channels (hardware supports %u)\n",
1645 GSI_CHANNEL_COUNT_MAX
, gsi
->channel_count
);
1646 gsi
->channel_count
= GSI_CHANNEL_COUNT_MAX
;
1649 gsi
->evt_ring_count
= u32_get_bits(val
, NUM_EV_PER_EE_FMASK
);
1650 if (!gsi
->evt_ring_count
) {
1651 dev_err(gsi
->dev
, "GSI reports zero event rings supported\n");
1654 if (gsi
->evt_ring_count
> GSI_EVT_RING_COUNT_MAX
) {
1656 "limiting to %u event rings (hardware supports %u)\n",
1657 GSI_EVT_RING_COUNT_MAX
, gsi
->evt_ring_count
);
1658 gsi
->evt_ring_count
= GSI_EVT_RING_COUNT_MAX
;
1661 /* Initialize the error log */
1662 iowrite32(0, gsi
->virt
+ GSI_ERROR_LOG_OFFSET
);
1664 /* Writing 1 indicates IRQ interrupts; 0 would be MSI */
1665 iowrite32(1, gsi
->virt
+ GSI_CNTXT_INTSET_OFFSET
);
1667 return gsi_channel_setup(gsi
, db_enable
);
1670 /* Inverse of gsi_setup() */
1671 void gsi_teardown(struct gsi
*gsi
)
1673 gsi_channel_teardown(gsi
);
1676 /* Initialize a channel's event ring */
1677 static int gsi_channel_evt_ring_init(struct gsi_channel
*channel
)
1679 struct gsi
*gsi
= channel
->gsi
;
1680 struct gsi_evt_ring
*evt_ring
;
1683 ret
= gsi_evt_ring_id_alloc(gsi
);
1686 channel
->evt_ring_id
= ret
;
1688 evt_ring
= &gsi
->evt_ring
[channel
->evt_ring_id
];
1689 evt_ring
->channel
= channel
;
1691 ret
= gsi_ring_alloc(gsi
, &evt_ring
->ring
, channel
->event_count
);
1693 return 0; /* Success! */
1695 dev_err(gsi
->dev
, "error %d allocating channel %u event ring\n",
1696 ret
, gsi_channel_id(channel
));
1698 gsi_evt_ring_id_free(gsi
, channel
->evt_ring_id
);
1703 /* Inverse of gsi_channel_evt_ring_init() */
1704 static void gsi_channel_evt_ring_exit(struct gsi_channel
*channel
)
1706 u32 evt_ring_id
= channel
->evt_ring_id
;
1707 struct gsi
*gsi
= channel
->gsi
;
1708 struct gsi_evt_ring
*evt_ring
;
1710 evt_ring
= &gsi
->evt_ring
[evt_ring_id
];
1711 gsi_ring_free(gsi
, &evt_ring
->ring
);
1712 gsi_evt_ring_id_free(gsi
, evt_ring_id
);
1715 /* Init function for event rings */
1716 static void gsi_evt_ring_init(struct gsi
*gsi
)
1718 u32 evt_ring_id
= 0;
1720 gsi
->event_bitmap
= gsi_event_bitmap_init(GSI_EVT_RING_COUNT_MAX
);
1721 gsi
->event_enable_bitmap
= 0;
1723 init_completion(&gsi
->evt_ring
[evt_ring_id
].completion
);
1724 while (++evt_ring_id
< GSI_EVT_RING_COUNT_MAX
);
1727 /* Inverse of gsi_evt_ring_init() */
1728 static void gsi_evt_ring_exit(struct gsi
*gsi
)
1733 static bool gsi_channel_data_valid(struct gsi
*gsi
,
1734 const struct ipa_gsi_endpoint_data
*data
)
1736 #ifdef IPA_VALIDATION
1737 u32 channel_id
= data
->channel_id
;
1738 struct device
*dev
= gsi
->dev
;
1740 /* Make sure channel ids are in the range driver supports */
1741 if (channel_id
>= GSI_CHANNEL_COUNT_MAX
) {
1742 dev_err(dev
, "bad channel id %u (must be less than %u)\n",
1743 channel_id
, GSI_CHANNEL_COUNT_MAX
);
1747 if (data
->ee_id
!= GSI_EE_AP
&& data
->ee_id
!= GSI_EE_MODEM
) {
1748 dev_err(dev
, "bad EE id %u (AP or modem)\n", data
->ee_id
);
1752 if (!data
->channel
.tlv_count
||
1753 data
->channel
.tlv_count
> GSI_TLV_MAX
) {
1754 dev_err(dev
, "channel %u bad tlv_count %u (must be 1..%u)\n",
1755 channel_id
, data
->channel
.tlv_count
, GSI_TLV_MAX
);
1759 /* We have to allow at least one maximally-sized transaction to
1760 * be outstanding (which would use tlv_count TREs). Given how
1761 * gsi_channel_tre_max() is computed, tre_count has to be almost
1762 * twice the TLV FIFO size to satisfy this requirement.
1764 if (data
->channel
.tre_count
< 2 * data
->channel
.tlv_count
- 1) {
1765 dev_err(dev
, "channel %u TLV count %u exceeds TRE count %u\n",
1766 channel_id
, data
->channel
.tlv_count
,
1767 data
->channel
.tre_count
);
1771 if (!is_power_of_2(data
->channel
.tre_count
)) {
1772 dev_err(dev
, "channel %u bad tre_count %u (not power of 2)\n",
1773 channel_id
, data
->channel
.tre_count
);
1777 if (!is_power_of_2(data
->channel
.event_count
)) {
1778 dev_err(dev
, "channel %u bad event_count %u (not power of 2)\n",
1779 channel_id
, data
->channel
.event_count
);
1782 #endif /* IPA_VALIDATION */
1787 /* Init function for a single channel */
1788 static int gsi_channel_init_one(struct gsi
*gsi
,
1789 const struct ipa_gsi_endpoint_data
*data
,
1790 bool command
, bool prefetch
)
1792 struct gsi_channel
*channel
;
1796 if (!gsi_channel_data_valid(gsi
, data
))
1799 /* Worst case we need an event for every outstanding TRE */
1800 if (data
->channel
.tre_count
> data
->channel
.event_count
) {
1801 dev_warn(gsi
->dev
, "channel %u limited to %u TREs\n",
1802 data
->channel_id
, data
->channel
.tre_count
);
1803 tre_count
= data
->channel
.event_count
;
1805 tre_count
= data
->channel
.tre_count
;
1808 channel
= &gsi
->channel
[data
->channel_id
];
1809 memset(channel
, 0, sizeof(*channel
));
1812 channel
->toward_ipa
= data
->toward_ipa
;
1813 channel
->command
= command
;
1814 channel
->use_prefetch
= command
&& prefetch
;
1815 channel
->tlv_count
= data
->channel
.tlv_count
;
1816 channel
->tre_count
= tre_count
;
1817 channel
->event_count
= data
->channel
.event_count
;
1818 init_completion(&channel
->completion
);
1820 ret
= gsi_channel_evt_ring_init(channel
);
1824 ret
= gsi_ring_alloc(gsi
, &channel
->tre_ring
, data
->channel
.tre_count
);
1826 dev_err(gsi
->dev
, "error %d allocating channel %u ring\n",
1827 ret
, data
->channel_id
);
1828 goto err_channel_evt_ring_exit
;
1831 ret
= gsi_channel_trans_init(gsi
, data
->channel_id
);
1836 u32 tre_max
= gsi_channel_tre_max(gsi
, data
->channel_id
);
1838 ret
= ipa_cmd_pool_init(channel
, tre_max
);
1841 return 0; /* Success! */
1843 gsi_channel_trans_exit(channel
);
1845 gsi_ring_free(gsi
, &channel
->tre_ring
);
1846 err_channel_evt_ring_exit
:
1847 gsi_channel_evt_ring_exit(channel
);
1849 channel
->gsi
= NULL
; /* Mark it not (fully) initialized */
1854 /* Inverse of gsi_channel_init_one() */
1855 static void gsi_channel_exit_one(struct gsi_channel
*channel
)
1858 return; /* Ignore uninitialized channels */
1860 if (channel
->command
)
1861 ipa_cmd_pool_exit(channel
);
1862 gsi_channel_trans_exit(channel
);
1863 gsi_ring_free(channel
->gsi
, &channel
->tre_ring
);
1864 gsi_channel_evt_ring_exit(channel
);
1867 /* Init function for channels */
1868 static int gsi_channel_init(struct gsi
*gsi
, bool prefetch
, u32 count
,
1869 const struct ipa_gsi_endpoint_data
*data
,
1875 gsi_evt_ring_init(gsi
);
1877 /* The endpoint data array is indexed by endpoint name */
1878 for (i
= 0; i
< count
; i
++) {
1879 bool command
= i
== IPA_ENDPOINT_AP_COMMAND_TX
;
1881 if (ipa_gsi_endpoint_data_empty(&data
[i
]))
1882 continue; /* Skip over empty slots */
1884 /* Mark modem channels to be allocated (hardware workaround) */
1885 if (data
[i
].ee_id
== GSI_EE_MODEM
) {
1887 gsi
->modem_channel_bitmap
|=
1888 BIT(data
[i
].channel_id
);
1892 ret
= gsi_channel_init_one(gsi
, &data
[i
], command
, prefetch
);
1901 if (ipa_gsi_endpoint_data_empty(&data
[i
]))
1903 if (modem_alloc
&& data
[i
].ee_id
== GSI_EE_MODEM
) {
1904 gsi
->modem_channel_bitmap
&= ~BIT(data
[i
].channel_id
);
1907 gsi_channel_exit_one(&gsi
->channel
[data
->channel_id
]);
1909 gsi_evt_ring_exit(gsi
);
1914 /* Inverse of gsi_channel_init() */
1915 static void gsi_channel_exit(struct gsi
*gsi
)
1917 u32 channel_id
= GSI_CHANNEL_COUNT_MAX
- 1;
1920 gsi_channel_exit_one(&gsi
->channel
[channel_id
]);
1921 while (channel_id
--);
1922 gsi
->modem_channel_bitmap
= 0;
1924 gsi_evt_ring_exit(gsi
);
1927 /* Init function for GSI. GSI hardware does not need to be "ready" */
1928 int gsi_init(struct gsi
*gsi
, struct platform_device
*pdev
, bool prefetch
,
1929 u32 count
, const struct ipa_gsi_endpoint_data
*data
,
1932 struct resource
*res
;
1933 resource_size_t size
;
1937 gsi_validate_build();
1939 gsi
->dev
= &pdev
->dev
;
1941 /* The GSI layer performs NAPI on all endpoints. NAPI requires a
1942 * network device structure, but the GSI layer does not have one,
1943 * so we must create a dummy network device for this purpose.
1945 init_dummy_netdev(&gsi
->dummy_dev
);
1947 /* Get the GSI IRQ and request for it to wake the system */
1948 ret
= platform_get_irq_byname(pdev
, "gsi");
1951 "DT error %d getting \"gsi\" IRQ property\n", ret
);
1952 return ret
? : -EINVAL
;
1956 ret
= request_irq(irq
, gsi_isr
, 0, "gsi", gsi
);
1958 dev_err(gsi
->dev
, "error %d requesting \"gsi\" IRQ\n", ret
);
1963 ret
= enable_irq_wake(gsi
->irq
);
1965 dev_warn(gsi
->dev
, "error %d enabling gsi wake irq\n", ret
);
1966 gsi
->irq_wake_enabled
= !ret
;
1968 /* Get GSI memory range and map it */
1969 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "gsi");
1972 "DT error getting \"gsi\" memory property\n");
1974 goto err_disable_irq_wake
;
1977 size
= resource_size(res
);
1978 if (res
->start
> U32_MAX
|| size
> U32_MAX
- res
->start
) {
1979 dev_err(gsi
->dev
, "DT memory resource \"gsi\" out of range\n");
1981 goto err_disable_irq_wake
;
1984 gsi
->virt
= ioremap(res
->start
, size
);
1986 dev_err(gsi
->dev
, "unable to remap \"gsi\" memory\n");
1988 goto err_disable_irq_wake
;
1991 ret
= gsi_channel_init(gsi
, prefetch
, count
, data
, modem_alloc
);
1995 mutex_init(&gsi
->mutex
);
1996 init_completion(&gsi
->completion
);
2002 err_disable_irq_wake
:
2003 if (gsi
->irq_wake_enabled
)
2004 (void)disable_irq_wake(gsi
->irq
);
2005 free_irq(gsi
->irq
, gsi
);
2010 /* Inverse of gsi_init() */
2011 void gsi_exit(struct gsi
*gsi
)
2013 mutex_destroy(&gsi
->mutex
);
2014 gsi_channel_exit(gsi
);
2015 if (gsi
->irq_wake_enabled
)
2016 (void)disable_irq_wake(gsi
->irq
);
2017 free_irq(gsi
->irq
, gsi
);
2021 /* The maximum number of outstanding TREs on a channel. This limits
2022 * a channel's maximum number of transactions outstanding (worst case
2023 * is one TRE per transaction).
2025 * The absolute limit is the number of TREs in the channel's TRE ring,
2026 * and in theory we should be able use all of them. But in practice,
2027 * doing that led to the hardware reporting exhaustion of event ring
2028 * slots for writing completion information. So the hardware limit
2029 * would be (tre_count - 1).
2031 * We reduce it a bit further though. Transaction resource pools are
2032 * sized to be a little larger than this maximum, to allow resource
2033 * allocations to always be contiguous. The number of entries in a
2034 * TRE ring buffer is a power of 2, and the extra resources in a pool
2035 * tends to nearly double the memory allocated for it. Reducing the
2036 * maximum number of outstanding TREs allows the number of entries in
2037 * a pool to avoid crossing that power-of-2 boundary, and this can
2038 * substantially reduce pool memory requirements. The number we
2039 * reduce it by matches the number added in gsi_trans_pool_init().
2041 u32
gsi_channel_tre_max(struct gsi
*gsi
, u32 channel_id
)
2043 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
2045 /* Hardware limit is channel->tre_count - 1 */
2046 return channel
->tre_count
- (channel
->tlv_count
- 1);
2049 /* Returns the maximum number of TREs in a single transaction for a channel */
2050 u32
gsi_channel_trans_tre_max(struct gsi
*gsi
, u32 channel_id
)
2052 struct gsi_channel
*channel
= &gsi
->channel
[channel_id
];
2054 return channel
->tlv_count
;