1 // SPDX-License-Identifier: GPL-2.0
3 * Linux for s390 qdio support, buffer handling, qdio API and module support.
5 * Copyright IBM Corp. 2000, 2008
6 * Author(s): Utz Bacher <utz.bacher@de.ibm.com>
7 * Jan Glauber <jang@linux.vnet.ibm.com>
8 * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com>
10 #include <linux/module.h>
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/delay.h>
14 #include <linux/gfp.h>
16 #include <linux/atomic.h>
17 #include <asm/debug.h>
25 #include "qdio_debug.h"
27 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\
28 "Jan Glauber <jang@linux.vnet.ibm.com>");
29 MODULE_DESCRIPTION("QDIO base support");
30 MODULE_LICENSE("GPL");
32 static inline int do_siga_sync(unsigned long schid
,
33 unsigned long out_mask
, unsigned long in_mask
,
47 : [fc
] "d" (fc
), [schid
] "d" (schid
),
48 [out
] "d" (out_mask
), [in
] "d" (in_mask
)
49 : "cc", "0", "1", "2", "3");
53 static inline int do_siga_input(unsigned long schid
, unsigned long mask
,
66 : [fc
] "d" (fc
), [schid
] "d" (schid
), [mask
] "d" (mask
)
67 : "cc", "0", "1", "2");
72 * do_siga_output - perform SIGA-w/wt function
73 * @schid: subchannel id or in case of QEBSM the subchannel token
74 * @mask: which output queues to process
75 * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer
76 * @fc: function code to perform
77 * @aob: asynchronous operation block
79 * Returns condition code.
80 * Note: For IQDC unicast queues only the highest priority queue is processed.
82 static inline int do_siga_output(unsigned long schid
, unsigned long mask
,
83 unsigned int *bb
, unsigned long fc
,
97 : [cc
] "=&d" (cc
), [fc
] "+&d" (fc
)
98 : [schid
] "d" (schid
), [mask
] "d" (mask
), [aob
] "d" (aob
)
99 : "cc", "0", "1", "2", "3");
105 * qdio_do_eqbs - extract buffer states for QEBSM
106 * @q: queue to manipulate
107 * @state: state of the extracted buffers
108 * @start: buffer number to start at
109 * @count: count of buffers to examine
110 * @auto_ack: automatically acknowledge buffers
112 * Returns the number of successfully extracted equal buffer states.
113 * Stops processing if a state is different from the last buffers state.
115 static int qdio_do_eqbs(struct qdio_q
*q
, unsigned char *state
,
116 int start
, int count
, int auto_ack
)
118 int tmp_count
= count
, tmp_start
= start
, nr
= q
->nr
;
119 unsigned int ccq
= 0;
124 nr
+= q
->irq_ptr
->nr_input_qs
;
126 ccq
= do_eqbs(q
->irq_ptr
->sch_token
, state
, nr
, &tmp_start
, &tmp_count
,
132 /* all done, or next buffer state different */
133 return count
- tmp_count
;
135 /* not all buffers processed */
136 qperf_inc(q
, eqbs_partial
);
137 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "EQBS part:%02x",
139 return count
- tmp_count
;
141 /* no buffer processed */
142 DBF_DEV_EVENT(DBF_WARN
, q
->irq_ptr
, "EQBS again:%2d", ccq
);
145 DBF_ERROR("%4x ccq:%3d", SCH_NO(q
), ccq
);
146 DBF_ERROR("%4x EQBS ERROR", SCH_NO(q
));
147 DBF_ERROR("%3d%3d%2d", count
, tmp_count
, nr
);
148 q
->handler(q
->irq_ptr
->cdev
, QDIO_ERROR_GET_BUF_STATE
, q
->nr
,
149 q
->first_to_check
, count
, q
->irq_ptr
->int_parm
);
155 * qdio_do_sqbs - set buffer states for QEBSM
156 * @q: queue to manipulate
157 * @state: new state of the buffers
158 * @start: first buffer number to change
159 * @count: how many buffers to change
161 * Returns the number of successfully changed buffers.
162 * Does retrying until the specified count of buffer states is set or an
165 static int qdio_do_sqbs(struct qdio_q
*q
, unsigned char state
, int start
,
168 unsigned int ccq
= 0;
169 int tmp_count
= count
, tmp_start
= start
;
177 nr
+= q
->irq_ptr
->nr_input_qs
;
179 ccq
= do_sqbs(q
->irq_ptr
->sch_token
, state
, nr
, &tmp_start
, &tmp_count
);
184 /* all done, or active buffer adapter-owned */
185 WARN_ON_ONCE(tmp_count
);
186 return count
- tmp_count
;
188 /* not all buffers processed */
189 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "SQBS again:%2d", ccq
);
190 qperf_inc(q
, sqbs_partial
);
193 DBF_ERROR("%4x ccq:%3d", SCH_NO(q
), ccq
);
194 DBF_ERROR("%4x SQBS ERROR", SCH_NO(q
));
195 DBF_ERROR("%3d%3d%2d", count
, tmp_count
, nr
);
196 q
->handler(q
->irq_ptr
->cdev
, QDIO_ERROR_SET_BUF_STATE
, q
->nr
,
197 q
->first_to_check
, count
, q
->irq_ptr
->int_parm
);
203 * Returns number of examined buffers and their common state in *state.
204 * Requested number of buffers-to-examine must be > 0.
206 static inline int get_buf_states(struct qdio_q
*q
, unsigned int bufnr
,
207 unsigned char *state
, unsigned int count
,
210 unsigned char __state
= 0;
214 return qdio_do_eqbs(q
, state
, bufnr
, count
, auto_ack
);
216 /* get initial state: */
217 __state
= q
->slsb
.val
[bufnr
];
219 /* Bail out early if there is no work on the queue: */
220 if (__state
& SLSB_OWNER_CU
)
223 for (; i
< count
; i
++) {
224 bufnr
= next_buf(bufnr
);
226 /* stop if next state differs from initial state: */
227 if (q
->slsb
.val
[bufnr
] != __state
)
236 static inline int get_buf_state(struct qdio_q
*q
, unsigned int bufnr
,
237 unsigned char *state
, int auto_ack
)
239 return get_buf_states(q
, bufnr
, state
, 1, auto_ack
);
242 /* wrap-around safe setting of slsb states, returns number of changed buffers */
243 static inline int set_buf_states(struct qdio_q
*q
, int bufnr
,
244 unsigned char state
, int count
)
249 return qdio_do_sqbs(q
, state
, bufnr
, count
);
251 /* Ensure that all preceding changes to the SBALs are visible: */
254 for (i
= 0; i
< count
; i
++) {
255 WRITE_ONCE(q
->slsb
.val
[bufnr
], state
);
256 bufnr
= next_buf(bufnr
);
259 /* Make our SLSB changes visible: */
265 static inline int set_buf_state(struct qdio_q
*q
, int bufnr
,
268 return set_buf_states(q
, bufnr
, state
, 1);
271 /* set slsb states to initial state */
272 static void qdio_init_buf_states(struct qdio_irq
*irq_ptr
)
277 for_each_input_queue(irq_ptr
, q
, i
)
278 set_buf_states(q
, 0, SLSB_P_INPUT_NOT_INIT
,
279 QDIO_MAX_BUFFERS_PER_Q
);
280 for_each_output_queue(irq_ptr
, q
, i
)
281 set_buf_states(q
, 0, SLSB_P_OUTPUT_NOT_INIT
,
282 QDIO_MAX_BUFFERS_PER_Q
);
285 static inline int qdio_siga_sync(struct qdio_q
*q
, unsigned int output
,
288 unsigned long schid
= *((u32
*) &q
->irq_ptr
->schid
);
289 unsigned int fc
= QDIO_SIGA_SYNC
;
292 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "siga-s:%1d", q
->nr
);
293 qperf_inc(q
, siga_sync
);
296 schid
= q
->irq_ptr
->sch_token
;
297 fc
|= QDIO_SIGA_QEBSM_FLAG
;
300 cc
= do_siga_sync(schid
, output
, input
, fc
);
302 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q
), cc
);
303 return (cc
) ? -EIO
: 0;
306 static inline int qdio_sync_input_queue(struct qdio_q
*q
)
308 return qdio_siga_sync(q
, 0, q
->mask
);
311 static inline int qdio_sync_output_queue(struct qdio_q
*q
)
313 return qdio_siga_sync(q
, q
->mask
, 0);
316 static inline int qdio_siga_sync_q(struct qdio_q
*q
)
319 return qdio_sync_input_queue(q
);
321 return qdio_sync_output_queue(q
);
324 static int qdio_siga_output(struct qdio_q
*q
, unsigned int count
,
325 unsigned int *busy_bit
, unsigned long aob
)
327 unsigned long schid
= *((u32
*) &q
->irq_ptr
->schid
);
328 unsigned int fc
= QDIO_SIGA_WRITE
;
332 if (queue_type(q
) == QDIO_IQDIO_QFMT
&& !multicast_outbound(q
)) {
334 fc
= QDIO_SIGA_WRITEM
;
336 fc
= QDIO_SIGA_WRITEQ
;
340 schid
= q
->irq_ptr
->sch_token
;
341 fc
|= QDIO_SIGA_QEBSM_FLAG
;
344 cc
= do_siga_output(schid
, q
->mask
, busy_bit
, fc
, aob
);
346 /* hipersocket busy condition */
347 if (unlikely(*busy_bit
)) {
351 start_time
= get_tod_clock_fast();
354 if (get_tod_clock_fast() - start_time
< QDIO_BUSY_BIT_PATIENCE
)
358 DBF_DEV_EVENT(DBF_WARN
, q
->irq_ptr
,
359 "%4x cc2 BB1:%1d", SCH_NO(q
), q
->nr
);
360 DBF_DEV_EVENT(DBF_WARN
, q
->irq_ptr
, "count:%u", retries
);
365 static inline int qdio_siga_input(struct qdio_q
*q
)
367 unsigned long schid
= *((u32
*) &q
->irq_ptr
->schid
);
368 unsigned int fc
= QDIO_SIGA_READ
;
371 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "siga-r:%1d", q
->nr
);
372 qperf_inc(q
, siga_read
);
375 schid
= q
->irq_ptr
->sch_token
;
376 fc
|= QDIO_SIGA_QEBSM_FLAG
;
379 cc
= do_siga_input(schid
, q
->mask
, fc
);
381 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q
), cc
);
382 return (cc
) ? -EIO
: 0;
385 int debug_get_buf_state(struct qdio_q
*q
, unsigned int bufnr
,
386 unsigned char *state
)
388 if (qdio_need_siga_sync(q
->irq_ptr
))
390 return get_buf_state(q
, bufnr
, state
, 0);
393 static inline void qdio_stop_polling(struct qdio_q
*q
)
395 if (!q
->u
.in
.batch_count
)
398 qperf_inc(q
, stop_polling
);
400 /* show the card that we are not polling anymore */
401 set_buf_states(q
, q
->u
.in
.batch_start
, SLSB_P_INPUT_NOT_INIT
,
402 q
->u
.in
.batch_count
);
403 q
->u
.in
.batch_count
= 0;
406 static inline void account_sbals(struct qdio_q
*q
, unsigned int count
)
408 q
->q_stats
.nr_sbal_total
+= count
;
409 q
->q_stats
.nr_sbals
[ilog2(count
)]++;
412 static void process_buffer_error(struct qdio_q
*q
, unsigned int start
,
415 /* special handling for no target buffer empty */
416 if (queue_type(q
) == QDIO_IQDIO_QFMT
&& !q
->is_input_q
&&
417 q
->sbal
[start
]->element
[15].sflags
== 0x10) {
418 qperf_inc(q
, target_full
);
419 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "OUTFULL FTC:%02x", start
);
423 DBF_ERROR("%4x BUF ERROR", SCH_NO(q
));
424 DBF_ERROR((q
->is_input_q
) ? "IN:%2d" : "OUT:%2d", q
->nr
);
425 DBF_ERROR("FTC:%3d C:%3d", start
, count
);
426 DBF_ERROR("F14:%2x F15:%2x",
427 q
->sbal
[start
]->element
[14].sflags
,
428 q
->sbal
[start
]->element
[15].sflags
);
431 static inline void inbound_handle_work(struct qdio_q
*q
, unsigned int start
,
432 int count
, bool auto_ack
)
434 /* ACK the newest SBAL: */
436 set_buf_state(q
, add_buf(start
, count
- 1), SLSB_P_INPUT_ACK
);
438 if (!q
->u
.in
.batch_count
)
439 q
->u
.in
.batch_start
= start
;
440 q
->u
.in
.batch_count
+= count
;
443 static int get_inbound_buffer_frontier(struct qdio_q
*q
, unsigned int start
,
446 unsigned char state
= 0;
449 q
->timestamp
= get_tod_clock_fast();
451 count
= atomic_read(&q
->nr_buf_used
);
455 if (qdio_need_siga_sync(q
->irq_ptr
))
456 qdio_sync_input_queue(q
);
458 count
= get_buf_states(q
, start
, &state
, count
, 1);
463 case SLSB_P_INPUT_PRIMED
:
464 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "in prim:%1d %02x", q
->nr
,
467 inbound_handle_work(q
, start
, count
, is_qebsm(q
));
468 if (atomic_sub_return(count
, &q
->nr_buf_used
) == 0)
469 qperf_inc(q
, inbound_queue_full
);
470 if (q
->irq_ptr
->perf_stat_enabled
)
471 account_sbals(q
, count
);
473 case SLSB_P_INPUT_ERROR
:
474 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "in err:%1d %02x", q
->nr
,
477 *error
= QDIO_ERROR_SLSB_STATE
;
478 process_buffer_error(q
, start
, count
);
479 inbound_handle_work(q
, start
, count
, false);
480 if (atomic_sub_return(count
, &q
->nr_buf_used
) == 0)
481 qperf_inc(q
, inbound_queue_full
);
482 if (q
->irq_ptr
->perf_stat_enabled
)
483 account_sbals_error(q
, count
);
485 case SLSB_CU_INPUT_EMPTY
:
486 if (q
->irq_ptr
->perf_stat_enabled
)
487 q
->q_stats
.nr_sbal_nop
++;
488 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "in nop:%1d %#02x",
491 case SLSB_P_INPUT_NOT_INIT
:
492 case SLSB_P_INPUT_ACK
:
493 /* We should never see this state, throw a WARN: */
495 dev_WARN_ONCE(&q
->irq_ptr
->cdev
->dev
, 1,
496 "found state %#x at index %u on queue %u\n",
497 state
, start
, q
->nr
);
502 static inline int qdio_inbound_q_done(struct qdio_q
*q
, unsigned int start
)
504 unsigned char state
= 0;
506 if (!atomic_read(&q
->nr_buf_used
))
509 if (qdio_need_siga_sync(q
->irq_ptr
))
510 qdio_sync_input_queue(q
);
511 get_buf_state(q
, start
, &state
, 0);
513 if (state
== SLSB_P_INPUT_PRIMED
|| state
== SLSB_P_INPUT_ERROR
)
514 /* more work coming */
520 static int get_outbound_buffer_frontier(struct qdio_q
*q
, unsigned int start
,
523 unsigned char state
= 0;
526 q
->timestamp
= get_tod_clock_fast();
528 count
= atomic_read(&q
->nr_buf_used
);
532 if (qdio_need_siga_sync(q
->irq_ptr
))
533 qdio_sync_output_queue(q
);
535 count
= get_buf_states(q
, start
, &state
, count
, 0);
540 case SLSB_P_OUTPUT_PENDING
:
541 *error
= QDIO_ERROR_SLSB_PENDING
;
543 case SLSB_P_OUTPUT_EMPTY
:
544 /* the adapter got it */
545 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
,
546 "out empty:%1d %02x", q
->nr
, count
);
548 atomic_sub(count
, &q
->nr_buf_used
);
549 if (q
->irq_ptr
->perf_stat_enabled
)
550 account_sbals(q
, count
);
552 case SLSB_P_OUTPUT_ERROR
:
553 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "out error:%1d %02x",
556 *error
= QDIO_ERROR_SLSB_STATE
;
557 process_buffer_error(q
, start
, count
);
558 atomic_sub(count
, &q
->nr_buf_used
);
559 if (q
->irq_ptr
->perf_stat_enabled
)
560 account_sbals_error(q
, count
);
562 case SLSB_CU_OUTPUT_PRIMED
:
563 /* the adapter has not fetched the output yet */
564 if (q
->irq_ptr
->perf_stat_enabled
)
565 q
->q_stats
.nr_sbal_nop
++;
566 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "out primed:%1d",
569 case SLSB_P_OUTPUT_HALTED
:
571 case SLSB_P_OUTPUT_NOT_INIT
:
572 /* We should never see this state, throw a WARN: */
574 dev_WARN_ONCE(&q
->irq_ptr
->cdev
->dev
, 1,
575 "found state %#x at index %u on queue %u\n",
576 state
, start
, q
->nr
);
581 static int qdio_kick_outbound_q(struct qdio_q
*q
, unsigned int count
,
585 unsigned int busy_bit
;
587 if (!qdio_need_siga_out(q
->irq_ptr
))
590 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "siga-w:%1d", q
->nr
);
592 qperf_inc(q
, siga_write
);
594 cc
= qdio_siga_output(q
, count
, &busy_bit
, aob
);
600 while (++retries
< QDIO_BUSY_BIT_RETRIES
) {
601 mdelay(QDIO_BUSY_BIT_RETRY_DELAY
);
604 DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q
), q
->nr
);
607 DBF_DEV_EVENT(DBF_INFO
, q
->irq_ptr
, "siga-w cc2:%1d", q
->nr
);
613 DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q
), cc
);
618 DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q
), q
->nr
);
619 DBF_ERROR("count:%u", retries
);
624 static inline void qdio_set_state(struct qdio_irq
*irq_ptr
,
625 enum qdio_irq_states state
)
627 DBF_DEV_EVENT(DBF_INFO
, irq_ptr
, "newstate: %1d", state
);
629 irq_ptr
->state
= state
;
633 static void qdio_irq_check_sense(struct qdio_irq
*irq_ptr
, struct irb
*irb
)
635 if (irb
->esw
.esw0
.erw
.cons
) {
636 DBF_ERROR("%4x sense:", irq_ptr
->schid
.sch_no
);
637 DBF_ERROR_HEX(irb
, 64);
638 DBF_ERROR_HEX(irb
->ecw
, 64);
642 /* PCI interrupt handler */
643 static void qdio_int_handler_pci(struct qdio_irq
*irq_ptr
)
645 if (unlikely(irq_ptr
->state
!= QDIO_IRQ_STATE_ACTIVE
))
648 qdio_deliver_irq(irq_ptr
);
649 irq_ptr
->last_data_irq_time
= S390_lowcore
.int_clock
;
652 static void qdio_handle_activate_check(struct qdio_irq
*irq_ptr
,
653 unsigned long intparm
, int cstat
,
658 DBF_ERROR("%4x ACT CHECK", irq_ptr
->schid
.sch_no
);
659 DBF_ERROR("intp :%lx", intparm
);
660 DBF_ERROR("ds: %2x cs:%2x", dstat
, cstat
);
662 if (irq_ptr
->nr_input_qs
) {
663 q
= irq_ptr
->input_qs
[0];
664 } else if (irq_ptr
->nr_output_qs
) {
665 q
= irq_ptr
->output_qs
[0];
671 q
->handler(q
->irq_ptr
->cdev
, QDIO_ERROR_ACTIVATE
,
672 q
->nr
, q
->first_to_check
, 0, irq_ptr
->int_parm
);
674 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_STOPPED
);
676 * In case of z/VM LGR (Live Guest Migration) QDIO recovery will happen.
677 * Therefore we call the LGR detection function here.
682 static void qdio_establish_handle_irq(struct qdio_irq
*irq_ptr
, int cstat
,
685 DBF_DEV_EVENT(DBF_INFO
, irq_ptr
, "qest irq");
689 if (dstat
& ~(DEV_STAT_DEV_END
| DEV_STAT_CHN_END
))
691 if (!(dstat
& DEV_STAT_DEV_END
))
693 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_ESTABLISHED
);
697 DBF_ERROR("%4x EQ:error", irq_ptr
->schid
.sch_no
);
698 DBF_ERROR("ds: %2x cs:%2x", dstat
, cstat
);
699 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_ERR
);
702 /* qdio interrupt handler */
703 void qdio_int_handler(struct ccw_device
*cdev
, unsigned long intparm
,
706 struct qdio_irq
*irq_ptr
= cdev
->private->qdio_data
;
707 struct subchannel_id schid
;
710 if (!intparm
|| !irq_ptr
) {
711 ccw_device_get_schid(cdev
, &schid
);
712 DBF_ERROR("qint:%4x", schid
.sch_no
);
716 if (irq_ptr
->perf_stat_enabled
)
717 irq_ptr
->perf_stat
.qdio_int
++;
720 DBF_ERROR("%4x IO error", irq_ptr
->schid
.sch_no
);
721 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_ERR
);
722 wake_up(&cdev
->private->wait_q
);
725 qdio_irq_check_sense(irq_ptr
, irb
);
726 cstat
= irb
->scsw
.cmd
.cstat
;
727 dstat
= irb
->scsw
.cmd
.dstat
;
729 switch (irq_ptr
->state
) {
730 case QDIO_IRQ_STATE_INACTIVE
:
731 qdio_establish_handle_irq(irq_ptr
, cstat
, dstat
);
733 case QDIO_IRQ_STATE_CLEANUP
:
734 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_INACTIVE
);
736 case QDIO_IRQ_STATE_ESTABLISHED
:
737 case QDIO_IRQ_STATE_ACTIVE
:
738 if (cstat
& SCHN_STAT_PCI
) {
739 qdio_int_handler_pci(irq_ptr
);
743 qdio_handle_activate_check(irq_ptr
, intparm
, cstat
,
746 case QDIO_IRQ_STATE_STOPPED
:
751 wake_up(&cdev
->private->wait_q
);
755 * qdio_get_ssqd_desc - get qdio subchannel description
756 * @cdev: ccw device to get description for
757 * @data: where to store the ssqd
759 * Returns 0 or an error code. The results of the chsc are stored in the
760 * specified structure.
762 int qdio_get_ssqd_desc(struct ccw_device
*cdev
,
763 struct qdio_ssqd_desc
*data
)
765 struct subchannel_id schid
;
767 if (!cdev
|| !cdev
->private)
770 ccw_device_get_schid(cdev
, &schid
);
771 DBF_EVENT("get ssqd:%4x", schid
.sch_no
);
772 return qdio_setup_get_ssqd(NULL
, &schid
, data
);
774 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc
);
776 static int qdio_cancel_ccw(struct qdio_irq
*irq
, int how
)
778 struct ccw_device
*cdev
= irq
->cdev
;
782 spin_lock_irq(get_ccwdev_lock(cdev
));
783 qdio_set_state(irq
, QDIO_IRQ_STATE_CLEANUP
);
784 if (how
& QDIO_FLAG_CLEANUP_USING_CLEAR
)
785 rc
= ccw_device_clear(cdev
, QDIO_DOING_CLEANUP
);
787 /* default behaviour is halt */
788 rc
= ccw_device_halt(cdev
, QDIO_DOING_CLEANUP
);
789 spin_unlock_irq(get_ccwdev_lock(cdev
));
791 DBF_ERROR("%4x SHUTD ERR", irq
->schid
.sch_no
);
792 DBF_ERROR("rc:%4d", rc
);
796 timeout
= wait_event_interruptible_timeout(cdev
->private->wait_q
,
797 irq
->state
== QDIO_IRQ_STATE_INACTIVE
||
798 irq
->state
== QDIO_IRQ_STATE_ERR
,
801 rc
= (timeout
== -ERESTARTSYS
) ? -EINTR
: -ETIME
;
807 * qdio_shutdown - shut down a qdio subchannel
808 * @cdev: associated ccw device
809 * @how: use halt or clear to shutdown
811 int qdio_shutdown(struct ccw_device
*cdev
, int how
)
813 struct qdio_irq
*irq_ptr
= cdev
->private->qdio_data
;
814 struct subchannel_id schid
;
820 WARN_ON_ONCE(irqs_disabled());
821 ccw_device_get_schid(cdev
, &schid
);
822 DBF_EVENT("qshutdown:%4x", schid
.sch_no
);
824 mutex_lock(&irq_ptr
->setup_mutex
);
826 * Subchannel was already shot down. We cannot prevent being called
827 * twice since cio may trigger a shutdown asynchronously.
829 if (irq_ptr
->state
== QDIO_IRQ_STATE_INACTIVE
) {
830 mutex_unlock(&irq_ptr
->setup_mutex
);
835 * Indicate that the device is going down.
837 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_STOPPED
);
839 qdio_shutdown_debug_entries(irq_ptr
);
841 rc
= qdio_cancel_ccw(irq_ptr
, how
);
842 qdio_shutdown_thinint(irq_ptr
);
843 qdio_shutdown_irq(irq_ptr
);
845 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_INACTIVE
);
846 mutex_unlock(&irq_ptr
->setup_mutex
);
851 EXPORT_SYMBOL_GPL(qdio_shutdown
);
854 * qdio_free - free data structures for a qdio subchannel
855 * @cdev: associated ccw device
857 int qdio_free(struct ccw_device
*cdev
)
859 struct qdio_irq
*irq_ptr
= cdev
->private->qdio_data
;
860 struct subchannel_id schid
;
865 ccw_device_get_schid(cdev
, &schid
);
866 DBF_EVENT("qfree:%4x", schid
.sch_no
);
867 DBF_DEV_EVENT(DBF_ERR
, irq_ptr
, "dbf abandoned");
868 mutex_lock(&irq_ptr
->setup_mutex
);
870 irq_ptr
->debug_area
= NULL
;
871 cdev
->private->qdio_data
= NULL
;
872 mutex_unlock(&irq_ptr
->setup_mutex
);
874 qdio_free_queues(irq_ptr
);
875 free_page((unsigned long) irq_ptr
->qdr
);
876 free_page(irq_ptr
->chsc_page
);
877 free_page((unsigned long) irq_ptr
);
880 EXPORT_SYMBOL_GPL(qdio_free
);
883 * qdio_allocate - allocate qdio queues and associated data
884 * @cdev: associated ccw device
885 * @no_input_qs: allocate this number of Input Queues
886 * @no_output_qs: allocate this number of Output Queues
888 int qdio_allocate(struct ccw_device
*cdev
, unsigned int no_input_qs
,
889 unsigned int no_output_qs
)
891 struct subchannel_id schid
;
892 struct qdio_irq
*irq_ptr
;
895 ccw_device_get_schid(cdev
, &schid
);
896 DBF_EVENT("qallocate:%4x", schid
.sch_no
);
898 if (no_input_qs
> QDIO_MAX_QUEUES_PER_IRQ
||
899 no_output_qs
> QDIO_MAX_QUEUES_PER_IRQ
)
902 /* irq_ptr must be in GFP_DMA since it contains ccw1.cda */
903 irq_ptr
= (void *) get_zeroed_page(GFP_KERNEL
| GFP_DMA
);
907 irq_ptr
->cdev
= cdev
;
908 mutex_init(&irq_ptr
->setup_mutex
);
909 if (qdio_allocate_dbf(irq_ptr
))
912 DBF_DEV_EVENT(DBF_ERR
, irq_ptr
, "alloc niq:%1u noq:%1u", no_input_qs
,
916 * Allocate a page for the chsc calls in qdio_establish.
917 * Must be pre-allocated since a zfcp recovery will call
918 * qdio_establish. In case of low memory and swap on a zfcp disk
919 * we may not be able to allocate memory otherwise.
921 irq_ptr
->chsc_page
= get_zeroed_page(GFP_KERNEL
);
922 if (!irq_ptr
->chsc_page
)
925 /* qdr is used in ccw1.cda which is u32 */
926 irq_ptr
->qdr
= (struct qdr
*) get_zeroed_page(GFP_KERNEL
| GFP_DMA
);
930 rc
= qdio_allocate_qs(irq_ptr
, no_input_qs
, no_output_qs
);
934 cdev
->private->qdio_data
= irq_ptr
;
935 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_INACTIVE
);
939 free_page((unsigned long) irq_ptr
->qdr
);
941 free_page(irq_ptr
->chsc_page
);
944 free_page((unsigned long) irq_ptr
);
947 EXPORT_SYMBOL_GPL(qdio_allocate
);
949 static void qdio_trace_init_data(struct qdio_irq
*irq
,
950 struct qdio_initialize
*data
)
952 DBF_DEV_EVENT(DBF_ERR
, irq
, "qfmt:%1u", data
->q_format
);
953 DBF_DEV_EVENT(DBF_ERR
, irq
, "qpff%4x", data
->qib_param_field_format
);
954 DBF_DEV_HEX(irq
, &data
->qib_param_field
, sizeof(void *), DBF_ERR
);
955 DBF_DEV_EVENT(DBF_ERR
, irq
, "niq:%1u noq:%1u", data
->no_input_qs
,
957 DBF_DEV_HEX(irq
, &data
->input_handler
, sizeof(void *), DBF_ERR
);
958 DBF_DEV_HEX(irq
, &data
->output_handler
, sizeof(void *), DBF_ERR
);
959 DBF_DEV_HEX(irq
, &data
->int_parm
, sizeof(long), DBF_ERR
);
960 DBF_DEV_HEX(irq
, &data
->input_sbal_addr_array
, sizeof(void *), DBF_ERR
);
961 DBF_DEV_HEX(irq
, &data
->output_sbal_addr_array
, sizeof(void *),
966 * qdio_establish - establish queues on a qdio subchannel
967 * @cdev: associated ccw device
968 * @init_data: initialization data
970 int qdio_establish(struct ccw_device
*cdev
,
971 struct qdio_initialize
*init_data
)
973 struct qdio_irq
*irq_ptr
= cdev
->private->qdio_data
;
974 struct subchannel_id schid
;
978 ccw_device_get_schid(cdev
, &schid
);
979 DBF_EVENT("qestablish:%4x", schid
.sch_no
);
984 if (init_data
->no_input_qs
> irq_ptr
->max_input_qs
||
985 init_data
->no_output_qs
> irq_ptr
->max_output_qs
)
988 if ((init_data
->no_input_qs
&& !init_data
->input_handler
) ||
989 (init_data
->no_output_qs
&& !init_data
->output_handler
))
992 if (!init_data
->input_sbal_addr_array
||
993 !init_data
->output_sbal_addr_array
)
996 if (!init_data
->irq_poll
)
999 mutex_lock(&irq_ptr
->setup_mutex
);
1000 qdio_trace_init_data(irq_ptr
, init_data
);
1001 qdio_setup_irq(irq_ptr
, init_data
);
1003 rc
= qdio_establish_thinint(irq_ptr
);
1008 irq_ptr
->ccw
.cmd_code
= irq_ptr
->equeue
.cmd
;
1009 irq_ptr
->ccw
.flags
= CCW_FLAG_SLI
;
1010 irq_ptr
->ccw
.count
= irq_ptr
->equeue
.count
;
1011 irq_ptr
->ccw
.cda
= (u32
) virt_to_phys(irq_ptr
->qdr
);
1013 spin_lock_irq(get_ccwdev_lock(cdev
));
1014 ccw_device_set_options_mask(cdev
, 0);
1016 rc
= ccw_device_start(cdev
, &irq_ptr
->ccw
, QDIO_DOING_ESTABLISH
, 0, 0);
1017 spin_unlock_irq(get_ccwdev_lock(cdev
));
1019 DBF_ERROR("%4x est IO ERR", irq_ptr
->schid
.sch_no
);
1020 DBF_ERROR("rc:%4x", rc
);
1024 timeout
= wait_event_interruptible_timeout(cdev
->private->wait_q
,
1025 irq_ptr
->state
== QDIO_IRQ_STATE_ESTABLISHED
||
1026 irq_ptr
->state
== QDIO_IRQ_STATE_ERR
, HZ
);
1028 rc
= (timeout
== -ERESTARTSYS
) ? -EINTR
: -ETIME
;
1029 goto err_ccw_timeout
;
1032 if (irq_ptr
->state
!= QDIO_IRQ_STATE_ESTABLISHED
) {
1037 qdio_setup_ssqd_info(irq_ptr
);
1039 /* qebsm is now setup if available, initialize buffer states */
1040 qdio_init_buf_states(irq_ptr
);
1042 mutex_unlock(&irq_ptr
->setup_mutex
);
1043 qdio_print_subchannel_info(irq_ptr
);
1044 qdio_setup_debug_entries(irq_ptr
);
1048 qdio_cancel_ccw(irq_ptr
, QDIO_FLAG_CLEANUP_USING_CLEAR
);
1051 qdio_shutdown_thinint(irq_ptr
);
1053 qdio_shutdown_irq(irq_ptr
);
1054 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_INACTIVE
);
1055 mutex_unlock(&irq_ptr
->setup_mutex
);
1058 EXPORT_SYMBOL_GPL(qdio_establish
);
1061 * qdio_activate - activate queues on a qdio subchannel
1062 * @cdev: associated cdev
1064 int qdio_activate(struct ccw_device
*cdev
)
1066 struct qdio_irq
*irq_ptr
= cdev
->private->qdio_data
;
1067 struct subchannel_id schid
;
1070 ccw_device_get_schid(cdev
, &schid
);
1071 DBF_EVENT("qactivate:%4x", schid
.sch_no
);
1076 mutex_lock(&irq_ptr
->setup_mutex
);
1077 if (irq_ptr
->state
== QDIO_IRQ_STATE_INACTIVE
) {
1082 irq_ptr
->ccw
.cmd_code
= irq_ptr
->aqueue
.cmd
;
1083 irq_ptr
->ccw
.flags
= CCW_FLAG_SLI
;
1084 irq_ptr
->ccw
.count
= irq_ptr
->aqueue
.count
;
1085 irq_ptr
->ccw
.cda
= 0;
1087 spin_lock_irq(get_ccwdev_lock(cdev
));
1088 ccw_device_set_options(cdev
, CCWDEV_REPORT_ALL
);
1090 rc
= ccw_device_start(cdev
, &irq_ptr
->ccw
, QDIO_DOING_ACTIVATE
,
1091 0, DOIO_DENY_PREFETCH
);
1092 spin_unlock_irq(get_ccwdev_lock(cdev
));
1094 DBF_ERROR("%4x act IO ERR", irq_ptr
->schid
.sch_no
);
1095 DBF_ERROR("rc:%4x", rc
);
1099 /* wait for subchannel to become active */
1102 switch (irq_ptr
->state
) {
1103 case QDIO_IRQ_STATE_STOPPED
:
1104 case QDIO_IRQ_STATE_ERR
:
1108 qdio_set_state(irq_ptr
, QDIO_IRQ_STATE_ACTIVE
);
1112 mutex_unlock(&irq_ptr
->setup_mutex
);
1115 EXPORT_SYMBOL_GPL(qdio_activate
);
1118 * handle_inbound - reset processed input buffers
1119 * @q: queue containing the buffers
1120 * @bufnr: first buffer to process
1121 * @count: how many buffers are emptied
1123 static int handle_inbound(struct qdio_q
*q
, int bufnr
, int count
)
1127 qperf_inc(q
, inbound_call
);
1129 /* If any processed SBALs are returned to HW, adjust our tracking: */
1130 overlap
= min_t(int, count
- sub_buf(q
->u
.in
.batch_start
, bufnr
),
1131 q
->u
.in
.batch_count
);
1133 q
->u
.in
.batch_start
= add_buf(q
->u
.in
.batch_start
, overlap
);
1134 q
->u
.in
.batch_count
-= overlap
;
1137 count
= set_buf_states(q
, bufnr
, SLSB_CU_INPUT_EMPTY
, count
);
1138 atomic_add(count
, &q
->nr_buf_used
);
1140 if (qdio_need_siga_in(q
->irq_ptr
))
1141 return qdio_siga_input(q
);
1147 * handle_outbound - process filled outbound buffers
1148 * @q: queue containing the buffers
1149 * @bufnr: first buffer to process
1150 * @count: how many buffers are filled
1151 * @aob: asynchronous operation block
1153 static int handle_outbound(struct qdio_q
*q
, unsigned int bufnr
, unsigned int count
,
1156 unsigned char state
= 0;
1159 qperf_inc(q
, outbound_call
);
1161 count
= set_buf_states(q
, bufnr
, SLSB_CU_OUTPUT_PRIMED
, count
);
1162 used
= atomic_add_return(count
, &q
->nr_buf_used
);
1164 if (used
== QDIO_MAX_BUFFERS_PER_Q
)
1165 qperf_inc(q
, outbound_queue_full
);
1167 if (queue_type(q
) == QDIO_IQDIO_QFMT
) {
1168 unsigned long phys_aob
= aob
? virt_to_phys(aob
) : 0;
1170 WARN_ON_ONCE(!IS_ALIGNED(phys_aob
, 256));
1171 rc
= qdio_kick_outbound_q(q
, count
, phys_aob
);
1172 } else if (qdio_need_siga_sync(q
->irq_ptr
)) {
1173 rc
= qdio_sync_output_queue(q
);
1174 } else if (count
< QDIO_MAX_BUFFERS_PER_Q
&&
1175 get_buf_state(q
, prev_buf(bufnr
), &state
, 0) > 0 &&
1176 state
== SLSB_CU_OUTPUT_PRIMED
) {
1177 /* The previous buffer is not processed yet, tack on. */
1178 qperf_inc(q
, fast_requeue
);
1180 rc
= qdio_kick_outbound_q(q
, count
, 0);
1187 * do_QDIO - process input or output buffers
1188 * @cdev: associated ccw_device for the qdio subchannel
1189 * @callflags: input or output and special flags from the program
1190 * @q_nr: queue number
1191 * @bufnr: buffer number
1192 * @count: how many buffers to process
1193 * @aob: asynchronous operation block (outbound only)
1195 int do_QDIO(struct ccw_device
*cdev
, unsigned int callflags
,
1196 int q_nr
, unsigned int bufnr
, unsigned int count
, struct qaob
*aob
)
1198 struct qdio_irq
*irq_ptr
= cdev
->private->qdio_data
;
1200 if (bufnr
>= QDIO_MAX_BUFFERS_PER_Q
|| count
> QDIO_MAX_BUFFERS_PER_Q
)
1206 DBF_DEV_EVENT(DBF_INFO
, irq_ptr
,
1207 "do%02x b:%02x c:%02x", callflags
, bufnr
, count
);
1209 if (irq_ptr
->state
!= QDIO_IRQ_STATE_ACTIVE
)
1213 if (callflags
& QDIO_FLAG_SYNC_INPUT
)
1214 return handle_inbound(irq_ptr
->input_qs
[q_nr
], bufnr
, count
);
1215 else if (callflags
& QDIO_FLAG_SYNC_OUTPUT
)
1216 return handle_outbound(irq_ptr
->output_qs
[q_nr
], bufnr
, count
, aob
);
1219 EXPORT_SYMBOL_GPL(do_QDIO
);
1222 * qdio_start_irq - enable interrupt processing for the device
1223 * @cdev: associated ccw_device for the qdio subchannel
1227 * 1 - irqs not started since new data is available
1229 int qdio_start_irq(struct ccw_device
*cdev
)
1232 struct qdio_irq
*irq_ptr
= cdev
->private->qdio_data
;
1238 for_each_input_queue(irq_ptr
, q
, i
)
1239 qdio_stop_polling(q
);
1241 clear_bit(QDIO_IRQ_DISABLED
, &irq_ptr
->poll_state
);
1244 * We need to check again to not lose initiative after
1245 * resetting the ACK state.
1247 if (test_nonshared_ind(irq_ptr
))
1250 for_each_input_queue(irq_ptr
, q
, i
) {
1251 if (!qdio_inbound_q_done(q
, q
->first_to_check
))
1258 if (test_and_set_bit(QDIO_IRQ_DISABLED
, &irq_ptr
->poll_state
))
1264 EXPORT_SYMBOL(qdio_start_irq
);
1266 static int __qdio_inspect_queue(struct qdio_q
*q
, unsigned int *bufnr
,
1267 unsigned int *error
)
1269 unsigned int start
= q
->first_to_check
;
1273 count
= q
->is_input_q
? get_inbound_buffer_frontier(q
, start
, error
) :
1274 get_outbound_buffer_frontier(q
, start
, error
);
1280 /* for the next time */
1281 q
->first_to_check
= add_buf(start
, count
);
1286 int qdio_inspect_queue(struct ccw_device
*cdev
, unsigned int nr
, bool is_input
,
1287 unsigned int *bufnr
, unsigned int *error
)
1289 struct qdio_irq
*irq_ptr
= cdev
->private->qdio_data
;
1294 q
= is_input
? irq_ptr
->input_qs
[nr
] : irq_ptr
->output_qs
[nr
];
1296 return __qdio_inspect_queue(q
, bufnr
, error
);
1298 EXPORT_SYMBOL_GPL(qdio_inspect_queue
);
1301 * qdio_stop_irq - disable interrupt processing for the device
1302 * @cdev: associated ccw_device for the qdio subchannel
1305 * 0 - interrupts were already disabled
1306 * 1 - interrupts successfully disabled
1308 int qdio_stop_irq(struct ccw_device
*cdev
)
1310 struct qdio_irq
*irq_ptr
= cdev
->private->qdio_data
;
1315 if (test_and_set_bit(QDIO_IRQ_DISABLED
, &irq_ptr
->poll_state
))
1320 EXPORT_SYMBOL(qdio_stop_irq
);
1322 static int __init
init_QDIO(void)
1326 rc
= qdio_debug_init();
1329 rc
= qdio_setup_init();
1332 rc
= qdio_thinint_init();
1344 static void __exit
exit_QDIO(void)
1346 qdio_thinint_exit();
1351 module_init(init_QDIO
);
1352 module_exit(exit_QDIO
);