source: mainline/uspace/drv/bus/usb/xhci/transfers.c@ ee820ff

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since ee820ff was 5fe3f954, checked in by Ondřej Hlavatý <aearsis@…>, 8 years ago

xhci: refactor splitting transfers

Allow nonzero offsets (not yet used bc. of other drivers), usage of
bigger chunks than PAGE_SIZE. The performance is now at 1.3 GB/s, when
using 16MB transfers (the current maximum) and without debug logs.
  • Property mode set to 100644
File size: 14.8 KB
Line 
1/*
2 * Copyright (c) 2017 Michal Staruch
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/** @addtogroup drvusbxhci
30 * @{
31 */
32/** @file
33 * @brief The host controller transfer ring management
34 */
35
36#include <usb/debug.h>
37#include <usb/request.h>
38#include "endpoint.h"
39#include "hc.h"
40#include "hw_struct/trb.h"
41#include "streams.h"
42#include "transfers.h"
43#include "trb_ring.h"
44
45typedef enum {
46 STAGE_OUT,
47 STAGE_IN,
48} stage_dir_flag_t;
49
50/** Get direction flag of data stage.
51 * See Table 7 of xHCI specification.
52 */
53static inline stage_dir_flag_t get_status_direction_flag(xhci_trb_t* trb,
54 uint8_t bmRequestType, uint16_t wLength)
55{
56 /* See Table 7 of xHCI specification */
57 return SETUP_REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType) && (wLength > 0)
58 ? STAGE_OUT
59 : STAGE_IN;
60}
61
62typedef enum {
63 DATA_STAGE_NO = 0,
64 DATA_STAGE_OUT = 2,
65 DATA_STAGE_IN = 3,
66} data_stage_type_t;
67
68/** Get transfer type flag.
69 * See Table 8 of xHCI specification.
70 */
71static inline data_stage_type_t get_transfer_type(xhci_trb_t* trb, uint8_t
72 bmRequestType, uint16_t wLength)
73{
74 if (wLength == 0)
75 return DATA_STAGE_NO;
76
77 /* See Table 7 of xHCI specification */
78 return SETUP_REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType)
79 ? DATA_STAGE_IN
80 : DATA_STAGE_NO;
81}
82
83static inline bool configure_endpoint_needed(usb_device_request_setup_packet_t *setup)
84{
85 usb_request_type_t request_type = SETUP_REQUEST_TYPE_GET_TYPE(setup->request_type);
86
87 return request_type == USB_REQUEST_TYPE_STANDARD &&
88 (setup->request == USB_DEVREQ_SET_CONFIGURATION
89 || setup->request == USB_DEVREQ_SET_INTERFACE);
90}
91
92/**
93 * Create a xHCI-specific transfer batch.
94 *
95 * Bus callback.
96 */
97usb_transfer_batch_t * xhci_transfer_create(endpoint_t* ep)
98{
99 xhci_transfer_t *transfer = calloc(1, sizeof(xhci_transfer_t));
100 if (!transfer)
101 return NULL;
102
103 usb_transfer_batch_init(&transfer->batch, ep);
104 return &transfer->batch;
105}
106
107/**
108 * Destroy a xHCI transfer.
109 */
110void xhci_transfer_destroy(usb_transfer_batch_t* batch)
111{
112 xhci_transfer_t *transfer = xhci_transfer_from_batch(batch);
113 free(transfer);
114}
115
116static xhci_trb_ring_t *get_ring(xhci_transfer_t *transfer)
117{
118 xhci_endpoint_t *xhci_ep = xhci_endpoint_get(transfer->batch.ep);
119 return xhci_endpoint_get_ring(xhci_ep, transfer->batch.target.stream);
120}
121
122#define MAX_CHUNK_SIZE (1 << 16)
123
124typedef struct {
125 /* Input parameters */
126 dma_buffer_t buf;
127 size_t chunk_size, packet_count, mps, max_trb_count;
128
129 /* Changing at runtime */
130 size_t transferred, remaining;
131 void *pos;
132} trb_splitter_t;
133
134static void trb_splitter_init(trb_splitter_t *ts, xhci_transfer_t *transfer)
135{
136 ts->buf = transfer->batch.dma_buffer;
137
138 const size_t chunk_mask = dma_policy_chunk_mask(ts->buf.policy);
139 ts->chunk_size = (chunk_mask > MAX_CHUNK_SIZE + 1)
140 ? MAX_CHUNK_SIZE : (chunk_mask + 1);
141
142 ts->remaining = transfer->batch.size;
143 ts->max_trb_count = (ts->remaining + ts->chunk_size - 1) / ts->chunk_size + 1;
144 ts->mps = transfer->batch.ep->max_packet_size;
145 ts->packet_count = (ts->remaining + ts->mps - 1) / ts->mps;
146
147 ts->transferred = 0;
148 ts->pos = ts->buf.virt + transfer->batch.offset;
149}
150
151static void trb_split_next(xhci_trb_t *trb, trb_splitter_t *ts)
152{
153 xhci_trb_clean(trb);
154
155 size_t size = min(ts->remaining, ts->chunk_size);
156
157 /* First TRB might be misaligned */
158 if (ts->transferred == 0) {
159 const size_t offset = (ts->pos - ts->buf.virt) % ts->chunk_size;
160 size = min(size, ts->chunk_size - offset);
161 }
162
163 ts->transferred += size;
164 ts->remaining -= size;
165
166 const size_t tx_packets = (ts->transferred + ts->mps - 1) / ts->mps;
167 const unsigned td_size = min(31, ts->packet_count - tx_packets);
168
169 /* Last TRB must have TD Size = 0 */
170 assert(ts->remaining > 0 || td_size == 0);
171
172 uintptr_t phys = dma_buffer_phys(&ts->buf, ts->pos);
173
174 trb->parameter = host2xhci(64, phys);
175 TRB_CTRL_SET_TD_SIZE(*trb, td_size);
176 TRB_CTRL_SET_XFER_LEN(*trb, size);
177 TRB_CTRL_SET_TRB_TYPE(*trb, XHCI_TRB_TYPE_NORMAL);
178
179 if (ts->remaining)
180 TRB_CTRL_SET_CHAIN(*trb, 1);
181
182 ts->pos += size;
183}
184
185static errno_t schedule_control(xhci_hc_t* hc, xhci_transfer_t* transfer)
186{
187 usb_transfer_batch_t *batch = &transfer->batch;
188 xhci_endpoint_t *xhci_ep = xhci_endpoint_get(transfer->batch.ep);
189
190 usb_device_request_setup_packet_t* setup = &batch->setup.packet;
191
192 trb_splitter_t splitter;
193 trb_splitter_init(&splitter, transfer);
194
195 xhci_trb_t trbs[splitter.max_trb_count + 2];
196 size_t trbs_used = 0;
197
198 xhci_trb_t *trb_setup = &trbs[trbs_used++];
199 xhci_trb_clean(trb_setup);
200
201 trb_setup->parameter = batch->setup.packed;
202
203 /* Size of the setup packet is always 8 */
204 TRB_CTRL_SET_XFER_LEN(*trb_setup, 8);
205
206 /* Immediate data */
207 TRB_CTRL_SET_IDT(*trb_setup, 1);
208 TRB_CTRL_SET_TRB_TYPE(*trb_setup, XHCI_TRB_TYPE_SETUP_STAGE);
209 TRB_CTRL_SET_TRT(*trb_setup,
210 get_transfer_type(trb_setup, setup->request_type, setup->length));
211
212 stage_dir_flag_t stage_dir = (transfer->batch.dir == USB_DIRECTION_IN)
213 ? STAGE_IN : STAGE_OUT;
214
215 /* Data stage - first TRB is special */
216 if (splitter.remaining > 0) {
217 xhci_trb_t *trb = &trbs[trbs_used++];
218 trb_split_next(trb, &splitter);
219 TRB_CTRL_SET_TRB_TYPE(*trb, XHCI_TRB_TYPE_DATA_STAGE);
220 TRB_CTRL_SET_DIR(*trb, stage_dir);
221 }
222 while (splitter.remaining > 0)
223 trb_split_next(&trbs[trbs_used++], &splitter);
224
225 /* Status stage */
226 xhci_trb_t *trb_status = &trbs[trbs_used++];
227 xhci_trb_clean(trb_status);
228
229 TRB_CTRL_SET_IOC(*trb_status, 1);
230 TRB_CTRL_SET_TRB_TYPE(*trb_status, XHCI_TRB_TYPE_STATUS_STAGE);
231 TRB_CTRL_SET_DIR(*trb_status, get_status_direction_flag(trb_setup,
232 setup->request_type, setup->length));
233
234 // Issue a Configure Endpoint command, if needed.
235 if (configure_endpoint_needed(setup)) {
236 const errno_t err = hc_configure_device(xhci_ep_to_dev(xhci_ep));
237 if (err)
238 return err;
239 }
240
241 return xhci_trb_ring_enqueue_multiple(get_ring(transfer), trbs,
242 trbs_used, &transfer->interrupt_trb_phys);
243}
244
245static errno_t schedule_bulk_intr(xhci_hc_t* hc, xhci_transfer_t *transfer)
246{
247 xhci_trb_ring_t * const ring = get_ring(transfer);
248 if (!ring)
249 return EINVAL;
250
251 /* The stream-enabled endpoints need to chain ED trb */
252 xhci_endpoint_t *ep = xhci_endpoint_get(transfer->batch.ep);
253 const bool use_streams = !!ep->primary_stream_data_size;
254
255 trb_splitter_t splitter;
256 trb_splitter_init(&splitter, transfer);
257
258 const size_t trb_count = splitter.max_trb_count + use_streams;
259 xhci_trb_t trbs[trb_count];
260 size_t trbs_used = 0;
261
262 while (splitter.remaining > 0)
263 trb_split_next(&trbs[trbs_used++], &splitter);
264
265 if (!use_streams) {
266 /* Set the interrupt bit for last TRB */
267 TRB_CTRL_SET_IOC(trbs[trbs_used - 1], 1);
268 }
269 else {
270 /* Clear the chain bit on the last TRB */
271 TRB_CTRL_SET_CHAIN(trbs[trbs_used - 1], 1);
272 TRB_CTRL_SET_ENT(trbs[trbs_used - 1], 1);
273
274 xhci_trb_t *ed = &trbs[trbs_used++];
275 xhci_trb_clean(ed);
276 ed->parameter = host2xhci(64, (uintptr_t) transfer);
277 TRB_CTRL_SET_TRB_TYPE(*ed, XHCI_TRB_TYPE_EVENT_DATA);
278 TRB_CTRL_SET_IOC(*ed, 1);
279 }
280
281 return xhci_trb_ring_enqueue_multiple(ring, trbs, trbs_used,
282 &transfer->interrupt_trb_phys);
283}
284
285static int schedule_isochronous(xhci_transfer_t* transfer)
286{
287 endpoint_t *ep = transfer->batch.ep;
288
289 return ep->direction == USB_DIRECTION_OUT
290 ? isoch_schedule_out(transfer)
291 : isoch_schedule_in(transfer);
292}
293
294errno_t xhci_handle_transfer_event(xhci_hc_t* hc, xhci_trb_t* trb)
295{
296 uintptr_t addr = trb->parameter;
297 const unsigned slot_id = XHCI_DWORD_EXTRACT(trb->control, 31, 24);
298 const unsigned ep_dci = XHCI_DWORD_EXTRACT(trb->control, 20, 16);
299
300 xhci_device_t *dev = hc->bus.devices_by_slot[slot_id];
301 if (!dev) {
302 usb_log_error("Transfer event on disabled slot %u", slot_id);
303 return ENOENT;
304 }
305
306 const usb_endpoint_t ep_num = ep_dci / 2;
307 const usb_endpoint_t dir = ep_dci % 2 ? USB_DIRECTION_IN : USB_DIRECTION_OUT;
308 /* Creating temporary reference */
309 endpoint_t *ep_base = bus_find_endpoint(&dev->base, ep_num, dir);
310 if (!ep_base) {
311 usb_log_error("Transfer event on dropped endpoint %u %s of device "
312 XHCI_DEV_FMT, ep_num, usb_str_direction(dir), XHCI_DEV_ARGS(*dev));
313 return ENOENT;
314 }
315 xhci_endpoint_t *ep = xhci_endpoint_get(ep_base);
316
317 usb_transfer_batch_t *batch;
318 xhci_transfer_t *transfer;
319
320 if (TRB_EVENT_DATA(*trb)) {
321 /* We schedule those only when streams are involved */
322 assert(ep->primary_stream_ctx_array != NULL);
323
324 /* We are received transfer pointer instead - work with that */
325 transfer = (xhci_transfer_t *) addr;
326 xhci_trb_ring_update_dequeue(get_ring(transfer),
327 transfer->interrupt_trb_phys);
328 batch = &transfer->batch;
329 }
330 else {
331 xhci_trb_ring_update_dequeue(&ep->ring, addr);
332
333 if (ep->base.transfer_type == USB_TRANSFER_ISOCHRONOUS) {
334 isoch_handle_transfer_event(hc, ep, trb);
335 /* Dropping temporary reference */
336 endpoint_del_ref(&ep->base);
337 return EOK;
338 }
339
340 fibril_mutex_lock(&ep->guard);
341 batch = ep->base.active_batch;
342 endpoint_deactivate_locked(&ep->base);
343 fibril_mutex_unlock(&ep->guard);
344
345 if (!batch) {
346 /* Dropping temporary reference */
347 endpoint_del_ref(&ep->base);
348 return ENOENT;
349 }
350
351 transfer = xhci_transfer_from_batch(batch);
352 }
353
354 const xhci_trb_completion_code_t completion_code = TRB_COMPLETION_CODE(*trb);
355 switch (completion_code) {
356 case XHCI_TRBC_SHORT_PACKET:
357 case XHCI_TRBC_SUCCESS:
358 batch->error = EOK;
359 batch->transferred_size = batch->size - TRB_TRANSFER_LENGTH(*trb);
360 break;
361
362 case XHCI_TRBC_DATA_BUFFER_ERROR:
363 usb_log_warning("Transfer ended with data buffer error.");
364 batch->error = EAGAIN;
365 batch->transferred_size = 0;
366 break;
367
368 case XHCI_TRBC_BABBLE_DETECTED_ERROR:
369 usb_log_warning("Babble detected during the transfer.");
370 batch->error = EAGAIN;
371 batch->transferred_size = 0;
372 break;
373
374 case XHCI_TRBC_USB_TRANSACTION_ERROR:
375 usb_log_warning("USB Transaction error.");
376 batch->error = EAGAIN;
377 batch->transferred_size = 0;
378 break;
379
380 case XHCI_TRBC_TRB_ERROR:
381 usb_log_error("Invalid transfer parameters.");
382 batch->error = EINVAL;
383 batch->transferred_size = 0;
384 break;
385
386 case XHCI_TRBC_STALL_ERROR:
387 usb_log_warning("Stall condition detected.");
388 batch->error = ESTALL;
389 batch->transferred_size = 0;
390 break;
391
392 case XHCI_TRBC_SPLIT_TRANSACTION_ERROR:
393 usb_log_error("Split transcation error detected.");
394 batch->error = EAGAIN;
395 batch->transferred_size = 0;
396 break;
397
398 default:
399 usb_log_warning("Transfer not successfull: %u", completion_code);
400 batch->error = EIO;
401 }
402
403 assert(batch->transferred_size <= batch->size);
404
405 usb_transfer_batch_finish(batch);
406 /* Dropping temporary reference */
407 endpoint_del_ref(&ep->base);
408 return EOK;
409}
410
411typedef errno_t (*transfer_handler)(xhci_hc_t *, xhci_transfer_t *);
412
413static const transfer_handler transfer_handlers[] = {
414 [USB_TRANSFER_CONTROL] = schedule_control,
415 [USB_TRANSFER_ISOCHRONOUS] = NULL,
416 [USB_TRANSFER_BULK] = schedule_bulk_intr,
417 [USB_TRANSFER_INTERRUPT] = schedule_bulk_intr,
418};
419
420/**
421 * Schedule a batch for xHC.
422 *
423 * Bus callback.
424 */
425errno_t xhci_transfer_schedule(usb_transfer_batch_t *batch)
426{
427 endpoint_t *ep = batch->ep;
428
429 xhci_hc_t *hc = bus_to_hc(endpoint_get_bus(batch->ep));
430 xhci_transfer_t *transfer = xhci_transfer_from_batch(batch);
431 xhci_endpoint_t *xhci_ep = xhci_endpoint_get(ep);
432 xhci_device_t *xhci_dev = xhci_ep_to_dev(xhci_ep);
433
434 if (!batch->target.address) {
435 usb_log_error("Attempted to schedule transfer to address 0.");
436 return EINVAL;
437 }
438
439 // FIXME: find a better way to check if the ring is not initialized
440 if (!xhci_ep->ring.segment_count) {
441 usb_log_error("Ring not initialized for endpoint " XHCI_EP_FMT,
442 XHCI_EP_ARGS(*xhci_ep));
443 return EINVAL;
444 }
445
446 // Isochronous transfer needs to be handled differently
447 if (batch->ep->transfer_type == USB_TRANSFER_ISOCHRONOUS) {
448 return schedule_isochronous(transfer);
449 }
450
451 const usb_transfer_type_t type = batch->ep->transfer_type;
452 assert(transfer_handlers[type]);
453
454 /*
455 * If this is a ClearFeature(ENDPOINT_HALT) request, we have to issue
456 * the Reset Endpoint command.
457 */
458 if (batch->ep->transfer_type == USB_TRANSFER_CONTROL
459 && batch->dir == USB_DIRECTION_OUT) {
460 const usb_device_request_setup_packet_t *request = &batch->setup.packet;
461 if (request->request == USB_DEVREQ_CLEAR_FEATURE
462 && request->request_type == USB_REQUEST_RECIPIENT_ENDPOINT
463 && request->value == USB_FEATURE_ENDPOINT_HALT) {
464 const uint16_t index = uint16_usb2host(request->index);
465 const usb_endpoint_t ep_num = index & 0xf;
466 const usb_direction_t dir = (index >> 7)
467 ? USB_DIRECTION_IN
468 : USB_DIRECTION_OUT;
469 endpoint_t *halted_ep = bus_find_endpoint(&xhci_dev->base, ep_num, dir);
470 if (halted_ep) {
471 /*
472 * TODO: Find out how to come up with stream_id. It might be
473 * possible that we have to clear all of them.
474 */
475 const errno_t err = xhci_endpoint_clear_halt(xhci_endpoint_get(halted_ep), 0);
476 endpoint_del_ref(halted_ep);
477 if (err) {
478 /*
479 * The endpoint halt condition failed to be cleared in HC.
480 * As it does not make sense to send the reset to the device
481 * itself, return as unschedulable answer.
482 *
483 * Furthermore, if this is a request to clear EP 0 stall, it
484 * would be gone forever, as the endpoint is halted.
485 */
486 return err;
487 }
488 } else {
489 usb_log_warning("Device(%u): Resetting unregistered endpoint"
490 " %u %s.", xhci_dev->base.address, ep_num,
491 usb_str_direction(dir));
492 }
493 }
494 }
495
496
497 errno_t err;
498 fibril_mutex_lock(&xhci_ep->guard);
499
500 if ((err = endpoint_activate_locked(ep, batch))) {
501 fibril_mutex_unlock(&xhci_ep->guard);
502 return err;
503 }
504
505 if ((err = transfer_handlers[batch->ep->transfer_type](hc, transfer))) {
506 endpoint_deactivate_locked(ep);
507 fibril_mutex_unlock(&xhci_ep->guard);
508 return err;
509 }
510
511 hc_ring_ep_doorbell(xhci_ep, batch->target.stream);
512 fibril_mutex_unlock(&xhci_ep->guard);
513 return EOK;
514}
Note: See TracBrowser for help on using the repository browser.