Index: uspace/drv/bus/usb/xhci/transfers.c =================================================================== --- uspace/drv/bus/usb/xhci/transfers.c (revision 1d758fc6d12dfedeea03efca816537794a056af2) +++ uspace/drv/bus/usb/xhci/transfers.c (revision 5fe3f954848e8edeb3e7a9741c25d0f620547e10) @@ -48,8 +48,4 @@ } stage_dir_flag_t; -#define REQUEST_TYPE_DTD (0x80) -#define REQUEST_TYPE_IS_DEVICE_TO_HOST(rq) ((rq) & REQUEST_TYPE_DTD) - - /** Get direction flag of data stage. * See Table 7 of xHCI specification. @@ -59,5 +55,5 @@ { /* See Table 7 of xHCI specification */ - return REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType) && (wLength > 0) + return SETUP_REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType) && (wLength > 0) ? STAGE_OUT : STAGE_IN; @@ -80,5 +76,5 @@ /* See Table 7 of xHCI specification */ - return REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType) + return SETUP_REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType) ? DATA_STAGE_IN : DATA_STAGE_NO; @@ -124,26 +120,65 @@ } -static int calculate_trb_count(xhci_transfer_t *transfer) -{ - const size_t size = transfer->batch.size; - return (size + PAGE_SIZE - 1 )/ PAGE_SIZE; -} - -static void trb_set_buffer(xhci_transfer_t *transfer, xhci_trb_t *trb, - size_t i, size_t total, size_t *remaining) -{ - const uintptr_t ptr = dma_buffer_phys(&transfer->batch.dma_buffer, - transfer->batch.dma_buffer.virt + i * PAGE_SIZE); - - trb->parameter = host2xhci(64, ptr); - TRB_CTRL_SET_TD_SIZE(*trb, max(31, total - i - 1)); - if (*remaining > PAGE_SIZE) { - TRB_CTRL_SET_XFER_LEN(*trb, PAGE_SIZE); - *remaining -= PAGE_SIZE; - } - else { - TRB_CTRL_SET_XFER_LEN(*trb, *remaining); - *remaining = 0; - } +#define MAX_CHUNK_SIZE (1 << 16) + +typedef struct { + /* Input parameters */ + dma_buffer_t buf; + size_t chunk_size, packet_count, mps, max_trb_count; + + /* Changing at runtime */ + size_t transferred, remaining; + void *pos; +} trb_splitter_t; + +static void trb_splitter_init(trb_splitter_t *ts, xhci_transfer_t *transfer) +{ + ts->buf = transfer->batch.dma_buffer; + + const size_t chunk_mask = dma_policy_chunk_mask(ts->buf.policy); + ts->chunk_size = (chunk_mask > MAX_CHUNK_SIZE + 1) + ? MAX_CHUNK_SIZE : (chunk_mask + 1); + + ts->remaining = transfer->batch.size; + ts->max_trb_count = (ts->remaining + ts->chunk_size - 1) / ts->chunk_size + 1; + ts->mps = transfer->batch.ep->max_packet_size; + ts->packet_count = (ts->remaining + ts->mps - 1) / ts->mps; + + ts->transferred = 0; + ts->pos = ts->buf.virt + transfer->batch.offset; +} + +static void trb_split_next(xhci_trb_t *trb, trb_splitter_t *ts) +{ + xhci_trb_clean(trb); + + size_t size = min(ts->remaining, ts->chunk_size); + + /* First TRB might be misaligned */ + if (ts->transferred == 0) { + const size_t offset = (ts->pos - ts->buf.virt) % ts->chunk_size; + size = min(size, ts->chunk_size - offset); + } + + ts->transferred += size; + ts->remaining -= size; + + const size_t tx_packets = (ts->transferred + ts->mps - 1) / ts->mps; + const unsigned td_size = min(31, ts->packet_count - tx_packets); + + /* Last TRB must have TD Size = 0 */ + assert(ts->remaining > 0 || td_size == 0); + + uintptr_t phys = dma_buffer_phys(&ts->buf, ts->pos); + + trb->parameter = host2xhci(64, phys); + TRB_CTRL_SET_TD_SIZE(*trb, td_size); + TRB_CTRL_SET_XFER_LEN(*trb, size); + TRB_CTRL_SET_TRB_TYPE(*trb, XHCI_TRB_TYPE_NORMAL); + + if (ts->remaining) + TRB_CTRL_SET_CHAIN(*trb, 1); + + ts->pos += size; } @@ -155,19 +190,14 @@ usb_device_request_setup_packet_t* setup = &batch->setup.packet; - size_t buffer_count = 0; - if (setup->length > 0) { - buffer_count = calculate_trb_count(transfer); - } - - xhci_trb_t trbs[buffer_count + 2]; - - xhci_trb_t *trb_setup = trbs; + trb_splitter_t splitter; + trb_splitter_init(&splitter, transfer); + + xhci_trb_t trbs[splitter.max_trb_count + 2]; + size_t trbs_used = 0; + + xhci_trb_t *trb_setup = &trbs[trbs_used++]; xhci_trb_clean(trb_setup); - TRB_CTRL_SET_SETUP_WVALUE(*trb_setup, setup->value); - TRB_CTRL_SET_SETUP_WLENGTH(*trb_setup, setup->length); - TRB_CTRL_SET_SETUP_WINDEX(*trb_setup, setup->index); - TRB_CTRL_SET_SETUP_BREQ(*trb_setup, setup->request); - TRB_CTRL_SET_SETUP_BMREQTYPE(*trb_setup, setup->request_type); + trb_setup->parameter = batch->setup.packed; /* Size of the setup packet is always 8 */ @@ -180,26 +210,19 @@ get_transfer_type(trb_setup, setup->request_type, setup->length)); - /* Data stage */ - if (setup->length > 0) { - int stage_dir = REQUEST_TYPE_IS_DEVICE_TO_HOST(setup->request_type) - ? STAGE_IN : STAGE_OUT; - size_t remaining = transfer->batch.size; - - for (size_t i = 0; i < buffer_count; ++i) { - xhci_trb_clean(&trbs[i + 1]); - trb_set_buffer(transfer, &trbs[i + 1], i, buffer_count, &remaining); - - TRB_CTRL_SET_DIR(trbs[i + 1], stage_dir); - TRB_CTRL_SET_TRB_TYPE(trbs[i + 1], XHCI_TRB_TYPE_DATA_STAGE); - - if (i == buffer_count - 1) break; - - /* Set the chain bit as this is not the last TRB */ - TRB_CTRL_SET_CHAIN(trbs[i], 1); - } - } + stage_dir_flag_t stage_dir = (transfer->batch.dir == USB_DIRECTION_IN) + ? STAGE_IN : STAGE_OUT; + + /* Data stage - first TRB is special */ + if (splitter.remaining > 0) { + xhci_trb_t *trb = &trbs[trbs_used++]; + trb_split_next(trb, &splitter); + TRB_CTRL_SET_TRB_TYPE(*trb, XHCI_TRB_TYPE_DATA_STAGE); + TRB_CTRL_SET_DIR(*trb, stage_dir); + } + while (splitter.remaining > 0) + trb_split_next(&trbs[trbs_used++], &splitter); /* Status stage */ - xhci_trb_t *trb_status = trbs + buffer_count + 1; + xhci_trb_t *trb_status = &trbs[trbs_used++]; xhci_trb_clean(trb_status); @@ -217,82 +240,44 @@ return xhci_trb_ring_enqueue_multiple(get_ring(transfer), trbs, - buffer_count + 2, &transfer->interrupt_trb_phys); -} - -static errno_t schedule_bulk(xhci_hc_t* hc, xhci_transfer_t *transfer) -{ + trbs_used, &transfer->interrupt_trb_phys); +} + +static errno_t schedule_bulk_intr(xhci_hc_t* hc, xhci_transfer_t *transfer) +{ + xhci_trb_ring_t * const ring = get_ring(transfer); + if (!ring) + return EINVAL; + /* The stream-enabled endpoints need to chain ED trb */ xhci_endpoint_t *ep = xhci_endpoint_get(transfer->batch.ep); - if (!ep->primary_stream_data_size) { - const size_t buffer_count = calculate_trb_count(transfer); - xhci_trb_t trbs[buffer_count]; - size_t remaining = transfer->batch.size; - - for (size_t i = 0; i < buffer_count; ++i) { - xhci_trb_clean(&trbs[i]); - trb_set_buffer(transfer, &trbs[i], i, buffer_count, &remaining); - TRB_CTRL_SET_TRB_TYPE(trbs[i], XHCI_TRB_TYPE_NORMAL); - - if (i == buffer_count - 1) break; - - /* Set the chain bit as this is not the last TRB */ - TRB_CTRL_SET_CHAIN(trbs[i], 1); - } + const bool use_streams = !!ep->primary_stream_data_size; + + trb_splitter_t splitter; + trb_splitter_init(&splitter, transfer); + + const size_t trb_count = splitter.max_trb_count + use_streams; + xhci_trb_t trbs[trb_count]; + size_t trbs_used = 0; + + while (splitter.remaining > 0) + trb_split_next(&trbs[trbs_used++], &splitter); + + if (!use_streams) { /* Set the interrupt bit for last TRB */ - TRB_CTRL_SET_IOC(trbs[buffer_count - 1], 1); - - xhci_trb_ring_t* ring = get_ring(transfer); - return xhci_trb_ring_enqueue_multiple(ring, &trbs[0], buffer_count, - &transfer->interrupt_trb_phys); + TRB_CTRL_SET_IOC(trbs[trbs_used - 1], 1); } else { - xhci_trb_ring_t* ring = get_ring(transfer); - if (!ring) { - return EINVAL; - } - - const size_t buffer_count = calculate_trb_count(transfer); - xhci_trb_t trbs[buffer_count + 1]; - size_t remaining = transfer->batch.size; - - for (size_t i = 0; i < buffer_count; ++i) { - xhci_trb_clean(&trbs[i]); - trb_set_buffer(transfer, &trbs[i], i, buffer_count + 1, &remaining); - TRB_CTRL_SET_TRB_TYPE(trbs[i], XHCI_TRB_TYPE_NORMAL); - TRB_CTRL_SET_CHAIN(trbs[i], 1); - } - TRB_CTRL_SET_ENT(trbs[buffer_count - 1], 1); - - xhci_trb_clean(&trbs[buffer_count]); - trbs[buffer_count].parameter = host2xhci(64, (uintptr_t) transfer); - TRB_CTRL_SET_TRB_TYPE(trbs[buffer_count], XHCI_TRB_TYPE_EVENT_DATA); - TRB_CTRL_SET_IOC(trbs[buffer_count], 1); - - return xhci_trb_ring_enqueue_multiple(ring, &trbs[0], buffer_count + 1, - &transfer->interrupt_trb_phys); - } -} - -static errno_t schedule_interrupt(xhci_hc_t* hc, xhci_transfer_t* transfer) -{ - const size_t buffer_count = calculate_trb_count(transfer); - xhci_trb_t trbs[buffer_count]; - size_t remaining = transfer->batch.size; - - for (size_t i = 0; i < buffer_count; ++i) { - xhci_trb_clean(&trbs[i]); - trb_set_buffer(transfer, &trbs[i], i, buffer_count, &remaining); - TRB_CTRL_SET_TRB_TYPE(trbs[i], XHCI_TRB_TYPE_NORMAL); - - if (i == buffer_count - 1) break; - - /* Set the chain bit as this is not the last TRB */ - TRB_CTRL_SET_CHAIN(trbs[i], 1); - } - /* Set the interrupt bit for last TRB */ - TRB_CTRL_SET_IOC(trbs[buffer_count - 1], 1); - - xhci_trb_ring_t* ring = get_ring(transfer); - return xhci_trb_ring_enqueue_multiple(ring, &trbs[0], buffer_count, + /* Clear the chain bit on the last TRB */ + TRB_CTRL_SET_CHAIN(trbs[trbs_used - 1], 1); + TRB_CTRL_SET_ENT(trbs[trbs_used - 1], 1); + + xhci_trb_t *ed = &trbs[trbs_used++]; + xhci_trb_clean(ed); + ed->parameter = host2xhci(64, (uintptr_t) transfer); + TRB_CTRL_SET_TRB_TYPE(*ed, XHCI_TRB_TYPE_EVENT_DATA); + TRB_CTRL_SET_IOC(*ed, 1); + } + + return xhci_trb_ring_enqueue_multiple(ring, trbs, trbs_used, &transfer->interrupt_trb_phys); } @@ -429,6 +414,6 @@ [USB_TRANSFER_CONTROL] = schedule_control, [USB_TRANSFER_ISOCHRONOUS] = NULL, - [USB_TRANSFER_BULK] = schedule_bulk, - [USB_TRANSFER_INTERRUPT] = schedule_interrupt, + [USB_TRANSFER_BULK] = schedule_bulk_intr, + [USB_TRANSFER_INTERRUPT] = schedule_bulk_intr, }; Index: uspace/lib/usb/include/usb/request.h =================================================================== --- uspace/lib/usb/include/usb/request.h (revision 1d758fc6d12dfedeea03efca816537794a056af2) +++ uspace/lib/usb/include/usb/request.h (revision 5fe3f954848e8edeb3e7a9741c25d0f620547e10) @@ -85,4 +85,5 @@ #define SETUP_REQUEST_TYPE_DEVICE_TO_HOST (1 << 7) #define SETUP_REQUEST_TYPE_HOST_TO_DEVICE (0 << 7) +#define SETUP_REQUEST_TYPE_IS_DEVICE_TO_HOST(rt) ((rt) & (1 << 7)) #define SETUP_REQUEST_TYPE_GET_TYPE(rt) ((rt >> 5) & 0x3) #define SETUP_REQUEST_TYPE_GET_RECIPIENT(rec) (rec & 0x1f)