Changeset 5fe3f954 in mainline

Timestamp:

2018-02-12T10:28:14Z (6 years ago)

Author:

Ondřej Hlavatý <aearsis@…>

Branches:

lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export

Children:

ee820ff

Parents:

1d758fc

git-author:

Ondřej Hlavatý <aearsis@…> (2018-02-12 10:23:51)

git-committer:

Ondřej Hlavatý <aearsis@…> (2018-02-12 10:28:14)

Message:

xhci: refactor splitting transfers

Allow nonzero offsets (not yet used bc. of other drivers), usage of
bigger chunks than PAGE_SIZE. The performance is now at 1.3 GB/s, when
using 16MB transfers (the current maximum) and without debug logs.

Location:

uspace

Files:

• drv/bus/usb/xhci/transfers.c (modified) (8 diffs)
• lib/usb/include/usb/request.h (modified) (1 diff)

uspace/drv/bus/usb/xhci/transfers.c

@@ -48 +48 @@
 } stage_dir_flag_t;
 
-#define REQUEST_TYPE_DTD (0x80)
-#define REQUEST_TYPE_IS_DEVICE_TO_HOST(rq) ((rq) & REQUEST_TYPE_DTD)
-
-
 /** Get direction flag of data stage.
  *  See Table 7 of xHCI specification.
@@ -59 +55 @@
 {
         /* See Table 7 of xHCI specification */
-        return REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType) && (wLength > 0)
+        return SETUP_REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType) && (wLength > 0)
                 ? STAGE_OUT
                 : STAGE_IN;
@@ -80 +76 @@
 
         /* See Table 7 of xHCI specification */
-        return REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType)
+        return SETUP_REQUEST_TYPE_IS_DEVICE_TO_HOST(bmRequestType)
                 ? DATA_STAGE_IN
                 : DATA_STAGE_NO;
@@ -124 +120 @@
 }
 
-static int calculate_trb_count(xhci_transfer_t *transfer)
-{
-        const size_t size = transfer->batch.size;
-        return (size + PAGE_SIZE - 1 )/ PAGE_SIZE;
-}
-
-static void trb_set_buffer(xhci_transfer_t *transfer, xhci_trb_t *trb,
-        size_t i, size_t total, size_t *remaining)
-{
-        const uintptr_t ptr = dma_buffer_phys(&transfer->batch.dma_buffer,
-                transfer->batch.dma_buffer.virt + i * PAGE_SIZE);
-
-        trb->parameter = host2xhci(64, ptr);
-        TRB_CTRL_SET_TD_SIZE(*trb, max(31, total - i - 1));
-        if (*remaining > PAGE_SIZE) {
-                TRB_CTRL_SET_XFER_LEN(*trb, PAGE_SIZE);
-                *remaining -= PAGE_SIZE;
-        }
-        else {
-                TRB_CTRL_SET_XFER_LEN(*trb, *remaining);
-                *remaining = 0;
-        }
+#define MAX_CHUNK_SIZE (1 << 16)
+
+typedef struct {
+        /* Input parameters */
+        dma_buffer_t buf;
+        size_t chunk_size, packet_count, mps, max_trb_count;
+
+        /* Changing at runtime */
+        size_t transferred, remaining;
+        void *pos;
+} trb_splitter_t;
+
+static void trb_splitter_init(trb_splitter_t *ts, xhci_transfer_t *transfer)
+{
+        ts->buf = transfer->batch.dma_buffer;
+
+        const size_t chunk_mask = dma_policy_chunk_mask(ts->buf.policy);
+        ts->chunk_size = (chunk_mask > MAX_CHUNK_SIZE + 1)
+                ? MAX_CHUNK_SIZE : (chunk_mask + 1);
+
+        ts->remaining = transfer->batch.size;
+        ts->max_trb_count = (ts->remaining + ts->chunk_size - 1) / ts->chunk_size + 1;
+        ts->mps = transfer->batch.ep->max_packet_size;
+        ts->packet_count = (ts->remaining + ts->mps - 1) / ts->mps;
+
+        ts->transferred = 0;
+        ts->pos = ts->buf.virt + transfer->batch.offset;
+}
+
+static void trb_split_next(xhci_trb_t *trb, trb_splitter_t *ts)
+{
+        xhci_trb_clean(trb);
+
+        size_t size = min(ts->remaining, ts->chunk_size);
+
+        /* First TRB might be misaligned */
+        if (ts->transferred == 0) {
+                const size_t offset = (ts->pos - ts->buf.virt) % ts->chunk_size;
+                size = min(size, ts->chunk_size - offset);
+        }
+
+        ts->transferred += size;
+        ts->remaining -= size;
+
+        const size_t tx_packets = (ts->transferred + ts->mps - 1) / ts->mps;
+        const unsigned td_size = min(31, ts->packet_count - tx_packets);
+
+        /* Last TRB must have TD Size = 0 */
+        assert(ts->remaining > 0 || td_size == 0);
+
+        uintptr_t phys = dma_buffer_phys(&ts->buf, ts->pos);
+
+        trb->parameter = host2xhci(64, phys);
+        TRB_CTRL_SET_TD_SIZE(*trb, td_size);
+        TRB_CTRL_SET_XFER_LEN(*trb, size);
+        TRB_CTRL_SET_TRB_TYPE(*trb, XHCI_TRB_TYPE_NORMAL);
+
+        if (ts->remaining)
+                TRB_CTRL_SET_CHAIN(*trb, 1);
+
+        ts->pos += size;
 }
 
@@ -155 +190 @@
         usb_device_request_setup_packet_t* setup = &batch->setup.packet;
 
-        size_t buffer_count = 0;
-        if (setup->length > 0) {
-                buffer_count = calculate_trb_count(transfer);
-        }
-
-        xhci_trb_t trbs[buffer_count + 2];
-
-        xhci_trb_t *trb_setup = trbs;
+        trb_splitter_t splitter;
+        trb_splitter_init(&splitter, transfer);
+
+        xhci_trb_t trbs[splitter.max_trb_count + 2];
+        size_t trbs_used = 0;
+
+        xhci_trb_t *trb_setup = &trbs[trbs_used++];
         xhci_trb_clean(trb_setup);
 
-        TRB_CTRL_SET_SETUP_WVALUE(*trb_setup, setup->value);
-        TRB_CTRL_SET_SETUP_WLENGTH(*trb_setup, setup->length);
-        TRB_CTRL_SET_SETUP_WINDEX(*trb_setup, setup->index);
-        TRB_CTRL_SET_SETUP_BREQ(*trb_setup, setup->request);
-        TRB_CTRL_SET_SETUP_BMREQTYPE(*trb_setup, setup->request_type);
+        trb_setup->parameter = batch->setup.packed;
 
         /* Size of the setup packet is always 8 */
@@ -180 +210 @@
             get_transfer_type(trb_setup, setup->request_type, setup->length));
 
-        /* Data stage */
-        if (setup->length > 0) {
-                int stage_dir = REQUEST_TYPE_IS_DEVICE_TO_HOST(setup->request_type)
-                                        ? STAGE_IN : STAGE_OUT;
-                size_t remaining = transfer->batch.size;
-
-                for (size_t i = 0; i < buffer_count; ++i) {
-                        xhci_trb_clean(&trbs[i + 1]);
-                        trb_set_buffer(transfer, &trbs[i + 1], i, buffer_count, &remaining);
-
-                        TRB_CTRL_SET_DIR(trbs[i + 1], stage_dir);
-                        TRB_CTRL_SET_TRB_TYPE(trbs[i + 1], XHCI_TRB_TYPE_DATA_STAGE);
-
-                        if (i == buffer_count - 1) break;
-
-                        /* Set the chain bit as this is not the last TRB */
-                        TRB_CTRL_SET_CHAIN(trbs[i], 1);
-                }
-        }
+        stage_dir_flag_t stage_dir = (transfer->batch.dir == USB_DIRECTION_IN)
+                                ? STAGE_IN : STAGE_OUT;
+
+        /* Data stage - first TRB is special */
+        if (splitter.remaining > 0) {
+                xhci_trb_t *trb = &trbs[trbs_used++];
+                trb_split_next(trb, &splitter);
+                TRB_CTRL_SET_TRB_TYPE(*trb, XHCI_TRB_TYPE_DATA_STAGE);
+                TRB_CTRL_SET_DIR(*trb, stage_dir);
+        }
+        while (splitter.remaining > 0)
+                trb_split_next(&trbs[trbs_used++], &splitter);
 
         /* Status stage */
-        xhci_trb_t *trb_status = trbs + buffer_count + 1;
+        xhci_trb_t *trb_status = &trbs[trbs_used++];
         xhci_trb_clean(trb_status);
 
@@ -217 +240 @@
 
         return xhci_trb_ring_enqueue_multiple(get_ring(transfer), trbs,
-            buffer_count + 2, &transfer->interrupt_trb_phys);
-}
-
-static errno_t schedule_bulk(xhci_hc_t* hc, xhci_transfer_t *transfer)
-{
+            trbs_used, &transfer->interrupt_trb_phys);
+}
+
+static errno_t schedule_bulk_intr(xhci_hc_t* hc, xhci_transfer_t *transfer)
+{
+        xhci_trb_ring_t * const ring = get_ring(transfer);
+        if (!ring)
+                return EINVAL;
+
         /* The stream-enabled endpoints need to chain ED trb */
         xhci_endpoint_t *ep = xhci_endpoint_get(transfer->batch.ep);
-        if (!ep->primary_stream_data_size) {
-                const size_t buffer_count = calculate_trb_count(transfer);
-                xhci_trb_t trbs[buffer_count];
-                size_t remaining = transfer->batch.size;
-
-                for (size_t i = 0; i < buffer_count; ++i) {
-                        xhci_trb_clean(&trbs[i]);
-                        trb_set_buffer(transfer, &trbs[i], i, buffer_count, &remaining);
-                        TRB_CTRL_SET_TRB_TYPE(trbs[i], XHCI_TRB_TYPE_NORMAL);
-
-                        if (i == buffer_count - 1) break;
-
-                        /* Set the chain bit as this is not the last TRB */
-                        TRB_CTRL_SET_CHAIN(trbs[i], 1);
-                }
+        const bool use_streams = !!ep->primary_stream_data_size;
+
+        trb_splitter_t splitter;
+        trb_splitter_init(&splitter, transfer);
+
+        const size_t trb_count = splitter.max_trb_count + use_streams;
+        xhci_trb_t trbs[trb_count];
+        size_t trbs_used = 0;
+
+        while (splitter.remaining > 0)
+                trb_split_next(&trbs[trbs_used++], &splitter);
+
+        if (!use_streams) {
                 /* Set the interrupt bit for last TRB */
-                TRB_CTRL_SET_IOC(trbs[buffer_count - 1], 1);
-
-                xhci_trb_ring_t* ring = get_ring(transfer);
-                return xhci_trb_ring_enqueue_multiple(ring, &trbs[0], buffer_count,
-                        &transfer->interrupt_trb_phys);
+                TRB_CTRL_SET_IOC(trbs[trbs_used - 1], 1);
         }
         else {
-                xhci_trb_ring_t* ring = get_ring(transfer);
-                if (!ring) {
-                        return EINVAL;
-                }
-
-                const size_t buffer_count = calculate_trb_count(transfer);
-                xhci_trb_t trbs[buffer_count + 1];
-                size_t remaining = transfer->batch.size;
-
-                for (size_t i = 0; i < buffer_count; ++i) {
-                        xhci_trb_clean(&trbs[i]);
-                        trb_set_buffer(transfer, &trbs[i], i, buffer_count + 1, &remaining);
-                        TRB_CTRL_SET_TRB_TYPE(trbs[i], XHCI_TRB_TYPE_NORMAL);
-                        TRB_CTRL_SET_CHAIN(trbs[i], 1);
-                }
-                TRB_CTRL_SET_ENT(trbs[buffer_count - 1], 1);
-
-                xhci_trb_clean(&trbs[buffer_count]);
-                trbs[buffer_count].parameter = host2xhci(64, (uintptr_t) transfer);
-                TRB_CTRL_SET_TRB_TYPE(trbs[buffer_count], XHCI_TRB_TYPE_EVENT_DATA);
-                TRB_CTRL_SET_IOC(trbs[buffer_count], 1);
-
-                return xhci_trb_ring_enqueue_multiple(ring, &trbs[0], buffer_count + 1,
-                        &transfer->interrupt_trb_phys);
-        }
-}
-
-static errno_t schedule_interrupt(xhci_hc_t* hc, xhci_transfer_t* transfer)
-{
-        const size_t buffer_count = calculate_trb_count(transfer);
-        xhci_trb_t trbs[buffer_count];
-        size_t remaining = transfer->batch.size;
-
-        for (size_t i = 0; i < buffer_count; ++i) {
-                xhci_trb_clean(&trbs[i]);
-                trb_set_buffer(transfer, &trbs[i], i, buffer_count, &remaining);
-                TRB_CTRL_SET_TRB_TYPE(trbs[i], XHCI_TRB_TYPE_NORMAL);
-
-                if (i == buffer_count - 1) break;
-
-                /* Set the chain bit as this is not the last TRB */
-                TRB_CTRL_SET_CHAIN(trbs[i], 1);
-        }
-        /* Set the interrupt bit for last TRB */
-        TRB_CTRL_SET_IOC(trbs[buffer_count - 1], 1);
-
-        xhci_trb_ring_t* ring = get_ring(transfer);
-        return xhci_trb_ring_enqueue_multiple(ring, &trbs[0], buffer_count,
+                /* Clear the chain bit on the last TRB */
+                TRB_CTRL_SET_CHAIN(trbs[trbs_used - 1], 1);
+                TRB_CTRL_SET_ENT(trbs[trbs_used - 1], 1);
+
+                xhci_trb_t *ed = &trbs[trbs_used++];
+                xhci_trb_clean(ed);
+                ed->parameter = host2xhci(64, (uintptr_t) transfer);
+                TRB_CTRL_SET_TRB_TYPE(*ed, XHCI_TRB_TYPE_EVENT_DATA);
+                TRB_CTRL_SET_IOC(*ed, 1);
+        }
+
+        return xhci_trb_ring_enqueue_multiple(ring, trbs, trbs_used,
                 &transfer->interrupt_trb_phys);
 }
@@ -429 +414 @@
         [USB_TRANSFER_CONTROL] = schedule_control,
         [USB_TRANSFER_ISOCHRONOUS] = NULL,
-        [USB_TRANSFER_BULK] = schedule_bulk,
-        [USB_TRANSFER_INTERRUPT] = schedule_interrupt,
+        [USB_TRANSFER_BULK] = schedule_bulk_intr,
+        [USB_TRANSFER_INTERRUPT] = schedule_bulk_intr,
 };
 

uspace/lib/usb/include/usb/request.h

@@ -85 +85 @@
 #define SETUP_REQUEST_TYPE_DEVICE_TO_HOST (1 << 7)
 #define SETUP_REQUEST_TYPE_HOST_TO_DEVICE (0 << 7)
+#define SETUP_REQUEST_TYPE_IS_DEVICE_TO_HOST(rt) ((rt) & (1 << 7))
 #define SETUP_REQUEST_TYPE_GET_TYPE(rt) ((rt >> 5) & 0x3)
 #define SETUP_REQUEST_TYPE_GET_RECIPIENT(rec) (rec & 0x1f)